import numpy as np
import matplotlib.pyplot as plt
from sklearn.mixture import GaussianMixture as GMM
from functools import partial

class Logs():
    cur_log_file = None
    log_time_tag = None
    log_dir = './logs'

    def set_log_file(tag, src_level= -2):
        Logs.cur_log_file = Logs.get_log_fn(tag=tag, src_level=src_level)

    def get_log_fn(tag=None, src_level=-2):
        if Logs.cur_log_file is None and tag is None:
            tag = "LOG_default"

        if tag is None:
            return Logs.cur_log_file

        import datetime, os

        Logs.log_time_tag = datetime.datetime.now().strftime("%m_%d_%H_%M")

        log_time_tag_parts = Logs.log_time_tag.split('_')
        log_start = 15 * (int(log_time_tag_parts[-1]) // 15)
        day = get_day_prefix(Logs.log_time_tag)

        log_time_tag_parts[-1] = str(log_start)

        fn = tag + '_log_' + '_'.join(log_time_tag_parts) + '.txt'
        target = os.sep.join([Logs.log_dir, day])
        if not os.path.exists(target):
            os.makedirs(target)

        log_fn = os.sep.join([Logs.log_dir, day, fn])
        return log_fn

def cur_step(st=None, store=[]):
    if st is not None:
        if not store:
            store.append(st)
        else:
            store[0] = st

    return store[0]

def log(s=None, d=None, src_level=-2):
    def col_print(d, cols=4, proc=lambda i, x, d:str(x)):
        cols_out = []
        last = len(d) - 1
        for i, val in enumerate(d):
            val = proc(i, val, d)
            cols_out.append(val)
            if i == 0 and i != last:
                continue
            if i % cols == cols - 1:
                log_file.write('\t' + '   '.join(cols_out) + '\n')
                cols_out = []
            elif i == last:
                if len(cols_out) == 1:
                    log_file.write('\t' + cols_out[0] + '\n')
                elif len(cols_out) > 1:
                    log_file.write('\t' + cols_out[0] + '\n')

    log_fn = Logs.get_log_fn()

    import inspect, os
    recs = inspect.stack()
    path = str(recs[1][0].f_code).split('"')[src_level]

    lineNum = recs[1][0].f_lineno
    where = os.path.basename(path)
    where += ":" + str(lineNum)
    if not Logs.log_time_tag:
        Logs.log_time_tag = datetime.datetime.now().strftime("%m_%d_%H_%M")

    when = Logs.log_time_tag + ": "
    when += where + "-"

    if log_fn is None:
        log_fn = Logs.get_log_fn()

    with open(log_fn, 'a+') as log_file:
        log_file.write(when + s + '\n')
        special_handling = ['m_params', 'vc_params_perf', 'train_scores', 'test_scores', 'plot_data']
        if d is None:
            pass
        elif type(d) is list or str(type(d)) == "<class 'numpy.ndarray'>":
            if len(d) <= 1:
                log_file.write(str(d))
            else:
                col_print(d)
        elif type(d) is dict:
            proc = lambda i, x, d : str(x) + ': ' + str(d[x])
            col_print(d, proc=proc)
            print(d)

    if type(d) is dict and len([x for x in d if x in special_handling]) > 0:
        if 'ds' in d:
            ds = d['ds']
        else:
            ds = 'no_ds_given'

        if 'plot' in d:
            plot_type = d['plot']
        else:
            plot_type = 'no_pt_specified'

        log_special = '.'.join(log_fn.split('.')[0:-1]) + '_' + list(d)[0] + '_' + ds + '_' + plot_type + '.csv'
        d = d[list(d)[0]]  # de-nest, get to the actual data

        if type(d) is list:
            out_cols = [i for i, x in enumerate(d)]
        elif str(type(d)) == "<class 'numpy.ndarray'>":
            out_cols = [i for i, x in enumerate(d)]
        else:
            out_cols = list(d)

        import csv
        if not os.path.exists(log_special):
            with open(log_special, 'w', newline='') as log_file:
                wr = csv.writer(log_file)
                wr.writerow(['where', 'when', 'msg'] + out_cols)

        col_2 = s  # ' ' * len(where)
        col_0 = where
        col_1 = Logs.log_time_tag
        with open(log_special, 'a+', newline='') as log_file:
            wr = csv.writer(log_file)

            if d is dict or isinstance(d, dict):
                out_vals = [d[k] for k in out_cols]
            elif type(d) is list:
                out_vals = d
            elif str(type(d)) == "<class 'numpy.ndarray'>":
                out_vals = d.tolist()
            else:
                hook = True

            wr.writerow([col_0, col_1, col_2] + out_vals)

def get_day_prefix(now_str=None):
    import datetime
    if now_str is None:
        now_str = datetime.datetime.now().strftime("%m_%d_%H_%M")

    time_tag_parts = now_str.split('_')
    return "_".join(time_tag_parts[:2])

def get_plot_fn(tag=None, root_path='./plots', ftype='.png'):
    import datetime, os
    time_tag = datetime.datetime.now().strftime("%m_%d_%H_%M")
    day = get_day_prefix(time_tag)
    target = os.sep.join([root_path, day])
    if not os.path.exists(target):
        os.makedirs(target)

    fn = tag + '_' + time_tag + ftype
    full_fn = os.sep.join([root_path, day, fn])

    no_overwrite = 0
    test_fn = full_fn
    while os.path.exists(test_fn):
        no_overwrite += 1
        pos = len(full_fn) - 1
        while pos >= 0 and full_fn[pos] != '.':
            pos += -1

        fn_parts = full_fn[0:pos]

        part1 = fn_parts + "_" + str(no_overwrite)
        test_fn = part1 + ftype

    full_fn = test_fn

    return full_fn

def get_ser_fn(tag=None, ftype='.npy'):
    return get_plot_fn(tag=tag, ftype=ftype)

def get_converge(npl, ep=.0001):
    cnv = 0
    cnv_val = npl[0]

    for i, x in enumerate(npl):
        if x > (1-ep)*cnv_val and x < (1+ep)*cnv_val:
            continue
        else:
            cnv = i
            cnv_val = npl[i]

    return cnv, cnv_val

def get_plot_xmax(x, y):
    y_max_idx = np.argmax(y)
    y_min_idx = np.argmin(y)
    conv_idx, val = get_converge(y)
    idxs = [y_max_idx, conv_idx, y_min_idx]
    which_idx = np.argmax( [x[ii] for ii in idxs] )
    target_idx = idxs[which_idx]
    # x vals should be monotonic, so included 2 extra points should not result in a sub-optimal x value
    if target_idx < len(x) - 2:
        target_idx += 2

    x_max = x[target_idx]

    return x_max

def pr_fl(f, len=8):
    return str(f)[0:len]

def get_ranges(data):
    hook = True

    for k in data:
        if len(data[k].shape) == 1:
            col2 = [[x+1] for x in range(data[k].shape[0])]
            data[k] = np.column_stack((data[k], col2))
            hook = True

    #massage = lambda x, l: x + 2 if x + 2 < len(l) else x
    y_idx_at_max = [get_plot_xmax(x=data[k][:, 1], y = data[k][:, 0]) for k in data]

    all_max_x = max(y_idx_at_max)
    all_min_x = min([np.min(data[k][:, 1]) for k in data])

    all_max_y = max([np.max(data[k][:, 0]) for k in data])
    all_min_y = min([np.min(data[k][:, 0]) for k in data])

    return all_min_x, all_max_x, all_min_y, all_max_y

BSK = 'best_state'
BFK = 'best_fitness'
BFCK = 'best_fc_curve'

def get_plot_title(pr, dm, dim):
    return f"{pr}-{dm} size:{dim}"

def make_plot(
        title=None,  data=None, fn=None, rows=1, extra_data=None,
        x_label="iterations", y_label="fitness score", trunc_converged=True,
        zero_x_bnd=False, zero_y_bnd=False, ebars=False
    ):

    plt.close()
    fig, axes = plt.subplots(rows, 1, squeeze=False, figsize=(10, 10))

    if hasattr(axes, 'shape'):
        subplot_1 = axes[0, 0]

    all_min_x, all_max_x, all_min_y, all_max_y = get_ranges(data)

    lb_factor = .95
    ub_factor = 1.05
    if zero_x_bnd:
        xp_min = min(0, lb_factor * all_min_x)
    else:
        xp_min = lb_factor * all_min_x

    if zero_y_bnd:
        yp_min = min(0, lb_factor * all_min_y)
    else:
        yp_min = lb_factor * all_min_y

    x_lb = xp_min
    x_ub = ub_factor * all_max_x
    y_lb = yp_min
    y_ub = ub_factor * all_max_y

    print(f"Plot bounds = x:{pr_fl(x_lb)},{pr_fl(x_ub)}, y:{pr_fl(y_lb)},{pr_fl(y_ub)}")
    subplot_1.set_xlim(x_lb, x_ub)
    subplot_1.set_ylim(y_lb, y_ub)
    subplot_1.set_title(title)
    subplot_1.set_xlabel(x_label)
    subplot_1.set_ylabel(y_label)

    subplot_1.grid()
    colors = ['r', 'g', 'b', 'c', 'm', 'k', 'y']
    syms = ['+', '1', '2', '3', '4']

    for i, k in enumerate(data):
        log(f"{title}raw plot {k}", {'plot_data': data[k]})

        cur_x = data[k][:, 1]
        cur_y = data[k][:, 0]
        if trunc_converged:
            y_idx_at_conv, _ = get_converge(cur_y)
            y_idx_at_conv = min(len(cur_y) - 1, y_idx_at_conv + 2)
            cur_x_data_2 = cur_x[0: y_idx_at_conv + 1]
            cur_y_data_2 = cur_y[0: y_idx_at_conv + 1]

            cur_x = cur_x_data_2
            cur_y = cur_y_data_2

        log(f"{title} processed plot {k} x:", cur_x)
        log(f"processed plot {k} y:", cur_y)

        extra_text = 'None'
        lbl = k
        if not extra_data:
            pass
        else:
            if k in extra_data:
                extra_text = extra_data[k]
                buffer = '' if extra_text[0].isspace() else ' '
                lbl = k + buffer + extra_text

        log(f"plot extra data {k}:" + extra_text)

        sylen = len(syms)
        pt = syms[i%sylen]
        clen = len(colors)
        if not ebars:
            subplot_1.plot(cur_x, cur_y, color=colors[i%clen], marker=pt, linestyle='solid', label=lbl)
        else:
            yerr = np.std(cur_y)
            subplot_1.errorbar(cur_x, cur_y, yerr=yerr, color=colors[i % clen], marker=pt, linestyle='solid', label=lbl)

    subplot_1.legend(loc="lower right")

    fig.tight_layout()
    plt.margins(10, 0)

    plt.savefig(fn)
    plt.show()

DS_1 = 'digits'
DS_2 = 'Bach'
DS_3 = 'Wine'

CODE_TEST_1 = 1
CODE_TEST_2 = 2
CODE_TEST_3 = 3

ds_names = {
    CODE_TEST_1:DS_1,
    CODE_TEST_2 : DS_2,
    CODE_TEST_3 : DS_3
}

def load_datasets(dset=1):
    from sklearn.datasets import load_digits
    from sklearn.datasets import load_wine

    from numpy import genfromtxt
    from datetime import datetime as dt

    def yn2num(val):
        assert (val[0].lower() in ['y', 'n'])
        res = 1 if val.lower()[0] == 'y' else 0
        return res

    def note2num(val):
        letter = val[0].lower()
        notes = ['c', 'd', 'e', 'f', 'g', 'a', 'b']
        index = notes.index(letter)
        res = float(index)

        if len(val) > 1:
            if val[1] == 'b':
                res -= .5
            elif val[1:3].lower() == 'sh':
                res += .5
            else:
                assert (False)

        assert (res >= 0)
        return res

    def colzero(val):
        decoded = val.decode("utf-8")
        return decoded

    loaded = []
    incl_digits = False
    incl_wine = False

    # CODE_TEST_2 for Bach
    if dset == CODE_TEST_1:
        incl_digits = True
    elif dset == CODE_TEST_3:
        incl_wine = False
    elif dset != CODE_TEST_2:
        assert(False)

    d1 = dict()
    cur_ds = d1

    if incl_digits:
        X, y = load_digits(return_X_y=True)
        cur_ds['X'] = X
        cur_ds['Y'] = y
        cur_ds['name'] = DS_1
        loaded.append(cur_ds)
        return loaded

    if incl_wine:
        X, y = load_wine(return_X_y=True)
        ds2 = dict()
        ds2['X'] = X
        ds2['Y'] = y
        ds2['name'] = DS_3
        # unq_1, cnts_1 = np.unique(ds1['Y'], return_counts=True)
        #nq_2, cnts_2 = np.unique(ds2['Y'], return_counts=True)

        #log(f'For dataset {DS_1} unique = {unq_1}, counts = {cnts_1}')
        #log(f'For dataset {DS_2} unique = {unq_2}, counts = {cnts_2}')
        loaded.append(ds2)

        return loaded

    # d1 = dict()

    converters = dict()
    for i in range(17):
        converters[i] = colzero

    try:
        temp = genfromtxt(
            'data/jsbach_chorals_harmony/jsbach_chorals_harmony.data', delimiter=',',
            converters=converters
        )
    except Exception as ex:
        debug_hook = True
        import sys
        _, _, tb = sys.exc_info()
        raise ()
        pass

    labels = []
    pieces = set()
    d3 = dict()
    cur_ds = d3
    cur_ds['X'] = np.zeros((len(temp), len(temp[0]) - 2))
    # cur_ds['Y'] = np.zeros((len(temp), 1))

    for i, row in enumerate(temp):
        for j in range(len(row) - 1):
            if j == 0:
                pieces.add(row[j])
                continue

            if j > 0:
                if j in (1, 15):
                    cur_ds['X'][i, j - 1] = float(row[j])
                elif 2 <= j <= 13:
                    cur_ds['X'][i, j - 1] = yn2num(row[j])
                elif j == 14:
                    cur_ds['X'][i, j - 1] = note2num(row[j])
                else:
                    assert (False)

        labels.append(row[-1])

    label_index = set(labels)
    label_index = list(label_index)
    label_index.sort()

    label_count_1 = dict()
    for s in label_index:
        label_count_1[s] = labels.count(s)

    from sklearn.preprocessing import LabelBinarizer
    from sklearn.preprocessing import LabelEncoder
    # lb = LabelBinarizer()
    le = LabelEncoder()
    cur_ds['Y'] = le.fit_transform(labels)
    unq_1, cnts_1 = np.unique(cur_ds['Y'], return_counts=True)
    cur_ds['name'] = DS_2
    loaded.append(cur_ds)

    return loaded

max_silh_km_bach = 1575
max_silh_km_digits = 15
max_silh_gmm_bach = 2056
max_silh_gmm_digits = 17

KMEANSK = 'kmeans'
GMMK = 'gmm'

STEP1 = 's1'
STEP2 = 's2'
STEP3 = 's3'
STEP4 = 's4'
STEP5 = 's5'

max_cluster_scores = {

    STEP1:{
        DS_1: {
            KMEANSK: {'max': max_silh_km_digits} ,
            GMMK: {'max': max_silh_gmm_digits}
        },
        DS_2: {
            KMEANSK: {'max': max_silh_km_bach},
            GMMK: {'max': max_silh_gmm_bach}
        }
    },
    STEP2:{
        DS_1: {
            KMEANSK: {'max': -1},
            GMMK: {'max': -1}
        },
        DS_2: {
            KMEANSK: {'max': -1},
            GMMK: {'max': -1}
        }
    }

}

def TEST_MODE(val=None, store=[]):
    if not store:
        if not val:
            store.append(False)
        else:
            store.append(val)

    if not val:
        pass
    else:
        store[0] = val

    return store[0]

def get_digits():
    from sklearn.preprocessing import MinMaxScaler
    from sklearn.preprocessing import OneHotEncoder

    datasets = load_datasets(CODE_TEST_1)
    cur_ds = datasets[0]

    # Normalize feature data
    scaler = MinMaxScaler()

    X_train_scaled = scaler.fit_transform(cur_ds['X'])
    y_train = cur_ds['Y']

    # One hot encode target values
    one_hot = OneHotEncoder()
    y_train_hot = one_hot.fit_transform(y_train.reshape(-1, 1)).todense()
    if TEST_MODE():
        #last = int(.1 * len(X_train_scaled))
        last = 250
        X_train_scaled = X_train_scaled[0:last]
        y_train_hot = y_train_hot[0:last]

    return X_train_scaled, y_train_hot

def get_bach():
    from sklearn.preprocessing import MinMaxScaler
    from sklearn.preprocessing import OneHotEncoder

    datasets = load_datasets(CODE_TEST_2)
    cur_ds = datasets[0]

    # Normalize feature data
    scaler = MinMaxScaler()

    X_train_scaled = scaler.fit_transform(cur_ds['X'])
    y_train = cur_ds['Y']

    # One hot encode target values
    one_hot = OneHotEncoder()
    y_train_hot = one_hot.fit_transform(y_train.reshape(-1, 1)).todense()

    if TEST_MODE():
        #last = int(.1 * len(X_train_scaled))
        last = 250
        X_train_scaled = X_train_scaled[0:last]
        y_train_hot = y_train_hot[0:last]

    return X_train_scaled, y_train_hot

GETK= 'get_func'
TAGK = 'tag'
GET_DIGITS ={
    GETK:get_digits
}

GET_BACH ={
    GETK:get_bach
}

datasets_info = {
    DS_1: GET_DIGITS,
    DS_2: GET_BACH
}

def make_x_points(starts=None, points=None, limits=None):
    assert (len(starts) == len(points))
    assert (len(starts) == len(limits))

    res_list = []
    regions = len(starts)
    for i, s in enumerate(starts):
        cur_start = s
        cur_points = points[i]

        #validate cur_points, make sure cur_points not too large
        if i == regions - 1:  # last
            if limits[i] is not None:
                cur_points = min(cur_points, limits[i] - cur_start + 1)
        else:
            next_start = starts[i + 1]
            temp_limit = next_start - 1
            cur_points = min(cur_points, temp_limit - cur_start + 1)

        if limits[i] is None:
            if i == regions - 1:  # last
                cur_limit = cur_start + cur_points - 1
            else:
                cur_limit = starts[i + 1] - 1
        else:
            cur_limit = limits[i]

        step = int((cur_limit - cur_start) / (cur_points - 1))
        assert (step > 0)
        cur_list = list(range(cur_start, cur_limit + 1, step))
        res_list += cur_list

    return res_list

def validate_limit_list(start, limit, instances):
    if limit is None:
        limit = [None for x in start]
        limit[-1] = instances
    elif len(limit) == 1:
        last_val = limit[0]
        last_pos = len(start) - 1
        limit = [None if i != last_pos else last_val for i in range(len(start)) ]
    elif len(start) != len(limit):
        assert(False)

    if limit[-1] is None:
        limit[-1] = instances

    return limit

def gmm_score(k, get_ds=None, plot=False, cl_tag=GMMK):
    from sklearn.metrics import silhouette_score

    X, y = get_ds[GETK]()
    tag = get_ds[TAGK]
    gmm_obj = GMM(k)
    gmm_obj.fit(X)
    label = gmm_obj.predict(X)

    if plot:
        st=cur_step()

        dr_tag = 'not_red'
        if REDK in get_ds:
            dr_tag = get_ds[REDK]

        cluster_plot(
            get_ds=get_ds, labels=label, dr_tag=dr_tag,
            cl_tag=cl_tag, fn=get_plot_fn(f'clu_plt_{cl_tag}_{dr_tag}')
        )
        fn_tag = f'{cl_tag}_{dr_tag}_{st}'

    return silhouette_score(X, label)

# def run_gmm(x_pts=None, get_ds=None):
#     import numpy as np
#     from sklearn.mixture import GaussianMixture as GMM
#
#     X, y = get_ds[GETK]()
#     tag = get_ds[TAGK]
#
#     instances = X.shape[0]
#
#     if x_pts is None:
#         cl_list = list(range(2, X.shape[0])) #all instance
#     else:
#         cl_list = x_pts
#
#     print(f"Evaluting cluster sizes ({tag}): {cl_list}")
#
#     results = np.zeros((len(cl_list), 2))
#
#     for i, k in enumerate(cl_list):
#         print(f'{k} centers')
#         sc = gmm_score(X, k)
#         results[i] = [sc, k]
#
#     return results

def save_np(a, st=None, ds_tag=None, cl_tag=None, extra=None,  fn=None):
    a_fn = fn
    if st is None:
        st = cur_step()
    if not fn:
        fn_tag = f'{st}@{str(ds_tag)}@{str(cl_tag)}@{str(extra)}'
        a_fn = get_ser_fn(fn_tag)

    np.save(a_fn, a)

def kmeans_score(k, get_ds=None, plot=False, cl_tag=KMEANSK):
    from sklearn.cluster import KMeans
    from sklearn.metrics import silhouette_score

    X, y= get_ds[GETK]()
    kMean = KMeans(n_clusters=k)
    kMean.fit(X)
    label = kMean.predict(X)

    if plot:
        st = cur_step()
        dr_tag = 'not_red'
        if REDK in get_ds:
            dr_tag = get_ds[REDK]

        cluster_plot(
            get_ds=get_ds, labels=label, dr_tag=dr_tag,
            cl_tag=cl_tag, fn=get_plot_fn(f'clu_plt_{cl_tag}_{dr_tag}')
        )

    return silhouette_score(X, label)

def incr_scoring(
        cl_list, get_ds=None, ds_tag=None,
        cl_tag=None, score_func= None, savepoint=100,
        skiplast=False, update=False, targ=None
):

    X, y = datasets_info[ds_tag][GETK]()
    all = [x for x in range(2, X.shape[0])]
    if not update:
        results = np.zeros((len(all), 2))
        visited = [False] * len(results)
    else:
        results = targ
        visited = [False] * len(results)
        for i, x in enumerate(results[:,1]):
            if int(x) > 0:
                visited[i] = True

    for i, k in enumerate(cl_list):
        if i == 0 or i % savepoint == 0:
            print(f'{k} clusters')
            if skiplast and i == len(cl_list) - 1:
                pass
            elif not visited[i]:
                sc = score_func(k, get_ds=get_ds)
                results[i] = [sc, k]
                visited[i] = True
                if update:
                    set_np_array(max_cluster_scores, ds_tag, cl_tag, results)  # will save to current step dictionary
                    set_np_array(
                        max_cluster_scores, ds_tag, cl_tag,
                        results, st=STEP1, custom_tag=CL_SCORING_RES
                    )
    set_np_array(max_cluster_scores, ds_tag, cl_tag, results) # will save to current step dictionary

    for i, k in enumerate(cl_list):
        print(f'{k} clusters')
        if skiplast and i == len(cl_list) - 1:
            pass
        else:
            if not visited[i]:
                sc = score_func(k, get_ds=get_ds)
                results[i] = [sc, k]
                visited[i] = True
                if update:
                    set_np_array(max_cluster_scores, ds_tag, cl_tag, results)  # will save to current step dictionary
                    set_np_array(
                        max_cluster_scores, ds_tag, cl_tag,
                        results, st=STEP1, custom_tag=CL_SCORING_RES
                    )
        if i > 0 and i % savepoint == 0:
            print("Savepoint")
            save_np(max_cluster_scores, ds_tag=ds_tag, cl_tag=cl_tag)

    set_np_array(max_cluster_scores, ds_tag, cl_tag, results)  # will save to current step dictionary

    return results

CL_SCORING_RES = 'cl-scoring'

def run_cluster_scoring(
        x_pts=None, get_ds=None, cl_tag=None,
        score_func=None, use_prev=False, update=False
):
    import numpy as np
    if use_prev and update:
        ds_tag = get_ds[TAGK]
        prev_found = check_for_previous_results(
            s1=cur_step(), ds_tag=ds_tag, cl_tag=cl_tag,
            custom_tag=CL_SCORING_RES
        )
        if prev_found is not None and not update:
            return prev_found

    X, y = get_ds[GETK]()
    ds_tag = get_ds[TAGK]

    instances = X.shape[0]
    #limit = validate_limit_list(start, limit, instances)
    if x_pts is None:
        cl_list = list(range(2, X.shape[0])) #all possible cluster sizes, 2 to instance - 1
    else:
        cl_list = x_pts

    print(f"Evaluating cluster sizes: {str(cl_list[0:4])[:-1]},... {cl_list[int(len(cl_list)/2)]},... {str(cl_list[-4:])[1:]}")
    if update:
        ds_tag = get_ds[TAGK]
        prev_found = check_for_previous_results(
            s1=cur_step(), ds_tag=ds_tag, cl_tag=cl_tag,
            custom_tag=CL_SCORING_RES
        )
        results = incr_scoring(
            cl_list, get_ds=get_ds, ds_tag=ds_tag,
            cl_tag= cl_tag, score_func=score_func, update=update, targ=prev_found)
    else:
        results = incr_scoring(
            cl_list, get_ds=get_ds, ds_tag=ds_tag,
            cl_tag=cl_tag, score_func=score_func, update=update)

    set_np_array(max_cluster_scores, ds_tag, cl_tag, results, custom_tag=CL_SCORING_RES)

    return results

def dr_rca_gp(X,y, ds_tag=None):
    import numpy as np
    from sklearn import random_projection
    X = np.random.rand(100, 10000)
    transformer = random_projection.GaussianRandomProjection()
    X_new = transformer.fit_transform(X)
    return transformer, X_new

def dr_rca_sm(X,y, ds_tag=None):
    import numpy as np
    from sklearn import random_projection
    X = np.random.rand(100, 10000)
    transformer = random_projection.SparseRandomProjection()
    X_new = transformer.fit_transform(X)
    return transformer, X_new

RCAGPK = 'rcagp'
RCASMK = 'rcasm'

def dr_ica(X,y, k=None, ds_tag=None, do_whitening=True, eff_score=False):
    from sklearn import decomposition
    from scipy.stats import norm
    from scipy.stats import kurtosis

    if k is not None:
        num_components = int(kur_results[kur_max_idx, 1])
        ica = decomposition.FastICA(n_components=num_components, whiten=do_whitening)

        return ica, ica.fit_transform(X)

    space_dim = X.shape[1]
    kur_results = np.zeros((space_dim, 2))
    for i in range(space_dim):
        num_components = i + 1
        #if num_components == space_dim:
        #    continue  # the max number of components tends to generate errors, evaluating not useful
        if num_components == 64:
            hook = True
        print(f"Evaluating {num_components} ICA components")
        ica = decomposition.FastICA(n_components=num_components, whiten=do_whitening,
            max_iter=800)

        try:
            ica.fit(X)
        except Exception as ex:
            hook = True
            raise

        A = ica.components_
        tally = np.zeros(A.shape[0])

        for ii, v in enumerate(ica.components_):
            one_axis = np.zeros(A.shape)
            one_axis[ii] = A[ii]
            proj = np.dot(X, one_axis.T)
            proj = proj[:, ii]
            kur_score = kurtosis(proj)
            tally[ii] = np.abs(kur_score)

        kur_results[i] = [np.min(tally), num_components]

    #kur_results = np.abs(kur_results)
    max_idx = np.argmax(kur_results[:,0])
    kur_max_idx  = max_idx
    plot_tag = 'kur'
    data = {plot_tag: kur_results}
    xdata = {plot_tag: f"max y:{pr_fl(kur_results[max_idx, 0])} x:{int(kur_results[max_idx, 1])}"}
    max_y = kur_results[max_idx, 0]
    max_red = kur_results[max_idx, 1]/space_dim
    max_vals = np.repeat(max_y, len(kur_results))
    std = np.std(max_vals - kur_results[:, 0])
    thresh = (max_y - std)/ max_y

    #norm_fact = max_y/max_red
    def score2(x, y,  thresh, max_store=[]):

        cur_sc_ratio = y/max_y
        cur_red_ratio = (x/space_dim)/max_red
        score2 = cur_sc_ratio/cur_red_ratio * max_y

        if not max_store:
            max_store.append(score2)

        if cur_sc_ratio > thresh and x > .4 * space_dim: # ignore larger reductions
            if score2 > max_store[0]:
                max_store[0] = score2
        else:
            score2 = .9 * max_store[0]

        return score2 # scale for graph

    #effectiveness_score
    test = score2(kur_results[max_idx, 1], kur_results[max_idx, 0], thresh)
    eff_score = [score2(r[1], r[0], thresh) for r in kur_results[0:max_idx + 1]]
    eff_score = np.array(eff_score)
    eff_score = np.column_stack((eff_score, kur_results[:,1][0:max_idx + 1]))
    data['eff_score'] = eff_score
    max_idx = np.argmax(eff_score[:, 0])
    highest_eff_score = max_idx
    xdata['eff_score'] = f"max y:{pr_fl(eff_score[max_idx, 0])} x:{int(eff_score[max_idx, 1])}"

    make_plot(
        title=f"Kurtosis Score for ICA components ({ds_tag})", data=data, x_label="ICA component",
        y_label="Kur Score", extra_data=xdata, fn=get_plot_fn(f'ICA_exp_plot_{ds_tag}'),
    )

    num_components = int(kur_results[kur_max_idx, 1])
    ica = decomposition.FastICA(n_components=num_components, whiten=do_whitening)

    return ica, ica.fit_transform(X)

ICAK = 'ica'
DR_ICA={
    GETK:dr_ica
}

def cluster_plot(get_ds=None, threshold=6, labels=None, dr_tag=None, cl_tag=None, fn=None):
    import numpy as np
    from main import get_plot_fn
    from main import get_digits
    import pandas as pd
    import seaborn as sns
    import matplotlib.pyplot as plt
    import matplotlib
    from sklearn.feature_selection import mutual_info_classif

    X, y = get_ds[GETK]()

    if labels is None:
        labels = y

    if type(y) is list or y is None:
        hook = True

    if len(labels.shape) > 1:  # flatten one hot encoding
        labels = np.array([np.argmax(x) for x in labels])

    try:
        res = mutual_info_classif(X, labels)
    except Exception as ex:
        hook = True
        raise

    columns = [str(x) for x in range(X.shape[1] + 1)]

    kept={}
    for score, f_name in sorted(zip(res, columns), reverse=True):
        print(f_name, score)
        kept[f_name]=score

    n1 = np.column_stack((X, labels))

    df = pd.DataFrame(n1, columns=columns)

    for c in columns[:-1]:
        if c not in list(kept)[:threshold]:
            df = df.drop(c, 1)

    matplotlib.rcParams['font.size'] = 18
    sns.set_context('talk', font_scale=1.2);
    ds_tag = get_ds[TAGK]

    dr_text = f'reduced by {dr_tag}'
    if dr_tag is None:
        dr_text = 'unreduced'
    if cl_tag is None:
        cl_text = "orig. labels"
    else:
        cl_text = f'{cl_tag} labels'

    title = f'{ds_tag}-{cl_text}, {dr_text}'

    try:
        pp=sns.pairplot(df, hue=columns[-1], corner=True, diag_kind='hist')
        pp.fig.suptitle(title)

    except Exception as ex:
        hook = True
        raise

    st = cur_step()
    if fn is None:
        fn = get_plot_fn(tag=f"reduced_{st}_{ds_tag}_{dr_tag}_{cl_tag}")

    plt.savefig(fn)

def reduced_plot(x_red):
    df1_filtered = df1[filter]
    matplotlib.rcParams['font.size'] = 18
    sns.set_context('talk', font_scale=1.2);

    plt.figure(figsize=(20, 20))
    sns.pairplot(df1_filtered, hue='Chord_label', corner=True)
    # plt.margins(10,10)

    plt.savefig(get_plot_fn(tag="bach_ds"))

def dr_pca(X,y, ds_tag=None):
    from sklearn import decomposition
    pca = decomposition.PCA(n_components=.95)
    pca.fit(X)

    x_red = pca.transform(X)
    #reduced_plot(x_red)

    return pca, x_red

DR_PCA={
    GETK:dr_pca
}

def ica_eval_plot(estimator=None, fn=None, ds_tag=None):
    import matplotlib.pyplot as plt
    from scipy.stats import kurtosis

    ica = estimator
    # for i, v in enumerate(ica.components_):

def rca_eval_plot(estimator=None, fn=None, ds_tag=None):
    pass

def pca_eval_plot(estimator=None, fn=None, ds_tag=None):
    import matplotlib.pyplot as plt
    pca = estimator
    plt.rcParams["figure.figsize"] = (12, 6)

    fig, ax = plt.subplots()

    y = np.cumsum(pca.explained_variance_ratio_)
    xi = np.arange(1, y.shape[0] + 1, step = 1)

    plt.ylim(0.0, 1.1)
    plt.plot(xi, y, marker='o', linestyle='--', color='b')

    plt.xlabel('Number of Components')
    plt.xticks(np.arange(0, 11, step=1))  # change from 0-based array index to 1-based human-readable label
    plt.ylabel('Cumulative variance (%)')
    cutoff = [(ii, v) for ii, v in enumerate(y) if v >= .95]
    last = cutoff[0][0]
    plt.title(f'The number of components needed to explain variance, cutoff component = {last}')

    plt.axhline(y=0.95, color='r', linestyle='-')
    plt.text(0.5, 0.85, '95% cut-off threshold', color='red', fontsize=16)

    ax.grid(axis='x')
    plt.savefig(fn)
    plt.show()

PCAK = 'pca'

EVALPLOTK = 'evp'
DRFUNCK = 'drfunc'
REDK = 'redmeth'

DR_TAGS = [PCAK, ICAK, RCAGPK, RCASMK]
dim_reduce_info = {
    PCAK: {DRFUNCK:dr_pca, EVALPLOTK:pca_eval_plot},
    ICAK: {DRFUNCK:dr_ica, EVALPLOTK:ica_eval_plot},
    RCAGPK: {DRFUNCK: dr_rca_gp, EVALPLOTK:rca_eval_plot},
    RCASMK: {DRFUNCK: dr_rca_sm, EVALPLOTK:rca_eval_plot}
}

def run_reduction_2(reduce_func=None, plot_func=None, get_ds=None, fn=None):
    import numpy as np
    from sklearn.cluster import KMeans
    from sklearn.metrics import silhouette_score

    X, y = get_ds[GETK]()
    tag = get_ds[TAGK]

    instances = X.shape[0]
    drfunc = reduce_func[DRFUNCK]
    estimator, reduced_x = drfunc(X, y, ds_tag=tag)
    #eval plots
    plot_func(estimator=estimator, fn=fn, ds_tag=tag)

    return reduced_x

STEP_RES = "_STEP_RESULTS"

def format_ranges(ranges):
    starts = []
    limits = []
    points = []
    for t in ranges:
        st = t[0]
        lim = t[1]
        starts.append(st)
        limits.append(lim)
        if len(t) > 2:
            pts = t[2]
        else:
            pts = 10

        points.append(pts)

    return starts, points, limits

def set_series(
    data, xdata, ptag=None,
    st=None, ds_tag=None, cl_tag=None, pts=None
):
    cur_s = get_np_array(max_cluster_scores, ds_tag, cl_tag, st=st)

    if cur_s is None:
        assert(False)
    else:
        max_idx = np.argmax(cur_s[:, 0])
        max_x = cur_s[max_idx,1]

    if pts is not None:
        assert (max_x in pts)
        fb = [int(x) in pts for x in cur_s[:, 1]]
        cur_s = cur_s[fb]
        cur_st = cur_step()

        set_np_array(max_cluster_scores, ds_tag, cl_tag, cur_s, custom_tag='trunc_plot_series', st=st)

    xdata[ptag] = f"max y:{pr_fl(cur_s[max_idx, 0])} x:{int(cur_s[max_idx, 1])}"
    data[ptag] = cur_s

def check_for_previous_results(s1=None, ds_tag=None, custom_tag=None, cl_tag=None):

    r = get_np_array(
        max_cluster_scores, ds_tag, cl_tag, custom_tag=custom_tag, st=s1
    )

    return r

def run_km_clustering(title=None, ranges=None, get_ds=None, cl_tag=KMEANSK, update=False):

    #starts, points, limits = format_ranges(ranges)
    results = run_cluster_scoring(
        x_pts=ranges, get_ds=get_ds, cl_tag=cl_tag,
        score_func=kmeans_score, update=update
    )
    ds_tag = get_ds[TAGK]
    data = {}
    xdata = {}

    st = cur_step()
    if st == STEP2:
        ptag = st + ' (reduced)'
        set_series(data, xdata, ptag=ptag, st=STEP2, ds_tag=ds_tag, cl_tag=cl_tag)
        set_series(data, xdata, ptag=STEP1, st=STEP1, ds_tag=ds_tag, cl_tag=cl_tag)
    elif st == STEP1:
        ptag = st
        set_series(data, xdata, ptag=ptag, st=STEP1, ds_tag=ds_tag, cl_tag=cl_tag)

    if ranges is None:
        range_text = "fullr"
    else:
        start = ranges[0]
        lim = ranges[-1]
        range_text = f'{start}-{lim}'

    ds_txt = ds_tag
    if REDK in get_ds:
        ds_txt = f'{ds_txt}_{get_ds[REDK]}'

    make_plot(
        title=title, fn=get_plot_fn(f'{cl_tag}_{ds_txt}_{st}_{range_text}'),
        data=data,
        x_label='k', y_label='Silh Scor', extra_data=xdata
    )

    return results

def run_gmm_clustering(title=None, get_ds=None, ranges=None, cl_tag=GMMK, update=False):

    #starts, points, limits = format_ranges(ranges)
    results = run_cluster_scoring(
        x_pts=ranges, get_ds=get_ds, cl_tag=cl_tag,
        score_func=gmm_score, update=update
    )

    st = cur_step()
    ds_tag = get_ds[TAGK]
    set_np_array(max_cluster_scores, ds_tag, cl_tag, results)
    title = f'{ds_tag} {cl_tag} Silhouette Scores'

    data = {}
    xdata = {}

    pt_list = [int(x) for x in results[:, 1]]

    if st == STEP2:
        ptag = st + ' (reduced)'
        set_series(data, xdata, ptag=ptag, st=STEP2, ds_tag=ds_tag, cl_tag=cl_tag)
        set_series(
            data, xdata, ptag=STEP1,
            st=STEP1, ds_tag=ds_tag, cl_tag=cl_tag
        )
    elif st == STEP1:
        ptag = st
        set_series(
            data, xdata, ptag=ptag,
            st=STEP1, ds_tag=ds_tag, cl_tag=cl_tag,
            pts=pt_list  #use the same x pts for both series
        )

    if ranges is None:
        range_text = "fullr"
    else:
        start = ranges[0]
        lim = ranges[0]
        range_text = f'{start}-{lim}'

    ds_txt = ds_tag
    if REDK in get_ds:
        ds_txt = f'{ds_txt}_{get_ds[REDK]}'

    make_plot(
        title=title, fn=get_plot_fn(f'{cl_tag}_{ds_txt}_{st}_{range_text}'),
        data=data,
        x_label='k', y_label='Silh Scor', extra_data=xdata
    )
    return results

SCOREK = 'score_func'
KM_CLUSTERING = {
    GETK: run_km_clustering,
    SCOREK: kmeans_score
}
GMM_CLUSTERING = {
    GETK: run_gmm_clustering,
    SCOREK: gmm_score
}

CLUSTERING_TAGS = [KMEANSK, GMMK]

clustering_info = {
    KMEANSK: KM_CLUSTERING,
    GMMK: GMM_CLUSTERING
}

def get_step1_range(ds_tag=None, cluster_tag=None):

    try:
        if STEP_RES in max_cluster_scores[STEP1][ds_tag][cluster_tag]:
            max_idx = max_cluster_scores[STEP1][ds_tag][cluster_tag][STEP_RES]['max_idx']
            step1_series = get_np_array(max_cluster_scores, ds_tag, cluster_tag, st=STEP1)
            max_x = step1_series[max_idx, 1]
        else:
            max_x = max_cluster_scores[STEP1][ds_tag][cluster_tag]['max']
            max_x = int(max_x)

    except Exception as ex:
        hook = True
        raise

    #max_y = step1_series[max_idx:0]

    start = 2
    if max_x < 20:
        last_pt = 20
    elif max_x < 40:
        last_pt = int(2.0 * max_x)
    else:
        last_pt = int(1.2 * max_x)

    pts = last_pt - start + 1
    if last_pt - start <= 40:
        ret_rng = list(range(2, last_pt + 1))
    elif last_pt - start <= 60:
        ret_rng = list(range(2, last_pt + 1, 2))
    else:
        ret_rng = make_x_points(starts=[start], limits=[last_pt], points=[20])

    ret_rng = list(set(ret_rng).union({max_x}))
    ret_rng.sort()
    assert(max_x in ret_rng)
    max_idx = ret_rng.index(max_x)

    return ret_rng, max_idx

def get_data_stub(d=None, y=None):
    return d, y

def run_reduction(
    get_ds=None, cluster_info=None, dr_func=None,
    plot_func=None
):
    ds_tag = get_ds[TAGK]
    dr_tag = dr_func[TAGK]
    ds_x_reduced = run_reduction_2(
        reduce_func=dr_func, plot_func=plot_func, get_ds=get_ds,
        fn=get_plot_fn(f'Step_2_PCA_{ds_tag}')
    )

    save_np(
        ds_x_reduced, ds_tag=ds_tag, cl_tag="na",
        extra=f'{ds_tag}_reduced_by_{dr_tag}'
    )

    # cluster plot (baseline) reduced  dataset against labels
    cluster_plot(
        get_ds=get_ds, dr_tag=dr_tag, cl_tag='baseline',
        fn=None
    )

    get_reduced = partial(get_data_stub, d=ds_x_reduced)
    GetReduced = {
        TAGK: get_ds[TAGK], GETK: get_reduced, REDK: dr_func[TAGK]
    }
    cl_tag = cluster_info[TAGK]
    cluster_func = cluster_info[GETK]
    ranges, s1_max = get_step1_range(ds_tag=ds_tag, cluster_tag=cl_tag)
    title = f'{ds_tag} {cl_tag} Silhouette Scores\nred. by {dr_tag} dim={ds_x_reduced.shape[1]}, s1 xmax = {s1_max}'
    cl_results = cluster_func(title=title, ranges=ranges, get_ds=GetReduced)
    max_idx = np.argmax(cl_results[:,0])
    max_x = cl_results[max_idx, 1]
    score_func = cluster_info[SCOREK]
    score_func(int(max_x), get_ds=GetReduced, plot=True, cl_tag=cl_tag) #
    #gmm_score

def set_np_array(info, k1, k2, a, custom_tag=None, st=None, sfn=None): # ds = k1, k2 = CL METHOD
    import os
    if st is None:
        st = cur_step()

    if k1 not in info[st]:
        info[st][k1] = {}

    if k2 not in info[st][k1]:
        info[st][k1][k2] = {}

    if STEP_RES not in info[st][k1][k2]:
        info[st][k1][k2][STEP_RES] = {}

    if custom_tag is not None:
        if custom_tag not in info[st][k1][k2]:
            info[st][k1][k2][custom_tag] = {}

        cur_d = info[st][k1][k2][custom_tag]
    else:
        cur_d = info[st][k1][k2][STEP_RES]

    if 'a' not in cur_d:
        old_a = None
    else:
        old_a = np.array(cur_d['a'])

    if old_a is None or len(old_a) != len(a):
        ba = False
    else:
        ba = old_a == a
        ba = ba.all()

    update = False
    if 'serialized' in cur_d:
        a_fn = cur_d['serialized']
    else:
        update = True
        if sfn is not None:
            a_fn = sfn
        else:
            a_fn = get_ser_fn(tag=f'saved_npa_{st}_{k1}_{k2}')

    if not ba:
        update = True

    if update:
        max_idx = np.argmax(a[:,0])
        cur_d['max_idx']= int(max_idx)
        cur_d['a'] = a.tolist()
        np.save(a_fn, a)
        if 'png' in a_fn:
            hook = True
        cur_d['serialized'] = a_fn

    save_cluster_scores(info)
    return a

def get_np_array(info, k1, k2, custom_tag=0, st=None):
    if st is None:
        st = cur_step()

    if k1 not in info[st] or k2 not in info[st][k1]:
        hook = True

    assert (k1 in info[st] and k2 in info[st][k1])
    if not custom_tag:
        # assert(STEP_RES in info[st][k1][k2])
        cur_k = STEP_RES
    else:
        #assert (custom_tag in info[st][k1][k2])
        cur_k = custom_tag

    if cur_k not in info[st][k1][k2]:
        return None

    cur_d = info[st][k1][k2][cur_k]
    a = None
    if 'serialized' in cur_d:
        a_fn = cur_d["serialized"]
        if 'png' in a_fn:
            hook = True
        a = np.load(a_fn)
        #assert ((np.array(cur_d['a']) == a).all())
        cur_d['a'] = a.tolist()
    else:
        print(f'serialized key not found in target node: {cur_d}')

    return a

def save_cluster_scores(scores, fn = './max_cluster_scores.json'):
    import json

    with open(fn, 'w') as jfp:
        json.dump(scores, jfp)

def read_cluster_scores(scores, fn='./max_cluster_scores.json'):
    import json
    with open(fn, 'r') as jfp:
        scores_pickled = json.load(jfp)

    src = scores_pickled
    targ = scores
    for k0 in [STEP1, STEP2]:
        keys = list(src[k0])
        for k1 in keys:
            for k2 in src[k0][k1]:
                cur_d = src[k0][k1][k2]
                assert(type(cur_d) is dict)
                if type(cur_d) is dict:
                    for cur_k in cur_d:
                        if type(cur_d[cur_k]) is dict and 'serialized' in cur_d[cur_k]:
                            a_fn = cur_d[STEP_RES]['serialized']
                            if STEP_RES == cur_k:
                                read_a = get_np_array(src, k1, k2, st=k0)
                            else:
                                read_a = get_np_array(src, k1, k2, custom_tag=cur_k, st=k0)

                            cur_d[cur_k]['safe_a'] = read_a.tolist()

                cur_targ = targ[k0][k1][k2]
                for k3 in cur_d:
                    new = cur_d[k3]
                    if k3 in cur_targ:
                        old = targ[k0][k1][k2][k3]
                    else:
                        old = None
                        cur_targ[k3] = None

                    if old != new:
                        print(f'updating {k0}-{k1}-{k2}-{k3}: {old} => {new}')
                        cur_targ[k3] = new
                    else:
                        print(f'{k0}-{k1}-{k2}-{k3}: original == new')

def set_serial_store(dump_dir, match_str='saved_npa', fn='./max_cluster_scores.json'):
    import json
    import os
    flist = os.listdir(dump_dir)
    with open(fn, 'r') as jfp:
        scores_pickled = json.load(jfp)
        found = [x for x in flist if match_str in x]

        st = ds_tag = None
        for f in found:
            toks = f.split('_')
            if '@' in toks[0]:
                continue
            else:
                hook = True
                for i, tok in enumerate(toks):
                    if i == 2 and tok in [STEP1, STEP2]:
                        st = tok
                    if i == 3 and tok in [DS_1, DS_2]:
                        ds_tag = tok
                    if i == 4 and tok in [KMEANSK, GMMK]:
                        cl_tag = tok

            fp = os.path.join(dump_dir, f)
            a = np.load(fp)
            set_np_array(scores_pickled, ds_tag, cl_tag, a, st=st, sfn=fp)

    save_cluster_scores(scores_pickled)

def main(args):
    cur_arg = 'update_stored'
    if cur_arg in args:
        pos = args.index(cur_arg)
        if pos <= len(args) - 2:
            set_serial_store(args[pos + 1])

    #def cluster_plot(get_ds=None, threshold=6, labels=None, dr_tag=None, cl_tag=None, fn=None):
    if 'cluster_plots' in args:
        cur_step(STEP3)
        for ds in [DS_1, DS_2]:
            ds_info = datasets_info[ds]
            ds_info[TAGK] = ds
            ds_tag= ds_info[TAGK]
            print('Dataset native cluster plot')
            cluster_plot(get_ds=ds_info, fn=get_plot_fn(f'{ds}_against_labels'))
            print('Dataset kmeans cluster plot')
            k = max_cluster_scores[STEP1][ds][KMEANSK]['max']
            kmeans_score(k, get_ds=ds_info, plot=True)
            print('Dataset gmm cluster plot')
            k = max_cluster_scores[STEP1][ds][GMMK]['max']
            gmm_score(k, get_ds=ds_info, plot=True)

            for dr_tag in DR_TAGS:
                if dr_tag in args:
                    print(f"Cluster plots on data reduced by {dr_tag}")
                    dr_info = dim_reduce_info[dr_tag]
                    dr_info[TAGK] = dr_tag

                    X, y = ds_info[GETK]()
                    if dr_tag == ICAK:
                        k = max_cluster_scores[STEP1][ds][KMEANSK]['max']
                        estimator, x_red = dr_info[DRFUNCK](X, y)
                    else:
                        estimator, x_red = dr_info[DRFUNCK](X, y)

                    save_np(
                        x_red, ds_tag=ds_tag, cl_tag="na",
                        extra=f'ds_reduced_by_{dr_tag}'
                    )

                    get_reduced = partial(get_data_stub, d=x_red, y=y)
                    GetReduced = {
                        TAGK: ds_info[TAGK], GETK: get_reduced, REDK: dr_info[TAGK]
                    }

                    cluster_plot(
                        get_ds=GetReduced, fn=get_plot_fn(f'{ds}_{dr_tag}_against_labels')
                    )
                    k = max_cluster_scores[STEP1][ds][KMEANSK]['max']
                    kmeans_score(k, get_ds=GetReduced, plot=True)
                    k = max_cluster_scores[STEP1][ds][GMMK]['max']
                    gmm_score(k, get_ds=GetReduced, plot=True)

    if 'test' in args:
        TEST_MODE(val=True)

    if 'reload' in args:
        read_cluster_scores(max_cluster_scores)

    if 'INIT' in args:
        get_ds = datasets_info[DS_2]
        get_ds[TAGK] = GMMK

        ds = get_ds[GETK]
        X, y = ds()
        lazy_search(X, y_func=gmm_score)

    if STEP1 in args or 'populate' in args:
        cur_step(st=STEP1)
        for cl_tag in CLUSTERING_TAGS:
            if cl_tag in args:
                Logs.set_log_file(f"{cl_tag}_Step_1")
                cluster_tag = cl_tag
                for arg in [DS_1, DS_2]:
                    if arg in args:
                        print(f"{cluster_tag} {arg} clustering")

                        get_ds = datasets_info[arg]
                        get_ds[TAGK] = arg

                        cl_info = clustering_info[cluster_tag]
                        cl_info[TAGK] = cluster_tag

                        if 'populate' in args:
                            X, y = get_ds[GETK]()
                            lim = X.shape[0]
                            start=2

                            rng_list=[]
                            rng_list.append(list(range(2,20)))
                            rng_list.append(list(range(2, lim, 500)))
                            rng_list.append(list(range(2, lim, 250)))
                            rng_list.append(list(range(2, lim, 50)))

                            for i, r in enumerate(rng_list):
                                if i == 0:
                                    results = cl_info[GETK](ranges=r, get_ds=get_ds)
                                    set_np_array(max_cluster_scores, arg, cluster_tag, results, st=STEP1)
                                    set_np_array(
                                        max_cluster_scores, arg, cluster_tag,
                                        results, st=STEP1, custom_tag=CL_SCORING_RES
                                    )
                                else:
                                    results = cl_info[GETK](ranges=r, get_ds=get_ds, update=True)
                                    set_np_array(max_cluster_scores, arg, cluster_tag, results, st=STEP1)
                                    set_np_array(
                                        max_cluster_scores, arg, cluster_tag,
                                        results, st=STEP1, custom_tag=CL_SCORING_RES
                                    )
                        else:
                            results = cl_info[GETK](ranges=None, get_ds=get_ds)

                        set_np_array(max_cluster_scores, arg, cluster_tag, results, st=STEP1)

            save_cluster_scores(max_cluster_scores)

    if STEP2 in args:
        cur_step(st=STEP2)
        scores_recorded = True
        use_original = True
        for k1 in max_cluster_scores[STEP1]:
            if use_original:
                pass
                scores_recorded = True
            else:
                k2_keys = list(max_cluster_scores[STEP1][k1])
                filtered = [x for x in k2_keys if 'serialized' in x]
                if not filtered:
                    scores_recorded = False
                    break

        if not scores_recorded:
            read_cluster_scores(max_cluster_scores)

        for dr_tag in DR_TAGS:
            Logs.set_log_file(f"Step_2_{dr_tag}")
            if dr_tag in args:
                dr_func = dim_reduce_info[dr_tag]
                dr_func[TAGK] = dr_tag

                plot_func = dr_func[EVALPLOTK]

                for cluster_tag in [KMEANSK, GMMK]:
                    if cluster_tag in args:
                        cluster_info = clustering_info[cluster_tag]
                        cluster_info[TAGK] = cluster_tag

                        for ds_tag in [DS_1, DS_2]:
                            if ds_tag in args:
                                ds_info = datasets_info[ds_tag]
                                ds_info[TAGK] = ds_tag
                                print(f'STEP2 {dr_tag} {cluster_tag} {ds_tag}')
                                run_reduction(
                                    get_ds=ds_info, cluster_info=cluster_info, dr_func=dr_func,
                                    plot_func=plot_func
                                )

def lazy_search(s, y_func=None, stop=10):
    def valid(x):
        return x >= 0 and x <= len(s) - 1

    def hillclimb(x, y_func, s, step=1, forward=None):
        if not s_visited[x]:
            s_map[x, 1] = s_pts[x]
            print(f'calculating y at {x}')
            s_map[x, 0] = y_func(s, s_pts[x])
            cur_val = s_map[x, 0]
            s_visited[x] = True
        else:
            cur_val = s_map[x, 0]

        if forward is None:
            n_stops = [x, x]
            step = 1
            neighbors = [x - step, x + step]
            for i, v in enumerate(neighbors):
                n_stops[i] = None
                if valid(v):
                    forward = True if i == 1 else False
                    ret = hillclimb(v, y_func, s, step=step * 2, forward=forward)
                    assert(type(ret) is not list)
                    n_stops[i] = ret
            return n_stops[0], n_stops[1]
        else:
            prev_step = step/2
            prev_pos = int(x + prev_step) if not forward else int(x - prev_step)
            prev_val = s_vals[prev_pos]
            assert(s_visited[prev_pos])

            neighbor = x + step if forward else x - step

            if valid(neighbor):
                stop = x
                if max_mode:
                    if cur_val >= prev_val :
                        stop = hillclimb(neighbor, y_func, s, step=step * 2, forward=forward)
                else:
                    if cur_val < prev_val:
                        stop = hillclimb(neighbor, y_func, s, step=step * 2, forward=forward)

                return stop
            else:
                return x

    def adjust(b, f, store, pos):
        tr = store[pos]
        if bs != fs:
            if not bs:
                pass
            else:
                tr.append(bs)
            if not fs:
                pass
            else:
                tr.append(fs)

            tr = list(set(tr))
            tr.sort()
            store[pos] = tr

    trails = [[], []]

    s_map = np.zeros((s.shape[0], 2))
    s_vals = s_map[:, 0] # reversed x, y order
    for i in range(len(s)):
        s_map[i, 1] = i + 2
    s_pts = [int(x) for x in s_map[:, 1]]
    s_visited = np.repeat(False, len(s))

    starts = {0: 1, 1: len(s.shape)//2, 2: len(s) - 2}
    max_mode = True
    step = 1
    for cycle in range(stop+1):
        print(f'cycle {cycle}')
        for i, a in enumerate(trails):
            if not a:
                a.append(starts[i])

            cur_start = a[0]
            cur_end = a[-1]
            int_len = len(a)
            print(f'starting search at {cur_start}')
            bs, fs = hillclimb(cur_start, y_func, s)
            adjust(bs, fs, trails, i)

            if  int_len > 1:
                print(f'starting search at {cur_end}')
                bs, fs = hillclimb(cur_end, y_func, s)
                adjust(bs, fs, trails, i)

                print(f'endpoints = {bs, fs}')

        max_mode = not max_mode
        make_plot(data={'lazy':s_map}, title=f"Lazy search, cycle {cycle}",
                  fn=get_plot_fn(f"Lz_srch_cy_{cycle}")
        )

if __name__ == '__main__':
    import sys
    import datetime

    main(sys.argv[1:])

    time_now = datetime.datetime.now().strftime("%m/%d %H:%M")
    print(f"Finished execution at {time_now}")
    log("Finished execution")

# See PyCharm help at https://www.jetbrains.com/help/pycharm/

{'datetime': 'datetime', 'os': 'os'}.log_time_tag.split

{'datetime': 'datetime', 'os': 'os'}.log_dir

{'datetime': 'datetime', 'os': 'os'}.get_log_fn

{'datetime': 'datetime', 'os': 'os'}.set_log_file


import display
from colorama import Back
import numpy as np

# Y =  1
# R = -1

class Connect4:
	plyTurn = -1
	win = 0
	plyStart = 0
	board = np.zeros((7, 6), dtype="int8")

	visual_board = None

	def __init__(self, show_board=True):
		self.board = np.zeros((7, 6), dtype="int8")
		# self.board = np.matrix([
		#     [ 0,  0,  0,  0, -1, -1],
		#     [ 0,  0,  1,  1,  1,  1],
		#     [ 0,  1, -1,  1,  1, -1],
		#     [ 0,  0,  0, -1, -1, -1],
		#     [ 0,  0, -1, -1,  1,  1],
		#     [ 0,  0,  0,  0,  1,  1],
		#     [ 0,  0,  0,  0, -1, -1],
		# ])
		self.win = 0
		self.plyStart = -1 if np.random.rand() > 0.5 else 1
		self.plyTurn = self.plyStart
		if show_board:
			self.visual_board = display.VisualBoard()
			self.visual_board.update_all(self.board)

	def drop_coin(self, x, color):
		for i in range(6):
			if self.board[x, i + 1] != 0 or i == 4:
				y = 5 if self.board[x, i + 1] == 0 else i
				self.board[x, y] = color
				if self.visual_board is not None:
					# self.visualBoard.updateCell(x , y, color)
					self.visual_board.update_all(self.board)
				break

	def play(self, play_column):
		if self.board[play_column, 0] != 0 and self.win == 0:
			self.win = self.plyTurn * -1
			return self.observation(), -10, True
		self.drop_coin(play_column, self.plyTurn)
		win = win_play(self.board, play_column)
		if win:
			self.win = self.plyTurn
		else:
			self.plyTurn *= -1
		return self.observation(), 10 if win and self.plyTurn != self.plyStart else 3 if win else 0, win

	def close_game(self):
		if self.visual_board is not None:
			self.visual_board.close()

	def observation(self):
		return self.board.copy().reshape((42, 1))

	def print(self):
		print()
		print_board(self.board)

def win_play(board, play_col):
	x = play_col
	y = 0
	for _y in range(6):
		if board[x, _y] != 0:
			y = _y
			break
	if board[x, y] != 0:
		dirs = [[1, 1], [0, 1], [-1, 1], [1, 0]]
		for curDir in dirs:
			align = 0
			for i in range(-3, 4):
				_x = x + (curDir[0] * i)
				_y = y + (curDir[1] * i)
				if _x < 0 or _y < 0 or _x > 6 or _y > 5:
					continue
				if board[_x, _y] == board[x, y]:
					align += 1
				else:
					align = 0
				if align == 4:
					return True
	return False

def print_board(board):
	for y in range(6):
		for i in range(2):
			pr = ""
			for x in range(7):
				color = Back.YELLOW if board[x, y] == 1 else Back.RED if board[x, y] == -1 else Back.WHITE
				pr = pr + f"{color}    {Back.RESET} "
			print(pr)
		print()


import json
from unittest import skip
from playwright.sync_api import sync_playwright
import re

# Create a dictionary of course requirements for each program major
def create_majors(major_dict):
    final_majors = {}
    final_majors["programs"] = {}

    # Iterate through program majors
    for major in major_dict["entry"]:
        try:
            table_text = major["table_obj"]

            # Find all courses in this program major
            courses = re.findall(r'[A-Z]{3,4}\*[0-9]{4}', table_text)

            # Parse out extra credit requirements
            total_extra_requirements = {}
            bad_words = ["*", "Select", "Semester", "work", "Exchange", "Option", "Stream", "following", "semesters", "admission"]
            for i in table_text.split('\n'):
                # Smallest length I found was around 14, this len check is used to cull non elective text
                if len(i) > 13:

                    # Skip the line if it contains information signifying its not a credit requirement
                    skip_line = False
                    for j in bad_words:
                        if j in i:
                            skip_line = True
                            break

                    if not skip_line:
                        # Strip whitespace, split into credit number and requirement
                        extra_requirements = i.strip()

                        # Remove duplicate credit value from the start, then extract credit weight of electives from the end of the string
                        extra_requirements = re.sub(r'^\d+\.\d+\s*', r'', extra_requirements)
                        weight = extra_requirements[len(extra_requirements) - 4:]

                        # Remove duplicated credit value on the end, and restrip whitespace
                        extra_requirements = re.sub(r'\d+\.\d+$', r'', extra_requirements)
                        extra_requirements = extra_requirements.strip()

                        # Case where text is like up to X total credits in the rest of the semester
                        if "total credits in this semester" in extra_requirements or "to a maximum of" in extra_requirements:
                            extra_requirements = "Electives or Restricted Electives"

                        # Capitalize first letter
                        if len(extra_requirements) > 0 and extra_requirements[0].isalpha:
                            extra_requirements = extra_requirements[0].upper() + extra_requirements[1:]

                        total_extra_requirements[extra_requirements] = weight

            # Build program major
            final_cur_major = {}
            final_cur_major["program_name"] = major["program_name"]
            final_cur_major["program_link"] = major["program_link"]
            final_cur_major["courses"] = sorted(list(set(courses)))
            final_cur_major["extra_requirements"] = total_extra_requirements

            code = re.sub(r'.*\((.*)\)$', r'\1',  major["program_name"])
            final_majors["programs"][code] = final_cur_major
        except Exception as e:
            print(e)
            pass
    return final_majors

def main():
    with sync_playwright() as scrape:
        browser = scrape.chromium.launch()
        page = browser.new_page()

        # Leaving 30 second default timeout while testing (i.e. not timeout = 0)
        page.goto('https://calendar.uoguelph.ca/undergraduate-calendar/degree-programs/')

        # Extracting all degree links and storing in degree_links
        links = page.query_selector('[id="textcontainer"]').query_selector_all("a")
        degree_links = []
        for link in links:
            degree_links.append(link.get_attribute("href"))

        # All links extracted will be appended to this
        root_url = 'https://calendar.uoguelph.ca/'

        # Loop through all the degrees, extract the major links, and store in major_links
        major_links = []

        for link in degree_links:
            page.goto(root_url + link)
            # print("Checking Degree Link: " + root_url + link)

            # Checking if page has a requirements tab
            try:
                requirements_tab = page.query_selector('[id="requirementstextcontainer"]')
                if(requirements_tab):
                    major_links.append(root_url + link)
            except:
                pass

            # Checking if page has a programs tab
            try:
                programs_tab = page.query_selector('[id="programstextcontainer"]')
                if(programs_tab):
                    temp_links = programs_tab.query_selector_all("a")
                    for n in temp_links:
                        major_links.append(root_url + n.get_attribute("href"))
            except:
                pass

            # Checking if page has a schedule of studies tab
            try:
                sched_of_stud = page.query_selector('[id="scheduleofstudiestextcontainer"]')
                if(sched_of_stud):
                    major_links.append(root_url + link)
            except:
                pass

        # Getting the list of pages that have majors
        # NOTE: major_links_table_dict is an array of programs, where each has attribute "program_name", "program_link", and "table_obj"
        major_links_table_dict = {}
        major_links_table_dict["entry"] = []
        for link in major_links:
            print("Visiting: " + link)
            page.goto(link)

            try:
                major_links_table_entry_dict = {}
                # Checking if the current tab is a requirements, programs, or schedule of studies tab
                current_tab = page.query_selector('[id="requirementstextcontainer"]')
                if(current_tab is None):
                    current_tab = page.query_selector('[id="programstextcontainer"]')
                if(current_tab is None):
                    current_tab = page.query_selector('[id="scheduleofstudiestextcontainer"]')
                if(current_tab is None):
                    continue

                # Checking if the entry contains a major or schedule of studies
                check_minor = current_tab.query_selector_all('h2,h3')
                for cur_degree in check_minor:
                    if("major" in cur_degree.inner_text().lower() or "schedule of studies" in cur_degree.inner_text().lower()):

                        # Getting the name of the major and link
                        major_title = page.query_selector('[class="page-title"]')
                        major_links_table_entry_dict["program_name"] = major_title.text_content()
                        major_links_table_entry_dict["program_link"] = link

                # Check to see if the page contains a Major header and a Schedule of Studies header
                check_headers = current_tab.query_selector_all('h2,h3')
                found_major = False
                found_sos = False
                for header in check_headers:
                    if "major" in header.inner_text().lower():
                        found_major = True
                    elif "schedule of studies" in header.inner_text().lower():
                        found_sos = True
                skip_sos = (found_major and found_sos)

                # Looking for all h2, h3, and tables since a table will almost always have a major prior to it
                current_tab_items = current_tab.query_selector_all('h2,h3,table');
                for i in range(len(current_tab_items)):
                    cur_item = current_tab_items[i].inner_text().lower()
                    cur_item_html = current_tab_items[i].inner_html().lower()

                    # Checking if an element that contains major is found, or schedule of studies (only if major isn't found on the page)
                    # After that check if its a table or not
                    if ("major" in cur_item or ("schedule of studies" in cur_item and not skip_sos)) and "thead" not in cur_item_html:
                        table_obj = current_tab_items[i+1]
                        major_links_table_entry_dict["table_obj"] = table_obj.inner_text()

                        # Check if there are any courses present in the inner text
                        if "*" in table_obj.text_content():
                            major_links_table_dict["entry"].append(major_links_table_entry_dict)
                        break
            except:
                pass

        # Dumping the parsed JSON data
        with open("src/scraper/data/scraped_majors_data.json", "w") as f:
            json.dump(create_majors(major_links_table_dict), f, sort_keys=True, indent=4)

        browser.close()
if __name__ == '__main__':
    main()


import random
import numpy as np
import pandas as pd

import tensorflow as tf

from tqdm import tqdm

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MultiLabelBinarizer

# Split dataset into train/test/validation
def split(x, test_size=0.2, val_size=0.0, seed=0):
    if val_size + test_size >= 1:
        return None
    x_train, x_test = train_test_split(
        x, test_size=test_size + val_size, random_state=seed
    )
    x_val = None
    if val_size > 0:
        x_test, x_val = train_test_split(
            x_test,
            test_size=val_size / (test_size + val_size),
            random_state=seed,
        )
    return x_train, x_val, x_test

# Loads an image given its path
def load_image_from_path(path):
    image = tf.io.read_file(path)
    image = tf.io.decode_jpeg(image, channels=3, dct_method="INTEGER_ACCURATE")

    # may need resizing
    #image = tf.image.resize(image, image_shape[:2])
    image = tf.cast(image, dtype=tf.float16)
    image = image / 255.0
    return image

# Loads all features of the given CLIP dataset given the image folder, captions/concepts files and a concept encoder
def load_features(image_folder, captions_file, concepts_file, concept_encoder, filter_percent=1):
    features = []

    # Import CSVs
    csv_caption_dataset = tf.data.experimental.CsvDataset(
        captions_file,
        field_delim='\t',
        record_defaults=[tf.string, tf.string],
        header=True,
        select_cols=[0, 1]
    )
    csv_concept_dataset = tf.data.experimental.CsvDataset(
        concepts_file,
        field_delim='\t',
        record_defaults=[tf.string, tf.string],
        header=True,
        select_cols=[0, 1]
    )

    # We make the assumption that CSV files contain the same key values (image names)
    # following the same ordering

    # Extract features from dataset
    print("Extracting features from CSV file(s)")
    for caption_el, concept_el in tqdm(zip(csv_caption_dataset, csv_concept_dataset)):
        filename_cap, caption = caption_el
        filename_con , concepts = concept_el

        # Sanity check
        assert filename_cap == filename_con

        image_path = image_folder + "/" + filename_cap + ".jpg"

        features.append({
            'caption': caption,
            'image path': image_path,
            'concepts': concept_encoder.transform([concepts.numpy().decode("utf-8").split(";")]),
        })

    # Filter elements
    if filter_percent != 1:
        n_features = int(len(features) * filter_percent)
        features = random.sample(features, n_features)

    return features

# Manual preprocessing of features
def preprocess_features(features, concept_encoder, filter_percent=1):
    print("Preprocessing features")

    # Filter elements
    if filter_percent != 1:
        n_features = int(len(features) * filter_percent)
        features = random.sample(features, n_features)

    return {
        'image paths': tf.convert_to_tensor([x["image path"] for x in tqdm(features)], dtype=tf.string),
        'captions': tf.convert_to_tensor([x["caption"] for x in tqdm(features)], dtype=tf.string),
        'concepts': tf.convert_to_tensor(np.vstack([concept_encoder.transform(x["concepts"]).flatten() for x in tqdm(features)]), dtype=tf.bool),
        # 'images': tf.convert_to_tensor([load_image(x["image path"]) for x in tqdm(features)], dtype=tf.float16),
    }

# Creates a tensorflow dataset given a CLIP dataset and other parameters
def create_dataset(
        features,                       # Dataset features to include in the dataset
        input_features_types,           # Dictionary containing the structure of the dataset features
        feature_shapes,                 # Dictionary containing the shapes of the dataset features
        x_features, y_features=None,    # Lists containing the names for the desired x and y features of the dataset
        x_dict=True, y_dict=True,       # Set to True in order to store x and y features into a dictionary format inside the dataset
        load_images=True,               # Set to True in order to add images to the dataset
        shuffle_buffer_size=1024,       # Shuffle buffer size for the dataset, set to 1 for no shuffling
        batch_size=10,                  # Batch size for the dataset
        cached=False                    # Set to True to enable caching
):
    # Generate dataset following initial input feature types
    dataset = tf.data.Dataset.from_generator(
        lambda: features, { x: input_features_types[x] for x in input_features_types }
    )

    # Preprocessing internal functions
    def setshape(e):
        for (k, v) in feature_shapes.items():
            if k in e:
                e[k].set_shape(v)
        return e
    def add_images(e):
        # Maybe parametrize
        img_from = "image path"
        img_to = "image"
        new_features = list(input_features_types.keys()) + [img_to]
        return {f:e[f] if f != img_to else load_image_from_path(e[img_from]) for f in new_features}
    def split_xy(e):
        e_x = {xf:tf.squeeze(e[xf]) for xf in x_features} if x_dict else tf.squeeze([e[xf] for xf in x_features])
        if y_features:
            e_y = {yf:tf.squeeze(e[yf]) for yf in y_features} if y_dict else tf.squeeze([e[yf] for yf in y_features])
            return (e_x, e_y)
        return e_x

    # Preprocess
    if load_images:
        dataset = dataset.map(add_images)
    dataset = dataset.map(setshape)
    dataset = dataset.map(split_xy)

    # Compile dataset
    if cached:
        dataset = dataset.cache()
    dataset = dataset.shuffle(shuffle_buffer_size).batch(batch_size).prefetch(tf.data.AUTOTUNE)

    return dataset

# Top level function to import datasets
# Returns (concept_list, concept_onehot_encoder), [(train_dataset, val_dataset, test_dataset)], (train_ds_size, val_ds_size, test_ds_size)
def import_datasets(
    image_folder,                   # Folder containing images
    captions_file,                  # File path for the captions file
    concepts_file,                  # File path for the concepts file (concept id <-> concept text)
    concepts_det_file,              # File path for the concepts detection train file (element id <-> concept id)
    input_features_types,           # Dictionary containing the structure of the dataset features
    feature_shapes,                 # Dictionary containing the shapes of the dataset features
    dataset_parameters,             # List containing dictionaries specifying dataset parameters for each desired output dataset collection, parameters are:
                                    # x_features, y_features:None,      Lists containing the names for the desired x and y features of the dataset
                                    # x_dict:True, y_dict:True,         Set to True in order to store x and y features into a dictionary format inside the dataset
                                    # load_images:True,                 Set to True in order to add images to the dataset
                                    # shuffle_buffer_size:1024,         Shuffle buffer size for the dataset, set to 1 for no shuffling
                                    # batch_size:10,                    Batch size for the dataset
                                    # cached:False,                     Set to True to enable caching
    filter_percent=1,               # Percentage of the dataset to retrieve
    test_size=0.2,                  # Percentage of test dataset
    val_size=0.0,                   # Percentage of validation dataset
    seed=0,                         # Random seed
):
    # Loads concepts
    concepts = pd.read_csv(concepts_file, sep='\t')
    concept_list = concepts.set_index('concept')['concept_name'].to_dict()
    # Concept one-hot encoder
    concepts_onehot = MultiLabelBinarizer(classes=list(concept_list.keys()))
    _ = concepts_onehot.fit([list(concept_list.keys())])

    # Load dataset features from csv files, split them and preprocess them
    features = load_features(image_folder, captions_file, concepts_det_file, concepts_onehot, filter_percent=filter_percent)
    feat_train, feat_val, feat_test = split(features, test_size=test_size, val_size=val_size, seed=seed)

    # Calculate dataset sizes
    train_ds_size = len(feat_train) if feat_train else 0
    val_ds_size = len(feat_val) if feat_val else 0
    test_ds_size = len(feat_test) if feat_test else 0

    datasets = []
    # Created datasets
    for parameters in dataset_parameters:
        train_dataset = create_dataset(feat_train, input_features_types, feature_shapes, **parameters) if feat_train else None
        val_dataset = create_dataset(feat_val, input_features_types, feature_shapes, **parameters) if feat_val else None
        test_dataset = create_dataset(feat_test, input_features_types, feature_shapes, **parameters) if feat_test else None
        datasets.append((train_dataset, val_dataset, test_dataset))

    return (concept_list, concepts_onehot), datasets, (train_ds_size, val_ds_size, test_ds_size)


import pandas as pd

from nltk.tokenize import word_tokenize
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection

if __name__ == '__main__':
    df = pd.read_csv('imdb.csv')

    df['kfold'] = -1

    df = df.sample(frac=1).reset_index(drop=True)

    y = df.label.values

    kf = model_selection.StratifiedKFold(n_splits=5)

    for f, (t_, v_) in enumerate(kf.split(X=df,y=y)):
        df.loc[v_, 'kfold'] = f

    for fold_ in range(5):

        test_df = df[df.kfold == fold_].reset_index(drop=True)
        train_df = df[df.kfold != fold_].reset_index(drop=True)

        tfidf_vec = TfidfVectorizer(tokenizer=word_tokenize, token_pattern=None, ngram_range=(2,2))
        tfidf_vec.fit(train_df.text)

        Xtrain = tfidf_vec.transform(train_df.text)
        Xtest = tfidf_vec.transform(test_df.text)

        model = LogisticRegression()
        model.fit(Xtrain, train_df.label)

        preds = model.predict(Xtest)
        accuracy = metrics.accuracy_score(test_df.label, preds)

        print(f"kfold : {fold_}")
        print(f"accuracy : {accuracy}")
        print("")


'''
' This file contains functions that will extract the HOG/HOF/MBH features from input frames
'
' The code is adapted from the Matlab code used in the papers:
' - J.R.R. Uijlings, I.C. Duta, E. Sangineto, and N. Sebe
' "Video Classification with Densely Extracted HOG/HOF/MBH Features: An Evaluation of the Accuracy/Computational Efficiency Trade-off"
' In International Journal of Multimedia Information Retrieval (IJMIR), 2015.
' - I.C. Duta, J.R.R. Uijlings, T.A. Nguyen, K. Aizawa, A.G. Hauptmann, B. Ionescu, N. Sebe
' "Histograms of Motion Gradients for Real-time Video Classification"
' In International Workshop on Content-based Multimedia Indexing (CBMI), 2016
'
' As the code is to be used on live videos on a per frame basis, it is slightly altered
' And does not take into account the videos as whole blocks
'''

import numpy as np
from numpy.lib.stride_tricks import as_strided
from numpy import einsum
import cv2 as cv
import queue
from scipy.ndimage.filters import convolve as filter2

'''
' This has been hard coded to provide the marix required for the dimensions we are using to save time
'
' Otherwise, it translates a diagonal matrix to a rectangular one, such that all the rows still add to one
' and the columns add to 1/shape[1]
'''
def diag_matrix_linear():
	matrix = np.load('diag_matrix.npy')

	return matrix

'''
' Strides the data, making block as in the paper
' Input:
'	data - data to make into a block
'	A - rectangular diagonal
'	B - rectangular diagonal
' Output:
'	block - the data strided into blocks
'
'''
def make_block(data, A, B):
	block = einsum('ab,bef,eh->fah',A,data,B)

	block = block.T.reshape(64,8)

	inds = np.arange(64).reshape(8,8)
	inds = as_strided(inds,(2,2,7,7),(64,8,8,64))
	inds = inds.transpose(0,1,3,2).reshape(-1)

	block = block[inds]

	block = block.T.reshape(32,-1).T.reshape(-1)

	return block

'''
' Calculate haar gradients of an image
' Input:
' 	im - image
' Output:
'	gr - haar gradients
'''
def haar_gradients(im):
	ogIm = np.zeros((im.shape[0],im.shape[1],8))

	rowIm = im[:-2, 1:-1] - im[2:,1:-1]
	colIm = im[1:-1,:-2] - im[1:-1,2:]

	magIm = np.sqrt(rowIm * rowIm + colIm * colIm)
	angleIm = np.arctan2(rowIm, colIm)

	binReal = (angleIm) * (8 / (2 * np.pi))
	binLow = np.floor(binReal)
	weightHigh = binReal - binLow
	weightLow = 1 - weightHigh
	binLow = (binLow % 8).astype(int)
	binHigh = (binLow + 1).astype(int)
	binHigh[binHigh == 8] = 0

	[colI, rowI] = np.meshgrid(range(1,im.shape[1]-1), range(1,im.shape[1]-1))
	ogIm[(rowI,colI,binLow)] += magIm * weightLow

	ogIm[(rowI,colI,binHigh)] += magIm * weightHigh

	return ogIm

'''
' Calculate histogram of gradients, repeatedly from a queue
' Input:
'	i - input queue
' Output:
'	o - output queue
'''
def HOG(i, o):
	arrayA = diag_matrix_linear()
	arrayB = arrayA.T
	if True:
		frame = i

		ogIm = haar_gradients(frame)

		hogs = make_block(ogIm,arrayA,arrayB)

		o.put(hogs)

'''
' Calculate horn-schunck optical flow between two images
' Input:
' 	im1 - image
' Output:
'	of - horn-schunck optical flow
'''
def horn_schunck(im1,im2,Niter=10):
	 #set up initial velocities
	U = np.zeros([im1.shape[0],im1.shape[1]])
	V = np.zeros([im1.shape[0],im1.shape[1]])

	sobelX = np.array([[-1,-2,-1],[0,0,0],[-1,-2,-1]])
	sobelY = sobelX.T

	Ix = cv.filter2D(im1,-1,sobelX)
	Iy = cv.filter2D(im1,-1,sobelY)

	It = im2 - im1

	avgK = np.array([[0,1,0],[1,0,1],[0,1,0]])

	for i in range(Niter):
		uAvg = cv.filter2D(U,-1,avgK)
		vAvg = cv.filter2D(V,-1,avgK)

		der = (Ix*uAvg + Iy*vAvg + It) / (1 + Ix**2 + Iy**2)

		U = uAvg - Ix * der
		V = vAvg - Iy * der

	return U, V

def HS(im1, im2, alpha=1, Niter=10):
	"""
	im1: image at t=0
	im2: image at t=1
	alpha: regularization constant
	Niter: number of iteration
	"""

	#set up initial velocities
	uInitial = np.zeros([im1.shape[0],im1.shape[1]])
	vInitial = np.zeros([im1.shape[0],im1.shape[1]])

	# Set initial value for the flow vectors
	U = uInitial
	V = vInitial

	# Estimate derivatives
	[fx, fy, ft] = computeDerivatives(im1, im2)

	# Averaging kernel
	kernel=np.array([[1/12, 1/6, 1/12],
	                  [1/6,    0, 1/6],
	                  [1/12, 1/6, 1/12]],float)

	# Iteration to reduce error
	for _ in range(Niter):
	#Compute local averages of the flow vectors
		uAvg = filter2(U,kernel)
		vAvg = filter2(V,kernel)
	#common part of update step
		der = (fx*uAvg + fy*vAvg + ft) / (alpha**2 + fx**2 + fy**2)
	#iterative step
		U = uAvg - fx * der
		V = vAvg - fy * der

	return U,V

def computeDerivatives(im1, im2):
#%% build kernels for calculating derivatives
	kernelX = np.array([[-1, 1],[-1, 1]]) * .25 #kernel for computing d/dx
	kernelY = np.array([[-1,-1],[ 1, 1]]) * .25 #kernel for computing d/dy
	kernelT = np.ones((2,2))*.25

	fx = filter2(im1,kernelX) + filter2(im2,kernelX)
	fy = filter2(im1,kernelY) + filter2(im2,kernelY)

	#ft = im2 - im1
	ft = filter2(im1,kernelT) + filter2(im2,-kernelT)

	return fx,fy,ft

'''
' Calculate histogram of optical flow, repeatedly from a queue
' Input:
'	i - input queue
' Output:
'	o - output queue
'''
def HOF(i, o):
	arrayA = diag_matrix_linear()
	arrayB = arrayA.T
	prev_frame = None
	if True:
		n,frame = i.get()

		if prev_frame is None:
			prev_frame = frame
			m = n
			n,frame = i.get()

		v1, v2 = HS(prev_frame,frame)

		mag = np.sqrt(v1**2 + v2**2)
		ang = np.arctan2(v1,v2)

		oM = np.zeros((mag.shape[0],mag.shape[1],8))

		binReal = (ang) * (8 / (2 * np.pi))
		binLow = np.floor(binReal)
		weightHigh = binReal - binLow
		weightLow = 1 - weightHigh
		binLow = (binLow % 8).astype(int)
		binHigh = (binLow + 1).astype(int)
		binHigh[binHigh == 8] = 0

		[colI, rowI] = np.meshgrid(range(mag.shape[1]), range(mag.shape[0]))
		oM[(rowI,colI,binLow)] += mag * weightLow

		oM[(rowI,colI,binHigh)] += mag * weightHigh

		hof = make_block(oM,arrayA,arrayB)

		o.put((m,hof))

def MBHr(i, o):
	arrayA = diag_matrix_linear()
	arrayB = arrayA.T
	prev_frame = None
	if True:
		n,frame = i.get()

		if prev_frame is None:
			prev_frame = frame
			m = n
			n,frame = i.get()

		v1, _ = HS(prev_frame,frame)

		ogIm = haar_gradients(v1)

		hogs = make_block(ogIm,arrayA,arrayB)

		o.put((m,hogs))

def MBHc(i, o):
	arrayA = diag_matrix_linear()
	arrayB = arrayA.T
	prev_frame = None
	if True:
		n,frame = i.get()

		if prev_frame is None:
			prev_frame = frame
			m = n
			n,frame = i.get()

		_, v2 = HS(prev_frame,frame)

		ogIm = haar_gradients(v2)

		hogs = make_block(ogIm,arrayA,arrayB)

		o.put((m,hogs))


from django.http import HttpResponseRedirect
from django.shortcuts import render, render_to_response,redirect, get_object_or_404
# Create your views here.
from django.template.context_processors import csrf

#from Gestion_Inst.forms import AlumnoForm,NivelEnsenanzaForm
from Gestion_Inst.models import *
from crispy_forms.bootstrap import StrictButton, FormActions
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Submit
from django.forms import forms
from django import forms
from Gestion_Inst.models import Alumno

class AlumnoForm(forms.ModelForm):

    class Meta:
        model = Alumno
        fields = ('apellido', 'nombres', 'fecha_nac', 'domicilio', 'telefono', 'dni', 'sexo', 'legajo', 'estado','nivel', 'curso','division',)

    def __init__(self,*args,**kwargs):
        super(AlumnoForm,self).__init__(*args,**kwargs)
        self.helper = FormHelper(self)
        self.helper.form_class = 'form-horizontal'
        self.helper.field_class = 'col-lg-6'
        self.helper.label_class = 'col-lg-2'
        self.helper.layout.append(
            FormActions(
                Submit('Guardar','Guardar'),
            )
        )

def alumnos_list(request, template_name='Gestion_Inst/Alumno/alumnos_list.html'):
    servers = Alumno.objects.all()
    data = {}
    data['object_list'] = servers
    return render(request, template_name, data)

def alumno_create(request, template_name='Gestion_Inst/Alumno/alumno_form.html'):
    form = AlumnoForm(request.POST or None)
    if form.is_valid():
        form.save()
        return redirect('alumnos_list')
    return render(request, template_name, {'form':form})

def alumno_update(request, pk, template_name='Gestion_Inst/Alumno/alumno_form.html'):
    server = get_object_or_404(Alumno, pk=pk)
    form = AlumnoForm(request.POST or None, instance=server)
    if form.is_valid():
        form.save()
        return redirect('alumnos_list')
    return render(request, template_name, {'form':form})

def alumno_delete(request, pk, template_name='Gestion_Inst/Alumno/alumno_confirm_delete.html'):
    server = get_object_or_404(Alumno, pk=pk)
    if request.method=='POST':
        server.delete()
        return redirect('alumno_list')
    return render(request, template_name, {'object':server})

def alumno_borrar(request, pk):
    server = get_object_or_404(Alumno, pk=pk)

    server.delete()
    return redirect('alumno_list')


from rich.panel import Panel

from gitlabtree.models import Info
from gitlabtree.gitlab_helper import GitLabHelper
from gitlabtree.runner import create_runner_info, get_tree_with_runner

from .fixture_helper import *

def test_create_runner_info(
    moked_data_runners: Dict[str, Any], gitlab_helper: GitLabHelper
) -> None:
    data = moked_data_runners["https://a/groups/start/projects?include_subgroups=true"][
        0
    ]
    gitlab_helper.gitlab.moked_data = moked_data_runners  # type: ignore
    info = create_runner_info(data, gitlab_helper)

    assert len(info) == 1
    runner1 = info[0]
    assert runner1.text == f"runner1 - True"
    assert isinstance(runner1.renderable, Panel)
    assert runner1.renderable.renderable == f"runner1 - True"
    assert runner1.renderable.title == "shared"
    assert runner1.renderable.style == "red"

    data2 = moked_data_runners[
        "https://a/groups/start/projects?include_subgroups=true"
    ][2]
    info2 = create_runner_info(data2, gitlab_helper)

    assert len(info2) == 2
    runner2 = info2[1]
    assert runner2.text == "runner2 - False"
    assert isinstance(runner2.renderable, Panel)
    assert runner2.renderable.renderable == f"runner2 - False"
    assert runner2.renderable.title == "group"
    assert runner2.renderable.style == "yellow"

def test_get_tree_with_runner(
    gitlab_helper: GitLabHelper, moked_data_runners: Dict[str, Any]
) -> None:
    gitlab_helper.gitlab.moked_data = moked_data_runners  # type: ignore

    tree = get_tree_with_runner(gitlab_helper, "start")

    assert isinstance(tree.repositories[0].info[0], Info)
    info = tree.repositories[0].info[0]
    assert info.text == "runner1 - True"
    assert len(tree.groups[0].groups[1].repositories[0].info) == 2
    assert tree.model_dump() == {
        "groups": [
            {
                "groups": [
                    {
                        "groups": [],
                        "info": [],
                        "name": "cat",
                        "repositories": [
                            {
                                "info": [{"text": "runner1 - True"}],
                                "name": "devon",
                            }
                        ],
                    },
                    {
                        "groups": [],
                        "info": [],
                        "name": "dog",
                        "repositories": [
                            {
                                "info": [
                                    {"text": "runner1 - True"},
                                    {"text": "runner2 - False"},
                                ],
                                "name": "leo",
                            }
                        ],
                    },
                ],
                "info": [],
                "name": "animal",
                "repositories": [],
            }
        ],
        "info": [],
        "name": "start",
        "repositories": [
            {
                "info": [{"text": "runner1 - True"}],
                "name": "banana",
            }
        ],
    }