text:
import random
import util
from numpy import mean

def init(matrix, num_clusters):
  # select num of clusters random matrix samples
  cluster_centers = []
  samples = range(len(matrix))
  cluster_indices = []
  while len(cluster_indices) < num_clusters:
    index = random.randint(0, len(samples) - 1)
    cluster_indices.append(samples.pop(index))

  for clusterIndex in cluster_indices:
    cluster_centers.append(matrix[clusterIndex])

  return cluster_centers

def assign_points(matrix, clusters):
  clusterPoints = []
  assignedClusters = []

  for sample in matrix:
    assignedCluster = 0
    minErr = float('inf')
    for clusterInd in range(len(clusters)):
      err = util.distance(sample, clusters[clusterInd])
      if (err < minErr):
        minErr = err
        assignedCluster = clusterInd

    clusterPoints.append([assignedCluster, sample])
    assignedClusters.append(assignedCluster)

  return (clusterPoints, assignedClusters)

def reselect_centers(clusterPoints, num_clusters):
  cluster_centers = []

  for cluster in range(num_clusters):
    assigned_points = []
    for point in clusterPoints:
      if point[0] is cluster:
        assigned_points.append(point[1])

    if len(assigned_points) is not 0:
      # convert tolist() so that it can be encoded back to node correctly
      cluster_centers.append(mean(assigned_points, axis=0).tolist())

  return cluster_centers

def converged(old_clusters, clusters):
  return old_clusters == clusters

ext: py

reason: from numpy


text:
import os
import cPickle
import time
import lasagne
import numpy as np
import theano
import theano.tensor as T
import six

import autosklearn.data.competition_data_manager

class FeedForwardNet(object):

    def __init__(self, input_shape=(1, 28, 28),
                 batch_size=100,
                 num_layers=4,
                 num_units_per_layer=(10, 10, 10),
                 dropout_per_layer=(0.5, 0.5, 0.5),
                 std_per_layer=(0.1, 0.1, 0.1),
                 num_output_units=10,
                 dropout_output=0.5,
                 learning_rate=0.01,
                 momentum=0.9,
                 beta1=0.9,
                 beta2=0.999,
                 rho=0.95,
                 solver="sgd"):

        assert len(num_units_per_layer) == num_layers - 1
        assert len(dropout_per_layer) == num_layers - 1

        self.batch_size = int(batch_size)
        input_var = T.tensor4('inputs')
        target_var = T.ivector('targets')
        print("... building network")
        print(input_shape)
        self.network = lasagne.layers.InputLayer(shape=input_shape,
                                                input_var=input_var)

        # Define each layer
        for i in range(num_layers - 1):
            self.network = lasagne.layers.DenseLayer(
                 lasagne.layers.dropout(self.network,
                                p=float(dropout_per_layer[i])),
                 num_units=int(num_units_per_layer[i]),
                 W=lasagne.init.Normal(std=float(std_per_layer[i]), mean=0),
                 b=lasagne.init.Constant(val=0.0),
                 nonlinearity=lasagne.nonlinearities.rectify)

        # Define output layer
        self.network = lasagne.layers.DenseLayer(
                 lasagne.layers.dropout(self.network, p=float(dropout_output)),
                 num_units=int(num_output_units),
                 W=lasagne.init.GlorotNormal(),
                 b=lasagne.init.Constant(),
                 nonlinearity=lasagne.nonlinearities.softmax)

        prediction = lasagne.layers.get_output(self.network)
        loss = lasagne.objectives.categorical_crossentropy(prediction,
                                                           target_var)
        loss = loss.mean()

        self.learning_rate = theano.shared(np.array(learning_rate,
                                                dtype=theano.config.floatX))

        params = lasagne.layers.get_all_params(self.network, trainable=True)

        if solver == "nesterov":
            updates = lasagne.updates.nesterov_momentum(loss, params,
                                                learning_rate=learning_rate,
                                                momentum=momentum)
        elif solver == "adam":
            updates = lasagne.updates.adam(loss, params,
                                           learning_rate=learning_rate,
                                           beta1=beta1, beta2=beta2)
        elif solver == "adadelta":
            updates = lasagne.updates.adadelta(loss, params,
                                               learning_rate=learning_rate,
                                               rho=rho)
        elif solver == "adagrad":
            updates = lasagne.updates.adagrad(loss, params,
                                              learning_rate=learning_rate)
        elif solver == "sgd":
            updates = lasagne.updates.sgd(loss, params,
                                          learning_rate=learning_rate)
        elif solver == "momentum":
            updates = lasagne.updates.momentum(loss, params,
                                               learning_rate=learning_rate,
                                               momentum=momentum)
        else:
            updates = lasagne.updates.sgd(loss, params,
                                          learning_rate=learning_rate)
        valid_prediction = lasagne.layers.get_output(self.network,
                                                     deterministic=True)
        valid_loss = lasagne.objectives.categorical_crossentropy(
                                                        valid_prediction,
                                                        target_var)
        valid_loss = valid_loss.mean()
        valid_acc = T.mean(T.eq(T.argmax(valid_prediction, axis=1),
                                target_var),
                                dtype=theano.config.floatX)

        print("... compiling theano functions")
        self.train_fn = theano.function([input_var, target_var], loss,
                                        updates=updates,
                                        allow_input_downcast=True)
        self.val_fn = theano.function([input_var, target_var],
                                      [valid_loss, valid_acc],
                                      allow_input_downcast=True)

def iterate_minibatches(inputs, targets, batchsize, shuffle=False):
    assert len(inputs) == len(targets)
    if shuffle:
        indices = np.arange(len(inputs))
        np.random.shuffle(indices)
    for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):
        if shuffle:
            excerpt = indices[start_idx:start_idx + batchsize]
        else:
            excerpt = slice(start_idx, start_idx + batchsize)
        yield inputs[excerpt], targets[excerpt]

def train(model, X_train, y_train, X_valid, y_valid, num_epochs=500,
          lr_policy="inv", gamma=0.1, power=0.5, step=30):

    learning_curve = np.zeros([num_epochs])

    lr_base = model.learning_rate.get_value()
    for epoch in range(num_epochs):

        train_err = 0
        train_batches = 0
        start_time = time.time()
        for batch in iterate_minibatches(X_train, y_train, model.batch_size, shuffle=True):
            inputs, targets = batch
            train_err += model.train_fn(inputs, targets)
            train_batches += 1

        # And a full pass over the validation data:
        val_err = 0
        val_acc = 0
        val_batches = 0

        # Learning rate adaption
        if lr_policy == "inv":
            decay = np.power((1 + gamma * epoch), (- power))
        elif lr_policy == "exp":
            decay = np.power(gamma, epoch)
        elif lr_policy == "step":
            decay = np.power(gamma, (np.floor(epoch / float(step))))
        elif lr_policy == "fixed":
            decay = 1

        decay = np.array([decay], dtype=np.float32)
        print ("decay factor %f" % decay)
        model.learning_rate.set_value(lr_base * decay)
        print ("Learning rate %f" % model.learning_rate.get_value())
        for batch in iterate_minibatches(X_valid, y_valid, model.batch_size,
                                         shuffle=False):
            inputs, targets = batch
            err, acc = model.val_fn(inputs, targets)
            val_err += err
            val_acc += acc
            val_batches += 1

        print("Epoch {} of {} took {:.3f}s".format(
            epoch + 1, num_epochs, time.time() - start_time))
        print("  training loss:\t\t{:.6f}".format(train_err / train_batches))
        print("  validation loss:\t\t{:.6f}".format(val_err / val_batches))
        print("  validation accuracy:\t\t{:.2f} %".format(
            val_acc / val_batches * 100))
        learning_curve[epoch] = (val_acc / float(val_batches) * 100)

    return 1 - (val_acc / val_batches), learning_curve

dataset = 'alexis'

output = "/home/kleinaa/experiments/automated_deep_learning/automl/alexis/reproduction"
path = os.path.join("/data/aad/automl_data/003", dataset)

dm = autosklearn.data.competition_data_manager.CompetitionDataManager(path)
X_train = dm.data["X_train"].toarray()
X_train = np.array(X_train, dtype=np.float32)
Y_train = np.argmax(np.array(dm.data["Y_train"], dtype=np.float32), axis=1)
X_valid = np.array(dm.data["X_valid"].toarray(), dtype=np.float32)
X_test = np.array(dm.data["X_test"].toarray(), dtype=np.float32)

# Replace the following array by a new ensemble
choices = \
    [({'num_layers': 3.0,
       'power': 0.9608086215808905,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.32350553193868387,
       'dropout_output': 0.19522565771560596,
       'learning_rate': 0.22087420991966428,
       'dropout_layer_2': 0.5416862005436668,
       'num_units_layer_2': 3429.0,
       'batch_size': 963.3316054786878,
       'num_units_layer_1': 4742.0,
       'step': 17.0,
       'momentum': 0.8358157637992942,
       'num_epochs': 490.0,
       'std_layer_2': 0.002570301444272789,
       'lr_policy': 'fixed',
       'std_layer_1': 0.00203675415789638,
       'gamma': 0.1982457569106505}, 0.02),
     ({'num_layers': 4.0,
       'power': 0.9543822789540591,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.32350553193868387,
       'dropout_output': 0.19522565771560596,
       'dropout_layer_3': 0.07073518892061528,
       'dropout_layer_2': 0.5416862005436668,
       'num_units_layer_2': 3429.0,
       'num_units_layer_3': 2921.0,
       'batch_size': 963.3316054786878,
       'num_units_layer_1': 4742.0,
       'step': 17.0,
       'std_layer_1': 0.00203675415789638,
       'momentum': 0.8358157637992942,
       'num_epochs': 490.0,
       'std_layer_3': 0.0027860724140478716,
       'std_layer_2': 0.002570301444272789,
       'lr_policy': 'step',
       'learning_rate': 0.22087420991966428,
       'gamma': 0.1982457569106505}, 0.02),
     ({'dropout_output': 0.2219836761125169,
       'num_units_layer_4': 503.0,
       'num_epochs': 249.0,
       'num_units_layer_2': 1798.0,
       'num_units_layer_3': 4435.0,
       'num_units_layer_1': 5785.0,
       'num_layers': 5.0,
       'beta2': 0.35170616682914413,
       'beta1': 0.3871526867361012,
       'std_layer_3': 0.0020693367447187833,
       'std_layer_2': 0.01626605159391926,
       'std_layer_1': 0.010029613006349311,
       'learning_rate': 0.161203081475512,
       'std_layer_4': 0.04266796418763915,
       'power': 0.7730778282076188,
       'dropout_layer_1': 0.5996870640229259,
       'dropout_layer_3': 0.016146856157454967,
       'dropout_layer_2': 0.049116767959870425,
       'dropout_layer_4': 0.6672322817450616,
       'batch_size': 741.3580222134027,
       'step': 21.0,
       'lr_policy': 'fixed',
       'solver': '"adam"',
       'gamma': 0.6788656698879015}, 0.040000000000000001),
     ({'num_layers': 4.0,
       'power': 0.7949015212265006,
       'solver': '"adadelta"',
       'dropout_layer_1': 0.4521497425726368,
       'dropout_output': 0.44804264005915573,
       'dropout_layer_3': 0.5015510137781847,
       'dropout_layer_2': 0.45385947764882123,
       'num_units_layer_2': 5059.0,
       'num_units_layer_3': 291.0,
       'batch_size': 808.4213834239384,
       'num_units_layer_1': 2468.0,
       'step': 39.0,
       'std_layer_1': 0.0014528596407236882,
       'rho': 0.5198404112822554,
       'num_epochs': 365.0,
       'std_layer_3': 0.018883281420906187,
       'std_layer_2': 0.060173127055786896,
       'lr_policy': 'inv',
       'learning_rate': 0.004445343676392594,
       'gamma': 0.5816489765526873}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.951501985775478,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.3653306482580645,
       'dropout_output': 0.22582421055308477,
       'learning_rate': 0.19202636600670817,
       'dropout_layer_2': 0.21819303981244112,
       'num_units_layer_2': 4648.0,
       'batch_size': 254.02504767576468,
       'num_units_layer_1': 5766.0,
       'step': 76.0,
       'momentum': 0.8768999623562101,
       'num_epochs': 486.0,
       'std_layer_2': 0.012236381842172813,
       'lr_policy': 'fixed',
       'std_layer_1': 0.012567729774967713,
       'gamma': 0.6722873907885709}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.951501985775478,
       'solver': '"adadelta"',
       'dropout_layer_1': 0.3653306482580645,
       'dropout_output': 0.3357527566089538,
       'learning_rate': 0.19202636600670817,
       'dropout_layer_2': 0.21819303981244112,
       'num_units_layer_2': 6138.0,
       'batch_size': 254.02504767576468,
       'num_units_layer_1': 5766.0,
       'step': 68.0,
       'rho': 0.6568839756050975,
       'num_epochs': 486.0,
       'std_layer_2': 0.012236381842172813,
       'lr_policy': 'fixed',
       'std_layer_1': 0.012567729774967713,
       'gamma': 0.6722873907885709}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.8196900527717096,
       'solver': '"sgd"',
       'dropout_layer_1': 0.2939609278437122,
       'dropout_output': 0.2553598975807717,
       'learning_rate': 0.0792359335703241,
       'dropout_layer_2': 0.17148792984538072,
       'num_units_layer_2': 6042.0,
       'batch_size': 68.01704776734499,
       'num_units_layer_1': 4337.0,
       'step': 37.0,
       'momentum': 0.7768588564816645,
       'num_epochs': 467.0,
       'std_layer_2': 0.09305184343463795,
       'lr_policy': 'inv',
       'std_layer_1': 0.0033240180436903202,
       'gamma': 0.7240342867167526}, 0.040000000000000001),
     ({'num_layers': 3.0,
       'power': 0.951501985775478,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.7836649078292658,
       'dropout_output': 0.21660605836335647,
       'learning_rate': 0.027610542766288576,
       'dropout_layer_2': 0.011586976261760496,
       'num_units_layer_2': 5787.0,
       'batch_size': 254.02504767576468,
       'num_units_layer_1': 4722.0,
       'step': 18.0,
       'momentum': 0.8768999623562101,
       'num_epochs': 486.0,
       'std_layer_2': 0.012236381842172813,
       'lr_policy': 'exp',
       'std_layer_1': 0.0005598439981691678,
       'gamma': 0.9667180658562067}, 0.040000000000000001),
     ({'num_layers': 3.0,
       'power': 0.9747702817998765,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.4051640090922492,
       'dropout_output': 0.2892451691812821,
       'learning_rate': 0.17394384345131372,
       'dropout_layer_2': 0.5231272729308043,
       'num_units_layer_2': 5608.0,
       'batch_size': 73.2292167635178,
       'num_units_layer_1': 4633.0,
       'step': 16.0,
       'momentum': 0.8358157637992942,
       'num_epochs': 491.0,
       'std_layer_2': 0.002570301444272789,
       'lr_policy': 'step',
       'std_layer_1': 0.00696310505976957,
       'gamma': 0.1982457569106505}, 0.040000000000000001),
     ({'dropout_output': 0.25306857146792566,
       'num_units_layer_4': 4733.0,
       'num_epochs': 486.0,
       'num_units_layer_2': 5797.0,
       'num_units_layer_3': 5620.0,
       'num_units_layer_1': 4722.0,
       'num_layers': 5.0,
       'std_layer_3': 0.0018624928976982682,
       'std_layer_2': 0.012236381842172813,
       'std_layer_1': 0.0013146995716357634,
       'learning_rate': 0.0528394424911447,
       'std_layer_4': 0.030949659698034787,
       'power': 0.951501985775478,
       'dropout_layer_1': 0.2602384986628831,
       'dropout_layer_3': 0.07918985321674664,
       'dropout_layer_2': 0.13607828166784158,
       'dropout_layer_4': 0.20468440586381545,
       'batch_size': 118.70898813118156,
       'step': 18.0,
       'rho': 0.6568839756050975,
       'lr_policy': 'exp',
       'solver': '"adadelta"',
       'gamma': 0.7090475445273005}, 0.040000000000000001),
     ({'num_layers': 3.0,
       'power': 0.951501985775478,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.3653306482580645,
       'dropout_output': 0.3795818048708614,
       'learning_rate': 0.19202636600670817,
       'dropout_layer_2': 0.21819303981244112,
       'num_units_layer_2': 4648.0,
       'batch_size': 254.02504767576468,
       'num_units_layer_1': 5766.0,
       'step': 76.0,
       'momentum': 0.8768999623562101,
       'num_epochs': 486.0,
       'std_layer_2': 0.012236381842172813,
       'lr_policy': 'fixed',
       'std_layer_1': 0.00019210721668567557,
       'gamma': 0.6722873907885709}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.951501985775478,
       'solver': '"sgd"',
       'dropout_layer_1': 0.3653306482580645,
       'dropout_output': 0.3357527566089538,
       'learning_rate': 0.19202636600670817,
       'dropout_layer_2': 0.21819303981244112,
       'num_units_layer_2': 6138.0,
       'batch_size': 254.02504767576468,
       'num_units_layer_1': 5766.0,
       'step': 68.0,
       'momentum': 0.8768999623562101,
       'num_epochs': 486.0,
       'std_layer_2': 0.012236381842172813,
       'lr_policy': 'fixed',
       'std_layer_1': 0.020725423936460784,
       'gamma': 0.6722873907885709}, 0.02),
     ({'dropout_output': 0.5544528745031847,
       'num_units_layer_4': 4835.0,
       'num_units_layer_5': 1207.0,
       'num_units_layer_2': 3532.0,
       'num_units_layer_3': 3265.0,
       'num_units_layer_1': 2341.0,
       'num_epochs': 79.0,
       'std_layer_3': 0.0003041749397972243,
       'std_layer_2': 0.015965838945860733,
       'lr_policy': 'step',
       'learning_rate': 0.003930848214214284,
       'momentum': 0.5508087063095997,
       'std_layer_4': 0.042866542378616,
       'power': 0.7392683450424904,
       'dropout_layer_1': 0.09142524602061186,
       'dropout_layer_3': 0.45551111081428075,
       'dropout_layer_2': 0.6039763657544621,
       'dropout_layer_5': 0.4194002794764725,
       'dropout_layer_4': 0.15409260770878827,
       'batch_size': 913.9939922328252,
       'step': 9.0,
       'std_layer_1': 0.018242827552014805,
       'num_layers': 6.0,
       'solver': '"momentum"',
       'std_layer_5': 0.080768263392991,
       'gamma': 0.12406835380828729}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.951501985775478,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.3653306482580645,
       'dropout_output': 0.3795818048708614,
       'learning_rate': 0.28665652540783937,
       'dropout_layer_2': 0.21819303981244112,
       'num_units_layer_2': 4648.0,
       'batch_size': 254.02504767576468,
       'num_units_layer_1': 5766.0,
       'step': 76.0,
       'momentum': 0.8768999623562101,
       'num_epochs': 486.0,
       'std_layer_2': 0.012236381842172813,
       'lr_policy': 'exp',
       'std_layer_1': 0.025068182392771483,
       'gamma': 0.6722873907885709}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.9501928234008037,
       'solver': '"adadelta"',
       'dropout_layer_1': 0.49417282390370604,
       'dropout_output': 0.1112655122924812,
       'learning_rate': 0.05647348442267938,
       'dropout_layer_2': 0.5668268697567368,
       'num_units_layer_2': 847.0,
       'batch_size': 69.43044908879082,
       'num_units_layer_1': 4509.0,
       'step': 62.0,
       'rho': 0.1412178971508795,
       'num_epochs': 325.0,
       'std_layer_2': 0.09245714688025959,
       'lr_policy': 'inv',
       'std_layer_1': 0.0030031803285322065,
       'gamma': 0.9880106805712771}, 0.02),
     ({'num_layers': 4.0,
       'power': 0.9982859561242945,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.3868459283104136,
       'dropout_output': 0.30578911000133563,
       'dropout_layer_3': 0.05910805094837215,
       'dropout_layer_2': 0.5346257876302651,
       'num_units_layer_2': 1555.0,
       'num_units_layer_3': 3709.0,
       'batch_size': 793.5704826524803,
       'num_units_layer_1': 5632.0,
       'step': 91.0,
       'std_layer_1': 0.0019427866308684189,
       'momentum': 0.47096093223996227,
       'num_epochs': 488.0,
       'std_layer_3': 0.004455993940858878,
       'std_layer_2': 0.07300638871271027,
       'lr_policy': 'exp',
       'learning_rate': 0.1414121563748416,
       'gamma': 0.1338226652313088}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.951501985775478,
       'solver': '"sgd"',
       'dropout_layer_1': 0.3653306482580645,
       'dropout_output': 0.3357527566089538,
       'learning_rate': 0.19202636600670817,
       'dropout_layer_2': 0.21819303981244112,
       'num_units_layer_2': 6138.0,
       'batch_size': 254.02504767576468,
       'num_units_layer_1': 5766.0,
       'step': 68.0,
       'momentum': 0.8768999623562101,
       'num_epochs': 486.0,
       'std_layer_2': 0.012236381842172813,
       'lr_policy': 'fixed',
       'std_layer_1': 0.012567729774967713,
       'gamma': 0.6722873907885709}, 0.02),
     ({'num_layers': 4.0,
       'power': 0.6102313152642265,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.5002438300235051,
       'dropout_output': 0.23534479317312496,
       'dropout_layer_3': 0.6828689388536276,
       'dropout_layer_2': 0.1762597535432976,
       'num_units_layer_2': 3647.0,
       'num_units_layer_3': 3044.0,
       'batch_size': 786.4476079179202,
       'num_units_layer_1': 5076.0,
       'step': 60.0,
       'std_layer_1': 0.013221036066045333,
       'momentum': 0.9814146239572572,
       'num_epochs': 478.0,
       'std_layer_3': 0.07660361796208646,
       'std_layer_2': 0.021796162268579933,
       'lr_policy': 'step',
       'learning_rate': 0.20931601325873997,
       'gamma': 0.29777339200275743}, 0.02),
     ({'dropout_output': 0.6986654447282298,
       'num_units_layer_4': 2724.0,
       'num_epochs': 70.0,
       'num_units_layer_2': 940.0,
       'num_units_layer_3': 176.0,
       'num_units_layer_1': 1554.0,
       'num_layers': 5.0,
       'beta2': 0.3437234235760527,
       'beta1': 0.5122595623529058,
       'std_layer_3': 0.001323462426190976,
       'std_layer_2': 0.08527672724824989,
       'std_layer_1': 0.06598031837899644,
       'learning_rate': 5.88849354815407e-05,
       'std_layer_4': 0.003244516882128532,
       'power': 0.8191416561238503,
       'dropout_layer_1': 0.12585693415903498,
       'dropout_layer_3': 0.2642128122930812,
       'dropout_layer_2': 0.07731697305608044,
       'dropout_layer_4': 0.016910080279172977,
       'batch_size': 803.0778892408711,
       'step': 82.0,
       'lr_policy': 'exp',
       'solver': '"adam"',
       'gamma': 0.9734274599579855}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.9839161318019856,
       'solver': '"adadelta"',
       'dropout_layer_1': 0.3286158737442102,
       'dropout_output': 0.2602697774913525,
       'learning_rate': 0.05647348442267938,
       'dropout_layer_2': 0.41223774560421556,
       'num_units_layer_2': 3119.0,
       'batch_size': 69.43044908879082,
       'num_units_layer_1': 4509.0,
       'step': 62.0,
       'rho': 0.1412178971508795,
       'num_epochs': 341.0,
       'std_layer_2': 0.07883910009093228,
       'lr_policy': 'inv',
       'std_layer_1': 0.002283362184875592,
       'gamma': 0.9880106805712771}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.8211518930492289,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.3288790841138845,
       'dropout_output': 0.2458740233193249,
       'learning_rate': 0.07112784792816951,
       'dropout_layer_2': 0.17005904025630333,
       'num_units_layer_2': 5787.0,
       'batch_size': 254.02504767576468,
       'num_units_layer_1': 4722.0,
       'step': 18.0,
       'momentum': 0.8768999623562101,
       'num_epochs': 486.0,
       'std_layer_2': 0.012236381842172813,
       'lr_policy': 'exp',
       'std_layer_1': 0.0005598439981691678,
       'gamma': 0.9667180658562067}, 0.02),
     ({'dropout_output': 0.6942222485334036,
       'num_units_layer_4': 4488.0,
       'num_epochs': 246.0,
       'num_units_layer_2': 360.0,
       'num_units_layer_3': 1762.0,
       'num_units_layer_1': 2452.0,
       'num_layers': 5.0,
       'std_layer_3': 0.000840782752525304,
       'std_layer_2': 0.053417714428946675,
       'std_layer_1': 2.49410460816067e-05,
       'learning_rate': 0.030725262843106595,
       'momentum': 0.7354775983645889,
       'std_layer_4': 0.014723488261112642,
       'power': 0.6825025080767805,
       'dropout_layer_1': 0.5430371683124818,
       'dropout_layer_3': 0.7178289875612346,
       'dropout_layer_2': 0.17679879849784874,
       'dropout_layer_4': 0.670500227550017,
       'batch_size': 848.7089176912435,
       'step': 15.0,
       'lr_policy': 'inv',
       'solver': '"nesterov"',
       'gamma': 0.6360825538930697}, 0.02),
     ({'num_layers': 4.0,
       'power': 0.7084104721089539,
       'solver': '"adam"',
       'dropout_layer_1': 0.3228804321714691,
       'dropout_output': 0.19123621059916907,
       'dropout_layer_3': 0.5991741560861912,
       'dropout_layer_2': 0.042708968496055696,
       'num_units_layer_2': 4473.0,
       'num_units_layer_3': 299.0,
       'batch_size': 662.3881155284046,
       'num_units_layer_1': 5531.0,
       'step': 75.0,
       'std_layer_1': 0.006831817706575724,
       'beta2': 0.5763734437755985,
       'beta1': 0.46578559803031666,
       'num_epochs': 454.0,
       'std_layer_3': 0.07534205836424511,
       'std_layer_2': 0.010541138669952833,
       'lr_policy': 'step',
       'learning_rate': 0.2286213615447043,
       'gamma': 0.9847806583536367}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.6074047952114839,
       'solver': '"sgd"',
       'dropout_layer_1': 0.3837602436079431,
       'dropout_output': 0.2586093771445448,
       'learning_rate': 0.1850908533669142,
       'dropout_layer_2': 0.2012363291904858,
       'num_units_layer_2': 2999.0,
       'batch_size': 252.36804895745078,
       'num_units_layer_1': 4742.0,
       'step': 98.0,
       'momentum': 0.5716625593124078,
       'num_epochs': 417.0,
       'std_layer_2': 0.04035506633883424,
       'lr_policy': 'fixed',
       'std_layer_1': 0.03471403541293925,
       'gamma': 0.9228259167471723}, 0.02),
     ({'dropout_output': 0.012311474913025122,
       'num_units_layer_4': 5022.0,
       'num_epochs': 425.0,
       'num_units_layer_2': 1134.0,
       'num_units_layer_3': 5339.0,
       'num_units_layer_1': 1766.0,
       'num_layers': 5.0,
       'std_layer_3': 0.022935273875549143,
       'std_layer_2': 0.03211092898082504,
       'std_layer_1': 0.0009145066420452281,
       'learning_rate': 0.020646689616918727,
       'momentum': 0.5669637108787287,
       'std_layer_4': 0.0777074650891919,
       'power': 0.9413926402897054,
       'dropout_layer_1': 0.8245359519955847,
       'dropout_layer_3': 0.5298513101402106,
       'dropout_layer_2': 0.8202560119711875,
       'dropout_layer_4': 0.627756689182714,
       'batch_size': 986.6136297054668,
       'step': 46.0,
       'lr_policy': 'inv',
       'solver': '"momentum"',
       'gamma': 0.965503310296916}, 0.059999999999999998),
     ({'num_layers': 4.0,
       'power': 0.7084104721089539,
       'solver': '"adam"',
       'dropout_layer_1': 0.3228804321714691,
       'dropout_output': 0.19123621059916907,
       'dropout_layer_3': 0.5991741560861912,
       'dropout_layer_2': 0.042708968496055696,
       'num_units_layer_2': 3844.0,
       'num_units_layer_3': 299.0,
       'batch_size': 962.2936600774519,
       'num_units_layer_1': 5531.0,
       'step': 75.0,
       'std_layer_1': 0.0017111040466369133,
       'beta2': 0.5763734437755985,
       'beta1': 0.46578559803031666,
       'num_epochs': 475.0,
       'std_layer_3': 0.07534205836424511,
       'std_layer_2': 0.010541138669952833,
       'lr_policy': 'step',
       'learning_rate': 0.2286213615447043,
       'gamma': 0.9847806583536367}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.7084104721089539,
       'solver': '"adam"',
       'dropout_layer_1': 0.3228804321714691,
       'dropout_output': 0.19123621059916907,
       'learning_rate': 0.2286213615447043,
       'dropout_layer_2': 0.042708968496055696,
       'num_units_layer_2': 3844.0,
       'batch_size': 962.2936600774519,
       'num_units_layer_1': 5531.0,
       'step': 75.0,
       'beta2': 0.5763734437755985,
       'beta1': 0.46578559803031666,
       'num_epochs': 475.0,
       'std_layer_2': 0.010541138669952833,
       'lr_policy': 'step',
       'std_layer_1': 0.0017111040466369133,
       'gamma': 0.9847806583536367}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.6102829556412925,
       'solver': '"sgd"',
       'dropout_layer_1': 0.04382776539965009,
       'dropout_output': 0.4008248943831341,
       'learning_rate': 0.24435066375760794,
       'dropout_layer_2': 0.692660907091548,
       'num_units_layer_2': 1375.0,
       'batch_size': 427.01583837951085,
       'num_units_layer_1': 5102.0,
       'step': 79.0,
       'momentum': 0.6475139577982485,
       'num_epochs': 320.0,
       'std_layer_2': 0.07399922539485788,
       'lr_policy': 'step',
       'std_layer_1': 0.09262827614546214,
       'gamma': 0.7776717365467021}, 0.02),
     ({'num_layers': 4.0,
       'power': 0.6419781172534461,
       'solver': '"momentum"',
       'dropout_layer_1': 0.9717726710481605,
       'dropout_output': 0.32908147435708823,
       'dropout_layer_3': 0.7055634222219525,
       'dropout_layer_2': 0.8546874464796694,
       'num_units_layer_2': 3851.0,
       'num_units_layer_3': 4517.0,
       'batch_size': 727.6171125587168,
       'num_units_layer_1': 4634.0,
       'step': 42.0,
       'std_layer_1': 0.0004299309081302,
       'momentum': 0.6321801704123293,
       'num_epochs': 371.0,
       'std_layer_3': 0.028926029974051224,
       'std_layer_2': 0.01145562004800013,
       'lr_policy': 'inv',
       'learning_rate': 0.029826318066865545,
       'gamma': 0.46436150446432656}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.7238401506457167,
       'solver': '"adadelta"',
       'dropout_layer_1': 0.3233543463764956,
       'dropout_output': 0.387692658865333,
       'learning_rate': 0.25055705607149903,
       'dropout_layer_2': 0.16537451177082238,
       'num_units_layer_2': 5217.0,
       'batch_size': 287.15312680026267,
       'num_units_layer_1': 2623.0,
       'step': 73.0,
       'rho': 0.3993788398015923,
       'num_epochs': 412.0,
       'std_layer_2': 0.005468903582612894,
       'lr_policy': 'inv',
       'std_layer_1': 0.03196564357281583,
       'gamma': 0.7225253602187486}, 0.02),
     ({'num_layers': 4.0,
       'power': 0.9084427363360542,
       'solver': '"sgd"',
       'dropout_layer_1': 0.47408618114026724,
       'dropout_output': 0.6375324638845747,
       'dropout_layer_3': 0.13532008977088042,
       'dropout_layer_2': 0.10867051992136886,
       'num_units_layer_2': 5200.0,
       'num_units_layer_3': 1764.0,
       'batch_size': 751.7233769160604,
       'num_units_layer_1': 1125.0,
       'step': 84.0,
       'std_layer_1': 0.004706707945838345,
       'momentum': 0.6579670949791995,
       'num_epochs': 418.0,
       'std_layer_3': 0.06703203686923141,
       'std_layer_2': 0.021364776685813765,
       'lr_policy': 'exp',
       'learning_rate': 1.006895930668558e-06,
       'gamma': 0.26589296318210975}, 0.02),
     ({'num_layers': 4.0,
       'power': 0.8808203083192006,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.31614542177669236,
       'dropout_output': 0.6128834950904959,
       'dropout_layer_3': 0.6734939376905885,
       'dropout_layer_2': 0.5921620400263008,
       'num_units_layer_2': 300.0,
       'num_units_layer_3': 3672.0,
       'batch_size': 241.71188208498393,
       'num_units_layer_1': 214.0,
       'step': 56.0,
       'std_layer_1': 0.05214945083378918,
       'momentum': 0.6720757416354959,
       'num_epochs': 194.0,
       'std_layer_3': 0.02020291659500061,
       'std_layer_2': 0.08489080217582551,
       'lr_policy': 'exp',
       'learning_rate': 0.28728739883922444,
       'gamma': 0.8087528472911654}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.9982859561242945,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.32925257461740814,
       'dropout_output': 0.1924704598298523,
       'learning_rate': 0.2135562949651281,
       'dropout_layer_2': 0.5346257876302651,
       'num_units_layer_2': 3885.0,
       'batch_size': 793.5704826524803,
       'num_units_layer_1': 4593.0,
       'step': 91.0,
       'momentum': 0.47096093223996227,
       'num_epochs': 488.0,
       'std_layer_2': 0.007225888809513894,
       'lr_policy': 'exp',
       'std_layer_1': 0.0019427866308684189,
       'gamma': 0.34356010533685133}, 0.02),
     ({'num_layers': 3.0,
       'power': 0.951501985775478,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.7836649078292658,
       'dropout_output': 0.2399093307780398,
       'learning_rate': 0.010929876171533532,
       'dropout_layer_2': 0.011586976261760496,
       'num_units_layer_2': 5787.0,
       'batch_size': 31.814264805340258,
       'num_units_layer_1': 4722.0,
       'step': 18.0,
       'momentum': 0.8768999623562101,
       'num_epochs': 486.0,
       'std_layer_2': 0.012236381842172813,
       'lr_policy': 'step',
       'std_layer_1': 0.0024717223220700455,
       'gamma': 0.9667180658562067}, 0.10000000000000001),
     ({'num_layers': 3.0,
       'power': 0.8196900527717096,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.3660063483123479,
       'dropout_output': 0.2553598975807717,
       'learning_rate': 0.0792359335703241,
       'dropout_layer_2': 0.17148792984538072,
       'num_units_layer_2': 4800.0,
       'batch_size': 242.74838348417052,
       'num_units_layer_1': 4749.0,
       'step': 37.0,
       'momentum': 0.7768588564816645,
       'num_epochs': 494.0,
       'std_layer_2': 0.08438075257417156,
       'lr_policy': 'exp',
       'std_layer_1': 0.0015575939075707984,
       'gamma': 0.7240342867167526}, 0.02),
     ({'num_layers': 4.0,
       'power': 0.9982859561242945,
       'solver': '"nesterov"',
       'dropout_layer_1': 0.32925257461740814,
       'dropout_output': 0.1924704598298523,
       'dropout_layer_3': 0.05910805094837215,
       'dropout_layer_2': 0.5346257876302651,
       'num_units_layer_2': 1555.0,
       'num_units_layer_3': 3709.0,
       'batch_size': 793.5704826524803,
       'num_units_layer_1': 4593.0,
       'step': 91.0,
       'std_layer_1': 0.0019427866308684189,
       'momentum': 0.47096093223996227,
       'num_epochs': 488.0,
       'std_layer_3': 0.004455993940858878,
       'std_layer_2': 0.07300638871271027,
       'lr_policy': 'fixed',
       'learning_rate': 0.1122446352398429,
       'gamma': 0.7159714609304522}, 0.02),
     ({'num_layers': 4.0,
       'power': 0.7038076018250163,
       'solver': '"adadelta"',
       'dropout_layer_1': 0.21049439968187753,
       'dropout_output': 0.37339465970155605,
       'dropout_layer_3': 0.4492220124816988,
       'dropout_layer_2': 0.2556332708684139,
       'num_units_layer_2': 2212.0,
       'num_units_layer_3': 5926.0,
       'batch_size': 934.4929652600218,
       'num_units_layer_1': 1350.0,
       'step': 23.0,
       'std_layer_1': 0.00041345564765364986,
       'rho': 0.6263095724704066,
       'num_epochs': 57.0,
       'std_layer_3': 0.058349495588107474,
       'std_layer_2': 0.04610450347271679,
       'lr_policy': 'fixed',
       'learning_rate': 7.219811931684516e-05,
       'gamma': 0.5372817819519713}, 0.02),
     ({'dropout_output': 0.05419711953824402,
       'num_units_layer_4': 503.0,
       'num_epochs': 249.0,
       'num_units_layer_2': 1798.0,
       'num_units_layer_3': 4435.0,
       'num_units_layer_1': 5785.0,
       'num_layers': 5.0,
       'std_layer_3': 0.0020693367447187833,
       'std_layer_2': 0.01626605159391926,
       'std_layer_1': 0.02160469296878519,
       'learning_rate': 0.161203081475512,
       'std_layer_4': 0.04266796418763915,
       'power': 0.7730778282076188,
       'dropout_layer_1': 0.5996870640229259,
       'dropout_layer_3': 0.2387361343390475,
       'dropout_layer_2': 0.049116767959870425,
       'dropout_layer_4': 0.6672322817450616,
       'batch_size': 719.3714049032842,
       'step': 14.0,
       'lr_policy': 'fixed',
       'solver': '"adagrad"',
       'gamma': 0.10402671251350482}, 0.02),
     ({'dropout_output': 0.1570217658484297,
       'num_units_layer_4': 5074.0,
       'num_units_layer_5': 4907.0,
       'num_units_layer_2': 5709.0,
       'num_units_layer_3': 5344.0,
       'num_units_layer_1': 2394.0,
       'num_epochs': 439.0,
       'std_layer_3': 0.07601832353856731,
       'std_layer_2': 0.020438631464011356,
       'lr_policy': 'step',
       'learning_rate': 0.0010928685578229672,
       'momentum': 0.7135023584881737,
       'std_layer_4': 0.05067105296151777,
       'power': 0.7904179532313478,
       'dropout_layer_1': 0.407964865575413,
       'dropout_layer_3': 0.1626118699682814,
       'dropout_layer_2': 0.8417694944058159,
       'dropout_layer_5': 0.5575846370595227,
       'dropout_layer_4': 0.7007827393164742,
       'batch_size': 702.817516948513,
       'step': 3.0,
       'std_layer_1': 0.011027585261841283,
       'num_layers': 6.0,
       'solver': '"nesterov"',
       'std_layer_5': 0.0986334443081331,
       'gamma': 0.41981866741792373}, 0.02)]

targets = []
predictions = []
predictions_valid = []
predictions_test = []

# Make predictions and weight them
iteration = 0
for config, weight in choices:
    print(config)
    print(weight)

    # Extract layer specific variables
    num_layers = int(config["num_layers"])
    num_units_per_layer = []
    dropout_per_layer = []
    std_per_layer = []
    for i in range(1, num_layers):
        num_units_per_layer.append(int(config["num_units_layer_" + str(i)]))
        dropout_per_layer.append(float(config["dropout_layer_" + str(i)]))
        std_per_layer.append(float(config["std_layer_" + str(i)]))

    # Extract solver specific variables
    if config["solver"] == "adam":
        beta1 = config["beta1"]
        beta2 = config["beta2"]
    else:
        beta1 = None
        beta2 = None

    if config["solver"] == "adadelta":
        rho = config["rho"]
    else:
        rho = None

    if config["solver"] == "momentum" or config["solver"] == "nesterov":
        momentum = config["momentum"]
    else:
        momentum = None
    iteration += 1

    n_feat = X_train.shape[1]
    num_output_units = dm.info["label_num"]

    input_shape = (int(config["batch_size"]), 1, 1, n_feat)

    net = FeedForwardNet(input_shape=input_shape,
                     batch_size=config["batch_size"],
                     num_layers=num_layers,
                     num_units_per_layer=num_units_per_layer,
                     dropout_per_layer=dropout_per_layer,
                     std_per_layer=std_per_layer,
                     num_output_units=num_output_units,
                     dropout_output=config["dropout_output"],
                     learning_rate=config["learning_rate"],
                     momentum=momentum,
                     beta1=beta1,
                     beta2=beta2,
                     rho=rho,
                     solver=config["solver"])

    print(X_train[:, np.newaxis, np.newaxis, :].shape)
    validation_error, learning_curve = train(net,
                                             X_train[:, np.newaxis, np.newaxis, :],
                                             Y_train,
                                             X_train[:, np.newaxis, np.newaxis, :],
                                             Y_train,
                                             num_epochs=int(config["num_epochs"]),
                                             lr_policy=config["lr_policy"],
                                             gamma=config["gamma"],
                                             power=config["power"],
                                             step=config["step"])
    print ("Learning Curve")
    print (learning_curve)

    # Compute predctions of this fold
    predictions_valid.append(lasagne.layers.get_output(net.network, X_valid[:, np.newaxis, np.newaxis, :], deterministic=True).eval() * weight)
    predictions_test.append(lasagne.layers.get_output(net.network, X_test[:, np.newaxis, np.newaxis, :], deterministic=True).eval() * weight)

# Output the predictions
for name, predictions in [('valid', predictions_valid),
                          ('test', predictions_test)]:
    predictions = np.array(predictions)
    predictions = np.sum(predictions, axis=0).astype(np.float32)

    filepath = os.path.join(output, '%s_%s_000.predict' % (dataset, name))
    np.savetxt(filepath, predictions, delimiter=' ', fmt='%.4e')

reason: import numpy,import theano


text:
import os
import sys
import numpy as np
import h5py

sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))

from ampere import SingleParticleFDSEI, SingleParticleFD, SingleParticleParabolic, PseudoTwoDimFD

def test_values_unchanged():
    filename = 'ampere/tests/test_data'
    with h5py.File(filename + ".hdf5", "r") as f:
        for model in [SingleParticleParabolic(), PseudoTwoDimFD(), SingleParticleFD(), SingleParticleFDSEI()]:

            print(model.__class__.__name__)
            data = model.discharge(current=1.0)
            name = f'data{model.__class__.__name__}_discharge'
            new = np.concatenate([[data.time], [data.voltage], [data.current]], axis=0)
            # f.create_dataset(name, data=new, compression="gzip")
            expected = f.get(name)
            np.testing.assert_almost_equal(new, expected)

            data = model.charge(current=1.0)
            name = f'data{model.__class__.__name__}_charge'
            new = np.concatenate([[data.time], [data.voltage], [data.current]], axis=0)
            # f.create_dataset(name, data=new, compression="gzip")
            expected = f.get(name)
            np.testing.assert_almost_equal(new, expected)

            data = model.discharge(current=1.0, internal=True)
            if data.internal is not None:
                name = f'data{model.__class__.__name__}_discharge_internal'
                # f.create_dataset(name, data=data.internal, compression="gzip")
                expected = f.get(name)
                np.testing.assert_almost_equal(data.internal, expected)

            data = model.piecewise_current([500,30,60,100], [1.0,-0.2,0.5,-2])
            name = f'data{model.__class__.__name__}_piecewise'
            new = np.concatenate([[data.time], [data.voltage], [data.current]], axis=0)
            # f.create_dataset(name, data=new, compression="gzip")
            expected = f.get(name)
            np.testing.assert_almost_equal(new, expected)

def test_optimization():
    labels = ['Rp', 'lp', 'kp']
    values = [2.0e-6, 78e-6, 2.334e-9]
    p1 = dict(zip(labels, values))

    spm1 = SingleParticleFDSEI()
    spm2 = SingleParticleFDSEI(initial_parameters=p1, estimate_parameters=['Rp', 'lp', 'kp'])

    # create the experimental data
    a = [
        spm1.discharge(current=0.5, trim=True),
        spm1.charge(current=1.0, trim=True)
    ]

    t_exp = [x.time for x in a]
    v_exp = [x.voltage for x in a]
    res = spm2.fit(t_exp, v_exp, currents=[0.5, -1.0], maxiter=1000)

    np.testing.assert_almost_equal(res.x, [7.84309206e-05, 2.01704867e-06, 2.36852421e-09])

def test_generate_data():
    spm = SingleParticleFDSEI()
    data = spm.generate_data(100, currents=[0.5, 1.0], parameters=['Rp', 'Rn'], bounds=[[1e-12, 1e-3], [1e-12, 1e-3]], sample_type='random')
    assert data.inputs.shape == (98, 2)
    assert data.voltages.shape == (98, 500, 2)
    assert data.failed.shape == (2, 2)

test_values_unchanged()
test_optimization()
test_generate_data()

max_stars_repo_name: nealde/Ampere

max_issues_repo_name: nealde/battery

max_forks_repo_name: nealde/battery

reason: import numpy


text:
#!/usr/bin/env python3
""" Handles the visible area of the :class:`~tools.manual.faceviewer.frame.FacesViewer` canvas. """

import logging
import tkinter as tk

import cv2
import numpy as np
from PIL import Image, ImageTk

from lib.align import AlignedFace

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

class Viewport():
    """ Handles the display of faces and annotations in the currently viewable area of the canvas.

    Parameters
    ----------
    canvas: :class:`tkinter.Canvas`
        The :class:`~tools.manual.faceviewer.frame.FacesViewer` canvas
    tk_edited_variable: :class:`tkinter.BooleanVar`
        The variable that indicates that a face has been edited
    """
    def __init__(self, canvas, tk_edited_variable):
        logger.debug("Initializing: %s: (canvas: %s, tk_edited_variable: %s)",
                     self.__class__.__name__, canvas, tk_edited_variable)
        self._canvas = canvas
        self._grid = canvas.grid
        self._centering = "face"
        self._tk_selected_editor = canvas._display_frame.tk_selected_action
        self._landmark_mapping = dict(mouth_inner=(60, 68),
                                      mouth_outer=(48, 60),
                                      right_eyebrow=(17, 22),
                                      left_eyebrow=(22, 27),
                                      right_eye=(36, 42),
                                      left_eye=(42, 48),
                                      nose=(27, 36),
                                      jaw=(0, 17),
                                      chin=(8, 11))
        self._landmarks = dict()
        self._tk_faces = dict()
        self._objects = VisibleObjects(self)
        self._hoverbox = HoverBox(self)
        self._active_frame = ActiveFrame(self, tk_edited_variable)
        self._tk_selected_editor.trace(
            "w", lambda *e: self._active_frame.reload_annotations())

    @property
    def face_size(self):
        """ int: The pixel size of each thumbnail """
        return self._grid.face_size

    @property
    def mesh_kwargs(self):
        """ dict: The color and state keyword arguments for the objects that make up a single
        face's mesh annotation based on the current user selected options. Key is the object
        type (`polygon` or `line`), value are the keyword arguments for that type. """
        state = "normal" if self._canvas.optional_annotations["mesh"] else "hidden"
        color = self._canvas.control_colors["Mesh"]
        kwargs = dict(polygon=dict(fill="", outline=color, state=state),
                      line=dict(fill=color, state=state))
        return kwargs

    @property
    def hover_box(self):
        """ :class:`HoverBox`: The hover box for the viewport. """
        return self._hoverbox

    @property
    def selected_editor(self):
        """ str: The currently selected editor. """
        return self._tk_selected_editor.get().lower()

    def toggle_mesh(self, state):
        """ Toggles the mesh optional annotations on and off.

        Parameters
        ----------
        state: ["hidden", "normal"]
            The state to set the mesh annotations to
        """
        logger.debug("Toggling mesh annotations to: %s", state)
        self._canvas.itemconfig("viewport_mesh", state=state)
        self.update()

    def toggle_mask(self, state, mask_type):
        """ Toggles the mask optional annotation on and off.

        Parameters
        ----------
        state: ["hidden", "normal"]
            Whether the mask should be displayed or hidden
        mask_type: str
            The type of mask to overlay onto the face
        """
        logger.debug("Toggling mask annotations to: %s. mask_type: %s", state, mask_type)
        for (frame_idx, face_idx), det_face in zip(
                self._objects.visible_grid[:2].transpose(1, 2, 0).reshape(-1, 2),
                self._objects.visible_faces.flatten()):
            if frame_idx == -1:
                continue

            key = "_".join([str(frame_idx), str(face_idx)])
            mask = None if state == "hidden" else self._obtain_mask(det_face, mask_type)
            self._tk_faces[key].update_mask(mask)
        self.update()

    @classmethod
    def _obtain_mask(cls, detected_face, mask_type):
        """ Obtain the mask for the correct "face" centering that is used in the thumbnail display.

        Parameters
        -----------
        detected_face: :class:`lib.align.DetectedFace`
            The Detected Face object to obtain the mask for
        mask_type: str
            The type of mask to obtain

        Returns
        -------
        :class:`numpy.ndarray` or ``None``
            The single channel mask of requested mask type, if it exists, otherwise ``None``
        """
        mask = detected_face.mask.get(mask_type)
        if not mask:
            return None
        if mask.stored_centering != "face":
            face = AlignedFace(detected_face.landmarks_xy)
            mask.set_sub_crop(face.pose.offset["face"] - face.pose.offset[mask.stored_centering],
                              centering="face")
        return mask.mask.squeeze()

    def reset(self):
        """ Reset all the cached objects on a face size change. """
        self._landmarks = dict()
        self._tk_faces = dict()

    def update(self, refresh_annotations=False):
        """ Update the viewport.

        Parameters
        ----------
        refresh_annotations: bool, optional
            ``True`` if mesh annotations should be re-calculated otherwise ``False``.
            Default: ``False``

        Obtains the objects that are currently visible. Updates the visible area of the canvas
        and reloads the active frame's annotations. """
        self._objects.update()
        self._update_viewport(refresh_annotations)
        self._active_frame.reload_annotations()

    def _update_viewport(self, refresh_annotations):
        """ Update the viewport

        Parameters
        ----------
        refresh_annotations: bool
            ``True`` if mesh annotations should be re-calculated otherwise ``False``

        Clear out cached objects that are not currently in view. Populate the cache for any
        faces that are now in view. Populate the correct face image and annotations for each
        object in the viewport based on current location. If optional mesh annotations are
        enabled, then calculates newly displayed meshes. """
        if not self._grid.is_valid:
            return
        self._discard_tk_faces()

        if self._canvas.optional_annotations["mesh"]:  # Display any hidden end of row meshes
            self._canvas.itemconfig("viewport_mesh", state="normal")

        for collection in zip(self._objects.visible_grid.transpose(1, 2, 0),
                              self._objects.images,
                              self._objects.meshes,
                              self._objects.visible_faces):
            for (frame_idx, face_idx, pnt_x, pnt_y), image_id, mesh_ids, face in zip(*collection):
                top_left = np.array((pnt_x, pnt_y))
                if frame_idx == self._active_frame.frame_index:
                    logger.trace("Skipping active frame: %s", frame_idx)
                    continue
                if frame_idx == -1:
                    logger.debug("Blanking non-existant face")
                    self._canvas.itemconfig(image_id, image="")
                    for area in mesh_ids.values():
                        for mesh_id in area:
                            self._canvas.itemconfig(mesh_id, state="hidden")
                    continue

                tk_face = self.get_tk_face(frame_idx, face_idx, face)
                self._canvas.itemconfig(image_id, image=tk_face.photo)
                if (self._canvas.optional_annotations["mesh"]
                        or frame_idx == self._active_frame.frame_index):
                    landmarks = self.get_landmarks(frame_idx, face_idx, face, top_left,
                                                   refresh=refresh_annotations)
                    self._locate_mesh(mesh_ids, landmarks)

    def _discard_tk_faces(self):
        """ Remove any :class:`TKFace` objects from the cache that are not currently displayed. """
        keys = ["{}_{}".format(pnt_x, pnt_y)
                for pnt_x, pnt_y in self._objects.visible_grid[:2].T.reshape(-1, 2)]
        for key in list(self._tk_faces):
            if key not in keys:
                del self._tk_faces[key]
        logger.trace("keys: %s allocated_faces: %s", keys, len(self._tk_faces))

    def get_tk_face(self, frame_index, face_index, face):
        """ Obtain the :class:`TKFace` object for the given face from the cache. If the face does
        not exist in the cache, then it is generated and added prior to returning.

        Parameters
        ----------
        frame_index: int
            The frame index to obtain the face for
        face_index: int
            The face index of the face within the requested frame
        face: :class:`~lib.align.DetectedFace`
            The detected face object, containing the thumbnail jpg

        Returns
        -------
        :class:`TKFace`
            An object for displaying in the faces viewer canvas populated with the aligned mesh
            landmarks and face thumbnail
        """
        is_active = frame_index == self._active_frame.frame_index
        key = "_".join([str(frame_index), str(face_index)])
        if key not in self._tk_faces or is_active:
            logger.trace("creating new tk_face: (key: %s, is_active: %s)", key, is_active)
            if is_active:
                image = AlignedFace(face.landmarks_xy,
                                    image=self._active_frame.current_frame,
                                    centering=self._centering,
                                    size=self.face_size).face
            else:
                image = AlignedFace(face.landmarks_xy,
                                    image=cv2.imdecode(face.thumbnail, cv2.IMREAD_UNCHANGED),
                                    centering=self._centering,
                                    size=self.face_size,
                                    is_aligned=True).face
            tk_face = self._get_tk_face_object(face, image, is_active)
            self._tk_faces[key] = tk_face
        else:
            logger.trace("tk_face exists: %s", key)
            tk_face = self._tk_faces[key]
        return tk_face

    def _get_tk_face_object(self, face, image, is_active):
        """ Obtain an existing unallocated, or a newly created :class:`TKFace` and populate it with
        face information from the requested frame and face index.

        If the face is currently active, then the face is generated from the currently displayed
        frame, otherwise it is generated from the jpg thumbnail.

        Parameters
        ----------
        face: :class:`lib.align.DetectedFace`
            A detected face object to create the :class:`TKFace` from
        image: :class:`numpy.ndarray`
            The jpg thumbnail or the 3 channel image for the face
        is_active: bool
            ``True`` if the face in the currently active frame otherwise ``False``

        Returns
        -------
        :class:`TKFace`
            An object for displaying in the faces viewer canvas populated with the aligned face
            image with a mask applied, if required.
        """
        get_mask = (self._canvas.optional_annotations["mask"] or
                    (is_active and self.selected_editor == "mask"))
        mask = self._obtain_mask(face, self._canvas.selected_mask) if get_mask else None
        tk_face = TKFace(image, size=self.face_size, mask=mask)
        logger.trace("face: %s, tk_face: %s", face, tk_face)
        return tk_face

    def get_landmarks(self, frame_index, face_index, face, top_left, refresh=False):
        """ Obtain the landmark points for each mesh annotation.

        First tries to obtain the aligned landmarks from the cache. If the landmarks do not exist
        in the cache, or a refresh has been requested, then the landmarks are calculated from the
        detected face object.

        Parameters
        ----------
        frame_index: int
            The frame index to obtain the face for
        face_index: int
            The face index of the face within the requested frame
        face: :class:`lib.align.DetectedFace`
            The detected face object to obtain landmarks for
        top_left: tuple
            The top left (x, y) points of the face's bounding box within the viewport
        refresh: bool, optional
            Whether to force a reload of the face's aligned landmarks, even if they already exist
            within the cache. Default: ``False``

        Returns
        -------
        dict
            The key is the tkinter canvas object type for each part of the mesh annotation
            (`polygon`, `line`). The value is a list containing the (x, y) coordinates of each
            part of the mesh annotation, from the top left corner location.
        """
        key = "{}_{}".format(frame_index, face_index)
        landmarks = self._landmarks.get(key, None)
        if not landmarks or refresh:
            aligned = AlignedFace(face.landmarks_xy,
                                  centering=self._centering,
                                  size=self.face_size)
            landmarks = dict(polygon=[], line=[])
            for area, val in self._landmark_mapping.items():
                points = aligned.landmarks[val[0]:val[1]] + top_left
                shape = "polygon" if area.endswith("eye") or area.startswith("mouth") else "line"
                landmarks[shape].append(points)
            self._landmarks[key] = landmarks
        return landmarks

    def _locate_mesh(self, mesh_ids, landmarks):
        """ Place the mesh annotation canvas objects in the correct location.

        Parameters
        ----------
        mesh_ids: list
            The list of mesh id objects to set coordinates for
        landmarks: dict
            The mesh point groupings and whether each group should be a line or a polygon
        """
        for key, area in landmarks.items():
            for coords, mesh_id in zip(area, mesh_ids[key]):
                self._canvas.coords(mesh_id, *coords.flatten())

    def face_from_point(self, point_x, point_y):
        """ Given an (x, y) point on the :class:`Viewport`, obtain the face information at that
        location.

        Parameters
        ----------
        point_x: int
            The x position on the canvas of the point to retrieve the face for
        point_y: int
            The y position on the canvas of the point to retrieve the face for

        Returns
        -------
        :class:`numpy.ndarray`
            Array of shape (4, ) containing the (`frame index`, `face index`, `x_point of top left
            corner`, `y point of top left corner`) of the face at the given coordinates.

            If the given coordinates are not over a face, then the frame and face indices will be
            -1
        """
        if point_x > self._grid.dimensions[0]:
            retval = np.array((-1, -1, -1, -1))
        else:
            x_idx = np.searchsorted(self._objects.visible_grid[2, 0, :], point_x, side="left") - 1
            y_idx = np.searchsorted(self._objects.visible_grid[3, :, 0], point_y, side="left") - 1
            if x_idx < 0 or y_idx < 0:
                retval = np.array((-1, -1, -1, -1))
            else:
                retval = self._objects.visible_grid[:, y_idx, x_idx]
        logger.trace(retval)
        return retval

    def move_active_to_top(self):
        """ Check whether the active frame is going off the bottom of the viewport, if so: move it
        to the top of the viewport. """
        self._active_frame.move_to_top()

class VisibleObjects():
    """ Holds the objects from the :class:`~tools.manual.faceviewer.frame.Grid` that appear in the
    viewable area of the :class:`Viewport`.

    Parameters
    ----------
    viewport: :class:`Viewport`
        The viewport object for the :class:`~tools.manual.faceviewer.frame.FacesViewer` canvas
    """
    def __init__(self, viewport):
        self._viewport = viewport
        self._canvas = viewport._canvas
        self._grid = viewport._grid
        self._size = viewport.face_size

        self._visible_grid = None
        self._visible_faces = None
        self._images = []
        self._meshes = []
        self._recycled = dict(images=[], meshes=[])

    @property
    def visible_grid(self):
        """ :class:`numpy.ndarray`: The currently visible section of the
        :class:`~tools.manual.faceviewer.frame.Grid`

        A numpy array of shape (`4`, `rows`, `columns`) corresponding to the viewable area of the
        display grid. 1st dimension contains frame indices, 2nd dimension face indices. The 3rd and
        4th dimension contain the x and y position of the top left corner of the face respectively.

        Any locations that are not populated by a face will have a frame and face index of -1. """
        return self._visible_grid

    @property
    def visible_faces(self):
        """ :class:`numpy.ndarray`: The currently visible :class:`~lib.align.DetectedFace`
        objects.

        A numpy array of shape (`rows`, `columns`) corresponding to the viewable area of the
        display grid and containing the detected faces at their currently viewable position.

        Any locations that are not populated by a face will have ``None`` in it's place. """
        return self._visible_faces

    @property
    def images(self):
        """ :class:`numpy.ndarray`: The viewport's tkinter canvas image objects.

        A numpy array of shape (`rows`, `columns`) corresponding to the viewable area of the
        display grid and containing the tkinter canvas image object for the face at the
        corresponding location. """
        return self._images

    @property
    def meshes(self):
        """ :class:`numpy.ndarray`: The viewport's tkinter canvas mesh annotation objects.

        A numpy array of shape (`rows`, `columns`) corresponding to the viewable area of the
        display grid and containing a dictionary of the corresponding tkinter polygon and line
        objects required to build a face's mesh annotation for the face at the corresponding
        location. """
        return self._meshes

    @property
    def _top_left(self):
        """ :class:`numpy.ndarray`: The canvas (`x`, `y`) position of the face currently in the
        viewable area's top left position. """
        return np.array(self._canvas.coords(self._images[0][0]), dtype="int")

    def update(self):
        """ Load and unload thumbnails in the visible area of the faces viewer. """
        self._visible_grid, self._visible_faces = self._grid.visible_area
        if (isinstance(self._images, np.ndarray) and isinstance(self._visible_grid, np.ndarray)
                and self._visible_grid.shape[-1] != self._images.shape[-1]):
            self._recycle_objects()

        required_rows = self._visible_grid.shape[1] if self._grid.is_valid else 0
        existing_rows = len(self._images)
        logger.trace("existing_rows: %s. required_rows: %s", existing_rows, required_rows)

        if existing_rows > required_rows:
            for image_id, mesh_ids in zip(self._images[required_rows: existing_rows].flatten(),
                                          self._meshes[required_rows: existing_rows].flatten()):
                logger.trace("Hiding image id: %s", image_id)
                self._canvas.itemconfig(image_id, image="")
                for ids in mesh_ids.values():
                    for mesh_id in ids:
                        self._canvas.itemconfig(mesh_id, state="hidden")

        if existing_rows < required_rows:
            self._add_rows(existing_rows, required_rows)

        self._shift()

    def _recycle_objects(self):
        """  On a column count change, place all existing objects into the recycle bin so that
        they can be used for the new grid shape and reset the objects size to the new size. """
        self._size = self._viewport.face_size
        images = self._images.flatten().tolist()
        meshes = self._meshes.flatten().tolist()

        for image_id in images:
            self._canvas.itemconfig(image_id, image="")
            self._canvas.coords(image_id, 0, 0)
        for mesh in meshes:
            for key, mesh_ids in mesh.items():
                coords = (0, 0, 0, 0) if key == "line" else (0, 0)
                for mesh_id in mesh_ids:
                    self._canvas.coords(mesh_id, *coords)

        self._recycled["images"].extend(images)
        self._recycled["meshes"].extend(meshes)
        logger.trace("Recycled objects: %s", self._recycled)

        self._images = []
        self._meshes = []

    def _add_rows(self, existing_rows, required_rows):
        """ Add rows to the viewport.

        Parameters
        ----------
        existing_rows: int
            The number of existing rows within the viewport
        required_rows: int
            The number of rows required by the viewport
        """
        columns = self._grid.columns_rows[0]
        if not isinstance(self._images, np.ndarray):
            base_coords = [(col * self._size, 0) for col in range(columns)]
        else:
            base_coords = [self._canvas.coords(item_id) for item_id in self._images[0]]
        logger.debug("existing rows: %s, required_rows: %s, base_coords: %s",
                     existing_rows, required_rows, base_coords)
        images = []
        meshes = []
        for row in range(existing_rows, required_rows):
            y_coord = base_coords[0][1] + (row * self._size)
            images.append(np.array([self._get_image((coords[0], y_coord))
                                    for coords in base_coords]))
            meshes.append(np.array([self._get_mesh() for _ in range(columns)]))
        images = np.array(images)
        meshes = np.array(meshes)

        if not isinstance(self._images, np.ndarray):
            logger.debug("Adding initial viewport objects: (image shapes: %s, mesh shapes: %s)",
                         images.shape, meshes.shape)
            self._images = images
            self._meshes = meshes
        else:
            logger.debug("Adding new viewport objects: (image shapes: %s, mesh shapes: %s)",
                         images.shape, meshes.shape)
            self._images = np.concatenate((self._images, images))
            self._meshes = np.concatenate((self._meshes, meshes))
        logger.debug("self._images: %s, self._meshes: %s", self._images.shape, self._meshes.shape)

    def _get_image(self, coordinates):
        """ Create or recycle a tkinter canvas image object with the given coordinates.

        Parameters
        ----------
        coordinates: tuple
            The (`x`, `y`) coordinates for the top left corner of the image

        Returns
        -------
        int
            The canvas object id for the created image
        """
        if self._recycled["images"]:
            image_id = self._recycled["images"].pop()
            self._canvas.coords(image_id, *coordinates)
            logger.trace("Recycled image: %s", image_id)
        else:
            image_id = self._canvas.create_image(*coordinates,
                                                 anchor=tk.NW,
                                                 tags=["viewport", "viewport_image"])
            logger.trace("Created new image: %s", image_id)
        return image_id

    def _get_mesh(self):
        """ Get the mesh annotation for the landmarks. This is made up of a series of polygons
        or lines, depending on which part of the face is being annotated. Creates a new series of
        objects, or pulls existing objects from the recycled objects pool if they are available.

        Returns
        -------
        dict
            The dictionary of line and polygon tkinter canvas object ids for the mesh annotation
        """
        kwargs = self._viewport.mesh_kwargs
        logger.trace("self.mesh_kwargs: %s", kwargs)
        if self._recycled["meshes"]:
            mesh = self._recycled["meshes"].pop()
            for key, mesh_ids in mesh.items():
                for mesh_id in mesh_ids:
                    self._canvas.itemconfig(mesh_id, **kwargs[key])
            logger.trace("Recycled mesh: %s", mesh)
        else:
            tags = ["viewport", "viewport_mesh"]
            mesh = dict(polygon=[self._canvas.create_polygon(0, 0,
                                                             width=1,
                                                             tags=tags + ["viewport_polygon"],
                                                             **kwargs["polygon"])
                                 for _ in range(4)],
                        line=[self._canvas.create_line(0, 0, 0, 0,
                                                       width=1,
                                                       tags=tags + ["viewport_line"],
                                                       **kwargs["line"])
                              for _ in range(5)])
            logger.trace("Created new mesh: %s", mesh)
        return mesh

    def _shift(self):
        """ Shift the viewport in the y direction if required

        Returns
        -------
        bool
            ``True`` if the viewport was shifted otherwise ``False``
        """
        current_y = self._top_left[1]
        required_y = self._visible_grid[3, 0, 0] if self._grid.is_valid else 0
        logger.trace("current_y: %s, required_y: %s", current_y, required_y)
        if current_y == required_y:
            logger.trace("No move required")
            return False
        shift_amount = required_y - current_y
        logger.trace("Shifting viewport: %s", shift_amount)
        self._canvas.move("viewport", 0, shift_amount)
        return True

class HoverBox():  # pylint:disable=too-few-public-methods
    """ Handle the current mouse location when over the :class:`Viewport`.

    Highlights the face currently underneath the cursor and handles actions when clicking
    on a face.

    Parameters
    ----------
    viewport: :class:`Viewport`
        The viewport object for the :class:`~tools.manual.faceviewer.frame.FacesViewer` canvas
    """
    def __init__(self, viewport):
        logger.debug("Initializing: %s (viewport: %s)", self.__class__.__name__, viewport)
        self._viewport = viewport
        self._canvas = viewport._canvas
        self._grid = viewport._canvas.grid
        self._globals = viewport._canvas._globals
        self._navigation = viewport._canvas._display_frame.navigation
        self._box = self._canvas.create_rectangle(0, 0, self._size, self._size,
                                                  outline="#0000ff",
                                                  width=2,
                                                  state="hidden",
                                                  fill="#0000ff",
                                                  stipple="gray12",
                                                  tags="hover_box")
        self._current_frame_index = None
        self._current_face_index = None
        self._canvas.bind("<Leave>", lambda e: self._clear())
        self._canvas.bind("<Motion>", self.on_hover)
        self._canvas.bind("<ButtonPress-1>", lambda e: self._select_frame())
        logger.debug("Initialized: %s", self.__class__.__name__)

    @property
    def _size(self):
        """ int: the currently set viewport face size in pixels. """
        return self._viewport.face_size

    def on_hover(self, event):
        """ Highlight the face and set the mouse cursor for the mouse's current location.

        Parameters
        ----------
        event: :class:`tkinter.Event` or ``None``
            The tkinter mouse event. Provides the current location of the mouse cursor. If ``None``
            is passed as the event (for example when this function is being called outside of a
            mouse event) then the location of the cursor will be calculated
        """
        if event is None:
            pnts = np.array((self._canvas.winfo_pointerx(), self._canvas.winfo_pointery()))
            pnts -= np.array((self._canvas.winfo_rootx(), self._canvas.winfo_rooty()))
        else:
            pnts = (event.x, event.y)

        coords = (int(self._canvas.canvasx(pnts[0])), int(self._canvas.canvasy(pnts[1])))
        face = self._viewport.face_from_point(*coords)
        frame_idx, face_idx = face[:2]

        if frame_idx == self._current_frame_index and face_idx == self._current_face_index:
            return

        is_zoomed = self._globals.is_zoomed
        if (-1 in face or (frame_idx == self._globals.frame_index
                           and (not is_zoomed or
                                (is_zoomed and face_idx == self._globals.tk_face_index.get())))):
            self._clear()
            self._canvas.config(cursor="")
            self._current_frame_index = None
            self._current_face_index = None
            return

        logger.debug("Viewport hover: frame_idx: %s, face_idx: %s", frame_idx, face_idx)

        self._canvas.config(cursor="hand2")
        self._highlight(face[2:])
        self._current_frame_index = frame_idx
        self._current_face_index = face_idx

    def _clear(self):
        """ Hide the hover box when the mouse is not over a face. """
        if self._canvas.itemcget(self._box, "state") != "hidden":
            self._canvas.itemconfig(self._box, state="hidden")

    def _highlight(self, top_left):
        """ Display the hover box around the face that the mouse is currently over.

        Parameters
        ----------
        top_left: tuple
            The top left point of the highlight box location
        """
        coords = (*top_left, *top_left + self._size)
        self._canvas.coords(self._box, *coords)
        self._canvas.itemconfig(self._box, state="normal")
        self._canvas.tag_raise(self._box)

    def _select_frame(self):
        """ Select the face and the subsequent frame (in the editor view) when a face is clicked
        on in the :class:`Viewport`.
        """
        frame_id = self._current_frame_index
        is_zoomed = self._globals.is_zoomed
        logger.debug("Face clicked. Global frame index: %s, Current frame_id: %s, is_zoomed: %s",
                     self._globals.frame_index, frame_id, is_zoomed)
        if frame_id is None or (frame_id == self._globals.frame_index and not is_zoomed):
            return
        face_idx = self._current_face_index if is_zoomed else 0
        self._globals.tk_face_index.set(face_idx)
        transport_id = self._grid.transport_index_from_frame(frame_id)
        logger.trace("frame_index: %s, transport_id: %s, face_idx: %s",
                     frame_id, transport_id, face_idx)
        if transport_id is None:
            return
        self._navigation.stop_playback()
        self._globals.tk_transport_index.set(transport_id)
        self._viewport.move_active_to_top()
        self.on_hover(None)

class ActiveFrame():
    """ Handles the display of faces and annotations for the currently active frame.

    Parameters
    ----------
    canvas: :class:`tkinter.Canvas`
        The :class:`~tools.manual.faceviewer.frame.FacesViewer` canvas
    tk_edited_variable: :class:`tkinter.BooleanVar`
        The tkinter callback variable indicating that a face has been edited
    """
    def __init__(self, viewport, tk_edited_variable):
        logger.debug("Initializing: %s (viewport: %s, tk_edited_variable: %s)",
                     self.__class__.__name__, viewport, tk_edited_variable)
        self._objects = viewport._objects
        self._viewport = viewport
        self._grid = viewport._grid
        self._tk_faces = viewport._tk_faces
        self._canvas = viewport._canvas
        self._globals = viewport._canvas._globals
        self._navigation = viewport._canvas._display_frame.navigation
        self._last_execution = dict(frame_index=-1, size=viewport.face_size)
        self._tk_vars = dict(selected_editor=self._canvas._display_frame.tk_selected_action,
                             edited=tk_edited_variable)
        self._assets = dict(images=[], meshes=[], faces=[], boxes=[])

        self._globals.tk_update_active_viewport.trace("w", lambda *e: self._reload_callback())
        tk_edited_variable.trace("w", lambda *e: self._update_on_edit())
        logger.debug("Initialized: %s", self.__class__.__name__)

    @property
    def frame_index(self):
        """ int: The frame index of the currently displayed frame. """
        return self._globals.frame_index

    @property
    def current_frame(self):
        """ :class:`numpy.ndarray`: A BGR version of the frame currently being displayed. """
        return self._globals.current_frame["image"]

    @property
    def _size(self):
        """ int: The size of the thumbnails displayed in the viewport, in pixels. """
        return self._viewport.face_size

    @property
    def _optional_annotations(self):
        """ dict: The currently selected optional annotations """
        return self._canvas.optional_annotations

    def _reload_callback(self):
        """ If a frame has changed, triggering the variable, then update the active frame. Return
        having done nothing if the variable is resetting. """
        if self._globals.tk_update_active_viewport.get():
            self.reload_annotations()

    def reload_annotations(self):
        """ Handles the reloading of annotations for the currently active faces.

        Highlights the faces within the viewport of those faces that exist in the currently
        displaying frame. Applies annotations based on the optional annotations and current
        editor selections.
        """
        logger.trace("Reloading annotations")
        if np.any(self._assets["images"]):
            self._clear_previous()

        self._set_active_objects()
        self._check_active_in_view()

        if not np.any(self._assets["images"]):
            logger.trace("No active faces. Returning")
            self._last_execution["frame_index"] = self.frame_index
            return

        if self._last_execution["frame_index"] != self.frame_index:
            self.move_to_top()
        self._create_new_boxes()

        self._update_face()
        self._canvas.tag_raise("active_highlighter")
        self._globals.tk_update_active_viewport.set(False)
        self._last_execution["frame_index"] = self.frame_index

    def _clear_previous(self):
        """ Reverts the previously selected annotations to their default state. """
        logger.trace("Clearing previous active frame")
        self._canvas.itemconfig("active_highlighter", state="hidden")

        for key in ("polygon", "line"):
            tag = "active_mesh_{}".format(key)
            self._canvas.itemconfig(tag, **self._viewport.mesh_kwargs[key], width=1)
            self._canvas.dtag(tag)

        if self._viewport.selected_editor == "mask" and not self._optional_annotations["mask"]:
            for key, tk_face in self._tk_faces.items():
                if key.startswith("{}_".format(self._last_execution["frame_index"])):
                    tk_face.update_mask(None)

    def _set_active_objects(self):
        """ Collect the objects that exist in the currently active frame from the main grid. """
        if self._grid.is_valid:
            rows, cols = np.where(self._objects.visible_grid[0] == self.frame_index)
            logger.trace("Setting active objects: (rows: %s, columns: %s)", rows, cols)
            self._assets["images"] = self._objects.images[rows, cols]
            self._assets["meshes"] = self._objects.meshes[rows, cols]
            self._assets["faces"] = self._objects.visible_faces[rows, cols]
        else:
            logger.trace("No valid grid. Clearing active objects")
            self._assets["images"] = []
            self._assets["meshes"] = []
            self._assets["faces"] = []

    def _check_active_in_view(self):
        """  If the frame has changed, there are faces in the frame, but they don't appear in the
        viewport, then bring the active faces to the top of the viewport. """
        if (not np.any(self._assets["images"]) and
                self._last_execution["frame_index"] != self.frame_index and
                self._grid.frame_has_faces(self.frame_index)):
            y_coord = self._grid.y_coord_from_frame(self.frame_index)
            logger.trace("Active not in view. Moving to: %s", y_coord)
            self._canvas.yview_moveto(y_coord / self._canvas.bbox("backdrop")[3])
            self._viewport.update()

    def move_to_top(self):
        """ Move the currently selected frame's faces to the top of the viewport if they are moving
        off the bottom of the viewer. """
        height = self._canvas.bbox("backdrop")[3]
        bot = int(self._canvas.coords(self._assets["images"][-1])[1] + self._size)

        y_top, y_bot = (int(round(pnt * height)) for pnt in self._canvas.yview())

        if y_top < bot < y_bot:  # bottom face is still in fully visible area
            logger.trace("Active faces in frame. Returning")
            return

        top = int(self._canvas.coords(self._assets["images"][0])[1])
        if y_top == top:
            logger.trace("Top face already on top row. Returning")
            return

        if self._canvas.winfo_height() > self._size:
            logger.trace("Viewport taller than single face height. Moving Active faces to top: %s",
                         top)
            self._canvas.yview_moveto(top / height)
            self._viewport.update()
        elif self._canvas.winfo_height() <= self._size and y_top != top:
            logger.trace("Viewport shorter than single face height. Moving Active faces to "
                         "top: %s", top)
            self._canvas.yview_moveto(top / height)
            self._viewport.update()

    def _create_new_boxes(self):
        """ The highlight boxes (border around selected faces) are the only additional annotations
        that are required for the highlighter. If more faces are displayed in the current frame
        than highlight boxes are available, then new boxes are created to accommodate the
        additional faces. """
        new_boxes_count = max(0, len(self._assets["images"]) - len(self._assets["boxes"]))
        if new_boxes_count == 0:
            return
        logger.debug("new_boxes_count: %s", new_boxes_count)
        for _ in range(new_boxes_count):
            box = self._canvas.create_rectangle(0,
                                                0,
                                                self._viewport.face_size, self._viewport.face_size,
                                                outline="#00FF00",
                                                width=2,
                                                state="hidden",
                                                tags=["active_highlighter"])
            logger.trace("Created new highlight_box: %s", box)
            self._assets["boxes"].append(box)

    def _update_on_edit(self):
        """ Update the active faces on a frame edit. """
        if not self._tk_vars["edited"].get():
            return
        self._set_active_objects()
        self._update_face()
        self._tk_vars["edited"].set(False)

    def _update_face(self):
        """ Update the highlighted annotations for faces in the currently selected frame. """
        for face_idx, (image_id, mesh_ids, box_id, det_face), in enumerate(
                zip(self._assets["images"],
                    self._assets["meshes"],
                    self._assets["boxes"],
                    self._assets["faces"])):
            top_left = np.array(self._canvas.coords(image_id))
            coords = (*top_left, *top_left + self._size)
            tk_face = self._viewport.get_tk_face(self.frame_index, face_idx, det_face)
            self._canvas.itemconfig(image_id, image=tk_face.photo)
            self._show_box(box_id, coords)
            self._show_mesh(mesh_ids, face_idx, det_face, top_left)
        self._last_execution["size"] = self._viewport.face_size

    def _show_box(self, item_id, coordinates):
        """ Display the highlight box around the given coordinates.

        Parameters
        ----------
        item_id: int
            The tkinter canvas object identifier for the highlight box
        coordinates: :class:`numpy.ndarray`
            The (x, y, x1, y1) coordinates of the top left corner of the box
        """
        self._canvas.coords(item_id, *coordinates)
        self._canvas.itemconfig(item_id, state="normal")

    def _show_mesh(self, mesh_ids, face_index, detected_face, top_left):
        """ Display the mesh annotation for the given face, at the given location.

        Parameters
        ----------
        mesh_ids: dict
            Dictionary containing the `polygon` and `line` tkinter canvas identifiers that make up
            the mesh for the given face
        face_index: int
            The face index within the frame for the given face
        detected_face: :class:`~lib.align.DetectedFace`
            The detected face object that contains the landmarks for generating the mesh
        top_left: tuple
            The (x, y) top left co-ordinates of the mesh's bounding box
        """
        state = "normal" if (self._tk_vars["selected_editor"].get() != "Mask" or
                             self._optional_annotations["mesh"]) else "hidden"
        kwargs = dict(polygon=dict(fill="", width=2, outline=self._canvas.control_colors["Mesh"]),
                      line=dict(fill=self._canvas.control_colors["Mesh"], width=2))

        edited = (self._tk_vars["edited"].get() and
                  self._tk_vars["selected_editor"].get() not in ("Mask", "View"))
        landmarks = self._viewport.get_landmarks(self.frame_index,
                                                 face_index,
                                                 detected_face,
                                                 top_left,
                                                 edited)
        for key, kwarg in kwargs.items():
            for idx, mesh_id in enumerate(mesh_ids[key]):
                self._canvas.coords(mesh_id, *landmarks[key][idx].flatten())
                self._canvas.itemconfig(mesh_id, state=state, **kwarg)
                self._canvas.addtag_withtag("active_mesh_{}".format(key), mesh_id)

class TKFace():
    """ An object that holds a single :class:`tkinter.PhotoImage` face, ready for placement in the
    :class:`Viewport`, Handles the placement of and removal of masks for the face as well as
    updates on any edits.

    Parameters
    ----------
    face: :class:`numpy.ndarray`
        The face, sized correctly as a 3 channel BGR image or an encoded jpg to create a
        :class:`tkinter.PhotoImage` from
    size: int, optional
        The pixel size of the face image. Default: `128`
    mask: :class:`numpy.ndarray` or ``None``, optional
        The mask to be applied to the face image. Pass ``None`` if no mask is to be used.
        Default ``None``
    """
    def __init__(self, face, size=128, mask=None):
        logger.trace("Initializing %s: (face: %s, size: %s, mask: %s)",
                     self.__class__.__name__,
                     face if face is None else face.shape,
                     size,
                     mask if mask is None else mask.shape)
        self._size = size
        if face.ndim == 2 and face.shape[1] == 1:
            self._face = self._image_from_jpg(face)
        else:
            self._face = face[..., 2::-1]
        self._photo = ImageTk.PhotoImage(self._generate_tk_face_data(mask))

        logger.trace("Initialized %s", self.__class__.__name__)

    # << PUBLIC PROPERTIES >> #
    @property
    def photo(self):
        """ :class:`tkinter.PhotoImage`: The face in a format that can be placed on the
         :class:`~tools.manual.faceviewer.frame.FacesViewer` canvas. """
        return self._photo

    # << PUBLIC METHODS >> #
    def update(self, face, mask):
        """ Update the :attr:`photo` with the given face and mask.

        Parameters
        ----------
        face: :class:`numpy.ndarray`
            The face, sized correctly as a 3 channel BGR image
        mask: :class:`numpy.ndarray` or ``None``
            The mask to be applied to the face image. Pass ``None`` if no mask is to be used
        """
        self._face = face[..., 2::-1]
        self._photo.paste(self._generate_tk_face_data(mask))

    def update_mask(self, mask):
        """ Update the mask in the 4th channel of :attr:`photo` to the given mask.

        Parameters
        ----------
        mask: :class:`numpy.ndarray` or ``None``
            The mask to be applied to the face image. Pass ``None`` if no mask is to be used
        """
        self._photo.paste(self._generate_tk_face_data(mask))

    # << PRIVATE METHODS >> #
    def _image_from_jpg(self, face):
        """ Convert an encoded jpg into 3 channel BGR image.

        Parameters
        ----------
        face: :class:`numpy.ndarray`
            The encoded jpg as a two dimension numpy array

        Returns
        -------
        :class:`numpy.ndarray`
            The decoded jpg as a 3 channel BGR image
        """
        face = cv2.imdecode(face, cv2.IMREAD_UNCHANGED)
        interp = cv2.INTER_CUBIC if face.shape[0] < self._size else cv2.INTER_AREA
        if face.shape[0] != self._size:
            face = cv2.resize(face, (self._size, self._size), interpolation=interp)
        return face[..., 2::-1]

    def _generate_tk_face_data(self, mask):
        """ Create the :class:`tkinter.PhotoImage` from the currant :attr:`_face`.

        Parameters
        ----------
        mask: :class:`numpy.ndarray` or ``None``
            The mask to add to the image. ``None`` if a mask is not being used

        Returns
        -------
        :class:`tkinter.PhotoImage`
            The face formatted for the  :class:`~tools.manual.faceviewer.frame.FacesViewer` canvas.
        """
        mask = np.ones(self._face.shape[:2], dtype="uint8") * 255 if mask is None else mask
        if mask.shape[0] != self._size:
            mask = cv2.resize(mask, self._face.shape[:2], interpolation=cv2.INTER_AREA)
        img = np.concatenate((self._face, mask[..., None]), axis=-1)
        return Image.fromarray(img)

max_stars_repo_name: erteck/textHighlighter

max_issues_repo_name: erteck/textHighlighter

max_forks_repo_name: erteck/textHighlighter


text:
import random

import numpy

import boards

def print_sudoku_board(board: list):
    for i in range(9):
        if i % 3 == 0:
            print("-------------------------")
        for j in range(9):
            if j % 3 == 0:
                print("|", end=" "),
            print(board[i][j], end=" ")
        print("|")
    print("-------------------------")

def get_first_empty_cell(board: list):
    for i in range(9):
        for j in range(9):
            if board[i][j] == "_":
                return i, j
    return None

def check_board(board: list) -> bool:
    # check rows
    for i in range(9):
        hist = []
        for j in range(9):
            if board[i][j] in hist and board[i][j] != "_":
                return False
            hist.append(board[i][j])

    # check cols
    for j in range(9):
        hist = []
        for i in range(9):
            if board[i][j] in hist and board[i][j] != "_":
                return False
            hist.append(board[i][j])

    # check squares
    for sq in range(9):
        x = sq // 3
        y = sq % 3
        hist = []
        for i in range(x * 3, x * 3 + 3):
            for j in range(y * 3, y * 3 + 3):
                if board[i][j] in hist and board[i][j] != "_":
                    return False
                hist.append(board[i][j])
    return True

def check_pos(board: list, y, x, num) -> bool:
    # check rows
    for j in range(9):
        if board[x][j] == num:
            return False

    # check cols
    for i in range(9):
        if board[i][y] == num:
            return False

    # check squares
    x = x // 3
    y = y % 3
    for i in range(x * 3, x * 3 + 3):
        for j in range(y * 3, y * 3 + 3):
            if board[i][j] == num:
                return False
    return True

def solve_board(board, tries=None):
    if tries is None:
        tries = []
    cell = get_first_empty_cell(board)
    if not cell:  # no empty cells -> finished
        return True
    else:
        for i in range(1, 10):
            board[cell[0]][cell[1]] = i
            if check_board(board): # check_pos cell[0], cell[1], i
                tries.append(i)
                solvable = solve_board(board, tries)
                if solvable:
                    return True
            board[cell[0]][cell[1]] = "_"
        return False

board_num = random.randint(0, 31)
sudoku_board = boards.boards[board_num]
ran = random.randint(0, 2)
if ran == 2:
    list(map(list, zip(*sudoku_board)))
elif ran == 1:
    sudoku_board = sudoku_board[::-1]


text:
import numpy as np
import numpy.random as npr

class MallowsPermutation(object):

    def __init__(self, n, q, seed=None):
        '''
        Create a random Mallows permutation with parameters 'n' and 'q'.
        '''
        self.n = n
        self.q = q
        self.seed = seed
        npr.seed(self.seed)

        self.params = 'n={};q={}'.format(self.n,self.q)

        self.__permutation__()

    def __permutation__(self):
        '''
        Generates the permutation.
        It first creates a vector 'G' such that 0 <= G[i] <= n-1-i and G[i] ~ Geometric(q).
        This vector is then used to create the permutation by choosing the image of i as the G[i]-th available value.
        '''
        if self.q == 0:
            self.permutation = np.arange(self.n)
        elif self.q_is_inf():
            self.permutation = np.arange(self.n-1, -1, -1)

        else:
            U = npr.rand(self.n)
            N = np.arange(self.n, 0, -1)

            if self.q == 1:
                G = np.floor(N*U).astype(int)
            else:
                G = np.floor(np.log(1 - U*(1 - self.q**N))/np.log(self.q)).astype(int)

            available_values = np.arange(self.n)
            self.permutation = np.zeros(self.n)
            for i in range(self.n):
                self.permutation[i] = available_values[G[i]]
                available_values = np.delete(available_values, G[i])

            self.permutation = np.argsort(self.permutation).astype(int)

    def __getitem__(self, key):
        '''
        Calling 'MallowsPermutation()[i]' returns the image of i through the permutation.
        '''
        return self.permutation[key]

    def __iter__(self):
        '''
        Iterating through 'MallowsPermutation()' goes through the images of 0, 1, 2, ...
        '''
        return iter(self.permutation)

    def q_is_inf(self):
        '''
        Checks if 'q' is infinity.
        '''
        return self.q > max(self.n**2, 1e10)

class MallowsTree(MallowsPermutation):

    def __init__(self, n, q, seed=None):
        '''
        Creates a random Mallows tree from a Mallows permutation.
        '''
        super().__init__(n, q, seed)
        self.__tree__()

    def __tree__(self):
        '''
        This function creates the tree corresponding to the permutation.
        It explores the entries of the permutation and insert them one after the other in the tree.
        The parameter 'self.int_to_node' is a dictionary mapping the entries to their nodes.
        The parameter 'self.node_to_int' is a dictionary mapping the nodes to their entries.
        The parameter 'self.E' is a list of edges of the tree.
        '''
        self.int_to_node = {}
        self.node_to_int = {}
        self.E = []

        for v in self:
            node_v = ''
            parent = -1
            while node_v in self.node_to_int:
                parent = self.node_to_int[node_v]
                if v > parent:
                    node_v += '1'
                else:
                    node_v += '0'
            self.int_to_node[v] = node_v
            self.node_to_int[node_v] = v
            if parent >= 0:
                self.E.append((parent, v))

    def __call__(self, key):
        '''
        Calling 'MallowsTree()(i)' returns the node of i.
        '''
        return self.int_to_node[key]

    def height(self):
        '''
        Computes the height of the tree.
        '''
        return max([len(m) for m in self.int_to_node.values()])

    def depth(self, v):
        '''
        Computes the depth of 'v'.
        '''
        return len(self.int_to_node[v])

    def parent(self, v):
        '''
        Finds the parent of 'v' in the tree.
        '''
        node = self.int_to_node[v]
        if node:
            return self.node_to_int[node[:-1]]
        else:
            raise Exception('The root has no parent')

    def path_from_root(self, v, edge_list=False):
        '''
        Computes the path from the root to the node 'v'.
        Either as a list of nodes, or as a list of edges.
        '''
        path = [self.node_to_int['']]
        node_v = self.int_to_node[v]
        for i in range(len(node_v)):
            path.append(self.node_to_int[node_v[:i+1]])

        if edge_list:
            edge_path = []
            if len(path) > 1:
                u = path[0]
                for v in path[1:]:
                    edge_path.append((u,v))
                    u = v
            return edge_path

        else:
            return path

    @staticmethod
    def c_star():
        '''
        Computes c*, solution to clog(2e/c)=1 with c>=2.
        This corresponds to the ratio height of random binary search tree over log(n).
        '''
        return 4.311070407001

max_stars_repo_name: BenoitCorsini/mallows-trees

max_issues_repo_name: BenoitCorsini/mallows-trees

max_forks_repo_name: BenoitCorsini/mallows-trees