get_ipython().system(u'pip install --upgrade --no-deps https://cntk.ai/PythonWheel/CPU-Only/cntk-2.4-cp35-cp35m-linux_x86_64.whl')

import cntk as C
print ("Using Microsoft Cognitive Toolkit version {}".format(C.__version__))

import numpy as np
print ("Using numpy version {}".format(np.__version__))

# In[2]:

import matplotlib.pyplot as plt
get_ipython().magic(u'matplotlib inline')
import time
import sys
import os
import cv2
from IPython.display import Image, display #, SVG
from cntk.learners import learning_rate_schedule, UnitType
from cntk.initializer import he_normal
from cntk.layers import AveragePooling, BatchNormalization, Convolution, Dense
from cntk.ops import element_times, relu, sigmoid
from cntk import load_model, placeholder, Constant
from tqdm import *          # tqdm allows us to pretty-print our loops later on...
import helper # some functions to plot images
import coco   # local class to read the COCO images and labels
import gc # Python garbage collector

# Paths relative to current python file.
abs_path  = os.path.dirname(os.path.abspath('.'))
data_path = os.path.join(abs_path, "../../data/M4")
zip_path = os.path.join(abs_path, "../data-zip")
output_path = os.path.join(abs_path, "lab1/output")

# Now, let's setup and call our function to setup the large dataset that is  needed for this lab.
#
# Why are we doing this? We need this step because Azure Notebooks has two storage areas: a persistent but slow area, and a transient but fast area. To ensure that this lab is able to execute as fast as possible on Azure Notebooks, we unzip our data set into this transient area once per session.  If it has already been unzipped then we'll automatically detect this and skip the step, so it is safe to run this next code block at any time.
#
# Depending on how you are executing the lab, this step can take a minute or two.  The output you should expect to see from the following code block will look something like:
#
#     Start unzipping data files in /home/nbuser/library/Module4/../data-zip to /home/nbuser/library/Module4/../../data/M4
#     zip file count:1
#     Destination folder /home/nbuser/library/Module4/../../data/M4 doesn't exist, creating it
#     Start unzipping files to /home/nbuser/library/Module4/../../data/M4
#        unzipping CocoMS.zip ...
#     Complete: Files have been unzipped to /home/nbuser/library/Module4/../../data/M4

# In[3]:

import sys
import os
import zipfile
import fnmatch

def hydrate(zip_path, dest_path):
    print("Start unzipping data files in {0} to {1}".format(zip_path,dest_path))
    if (os.path.exists(zip_path) == False):
        print("The source folder {0} doesn't exist, so quitting".format(zip_path))
        quit()

    zipfile_count = len(fnmatch.filter(os.listdir(zip_path), '*.zip'))
    if (zipfile_count == 0):
        print("No zip (.zip) files in {0}, so quitting ".format(zip_path))

    print("zip file count:%s" % zipfile_count)

    if (os.path.exists(dest_path) == False):
        print("Destination folder {0} doesn't exist, creating it".format(dest_path))
        os.makedirs(dest_path)

        # Extract all zip files from zip_path to dest_path
        print("Start unzipping files to {0}".format(dest_path))
        for item in os.listdir(zip_path): # loop through items in dir
            if item.endswith(".zip"): # check for ".zip" extension
                print("   unzipping {0} ...".format(item))
                file_name = os.path.join(zip_path,item) # get full path of files
                zip_ref = zipfile.ZipFile(file_name) # create zipfile object
                zip_ref.extractall(dest_path) # extract file to dir
                zip_ref.close() # close file
    else:
        print("data folder already populated")

    print("Complete: Files have been unzipped to {0}".format(dest_path))

hydrate(zip_path, data_path)

# In the next code block, we check to see whether the Microsoft Cognitive Toolkit can use a GPU for its work. We are going to use the best available device (GPU, if available, else CPU).

# In[4]:

try:
    isUsingGPU # if this is our first time running, this will cause an exception as undefined
except NameError:
    try:
        isUsingGPU = C.device.try_set_default_device(C.device.gpu(0))
    except ValueError:
        isUsingGPU = False
        C.device.try_set_default_device(C.device.cpu())

print ("The Cognitive Toolkit is using the {} for processing.".format("GPU" if isUsingGPU else "CPU"))

# Next, we're going to implement functions to split our data into minibatches. We discussed in Module 3, Lab 2 how minibatches offer a number of advantages over single batch for training.
#
# To split our data into minibatches, we will use Python's array slice notation.
#
# By way of a quick explanation of this, the notation works as follows:
#
#     array[start:end] # items start through end-1
#     array[start:]    # items start through the rest of the array
#     array[:end]      # items from the beginning through end-1
#     array[:]         # a copy of the whole array
#
# So, we'll slice our training data (images in `data_x`, pixel-labels in `data_y`) into `minibatch_size` chunks, where `i` is the chunk number.
#
# **Add your code here to replace the commented section in the cell below** -- your code needs to take `data_x` and slice it into `sx` and `data_y` and slice it into `sy`.  As a hint, for each slice, you'll want to take elements from `i * minibatch_size` (inclusive) to `(i + 1) * minibatch_size` (exclusive), using the Python array slice notation explained above.

# In[7]:

def slice_minibatch(data_x, data_y, i, minibatch_size):
    sx = []
    sy = []
    # Use the Python array slice notation the divide up the dataset into minibatches
    #
    # *** YOUR CODE HERE ***
    # Set sx to the slice of data_x from element (i * minibatch_size) to (((i + 1) * minibatch_size)-1)
    # Set sy to the slice of data_y from element (i * minibatch_size) to (((i + 1) * minibatch_size)-1)
    # Hint: You'll find answers at the bottom of this lab.
    # *** YOUR CODE HERE ***
    sx = data_x[(i * minibatch_size):((i + 1) * minibatch_size)]
    sy = data_y[(i * minibatch_size):((i + 1) * minibatch_size)]
    return sx, sy

# Our next function will measure and record the test error per mini-batch, appending it to an errors array. We will use this to calculate the mean error across all mini-batches.
#
# **Again, you will need to fill in some code in the cell below**. Use the Python numpy `mean()` method on the `errors` array to calculate the average error across the minibatch and store the value in the variable named `result`.

# In[8]:

def measure_error(data_x_files, data_y_files, x, y, trainer, minibatch_size):
    errors = []
    for i in range(0, int(len(data_x_files) / minibatch_size)):
        data_sx_files, data_sy_files = slice_minibatch(data_x_files, data_y_files, i, minibatch_size)
        data_sx, data_sy = source.files_to_data(data_sx_files, data_sy_files)
        errors.append(trainer.test_minibatch({x: data_sx, y: data_sy}))

    result = 0
    #
    # *** YOUR CODE HERE ***
    # Use np.mean to set result to the average of the errors across each minibatch.
    # Hint: You'll find answers at the bottom of this lab.
    # *** YOUR CODE HERE ***
    result = np.mean(errors)
    return result

# ## Augmenting our ResNet model from Classification to Semantic Segmentation
#
# Now, we need to add additional Fully-Convolutional layers to our ResNet classification model to allow it to perform Semantic Segmentation.
#
# First, let's create the functions for our usual ResNet building blocks, as per Module 3 Lab 2:

# In[9]:

# Built on top of the ResNet model in CNTK repository
# https://github.com/Microsoft/CNTK/blob/master/Examples/Image/Classification/ResNet/Python/resnet_models.py

#
# ResNet building blocks
#
def conv_bn(input, filter_size, num_filters, strides=(1,1), init=C.he_normal()):
    c = Convolution(filter_size, num_filters, activation=None, init=init, pad=True, strides=strides, bias=False)(input)
    r = BatchNormalization(map_rank=1, normalization_time_constant=4096, use_cntk_engine=False)(c)
    return r

def conv_bn_relu(input, filter_size, num_filters, strides=(1,1), init=C.he_normal()):
    r = conv_bn(input, filter_size, num_filters, strides, init)
    return relu(r)

def resnet_basic(input, num_filters):
    c1 = conv_bn_relu(input, (3,3), num_filters)
    c2 = conv_bn(c1, (3,3), num_filters)
    p  = c2 + input
    return relu(p)

def resnet_basic_inc(input, num_filters, strides=(2,2)):
    c1 = conv_bn_relu(input, (3,3), num_filters, strides)
    c2 = conv_bn(c1, (3,3), num_filters)
    s  = conv_bn(input, (1,1), num_filters, strides)
    p  = c2 + s
    return relu(p)

def resnet_basic_stack(input, num_stack_layers, num_filters):
    assert (num_stack_layers >= 0)
    l = input
    for _ in range(num_stack_layers):
        l = resnet_basic(l, num_filters)
    return l

# Now, we need to add some functions to help us augment our ResNet model with fully-convolutional layers.
#
# This is where the magic happens, where we convert ResNet for object classification to ResNet for semantic segmentation.
#
# In 2015 (https://arxiv.org/pdf/1411.4038.pdf) and 2016 (https://arxiv.org/pdf/1605.06211.pdf), Long, Shelhamer, and Darrell presented a new method for object boundaries detection using Fully Convolutional Networks (FCN). They discovered that fully convolutional networks, trained end-to-end, pixels-to-pixels, exceeded the state-of-the-art in semantic segmentation.
#
# The main difference from a regular convolutional network is that the final fully-connected classifier is removed and replaced by a combination of all the convolutional features maps extracted from ResNet skipped connections. This allows:
#
#  * combining of features from different stages which vary in accuracy of semantic information;
#  * upsampling of learned low resolution semantic features maps through learnable one-by-one (1x1) convolutions and 2D upsampling.
#
#
# The following diagram shows the evolution of our model, from the basic ResNet20 classifier in Module 3, Lab 2, (on the left-hand side) to our fully convolutional network architecture for semantic segmentation in this lab (on the right-hand side):
# ![ResNet20 Fully Convolutional Semantic Segmentation Network Architecture](images/resnet20_fcn_semantic_segmentation_network_architecture.svg "ResNet20 Fully Convolutional Semantic Segmentation Network Architecture")
#
# In the right-hand side of the diagram above, the extra convolutional layers coming from the FCN model are actually deconvolutional layers - that is, they are deconvolving back to prediction masks, which are then reshaped and fused by the upsampling layers.
#
# The function `create_model()` in the following code block shows how this is implemented using the Microsoft Cognitive Toolkit. We take connections from `r1`, `r2_2` and `r3_2` with the feature maps we are interested in to finally combine them in the 'merged' layer after upsampling. A convolution with a sigmoid function classifies the output depending on the number of object classes - in our case, just for airplanes.
#
# The function `OneByOneConvAndUpSample()`  performs a 1x1 convolution over the input and and upsamples the output using basic Cognitive Toolkit primitives for operating with tensors.
#
# We also need a dice coefficient function, which is a statistic function commonly used to compare the similarity between two samples. This is what we will use to compare our predicted semantic segmentation labelling against the ground truth for our training and test images.

# In[10]:

def UpSampling2D(x):
    xr = C.reshape(x, (x.shape[0], x.shape[1], 1, x.shape[2], 1))
    xx = C.splice(xr, xr, axis=-1) # axis=-1 refers to the last axis
    xy = C.splice(xx, xx, axis=-3) # axis=-3 refers to the middle axis
    r = C.reshape(xy, (x.shape[0], x.shape[1] * 2, x.shape[2] * 2))

    return r

def UpSampling2DPower(x, k_power):
    for i in range(k_power):
        x = UpSampling2D(x)

    return x

def OneByOneConvAndUpSample(x, k_power, num_channels):
    x = Convolution((1, 1), num_channels, init=he_normal(), activation=relu, pad=True)(x)
    x = UpSampling2DPower(x, k_power)

    return x

def dice_coefficient(x, y):
    # average of per-channel dice coefficient
    # global dice coefificnet doesn't work as class with larger region dominates the metrics
    # https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient
    intersection = C.reduce_sum(x * y, axis=(1,2))

    return C.reduce_mean(2.0 * intersection / (C.reduce_sum(x, axis=(1,2)) + C.reduce_sum(y, axis=(1,2)) + 1.0))

# With these functions, we can create a new model that is a combination of ResNet plus our additional layers of convolution and upsampling.

# In[11]:

#
# Defines the fully convolutional models for image segmentation
#
def create_model(input, num_classes):
    c_map = [16, 32, 64]
    num_stack_layers = 3

    conv = conv_bn_relu(input, (3,3), c_map[0])
    r1 = resnet_basic_stack(conv, num_stack_layers, c_map[0])

    r2_1 = resnet_basic_inc(r1, c_map[1])
    r2_2 = resnet_basic_stack(r2_1, num_stack_layers-1, c_map[1])

    r3_1 = resnet_basic_inc(r2_2, c_map[2])
    r3_2 = resnet_basic_stack(r3_1, num_stack_layers-1, c_map[2])

    # now we'll add our "deconvolutional" layers (1x1 convolution with upsampling)

    up_r1 = OneByOneConvAndUpSample(r1, 0, num_classes)
    up_r2_2 = OneByOneConvAndUpSample(r2_2, 1, num_classes)
    up_r3_2 = OneByOneConvAndUpSample(r3_2, 2, num_classes)

    merged = C.splice(up_r1, up_r3_2, up_r2_2, axis=0)

    resnet_fcn_out = Convolution((1, 1), num_classes, init=he_normal(), activation=sigmoid, pad=True)(merged)

    return resnet_fcn_out

# As before in Module 3 Lab2, we can look at the Cognitive Toolkit generated view of this model, as rendered by GraphViz:
#
# ![ResNet20_Semantic_Segmentation_Model](images/ResNet20_Semantic_Segmentation_Model.svg "ResNet20 Fully Convolutional Semantic Segmentation Network Model")
#

# As this model is generated directly from the Microsoft Cognitive Toolkit, it represents the network that we have actually
# programmed the toolkit to construct.  If you are interested, you can download this model from the images/ subdirectory of this lab, and view it separately, comparing it against our reference diagram above.

# ## Training
#
# We will again create a training function to train our network using mini-batches.
#
# It is worth repeating that training without a GPU is really slow. Our code here checks whether we are going to be running on a GPU or CPU. If we're running on a CPU, we'll deliberately shrink the amount of training to just a single epoch to give you a *flavor* of what happens during training, but that model will perform pretty badly. In the case of running from a CPU and training for a single epoch, we will read in a pre-trained model for inference instead.
#

# In[12]:

def train(train_image_files, train_mask_files, val_image_files, val_mask_files):
    # Create model
    sample_img, sample_mask = source.files_to_data([train_image_files[0]], [val_image_files[0]])
    x = C.input_variable(sample_img[0].shape)
    y = C.input_variable(sample_mask[0].shape)

    z = create_model(x, source.num_classes)
    dice_coef = dice_coefficient(z, y)

    # Prepare model and trainer
    if (isUsingGPU):
        lr_mb = [0.001] * 5 + [0.0001] * 5 + [0.00001]*5 + [0.000001]*5 + [0.0000001]*5
    else:
        # training without a CPU is really slow, so we'll deliberatly shrink the amount of training
        # to just an epoch if we're on a CPU - just to give a flavor of what happens during training
        # and then read in a pre-trained model for inference instead.
        lr_mb = [0.0001] * 1 # deliberately shrink if training on CPU...
    lr = learning_rate_schedule(lr_mb, UnitType.sample)
    momentum = C.learners.momentum_as_time_constant_schedule(0.9)
    trainer = C.Trainer(z, (-dice_coef, -dice_coef), C.learners.adam(z.parameters, lr=lr, momentum=momentum))

    training_errors = []
    test_errors = []

    # Get minibatches of training data and perform model training
    minibatch_size = 8
    num_epochs = len(lr_mb)

    for e in range(0, num_epochs):
        for i in tqdm(range(0, int(len(train_image_files) / minibatch_size)), ascii=True,
                               desc="[i] Processing epoch {}/{}".format(e, num_epochs-1)):
            data_x_files, data_y_files = slice_minibatch(train_image_files, train_mask_files, i, minibatch_size)
            data_x, data_y = source.files_to_data(data_x_files, data_y_files)
            trainer.train_minibatch({x: data_x, y: data_y})
            gc.collect()

        # Measure training error
        training_error = measure_error(data_x_files, data_y_files, x, y, trainer, minibatch_size)
        training_errors.append(training_error)

        # Measure test error
        test_error = measure_error(val_image_files, val_mask_files, x, y, trainer, minibatch_size)
        test_errors.append(test_error)

        print("    epoch #{}: training_error={}, test_error={}".format(e, training_errors[-1], test_errors[-1]))

    return trainer, training_errors, test_errors

# # Initialise the Model
#
# Now, let's put it all to the test, and see our solution in action!  First, we need to configure our data source, which involves parsing to figure out how many training and validation samples we have.

# In[13]:

# Configure the data source

print('[i] Configuring data source...')
try:
    source = coco.CocoMs(os.path.join(data_path, "CocoMS"))
    training_input_image_files, training_target_mask_files = source.get_data(train_data_folder='/Training')
    validation_input_image_files, validation_target_mask_files = source.get_data(train_data_folder='/Validation')
    print('[i] # training samples:   ', len(training_input_image_files))
    print('[i] # validation samples: ', len(validation_input_image_files))
    print('[i] # classes:            ', source.num_classes)
    print('[i] Image size:           ', (224,224))
except (ImportError, AttributeError, RuntimeError) as e:
    print('[!] Unable to load data source:', str(e))

# In the next code block, we'll train our model. We'll need to either train from scratch or load a model that we made earlier...
#
# ## If training on a CPU
#
# If we are training on a CPU (like we will be in Azure Notebooks), we deliberately limit the training to a single epoch for time reasons. This gives an example of what training is like, but the resulting model will be very innaccurate.
#
# Try completing the lab using the pre-trained model, and viewing the results. This was trained on a system with a powerful GPU, and run for a number of epochs.
#
# Then restart the Jupyter Notebook and clear all outputs - run to this point again, but set `make_model` to `True` in the code block below below. Continue running on to the end, and you will quickly see how much worse the results are, and get an intuition for how much better the results get with increasing the number of epochs of training.
#
# You can expect even our very limited training in Azure Notebooks to take 50 minutes or so to complete. By comparison, a GPU could be anywhere from 10x to 100x faster.

# In[14]:

# And now do some training

make_model = False

# Load the saved model if specified
model_file = "cntk-resnet-fcn.dnn"

if (make_model):
    print("[i] Starting training...", flush=True)
    if (not isUsingGPU):
        print("    Have you tried running again with make_model set to False to compare output?", flush=True)

    tic = time.time()
    trainer, training_errors, test_errors = train(training_input_image_files, training_target_mask_files,
                                              validation_input_image_files, validation_target_mask_files)
    print('Training time: {} s.'.format(time.time() - tic))

    print("[i] Training finished...")
    model = trainer.model
    del training_input_image_files
    del training_target_mask_files

else:
    # load model
    if os.path.isfile(model_file):
        print("[i] Skipping training, using pre-trained model...")
        model = C.load_model(model_file)
    else:
        print("[!] model files {} does not exist. This will force training".format(model_file))
        tic = time.time()
        trainer, training_errors, test_errors = train(training_input_image_files, training_target_mask_files,
                                                  validation_input_image_files, validation_target_mask_files)
        print('Training time: {} s.'.format(time.time() - tic))
        print("[i] Training finished...")
        model = trainer.model
        # clean-up no longer needed variables
        del training_input_image_files
        del training_target_mask_files

# ## At last, some Semantic Segmentation predictions!
#
# With our model initialised (either trained or using a downloaded pre-trained model, we can now start making predictions on test images, and try to label them.  We will do this in two steps:
#
#  * First, we will do a forward pass of the input image through our network in order to calculate the actual prediction masks, which contain a class label for each image in the input image;
#  * Second, we will create some graphics, allowing us to quickly visualize how well we did.

# In[15]:

#
# We need to convert our validation filenames to image data for the predictor...
#

print ("[i] Converting file lists to image data...", flush=True)
tic = time.time()
validation_input_images, validation_target_masks =         source.files_to_data(validation_input_image_files, validation_target_mask_files)
print('Converting validation image data time: {} s.'.format(time.time() - tic))
print("[i] Converting file lists finished...")

# In[16]:

#
# First, let's do our semantic segmentation prediction
#

print("[i] Starting prediction...", flush=True)
tic = time.time()

pred = []
for idx in tqdm(range(0, len(validation_input_images)),ascii=True, desc='[i] Predicting...'):
    pred += list(model.eval(validation_input_images[idx]))
print('Prediction time: {} s.'.format(time.time() - tic))
print('[i] {} images predicted.'.format(len(pred)))
print("[i] Prediction finished...")

# If you have run the Python code above, you'll again see how much faster predicting with a network is compared to training the network.
#
# And now some image processing to allow us better understand the results... We are going to blend our input images and predictions together, so we can see where we labeled the pixels correctly and where we did not.
#

# In[17]:

#
# Second, let's do our image processing to create some graphics for ease of visualisation
#
print("[i] Started image processing...", flush=True)

tic = time.time()

input_images_rgb = []

for x in tqdm(validation_input_images, ascii=True, desc='[i] Converting input images (BGR2RGB)...'):
    img = np.moveaxis(x,0,2).astype(np.uint8)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    input_images_rgb.append(img)

target_masks_rgb=[]
for x in tqdm(validation_target_masks, ascii=True, desc='[i] Coloring ground truth images...'):
    target_masks_rgb.append(helper.masks_to_colorimg(x))

pred_rgb=[]
for x in tqdm(pred, ascii=True, desc='[i] Coloring prediction images...'):
    pred_rgb.append(helper.masks_to_colorimg(x))

output_images_rgb = []
for index in tqdm(range(len(input_images_rgb)), ascii=True, desc='[i] Combining input images + predictions...'):
    img = cv2.bitwise_or(input_images_rgb[index], pred_rgb[index])
    output_images_rgb.append(img)

print('Image Processing time: {} s.'.format(time.time() - tic))

print("Image processing finished...")

# If we have trained, we can view the learning curves for our training and our testing. All going well, these will have converged to a reasonable error rate, based on our dice coefficient. Ideally, we want the curves for test and training sets to be as close together as possible. Normally we'll see the curve for the training set is slightly more accurate (lower `dice_coef` value) than the test set.
#
# If you have trained locally on a GPU, they should look something like:
#
# ![sample training curve](images/sample_training_curve.png "Sample Training Curve")
#
# If you have trained on a CPU and only for one epoch, you'll only have one reading for test and training error, so you might get a funny graph like:
#
# ![cpu limited_training curve](images/cpu_limited_training_curve.png "CPU Limited Training Curve")
#
# If you have trained on CPU and see that type of curve, don't worry about it - it makes perfect sense for the limited (but costly in terms of time) amount of training you performed!  Your network has not converged *yet* and it would not be expected perform well. If you had plenty of patience, you could continue to train it for longer and see the curves converging.
#

# In[18]:

if (make_model):
    helper.plot_errors({"training": training_errors, "test": test_errors}, title="Simulation Learning Curve")

    # clean-up some variables we no longer need to reduce our memory footprint...
    # otherwise our Azure Notebook might run out of memory
    del validation_input_images
    del validation_input_image_files
    del validation_target_mask_files
    del validation_target_masks
    del trainer
    del training_errors
    del test_errors
    print("Garbage collection reclaimed {} objects".format(gc.collect()))
    plt.close()

# ## *A picture is worth a thousand words*
#
# And now, some images. We'll plot an original test image, with its associated ground truth mask/label. Alongside these, we'll show our predicted mask, and finally our predicted mask super-imposed on the original image, allowing us to visually see how well our model did. **Note: this plot function takes a while!**

# In[19]:

helper.plot_side_by_side([input_images_rgb, target_masks_rgb, pred_rgb, output_images_rgb])

# If you took the pre-trained model or trained via a GPU, we did well, considering the network learned *from scratch* how to label planes of all sorts, and from all viewpoints and angles. However, we can see that in some cases the network did make mistakes. This is mostly due to our network being small and our training set being very small -- to make this lab run in a reasonable amount of time.
#
# If you trained on a CPU, the very limited training will mean we're just barely detecting the planes and our model has a lot more *learning* to do to be useful.
#
# A larger network would be more expressive (e.g., ResNet-150), and could capture greater detail. If running on a GPU, we could relatively easily switch in a larger/deeper ResNet model and benefit from its greater capabilities.
#
# Additionally, a larger corpus of training examples would allow the network to better learn the objective, and to generalise much more accurately to new examples.
#
# There is something we can do to help tremendously with the limited training set by making use of *Transfer Learning*.  We will see in in the next lab.
#
# For now, let's take a look at some of the more interesting samples.

# In[20]:

plt.imshow(output_images_rgb[89])

# In[21]:

plt.imshow(output_images_rgb[2])

# In[22]:

plt.imshow(output_images_rgb[6])

# In[23]:

plt.imshow(output_images_rgb[7])

# In[24]:

plt.imshow(output_images_rgb[8])

# In[25]:

plt.imshow(output_images_rgb[11])

# # Conclusions
#
# In this lab, we used the Microsoft Cognitive Toolkit to build a ResNet20 model and modified it to perform semantic segmentation.
#
# We used the Coco dataset, which provides labels and masks for a variety of object classes. We focused on a subset containing airplanes.  We either used a pre-trained model or used this subset to train our model using the training data in mini-batches. Then we used this trained model to perform segmentation on some test images and plotted the results.
#
# Despite the fact that our accuracy was limited by the size of the base model (ResNet20, vs its
# larger cousin ResNet150, for example) and the size of the training set (~3K images), we demonstrated
# segmentation results far superior than could be expected of the traditional hand-crafted methods we saw in Module 2.
#
# In the next lab, we'll look at transfer learning -- a way of bringing in extra smarts from another
# model already trained on a much larger but different dataset. These extra smarts can help boost the
# performance of our semantic segmentation considerably.

# **Solution to missing slice_minibatch code**
#
#     sx = data_x[i \* minibatch_size: (i+1) \* minibatch_size]
#     sy = data_y[i \* minibatch_size: (i+1) \* minibatch_size]
#

# **Solution to missing measure_error code**
#
#     result = np.mean(errors)
#
#

courses/Microsoft/DEV290x - Computer Vision and Image Analysis (2018)/Labs/Module4_Lab1_Using_ResNet_with_FC_Layers_to_implement_Semantic_Segmentation.py


import os
import glob
import cv2 as cv
from PIL import Image
import numpy as np
from tqdm import tqdm

from datasets.voc import VOCSegmentation
from datasets.utils.random_sampling import sample_points_for_singal_object

class GenerateDataPairs():
    def __init__(self, root_dir, save_dir='./voc2012', image_set='train', n_pairs:int=1, minimum_object_area:int=100) -> None:

        assert image_set in ['train', 'val', 'trainval']
        assert isinstance(n_pairs, int)
        self.n_pairs = n_pairs if n_pairs > 0 else 1
        self.minimum_object_area = minimum_object_area if minimum_object_area > 0 else 100

        if not os.path.exists(save_dir):
            os.mkdir(save_dir)
        save_dir = os.path.join(save_dir, image_set) # save_dir/image_set
        if not os.path.exists(save_dir):
            os.mkdir(save_dir)
        image_dir = os.path.join(save_dir, 'images') # image_dir
        if not os.path.exists(image_dir):
            os.mkdir(image_dir)
        label_dir = os.path.join(save_dir, 'labels') # label_dir
        if not os.path.exists(label_dir):
            os.mkdir(label_dir)
        interactive_dir = os.path.join(save_dir, 'interactives') # interactive_dir
        if not os.path.exists(interactive_dir):
            os.mkdir(interactive_dir)
        self.image_dir = image_dir
        self.label_dir = label_dir
        self.interactive_dir = interactive_dir

        self.dataset = VOCSegmentation(root_dir, image_set=image_set)

        self.generate()

    def check_object_area(self, object_binary_mask, minimum_object_area=100):
        contours, hierarchy = cv.findContours(object_binary_mask, cv.RETR_LIST, cv.CHAIN_APPROX_SIMPLE)
        max_area = 0
        for c in contours:
            cur_area = cv.contourArea(c)
            if cur_area > max_area:
                max_area = cur_area
        return (max_area > minimum_object_area)

    def generate(self):
        for i, data in enumerate(tqdm(self.dataset)):
            # if i > 0:
            #     break
            image, label, name = data
            label_np = np.array(label)
            have_object_flag = False
            for id in np.unique(label_np):
                if id == 0 or id == 255: # skip background and object's contour
                    continue
                for k in range(self.n_pairs):
                    if self.check_object_area(np.uint8((label_np == id) * 255), 400):
                        object_interative_map = sample_points_for_singal_object(label_np, id) # [2, h, w]
                        if (not np.any(object_interative_map[0])) or (not np.any(object_interative_map[1])):
                            break
                        have_object_flag = True
                        fg_save_path = os.path.join(self.interactive_dir, f'{name}-{id}-fg-{k+1}.png')
                        Image.fromarray(object_interative_map[0]).save(fg_save_path)
                        bg_save_path = os.path.join(self.interactive_dir, f'{name}-{id}-bg-{k+1}.png')
                        Image.fromarray(object_interative_map[1]).save(bg_save_path)

            if have_object_flag:
                image_save_path = os.path.join(self.image_dir, f'{name}.jpg')
                image.save(image_save_path)
                label_save_path = os.path.join(self.label_dir, f'{name}.png')
                label.save(label_save_path)

# GenerateDataPairs(root_dir='/raid/home/guiyan/datasets', save_dir='./voc2012', image_set='val', n_pairs=1)
# GenerateDataPairs(root_dir='/raid/home/guiyan/datasets', save_dir='./voc2012', image_set='train', n_pairs=15)


from flask import Flask, render_template, url_for, request, jsonify
import pandas as pd
import numpy as np
import data as dt
import datetime
app = Flask(__name__)

@app.route('/shinobu', methods = ['GET'])
def kawaii():
    return render_template("index.html")
#mainblock
@app.route('/', methods = ['GET'])
def profile():
    return render_template("profile.html")
@app.route('/data', methods = ['GET'])
def data():
    data = pd.read_csv('s_data.csv')
    dataset = []
    for i in range(len(data)):
        temp = []
        for x in data.iloc[i]:
            temp.append(x)
        dataset.append(temp)
    dataset2 = dt.data_to_analysis
    return render_template("data.html",dataset = dataset,data=dataset2)
@app.route('/analysis', methods = ['GET'])
def analysis():
    dataset = dt.data_to_analysis
    return render_template("analysis.html",data = dataset)

@app.route('/notifications', methods = ['GET'])
def notifications():
    return render_template("notifications.html")
@app.route('/payment', methods = ['GET'])
def payment():
    return render_template("payment.html")

@app.route('/dataformat')
def send_to():
    return '{"id": 0, "I": 1.5, "U":3.1, "P":36, "T":1.2, "I_max":3.6, "U_max":36, "timecode": "2020-08-14-19:48"}'
@app.route('/postdata', methods = ['POST'])
def postJSON():
    response = request.get_json(force=True)
    data = pd.read_csv('s_data.csv')
    data = dt.data_add(data,response)
    dt.save_data(data,'s_data.csv','csv')
    #dt.html_update(response)
    return 'Recieved'

@app.route('/charts')
def charts():
    dataset = dt.data_to_analysis
    return render_template("charts.html",data = dataset)

@app.route('/background_plot')#This page accessable only for js script
def background_process():
    date_start = request.args.get('date_start', 0, type= str)
    date_end = request.args.get('date_end', 0, type= str)
    region = request.args.get('region', 0, type= int)
    city = request.args.get('city', 0, type= str)
    street = request.args.get('street', 0, type= str)
    building = request.args.get('building', 0, type= int)
    flat = request.args.get('flat', 0, type= int)
    dep = int(request.args.get('dep', 0, type= int))
    dep_value = int(request.args.get('dep_value', 0, type= int))
    typee = request.args.get('typee', 0, type= int)
    if dep_value == 1:
        id_req = dt.get_id(region, city, street, building, flat)
        return jsonify(dt.get_temperature(dt.load_id(id_req),dep))
    else:
        if typee == 2:
            if city == "Город":
                return jsonify(dt.get_mean_region(dt.load_region(region),dep))
            else:
                id_req = dt.get_id(region, city, street, building, flat)
                return jsonify(dt.get_mean(dt.load_id(id_req),id_req,dep))
        else:
            if city == "Город":
                return jsonify(dt.get_plot_data_region(dt.load_region(region),date_start,date_end,dep))
            else:
                id_req = dt.get_id(region, city, street, building, flat)
                return jsonify(dt.get_plot_data(dt.load_id(id_req),date_start,date_end,id_req,dep))

    #return jsonify(dt.get_plot_data(dt.load_id(id_req),date_start,date_end,id_req,dep))

@app.route('/background_select', methods = ['GET'])
def select():
    regions_cities = dt.regions_cities
    region = request.args.get('region', 0, type= int)
    if region == 0:
        cities = regions_cities[region]
        return jsonify(city = data)
    else:
        cities = regions_cities[region]
        return jsonify(city = cities)
@app.route('/background_analysis', methods = ['GET'])
def do_analysis():
    date_start = request.args.get('date_start', 0, type= str)
    output = {'y': ['6', '1', '2'], 'x1': [1, 2], 'x2': [1, 2,5, 4]}
    return jsonify(dt.get_neural(date_start))
@app.route('/background_data', methods = ['GET'])
def bg_data():
    date_start = request.args.get('date_start', 0, type= str)
    date_end = request.args.get('date_end', 0, type= str)
    region = request.args.get('region', 0, type= int)
    city = request.args.get('city', 0, type= str)
    street = request.args.get('street', 0, type= str)
    building = request.args.get('building', 0, type= int)
    flat = request.args.get('flat', 0, type= int)

    if city == "Город":
        return jsonify(dt.get_data_region(dt.load_region(region),date_start,date_end))
    else:
        id_req = dt.get_id(region, city, street, building, flat)
        return jsonify(dt.get_data(dt.load_id(id_req),date_start,date_end,id_req,1))

@app.route('/time', methods = ['GET'])
def time():
   return(datetime.datetime.now().isoformat()[:10] + '-' + datetime.datetime.now().isoformat()[11:-7])
if __name__ == '__main__':
    app.jinja_env.auto_reload = True
    app.debug = True
    app.run(host = '0.0.0.0')


from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import tensorflow as tf
from transformers import (
    DistilBertTokenizerFast, TFDistilBertForSequenceClassification, TFTrainer,
    TFTrainingArguments
)

from prep_data import load_data

tokenizer_config = {
    'truncation': True,
    'padding': 'max_length',
    'max_length': 128
}
tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-cased')

def prepare_dataset(dataset, n_examples):
    text, prices = load_data(dataset, n_examples)
    encoded_text = tokenizer(text, **tokenizer_config)
    return tf.data.Dataset.from_tensor_slices((
        dict(encoded_text),
        prices
    ))

def train():
    train_dataset = prepare_dataset('train', 100000)
    val_dataset = prepare_dataset('validate', 30000)

    training_args = TFTrainingArguments(
        output_dir='./results',
        num_train_epochs=1,
        per_device_train_batch_size=16,
        per_device_eval_batch_size=64,
        warmup_steps=500,
        weight_decay=0.01,
        logging_dir='./logs',
        logging_steps=10,
    )

    with training_args.strategy.scope():
        model = TFDistilBertForSequenceClassification.from_pretrained(
            "distilbert-base-cased", num_labels=1
        )

    trainer = TFTrainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset,
        eval_dataset=val_dataset
    )

    trainer.train()
    trainer.save_model()

    return trainer

def evaluate(trainer):
    test_dataset = prepare_dataset('test', 30000)
    preds = trainer.predict(test_dataset)
    labels = [1, 2, 3, 4]

    conf_matrix = confusion_matrix(
        preds.label_ids, preds.predictions.round(), labels=labels
    )
    (
        ConfusionMatrixDisplay(conf_matrix, display_labels=labels)
        .plot().figure_.savefig('conf-matrix.png')
    )

if __name__ == '__main__':
    trainer = train()
    evaluate(trainer)


from panda3d.core import *
from toontown.toonbase.ToonBaseGlobal import *
from .DistributedMinigame import *
from direct.interval.IntervalGlobal import *
from .OrthoWalk import *
from direct.showbase.PythonUtil import Functor, bound, lineupPos, lerp
from direct.fsm import ClassicFSM, State
from direct.fsm import State
from toontown.toonbase import TTLocalizer
from . import CatchGameGlobals
from direct.task.Task import Task
from toontown.toon import Toon
from toontown.suit import Suit
from . import MinigameAvatarScorePanel
from toontown.toonbase import ToontownTimer
from toontown.toonbase import ToontownGlobals
from . import CatchGameToonSD
from . import Trajectory
import math
from direct.distributed import DistributedSmoothNode
from direct.showbase.RandomNumGen import RandomNumGen
from . import MinigameGlobals
from toontown.toon import ToonDNA
from toontown.suit import SuitDNA
from .CatchGameGlobals import DropObjectTypes
from .CatchGameGlobals import Name2DropObjectType
from .DropPlacer import *
from .DropScheduler import *
from functools import reduce

class DistributedCatchGame(DistributedMinigame):
    DropTaskName = 'dropSomething'
    EndGameTaskName = 'endCatchGame'
    SuitWalkTaskName = 'catchGameSuitWalk'
    DropObjectPlurals = {'apple': TTLocalizer.CatchGameApples,
     'orange': TTLocalizer.CatchGameOranges,
     'pear': TTLocalizer.CatchGamePears,
     'coconut': TTLocalizer.CatchGameCoconuts,
     'watermelon': TTLocalizer.CatchGameWatermelons,
     'pineapple': TTLocalizer.CatchGamePineapples,
     'anvil': TTLocalizer.CatchGameAnvils}

    def __init__(self, cr):
        DistributedMinigame.__init__(self, cr)
        self.gameFSM = ClassicFSM.ClassicFSM('DistributedCatchGame', [State.State('off', self.enterOff, self.exitOff, ['play']), State.State('play', self.enterPlay, self.exitPlay, ['cleanup']), State.State('cleanup', self.enterCleanup, self.exitCleanup, [])], 'off', 'cleanup')
        self.addChildGameFSM(self.gameFSM)
        self.setUsesSmoothing()
        self.setUsesLookAround()

    def getTitle(self):
        return TTLocalizer.CatchGameTitle

    def getInstructions(self):
        return TTLocalizer.CatchGameInstructions % {'fruit': self.DropObjectPlurals[self.fruitName],
         'badThing': self.DropObjectPlurals['anvil']}

    def getMaxDuration(self):
        return CatchGameGlobals.GameDuration + 5

    def load(self):
        self.notify.debug('load')
        DistributedMinigame.load(self)
        self.defineConstants()
        groundModels = ['phase_4/models/minigames/treehouse_2players',
         'phase_4/models/minigames/treehouse_2players',
         'phase_4/models/minigames/treehouse_3players',
         'phase_4/models/minigames/treehouse_4players']
        index = self.getNumPlayers() - 1
        self.ground = loader.loadModel(groundModels[index])
        self.ground.setHpr(180, -90, 0)
        self.dropShadow = loader.loadModel('phase_3/models/props/drop_shadow')
        self.dropObjModels = {}
        for objType in DropObjectTypes:
            if objType.name not in ['anvil', self.fruitName]:
                continue
            model = loader.loadModel(objType.modelPath)
            self.dropObjModels[objType.name] = model
            modelScales = {'apple': 0.7,
             'orange': 0.7,
             'pear': 0.5,
             'coconut': 0.7,
             'watermelon': 0.6,
             'pineapple': 0.45}
            if objType.name in modelScales:
                model.setScale(modelScales[objType.name])
            if objType == Name2DropObjectType['pear']:
                model.setZ(-.6)
            if objType == Name2DropObjectType['coconut']:
                model.setP(180)
            if objType == Name2DropObjectType['watermelon']:
                model.setH(135)
                model.setZ(-.5)
            if objType == Name2DropObjectType['pineapple']:
                model.setZ(-1.7)
            if objType == Name2DropObjectType['anvil']:
                model.setZ(-self.ObjRadius)
            model.flattenMedium()

        self.music = base.loader.loadMusic('phase_4/audio/bgm/MG_toontag.ogg')
        self.sndGoodCatch = base.loader.loadSfx('phase_4/audio/sfx/SZ_DD_treasure.ogg')
        self.sndOof = base.loader.loadSfx('phase_4/audio/sfx/MG_cannon_hit_dirt.ogg')
        self.sndAnvilLand = base.loader.loadSfx('phase_4/audio/sfx/AA_drop_anvil_miss.ogg')
        self.sndPerfect = base.loader.loadSfx('phase_4/audio/sfx/ring_perfect.ogg')
        self.toonSDs = {}
        avId = self.localAvId
        toonSD = CatchGameToonSD.CatchGameToonSD(avId, self)
        self.toonSDs[avId] = toonSD
        toonSD.load()
        if self.WantSuits:
            suitTypes = ['f',
             'tm',
             'pp',
             'dt']
            self.suits = []
            for type in suitTypes:
                suit = Suit.Suit()
                d = SuitDNA.SuitDNA()
                d.newSuit(type)
                suit.setDNA(d)
                suit.pose('walk', 0)
                self.suits.append(suit)

        self.__textGen = TextNode('ringGame')
        self.__textGen.setFont(ToontownGlobals.getSignFont())
        self.__textGen.setAlign(TextNode.ACenter)
        self.introMovie = self.getIntroMovie()

    def unload(self):
        self.notify.debug('unload')
        DistributedMinigame.unload(self)
        self.removeChildGameFSM(self.gameFSM)
        del self.gameFSM
        self.introMovie.finish()
        del self.introMovie
        del self.__textGen
        for avId in list(self.toonSDs.keys()):
            toonSD = self.toonSDs[avId]
            toonSD.unload()

        del self.toonSDs
        for suit in self.suits:
            suit.reparentTo(hidden)
            suit.delete()

        del self.suits
        self.ground.removeNode()
        del self.ground
        self.dropShadow.removeNode()
        del self.dropShadow
        for model in list(self.dropObjModels.values()):
            model.removeNode()

        del self.dropObjModels
        del self.music
        del self.sndGoodCatch
        del self.sndOof
        del self.sndAnvilLand
        del self.sndPerfect

    def getObjModel(self, objName):
        return self.dropObjModels[objName].copyTo(hidden)

    def __genText(self, text):
        self.__textGen.setText(text)
        return self.__textGen.generate()

    def calcDifficultyConstants(self, difficulty, numPlayers):
        ToonSpeedRange = [16.0, 25.0]
        self.ToonSpeed = lerp(ToonSpeedRange[0], ToonSpeedRange[1], difficulty)
        self.SuitSpeed = self.ToonSpeed / 2.0
        self.SuitPeriodRange = [lerp(5.0, 3.0, self.getDifficulty()), lerp(15.0, 8.0, self.getDifficulty())]

        def scaledDimensions(widthHeight, scale):
            w, h = widthHeight
            return [math.sqrt(scale * w * w), math.sqrt(scale * h * h)]

        BaseStageDimensions = [20, 15]
        areaScales = [1.0,
         1.0,
         3.0 / 2,
         4.0 / 2]
        self.StageAreaScale = areaScales[numPlayers - 1]
        self.StageLinearScale = math.sqrt(self.StageAreaScale)
        self.notify.debug('StageLinearScale: %s' % self.StageLinearScale)
        self.StageDimensions = scaledDimensions(BaseStageDimensions, self.StageAreaScale)
        self.notify.debug('StageDimensions: %s' % self.StageDimensions)
        self.StageHalfWidth = self.StageDimensions[0] / 2.0
        self.StageHalfHeight = self.StageDimensions[1] / 2.0
        MOHs = [24] * 2 + [26, 28]
        self.MinOffscreenHeight = MOHs[self.getNumPlayers() - 1]
        distance = math.sqrt(self.StageDimensions[0] * self.StageDimensions[0] + self.StageDimensions[1] * self.StageDimensions[1])
        distance /= self.StageLinearScale
        if self.DropPlacerType == PathDropPlacer:
            distance /= 1.5
        ToonRunDuration = distance / self.ToonSpeed
        offScreenOnScreenRatio = 1.0
        fraction = 1.0 / 3 * 0.85
        self.BaselineOnscreenDropDuration = ToonRunDuration / (fraction * (1.0 + offScreenOnScreenRatio))
        self.notify.debug('BaselineOnscreenDropDuration=%s' % self.BaselineOnscreenDropDuration)
        self.OffscreenTime = offScreenOnScreenRatio * self.BaselineOnscreenDropDuration
        self.notify.debug('OffscreenTime=%s' % self.OffscreenTime)
        self.BaselineDropDuration = self.BaselineOnscreenDropDuration + self.OffscreenTime
        self.MaxDropDuration = self.BaselineDropDuration
        self.DropPeriod = self.BaselineDropDuration / 2.0
        scaledNumPlayers = (numPlayers - 1.0) * 0.75 + 1.0
        self.DropPeriod /= scaledNumPlayers
        typeProbs = {'fruit': 3,
         'anvil': 1}
        probSum = reduce(lambda x, y: x + y, list(typeProbs.values()))
        for key in list(typeProbs.keys()):
            typeProbs[key] = float(typeProbs[key]) / probSum

        scheduler = DropScheduler(CatchGameGlobals.GameDuration, self.FirstDropDelay, self.DropPeriod, self.MaxDropDuration, self.FasterDropDelay, self.FasterDropPeriodMult)
        self.totalDrops = 0
        while not scheduler.doneDropping():
            scheduler.stepT()
            self.totalDrops += 1

        self.numFruits = int(self.totalDrops * typeProbs['fruit'])
        self.numAnvils = int(self.totalDrops - self.numFruits)

    def getNumPlayers(self):
        return self.numPlayers

    def defineConstants(self):
        self.notify.debug('defineConstants')
        self.DropPlacerType = RegionDropPlacer
        fruits = {ToontownGlobals.ToontownCentral: 'apple',
         ToontownGlobals.DonaldsDock: 'orange',
         ToontownGlobals.DaisyGardens: 'pear',
         ToontownGlobals.MinniesMelodyland: 'coconut',
         ToontownGlobals.TheBrrrgh: 'watermelon',
         ToontownGlobals.DonaldsDreamland: 'pineapple'}
        self.fruitName = fruits[self.getSafezoneId()]
        self.ShowObjSpheres = 0
        self.ShowToonSpheres = 0
        self.ShowSuitSpheres = 0
        self.PredictiveSmoothing = 1
        self.UseGravity = 1
        self.TrickShadows = 1
        self.WantSuits = 1
        self.FirstDropDelay = 0.5
        self.FasterDropDelay = int(2.0 / 3 * CatchGameGlobals.GameDuration)
        self.notify.debug('will start dropping fast after %s seconds' % self.FasterDropDelay)
        self.FasterDropPeriodMult = 0.5
        self.calcDifficultyConstants(self.getDifficulty(), self.getNumPlayers())
        self.notify.debug('ToonSpeed: %s' % self.ToonSpeed)
        self.notify.debug('total drops: %s' % self.totalDrops)
        self.notify.debug('numFruits: %s' % self.numFruits)
        self.notify.debug('numAnvils: %s' % self.numAnvils)
        self.ObjRadius = 1.0
        dropGridDimensions = [[5, 5],
         [5, 5],
         [6, 6],
         [7, 7]]
        self.DropRows, self.DropColumns = dropGridDimensions[self.getNumPlayers() - 1]
        self.cameraPosTable = [[0, -29.36, 28.17]] * 2 + [[0, -32.87, 30.43], [0, -35.59, 32.1]]
        self.cameraHpr = [0, -35, 0]
        self.CameraPosHpr = self.cameraPosTable[self.getNumPlayers() - 1] + self.cameraHpr
        for objType in DropObjectTypes:
            self.notify.debug('*** Object Type: %s' % objType.name)
            objType.onscreenDuration = objType.onscreenDurMult * self.BaselineOnscreenDropDuration
            self.notify.debug('onscreenDuration=%s' % objType.onscreenDuration)
            v_0 = 0.0
            t = objType.onscreenDuration
            x_0 = self.MinOffscreenHeight
            x = 0.0
            g = 2.0 * (x - x_0 - v_0 * t) / (t * t)
            self.notify.debug('gravity=%s' % g)
            objType.trajectory = Trajectory.Trajectory(0, Vec3(0, 0, x_0), Vec3(0, 0, v_0), gravMult=abs(g / Trajectory.Trajectory.gravity))
            objType.fallDuration = objType.onscreenDuration + self.OffscreenTime

    def grid2world(self, column, row):
        x = column / float(self.DropColumns - 1)
        y = row / float(self.DropRows - 1)
        x = x * 2.0 - 1.0
        y = y * 2.0 - 1.0
        x *= self.StageHalfWidth
        y *= self.StageHalfHeight
        return (x, y)

    def showPosts(self):
        self.hidePosts()
        self.posts = [Toon.Toon(),
         Toon.Toon(),
         Toon.Toon(),
         Toon.Toon()]
        for i in range(len(self.posts)):
            toon = self.posts[i]
            toon.setDNA(base.localAvatar.getStyle())
            toon.reparentTo(render)
            x = self.StageHalfWidth
            y = self.StageHalfHeight
            if i > 1:
                x = -x
            if i % 2:
                y = -y
            toon.setPos(x, y, 0)

    def hidePosts(self):
        if hasattr(self, 'posts'):
            for toon in self.posts:
                toon.removeNode()

            del self.posts

    def showDropGrid(self):
        self.hideDropGrid()
        self.dropMarkers = []
        print('dropRows: %s' % self.DropRows)
        print('dropCols: %s' % self.DropColumns)
        for row in range(self.DropRows):
            self.dropMarkers.append([])
            rowList = self.dropMarkers[row]
            for column in range(self.DropColumns):
                toon = Toon.Toon()
                toon.setDNA(base.localAvatar.getStyle())
                toon.reparentTo(render)
                toon.setScale(1.0 / 3)
                x, y = self.grid2world(column, row)
                toon.setPos(x, y, 0)
                rowList.append(toon)

    def hideDropGrid(self):
        if hasattr(self, 'dropMarkers'):
            for row in self.dropMarkers:
                for marker in row:
                    marker.removeNode()

            del self.dropMarkers

    def onstage(self):
        self.notify.debug('onstage')
        DistributedMinigame.onstage(self)
        self.ground.reparentTo(render)
        self.scorePanels = []
        camera.reparentTo(render)
        camera.setPosHpr(*self.CameraPosHpr)
        lt = base.localAvatar
        lt.reparentTo(render)
        self.__placeToon(self.localAvId)
        lt.setSpeed(0, 0)
        toonSD = self.toonSDs[self.localAvId]
        toonSD.enter()
        toonSD.fsm.request('normal')
        self.orthoWalk.stop()
        radius = 0.7
        handler = CollisionHandlerEvent()
        handler.setInPattern('ltCatch%in')
        self.ltLegsCollNode = CollisionNode('catchLegsCollNode')
        self.ltLegsCollNode.setCollideMask(ToontownGlobals.CatchGameBitmask)
        self.ltHeadCollNode = CollisionNode('catchHeadCollNode')
        self.ltHeadCollNode.setCollideMask(ToontownGlobals.CatchGameBitmask)
        self.ltLHandCollNode = CollisionNode('catchLHandCollNode')
        self.ltLHandCollNode.setCollideMask(ToontownGlobals.CatchGameBitmask)
        self.ltRHandCollNode = CollisionNode('catchRHandCollNode')
        self.ltRHandCollNode.setCollideMask(ToontownGlobals.CatchGameBitmask)
        legsCollNodepath = lt.attachNewNode(self.ltLegsCollNode)
        legsCollNodepath.hide()
        head = base.localAvatar.getHeadParts().getPath(2)
        headCollNodepath = head.attachNewNode(self.ltHeadCollNode)
        headCollNodepath.hide()
        lHand = base.localAvatar.getLeftHands()[0]
        lHandCollNodepath = lHand.attachNewNode(self.ltLHandCollNode)
        lHandCollNodepath.hide()
        rHand = base.localAvatar.getRightHands()[0]
        rHandCollNodepath = rHand.attachNewNode(self.ltRHandCollNode)
        rHandCollNodepath.hide()
        lt.cTrav.addCollider(legsCollNodepath, handler)
        lt.cTrav.addCollider(headCollNodepath, handler)
        lt.cTrav.addCollider(lHandCollNodepath, handler)
        lt.cTrav.addCollider(lHandCollNodepath, handler)
        if self.ShowToonSpheres:
            legsCollNodepath.show()
            headCollNodepath.show()
            lHandCollNodepath.show()
            rHandCollNodepath.show()
        self.ltLegsCollNode.addSolid(CollisionSphere(0, 0, radius, radius))
        self.ltHeadCollNode.addSolid(CollisionSphere(0, 0, 0, radius))
        self.ltLHandCollNode.addSolid(CollisionSphere(0, 0, 0, 2 * radius / 3.0))
        self.ltRHandCollNode.addSolid(CollisionSphere(0, 0, 0, 2 * radius / 3.0))
        self.toonCollNodes = [legsCollNodepath,
         headCollNodepath,
         lHandCollNodepath,
         rHandCollNodepath]
        if self.PredictiveSmoothing:
            DistributedSmoothNode.activateSmoothing(1, 1)
        self.introMovie.start()

    def offstage(self):
        self.notify.debug('offstage')
        DistributedSmoothNode.activateSmoothing(1, 0)
        self.introMovie.finish()
        for avId in list(self.toonSDs.keys()):
            self.toonSDs[avId].exit()

        self.hidePosts()
        self.hideDropGrid()
        for collNode in self.toonCollNodes:
            while collNode.node().getNumSolids():
                collNode.node().removeSolid(0)

            base.localAvatar.cTrav.removeCollider(collNode)

        del self.toonCollNodes
        for panel in self.scorePanels:
            panel.cleanup()

        del self.scorePanels
        self.ground.reparentTo(hidden)
        DistributedMinigame.offstage(self)

    def handleDisabledAvatar(self, avId):
        self.notify.debug('handleDisabledAvatar')
        self.notify.debug('avatar ' + str(avId) + ' disabled')
        self.toonSDs[avId].exit(unexpectedExit=True)
        del self.toonSDs[avId]
        DistributedMinigame.handleDisabledAvatar(self, avId)

    def __placeToon(self, avId):
        toon = self.getAvatar(avId)
        idx = self.avIdList.index(avId)
        x = lineupPos(idx, self.numPlayers, 4.0)
        toon.setPos(x, 0, 0)
        toon.setHpr(180, 0, 0)

    def setGameReady(self):
        if not self.hasLocalToon:
            return
        self.notify.debug('setGameReady')
        if DistributedMinigame.setGameReady(self):
            return
        headCollNP = base.localAvatar.find('**/catchHeadCollNode')
        if headCollNP and not headCollNP.isEmpty():
            headCollNP.hide()
        for avId in self.remoteAvIdList:
            toon = self.getAvatar(avId)
            if toon:
                toon.reparentTo(render)
                self.__placeToon(avId)
                toonSD = CatchGameToonSD.CatchGameToonSD(avId, self)
                self.toonSDs[avId] = toonSD
                toonSD.load()
                toonSD.enter()
                toonSD.fsm.request('normal')
                toon.startSmooth()

    def setGameStart(self, timestamp):
        if not self.hasLocalToon:
            return
        self.notify.debug('setGameStart')
        DistributedMinigame.setGameStart(self, timestamp)
        self.introMovie.finish()
        camera.reparentTo(render)
        camera.setPosHpr(*self.CameraPosHpr)
        self.gameFSM.request('play')

    def enterOff(self):
        self.notify.debug('enterOff')

    def exitOff(self):
        pass

    def enterPlay(self):
        self.notify.debug('enterPlay')
        self.orthoWalk.start()
        for suit in self.suits:
            suitCollSphere = CollisionSphere(0, 0, 0, 1.0)
            suit.collSphereName = 'suitCollSphere%s' % self.suits.index(suit)
            suitCollSphere.setTangible(0)
            suitCollNode = CollisionNode(self.uniqueName(suit.collSphereName))
            suitCollNode.setIntoCollideMask(ToontownGlobals.WallBitmask)
            suitCollNode.addSolid(suitCollSphere)
            suit.collNodePath = suit.attachNewNode(suitCollNode)
            suit.collNodePath.hide()
            if self.ShowSuitSpheres:
                suit.collNodePath.show()
            self.accept(self.uniqueName('enter' + suit.collSphereName), self.handleSuitCollision)

        self.scores = [0] * self.numPlayers
        spacing = 0.4
        for i in range(self.numPlayers):
            avId = self.avIdList[i]
            avName = self.getAvatarName(avId)
            scorePanel = MinigameAvatarScorePanel.MinigameAvatarScorePanel(avId, avName)
            scorePanel.setScale(0.9)
            scorePanel.setPos(0.75 - spacing * (self.numPlayers - 1 - i), 0.0, 0.85)
            scorePanel.makeTransparent(0.75)
            self.scorePanels.append(scorePanel)

        self.fruitsCaught = 0
        self.droppedObjCaught = {}
        self.dropIntervals = {}
        self.droppedObjNames = []
        self.dropSchedule = []
        self.numItemsDropped = 0
        self.scheduleDrops()
        self.startDropTask()
        if self.WantSuits:
            self.startSuitWalkTask()
        self.timer = ToontownTimer.ToontownTimer()
        self.timer.posInTopRightCorner()
        self.timer.setTime(CatchGameGlobals.GameDuration)
        self.timer.countdown(CatchGameGlobals.GameDuration, self.timerExpired)
        self.timer.setTransparency(1)
        self.timer.setColorScale(1, 1, 1, 0.75)
        base.playMusic(self.music, looping=0, volume=0.9)

    def exitPlay(self):
        self.stopDropTask()
        self.stopSuitWalkTask()
        if hasattr(self, 'perfectIval'):
            self.perfectIval.pause()
            del self.perfectIval
        self.timer.stop()
        self.timer.destroy()
        del self.timer
        self.music.stop()
        for suit in self.suits:
            self.ignore(self.uniqueName('enter' + suit.collSphereName))
            suit.collNodePath.removeNode()

        for ival in list(self.dropIntervals.values()):
            ival.finish()

        del self.dropIntervals
        del self.droppedObjNames
        del self.droppedObjCaught
        del self.dropSchedule
        taskMgr.remove(self.EndGameTaskName)

    def timerExpired(self):
        pass

    def __handleCatch(self, objNum):
        self.notify.debug('catch: %s' % objNum)
        self.showCatch(self.localAvId, objNum)
        objName = self.droppedObjNames[objNum]
        objTypeId = CatchGameGlobals.Name2DOTypeId[objName]
        self.sendUpdate('claimCatch', [objNum, objTypeId])
        self.finishDropInterval(objNum)

    def showCatch(self, avId, objNum):
        isLocal = avId == self.localAvId
        objName = self.droppedObjNames[objNum]
        objType = Name2DropObjectType[objName]
        if objType.good:
            if objNum not in self.droppedObjCaught:
                if isLocal:
                    base.playSfx(self.sndGoodCatch)
                fruit = self.getObjModel(objName)
                toon = self.getAvatar(avId)
                rHand = toon.getRightHands()[0]
                self.toonSDs[avId].eatFruit(fruit, rHand)
        else:
            self.toonSDs[avId].fsm.request('fallForward')
        self.droppedObjCaught[objNum] = 1

    def setObjectCaught(self, avId, objNum):
        if not self.hasLocalToon:
            return
        if self.gameFSM.getCurrentState().getName() != 'play':
            self.notify.warning('ignoring msg: object %s caught by %s' % (objNum, avId))
            return
        isLocal = avId == self.localAvId
        if not isLocal:
            self.notify.debug('AI: avatar %s caught %s' % (avId, objNum))
            self.finishDropInterval(objNum)
            self.showCatch(avId, objNum)
        objName = self.droppedObjNames[objNum]
        if Name2DropObjectType[objName].good:
            i = self.avIdList.index(avId)
            self.scores[i] += 1
            self.scorePanels[i].setScore(self.scores[i])
            self.fruitsCaught += 1

    def finishDropInterval(self, objNum):
        if objNum in self.dropIntervals:
            self.dropIntervals[objNum].finish()

    def scheduleDrops(self):
        self.droppedObjNames = [self.fruitName] * self.numFruits + ['anvil'] * self.numAnvils
        self.randomNumGen.shuffle(self.droppedObjNames)
        dropPlacer = self.DropPlacerType(self, self.getNumPlayers(), self.droppedObjNames)
        while not dropPlacer.doneDropping():
            self.dropSchedule.append(dropPlacer.getNextDrop())

    def startDropTask(self):
        taskMgr.add(self.dropTask, self.DropTaskName)

    def stopDropTask(self):
        taskMgr.remove(self.DropTaskName)

    def dropTask(self, task):
        curT = self.getCurrentGameTime()
        while self.dropSchedule[0][0] <= curT:
            drop = self.dropSchedule[0]
            self.dropSchedule = self.dropSchedule[1:]
            dropTime, objName, dropCoords = drop
            objNum = self.numItemsDropped
            lastDrop = len(self.dropSchedule) == 0
            x, y = self.grid2world(*dropCoords)
            dropIval = self.getDropIval(x, y, objName, objNum)

            def cleanup(self = self, objNum = objNum, lastDrop = lastDrop):
                del self.dropIntervals[objNum]
                if lastDrop:
                    self.sendUpdate('reportDone')

            dropIval.append(Func(cleanup))
            self.dropIntervals[objNum] = dropIval
            self.numItemsDropped += 1
            dropIval.start(curT - dropTime)
            if lastDrop:
                return Task.done

        return Task.cont

    def setEveryoneDone(self):
        if not self.hasLocalToon:
            return
        if self.gameFSM.getCurrentState().getName() != 'play':
            self.notify.warning('ignoring setEveryoneDone msg')
            return
        self.notify.debug('setEveryoneDone')

        def endGame(task, self = self):
            if not CatchGameGlobals.EndlessGame:
                self.gameOver()
            return Task.done

        self.notify.debug('num fruits: %s' % self.numFruits)
        self.notify.debug('num catches: %s' % self.fruitsCaught)
        self.timer.hide()
        if self.fruitsCaught >= self.numFruits:
            self.notify.debug('perfect game!')
            perfectTextSubnode = hidden.attachNewNode(self.__genText(TTLocalizer.CatchGamePerfect))
            perfectText = hidden.attachNewNode('perfectText')
            perfectTextSubnode.reparentTo(perfectText)
            frame = self.__textGen.getCardActual()
            offsetY = -abs(frame[2] + frame[3]) / 2.0
            perfectTextSubnode.setPos(0, 0, offsetY)
            perfectText.setColor(1, 0.1, 0.1, 1)

            def fadeFunc(t, text = perfectText):
                text.setColorScale(1, 1, 1, t)

            def destroyText(text = perfectText):
                text.removeNode()

            textTrack = Sequence(Func(perfectText.reparentTo, aspect2d), Parallel(LerpScaleInterval(perfectText, duration=0.5, scale=0.3, startScale=0.0), LerpFunctionInterval(fadeFunc, fromData=0.0, toData=1.0, duration=0.5)), Wait(2.0), Parallel(LerpScaleInterval(perfectText, duration=0.5, scale=1.0), LerpFunctionInterval(fadeFunc, fromData=1.0, toData=0.0, duration=0.5, blendType='easeIn')), Func(destroyText), WaitInterval(0.5), Func(endGame, None))
            soundTrack = SoundInterval(self.sndPerfect)
            self.perfectIval = Parallel(textTrack, soundTrack)
            self.perfectIval.start()
        else:
            taskMgr.doMethodLater(1, endGame, self.EndGameTaskName)
        return

    def getDropIval(self, x, y, dropObjName, num):
        objType = Name2DropObjectType[dropObjName]
        dropNode = hidden.attachNewNode('catchDropNode%s' % num)
        dropNode.setPos(x, y, 0)
        shadow = self.dropShadow.copyTo(dropNode)
        shadow.setZ(0.2)
        shadow.setColor(1, 1, 1, 1)
        object = self.getObjModel(dropObjName)
        object.reparentTo(dropNode)
        if dropObjName in ['watermelon', 'anvil']:
            objH = object.getH()
            absDelta = {'watermelon': 12,
             'anvil': 15}[dropObjName]
            delta = (self.randomNumGen.random() * 2.0 - 1.0) * absDelta
            newH = objH + delta
        else:
            newH = self.randomNumGen.random() * 360.0
        object.setH(newH)
        sphereName = 'FallObj%s' % num
        radius = self.ObjRadius
        if objType.good:
            radius *= lerp(1.0, 1.3, self.getDifficulty())
        collSphere = CollisionSphere(0, 0, 0, radius)
        collSphere.setTangible(0)
        collNode = CollisionNode(sphereName)
        collNode.setCollideMask(ToontownGlobals.CatchGameBitmask)
        collNode.addSolid(collSphere)
        collNodePath = object.attachNewNode(collNode)
        collNodePath.hide()
        if self.ShowObjSpheres:
            collNodePath.show()
        catchEventName = 'ltCatch' + sphereName

        def eatCollEntry(forward, collEntry):
            forward()

        self.accept(catchEventName, Functor(eatCollEntry, Functor(self.__handleCatch, num)))

        def cleanup(self = self, dropNode = dropNode, num = num, event = catchEventName):
            self.ignore(event)
            dropNode.removeNode()

        duration = objType.fallDuration
        onscreenDuration = objType.onscreenDuration
        dropHeight = self.MinOffscreenHeight
        targetShadowScale = 0.3
        if self.TrickShadows:
            intermedScale = targetShadowScale * (self.OffscreenTime / self.BaselineDropDuration)
            shadowScaleIval = Sequence(LerpScaleInterval(shadow, self.OffscreenTime, intermedScale, startScale=0))
            shadowScaleIval.append(LerpScaleInterval(shadow, duration - self.OffscreenTime, targetShadowScale, startScale=intermedScale))
        else:
            shadowScaleIval = LerpScaleInterval(shadow, duration, targetShadowScale, startScale=0)
        targetShadowAlpha = 0.4
        shadowAlphaIval = LerpColorScaleInterval(shadow, self.OffscreenTime, Point4(1, 1, 1, targetShadowAlpha), startColorScale=Point4(1, 1, 1, 0))
        shadowIval = Parallel(shadowScaleIval, shadowAlphaIval)
        if self.UseGravity:

            def setObjPos(t, objType = objType, object = object):
                z = objType.trajectory.calcZ(t)
                object.setZ(z)

            setObjPos(0)
            dropIval = LerpFunctionInterval(setObjPos, fromData=0, toData=onscreenDuration, duration=onscreenDuration)
        else:
            startPos = Point3(0, 0, self.MinOffscreenHeight)
            object.setPos(startPos)
            dropIval = LerpPosInterval(object, onscreenDuration, Point3(0, 0, 0), startPos=startPos, blendType='easeIn')
        ival = Sequence(Func(Functor(dropNode.reparentTo, render)), Parallel(Sequence(WaitInterval(self.OffscreenTime), dropIval), shadowIval), Func(cleanup), name='drop%s' % num)
        landSound = None
        if objType == Name2DropObjectType['anvil']:
            landSound = self.sndAnvilLand
        if landSound:
            ival.append(SoundInterval(landSound))
        return ival

    def startSuitWalkTask(self):
        ival = Parallel(name='catchGameMetaSuitWalk')
        rng = RandomNumGen(self.randomNumGen)
        delay = 0.0
        while delay < CatchGameGlobals.GameDuration:
            delay += lerp(self.SuitPeriodRange[0], self.SuitPeriodRange[0], rng.random())
            walkIval = Sequence(name='catchGameSuitWalk')
            walkIval.append(Wait(delay))

            def pickY(self = self, rng = rng):
                return lerp(-self.StageHalfHeight, self.StageHalfHeight, rng.random())

            m = [2.5,
             2.5,
             2.3,
             2.1][self.getNumPlayers() - 1]
            startPos = Point3(-(self.StageHalfWidth * m), pickY(), 0)
            stopPos = Point3(self.StageHalfWidth * m, pickY(), 0)
            if rng.choice([0, 1]):
                startPos, stopPos = stopPos, startPos
            walkIval.append(self.getSuitWalkIval(startPos, stopPos, rng))
            ival.append(walkIval)

        ival.start()
        self.suitWalkIval = ival

    def stopSuitWalkTask(self):
        self.suitWalkIval.finish()
        del self.suitWalkIval

    def getSuitWalkIval(self, startPos, stopPos, rng):
        data = {}
        lerpNP = render.attachNewNode('catchGameSuitParent')

        def setup(self = self, startPos = startPos, stopPos = stopPos, data = data, lerpNP = lerpNP, rng = rng):
            if len(self.suits) == 0:
                return
            suit = rng.choice(self.suits)
            data['suit'] = suit
            self.suits.remove(suit)
            suit.reparentTo(lerpNP)
            suit.loop('walk')
            suit.setPlayRate(self.SuitSpeed / ToontownGlobals.SuitWalkSpeed, 'walk')
            suit.setPos(0, 0, 0)
            lerpNP.setPos(startPos)
            suit.lookAt(stopPos)

        def cleanup(self = self, data = data, lerpNP = lerpNP):
            if 'suit' in data:
                suit = data['suit']
                suit.reparentTo(hidden)
                self.suits.append(suit)
            lerpNP.removeNode()

        distance = Vec3(stopPos - startPos).length()
        duration = distance / self.SuitSpeed
        ival = Sequence(FunctionInterval(setup), LerpPosInterval(lerpNP, duration, stopPos), FunctionInterval(cleanup))
        return ival

    def handleSuitCollision(self, collEntry):
        self.toonSDs[self.localAvId].fsm.request('fallBack')
        timestamp = globalClockDelta.localToNetworkTime(globalClock.getFrameTime())
        self.sendUpdate('hitBySuit', [self.localAvId, timestamp])

    def hitBySuit(self, avId, timestamp):
        if not self.hasLocalToon:
            return
        if self.gameFSM.getCurrentState().getName() != 'play':
            self.notify.warning('ignoring msg: av %s hit by suit' % avId)
            return
        toon = self.getAvatar(avId)
        if toon == None:
            return
        self.notify.debug('avatar %s hit by a suit' % avId)
        if avId != self.localAvId:
            self.toonSDs[avId].fsm.request('fallBack')
        return

    def enterCleanup(self):
        self.notify.debug('enterCleanup')

    def exitCleanup(self):
        pass

    def initOrthoWalk(self):
        self.notify.debug('startOrthoWalk')

        def doCollisions(oldPos, newPos, self = self):
            x = bound(newPos[0], self.StageHalfWidth, -self.StageHalfWidth)
            y = bound(newPos[1], self.StageHalfHeight, -self.StageHalfHeight)
            newPos.setX(x)
            newPos.setY(y)
            return newPos

        orthoDrive = OrthoDrive(self.ToonSpeed, customCollisionCallback=doCollisions)
        self.orthoWalk = OrthoWalk(orthoDrive, broadcast=not self.isSinglePlayer())

    def destroyOrthoWalk(self):
        self.notify.debug('destroyOrthoWalk')
        self.orthoWalk.destroy()
        del self.orthoWalk

    def getIntroMovie(self):
        locNode = self.ground.find('**/locator_tree')
        treeNode = locNode.attachNewNode('treeNode')
        treeNode.setHpr(render, 0, 0, 0)

        def cleanupTree(treeNode = treeNode):
            treeNode.removeNode()

        initialCamPosHpr = (-0.21,
         -19.56,
         13.94,
         0.0,
         26.57,
         0.0)
        suitViewCamPosHpr = (0, -11.5, 13, 0, -35, 0)
        finalCamPosHpr = self.CameraPosHpr
        cameraIval = Sequence(Func(camera.reparentTo, render), Func(camera.setPosHpr, treeNode, *initialCamPosHpr), WaitInterval(4.0), LerpPosHprInterval(camera, 2.0, Point3(*suitViewCamPosHpr[:3]), Point3(*suitViewCamPosHpr[3:]), blendType='easeInOut', name='lerpToSuitView'), WaitInterval(4.0), LerpPosHprInterval(camera, 3.0, Point3(*finalCamPosHpr[:3]), Point3(*finalCamPosHpr[3:]), blendType='easeInOut', name='lerpToPlayView'))

        def getIntroToon(toonProperties, parent, pos):
            toon = Toon.Toon()
            dna = ToonDNA.ToonDNA()
            dna.newToonFromProperties(*toonProperties)
            toon.setDNA(dna)
            toon.reparentTo(parent)
            toon.setPos(*pos)
            toon.setH(180)
            toon.startBlink()
            return toon

        def cleanupIntroToon(toon):
            toon.detachNode()
            toon.stopBlink()
            toon.delete()

        def getThrowIval(toon, hand, object, leftToon, isAnvil = 0):
            anim = 'catch-intro-throw'
            grabFrame = 12
            fullSizeFrame = 30
            framePeriod = 1.0 / toon.getFrameRate(anim)
            objScaleDur = (fullSizeFrame - grabFrame) * framePeriod
            releaseFrame = 35
            trajDuration = 1.6
            trajDistance = 4
            if leftToon:
                releaseFrame = 34
                trajDuration = 1.0
                trajDistance = 1
            animIval = ActorInterval(toon, anim, loop=0)

            def getThrowDest(object = object, offset = trajDistance):
                dest = object.getPos(render)
                dest += Point3(0, -offset, 0)
                dest.setZ(0)
                return dest

            if leftToon:
                trajIval = ProjectileInterval(object, startVel=Point3(0, 0, 0), duration=trajDuration)
            else:
                trajIval = ProjectileInterval(object, endPos=getThrowDest, duration=trajDuration)
            trajIval = Sequence(Func(object.wrtReparentTo, render), trajIval, Func(object.wrtReparentTo, hidden))
            if isAnvil:
                trajIval.append(SoundInterval(self.sndAnvilLand))
            objIval = Track((grabFrame * framePeriod, Sequence(Func(object.reparentTo, hand), Func(object.setPosHpr, 0.05, -.13, 0.62, 0, 0, 336.8), LerpScaleInterval(object, objScaleDur, 1.0, startScale=0.1, blendType='easeInOut'))), (releaseFrame * framePeriod, trajIval))

            def cleanup(object = object):
                object.reparentTo(hidden)
                object.removeNode()

            throwIval = Sequence(Parallel(animIval, objIval), Func(cleanup))
            return throwIval

        tY = -4.0
        tZ = 19.5
        props = ['css',
         'md',
         'm',
         'f',
         9,
         0,
         9,
         9,
         13,
         5,
         11,
         5,
         8,
         7]
        leftToon = getIntroToon(props, treeNode, [-2.3, tY, tZ])
        props = ['mss',
         'ls',
         'l',
         'm',
         6,
         0,
         6,
         6,
         3,
         5,
         3,
         5,
         5,
         0]
        rightToon = getIntroToon(props, treeNode, [1.8, tY, tZ])
        fruit = self.getObjModel(self.fruitName)
        if self.fruitName == 'pineapple':
            fruit.setZ(0.42)
            fruit.flattenMedium()
        anvil = self.getObjModel('anvil')
        anvil.setH(100)
        anvil.setZ(0.42)
        anvil.flattenMedium()
        leftToonIval = getThrowIval(leftToon, leftToon.getRightHands()[0], fruit, leftToon=1)
        rightToonIval = getThrowIval(rightToon, rightToon.getLeftHands()[0], anvil, leftToon=0, isAnvil=1)
        animDur = leftToon.getNumFrames('catch-intro-throw') / leftToon.getFrameRate('catch-intro-throw')
        toonIval = Sequence(Parallel(Sequence(leftToonIval, Func(leftToon.loop, 'neutral')), Sequence(Func(rightToon.loop, 'neutral'), WaitInterval(animDur / 2.0), rightToonIval, Func(rightToon.loop, 'neutral')), WaitInterval(cameraIval.getDuration())), Func(cleanupIntroToon, leftToon), Func(cleanupIntroToon, rightToon))
        self.treeNode = treeNode
        self.fruit = fruit
        self.anvil = anvil
        self.leftToon = leftToon
        self.rightToon = rightToon
        introMovie = Sequence(Parallel(cameraIval, toonIval), Func(cleanupTree))
        return introMovie


# -*- coding: utf-8 -*-
"""
Created on Tue Dec 06 21:45:09 2016

@author: Juan
"""

import numpy as np
import matplotlib.pyplot as plt
import scipy.io.wavfile as wav
from scipy import signal

if __name__=='__main__':

    melcoeff = 20
    melbands = 40

    artist = 'claire'
    audio_file = "../audio/" + artist + "_mono.wav"
    proba_csv = "../prediction/" + artist + "_mfcc_" + str(melcoeff) + str(melbands) + "_proba.csv"
    prediction_csv= "../prediction/" + artist + "_mfcc_" + str(melcoeff) + str(melbands) + "_prediction.csv"
#    audio_file='../audio/LP-mem-6-a.wav'

    fs, x = wav.read(audio_file)
    t = np.arange(len(x)) * (1/fs)

    proba = np.loadtxt(proba_csv)
    prediction = np.loadtxt(prediction_csv)

    from PIL import Image
    import numpy as np

    aux=prediction[np.argsort(prediction[:,2]),1]
    r_gt=np.zeros(aux.shape,dtype='uint8')
    g_gt=np.zeros(aux.shape,dtype='uint8')
    b_gt=np.zeros(aux.shape,dtype='uint8')
    r_gt[aux==1]=1
    g_gt[aux==2]=1
    b_gt[aux==3]=1

    for i in range(0,9):
        r_gt=np.c_[r_gt,r_gt]
        g_gt=np.c_[g_gt,g_gt]
        b_gt=np.c_[b_gt,b_gt]

    rgbArray_gt = np.zeros((r_gt.shape[1],r_gt.shape[0],3), 'uint8')
    rgbArray_gt[..., 0] = r_gt.T*255
    rgbArray_gt[..., 1] = g_gt.T*255
    rgbArray_gt[..., 2] = b_gt.T*255

    r=np.c_[proba[np.argsort(proba[:,3]),0],proba[np.argsort(proba[:,3]),0]]
    g=np.c_[proba[np.argsort(proba[:,3]),1],proba[np.argsort(proba[:,3]),1]]
    b=np.c_[proba[np.argsort(proba[:,3]),2],proba[np.argsort(proba[:,3]),2]]
    for i in range(0,8):
        r=np.c_[r,r]
        g=np.c_[g,g]
        b=np.c_[b,b]

    rgbArray = np.zeros((r.shape[1],r.shape[0],3), 'uint8')

    rgbArray[..., 0] = r.T*255
    rgbArray[..., 1] = g.T*255
    rgbArray[..., 2] = b.T*255

    rgbArray_mix=np.zeros((2*r.shape[1],r.shape[0],3), 'uint8')

    rgbArray_mix[0:512,:,:]=rgbArray
    rgbArray_mix[512:1024,:,:]=rgbArray_gt
    img_mix = Image.fromarray(rgbArray_mix)
    img_mix.save(artist+'_mix.jpeg')


import cv2
import imutils
import numpy as np

class EyeTracker:
    def __init__(self, faceCascadePath, eyeCascadePath):
        # Initializerar 2 objekt, av Cascade Classifiers,
        self.faceCascade = cv2.CascadeClassifier(faceCascadePath)
        self.eyeCascade = cv2.CascadeClassifier(eyeCascadePath)

    def track(self, img):
        face_rectangles = self.faceCascade.detectMultiScale(img, scaleFactor=1.1,
                                                  minNeighbors=5, minSize=(30,30),
                                                  flags=cv2.CASCADE_SCALE_IMAGE)
        rects = []

        #Börjar leta efter ögonen
        for (fX, fY, fW, fH) in face_rectangles:
            #vi skapar ett område för ansiktsregionen via rektangeln
            faceROI = img[fY:fY + fH, fX:fX+fW]
            #Appendar rektangel till rects, för senare användning
            # rects.append((fX, fY, fX + fW, fY + fH))

            #Därefter letar vi nu efter ögonen inuti ansiktsregionen, fast med andra parametervärden, för att reducera false positives.
            eye_rectangles = self.eyeCascade.detectMultiScale(faceROI, scaleFactor=1.1, minNeighbors= 10, minSize=(20,20), flags=cv2.CASCADE_SCALE_IMAGE)

            #Vi appendar rektanglarna ifrån eye rektangels, till listan över rektangles.
            for (eX, eY, eW, eH) in eye_rectangles:
                rects.append((fX + eX, fY + eY, fX + eX + eW, fY + eY + eH))

        return rects

ec = EyeTracker("haarcascade_frontalface_default.xml", "haarcascade_eye.xml")
cam = cv2.VideoCapture(0)

while True:
    (grabbed, frame) = cam.read()

    # standard korrigeringar
    frame = imutils.resize(frame, 400)
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    #Svart canavas
    canavas = np.zeros(frame.shape[:2], dtype="uint8")

    #Hämtar rektanglarna som returneras utav Cascade Classifiersen i track()
    rectangles = ec.track(gray)

    #Vi börjar loopa över rektanglarna och målar upp dem,
    # for rect in rectangles:
    #     cv2.rectangle(canavas, (rect[0],rect[1]),(rect[2],rect[3]), (np.random.randint(0,256),np.random.randint(0,256),np.random.randint(0,256)), 2)

    #Alternativ
    #Måla upp cirklar som varierar i färg
    for rect in rectangles:
        centerX = (rect[0]+ rect[2]) // (2)
        centerY = (rect[1] + rect[3]) // (2)

        radius = rect[2] - rect[0]
        radius = int(radius / 2)
        print(radius)

        cv2.circle(canavas, (centerX, centerY), radius , (np.random.randint(0,256),np.random.randint(0,256), np.random.randint(0,256)),-1)

    #Visar
    cv2.imshow("Tracking",cv2.bitwise_and(frame,frame, mask=canavas))

    #bryter
    if cv2.waitKey(1) & 0xFF == ord("q"):
        break

cam.release()
cv2.destroyAllWindows()