import numpy as np
import cv2
import csv
import matplotlib.pyplot as plt
import os
import sys
from numba import jit, typed, types
import time
from tqdm import tqdm
from scipy import signal
from sep_auto import generate_toeplitz, ac_tensor_uni
from motion import motion_est, motion_est2
import math
from cuda_auto_uni import get_c, get_C, get_c_m, get_C_m

def edge_detect(image):
    sobel = np.array([[1, 0, -1], [2, 0, -2], [1, 0, -1]])
    image_x = signal.convolve2d(image, sobel, mode = 'same')
    image_y = signal.convolve2d(image, sobel.T, mode = 'same')
    mag = np.sqrt(image_x ** 2 + image_y ** 2)
    mag[mag > 255] = 255
    mag[mag < 0] = 0
    return mag.astype(np.uint8)

def q_dict(Q):
    qd = typed.Dict.empty(key_type = types.UniTuple(types.int64, 2), value_type=types.int64, )
    for i in range(Q.shape[0]):
        key = np.asarray([Q[i, 0], Q[i, 1]], dtype = np.int64)
        qd[tuple(key)] = i
    return qd

def get_psnr(image1, image2):
    mse = np.sum((image1 - image2) ** 2)/(image1.shape[0] * image2.shape[1])
    psnr = 10 * math.log10(255 ** 2/mse)
    return psnr

def generate_Q_auto(width):
    Q = np.zeros((width, width, 2))
    #maximum offset
    off = int(width / 2)
    #e.g for a 3x3 kernel centered at current pixel max offset is
    for i in range(0, width):
        Q[i, :, 0] = i - off
        Q[:, i, 1] = i - off
    Q = Q.reshape((width * width, 2)).astype(np.int64)
    return Q

def estimate_coefficients_motion2(c_array, C_array):
    A = np.zeros(c_array.shape)
    print("Estimating coefficients...")
    for i in tqdm(range(0, A.shape[0])):
        for j in range(0, A.shape[1]):
            A[i, j] = np.linalg.lstsq((C_array[i, j]).astype(np.float32), c_array[i,j].astype(np.float32))[0]
    return A

def generate_Q(width):
    Q = np.zeros((2 * width - 1, 2))
    #maximum offset
    off = int(width / 2)
    #generate 1d array of offset vectors
    #for kernel width of 3 should be [-1,0], [0,0], [1,0], [0, -1], [0,0] -- plus shape without repeating origin
    for i in range(0, width):
        Q[i] = np.array([i - off, 0])

    for i in range(0, off):
        Q[width + i] = np.array([0, i - off])

    for i in range(0, off):
        Q[width + off + i] = np.array([0, i + 1])
    Q = Q.astype(np.int64)
    return Q

@jit(nopython=True)
def estimate_frame_motion(I1, I2, A, A2, k_width, k_off, b, motion, mvs1, mvs2, mvs):
    #p is past, f is future
    predicted = np.zeros((A.shape[0], A.shape[1], 3))
    #for each colour channel, use the corresponding AR coefficients to estimate future frame
    for c in range(0, 3):
        for i in range(0, A.shape[0]):
            for j in range(0, A.shape[1]):

                kernel_p                   = np.zeros((k_width, k_width))
                a_p                         = A[i, j, 0 : 2 * k_width - 1]
                kernel_p[:, k_off]          = a_p[ 0 : k_width]
                kernel_p[k_off, 0 : k_off ] = a_p[k_width : k_width + k_off]
                kernel_p[k_off, k_off + 1:] = a_p[k_width + k_off:]

                kernel_f                   = np.zeros((k_width, k_width))
                a_f                         = A2[i, j, 0 : 2 * k_width - 1]
                kernel_f[:, k_off]          = a_f[ 0 : k_width]
                kernel_f[k_off, 0 : k_off ] = a_f[k_width : k_width + k_off]
                kernel_f[k_off, k_off + 1:] = a_f[k_width + k_off:]

                k_sum = np.sum(kernel_p + kernel_f)
                if(k_sum != 0):
                   kernel_p = kernel_p/k_sum
                   kernel_f = kernel_f/k_sum

                y1 = i + b + mvs[i + b, j + b, 0]
                x1 = j + b + mvs[i + b, j + b, 1]
                y2 = i + b - mvs[i + b, j + b, 0]
                x2 = j + b - mvs[i + b, j + b, 1]

                patch1 = I1[y1 - k_off: y1 + k_off + 1, x1 - k_off: x1 + k_off + 1, c]
                patch2 = I2[y2 - k_off: y2 + k_off + 1, x2 - k_off: x2 + k_off + 1, c]
                mask = kernel_p * patch1 + kernel_f * patch2
                predicted[i, j, c] = np.sum(mask)

    return predicted

def predict_frame_uni(image1, image2, k_width, ac_block, motion):
    #constants
    k_off  = int(k_width  / 2)
    max_motion = 10
    b      = int(ac_block / 2) + int(k_width / 2) + max_motion
    I1 = cv2.copyMakeBorder(cv2.imread(image1), b, b, b, b, cv2.BORDER_REFLECT).astype(np.int32)
    I2 = cv2.copyMakeBorder(cv2.imread(image2), b, b, b, b, cv2.BORDER_REFLECT).astype(np.int32)

    Q = generate_Q(k_width)

    i_g = np.zeros((I1.shape[0], I1.shape[1], 2), dtype = np.int32)
    i_g[:, :, 0] = I1[:, :, 1]
    i_g[:, :, 1] = I2[:, :, 1]

    i_g2 = np.zeros((I1.shape[0], I1.shape[1], 2), dtype = np.int32)
    i_g2[:, :, 1] = I1[:, :, 1]
    i_g2[:, :, 0] = I2[:, :, 1]

    # i_g_edge = np.zeros((I1.shape[0], I1.shape[1], 2), dtype = np.int32)
    # i_g_edge[:, :, 0] = edge_detect(i_g[:, :, 0])
    # i_g_edge[:, :, 1] = edge_detect(i_g[:, :, 1])

    # i_g_edge2 = np.zeros((I1.shape[0], I1.shape[1], 2), dtype = np.int32)
    # i_g_edge2[:, :, 1] = edge_detect(i_g[:, :, 0])
    # i_g_edge2[:, :, 0] = edge_detect(i_g[:, :, 1])

    if(motion == 1):
        mvs1 = motion_est(i_g[:, :, 0], i_g[:, :, 1], b, max_motion)
        mvs2 = motion_est(i_g[:, :, 1], i_g[:, :, 0], b, max_motion)
        mvs = motion_est2(i_g[:, :, 0], i_g[:, :, 1], b, max_motion)

    else:
        mvs1 = np.zeros((i_g.shape[0], i_g.shape[1], 2), dtype = np.int64)
        mvs2 = np.zeros((i_g.shape[0], i_g.shape[1], 2), dtype = np.int64)
        mvs =  np.zeros((i_g.shape[0], i_g.shape[1], 2), dtype = np.int64)

    c_g = get_c_m(i_g, Q, int(ac_block/2), mvs1, b)
    C_g = get_C_m(i_g, Q, int(ac_block/2), mvs1, b)
    A_g = estimate_coefficients_motion2(c_g, C_g)

    c_g2 = get_c_m(i_g2, Q, int(ac_block/2), mvs2, b)
    C_g2 = get_C_m(i_g2, Q, int(ac_block/2), mvs2, b)
    A_g2 = estimate_coefficients_motion2(c_g2, C_g2)

    # c_g_edge = get_c_m(i_g_edge, Q, int(ac_block/2), mvs1, b)
    # C_g_edge = get_C_m(i_g_edge, Q, int(ac_block/2), mvs1, b)
    # A_g_edge = estimate_coefficients_motion2(c_g_edge, C_g_edge)

    # c_g_edge2 = get_c_m(i_g_edge2, Q, int(ac_block/2), mvs1, b)
    # C_g_edge2 = get_C_m(i_g_edge2, Q, int(ac_block/2), mvs1, b)
    # A_g_edge2 = estimate_coefficients_motion2(c_g_edge2, C_g_edge2)

    predicted = estimate_frame_motion(I1, I2, A_g, A_g2, k_width, k_off, b, motion, mvs1, mvs2, mvs)

    predicted[predicted > 255] = 255
    predicted[predicted < 0] = 0
    return predicted

#global variables
def main():
    if(len(sys.argv) == 8):
        image1 = sys.argv[1]
        image2 = sys.argv[2]
        image3 = sys.argv[3]
        out = sys.argv[4]
        k_width = int(sys.argv[5])
        ac_block = int(sys.argv[6])
        motion = int(sys.argv[7])

    else:
        image1 = '../football/image4.png'
        image2 = '../football/image5.png'
        image3 = '../football/image6.png'
        out = '../output/out.png'
        k_width = 5
        ac_block = 23
        motion = 1

    print("Kernel size:", k_width)
    print("Autocorrelation kernel size:", ac_block)
    #kernel max offsets (the max index to be used)
    print("Predicting frames")
    predicted = predict_frame_uni(image1, image3, k_width, ac_block, motion)
    print("PSNR is :", get_psnr(cv2.imread(image2), predicted))
    if(cv2.imwrite(out, predicted) == False):
        print("Error writing file!")
    else:
        print("Image written to file")

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
    shipment.py

"""
from trytond.model import fields, ModelView, ModelSQL
from trytond.pool import PoolMeta, Pool
from trytond.wizard import Wizard, StateView, Button, StateTransition
from trytond.pyson import Eval, Or, Bool, Id
from trytond.transaction import Transaction

__metaclass__ = PoolMeta
__all__ = [
    'ShipmentOut', 'GenerateShippingLabelMessage',
    'GenerateShippingLabel', 'ShippingCarrierSelector',
    'ShippingLabelNoModules', 'Package', 'ShipmentTracking'
]

class Package:
    __name__ = 'stock.package'

    tracking_number = fields.Function(
        fields.Many2One('shipment.tracking', 'Tracking Number'),
        'get_tracking_number', searcher="search_tracking_number"
    )

    weight = fields.Function(
        fields.Float(
            "Weight", digits=(16,  Eval('weight_digits', 2)),
            depends=['weight_digits'],
        ),
        'get_weight'
    )
    weight_uom = fields.Function(
        fields.Many2One('product.uom', 'Weight UOM'),
        'get_weight_uom'
    )
    weight_digits = fields.Function(
        fields.Integer('Weight Digits'), 'on_change_with_weight_digits'
    )

    computed_weight = fields.Function(
        fields.Float(
            "Computed Weight", digits=(16,  Eval('weight_digits', 2)),
            depends=['weight_digits'],
        ),
        'get_computed_weight'
    )

    override_weight = fields.Float(
        "Override Weight", digits=(16,  Eval('weight_digits', 2)),
        depends=['weight_digits'],
    )

    available_box_types = fields.Function(
        fields.One2Many("carrier.box_type", None, "Available Box Types"),
        getter="on_change_with_available_box_types"
    )
    box_type = fields.Many2One(
        'carrier.box_type', 'Box Types', domain=[
            ('id', 'in', Eval('available_box_types'))
        ], depends=["available_box_types"]
    )

    length = fields.Float('Length')
    width = fields.Float('Width')
    height = fields.Float('Height')
    distance_unit = fields.Many2One(
        'product.uom', 'Distance Unit', states={
            'required': Or(Bool(Eval('length')), Bool(
                Eval('width')), Bool(Eval('height')))
        },
        domain=[
            ('category', '=', Id('product', 'uom_cat_length'))
        ], depends=['length', 'width', 'height']
    )

    @fields.depends("_parent_shipment", "_parent_shipment.carrier")
    def on_change_with_available_box_types(self, name=None):
        if self.shipment.carrier:
            return map(int, self.shipment.carrier.box_types)
        return []

    def get_tracking_number(self, name):
        """
        Return first tracking number for this package
        """
        Tracking = Pool().get('shipment.tracking')

        tracking_numbers = Tracking.search([
            ('origin', '=', '%s,%s' % (self.__name__, self.id)),
        ], limit=1)

        return tracking_numbers and tracking_numbers[0].id or None

    @classmethod
    def search_tracking_number(cls, name, clause):
        Tracking = Pool().get('shipment.tracking')

        tracking_numbers = Tracking.search([
            ('origin', 'like', 'stock.package,%'),
            ('tracking_number', ) + tuple(clause[1:])
        ])
        return [
            ('id', 'in', map(lambda x: x.origin.id, tracking_numbers))
        ]

    @fields.depends('weight_uom')
    def on_change_with_weight_digits(self, name=None):
        if self.weight_uom:
            return self.weight_uom.digits
        return 2

    def get_weight_uom(self, name):
        """
        Returns weight uom for the package from shipment
        """
        return self.shipment.weight_uom.id

    def get_weight(self, name):
        """
        Returns package weight if weight is not overriden
        otherwise returns overriden weight
        """
        return self.override_weight or self.get_computed_weight()

    def get_computed_weight(self, name=None):
        """
        Returns sum of weight associated with each move line
        """
        return sum(map(
            lambda move: move.get_weight(self.weight_uom, silent=True),
            self.moves
        ))

    @staticmethod
    def default_type():
        ModelData = Pool().get('ir.model.data')
        return ModelData.get_id(
            'shipping', 'shipment_package_type'
        )

    @staticmethod
    def default_distance_unit():
        ModelData = Pool().get('ir.model.data')
        return ModelData.get_id(
            'product', 'uom_inch'
        )

class ShipmentOut:
    "Shipment Out"
    __name__ = 'stock.shipment.out'

    is_international_shipping = fields.Function(
        fields.Boolean("Is International Shipping"),
        'on_change_with_is_international_shipping'
    )

    weight = fields.Function(
        fields.Float(
            "Weight", digits=(16,  Eval('weight_digits', 2)),
            depends=['weight_digits'],
        ),
        'get_weight'
    )
    weight_uom = fields.Function(
        fields.Many2One('product.uom', 'Weight UOM'),
        'get_weight_uom'
    )
    weight_digits = fields.Function(
        fields.Integer('Weight Digits'), 'on_change_with_weight_digits'
    )

    tracking_number = fields.Many2One(
        'shipment.tracking', 'Tracking Number', select=True,
        states={'readonly': Eval('state') == 'done'}, depends=['state']
    )

    shipping_instructions = fields.Text(
        'Shipping Instructions', states={
            'readonly': Eval('state').in_(['cancel', 'done']),
        }, depends=['state']
    )

    available_carrier_services = fields.Function(
        fields.One2Many("carrier.service", None, "Available Carrier Services"),
        getter="on_change_with_available_carrier_services"
    )
    carrier_service = fields.Many2One(
        "carrier.service", "Carrier Service", domain=[
            ('id', 'in', Eval('available_carrier_services'))
        ], depends=['available_carrier_services', 'state']
    )
    carrier_cost_method = fields.Function(
        fields.Char('Carrier Cost Method'),
        "on_change_with_carrier_cost_method"
    )
    shipping_manifest = fields.Many2One(
        "shipping.manifest", "Shipping Manifest", readonly=True, select=True
    )

    @fields.depends("carrier")
    def on_change_with_carrier_cost_method(self, name=None):
        if self.carrier:
            return self.carrier.carrier_cost_method

    @fields.depends('carrier')
    def on_change_with_available_carrier_services(self, name=None):
        if self.carrier:
            return map(int, self.carrier.services)
        return []

    def on_change_inventory_moves(self):
        with Transaction().set_context(ignore_carrier_computation=True):
            return super(ShipmentOut, self).on_change_inventory_moves()

    def get_weight(self, name=None):
        """
        Returns sum of weight associated with each move line
        """
        return sum(map(
            lambda move: move.get_weight(self.weight_uom, silent=True),
            self.outgoing_moves
        ))

    @fields.depends('weight_uom')
    def on_change_with_weight_digits(self, name=None):
        if self.weight_uom:
            return self.weight_uom.digits
        return 2

    @classmethod
    def __setup__(cls):
        super(ShipmentOut, cls).__setup__()
        readonly_when_done = {
            'readonly': Eval('state') == 'done',
        }
        cls.carrier.states = readonly_when_done
        cls.carrier_service.states = readonly_when_done
        cls._buttons.update({
            'label_wizard': {
                'invisible': Or(
                    (~Eval('state').in_(['packed', 'done'])),
                    (Bool(Eval('tracking_number')))
                ),
                'icon': 'tryton-executable',
            },
        })
        cls._error_messages.update({
            'no_shipments': 'There must be atleast one shipment.',
            'too_many_shipments':
                'The wizard can be called on only one shipment',
            'tracking_number_already_present':
                'Tracking Number is already present for this shipment.',
            'invalid_state': 'Labels can only be generated when the '
                'shipment is in Packed or Done states only',
            'wrong_carrier':
                'Carrier for selected shipment is not of %s',
            'no_packages': 'Shipment %s has no packages',
            'warehouse_address_missing': 'Warehouse address is missing',
        })

    @classmethod
    @ModelView.button_action('shipping.wizard_generate_shipping_label')
    def label_wizard(cls, shipments):
        if len(shipments) == 0:
            cls.raise_user_error('no_shipments')
        elif len(shipments) > 1:
            cls.raise_user_error('too_many_shipments')

    @classmethod
    def copy(cls, shipments, default=None):
        if default is None:
            default = {}
        default = default.copy()
        default['tracking_number'] = None
        return super(ShipmentOut, cls).copy(shipments, default=default)

    @fields.depends('delivery_address', 'warehouse')
    def on_change_with_is_international_shipping(self, name=None):
        """
        Return True if international shipping
        """
        from_address = self._get_ship_from_address(silent=True)
        if self.delivery_address and from_address and \
           from_address.country and self.delivery_address.country and \
           from_address.country != self.delivery_address.country:
            return True
        return False

    def get_weight_uom(self, name):
        """
        Returns weight uom for the shipment
        """
        ModelData = Pool().get('ir.model.data')

        return ModelData.get_id('product', 'uom_pound')

    def _get_ship_from_address(self, silent=False):
        """
        Usually the warehouse from which you ship
        """
        if not self.warehouse.address and not silent:
            return self.raise_user_error('warehouse_address_missing')
        return self.warehouse and self.warehouse.address

    def allow_label_generation(self):
        """
        Shipment must be in the right states and tracking number must not
        be present.
        """
        if self.state not in ('packed', 'done'):
            self.raise_user_error('invalid_state')

        if self.tracking_number:
            self.raise_user_error('tracking_number_already_present')

        return True

    def _create_default_package(self, box_type=None):
        """
        Create a single stock package for the whole shipment
        """
        Package = Pool().get('stock.package')

        package, = Package.create([{
            'shipment': '%s,%d' % (self.__name__, self.id),
            'box_type': box_type and box_type.id,
            'moves': [('add', self.outgoing_moves)],
        }])
        return package

    def get_shipping_rates(self, carriers=None, silent=False):
        """
        Gives a list of rates from carriers provided. If no carriers provided,
        return rates from all the carriers.

        List contains dictionary with following minimum keys:
            [
                {
                    'display_name': Name to display,
                    'carrier_service': carrier.service active record,
                    'cost': cost,
                    'cost_currency': currency.currency active repord,
                    'carrier': carrier active record,
                }..
            ]
        """
        Carrier = Pool().get('carrier')

        if carriers is None:
            carriers = Carrier.search([])

        rates = []
        for carrier in carriers:
            rates.extend(self.get_shipping_rate(carrier, silent))
        return rates

    def get_shipping_rate(self, carrier, carrier_service=None, silent=False):
        """
        Gives a list of rates from provided carrier and carrier service.

        List contains dictionary with following minimum keys:
            [
                {
                    'display_name': Name to display,
                    'carrier_service': carrier.service active record,
                    'cost': cost,
                    'cost_currency': currency.currency active repord,
                    'carrier': carrier active record,
                }..
            ]
        """
        Company = Pool().get('company.company')

        if carrier.carrier_cost_method == 'product':
            currency = Company(Transaction().context['company']).currency
            rate_dict = {
                'display_name': carrier.rec_name,
                'carrier_service': carrier_service,
                'cost': carrier.carrier_product.list_price,
                'cost_currency': currency,
                'carrier': carrier,
            }
            return [rate_dict]

        return []

    def apply_shipping_rate(self, rate):
        """
        This method applies shipping rate. Rate is a dictionary with
        following minimum keys:

            {
                'display_name': Name to display,
                'carrier_service': carrier.service active record,
                'cost': cost,
                'cost_currency': currency.currency active repord,
                'carrier': carrier active record,
            }
        """
        Currency = Pool().get('currency.currency')

        shipment_cost = rate['cost_currency'].round(rate['cost'])
        if self.cost_currency != rate['cost_currency']:
            shipment_cost = Currency.compute(
                rate['cost_currency'], shipment_cost, self.cost_currency
            )

        self.cost = shipment_cost
        self.cost_currency = self.cost_currency
        self.carrier = rate['carrier']
        self.carrier_service = rate['carrier_service']
        self.save()

    def generate_shipping_labels(self, **kwargs):
        """
        Generates shipment label for shipment and saves labels,
        tracking numbers.
        """
        self.raise_user_error(
            "Shipping label generation feature is not available"
        )

class ShippingCarrierSelector(ModelView):
    'View To Select Carrier'
    __name__ = 'shipping.label.start'

    carrier = fields.Many2One(
        "carrier", "Carrier", required=True
    )
    override_weight = fields.Float("Override Weight", digits=(16,  2))
    no_of_packages = fields.Integer('Number of packages', readonly=True)
    box_type = fields.Many2One(
        "carrier.box_type", "Box Type", required=True, domain=[
            ('id', 'in', Eval("available_box_types"))
        ], depends=["available_box_types"]
    )

    shipping_instructions = fields.Text('Shipping Instructions', readonly=True)
    carrier_service = fields.Many2One(
        "carrier.service", "Carrier Service", domain=[
            ('id', 'in', Eval("available_carrier_services"))
        ], depends=["available_carrier_services"]
    )

    available_box_types = fields.Function(
        fields.One2Many("carrier.box_type", None, 'Available Box Types'),
        getter="on_change_with_available_box_types"
    )
    available_carrier_services = fields.Function(
        fields.One2Many("carrier.service", None, 'Available Carrier Services'),
        getter="on_change_with_available_carrier_services"
    )

    @fields.depends("carrier")
    def on_change_with_available_box_types(self, name=None):
        if self.carrier:
            return map(int, self.carrier.box_types)
        return []

    @fields.depends("carrier")
    def on_change_with_available_carrier_services(self, name=None):
        if self.carrier:
            return map(int, self.carrier.services)
        return []

class GenerateShippingLabelMessage(ModelView):
    'Generate Labels Message'
    __name__ = 'shipping.label.end'

    tracking_number = fields.Many2One(
        "shipment.tracking", "Tracking number", readonly=True
    )
    message = fields.Text("Message", readonly=True)
    attachments = fields.One2Many(
        'ir.attachment', None, 'Attachments', readonly=True
    )
    cost = fields.Numeric("Cost", digits=(16, 2), readonly=True)
    cost_currency = fields.Many2One(
        'currency.currency', 'Cost Currency', readonly=True
    )

class ShippingLabelNoModules(ModelView):
    'Wizard State for No Modules'
    __name__ = 'shipping.label.no_modules'

    no_module_msg = fields.Text("Message", readonly=True)

    @staticmethod
    def default_no_module_msg():
        """
        Returns default message.
        """
        return (
            'No shipping module is available for label generation.'
            'Please install a shipping module first.'
        )

class GenerateShippingLabel(Wizard):
    'Generate Labels'
    __name__ = 'shipping.label'

    #: This is the first state of wizard to generate shipping label.
    #: It asks for carrier, carrier_service, box_type and override weight,
    #: once entered, it move to `next` transition where it saves all the
    #: values on shipment.
    start = StateView(
        'shipping.label.start',
        'shipping.select_carrier_view_form',
        [
            Button('Cancel', 'end', 'tryton-cancel'),
            Button('Continue', 'next', 'tryton-go-next'),
        ]
    )

    #: Transition saves values from `start` state to the shipment.
    next = StateTransition()

    #: This state is showed up when shipping provider doesnt give facility
    #: to generate labels.
    no_modules = StateView(
        'shipping.label.no_modules',
        'shipping.no_module_view_form',
        [
            Button('Ok', 'end', 'tryton-ok')
        ]
    )

    #: Transition generates shipping labels.
    generate_labels = StateTransition()

    #: State shows the generated label, tracking number, cost and cost
    #: currency.
    generate = StateView(
        'shipping.label.end',
        'shipping.generate_shipping_label_message_view_form',
        [
            Button('Ok', 'end', 'tryton-ok'),
        ]
    )

    @property
    def shipment(self):
        "Gives the active shipment."
        Shipment = Pool().get('stock.shipment.out')
        return Shipment(Transaction().context.get('active_id'))

    @classmethod
    def __setup__(cls):
        super(GenerateShippingLabel, cls).__setup__()
        cls._error_messages.update({
            'tracking_number_already_present':
                'Tracking Number is already present for this shipment.',
            'invalid_state': (
                'Labels can only be generated when the shipment is in Packed or'
                ' Done states only'
            ),
            'no_packages': 'Shipment %s has no packages',
        })

    def default_start(self, data):
        """Fill the default values for `start` state.
        """
        if self.shipment.allow_label_generation():
            values = {
                'no_of_packages': len(self.shipment.packages),
                'shipping_instructions': self.shipment.shipping_instructions,
            }

        if self.shipment.carrier:
            values.update({
                'carrier': self.shipment.carrier.id,
            })
        if self.shipment.packages:
            package_weights = [
                p.override_weight
                for p in self.shipment.packages if p.override_weight
            ]
            values['override_weight'] = sum(package_weights)

        if self.shipment.carrier_service:
            values['carrier_service'] = self.shipment.carrier_service.id

        return values

    def transition_next(self):
        Company = Pool().get('company.company')

        shipment = self.shipment
        company = Company(Transaction().context['company'])
        shipment.carrier = self.start.carrier
        shipment.cost_currency = company.currency
        shipment.carrier_service = self.start.carrier_service
        shipment.save()

        if not shipment.packages:
            shipment._create_default_package(self.start.box_type)

        default_values = self.default_start({})
        per_package_weight = None
        if self.start.override_weight and \
                default_values['override_weight'] != self.start.override_weight:
            # Distribute weight equally
            per_package_weight = (
                self.start.override_weight / len(shipment.packages)
            )

        for package in shipment.packages:
            if per_package_weight:
                package.override_weight = per_package_weight
            if self.start.box_type != package.box_type:
                package.box_type = self.start.box_type
            package.save()

        return 'no_modules'

    def transition_generate_labels(self):
        "Generates shipping labels from data provided by earlier states"
        self.shipment.generate_shipping_labels()

        return "generate"

    def get_attachments(self):  # pragma: no cover
        """
        Returns list of attachments corresponding to shipment.
        """
        Attachment = Pool().get('ir.attachment')

        return map(int, Attachment.search([
            ('resource', '=', '%s,%d' %
                (self.shipment.__name__, self.shipment.id))
            ])
        )

    def get_message(self):
        """
        Returns message to be displayed on wizard
        """
        message = 'Shipment labels have been generated via %s and saved as ' \
            'attachments for the shipment' % (
                self.shipment.carrier.carrier_cost_method.upper()
            )
        return message

    def default_generate(self, data):
        return {
            'tracking_number': self.shipment.tracking_number and
            self.shipment.tracking_number.id,
            'message': self.get_message(),
            'attachments': self.get_attachments(),
            'cost': self.shipment.cost,
            'cost_currency': self.shipment.cost_currency.id,
        }

class ShipmentTracking(ModelSQL, ModelView):
    """Shipment Tracking
    """
    __name__ = 'shipment.tracking'
    _rec_name = 'tracking_number'

    #: Boolean to indicate if tracking number is master.
    is_master = fields.Boolean("Is Master ?", readonly=True, select=True)

    origin = fields.Reference(
        'Origin', selection='get_origin', select=True, readonly=True
    )
    tracking_number = fields.Char(
        "Tracking Number", required=True, select=True, readonly=True
    )
    carrier = fields.Many2One(
        'carrier', 'Carrier', required=True, readonly=True
    )
    tracking_url = fields.Char("Tracking Url", readonly=True)
    state = fields.Selection([
        ('waiting', 'Waiting'),
        ('in_transit', 'In Transit'),
        ('delivered', 'Delivered'),
        ('failure', 'Failure'),
        ('returned', 'Returned'),
        ('cancelled', 'Cancelled'),
        ('pending_cancellation', 'Pending Cancellation'),
        ], 'State', readonly=True, required=True, select=True)

    @staticmethod
    def default_state():
        return 'waiting'

    @classmethod
    def __setup__(cls):
        """
        Setup the class before adding to pool
        """
        super(ShipmentTracking, cls).__setup__()
        cls._buttons.update({
            'cancel_tracking_number_button': {
                'invisible': Eval('state') == 'cancelled',
            },
            'refresh_status_button': {},
        })

    def cancel_tracking_number(self):
        "Cancel tracking number"
        self.state = 'cancelled'
        self.save()

    @classmethod
    @ModelView.button
    def cancel_tracking_number_button(cls, tracking_numbers):
        """
        Cancel tracking numbers
        """
        for tracking_number in tracking_numbers:
            tracking_number.cancel_tracking_number()

    def refresh_status(self):
        """
        Downstream module can implement this
        """
        pass

    @classmethod
    @ModelView.button
    def refresh_status_button(cls, tracking_numbers):
        """
        Update tracking numbers state
        """
        for tracking_number in tracking_numbers:
            tracking_number.refresh_status()

    @classmethod
    def refresh_tracking_numbers_cron(cls):
        """
        This is a cron method, responsible for updating state of
        shipments.
        """
        states_to_refresh = [
            'pending_cancellation',
            'failure',
            'waiting',
            'in_transit',
        ]

        tracking_numbers = cls.search([
            ('state', 'in', states_to_refresh),
        ])
        for tracking_number in tracking_numbers:
            tracking_number.refresh_status()

    @classmethod
    def _get_origin(cls):
        'Return list of Model names for origin Reference'
        return ['stock.shipment.out', 'stock.package']

    @classmethod
    def get_origin(cls):
        Model = Pool().get('ir.model')
        models = cls._get_origin()
        models = Model.search([
            ('model', 'in', models),
        ])
        return [(None, '')] + [(m.model, m.name) for m in models]


import datetime

from connexion import NoContent
from flask import request
from flask.views import MethodView

class PetsView(MethodView):
    """ Create Pets service
    """
    method_decorators = []
    pets = {}

    def post(self):
      body= request.json
      name = body.get("name")
      tag = body.get("tag")
      count = len(self.pets)
      pet = {}
      pet['id'] = count + 1
      pet["tag"] = tag
      pet["name"] = name
      pet['last_updated'] = datetime.datetime.now()
      self.pets[pet['id']] = pet
      return pet, 201

    def put(self, petId):
      body = request.json
      name = body["name"]
      tag = body.get("tag")
      id_ = int(petId)
      pet = self.pets.get(petId, {"id": id_})
      pet["name"] = name
      pet["tag"] = tag
      pet['last_updated'] = datetime.datetime.now()
      self.pets[id_] = pet
      return self.pets[id_], 201

    def delete(self, petId):
      id_ = int(petId)
      if self.pets.get(id_) is None:
          return NoContent, 404
      del self.pets[id_]
      return NoContent, 204

    def get(self, petId):
      id_ = int(petId)
      if self.pets.get(id_) is None:
          return NoContent, 404
      return self.pets[id_]

    def search(self, limit=100):
      # NOTE: we need to wrap it with list for Python 3 as dict_values is not JSON serializable
      return list(self.pets.values())[0:limit]


#!/usr/bin/python

from neopixel import *
import random
import datetime

MIN_SIZE = 2
MAX_SIZE = 10

class Stick:

    def __init__(self, startPos, size, ledCount):
        self.size = size
        self.ledCount = ledCount
        self.startPos = startPos
        self.endPos = self.startPos + size
        self.velocity = random.randint(1, 1000)
        self.direction = bool(random.getrandbits(1))
        self.lastMove = datetime.datetime.now()
        self.setRandomColor()

    def edgeDetection(self):
        if ((self.startPos <= 0 and not self.direction) or (self.endPos >= self.ledCount and self.direction)):
            self.direction = not self.direction

    def setRandomColor(self):
        self.color = Color(random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))

    def step(self):
        timeDelta = datetime.datetime.now() - self.lastMove
        millSecs = int(timeDelta.total_seconds() * 1000)
        if (millSecs > self.velocity):
            self.edgeDetection()

            if (self.direction):
                self.startPos += 1
                self.endPos += 1
            else:
                self.startPos -= 1
                self.endPos -= 1
            self.lastMove = datetime.datetime.now()

class Colliders:

    def __init__(self, settings):
        self.settings = settings
        self.sticks = []
        #Temp
        for i in range(15):
            self.spawnSingle()

    def isSpaceAvailable(self, startPos, endPos):
        for stick in self.sticks:
            if ((startPos >= stick.startPos or endPos >= stick.startPos)
                and (startPos <= stick.endPos or endPos <= stick.endPos)):
                return False
        return True

    def spawnSingle(self):
        retries = 10
        spaceIsFree = False

        while (not spaceIsFree and retries > 0):
            size = random.randint(MIN_SIZE, MAX_SIZE)
            startPos = random.randint(0, self.settings.ledCount)
            spaceIsFree = self.isSpaceAvailable(startPos, startPos + size)
            if (spaceIsFree):
                stick = Stick(startPos, size, self.settings.ledCount)
                self.sticks.append(stick)
                break
            retries -= 1

    def deleteSingle(self):
        self.sticks.pop()

    def drawAll(self):
        self.clear()
        for stick in self.sticks:
            for i in range(stick.startPos, stick.endPos):
                self.settings.strip.setPixelColor(i, stick.color)

    def clear(self):
        for i in range(0, self.settings.ledCount):
            self.settings.strip.setPixelColorRGB(i, 0, 0, 0)

    def collisionDetection(self, cStick, currentIndex):
        for i, stick in enumerate(self.sticks):
            if ((i != currentIndex)
                and (((cStick.endPos >= stick.startPos and cStick.startPos < stick.startPos)
                    and ((cStick.direction and (not stick.direction
                             or (stick.direction and cStick.velocity < stick.velocity)))
                        or (not cStick.direction and not stick.direction and cStick.velocity > stick.velocity)))
                    or ((cStick.startPos <= stick.endPos and cStick.endPos > stick.endPos)
                    and ((not cStick.direction and (stick.direction
                            or (not stick.direction and cStick.velocity < stick.velocity)))
                        or (cStick.direction and stick.direction and cStick.velocity > stick.velocity))))):
                if (cStick.direction != stick.direction):
                    currentDirection = cStick.direction
                    cStick.direction = not currentDirection
                    stick.direction = currentDirection
                currentLastMov = cStick.lastMove
                currentVelocity = cStick.velocity
                cStick.velocity = stick.velocity
                cStick.lastMove = stick.lastMove
                stick.velocity = currentVelocity
                stick.lastMove = currentLastMov
                stick.setRandomColor()
                cStick.setRandomColor()

    def step(self):
        for i, stick in enumerate(self.sticks):
            self.collisionDetection(stick, i)
            stick.step()
        self.drawAll()
        self.settings.strip.show()

    def show(self, settings = None):
        if (settings != None):
            self.settings = settings
        self.step()


'''

'''
import os, subprocess, time
from sched import scheduler
import RunVasp
from comMethods import finishCheck, convergeCheck, getNSW, joinLists, flat

class JobManager:
    """ This class is responsible for scheduling and watching the VASP calculations.  It starts
        the low-precision relaxation, waits for all structures to either timeout or complete,
        reports how many structures converged and how many did not, then starts the
        normal-precision relaxation for all structures that converged.  It repeats this process
        for the normal-precision relaxation and the DOS runs.  An instance of the RunVasp class
        is used to prepare all the directories, retrieve the needed files, and submit the jobs to
        the supercomputer. """

    def __init__(self, user, atomSets,siteConcList,ediffLowScale,jobstime):
        """ CONSTRUCTOR """
        self.jobstime = jobstime
        self.atomSets = atomSets
        self.vaspRunner = RunVasp.RunVasp(user, self.atomSets, siteConcList,ediffLowScale,jobstime)

    def reportFinished(self, jobIds):
        """ Returns true if all the jobs with IDs given in 'jobIds' have finished VASP
            calculations, false otherwise. """
        devnull = open(os.devnull, 'w')
        for jobid in jobIds:
            proc = subprocess.Popen(['squeue', '--job', jobid], stdout=subprocess.PIPE, stderr=devnull)
            output = proc.communicate()[0].split()
            if len(output) != 8 and len(output) != 0:   # It will list either all the headers or
                return False                            # sometimes an error and outputs nothing.
                                                        # The error in this case is an "invalid
                                                        # job id" error because the job is no
        return True                                     # longer on the supercomputer.

    def reportLowStats(self, vstructsToStart):
        """ Displays the percentage of structures that converged during the low-precision VASP
            calculations.  Also displays the number of structures that converged and the number
            of structures that didn't. """
        for i in xrange(len(self.atomSets)):
            subprocess.call(['echo','\nFor atom ' + self.atomSets[i] + ':'])
            total = 0
            converged = 0
            atomSetDir = os.getcwd() + '/' + self.atomSets[i]
            if os.path.isdir(atomSetDir):
                for structure in vstructsToStart[i]:
                    structureDir = atomSetDir + '/' + structure
                    if os.path.isdir(structureDir):
                        if finishCheck(structureDir) and convergeCheck(structureDir, getNSW(structureDir)):
                            total += 1
                            converged += 1
                        else:
                            total += 1

            if total == 0:
                subprocess.call(['echo','\tNo structures were submitted for ' + self.atomSets[i]])
            else:
                percent = float(float(converged) / float(total)) * 100.0
                notConverged = total - converged
                subprocess.call(['echo','\t%5.2f %% of the structures converged during low-precision relaxation.' % percent])
                subprocess.call(['echo','\t%d structures converged.' % converged])
                subprocess.call(['echo','\t%d structures did not converge.' % notConverged])

    def reportDOSStats(self, vstructsToStart):
        """ Reports the percentage of structures that converged during the Density of States VASP
            calculations.  Also reports the number of structures that converged and the number
            that didn't. """
        for i in xrange(len(self.atomSets)):
            subprocess.call(['echo','\nFor atom ' + self.atomSets[i] + ':'])
            total = 0
            converged = 0
            atomSetDir = os.getcwd() + '/' + self.atomSets[i]
            if os.path.isdir(atomSetDir):
                for struct in vstructsToStart[i]:
                    structDir = atomSetDir + '/' + struct
                    if os.path.isdir(structDir):
                        dosDir = structDir + '/DOS'
                        if os.path.isdir(dosDir):
			    #if os.path.isdir(dosDir + '/vasp.x'):
                            #    subprocess.call(['rm',dosDir + '/vasp.x'])
                            if finishCheck(dosDir) and convergeCheck(dosDir, 2):
                                total += 1
                                converged += 1
                            else:
                                total += 1
            if total == 0:
                subprocess.call(['echo','\tNo structures were submitted for ' + self.atomSets[i]])
            else:
                percent = float(float(converged) / float(total)) * 100.0
                notConverged = total - converged
                subprocess.call(['echo','\t%5.2f %% of the structures converged during the DOS run.' % percent])
                subprocess.call(['echo','\t%d structures converged.' % converged])
                subprocess.call(['echo','\t%d structures did not converge.' % notConverged])

    def reportNormalStats(self, vstructsToStart):
        """ Reports the percentage of structures that converged during normal-precision VASP
            calculations.  Also reports the number of structures that converged and the number
            that didn't. """
        for i in xrange(len(self.atomSets)):
            subprocess.call(['echo','\nFor atom ' + self.atomSets[i] + ':'])
            total = 0
            converged = 0
            atomSetDir = os.getcwd() + '/' + self.atomSets[i]
            if os.path.isdir(atomSetDir):
                for struct in vstructsToStart[i]:
                    structDir = atomSetDir + '/' + struct
                    if os.path.isdir(structDir):
                        normalDir = structDir + '/normal'
                        if os.path.isdir(normalDir):
                            if finishCheck(normalDir) and convergeCheck(normalDir, getNSW(normalDir)):
                                total += 1
                                converged += 1
                            else:
                                total += 1
            if total == 0:
                subprocess.call(['echo','\tNo structures were submitted for ' + self.atomSets[i]])
            else:
                percent = float(float(converged) / float(total)) * 100.0
                notConverged = total - converged
                subprocess.call(['echo','\t%5.2f %% of the structures converged during normal-precision relaxation.' % percent])
                subprocess.call(['echo','\t%d structures converged.' % converged])
                subprocess.call(['echo','\t%d structures did not converge.' % notConverged])

    def runDOSJobs(self, vstructsToStart, vstructsRestart):
        """ Starts the Density of States VASP calculations for all of the structures in
            'vstructsToStart' and waits for all of the jobs to finish. It checks on the jobs every ten
            minutes. """
        subprocess.call(['echo','\nStarting DOS run. . .\n'])
        vstructsToRun = joinLists([vstructsRestart,vstructsToStart])
        self.vaspRunner.runJobs('DOS', vstructsToStart, vstructsToRun)
        s = scheduler(time.time, time.sleep)
        finished = False
        start_time = time.time()
        event_time = start_time
        while not finished:
            event_time += 60 #seconds
            s.enterabs(event_time, 1, self.reportFinished, ([self.vaspRunner.getCurrentJobIds()]))
            s.run()
            finished = self.reportFinished(self.vaspRunner.getCurrentJobIds())
        self.reportDOSStats(vstructsToRun)

    def runHexMono(self):
        subprocess.call(['echo','\nPreparing and running hexagonal metal monolayers directories\n']) #bch
        jobs = self.vaspRunner.makeRunHexMono()
        return jobs

    def runLowJobs(self, vstructsToStart,vstructsRestart):
        """ Starts the low-precision VASP calculations for all of the structures in 'vstructsToStart'
            and waits for all of the jobs to finish. It checks on the jobs every ten minutes. """
        subprocess.call(['echo','\nPreparing directories for VASP. . .\n'])
        vstructsToRun = joinLists([vstructsRestart,vstructsToStart])
        self.vaspRunner.prepareForVasp(vstructsToStart)
        self.vaspRunner.prepareRestarts(vstructsRestart) #contcar->poscar, incar and job file

        s = scheduler(time.time, time.sleep)

        if len(flat(vstructsToRun)) != 0:
            subprocess.call(['echo','\nStarting low-precision ionic relaxation. . .\n'])
            self.vaspRunner.runJobs('low',vstructsToStart,vstructsToRun)

            finished = False
            start_time = time.time()
            event_time = start_time
            while not finished:
                event_time += 5
                s.enterabs(event_time, 1, self.reportFinished, ([self.vaspRunner.getCurrentJobIds()]))
                s.run()
                finished = self.reportFinished(self.vaspRunner.getCurrentJobIds())

        #self.reportLowStats(vstructsToRun)

    def runNormalJobs(self, vstructsToStart,vstructsRestart):
        """ Starts the normal-precision VASP calculations for all of the structures in 'vstructsToStart'
            and waits for all of the jobs to finish. It checks on the jobs every ten minutes. """
        subprocess.call(['echo','\nStarting normal-precision ionic relaxation. . .\n'])
        vstructsToRun = joinLists([vstructsRestart,vstructsToStart])
        self.vaspRunner.runJobs('accurate', vstructsToStart,vstructsToRun)

        s = scheduler(time.time, time.sleep)

        finished = False
        start_time = time.time()
        event_time = start_time
        while not finished or len(self.vaspRunner.getCurrentJobIds())==0:
            event_time += 60
            s.enterabs(event_time, 1,self.reportFinished(self.vaspRunner.getCurrentJobIds())) #bch: was self.reportFinished, ([self.vaspRunner.getCurrentJobIds()]))
            s.run()
            finished = self.reportFinished(self.vaspRunner.getCurrentJobIds())

        self.reportNormalStats(vstructsToRun)

    def runSingleAtoms(self):
        subprocess.call(['echo','\nPreparing and running single atom directories\n']) #bch
        jobs = self.vaspRunner.makeRunSingleDirectories()
        return jobs


import numpy as np
import matplotlib.pyplot as plt
from sklearn.utils import resample
from scipy.stats import gaussian_kde

from .plotting import splot, oplot

def initial_parsing(df):
    """
    TBD
    """

    # Load dataset
    cells = df.copy().reset_index()

    # %% Check out columns, keep a couple
    keepcolumns = [
        "CellId",
        "structure_name",
        "mem_roundness_surface_area_lcc",
        "mem_shape_volume_lcc",
        "dna_roundness_surface_area_lcc",
        "dna_shape_volume_lcc",
        "str_connectivity_number_cc",
        "str_shape_volume",
        "mem_position_depth_lcc",
        "mem_position_height_lcc",
        "mem_position_width_lcc",
        "dna_position_depth_lcc",
        "dna_position_height_lcc",
        "dna_position_width_lcc",
    ]

    cells = cells[keepcolumns]

    # %% Rename columns
    cells = cells.rename(
        columns={
            "mem_roundness_surface_area_lcc": "Cell surface area",
            "mem_shape_volume_lcc": "Cell volume",
            "dna_roundness_surface_area_lcc": "Nuclear surface area",
            "dna_shape_volume_lcc": "Nuclear volume",
            "str_connectivity_number_cc": "Number of pieces",
            "str_shape_volume": "Structure volume",
            "str_shape_volume_lcc": "Structure volume alt",
            "mem_position_depth_lcc": "Cell height",
            "mem_position_height_lcc": "Cell xbox",
            "mem_position_width_lcc": "Cell ybox",
            "dna_position_depth_lcc": "Nucleus height",
            "dna_position_height_lcc": "Nucleus xbox",
            "dna_position_width_lcc": "Nucleus ybox",
        }
    )

    # %% Add a column
    cells["Cytoplasmic volume"] = cells["Cell volume"] - cells["Nuclear volume"]

    return cells

def outliers_removal(df, output_dir):
    """
    TBD
    """

    # Load dataset
    cells = initial_parsing(df=df)

    # %% Threshold for determing outliers
    cell_dens_th_CN = 1e-20  # for cell-nucleus metrics across all cells
    cell_dens_th_S = 1e-10  # for structure volume metrics

    # Remove outliers

    # %% Remove cells that lack a Structure Volume value
    print(np.any(cells.isnull()))
    cells_ao = cells[["CellId", "structure_name"]].copy()
    cells_ao["Outlier"] = "No"
    print(cells.shape)
    CellIds_remove = (
        cells.loc[cells["Structure volume"].isnull(), "CellId"].squeeze().to_numpy()
    )
    cells_ao.loc[
        cells_ao["CellId"].isin(CellIds_remove), "Outlier"
    ] = "yes_missing_structure_volume"
    cells = cells.drop(cells[cells["CellId"].isin(CellIds_remove)].index)
    cells.reset_index(drop=True)
    print(
        f"Removing {len(CellIds_remove)} cells that lack a Structure Volume measurement value"
    )
    print(cells.shape)
    print(np.any(cells.isnull()))

    # %%
    print("FIX LINE BELOW")
    # cells["Piece std"] = cells["Piece std"].replace(np.nan, 0)

    # %% Feature set for cell and nuclear features
    cellnuc_metrics = [
        "Cell surface area",
        "Cell volume",
        "Cell height",
        "Nuclear surface area",
        "Nuclear volume",
        "Nucleus height",
        "Cytoplasmic volume",
    ]
    cellnuc_abbs = [
        "Cell area",
        "Cell vol",
        "Cell height",
        "Nuc area",
        "Nuc vol",
        "Nuc height",
        "Cyto vol",
    ]

    # %% All metrics including height
    L = len(cellnuc_metrics)
    pairs = np.zeros((int(L * (L - 1) / 2), 2)).astype(np.int)
    i = 0
    for f1 in np.arange(L):
        for f2 in np.arange(L):
            if f2 > f1:
                pairs[i, :] = [f1, f2]
                i += 1

    # %% The typical six scatter plots
    xvec = [1, 1, 6, 1, 4, 6]
    yvec = [4, 6, 4, 0, 3, 3]
    pairs2 = np.stack((xvec, yvec)).T

    # %% Just one
    xvec = [1]
    yvec = [4]

    # %% Parameters
    nbins = 100
    N = 10000
    fac = 1000
    Rounds = 5

    # %% For all pairs compute densities
    remove_cells = cells["CellId"].to_frame().copy()
    for i, xy_pair in enumerate(pairs):

        metricX = cellnuc_metrics[xy_pair[0]]
        metricY = cellnuc_metrics[xy_pair[1]]
        print(f"{metricX} vs {metricY}")

        # data
        x = cells[metricX].to_numpy() / fac
        y = cells[metricY].to_numpy() / fac
        # x = cells[metricX].sample(1000,random_state = 1117).to_numpy() / fac
        # y = cells[metricY].sample(1000,random_state = 1117).to_numpy() / fac

        # density estimate, repeat because of probabilistic nature of density estimate
        # used here
        for r in np.arange(Rounds):
            remove_cells[f"{metricX} vs {metricY}_{r}"] = np.nan
            print(f"Round {r + 1} of {Rounds}")
            rs = int(r)
            xS, yS = resample(
                x, y, replace=False, n_samples=np.amin([N, len(x)]), random_state=rs
            )
            k = gaussian_kde(np.vstack([xS, yS]))
            cell_dens = k(np.vstack([x.flatten(), y.flatten()]))
            cell_dens = cell_dens / np.sum(cell_dens)
            remove_cells.loc[
                remove_cells.index[np.arange(len(cell_dens))],
                f"{metricX} vs {metricY}_{r}",
            ] = cell_dens

    # remove_cells = pd.read_csv(data_root_extra / 'cell_nucleus.csv')

    # %% Summarize across repeats
    remove_cells_summary = cells["CellId"].to_frame().copy()
    for i, xy_pair in enumerate(pairs):
        metricX = cellnuc_metrics[xy_pair[0]]
        metricY = cellnuc_metrics[xy_pair[1]]
        print(f"{metricX} vs {metricY}")
        metricX = cellnuc_metrics[xy_pair[0]]
        metricY = cellnuc_metrics[xy_pair[1]]
        filter_col = [
            col for col in remove_cells if col.startswith(f"{metricX} vs {metricY}")
        ]
        x = remove_cells[filter_col].to_numpy()
        pos = np.argwhere(np.any(x < cell_dens_th_CN, axis=1))
        y = x[pos, :].squeeze()

        fig, axs = plt.subplots(1, 2, figsize=(16, 9))
        xr = np.log(x.flatten())
        xr = np.delete(xr, np.argwhere(np.isinf(xr)))
        axs[0].hist(xr, bins=100)
        axs[0].set_title("Histogram of cell probabilities (log scale)")
        axs[0].set_yscale("log")
        im = axs[1].imshow(np.log(y), aspect="auto")
        plt.colorbar(im)
        axs[1].set_title("Heatmap with low probability cells (log scale)")

        plot_save_path = f"{output_dir}/{metricX}_vs_{metricY}_cellswithlowprobs.png"
        plt.savefig(plot_save_path, format="png", dpi=150)
        plt.close("all")

        remove_cells_summary[f"{metricX} vs {metricY}"] = np.median(x, axis=1)

    # %% Identify cells to be removed
    CellIds_remove_dict = {}
    CellIds_remove = np.empty(0, dtype=int)
    for i, xy_pair in enumerate(pairs):
        metricX = cellnuc_metrics[xy_pair[0]]
        metricY = cellnuc_metrics[xy_pair[1]]
        CellIds_remove_dict[f"{metricX} vs {metricY}"] = np.argwhere(
            remove_cells_summary[f"{metricX} vs {metricY}"].to_numpy() < cell_dens_th_CN
        )
        CellIds_remove = np.union1d(
            CellIds_remove, CellIds_remove_dict[f"{metricX} vs {metricY}"]
        )
        print(len(CellIds_remove))

    # %% Plot and remove outliers
    plotname = "CellNucleus"
    oplot(
        cellnuc_metrics,
        cellnuc_abbs,
        pairs2,
        cells,
        True,
        output_dir,
        f"{plotname}_6_org_fine",
        0.5,
        [],
    )
    oplot(
        cellnuc_metrics,
        cellnuc_abbs,
        pairs2,
        cells,
        True,
        output_dir,
        f"{plotname}_6_org_thick",
        2,
        [],
    )
    oplot(
        cellnuc_metrics,
        cellnuc_abbs,
        pairs2,
        cells,
        True,
        output_dir,
        f"{plotname}_6_outliers",
        2,
        CellIds_remove_dict,
    )
    oplot(
        cellnuc_metrics,
        cellnuc_abbs,
        pairs,
        cells,
        True,
        output_dir,
        f"{plotname}_21_org_fine",
        0.5,
        [],
    )
    oplot(
        cellnuc_metrics,
        cellnuc_abbs,
        pairs,
        cells,
        True,
        output_dir,
        f"{plotname}_21_org_thick",
        2,
        [],
    )
    oplot(
        cellnuc_metrics,
        cellnuc_abbs,
        pairs,
        cells,
        True,
        output_dir,
        f"{plotname}_21_outliers",
        2,
        CellIds_remove_dict,
    )
    print(cells.shape)
    CellIds_remove = (
        cells.loc[cells.index[CellIds_remove], "CellId"].squeeze().to_numpy()
    )
    cells_ao.loc[
        cells_ao["CellId"].isin(CellIds_remove), "Outlier"
    ] = "yes_abnormal_cell_or_nuclear_metric"
    cells = cells.drop(cells.index[cells["CellId"].isin(CellIds_remove)])
    print(
        f"Removing {len(CellIds_remove)} cells due to abnormal cell or nuclear metric"
    )
    print(cells.shape)
    oplot(
        cellnuc_metrics,
        cellnuc_abbs,
        pairs2,
        cells,
        True,
        output_dir,
        f"{plotname}_6_clean_thick",
        2,
        [],
    )
    oplot(
        cellnuc_metrics,
        cellnuc_abbs,
        pairs2,
        cells,
        True,
        output_dir,
        f"{plotname}_6_clean_fine",
        0.5,
        [],
    )
    oplot(
        cellnuc_metrics,
        cellnuc_abbs,
        pairs,
        cells,
        True,
        output_dir,
        f"{plotname}_21_clean_thick",
        2,
        [],
    )
    oplot(
        cellnuc_metrics,
        cellnuc_abbs,
        pairs,
        cells,
        True,
        output_dir,
        f"{plotname}_21_clean_fine",
        0.5,
        [],
    )

    # %% Feature sets for structures
    selected_metrics = [
        "Cell volume",
        "Cell surface area",
        "Nuclear volume",
        "Nuclear surface area",
    ]
    selected_metrics_abb = ["Cell Vol", "Cell Area", "Nuc Vol", "Nuc Area"]
    selected_structures = [
        "LMNB1",
        "ST6GAL1",
        "TOMM20",
        "SEC61B",
        "ATP2A2",
        "LAMP1",
        "RAB5A",
        "SLC25A17",
        "TUBA1B",
        "TJP1",
        "NUP153",
        "FBL",
        "NPM1",
        "SON",
    ]
    structure_metric = "Structure volume"

    # %% Parameters
    N = 1000
    fac = 1000
    Rounds = 5

    # %% For all pairs compute densities
    remove_cells = cells["CellId"].to_frame().copy()
    for xm, metric in enumerate(selected_metrics):
        for ys, struct in enumerate(selected_structures):

            # data
            x = (
                cells.loc[cells["structure_name"] == struct, [metric]]
                .squeeze()
                .to_numpy()
                / fac
            )
            y = (
                cells.loc[cells["structure_name"] == struct, [structure_metric]]
                .squeeze()
                .to_numpy()
                / fac
            )

            # density estimate, repeat because of probabilistic nature of density
            # estimate used here
            for r in np.arange(Rounds):
                if ys == 0:
                    remove_cells[f"{metric} vs {structure_metric}_{r}"] = np.nan
                # print(f"Round {r+1} of {Rounds}")
                rs = int(r)
                xS, yS = resample(
                    x, y, replace=False, n_samples=np.amin([N, len(x)]), random_state=rs
                )
                k = gaussian_kde(np.vstack([xS, yS]))
                cell_dens = k(np.vstack([x.flatten(), y.flatten()]))
                cell_dens = cell_dens / np.sum(cell_dens)
                remove_cells.loc[
                    cells["structure_name"] == struct,
                    f"{metric} vs {structure_metric}_{r}",
                ] = cell_dens

    # remove_cells = pd.read_csv(data_root_extra / 'structures.csv')

    # %% Summarize across repeats
    remove_cells_summary = cells["CellId"].to_frame().copy()
    for xm, metric in enumerate(selected_metrics):
        print(metric)

        filter_col = [
            col
            for col in remove_cells
            if col.startswith(f"{metric} vs {structure_metric}")
        ]
        x = remove_cells[filter_col].to_numpy()
        pos = np.argwhere(np.any(x < cell_dens_th_S, axis=1))
        y = x[pos, :].squeeze()

        fig, axs = plt.subplots(1, 2, figsize=(16, 9))
        xr = np.log(x.flatten())
        xr = np.delete(xr, np.argwhere(np.isinf(xr)))
        axs[0].hist(xr, bins=100)
        axs[0].set_title("Histogram of cell probabilities (log scale)")
        axs[0].set_yscale("log")
        im = axs[1].imshow(np.log(y), aspect="auto")
        plt.colorbar(im)
        axs[1].set_title("Heatmap with low probability cells (log scale)")

        plot_save_path = (
            f"{output_dir}/{metric}_vs_{structure_metric}_cellswithlowprobs.png"
        )
        plt.savefig(plot_save_path, format="png", dpi=150)

        remove_cells_summary[f"{metric} vs {structure_metric}"] = np.median(x, axis=1)

    # %% Identify cells to be removed
    CellIds_remove_dict = {}
    CellIds_remove = np.empty(0, dtype=int)
    for xm, metric in enumerate(selected_metrics):
        print(metric)
        CellIds_remove_dict[f"{metric} vs {structure_metric}"] = np.argwhere(
            remove_cells_summary[f"{metric} vs {structure_metric}"].to_numpy()
            < cell_dens_th_S
        )
        CellIds_remove = np.union1d(
            CellIds_remove, CellIds_remove_dict[f"{metric} vs {structure_metric}"]
        )
        print(len(CellIds_remove))

    # %% Plot and remove outliers
    plotname = "Structures"
    splot(
        selected_metrics,
        selected_metrics_abb,
        selected_structures[0:7],
        structure_metric,
        cells,
        True,
        output_dir,
        f"{plotname}_1_org_fine",
        0.5,
        [],
    )
    splot(
        selected_metrics,
        selected_metrics_abb,
        selected_structures[7:14],
        structure_metric,
        cells,
        True,
        output_dir,
        f"{plotname}_2_org_fine",
        0.5,
        [],
    )
    splot(
        selected_metrics,
        selected_metrics_abb,
        selected_structures[0:7],
        structure_metric,
        cells,
        True,
        output_dir,
        f"{plotname}_1_org_thick",
        2,
        [],
    )
    splot(
        selected_metrics,
        selected_metrics_abb,
        selected_structures[7:14],
        structure_metric,
        cells,
        True,
        output_dir,
        f"{plotname}_2_org_thick",
        2,
        [],
    )
    splot(
        selected_metrics,
        selected_metrics_abb,
        selected_structures[0:7],
        structure_metric,
        cells,
        True,
        output_dir,
        f"{plotname}_1_outliers",
        2,
        CellIds_remove_dict,
    )
    splot(
        selected_metrics,
        selected_metrics_abb,
        selected_structures[7:14],
        structure_metric,
        cells,
        True,
        output_dir,
        f"{plotname}_2_outliers",
        2,
        CellIds_remove_dict,
    )
    print(cells.shape)
    CellIds_remove = (
        cells.loc[cells.index[CellIds_remove], "CellId"].squeeze().to_numpy()
    )
    cells_ao.loc[
        cells_ao["CellId"].isin(CellIds_remove), "Outlier"
    ] = "yes_abnormal_structure_volume_metrics"
    cells = cells.drop(cells.index[cells["CellId"].isin(CellIds_remove)])
    print(f"Removing {len(CellIds_remove)} cells due to structure volume metrics")
    print(cells.shape)
    splot(
        selected_metrics,
        selected_metrics_abb,
        selected_structures[0:7],
        structure_metric,
        cells,
        True,
        output_dir,
        f"{plotname}_1_clean_fine",
        0.5,
        [],
    )
    splot(
        selected_metrics,
        selected_metrics_abb,
        selected_structures[7:14],
        structure_metric,
        cells,
        True,
        output_dir,
        f"{plotname}_2_clean_fine",
        0.5,
        [],
    )
    splot(
        selected_metrics,
        selected_metrics_abb,
        selected_structures[0:7],
        structure_metric,
        cells,
        True,
        output_dir,
        f"{plotname}_1_clean_thick",
        2,
        [],
    )
    splot(
        selected_metrics,
        selected_metrics_abb,
        selected_structures[7:14],
        structure_metric,
        cells,
        True,
        output_dir,
        f"{plotname}_2_clean_thick",
        2,
        [],
    )

    # %% Final diagnostic plot
    cells = initial_parsing(df=df)
    CellIds_remove_dict = {}

    for i, xy_pair in enumerate(pairs):
        metricX = cellnuc_metrics[xy_pair[0]]
        metricY = cellnuc_metrics[xy_pair[1]]
        CellIds_remove_dict[f"{metricX} vs {metricY}"] = np.argwhere(
            (cells_ao["Outlier"] == "yes_abnormal_cell_or_nuclear_metric").to_numpy()
        )
    oplot(
        cellnuc_metrics,
        cellnuc_abbs,
        pairs2,
        cells,
        True,
        output_dir,
        "Check_cellnucleus",
        2,
        CellIds_remove_dict,
    )

    CellIds_remove_dict = {}
    for xm, metric in enumerate(selected_metrics):
        CellIds_remove_dict[f"{metric} vs {structure_metric}"] = np.argwhere(
            (
                (cells_ao["Outlier"] == "yes_abnormal_structure_volume_metrics")
                | (cells_ao["Outlier"] == "yes_abnormal_cell_or_nuclear_metric")
            ).to_numpy()
        )
    splot(
        selected_metrics,
        selected_metrics_abb,
        selected_structures[0:7],
        structure_metric,
        cells,
        True,
        output_dir,
        "Check_structures_1",
        2,
        CellIds_remove_dict,
    )
    splot(
        selected_metrics,
        selected_metrics_abb,
        selected_structures[7:14],
        structure_metric,
        cells,
        True,
        output_dir,
        "Check_structures_2",
        2,
        CellIds_remove_dict,
    )

    cells_ao = cells_ao.set_index("CellId", drop=True)

    return cells_ao


#!/usr/bin/env python
import sys
import matplotlib.pyplot as plt
import spada

def main():
    trackname = sys.argv[1]
    colors = 'rgbc'
    fig = plt.figure(figsize=(8,6))
    ax = fig.gca()
    for imass,mass in enumerate([1.0,2.0,3.0,4.0]):
        for ifeh,feh in enumerate([0.0,-0.2,-0.5,-0.8]):
            track = spada.SETrack(trackname,mass=mass,FeH=feh)
            if imass == 0:
                ax.plot(track.logTeff_lst,track.logL_lst,
                    color=colors[ifeh],ls='-',label='[Fe/H]=%+3.2f'%feh)
            else:
                ax.plot(track.logTeff_lst,track.logL_lst,
                    color=colors[ifeh],ls='-')
    t1,t2 = ax.get_xlim()
    ax.set_xlim(t2,t1)
    ax.set_xlabel('$\log{T_\mathrm{eff}}\,(\mathrm{K})$')
    ax.set_ylabel('$\log{\left(L/L_\odot\\right)}$')
    leg = ax.legend(loc='lower left')
    for t in leg.get_texts():
        t.set_fontsize(10)
    leg.get_frame().set_alpha(0.1)
    plt.show()

if __name__=='__main__':
    main()


# ----------------------------------------------------------------------
# |
# |  FuncDeclarationStatement.py
# |
# |  David Brownell <db@DavidBrownell.com>
# |      2021-06-13 12:21:15
# |
# ----------------------------------------------------------------------
# |
# |  Copyright David Brownell 2021
# |  Distributed under the Boost Software License, Version 1.0. See
# |  accompanying file LICENSE_1_0.txt or copy at
# |  http://www.boost.org/LICENSE_1_0.txt.
# |
# ----------------------------------------------------------------------
"""Contains the FuncDeclarationStatement object"""

import itertools
import os
import textwrap

from enum import auto, Enum
from typing import List, Optional, Union

from dataclasses import dataclass

import CommonEnvironment
from CommonEnvironment import Interface
from CommonEnvironment import StringHelpers

from CommonEnvironmentEx.Package import InitRelativeImports

# ----------------------------------------------------------------------
_script_fullpath                            = CommonEnvironment.ThisFullpath()
_script_dir, _script_name                   = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------

with InitRelativeImports():
    from .Common import Tokens as CommonTokens
    from .Common import Statements as CommonStatements
    from .Common import NamingConventions

    from ..GrammarStatement import (
        DynamicStatements,
        GrammarStatement,
        Leaf,
        Node,
        Statement,
        ValidationError,
    )

# ----------------------------------------------------------------------
@dataclass(frozen=True)
class InvalidFunctionNameError(ValidationError):
    FunctionName: str

    MessageTemplate                         = Interface.DerivedProperty(
        textwrap.dedent(
            """\
            '{{FunctionName}}' is not a valid function name.

            Function names must:
                {}
            """,
        ).format(
            StringHelpers.LeftJustify("\n".join(NamingConventions.Function.Constraints).rstrip(), 4),
        ),
    )

    # ----------------------------------------------------------------------
    @classmethod
    def FromNode(
        cls,
        node: Node,
        function_name: str,
    ):
        return cls(
            node.IterBefore.Line,
            node.IterBefore.Column,
            node.IterAfter.Line,
            node.IterAfter.Column,
            function_name,
        )

# ----------------------------------------------------------------------
@dataclass(frozen=True)
class PositionalParameterAfterDefaultValueParameterError(ValidationError):
    MessageTemplate                         = Interface.DerivedProperty("Positional parameters may not appear after a parameter with a default value has been defined")

# ----------------------------------------------------------------------
@dataclass(frozen=True)
class InvalidNewStyleParameterGroupOrderingError(ValidationError):
    MessageTemplate                         = Interface.DerivedProperty("Parameter groups must appear in the order {}".format(", ".join(token.Name for token in CommonTokens.AllNewStyleParameters)))

# ----------------------------------------------------------------------
@dataclass(frozen=True)
class InvalidTraditionalDelimiterOrderError(ValidationError):
    MessageTemplate                         = Interface.DerivedProperty(
        "The positional delimiter ({}) must appear before the keyword delimiter ({})".format(
            CommonTokens.FunctionParameterPositionalDelimiter.Name,
            CommonTokens.FunctionParameterKeywordDelimiter.Name,
        ),
    )

# ----------------------------------------------------------------------
@dataclass(frozen=True)
class InvalidTraditionalPositionalDelimiterError(ValidationError):
    MessageTemplate                         = Interface.DerivedProperty(
        "The positional delimiter ({}) must appear after at least 1 parameter".format(
            CommonTokens.FunctionParameterPositionalDelimiter.Name,
        ),
    )

# ----------------------------------------------------------------------
@dataclass(frozen=True)
class InvalidTraditionalKeywordDelimiterError(ValidationError):
    MessageTemplate                         = Interface.DerivedProperty(
        "The keyword delimiter ({}) must appear before at least 1 parameter".format(
            CommonTokens.FunctionParameterKeywordDelimiter.Name,
        ),
    )

# ----------------------------------------------------------------------
@dataclass(frozen=True)
class InvalidTraditionalDuplicatePositionalDelimiterError(ValidationError):
    MessageTemplate                         = Interface.DerivedProperty(
        "The positional delimiter ({}) may only appear once in a list of parameters".format(
            CommonTokens.FunctionParameterPositionalDelimiter.Name,
        ),
    )

# ----------------------------------------------------------------------
@dataclass(frozen=True)
class InvalidTraditionalDuplicateKeywordDelimiterError(ValidationError):
    MessageTemplate                         = Interface.DerivedProperty(
        "The keyword delimiter ({}) may only appear once in a list of parameters".format(
            CommonTokens.FunctionParameterKeywordDelimiter.Name,
        ),
    )

# ----------------------------------------------------------------------
class FuncDeclarationStatement(GrammarStatement):
    """\
    Declares a function.

    <type> <name> '(' <args> ')' ':'
        <dynamic_statements>
    """

    # ----------------------------------------------------------------------
    def __init__(self):
        parameter_statement = Statement(
            "Parameter",
            CommonStatements.Type,
            [
                # <name> = <rhs>
                Statement(
                    "With Default",
                    CommonTokens.Name,
                    CommonTokens.Equal,
                    DynamicStatements.Expressions,
                ),

                # <name>
                CommonTokens.Name,
            ],
        )

        traditional_parameter_statement = [
            parameter_statement,
            CommonTokens.FunctionParameterPositionalDelimiter,
            CommonTokens.FunctionParameterKeywordDelimiter,
        ]

        super(FuncDeclarationStatement, self).__init__(
            GrammarStatement.Type.Statement,
            Statement(
                "Function Declaration",
                (CommonTokens.Export, 0, 1),
                CommonStatements.Type,
                CommonTokens.Name,
                CommonTokens.LParen,
                CommonTokens.PushIgnoreWhitespaceControl,

                # Parameters are optional
                (
                    [
                        # New Style:
                        #
                        #       <type> Func(
                        #           pos: <type> a, <type> b, <type> c,
                        #           any: <type> d, <type> e=100
                        #           key: <type> f=200, <type> g=300
                        #       ):
                        #           <statements>
                        #
                        (
                            Statement(
                                "New Style",
                                CommonTokens.AllNewStyleParameters,
                                CommonTokens.Colon,
                                parameter_statement,
                                (
                                    Statement(
                                        "Comma and Parameter",
                                        CommonTokens.Comma,
                                        parameter_statement,
                                    ),
                                    0,
                                    None,
                                ),
                                (CommonTokens.Comma, 0, 1),
                            ),
                            1,
                            len(CommonTokens.AllNewStyleParameters),
                        ),

                        # Traditional:
                        #
                        #       # Exactly matches the new-style definition above:
                        #       #
                        #       #   - Everything before '/' is positional
                        #       #   - Everything after '/' and before '*' is any
                        #       #   - Everything after '*' is keyword
                        #       #
                        #       <type> Func(<type> a, <type> b, <type> c, /, <type> d, <type> e=100, *, <type> f=200, <type> g=300):
                        #           <statements>
                        #
                        #       # No restriction on positional or keyword parameters
                        #       <type> Func(<type> a, <type> b, <type> c, <type> d, <type> e=100, <type> f=200, <type> g=300):
                        #           <statement>
                        Statement(
                            "Traditional",
                            traditional_parameter_statement,
                            (
                                Statement(
                                    "Comma and Parameter",
                                    CommonTokens.Comma,
                                    traditional_parameter_statement,
                                ),
                                0,
                                None,
                            ),
                            (CommonTokens.Comma, 0, 1),
                        ),
                    ],
                    0,
                    1,
                ),

                CommonTokens.PopIgnoreWhitespaceControl,
                CommonTokens.RParen,
                CommonTokens.Colon,
                CommonTokens.NewlineToken(),

                CommonTokens.Indent,
                (DynamicStatements.Statements, 1, None),
                CommonTokens.Dedent,
            ),
        )

    # ----------------------------------------------------------------------
    @classmethod
    @Interface.override
    def ValidateNodeSyntax(
        cls,
        node: Node,
    ):
        # The export keyword may or may not appear, which will impact the parameter index
        if (
            isinstance(node.Children[0].Type, tuple)
            and node.Children[0].Type[0] == CommonTokens.Export
        ):
            parameter_offset = 1
        else:
            parameter_offset = 0

        # Validate the function name
        assert len(node.Children) > (2 + parameter_offset)

        name_node = node.Children[1 + parameter_offset]
        function_name = name_node.Value.Match.group("value")

        if not NamingConventions.Function.Regex.match(function_name):
            raise InvalidFunctionNameError.FromNode(name_node, function_name)

        # Validate the parameters
        assert len(node.Children) > (4 + parameter_offset)
        parameters = cls._GetParameters(node.Children[3 + parameter_offset])

        # Commit the values for later
        node.function_name = function_name
        node.parameters = parameters

    # ----------------------------------------------------------------------
    # ----------------------------------------------------------------------
    # ----------------------------------------------------------------------
    @dataclass(frozen=True)
    class _ParameterInfo(object):
        Parameter: Node
        Type: str
        TypeModifier: CommonTokens.RegexToken
        Name: str
        DefaultValue: Optional[Node]

    # ----------------------------------------------------------------------
    @dataclass(frozen=True)
    class _Parameters(object):
        Positional: List["FuncDeclarationStatement._ParameterInfo"]
        PositionalOrKeyword: List["FuncDeclarationStatement._ParameterInfo"]
        Keyword: List["FuncDeclarationStatement._ParameterInfo"]

    # ----------------------------------------------------------------------
    @classmethod
    def _GetParameters(
        cls,
        node: Union[Node, Leaf],
    ) -> "FuncDeclarationStatement._Parameters":

        if isinstance(node, Leaf):
            # The leaf is the closing ')' token; this indicates that there aren't any
            # parameters
            return cls._Parameters([], [], [])

        # Drill into the Optional node
        assert isinstance(node.Type, tuple)
        assert len(node.Children) == 1
        node = node.Children[0]

        # Drill into the Or node
        assert isinstance(node.Type, list)
        assert len(node.Children) == 1
        node = node.Children[0]

        # Arrive at the content node
        if isinstance(node.Type, tuple):
            result = cls._GetNewStyleParameters(node)
        elif node.Type.Name == "Traditional":
            result = cls._GetTraditionalParameters(node)
        else:
            assert False, node  # pragma: no cover

        # Ensure that all parameters after the first with a default value
        # also have a default value.
        encountered_default = False

        for parameter_info in itertools.chain(
            result.Positional,
            result.PositionalOrKeyword,
            result.Keyword,
        ):
            if parameter_info.DefaultValue:
                encountered_default = True
            elif encountered_default:
                raise PositionalParameterAfterDefaultValueParameterError.FromNode(
                    parameter_info.Parameter,
                )

        return result

    # ----------------------------------------------------------------------
    @classmethod
    def _GetTraditionalParameters(
        cls,
        node: Node,
    ) -> "FuncDeclarationStatement._Parameters":

        positional_parameters: List["_ParameterInfo"] = []
        positional_or_keyword_parameters: List["_ParameterInfo"] = []
        keyword_parameters: List["_ParameterInfo"] = []

        encountered_positional_delimiter = False
        encountered_keyword_delimiter = False

        # ----------------------------------------------------------------------
        class State(Enum):
            PositionalOrKeyword     = auto()
            Keyword                 = auto()

        # ----------------------------------------------------------------------

        state = State.PositionalOrKeyword

        # ----------------------------------------------------------------------
        def ProcessParameter(
            parameter: Union[Statement, CommonTokens.RegexToken],
            is_last_parameter: bool,
        ):
            nonlocal positional_parameters
            nonlocal positional_or_keyword_parameters
            nonlocal keyword_parameters

            nonlocal encountered_positional_delimiter
            nonlocal encountered_keyword_delimiter

            nonlocal state

            if parameter.Type == CommonTokens.FunctionParameterPositionalDelimiter:
                # We shouldn't see this as the first parameter
                if (
                    not positional_parameters
                    and not positional_or_keyword_parameters
                    and not keyword_parameters
                ):
                    raise InvalidTraditionalPositionalDelimiterError.FromNode(parameter)

                # We shouldn't see this if we have already seen a keyword delimiter
                if encountered_keyword_delimiter:
                    raise InvalidTraditionalDelimiterOrderError.FromNode(parameter)

                # We shouldn't see this delimiter more than once
                if encountered_positional_delimiter:
                    raise InvalidTraditionalDuplicatePositionalDelimiterError.FromNode(parameter)

                assert not positional_parameters
                positional_parameters = positional_or_keyword_parameters
                positional_or_keyword_parameters = []

                encountered_positional_delimiter = True

            elif parameter.Type == CommonTokens.FunctionParameterKeywordDelimiter:
                # We shouldn't see this delimiter as the last parameter
                if is_last_parameter:
                    raise InvalidTraditionalKeywordDelimiterError.FromNode(parameter)

                # We shouldn't see this delimiter more than once
                if encountered_keyword_delimiter:
                    raise InvalidTraditionalDuplicateKeywordDelimiterError.FromNode(parameter)

                encountered_keyword_delimiter = True
                state = State.Keyword

            elif parameter.Type.Name == "Parameter":
                parameter_info = cls._CreateParameterInfo(parameter)

                if state == State.PositionalOrKeyword:
                    positional_or_keyword_parameters.append(parameter_info)
                elif state == State.Keyword:
                    keyword_parameters.append(parameter_info)
                else:
                    assert False, state  # pragma: no cover

            else:
                assert False, parameter  # pragma: no cover

        # ----------------------------------------------------------------------

        has_comma_delimited_parameters = (
            len(node.Children) > 1
            and isinstance(node.Children[1].Type, tuple)
            and isinstance(node.Children[1].Type[0], Statement)
        )

        # Initial parameter
        assert len(node.Children[0].Children) == 1

        ProcessParameter(
            node.Children[0].Children[0],
            is_last_parameter=not has_comma_delimited_parameters,
        )

        # Following parameters
        if has_comma_delimited_parameters:
            node = node.Children[1]

            for child_index, child in enumerate(node.Children):
                # First value is the comma, second value is the parameter
                assert len(child.Children) == 2
                child = child.Children[1]

                # Drill into the Or node
                assert isinstance(child.Type, list)
                assert len(child.Children) == 1
                child = child.Children[0]

                ProcessParameter(
                    child,
                    is_last_parameter=child_index + 1 == len(node.Children),
                )

        return cls._Parameters(
            positional_parameters,
            positional_or_keyword_parameters,
            keyword_parameters,
        )

    # ----------------------------------------------------------------------
    @classmethod
    def _GetNewStyleParameters(
        cls,
        node: Node,
    ) -> "FuncDeclarationStatement._Parameters":

        positional_parameters: List["_ParameterInfo"] = []
        positional_or_keyword_parameters: List["_ParameterInfo"] = []
        keyword_parameters: List["_ParameterInfo"] = []

        for child in node.Children:
            # Get the type
            the_type = child.Children[0]

            # Drill into the Or node
            assert isinstance(the_type.Type, list)
            assert len(the_type.Children) == 1
            the_type = the_type.Children[0]

            # Set the result list and check for errors
            if the_type.Type == CommonTokens.FunctionParameterPositional:
                parameters_list = positional_parameters
                has_error = positional_or_keyword_parameters or keyword_parameters
            elif the_type.Type == CommonTokens.FunctionParameterAny:
                parameters_list = positional_or_keyword_parameters
                has_error = bool(keyword_parameters)
            elif the_type.Type == CommonTokens.FunctionParameterKeyword:
                parameters_list = keyword_parameters
                has_error = False
            else:
                assert False, the_type  # pragma: no cover

            if has_error:
                raise InvalidNewStyleParameterGroupOrderingError.FromNode(the_type)

            # Get the parameters
            parameters_list.append(cls._CreateParameterInfo(child.Children[2]))

            if (
                len(child.Children) > 4
                and isinstance(child.Children[3].Type, tuple)
                and isinstance(child.Children[3].Type[0], Statement)
            ):
                parameter_nodes = child.Children[3]

                for parameter_node in parameter_nodes.Children:
                    # First value is the comma, second is the parameter
                    assert len(parameter_node.Children) == 2
                    parameter_node = parameter_node.Children[1]

                    parameters_list.append(cls._CreateParameterInfo(parameter_node))

        return cls._Parameters(
            positional_parameters,
            positional_or_keyword_parameters,
            keyword_parameters,
        )

    # ----------------------------------------------------------------------
    @classmethod
    def _CreateParameterInfo(
        cls,
        node: Node,
    ) -> "FuncDeclarationStatement._ParameterInfo":
        assert len(node.Children) == 2

        type_info = CommonStatements.TypeInfo.FromNode(node.Children[0])

        # Drill into the Parameter node
        node = node.Children[1]

        # Drill into the Or node
        assert isinstance(node.Type, list)
        assert len(node.Children) == 1
        node = node.Children[0]

        if isinstance(node, Leaf):
            # Name
            name_node = node
            default_value = None
        else:
            # Name with default
            assert len(node.Children) == 3

            name_node = node.Children[0]
            default_value = CommonStatements.ExtractDynamicExpressionsNode(node.Children[2])

        return cls._ParameterInfo(
            node,
            type_info.Name,
            type_info.Modifier,
            name_node.Value.Match.group("value"),
            default_value,
        )


from setuptools import setup, find_packages
from distutils.sysconfig import get_python_lib
from setuptools.command.install import install as _install
from setuptools.command.develop import develop as _develop
import os

def _post_install(libname, libpath):
    from JumpScale9 import j  # here its still the boostrap JumpScale9

    # remove leftovers
    for item in j.sal.fs.find("/usr/local/bin/", fileregex="js9*"):
        j.sal.fs.remove("/usr/local/bin/%s" % item)

    j.tools.executorLocal.initEnv()
    j.tools.jsloader.generate()

class install(_install):

    def run(self):
        _install.run(self)
        libname = self.config_vars['dist_name']
        libpath = os.path.join(os.path.dirname(os.path.abspath(__file__)), libname)
        self.execute(_post_install, (libname, libpath), msg="Running post install task")

class develop(_develop):

    def run(self):
        _develop.run(self)
        libname = self.config_vars['dist_name']
        libpath = os.path.join(os.path.dirname(os.path.abspath(__file__)), libname)
        self.execute(_post_install, (libname, libpath), msg="Running post install task")

long_description = ""
try:
    from pypandoc import convert
    long_description = convert('README.md', 'rst')
except ImportError:
    long_description = ""

setup(
    name='JumpScale9',
    version='9.4.0',
    description='Automation framework for cloud workloads',
    long_description=long_description,
    url='https://github.com/Jumpscale/core9',
    author='GreenItGlobe',
    author_email='info@gig.tech',
    license='Apache',
    packages=find_packages(),

    # IF YOU CHANGE ANYTHING HERE, LET DESPIEGK NOW (DO NOT INSTALL ANYTHING WHICH NEEDS TO COMPILE)
    install_requires=[
        'GitPython>=2.1.3',
        'click>=6.7',
        'colored_traceback',
        'colorlog>=2.10.0',
        'httplib2>=0.10.3',
        'ipython>=6.0.0',
        'libtmux>=0.7.1',
        'netaddr>=0.7.19',
        'path.py>=10.3.1',
        'pystache>=0.5.4',
        'python-dateutil>=2.6.0',
        'pytoml>=0.1.12',
        'toml',
        'redis>=2.10.5',
        'requests>=2.13.0',
        'future>=0.16.0',
        'watchdog',
        'msgpack-python',
        'npyscreen',
        'pyyaml',
        'pyserial>=3.0'
        'docker>=3',
        'fakeredis',
        'ssh2-python',
        'parallel_ssh>=1.4.0',
        'psutil>=5.4.3',
        'Unidecode>=1.0.22',

    ],
    cmdclass={
        'install': install,
        'develop': develop,
        'developement': develop
    },
    scripts=[
        'cmds/js9',
        'cmds/js9_code',
        'cmds/js9_docker',
        'cmds/js9_doc',
    ],
)


from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import accuracy_score
from sklearn.neighbors import KNeighborsClassifier
from nltk.corpus import stopwords
from sklearn.grid_search import GridSearchCV
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier

from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import MultinomialNB

import time

import matplotlib.pyplot as plt; plt.rcdefaults()
import numpy as np
import matplotlib.pyplot as plt
from sklearn import tree

start_time = time.time()
log=open('classification_demo.txt','w') # test file

def loadData(fname):

    labels=[]
    reviews=[]
    f=open(fname)
    for line in f:
        rating,vote,review=line.strip().split('\t')
        reviews.append(review.lower())
        labels.append(rating.lower())
    f.close()
    return reviews,labels

question_train,labels_train=loadData('train.txt')
question_test,labels_test=loadData('test.txt')

#Build a counter based on the training dataset
counter = CountVectorizer(stop_words=stopwords.words('english'))
counter.fit(question_train)

#count the number of times each term appears in a document and transform each doc into a count vector
counts_train = counter.transform(question_train)#transform the training data
counts_test = counter.transform(question_test)#transform the testing data

#    Classifiers Used:
#    1. KNN
#    2.Logistic Regression
#    3.DecisionTreeClassifier
#    4.RandomForestClassifier
#    5. Bayesian

KNN_classifier=KNeighborsClassifier()
LREG_classifier=LogisticRegression()
DT_classifier = DecisionTreeClassifier()
RF_classifier=RandomForestClassifier()
NB_classifier=MultinomialNB()

predictors=[('knn',KNN_classifier),('lreg',LREG_classifier),('dt',DT_classifier),('rf', RF_classifier),('nb',NB_classifier)]
VT=VotingClassifier(predictors)

#=======================================================================================
#build the parameter grid
KNN_grid = [{'n_neighbors': [1,3,5,7,9,11,13,15,17], 'weights':['uniform','distance']}]

#build a grid search to find the best parameters
gridsearchKNN = GridSearchCV(KNN_classifier, KNN_grid, cv=5)

#run the grid search
gridsearchKNN.fit(counts_train,labels_train)

#=======================================================================================

#build the parameter grid
DT_grid = [{'max_depth': [3,4,5,6,7,8,9,10,11,12],'criterion':['gini','entropy']}]

#build a grid search to find the best parameters
gridsearchDT  = GridSearchCV(DT_classifier, DT_grid, cv=5)

#run the grid search
gridsearchDT.fit(counts_train,labels_train)

DT = tree.DecisionTreeClassifier()
treeClf = DT.fit(counts_train,labels_train)
tree.export_graphviz(treeClf, out_file='DTree.dot')
#=======================================================================================

#build the parameter grid
LREG_grid = [ {'C':[0.5,1,1.5,2],'penalty':['l1','l2']}]

#build a grid search to find the best parameters
gridsearchLREG  = GridSearchCV(LREG_classifier, LREG_grid, cv=5)

#run the grid search
gridsearchLREG.fit(counts_train,labels_train)

#=======================================================================================
#build the parameter grid
RF_grid = [{'n_estimators':[500,1000,1500,2000,2500,3000], 'criterion':['gini','entropy']}]

#build a grid search to find the best parameters
gridsearchRF = GridSearchCV(RF_classifier, RF_grid, cv=5)

#run the grid search
gridsearchRF.fit(counts_train,labels_train)

#=======================================================================================
#build the parameter grid
NB_grid = [{'alpha':[0,1.0,5.0,10.0,20.0,30.0,40.0], 'fit_prior':[True,False]}]

#build a grid search to find the best parameters
gridsearchNB = GridSearchCV(NB_classifier, NB_grid, cv=5)

#run the grid search
gridsearchNB.fit(counts_train,labels_train)

#=======================================================================================

VT.fit(counts_train,labels_train)

#use the VT classifier to predict
predicted=VT.predict(counts_test)

#print the accuracy
print (accuracy_score(predicted,labels_test))
log.write(str(accuracy_score(predicted,labels_test))+"\n");

algos=[];
acc = [];
m = 0;

for params, mean_score, scores in gridsearchKNN.grid_scores_:
    print (params, mean_score)
    log.write(str(params)+str(mean_score)+"\n")
    if mean_score>m:
        m = mean_score

acc.append(m)
m=0

for params, mean_score, scores in gridsearchDT.grid_scores_:
    print (params, mean_score)
    log.write(str(params)+str(mean_score)+"\n")
    if mean_score>m:
        m = mean_score

acc.append(m)
m=0

for params, mean_score, scores in gridsearchLREG.grid_scores_:
    print (params, mean_score)
    log.write(str(params)+str(mean_score)+"\n")
    if mean_score>m:
        m = mean_score

acc.append(m)
m=0

for params, mean_score, scores in gridsearchRF.grid_scores_:
    print (params, mean_score)
    log.write(str(params)+str(mean_score)+"\n")
    if mean_score>m:
        m = mean_score

acc.append(m)
m=0

for params, mean_score, scores in gridsearchNB.grid_scores_:
    print (params, mean_score)
    log.write(str(params)+str(mean_score)+"\n")
    if mean_score>m:
        m = mean_score

acc.append(m)
m=0

print(str(algos))
print(str(acc))

print("--- %s seconds ---" % round(time.time() - start_time, 2))

algos = ['KNN', 'DT', 'LREG', 'RF', 'NB']
y_pos = np.arange(len(algos))

plt.bar(y_pos, acc, align='center', alpha=0.5)
plt.xticks(y_pos, algos)
plt.ylabel('Accuracy')
plt.title('Algorithms v Accuracy')

plt.show()