import torch
from torch.nn.parameter import Parameter

from allennlp.modules.attention.attention import Attention

@Attention.register("additive")
class AdditiveAttention(Attention):
    """
    Computes attention between a vector and a matrix using an additive attention function.  This
    function has two matrices `W`, `U` and a vector `V`. The similarity between the vector
    `x` and the matrix `y` is computed as `V tanh(Wx + Uy)`.

    This attention is often referred as concat or additive attention. It was introduced in
    [Neural Machine Translation by Jointly Learning to Align and Translate (Bahdanau et al, 2015)]
    (https://api.semanticscholar.org/CorpusID:11212020).

    Registered as an `Attention` with name "additive".

    # Parameters

    vector_dim : `int`, required
        The dimension of the vector, `x`, described above.  This is `x.size()[-1]` - the length
        of the vector that will go into the similarity computation.  We need this so we can build
        the weight matrix correctly.
    matrix_dim : `int`, required
        The dimension of the matrix, `y`, described above.  This is `y.size()[-1]` - the length
        of the vector that will go into the similarity computation.  We need this so we can build
        the weight matrix correctly.
    normalize : `bool`, optional (default = `True`)
        If true, we normalize the computed similarities with a softmax, to return a probability
        distribution for your attention.  If false, this is just computing a similarity score.
    """

    def __init__(self, vector_dim: int, matrix_dim: int, normalize: bool = True) -> None:
        super().__init__(normalize)
        self._w_matrix = Parameter(torch.Tensor(vector_dim, vector_dim))
        self._u_matrix = Parameter(torch.Tensor(matrix_dim, vector_dim))
        self._v_vector = Parameter(torch.Tensor(vector_dim, 1))
        self.reset_parameters()

    def reset_parameters(self):
        torch.nn.init.xavier_uniform_(self._w_matrix)
        torch.nn.init.xavier_uniform_(self._u_matrix)
        torch.nn.init.xavier_uniform_(self._v_vector)

    def _forward_internal(self, vector: torch.Tensor, matrix: torch.Tensor) -> torch.Tensor:
        intermediate = vector.matmul(self._w_matrix).unsqueeze(1) + matrix.matmul(self._u_matrix)
        intermediate = torch.tanh(intermediate)
        return intermediate.matmul(self._v_vector).squeeze(2)


# coding: utf-8

# In[ ]:

import scrapy

class ShopcluesSpider(scrapy.Spider):
    # name of spider
    name = 'shopclues'

    # list of allowed domains
    allowed_domains = ['www.shopclues.com/mobiles-feature-phones.html']
    #starting url
    start_urls = ['https://www.shopclues.com/mobiles-feature-phones.html?sort_by=sort_price&sort_order=asc&facet_brand[]=Ikall&fsrc=facet_brand/']
    # location of csv file
    custom_settings = {
        'FEED_URI' : 'tmp/shopclues.csv',
        'FEED_FORMAT': 'csv'
    }

    def parse(self, response):
        print('starts crawling...')

        # extract product information
        titles = response.css('.column.col3 h2::text').extract()
        prices = response.css('.column.col3 .p_price::text').extract()
        discounts = response.css('.column.col3 .prd_discount::text').extract()
        images = response.css('img::attr(data-img)').extract()

        for item in zip(titles, prices, discounts, images):
            scraped_info = {
                'titles' : item[0],
                'prices' : item[1],
                'discounts' : item[2],
                'image_urls' : [item[3]],
            }

            yield scraped_info


from itertools import permutations

MAX_LIMIT = 100

def max_weight_for_hike():
    hike_things = {
        "зажигалка": 5,
        "одежда": 50,
        "нож": 25,
        "палатка": 37,
        "еда": 10,
        "вода": 18,
        "удочка": 35,
        "мангал": 24,
        "стулья": 29,
        "приборы": 13,
    }

    all_choise = permutations(hike_things.items())
    result_dict = {}
    for temp_set in all_choise:
        temp_available_weight = MAX_LIMIT
        possible_variant_set = set()
        for thing, weight in temp_set:
            if temp_available_weight >= weight:
                temp_available_weight -= weight
                possible_variant_set.add(thing)
        result_dict[tuple(possible_variant_set)] = MAX_LIMIT-temp_available_weight

    print("Все доступные варианты: ")
    for thing, weight in result_dict.items():
        print(f'Вещи = {thing} занимают {weight} места в рюкзаке')


#!/usr/bin/python
import requests
url = 'http://158.69.76.135/level2.php'
x = requests.get(url)
cookie = x.cookies.get('HoldTheDoor')
print(cookie)
url = 'http://158.69.76.135/level2.php'
headers = {
  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36',
  'Referer': 'http://158.69.76.135/level2.php'
}
data = {
  'id': '1033',
  'holdthedoor': 'Valider',
  'key': cookie
}

cookies = {
  'HoldTheDoor' : cookie,
}
i = 0
while i < 1024:
    requests.post(url, headers=headers, data=data, cookies=cookies)
    i += 1


import itertools

import itertools as it
import operator

'''
mylist = ['a', 'b', 'c', 'd']

for combination in it.combinations(mylist, 3):
    print(combination)

print("*"*30)

for combination in it.permutations(mylist, 3):
    print(combination)

print("*"*30)

for combination in it.combinations_with_replacement(mylist, 3):
    print(combination)
'''

money = [20, 20, 20, 20, 10, 10, 10, 5, 5, 1, 1, 1, 1, 1]

results = []

for i in range(1, 101):
    for combination in it.combinations(money, i):
        if sum(combination) == 100:
            results.append(combination)

results = set(results)

for results in results:
    print(results)
print("*"*30)
print("*"*30)

money = [50, 20, 10]

for i in range(1, 101):
    for combination in it.combinations_with_replacement(money, i):
        if sum(combination) == 100:
            print(combination)
print("*"*30)
print("*"*30)
data = [1, 2, 3, 4, 5]
result = it.accumulate(data, operator.mul)
for each in result:
    print(each)

print("*"*30)
print("*"*30)


import threading
import time
import psycopg2
import re

from datetime import datetime , timedelta

from openerp import models, fields, api ,  SUPERUSER_ID, netsvc
from openerp import tools

from urllib import urlencode, quote as quote

import logging
_logger = logging.getLogger(__name__)

#from https://github.com/SythilBlade/sythil-odoo-test/blob/8.0/entity_sms/esms_templates.py

try:
    # We use a jinja2 sandboxed environment to render mako templates.
    # Note that the rendering does not cover all the mako syntax, in particular
    # arbitrary Python statements are not accepted, and not all expressions are
    # allowed: only "public" attributes (not starting with '_') of objects may
    # be accessed.
    # This is done on purpose: it prevents incidental or malicious execution of
    # Python code that may break the security of the server.
    from jinja2.sandbox import SandboxedEnvironment
    mako_template_env = SandboxedEnvironment(
        block_start_string="<%",
        block_end_string="%>",
        variable_start_string="${",
        variable_end_string="}",
        comment_start_string="<%doc>",
        comment_end_string="</%doc>",
        line_statement_prefix="%",
        line_comment_prefix="##",
        trim_blocks=True,               # do not output newline after blocks
        autoescape=True,                # XML/HTML automatic escaping
    )
    mako_template_env.globals.update({
        'str': str,
        'quote': quote,
        'urlencode': urlencode,
        'datetime': datetime,
        'len': len,
        'abs': abs,
        'min': min,
        'max': max,
        'sum': sum,
        'filter': filter,
        'reduce': reduce,
        'map': map,
        'round': round,

        # dateutil.relativedelta is an old-style class and cannot be directly
        # instanciated wihtin a jinja2 expression, so a lambda "proxy" is
        # is needed, apparently.
        'relativedelta': lambda *a, **kw : relativedelta.relativedelta(*a, **kw),
    })
except ImportError:
    _logger.warning("jinja2 not available, templating features will not work!")

class gammu_campaign(models.Model):
    _name = 'gammu.campaign'

    _description = "gammu campaign"

    name=fields.Char(string="Name")
    template_text=fields.Text(string="template")
    state=fields.Selection(selection=[('draft', 'Borrador'),('sending', 'Enviando'),('send', 'Enviado')],default='draft')
    sms=fields.One2many(comodel_name='gammu.campaign.sms', inverse_name='gammu_campaign_id')
    opportunity_user_id =  fields.Many2one(comodel_name='res.users',string="Crear una oportunidad a")

    @api.one
    def send(self):
        self.state='sending'
        item_minute = 10
        item_count = 0

        for sms in self.sms :
            if sms.name:
                item_count = item_count + 1

                delay = datetime.now() + timedelta(minutes=int(item_count/item_minute))

                outbox={'name':sms.name,
                        'text':sms.text,
                        'sending_datetime':delay,'creatorid':'odoo','sending_time_out':delay,
                        'send_before':'23:00:00','send_after':'00:00:00','multipart':False}
                outbox_id=self.env['gammu.outbox'].sudo().create(outbox)

                if self.opportunity_user_id:
                    response=self.env['gammu.expected.responses'].sudo().create({
                        'name':sms.name,
                        'timeout':datetime.now(),
                        'model':'gammu.campaign',
                        'function':'response',
                        'args':'([%d,"%s",%d])' % (self.opportunity_user_id.id,sms.name,sms.partner_id.id) })

        self.state='send'

    @api.one
    def response(self,user_id,phone,partner_id):
        lead=self.env['crm.lead'].create({
            'name':'Respuesta la la campaña',
            'user_id':user_id,
            'partner_id':partner_id,
            'phone':phone,
            'type':'opportunity',
            'description' : self[0],
            })

            #return True
        return False

    @api.one
    def presupuestos(self):

        self.env.cr.execute("""select distinct partner_id
                       from sale_order where date_order > '2016-09-01'  and partner_id not in (
                            select distinct partner_id from account_invoice  where date_due > '2016-09-01'
        )""")

        partner_list = self.env.cr.fetchall()
        _logger.info("partner_list %r" , partner_list)
        if partner_list :
            for partner_id in partner_list :
                    sms = self.env['gammu.campaign.sms'].create({'gammu_campaign_id': self.id,'partner_id':partner_id[0]})
                    sms._partner_get_phone()

class gammu_campaign_sms(models.Model):
    _name = 'gammu.campaign.sms'

    _description = "gammu campaign sms"

    @api.depends('gammu_campaign_id')
    @api.one
    def _compute_text (self):
        self.text = self.render_template(self.gammu_campaign_id.template_text   , 'res.partner', self.partner_id.id)

    gammu_campaign_id =  fields.Many2one(comodel_name='gammu.campaign')
    name=fields.Char(string="To")
    partner_id=fields.Many2one( comodel_name='res.partner',string="partner" )
    text=fields.Text(string="msg" , compute=_compute_text)

    def render_template(self, template, model, res_id):
        """Render the given template text, replace mako expressions ``${expr}``
           with the result of evalua_compute_textting these expressions with
           an evaluation context containing:
                * ``user``: browse_record of the current user
                * ``object``: browse_record of the document record this mail is
                              related to
                * ``context``: the context passed to the mail composition wizard
           :param str template: the template text to render
           :param str model: model name of the document record this mail is related to.
           :param int res_id: id of document records those mails are related to.
        """

        # try to load the template
        #try:
        template = mako_template_env.from_string(tools.ustr(template))
        #except Exception:
        #    _logger.error("Failed to load template %r", template)
        #    return False

        # prepare template variables
        user = self.env.user
        record = self.env[model].browse(res_id)

        variables = {
            'user': user
        }

        variables['object'] = record
        try:
            render_result = template.render(variables)
        except Exception:
            _logger.error("Failed to render template %r using values %r" % (template, variables))
            render_result = u""
        if render_result == u"False":
            render_result = u""

        return render_result

    @api.onchange('partner_id')
    def _partner_get_phone(self):

        if self.partner_id.mobile:
            phone=self.clean_mobile(self.partner_id.mobile)
            if phone :
                self.name =  phone
                return

        if self.partner_id.phone :
            phone=self.clean_phone(self.partner_id.phone)
            if phone :
                self.name = phone
                return

        self.name = self.partner_id.mobile

    def clean_mobile(self,phone):

        mob=re.compile('(\+)*(54)*(9)*(0)*(299|291|11|294)*(15)*([4|5|6])([0-9][0-9][0-9][0-9][0-9][0-9])')
        mobiles=mob.findall(re.sub("\D", "",phone))

        for mobile in mobiles:
            if  mobile[6] != '' and len(mobile[7])==6:
                caracteristica = "299" if  mobile[4] == '' else  mobile[4]

                return '+549' + str(caracteristica)  + str(mobile[6])+ str(mobile[7])
        return False

    def clean_phone(self,phone):

        mob=re.compile('(\+)*(54)*(9)*(0)*(299|291|11|294)*(15)*([4|5|6])([0-9][0-9][0-9][0-9][0-9][0-9])')
        mobiles=mob.findall(re.sub("\D", "",phone))

        for mobile in mobiles:
            if mobile[5] != '' and mobile[6] != '' and len(mobile[7])==6:
                caracteristica = "299" if  mobile[4] == '' else  mobile[4]

                return '+549' + str(caracteristica)  + str(mobile[6])+ str(mobile[7])
        return False

class campain_from_dni(models.TransientModel):
    _name = 'campain.from.dni'
    phone_list=fields.Text(string="phones",required=True)
    gammu_campaign_id =  fields.Many2one(comodel_name='gammu.campaign',required=True)

    @api.multi
    def by_phone(self):
        phones=[]
        phones= filter(None,[x.strip() for x in self.phone_list.split('\n')])
        if phones :

            partner_ids = self.env['res.partner'].search(['|',('phone' ,'in',phones),('mobile' ,'in',phones)])
            for partner_id in partner_ids:
                sms = self.env['gammu.campaign.sms'].create({'gammu_campaign_id': self.gammu_campaign_id.id,'partner_id':partner_id.id})
                sms._partner_get_phone()

    @api.multi
    def by_dni(self):
        phones=[]

        phones= filter(None,[re.sub("[^0-9]", "",x) for x in self.phone_list.split('\n')])
        if phones :
            partner_ids = self.env['res.partner'].search([('document_number' ,'in',phones)])
            for partner_id in partner_ids:
                sms = self.env['gammu.campaign.sms'].create({'gammu_campaign_id': self.gammu_campaign_id.id,'partner_id':partner_id.id})
                sms._partner_get_phone()


# -*- coding: utf-8 -*-
"""
@author: Evangelos Tzardis
"""

import PySpin
import scipy.misc as misc
import numpy as np
import rw_config as cfg
from multiprocessing import Queue

#from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2TkAgg
#from matplotlib.figure import Figure

def PIL2array(imgptr):
    bpp = imgptr.GetBitsPerPixel()
    if bpp == 8:
        bppnumpy = np.uint8

    elif bpp == 16:
        bppnumpy = np.uint16

    return np.array(imgptr.GetData(), bppnumpy).reshape(imgptr.GetHeight(), imgptr.GetWidth())

def print_device_info(nodemap):
    """
    This function prints the device information of the camera from the transport
    layer

    :param nodemap: Transport layer device nodemap.
    :type nodemap: INodeMap
    :returns: True if successful, False otherwise.
    :rtype: bool
    """

    print('*** DEVICE INFORMATION ***\n')

    try:
        result = True
        node_device_information = PySpin.CCategoryPtr(nodemap.GetNode('DeviceInformation'))

        if PySpin.IsAvailable(node_device_information) and PySpin.IsReadable(node_device_information):
            features = node_device_information.GetFeatures()
            for feature in features:
                node_feature = PySpin.CValuePtr(feature)
                print('%s: %s' % (node_feature.GetName(),
                                  node_feature.ToString() if PySpin.IsReadable(node_feature) else 'Node not readable'))

        else:
            print('Device control information not available.')

    except PySpin.SpinnakerException as ex:
        print('Error: %s' % ex)
        return False

    return result

def reset_exposure(cam):
    """
    This function returns the camera to a normal state by re-enabling automatic exposure.

    :param cam: Camera to reset exposure on.
    :type cam: CameraPtr
    :return: True if successful, False otherwise.
    :rtype: bool
    """
    try:
        result = True

        if cam.ExposureAuto.GetAccessMode() != PySpin.RW:
            print('Unable to enable automatic exposure (node retrieval). Non-fatal error...')
            return False

        cam.ExposureAuto.SetValue(PySpin.ExposureAuto_Continuous)

        print('Automatic exposure enabled...')

    except PySpin.SpinnakerException as ex:
        print('Error: %s' % ex)
        result = False

    return result

def reset_trigger(cam):
    """
    This function returns the camera to a normal state by turning off trigger mode.

    :param cam: Camera to acquire images from.
    :type cam: CameraPtr
    :returns: True if successful, False otherwise.
    :rtype: bool
    """
    try:
        result = True
        # Ensure trigger mode off
        # The trigger must be disabled in order to configure whether the source
        # is software or hardware.
        if cam.TriggerMode.GetAccessMode() != PySpin.RW:
            print('Unable to disable trigger mode (node retrieval). Aborting...')
            return False

        cam.TriggerMode.SetValue(PySpin.TriggerMode_Off)

        print('Trigger mode disabled...')

    except PySpin.SpinnakerException as ex:
        print('Error: %s' % ex)
        result = False

    return result

def reset_offsets(cam):

    if cam.OffsetX.GetAccessMode() == PySpin.RW:
        cam.OffsetX.SetValue(0)

    if cam.OffsetY.GetAccessMode() == PySpin.RW:
        cam.OffsetY.SetValue(0)

def correct_type(string):
    # convert to original data types.
    # eval returns its string argument with its correct data type,
    # except for the case in which the arguement is an incomprehensible string
    # so a NameError appears
    try:
        t = eval(string)
    except:
        t = string

    return t

def configure_custom_image_settings(cam, section):
    """
    This function accesses some nodes of settings via QuickSpin.
    :param cam: Camera to configure settings for.
    :type cam: CameraPtr
    :param section: section of 'config.ini' file to configure camera from
    :type section: SectionProxy
    :return: True if successful, False otherwise.
    :rtype: bool
    """
    try:
        result = True

        for key in section:

            if key == 'acquisition mode':
                config_acquisitionmode = correct_type( section[key] )
            elif key == 'frame count':
                config_framecount = correct_type( section[key] )
            elif key == 'frame rate enable':
                config_framerateenable = correct_type( section[key] )
            elif key == 'frame rate':
                config_framerate = correct_type( section[key] )
            elif key == 'pixel format':
                config_pixelformat = correct_type( section[key] )
            elif key == 'width':
                config_width = correct_type( section[key] )
            elif key == 'height':
                config_height = correct_type( section[key] )
            elif key == 'offset x':
                config_offsetx = correct_type( section[key] )
            elif key == 'offset y':
                config_offsety = correct_type( section[key] )
            elif key == 'exposure auto':
                config_exposureauto = correct_type( section[key] )
            elif key == 'exposure time':
                config_exposuretime = correct_type( section[key] )
            elif key == 'gain':
                config_gain = correct_type( section[key] )
            elif key == 'gamma enable':
                config_gammaenable = correct_type( section[key] )
            elif key == 'gamma':
                config_gamma = correct_type( section[key] )
            elif key == 'trigger mode':
                config_triggermode = correct_type( section[key] )
            elif key == 'trigger source':
                config_triggersource = correct_type( section[key] )

        ###### CONFIGURE ACQUISITION MODE ######
        if cam.AcquisitionMode.GetAccessMode() == PySpin.RW:

            if config_acquisitionmode == 'single frame':
                cam.AcquisitionMode.SetValue(PySpin.AcquisitionMode_SingleFrame)
            elif config_acquisitionmode == 'continuous':
                cam.AcquisitionMode.SetValue(PySpin.AcquisitionMode_Continuous)
            elif config_acquisitionmode == 'multiframe':
                cam.AcquisitionMode.SetValue(PySpin.AcquisitionMode_MultiFrame)

            print('Acquisition mode set to %s...' % cam.AcquisitionMode.GetCurrentEntry().GetSymbolic())

            if config_acquisitionmode == 'multiframe':
                ###### CONFIGURE ACQUISITION FRAME COUNT ######
                if cam.AcquisitionFrameCount.GetAccessMode() == PySpin.RW:
                    cam.AcquisitionFrameCount.SetValue(config_framecount)
                else:
                    print('Acquisition frame count not available...')
                    result = False

            if config_acquisitionmode == 'continuous' or config_acquisitionmode == 'multiframe':

                ###### CONFIGURE ACQUISITION FRAME RATE ######
                if cam.AcquisitionFrameRateEnable.GetAccessMode() == PySpin.RW:
                    cam.AcquisitionFrameRateEnable.SetValue(config_framerateenable)

                if cam.AcquisitionFrameRate.GetAccessMode() == PySpin.RW:
                    cam.AcquisitionFrameRate.SetValue(config_framerate)
                else:
                    if cam.AcquisitionFrameRateEnable.GetValue() == 'on':
                        print('Acquisition frame rate not available...')
                        result = False

        else:
            print('Acquisition mode not available...')
            result = False

        ###### CONFIGURE PIXEL FORMAT ######
        if cam.PixelFormat.GetAccessMode() == PySpin.RW:

            if config_pixelformat == 'mono8':
                cam.PixelFormat.SetValue(PySpin.PixelFormat_Mono8)
            elif config_pixelformat == 'mono16':
                cam.PixelFormat.SetValue(PySpin.PixelFormat_Mono16)

            print('Pixel format set to %s...' % cam.PixelFormat.GetCurrentEntry().GetSymbolic())

        else:
            print('Pixel format not available...')
            result = False

        # reset OffsetX and OffsetY first
        reset_offsets(cam)

        ###### CONFIGURE WIDTH ######
        if cam.Width.GetAccessMode() == PySpin.RW and cam.Width.GetInc() != 0 and cam.Width.GetMax != 0:
            cam.Width.SetValue(config_width)
            print('Width set to %i...' % cam.Width.GetValue())

        else:
            print('Width not available...')
            result = False

        ###### CONFIGURE HEIGHT ######
        if cam.Height.GetAccessMode() == PySpin.RW and cam.Height.GetInc() != 0 and cam.Height.GetMax != 0:
            cam.Height.SetValue(config_height)
            print('Height set to %i...' % cam.Height.GetValue())

        else:
            print('Height not available...')
            result = False

        ######## CONFIGURE OFFSET X ####
        if cam.OffsetX.GetAccessMode() == PySpin.RW:

            if config_offsetx >= cam.OffsetX.GetMin() & config_offsetx <= cam.OffsetX.GetMax():
                cam.OffsetX.SetValue(config_offsetx)
                print('Offset X set to %d...' % cam.OffsetX.GetValue())

            else:
                print('Offset X value not possible...')

        else:
            print('Offset X not available...')
            result = False

        ###### CONFIGURE OFFSET Y ######
        if cam.OffsetY.GetAccessMode() == PySpin.RW:

            if config_offsety >= cam.OffsetY.GetMin() & config_offsety <= cam.OffsetY.GetMax():
                cam.OffsetY.SetValue(config_offsety)
                print('Offset Y set to %d...' % cam.OffsetY.GetValue())

            else:
                print('Offset Y value not possible...')

        else:
            print('Offset Y not available...')
            result = False

        ###### CONFIGURE EXPOSURE ######
        if cam.ExposureAuto.GetAccessMode() != PySpin.RW:
            print('Unable to edit automatic exposure. Aborting...')
            return False

        if config_exposureauto == 'continuous':
            cam.ExposureAuto.SetValue(PySpin.ExposureAuto_Continuous)
        elif config_exposureauto == 'off':
            cam.ExposureAuto.SetValue(PySpin.ExposureAuto_Off)

        print('Automatic exposure set to %s...' % cam.ExposureAuto.GetCurrentEntry().GetSymbolic())

        # If auto exposure is continuous: set exposure time manually; exposure time recorded in microseconds
        if config_exposureauto == 'off':
            if cam.ExposureTime.GetAccessMode() != PySpin.RW:
                print('Unable to set exposure time. Aborting...')
                return False

            # Ensure desired exposure time does not exceed the maximum
            exposure_time_to_set = min(cam.ExposureTime.GetMax(), config_exposuretime)
            cam.ExposureTime.SetValue(exposure_time_to_set)
            print('Exposure time set to %f microseconds...' % exposure_time_to_set)

        ###### CONFIGURE GAIN ######
        if cam.GainAuto.GetAccessMode() != PySpin.RW:
            print('Unable to disable automatic gain. Aborting...')
            return False

        cam.GainAuto.SetValue(PySpin.GainAuto_Off)
        print('Automatic gain disabled...')

        if cam.Gain.GetAccessMode() != PySpin.RW:
            print('Unable to set gain. Aborting...')
            return False

        gain_value = config_gain
        cam.Gain.SetValue(gain_value)
        print('Gain value set to %f...' % gain_value)

        ###### CONFIGURE GAMMA ######

        if cam.GammaEnable.GetAccessMode() != PySpin.RW:
            print('Unable to enable/disable gamma. Aborting...')
            return False

        if config_gammaenable == False:
            cam.GammaEnable.SetValue(False)
            print('Gamma disabled...')

        else:
            cam.GammaEnable.SetValue(True)
            print('Gamma enabled...')
            cam.Gamma.SetValue(config_gamma)
            print('Gamma value set to %f...' % config_gamma)

        ###### CONFIGURE TRIGGER SOURCE ######

        # Ensure trigger mode off
        # The trigger must be disabled in order to configure whether the source
        # is software or hardware.
        if cam.TriggerMode.GetAccessMode() != PySpin.RW:
            print('Unable to disable trigger mode (node retrieval). Aborting...')
            return False

        cam.TriggerMode.SetValue(PySpin.TriggerMode_Off)

        # Select trigger source
        # The trigger source must be set to hardware or software while trigger
    	  # mode is off.
        if cam.TriggerSource.GetAccessMode() != PySpin.RW:
            print('Unable to get trigger source (node retrieval). Aborting...')
            return False

        if config_triggermode == 'on':

            if config_triggersource == 'software':
                cam.TriggerSource.SetValue(PySpin.TriggerSource_Software)
                print('Software trigger chosen...')
            elif config_triggersource == 'hardware':
                cam.TriggerSource.SetValue(PySpin.TriggerSource_Line0)
                print('Hardware trigger chosen...')

            # Turn trigger mode on
            # Once the appropriate trigger source has been set, turn trigger mode
            # on in order to retrieve images using the trigger.
            cam.TriggerMode.SetValue(PySpin.TriggerMode_On)
            print('Trigger mode turned on...')

    except PySpin.SpinnakerException as ex:
        print('Error: %s' % ex)
        return False

    return result

#def live_view():
#

def acquire_images(cam, nodemap, nodemap_tldevice, q, stoplive_event):
    """
    :param cam: Camera to acquire images from.
    :param nodemap: Device nodemap.
    :param nodemap_tldevice: Transport layer device nodemap.
    :type cam: CameraPtr
    :type nodemap: INodeMap
    :type nodemap_tldevice: INodeMap
    :return: True if successful, False otherwise.
    :rtype: bool
    """

    print('*** IMAGE ACQUISITION ***\n')
    try:
        # Begin acquiring images
        cam.BeginAcquisition()

        print('Acquiring image...')

#        disting_name = ''
#        node_device_serial_number = PySpin.CStringPtr(nodemap_tldevice.GetNode('DeviceSerialNumber'))
#        if PySpin.IsAvailable(node_device_serial_number) and PySpin.IsReadable(node_device_serial_number):
#            disting_name = node_device_serial_number.GetValue()

        if cam.AcquisitionMode.GetValue() == PySpin.AcquisitionMode_Continuous:

            # while loop until flag gets 1 from GUI func: 'stop_live'
            while(stoplive_event.is_set() == False):

                try:
                    image_result = cam.GetNextImage()
                    q.put(image_result.GetData(), block=True, timeout=0.2)
                    image_result.Release()

                except PySpin.SpinnakerException as ex1:
                    print('Error: %s' % ex1)
                    return False

                except Queue.Full as ex2:
                    print('Error: %s' % ex2)
                    return False

        # send signal for finishing
        q.put('done')

        print('Image acquisition finished...')
        cam.EndAcquisition()

    except PySpin.SpinnakerException as ex:
        print('Error: %s' % ex)
        return False

def run_camera(cam_num, config_with):
    """
    cam_num: int | [0,1,...]
    config_with: int | [0,1,2]
    """

    global config_sections

    # Retrieve singleton reference to system object
    system = PySpin.System.GetInstance()

    # Get current library version
    version = system.GetLibraryVersion()
    print('Library version: %d.%d.%d.%d' % (version.major, version.minor, version.type, version.build))

    # Retrieve list of cameras from the system
    cam_list = system.GetCameras()

    num_cameras = cam_list.GetSize()

    print('Number of cameras detected: %d' % num_cameras)

    # Finish if there are no cameras
    if num_cameras == 0:

        # Clear camera list before releasing system
        cam_list.Clear()

        # Release system instance
        system.ReleaseInstance()

        print('Not enough cameras!')

    cam = cam_list[cam_num]

    # Retrieve TL device nodemap and print device information
    nodemap_tldevice = cam.GetTLDeviceNodeMap()

    print_device_info(nodemap_tldevice)

    # Initialize camera
    cam.Init()

    # Retrieve GenICam nodemap
    nodemap = cam.GetNodeMap()

    # load all configurations
    config_sections = cfg.load_config('ALL')
    # select one configuration section out of all
    configcam_with = config_sections[config_with]
    if not configure_custom_image_settings(cam, configcam_with):
        return False

    print('*** CAMERA CONNECTION ESTABLISHED ***\n')
    cam_nodes = [cam, nodemap, nodemap_tldevice, cam_list, system]

    return cam_nodes

def begin_acquisition(cam):

    try:
        cam.BeginAcquisition()
        return True

    except PySpin.SpinnakerException as ex:
        print('Error: %s' % ex)
        return False

def end_acquisition(cam_nodes):
    cam = cam_nodes[0]

    # End camera acquisition
    cam.EndAcquisition()

    del cam
    release_camera(cam_nodes)

def release_camera(cam_nodes):
    cam = cam_nodes[0]
    cam_list = cam_nodes[3]
    system = cam_nodes[4]

    # Reset exposure
    reset_exposure(cam)

    # Reset trigger
    reset_trigger(cam)

    # Deinitialize camera
    cam.DeInit()

    # Release reference to camera
    del cam

    # Clear camera list before releasing system
    cam_list.Clear()
    del cam_nodes[:]

    # Release system instance
    system.ReleaseInstance()

    print('Camera release successful')

def grab_next_image_by_trigger(cam):
    """
    This function acquires an image by executing the trigger node.

    :param cam: Camera to acquire images from.
    :type cam: CameraPtr
    :return: True if successful, False otherwise.
    :rtype: bool
    """
    global trigger_flag

    try:
        result = True

        if cam.TriggerSource.GetValue() == PySpin.TriggerSource_Software:

            # Execute software trigger
            if cam.TriggerSoftware.GetAccessMode() != PySpin.WO:
                print('Unable to execute trigger. Aborting...')
                return False

            try:
                cam.TriggerSoftware.Execute()
                trigger_flag = 1
                image_result = cam.GetNextImage()
                image_array = PIL2array(image_result)
                image_result.Release()

                result = image_array
            except PySpin.SpinnakerException as ex:
                print('Error: %s' % ex)
                result = False

            # TODO: Blackfly and Flea3 GEV cameras need 2 second delay after software trigger

        elif cam.TriggerSource.GetValue() == PySpin.TriggerSource_Line0:
            print('Use the hardware to trigger image acquisition.')

    except PySpin.SpinnakerException as ex:
        print('Error: %s' % ex)
        return False

    return result

def save_profile(filename, profile):
    misc.imsave(filename, profile)

def save_image(disting_name):

    filename = '%s.tif' % disting_name

    # Save in image format
    misc.imsave(filename, image_array)
    # Save image array in txt
    #np.savetxt(('%s.txt' % disting_name), image_array, fmt = '%d')
    print('Image saved as %s' % filename)

#def stoplive():
#    global stoplive_flag
#
#    stoplive_flag = 1

##########################################
##                                       #
##   FUNCTIONS USED SPECIFICALLY FOR GUI #
##                                       #
##########################################
#
#def acquire_images_gui(cam, nodemap, nodemap_tldevice, gui_param):
#    """
#    :param cam: Camera to acquire images from.
#    :param nodemap: Device nodemap.
#    :param nodemap_tldevice: Transport layer device nodemap.
#    :type cam: CameraPtr
#    :type nodemap: INodeMap
#    :type nodemap_tldevice: INodeMap
#    :return: True if successful, False otherwise.
#    :rtype: bool
#    """
#
#    global image_array
#    global disting_name
#    global stoplive_flag
#
#    print('*** IMAGE ACQUISITION ***\n')
#    try:
#        result = True
#
#        # Retrieve gui parameters needed for image displaying
#        figplot = gui_param[0]
#        fig = gui_param[1]
#        canvas = gui_param[2]
#        single_feed = gui_param[3]
#        livefeed_img = gui_param[4]
#        livefeed_midline = gui_param[5]
#
#        # Begin acquiring images
#        cam.BeginAcquisition()
#
#        print('Acquiring image...')
#
#        disting_name = ''
#        node_device_serial_number = PySpin.CStringPtr(nodemap_tldevice.GetNode('DeviceSerialNumber'))
#        if PySpin.IsAvailable(node_device_serial_number) and PySpin.IsReadable(node_device_serial_number):
#            disting_name = node_device_serial_number.GetValue()
#            #print('Device serial number retrieved as %s...' % disting_name)
#
##        if cam.AcquisitionMode.GetValue() == PySpin.AcquisitionMode_SingleFrame:
##            NUM_IMAGES = 1
##        elif cam.AcquisitionMode.GetValue() == PySpin.AcquisitionMode_Continuous:
##            NUM_IMAGES = 1000
#
#        try:
#            image_result = cam.GetNextImage()
#            image_array = PIL2array(image_result)
#            image_result.Release()
#
#            ### SINGLE CAPTURE - IMAGE DISPLAY ###
#            if single_feed:
#                fig.imshow(image_array, aspect='auto', cmap='gray')
#                canvas.draw()
#
#            ### LIVE IMAGE DISPLAY ON GUI ###
#            if livefeed_img:
#                fimshow = fig.imshow(image_array, aspect='auto', cmap='gray')
#                canvas.draw()
#
#        except PySpin.SpinnakerException as ex:
#                print('Error: %s' % ex)
#                return False
#
#        if cam.AcquisitionMode.GetValue() == PySpin.AcquisitionMode_Continuous:
#
#            # Reset flag
#            stoplive_flag = 0
#            ctr = 0
#
#            # while loop until flag gets 1 from GUI func: 'stop_live'
#            while(stoplive_flag == 0):
#                ctr += 1
#
#                try:
#                    image_result = cam.GetNextImage()
#                    image_array = PIL2array(image_result)
#                    image_result.Release()
#
#                    ### LIVE IMAGE DISPLAY ON GUI ###
#                    if livefeed_img & (ctr % 20 == 0):
#                        fimshow.set_data(image_array)
#                        canvas.draw()
#
#                    ### MIDDLE LINE DISPLAY ON GUI ###
#                    if livefeed_midline & (ctr % 20 == 0):
#                        fig.clear()
#                        midline = (np.shape(image_array))[0]//2
#                        fig.plot(image_array[midline])
#                        canvas.draw()
#                        fig.clear()
#
#                    if ctr == 100:
#                        ctr = 0
#
#                except PySpin.SpinnakerException as ex:
#                    print('Error: %s' % ex)
#                    return False
#
#            # stoplive_flag value became '1', so reset it
#            stoplive_flag = 0
#
#        cam.EndAcquisition()
#        print('Image(s) acquisition completed...')
#
#    except PySpin.SpinnakerException as ex:
#        print('Error: %s' % ex)
#        return False
#
#    return result
#
#def run_single_camera_gui(cam, gui_param):
#    """
#    :param cam: Camera to run on.
#    :type cam: CameraPtr
#    :return: True if successful, False otherwise.
#    :rtype: bool
#    """
#    try:
#        result = True
#
#        # Retrieve TL device nodemap and print device information
#        nodemap_tldevice = cam.GetTLDeviceNodeMap()
#
#        result &= print_device_info(nodemap_tldevice)
#
#        # Initialize camera
#        cam.Init()
#
#        # Retrieve GenICam nodemap
#        nodemap = cam.GetNodeMap()
#
#        if not configure_custom_image_settings(cam):
#            return False
#
#        # Acquire images
#        result &= acquire_images_gui(cam, nodemap, nodemap_tldevice, gui_param)
#
#        # Reset exposure
#        result &= reset_exposure(cam)
#
#        # Reset trigger
#        result &= reset_trigger(cam)
#
#        # Deinitialize camera
#        cam.DeInit()
#
#    except PySpin.SpinnakerException as ex:
#        print('Error: %s' % ex)
#        result = False
#
#    return result
#
#def run_camera_gui(cam_num, gui_param):
#
#    # Retrieve singleton reference to system object
#    system = PySpin.System.GetInstance()
#
#    # Get current library version
#    version = system.GetLibraryVersion()
#    print('Library version: %d.%d.%d.%d' % (version.major, version.minor, version.type, version.build))
#
#    # Retrieve list of cameras from the system
#    cam_list = system.GetCameras()
#
#    num_cameras = cam_list.GetSize()
#
#    print('Number of cameras detected: %d' % num_cameras)
#
#    # Finish if there are no cameras
#    if num_cameras == 0:
#
#        # Clear camera list before releasing system
#        cam_list.Clear()
#
#        # Release system instance
#        system.ReleaseInstance()
#
#        print('Not enough cameras!')
#
#    cam = cam_list[cam_num]
#    run_single_camera_gui(cam, gui_param)
#    #mpl.pyplot.pcolormesh(image_array,cmap='gray')
#    # Release reference to camera
#    del cam
#
#    # Clear camera list before releasing system
#    cam_list.Clear()
#
#    # Release system instance
#    system.ReleaseInstance()
#
#def updateFrame(fig_img, image_array):
#    fig_img.set_data(image_array)
#    #canvas.draw()
#
#def save_image_gui(savedir, img_case):
#    """
#    Wrapper function called from gui
#    """
#    # img_case is the name that corresponds to one of the four different images
#    # acquired from the interferometer
#    if img_case:
#        save_image(savedir + '\\' + img_case)
#    # disting_name is a global variable assigned at image acquisition process
#    else:
#        save_image(savedir + '\\' + disting_name)
#
#def stoplive():
#    global stoplive_flag
#
#    stoplive_flag = 1


from __future__ import absolute_import, division, print_function

__metaclass__ = type

ANSIBLE_METADATA = {
    "metadata_version": "1.1",
    "status": ["preview"],
    "supported_by": "community",
}

DOCUMENTATION = """
---
module: oci_object_storage_namespace_metadata_facts
short_description: Fetches details about a NamespaceMetadata resource in Oracle Cloud Infrastructure
description:
    - Fetches details about a NamespaceMetadata resource in Oracle Cloud Infrastructure
    - Gets the metadata for the Object Storage namespace, which contains defaultS3CompartmentId and
      defaultSwiftCompartmentId.
    - Any user with the OBJECTSTORAGE_NAMESPACE_READ permission will be able to see the current metadata. If you are
      not authorized, talk to an administrator. If you are an administrator who needs to write policies
      to give users access, see
      L(Getting Started with Policies,https://docs.cloud.oracle.com/Content/Identity/Concepts/policygetstarted.htm).
version_added: "2.9.0"
author: Oracle (@oracle)
options:
    namespace_name:
        description:
            - The Object Storage namespace used for the request.
        type: str
        required: true
extends_documentation_fragment: [ oracle.oci.oracle ]
"""

EXAMPLES = """
- name: Get a specific namespace_metadata
  oci_object_storage_namespace_metadata_facts:
    # required
    namespace_name: namespace_name_example

"""

RETURN = """
namespace_metadata:
    description:
        - NamespaceMetadata resource
    returned: on success
    type: complex
    contains:
        namespace:
            description:
                - The Object Storage namespace to which the metadata belongs.
            returned: on success
            type: str
            sample: namespace_example
        default_s3_compartment_id:
            description:
                - If the field is set, specifies the default compartment assignment for the Amazon S3 Compatibility API.
            returned: on success
            type: str
            sample: "ocid1.defaults3compartment.oc1..xxxxxxEXAMPLExxxxxx"
        default_swift_compartment_id:
            description:
                - If the field is set, specifies the default compartment assignment for the Swift API.
            returned: on success
            type: str
            sample: "ocid1.defaultswiftcompartment.oc1..xxxxxxEXAMPLExxxxxx"
    sample: {
        "namespace": "namespace_example",
        "default_s3_compartment_id": "ocid1.defaults3compartment.oc1..xxxxxxEXAMPLExxxxxx",
        "default_swift_compartment_id": "ocid1.defaultswiftcompartment.oc1..xxxxxxEXAMPLExxxxxx"
    }
"""

from ansible_collections.oracle.oci.plugins.module_utils import oci_common_utils
from ansible_collections.oracle.oci.plugins.module_utils.oci_resource_utils import (
    OCIResourceFactsHelperBase,
    get_custom_class,
    OCIAnsibleModule,
)

try:
    from oci.object_storage import ObjectStorageClient

    HAS_OCI_PY_SDK = True
except ImportError:
    HAS_OCI_PY_SDK = False

class NamespaceMetadataFactsHelperGen(OCIResourceFactsHelperBase):
    """Supported operations: get"""

    def get_required_params_for_get(self):
        return [
            "namespace_name",
        ]

    def get_resource(self):
        return oci_common_utils.call_with_backoff(
            self.client.get_namespace_metadata,
            namespace_name=self.module.params.get("namespace_name"),
        )

NamespaceMetadataFactsHelperCustom = get_custom_class(
    "NamespaceMetadataFactsHelperCustom"
)

class ResourceFactsHelper(
    NamespaceMetadataFactsHelperCustom, NamespaceMetadataFactsHelperGen
):
    pass

def main():
    module_args = oci_common_utils.get_common_arg_spec()
    module_args.update(dict(namespace_name=dict(type="str", required=True),))

    module = OCIAnsibleModule(argument_spec=module_args)

    if not HAS_OCI_PY_SDK:
        module.fail_json(msg="oci python sdk required for this module.")

    resource_facts_helper = ResourceFactsHelper(
        module=module,
        resource_type="namespace_metadata",
        service_client_class=ObjectStorageClient,
        namespace="object_storage",
    )

    result = []

    if resource_facts_helper.is_get():
        result = resource_facts_helper.get()
    else:
        resource_facts_helper.fail()

    module.exit_json(namespace_metadata=result)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""

## IMPORTANT: FOLDER STRUCTURE ##
Below mentioned folder structure must be followed for getting the results from this script:
01. Script must be placed into a folder known as main folder
02. Source files (3 lists) must be placed into the Source folder under main folder
03. a folder named as IntermediateFiles must be created under main folder with below mentioned sub-folders
    A. Source
    B. Preprocessed
    C. IntermediateFiles
        i.  DDM
        ii. PDM

## IMPORTANT: EXECUTION ORDER ##
Below scripts must be exected in the mentioned order:
01. 01_RecordLinkageDDM.py
02. 02_RecordLinkagePDM.py
03. 03_RecordLinkageDDMScore.py
04. 04_RecordLinkagePDMScore.py

This script performs determistics data matching between customer list and negative/positive list. It performs below mentioned steps:
01. Reads data from the source files - 00_List_Customer_Monitoring.csv, 01a_List_Negative.csv and 01b_List_Positive.csv, placed under Source folder
02. Performs data preprocessing and cleaning on the loaded data
03. Generates the files under Preprocessed folder present under IntermediateFiles folder
03. Matches data using different determistics record linkage rules
04. Generates files under DDM folder present under IntermediateFiles folder

"""

# Load required packages
import os
import pandas as pd
import numpy as np
from datetime import datetime
import unicodedata

def extractSource(dir, files):
    '''
    Function to extract Data from files in a specific format
    '''
    t = {"FIRST_NAME": object, "LAST_NAME": object, "DOB": object, "STREET": object, "HNR": object, "HNRADD": object, "ZIP": object, "CITY": object}
    filename = dir + files
    df = pd.read_csv(filename, index_col="ID", na_values="0000-00-00", parse_dates=[3], dtype=t)
    return df

def caseConvertion(df):
    '''
    Function to convert string values present into columns to lowercase letters
    '''
    df = df.applymap(lambda s:s.lower() if type(s) == str else s)
    return df

def stripList(df):
    '''
    Function to trim the string values present into columns
    '''
    df = df.applymap(lambda s:s.strip() if type(s) == str else s)
    return df

def removeSpecialChar(s):
    '''
    Function to replace special symbols present with a space
    '''
    spec_chars = ["!", '"', "#", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/", ":", ";", "<", "=", ">", "?", "@", "[", "\\", "]", "^", "_", "`", "{", "|", "}", "~", "–"]
    for char in spec_chars:
        s = s.replace(char, " ")
        s = " ".join(s.split())
    return s

def removeSpecial(df):
    '''
    Function to replace special characters from all the values present into different columns
    '''
    df = df.applymap(lambda s:removeSpecialChar(s) if type(s) == str else s)
    return df

def removeTitleName(s):
    '''
    Function to remove titles and honorifics from a series
    '''
    titles = ["ms ", "mr ", "mrs ", "miss ", "master ", "professor ", "dr ", "herr ", "frau ", "prof "]
    for title in titles:
        s = s.str.replace(title, "").str.strip()
    return s

def removeTitle(df):
    '''
    Function to remove titles and honorifics from first name and last name
    '''
    df["FIRST_NAME"] = removeTitleName(df["FIRST_NAME"])
    df["LAST_NAME"] = removeTitleName(df["LAST_NAME"])
    return df

def replaceUmlaut(df):
    '''
    Function to replace umlauts (ä --> ae, ö --> oe, ü --> ue, ß --> ss)
    '''
    df = df.applymap(lambda s:s.replace('ä','ae') if type(s) == str else s)
    df = df.applymap(lambda s:s.replace('ö','oe') if type(s) == str else s)
    df = df.applymap(lambda s:s.replace('ü','ue') if type(s) == str else s)
    df = df.applymap(lambda s:s.replace('ß','ss') if type(s) == str else s)
    return df

def removeAccentedChars(s):
    '''
    Function to replace special characters like accents to ASCII characters from a series
    '''
    s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore').decode('utf-8', 'ignore')
    return s

def removeAccented(df):
    '''
    Function to replace special characters like accents to ASCII characters from the dataframe
    '''
    df = df.applymap(lambda s:removeAccentedChars(s) if type(s) == str else s)
    return df

def formatZip(df):
    '''
    Function to standardize ZIP to a five characters format by appending leading zeroes.
    '''
    df['ZIP'] = df['ZIP'].str.zfill(5)
    return df

def formatCity(df):
    '''
    Function to correct common spelling mistakes in CITY
    '''
    df['CITY'] = df['CITY'].str.replace('mainz a r', 'mainz')
    df['CITY'] = df['CITY'].str.replace('frankfurt a m', 'frankfrut am main')
    df['CITY'] = df['CITY'].str.replace('frankfurt am', 'frankfrut am main')
    df['CITY'] = df['CITY'].str.replace('frankfurt a main', 'frankfrut am main')
    df['CITY'] = df['CITY'].str.replace('frankfurt m', 'frankfrut a main')
    df.loc[(df['CITY'] == 'frankfurt') & (df['ZIP'].str.startswith('1')), 'CITY'] = 'frankfurt oder'
    df.loc[(df['CITY'] == 'frankfurt') & (df['ZIP'].str.startswith('6')), 'CITY'] = 'frankfurt am main'
    return df

def extractHNR(df):
    '''
    Function to extract HNR from STREET if it contains any number
    '''
    pattern1 = r'([\d]+)'
    pattern2 = r'([\d]+[\w])'
    pattern3 = r'([\d]+\s[\w])'
    pattern4 = r'([\d]+\s[\d]+\s[\w]\s[\d]+)'
    df["PAT3"] = df["STREET"].str.extract(pattern3).replace(' ', '')
    df["PAT2"] = df["STREET"].str.extract(pattern2).replace(' ', '')
    df["PAT1"] = df["STREET"].str.extract(pattern1).replace(' ', '')
    df['HNRNEW'] = df['PAT3'].fillna(df["PAT2"].fillna(df["PAT1"]))
    df['HNRNEW'] = df['HNRNEW'].str.replace(' ', '')
    df["STREET"] = df["STREET"].str.replace(pattern4, '').str.strip()
    df["STREET"] = df["STREET"].str.replace(pattern3, '').str.strip()
    df["STREET"] = df["STREET"].str.replace(pattern2, '').str.strip()
    df["STREET"] = df["STREET"].str.replace(pattern1, '').str.strip()
    df = df.drop(columns=['PAT3'], axis = 1)
    df = df.drop(columns=['PAT2'], axis = 1)
    df = df.drop(columns=['PAT1'], axis = 1)
    return df

def joinColumns(df):
    '''
    Function to combine HNR and HNRADD to form a new column as HNRNEW and dropping the HNR and HNRADD columns from the dataframe
    '''
    df["HNR"] = df["HNR"].astype(str)
    df["HNRADD"] = df["HNRADD"].astype(str)
    df["HNR"] = df["HNR"].str.replace(' ', '')
    df["HNR"] = df["HNR"].str.replace('nan', '')
    df["HNRADD"] = df["HNRADD"].str.replace(' ', '')
    df["HNRADD"] = df["HNRADD"].str.replace('nan', '')
    df["HNRNEW"] = df['HNR'].str.strip().fillna('') + df['HNRADD'].str.strip().fillna('')
    df["HNRNEW"] = df["HNRNEW"].str.strip().replace(r'^\s*$', np.nan, regex=True)
    df = df.drop(columns=['HNR'], axis = 1)
    df = df.drop(columns=['HNRADD'], axis = 1)
    return df

def formatStreet(df):
    '''
    Function to standardise street name (' str', ' strsse', ' srasse', 'str$', 'strsse$', 'srasse$' --> 'strasse')
    '''
    df['STREET'] = df['STREET'].str.replace(' str','strasse')
    df['STREET'] = df['STREET'].str.replace(' strsse','strasse')
    df['STREET'] = df['STREET'].str.replace(' srasse','strasse')
    df['STREET'] = df['STREET'].str.replace('str$','strasse')
    df['STREET'] = df['STREET'].str.replace('strsse$','strasse')
    df['STREET'] = df['STREET'].str.replace('srasse$','strasse')
    return df

def dataPreprocessing(df):
    '''
    Function to perfrom data preprocessing and cleaning on customer list as per below mentioned order:
    01. Convert case to lowercase
    02. Remove the whitespace from the string columns
    03. Remove special symbols by a single space
    04. Remove titles and honorifics from the name
    05. Replace german umlaut to english equivalents
    06. Replace special characters with ASCII characters
    07. Standardized the Zip
    08. Standardized the city name
    09. Extract HNR from STREET name
    10. Formation of HNRNEW
    11. Standardized the Street name
    '''
    df = caseConvertion(df)
    df = stripList(df)
    df = removeSpecial(df)
    df = removeTitle(df)
    df = replaceUmlaut(df)
    df = removeAccented(df)
    df = formatZip(df)
    df = formatCity(df)
    df = extractHNR(df)
    df = joinColumns(df)
    df = formatStreet(df)
    return df

def dataPreprocessing1(df):
    '''
    Function to perfrom data preprocessing and cleaning on positive and negative lists as per below mentioned order:
    01. Convert case to lowercase
    02. Remove the whitespace from the string columns
    03. Remove special symbols by a single space
    04. Remove titles and honorifics from the name
    05. Replace german umlaut to english equivalents
    06. Replace special characters with ASCII characters
    07. Standardized the Zip
    08. Standardized the city name
    09. Extract HNR from STREET name
    10. Formation of NEWHNR
    11. Standardized the Street name
    12. Drop duplicate rows
    '''
    df = caseConvertion(df)
    df = stripList(df)
    df = removeSpecial(df)
    df = removeTitle(df)
    df = replaceUmlaut(df)
    df = removeAccented(df)
    df = formatZip(df)
    df = formatCity(df)
    df = extractHNR(df)
    df = joinColumns(df)
    df = formatStreet(df)
    df.drop_duplicates(inplace=True)
    return df

def IntermediateFiles(dir, file, df):
    '''
    Function to create intermediate files post a specific operation like Data Preprocessing with index values
    '''
    originalFiles = ["00_List_Customer_Monitoring.csv", "01a_List_Negative.csv", "01b_List_Positive.csv"]
    if file in originalFiles:
        file = file
    else:
        file = datetime.now().strftime("%Y%m%d") + "_" + file
    filename = dir + file
    df.to_csv(filename)

def MatchedFiles(dir, file, df):
    '''
    Function to create intermediate files post a specific operation like Data Preprocessing without index values
    '''
    originalFiles = ["00_List_Customer_Monitoring.csv", "01a_List_Negative.csv", "01b_List_Positive.csv"]
    if file in originalFiles:
        file = file
    else:
        file = datetime.now().strftime("%Y%m%d") + "_" + file
    filename = dir + file
    df.to_csv(filename, index=False)

def matchedIndex(nmatch, pmatch, rule):
    '''
    Function to append customers matched with negative list to the customers matched with positive list
    '''
    match = nmatch.append(pmatch, ignore_index=True, sort=False)
    match = match[["ID_CUST", "ID_NEG", "ID_POS", "MATCH_CRITERIA", "MATCH_SCORE"]]
    match.sort_values(by=["ID_CUST"], inplace=True)
    match = match.reset_index()
    match = match.drop("index", axis = 1)
    return match

def colMatchDDMPOS(cust, pos, condition, i):
    '''
    Function to match customer with the positive list based on a condition
    '''
    print("Start of Rule" + str(i) + ": [" + ', '.join(condition[:-1]) + "]")
    pos_match = pd.merge(cust, pos, how="inner", on=condition[:-1], suffixes=["_CUST", "_POS"])
    matchCriteria = "DDM RULE" + str(i) + ": " + ', '.join(condition[:-1])
    pos_match["MATCH_CRITERIA"] = matchCriteria
    pos_match["MATCH_SCORE"] = float(condition[-1])
    matched_index = pos_match[["ID_CUST", "ID_POS", "MATCH_CRITERIA", "MATCH_SCORE"]]
    matched_index.sort_values(by=["ID_CUST"], inplace=True)
    matched_index = matched_index.reset_index()
    matched_index = matched_index.drop("index", axis = 1)
    cust = cust[~cust["ID"].isin(matched_index["ID_CUST"])]
    print("End of Rule" + str(i) +": [" + ', '.join(condition[:-1]) + "]")
    return matched_index, cust

def colMatchDDMNEG(cust, neg, condition, i):
    '''
    Function to match customer with the negative list based on a condition
    '''
    print("Start of Rule" + str(i) + ": [" + ', '.join(condition[:-1]) + "]")
    neg_match = pd.merge(cust, neg, how="inner", on=condition[:-1], suffixes=["_CUST", "_NEG"])
    matchCriteria = "DDM RULE " + str(i) + ": " + ', '.join(condition[:-1])
    neg_match["MATCH_CRITERIA"] = matchCriteria
    neg_match["MATCH_SCORE"] = float(condition[-1])
    matched_index = neg_match[["ID_CUST", "ID_NEG", "MATCH_CRITERIA", "MATCH_SCORE"]]
    matched_index.sort_values(by=["ID_CUST"], inplace=True)
    matched_index = matched_index.reset_index()
    matched_index = matched_index.drop("index", axis = 1)
    cust = cust[~cust["ID"].isin(matched_index["ID_CUST"])]
    print("End of Rule" + str(i) +": [" + ', '.join(condition[:-1]) + "]")
    return matched_index, cust

def DDM(cust_df, neg_df, pos_df):
    '''
    Function to iteratively perform DDM for all the defined rules
    '''
    print("Start DDM")
    # Replace 0000-00-00 with 1900-00-00 in customer list to avoid invalid matches
    cust_df["DOB"] = cust_df["DOB"].fillna('1900-00-00')
    cust_df = cust_df.fillna('-99999')
    # Replace 0000-00-00 with 1800-00-00 in negative list to avoid invalid matches
    neg_df["DOB"] = neg_df["DOB"].fillna('1800-00-00')
    neg_df = neg_df.fillna('-88888')
    # Replace 0000-00-00 with 1700-00-00 in negative list to avoid invalid matches
    pos_df["DOB"] = pos_df["DOB"].fillna('1700-00-00')
    pos_df = pos_df.fillna('-77777')
    cust_pos_df = cust_df.copy()
    cust_neg_df = cust_df.copy()
    cust_pos_df['FN'] = cust_pos_df['FIRST_NAME']
    cust_pos_df['LN'] = cust_pos_df['LAST_NAME']
    cust_neg_df['FN'] = cust_neg_df['FIRST_NAME']
    cust_neg_df['LN'] = cust_neg_df['LAST_NAME']
    neg_df['FN'] = neg_df['LAST_NAME']
    neg_df['LN'] = neg_df['FIRST_NAME']
    pos_df['FN'] = pos_df['LAST_NAME']
    pos_df['LN'] = pos_df['FIRST_NAME']
    cust_df = cust_df.reset_index()
    neg_df = neg_df.reset_index()
    pos_df = pos_df.reset_index()
    cust_pos_df = cust_pos_df.reset_index()
    cust_neg_df = cust_neg_df.reset_index()
    # Rules or conditions to perform DDM along with rule based matching score
    condition1 = ['FIRST_NAME', 'LAST_NAME', 'DOB', 'ZIP', 'CITY', 'STREET', 'HNRNEW', 100]
    condition2 = ['FIRST_NAME', 'LAST_NAME', 'DOB', 'ZIP', 'STREET', 'HNRNEW', 99.4]
    condition3 = ['FIRST_NAME', 'LAST_NAME', 'DOB', 'CITY', 'STREET', 'HNRNEW', 97.9]
    condition4 = ['FIRST_NAME', 'LAST_NAME', 'DOB', 97.2]
    condition5 = ['LAST_NAME', 'DOB', 'ZIP', 'CITY', 'STREET', 'HNRNEW', 95.6]
    condition6 = ['FIRST_NAME', 'DOB', 'ZIP', 'CITY', 'STREET', 'HNRNEW', 91]
    condition7 = ['FIRST_NAME', 'LAST_NAME', 'ZIP', 'CITY', 'STREET', 'HNRNEW', 90.2]
    condition8 = ['FIRST_NAME', 'LAST_NAME', 'ZIP', 'STREET', 'HNRNEW', 89]
    condition9 = ['FIRST_NAME', 'LAST_NAME', 'CITY', 'STREET', 'HNRNEW', 87]
    condition10 = ['FIRST_NAME', 'LAST_NAME', 'ZIP', 'CITY', 'STREET', 84]
    condition11 = ['FN', 'LN', 'DOB', 83.5]
    condition12 = ['FN', 'LN', 'ZIP', 'CITY', 'STREET', 'HNRNEW', 83]
    condition13 = ['LAST_NAME', 'DOB', 'ZIP', 81.6]
    condition14 = ['DOB', 'ZIP', 'CITY', 'STREET', 'HNRNEW', 78]
    condition15 = ['FIRST_NAME', 'DOB', 'ZIP', 76]
    matchConditions = [condition1, condition2, condition3, condition4, condition5, condition6, condition7, condition8, condition9, condition10, condition11, condition12, condition13, condition14, condition15]
    cwd = os.getcwd()
    intFileDir = cwd + r"\\IntermediateFiles\\DDM\\"
    i = 1
    custFile = r"00_List_Customer_Monitoring.csv"
    matched_idx = pd.DataFrame()
    for matchCondition in matchConditions:
        FileNamePOS = "DDM_POS_Rule" + str(i) + ".csv"
        FileNameNEG = "DDM_NEG_Rule" + str(i) + ".csv"
        custFilePOSPostMatch = "DDM_POS_Rule" + str(i) + "_" + custFile
        custFileNEGPostMatch = "DDM_NEG_Rule" + str(i) + "_" + custFile
        idxPOS, cust_pos_df = colMatchDDMPOS(cust_pos_df, pos_df, matchCondition, i)
        idxNEG, cust_neg_df = colMatchDDMNEG(cust_neg_df, neg_df, matchCondition, i)
        MatchedFiles(intFileDir, FileNamePOS, idxPOS)
        MatchedFiles(intFileDir, custFilePOSPostMatch, cust_pos_df)
        MatchedFiles(intFileDir, FileNameNEG, idxNEG)
        MatchedFiles(intFileDir, custFileNEGPostMatch, cust_neg_df)
        matched_idx = matched_idx.append(matchedIndex(idxNEG, idxPOS, matchCondition), ignore_index=True, sort=False)
        matched_idx.sort_values(by=["ID_CUST"], inplace=True)
        i += 1
    cust_df = cust_df[~cust_df["ID"].isin(matched_idx["ID_CUST"])]
    print("End of DDM")
    return matched_idx, cust_df

# Main function - starting point of the script
if __name__ == "__main__":
    print("Data Load started: " + str(datetime.now()))
    cwd = os.getcwd()
    srcFolder = cwd + r"\\Source\\"
    print("CUSTOMER MONITORING LIST")
    custFile = r"00_List_Customer_Monitoring.csv"
    df_cust = extractSource(srcFolder, custFile)
    print("NEGATIVE LIST")
    negFile = r"01a_List_Negative.csv"
    df_neg = extractSource(srcFolder, negFile)
    print("POSITIVE LIST")
    posFile = r"01b_List_Positive.csv"
    df_pos = extractSource(srcFolder, posFile)
    print("Data Load Completed: " + str(datetime.now()))
    print(len(df_cust), len(df_neg), len(df_pos))
    print("Data Preprocessing Started: " + str(datetime.now()))
    print("CUSTOMER MONITORING LIST")
    df_cust = dataPreprocessing(df_cust)
    print("NEGATIVE LIST")
    df_neg = dataPreprocessing1(df_neg)
    print("POSITIVE LIST")
    df_pos = dataPreprocessing1(df_pos)
    print("Data Pre-processing Completed!!! " + str(datetime.now()))
    print(len(df_cust), len(df_neg), len(df_pos))
    print("Data load of preprocessed file started: " + str(datetime.now()))
    intFileDir = cwd + r"\\IntermediateFiles\\Preprocessed\\"
    ppCustFile = "PP_" + custFile
    IntermediateFiles(intFileDir, ppCustFile, df_cust)
    ppNegFile = "PP_" + negFile
    IntermediateFiles(intFileDir, ppNegFile, df_neg)
    ppPosFile = "PP_" + posFile
    IntermediateFiles(intFileDir, ppPosFile, df_pos)
    print("Data load of preprocessed file completed!!! " + str(datetime.now()))
    print("Determistics Data Match started: " + str(datetime.now()))
    index_df, df_cust = DDM(df_cust, df_neg, df_pos)
    print("Determistics Data Match completed!!! " + str(datetime.now()))
    print(len(df_cust), len(df_neg), len(df_pos))
    print("Data load of DDM file started: " + str(datetime.now()))
    intFileDir = cwd + r"\\IntermediateFiles\\DDM\\"
    DDMFile = r"DDM.csv"
    peCustFile = "DDM_" + custFile
    MatchedFiles(intFileDir, DDMFile, index_df)
    MatchedFiles(intFileDir, peCustFile, df_cust)
    MatchedFiles(intFileDir, custFile, df_cust)
    IntermediateFiles(intFileDir, negFile, df_neg)
    IntermediateFiles(intFileDir, posFile, df_pos)
    print("Data load of DDM file completed!!! " + str(datetime.now()))


"""
Parser for a Type / Class Tree

After a call to build_type_hirarchy, the types are put into a hirarchy, then
the class parsing starts.

The main features this thing provides is to detect type derived superclasses and
rearanges of them as new classes within the defining one.

        class Foo(Bar, Baz) ->
        class Foo(Bar):
                class Baz(Baz)

where in this not yet built when doing this,
we are in its metacls __new__ yet.

later Foo.Baz can be getting special settings without influencing the other
Baz classes.

This allows for very small definition specs.
"""

# TODO: Check this: https://www.stat.washington.edu/~hoytak/code/treedict/

# -------------------------------------------------------------- Build the Tree
# py 2 version. py2 syntax is compat with py3, so py3 can override
# cls_with_meta
import os
import sys

# from tree_builder.py2 import cls_with_meta
from collections import OrderedDict

def sys_path_insert(fn):
    """ helper, often required in SPECs, which does import us only"""
    sys.path.insert(0, os.path.abspath(os.path.dirname(fn)))

try:
    is_str = lambda s, t=str: isinstance(s, t)
except:
    is_str = lambda s: isinstance(s, (str, bytes))  # v3

try:
    # have?
    from theming.pretty_print import dict_to_txt

    class ODict(OrderedDict):
        def __repr__(self):
            return dict_to_txt(self)

except:
    import json

    ODict = OrderedDict
    dict_to_txt = lambda a, **kw: json.dumps(a, indent=4, default=str)

def pretty(o):
    from tree_builder.render import to_dict

    return dict_to_txt(to_dict(o), fmt={'ax': 1})

class ObjKeyDict(dict):
    set = dict.__setitem__

have_tty = lambda: sys.stdin.isatty() and sys.stdout.isatty()

class UserInput:
    """Avoids repeatedly asking same questions over repeated spec runs
    Stores answered questions in DA_DIR/.stored_input
    """

    fn = lambda: os.environ.get('DA_DIR', 'no_da_dir') + '/.stored_input'
    # when loaded (oncee from fn) then this keeps all previous user input:
    # inp e.g. = {'env': {'registration_token': 'foo'}}
    inp = None

    @classmethod
    def get_stored_if_build(ui, val_pth):
        """We do not want to prevent parsing of spec if env vars are missing.

        Only when its a spec build we return None, resulting in questions on the tty
        (or failures if non interactive)
        """

        if ui.inp is None:
            ui.load()
        m = ui.inp
        # val_pth e.g. ['env', 'reg_token']:
        while val_pth:
            m = m.get(val_pth.pop(0)) or {}
        if m != {}:
            return m

        # are we a spec build? If not return a placeholder:
        from devapp.app import app_func
        from devapp.spec.build import build

        if app_func(inner=True) != build:
            # placholder, this is not a spec build:
            return 'Input Value(%s)' % '.'.join(val_pth)

    @classmethod
    def load(ui):
        if not os.path.exists(ui.fn()):
            ui.inp = {}
            return
        with open(ui.fn()) as fd:
            ui.inp = json.loads(fd.read())

    @classmethod
    def store(ui):
        """called by successfull spec.build, storing last inputs"""
        if ui.inp:
            with open(ui.fn(), 'w') as fd:
                fd.write(json.dumps(ui.inp, indent=2))

def Env(key, dflt=None, suggest=None):
    """env access bridge for specs - here we can in a build process
    parametrize the constructor query session
    """
    import os

    res = os.environ.get(key, dflt)
    if res is None:
        res = UserInput.get_stored_if_build(['env', key])
        if res is None:
            if have_tty():
                print('Require value for: %s' % key)
                print(
                    'Note: Specifying a command (format: json) will delay value acquisition to process start time and prevent putting the value into process environ.'
                )
                print('Example: ["cat", "/tmp/mypw"]')
                if suggest:
                    print('Suggested (enter to confirm): %s' % suggest)
                res = UserInput.inp.setdefault('env', {})[key] = (
                    input('Value for %s: ' % key) or suggest
                )

            if not res:
                raise Exception('Required but not provided: $%s' % key)

    return res

def mro(c):
    return c.mro()

class Reg:
    """
    stores global registries,to allow isolated load of more specs in one
    process (clients could pass maps as 'env', then not using os.environ)
    """

    def __init__(self, env):
        self.env = env
        self.mixins = {}
        self.hirarchy = []
        self.cls_counter = 0
        self.hir_postconfigured = 2
        self.hirarchy_built = False
        self.classes = ObjKeyDict({})

R = Reg(os.environ)

def reinit(env=os.environ):
    global R
    R = Reg(env)
    return R

def load_isolated(spec, env=None):
    """execing a spec file (as string), should not collide
    only problem I see is that T's _allowed childs could differ,
    have to make an exception for it

    Note: R
    """
    global R
    if env is None:
        env = os.environ
    R = Reg(env=env)
    R.hirarchy.append(T)
    exec(spec, globals())
    return R.Root

pystring = str

def out(*a):
    pass
    # print ' '.join([str(k) for k in a])

class MixinTypeMetaClass(type):
    def __new__(cls, name, bases, attrs):
        ncls = type.__new__(cls, name, bases, attrs)
        # ncls = super(MixinTypeMetaClass, cls).__new__(cls, name, bases, attrs)
        R.mixins.setdefault(ncls, [])
        return ncls

def out_clses(cs):
    r = ''
    for c in cs:
        r += ', name: ' + c.__name__
    return 'Classes: ' + r[1:]

class MT(object, metaclass=MixinTypeMetaClass):
    pass

def build_type_hirarchy(root):
    """ e.g. Project"""
    while 1:
        if R.hirarchy[0] != root:
            R.hirarchy.pop(0)
        else:
            break
    # that hack now is necessary since transcypt requires the base with the
    # metaclass as FIRST base. Our Py2/Py3 compliance hack (cls_with_meta)
    # inserts an intermediate class - with a constructor which would overwrite
    # the actual wanted one (and we don't have super in Transcrypt, so that we
    # could fix it in the intermediate's __init__).
    # In short: We have to bypass __init__ of the intermediate helper base class:
    pfh = globals().get('post_fix_hirarchy')
    pfh() if pfh else None
    out('have hirarchy', [n.__name__ for n in R.hirarchy])
    R.hirarchy_built = True  # close hirachy, following are specific classes

class HirarchyErr(Exception):
    pass

def check_allow_add(cls, sub):
    if cls.__name__ == 'xProject':
        import pdb

        pdb.set_trace()
    try:
        if not sub._hirarchy == cls._hirarchy + 1:
            for c in cls._allowed_childs:
                if sub.type == c.type:
                    return
            raise HirarchyErr("Can't add %s directly within %s" % (sub, cls))
    except Exception as ex:
        print(ex)
        import pdb

        pdb.set_trace()

# ----------------------------------------------------------------------------
def cast_str(v):
    for t in float, int:
        try:
            return t(v)
        except:
            pass
    if v.lower() == 'true':
        return True
    if v.lower() == 'false':
        return False
    return v

nil = '\x01'

class ItemGetterMetaType(type):
    """with env to attr mapping (for inheritance. => DONT do a dict here)"""

    def __getitem__(cls, k, dflt=None):
        if isinstance(k, str) and k.startswith('$'):
            'could be env var pointing to our var OR env var'
            # we have it directly?:
            k = k[1:]
            ck = getattr(cls, '_env_%s' % k, nil)
            if ck != nil:
                return ck
            k = R.env.get(k, nil)
            if k == nil:
                return dflt
        return getattr(cls, k, dflt)

    def __setitem__(cls, k, v):
        # support cls['foo,bar'] = 'bar', 'foo'
        if isinstance(k, str):
            if k.startswith('$'):
                setattr(cls, '_env_%s' % k[1:], v)
            else:
                setattr(cls, k, v)
        else:
            # n.Consul['is_server', 'port_http', 'port_dns'] = True, 8880, 53
            for k, v in zip(k, v):
                setattr(cls, k, v)

    def get(cls, k, dflt=None):
        return cls.__getitem__(k, dflt)

    def get_env(cls):
        r = ODict()
        for k in [i for i in dir(cls) if i.startswith('_env_')]:
            r[k[5:]] = getattr(cls, k)
        return r

class Group(object, metaclass=ItemGetterMetaType):
    pass

class TypeMetaClass(ItemGetterMetaType):
    def __new__(cls, name, bases, attrs):
        try:
            attrs = dict(attrs)  # 0
        except:
            pass
        attrs['name'] = name
        if not bases:
            bases = (object,)
        b0, orig_bases = bases[0], None
        if b0 in R.mixins and not R.hirarchy_built:
            raise Exception("First Base Class can't be a mixin", b0)

        if R.hirarchy_built:
            out('adding a class', name, bases)
            if not '_parent' in attrs and not hasattr(b0, '_parent'):
                while not b0 in R.hirarchy:
                    # a helper class (like class Node(Node, Role.Foo) and inner
                    # Node is
                    # R.hirarchy type -> act as if we'd defined all:
                    inh = R.classes.get(b0)
                    b0, bases = inh[0], inh + bases[1:]
                attrs['type'] = b0.__name__
                attrs['_cls'] = name
                attrs['_hir'] = R.hirarchy.index(b0)
                out('adding {0}({1})'.format(name, b0.__name__))
                bases, orig_bases, mxins, attrs = set_bases_as_attrs(
                    name, b0, bases, attrs
                )
        else:
            # out ('Registering type', name)
            attrs['type'] = name
            attrs.setdefault('_allowed_childs', [])

        R.cls_counter += 1
        attrs['_id'] = R.cls_counter
        # ncls = super(TypeMetaClass, cls).__new__(cls, name, bases, attrs)
        ncls = type.__new__(cls, name, bases, attrs)
        if orig_bases:
            out(ncls.__name__, 'orig_bases', out_clses(orig_bases))
        if R.hirarchy_built:
            # if name == 'NB':
            #    import pdb; pdb.set_trace()
            if '_parent' in attrs:
                out('returning', ncls.__name__)
                return ncls
        if orig_bases:
            R.classes.set(ncls, orig_bases)
            for m in mxins:
                if hasattr(m, 'func_name'):
                    m(ncls)
                else:
                    out(
                        'Adding %s to %s %s'
                        % (name, R.mixins[m]['orig'].__name__, m.__name__)
                    )
                    R.mixins[m]['members'].append(ncls)
            out('built class', ncls.__name__, out_clses(R.classes.get(ncls)))
            return ncls
        if not R.hirarchy_built:  # and len(R.hirarchy):
            if R.hirarchy:
                R.hirarchy[-1:][0]._allowed_childs.append(ncls)
            ncls._hirarchy = len(R.hirarchy)
        R.hirarchy.append(ncls)
        return ncls

    def __repr__(cls):
        p = getattr(cls, '_parent', None)
        if p:
            # an object within an exploded tree:
            return fidstr(cls)

        if isinstance(p, pystring):
            p = 'prebound(%s)' % p
        elif isinstance(p, type(None)):
            p = 'type'
        else:
            if not getattr(p, '_parent', None):
                p = ''
            else:
                p = ' [%s]' % p
        return '%s.%s.%s%s' % (
            cls.type,
            getattr(cls, 'name', ''),
            getattr(cls, '_id', 0),
            p,
        )

class T(object, metaclass=TypeMetaClass):
    descr = None

    @classmethod
    def all(cls, type, **kw):
        def filter(o, kw):
            for k, v in kw:
                if getattr(o, k, None) == v:
                    return True

        # depends on root._all_<type>s to be set in build time:
        res = [
            o for o in getattr(cls._parents[0], '_all_%ss' % type) if cls in o._parents
        ]
        if not kw:
            return res
        kw = list(kw.items())
        return [o for o in res if list(filter(o, kw))]

class AXCTreeObj(T):
    pass

# T = cls_with_meta(TypeMetaClass, {'descr': None})

def has_parent(c):
    return getattr(c, '_parent', None)

def descr(c):
    'up the mro to find any descr or __doc__'
    d = []
    for b in mro(c):
        if b in R.hirarchy:  # this is a type not a class
            break
        dd = getattr(b, 'descr', None)
        if dd and not dd in d:
            d.append(dd)
        else:
            dd = b.__doc__
            if dd and not dd in d:
                d.append(dd)
    return '.'.join(d)

def has_typ(c):
    try:
        return hasattr(c, 'type')
    except:
        pass

def add_to(*dests, **kw):
    'convenience wrapper for add_cls - from spec'
    dests = _list(dests)
    for parent in dests:
        for k, childs in list(kw.items()):
            for child in _list(childs):
                if not child.type == k:
                    raise Exception("Can't add: %s is no %s" % (child, k))
                add_cls(parent, child)

def add_cls(parent, child):
    """ called from the spec """
    check_allow_add(parent, child)
    out('    adding %s to %s' % (child, parent))
    # R.cls_counter += 1
    new_sub = type.__new__(
        TypeMetaClass,
        child.__name__,
        (child,),
        {'_parent': parent, '_id': R.cls_counter},
    )
    # new_sub = type(child.__name__, (child,), {'_parent': parent})
    setattr(parent, child.__name__, new_sub)
    return new_sub

def set_bases_as_attrs(name, b0, orig_bases, attrs):
    """ type hirarchy is built at this point
    Our job now is to pop those bases (and add them as subclasses which
    registered already).
    If registered as mixin then we create new R.mixins or resolve - see below
    """

    def add_local(b0, name, sub, parent_attrs, lms=None):
        """
        """
        msg = 'Error trying to add %s(%s) to %s: ' % (sub, name, b0)
        try:
            check_allow_add(b0, sub)
        except HirarchyErr:
            if not lms:
                raise Exception(msg + ' no mro based sub')
            for i in range(len(lms), 0, -1):
                l = lms[i - 1]
                try:
                    t = add_cls(l, sub)
                    return t
                except HirarchyErr:
                    lms.pop(i - 1)
            raise Exception(msg + 'check_allow_add failed')
        bn = sub.__name__
        out('    adding {0} to {1}'.format(sub.__name__, name))
        # parent_attrs[bn] = t = type(bn, (sub,), {'_parent': name})
        # won't work with R.mixins, is basically creating a class also for python=0:
        parent_attrs[bn] = t = type.__new__(
            TypeMetaClass, bn, (sub,), {'_parent': name}
        )
        parent_attrs.setdefault('_from_mro', []).append(t)
        out('parent_attrs', len(parent_attrs), list(parent_attrs.keys()))
        return t

    mxins = []

    def new_mixin(ncls, b0, orig_mixin):
        if ncls in R.mixins:
            raise Exception('mixin %s already defined' % ncls)
        R.mixins[ncls] = {'members_cls': b0, 'members': [], 'orig': orig_mixin}

    mx = R.mixins.get(b0)
    if mx:
        # class NBI(AXESS) where AXESS is a group
        # -> we set group='AXESS' and b0 to Service
        mxins.append(b0)  # this will add this new cls to b0's members,
        # we don't have it yet
        attrs[mx['orig'].__name__] = b0.__name__
        b0 = mx['members_cls']
        attrs['type'] = b0.type

    new_bases = [b0]
    lms = []  # last added mro based sub
    out('orig_bases', out_clses(orig_bases))
    out('orig_bases1', out_clses(orig_bases[1:]))
    for b in orig_bases[1:]:
        out('base', b.__name__)
        if b in R.mixins:
            # a mixin type in the bases: class AXESS(Service, Group)
            # -> we'll create a NEW mixin, but with a members_type,
            # e.g. Service
            # if such a members_cls is alredy set, then we resolve
            # e.g. class Foo(Role, AXESS), AXESS is such a mx-> resolve
            mx = R.mixins[b]
            if 'members_cls' in mx:
                out('    resolving all members of %s' % b.__name__)
                for m in mx['members']:
                    # lms = [add_local(b0, name, sub=m, parent_attrs=attrs)]
                    lms = [add_local(b0, name, m, attrs)]
            else:
                # create new
                mxins.append(lambda ncls: new_mixin(ncls, b0, b))
        elif b in R.hirarchy:
            out(
                'you should instantiate your inner classes from R.hirarchy type classes'
            )
            import pdb

            pdb.set_trace()
        elif R.classes.get(b):
            # lms.append(add_local(b0, name, sub=b, parent_attrs=attrs, lms=lms))
            lms.append(add_local(b0, name, b, attrs, lms))
        else:
            # normal superclass:
            new_bases.append(b)
    return tuple(new_bases), orig_bases, mxins, attrs

def allow(*what, **on):
    # def allow(what):
    # 'allow((this, that), (onthis, onthat))'
    # what, on = what[0], what[1]
    on = on['on'] if isinstance(on['on'], (list, tuple)) else (on['on'],)
    for o in on:
        for w in what:
            o._allowed_childs.append(w)

def _list(s):
    if s and isinstance(s, tuple) and isinstance(s[0], (tuple, list)):
        s = s[0]
    return list(s) if isinstance(s, (list, tuple)) else [s]

# ------------------------------------------------------------ Tools
def lazy(d, key, func_if_missing, args):
    """ a lazy setdefault effectively. Often useful"""
    if key in d:
        return d[key]
    d[key] = func_if_missing(*args)
    return d[key]

# ------------------------------------------------------------ Explore the Tree
def props(c):
    # keys = [k for k in dir(c) if not k.startswith('_')]
    l = [(k, getattr(c, k)) for k in dir(c) if not k.startswith('_')]
    return [i for i in l if not hasattr(i[1], 'func_name') and not i[0] == 'all']

simple_types = (float, int, bool, pystring)

def simple_props(c):
    res = [(k, v) for k, v in props(c) if isinstance(v, simple_types)]
    return res

struct_types = (float, int, bool, pystring, dict, list, tuple)

def struct_props(c):
    'still serializable but with dicts and lists'
    res = [(k, v) for k, v in props(c) if isinstance(v, struct_types)]
    return res

def walk_tree(cur, pre=None, post=None, match=None, res=None, cfg=None, level=0):
    """
    Recursing the class tree, starting from a given root node (cur).
    the result is mutated
    pre, post, match = callables
    """
    if not match:
        match = has_typ
    if res is None:
        res = []
    level += 1
    if pre:
        pre(cur, res, level, cfg)
    if not cfg or cfg.get('stop_recursion') != cur.type:
        for k in ordered_childs(cur, match):
            walk_tree(k, pre, post, match, res, cfg, level=level)
    if post:
        post(cur, res, level, cfg)
    return res

def setup(root):
    """ configure all prebound classes """
    cfg = {'hashes': [], 'root': root}
    out('walking tree', root.__name__)
    root._level = root._hir = 0
    root._parents = [root]

    # register (API): _all_<type>s, e.g. _all_clusters
    def type_list_n(c):
        return '_all_' + c.type.lower() + 's'

    for h in R.hirarchy:
        setattr(root, type_list_n(h), [])

    def pre(cur, res, level, cfg):
        root = cur._root = cfg['root']
        getattr(root, type_list_n(cur)).append(cur)
        if cur != root:
            cur._parents = list(cur._parent._parents)
            # c._cluster -> Cluster.foo
            for c in cur._parents:
                setattr(cur, '_%s' % c.type.lower(), c)

            cur._parents.append(cur)

        return unordered_childs(cur, has_typ, level)

    walk_tree(root, pre=pre, post=None, match=has_typ, cfg=cfg)
    R.hirarchy_built = R.hir_postconfigured
    root.descr = descr(root)
    R.Root = root
    root._hirarchy_list = R.hirarchy
    return root

create_tree = setup

def scale(c, into, n):
    for i0 in range(n):
        i = i0 + 1
        name = c.name + str(i)
        t = type(name, (c,), {})
        t.__doc__ = c.__doc__
        into[name] = t

def unordered_childs(parent, match, level=None):

    # if parent.name == 'NB':
    #    import pdb; pdb.set_trace()
    c = getattr(parent, '_childs', None)
    if c:
        return c
    childs = [(k, v) for k, v in props(parent) if match(v)]
    if not childs:
        _childs = childs
    else:
        # we sort class childs by mro order, e.g. NB(N, A, B, C) ->
        # _childs = [A, B, C] even if C._id < A._id since defined earlier:
        try:
            pmro = parent._from_mro
            childs.sort(key=lambda x: pmro.index(x[1]))
        except Exception as ex:
            # some parents have no _from_mro
            childs.sort(key=lambda x: x[1]._id)

        _childs = []
        for k, v in childs:
            # v = TypeMetaClass.__new__(TypeMetaClass, v.__name__, (v,), {})
            R.cls_counter += 1
            # v = TypeMetaClass.__new__(TypeMetaClass, v.__name__, (v,), {'_id': R.cls_counter})
            # if not isinstance(v._parent, basestring):
            #    import pdb; pdb.set_trace()
            #    v = v.__bases__[0]
            v = type.__new__(
                TypeMetaClass,
                v.__name__,
                (v,),
                {'_id': R.cls_counter, '_level': level, '_parent': parent},
            )
            setattr(parent, k, v)
            v.descr = descr(v)
            _childs.append(v)
    parent._childs = _childs
    # parent._childs_ordered = False
    return _childs

# spec API Functions:
def ordered_childs(parent, match):
    # match is always the same,thats why we can store the result:
    return unordered_childs(parent, match)

def full_id(c, sep=None):
    res = [o.name for o in c._parents]
    return sep.join(res) if sep else res

def assign(key, objs, vals):
    [setattr(_[0], key, _[1]) for _ in zip(objs, vals)]

def setter(key):
    def set(key, v, *o):
        for obj in o:
            setattr(obj, key, v)

    return lambda v, *o: set(key, v, *o)

# full id as string:
fidstr = lambda cls: '.'.join([c.name for c in cls._parents])

def get_spec_root(spec):
    for k in dir(spec):
        root = getattr(getattr(spec, k), '_root', None)
        if root:
            return root
    die('Could not determine spec root', spec)

if __name__ == '__main__':

    class Project(T):
        ''

    class Cluster(T):
        ''

    build_type_hirarchy(root=Project)

    class cluster(Group):
        class A(Cluster):
            'A cluster'

        class B(Cluster):
            'B cluster'

    class MyP(Project, cluster['A'], cluster.B):
        'Pro'

    setup(MyP)
    print(('have ', dir(MyP)))

{'app_func': 'devapp.app.app_func', 'build': 'devapp.spec.build.build'}.get_stored_if_build

{'app_func': 'devapp.app.app_func', 'build': 'devapp.spec.build.build'}.inp.setdefault


import datetime
from time import sleep
nada = ("\033[0;0m")
vermelho = ("\033[0;31m")
amarelo = ("\033[0;33m")
verde = ("\033[0;32m")
ciano = ("\033[0;36m")
azul = ("\033[0;34m")
roxo = ("\033[0;35m")
print("CONTAGEM REGRESSIVA PARA {}".format(datetime.date.today().year + 1))
for c in range(10, 0, -1):
    print(c)
    sleep(1)
print("{}F{}{}E{}{}L{}{}I{}{}Z{} {}A{}{}N{}{}O{} {}N{}{}O{}{}V{}{}O{}".format(vermelho, nada, amarelo, nada, verde, nada, ciano, nada, azul, nada, roxo, nada, vermelho, nada, amarelo, nada, verde, nada, ciano, nada, azul, nada, roxo, nada))


import numpy as np
import matplotlib.pyplot as plt
import pickle

from CalculatePolarization3 import comb_plot

with open("pol3_matrix.pickle", "rb") as f:
    data = pickle.load(f)

frequency = data['freq']
pol3_mat = data['pol3_matrix'].real

def plot_pol3():

    fig, axes = plt.subplots(nrows=2, ncols=1)
    comb_plot(frequency, pol3_mat[:, 0].real, axes[0], 'k')
    comb_plot(frequency, pol3_mat[:, 1].real, axes[0], 'b')
    comb_plot(frequency, pol3_mat[:, 2].real, axes[0], 'r')
    axes[0].plot(frequency, np.zeros_like(frequency), 'k')

    comb_plot(frequency, pol3_mat[:, 0].imag, axes[1], 'k')
    comb_plot(frequency, pol3_mat[:, 1].imag, axes[1], 'b')
    comb_plot(frequency, pol3_mat[:, 2].imag, axes[1], 'r')
    axes[1].plot(frequency, np.zeros_like(frequency), 'k')

N_comb = 10
rows, cols = pol3_mat.shape
print(rows, cols)

sigma = 0.16
x = np.linspace(0., 1., rows)
x = np.linspace(0., 1., 2 * N_comb)

colors = ['r', 'b', 'k']

det = np.empty((3, 3))
det_noise = np.empty((3, 3))

het_field = np.empty([cols, 2 * N_comb])

def pol2_basis_change_freq2comb():
    """
    CALCULATES BASIS CHANGE MATRIX FROM FREQUENCY BASIS TO COMB BASIS: DIM - (N_frequency x N_comb)
    :return: POLARIZATION MATRIX IN THE COMB BASIS: DIM - (N_frequency x N_molecules)
    """
    omega_cb = frequency[:, np.newaxis]
    del_omega1_cb = 1 * np.arange(- N_comb, N_comb)
    comb_omega1_cb = del_omega1_cb[np.newaxis, :]

    freq2comb_basis = 5e-6 / (
            (omega_cb - 0.07 - 0.03 - comb_omega1_cb) ** 2 + 5e-6 ** 2
    )
    freq2comb_basis /= freq2comb_basis.max()
    pol3_comb_matrix = freq2comb_basis.T.dot(pol3_mat)
    return pol3_comb_matrix

for i in range(3):
    Q_mat, R_mat = np.linalg.qr(np.delete(pol2_basis_change_freq2comb(), i, 1), mode='complete')

    print(Q_mat.real)
    gaussian = np.sin(2 * (1+i) * np.pi * (x - x.min()) / x.max()) * np.exp(-(x - 0.5) ** 2 / (2 * sigma ** 2))
    het_field[i] = sum(q * np.vdot(q, gaussian) for q in Q_mat[:, 3:].T)
    het_field[i] = sum(q for q in Q_mat[:, 3:].T)

fig, axes = plt.subplots(nrows=2, ncols=3)

for i in range(3):
    comb_plot(frequency, pol3_mat[:, i] / pol3_mat.max(), axes[0, i], colors[i], alpha=0.75, linewidth=2.)
    comb_plot(np.arange(20), het_field[i], axes[1, i], colors[i], alpha=0.75, linewidth=2.)

    axes[0, i].set_ylim(-0.025, 1.05)
    axes[1, i].set_ylim(-3.525, 1.625)
plt.show()