# Copyright 2011, Dan Gindikin <dgindikin@gmail.com>
# Copyright 2012, Jono Finger <jono@foodnotblogs.com>
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
"""textmap is a Python3 plugin for Gedit.

This plugin is used for displaying a 10,000 foot view of your code in the
sidebar.  It is based on Dan Gindikin's original code, but has been updated
to work with Gedit 3.

To install this plugin copy the files into ~/.local/share/gedit/plugins/ and
restart Gedit. Then activate the plugin through the preferences.

"""

__version__ = "0.2 beta - gtk3"

import cairo

from gi.repository import Gtk, GdkPixbuf, Gdk, GtkSource, Gio, Gedit, GObject

def document_lines(document):
    if not document:
        return None

    return document.get_property('text').split('\n')

def visible_lines_top_bottom(geditwin):
    view = geditwin.get_active_view()
    rect = view.get_visible_rect()
    topiter = view.get_line_at_y(rect.y)[0]
    botiter = view.get_line_at_y(rect.y + rect.height)[0]
    return topiter.get_line(), botiter.get_line()

def is_dark(red, green, blue):
    """ This function determines if an rgb value is dark or not.
    """
    if red + green + blue < 1.5:
        return True
    else:
        return False

def darken(fraction, r, g, b):
    return (
        r - fraction * r,
        g - fraction * g,
        b - fraction * b
    )

def lighten(fraction, r, g, b):
    return (
        r + (1 - r) * fraction,
        g + (1 - g) * fraction,
        b + (1 - b) * fraction
    )

def queue_refresh(textmapview):
    try:
        win = textmapview.darea.get_window()
    except AttributeError:
        win = textmapview.darea.window
    if win:
        textmapview.darea.queue_draw_area(0, 0, win.get_width(),
                                          win.get_height())

def str2rgb(s):
    assert s.startswith('#') and len(s) == 7, ('not a color string', s)
    r = int(s[1:3], 16) / 256.
    g = int(s[3:5], 16) / 256.
    b = int(s[5:7], 16) / 256.
    return r, g, b

class TextmapView(Gtk.VBox):
    def __init__(self, geditwin):
        Gtk.VBox.__init__(self)

        self.geditwin = geditwin

        self.geditwin.connect("active-tab-changed", self.tab_changed)
        self.geditwin.connect("tab-added", self.tab_added)

        darea = Gtk.DrawingArea()
        darea.connect("draw", self.draw)

        darea.add_events(Gdk.EventMask.BUTTON_PRESS_MASK)
        darea.connect("button-press-event", self.button_press)
        darea.connect("scroll-event", self.on_darea_scroll_event)

        darea.add_events(Gdk.EventMask.POINTER_MOTION_MASK)
        darea.connect("motion-notify-event", self.on_darea_motion_notify_event)

        self.pack_start(darea, True, True, 0)

        self.show_all()

        self.topL = None
        self.botL = None
        self.scale = 4  # TODO: set this smartly somehow
        self.darea = darea

        self.winHeight = 0
        self.winWidth = 0
        self.linePixelHeight = 0

        self.currentDoc = None
        self.currentView = None

    def tab_added(self, window, tab):
        self.currentView = tab.get_view()
        self.currentDoc = tab.get_document()

        self.currentDoc.connect(
            'changed',
            self.on_doc_changed
        )

        self.currentView.get_vadjustment().connect(
            'value-changed',
            self.on_vadjustment_changed
        )

# TODO: make sure value-changed is not conflicting with darea move events

    def tab_changed(self, window, event):
        self.currentView = self.geditwin.get_active_view()
        self.currentDoc = self.geditwin.get_active_tab().get_document()

        self.lines = document_lines(self.currentDoc)
        queue_refresh(self)

    def on_doc_changed(self, buffer):
        self.lines = document_lines(self.currentDoc)
        queue_refresh(self)

    def on_vadjustment_changed(self, adjustment):
        queue_refresh(self)

    def on_darea_motion_notify_event(self, widget, event):
        "used for clicking and dragging"

        if event.state & Gdk.ModifierType.BUTTON1_MASK:
            self.scroll_from_y_mouse_pos(event.y)

    def on_darea_scroll_event(self, widget, event):

# TODO: match this to self.currentView.get_vadjustment().get_page_size()
        pagesize = 12
        topL, botL = visible_lines_top_bottom(self.geditwin)
        if event.direction == Gdk.ScrollDirection.UP and topL > pagesize:
            newI = topL - pagesize
        elif event.direction == Gdk.ScrollDirection.DOWN:
            newI = botL + pagesize
        else:
            return

        self.currentView.scroll_to_iter(
            self.currentDoc.get_iter_at_line_index(newI, 0), 0, False, 0, 0)

        queue_refresh(self)

    def scroll_from_y_mouse_pos(self, y):

        self.currentView.scroll_to_iter(
            self.currentDoc.get_iter_at_line_index(
                int(
                    (
                        len(self.lines) + (self.botL - self.topL)
                    ) * y / self.winHeight
                ),
                0
            ),
            0,
            True,
            0,
            0.5
        )
        queue_refresh(self)

    def button_press(self, widget, event):
        self.scroll_from_y_mouse_pos(event.y)

    def draw(self, widget, cr):

        if not self.currentDoc or not self.currentView:  # nothing open yet
            return

        bg = (0, 0, 0)
        fg = (1, 1, 1)
        try:
            style = self.currentDoc.get_style_scheme().get_style('text')
            # there is a style scheme, but it does not specify default
            if style is None:
                bg = (1, 1, 1)
                fg = (0, 0, 0)
            else:
                fg, bg = map(
                    str2rgb,
                    style.get_properties('foreground', 'background'))

        except:
            pass  # probably an older version of gedit, no style schemes yet

        try:
            win = widget.get_window()
        except AttributeError:
            win = widget.window

        cr = win.cairo_create()

        self.winHeight = win.get_height()
        self.winWidth = win.get_width()

        cr.push_group()

        # draw the background
        cr.set_source_rgb(*bg)
        cr.move_to(0, 0)
        cr.rectangle(0, 0, self.winWidth, self.winHeight)
        cr.fill()
        cr.move_to(0, 0)

        if not self.lines:
            return

        # draw the text
        cr.select_font_face('monospace', cairo.FONT_SLANT_NORMAL,
                            cairo.FONT_WEIGHT_NORMAL)
        cr.set_font_size(self.scale)

        if self.linePixelHeight == 0:
            self.linePixelHeight = cr.text_extents("L")[
                3]  # height # TODO: make this more global

        self.topL, self.botL = visible_lines_top_bottom(self.geditwin)

        if is_dark(*fg):
            faded_fg = lighten(.5, *fg)
        else:
            faded_fg = darken(.5, *fg)

        cr.set_source_rgb(*fg)

        textViewLines = int(self.winHeight / self.linePixelHeight)

        firstLine = self.topL - int(
            (textViewLines - (self.botL - self.topL)) *
            float(self.topL) / float(len(self.lines))
        )

        if firstLine < 0:
            firstLine = 0

        lastLine = firstLine + textViewLines
        if lastLine > len(self.lines):
            lastLine = len(self.lines)

        sofarH = 0

        for i in range(firstLine, lastLine, 1):
            cr.show_text(self.lines[i])
            sofarH += self.linePixelHeight
            cr.move_to(0, sofarH)

        cr.set_source(cr.pop_group())
        cr.rectangle(0, 0, self.winWidth, self.winHeight)
        cr.fill()

        # draw the scrollbar
        topY = (self.topL - firstLine) * self.linePixelHeight

        if topY < 0:
            topY = 0

        botY = topY + self.linePixelHeight * (self.botL - self.topL)
        # TODO: handle case   if botY > ?

        cr.set_source_rgba(.3, .3, .3, .35)
        cr.rectangle(0, topY, self.winWidth, botY - topY)
        cr.fill()
        cr.stroke()

class TextmapWindowHelper:
    def __init__(self, plugin, window):
        self.window = window
        self.plugin = plugin

        panel = self.window.get_side_panel()
        image = Gtk.Image()
        image.set_from_stock(Gtk.STOCK_DND_MULTIPLE, Gtk.IconSize.BUTTON)
        self.textmapview = TextmapView(self.window)
        self.ui_id = panel.add_titled(self.textmapview, "TextMap", "textMap")

        self.panel = panel

    def deactivate(self):
        self.window = None
        self.plugin = None
        self.textmapview = None

    def update_ui(self):
        queue_refresh(self.textmapview)

class WindowActivatable(GObject.Object, Gedit.WindowActivatable):
    window = GObject.property(type=Gedit.Window)

    def __init__(self):
        GObject.Object.__init__(self)
        self._instances = {}

    def do_activate(self):
        self._instances[self.window] = TextmapWindowHelper(self, self.window)

    def do_deactivate(self):
        if self.window in self._instances:
            self._instances[self.window].deactivate()

    def update_ui(self):
        if self.window in self._instances:
            self._instances[self.window].update_ui()


"""--------------------------------------------------------------------------------------
Created on Thu May 29 18:24:20 2014

@author: salman Khan

     Conductance model based on maximum likelihood decoding - 1D stimulus

     Excitatory conductance = numerator ML decoding   = sum (rate x preferred stimulus)
     Inhibitory conductance = denominator ML decoding = sum (rate)

--------------------------------------------------------------------------------------"""
import numpy as np
import matplotlib.pyplot as plt

def main(s):

    ''' -------------------------------------------------------------
    INPUT
    --------------------------------------------------------------'''

    ''' Stimulus '''
    s_range = 150
    #TODO add some error checks
#    print ("Input Stimulus %i" %(s))

    ''' Time '''
    t_stop = 1000 # time im ms

    ''' -------------------------------------------------------------
    PRESYNAPTIC NEURON POPULATION
    --------------------------------------------------------------'''
    pre_n       = 100
    pre_rMax    = 100      # per second

    ''' Preferred Direction Vectors Distribution '''
    # Uniform Distribution
    pre_preferredStim = np.linspace(0, s_range, pre_n, endpoint = False)

    ''' Firing Rates '''
    # Gaussian Tuning Curves
    pre_var = s_range * 0.1  # width of tuning curve
    pre_fireRates = pre_rMax * np.exp( -((s - pre_preferredStim) / pre_var)**2 / 2 )

    # Plot Firing Rates
#    plt.figure("Firing Rates")
#    plt.title("Firing Rate", y = 1.05)
#    plt.stem(np.arange(0, pre_n), pre_fireRates, label = 'pre_fireRates')
#    fig2Ax = plt.gca()
#    fig2Ax.set_xlabel('Neuron Index', fontsize = 9, x = 1)
#    fig2Ax.set_xlim([0, (1.1 * pre_n)])
#    fig2Ax.set_ylabel('Firing Rate', fontsize = 9)
#    fig2Ax.set_ylim([0, (1.1 * pre_fireRates.max())])
#    fig2Ax.legend()

    ''' Spike Generation  '''
    # Poisson Firing - Commulative expontential inter-spike times method
    pre_spikeTimes = list()

    for rate in pre_fireRates:
        spikeTimes = []

        if rate != 0:
            nSpikes = np.floor(2 * t_stop * rate / 1000.0)
            spikeTimes = np.cumsum( \
            np.random.exponential(scale = 1000.0/rate, size = nSpikes) )

        pre_spikeTimes.append(spikeTimes)

    # Convert to indices and eliminate invalid entries
    for row in np.arange(len(pre_spikeTimes)):
        pre_spikeTimes[row] = \
        [ int(x) for x in pre_spikeTimes[row] if x < t_stop ]

#    # Print number of spikes for each neuron
#    for idx, item in enumerate(pre_spikeTimes):
#        print ("preferred Stimulus %i, numSpikes  %i" %((pre_preferredStim[idx]), len(item)))

    pre_out = np.zeros(shape =( pre_n, t_stop))
    for row in np.arange(len(pre_spikeTimes)):
        pre_out[row, pre_spikeTimes[row]] = 1

     # Plot the Spikes of each neuron over Time
#    fig3 = plt.figure("PostSynaptic Input Spikes")
#    plt.title("PostSynaptic Input Spikes")
#    plt.ylabel('Neuron Index')
#    if pre_n <= 25:
#        plt.yticks(np.arange(pre_n))
#    plt.xlabel('Time(ms)')
#    im = plt.imshow(pre_out, aspect = 'auto', cmap = plt.cm.binary)

    ''' -------------------------------------------------------------
    POSTSYNAPTIC NEURON
    --------------------------------------------------------------'''

    ''' Fire Rates Estimation '''
    # Average incoming spikes using a window to get instantaneous rate
    # Alpha function Window = alpha^2 * x * exp(-alpha * x)
    post_alpha = 1/16.
    post_windowLen = 100

    tmp =  np.linspace(0, post_windowLen-1, post_windowLen)
    post_window = np.power(post_alpha, 2) * tmp * np.exp(-post_alpha * tmp)

#    # Plot spike rate estimation window
#    plt.figure("Firing Rate Estimation Averaging Window")
#    plt.plot(tmp, post_window)
#    plt.title("Alpha Function Window (alpha = %f)" %post_alpha)
#    plt.xlabel('Time (ms)', fontsize = 9, x = 1)
#    plt.ylabel('Weight', fontsize = 9)

    # Convolve spikes with window to get instantaneous fire rate
    post_instRates = np.zeros(shape = pre_out.shape)
    for time in np.arange(t_stop):
        for synapse in np.arange(pre_n):
            spikes = pre_out[synapse, time::-1]

            if len(spikes) > post_windowLen:
                spikes = spikes[0:post_windowLen]

            # 1000/post_windowLen factor = for getting rates per second
            post_instRates[synapse, time] = np.convolve(spikes, post_window).sum() \
                                            * 1000 / post_windowLen

    # Override instantaneous firing rates with actual firing rates
#    post_instRates = np.repeat(pre_fireRates[:,np.newaxis], repeats = t_stop, axis = 1)

    # Plot estimated firing rates of synapses of post-neuron at some instant of time
#    plt.figure('Post Synaptic Neuron Estimated Synaptic Firing Rates')
#    plt.title('Estimated Firing Rates of Synapses at Post-Synaptic Neuron')
#    tmp_avg = post_instRates.sum(axis = 1)/t_stop
#    tmp_std = post_instRates.std(axis = 1)
#    plt.plot(np.arange(pre_n), tmp_avg, label = 'Mean')
#    plt.plot(np.arange(pre_n), tmp_avg + tmp_std, 'r.-', label = r'$1 \alpha deviation$')
#    plt.plot(np.arange(pre_n), [x if x > 0 else 0 for x in (tmp_avg - tmp_std)], 'r.-')
#    plt.legend ()

    ''' Maximum Likelihood Decoding '''
    post_sEstML = np.zeros(shape = (t_stop, 1))

    for time in np.arange(t_stop):
        rates = post_instRates[:, time]
        if rates.sum() != 0:
            post_sEstML[time] = (rates * pre_preferredStim.T).sum() / rates.sum()

#    # Plot Stimulus Estimates
#    plt.figure("Stimulus Estimates")
#    plt.title("Stimulus Estimates")
#    plt.plot (np.arange(t_stop), s*np.ones(shape = (t_stop, 1)), 'r', label = 'stimulus')
#    plt.plot( np.arange(t_stop), post_sEstML,  label = 'Estimates')

    ''' Conductance Model '''

    # Reversal Potential (mV)
    post_XE =   0.0 * np.power(10.0, -3)
    post_IE = -70.0 * np.power(10.0, -3)

    # Voltage Thresholds
    post_thReset = -80.0 * np.power(10.0, -3)
    post_thAp    = -50.0 * np.power(10.0, -3)
    post_vAp     =  50.0 * np.power(10.0, -3)  # Peak Action Potential voltage

    # Specific Capacitance
    post_cm = 10 * np.power(10.0, 3)

    post_v        = np.zeros(shape = (t_stop, 1))
    post_vSpikes  = np.ones(shape = (t_stop, 1)) * post_thAp
    post_rateCalc = np.zeros(shape = (t_stop, 1))

    # Membrane Potential
    t_lastSpike  = 0
    post_nSpikes = 0
    for time in np.arange(t_stop):

#        raw_input('Continue?')

        rates = post_instRates[:, time] / pre_rMax
        timeLastSpike = time - t_lastSpike

        XconBas = (rates * pre_preferredStim.T).sum()
        IconBas = rates.sum()/ pre_n

        ''' Modified Conductances '''
        alpha2 = 0.0
        alpha1 = 1.00
        alpha0 = 0.00

        Icon2 = alpha2*np.power(XconBas, 2.0) + alpha1*XconBas + alpha0*IconBas
        Xcon2 = XconBas - ( (post_thReset - post_IE) / (post_thReset - post_XE) ) * \
                 ( alpha2*np.power(XconBas, 2.0) + alpha1*XconBas + (alpha0 - 1)*IconBas )

#        print  ("XconBas %0.2f, IconBase = %0.2f, Xcon2 = %0.2f, Icon2 = %0.2f" \
#            %(XconBas, IconBas, Xcon2,Icon2 ))

        Xcon = Xcon2
        Icon = Icon2
#        Xcon = XconBas
#        Icon = IconBas

        # Membrane Potential Calculation
        totalCon = Xcon + Icon

        vInf = (Xcon * post_XE + Icon * post_IE) / totalCon
        vK = (Xcon * (post_thReset - post_XE) + Icon * (post_thReset - post_IE)) / totalCon
        vExp = np.exp(-totalCon * timeLastSpike / post_cm)

        post_v[time] = vInf + vK*vExp
#        print( "time=%i: post Membrane Potential=%f timeLastSpike=%i, Xcon=%f, Icon=%f, vInf=%f, vK=%f, vExp=%f" \
#            %(time, post_v[time], timeLastSpike, Xcon, Icon, vInf, vK, vExp) )

        if post_v[time] >= post_thAp:
#            post_v[time] = post_vAp
            # For greater time resolution, calculate  time of actual threshold crossing
            # (within the ms) and add the time to time of last spike.
            # Without this change, all stimuli that cause spiking within a milisecond will
            # spike at the same time resulting in the same measured spiking rate
            over = (post_v[time] - post_thAp) / (post_v[time] - post_v[time - 1])

            post_vSpikes[time] = post_vAp
            t_lastSpike = time - over
            post_nSpikes += 1.0

        ''' Firing Rate '''
        # Theoretical Firing Rate - Actual
        post_tIsi = np.log( (post_thAp*totalCon - Xcon*post_XE - Icon*post_IE ) / \
                            (post_thReset*totalCon - Xcon*post_XE - Icon*post_IE ) ) \
                    *(-post_cm / totalCon)
        post_rateCalc[time] = 1 / (post_tIsi * np.power(10.0, -3))

        # Estimate using approx ln(1+x) = x
#        post_rateCalc[time] = \
#            ( Xcon*(post_thReset - post_XE) + Icon*(post_thReset - post_IE) ) \
#            / ( post_cm *(post_thReset - post_thAp) ) * np.power(10.0, 3)

    # Measured Firing Rate - Simple Binning
    post_rateMeas = post_nSpikes / t_stop * np.power(10.0, 3)
    # average over [post_window: end], include only reliable rate estimates
    post_rateCalcMean = np.mean(post_rateCalc)
    post_rateCalcStd = np.std(post_rateCalc[post_windowLen/2:])

    print ("Stimulus %i, Post Synaptic Spikes %i, Measured Mean Rate %0.2f, Mean Calculated Rate %0.2f" \
            %(s, post_nSpikes, post_rateMeas, post_rateCalcMean) )

    #Plot post-synaptic Neuron Voltage
#    plt.figure("Post Synaptic Membrane Potential")
#    plt.title("Post Synaptic Membrane Potential, Stimulus %i, Avg Stimulus ML Est %0.2f" \
#              %(s, np.mean(post_sEstML[post_windowLen/2:]) ))
#    plt.plot(np.arange(t_stop), post_v * np.power(10.0,3))
#    plt.stem(np.arange(t_stop), post_vSpikes * np.power(10.0,3), bottom = -50, \
#             linestyle ='-', markerfmt ='.')
#
#    plt.plot(np.arange(t_stop), post_thAp * np.ones(shape = (t_stop, 1)) * np.power(10.0,3), 'r', label = 'AP Thresh')
#    ax = plt.gca()
#    ax.set_xlabel('Time(ms)', x = 1)
#    ax.set_ylabel("Membrane Potential(mV)")
#    ax.text(100, -75, 'Num Spikes %i, Mean Firing Rate = %0.2f' \
#            %(post_nSpikes, post_rateCalcMean), fontsize = 12)
#    ax.legend()
#
#    plt.figure("Post Synaptic Firing Rates")
#    plt.plot(np.arange(t_stop), post_rateCalc, label='Calculated Rate')
#    ax = plt.gca()
#    ax.set_xlabel('Time(ms)', x = 1)
#    ax.set_ylabel("Firing Rate")
#    plt.axhline(y=post_rateMeas, color='red', linewidth=2, label='Measured Rate')
#    plt.legend()

    return(post_rateMeas, post_rateCalcMean, post_rateCalcStd)

if __name__ == "__main__":

    #pyplot interactive mode - do not need to close windows after each run
    plt.ion()
    stimulus_set = np.arange(100)
#    stimulus_set = [34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45]
#    stimulus_set = [30]

    iterations = 2;

    measRate = np.zeros(shape = (len(stimulus_set), iterations))
    calcRate = np.zeros(shape = (len(stimulus_set), iterations))
    calcRateStd =  np.zeros(shape = (len(stimulus_set), iterations))

    for ii in np.arange(iterations):
        print("Iteration no %i" %ii)

        for idx, stimulus in enumerate(stimulus_set):
            measRate[idx][ii], calcRate[idx][ii], calcRateStd[idx][ii] = main(stimulus)

    plt.figure("Post Synaptic Neuron")
    plt.title("Post-Synaptic Neuron Tuning Curve (Conductance Model)")
    plt.plot(stimulus_set, measRate, label = 'Mean Firing Rate (measured)')
    plt.plot(stimulus_set, calcRate, 'r--', label = 'Mean FiringRate (calculated)' )
    ax = plt.gca()
    ax.set_xlabel('Stimulus')
    ax.set_ylabel('Firing Rate')
#    plt.legend()

    # Plot Variance Vertical Lines
    for idx, stimulus in enumerate(stimulus_set):
        plt.plot([stimulus, stimulus], [ (calcRate[idx] + calcRateStd[idx]), \
                                         max((calcRate[idx] - calcRateStd[idx]), 0) ], 'k')

    plt.figure('Post Synaptic Firing Rate Variance')
    ax = plt.gca()
    ax.set_xlabel('Stimulus')
    ax.set_ylabel('Variance')
    plt.plot(stimulus_set, calcRateStd)
    plt.title('Post Synaptic Firing Rate standard Deviation')

    plt.figure("Mean Firing Rates")
    plt.plot(stimulus_set, measRate.mean(axis=1), label ='Mean')
    plt.figure("Firing Rate standard deviation")
    plt.plot(stimulus_set, measRate.std(axis=1), label ='standard deviation')


from __future__ import print_function

from keras.models import Model, load_model
from keras.layers import Input, Embedding, LSTM, Dense
from nltk.tokenize import word_tokenize
# import tensorflow as tf

from keras import optimizers

import numpy as np
import os, json, pickle

BATCH_SIZE      = 64        # Batch size for training.
EPOCHS          = 100       # Number of EPOCHS to train for.
LATENT_DIM      = 256       # Latent dimensionality of the encoding space.
NUM_SAMPLES     = 10000     # Number of samples to train on.
                            # Path to the data txt file on disk.

class SeqToSeq:

    MODEL_PATH = os.path.abspath('../seq2seq/model')

    def __init__(self):

        # Vectorize the data.
        self.input_texts  = []
        self.target_texts = []

        self.input_vocab  = set()
        self.target_vocab = set(['<START>', '<END>'])

        file_path = self.MODEL_PATH + '/s2s.h5'

        if not os.path.isdir(self.MODEL_PATH):

            os.mkdir(self.MODEL_PATH)

        if os.path.isfile(file_path):

            self.model = load_model(file_path)

            extra_data = pickle.load(open(self.MODEL_PATH + '/extra_data.bin', 'rb'))

            self.encoder_input_data = extra_data['encoder_input_data']
            self.num_encoder_vocab  = extra_data['num_encoder_vocab']
            self.input_word_index   = extra_data['input_word_index']

            self.max_encoder_seq_length = extra_data['max_encoder_seq_length']
            self.max_decoder_seq_length = extra_data['max_decoder_seq_length']

            self.num_decoder_vocab = extra_data['num_decoder_vocab']
            self.target_word_index = extra_data['target_word_index']

            self.reverse_target_word_index = extra_data['reverse_target_word_index']

            encoder_inputs = self.model.input[0]   # input_1
            encoder_outputs, state_h_enc, state_c_enc = self.model.layers[2].output   # lstm_1
            encoder_states = [state_h_enc, state_c_enc]
            self.encoder_model  = Model(encoder_inputs, encoder_states)

            decoder_inputs = self.model.input[1]   # input_2
            decoder_state_input_h = Input(shape=(LATENT_DIM,), name='input_3')
            decoder_state_input_c = Input(shape=(LATENT_DIM,), name='input_4')
            decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]
            decoder_lstm = self.model.layers[3]

            decoder_outputs, state_h_dec, state_c_dec = decoder_lstm(
                decoder_inputs, initial_state=decoder_states_inputs)

            decoder_states  = [state_h_dec, state_c_dec]
            decoder_dense   = self.model.layers[4]
            decoder_outputs = decoder_dense(decoder_outputs)
            self.decoder_model   = Model(
                [decoder_inputs] + decoder_states_inputs,
                [decoder_outputs] + decoder_states)

    def train(self, data):

        self.__get_data(data)

        self.__create_useful_matrix()

        self.__create_one_hot_vector()

        self.__create_model()

    def __get_data(self, data):

        # Tìm kiếm text có độ dài lớn nhất trong dữ liệu question và answer

        self.max_encoder_seq_length = 0

        self.max_decoder_seq_length = 0

        for index, text in enumerate(data[: min(NUM_SAMPLES * 2, len(data) - 1)]):

            word_array = word_tokenize(text)

            if index % 2 == 0:
                self.input_texts.append(text)

                if len(word_array) > self.max_encoder_seq_length:
                    self.max_encoder_seq_length = len(word_array)

                for word in word_array:
                    if word not in self.input_vocab:
                        self.input_vocab.add(word)

            else:
                self.target_texts.append(text)

                if len(word_array) > self.max_decoder_seq_length:

                    # Chiều dài lớn nhất của tập câu tring decoder phải + thêm 2 từ <START>, <END>
                    self.max_decoder_seq_length = len(word_array) + 2

                for word in word_array:
                    if word not in self.target_vocab:
                        self.target_vocab.add(word)

    def __create_useful_matrix(self):

        self.input_vocab  = sorted(list(self.input_vocab))

        self.target_vocab = sorted(list(self.target_vocab))

        self.num_encoder_vocab = len(self.input_vocab)

        self.num_decoder_vocab = len(self.target_vocab)

        # Khởi tạo dict lưu các từ xuất hiện trong corpus cùng index của các ký tự đó

        self.input_word_index = dict(
            [(word, i) for i, word in enumerate(self.input_vocab)])

        self.target_word_index = dict(
            [(word, i) for i, word in enumerate(self.target_vocab)])

        # Tạo ma trận 0 3 chiều với kích thước
        # row = Số lượng text trong corpus
        # column = Chiều dài tối đa của text
        # depth = Số lượng ký tự độc lập trong corpus

        self.encoder_input_data = np.zeros(
            (len(self.input_texts), self.max_encoder_seq_length, self.num_encoder_vocab),
            dtype='float32')

        self.decoder_input_data = np.zeros(
            (len(self.input_texts), self.max_decoder_seq_length, self.num_decoder_vocab),
            dtype='float32')

        self.decoder_target_data = np.zeros(
            (len(self.input_texts), self.max_decoder_seq_length, self.num_decoder_vocab),
            dtype='float32')

        self.reverse_target_word_index = dict(
            (i, word) for word, i in self.target_word_index.items())

    def __create_one_hot_vector(self):

        for i, (input_text, target_text) in enumerate(zip(self.input_texts, self.target_texts)):

            input_array  = word_tokenize(input_text)

            target_array = ['<START>']  + word_tokenize(target_text) + ['<END>']

            for t, word in enumerate(input_array):

                self.encoder_input_data[i, t, self.input_word_index[word]] = 1.

            for t, word in enumerate(target_array):

                self.decoder_input_data[i, t, self.target_word_index[word]] = 1.

                if t > 0:

                    self.decoder_target_data[i, t - 1, self.target_word_index[word]] = 1.

    def __create_model(self):

        # Define an input sequence and process it.
        encoder_inputs = Input(shape=(None, self.num_encoder_vocab))
        encoder        = LSTM(LATENT_DIM, return_state=True)
        encoder_outputs, state_h, state_c = encoder(encoder_inputs)

        # We discard `encoder_outputs` and only keep the states.

        encoder_states = [state_h, state_c]

        # embedding_layer = Embedding(
        #     input_dim    = vocab_size,
        #     output_dim   = embedding_dimension,
        #     input_length = sequence_length)

        # Set up the decoder, using `encoder_states` as initial state.
        decoder_inputs = Input(shape=(None, self.num_decoder_vocab))

        # We set up our decoder to return full output sequences,
        # and to return internal states as well. We don't use the
        # return states in the training model, but we will use them in inference.

        decoder_lstm = LSTM(LATENT_DIM, return_sequences=True, return_state=True)

        decoder_outputs, _, _ = decoder_lstm(
            decoder_inputs,
            initial_state=encoder_states)

        decoder_dense = Dense(self.num_decoder_vocab, activation='softmax')

        decoder_outputs = decoder_dense(decoder_outputs)

        # Define the model that will turn
        # `self.encoder_input_data` & `self.decoder_input_data` into `self.decoder_target_data`
        self.model = Model([encoder_inputs, decoder_inputs], decoder_outputs)

        # Run training
        rmsprop = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)

        self.model.compile(optimizer=rmsprop, loss='categorical_crossentropy')

        self.model.fit([self.encoder_input_data, self.decoder_input_data], self.decoder_target_data,
            batch_size       = BATCH_SIZE,
            epochs           = EPOCHS,
            validation_split = 0.2)
        # Save model

        file_path = self.MODEL_PATH + '/s2s.h5'

        self.model.save(file_path)

        file_instance = open(self.MODEL_PATH + '/extra_data.bin', 'wb')

        pickle.dump({
            'encoder_input_data'        : self.encoder_input_data,
            'num_encoder_vocab'         : self.num_encoder_vocab,
            'input_word_index'          : self.input_word_index,
            'max_encoder_seq_length'    : self.max_encoder_seq_length,
            'num_decoder_vocab'         : self.num_decoder_vocab,
            'target_word_index'         : self.target_word_index,
            'reverse_target_word_index' : self.reverse_target_word_index,
            'max_decoder_seq_length'    : self.max_decoder_seq_length,
        }, file_instance)

    def search(model, src_input, k=1, sequence_max_len=25):
        # (log(1), initialize_of_zeros)
        k_beam = [(0, [0]*(sequence_max_len+1))]

        # l : point on target sentence to predict
        for l in range(sequence_max_len):
            all_k_beams = []
            for prob, sent_predict in k_beam:
                predicted = model.predict([np.array([src_input]), np.array([sent_predict])])[0]
                # top k!
                possible_k = predicted[l].argsort()[-k:][::-1]

                # add to all possible candidates for k-beams
                all_k_beams += [
                    (
                        sum(np.log(predicted[i][sent_predict[i+1]]) for i in range(l)) + np.log(predicted[l][next_wid]),
                        list(sent_predict[:l+1])+[next_wid]+[0]*(sequence_max_len-l-1)
                    )
                    for next_wid in possible_k
                ]

            # top k
            k_beam = sorted(all_k_beams)[-k:]

        return k_beam

    def __create_text_one_hot_vector(self, text):

        one_hot_vector = np.zeros(
            (1, self.max_encoder_seq_length, self.num_encoder_vocab),
            dtype='float32')

        for t, word in enumerate(text.split()):

            one_hot_vector[0, t, self.input_word_index[word]] = 1.

        return one_hot_vector

    def decode_sequence(self, input_seq):

        input_seq = self.__create_text_one_hot_vector(input_seq)

        # Encode the input as state vectors.
        states_value = self.encoder_model.predict(input_seq)

        # Generate empty target sequence of length 1.
        target_seq = np.zeros((1, 1, self.num_decoder_vocab))
        # Populate the first character of target sequence with the start character.
        target_seq[0, 0, self.target_word_index['<START>']] = 1.

        # Sampling loop for a batch of sequences
        # (to simplify, here we assume a batch of size 1).
        stop_condition = False
        decoded_sentence = []
        while not stop_condition:
            output_words, h, c = self.decoder_model.predict(
                [target_seq] + states_value)

            # Sample a word
            sampled_word_index = np.argmax(output_words[0, -1, :])
            sampled_char = self.reverse_target_word_index[sampled_word_index]
            decoded_sentence.append(sampled_char)

            # Exit condition: either hit max length
            # or find stop character.
            if (sampled_char == '<END>' or
            len(decoded_sentence) > self.max_decoder_seq_length):
                stop_condition = True

            # Update the target sequence (of length 1).
            target_seq = np.zeros((1, 1, self.num_decoder_vocab))
            target_seq[0, 0, sampled_word_index] = 1.

            # Update states
            states_value = [h, c]

        return ' '.join(decoded_sentence)


# -*- coding: utf-8 -*-

from waldis.common_utils import AttributeType
import matplotlib.pyplot as plt
import numpy as np
import math

class Pattern:
    def __init__(self, pattern_edges, pattern_attributes, pattern_timestamps, pattern_directions, pattern_scores, edge_schema):
        self.pattern_edges = pattern_edges
        self.pattern_attributes = pattern_attributes
        self.pattern_timestamps = pattern_timestamps
        self.pattern_directions = pattern_directions
        self.pattern_scores = pattern_scores
        self.edge_schema = edge_schema

    def plot_statistics(self, add_one_to_timestamp=False):
        num_of_plots = len(self.edge_schema) + 2
        plt.figure(1)
        index = 1

        xticks = np.arange(len(self.pattern_edges))

        for attr_name, attr_type in self.edge_schema:
            if attr_type == AttributeType.NUMERIC:
                values = [[y[attr_name] for y in x] for x in self.pattern_attributes]

                plt.subplot(num_of_plots, 1, index)
                plt.subplots_adjust(hspace=0.5)
                plt.title("Attribute: " + attr_name)
                plt.xlabel('pattern edge id')
                plt.xticks(xticks, xticks + 1)
                plt.boxplot(values)
            elif attr_type == AttributeType.NOMINAL:
                par_edges_cnt = len(self.pattern_attributes)
                possible_values = list(set.union(*[set([y[attr_name] for y in x]) for x in self.pattern_attributes]))
                value_counts = {p: [0]*par_edges_cnt for p in possible_values}

                for i, par_edge in enumerate(self.pattern_attributes):
                    for e in par_edge:
                        value_counts[e[attr_name]][i] += 1

                n_rows = len(value_counts)
                colors = plt.cm.jet(np.linspace(0, 1.0, n_rows))
                i_value = 0
                y_offset = np.array([0.0] * par_edges_cnt)
                bar_width = 0.4
                plt.subplot(num_of_plots, 1, index)
                plt.subplots_adjust(hspace=0.5)
                nrows = math.ceil(len(value_counts) / 4)
                title_y_shift = (1.15 if nrows == 1 else (1.25 if nrows == 2 else (1.4 if nrows >= 3 else 1.6)))
                plt.title("Attribute: " + attr_name, y=title_y_shift)
                plt.xlabel('pattern edge id')

                for k, v in value_counts.items():
                    plt.bar(np.arange(par_edges_cnt), v, bar_width, bottom=y_offset, color=colors[i_value],
                            label=k)
                    y_offset = y_offset + np.array(v)
                    i_value += 1
                plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, ncol=4, mode="expand", borderaxespad=0.)
                plt.xticks(xticks, xticks + 1)
                plt.show()
            index += 1

        plt.subplot(num_of_plots, 1, index)
        plt.subplots_adjust(hspace=0.5)
        plt.title("relative time before events")
        plt.xlabel('pattern edge id')
        plt.xticks(xticks, xticks + 1)
        if add_one_to_timestamp:
            new_timestamps = [[y + 1 for y in x] for x in self.pattern_timestamps]
            plt.boxplot(new_timestamps)
        else:
            plt.boxplot(self.pattern_timestamps)
        index += 1
        plt.subplot(num_of_plots, 1, index)
        plt.subplots_adjust(hspace=0.5)
        plt.title("edge scores")
        plt.xlabel('pattern edge id')
        plt.xticks(xticks, xticks + 1)
        plt.bar(np.arange(len(self.pattern_scores)), self.pattern_scores, align='center', alpha=0.5)

    def __str__(self):
        return str(self.pattern_edges) + "\n" + str(self.pattern_attributes) + "\n" + str(self.pattern_timestamps) \
            + "\n" + str(self.pattern_directions) + "\n" + str(self.pattern_scores)

    def __repr__(self):
        return self.__str__()


# coding=utf-8
import marshmallow as marshmallow
from marshmallow import ValidationError
from marshmallow import post_load, post_dump
from marshmallow import validates_schema
from marshmallow.compat import with_metaclass

from marshmallow_arango.common import SPECIAL_FIELDS
from marshmallow_arango.model import Document

class ModelSchemaOpts(marshmallow.SchemaOpts):
    """Options class for `ModelSchema`.
    Adds the following options:
    - ``model``: The database model to generate the `Schema` from (required).
    """

    def __init__(self, meta):
        super().__init__(meta)
        self.include.update(SPECIAL_FIELDS)
        self.model = getattr(meta, 'model', Document)

class ModelSchema(with_metaclass(marshmallow.schema.SchemaMeta, marshmallow.Schema)):
    """Base class for Arango model-based Schemas.
    """
    OPTIONS_CLASS = ModelSchemaOpts

    def __init__(self, *args, **kwargs):
        super(ModelSchema, self).__init__(*args, **kwargs)
        self.model = self.opts.model

    @post_load
    def make_instance(self, data):
        return self.model(**data)

    @post_dump
    def remove_none(self, data):
        for key in SPECIAL_FIELDS.keys():
            if data.get(key) is None:
                del data[key]
        return data

    def _check_unknown_fields(self, data):
        for key in data:
            if key not in self.fields:
                raise ValidationError('Unknown field name {}'.format(key))

    @validates_schema(pass_original=True, pass_many=True)
    def check_unknown_fields(self, data, original_data, many):
        if many:
            for original_datum in original_data:
                self._check_unknown_fields(data=original_datum)
        else:
            self._check_unknown_fields(data=original_data)


from collections import OrderedDict
from copy import deepcopy
from typing import Union, Tuple, List, Any, Dict

import pandas as pd
from frozendict import frozendict

from autoflow.constants import PHASE1, PHASE2, SERIES_CONNECT_LEADER_TOKEN, SERIES_CONNECT_SEPARATOR_TOKEN
from autoflow.hdl.utils import get_hdl_bank, get_default_hdl_bank
from autoflow.utils.dict import add_prefix_in_dict_keys
from autoflow.utils.graphviz import ColorSelector
from autoflow.utils.klass import StrSignatureMixin
from autoflow.utils.logging_ import get_logger
from autoflow.utils.math_ import get_int_length

class HDL_Constructor(StrSignatureMixin):
    '''
    ``HDL`` is abbreviation of Hyper-parameter Descriptions Language.
    It describes an abstract hyperparametric space that independent with concrete implementation.

    ``HDL_Constructor`` is a class who is responsible for translating dict-type ``DAG-workflow`` into ``H.D.L`` .

    If ``DAG-workflow`` didn't be explicit assign (str "generic_recommend" is default ),
    a generic ``DAG-workflow`` will be recommend by analyzing input data in doing :meth:`run`.

    And then, by using function :meth:`run` , ``DAG-workflow`` will be translated to ``HDL``.

    '''

    def __init__(
            self,
            DAG_workflow: Union[str, Dict[str, Any]] = "generic_recommend",
            hdl_bank_path=None,
            hdl_bank=None,
            hdl_metadata=frozendict(),
            included_classifiers=(
                    "adaboost", "catboost", "decision_tree", "extra_trees", "gaussian_nb", "k_nearest_neighbors",
                    "liblinear_svc", "libsvm_svc", "lightgbm", "logistic_regression", "random_forest", "sgd"),
            included_regressors=(
                    "adaboost", "bayesian_ridge", "catboost", "decision_tree", "elasticnet", "extra_trees",
                    "gaussian_process", "k_nearest_neighbors", "kernel_ridge",
                    "liblinear_svr", "lightgbm", "random_forest", "sgd"),
            included_highR_nan_imputers=("operate.drop", {"_name": "operate.merge", "__rely_model": "boost_model"}),
            included_cat_nan_imputers=(
                    "impute.fill_cat", {"_name": "impute.fill_abnormal", "__rely_model": "boost_model"}),
            included_num_nan_imputers=(
                    "impute.fill_num", {"_name": "impute.fill_abnormal", "__rely_model": "boost_model"}),
            included_highR_cat_encoders=("operate.drop", "encode.label", "encode.cat_boost"),
            included_lowR_cat_encoders=("encode.one_hot", "encode.label", "encode.cat_boost"),

    ):
        '''

        Parameters
        ----------
        DAG_workflow: str or dict, default="generic_recommend"

            directed acyclic graph (DAG) workflow to describe the machine-learning procedure.

            By default, this value is  "generic_recommend", means HDL_Constructor will analyze the training data
            to recommend a valid DAG workflow.

            If you want design DAG workflow by yourself, you can seed a dict .

        hdl_bank_path: str, default=None

            ``hdl_bank`` is a json file which contains  all the hyper-parameters of the algorithm.

            ``hdl_bank_path`` is this file's path. If it is None, ``autoflow/hdl/hdl_bank.json`` will be choosed.

        hdl_bank: dict, default=None

            If you pass param ``hdl_bank_path=None`` and pass  ``hdl_bank`` as a dict,
            program will not load ``hdl_bank.json``, it uses passed ``hdl_bank`` directly.

        included_classifiers: list or tuple

            active if ``DAG_workflow="generic_recommend"``, and all of the following params will active in such situation.

            It decides which **classifiers** will consider in the algorithm selection.

        included_regressors: list or tuple

            It decides which **regressors** will consider in the algorithm selection.

        included_highR_nan_imputers: list or tuple

            ``highR_nan`` is a feature_group, means ``NaN`` has a high ratio in a column.

            for example:

            >>> from numpy import NaN
            >>> column = [1, 2, NaN, NaN, NaN]    # nan ratio is 60% , more than 50% (default highR_nan_threshold)

            ``highR_nan_imputers`` algorithms will handle such columns contain high ratio missing value.

        included_cat_nan_imputers: list or tuple

            ``cat_nan`` is a feature_group, means a categorical feature column contains ``NaN`` value.

            for example:

            >>> column = ["a", "b", "c", "d", NaN]

            ``cat_nan_imputers`` algorithms will handle such columns.

        included_num_nan_imputers: list or tuple

            ``num_nan`` is a feature_group, means a numerical feature column contains ``NaN`` value.

            for example:

            >>> column = [1, 2, 3, 4, NaN]

            ``num_nan_imputers`` algorithms will handle such columns.

        included_highR_cat_encoders: list or tuple

            ``highR_cat`` is a feature_group, means a categorical feature column contains highly cardinality ratio.

            for example:

            >>> import numpy as np
            >>> column = ["a", "b", "c", "d", "a"]
            >>> rows = len(column)
            >>> np.unique(column).size / rows  # result is 0.8 , is higher than 0.5 (default highR_cat_ratio)
            0.8

            ``highR_cat_imputers`` algorithms will handle such columns.

        included_lowR_cat_encoders: list or tuple

            ``lowR_cat`` is a feature_group, means a categorical feature column contains lowly cardinality ratio.

            for example:

            >>> import numpy as np
            >>> column = ["a", "a", "a", "d", "a"]
            >>> rows = len(column)
            >>> np.unique(column).size / rows  # result is 0.4 , is lower than 0.5 (default lowR_cat_ratio)
            0.4

            ``lowR_cat_imputers`` algorithms will handle such columns.

        Attributes
        ----------
        random_state: int

        ml_task: :class:`autoflow.utils.ml_task.MLTask`

        data_manager: :class:`autoflow.manager.data_manager.DataManager`

        hdl: dict
            construct by :meth:`run`

        Examples
        ----------
        >>> import numpy as np
        >>> from autoflow.manager.data_manager import DataManager
        >>> from autoflow.hdl.hdl_constructor import  HDL_Constructor
        >>> hdl_constructor = HDL_Constructor(DAG_workflow={"num->target":["lightgbm"]},
        ...   hdl_bank={"classification":{"lightgbm":{"boosting_type":  {"_type": "choice", "_value":["gbdt","dart","goss"]}}}})
        >>> data_manager = DataManager(X_train=np.random.rand(3,3), y_train=np.arange(3))
        >>> hdl_constructor.run(data_manager, 42, 0.5)
        >>> hdl_constructor.hdl
        {'preprocessing': {}, 'estimating(choice)': {'lightgbm': {'boosting_type': {'_type': 'choice', '_value': ['gbdt', 'dart', 'goss']}}}}

        '''
        self.hdl_metadata = dict(hdl_metadata)
        self.included_lowR_cat_encoders = included_lowR_cat_encoders
        self.included_highR_cat_encoders = included_highR_cat_encoders
        self.included_num_nan_imputers = included_num_nan_imputers
        self.included_cat_nan_imputers = included_cat_nan_imputers
        self.included_highR_nan_imputers = included_highR_nan_imputers
        self.included_regressors = included_regressors
        self.included_classifiers = included_classifiers
        self.logger = get_logger(self)
        self.hdl_bank_path = hdl_bank_path
        self.DAG_workflow = DAG_workflow
        if hdl_bank is None:
            if hdl_bank_path:
                hdl_bank = get_hdl_bank(hdl_bank_path)
            else:
                hdl_bank = get_default_hdl_bank()
        if hdl_bank is None:
            hdl_bank = {}
            self.logger.warning("No hdl_bank, will use DAG_descriptions only.")
        self.hdl_bank = hdl_bank
        self.random_state = 42
        self.ml_task = None
        self.data_manager = None

    def parse_item(self, value: Union[dict, str]) -> Tuple[str, dict, bool]:
        if isinstance(value, dict):
            packages = value.pop("_name")
            if "_vanilla" in value:
                is_vanilla = value.pop("_vanilla")
            else:
                is_vanilla = False
            addition_dict = value
        elif isinstance(value, str):
            packages = value
            addition_dict = {}
            is_vanilla = False
        elif value is None:
            packages = "None"
            addition_dict = {}
            is_vanilla = False
        else:
            raise TypeError
        return packages, addition_dict, is_vanilla

    def purify_DAG_describe(self):
        DAG_describe = {}
        for k, v in self.DAG_workflow.items():
            if not isinstance(v, (list, tuple)):
                v = [v]
            DAG_describe[k.replace(" ", "").replace("\n", "").replace("\t", "")] = v
        self.DAG_workflow = DAG_describe

    def _get_params_in_dict(self, dict_: dict, package: str) -> dict:
        result = deepcopy(dict_)
        for path in package.split("."):
            result = result.get(path, {})
        return result

    def get_params_in_dict(self, hdl_bank: dict, packages: str, phase: str, mainTask):
        assert phase in (PHASE1, PHASE2)
        packages: list = packages.split(SERIES_CONNECT_SEPARATOR_TOKEN)
        params_list: List[dict] = [self._get_params_in_dict(hdl_bank[PHASE1], package) for package in
                                   packages[:-1]]
        last_phase_key = PHASE1 if phase == PHASE1 else mainTask
        params_list += [self._get_params_in_dict(hdl_bank[last_phase_key], packages[-1])]
        if len(params_list) == 0:
            raise AttributeError
        elif len(params_list) == 1:
            return params_list[0]
        else:
            result = {}
            for params, package in zip(params_list, packages):
                result.update(add_prefix_in_dict_keys(params, package + SERIES_CONNECT_LEADER_TOKEN))
            return result

    def generic_recommend(self) -> Dict[str, List[Union[str, Dict[str, Any]]]]:
        '''
        Recommend a generic DAG workflow-space.

        Returns
        -------
        DAG_workflow: dict

        '''
        DAG_workflow = OrderedDict()
        contain_highR_nan = False
        contain_nan = False
        # todo: 对于num特征进行scale transform
        # --------Start imputing missing(nan) value--------------------
        if "highR_nan" in self.data_manager.feature_groups:
            DAG_workflow["highR_nan->nan"] = self.included_highR_nan_imputers
            contain_highR_nan = True
        if contain_highR_nan or "nan" in self.data_manager.feature_groups:
            # split nan to cat_nan and num_nan
            DAG_workflow["nan->{cat_name=cat_nan,num_name=num_nan}"] = "operate.split.cat_num"
            DAG_workflow["num_nan->num"] = self.included_num_nan_imputers
            DAG_workflow["cat_nan->cat"] = self.included_cat_nan_imputers
            contain_nan = True
        # --------Start encoding categorical(cat) value --------------------
        if contain_nan or "cat" in self.data_manager.feature_groups:
            DAG_workflow["cat->{highR=highR_cat,lowR=lowR_cat}"] = {"_name": "operate.split.cat",
                                                                    "threshold": self.highR_cat_threshold}
            DAG_workflow["highR_cat->num"] = self.included_highR_cat_encoders
            DAG_workflow["lowR_cat->num"] = self.included_lowR_cat_encoders
        # --------Start estimating--------------------
        mainTask = self.ml_task.mainTask
        if mainTask == "classification":
            DAG_workflow["num->target"] = self.included_classifiers
        elif mainTask == "regression":
            DAG_workflow["num->target"] = self.included_regressors
        else:
            raise NotImplementedError
        # todo: 如果特征多，做特征选择或者降维。如果特征少，做增维
        return DAG_workflow

    def draw_workflow_space(self):
        '''

        Notes
        ------
        You must install graphviz in your compute.

        if you are using Ubuntu or another debian Linux, you should run::

            $ sudo apt-get install graphviz

        You can also install graphviz by conda::

            $ conda install -c conda-forge graphviz

        Returns
        -------
        graph: :class:`graphviz.dot.Digraph`
            You can find usage of :class:`graphviz.dot.Digraph` in https://graphviz.readthedocs.io/en/stable/manual.html

        '''
        candidates_colors = ["#663366", "#663300", "#666633", "#333366", "#660033"]
        feature_groups_colors = ["#0099CC", "#0066CC", "#339933", "#FFCC33", "#33CC99", "#FF0033", "#663399", "#FF6600"]
        cand2c = ColorSelector(candidates_colors)
        feat2c = ColorSelector(feature_groups_colors)

        def parsed_candidates(candidates):
            parsed_candidates = []
            for candidate in candidates:
                if isinstance(candidate, dict):
                    parsed_candidates.append(candidate["_name"])
                else:
                    parsed_candidates.append(candidate)
            return parsed_candidates

        def get_node_label(node_name):
            return f'''<{node_name}>'''

        from graphviz import Digraph
        DAG_workflow = deepcopy(self.DAG_workflow)
        graph = Digraph("workflow_space")
        graph.node("data")
        for parsed_node in pd.unique(self.data_manager.feature_groups):
            graph.node(parsed_node, color=feat2c[parsed_node], label=get_node_label(parsed_node))
            graph.edge("data", parsed_node,
                       label=f'''<data_manager: {parsed_node}>''')
        for indicate, candidates in DAG_workflow.items():
            if "->" not in indicate:
                continue
            _from, _to = indicate.split("->")
            graph.node(_from, color=feat2c[_from], label=get_node_label(_from))
            candidates = parsed_candidates(candidates)
            if _to.startswith("{") and _to.endswith("}"):
                candidate = candidates[0]
                _to = _to[1:-1]
                tails = []
                for item in _to.split(","):
                    tails.append(item.split("=")[-1])
                for tail in tails:
                    graph.node(tail, color=feat2c[tail], label=get_node_label(tail))
                    graph.edge(_from, tail, f'''<{candidate}: {tail}>''')
            else:
                graph.node(_to, color=feat2c[_to], label=get_node_label(_to))
                if len(candidates) == 1:
                    candidates_str = f'<{candidates[0]}>'
                else:
                    candidates_str = "<{" + ", ".join(
                        [f'{candidate}' for candidate in candidates]) + "}>"
                graph.edge(_from, _to, candidates_str)
        graph.attr(label=r'WorkFlow Space')
        return graph

    def run(self, data_manager, random_state, highR_cat_threshold):
        '''

        Parameters
        ----------
        data_manager: :class:`autoflow.manager.data_manager.DataManager`
        random_state: int
        highR_cat_threshold: float

        '''
        self.highR_cat_threshold = highR_cat_threshold
        self.data_manager = data_manager
        self.ml_task = data_manager.ml_task
        self.random_state = random_state
        if isinstance(self.DAG_workflow, str):
            if self.DAG_workflow == "generic_recommend":
                self.hdl_metadata.update({"source": "generic_recommend"})
                self.logger.info("Using 'generic_recommend' method to initialize a generic DAG_workflow, \n"
                                 "to Adapt to various data such like NaN and categorical features.")
                self.DAG_workflow = self.generic_recommend()
            else:
                raise NotImplementedError
        elif isinstance(self.DAG_workflow, dict):
            self.hdl_metadata.update({"source": "user_defined"})
            self.logger.info("DAG_workflow is specifically set by user.")
        else:
            raise NotImplementedError

        target_key = None
        self.purify_DAG_describe()

        # ---- 开始将 DAG_workflow 解析成 HDL
        DAG_workflow = deepcopy(self.DAG_workflow)
        for key in DAG_workflow.keys():
            if key.split("->")[-1] == "target":
                target_key = key
        estimator_values = DAG_workflow.pop(target_key)
        if not isinstance(estimator_values, (list, tuple)):
            estimator_values = [estimator_values]
        preprocessing_dict = {}
        mainTask = self.ml_task.mainTask
        hdl_bank = deepcopy(self.hdl_bank)
        # 遍历DAG_describe，构造preprocessing
        n_steps = len(DAG_workflow)
        int_len = get_int_length(n_steps)
        for i, (key, values) in enumerate(DAG_workflow.items()):
            formed_key = f"{i:0{int_len}d}{key}(choice)"
            sub_dict = {}
            for value in values:
                packages, addition_dict, is_vanilla = self.parse_item(value)
                params = {} if is_vanilla else self.get_params_in_dict(hdl_bank, packages, PHASE1, mainTask)
                sub_dict[packages] = params
                sub_dict[packages].update(addition_dict)
            preprocessing_dict[formed_key] = sub_dict
        # 构造estimator
        estimator_dict = {}
        for estimator_value in estimator_values:
            packages, addition_dict, is_vanilla = self.parse_item(estimator_value)
            params = {} if is_vanilla else self.get_params_in_dict(hdl_bank, packages, PHASE2, mainTask)
            estimator_dict[packages] = params
            estimator_dict[packages].update(addition_dict)
        final_dict = {
            PHASE1: preprocessing_dict,
            f"{PHASE2}(choice)": estimator_dict
        }
        self.hdl = final_dict

    def get_hdl(self) -> Dict[str, Any]:
        '''

        Returns
        -------
        hdl: dict

        '''
        # 获取hdl
        return self.hdl


"""himalaya URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path,include
from django.conf import settings
from django.conf.urls.static import static
from rest_framework.decorators import api_view,renderer_classes,permission_classes
from rest_framework.permissions import IsAuthenticated,IsAdminUser
from rest_framework.response import Response
from rest_framework.authtoken.models import Token
from django.views.static import serve
from django.contrib.auth.models import Group, User

admin.autodiscover()
# admin.site.unregister(Token)
# admin.site.unregister(User)
# admin.site.unregister(Group)

admin.site.site_header = "Denim Factory Admin"
admin.site.site_title = "Denim Factory Admin Portal"
admin.site.index_title = "Welcome to Denim Factory Admin Portal"
admin.site.site_url="https://shield-1712.firebaseapp.com/"
admin.empty_value_display="Nothing to Display, Add by clicking on right side"

urlpatterns = [
    path('admin/', admin.site.urls),
    path('user/', include("users.urls","users")),
    path('treatment/',include("treatment.urls"))
]

@api_view(['GET'])
@permission_classes([])
def protected_serve(request, path, document_root=None, show_indexes=False):
    path=path.split("?Token=")
    if(len(path)>1):
        try:
            token=Token.objects.get(key=path[1])
            return serve(request, path[0], document_root, show_indexes)
        except Token.DoesNotExist:
            return Response({'authentication':'Authentication Credentials not provided/ Wrong Credentials'})
    else:
        return Response({'authentication':'Token should be provided with URL'})

if settings.DEBUG:
    urlpatterns+=static(settings.MEDIA_URL,document_root=settings.MEDIA_ROOT)


from models.mlp import model
import torch
import numpy as np
from sklearn import preprocessing

def get_prediction(data):
    data = np.array(data).reshape(1, len(data))

    data = preprocessing.normalize(data)
    data = torch.from_numpy(data)

    output = model(data.float())
    _, predicted = torch.max(output.data, 1)
    prediction = int(predicted[0])

    if prediction == 0:
        print('Пациент здоров')
    else:
        print('Имеется заболевание сердца')

# Loading model parametrs
checkpoint = torch.load('stats\\model800.ckpt')
model.load_state_dict(checkpoint)
model.eval()

# Number of features
n = 13

data = []
for i in range(n):
    x = float(input())
    data.append(x)

get_prediction(data)


import logging
import settings
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters

#proxy setting

#logs дата время--- уровень важности события -- само сообщение события
logging.basicConfig(format ='%(asctime)s - %(levelname)s - %(message)s',
                    #тип логирования:
                    level=logging.INFO,
                    filename='bot.log'
                    )
'''
def greet_user(bot, update):
        print ('called /allo')
        update.message.reply_text('ALLO KISUN!! HOW ARE U?')

def echo(bot, update):
    print(update.message)
    user_text = update.message.text
    logging.info(f"User: {update.message.chat.first_name}, Chat ID: {update.message.chat.id}, Message: {update.message.text}")
    print(user_text)

    if user_text == 'genji':
        update.message.reply_text('ryujin no ken wo kurae!!!!')
    else:
        update.message.reply_text('DUPLICATING: '+user_text)
'''
def greet_user(update, context):
        print ('called /allo')
        update.message.reply_text('ALLO KISUN!! HOW ARE U?')

def echo(update, context):
    print(update.message)
    user_text = update.message.text
    logging.info(f"User: {update.message.chat.first_name}, Chat ID: {update.message.chat.id}, Message: {update.message.text}")
    print(user_text)

    if user_text == 'genji':
        update.message.reply_text('ryujin no ken wo kurae!!!!')
    else:
        update.message.reply_text('DUPLICATING: '+user_text)
#главная функция для взаимодейсвия с ботом
def main():
    #объект класс апдейтер который будет проверять сообщения боту по указанному ключу
    paranoidBot = Updater(settings.API_KEY, use_context=True, request_kwargs= settings.PROXY)

    #тупой препод не заметил что info нужно именно в lower case писать
    logging.info('Started bot')

    #диспетчер принимает сообщения, и распределяет их, например в команд хэндлер
    dp = paranoidBot.dispatcher

    # регистрируем процедуру start как обработчик команды start
    dp.add_handler(CommandHandler('allo', greet_user))
    # регистрируем процедуру echo как обработчик текстового сообщения
    dp.add_handler(MessageHandler(Filters.text, echo))

    paranoidBot.start_polling()# начать регулярную проверку сообщений боту
    paranoidBot.idle() #работа до принудительной остановки
# в ботах используется собыйтийная модель

main()


#! /usr/bin/env python

# Script to craft a packet for testing flow entries
# February 2017 - Martin Roedvand

import sys
import argparse
import logging
import random
import datetime
import os
from scapy.all import *
from subprocess import call

# Commandline arguments
parser = argparse.ArgumentParser(description='Craft a packet to test flow entries.')
parser.add_argument('-d', nargs='?', required=True, help='the destination address of the packet', metavar='DST')
parser.add_argument('-p', nargs=1, choices=['icmp', 'udp', 'tcp'], default=['tcp'], help='the protocol used in the packet')
parser.add_argument('-dp', nargs=1, help='the destination port to be used in the packet', metavar='DPORT')
parser.add_argument('-sp', nargs=1, help='the source port to be used in the packet', metavar='SPORT')
parser.add_argument('-t', action='store_true', help='show packet trace')
parser.add_argument('-v', action='store_true', help='for verbose output. May be helpful under debugging')

args = parser.parse_args()

dest = args.d
prot = args.p[0]
trace = args.t

if args.dp:
	port = int(args.dp[0])
else:
	port = '-1'

if args.sp:
	sourceport = int(args.sp[0])
else:
	sourceport = random.randint(1, 65535)

# Logging
verbose = args.v

if verbose:
	logging.basicConfig(level=logging.DEBUG)
else:
	logging.basicConfig(level=logging.INFO)

# Start crafting a packet with SCAPY
logging.debug("Attempting to craft packet")
logging.debug("CMD arguments: " + str(args))

if prot == 'icmp':
	logging.debug("We're in ICMP")
	packet = IP(dst=dest)/ICMP()/"Hello World"

if prot == 'tcp':
	logging.debug("We're in TCP")
	logging.debug(port)
	if port == '-1':
		logging.warning("Port is not defined. Port is required for TCP.")
		sys.exit(1)
	packet = IP(dst=dest)/TCP(sport=sourceport,dport=port)

if prot == 'udp':
	print

logging.debug("Going for a send")

# Start the trace with ./dump.sh
if trace:
	# Create a time stamp for logging
	timestamp = '{:%Y%m%d%H%M}'.format(datetime.now())
	outfile = open("logs/" + timestamp, 'w')
	logging.debug("Timestamp: " + timestamp)
	logging.debug("Current directory " + os.getcwd())
	logCode = call(["./dump.sh dst host " + dest + "> logs/" + timestamp + " &"], shell=True)
	logging.debug("Call Error Code: " + str(logCode))

	if logCode != 0:
		logging.info("TCPdump returned with error code " + str(logCode))

sr(packet, timeout=1)