# 1. import libraries and modules
from config import *
from tkinter import *
from tkinter import messagebox
from tkinter.filedialog import asksaveasfile, askopenfile
from PIL import ImageTk, Image
import cv2
import numpy as np
import json
from a_star import astar

# 2. init main variables
# cells data structure: { key( (row_index, col_index) ): value( canvas_rect_id ) }
wallCells = {}
exitCells = {}
humanCells = {}

# flags
wallsHighlighted = False
gridDisplayed = False
wallsDisplayed = True
childIsOpen = False

# variables for selecting multiple cells
selectionRect = None
startX, startY = 0, 0
endX, endY = 0, 0
startCol, endCol = 0, 0
startRow, endRow = 0, 0
modelFilepath = None

# 3. init supporting functions
def refreshCells():
    global canvas, wallCells, humanCells, exitCells, wallsHighlighted, sbWidthVar, sbHeightVar

    cellWidth, cellHeight = sbWidthVar.get(), sbHeightVar.get()

    for pos in wallCells.keys():
        col, row = pos
        canvas.delete(wallCells[(col, row)])
        x, y = col * cellWidth, row * cellHeight
        fill = 'black' if wallsHighlighted and wallsDisplayed else ''
        outline = 'black' if wallsDisplayed else ''
        wallCells[(col, row)] = canvas.create_rectangle((x, y), (x + cellWidth, y + cellHeight),
                                                        outline=outline, fill=fill, tag='cell')

    for pos in exitCells.keys():
        col, row = pos
        tp = exitCells[pos][1]
        canvas.delete(exitCells[(col, row)][0])
        x, y = col * cellWidth, row * cellHeight
        exitCells[(col, row)] = (canvas.create_rectangle((x, y), (x + cellWidth, y + cellHeight),
                                                         outline='black', fill='lime', tag='cell'), tp)

    for pos in humanCells.keys():
        col, row = pos
        canvas.delete(humanCells[(col, row)])
        x, y = col * cellWidth, row * cellHeight
        humanCells[(col, row)] = canvas.create_rectangle((x, y), (x + cellWidth, y + cellHeight),
                                                         outline='black', fill='red', tag='cell')

def markingCell(col, row):
    global canvas, wallCells, exitCells, humanCells, sbWidthVar, sbHeightVar, rbVar

    cellWidth, cellHeight = sbWidthVar.get(), sbHeightVar.get()

    try:
        if rbVar.get() == 0:  # is wall
            canvas.delete(wallCells[(col, row)])
            del wallCells[(col, row)]
        elif rbVar.get() == 2:  # is exit
            canvas.delete(exitCells[(col, row)][0])
            del exitCells[(col, row)]
        elif rbVar.get() == 3:  # is human
            canvas.delete(humanCells[(col, row)])
            del humanCells[(col, row)]
    except KeyError:
        x, y = col * cellWidth, row * cellHeight
        if rbVar.get() == 0:  # is wall
            fill = 'black' if wallsHighlighted and wallsDisplayed else ''
            outline = 'black' if wallsDisplayed else ''
            wallCells[(col, row)] = canvas.create_rectangle((x, y), (x + cellWidth, y + cellHeight),
                                                            outline=outline, fill=fill, tag='cell')
        elif rbVar.get() == 2:  # is exit
            tpVar = []
            setThroughput(tpVar)
            tp = tpVar[0]
            if tp > 0:
                exitCells[(col, row)] = (canvas.create_rectangle((x, y), (x + cellWidth, y + cellHeight),
                                                                 outline='black', fill='lime', tag='cell'), tp)
        elif rbVar.get() == 3:  # is human
            humanCells[(col, row)] = canvas.create_rectangle((x, y), (x + cellWidth, y + cellHeight),
                                                             outline='black', fill='red', tag='cell')

def canvasRefresh():
    global arrayImage, panel, canvasImage, imagePosition
    imageTK = ImageTk.PhotoImage(Image.fromarray(obj=arrayImage, mode='RGBA'))  # convert numpy image to tkinter image
    canvas.delete(canvasImage)
    canvasImage = canvas.create_image(imagePosition, image=imageTK, anchor=NW)
    panel = Label(window, image=imageTK)
    panel.image = imageTK

def cellsFill():
    global wallsHighlighted
    wallsHighlighted = not wallsHighlighted
    refreshCells()

def displayGrid(isRefresh=None):
    global canvas, cursorCell, CANVAS_WIDTH, CANVAS_HEIGHT, gridDisplayed, sbWidthVar, sbHeightVar

    cellWidth, cellHeight = sbWidthVar.get(), sbHeightVar.get()

    if isRefresh:
        canvas.delete('grid_line')
        for i in range(0, CANVAS_WIDTH, cellWidth):
            canvas.create_line([(i, 0), (i, CANVAS_HEIGHT)], fill='gray', tag='grid_line')

        for i in range(0, CANVAS_HEIGHT, cellHeight):
            canvas.create_line([(0, i), (CANVAS_WIDTH, i)], fill='gray', tag='grid_line')
    else:
        if gridDisplayed:
            canvas.delete('grid_line')
            gridDisplayed = False
        else:
            for i in range(0, CANVAS_WIDTH, cellWidth):
                canvas.create_line([(i, 0), (i, CANVAS_HEIGHT)], fill='gray', tag='grid_line')

            for i in range(0, CANVAS_HEIGHT, cellHeight):
                canvas.create_line([(0, i), (CANVAS_WIDTH, i)], fill='gray', tag='grid_line')

            gridDisplayed = True

        canvas.delete((cursorCell))
        cursorCell = canvas.create_rectangle((0, 0), (cellWidth, cellWidth), outline='blue')
    refreshCells()

# 4. init basic UI event handlers
def mouseMove(event):
    global canvas, cursorCell, sbWidthVar, sbHeightVar

    cellWidth, cellHeight = sbWidthVar.get(), sbHeightVar.get()
    col, row = event.x // cellWidth, event.y // cellHeight
    x, y = col * cellWidth - 1, row * cellHeight - 1

    if rbVar.get() != 1:
        canvas.delete(cursorCell)
        cursorCell = canvas.create_rectangle((0, 0), (sbWidthVar.get(), sbHeightVar.get()), outline='blue')
        canvas.moveto(cursorCell, x, y)
    else:
        canvas.moveto(cursorCell, -cellWidth, -cellHeight)

def mousePress(event):
    global canvas, wallCells, wallsHighlighted, sbWidthVar, sbHeightVar, startX, startY, rbVar

    cellWidth, cellHeight = sbWidthVar.get(), sbHeightVar.get()
    startX, startY = event.x, event.y
    col, row = event.x // cellWidth, event.y // cellHeight

    canvas.delete('select_grid_line')
    if wallsDisplayed and rbVar.get() != 1:
        markingCell(col, row)

def mousePressMove(event):
    global canvas, selectionRect, startX, startY, endX, endY, sbWidthVar, sbHeightVar, imagePosition, arrayImage

    if rbVar.get() == 0:
        cellWidth, cellHeight = sbWidthVar.get(), sbHeightVar.get()
        endX, endY = event.x, event.y

        canvas.delete(selectionRect)
        selectionRect = canvas.create_rectangle((startX, startY), (endX, endY), outline='blue')

        try:
            col, row = startX // cellWidth, startY // cellHeight
            canvas.delete(wallCells[(col, row)])
            del wallCells[(col, row)]
        except KeyError:
            pass
    elif rbVar.get() == 1:
        h, w = arrayImage.shape[:2]
        imagePosition = event.x - w // 2, event.y - h // 2
        canvasRefresh()

def mouseUp(event):
    global canvas, CANVAS_WIDTH, CANVAS_HEIGHT, selectionRect, wallCells, startX, startY, endX, endY, \
        sbWidthVar, sbHeightVar, startCol, endCol, startRow, endRow

    if rbVar.get() == 0:
        cellWidth, cellHeight = sbWidthVar.get(), sbHeightVar.get()
        endX, endY = event.x, event.y

        canvas.delete(selectionRect)

        startCol = max(min(startX, endX), -cellWidth) // cellWidth + 1
        endCol = min(max(startX, endX), CANVAS_WIDTH) // cellWidth
        startRow = max(min(startY, endY), -cellHeight) // cellHeight + 1
        endRow = min(max(startY, endY), CANVAS_HEIGHT) // cellHeight

        if (endCol - startCol < 1) or (endRow - startRow < 1):
            return

        x1, x2 = startCol * cellWidth, endCol * cellWidth
        y1, y2 = startRow * cellHeight, endRow * cellHeight

        for i in range(x1, x2 + cellWidth, cellWidth):
            canvas.create_line([(i, y1), (i, y2)], fill='blue', tag='select_grid_line')

        for i in range(y1, y2 + cellHeight, cellHeight):
            canvas.create_line([(x1, i), (x2, i)], fill='blue', tag='select_grid_line')

def markingSelectionCells(event):
    global canvas, selectionRect, startRow, startCol, endRow, endCol

    if (endCol - startCol < 1) or (endRow - startRow < 1):
        return

    for row in range(startRow, endRow):
        for col in range(startCol, endCol):
            markingCell(col, row)

    startCol, endCol = 0, 0
    startRow, endRow = 0, 0

def resizeImage(event):
    global arrayImage, scaleValue, childIsOpen

    if childIsOpen:
        return

    scaleValue += event.delta // 120 * 0.01
    h, w = SOURCE_IMAGE.shape[:2]
    h, w = int(h * scaleValue), int(w * scaleValue)
    arrayImage = cv2.resize(SOURCE_IMAGE.copy(), (w, h))
    canvasRefresh()

def loadImage(event=None):
    global arrayImage, SOURCE_IMAGE, CANVAS_WIDTH, CANVAS_HEIGHT
    file = askopenfile(filetypes=(('PNG', '*png'), ('JPEG', '*jpeg *jpg'), ('All Files', '*.*')))
    image = np.asarray(Image.open(file.name))
    h, w = image.shape[:2]
    k = CANVAS_WIDTH / w if w > h else CANVAS_HEIGHT / h
    h, w = int(h * k), int(w * k)
    image = cv2.resize(image, (w, h))
    B, G, R = cv2.split(image)[:3]  # get color channels as lists
    alpha = 100  # transparency value
    A = np.ones(B.shape, dtype=B.dtype) * alpha  # image alpha-channel as list
    arrayImage = cv2.merge((R, G, B, A))  # merging channel lists into image
    SOURCE_IMAGE = arrayImage.copy()
    canvasRefresh()

def openFile(event=None):
    global imagePosition, arrayImage, scaleValue, SOURCE_IMAGE, canvasImage, wallCells, exitCells, humanCells, \
        sbWidthVar, sbHeightVar, modelFilepath

    if SOURCE_IMAGE is not None:
        ans = messagebox.askyesnocancel('Внимание', 'Сохранить открытый проект?')
        if ans is None:
            return
        else:
            if ans:
                save()
            else:
                pass

    file = askopenfile(filetypes=(('JSON Files', '*.json'), ('All Files', '*.*')))
    with open(file.name, 'r', encoding='utf-8') as j:
        data = json.loads(j.read())

    imageFilename = file.name.removesuffix('json') + 'png'
    SOURCE_IMAGE = np.asarray(Image.open(imageFilename, 'r'))
    if SOURCE_IMAGE is None:
        messagebox.showinfo('Ошибка', f'Изображение {imageFilename} не найдено')
        return

    canvas.delete(canvasImage)

    modelFilepath = file.name

    scaleValue = data['scale']
    imagePosition = data['image_position']['x'], data['image_position']['y']
    sbWidthVar.set(data['cell_size']['width'])
    sbHeightVar.set(data['cell_size']['height'])

    h, w = SOURCE_IMAGE.shape[:2]
    h, w = int(h * scaleValue), int(w * scaleValue)
    arrayImage = cv2.resize(SOURCE_IMAGE.copy(), (w, h))
    B, G, R = cv2.split(arrayImage)[:3]  # get color channels as lists
    alpha = 100  # transparency value
    A = np.ones(B.shape, dtype=B.dtype) * alpha  # image alpha-channel as list
    arrayImage = cv2.merge((R, G, B, A))  # merging channel lists into image
    SOURCE_IMAGE = arrayImage.copy()
    imageTK = ImageTk.PhotoImage(Image.fromarray(obj=arrayImage, mode='RGBA'))  # convert numpy image to tkinter image
    canvasImage = canvas.create_image(imagePosition, image=imageTK, anchor=NW)

    wallCells = {}
    exitCells = {}
    humanCells = {}
    canvas.delete('cell')

    for pos in data['walls']:
        col, row = pos['col'], pos['row']
        x, y = col * sbWidthVar.get(), row * sbHeightVar.get()
        fill = 'black' if wallsHighlighted and wallsDisplayed else ''
        outline = 'black' if wallsDisplayed else ''
        wallCells[(col, row)] = canvas.create_rectangle((x, y), (x + sbWidthVar.get(),
                                                                 y + sbHeightVar.get()),
                                                        outline=outline, fill=fill, tag='cell')
    for pos in data['exits']:
        col, row, tp = pos['col'], pos['row'], pos['throughput']
        x, y = col * sbWidthVar.get(), row * sbHeightVar.get()
        exitCells[(col, row)] = (canvas.create_rectangle((x, y), (x + sbWidthVar.get(),
                                                                  y + sbHeightVar.get()),
                                                         outline='black', fill='lime', tag='cell'), tp)

    canvasRefresh()
    refreshCells()

def save(event=None):
    global imagePosition, scaleValue, SOURCE_IMAGE, wallCells, exitCells, sbWidthVar, sbHeightVar

    data = {}
    data['image_position'] = {
        'x': imagePosition[0],
        'y': imagePosition[1]
    }
    data['scale'] = scaleValue
    data['cell_size'] = {
        'width': sbWidthVar.get(),
        'height': sbHeightVar.get()
    }

    data['walls'] = []
    for pos in wallCells.keys():
        col, row = pos
        data['walls'].append({
            'col': col,
            'row': row
        })
    data['exits'] = []
    for pos in exitCells.keys():
        col, row = pos
        tp = exitCells[pos][1]
        data['exits'].append({
            'col': col,
            'row': row,
            'throughput': tp
        })

    if SOURCE_IMAGE is None:
        messagebox.showinfo('Отмена', 'Сохранение невозможно. Отсутствует изображение проекта.')
        return

    if modelFilepath:
        with open(modelFilepath, 'w') as file:
            file.write(json.dumps(data, indent=4))
        imageFilename = file.name.removesuffix('json') + 'png'
        cv2.imwrite(imageFilename, SOURCE_IMAGE)
        messagebox.showinfo('Сохранение', f'Данные успешно сохранены в {modelFilepath}')
    else:
        file = asksaveasfile(initialfile='Untitled.json',
                             defaultextension='.json',
                             filetypes=(('All Files', '*.*'), ('JSON Files', '*.json')))

        imageFilename = file.name.removesuffix('json') + 'png'
        cv2.imwrite(imageFilename, SOURCE_IMAGE)
        file.write(json.dumps(data, indent=4))
        file.close()

def searchPaths():
    global canvas, wallCells, exitCells, humanCells, sbWidthVar, sbHeightVar
    cellSize = sbWidthVar.get(), sbHeightVar.get()

    exitTps = {pos: exitCells[pos][1] for pos in exitCells.keys()}
    if exitCells and humanCells:
        for human in humanCells.keys():
            variants = sorted([astar(wallCells, human, exit, cellSize) for exit in exitCells.keys()],
                              key=lambda item: item[2])
            path = []
            for v in variants:
                t, p, _ = v
                if exitTps[t] > 0:
                    path = p
                    exitTps[t] -= 1
                    break
                else:
                    continue

            # path = min([astar(wallCells, human, exit, cellSize) for exit in exitCells.keys()],
            #            key=lambda item: item[2])[1]

            for i in range(len(path) - 1):
                row1, col1 = path[i]
                row2, col2 = path[i + 1]
                x1, y1 = row1 * cellSize[0] + cellSize[0] // 2, col1 * cellSize[1] + cellSize[1] // 2
                x2, y2 = row2 * cellSize[0] + cellSize[0] // 2, col2 * cellSize[1] + cellSize[1] // 2
                canvas.create_line([(x1, y1), (x2, y2)], fill='darkgreen', width=2, tag='path')

def clearPaths():
    canvas.delete('path')
    canvasRefresh()

def openChildWindow():
    global window, rbVar, scaleValue, imagePosition, arrayImage, childIsOpen

    walls = set()
    childIsOpen = True

    rbVar.set(0)

    # set child window
    childWindow = Toplevel(window)
    childWindow.title('SEMI-AUTOMATIC BUILDING PLAN')
    childWindow.geometry(f'{WINDOW_WIDTH}x{WINDOW_HEIGHT}')
    childWindow.resizable(False, False)

    def update(a=None, b=None, c=None):
        global arrayImage, imagePosition

        if sbWidthVar2.get() < 2:
            sbWidthVar2.set(2)
        if sbWidthVar2.get() > 50:
            sbWidthVar2.set(50)

        if sbHeightVar2.get() < 2:
            sbHeightVar2.set(2)
        if sbHeightVar2.get() > 50:
            sbHeightVar2.set(50)

        offsetX, offsetY = abs(imagePosition[0]), abs(imagePosition[1])
        img = arrayImage[offsetY:offsetY + CANVAS_HEIGHT, offsetX:offsetX + CANVAS_WIDTH]
        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        _, thresh_img = cv2.threshold(img, scThreshVar2.get(), 255, cv2.THRESH_BINARY)

        childCanvas.delete('cell')

        walls.clear()
        stepX, stepY = sbWidthVar2.get(), sbHeightVar2.get()
        for x in range(0, CANVAS_WIDTH, stepX):
            for y in range(0, CANVAS_HEIGHT, stepY):
                cell = thresh_img[y:y + stepY, x:x + stepX]
                if len(cell[cell == 0]) / (stepX * stepY) > 0.5:
                    childCanvas.create_rectangle([(x, y), (x + stepX, y + stepY)], outline='black', tag='cell')
                    col, row = x // stepX, y // stepY
                    walls.add((col, row))

    # add canvas to child window
    childCanvas = Canvas(childWindow, width=CANVAS_WIDTH, height=CANVAS_HEIGHT, bg='white')
    childCanvas.place(x=CANVAS_OFFSET_X, y=CANVAS_OFFSET_Y)
    imageTK = ImageTk.PhotoImage(Image.fromarray(obj=arrayImage, mode='RGBA'))
    childCanvas.create_image(imagePosition, image=imageTK, anchor=NW)

    # add slider
    lbThresh2 = Label(childWindow, text='Пороговое значение')
    lbThresh2.place(x=818, y=230)
    scThreshVar2 = IntVar()
    scThreshVar2.set(50)
    scThresh2 = Scale(childWindow, from_=0, to=255, variable=scThreshVar2,
                      orient=HORIZONTAL, length=160, command=update)
    scThresh2.place(x=818, y=250)

    # add spinboxes
    lbSetCellWidth2 = Label(childWindow, text='Длина ячейки')
    lbSetCellWidth2.place(x=818, y=319)
    sbWidthVar2 = IntVar()
    sbWidthVar2.set(5)
    sbWidthVar2.trace('w', update)
    sbSetCellWidth2 = Spinbox(childWindow, from_=2, to=50, width=8, textvariable=sbWidthVar2, justify='center')
    sbSetCellWidth2.place(x=918, y=320)

    lbSetCellHeight2 = Label(childWindow, text='Высота ячейки')
    lbSetCellHeight2.place(x=818, y=349)
    sbHeightVar2 = IntVar()
    sbHeightVar2.set(5)
    sbHeightVar2.trace('w', update)
    sbSetCellHeight2 = Spinbox(childWindow, from_=2, to=50, width=8, textvariable=sbHeightVar2, justify='center')
    sbSetCellHeight2.place(x=918, y=350)

    # add buttons
    def apply():
        global sbWidthVar, sbHeightVar, wallCells

        sbWidthVar.set(sbWidthVar2.get())
        sbHeightVar.set(sbHeightVar2.get())

        wallCells = dict.fromkeys(walls, None)
        closingTop()
        refreshCells()

    def closingTop():
        global childIsOpen
        childIsOpen = False
        childWindow.destroy()

    btApply = Button(childWindow, text='Применить', command=lambda: apply(), width=20)
    btApply.place(x=830, y=511)

    btCancel = Button(childWindow, text='Отмена', command=lambda: closingTop(), width=20)
    btCancel.place(x=830, y=550)

    childWindow.protocol("WM_DELETE_WINDOW", closingTop)

    # first call
    update()

    childWindow.transient(window)
    childWindow.grab_set()
    childWindow.focus_set()
    childWindow.wait_window()

def setThroughput(tpVar: list):
    global window, childIsOpen

    childIsOpen = True

    # set child window
    childWindow = Toplevel(window)
    childWindow.title('INPUT THROUGHPUT')
    childWindow.geometry(f'300x150')
    childWindow.resizable(False, False)

    def update(a=None, b=None, c=None):
        if sbTPVar.get() < 1:
            sbTPVar.set(1)
        if sbTPVar.get() > 100:
            sbTPVar.set(100)

        if sbTPVar.get() < 1:
            sbTPVar.set(1)
        if sbTPVar.get() > 100:
            sbTPVar.set(100)

    # add spinbox
    lbSetTP = Label(childWindow, text='Введите пропускную способность:')
    lbSetTP.place(x=20, y=20)
    sbTPVar = IntVar()
    sbTPVar.set(1)
    sbTPVar.trace('w', update)
    sbSetTP = Spinbox(childWindow, from_=1, to=100, width=8, textvariable=sbTPVar, justify='center')
    sbSetTP.place(x=225, y=20)

    def apply():
        global childIsOpen
        tpVar.append(sbTPVar.get())
        childIsOpen = False
        childWindow.destroy()

    def closingTop():
        global childIsOpen
        tpVar.append(0)
        childIsOpen = False
        childWindow.destroy()

    btApply = Button(childWindow, text='Применить', command=lambda: apply(), width=20)
    btApply.place(x=80, y=65)

    btCancel = Button(childWindow, text='Отмена', command=lambda: closingTop(), width=20)
    btCancel.place(x=80, y=100)

    childWindow.protocol("WM_DELETE_WINDOW", closingTop)
    childWindow.transient(window)
    childWindow.grab_set()
    childWindow.focus_set()
    childWindow.wait_window()

# 5. init windows form elements and widgets
window = Tk()
window.title('BUILDING PLAN EDITOR')
window.geometry(f'{WINDOW_WIDTH}x{WINDOW_HEIGHT}')
window.resizable(False, False)

# add canvas
canvas = Canvas(window, width=CANVAS_WIDTH, height=CANVAS_HEIGHT, bg='white')
canvas.place(x=CANVAS_OFFSET_X, y=CANVAS_OFFSET_Y)
canvasImage = None
arrayImage: np.array = None
SOURCE_IMAGE: np.array = None
imagePosition = (0, 0)
panel = None
scaleValue: float = 1.00

# add checkboxes
cbShowGridVar = BooleanVar()
cbShowGridVar.set(gridDisplayed)
cbShowGrid = Checkbutton(window, text='Отображать сетку', variable=cbShowGridVar, command=displayGrid)
cbShowGrid.place(x=818, y=12)

cbShowWallsVar = BooleanVar()
cbShowWallsVar.set(wallsDisplayed)

def cbShowWallsCommand():
    global wallsDisplayed
    wallsDisplayed = not wallsDisplayed
    refreshCells()

cbShowWalls = Checkbutton(window, text='Отображать препятствия', variable=cbShowWallsVar, command=cbShowWallsCommand)
cbShowWalls.place(x=818, y=35)

cbFillWallsVar = BooleanVar()
cbFillWallsVar.set(wallsHighlighted)
cbFillWalls = Checkbutton(window, text='Закрашивать препятствия', variable=cbFillWallsVar, command=cellsFill)
cbFillWalls.place(x=818, y=58)

# add radiobuttons
rbVar = IntVar()
rbVar.set(0)

rbSetWalls = Radiobutton(window, text='Установить препятствия', value=0, variable=rbVar)
rbSetWalls.place(x=818, y=130)

rbSetImagePos = Radiobutton(window, text='Установить положение\nплана', value=1, variable=rbVar)
rbSetImagePos.place(x=818, y=170)

rbSetExits = Radiobutton(window, text='Установить\nэвакуационные выходы', value=2, variable=rbVar)
rbSetExits.place(x=818, y=210)

rbSetHumans = Radiobutton(window, text='Установить положение\nлюдей', value=3, variable=rbVar)
rbSetHumans.place(x=818, y=250)

# add spinboxes
def refreshCellSize(a=None, b=None, c=None):
    global sbWidthVar, sbHeightVar, sbSetCellHeight, sbSetCellWidth

    if sbWidthVar.get() < 2:
        sbWidthVar.set(2)
    if sbWidthVar.get() > 50:
        sbWidthVar.set(50)

    if sbHeightVar.get() < 2:
        sbHeightVar.set(2)
    if sbHeightVar.get() > 50:
        sbHeightVar.set(50)

    if cbShowGridVar.get():
        displayGrid(True)
    refreshCells()

lbSetCellWidth = Label(window, text='Длина ячейки')
lbSetCellWidth.place(x=818, y=319)
sbWidthVar = IntVar()
sbWidthVar.set(5)
sbWidthVar.trace('w', refreshCellSize)
sbSetCellWidth = Spinbox(window, from_=2, to=50, width=8, textvariable=sbWidthVar, justify='center')
sbSetCellWidth.place(x=918, y=320)

lbSetCellHeight = Label(window, text='Высота ячейки')
lbSetCellHeight.place(x=818, y=349)
sbHeightVar = IntVar()
sbHeightVar.set(5)
sbHeightVar.trace('w', refreshCellSize)
sbSetCellHeight = Spinbox(window, from_=2, to=50, width=8, textvariable=sbHeightVar, justify='center')
sbSetCellHeight.place(x=918, y=350)

# add buttons
btOpenWindow = Button(window, text='Авто-конструктор', command=lambda: openChildWindow(), width=20)
btOpenWindow.place(x=830, y=394)

btTest = Button(window, text='Найти пути', command=lambda: searchPaths(), width=20)
btTest.place(x=830, y=433)

btClear = Button(window, text='Очистить пути', command=lambda: clearPaths(), width=20)
btClear.place(x=830, y=472)

btLoad = Button(window, text='Загрузить изображение', command=lambda: loadImage(), width=20)
btLoad.place(x=830, y=511)

btOpen = Button(window, text='Открыть', command=lambda: openFile(), width=20)
btOpen.place(x=830, y=550)

btSave = Button(window, text='Сохранить', command=lambda: save(), width=20)
btSave.place(x=830, y=589)

# graphic moving cursor cell
cursorCell = canvas.create_rectangle((0, 0), (sbWidthVar.get(), sbHeightVar.get()), outline='blue')

# 6. binding UI event handlers
# - keyboard events
window.bind('<space>', markingSelectionCells)
window.bind('<Control-s>', save)
window.bind('<Control-o>', openFile)

# - mouse events
canvas.bind('<MouseWheel>', resizeImage)
canvas.bind('<Button-1>', mousePress)
canvas.bind('<Motion>', mouseMove)
canvas.bind('<B1-Motion>', mousePressMove)
canvas.bind('<ButtonRelease-1>', mouseUp)

# 7. starting system/program
window.mainloop()


import json
import random

with open('data.json') as file:
    data = json.load(file)
print(data)

class DOM(object):
    """ Class representing the entire
        DOM tree
    """

    class Node:
        """ Node Element of the DOM
        """
        def __init__(self, elem):
            self.tag = None if "tag" not in elem else elem['tag']
            self.id = None if "id" not in elem else elem['id']
            self.classes = [] if 'classes' not in elem else elem['classes']
            self.adj = []
            self.parent = None

        def is_selected(self, selector):
            # Selector method for the instance node
            tag, _id, classes = None, None, []
            s_classes = selector.split(".")
            if len(s_classes) > 1:
                classes = s_classes[1:]
            s_id = s_classes[0].split("#")
            if len(s_id) > 1:
                _id = s_id[1]
            tag = s_id[0]
            # Compare
            req = 0
            obs = 0
            if tag: req+=1
            if _id: req+=1
            if len(classes) > 0: req +=1
            # Compare
            if tag and self.tag == tag: obs+=1
            if _id and self.id == _id: obs+=1
            if len(classes) > 0 and set(classes).issubset(self.classes): obs+=1
            return req == obs

    def __init__(self, hierarchy=None):
        self.nodes = {}
        self.count = 0
        self.root = None
        if hierarchy:
            self.add(hierarchy)

    def __str__(self):
        s = ""
        for k in self.nodes:
            s+="{} => {}".format(
                k,
                [ a.key for a in self.nodes[k].adj ]
            )
            s+="\n"
        return s

    def add_node(self, key, elem):
        new_node = self.Node(elem)
        self.nodes[key] = new_node
        new_node.parent = self.nodes[key]
        self.nodes[key].key = key
        if not self.root:
            self.root = self.nodes[key]

    def add(self, elem, parent_key=""):
        # Unique node key
        if not parent_key:
            node_key = self.gen_key(elem, parent_key)
        else:
            node_key=parent_key
        if node_key not in self.nodes:
            self.add_node(node_key, elem)
        # Check children to recurse
        if 'children' in elem:
            for child in elem['children']:
                child_key = self.gen_key(child, self.nodes[node_key].key)
                self.add_node(child_key, child)
                self.nodes[node_key].adj.append(self.nodes[child_key])
                self.add(child, parent_key=child_key)

    def gen_key(self, elem, path):
        key="({})".format(random.randint(0,1000))
        if "tag" in elem:
            key+=elem["tag"]
        if "id" in elem:
            key+="#{}".format(elem["id"])
        if "classes" in elem:
            key+="".join( [".{}".format(c) for c in elem['classes']] )
        return "{}>{}".format(path,key)

    def find(self, selector):
        # Count the times the conditions are fullfilled
        count = 0
        selector = selector.split(" ")
        # Traverse tree to find node
        def find_compare(node=None, selector_path=None):
            nonlocal count
            if len(selector_path) <= 0:
                return
            # If length of the condition == 1 and node fulfill the condition, increment
            if node.is_selected(selector_path[0]):
                if len(selector_path) == 1:
                    count+=1
                selector_path = selector_path[1:]

            # Traverse the children
            for adj in node.adj:
                find_compare(adj, selector_path=selector_path[:])

        find_compare(self.root, selector_path=selector[:])
        return count

def find_selectors(dom, tests):
    results = []
    for selector in tests:
        results.append(dom.find(selector))
    return results

if __name__=='__main__':
    dom = DOM(data['hierarchy'])
    print("DOM tree:")
    print(dom)
    res = find_selectors(dom, data['tests'])
    print("Count: {}".format(res))


import csv
import json

for i in range(1,8):
	name=str(i)+"-1-2017"
	infile = open(name+".log", "r")
	outfile = open(name+".csv", "w")

	f = csv.writer(outfile)

	logs=[]

	for line in infile:
		logs.append(json.loads(line))

	f.writerow(["username","event_source","name","accept_language","time","agent","page","host","referer","context_user_id","context_org_id","context_path","ip","event","event_type"])

	for log in logs:
		if log.has_key("name")!=True:
			log["name"]=None
		f.writerow([log["username"],log["event_source"],log["name"],log["accept_language"],log["time"],log["agent"],log["page"],log["host"],log["referer"],log["context"]["user_id"],log["context"]["org_id"],log["context"]["course_id"],log["context"]["path"],log["ip"],log["event"],log["event_type"]])
	#print log["event_type"].replace(";","_")
#print logs[81]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#%% ALBUM DE FIGURITAS - COMPRANDO DE 1 EN 1.

import random
import numpy as np

def crear_album(figus_total):
    album = np.zeros(figus_total)
    return album

def album_incompleto(A):
    if 0 in A:
        return True
    else:
        return False

def comprar_figu(figus_total):
    fig = random.randint(0,figus_total-1)
    return fig

def cuantas_figus(figus_total):
    alb = crear_album(figus_total)
    figus = 0
    while album_incompleto(alb) == True:
        me_toco_esta = comprar_figu(figus_total)
        alb[me_toco_esta] = 1
        figus += 1
    return figus

# n_repeticiones = 1000
# figus_total = 6
# simulaciones = [cuantas_figus(figus_total) for _ in range(n_repeticiones)]
# print(f'Si comprás de a 1 figurita, para completar el album de 6 deberías comprar {int(np.mean(simulaciones))} en promedio')

n_repeticiones=100
figus_total = 670
simulaciones = [cuantas_figus(figus_total) for _ in range(n_repeticiones)]
print(f'Si comprás de a 1 figurita, para completar el album de 670 deberías comprar {int(np.mean(simulaciones))} en promedio')
#%% AHORA CON PAQUETES DE 5 FIGUS

import random
import numpy as np

def crear_album(figus_total):
    album = np.zeros(figus_total)
    return album

def album_incompleto(A):
    if 0 in A:
        return True
    else:
        return False

def comprar_paquete(figus_total, figus_paquete):
    paquete =[random.randint(0,figus_total - 1) for _ in range(figus_paquete)]
    return paquete

def cuantos_paquetes(figus_total, figus_paquete):
    alb = crear_album(figus_total)
    paq = 0
    while album_incompleto(alb) == True:
        me_tocaron = comprar_paquete(figus_total,figus_paquete)
        for fig in me_tocaron:
            alb[fig] = 1
        paq += 1
    return paq

n_repeticiones = 100
figus_total = 670
figus_paquete = 5
simulaciones = [cuantos_paquetes(figus_total, figus_paquete) for _ in range(n_repeticiones)]
print(f'Para completar el album se necesitaron en promedio {int(np.mean(simulaciones))} paquetes de figus.')

#%% PLOT QUE EXPLICITARON EN EL GIT
import matplotlib.pyplot as plt
def calcular_historia_figus_pegadas(figus_total, figus_paquete):
    album = crear_album(figus_total)
    historia_figus_pegadas = [0]
    while album_incompleto(album):
        paquete = comprar_paquete(figus_total, figus_paquete)
        while paquete:
            album[paquete.pop()] = 1
        figus_pegadas = (album>0).sum()
        historia_figus_pegadas.append(figus_pegadas)
    return historia_figus_pegadas

#COMENTO LAS SIGUIENTES PORQUE LAS TENGO DE ANTES
#figus_total = 670
#figus_paquete = 5

plt.plot(calcular_historia_figus_pegadas(figus_total, figus_paquete))
plt.xlabel("Cantidad de paquetes comprados.")
plt.ylabel("Cantidad de figuritas pegadas.")
plt.title("La curva de llenado se desacelera al final")
plt.show()

#%%


import pytest
from unittest import mock

from tests_backend.conftest import amock

from wishlist_optimizer.mkm_api import HttpClient, MkmApi

def get_product(
        product_id=123,
        name='Shock',
        expansion='Kaladesh',
        category='Magic Single'):
    return {
        'idProduct': product_id,
        'expansionName': expansion,
        'categoryName': category,
        'enName': name
    }

def get_article(article_id=123, language_id=1, price=0.1, count=1):
    return {
        'idArticle': article_id,
        'language': {'idLanguage': language_id},
        'price': price,
        'seller': {
            'address': {'country': 'D'},
            'username': 'some_seller',
            'idUser': 'some_id',
        },
        'count': count,
    }

@pytest.yield_fixture
def http_client(event_loop):
    with mock.patch.object(HttpClient, 'get'):
        client = HttpClient(event_loop, {'url': 'http://some.url'})
        yield client
        event_loop.run_until_complete(client.close())

@pytest.mark.parametrize(['name'], (
    ('Fire',),
    ('Ice',),
    ('Fire Ice',),
    ('Fire / Ice',),
    ('Fire/Ice',),
    ('Fire//Ice',),
    ('Fire // Ice',)
))
@pytest.mark.asyncio
async def test_handles_split_cards(name, http_client):
    response = [get_product(name='Fire // Ice')]
    http_client.get.return_value = amock(response)
    api = MkmApi(http_client)
    result = await api.get_product_ids(name, expansions=None)
    assert result == [123]

@pytest.mark.parametrize(['name'], (
    ('Search for Azcanta',),
    ('Azcanta, the Sunken Ruin',),
    ('Search for Azcanta Azcanta, the Sunken Ruin',),
    ('Search for Azcanta / Azcanta, the Sunken Ruin',),
    ('Search for Azcanta/Azcanta, the Sunken Ruin',),
    ('Search for Azcanta//Azcanta, the Sunken Ruin',),
    ('Search for Azcanta // Azcanta, the Sunken Ruin',)
))
@pytest.mark.asyncio
async def test_handles_flip_cards(name, http_client):
    response = [
        get_product(name='Search for Azcanta / Azcanta, the Sunken Ruin')
    ]
    http_client.get.return_value = amock(response)
    api = MkmApi(http_client)
    result = await api.get_product_ids(name, expansions=None)
    assert result == [123]

@pytest.mark.asyncio
async def test_filters_out_product_that_are_not_cards(http_client):
    response = amock([
        get_product(category='Magic Single', product_id=123),
        get_product(category='some other category', product_id=456),
    ])
    http_client.get.return_value = response
    api = MkmApi(http_client)
    result = await api.get_product_ids('Shock', expansions=None)
    assert result == [123]

@pytest.mark.parametrize(['name', 'exact'], (
    ('Shock', "false"), ('Opt', "true")
))
@pytest.mark.asyncio
async def test_calls_api_with_correct_params(name, exact, http_client):
    http_client.get.return_value = amock([get_product()])
    api = MkmApi(http_client)
    await api.get_product_ids(name, expansions=None)
    params = {
        'search': name.lower(),
        'exact': exact,
        'idGame': 1
    }
    http_client.get.assert_called_once_with(
        'products/find', params=params, headers={}, field='product'
    )

@pytest.mark.asyncio
async def test_filters_products_by_expansion(http_client):
    response = amock([
        get_product(expansion='Kaladesh', product_id=123),
        get_product(expansion='Dominaria', product_id=456),
        get_product(expansion='Magic 2019', product_id=789),
    ])
    http_client.get.return_value = response
    api = MkmApi(http_client)
    result = await api.get_product_ids(
        'Shock', expansions=['Magic 2019', 'Dominaria']
    )
    assert set(result) == {456, 789}

@pytest.mark.asyncio
async def test_get_article(http_client, app):
    response = amock([get_article()])
    http_client.get.return_value = response
    api = MkmApi(http_client)
    result = await api.get_articles('some_product_id')
    http_client.get.assert_called_once_with(
        'articles/some_product_id',
        params={}, headers={}, field='article'
    )
    assert len(result) == 1
    assert result[0] == {
        'language': 1,
        'price': 0.1,
        'seller_username': 'some_seller',
        'seller_country': 'D',
        'seller_url': 'http://dummy.user.url/some_seller',
        'seller_id': 'some_id',
        'count': 1,
        'id': 123,
    }

@pytest.mark.asyncio
async def test_filters_articles_by_language(http_client, app):
    response = amock([get_article()])
    http_client.get.return_value = response
    api = MkmApi(http_client)
    await api.get_articles('some_product_id', language_id=10)
    params = {'idLanguage': 10}
    http_client.get.assert_called_once_with(
        'articles/some_product_id',
        params=params, headers={}, field='article'
    )

@pytest.mark.parametrize(['foil', 'params'], (
    (True, {'isFoil': 'true'}),
    (False, {'isFoil': 'false'}),
    (None, {})
))
@pytest.mark.asyncio
async def test_filters_articles_by_foil(foil, params, http_client, app):
    response = amock([get_article()])
    http_client.get.return_value = response
    api = MkmApi(http_client)
    await api.get_articles('some_product_id', foil=foil)
    http_client.get.assert_called_once_with(
        'articles/some_product_id',
        params=params, headers={}, field='article'
    )

@pytest.mark.asyncio
async def test_filters_articles_by_min_condition(http_client, app):
    response = amock([get_article()])
    http_client.get.return_value = response
    api = MkmApi(http_client)
    await api.get_articles('some_product_id', min_condition='NM')
    params = {'minCondition': 'NM'}
    http_client.get.assert_called_once_with(
        'articles/some_product_id',
        params=params, headers={}, field='article'
    )


from typing import TYPE_CHECKING

from src import utilities
from src.LevelObject import LevelObject
from src.utilities import Colors, Position
from src.WaterObject import WaterObject

if TYPE_CHECKING:
    from src.level import Level

class WaterFactory:
    poss_water: list[LevelObject] = []

    left: int = 0  # for Rect
    top: int = 0  #  for Rect

    color = Colors.BLUE_LIGHT.value  # for draw rect

    def __init__(self, level: "Level") -> None:
        print("init Water")

        self.HEIGHT_GS = level.HEIGHT_GS
        self.GRID_SIZE = level.GRID_SIZE

        self.width: int = level.GRID_SIZE  # for Rect and Surface
        self.height: int = level.GRID_SIZE  # for Rect ans Surface
        self.rect = (self.width, self.height)

        positions = level.paths.add_water_above_rock(level.GRID_SIZE)

        for position in positions:
            water_object = WaterObject(self.rect, position)
            self.poss_water.append(water_object)

        self.water_collect_positions = self.get_water_collect_positions(level)

        self.water_waterline_positions = self.get_water_waterline_positions(level)

    @property
    def water_objects(self) -> list[LevelObject]:
        res_set = set(self.water_collect_positions + self.water_waterline_positions)
        return list(res_set)

    @property
    def water_line(self) -> float:
        grid_height = self.HEIGHT_GS - self.GRID_SIZE * 2
        water_level = grid_height * (2 / 3)
        return water_level

    # def add_water_above_rock(self, level: "Level") -> None:
    #     for path in level.paths:
    #         new_position = utilities.get_distance_in_direction(
    #             path, "DOWN", level.GRID_SIZE
    #         )
    #         if new_position not in level.paths:
    #             position = Position(path.x, path.y)
    #             water_object = WaterObject(self.rect, position)
    #             self.poss_water.append(water_object)

    def get_position_either_side(self, position: Position, level: "Level"):
        direction = "LEFT"
        grid_size = level.GRID_SIZE
        position_left = utilities.get_distance_in_direction(
            position, direction, grid_size
        )

        direction = "RIGHT"
        grid_size = level.GRID_SIZE
        position_right = utilities.get_distance_in_direction(
            position, direction, grid_size
        )

        return [position_left, position_right]

    def get_water_collect_positions(self, level: "Level") -> list[LevelObject]:
        positions = level.paths.get_list_position_difference(self.poss_water)
        run = True
        while run:
            start = len(self.poss_water)
            for index, p in enumerate(self.poss_water):
                if any(
                    item in self.get_position_either_side(p.position, level)
                    for item in positions
                ):
                    self.poss_water.pop(index)
            end = len(self.poss_water)

            run = start != end

        return self.poss_water

    def get_water_waterline_positions(self, level: "Level") -> list[LevelObject]:
        positions = level.paths.filter_positions_above_height(self.water_line)

        self.water_waterline_positions: list[LevelObject] = [
            WaterObject(self.rect, p) for p in positions
        ]

        return self.water_waterline_positions


import datetime
from dateutil.relativedelta import relativedelta
def minus_one_week():
    return datetime.date.today() - relativedelta(weeks = 1)

from django import template
from courses.wagtail_hooks import SskStudentAttendeeMA
from courses.models import CourseAttendee

register = template.Library()

@register.inclusion_tag(
    'tags/next_dates.html'
)
def render_next_dates( page ):
    children = page.get_upcoming_courses()
    return {
        'page' : page,
        'children' : children
    }

@register.simple_tag()
def get_n_attendees( at, course ):
    return get_attendees_by_type(at, course).count()

@register.simple_tag()
def get_n_attendees_waitlist( at, course ):
    return get_waitlist_by_type(at, course).filter(is_validated = True).count()

@register.simple_tag()
def get_n_waitlist( course ):
    return CourseAttendee.objects.filter(
        waitlist_course = course,
        is_validated = True
    ).count()

@register.simple_tag()
def get_n_confirmed_attendees( at, course, payed = None ):
    qs = get_attendees_by_type(at, course).filter(is_validated = True)
    if payed is None:
        return qs.count()
    if payed is True:
        if at.price:
            return (
                qs.filter(amount__isnull = True).count()
                + qs.filter(amount = 0).count()
                + qs.filter(amount__gt = 0, payed = True).count()
            )
        else:
            return qs.count()
    if payed is False:
        if at.price:
            return qs.filter(amount__gt = 0, payed = False).count()
        else:
            return 0

@register.simple_tag()
def get_n_recent_unconfirmed_attendees_waitlist(at, course ):
    return (
        get_waitlist_by_type(at, course).filter(
            is_validated = False,
            add2course_mail_sent = True,
            add2course_mail_sent_at__gte = minus_one_week()
        ).count()
    )
    # TODO!

@register.simple_tag()
def get_n_recent_unconfirmed_attendees( at, course ):
    return (
        get_attendees_by_type(at, course).
        filter(
            is_validated = False,
            validation_mail_sent = True,
            validation_mail_sent_at__gte = minus_one_week()
        ).count()
    )

@register.simple_tag()
def get_n_total_recent_unconfirmed( course ):
    return (
        CourseAttendee.objects.
        filter(
            related_course = course,
            is_validated = False,
            validation_mail_sent = True,
            validation_mail_sent_at__gte = minus_one_week()
        ).count()
    )

@register.simple_tag()
def get_n_total_confirmed( course, payed = None ):
    n = 0
    for at in course.get_attendee_types().all():
        n = n + get_n_confirmed_attendees( at, course, payed )
    return n

@register.simple_tag()
def get_n_expired_unconfirmed_attendees( at, course ):
    return (
        get_attendees_by_type(at, course).
        filter(
            is_validated = False,
            validation_mail_sent = True,
            validation_mail_sent_at__lt = minus_one_week()
        ).count()
    )
@register.simple_tag()
def get_n_expired_unconfirmed( course ):
    return (
        CourseAttendee.objects.
        filter(
            related_course = course,
            is_validated = False,
            validation_mail_sent = True,
            validation_mail_sent_at__lt = minus_one_week()
        ).count()
    )

@register.simple_tag()
def get_attendees_by_type( at, course ):
    Kls = at.get_attendee_class()
    return Kls.objects.filter( related_course = course )

def get_waitlist_by_type( at, course ):
    Kls = at.get_attendee_class()
    return Kls.objects.filter( waitlist_course = course )

@register.simple_tag()
def get_value(obj, field):
    return getattr(obj,field)

@register.inclusion_tag(
    'tags/courses/specific_attendee_list.html',
    takes_context = True
)
def specific_attendee_list( context, at, instance ):
    additional_fields = []
    additional_fields_title = []
    if hasattr(at.get_attendee_class(), 'additional_admin_fields'):
        for f in at.get_attendee_class().additional_admin_fields:
            additional_fields.append(f[0])
            additional_fields_title.append(f[1])
            print('f is {}'.format(f[1]))

    context.update({
        'has_attendees' : get_n_attendees(at, instance) > 0,
        'attendees' : get_attendees_by_type(at, instance),
        'has_waitlist' : at.waitlist,
        'waitlist' : get_waitlist_by_type( at ,instance),
        'additional_fields' : additional_fields,
        'additional_fields_title' : additional_fields_title,
    })
    return context

@register.inclusion_tag(
    'tags/courses/attendee_in_list.html',
    takes_context = True
)
def attendee_in_list( context, course, Attendee ):
    print (Attendee)
    return {
        'available' : course.get_free_slots(Attendee.get_attendee_class()) > 0,
        'type' : Attendee,
        'page' : course,
        'request' : context['request']
    }

@register.inclusion_tag(
    'tags/courses/attendee_buttons.html',
    takes_context = True
)
def attendee_buttons( context, attendee ):
    ma = SskStudentAttendeeMA()
    BH = ma.get_button_helper_class()
    bh = BH(context['view'] , context['request'], modeladmin = ma )
    if attendee.related_course == context['instance'] :
        btns = bh.get_buttons_for_obj(
            attendee,
            classnames_add=['button-small button-secondary'],
            exclude = ['delete_from_waitlist', 'add_to_course']
        )
    else:
        if attendee.add2course_mail_sent:
            exclude = ['delete', 'add_to_course']
        else:
            exclude = ['delete']
        btns = bh.get_buttons_for_obj(
            attendee,
            classnames_add=['button-small button-secondary'],
            exclude = exclude
        )
#    print (context['view'].model)
#    buttonhelper = ButtonHelper( context['view'], context['request'])
    context.update({'buttons' : btns })
    return context
# @register.inclusion_tag(
#     'tags/registration_overview.html'
# )
# def render_registration_overview( page ):
#     context = { 'page' : page }
#     if page.start_date < date.today():
#         context['in_past'] = True
#         context['available'] = False
#         return context

#     plist = []
#     is_full = page.is_full()

#     for pt in PARTICIPANT_TYPES:
#         if not page.can_register( pt[0] ):
#             continue
#         dat = {
#             'ptype'   : pt[0],
#             'full'    : page.is_full_for( pt[0] ),
#             'waitlist': page.allows_waitlist_for( pt[0] ),
#             'price'   : page.get_prize ( pt[0] )
#         }
#         plist += [ dat ]
#     context['registration_list'] = plist

#     return context

# @register.filter
# def long_ptype( ptype ):
#     d = dict(PARTICIPANT_TYPES)
#     return d[ptype]

# @register.inclusion_tag(
#     'courses/rub_login_form.html'
# )
# def rub_login_form( form ):
#     return { 'form' : form }
# @register.inclusion_tag(
#     'courses/personal_data_form.html'
# )
# def personal_data_form( form ):
#     return { 'form' : form }
# @register.inclusion_tag(
#     'courses/student_data_form.html'
# )
# def student_data_form( form ):
#     return { 'form' : form }

# @register.inclusion_tag(
#     'courses/extended_personal_data_form.html'
# )
# def extended_personal_data_form( form ):
#     return { 'form' : form }

# @register.filter
# def is_container( page ):
#     if issubclass( type(page), AbstractParticipantsContainer ):
#         return True
#     return False


import logging
from pathlib import Path

from rich.logging import RichHandler

try:
    from dotenv import find_dotenv, load_dotenv

    load_dotenv(find_dotenv())
except ModuleNotFoundError:
    pass

# Configure logging
logging.basicConfig(
    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)
rich_handler = RichHandler(rich_tracebacks=True)
logging.getLogger().handlers = [rich_handler]

def get_filepath(filename: str, folder_name: str = "examples") -> str:
    """Function created to take the full path of the base files to
    execute in the scripts.
    Args:
        filename (str): The file name
        folder_name (str, optional): Folder where files are stored..
        Defaults to "examples".
    Returns:
        str: Returns the name of the file, assuming that it has been
        placed in the root folder or in the folder where it should be placed.
    """
    result = filename
    source = Path(folder_name).resolve()
    list_files = [file for file in source.iterdir()]
    for file in list_files:
        if file.name.strip() == filename.strip():
            result = file.as_posix()
    return result


from django.core.exceptions import ValidationError
from django.db.models import Sum
from rest_framework import serializers

from features.models import Feature
from features.multivariate.models import (
    MultivariateFeatureOption,
    MultivariateFeatureStateValue,
)

class NestedMultivariateFeatureOptionSerializer(serializers.ModelSerializer):
    class Meta:
        model = MultivariateFeatureOption

        fields = (
            "id",
            "uuid",
            "type",
            "integer_value",
            "string_value",
            "boolean_value",
            "default_percentage_allocation",
        )
        read_only_fields = ("uuid",)

class MultivariateFeatureOptionSerializer(NestedMultivariateFeatureOptionSerializer):
    class Meta(NestedMultivariateFeatureOptionSerializer.Meta):
        fields = NestedMultivariateFeatureOptionSerializer.Meta.fields + ("feature",)

    def validate(self, attrs):
        attrs = super().validate(attrs)
        total_sibling_percentage_allocation = (
            self._get_siblings(attrs["feature"]).aggregate(
                total_percentage_allocation=Sum("default_percentage_allocation")
            )["total_percentage_allocation"]
            or 0
        )
        total_percentage_allocation = (
            total_sibling_percentage_allocation + attrs["default_percentage_allocation"]
        )

        if total_percentage_allocation > 100:
            raise ValidationError(
                {"default_percentage_allocation": "Invalid percentage allocation"}
            )

        return attrs

    def _get_siblings(self, feature: Feature):
        siblings = feature.multivariate_options.all()
        if self.instance:
            siblings = siblings.exclude(id=self.instance.id)

        return siblings

class MultivariateFeatureStateValueSerializer(serializers.ModelSerializer):
    class Meta:
        model = MultivariateFeatureStateValue
        fields = (
            "id",
            "multivariate_feature_option",
            "percentage_allocation",
        )


import boto3
import json
import os
import sys

readme = """
	A Python script that can be used to upload a directory contents to a bucket in S3
	Used for managing simple S3-hosted websites.

	Usage:

		python upload.py

	This uploads all valid files (except those in the excluced directories specified in config)

		python upload.py index.html error.html css/main.css

	This only uploads the specified files
"""

BASE_DIR = os.path.dirname(os.path.abspath(__file__))

CFG_FILE = "upload_cfg.json"
# CFG file needs to look like so:
demo_cfg_file = """
	{
		"site_name": "<name of the S3 bucket (also domain name of your site",
		"region_name": "<region in which your S3 bucket exists",
		"aws_secret": "<your aws secret>",
		"aws_access_key_id": "<your aws key id>"
		"excluded_dirs": ["assets"]
	}
"""

# map file extensions to ContentTypes
CONTENT_TYPES = {
	'jpg': 'image/jpeg',
	'jpeg': 'image/jpeg',
	'css': 'text/css',
	'html': 'text/html',
	'png': 'image/png',
	'js': 'text/javascript',
	'ico': 'image/x-icon',
	'gif': 'image/gif',
	'svg': 'image/svg+xml',
	'txt': 'text/plain'
}

def get_uploadable_files(directory, excluded_dirs):
	files = []
	for f in os.listdir(directory):
		if f != sys.argv[0] and not f.startswith('.'):
			dir_file = os.path.join(directory.replace('.', ''), f)
			if os.path.isfile(dir_file):
				files.append(dir_file)
			if os.path.isdir(dir_file):
				if dir_file not in excluded_dirs:
					files = files + get_uploadable_files(dir_file, excluded_dirs)
	return files

def get_content_type(filepath):
	suffix = filepath.split('.')[-1]
	return CONTENT_TYPES.get(suffix, None)

def load_config():
	try:
		f = open(os.path.join(BASE_DIR, CFG_FILE), 'r')
	except IOError:
		print("Failed to load config file: %s" % CFG_FILE)
		sys.exit()
	try:
		cfg = json.loads(f.read())
	except ValueError:
		print("Could not parse config file: %s" % CFG_FILE)
		sys.exit()
	return cfg

if __name__ == "__main__":
	cfg = load_config()
	s3 = boto3.client('s3',
		aws_access_key_id=cfg['aws_access_key_id'],
		aws_secret_access_key=cfg['aws_secret'],
		region_name=cfg['region_name']
	)
	files = sys.argv[1:] if len(sys.argv) > 1 else []
	files = files or get_uploadable_files('.', cfg['excluded_dirs'])
	if files:
		print("Uploading files to: %s" % cfg['site_name'])
	for f in files:
		ct = get_content_type(f)
		if ct:
			print("Uploading file: %s" % f)
			try:
				s3.upload_file(os.path.join(BASE_DIR, f), cfg['site_name'], f,  ExtraArgs={'ACL':'public-read', 'ContentType': ct})
			except OSError:
				print("ERROR! Could not find file to upload: %s" % f)
		else:
			print("Not uploading unknown file type: %s" % f)
	print("Done!")


import datetime
from mongoengine import Q
from base.utils.fil import _d, height_to_datetime, datetime_to_height, bson_to_decimal
from explorer.models.deal import Deal, Dealinfo,DealDay,DealStat
from explorer.models.miner import MinerDealPriceHistory
from base.utils.paginator import mongo_paginator
from mongoengine.connection import get_db
from bson.decimal128 import Decimal128

class DealService(object):
    """
    订单服务
    """

    @classmethod
    def get_deal_list(cls, key_words=None, is_verified=0, start_time_height=None, end_time_height=None, page_index=1,
                      page_size=20):
        """
        获取订单列表
        :param key_words:
        :param is_verified:
        :param start_time_height:
        :param end_time_height:
        :param page_index:
        :param page_size:
        :return:
        """
        query = Q()
        if key_words:
            if key_words.isdigit():
                query = Q(deal_id=key_words)
            else:
                query = (Q(client=key_words) | Q(provider=key_words))
        if is_verified == 1:
            query &= Q(is_verified=True)
        if is_verified == 0:
            query &= Q(is_verified=False)
        if start_time_height:
            query &= Q(height__gte=start_time_height)
        if end_time_height:
            query &= Q(height__lte=end_time_height)
        query = Deal.objects(query).order_by("-deal_id")
        result = mongo_paginator(query, page_index, page_size)
        result['objects'] = [info.to_dict(only_fields=("deal_id", "height_time", "client", "provider", "piece_size",
                                                       "is_verified", "storage_price_per_epoch"))
                             for info in result['objects']]

        return result

    @classmethod
    def get_deal_info(cls, deal_id):
        """
        获取订单详情
        :param deal_id:
        :return:
        """

        wallet = Deal.objects(deal_id=deal_id).first()
        data = wallet.to_dict()
        data["start_time"] = height_to_datetime(data["start_epoch"], need_format=True)
        data["end_time"] = height_to_datetime(data["end_epoch"], need_format=True)
        return data

    @classmethod
    def get_deal_info_list(cls, key_words=None, is_verified=0, start_time_height=None, end_time_height=None,
                           page_index=1, page_size=20):
        """
        获取订单列表
        :param key_words:
        :param is_verified:
        :param start_time_height:
        :param end_time_height:
        :param page_index:
        :param page_size:
        :return:
        """
        query = Q()
        if key_words:
            if key_words.isdigit():
                query = Q(deal_id=key_words)
            else:
                query = (Q(client=key_words) | Q(provider=key_words))
        if is_verified == 1:
            query &= Q(verified_deal=True)
        if is_verified == 0:
            query &= Q(verified_deal=False)
        if start_time_height:
            query &= Q(sector_start_epoch__gte=start_time_height)
        if end_time_height:
            query &= Q(sector_start_epoch__lte=end_time_height)
        query = Dealinfo.objects(query).order_by("-sector_start_epoch")
        result = mongo_paginator(query, page_index, page_size)
        result['objects'] = [info.to_dict() for info in result['objects']]
        return result

class DealStatService(object):

    @classmethod
    def sync_deal_stat(cls):
        """
        24小订单统计
        :return:
        """
        obj_dict = {}
        now_height = datetime_to_height(datetime.datetime.now())
        height = now_height - 2880
        # 所有数据
        deal_data = Deal.objects(height__gte=height).aggregate([
            {"$group": {"_id": 0,
                        "data_size": {"$sum": "$piece_size"},
                        "count": {"$sum": 1}}},
        ])
        deal_data = list(deal_data)
        deal_data = deal_data[0] if deal_data else {}
        obj_dict["deal_count"] = deal_data.get("count", 0)
        obj_dict["data_size"] = bson_to_decimal(deal_data.get("data_size", Decimal128("0")))
        # 验证的数据
        verified_deal_data = Deal.objects(height__gte=height, is_verified=True).aggregate([
            {"$group": {"_id": 0,
                        "data_size": {"$sum": "$piece_size"},
                        "count": {"$sum": 1}}},
        ])
        verified_deal_data = list(verified_deal_data)
        verified_deal_data = verified_deal_data[0] if verified_deal_data else {}
        obj_dict["verified_deal_count"] = verified_deal_data.get("count", 0)
        obj_dict["verified_data_size"] = bson_to_decimal(verified_deal_data.get("data_size", Decimal128("0")))
        # 活跃客户
        client_data = Deal.objects(height__gte=height).aggregate([
            {"$project": {"client": 1}},
            {"$group": {"_id": "$client"}},
            {"$group": {"_id": 0,
                        "count": {"$sum": 1}}},
        ])
        client_data = list(client_data)
        client_data = client_data[0] if client_data else {}
        obj_dict["client_count"] = client_data.get("count", 0)
        # 文件数
        piece_cid_data = Deal.objects(height__gte=height).aggregate([
            {"$project": {"piece_cid": 1}},
            {"$group": {"_id": "$piece_cid"}},
            {"$group": {"_id": 0,
                        "count": {"$sum": 1}}},
        ])
        piece_cid_data = list(piece_cid_data)
        piece_cid_data = piece_cid_data[0] if piece_cid_data else {}
        obj_dict["piece_cid_count"] = piece_cid_data.get("count", 0)
        # 成本
        total_gas = _d(0)
        pipeline = [
            {"$match": {"height": {"$gte": height}, "msg_method_name": "PublishStorageDeals"}},
            {"$group": {"_id": 0,
                        "gascost_total_cost": {"$sum": "$gascost_total_cost"}}},
        ]
        table_names = set()
        table_names.add(height_to_datetime(height).strftime("%Y%m"))
        table_names.add(height_to_datetime(now_height).strftime("%Y%m"))
        for table_name in table_names:
            for per in get_db("base")["messages@zone_" + table_name].aggregate(pipeline):
                total_gas += bson_to_decimal(per.get("gascost_total_cost"))
        deal_gas_by_t = total_gas / (deal_data.get("data_size") / _d(1024 ** 4))
        obj_dict["deal_gas_by_t"] = deal_gas_by_t
        DealStat.objects(height=height).upsert_one(**obj_dict)
        # 删除7天前的数据
        DealStat.objects(height__lte=height-(7*2880)).delete()

    @classmethod
    def sync_deal_day(cls, date_str=None):
        now_date = datetime.datetime.now().date()
        if date_str:
            date = datetime.datetime.strptime(date_str, "%Y-%m-%d").date()

        else:
            day_date = "2020-10-15"
            deal_day = DealDay.objects().order_by("-date").first()
            if deal_day:
                day_date = deal_day.date
            date = (datetime.datetime.strptime(day_date, "%Y-%m-%d") + datetime.timedelta(days=1)).date()
            if date > now_date - datetime.timedelta(days=1):
                return
            date_str = date.strftime("%Y-%m-%d")
        # 当天的最后一个高度数据
        obj_dict={}
        height = datetime_to_height(date)
        client_count = len(Deal.objects(height__gte=height, height__lt=height + 2880).distinct(
            "client"))
        obj_dict["client_count"]=client_count
        # 所有数据
        deal_data = Deal.objects(height__gte=height, height__lt=height + 2880).aggregate([
            {"$group": {"_id": 0,
                        "data_size": {"$sum": "$piece_size"},
                        "count": {"$sum": 1}}},
        ])
        deal_data = list(deal_data)
        deal_data = deal_data[0] if deal_data else {}
        obj_dict["deal_count"] = deal_data.get("count", 0)
        obj_dict["data_size"] = bson_to_decimal(deal_data.get("data_size", Decimal128("0")))
        # 验证的数据
        verified_deal_data = Deal.objects(height__gte=height, height__lt=height + 2880, is_verified=True).aggregate([
            {"$group": {"_id": 0,
                        "data_size": {"$sum": "$piece_size"},
                        "count": {"$sum": 1}}},
        ])
        verified_deal_data = list(verified_deal_data)
        verified_deal_data = verified_deal_data[0] if verified_deal_data else {}
        obj_dict["verified_deal_count"] = verified_deal_data.get("count", 0)
        obj_dict["verified_data_size"] = bson_to_decimal(verified_deal_data.get("data_size", Decimal128("0")))
        # 成本
        total_gas = _d(0)
        pipeline = [
            {"$match": {"height": {"$gte": height, "$lt": height + 2880}, "msg_method_name": "PublishStorageDeals"}},
            {"$group": {"_id": 0,
                        "gascost_total_cost": {"$sum": "$gascost_total_cost"}}},
        ]
        table_names = set()
        table_names.add(height_to_datetime(height).strftime("%Y%m"))
        table_names.add(height_to_datetime(height + 2880).strftime("%Y%m"))
        for table_name in table_names:
            for per in get_db("base")["messages@zone_" + table_name].aggregate(pipeline):
                total_gas += bson_to_decimal(per.get("gascost_total_cost"))
        deal_gas_by_t = total_gas / (deal_data.get("data_size") / _d(1024 ** 4))
        obj_dict["deal_gas_by_t"] = deal_gas_by_t
        return DealDay.objects(date=date_str).upsert_one(**obj_dict).id

    @classmethod
    def get_deal_stat(cls):
        """
        24小时的订单数据
        :return:
        """
        deal_stat = DealStat.objects().order_by("-height").first()
        result = deal_stat.to_dict()
        # result["avg_price"] = bson_to_decimal(MinerDealPriceHistory.objects(price__gt=0).average("price"))
        return result

    @classmethod
    def get_deal_day(cls, stats_type):
        """
        按天获取订单数据
        :param stats_type:
        :return:
        """
        limit = int(stats_type[0:stats_type.find("d")])
        days = DealDay.objects().order_by("-date").limit(limit)
        data = []
        for info in days:
            tmp = info.to_dict()
            data.append(tmp)
        if limit == 90:
            data = data[::3]
        if limit == 180:
            data = data[::6]
        if limit == 360:
            data = data[::120]
        return data


"""
Utils that support fringe DataSet features.
"""
import os
import tempfile
import logging
import json
from pbcore.util.Process import backticks

log = logging.getLogger(__name__)

def consolidateBams(inFiles, outFile, filterDset=None):
    """Take a list of infiles, an outFile to produce, and optionally a dataset
    (filters) to provide the definition and content of filtrations."""
    if filterDset and filterDset.filters:
        finalOutfile = outFile
        outFile = _infixFname(outFile, "_unfiltered")
    _mergeBams(inFiles, outFile)
    if filterDset and filterDset.filters:
        _filterBam(outFile, finalOutfile, filterDset)
        outFile = finalOutfile
    _indexBam(outFile)
    _pbindexBam(outFile)

def _pbindexBam(fname):
    cmd = "pbindex {i}".format(i=fname)
    log.info(cmd)
    o, r, m = backticks(cmd)
    if r != 0:
        raise RuntimeError(m)

def _sortBam(fname):
    tmpname = _infixFname(fname, "_sorted")
    cmd = "bamtools sort -in {i} -out {o}".format(i=fname, o=tmpname)
    log.info(cmd)
    o, r, m = backticks(cmd)
    if r != 0:
        raise RuntimeError(m)
    _mvFile(tmpname, fname)

def _mvFile(inFname, outFname):
    cmd = "mv {i} {o}".format(i=inFname,
                              o=outFname)
    log.info(cmd)
    o, r, m = backticks(cmd)
    if r != 0:
        raise RuntimeError(m)

def _cpFile(inFname, outFname):
    cmd = "cp {i} {o}".format(i=inFname,
                              o=outFname)
    log.info(cmd)
    o, r, m = backticks(cmd)
    if r != 0:
        raise RuntimeError(m)

def _indexBam(fname):
    cmd = "samtools index {i}".format(i=fname)
    log.info(cmd)
    o, r, m = backticks(cmd)
    if r != 0:
        raise RuntimeError(m)

def _mergeBams(inFiles, outFile):
    if len(inFiles) > 1:
        cmd = "bamtools merge -in {i} -out {o}".format(i=' -in '.join(inFiles),
                                                       o=outFile)
        log.info(cmd)
        o, r, m = backticks(cmd)
        if r != 0:
            raise RuntimeError(m)
    else:
        _cpFile(inFiles[0], outFile)

def _maxDigits(numItems):
    return len(str(numItems))

def _filterBam(inFile, outFile, filterDset):
    tmpout = tempfile.mkdtemp(suffix="consolidation-filtration")
    filtScriptName = os.path.join(tmpout, "filtScript.json")
    _emitFilterScript(filterDset, filtScriptName)
    cmd = "bamtools filter -in {i} -out {o} -script {s}".format(
        i=inFile, o=outFile, s=filtScriptName)
    log.info(cmd)
    o, r, m = backticks(cmd)
    if r != 0:
        raise RuntimeError(m)

def _infixFname(fname, infix):
    prefix, extension = os.path.splitext(fname)
    return prefix + infix + extension

def _emitFilterScript(filterDset, filtScriptName):
    bamtoolsFilters = ['reference', 'insertSize', 'mapQuality', 'name',
                       'position', 'queryBases']
    filterMap = {'rname': 'reference',
                 'pos': 'position',
                 'length': 'queryBases',
                 'qname': 'name',
                 'rq': 'mapQuality'}
    cheapFilters = True
    for filt in filterDset.filters:
        for req in filt:
            if not filterMap.get(req.name):
                cheapFilters = False
    if cheapFilters:
        script = {"filters":[]}
        for filt in filterDset.filters:
            filtDict = {}
            for req in filt:
                name = filterMap[req.name]
                if name == 'reference':
                    if req.operator == '=' or req.operator == '==':
                        filtDict[name] = req.value
                    else:
                        raise NotImplementedError()
                else:
                    filtDict[name] = req.operator + req.value
            script['filters'].append(filtDict)
    else:
        names = [rec.qName for rec in filterDset]
        script = {"filters":[{"name": name} for name in names]}
    with open(filtScriptName, 'w') as scriptFile:
        scriptFile.write(json.dumps(script))


from flask import Flask, request, jsonify
import string
from tensorflow.keras.models import load_model
import pickle
import spacy

# Initialisation des stopwords
nlp = spacy.load('en_core_web_sm')
stopwords = nlp.Defaults.stop_words

def Preprocess_Sentence(Sentence):
    # Enlever la ponctuation
    Sentence = "".join([i.lower() for i in Sentence if i not in string.punctuation])
    # Enlever les chiffres
    Sentence = ''.join(i for i in Sentence if not i.isdigit())
    # Tokenization : Transformer les phrases en liste de tokens (en liste de mots)
    Sentence = nlp(Sentence)
    # Enlever les stopwords
    Sentence = [i.text for i in Sentence if i.text not in stopwords]
    # Enlever les mots qui ne sont pas alphabétiques ou qui ne sont pas dans le dictionnaire
    Sentence = ' '.join(w for w in Sentence if w.isalpha() and w.lower() in nlp.vocab)
    # Lemmatisation
    Sentence = ' '.join([token.lemma_ for token in nlp(Sentence)])

    return Sentence

# Chargement du tokenizer
with open('tokenizer.pickle', 'rb') as handle:
    tokenizer = pickle.load(handle)

# Chargement du modèle
loaded_model = load_model('advance_model.h5')

app = Flask(__name__)

@app.route('/prediction', methods=['POST'])
def prediction():
    phrase = request.form['phrase']
    sequence = Preprocess_Sentence(phrase)
    sequence = tokenizer.texts_to_sequences([sequence])
    while len(sequence[0]) < 35:
        sequence[0].insert(0, 0)
    prediction = loaded_model.predict(sequence)
    stringprediction = str(prediction[0][0])
    return jsonify({'prediction': stringprediction})

if __name__ == '__main__':
    app.run()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import gzip
import os
import zlib

from tensorflow.python.data.kernel_tests import test_base
from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.data.ops import readers
from tensorflow.python.eager import context
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test
from tensorflow.python.util import compat

try:
  import psutil  # pylint: disable=g-import-not-at-top
  psutil_import_succeeded = True
except ImportError:
  psutil_import_succeeded = False

@test_util.run_all_in_graph_and_eager_modes
class TextLineDatasetTest(test_base.DatasetTestBase):

  def _lineText(self, f, l):
    return compat.as_bytes("%d: %d" % (f, l))

  def _createFiles(self,
                   num_files,
                   num_lines,
                   crlf=False,
                   compression_type=None):
    filenames = []
    for i in range(num_files):
      fn = os.path.join(self.get_temp_dir(), "text_line.%d.txt" % i)
      filenames.append(fn)
      contents = []
      for j in range(num_lines):
        contents.append(self._lineText(i, j))
        # Always include a newline after the record unless it is
        # at the end of the file, in which case we include it
        if j + 1 != num_lines or i == 0:
          contents.append(b"\r\n" if crlf else b"\n")
      contents = b"".join(contents)

      if not compression_type:
        with open(fn, "wb") as f:
          f.write(contents)
      elif compression_type == "GZIP":
        with gzip.GzipFile(fn, "wb") as f:
          f.write(contents)
      elif compression_type == "ZLIB":
        contents = zlib.compress(contents)
        with open(fn, "wb") as f:
          f.write(contents)
      else:
        raise ValueError("Unsupported compression_type", compression_type)

    return filenames

  def _testTextLineDataset(self, compression_type=None):
    test_filenames = self._createFiles(
        2, 5, crlf=True, compression_type=compression_type)

    def dataset_fn(filenames, num_epochs, batch_size=None):
      repeat_dataset = readers.TextLineDataset(
          filenames, compression_type=compression_type).repeat(num_epochs)
      if batch_size:
        return repeat_dataset.batch(batch_size)
      return repeat_dataset

    # Basic test: read from file 0.
    expected_output = [self._lineText(0, i) for i in range(5)]
    self.assertDatasetProduces(
        dataset_fn([test_filenames[0]], 1), expected_output=expected_output)

    # Basic test: read from file 1.
    self.assertDatasetProduces(
        dataset_fn([test_filenames[1]], 1),
        expected_output=[self._lineText(1, i) for i in range(5)])

    # Basic test: read from both files.
    expected_output = [self._lineText(0, i) for i in range(5)]
    expected_output.extend([self._lineText(1, i) for i in range(5)])
    self.assertDatasetProduces(
        dataset_fn(test_filenames, 1), expected_output=expected_output)

    # Test repeated iteration through both files.
    expected_output = [self._lineText(0, i) for i in range(5)]
    expected_output.extend([self._lineText(1, i) for i in range(5)])
    self.assertDatasetProduces(
        dataset_fn(test_filenames, 10), expected_output=expected_output * 10)

    # Test batched and repeated iteration through both files.
    self.assertDatasetProduces(
        dataset_fn(test_filenames, 10, 5),
        expected_output=[[self._lineText(0, i) for i in range(5)],
                         [self._lineText(1, i) for i in range(5)]] * 10)

  def testTextLineDatasetParallelRead(self):
    test_filenames = self._createFiles(10, 10)
    files = dataset_ops.Dataset.from_tensor_slices(test_filenames).repeat(10)
    expected_output = []
    for j in range(10):
      expected_output.extend([self._lineText(j, i) for i in range(10)])
    dataset = readers.TextLineDataset(files, num_parallel_reads=4)
    self.assertDatasetProduces(
        dataset, expected_output=expected_output * 10, assert_items_equal=True)

  def testTextLineDatasetNoCompression(self):
    self._testTextLineDataset()

  def testTextLineDatasetGzipCompression(self):
    self._testTextLineDataset(compression_type="GZIP")

  def testTextLineDatasetZlibCompression(self):
    self._testTextLineDataset(compression_type="ZLIB")

  def testTextLineDatasetBuffering(self):
    test_filenames = self._createFiles(2, 5, crlf=True)

    repeat_dataset = readers.TextLineDataset(test_filenames, buffer_size=10)
    expected_output = []
    for j in range(2):
      expected_output.extend([self._lineText(j, i) for i in range(5)])
    self.assertDatasetProduces(repeat_dataset, expected_output=expected_output)

  def testIteratorResourceCleanup(self):
    filename = os.path.join(self.get_temp_dir(), "text.txt")
    with open(filename, "wt") as f:
      for i in range(3):
        f.write("%d\n" % (i,))
    with context.eager_mode():
      first_iterator = iter(readers.TextLineDataset(filename))
      self.assertEqual(b"0", next(first_iterator).numpy())
      second_iterator = iter(readers.TextLineDataset(filename))
      self.assertEqual(b"0", next(second_iterator).numpy())
      # Eager kernel caching is based on op attributes, which includes the
      # Dataset's output shape. Create a different kernel to test that they
      # don't create resources with the same names.
      different_kernel_iterator = iter(
          readers.TextLineDataset(filename).repeat().batch(16))
      self.assertEqual([16], next(different_kernel_iterator).shape)
      # Remove our references to the Python Iterator objects, which (assuming no
      # reference cycles) is enough to trigger DestroyResourceOp and close the
      # partially-read files.
      del first_iterator
      del second_iterator
      del different_kernel_iterator
      if not psutil_import_succeeded:
        self.skipTest(
            "psutil is required to check that we've closed our files.")
      open_files = psutil.Process().open_files()
      self.assertNotIn(filename, [open_file.path for open_file in open_files])

if __name__ == "__main__":
  test.main()


from create_AWS_cluster import aws_client, aws_resource, parse_configFile
import configparser
import pandas as pd
import boto3
import json
import time

def persist_cluster_configInfo(Redshift, DWH_CLUSTER_IDENTIFIER):
    """
    Writes the cluster endpoint and IAM ARN to the dwh.cfg config file
    @type redshift -- client
    @param redshift -- Redshift resource client

    @type DWH_CLUSTER_IDENTIFIER -- string
    @param DWH_CLUSTER_IDENTIFIER -- value from config file

    @return -- None
    """
    print('Cluster address and IamRoleARN is being written to the config file.')
    cluster_properties = Redshift.describe_clusters(ClusterIdentifier = DWH_CLUSTER_IDENTIFIER)['Clusters'][0]
    config = configparser.ConfigParser()

    config.read_file(open('dwh.cfg'))

    config.set('CLUSTER','HOST',cluster_properties['Endpoint']['Address'])
    config.set('IAM_ROLE','ARN',cluster_properties['IamRoles'][0]['IamRoleArn'])

    with open('dwh.cfg','w+') as configFile:
        config.write(configFile)
    parse_configFile()

def get_cluster_status(redshift,DWH_CLUSTER_IDENTIFIER):
    """
    Checks the Redshift cluster status, retuns True or False
    @type redshift -- redshift
    @param redshift -- AWS Redshift resource client

    @type DWH_CLUSTER_IDENTIFIER -- string
    @param DWH_CLUSTER_IDENTIFIER -- value from config file

    @return -- boolean True or False
    """
    cluster_properties = redshift.describe_clusters(ClusterIdentifier = DWH_CLUSTER_IDENTIFIER)['Clusters'][0]
    clusters_status = cluster_properties['ClusterStatus']
    return clusters_status.lower()

def describe_clust(redshift,DWH_CLUSTER_IDENTIFIER):
    return redshift.describe_clusters(ClusterIdentifier = DWH_CLUSTER_IDENTIFIER)['Clusters'][0]

def prettyRedshiftProps(props):
    pd.set_option('display.max_colwidth', -1)
    keysToShow = ["ClusterIdentifier", "NodeType", "ClusterStatus", "MasterUsername", "DBName", "Endpoint", "NumberOfNodes", 'VpcId']
    x = [(k, v) for k,v in props.items() if k in keysToShow]
    return pd.DataFrame(data=x, columns=["Key", "Value"])

def open_redshift_port (ec2,redshift, DWH_CLUSTER_IDENTIFIER, DB_PORT):
    """
    Redshift post on VPC security group is opened
    @type ec2 -- clinet
    @param ec2 -- AWS EC2 resource client

    @type redshift -- client
    @param redshift -- Redshift resource client

    @return -- none
    """
    myClusterProps = redshift.describe_clusters(ClusterIdentifier = DWH_CLUSTER_IDENTIFIER)['Clusters'][0]
    try:
        vpc = ec2.Vpc(id=myClusterProps['VpcId'])
        secur_group = list(vpc.security_groups.all())
        print(secur_group)
        defaultSg = secur_group[1]

        defaultSg.authorize_ingress(
            GroupName= defaultSg.group_name,
            CidrIp='0.0.0.0/0',
            IpProtocol='TCP',
            FromPort=int(DB_PORT),
            ToPort=int(DB_PORT)
        )
    except Exception as e:
        print(e)

def main():
    """
    Checks the Redshift cluster status and opens the Port
    """
    # parse config file
    configs=parse_configFile()

    #get redshift client
    redshift = aws_client('redshift','us-east-2',configs[0],configs[1])

    # check if cluster was created
        # add port thing, and create ec2 resource object to do so
    if get_cluster_status(redshift,configs[10]):
        print('Cluster is available:')
        print(prettyRedshiftProps(describe_clust(redshift,configs[10])))
        ec2 = aws_resource('ec2','us-east-2',configs[0],configs[1])
        persist_cluster_configInfo(redshift,configs[10])
        open_redshift_port(ec2,redshift,configs[10],configs[5])
        print('Cluster is up and running')
    else:
        print('Cluster is not available yet')

if __name__ =='__main__':
    main()


# import libraries
import time
import datetime
import pandas as pd
from rich import print as rprint
from rich.pretty import pprint
from rich.progress import Progress
import re
from rich.progress import track
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import ElementClickInterceptedException, TimeoutException
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.firefox.options import Options
from selenium.webdriver import Firefox
# ------------------------------------------------------------------
# TODO: functions
# ------------------------------------------------------------------

# This function search each element of the document DOM

# sudo apt install firefox
# pip install webdriver-manager
# sudo apt install firefox-geckodriver

def findElementTextBySelector(selector, exception):
    try:
        element = i.find_element(
            By.CSS_SELECTOR, selector).text.strip().upper()
    except:
        element = exception
    return element

def findElementNumberBySelector(selector, exception):
    try:
        element = i.find_element(
            By.CSS_SELECTOR, selector).text.replace('$', '')
        element = "".join([ch for ch in element if ch.isdigit()])
    except:
        element = exception
    return element

def findElementBy(by, selector, t):
    open_modal = driver.find_element(by, selector)
    open_modal.click()
    time.sleep(t)

def findElementByAndSendKey(by, selector, key, t):
    open_modal = driver.find_element(by, selector)
    open_modal.click()
    open_modal.send_keys(key)
    open_modal.send_keys(Keys.TAB)
    time.sleep(t)

def scrollDownPage(driver, t):
    time.sleep(t)
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

def scrollDownFullPage(driver):
    height = driver.execute_script("return document.body.scrollHeight")
    for i in range(height):
        # scroll by 10 on each iteration
        driver.execute_script('window.scrollBy(0,10)')
        # reset height to the new height after scroll-triggered elements have been loaded.
        height = driver.execute_script("return document.body.scrollHeight")
        time.sleep(0.0005)

# Function Beatiful View

def process_data():
    time.sleep(0.02)

# Date

today = datetime.date.today()

# Categories of brands that should be considered for search results
categories = ['vinos', 'licores', 'tequilas']

shops = {'Bogotá, D.c.', 'Medellín', 'Barranquilla'}

# ------------------------------------------------------------------
# TODO: Extract the data for shop EXITO
# ------------------------------------------------------------------

for city in shops:
    for category in categories:
        # Bar progress -> comment
        for _ in track(range(100), description=f'[green]Iniciando Scraping en Dislicores ciudad {city} categoria: {category}'):
            process_data()
        # Initialized by selenium driver with options and optmizer
        options = Options()
        options.set_preference("network.http.pipelining", True)
        options.set_preference("network.http.proxy.pipelining", True)
        # options.set_preference("network.http.pipelining.maxrequests", 8)
        # options.set_preference("content.switch.threshold", 250000)
        # options.set_preference("browser.cache.memory.capacity", 65536)
        options.set_preference("general.startup.browser", False)
        # Disable reader, we won't need that.
        options.set_preference("reader.parse-on-load.enabled", False)
        options.set_preference("browser.pocket.enabled", False)
        options.set_preference("loop.enabled", False)
        # Text on Toolbar instead of icons
        options.set_preference("browser.chrome.toolbar_style", 1)
        # Don't show thumbnails on not loaded images.
        options.set_preference(
            "browser.display.show_image_placeholders", False)
        # Don't show document colors.
        options.set_preference("browser.display.use_document_colors", False)
        # Don't load document fonts.
        options.set_preference("browser.display.use_document_fonts", 0)
        # Use system colors.
        options.set_preference("browser.display.use_system_colors", True)
        # Autofill on forms disabled.
        options.set_preference("browser.formfill.enable", False)
        # Delete temprorary files.
        options.set_preference("browser.helperApps.deleteTempFileOnExit", True)
        options.set_preference("permissions.default.image", 2)
        # Disable tabs, We won't need that.
        options.set_preference("browser.tabs.forceHide", True)
        # Disable autofill on URL bar.
        options.set_preference("browser.urlbar.autoFill", False)
        # Disable autocomplete on URL bar.
        options.set_preference("browser.urlbar.autocomplete.enabled", False)

        driver = webdriver.Firefox(options=options)
        # driver.maximize_window()
        # driver.set_window_position(2000,0)
        # driver.set_window_size(1180, 1000)
        driver.set_window_position(-50, -50)
        # driver.set_window_size(1500, 1050)
        driver.maximize_window()

        # Open the Page

        driver.get(f"https://www.dislicores.com/{category}")

        time.sleep(10)

        # Click on Modal Window
        try:
            findElementBy(
                By.XPATH, "//span[@class='vtex-minicart-2-x-minicartIconContainer gray relative']//*[@class=' ']", 2)
        except:
            pass
        # ActionChains(driver)

        # Click for city selection
        findElementBy(
            By.XPATH, "//div[@class=' css-1fvtkye']", 2)
        # Select City
        findElementBy(
            By.XPATH, "//div[@id='react-select-2-option-9']", 2)
        # Click Box
        findElementBy(
            By.XPATH, "//span[@class='dislicoresqa-custom-app-2-x-m_custom_age_verification__checkbox mr3 ']", 2)
        # Click button continue
        findElementBy(
            By.XPATH, "//button[normalize-space()='Continuar']", 10)

        scrollDownPage(driver, 5)
        # scrollDownFullPage(driver)

        initial_XPATH = "//div[contains(@class,'vtex-button__label flex items-center justify-center h-100 ph5')]"
        # define the max clicks for page for default 30
        max_click_SHOW_MORE = 45
        # count the number of clicks
        count = 1
        # This loop search the button load more and apply the click until the end of page
        while count <= max_click_SHOW_MORE:
            try:
                WebDriverWait(driver, 30).until(
                    EC.visibility_of_all_elements_located((By.XPATH, initial_XPATH)))
                WebDriverWait(driver, 20).until(
                    EC.element_to_be_clickable((By.XPATH, initial_XPATH))).click()
                count += 1
                time.sleep(10)
                # Bar progress -> comment
                for i in track(range(4), description=f"[red]Explorando Pagina Web iter {count - 1}.........."):
                    time.sleep(1)

            except TimeoutException:
                break

        # Search the elements of the page
        items = driver.find_elements(
            By.CSS_SELECTOR,  ".vtex-product-summary-2-x-element.pointer.pt3.pb4.flex.flex-column.h-100")
        # Create a frame empty for the data
        data = []
        # iterate over each element
        for i in items:
            name = findElementTextBySelector(
                ".vtex-product-summary-2-x-productBrand.vtex-product-summary-2-x-brandName.t-body", "SIN DESCRIPCION")
            unit_measure = findElementNumberBySelector(
                ".vtex-product-summary-2-x-valueWrapper.vtex-product-summary-2-x-valueWrapper--grid.vtex-store-components-3-x-valueWrapper.vtex-product-summary-2-x-skuSelectorItemTextValue.vtex-product-summary-2-x-skuSelectorItemTextValue--grid.vtex-store-components-3-x-skuSelectorItemTextValue.c-on-base.center.pl5.pr5.z-1.t-body", "0")
            brand = findElementTextBySelector(
                ".vtex-product-summary-2-x-productBrandName", "SIN MARCA")
            price_prime = findElementNumberBySelector(
                ".exito-vtex-components-4-x-valuePLPAllied", "0")
            price_regular = findElementNumberBySelector(
                ".vtex-store-components-3-x-listPriceValue.ph2.dib.strike.vtex-store-components-3-x-price_listPrice", "0")
            price_now = findElementNumberBySelector(
                ".vtex-store-components-3-x-sellingPrice.vtex-store-components-3-x-sellingPriceValue.t-heading-2-s.dib.ph2.vtex-store-components-3-x-price_sellingPrice", "0")
            conditional_discount = findElementTextBySelector(
                ".dislicoresqa-custom-app-2-x-a_layout.dislicoresqa-custom-app-2-x-a_layout--productSummaryTagsGrid.flex.flex-column.justify-around.items-stretch.h-100.bg-base", "")
            discount = findElementNumberBySelector(
                ".vtex-store-components-3-x-discountInsideContainer.t-mini.white.absolute.right-0.pv2.ph3.bg-emphasis.z-1", "0")

            data.append({f"shop": "DISLICORES",
                         "city": city,
                         "location": city,
                         "category": category,
                         "name": name,
                         "unit_measure": unit_measure,
                         "brand": brand,
                         "price_prime": price_prime,
                         "price_regular": price_regular,
                         "price_now": price_now,
                         "conditional_discount": conditional_discount,
                         "conditional_discount_2": "",
                         "discount": discount,
                         "date": today
                         })

        df = pd.DataFrame(data)
        df.to_csv(f'/home/pydev/workflow/dt_web_scraping/prod/data/dislicores_{city}_{category}_data_{today}.txt',
                  index=False, encoding='utf-8')

        time.sleep(1)
        driver.quit()

time.sleep(10)
driver.quit()


from bs4 import BeautifulSoup
import requests
import re
from movieapp.models import Movie
from movieapp import models
from django.core.management.base import BaseCommand, CommandError

class Command(BaseCommand):
    url = 'http://www.imdb.com/chart/top'

    response = requests.get(url)

    soup = BeautifulSoup(response.text, 'lxml')

    movies = soup.select('td.titleColumn')
    crew = [a.attrs.get('title') for a in soup.select('td.titleColumn a')]
    ratings = [b.attrs.get('data-value') for b in soup.select('td.posterColumn span[name=ir]')]
    votes = [b.attrs.get('data-value') for b in soup.select('td.ratingColumn strong')]

    for index in range(0, len(movies)):
        movie_string = movies[index].get_text()
        movie = (' '.join(movie_string.split()).replace('.', ''))
        movie_title = movie[len(str(index))+1:-7]
        year = re.search('\((.*?)\)', movie_string).group(1)
        place = movie[:len(str(index))-(len(movie))]

        try:
            new_movie = Movie.objects.get_or_create(movie_name=movie_title,
                                                    release_year = year,
                                                    imdb_rating=ratings[index],
                                                    movie_crew = crew[index],
                                                    total_votes= votes[index],
                                                    movie_location=place)[0]
            new_movie.save()
            print('%s movie added' % (movie_title,))
        except:
            print('%s movie already exists and has been updated' % (movie_title,))

    self.stdout.write('Movie Added/Updated')


import requests
from urllib.request import urlopen
from bs4 import BeautifulSoup

from selenium import webdriver

import time

def get_(link):

    data = open("data.txt","a")

    page = urlopen(link)
    soup = BeautifulSoup(page, 'html.parser')

    title = soup.find(class_ = 'title')
    time = soup.find(class_ = "content-publication-data__updated")

    if title == None:
        title = soup.find(class_ = "playkit-video-info__ep-section")
        time = soup.find(class_ = "playkit-video-info__published-at")

    if title == None:

        title = soup.find(class_ = "catalog-video-title")
        time = soup.find(class_ = "catalog-metadata-published-time")

    data.write("{}\n{}\n".format(title.text, time.text))

    data.close()

def get_all(link):

    driver = webdriver.Chrome()

    driver.get(link)
    def roll():
        #Rola a barra do navegador pra baixo
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        # Eespera 2s par rolar novamente
        time.sleep(2)
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(1)

    roll()
    roll()
    roll()

    # Pega as div's com a classe igual á abaixo
    posts = driver.find_elements_by_xpath("//div[@class='feed-post-body']")

    #salva a página html
    page_source = driver.page_source

    try:
        soup = BeautifulSoup(page_source, 'lxml')
    except Exception as e:
        print(e)

    reviews = []
    reviews_selector = soup.find_all('div', class_='feed-post-body')
    botao = soup.find_all('div', class_="load-more gui-color-primary-bg")
    botao = botao[0].a.get('href')
    print(botao)
    #try para caso de erro o selenium fechar o driver do navegador
    try:
        links = []
        i = 0
        for count, review_selector in enumerate(reviews_selector):

            title = review_selector.find('div', class_='feed-post-body-title gui-color-primary gui-color-hover')
            links.append(title.a.get('href'))
            print(links[i])
            get_(links[i])

            i += 1
            reviews.append(title)

    except Exception as e:
        print(e)
    #fecha o driver do navegador
    driver.close()
    return botao

link = "https://g1.globo.com/tudo-sobre/petrobras/"
link_vale = "https://g1.globo.com/tudo-sobre/vale/"
link_economics = "https://g1.globo.com/economia/"
link_ibm = "https://g1.globo.com/tudo-sobre/ibm/"
link_bb = "https://g1.globo.com/tudo-sobre/banco-do-brasil/"
link_oi = "https://g1.globo.com/tudo-sobre/oi/"

botao = get_all(link)

for i in range(60):

    botao = get_all(botao)


import requests
from bs4 import BeautifulSoup

URL ='https://crawlingstudy-dd3c9.web.app/01'

response = requests.get(URL)

bs = BeautifulSoup(response.text, 'html.parser')
result = bs.find(id='cook')

#print(result)
print(result.text)