#!/usr/bin/env python2.7

import cauv
import cauv.messaging as messaging
import cauv.pipeline as pipeline
import cauv.node

from utils.control import PIDController
from utils.timeaverage import TimeAverage
from cauv.debug import debug, info, warning, error
import AI

import threading
from math import degrees, cos, sin, pi
import time

class FollowPipe(AI.Script):
    class DefaultOptions(AI.Script.DefaultOptions):
        def __init__(self):
            #Pipeline details
            self.ellipses_name = 'pipe'
            self.lines_name = 'pipe'
            self.follows = 2
            #Timeouts
            self.ready_timeout = 30
            self.lost_timeout = 15
            # Calibration
            self.pipe_end     = 0.1 # of image (range +0.5 to -0.5)
            self.use_depth = False
            self.target_width = 0.05 # of image
            self.width_error  = 0.1 # of image
            self.centre_error = 0.2 # of image
            self.align_error  = 10   # degrees
            self.average_time = 1   # seconds
            self.angle_consistency = 0.4
            # Control
            self.prop_speed = 40
            self.strafe_kP  = -100 # if we are to the left of the pipe then the pipe is on the right, so has positive center error
            #we want to move right, ie negative strafe, so set kP negative
            self.depth_kP = -1 # if we are too high the width is too small so the error appears negative
            #we want to dive, ie positive value to add to depth, so kP is negative
            self.initial_direction = 260

    class Debug:
        def __init___(self):
            self.centre_error = 0
            self.angle_error = 0
            self.depth_error = 0
            self.angle = 0
            self.centred = 0
            self.depthed = 0
            self.aligned = 0
            self.pipeEnded = 0
            self.yVal = 0

    def __init__(self, *args, **kwargs):
        aiScript.__init__(self, *args, **kwargs)
        self.node.join("processing")

        # parameters to say if the auv is above the pipe
        self.centred = threading.Event()
        self.aligned = threading.Event()
        self.depthed = threading.Event()
        self.pipeEnded = threading.Event()
        self.ready = threading.Event()
        self.enabled = True

        #debug values
        self.centre_error = 0
        self.angle_error = 0
        self.depth_error = 0
        self.angle = 0
        self.yVal = 0

        # controllers for staying above the pipe
        # proportional: if the value we give is positive, then the values we get is positive*sign(kP)
        self.depthControl = PIDController((self.options.depth_kP,0,0))
        self.strafeControl = PIDController((self.options.strafe_kP,0,0))

        self.yAverage = TimeAverage(1)

    def onLinesMessage(self, m):
        if m.name != self.options.lines_name:
            return
        if len(m.lines):
            #
            #          /
            #  angle  /
            #________/line
            #       /
            #
            # Adjust the AUV's bearing to align with the pipe
            # calculate the average angle of the lines (assumes good line finding)
            self.angle = sum([x.angle for x in m.lines])/len(m.lines)

            # calculate the bearing of the pipe (relative to the sub),
            # mod 180 as we dont want to accidentally turn the sub around
            corrected_angle=90-degrees(self.angle)%180 #TODO this mod 180 probably is not needed from observation of cam following

            #debug
            debug('Average lines direction: %g (uncorrected=%g)' % (corrected_angle, self.angle))
            self.angle_error = corrected_angle

            # work out the actual bearing of the pipe using the current bearing of the AUV
            if self.auv.current_bearing and self.enabled:
                self.auv.bearing((self.auv.current_bearing-corrected_angle)%360) #- as angle is opposite direction to bearing

            if abs(corrected_angle) < self.options.align_error:
                self.aligned.set()
            else:
                self.aligned.clear()

            # set the flags that show if we're above the pipe
            debug('Centred:%s Aligned:%s Depthed:%s' %
                    (self.centred.is_set(), self.aligned.is_set(), self.depthed.is_set())
            )
            if self.centred.is_set() and self.aligned.is_set() and (not self.options.use_depth or self.depthed.is_set()):
                self.ready.set()
                self.log('Pipeline follower managed to align itself over the pipe.')
            else:
                self.ready.clear()

    def onEllipsesMessage(self, ms):
        if ms.name != self.options.ellipses_name:
            return
        try:
            m = ms.ellipses[0]
        except IndexError:
            warning('Incorrect ellipses message received')
        #centering
        debug('Centre error: %f' %(m.centre.x - 0.5))
        if abs(m.centre.x) == 0: #probably an error
            warning('Centre at 0, ignoring')
            self.centred.clear()
            return

        strafe = int(self.strafeControl.update(m.centre.x - 0.5))
        if strafe < -127:
            strafe = -127
        elif strafe > 127:
            strafe = 127
        if self.enabled:
            self.auv.strafe(strafe)
        info('Strafing %i' %(strafe))

        #debug info
        self.centre_error = m.centre.x - 0.5

        # y average is used to find the end of the pipe
        yCoord = self.yAverage.update(m.centre.y - 0.5)
        info("Averaged centre y location coord = %f" % (yCoord,))
        #if y is below a certain point and we are above the pipe
        if(yCoord > self.options.pipe_end) and self.ready.is_set():
            self.pipeEnded.set()
        else: self.pipeEnded.clear()

        #debug
        self.yVal = yCoord

        # set the flag used for determining if we're above the pipe
        if abs(m.centre.x -0.5) < abs(self.options.centre_error):
            self.centred.set() #ie within circle radius centre error
        else:
            self.centred.clear()

        #depthing
        if abs(self.angle%pi-m.angle%pi)>self.options.angle_consistency and \
        abs(self.angle%pi-m.angle%pi)<pi-self.options.angle_consistency:
            debug('Too much variation in angle between lines and ellipse (angles %f, %f respectively)' %(self.angle, m.angle))
            return
        depth = self.depthControl.update(m.minorRadius-self.options.target_width)
        debug(str(m.minorRadius))
        self.depth_error = depth
        if self.options.use_depth:
            debug('Width error: %f, diving: %f' %(m.minorRadius-self.options.target_width, depth))
            self.auv.depth(self.auv.current_depth+depth)
        if abs(self.options.target_width-m.minorRadius) < self.options.width_error:
            self.depthed.set()
        else:
            self.depthed.clear()

    def followPipeUntil(self, condition):
        debug('followPipeUntil: %s', condition.is_set())
        while True:
            self.auv.prop(0)
            # check we're still good to go, giving a little time to re-align the
            # pipe if needed
            debug("Re-aligning with pipe...")
            if not self.ready.wait(self.options.lost_timeout):
                return False

            debug("Above pipe, heading forward...")
            self.auv.prop(self.options.prop_speed)

            # go forward for a bit while we're still above the pipe
            while self.ready.is_set():
                time.sleep(0.2)
                if condition.is_set():
                    return True

    def run(self):
        self.load_pipeline('follow_pipe2')
        # first we need to check we've actually found the pipe
        # the detector doesn't really look for the pipe it just looks
        # for yellow things in the downward camera, so there might be
        # a few false positives
        self.log('Attempting to align over the pipe.')

        # now we wait for messages allowing us to work out how to align with
        # the pipe, but if this is taking too long then just give up as we've
        # probably drifted away from the pipe
        debug('Waiting for ready...')
        if not self.ready.wait(self.options.ready_timeout):
            self.log('Pipeline follower could not position itself over the pipe (timed out).')
            error("Took too long to become ready, aborting")
            return 'ABORT'
        #now make sure we're facing in (roughly) the right direction
        if abs((self.auv.current_bearing - self.options.initial_direction)%360) > 90:
            self.log("Switching direction (was facing wrong way)")
            self.enabled = False
            self.auv.prop(0)
            self.auv.strafe(0)
            self.auv.bearingAndWait((self.auv.current_bearing+180)%360)
            self.enabled = True

        for i in range(self.options.follows):
            self.log('Attempting to follow pipe.')
            self.auv.prop(self.options.prop_speed)
            # follow the pipe along until the end
            if not self.followPipeUntil(self.pipeEnded):
                self.log('Lost the pipeline...')
                error("Pipeline lost on pass %d" %(i,))
                return 'LOST'

            debug('Reached end of pipe')
            self.log("Reached end of pipe.")
            self.auv.prop(0)
            self.auv.strafe(0)
            #if this is the last run, dont turn
            if i == self.options.follows-1:
                break
            debug('Turning (turn %i)' %i)
            self.enabled = False
            self.auv.bearingAndWait((self.auv.current_bearing+180)%360)
            self.enabled = True

        self.log('Finished following the pipe.')
        info('Finished pipe following')
        return 'SUCCESS'

Script = FollowPipe

if __name__ == "__main__":
    FollowPipe.entry()


import os
from collections import Counter

#собираем пути и приводим их к виду 'папка?расширение'
#символ '?' нужен для того, чтобы потом отделить имя папки от расширения

def collect_paths():
    dir_ext = []
    start_path = '.'
    for root, dirs, files in os.walk(start_path):
     for file in files:
       filepath = os.path.join(root, file)
       dr = os.path.split(filepath)[0][2::]
       ext = os.path.splitext(filepath)[1]
       dic = dr + '?' + ext
       dir_ext.append(dic)
    return dir_ext

#делаем словарь частотности вхождений типа 'папка?расширение'
#оставляем только те, которые встречаются больше 1 раза

def freq_dic(dir_ext):
    dic = {}
    freq = dict(Counter(dir_ext))
    for k, v in freq.items():
        if v != 1:
            dic.update({k: v})
    return dic

#отделяем имена папок от расширений
#считаем, в скольких папках есть файлы с одним расширением

def count_dirs(dic):
    spis2 = []
    for k in dic.keys():
        if k.split('?')[0] not in spis2:
         spis2.append(k.split('?')[0])
    print('Папок, в которых встречаются файлы с одинаковыми расширениями: ', len(spis2))

def main():
    dir_ext = collect_paths()
    dic = freq_dic(dir_ext)
    count_dirs(dic)

if __name__ == '__main__':
    main()


#/usr/bin/env python
import datetime
import logging
import random
import time

from slackclient import SlackClient

NUDGE_TEXT = ['hello?', 'any news?', 'could you please click one of the buttons?',
              'still there?']

class SlackWorker(object):
    """
    Iterates over tickets, creates/opens conversations
    """
    def __init__(self, bot_token, domain, workflow):
        self.client = SlackClient(bot_token)
        self.domain = domain
        info = self.client.api_call("auth.test")
        self.bot_id = info['user_id']
        self.workflow = workflow

    def process_tickets(self, tickets):
        """
        Iterate over tickets and engage with users
        """
        for ticket_id, ticket_data in tickets.items():
            slack_user = self.get_slack_user(ticket_data.username)
            # check if there was a conversation with the user regarding this ticket
            conversation = self.client.api_call("conversations.open", users=[slack_user])
            channel_id = conversation['channel']['id']
            some_time_ago = datetime.datetime.now() - datetime.timedelta(hours=3)
            timestamp = time.mktime(some_time_ago.timetuple())
            history = self.client.api_call("conversations.history", channel=channel_id,
                                           oldest=timestamp)
            # search for a last message sent by bot
            last_message = {}
            for message in history['messages']:
                # TODO: find a way to check bot_id
                if 'bot_id' in message and 'blocks' in message and (
                        any([True for b in message['blocks']
                             if b['block_id'] == self.workflow['label']])):
                    last_message = message
                    break
            # first time message
            if not last_message:
                text = self.workflow['message'].format(barcode=ticket_data.barcode)
                choices = self.workflow['choices']
                blocks = self.build_blocks(text, choices, ticket_id)
                logging.debug(blocks)
                self.send_block_message(channel_id, blocks)
            # nudge a user if last message is an unanswered question
            # TODO: needs work
            #if last_message:
            #    history = self.client.api_call("conversations.history", channel=channel_id,
            #                                   oldest=timestamp)
            #    if (not history['messages'][0]['text'] in NUDGE_TEXT and
            #        any(['blocks' in message
            #           for message in history['messages'][:len(NUDGE_TEXT)]])):
            #        self.send_text_message(channel_id, random.choice(NUDGE_TEXT))

    def generate_response(self, json_data):
        """
        Generates a response for an action performed by a user, e.g. button click.
        Returns a dict with ticket id and a message if a ticket needs to ne updated.
        """
        # delete all actions
        logging.debug("incoming %s", json_data)
        no_button_blocks = [
            block for block in json_data['message']['blocks'] if block['type'] != "actions"]
        actions = json_data['actions'][0]
        ticket_id = actions['value']
        action_id = actions['action_id']
        channel_id = json_data['container']['channel_id']
        message_ts = json_data['container']['message_ts']
        self.client.api_call("chat.update", channel=channel_id, ts=message_ts,
                             blocks=no_button_blocks)
        reply_text = "You answered: _{}_".format(actions['text']['text'])
        self.send_text_message(channel_id, reply_text)
        current_step = self.recurse_workflow(self.workflow, action_id)
        update_ticket = ()
        if "choices" in current_step:
            blocks = self.build_blocks(current_step['message'], current_step['choices'],
                                                   ticket_id)
            self.send_block_message(channel_id, blocks)
        elif "update_ticket" in current_step:
            update_ticket = (ticket_id, current_step["update_ticket"])
            self.send_text_message(channel_id, current_step['message'])
        return update_ticket

    def recurse_workflow(self, step, key):
        """
        Recurses and returns requested workflow step
        """
        if not isinstance(step, dict):
            return step
        if "choices" in step.keys():
            if key in step["choices"].keys():
                return step["choices"][key]
            else:
                for choice in step["choices"].keys():
                    result = self.recurse_workflow(step["choices"][choice], key)
                    if result:
                        return result

    def create_button(self, label, action, ticket_id):
        """
        Generates necessary structure for a button.
        """
        return {
            "type": "button",
            "text": {
                "type": "plain_text",
                "text": label,
            },
            "value": str(ticket_id),
            "action_id": str(action)
        }

    def build_blocks(self, text, choices, ticket_id):
        return [
            {
                "type": "section",
                "text": {
                    "type": "plain_text",
                    "text": text,
                },
                "block_id": self.workflow['label']  # this is a key to determine a workflow
            },
            {
                "type": "actions",
                "elements": [self.create_button(data['label'], action, ticket_id)
                             for action, data in choices.items()]
            },
        ]

    def get_slack_user(self, username):
        """
        Returns slack user id based on username
        """
        return self.client.api_call("users.lookupByEmail",
            email="{}@{}".format(username, self.domain))['user']['id']

    def send_text_message(self, channel_id, text):
        """
        Sends a basic text message to a given channel.
        """
        self.client.api_call("chat.postMessage", channel=channel_id, as_user="false", text=text)

    def send_block_message(self, channel_id, blocks):
        """
        Sends given list of blocks to a channel with id.
        """
        self.client.api_call("chat.postMessage", channel=channel_id, as_user="false",
                             blocks=blocks)


import os
import json

from _thread import start_new_thread

from gi.repository import GLib, Gtk, GdkPixbuf, Notify
from .ChatController import ChatController
from .CreateTeamController import CreateTeamController
from .SearchChannelsController import SearchChannelsController
from Mattermost.ChannelModel import ChannelModel

class TeamController:
    __gladeBuilder = None           # Gtk.Builder
    __application = None            # Application
    __window = None                 # Gtk.Window
    __windowTitleTemplate = None    # string
    __serverModel = None            # Mattermost.ServerModel
    __loggedInModel = None          # Mattermost.ServerLoggedInModel
    __teamModel = None              # Mattermost.TeamModel
    __channelControllers = {}       # dict(Mattermost.ChatController)
    __channelTreeIterMap = {}       # dict(Gtk.TreeIter)
    __privateGroupTreeIterMap = {}  # dict(Gtk.TreeIter)
    __directMessageTreeIterMap = {} # dict(Gtk.TreeIter)

    FONT_WEIGHT_NORMAL=400
    FONT_WEIGHT_BOLD=700

    def __init__(self, application, url, username, password, teamName):
        self.__application = application

        self.__gladeBuilder = application.createGladeBuilder('team')
        self.__gladeBuilder.connect_signals(self)

        self.__window = self.__gladeBuilder.get_object('windowTeam')
        self.__windowTitleTemplate = self.__window.get_title()

        self.__serverModel = application.getServerModel(url)

        self.__loggedInModel = self.__serverModel.login(username, password, self.__onConnectionClosed)

        self.__teamModel = self.__loggedInModel.getTeam(teamName)

        self.__loggedInModel.registerPostedListener(self.onMessagePosted)

        # Mattermost.UserModel
        selfUser = self.__loggedInModel.getSelfUser()

        self.__teamModel.registerChannelCreatedListener(self.onChannelCreated, {
            'user_id': selfUser.getId()
        })

    def show(self):
        self.__reload()
        self.__window.show_all()

    def getChatController(self, channel):
        channelId = channel
        if type(channel) == ChannelModel:
            channelId = channel.getId()
        if channelId not in self.__channelControllers:
            if type(channel) != ChannelModel:
                # Mattermost.ChannelModel
                channel = self.__teamModel.getChannel(channelId)

            # ChatController
            chatController = None

            if channel != None:
                chatController = ChatController(self.__application, channel)

            self.__channelControllers[channelId] = chatController
        return self.__channelControllers[channelId]

    def onChannelCreated(self, data=None):
        channelId = data['channel_id']
        teamId = data['team_id']

        # Mattermost.TeamModel
        teamModel = self.__teamModel

        if teamId == teamModel.getId():
            # Mattermost.ChannelModel
            channelModel = teamModel.getChannel(channelId)

            self.__addChannelToView(channelModel)

    def onMessagePosted(self, data=None):

        # Gtk.Builder
        gladeBuilder = self.__gladeBuilder

        # Gtk.ListStore
        liststoreTeamChannels = gladeBuilder.get_object('liststoreTeamChannels')

        # Gtk.ListStore
        liststoreTeamPrivateGroups = gladeBuilder.get_object('liststoreTeamPrivateGroups')

        # Gtk.ListStore
        liststoreTeamDirectMessages = gladeBuilder.get_object('liststoreTeamDirectMessages')

        postJson = data['post']
        postData = json.loads(postJson)

        channelId = postData['channel_id']

        # Mattermost.ChatController
        chatController = self.getChatController(channelId)

        # Mattermost.ChannelModel
        channel = self.__teamModel.getChannel(channelId)

        # Mattermost.UserModel
        user = self.__loggedInModel.getUserById(postData['user_id'])

        # Mattermost.UserModel
        selfUser = self.__loggedInModel.getSelfUser()

        GLib.idle_add(chatController.show)

        if selfUser.getId() != user.getId():
            # Notify.Notification
            notification = Notify.Notification.new("gMattermost", "%s - %s: %s" % (
                channel.getDisplayName(),
                user.getUseName(),
                postData['message']
            ))
            success = notification.show()

        if postData['channel_id'] in self.__channelTreeIterMap:
            # Gtk.TreeIter
            treeIter = self.__channelTreeIterMap[channelId]

            liststoreTeamChannels.set_value(treeIter, 2, self.FONT_WEIGHT_BOLD)

        if postData['channel_id'] in self.__privateGroupTreeIterMap:
            # Gtk.TreeIter
            treeIter = self.__privateGroupTreeIterMap[channelId]

            liststoreTeamPrivateGroups.set_value(treeIter, 2, self.FONT_WEIGHT_BOLD)

        if postData['channel_id'] in self.__directMessageTreeIterMap:
            # Gtk.TreeIter
            treeIter = self.__directMessageTreeIterMap[channelId]

            liststoreTeamDirectMessages.set_value(treeIter, 2, self.FONT_WEIGHT_BOLD)

    def onSearchMoreChannelsClicked(self, button, data=None):
        searchChannels = SearchChannelsController(self.__application, self.__teamModel)
        searchChannels.show()

    def onCreateChannelClicked(self, button, data=None):
        createChannel = CreateTeamController(self.__application, self.__teamModel)
        createChannel.show()

    def onSearchMoreDirectMessagesClicked(self, button, data=None):
        searchChannels = SearchChannelsController(self.__application, self.__teamModel, "direct-message")
        searchChannels.show()

    def onCreatePrivateGroupClicked(self, button, data=None):
        createChannel = CreateTeamController(self.__application, self.__teamModel, "private-group")
        createChannel.show()

    def onTeamChannelRowActivated(self, treeView, treePath, column, data=None):
        # Gtk.TreeView
        # Gtk.TreePath
        # Gtk.TreeViewColumn

        # Gtk.Builder
        gladeBuilder = self.__gladeBuilder

        # Gtk.ListStore
        liststoreTeamChannels = gladeBuilder.get_object('liststoreTeamChannels')

        # Gtk.TreeIter
        treeIter = liststoreTeamChannels.get_iter(treePath)

        channelId = liststoreTeamChannels.get_value(treeIter, 1)

        # ChatController
        chatController = self.getChatController(channelId)
        chatController.show()

    def onTeamDirectMessageRowActivated(self, treeView, treePath, column, data=None):
        # Gtk.TreeView
        # Gtk.TreePath
        # Gtk.TreeViewColumn

        # Gtk.Builder
        gladeBuilder = self.__gladeBuilder

        # Gtk.ListStore
        liststoreTeamDirectMessages = gladeBuilder.get_object('liststoreTeamDirectMessages')

        # Gtk.TreeIter
        treeIter = liststoreTeamDirectMessages.get_iter(treePath)

        remoteUserId = liststoreTeamDirectMessages.get_value(treeIter, 1)

        for channelModelCandidate in self.__teamModel.getChannels():
            if channelModelCandidate.isDirectMessage():
                remoteUser = channelModelCandidate.getDirectMessageRemoteUser()
                if remoteUser != None and remoteUser.getId() == remoteUserId:
                    channelModel = channelModelCandidate

        if channelModel != None:
            # ChatController
            chatController = self.getChatController(channelModel)
            chatController.show()

    def onTeamPrivateGroupRowActivated(self, treeView, treePath, column, data=None):
        # Gtk.TreeView
        # Gtk.TreePath
        # Gtk.TreeViewColumn

        # Gtk.Builder
        gladeBuilder = self.__gladeBuilder

        # Gtk.ListStore
        liststoreTeamPrivateGroups = gladeBuilder.get_object('liststoreTeamPrivateGroups')

        # Gtk.TreeIter
        treeIter = liststoreTeamPrivateGroups.get_iter(treePath)

        channelId = liststoreTeamPrivateGroups.get_value(treeIter, 1)

        # ChatController
        chatController = self.getChatController(channelId)
        chatController.show()

    def __reload(self):
        # MattermostTeamModel
        teamModel = self.__teamModel

        # Gtk.Builder
        gladeBuilder = self.__gladeBuilder

        # Gtk.Image
        imageTeamAvatar = gladeBuilder.get_object('imageTeamAvatar')

        # Gtk.Window
        window = self.__window

        # UserModel
        selfUser = self.__loggedInModel.getSelfUser()

        cacheId = "avatar." + selfUser.getId() + ".png"
        avatarImagePath = self.__application.getCacheFilePath(cacheId)

        if not os.path.exists(avatarImagePath):
            avatarImageData = selfUser.getImage()
            self.__application.putCache(cacheId, avatarImageData)

        if os.path.exists(avatarImagePath):
            # GdkPixbuf.Pixbuf
            pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(
                avatarImagePath,
                width=48,
                height=48,
                preserve_aspect_ratio=False
            )

            imageTeamAvatar.set_from_pixbuf(pixbuf)

        variables = {
            'USERNAME': selfUser.getUseName(),
            'TEAMNAME': teamModel.getName()
        }

        windowTitle = self.__windowTitleTemplate
        for variableKey in variables:
            windowTitle = windowTitle.replace("%"+variableKey+"%", variables[variableKey])
        window.set_title(windowTitle)

        for labelId in ["labelTeamNamesUsername", "labelTeamNamesTeam"]:
            # Gtk.Label
            label = gladeBuilder.get_object(labelId)

            templateText = label.get_text()
            for variableKey in variables:
                templateText = templateText.replace("%"+variableKey+"%", variables[variableKey])
            label.set_text(templateText)

        self.__addChannelsToView(teamModel.getChannels())

    def __addChannelsToView(self, channels):

        # Gtk.Builder
        gladeBuilder = self.__gladeBuilder

        # Gtk.ListStore
        liststoreTeamDirectMessages = gladeBuilder.get_object('liststoreTeamDirectMessages')

        directMessageUserIdToTreeIterMap = {}

        for channel in channels:
            # Mattermost.ChannelModel

            self.__addChannelToView(channel, directMessageUserIdToTreeIterMap)

        # Load all direct-message remote users at once
        # list(Mattermost.UserModel)
        remoteUsers = self.__loggedInModel.getUsersByIds(list(directMessageUserIdToTreeIterMap))

        for remoteUser in remoteUsers:
            # Mattermost.UserModel

            treeIter = directMessageUserIdToTreeIterMap[remoteUser.getId()]
            liststoreTeamDirectMessages.set_value(treeIter, 0, remoteUser.getUseName())

    def __addChannelToView(self, channel, directMessageUserIdToTreeIterMap=None):
        # Mattermost.ChannelModel

        # Gtk.Builder
        gladeBuilder = self.__gladeBuilder

        # Gtk.ListStore
        liststoreTeamChannels = gladeBuilder.get_object('liststoreTeamChannels')

        # Gtk.ListStore
        liststoreTeamPrivateGroups = gladeBuilder.get_object('liststoreTeamPrivateGroups')

        # Gtk.ListStore
        liststoreTeamDirectMessages = gladeBuilder.get_object('liststoreTeamDirectMessages')

        doSetDirectMessageNames = False
        if directMessageUserIdToTreeIterMap == None:
            directMessageUserIdToTreeIterMap = {}
            doSetDirectMessageNames = True

        # TODO: find out if channel's have unread messages

        if channel.isOpen():
            # Gtk.TreeIter
            treeIter = liststoreTeamChannels.append()

            self.__channelTreeIterMap[channel.getId()] = treeIter

            liststoreTeamChannels.set_value(treeIter, 0, channel.getDisplayName())
            liststoreTeamChannels.set_value(treeIter, 1, channel.getId())
            liststoreTeamChannels.set_value(treeIter, 2, self.FONT_WEIGHT_NORMAL) # hasUnreadMessages

        if channel.isDirectMessage():
            # Gtk.TreeIter
            treeIter = liststoreTeamDirectMessages.append()

            self.__directMessageTreeIterMap[channel.getId()] = treeIter

            remoteUserId = channel.getDirectMessageRemoteUserId()

            directMessageUserIdToTreeIterMap[remoteUserId] = treeIter

            if doSetDirectMessageNames:
                # Mattermost.UserModel
                remoteUser = self.__loggedInModel.getUserById(remoteUserId)

                liststoreTeamDirectMessages.set_value(treeIter, 0, remoteUser.getUseName())

            liststoreTeamDirectMessages.set_value(treeIter, 1, remoteUserId)
            liststoreTeamDirectMessages.set_value(treeIter, 2, self.FONT_WEIGHT_NORMAL) # hasUnreadMessages

        if channel.isPrivateGroup():
            # Gtk.TreeIter
            treeIter = liststoreTeamPrivateGroups.append()

            self.__privateGroupTreeIterMap[channel.getId()] = treeIter

            liststoreTeamPrivateGroups.set_value(treeIter, 0, channel.getDisplayName())
            liststoreTeamPrivateGroups.set_value(treeIter, 1, channel.getId())
            liststoreTeamPrivateGroups.set_value(treeIter, 2, self.FONT_WEIGHT_NORMAL) # hasUnreadMessages

    def __onConnectionClosed(self, serverLoggedInModel, exception):
        GLib.idle_add(self.__handleLostConnectionUI, serverLoggedInModel, exception)
        return False

    def __handleLostConnectionUI(self, serverLoggedInModel, exception):
        url = self.__serverModel.getUrl()

        self.__window.hide()

        dialog = Gtk.Dialog(
            "Lost connection to mattermost-server %s" % (url),
            self.__window,
            0,
            (Gtk.STOCK_OK, Gtk.ResponseType.OK)
        )
        dialog.get_content_area().add(Gtk.Label(str(exception)))
        dialog.show_all()
        dialog.run()
        dialog.destroy()

#


# -*- coding: utf-8 -*-

import numpy as np
import copy
import skimage.io as io
import os
import sys
from skimage import morphology
from skimage.morphology import watershed
from skimage import transform
from scipy import ndimage
from argparse import ArgumentParser
from utils import Utils

class Evaluator():
    def __init__(self):
        pass

    def mucov(self, y, y_ans):
        sum_iou = 0
        for i in range(1, y.max()+1):
            mask_y = np.array((y == i) * 1).astype(np.uint8)
            best_iou = 0
            for j in range(1, y_ans.max()+1):
                mask_y_ans = np.array((y_ans == j) * 1).astype(np.uint8)
                best_iou = np.max([self.iou(mask_y, mask_y_ans), best_iou])
            print('best IoU: {}'.format(best_iou))
            sum_iou += best_iou
        return sum_iou / y.max()

    def iou(self, pred, gt):
        countListPos = copy.deepcopy(pred + gt)
        countListNeg = copy.deepcopy(pred - gt)
        TP = len(np.where(countListPos.reshape(countListPos.size)==2)[0])
        FP = len(np.where(countListNeg.reshape(countListNeg.size)==1)[0])
        FN = len(np.where(countListNeg.reshape(countListNeg.size)==-1)[0])
        try:
            iou = TP / float(TP + FP + FN)
        except:
            iou = 0
        return iou

    def qca_watershed(self, seg, det):
        distance = ndimage.distance_transform_edt(seg)
        markers = morphology.label(det, neighbors=4)
        wsimg = watershed(-distance, markers, mask=seg)
        return wsimg

    def detection_evaluator(self, PR, GT, thr):
        numPR, numGT = len(PR), len(GT)
        pare = []
        for gn in range(numGT):
            tmp = 0
            chaild = []
            for pn in range(numPR):
                if np.sum((GT[gn] - PR[pn])**2) < thr**2:
                    chaild.append(pn)
                    tmp += 1
            if tmp > 0:
                pare.append(chaild)
        used = np.zeros(numPR)
        TP = self._search_list(pare, used, 0)

        evals = {}
        FP = numPR - TP
        FN = numGT - TP
        try:
            evals['recall'] = TP / float(TP + FN)
        except:
            evals['recall'] = 0.0
        try:
            evals['precision'] = TP / float(TP + FP)
        except:
            evals['precision'] = 0.0
        try:
            evals['F-measure'] = 2 * evals['recall'] * evals['precision'] / (evals['recall'] + evals['precision'])
        except:
            evals['F-measure'] = 0.0
        try:
            evals['IoU'] = TP / float(TP + FP + FN)
        except:
            evals['IoU'] = 0.0
        return evals

    def _search_list(self, node, used, idx):
        if len(node) == idx:
            return 0
        else:
            tmp = []
            for i in range(len(node[idx])):
                if used[node[idx][i]] == 0:
                    used[node[idx][i]] += 1
                    tmp.append(self._search_list(node, used, idx+1) + 1)
                    used[node[idx][i]] -= 1
                else:
                    tmp.append(self._search_list(node, used, idx+1))
            return np.max(tmp)

if __name__ == '__main__':
    ap = ArgumentParser(description='python evaluation.py')
    ap.add_argument('--indir', '-i', nargs='?', default='../InstanceSegmentationResult/ROI2_Feb25Sun_2018_001348/WatershedSegmentationImages', help='Specify input files (format : csv)')
    ap.add_argument('--outdir', '-o', nargs='?', default='EvaluationMUCov', help='Specify output files directory for create figures')
    ap.add_argument('--labeling4', action='store_true', help='Specify Labeling Flag (Gray Scale Image)')
    ap.add_argument('--labeling8', action='store_true', help='Specify Labeling Flag (Gray Scale Image)')
    args = ap.parse_args()
    argvs = sys.argv
    util = Utils()
    psep = '/'
    opbase = util.createOpbase(args.outdir)

    with open(opbase + psep + 'result.txt', 'w') as f:
        f.write('python ' + ' '.join(argvs) + '\n')

    t_list = [1, 50, 100, 150, 188, 250, 256, 257, 258, 262, 320, 335, 338, 350, 398, 450, 474, 502]
    n_list = [2, 2, 2, 2, 3, 4, 5, 7, 7, 8, 9, 13, 14, 16, 20, 32, 40, 50]

    evaluation = Evaluator()
    all_iou, all_mucov = [], []

    with open(opbase + psep + 'result.txt', 'a') as f:
        for t in t_list:
            if args.labeling4:
                seg = io.imread(args.indir + psep + 'segimg_{0:03d}.tif'.format(t))
                seg = morphology.label(seg, neighbors=4)
            elif args.labeling8:
                seg = io.imread(args.indir + psep + 'segimg_{0:03d}.tif'.format(t))
                seg = morphology.label(seg, neighbors=8)
            else:
                seg = io.imread(args.indir + psep + 'ws_{0:03d}.tif'.format(t))
            labels = np.unique(seg)
            seg = np.searchsorted(labels, seg)
            seg_bin = np.array((seg > 0) * 1).astype(np.uint8)

            gt = io.imread('../images/instance_segmentation_gt/gt_t{}.tif'.format(t))
            gt_bin = np.array((gt > 0) * 1).astype(np.uint8)

            iou = evaluation.iou(seg_bin, gt_bin)
            mucov = evaluation.mucov(seg, gt)
            all_iou.append(iou)
            all_mucov.append(mucov)
            print('tp{}: IoU={}'.format(t, iou))
            print('tp{}: MUCov={}'.format(t, mucov))
            f.write('tp{}: IoU={}\n'.format(t, iou))
            f.write('tp{}: MUCov={}\n'.format(t, mucov))
        mean_iou = np.mean(all_iou)
        std_iou = np.std(all_iou)
        mean_mucov = np.mean(all_mucov)
        std_mucov = np.std(all_mucov)
        print('Mean IoU: {}'.format(mean_iou))
        print('Std IoU: {}'.format(std_iou))
        print('Mean MUCov: {}'.format(mean_mucov))
        print('Std MUCov: {}'.format(std_mucov))
        f.write('Mean IoU={}\n'.format(mean_iou))
        f.write('Std IoU={}\n'.format(std_iou))
        f.write('Mean MUCov={}\n'.format(mean_mucov))
        f.write('Std MUCov={}\n'.format(std_mucov))


# encoding: utf-8

import ntpath
import os

import numpy as np
import PIL
import pytest

from histolab.exceptions import LevelError, SlidePropertyError
from histolab.masks import BiggestTissueBoxMask, TissueMask
from histolab.slide import Slide

from ..fixtures import EXTERNAL_SVS, SVS
from ..unitutil import on_ci
from ..util import load_expectation, load_python_expression

class Describe_Slide:
    def it_knows_its_name(self):
        slide = Slide(
            SVS.CMU_1_SMALL_REGION, os.path.join(SVS.CMU_1_SMALL_REGION, "processed")
        )

        name = slide.name

        assert name == ntpath.basename(SVS.CMU_1_SMALL_REGION).split(".")[0]

    def it_calculate_resampled_nparray_from_small_region_svs_image(self):
        slide = Slide(
            SVS.CMU_1_SMALL_REGION, os.path.join(SVS.CMU_1_SMALL_REGION, "processed")
        )

        resampled_array = slide.resampled_array(scale_factor=32)

        expected_value = load_expectation(
            "svs-images/small-region-svs-resampled-array", type_="npy"
        )
        np.testing.assert_almost_equal(resampled_array, expected_value)

    def it_knows_the_right_slide_dimension(self):
        slide = Slide(
            SVS.CMU_1_SMALL_REGION, os.path.join(SVS.CMU_1_SMALL_REGION, "processed")
        )
        image = PIL.Image.open(SVS.CMU_1_SMALL_REGION)

        dimensions = slide.dimensions

        assert image.size == dimensions
        assert slide.dimensions == (2220, 2967)
        assert image.size == (2220, 2967)

    @pytest.mark.parametrize(
        "fixture_slide, level, expected_value",
        (
            pytest.param(
                EXTERNAL_SVS.LIVER_1,
                0,
                "20.0X",
                marks=pytest.mark.skipif(not on_ci(), reason="To run only on CI"),
            ),
            pytest.param(
                EXTERNAL_SVS.LIVER_1,
                1,
                "5.0X",
                marks=pytest.mark.skipif(not on_ci(), reason="To run only on CI"),
            ),
            pytest.param(
                EXTERNAL_SVS.LIVER_1,
                2,
                "1.25X",
                marks=pytest.mark.skipif(not on_ci(), reason="To run only on CI"),
            ),
            (SVS.CMU_1_SMALL_REGION, 0, "20.0X"),
        ),
    )
    def it_knows_its_magnification_factors(self, fixture_slide, level, expected_value):
        slide = Slide(fixture_slide, "")

        magnification_factor = slide.level_magnification_factor(level=level)

        assert magnification_factor == expected_value

    @pytest.mark.parametrize(
        "fixture_slide, level",
        (
            (SVS.TCGA_CR_7395_01A_01_TS1, 0),
            (SVS.TCGA_CR_7395_01A_01_TS1, 1),
        ),
    )
    def but_it_raises_exception_when_magnification_cannot_be_computed(
        self, fixture_slide, level
    ):
        slide = Slide(fixture_slide, "")

        with pytest.raises(SlidePropertyError) as err:
            slide.level_magnification_factor(level=level)

        assert isinstance(err.value, SlidePropertyError)
        assert (
            str(err.value)
            == f"Native magnification not available. Available slide properties: "
            f"{list(slide.properties.keys())}"
        )

    @pytest.mark.parametrize(
        "fixture_slide, level",
        [
            pytest.param(
                EXTERNAL_SVS.LIVER_1,
                4,
                marks=pytest.mark.skipif(not on_ci(), reason="To run only on CI"),
            ),
        ],
    )
    def and_when_level_is_incorrect(self, fixture_slide, level):
        slide = Slide(fixture_slide, "")

        with pytest.raises(LevelError) as err:
            slide.level_magnification_factor(level=level)

        assert isinstance(err.value, LevelError)
        assert (
            str(err.value)
            == f"Level {level} not available. Number of available levels: 3"
        )

    def it_raises_openslideerror_with_broken_wsi(self):
        slide = Slide(SVS.BROKEN, os.path.join(SVS.BROKEN, "processed"))

        with pytest.raises(PIL.UnidentifiedImageError) as err:
            slide._wsi

        assert isinstance(err.value, PIL.UnidentifiedImageError)
        assert (
            str(err.value) == "Your wsi has something broken inside, a doctor is needed"
        )

    @pytest.mark.parametrize(
        "slide_fixture, tissue_mask, binary_mask, expectation",
        [
            (
                SVS.CMU_1_SMALL_REGION,
                True,
                BiggestTissueBoxMask(),
                "cmu-1-small-region-bbox-location-tissue-mask-true",
            ),
            (
                SVS.CMU_1_SMALL_REGION,
                False,
                BiggestTissueBoxMask(),
                "cmu-1-small-region-bbox-location-tissue-mask-false",
            ),
            (
                SVS.TCGA_CR_7395_01A_01_TS1,
                True,
                BiggestTissueBoxMask(),
                "tcga-cr-7395-01a-01-ts1-bbox-location-tissue-mask-true",
            ),
            (
                SVS.TCGA_CR_7395_01A_01_TS1,
                False,
                BiggestTissueBoxMask(),
                "tcga-cr-7395-01a-01-ts1-bbox-location-tissue-mask-false",
            ),
            (
                SVS.CMU_1_SMALL_REGION,
                True,
                TissueMask(),
                "cmu-1-small-region-tissue-location-tissue-mask-true",
            ),
            (
                SVS.CMU_1_SMALL_REGION,
                False,
                TissueMask(),
                "cmu-1-small-region-tissue-location-tissue-mask-false",
            ),
            (
                SVS.TCGA_CR_7395_01A_01_TS1,
                True,
                TissueMask(),
                "tcga-cr-7395-01a-01-ts1-tissue-location-tissue-mask-true",
            ),
            (
                SVS.TCGA_CR_7395_01A_01_TS1,
                False,
                TissueMask(),
                "tcga-cr-7395-01a-01-ts1-tissue-location-tissue-mask-false",
            ),
        ],
    )
    def it_locates_the_mask(
        self, tmpdir, slide_fixture, tissue_mask, binary_mask, expectation
    ):
        slide = Slide(slide_fixture, os.path.join(tmpdir, "processed"))
        expected_img = load_expectation(
            os.path.join("mask-location-images", expectation),
            type_="png",
        )

        mask_location_img = slide.locate_mask(
            binary_mask, tissue_mask=tissue_mask, scale_factor=3
        )

        np.testing.assert_array_almost_equal(
            np.asarray(mask_location_img), expected_img
        )

    def it_knows_its_properties(self):
        slide = Slide(SVS.CMU_1_SMALL_REGION, "processed")

        properties = slide.properties

        assert isinstance(properties, dict)
        assert properties == load_python_expression("python-expr/slide_properties_dict")


import contextlib
import datetime
import os
import sqlite3

import openpyxl
from dotenv import load_dotenv

class FileFormatError(Exception):
    pass

def validate_row(
        row,
        type_mask=(
            (int, float), str, (int, float), (int, float), (int, float),
            (int, float), (int, float), (int, float), (int, float), (int, float)
        )
):
    if not all(
        [
            isinstance(value, value_type)
            for value, value_type in zip(row, type_mask)
        ]
    ):
        raise FileFormatError

def parse_excel(filename):
    workbook = openpyxl.load_workbook(filename)
    worksheet = workbook.active
    parsed_data = []
    for row in worksheet.iter_rows(4, worksheet.max_row, values_only=True):
        validate_row(row)
        parsed_data.append(list(row))
    return parsed_data

def create_db(cursor):
    cursor.execute(
        '''
        CREATE TABLE parsed_data(
            row_id INT,
            date DATE,
            company VARCHAR(50),
            fact_qliq_data1 INT,
            fact_qliq_data2 INT,
            fact_qoil_data1 INT,
            fact_qoil_data2 INT,
            forecast_qliq_data1 INT,
            forecast_qliq_data2 INT,
            forecast_qoil_data1 INT,
            forecast_qoil_data2 INT
        )
        '''
    )

def add_random_dates(data):
    start_date = datetime.date(2023, 3, 1)
    rows_per_date = max(len(data) // 30 + 1, 2)
    for n, row in enumerate(data):
        date = start_date + datetime.timedelta(days=n // rows_per_date)
        row.insert(1, date)
    return data

def get_table_with_subtotals(cursor):
    query = cursor.execute(
        '''
            SELECT * FROM
            (SELECT * FROM parsed_data
            UNION
            SELECT "", date, "SUBTOTAL", SUM(fact_qliq_data1), SUM(fact_qliq_data2),
            SUM(fact_qoil_data1), SUM(fact_qoil_data2), SUM(forecast_qliq_data1),
            SUM(forecast_qliq_data2), SUM(forecast_qoil_data1),
            SUM(forecast_qoil_data2) FROM parsed_data GROUP BY date
            ORDER BY date)
            UNION ALL
            SELECT "", "", "GRAND TOTAL", SUM(fact_qliq_data1), SUM(fact_qliq_data2),
            SUM(fact_qoil_data1), SUM(fact_qoil_data2), SUM(forecast_qliq_data1),
            SUM(forecast_qliq_data2), SUM(forecast_qoil_data1),
            SUM(forecast_qoil_data2) FROM parsed_data;
            '''
    )
    return query.fetchall()

def print_table(table):
    print(f'{"":^28}{"fact":^24}{"forecast":^24}')
    print(f'{"" :^28}{"Qliq":^12}{"Qoil":^12}{"Qliq":^12}{"Qoil":^12}')
    print(f'{"id":^4}{"date":^12}{"company":^12}{"data1":>6}{"data2":>6}{"data1":>6}'
          f'{"data2":>6}{"data1":>6}{"data2":>6}{"data1":>6}{"data2":>6}')
    for row in table:
        print(f'{row[0]:>4}{row[1]:^12}{row[2]:<12}', end='')
        for value in row[3:]:
            print(f'{value:>6}', end='')
        print()

def main():
    load_dotenv()
    db_path = os.getenv('DB_PATH', 'db.sqlite3')
    raw_data_path = os.getenv('RAW_DATA_PATH', 'raw_data/')
    parsed_data = []
    for file in os.listdir(raw_data_path):
        filename, extension = os.path.splitext(file)
        if extension != '.xlsx' or filename.endswith('_processed'):
            continue
        try:
            parsed_data.extend(parse_excel(os.path.join(raw_data_path, file)))
            os.rename(
                os.path.join(raw_data_path, file),
                os.path.join(raw_data_path, f'{filename}_processed.xlsx'))
        except FileFormatError:
            print(f'Неверный формат данных в файле {file}. Файл не обработан.')

    parsed_data_with_dates = add_random_dates(parsed_data)

    con = sqlite3.connect(db_path)
    cur = con.cursor()
    with contextlib.suppress(sqlite3.OperationalError):
        create_db((cur))

    cur.executemany(
        "INSERT INTO parsed_data VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
        parsed_data_with_dates
    )
    con.commit()
    table_with_subtotals = get_table_with_subtotals(cur)
    con.close()
    print_table(table_with_subtotals)

if __name__ == '__main__':
    main()


# vim: set fileencoding=utf8:
from setuptools import setup, find_packages

version = '0.1.0'

def read(filename):
    import os.path
    return open(os.path.join(os.path.dirname(__file__), filename)).read()
setup(
    name="django-packageutils",
    version=version,
    description = "Utility collection of Django package",
    long_description=read('README.rst'),
    classifiers = [
        'Framework :: Django',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python',
        'Topic :: Internet :: WWW/HTTP',
    ],
    keywords = "django package utilty",
    author = "Alisue",
    author_email = "lambdalisue@hashnote.net",
    url=r"https://github.com/lambdalisue/django-packageutils",
    download_url = r"https://github.com/lambdalisue/django-packageutils/tarball/master",
    license = 'MIT',
    packages = find_packages(),
    include_package_data = True,
    install_requires=[
        'distribute',
        'setuptools-git',
    ],
    test_suite='tests.runtests.runtests',
    tests_require=[
        'django>=1.3',
        'PyYAML',
        'django-author',
    ],
)


#!/usr/bin/env python3

import os
import argparse
import logging
import warnings
from common import (start_container, stop_container, run_in_container, get_container_id, container_mkdir, container_log, copy_log_from_container, image_name,
                    copy_git_to_container, copy_to_container)

warnings.filterwarnings("ignore")
logger = logging.getLogger()
current = os.path.dirname(os.path.realpath(__file__))
parent = os.path.dirname(current)

class Params(object):

    def __init__(self):
        parser = argparse.ArgumentParser()
        parser.add_argument("--container", action="store", help="Container", default="redhat/ubi8")
        parser.add_argument("--script", action="store", help="Script", default="setup.sh")
        parser.add_argument("--log", action="store", help="Script", default="setup.log")
        parser.add_argument("--run", action="store_true")
        parser.add_argument("--start", action="store_true")
        parser.add_argument("--stop", action="store_true")
        parser.add_argument("--refresh", action="store_true")
        parser.add_argument("--minimal", action="store_true")
        parser.add_argument("--logs", action="store_true")
        parser.add_argument("--arm", action="store_true")
        self.args = parser.parse_args()

    @property
    def parameters(self):
        return self.args

def minimal_1(args: argparse.Namespace):
    start_container(args.container)

def manual_1(args: argparse.Namespace):
    global parent
    if args.arm:
        platform = "linux/arm64"
    else:
        platform = "linux/amd64"
    volume = "/opt/couchbase"
    destination = "/usr/local/hostprep"

    container_id = start_container(args.container, platform, volume)
    try:
        container_mkdir(container_id, destination)
        copy_git_to_container(container_id, parent, destination)
        run_in_container(container_id, destination, ["bin/setup.sh", "-s"])
    except Exception:
        raise

def manual_2(args: argparse.Namespace):
    global parent
    if args.arm:
        platform = "linux/arm64"
    else:
        platform = "linux/amd64"
    volume = "/opt/couchbase"
    destination = "/var/tmp"
    container_image = args.container
    hostprep_log_file = "/var/log/hostprep.log"
    setup_log_file = "/var/tmp/setup.log"
    setup_script = os.path.join(parent, 'bin', 'setup.sh')

    container_id = start_container(container_image, platform, volume)
    log_dest = f"{parent}/tests/output/{image_name(container_id)}"
    try:
        copy_to_container(container_id, setup_script, destination)
        run_in_container(container_id, destination, ["./setup.sh", "-s", "-g", "https://github.com/mminichino/host-prep-lib"])
        run_in_container(container_id, destination, ["bundlemgr", "-b", "CBS"])
    except Exception:
        container_log(container_id, log_dest)
        copy_log_from_container(container_id, hostprep_log_file, log_dest)
        copy_log_from_container(container_id, setup_log_file, log_dest)
        raise

def get_logs():
    global parent
    container_id = get_container_id()
    hostprep_log_file = "/var/log/hostprep.log"
    setup_log_file = "/usr/local/hostprep/setup.log"
    log_dest = f"{parent}/tests/output/{image_name(container_id)}"
    container_log(container_id, log_dest)
    copy_log_from_container(container_id, hostprep_log_file, log_dest)
    copy_log_from_container(container_id, setup_log_file, log_dest)

def refresh():
    global parent
    destination = "/usr/local/hostprep"

    container_id = get_container_id()
    try:
        copy_git_to_container(container_id, parent, destination)
    except Exception:
        raise

p = Params()
options = p.parameters

try:
    debug_level = int(os.environ['DEBUG_LEVEL'])
except (ValueError, KeyError):
    debug_level = 3

if debug_level == 0:
    logger.setLevel(logging.DEBUG)
elif debug_level == 1:
    logger.setLevel(logging.ERROR)
elif debug_level == 2:
    logger.setLevel(logging.INFO)
else:
    logger.setLevel(logging.CRITICAL)

logging.basicConfig()

if options.stop:
    container = get_container_id()
    if container:
        stop_container(container)

if options.run:
    manual_1(options)

if options.start:
    manual_2(options)

if options.refresh:
    refresh()

if options.minimal:
    minimal_1(options)

if options.logs:
    get_logs()


from decimal import Decimal
from typing import Union

from django.conf import settings
from django.http import HttpRequest

from apps.coupons.models import Coupon
from apps.shop.models import Product

class Cart:
    """
    Class representing a shopping cart for a user.

    Attributes:
    - session: session data for the cart
    - cart: dictionary of cart items
    - coupon_id: id of the coupon applied to the cart

    Methods:
    - __init__(self, request): Constructor that initializes the cart object
    - __iter__(self): Allows the cart object to be iterated over
    - __len__(self): Returns the total number of items in the cart
    - add(self, product, quantity=1, override_quantity=False): Adds a product to the cart
    - save(self): Saves the cart to the session
    - remove(self, product): Removes a product from the cart
    - clear(self): Clears the cart by removing it from the session
    - get_total_price(self): Returns the total price of all items in the cart
    - coupon(self): Getter method for the coupon applied to the cart
    - get_discount(self): Returns the discount applied to the cart based on the coupon
    - get_total_price_after_discount(self): Returns the total price of all items in the cart
        after applying the coupon discount
    """

    def __init__(self, request: HttpRequest) -> None:
        """
        Constructor that initializes the cart object.

        Parameters:
        - request: the request object
        """
        self.session = request.session
        cart = self.session.get(settings.CART_SESSION_ID)
        if not cart:
            cart = self.session[settings.CART_SESSION_ID] = {}
        self.cart = cart
        self.coupon_id = self.session.get("coupon_id")

    def __iter__(self) -> dict[str, Union[Product, Decimal, int]]:
        """
        Allows the cart object to be iterated over.
        """
        product_ids = self.cart.keys()
        products = Product.objects.filter(id__in=product_ids)

        cart = self.cart.copy()
        for product in products:
            cart[str(product.id)]["product"] = product
        for item in cart.values():
            item["price"] = Decimal(item["price"])
            item["total_price"] = item["price"] * item["quantity"]
            yield item

    def __len__(self) -> int:
        """
        Returns the total number of items in the cart.
        """
        return sum(item["quantity"] for item in self.cart.values())

    def add(self, product: Product, quantity: int = 1, override_quantity: bool = False) -> None:
        """
        Adds a product to the cart.

        Parameters:
        - product: the product to add
        - quantity: the quantity of the product to add (default=1)
        - override_quantity: whether to override the quantity of an
            existing product in the cart (default=False)
        """
        product_id = str(product.id)
        if product_id not in self.cart:
            self.cart[product_id] = {"quantity": 0, "price": str(product.price)}
        if override_quantity:
            self.cart[product_id]["quantity"] = quantity
        else:
            self.cart[product_id]["quantity"] += quantity
        self.save()

    def save(self) -> None:
        """
        Saves the cart to the session.
        """
        self.session.modified = True

    def remove(self, product: Product) -> None:
        """
        Removes a product from the cart.

        Parameters:
        - product: the product to remove
        """
        product_id = str(product.id)
        if product_id in self.cart:
            del self.cart[product_id]
            self.save()

    def clear(self) -> None:
        """
        Remove all items from the cart session.
        """
        del self.session[settings.CART_SESSION_ID]
        self.save()

    def get_total_price(self) -> Decimal:
        """
        Calculate the total price of all items in the cart.

        Returns:
            Decimal: The total price.
        """
        return sum(Decimal(item["price"]) * item["quantity"] for item in self.cart.values())

    @property
    def coupon(self) -> Union[Coupon, None]:
        """
        Get the coupon object associated with the cart.

        Returns:
            Coupon or None: The Coupon object or None if no coupon is associated with the cart.
        """
        if self.coupon_id:
            return Coupon.objects.get(id=self.coupon_id)
        return None

    def get_discount(self) -> Decimal:
        """
        Calculate the discount amount based on the coupon associated with the cart.

        Returns:
            Decimal: The discount amount.
        """
        if self.coupon:
            return self.coupon.discount / Decimal("100") * self.get_total_price()
        return Decimal("0")

    def get_total_price_after_discount(self) -> Decimal:
        """
        Calculate the total price after applying the coupon discount.

        Returns:
            Decimal: The total price after the discount.
        """
        return self.get_total_price() - self.get_discount()


from collections import deque

class MyQueue:

    def __init__(self):
        self.input = deque()
        self.output = deque()

    def push(self, x: int) -> None:
        self.input.append(x)

    def pop(self) -> int:
        if self.empty():
            return -1

        if len(self.output) == 0:
            while self.input:
                self.output.append(self.input.pop())

        return self.output.pop()

    def peek(self) -> int:
        if self.empty():
            return -1

        if len(self.output) == 0:
            while self.input:
                self.output.append(self.input.pop())

        return self.output[-1]

    def empty(self) -> bool:
        return True if len(self.input) + len(self.output) == 0 else False

if __name__ == '__main__':
    obj = MyQueue()
    obj.push(1)
    obj.push(2)
    assert obj.peek() == 1
    assert obj.pop() == 1
    assert obj.empty() == False


import torch

def pos_int_to_base(x, b=2, extend_zero_to_length=0):
	if x == 0:
		return [0]
	out = []
	cur = x
	while cur > 0:
		out.append(int(cur % b))
		cur = int(cur / b)
	if extend_zero_to_length>0 and len(out) < extend_zero_to_length:
		out += [0]*(extend_zero_to_length - len(out))
	return out[::-1]

def compute_index_from_pad_region(region_code, dim_shape, dim_pad):
	original_lower = 0
	original_upper = 0
	out_lower = 0
	out_upper = 0
	if region_code == 1:
		out_upper = dim_pad[0]
		original_lower = dim_shape - dim_pad[0]
		original_upper = dim_shape
		return original_lower, original_upper, out_lower, out_upper

	if region_code == 2:
		out_lower = dim_shape + dim_pad[0]
		out_upper = dim_shape + dim_pad[0] + dim_pad[1]
		original_upper = dim_pad[1]
		return original_lower, original_upper, out_lower, out_upper

	original_upper = dim_shape
	out_lower = dim_pad[0]
	out_upper = dim_shape + dim_pad[0]
	return original_lower, original_upper, out_lower, out_upper

def periodic_padding_3d(x, pad):
	ndim = 3
	m = torch.nn.ConstantPad3d(pad,0)
	out = m(x)
	for i in range(1, 3**ndim):
		region_code = pos_int_to_base(i, 3, ndim)
		x_original_lower, x_original_upper, x_out_lower, x_out_upper =\
			compute_index_from_pad_region(region_code[0], int(x.shape[2]), pad[0:2])
		y_original_lower, y_original_upper, y_out_lower, y_out_upper =\
			compute_index_from_pad_region(region_code[1], int(x.shape[3]), pad[2:4])
		z_original_lower, z_original_upper, z_out_lower, z_out_upper =\
			compute_index_from_pad_region(region_code[2], int(x.shape[4]), pad[4:6])
		if x_out_lower != x_out_upper and\
			y_out_lower != y_out_upper and\
			z_out_lower != z_out_upper:

			out[:,:,x_out_lower: x_out_upper,
			y_out_lower: y_out_upper,
			z_out_lower: z_out_upper] =\
			x[:,:,x_original_lower: x_original_upper,
				y_original_lower: y_original_upper,
				z_original_lower: z_original_upper]
	return out

if __name__ == "__main__":
	x = torch.rand(3,4,2, 2, 2)
	out = periodic_padding_3d(x, pad=(1,1,1,1,1,1))
	print (out.shape)


import numpy as np
import time
import pickle
import json
from copy import deepcopy
from pycvi.cluster import compute_cluster_params, prepare_data

from typing import List, Sequence, Tuple, Union, Any, Dict

from . import Vertex
from . import Edge
from . import Component
from ._set_default_properties import (
    _set_members, _set_zero, _set_model_class, _set_score, _set_k_range,
    _set_sliding_window, _set_transformer, _set_scaler
)
from ._clustering_model import generate_all_clusters, merge_clusters
from ._scores import _compute_ratio_scores, _compute_score_bounds
from ._analysis import k_info, get_relevant_k

from ..utils.sorted_lists import ( has_element )
from ..utils.d3 import jsonify

class PersistentGraph():

    def __init__(
        self,
        members: np.ndarray = None,
        time_axis: np.ndarray = None,
        w: int = 1,
        weights: np.ndarray = None,
        precision: int = 13,
        score: str = None,
        transformer = None,
        scaler = None,
        DTW: bool = False,
        zero_type: str = 'variance',
        model_class = None,
        k_max : int = 5,
        name: str = None,
        model_kw: dict = {},
        fit_predict_kw: dict = {},
        model_class_kw: dict = {},
    ):
        """
        Initialize an empty graph

        :param members: (N, T, d)-array. Original data, ensemble of time series

        - N: number of members
        - d: number of variables (default: 1)
        - T: length of the time series (default: 1)

        :type members: np.ndarray

        :param time_axis: (T)-array. Time axis, mostly used for plotting,
        defaults to None, in that case, ``time_axis = np.arange(T)``
        :type time_axis: np.ndarray, optional

        :param weights: Only here for parameter compatibility with the naive
        version, defaults to None
        :type weights: np.ndarray, optional

        :param precision: Score precision, defaults to 13
        :type precision: int, optional

        :param score: Define how the score is computed,
        defaults to 'inertia'. Scores available:

        - inertia
        - min / max_inertia
        - variance
        - min / max_variance

        :type score: optional
        :param zero_type: Define the "0" cluster, defaults to 'uniform'
        :type zero_type: str, optional
        :param verbose: Level of verbosity (defaults to False):

        - 0 (False) Nothing
        - 1 (True) Steps overview
        - 2 Steps overview + more details about vertices and edges
        :type verbose: Union[bool,int], optional

        :param name: Name of the graph object, mostly used for saving,
        defaults to None
        :type name: str, optional
        :raises ValueError: [description]
        """

        # --------------------------------------------------------------
        # ---------------------- About the data ------------------------
        # --------------------------------------------------------------

        if members is not None:
            # After that pg._members is of shape (N, T, d) even if
            # d and T were initially omitted
            _set_members(self, members)
            _set_sliding_window(self, w)
            _set_zero(self, zero_type)

            # Determines how to scale the clustering data
            _set_scaler(self, scaler)
            # Determines how to transform the clustering data
            _set_transformer(self, transformer)

            # Shared x-axis values among the members
            if time_axis is None:
                self._time_axis = np.arange(self.T)
            else:
                self._time_axis = np.copy(time_axis)

            if weights is None:
                self._weights = np.ones((self.T, self.d), dtype = float)
            else:
                self._weights = np.array(weights)
                if len(self._weights.shape) < 2:
                    self._weights = np.expand_dims(self._weights, axis=0)

            # --------------------------------------------------------------
            # ---------------- About the clustering method  ----------------
            # --------------------------------------------------------------

            # Determines how to cluster the members
            _set_model_class(
                self, model_class, DTW, model_kw, fit_predict_kw,
                model_class_kw)
            # Score type, determines how to measure how good a model is
            _set_score(self, score)
            # Set k_range while taking into account the condition on k
            # of the given score.
            _set_k_range(self, k_max)

            if name is None:
                self._name = self._model_type + "_" + str(self._score)
            else:
                self._name = name

            # --------------------------------------------------------------
            # --------- About the graph:   graph's attributes --------------
            # --------------------------------------------------------------

            # Total number of vertices/edges created at each time step
            self._nb_vertices = np.zeros((self.T), dtype=int)
            self._nb_edges = np.zeros((self.T-1), dtype=int)
            # Nested list (time, nb_vertices/edges) of vertices/edges
            # Here are stored Vertices/Edges themselves, not only their num
            self._vertices = [[] for _ in range(self._T_w)]
            self._edges = [[] for _ in range(self._T_w-1)]
            # Nested list (time, nb_local_steps) of dict storing info about
            # The successive steps
            self._local_steps = [[] for _ in range(self._T_w)]
            self._k_info = None
            self._life_span_max = None
            self._life_span_min = None
            self._relevant_k = None
            self._max = None
            self._min = None

            # Score precision
            if precision <= 0:
                raise ValueError("precision must be a > 0 int")
            else:
                self._precision = int(precision)

            # --------------------------------------------------------------
            # --------- About the graph:   algo's helpers ------------------
            # --------------------------------------------------------------

            # To find members' vertices and edges more efficiently
            #
            # List of dictionary of arrays of length N
            # members_v_distrib[t][k][i] is the vertex num of the
            # ith member at t for the assumption k.
            self._members_v_distrib = [
                {
                    k: np.zeros(self.N)
                    # Take into account the available k and ignore the
                    # case k=0
                    for k in self._k_range if k>=1
                }
                for _ in range(self._T_w)
            ]

            # List of length T_w of scores
            self._best_scores = None
            self._worst_scores = None

            self._verbose = False
            self._quiet = False

    def _add_vertex(
        self,
        list_k: List[int],
        info: Dict[str, Any],
        t: int,
        members: List[int],
    ):
        """
        Add a vertex to the current graph

        :param list_k: Number(s) of clusters in which this vertex exist
        :type list_k: List[int]
        :param t: time step at which the vertex should be added
        :type t: int
        :param members: Ordered list of members indices represented by the
        vertex
        :type members: List[int]
        :return: The newly added vertex
        :rtype: Vertex
        """

        info_complete = {
            **{"k": list_k},
            **info
        }

        # Compute the ratio intervals
        score_ratios = []
        for k in list_k:
            score_ratios = Component.ratio_union(
                score_ratios, [self._k_info[k]["score_ratios"][t]]
            )

        # Create the vertex
        num = self._nb_vertices[t]
        v = Vertex(
            info = info_complete,
            t = t,
            num = num,
            members = members,
            score_ratios = score_ratios,
            total_nb_members = self.N,
        )

        # Update the graph with the new vertex
        self._nb_vertices[t] += 1
        self._vertices[t].append(v)
        for k in info_complete["k"]:
            self._members_v_distrib[t][k][members] = num

        return v

    def _add_edge(
        self,
        v_start: Vertex,
        v_end: Vertex,
    ):
        """
        Add an edge to the current graph

        Return None if the edge is malformed

        :param v_start: Vertex from which the edge comes
        :type v_start: Vertex
        :param v_end: Vertex to which the edge goes
        :type v_end: Vertex
        :return: The newly added edge
        :rtype: Edge
        """
        t = v_start.time_step

        members = v_start.get_common_members(v_end)
        if not members:
            if not self._quiet:
                print("WARNING: No members in edge")
            return None

        # --------- Compute scores and ratios info --------------
        ratios = Component.ratio_intersection(
            v_start.score_ratios, v_end.score_ratios
        )

        # -------------- Compute edge info --------------

        # Compute info (mean, std inf/sup at start and end)
        # Option 1: Re-compute cluster params naively from original members
        # Note that this is not very consistent with DTW.....
        # X_start = self._members[members, :, t]
        # info_start = compute_cluster_params(X_start)
        # X_end = self._members[members, :, t+1]
        # info_end = compute_cluster_params(X_end)

        # Option 2: Use v.info["X"] that we are now storing!
        info_start = Edge.info(v_start, members)
        info_end = Edge.info(v_end, members)

        e = Edge(
            info_start = info_start,
            info_end = info_end,
            v_start = v_start.num,
            v_end = v_end.num,
            t = t,
            num = self._nb_edges[t],
            members = members,
            score_ratios = ratios,
            total_nb_members = self.N,
        )

        # Add edge number to v_start and v_end
        v_start.add_edge_from(e.num)
        v_end.add_edge_to(e.num)

        # Update the graph with the new edge
        self._nb_edges[t] += 1
        self._edges[t].append(e)
        return e

    def get_alive_vertices(
        self,
        ratio: float = None,
        t: Union[Sequence[int],int] = None,
        get_only_num: bool = True
    ) -> Union[List[Vertex], List[int]]:
        """
        Extract alive vertices

        If ``t`` is not specified then returns a nested list of
        alive vertices for each time steps

        :param ratio: Ratio at which vertices should be alive
        :type ratio: float, optional
        :param t: Time step from which the vertices should be extracted
        :type t: int, optional
        :param get_only_num: Return the component or its num only?
        :type get_only_num: bool, optional
        """
        # -------------------- Initialization --------------------------
        v_alive = []
        if t is None:
            return_nested_list = True
            t_range = range(self._T_w)
        else:
            if isinstance(t, int):
                return_nested_list = False
                t_range = [t]
            else:
                return_nested_list = True
                t_range = t

        # -------------------- Using ratio ---------------------------
        for t in t_range:

            v_alive_t = [
                v for v in self._vertices[t] if v.is_alive(ratio)
            ]

            if not get_only_num:
                v_alive_t = [
                    self._vertices[t][v_num] for v_num in v_alive_t
                ]
            v_alive.append(v_alive_t)

        if not return_nested_list:
            v_alive = v_alive[0]
        return v_alive

    def get_alive_edges(
        self,
        ratio: float = None,
        t: Union[Sequence[int],int] = None,
        get_only_num: bool = True
    ) -> Union[List[Vertex], List[int]]:
        """
        Extract alive edges

        If ``t`` is not specified then returns a nested list of
        alive edges for each time steps

        :param ratio: Ratio at which edges should be alive
        :type ratio: float, optional
        :param t: Time step from which the edges should be extracted
        :type t: int, optional
        :param get_only_num: Return the component or its num only?
        :type get_only_num: bool, optional
        """
        # -------------------- Initialization --------------------------
        e_alive = []
        if t is None:
            return_nested_list = True
            t_range = range(self._T_w-1)
        else:
            if isinstance(t, int):
                return_nested_list = False
                t_range = [t]
            else:
                return_nested_list = True
                t_range = t

        # -------------------- Using ratio ---------------------------
        for t in t_range:

            e_alive_t = [
                e for e in self._edges[t] if e.is_alive(ratio)
            ]

            if not get_only_num:
                e_alive_t = [
                    self._edges[t][e_num] for e_num in e_alive_t
                ]
            e_alive.append(e_alive_t)

        if not return_nested_list:
            e_alive = e_alive[0]
        return e_alive

    def _construct_vertices(
        self,
        clusterings_t_k: List[Dict[int, List[List[int]]]],
        listk_t_k: List[Dict[int, List[List[int]]]],
    ) -> None:
        # list of T (if sliding window) or 1 array(s) of shape:
        # (N, T|w_t, d) if DTW
        # (N, (T|w_t)*d) if not DTW
        data_params = prepare_data(
            self._members, DTW=self._DTW, window=self._sliding_window,
            transformer=None, scaler=None)

        # `clusterings_t_k[t_w][k][i]` is a list of members indices
        # contained in cluster i for the clustering assuming k clusters
        # for the extracted time window t_w.
        for t_w in range(len(clusterings_t_k)):
            if self._sliding_window is None:
                midpoint = None
            else:
                midpoint = self._sliding_window["midpoint_w"][t_w]
            for k, clustering_t in clusterings_t_k[t_w].items():
                    # create all v
                    new_vertices = [
                        self._add_vertex(
                                list_k=listk_t_k[t_w][k][i_c],
                                info=compute_cluster_params(
                                    data_params[t_w][cluster],
                                    midpoint
                                ),
                                t=t_w,
                                members=cluster,
                            ) for i_c, cluster in enumerate(clustering_t)
                        ]

    def _construct_edges(self):
        for t in range(self._T_w-1):
            v_starts = self._vertices[t]
            v_ends = self._vertices[t+1]

            for v_start in v_starts:
                nb_new_edges = 0
                # Take all vertices at t and all at t+1 If they are
                # contemporaries and have common members then draw an
                # edge
                v_ends_contemp = [
                    v_end for v_end in v_ends
                    if (Component.contemporaries(v_start, v_end)
                    and Component.have_common_members(v_start, v_end))
                ]
                for v_end in v_ends_contemp:

                    e = self._add_edge(
                        v_start = v_start,
                        v_end = v_end,
                    )
                    if e is not None:
                        nb_new_edges += 1

                if self._verbose == 2 and nb_new_edges == 0:
                    print(
                        "WARNING! \nt: {}, no new edges going from v: {}"
                        %(t, v_start.num)
                    )

    def _compute_statistics(self):
        # Max/min (N, T, d)
        self._max = np.amax(self._members, axis=0)
        self._min = np.amin(self._members, axis=0)
        # max/min life span
        life_spans = []
        for v_t in self._vertices:
            life_spans += [v.life_span for v in v_t]
        self._life_span_max = np.amax(life_spans)
        self._life_span_min = np.amin(life_spans)
        return None

    def construct_graph(
        self,
        k_max : int = None,
        pre_prune: bool = False,
        pre_prune_threshold: float = 0.30,
        post_prune: bool = False,
        post_prune_threshold: float = 0.05,
        verbose: Union[bool,int] = False,
        quiet: bool = False,
    ):

        self._verbose = verbose
        self._quiet = (quiet or not verbose)
        self._pre_prune = pre_prune
        self._pre_prune_threshold = pre_prune_threshold
        self._post_prune = post_prune
        self._post_prune_threshold = post_prune_threshold

        # ================== Cluster all the data ======================
        t_start = time.time()
        if self._verbose:
            print("Clustering data...")
        clusterings_t_k = generate_all_clusters(self)
        t_end = time.time()
        if self._verbose:
            print('Data clustered in %.2f s' %(t_end - t_start))

        # ================== Compute score bounds ======================
        _compute_score_bounds(self)

        # ================= Compute ratio scores =======================
        _compute_ratio_scores(self)

        # =================== default k values =========================
        self._k_info = k_info(self)
        relevant_k = get_relevant_k(self)
        dict_relevant_k = {}
        dict_relevant_k["k"] = [x[0] for x in relevant_k]
        dict_relevant_k["life_span"] = [x[1] for x in relevant_k]
        self._relevant_k = dict_relevant_k

        # ================== Construct vertices ========================
        # Remove the case k=0 from the clusters and list_k now that
        # k_info, and ratios and score bounds have been computed
        for clusterings_t in clusterings_t_k:
            clusterings_t.pop(0, None)

        # Merge clusters that are identical
        clusterings_t_k, listk_t_k = merge_clusters(clusterings_t_k)

        t_start = time.time()
        if self._verbose:
            print("Construct vertices...")
        self._construct_vertices(clusterings_t_k, listk_t_k)
        t_end = time.time()
        if self._verbose:
            print('Vertices constructed in %.2f s' %(t_end - t_start))

        # =================== Construct edges ==========================
        if self._verbose:
            print("Construct edges...")
        t_start = time.time()
        self._construct_edges()
        t_end = time.time()
        if self._verbose:
            print('Edges constructed in %.2f s' %(t_end - t_start))

        # =================== Compute statistics =======================
        self._compute_statistics()

    def get_relevant_components(
        self,
        selected_k: List[int] = None,
        k_max: int = 8,
        fill_holes: bool = True,
    ) -> Tuple[List[Vertex], List[Edge]]:
        """
        Return a deep copy of most relevant vertices and edges

        Potentially fills holes in edges

        :type selected_k: List[int], optional
        :param k_max: Max value of k considered, defaults to 8
        :type k_max: int, optional
        :return: Relevant vertices and edges
        :rtype: Tuple[List[Vertex], List[Edge]]
        """
        k_max = min(k_max, self.k_max)
        # For each time step, get the most relevant number of clusters
        if selected_k is None:
            selected_k = self._relevant_k["k"]

        # ------------ Find vertices that represent such a k -----------
        relevant_vertices = [
            [
                deepcopy(v) for v in self._vertices[t]
                if has_element(v.info['k'], selected_k[t])
            ] for t in range(self._T_w)
        ]

        # ------------- Find edges between 2 relevant k ----------------
        if not fill_holes:
            relevant_edges = [
                [
                    deepcopy(e) for e in self._edges[t]
                    if (
                        has_element(
                            self._vertices[e.time_step][e.v_start].info['k'],
                            selected_k[t]
                        ) and has_element(
                            self._vertices[e.time_step + 1][e.v_end].info['k'],
                            selected_k[t+1]
                        ))
                ] for t in range(self._T_w-1)
            ]

        # Some edges might be non-existant (edge k1 -> k2 does not exist)
        # So we will create edges that won't be attached to the graph but with
        # the necessary information so that they can be visualized
        else:
            relevant_edges = []
            for t in range(self._T_w-1):
                edges = []

                # keep track of edge num for each t
                e_num = self._nb_edges[t]

                # Get start relevant vertices
                v_starts = relevant_vertices[t]
                v_ends = relevant_vertices[t+1]
                for v_start in v_starts:

                    # Find common members between v_start and v_end
                    v_end_members = [
                        (v, v.get_common_members(v_start)) for v in v_ends
                        if v.get_common_members(v_start) != []
                    ]
                    for (v_end, members) in v_end_members:

                        # Compute info (mean, std inf/sup at start and end)
                        info_start = Edge.info(v_start, members)
                        info_end = Edge.info(v_end, members)

                        edges.append(Edge(
                            info_start=info_start,
                            info_end=info_end,
                            v_start = v_start.num,
                            v_end = v_end.num,
                            t = t,
                            num = e_num,
                            members = members,
                            total_nb_members = self.N,
                            score_ratios = [[0, 1]],
                        ))

                        # Add edge number to v_start and v_end
                        # Potentially useful for plotting
                        v_start.add_edge_from(e_num)
                        v_end.add_edge_to(e_num)

                        e_num += 1

                relevant_edges.append(edges)

        return relevant_vertices, relevant_edges

    def save(
        self,
        filename: str = None,
        path: str = '',
        type:str = 'pg'
    ) -> None:
        """
        Save the graph (either as a class or a JSON file)

        :param filename: filename of the saved graph, defaults to None
        :type filename: str, optional
        :param path: path to the saved graph, defaults to ''
        :type path: str, optional
        :param type: type of the graph, either as a python class (.pg)
        or as a JSON file, defaults to 'pg'
        :type type: str, optional
        """
        if filename is None:
            filename = self.name
        if type == 'json':
            class_dict = jsonify(self)
            json_str = json.dumps(class_dict, indent=4)
            with open(path + filename + '.json', 'w', encoding='utf-8') as f:
                f.write(json_str)
        else:
            with open(path + filename + '.pg', 'wb') as f:
                pickle.dump(self, f)

    def load(
        self,
        filename: str = None,
        path: str = '',
    ) -> None:
        """
        Load a graph from a PersistentGraph file (.pg)

        :param filename: filename of the saved graph, defaults to None
        :type filename: str, optional
        :param path: path to the saved graph, defaults to ''
        :type path: str, optional
        """
        raise NotImplementedError("pg.load not working, use pickle instead")
        with open(path + filename, 'rb') as f:
            self = pickle.load(f)

    @property
    def name(self) -> str :
        """Name of the graph object, used as filename by default when saved

        :rtype: str
        """
        return self._name

    @property
    def N(self) -> int :
        """Number of members in the ensemble

        :rtype: int
        """
        return self._N

    @property
    def T(self) -> int :
        """Length of the time series

        :rtype: int
        """
        return self._T

    @property
    def d(self) -> int :
        """Number of variables studied
        :rtype: int
        """
        return self._d

    @property
    def w(self) -> int :
        """Size of the sliding time window
        :rtype: int
        """
        return self._w

    @property
    def k_max(self) -> int:
        """
        Max value of k considered

        :rtype: int
        """
        return self._k_max

    @property
    def members(self) -> np.ndarray:
        """Original data, ensemble of time series

        :rtype: np.ndarray[float], shape: (N, T, d)
        """
        return np.copy(self._members)

    @property
    def time_axis(self) -> np.ndarray:
        """
        Time axis, mostly used for plotting

        :rtype: np.ndarray[float], shape: T
        """
        return self._time_axis

    @property
    def max(self) -> np.ndarray:
        """
        Max member values for each variable and each t

        :rtype: np.ndarray[float], shape: (T, d)
        """
        return self._max

    @property
    def min(self) -> np.ndarray:
        """
        Min member values for each variable and each t

        :rtype: np.ndarray[float], shape: (T, d)
        """
        return self._min

    @property
    def k_info(self) -> Dict[int, List[float]]:
        """
        Life span and ratios of each assumptions k for all k and t.

        Available keys:

        - "score_ratios"
        - "life_span"

        Example: k_info[3]["life_span"][10] is the life span of k=3 and
        t = 10

        :rtype: Dict[int, List[float]]
        """
        return self._k_info

    @property
    def life_span_max(self) -> float:
        """
        Max life span

        :rtype: float
        """
        return self._life_span_max

    @property
    def life_span_min(self) -> float:
        """
        Min life span

        :rtype: float
        """
        return self._life_span_min

    @property
    def relevant_k(self) -> Dict[str, List]:
        """
        Dict of lists with 2 keys, "k" and "life_span", both lists
        are of length T and represent respectively the most relevant k
        and its life span at each time step.

        :rtype: Dict[str, List]
        """
        return self._relevant_k

    @property
    def nb_vertices(self) -> np.ndarray:
        """
        Total number of vertices created at each time step (T)

        :rtype: np.ndarray[int], shape: T
        """
        return np.copy(self._nb_vertices)

    @property
    def nb_edges(self) -> np.ndarray:
        """
        Total number of edges created at each time step (T-1)

        ..note::
          ``nb_edges[t]`` are the edges going from vertices at ``t`` to
          vertices at ``t+1``

        :rtype: np.ndarray[int], shape: T-1
        """
        return np.copy(self._nb_edges)

    @property
    def edges(self) -> List[List[Edge]]:
        """
        Nested list of edges of the graph (T-1, nb_edges[t])

        .. note::
          This includes dead and alive vertices

        :rtype: List[List[Edge]]
        """
        return self._edges

    @property
    def vertices(self) -> List[List[Vertex]]:
        """
        Nested list of vertices of the graph (T, nb_vertices[t])

        .. note::
          This includes dead and alive vertices

        :rtype: List[List[Vertex]]
        """
        return self._vertices

    @property
    def local_steps(self) -> List[List[dict]]:
        """
        Nested list (time and steps) of information on local steps.

        Available keys of self._local_steps[t][s]:
        - `k`
        - `score`
        - `ratio_score`

        :rtype: List[List[dict]]
        """
        return self._local_steps

    @property
    def k_range(self) -> Sequence[int]:
        """
        Ordered range of number of clusters studied, satisfying the
        condition of the score and the condition on k_max

        :rtype: Sequence[int]
        """
        return self._k_range

    @property
    def parameters(self) -> dict:
        """
        Parameters of the graph

        :rtype: dict
        """
        dic = {
            "model_type" : self._model_type,
            "zero_type" : self._zero_type,
            "score" : self._score,
            "transformer" : self._transformer.__name__,
            "scaler" : self._scaler.__class__,
            "DTW" : self._DTW,
            "model_class_kw" : self._model_class_kw,
            "model_kw" : self._model_kw,
            "fit_predict_kw" : self._fit_predict_kw,
        }
        return dic


import os
from dotenv import load_dotenv
import ptbot as alien_bot
from pytimeparse import parse

def start_countdown(chat_id, text):
    time = parse(text)
    message_id = bot.send_message(chat_id, f'Осталось {time}')
    bot.create_countdown(
        time,
        update_telegram_message,
        chat_id=chat_id,
        id=message_id,
        start_time=time
    )
    bot.create_timer(time, send_finish, chat_id=chat_id)

def send_finish(chat_id):
    bot.send_message(chat_id,'Время вышло!')

def update_telegram_message(secs_left,chat_id,id,start_time):
    bot.update_message(
        chat_id,
        id,
        f'Осталось {secs_left}'
        f'\n{render_progressbar(start_time,start_time-secs_left)}'
    )

def render_progressbar(total, iteration, prefix='', suffix='', length=30, fill='█', zfill='░'):
    iteration = min(total, iteration)
    percent = "{0:.1f}"
    percent = percent.format(100 * (iteration / float(total)))
    filled_length = int(length * iteration // total)
    pbar = fill * filled_length + zfill * (length - filled_length)
    return '{0} |{1}| {2}% {3}'.format(prefix, pbar, percent, suffix)

if __name__ == '__main__':
    load_dotenv()
    tg_token = os.getenv('TG_TOKEN')
    bot = alien_bot.Bot(tg_token)
    bot.reply_on_message(start_countdown)
    bot.run_bot()


#!/usr/bin/env python
# -*- coding: UTF-8 -*-
'''=================================================
@Project -> File   ：qiyangPythonCrawler -> start
@IDE    ：PyCharm
@Author ：Mr. cyh
@Date   ：2021-05-31 10:41
@Desc   ：https://www.chyxx.com/minglu/list_12832.html
=================================================='''
import time

import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from lxml import etree

from myJDBC.jdbc import mySqlJdbc

jdbc = mySqlJdbc()

ua = UserAgent()

proxiesNone = {
    "http": None,
    "https": None
}

urlLBCS = "/minglu/list_{}.html"
url = "https://www.chyxx.com"
dataBD = "chyxx_com"

GJZList = {"水产", "海产", "土产", "包子", "凉皮", "罐头", "消毒",
           "灌装", "封尾", "旋盖", "贴标", "包装", "输送", "喷码", "封口", "速冻",
           "咖啡", "调味", "汉堡", "消毒", "清洗", "灭菌", "切片", "花生", "咖喱", "添加剂", "膨化", "搅拌",
           "农", "葵", "豆", "薯", "果", "菌", "肉", "稻"
    , "猪", "牛", "羊", "鱼", "鸟", "鸡", "蛋", "米", "麦", "酱", "谷", "鸭", "鹅", "兔", "瓜", "食"
    , "饼", "菜", "粮", "餐", "素", "骨", "枣", "栗", "面", "糖", "奶", "饺", "茄", "麸", "糯",
           "炒", "煎", "炸", "炖", "煮", "泡", "煸", "煲", "烘", "蒸", "烤", "炖", "烧", "腌", "熏",
           "汤", "粥", "料", "渔", "筋", "萝", "姜", "蒜", "葱", "辣", "冰", "浆", "糕", "厨", "盐",
           "糖", "笋", "炉", "榨", "碗", "屠", "吃", "腐", "禽", "畜", "麻"  "馍", "馒", "蹄", "饮", "盆", "勺", "筷", "锅", "叉",
           "蜜", "孜", "馅", "丸", "罐", "汁水", "蘸", "醒", "酥", "椒", "茶", "酸", "菇"
           }

"""
清洗详情数据的函数
"""

def getData_QX(dataList):
    i = 0
    indx = 0;
    retDataList = None
    for data in dataList:
        indx += 1
        # print(data)
        if data.find('：') > -1:
            i += 1
            if i == 2:
                retDataList = dataList[indx - 1:]
    return retDataList

def getHtml(pathurl):
    global url
    time.sleep(2.5)
    t = int(time.time())
    print(pathurl)
    header = {
        'User-Agent': ua.random,
        'path': pathurl,
        'cookie': 'Hm_lpvt_92877e89b7b6871e4dbbf66d5839d430={}; Hm_lpvt_303541224a8d81c65040eb747f5ee614={}'.format(
            str(t), str(t + 1))
    }
    url_ = url + pathurl
    req = requests.get(url=url_, headers=header, proxies=proxiesNone, timeout=14,verify=False)
    reqHtml = req.text
    # print(reqHtml)
    # exit()
    return reqHtml

"""
清洗   说明 list
"""

def get_SM(list_sm):
    reList = list()
    for sm in list_sm:
        reList.append(sm.replace("\u3000", ""))
    return reList

def get_DZDZ(DZDZ_List):
    pass
    dz_All = ""
    for dz in DZDZ_List:
        dz_All += dz
    return dz_All

def get_xq_data(K_List, V_list, Qdz, url_path, name):
    global url
    lxr = ""
    dh = ""
    sj = ""
    dz = ""
    zy = ""
    cent = 0
    for my_data_k in K_List:
        if my_data_k.find("地址") > -1:
            dz = Qdz + V_list[cent].replace("：", "")
        if my_data_k.find("电话") > -1:
            dh = V_list[cent].replace("：", "")
        if my_data_k.find("经营范围") > -1:
            zy = V_list[cent].replace("：", "")
        cent += 1
    jdbc.inXqDate(name, url_path, zy, lxr, dh, sj, dz, dataBD)

def getXQ(pathurl):
    HtmlXQ = getHtml(pathurl)
    # print(HtmlXQ)
    selector = etree.HTML(HtmlXQ)
    # 说明
    K_List = selector.xpath("//*[@id=\"contentBody\"]/p/strong/text()")
    # 大致地址
    DZDZList = selector.xpath("//div[@id='contentBody']/p/a/text()")
    # 数据
    dataList = selector.xpath("//*[@id='contentBody']/p/text()")
    # 名字
    name = selector.xpath("//h1[@class='title']/text()")
    if len(name) > 0:
        name = name[0]
    # exit()
    K_List = K_List[1:]
    K_List = get_SM(K_List)
    V_list = getData_QX(dataList)
    dz = get_DZDZ(DZDZList)
    get_xq_data(K_List, V_list, dz, pathurl, name)

if __name__ == '__main__':
    for i in range(192+1, 12832 + 1):
        print("<><><><><><<><><><><>",i,"<><><><><><<><><><><>")
        ymStr = str(i)
        path = urlLBCS.format(ymStr)
        HtmlLB = getHtml(path)
        selector = etree.HTML(HtmlLB)
        xqymURlList = set(selector.xpath("//div[@class='compnay-list']/div/h3/a/@href"))
        for pathurl in xqymURlList:
            isOrNo = jdbc.getUrlGs(pathurl, dataBD)
            if isOrNo < 1:
                getXQ(pathurl)
            else:
                print("已经有了这个url的数据了", pathurl)
            print("--------------------------------------------------------------------------------------------")
        # if i == 102:
        #     exit()


import scipy.io as io
import numpy as np

self = io.netcdf.netcdf_file('/home/hendrik/data/global/AHF_2005_2.5min.nc','r')
self.fp.seek(0)
magic = self.fp.read(3)
self.__dict__['version_byte'] = np.fromstring(self.fp.read(1), '>b')[0]

# Read file headers and set data.
# stolen from scipy: /usr/lib/python2.7/dist-packages/scipy/io/netcdf.py
self._read_numrecs()
self._read_dim_array()
self._read_gatt_array()
# self._read_var_array()
header = self.fp.read(4)
begin = 0
count = self._unpack_int()
vars = []
for ivar in range(count):
    vars.append(self._read_var())


import boto3
import sys
db = sys.argv[1]
rds = boto3.client('rds')

try:
    resp = rds.delete_db_instance(
        DBInstanceIdentifier = db,
        SkipFinalSnapshot = True
        )
    print(resp)
except Exception as e:
    print(e)


from trped_main import MainWindow as trpedmw
from aiced_main import MainWindow as aicedmw
from stged_main import MainWindow as stgedmw
from UI.ucpre_allinone import Ui_MainWindow as LauncherMain
from PyQt5 import QtWidgets
from PyQt5.QtCore import pyqtSlot
import sys

class UCPAllInOne(QtWidgets.QMainWindow, LauncherMain):
    def __init__(self):
        super().__init__()
        self.setupUi(self)
        self.windows = {
            "trp": trpedmw,
            "aic": aicedmw,
            "stg": stgedmw
        }
        self.trp_window = None
        self.aic_window = None
        self.stg_window = None

        self.trp.clicked.connect(lambda: self.launch("trp"))
        self.aic.clicked.connect(lambda: self.launch("aic"))
        self.stg.clicked.connect(lambda: self.launch("stg"))

    @pyqtSlot(name="Kek")
    def launch(self, name):
        wdw: QtWidgets.QMainWindow = getattr(self, name + "_window")
        if wdw:
            wdw.showNormal()
        else:
            setattr(self, name+"_window", self.windows[name]())

if __name__ == "__main__":
    app = QtWidgets.QApplication([])
    mainwindow = UCPAllInOne()
    mainwindow.show()
    sys.exit(app.exec_())