from dotenv import load_dotenv
from requests import post, get
import os
import base64
import json

load_dotenv()

ID = os.getenv("ID")
secret = os.getenv("SECRET")

def get_token():
    auth_string = ID + ":" + secret
    auth_bytes = auth_string.encode("utf-8")
    auth_base64 = str(base64.b64encode(auth_bytes), "utf-8")

    url = "https://accounts.spotify.com/api/token"
    headers = {
        "Authorization": "Basic " + auth_base64,
        "Content-Type": "application/x-www-form-urlencoded"
    }
    data = {"grant_type": "client_credentials"}
    result = post(url, headers=headers, data=data)
    json_result = json.loads(result.content)
    token = json_result["access_token"]

    return token

def get_auth_header(token):
    return {"Authorization": "Bearer " + token}

def search_for_artist(token, artist_name):
    url = "https://api.spotify.com/v1/search"
    headers = get_auth_header(token)
    query = f"?q={artist_name}&type=artist&limit=1"

    query_url = url + query
    result = get(query_url, headers=headers)
    json_result = json.loads(result.content)["artists"]["items"]

    return json_result[0]

def get_songs_by_artist(token, artist_id):
    url = f"https://api.spotify.com/v1/artists/{artist_id}/top-tracks?country=US"
    headers = get_auth_header(token)
    result = get(url, headers=headers)
    json_result = json.loads(result.content)["tracks"]

    return json_result


from collections import deque

#뱀. 머리좌표, 꼬리좌표
hx, hy = 0, 0
tx, ty = 0, 0
snake = deque([[hx, hy]])
turn = "r"

#보드크기, 뱀 = 1, 사과 = 2. 빈공간 = 0
N = int(input())
board = [[0 for i in range (N)] for i in range (N)]
board[hx][hy] = 1

#사과개수, 위치설정
apple = int(input())
for i in range(apple):
    aX, aY = map(int, input().split())
    board[aX - 1][aY - 1] = 2

#뱀 방향변환 정보
L = int(input())
tX = [0 for _ in range(L)]
tC = [0 for _ in range(L)]
for i in range(L):
    tmp = input().split()
    tX[i] = int(tmp[0])
    tC[i] = tmp[1]

time = 1
while(True):
    #머리이동
    if turn == "r":
        hy = hy + 1
    elif turn == "l":
        hy = hy - 1
    elif turn == "u":
        hx = hx - 1
    else:
        hx = hx + 1

    #머리가 보드 밖으로 진출시
    if not 0 <= hx < N or not 0 <= hy < N:
        break

    #머리 위치한곳 확인(빈곳 or 사과 or 몸통)
    if board[hx][hy]  == 0:
        board[hx][hy] = 1
        snake.append([hx,hy])
        tx, ty = snake.popleft()
        board[tx][ty] = 0
    elif board[hx][hy] == 2:
        board[hx][hy] = 1
        snake.append([hx,hy])
    else:
        break

    #뱀 방향전환
    for i in range(L):
        if tX[i] == time:
            if turn == "r" and tC[i] == "D":
                turn = "d"
            elif turn == "r" and tC[i] == "L":
                turn = "u"
            elif turn == "l" and tC[i] == "D":
                turn = "u"
            elif turn == "l" and tC[i] == "L":
                turn = "d"
            elif turn == "u" and tC[i] == "D":
                turn = "r"
            elif turn == "u" and tC[i] == "L":
                turn = "l"
            elif turn == "d" and tC[i] == "D":
                turn = "l"
            elif turn == "d" and tC[i] == "L":
                turn = "r"
    time = time + 1

print(time)


# -*- encoding: UTF-8 -*-
import requests
from bs4 import BeautifulSoup
import sys

sys.path.insert(0, './speech_recognition')
import audio

class saju(object):
    def __init__(self, srv):
        self.tts = srv['tts']
        self.asr = srv['asr']
        srv['asr'].setLanguage("English")
        self.exit_flag = False

    def asr_callback(self, msg):
        # Threshold
        if msg[1] > 0.4:
            print(msg[0], msg[1], " is returned")
            self.ret['type'] = 'speech'
            self.ret['word'] = msg[0]
            self.exit_flag = True

def calculate(value):
    print("calculate:", value)
    # value가 '1998 December 10th Female' 이라고 가정하면
    rem = value.translate({ord(i): None for i in 'th'})
    sent = rem.split()

    mon = sent[1]
    if mon == 'January':
        month = '1'
    elif mon == 'February':
        month = '2'
    elif mon == 'March':
        month = '3'
    elif mon == 'April':
        month = '4'
    elif mon == 'May':
        month = '5'
    elif mon == 'June':
        month = '6'
    elif mon == 'July':
        month = '7'
    elif mon == 'August':
        month = '8'
    elif mon == 'September':
        month = '9'
    elif mon == 'October':
        mon = '10'
    elif mon == 'November':
        month = '11'
    if mon == 'December':
        month = '12'

    if "여자" or "female" in value:
        gen = 2
    elif '남자' or 'male' in value:
        gen = 1

    gender = "% s" % gen
    year = sent[0]
    # month = sent[1]
    day = sent[2]
    return gender, year, month, day

def main(srv):
    print("2021 saju mode")
    srv['tts'].say("Hello!")

    value = audio.get_string()
    gender, year, month, day = calculate(value)

    year = '2021'  # 2021
    gender = gender  # 1:male, 2:female
    birth_year = year  # '1994'
    birth_month = month  # '04'
    birth_day = day  # '11'

    birth_hour = 'N'  # default is N
    solun = '01'  # 양력
    lun_yn = '01'  # 평달
    url = 'http://tojung.freeunse.funstory.biz/sub/tradition4.php?unse_yy=' + year + '&unse1_sex=' + gender + '&unse1_yy=' + birth_year + '&unse1_mm=' + birth_month + '&unse1_dd=' + birth_day + '&unse1_hh=' + birth_hour + '&unse1_solun=' + solun + '&unse1_lun_yn=' + lun_yn
    headers = {'User-Agent': 'Mozilla/5.0'}
    req = requests.get(url, headers=headers)
    soup = BeautifulSoup(req.text, 'html.parser')
    my_titles = soup.select('.box_1111')
    response = my_titles[0].text
    print(response)
    return response


import numpy as np
import cv2
# ________色抽出__________#
image = cv2.imread('./driver.jpg') # ファイル読み込み
hsvLower = np.array([150, 0, 0])    # 抽出する色の下限
hsvUpper = np.array([179, 255, 255])    # 抽出する色の上限
hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) # 画像をHSVに変換
hsv_mask = cv2.inRange(hsv, hsvLower, hsvUpper)    # HSVからマスクを作成
# result = cv2.bitwise_and(image, image, mask=hsv_mask) # 元画像とマスクを合成

# ________ノイズ処理__________#
blur = cv2.medianBlur(hsv_mask,9)
kernel = np.ones((15,15),np.uint8)# カーネルサイズ
opening = cv2.morphologyEx(blur, cv2.MORPH_OPEN, kernel)# オープニング処理
mask_opening = cv2.bitwise_and(image, image, mask=opening) # マスク処理
closing = cv2.morphologyEx(blur, cv2.MORPH_CLOSE, kernel)# クロージング処理
mask_closing = cv2.bitwise_and(image, image, mask=closing) # マスク処理

# ________輪郭検出__________#
img,contours, hierarchy = cv2.findContours(closing, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
area = []
for cnt in contours:
    area.append(cv2.contourArea(cnt))
area_max = area.index(max(area))
image_copy = image.copy()
image_outline = cv2.drawContours(image_copy, contours,area_max, (0,255,0), 3)

# ________重心位置と描画__________#
M = cv2.moments(contours[area_max])
cx = int(M['m10']/M['m00'])
cy = int(M['m01']/M['m00'])
cv2.circle(image_outline, (cx,cy), 5, (0, 255, 255), -1)

#_________上下左右の端点位置と描画＿＿＿＿#
leftmost = tuple(contours[area_max][contours[area_max][:,:,0].argmin()][0])
rightmost = tuple(contours[area_max][contours[area_max][:,:,0].argmax()][0])
topmost = tuple(contours[area_max][contours[area_max][:,:,1].argmin()][0])
bottommost = tuple(contours[area_max][contours[area_max][:,:,1].argmax()][0])
cv2.circle(image_outline,(leftmost), 5, (255, 0, 0), -1)
cv2.circle(image_outline,(rightmost), 5, (255, 0, 0), -1)
cv2.circle(image_outline,(topmost), 5, (255, 0, 0), -1)
cv2.circle(image_outline,(bottommost), 5, (255, 0, 0), -1)

while True:
    # キー入力を1ms待って、keyが「q」だったらbreak
    cv2.imshow('blur',blur)
    cv2.imshow('openig_mask', opening)
    cv2.imshow('opening_result',mask_opening)
    cv2.imshow('closing_mask', closing)
    cv2.imshow('closing_result',mask_closing)
    cv2.imshow('image_outline',image_outline)

    key = cv2.waitKey(1)&0xff
    if key == ord('q'):
        break
cv2.destroyAllWindows()


import os.path
import logging
log = logging.getLogger(__name__)

# third party imports
from PySide import QtGui, QtCore
import numpy
from scipy.optimize import minimize
import pyqtgraph as pg
from pyqtgraph.graphicsItems.PlotItem import PlotItem
# Use black text on white background
pg.setConfigOption('background', 'w')
pg.setConfigOption('foreground', 'k')
# Enable antialiasing for prettier plots
pg.setConfigOptions(antialias=True)

# local imports
from .version import __version__
from . import parser
from . import linear_regression

class MainWindow(QtGui.QMainWindow):

    def __init__(self):
        super(MainWindow, self).__init__()

        # Initialize private variables
        self.plot = None

        # Initialize QSettings object
        self.settings = QtCore.QSettings()

        # Initialize GUI stuff
        self.initUI()

    @QtCore.Slot(str)
    def setStatusText(self, status):
        self.statusLabel.setText(status)

    def initUI(self):
        self.aboutAction = QtGui.QAction('&About', self)
        self.aboutAction.triggered.connect(self.about)

        self.openAction = QtGui.QAction('&Open', self)
        self.openAction.setStatusTip('Open a data set')
        self.openAction.setToolTip('Open a data set')
        self.openAction.setShortcut('Ctrl+O')
        self.openAction.triggered.connect(self.openFile)

        self.closeAction = QtGui.QAction('Close &Window', self)
        self.closeAction.setStatusTip('Close the Window')
        self.closeAction.setToolTip('Close the Window')
        self.closeAction.setShortcut('Ctrl+W')
        self.closeAction.triggered.connect(self.close)

        self.fitLSAction = QtGui.QAction('Least Squares', self)
        self.fitLSAction.setStatusTip('Fit using the least squares')
        self.fitLSAction.setToolTip('Fit using the least squares')
        self.fitLSAction.triggered.connect(self.plotLinearLeastSquares)

        self.fitLMSAction = QtGui.QAction('Least Median of Squares', self)
        self.fitLMSAction.setStatusTip('Fit using the least median of squares')
        self.fitLMSAction.setToolTip('Fit using the least median of squares')
        self.fitLMSAction.triggered.connect(self.plotLinearLeastMedianOfSquares)

        menubar = self.menuBar()
        fileMenu = menubar.addMenu('&File')
        fileMenu.addAction(self.openAction)
        fileMenu.addAction(self.closeAction)
        fitMenu = menubar.addMenu('Fi&t')
        fitMenu.addAction(self.fitLSAction)
        fitMenu.addAction(self.fitLMSAction)
        aboutMenu = menubar.addMenu('&About')
        aboutMenu.addAction(self.aboutAction)

        statusBar = self.statusBar()
        self.statusLabel = QtGui.QLabel('')
        statusBar.addWidget(self.statusLabel, stretch=1)

        self.view = pg.GraphicsLayoutWidget()
        self.setCentralWidget(self.view)
        self.plot = self.view.addPlot()
        self.view.addItem(self.plot, 0, 0)

        self.setWindowTitle('VisFitter')
        from .resources.icons import logoIcon
        self.setWindowIcon(logoIcon)
        self.setMinimumSize(576, 432)
        self.readWindowSettings()

    def fitLS(self):
        pass

    def openFile(self):
        lastOpened = self.settings.value('lastOpened', '')
        filepath, _filter = QtGui.QFileDialog.getOpenFileName(parent=self,
                                caption='Open a PL spectrum file',
                                dir=lastOpened)
        if not filepath:
            return
        self.settings.setValue('lastOpened', filepath)

        # Read in the data, and plot it
        self.x, self.y = parser.getXY(filepath)
        self.plot.clearPlots()
        self.plotXY()
#         self.plotLinearLeastSumOfSquares()
#         self.plotLinearLeastMedianOfSquares()

    def plotXY(self):
        self.scatter = self.plot.plot(self.x, self.y,
                                      pen=None, symbol='o')

    def plotLinearLeastSquares(self):
        alpha, beta = linear_regression.leastSquares(self.x, self.y)
        x = numpy.array([self.x.min(), self.x.max()])
        y = alpha + beta * x
        self.scatter = self.plot.plot(x, y, pen=pg.mkPen('r'))

    def plotLinearLeastMedianOfSquares(self):
        alpha, beta = linear_regression.leastMedianOfSquaresCrude(self.x,
                                                                  self.y)
        x = numpy.array([self.x.min(), self.x.max()])
        y = alpha + beta * x
        self.scatter = self.plot.plot(x, y, pen=pg.mkPen('b'))

    def about(self):
        title = 'About VisFitter'
        text = ('VisFitter\n'
                'Version {}\n'
                '\n'
                'Copyright (c) 2013-2014, Scott J Maddox\n'
                '\n'
                'VisFitter is free software: you can redistribute it'
                ' and/or modify it under the terms of the GNU Affero General'
                ' Public License as published by the Free Software Foundation,'
                ' either version 3 of the License, or (at your option) any'
                ' later version.\n'
                '\n'
                'VisFitter is distributed in the hope that it will be'
                ' useful, but WITHOUT ANY WARRANTY; without even the implied'
                ' warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR'
                ' PURPOSE.  See the GNU Affero General Public License for'
                ' more details.\n'
                '\n'
                'You should have received a copy of the GNU Affero General'
                ' Public License along with VisFitter.  If not, see'
                ' <http://www.gnu.org/licenses/>.'
                ''.format(__version__))
        QtGui.QMessageBox.about(self, title, text)

    def moveCenter(self):
        qr = self.frameGeometry()
        cp = QtGui.QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

    def moveTopLeft(self):
        p = QtGui.QDesktopWidget().availableGeometry().topLeft()
        self.move(p)

    def closeEvent(self, event):
        reply = QtGui.QMessageBox.question(self, 'Quit?',
                'Are you sure you want to quit?',
                QtGui.QMessageBox.Yes | QtGui.QMessageBox.No,
                QtGui.QMessageBox.No)

        if reply == QtGui.QMessageBox.Yes:
            self.writeWindowSettings()
            event.accept()
        else:
            event.ignore()

    def writeWindowSettings(self):
        self.settings.setValue("MainWindow/size", self.size())
        self.settings.setValue("MainWindow/pos", self.pos())

    def readWindowSettings(self):
        self.resize(self.settings.value("MainWindow/size",
                                         QtCore.QSize(1280, 800)))
        pos = self.settings.value("MainWindow/pos")
        if pos is None:
            self.moveCenter()  # default to centered
        else:
            self.move(pos)


#! /usr/local/env python

# 06 May 2018
# ADW

import numpy as np
from scipy.interpolate import interp1d
import warnings

class Segment(object):

    def __init__(self, _id, to_ids=None, segment_type='stream',
                 direction='to_downstream'):
        """
        A segment of a stream, between tributary junctions;
        ID of the stream should be an integer >0
        """
        self.id = int(_id)
        self.to_ids = np.array([to_ids]).astype(int)
        self.x = None
        self.z = None
        self.XP = None
        self.B = None
        self.z_smoothed = None
        self.channel_flow_accumulation = None
        self.channel_slope = None
        self.channel_flow_accumulation_smoothed = None
        self.channel_slope_smoothed = None
        #self.E = None
        #self.N = None
        self.target_dx_downstream = None
        self.segment_type = segment_type
        self.allowed_segment_types = ['stream',
                                      'lake']
        self.direction = direction
        self.from_ids = []

        if self.id == 0:
            warnings.warn('ID of 0 by default indicates offmap flow')

    ###########
    # SETTERS #
    ###########

    def set_x(self, x):
        """
        Set distances downstream
        x can be overwritten with the network
        x_local stays local
        """
        self.x = np.array(x)
        self.x_local = self.x.copy()

    def set_z(self, z):
        """
        Set elevation
        """
        self.z = np.array(z)

    def set_XP(self, XP):
        """
        Set cross-sectional elevation profile; from this, the at-channel
        valley width can always be determined.
        """
        self.XP = XP

    def set_B(self, B):
        """
        Set valley width at the elevation of the channel.
        Just a scalar value at each downstream position
        """
        self.B = B

    def set_B_from_XP(self):
        """
        Use the elevation and cross-sectional profile to set the width
        of the channel
        """
        # Something with interpolation
        # Look at TerraPIN for how this may work
        pass

    def set_type(self, segment_type):
        if segment_type in self.allowed_segment_types:
            self.segment_type = segment_type
        else:
            warnings.warn('Segment type not valid: please choose one of'+
                         ', '.join(a))

    def set_Q(self, Q):
        """
        Set discharge with distance downstream
        """
        self.Q = Q

    def set_A(self, A):
        """
        Set drainage area with distance downstream
        """
        self.A = A

    def set_target_dx_downstream(self, target_dx_downstream):
        """
        Set spacing between points in a downstream direction
        This will almost certainly not divide evenly into the length of a
        stream segment, and will therefore be used as the target distance
        between nodes, which will be adjusted for an integer number of
        divisible distances using np.ceil
        """
        self.target_dx_downstream = target_dx_downstream

    def set_fromids(self, from_id):
        """
        Set the ID of the stream from which
        """
        self.from_ids = from_id

    def append_fromids(self, from_id):
        """
        Set the ID of the stream from which
        """
        self.from_ids.append(from_id)

    #def set_E(self, E):
    #    """
    #    Set easting of points
    #    """
    #    self.E = E

    #def set_N(self, N):
    #    """
    #    Set northing of points
    #    """
    #    self.N = N

    def set_EastingNorthing(self, ENarray=None, Easting=None, Northing=None):
        """
        Set Northing and Easting at once
        """
        if ENarray is not None:
            self.EastingNorthing = np.array(ENarray)
            self.Easting = self.EastingNorthing[:,0]
            self.Northing = self.EastingNorthing[:,1]
        elif (Easting is not None) and (Northing is not None):
            self.Easting = Easting
            self.Northing = Northing
        else:
            warnings.warn("Improper inputs")

    def calc_x_from_EastingNorthing(self):
        """
        Calculate downstream distances from Easting and Northing, relative
        only to the individual segment
        """
        if (self.Easting is not None) and (self.Northing is not None):
            dx = ( np.diff(self.Easting)**2 + np.diff(self.Northing)**2 )**.5
            self.x = np.hstack((0., np.cumsum(dx)))
            self.x_local = self.x.copy()
        else:
            warnings.warn("Both Easting and Northing must be defined")

    def densify_x_E_N(self):
        """
        Especially if E, N, are taken only at vertices, there can be a
        signficant lack of resolution in the network. This seeks to densify
        the network based on a target downstream dx.

        This will not preserve points at the coordinates of all of the corners
        of the fluvial network. It will create backups of the original grid
        to do this. It could be reprogrammed to optionally preserve these
        corners.

        Run this after calc_x_from_EastingNorthing and set_target_dx_downstream
        """
        if self.target_dx_downstream is None:
            warnings.warn("Nothing will be done; run set_target_dx_downstream")
        elif (self.Easting is None) or (self.Northing is None):
            warnings.warn("Nothing will be done; Set Easting and Northing")
        elif self.x_local is None:
            warnings.warn("Nothing will be done; Set x (and x_local)")
        else:
            # Backup original
            self.Easting_original = self.Easting.copy()
            self.Northing_original = self.Northing.copy()
            self.x_local_original = self.x_local.copy()
            ixE = interp1d(self.x_local_original, self.Easting_original)
            ixN = interp1d(self.x_local_original, self.Northing_original)
            #print self.Easting[-1], self.Northing[-1]
            # +1 because it includes the bookends
            # Ceil instead of round for short segments
            nx = np.ceil( ( np.max(self.x_local) - np.min(self.x_local) )
                            / self.target_dx_downstream ) + 1.
            self.x_local = np.linspace( np.min(self.x_local_original),
                                        np.max(self.x_local_original),
                                        nx )
            self.Easting = ixE(self.x_local)
            self.Northing = ixN(self.x_local)
            self.EastingNorthing = \
                 np.vstack((self.Easting, self.Northing)).transpose()

class BoundingBox(object):
    """
    Easily define a bounding box around your data source, padded to include
    the raster grid cells (if these are important)
    """
    def __init__(self, points_xy=None):
        points = np.array(points_xy)
        self.west = np.min(points[:,0])
        self.east = np.max(points[:,0])
        self.south = np.min(points[:,1])
        self.north = np.max(points[:,1])

class Network(object):

    def __init__(self, segment_list, downstream_x=0., offmap_id=0):
        """
        Tools to build a river network
        Downstream_x = the x-value or values for the offmap cell(s);
        they default to 0 and will be expanded to a list in this case
        offmap_id is the ID given to offmap segments (to_id = offmap_id if a
        segment flows offmap). This defaults to 0 following GRASS GIS
        convention.
        """
        self.segment_list = np.array(segment_list)
        # Convenient info to have in lists
        # identifiers and connections
        self.ids = []
        self.to_ids = []
        for segment in self.segment_list:
            self.ids.append(segment.id)
            self.to_ids.append(segment.to_ids)
        self.ids = np.array(self.ids)
        #self.to_ids = np.array(self.to_ids)
        self.offmap_id = offmap_id # 0 default following GRASS GIS
        # Get ID of downstream-most segment
        # Is downstream-most if "tosegment" is not in the list
        for segment in self.segment_list:
            if np.sum(self.ids == segment.to_ids[0][0]) == 0:
                self.downstream_most_id = segment.id
                break
        self.downstream_x = downstream_x

    def compute_fromstreams(self):
        """
        Compute IDs of flow contributors to each segment.
        This will be redundant with to_id, from which it is derived, but
        will also be convenient.
        """
        for segment in self.segment_list:
            from_ids = []
            for _i in range(len(self.to_ids)):
                if (self.to_ids[_i] == segment.id).any():
                    from_ids.append(self.ids[_i])
            segment.set_fromids(from_ids)

    def segments_xy_flattened(self):
        """
        Return all points in segments, flattened into a single Numpy array
        """
        allpoints = []
        for segment in self.segment_list:
            allpoints.append(segment.EastingNorthing)
        return np.vstack(allpoints)

    def get_bounding_box(self):
        """
        Produces a bounding box in
        """
        allpoints = segments_xy_flattened(self)
        self.BoundingBox = BoundingBox(allpoints)

    def get_segment_ids(self):
        """
        Create a list of all segment IDs
        """
        self.ids = []
        for segment in self.segment_list:
            self.ids.append(segment.id)
        self.ids = np.array(self.ids)

    def get_to_segment_ids(self):
        """
        Create a list of all segment IDs
        """
        self.to_ids = []
        for segment in self.segment_list:
            self.to_ids.append(segment.to_ids)
        self.to_ids_flattened = np.squeeze(np.hstack(self.to_ids))

    def compute_offmap_segments(self):
        """
        get list of all segments that flow offmap
        """
        self.get_segment_ids()
        self.offmap_segments = []
        # With the below, we can probbaly just ditch this top part of code
        for segment in self.segment_list:
            if segment.to_ids == 0:
                self.offmap_segments.append(segment.id)
        # More robust -- see which ones do not go to an on-map segment
        for segment in self.segment_list:
            if len([i for i in segment.to_ids if i in self.ids]) == 0:
                self.offmap_segments.append(segment.id)

    def compute_headwater_segments(self):
        """
        get list of segments that are at the headwaters of a drainage network
        (or the one that is the upstream-most in the case in which an
        upstream-most segment was defined)
        """
        self.get_to_segment_ids()
        self.headwater_segments = []
        for segment in self.segment_list:
            if segment.id not in self.to_ids_flattened:
                self.headwater_segments.append(segment.id)

    def set_hierarchy_order(self):
        """
        Get IDs in order going upstream.
        Helps with calculating distance downstream
        """
        self.get_to_segment_ids()
        self.hierarchy_ids = []
        _ids = [self.downstream_most_id]
        while(len(_ids) > 0):
            self.hierarchy_ids.append(_ids)
            __ids = []
            for _id in _ids:
                __ids.append(self.ids[self.to_ids_flattened == _id])
            __ids = np.hstack(__ids)
            _ids = list(__ids)

    def compute_x_in_network(self, overlapping_termini=True):
        """
        Get distance along each stream path
        * Straightforward for downstream at map resolution
        * For upstream, need to march through tributary network to
          obtain connectivity, and then turn this into a set of distances
        * For reduced resolution (for compute time): need to keep track of
          endpoints from initial survey
        "Ovelapping termini" means that the nodes between each segment are
        shared among adjacent segments

        Can move from bottom up in all cases.
        """

        self.overlapping_termini = overlapping_termini
        self.compute_offmap_segments()
        self.set_hierarchy_order()

        # This is ordered from downstream to upstream
        for _id_list in self.hierarchy_ids:
            for _id in _id_list:
                segment = self.segment_list[self.ids == _id][0]
                # Will break in a branching network
                downstream_segment_id = int(segment.to_ids)
                downstream_segment = self.segment_list[self.ids ==
                                                       downstream_segment_id]
                if len(downstream_segment) == 0:
                    x_downstream = self.downstream_x
                elif len(downstream_segment) > 1:
                    sys.exit("Downstream-branching network: not supported!")
                else:
                    downstream_segment = downstream_segment[0]
                    x_downstream = downstream_segment.x[0]
                segment.x = segment.x_local - segment.x_local[-1] \
                                + x_downstream

    #def interpolate_xEN_in_network(self):

    def compute_downstream_paths_from_headwaters(self):
        """
        Non-parsimonious: every full long profile
        """
        self.headwaters_segment_ids = []
        for _id in self.ids:
            if _id not in self.to_ids:
                self.headwaters_segment_ids.append(_id)
        self.downstream_paths_from_headwaters = []
        for _hwid in self.headwaters_segment_ids:
            pathlist = []
            segment_list = self.segment_list[self.ids == _hwid]
            while len(segment_list) == 1:
                segment = segment_list[0]
                pathlist.append(segment.id)
                segment_list = self.segment_list[self.ids ==
                                                  int(segment.to_ids)]
            self.downstream_paths_from_headwaters.append(pathlist)

    def compute_index_per_point_from_headwaters(self):
        self.index_per_point_from_headwaters = []
        for _id_list in self.downstream_paths_from_headwaters:
            tmplist = []
            for _id in _id_list:
                segment = self.segment_list[self.ids == _id][0]
                tmplist += [segment.id] * len(segment.x)
            self.index_per_point_from_headwaters.append(np.array(tmplist))

    def compute_x_from_headwaters(self):
        self.x_from_headwaters = []
        for _id_list in self.downstream_paths_from_headwaters:
            tmplist = []
            for _id in _id_list:
                segment = self.segment_list[self.ids == _id][0]
                tmplist += list(segment.x)
            self.x_from_headwaters.append(np.array(tmplist))

    def compute_z_from_headwaters(self):
        self.z_from_headwaters = []
        for _id_list in self.downstream_paths_from_headwaters:
            tmplist = []
            for _id in _id_list:
                segment = self.segment_list[self.ids == _id][0]
                tmplist += list(segment.z)
            self.z_from_headwaters.append(np.array(tmplist))

    def compute_slope_from_headwaters(self):
        self.slope_from_headwaters = []
        # NEED SETTER!
        for _id_list in self.downstream_paths_from_headwaters:
            tmplist = []
            for _id in _id_list:
                segment = self.segment_list[self.ids == _id][0]
                tmplist += list(segment.channel_slope)
            self.slope_from_headwaters.append(np.array(tmplist))

    def compute_accum_from_headwaters(self):
        self.accum_from_headwaters = []
        # NEED SETTER!
        for _id_list in self.downstream_paths_from_headwaters:
            tmplist = []
            for _id in _id_list:
                segment = self.segment_list[self.ids == _id][0]
                tmplist += list(segment.channel_flow_accumulation)
            self.accum_from_headwaters.append(np.array(tmplist))

    def smooth_from_headwaters(self, var):
        #var = self.z_from_headwaters
        _y_smoothed = []
        for i in range(len(var)):
            _y_smoothed.append( np.array(self.moving_average(
                            self.x_from_headwaters[i], var[i], self.window)) )
        return _y_smoothed

    def moving_average(self, _x, _y, window):
        """
        Create a moving average at every point over a distance of window/2
        """
        _x = np.array(_x)
        _y = np.array(_y)
        out_y = []
        for _xi in _x:
            out_y.append( np.mean(_y[ (_x < _xi + window/2.) *
                                      (_x > _xi - window/2.) ]))
        return out_y
        #return out_x, out_y

    def smooth_window(self, window):
        """
        Smoothes using a moving window
        written as a sketch here
        """
        self.window = window
        self.compute_downstream_paths_from_headwaters()
        self.compute_index_per_point_from_headwaters()
        self.compute_x_from_headwaters()

        try:
            self.compute_z_from_headwaters()
            _sfh = self.smooth_from_headwaters(self.z_from_headwaters)
            for segment in self.segment_list:
                tmp = []
                for i in range(len(_sfh)):
                    _path = _sfh[i]
                    _ipp = self.index_per_point_from_headwaters[i]
                    _append = _path[_ipp == segment.id]
                    if len(_append) > 0:
                        tmp.append(_append)
                # USE SETTER
                segment.z_smoothed = np.average(tmp, axis=0)
        except:
            pass
        try:
            self.compute_slope_from_headwaters()
            _sfh = self.smooth_from_headwaters(self.slope_from_headwaters)
            for segment in self.segment_list:
                tmp = []
                for i in range(len(_sfh)):
                    _path = _sfh[i]
                    _ipp = self.index_per_point_from_headwaters[i]
                    _append = _path[_ipp == segment.id]
                    if len(_append) > 0:
                        tmp.append(_append)
                # USE SETTER
                segment.channel_slope_smoothed = np.average(tmp, axis=0)
        except:
            pass
        try:
            self.compute_accum_from_headwaters()
            _sfh = self.smooth_from_headwaters(self.accum_from_headwaters)
            for segment in self.segment_list:
                tmp = []
                for i in range(len(_sfh)):
                    _path = _sfh[i]
                    _ipp = self.index_per_point_from_headwaters[i]
                    _append = _path[_ipp == segment.id]
                    if len(_append) > 0:
                        tmp.append(_append)
                # USE SETTER
                segment.channel_flow_accumulation_smoothed = \
                                               np.average(tmp, axis=0)
        except:
            pass

        # Average across all paths that include the segment

    def compute_profile_from_starting_segment(self):
        """
        Compute a single long profile from a starting river segment
        This code currently assumes that you have built the network from
        just a starting segment, and so doesn't walk downstream, but rather
        takes the segments in order. This could be easily fixed by checking
        the indices of the next river downstream.
        """
        outlist = []
        _zbarflag = False
        _Sflag = False
        _Sbarflag = False
        _Aflag = False
        _Abarflag = False
        header = ['id', 'E', 'N', 'x_downstream', 'z']
        segment = self.segment_list[0] # All should have the same parameters
        if segment.z_smoothed is not None:
            header.append('z_avg')
            _zbarflag = True
        if segment.channel_slope is not None:
            header.append('S')
            _Sflag = True
        if segment.channel_slope_smoothed is not None:
            header.append('S_avg')
            _Sbarflag = True
        if segment.channel_flow_accumulation is not None:
            header.append('Accum')
            _Aflag = True
        if segment.channel_flow_accumulation_smoothed is not None:
            header.append('Accum_avg')
            _Abarflag = True
        for segment in self.segment_list:
            # Seems a clunky way to do this
            for i in range(len(segment.z)):
                #print [segment.x[i], segment.Easting[i], segment.Northing[i], segment.z[i]]
                _out_i = [int(segment.id), \
                          segment.x[i], segment.Easting[i], \
                          segment.Northing[i], segment.z[i]]
                if _zbarflag:
                    _out_i.append(segment.z_smoothed[i])
                if _Sflag:
                    _out_i.append(segment.channel_slope[i])
                if _Sbarflag:
                    _out_i.append(segment.channel_slope_smoothed[i])
                if _Aflag:
                    _out_i.append(segment.channel_flow_accumulation[i])
                if _Abarflag:
                    _out_i.append(segment.channel_flow_accumulation_smoothed[i])
                outlist.append(_out_i)
        self.long_profile_header = header
        self.long_profile_output = outlist


import tensorflow as tf
from tensorflow import keras
from keras.utils.layer_utils import count_params
import os
from os import path
import matplotlib.pyplot as plt
import numpy as np
import yaml

from cnn.file_inout import *
from cnn.CNN_Deconvolution.DeblurCNNModel2D import *
from cnn.CNN_Deconvolution.DeblurCNNModelMini3D import *

def SplitOnTrainAndVal(blured_list, clear_list, train_val_coef=0.8):
    # total count of each list
    total_elements = len(blured_list)
    train_elements = int(total_elements * train_val_coef)
    val_elements = total_elements - train_elements

    # make randomize indexes
    list1 = [True] * train_elements
    list2 = [False] * val_elements
    listcomb = list1 + list2
    listcomb = np.random.choice(listcomb, total_elements, replace=False)
    train_indices = np.argwhere(listcomb == True)
    valid_indices = np.argwhere(listcomb == False)

    # make split
    blured_train_list = [blured_list[i[0]] for i in train_indices]
    blured_val_list = [blured_list[i[0]] for i in valid_indices]

    clear_train_list = [clear_list[i[0]] for i in train_indices]
    clear_val_list = [clear_list[i[0]] for i in valid_indices]

    return blured_train_list, blured_val_list, clear_train_list, clear_val_list

# Class, which provides training model
class DeblurTrainer:
    # constructor
    def __init__(self, _blured_images, _clear_images,
                 _mode, _layers, _rows, _cols,
                 _learningRate, _epoches,
                 _isNeedToSaveDebluredDataset, _debluredPath):
        # CONSTANTS
        self.MODEL_NAME_2d = "2d_gaus_blur.h5"
        self.MODEL_NAME_3d = "3d_gaus_blur.h5"

        self.blured = _blured_images
        self.clear = _clear_images

        self.mode = _mode
        self.layers = _layers
        self.rows = _rows
        self.cols = _cols

        self.learningRate = _learningRate
        self.epoches = _epoches

        self.isNeedToSaveDebluredDataset = _isNeedToSaveDebluredDataset
        self.debluredImagesDatasetPath = _debluredPath
        return

    # Method which provides network model initialization
    def initTrainableModel(self):
        if self.mode == "2d learning":
            model = DeblurCNNModel2D.ModelBuilder((self.rows, self.cols, 1), self.learningRate)
        else:
            #model = PSFCNNModel3D.ModelBuilder(self.shape, self.learningRate)
            model = DeblurCNNModelMini3D.ModelBuilder((self.layers, self.rows, self.cols, 1), self.learningRate)
        return model

    # Method, which provides reshaping of inputed array to prediction
    def reshapeInput(self, input):
        if self.mode == "2d learning":
            input = input.reshape(1, self.rows, self.cols, 1)
        else:
            input = input.reshape(1, self.layers, self.rows, self.cols, 1)
        return input

    # Method which provides saving images of outputed data
    def deblurTrainingDataset(self):
        # Init model
        model = self.initTrainableModel()
        model.load_weights(self.saveWeightsPath)

        for i in range(len(self.blured)):
            bluredInto = self.reshapeInput(self.blured[i])
            clearResult = model.predict(bluredInto)
            clearResult = clearResult / np.amax(clearResult) * 255.
            clearResult = clearResult.reshape(self.layers, self.rows, self.cols)
            SaveTiffStack(clearResult, self.debluredImagesDatasetPath, "predicted_{}".format(i + 1))
        return

    def GenerateModelPath(self):
        dirId = -1
        while True:
            dirId += 1
            print(dirId)
            txt_folder = str(os.getcwd()) + "\\" + self.mode + "_" + str(dirId)
            if not path.isdir(txt_folder):
                os.mkdir(txt_folder)
                break
        return txt_folder

    # Method which provides plotting training history
    def plotHist(self, history):
        loss = history.history['loss']
        valLoss = history.history['val_loss']
        epochs = range(len(loss))

        plt.figure()

        plt.semilogy(epochs, loss, 'b', label = 'Training loss')
        plt.semilogy(epochs, valLoss, 'r', label = 'Validation loss')
        plt.title('Training and validation loss')
        plt.legend()
        plt.savefig(self.dirPath + "/history.png")
        #plt.show()
        return

    # Method which provides Resnet CNN training
    def train(self):
        try:
            # Init model
            model = self.initTrainableModel()

            # Init train test images
            bluredTrainList, bluredValList, clearTrainList, clearValList = SplitOnTrainAndVal(self.blured, self.clear)

            # train model
            self.batchSize = 32 if self.mode == "2d learning" else 2
            hist = model.fit(x = np.array(bluredTrainList), y = np.array(clearTrainList), validation_data=(np.array(bluredValList), np.array(clearValList)), epochs = self.epoches, batch_size=self.batchSize, use_multiprocessing=True)
            self.dirPath = self.GenerateModelPath()
            self.saveWeightsPath = self.dirPath + "/" + (self.MODEL_NAME_2d if self.mode == "2d learning" else self.MODEL_NAME_3d)
            model.save_weights(self.saveWeightsPath)

            # plot hist graph
            self.plotHist(hist)

            # outpur deblured images from dataset
            if self.isNeedToSaveDebluredDataset:
                self.deblurTrainingDataset()

        except Exception as e:
            print(e)
        return


import nextcord
from nextcord.ext import commands

from decouple import config

import nextwave
from nextwave.ext import spotify

import datetime

from utils import music_player_utils, guild_utils
from music_player.player_handlers.control_panel_handler import ControlPanelHandler
from utils.commands_utils import command_modules, module_disabled_message

class PlayerSkipCommand(commands.Cog):

    def __init__(self, bot: commands.Bot):
        self.bot = bot

    @nextcord.slash_command(description="Pula para a próxima música da queue", guild_ids=guild_utils.guild_ids)
    async def skip(self, interaction: nextcord.Interaction):
        if command_modules['Music Player'] == "Off":
            await interaction.send(embed=module_disabled_message)
            return

        if not getattr(interaction.user.voice, "channel", None):
            return await interaction.send(embed=music_player_utils.connect_first, ephemeral=True)
        elif not interaction.guild.voice_client:
            return await interaction.send(embed=music_player_utils.no_music_playing, ephemeral=True)
        else:
            vc: nextwave.Player = interaction.guild.voice_client

        if vc.channel == interaction.user.voice.channel:
            try:
                await ControlPanelHandler.check_if_panel_exists(self, interaction=interaction)
                return await interaction.send(embed=music_player_utils.use_use_the_panel, ephemeral=True)
            except NameError:
                pass
            except nextcord.NotFound:
                pass
            if vc.loop:
                return await interaction.send(embed=music_player_utils.music_is_looping, ephemeral=True)

            if not vc.is_playing():
                return await interaction.send("Não tem nada tocando no momento", ephemeral=True)

            if vc.queue.count != 0:

                next_playing_embed = nextcord.Embed(
                title="pulando para próxima música  <a:loading:747680523459231834> ..",
                color=0x2494f4
                )

                await interaction.send(embed=next_playing_embed, delete_after=5)
            else:
                await interaction.send(embed=music_player_utils.no_next_songs, ephemeral=True)

            try:
                await vc.stop()
            except:
                return

        else:
            await interaction.send(embed=music_player_utils.command_not_in_same_vc, ephemeral=True)

def setup(bot):
    bot.add_cog(PlayerSkipCommand(bot))


from __future__ import print_function

"""
This script is used to analyze VASP AIMD result. It currently supports two options:

1. Analyze diffusivity from a series vasprun.xml (or vasprun.xml.gz) files at one
    temperature. It will output diffusivity and corresponding error analysis result.

2. Fitting Arrhenius relationship from diffusivities at different temperatures. It
    will output activation energy and corresponding error analysis result. It can
    also be used to predict diffusivity/conductivity at different temperature
    from the fitted Arrehenius relationship. The conversion factor from diffusivity
    to conductivity is (z*e)^2*c/(K*T), details can be found in paper
"""
import os
import argparse
from argparse import RawTextHelpFormatter
import csv
from prettytable import PrettyTable
from aimd.diffusion import DiffusivityAnalyzer, ErrorAnalysisFromDiffusivityAnalyzer, \
    ArreheniusAnalyzer, get_conversion_factor
from pymatgen.core.periodic_table import Specie
from pymatgen.io.vasp import Poscar

citing_info = """
-----------------------------------------
Thanks for using this script.
Please consider citing our papers: https://github.com/mogroupumd/aimd#citing.
-----------------------------------------
"""

def Analyze_VASP_MD(args):
    """
    Analyze diffusivity from a series vasprun.xml (or vasprun.xml.gz) files at one
            temperature
    :param args: please check main function for details of args
    :return:
    """
    vasprun_dirs = []
    for i in range(args.runs_start, args.runs_end + 1):
        if os.path.exists(os.path.join(args.folder_feature + str(i), 'vasprun.xml.gz')):
            vasprun_dirs.append(os.path.join(args.folder_feature + str(i), 'vasprun.xml.gz'))
        elif os.path.exists(os.path.join(args.folder_feature + str(i), 'vasprun.xml')):
            vasprun_dirs.append(os.path.join(args.folder_feature + str(i), 'vasprun.xml'))
        else:
            raise Exception("No vasprun.xml or vasprun.xml.gz in folder {}".
                            format(args.folder_feature + str(i)))

    # In analyzing Arrhenius relationship, it is required to provide charged specie. To keep consistent, I also
    # require charged specie, even it is not necessary
    specie = Specie.from_string(args.specie)
    da = DiffusivityAnalyzer.from_files(vasprun_dirs, str(specie.element), step_skip=args.step_skip,
                                        ncores=args.ncores,
                                        time_intervals_number=args.time_intervals_number,
                                        spec_dict={'lower_bound': args.lower_bound_in_a_square \
                                                                  * args.site_distance \
                                                                  * args.site_distance,
                                                   'upper_bound': args.upper_bound,
                                                   'minimum_msd_diff': args.minimum_msd_diff_in_a_square \
                                                                       * args.site_distance \
                                                                       * args.site_distance,
                                                   }
                                        )
    ea = ErrorAnalysisFromDiffusivityAnalyzer(da, site_distance=args.site_distance)
    if da.diffusivity > 0:  # The linear fitting succeed
        summary_info = ea.get_summary_dict(oxidized_specie=args.specie)
    # if the msd profile of the MD doesn't fulfill the fitting requirements,
    # da.diffusivity is set  to be negative
    else:
        summary_info = {"diffusion result": "MSD calculated from MD doesn't fulfill the fitting requirement",
                        "max msd": max(da.msd),
                        'msd': da.msd,
                        'dt': da.dt,
                        'msd_component': da.msd_component
                        }
        print("Output msd-dt into {}K_msd-dt.csv".format(int(da.temperature)))
        args.msd_file = "{}K_msd-dt.csv".format(int(da.temperature))

    # output
    print("=" * 40)
    print("Used vasprun.xml files")
    print("Start run: {}, end run: {}".format(vasprun_dirs[0], vasprun_dirs[-1]))
    print("=" * 40)

    # If drift larger than 3 angstrom, raise warning
    if 'drift_maximum' in summary_info.keys():
        if max(summary_info['drift_maximum']) > 3:
            print("{} WARNING {}".format('='*20, '='*20))
            print("The entire cell has significant drift, please check the MD data")
            print("Maximum drift distance in xyz (Angstrom): {}".format(summary_info['drift_maximum']))
            print("{} WARNING {}".format('=' * 20, '=' * 20))
    # If max framework displacement larger than 5 angstrom, raise warning
    if 'max_framework_displacement' in summary_info.keys():
        if summary_info['max_framework_displacement'] > 5:
            print("{} WARNING {}".format('=' * 20, '=' * 20))
            print("There are significant movement for framework ions, may be melt. pLease check")
            print("Maximum framework_displacement (Angstrom): {}".format(summary_info['max_framework_displacement']))
            print("{} WARNING {}".format('=' * 20, '=' * 20))
    # results table
    header_result = ("Parameter", "Value")
    result_table = PrettyTable(header_result)
    result_table.align["Parameter"] = "l"
    for k, v in summary_info.items():
        if k not in ['msd', 'dt', 'msd_component']:
            result_table.add_row([k, str(v)])
    result_table.add_row(['composition', str(da.structure.composition)])
    print("Results table: ")
    print("Diffusivity unit: cm^2/s, Conductivity unit: mS/cm, Drift unit: Angstrom")
    print(result_table.get_string(sortby='Parameter'))
    print(citing_info)
    # whether output msd
    if args.msd_file:
        print("Output msd-dt into file: {}".format(args.msd_file))
        with open(args.msd_file, 'w') as fp:
            w_csv = csv.writer(fp, delimiter=',')
            data = [summary_info['dt'], summary_info['msd'],
                    summary_info['msd_component'][0], summary_info['msd_component'][1],
                    summary_info['msd_component'][2]]
            w_csv.writerows([["dt (fs)", "msd (A^2)", "msd_component_0", "msd_component_1", "msd_component_2"]])
            w_csv.writerows(zip(*data))

def Analyze_arrhenius(args):
    """
    Fitting Arrhenius relationship from diffusivities at different temperatures.
    :param args:  please check main function for details of args
    :return:
    """
    aa = ArreheniusAnalyzer.from_csv(args.D_T_csv_file)
    # results table
    header_result = ("Parameter", "Value")
    result_table = PrettyTable(header_result)
    result_table.align["Parameter"] = "l"
    result_table.add_row(["Ea (eV) ", aa.Ea])
    result_table.add_row(["Ea error (eV)", aa.Ea_error])

    if args.temperature:
        result_table.add_row(["D at {} K (cm^2/s)".format(args.temperature),
                              aa.predict_diffusivity(args.temperature)[0]])
        result_table.add_row(["D range at {} K (cm^2/s)".format(args.temperature),
                              aa.predict_diffusivity(args.temperature)[1]])
    if args.poscar:
        if not args.specie:
            raise Exception("Please provide diffusion specie, with charged information, like Li+.")
        if not args.temperature:
            raise Exception("Please provide target temperature, like 300")
        poscar = Poscar.from_file(args.poscar)
        struct = poscar.structure
        result_table.add_row(["Convertion factor from D (cm^2/s) to S (mS/cm)",
                              get_conversion_factor(struct, args.specie, args.temperature)])
        result_table.add_row(["Conductivity at {} K (mS/cm)".format(args.temperature),
                              aa.predict_conductivity(args.temperature, struct, args.specie)[0]])
        result_table.add_row(["Conductivity range at {} K (mS/cm)".format(args.temperature),
                              aa.predict_conductivity(args.temperature, struct, args.specie)[1]])
    print("=" * 40)
    if aa.diffusivity_errors:
        print("Considering diffusivity errors as weight during analysis.")
    else:
        print("No diffusivity errors are provided, use ordinary fitting.")
    print("=" * 40)
    print("Results table: ")
    print(result_table.get_string())

    if args.verbose:
        verbose_table = PrettyTable(("Patameter", "Value"))
        verbose_table.add_row(["1000/T (1/K)", aa.x])
        verbose_table.add_row(["log10(D) (cm^2/s)", aa.y])
        if aa.y_error:
            verbose_table.add_row(["Error bar of log10(D) (cm^2/s)", aa.y_error])
        verbose_table.add_row(["Slope in arrhenius relationship", aa.slope])
        verbose_table.add_row(["Standard deviation of slope in arrhenius relationship", aa.slope_sigma])
        verbose_table.add_row(["Intercept in arrhenius relationship", aa.intercept])
        verbose_table.add_row(["Standard deviation of intercept in arrhenius relationship", aa.intercept_sigma])
        print("All details in arrhenius fitting")
        print(verbose_table.get_string())

    print(citing_info)
    if args.plot:
        plt = aa.get_arrhenius_plot()
        plt.show()

def main():
    parser = argparse.ArgumentParser(
        description="""
        This script is used to analyze VASP AIMD result. It currently supports two options:

        1. Analyze diffusivity from a series vasprun.xml (or vasprun.xml.gz) files at one
            temperature. It will output diffusivity and corresponding error analysis result.

        2. Fitting Arrhenius relationship from diffusivities at different temperatures. It
            will output activation energy and corresponding error analysis result. It can
            also be used to predict diffusivity/conductivity at different temperature
            from the fitted Arrehenius relationship. The conversion factor from diffusivity
            to conductivity is (z*e)^2*c/(K*T), details can be found in paper ...
        """,
        formatter_class=RawTextHelpFormatter
    )

    subparsers = parser.add_subparsers()

    parser_diffusivity = subparsers.add_parser(
        "diffusivity",
        help="Analyze diffusivity from a series vasprun.xml (or vasprun.xml.gz) files"
    )
    parser_diffusivity.add_argument(
        "specie", type=str,
        help="The diffusing species. "
             "Pleae provide species with charge, like Li+, not Li. Useful for calculation "
             "of conductivity from diffusivity."
    )
    parser_diffusivity.add_argument(
        "folder_feature", type=str,
        help="The shared folder feature of all MD result folders. "
             "For example, you want to analyze ./RUN_10/vasprun.xml.gz to ./RUN_100/vasprun.xml.gz. "
             "Then you should provide ./RUN_ or just RUN_. "
             "If your folders are run_000 to run_100, please change to format folder_feature + index_number, "
             "like run_0, ... run_100"
    )
    parser_diffusivity.add_argument(
        "runs_start", type=int,
        help="The start index number of the folders you want to analyze. "
    )
    parser_diffusivity.add_argument(
        "runs_end", type=int,
        help="The end index number of the folders you want to analyze. "
             "The end folder is included in the analysis. "
             "For example, if you want to analyze vasprun in folders [RUN_10, ..., RUN_20], "
             "you should provide runs_end as 20."
    )
    parser_diffusivity.add_argument(
        "site_distance", type=float,
        help="The distance between neighboring sites for the diffusion specie. "
             "If there are many different paths, you can provide an averaged value"
    )

    parser_diffusivity.add_argument(
        "-msd", "--msd_file", dest="msd_file", type=str, default=None,
        help="Option to save msd-dt file in csv format. "
             "If a file name is not provided, it will not output the msd-dt data"
    )
    parser_diffusivity.add_argument(
        "--step_skip", type=int, default=10,
        help="How many steps to skip in reading MD data, default is 10."
    )
    parser_diffusivity.add_argument(
        "-dt", "--time_intervals_number", type=int, default=1000,
        help="Spec of the time interval (dt) setting, the number of time_intervals. Default is 1000. "
    )
    parser_diffusivity.add_argument(
        "-l", "--lower_bound_in_a_square", type=float, default=0.5,
        help="The lower fitting bound in unit of a^2, a is the site distance. "
             "Default is 0.5. Typical lower bound is in unit of A^2"
    )
    parser_diffusivity.add_argument(
        "-u", "--upper_bound", type=float, default=0.5,
        help="The upper fitting bound, unit is total time. "
             "Default is 0.5 t_total"
    )
    parser_diffusivity.add_argument(
        "-min", "--minimum_msd_diff_in_a_square", type=float, default=0.5,
        help="The minimum msd range for fitting range, in unit of a^2. "
             "The fitting range should cover a range of MSD, default is 0.5. "
    )
    parser_diffusivity.add_argument(
        "-n", "--ncores", type=int, default=1,
        help="Number of cores used to do the analysis, default is 1. "
    )
    parser_diffusivity.set_defaults(
        func=Analyze_VASP_MD
    )

    parser_arrhenius = subparsers.add_parser(
        "arrhenius",
        help="Arrhenius relationship from diffusivities at different temperatures. "
             "It will read csv file containing diffusivities, temperatures and diffusivities error (optional). "
             "The header of csv MUST be D,T,D_error"
             "You can also predict diffusivity at different temperatures. "
             "More over, if you can also provide structure by POSCAR and the diffusion specie, "
             "you can obtain the corresponding conductivity."
    )  # Add this line to arrhenius -h output

    parser_arrhenius.add_argument(
        "D_T_csv_file", type=str,
        help="The csv file containing temperatures, diffusivities, and diffusivities error (error is optional). "
             "The csv header must be D,T, D_error, and the columns must follow that form."
    )
    parser_arrhenius.add_argument(
        "-T", "--temperature", type=float, default=None,
        help="new temperature to predict diffusivity/conductivity"
    )
    parser_arrhenius.add_argument(
        "-p", "--poscar", type=str, default=None,
        help="The POSCAR file of structure, any structure file is okay, "
             "as long as the volume and number of diffusion specie are same."
    )
    parser_arrhenius.add_argument(
        "-s", "--specie", type=str, default=None,
        help="The interested diffusion specie. "
             "It must charge information like Li+, not Li."
    )
    parser_arrhenius.add_argument(
        "-v", "--verbose", default=False, action="store_true",
        help="Whether output details of fitting"
    )
    parser_arrhenius.add_argument(
        "-pl", "--plot", default=False, action='store_true',
        help="Whether plot the arrhenius plot."
    )
    parser_arrhenius.set_defaults(
        func=Analyze_arrhenius
    )

    args = parser.parse_args()
    args.func(args)

if __name__ == '__main__':
    main()


import cv2
import os

from pathlib import Path
from glob import glob
from tqdm import tqdm
import shutil
def convert_images(input_folder,output_folder):
    # Path(output_folder).mkdir(parents=True, exist_ok=True)
    input_files = glob(os.path.join(input_folder, "*"))
    for f in input_files:
        image = cv2.imread(f, cv2.IMREAD_GRAYSCALE)
        # quantize
        image = (image // 43) * 43
        image[image > 43] = 255
        cv2.imwrite(os.path.join(output_folder, os.path.basename(f)), image)

if __name__ == "__main__":
    folder_name = "./Submission_4/train"
    folders_output = glob(f"{folder_name}/*")
    folders_input = [folder+'\\augmented' for folder in folders_output]
    # folders_input,folders_output = ['./Submission_4\\train\\ix\\Manual'], ['./Submission_4\\train\\ix\\']
    for inp,out in tqdm(zip(folders_input,folders_output)):
        convert_images(inp,out)
    for f in folders_input:
        try:
            shutil.rmtree(f)
        except:
            print(f)
    # img_dir  = './test/train/'
    # img_augmentation(img_dir)


from colorama import Fore, Back, Style
def remover_emails_repetidos(lista_emails):
    emails_unicos = list(set(lista_emails))
    emails_removidos = len(lista_emails) - len(emails_unicos)
    return emails_unicos, emails_removidos

lista_emails = ['']
emails_unicos, emails_removidos = remover_emails_repetidos(lista_emails)

#print(emails_unicos)
print('\n'.join([f"{email}" for email in lista_emails]))
print(f'Quantidade de emails removidos: {Back.RED}{Fore.WHITE}  {emails_removidos}  {Style.RESET_ALL}')


# use of argparse for handling the %commands in the cells
import argparse
import logging
import os
import re
import shlex
import time
import nbconvert
import nbformat
import websocket  # only for WebSocketConnectionClosedException
from ipykernel.kernelbase import Kernel

from . import deviceconnector

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

ap_serial_connect = argparse.ArgumentParser(prog="%serialconnect", add_help=False)
ap_serial_connect.add_argument('--raw', help='Just open connection', action='store_true')
ap_serial_connect.add_argument('--port', type=str, default=0)
ap_serial_connect.add_argument('--baud', type=int, default=115200)
ap_serial_connect.add_argument('--verbose', action='store_true')
ap_serial_connect.add_argument('--clear', '-c',
                               help='clears the state of the MicroPython virtual machine after connected',
                               action='store_true')

ap_socket_connect = argparse.ArgumentParser(prog="%socketconnect", add_help=False)
ap_socket_connect.add_argument('--raw', help='Just open connection', action='store_true')
ap_socket_connect.add_argument('ipnumber', type=str)
ap_socket_connect.add_argument('portnumber', type=int)

ap_disconnect = argparse.ArgumentParser(prog="%disconnect", add_help=False)
ap_disconnect.add_argument('--raw', help='Close connection without exiting paste mode', action='store_true')

ap_websocket_connect = argparse.ArgumentParser(prog="%websocketconnect", add_help=False)
ap_websocket_connect.add_argument('--raw', help='Just open connection', action='store_true')
ap_websocket_connect.add_argument('websocketurl', type=str, default="ws://192.168.4.1:8266", nargs="?")
ap_websocket_connect.add_argument("--password", type=str)
ap_websocket_connect.add_argument('--verbose', action='store_true')

ap_write_bytes = argparse.ArgumentParser(prog="%writebytes", add_help=False)
ap_write_bytes.add_argument('--binary', '-b', action='store_true')
ap_write_bytes.add_argument('--verbose', '-v', action='store_true')
ap_write_bytes.add_argument('stringtosend', type=str)

ap_read_bytes = argparse.ArgumentParser(prog="%readbytes", add_help=False)
ap_read_bytes.add_argument('--binary', '-b', action='store_true')

ap_send_to_file = argparse.ArgumentParser(prog="%sendfile",
                                          description="send a file to the microcontroller's file system",
                                          add_help=False)
ap_send_to_file.add_argument('--append', '-a', action='store_true')
ap_send_to_file.add_argument('--mkdir', '-d', action='store_true')
ap_send_to_file.add_argument('--binary', '-b', action='store_true')
ap_send_to_file.add_argument('--execute', '-x', action='store_true')
ap_send_to_file.add_argument('--source', help="source file", type=str, default="<<cellcontents>>", nargs="?")
ap_send_to_file.add_argument('--quiet', '-q', action='store_true')
ap_send_to_file.add_argument('--QUIET', '-Q', action='store_true')
ap_send_to_file.add_argument('destinationfilename', type=str, nargs="?")

ap_upload_main = argparse.ArgumentParser(prog="%uploadmain",
                                         description="upload file(.py/.ipynb) as 'main.py' to the microcontroller's "
                                                     "file system",
                                         add_help=False)
ap_upload_main.add_argument('--source', help='source file(.py/.ipynb)', type=str)
ap_upload_main.add_argument('--reboot', '-r', help='hard reset after uploaded', action='store_true')

ap_upload_project = argparse.ArgumentParser(prog="%uploadproject",
                                            description="upload all files in the specified folder to the"
                                                        " microcontroller's file system"
                                                        " while convert all .ipynb files to .py files")
ap_upload_project.add_argument('--source', help='project source directory', type=str, default=".")
ap_upload_project.add_argument('--reboot', '-r', help='hard reset after uploaded', action='store_true')
ap_upload_project.add_argument('--emptydevice', '-e', help='empty device before uploaded', action='store_true')
ap_upload_project.add_argument('--onlypy', '-py', help='Only upload all .py and .ipynb files', action='store_true')

ap_ls = argparse.ArgumentParser(prog="%ls", description="list directory of the microcontroller's file system",
                                add_help=False)
ap_ls.add_argument('--recurse', '-r', action='store_true')
ap_ls.add_argument('dirname', type=str, nargs="?")

ap_meminfo = argparse.ArgumentParser(prog="%meminfo", add_help=False)

ap_remove = argparse.ArgumentParser(prog="%remove",
                                    description="remove a file of the microcontroller's file system",
                                    add_help=False)
ap_remove.add_argument('filename', type=str)

ap_rmdir = argparse.ArgumentParser(prog="%rmdir",
                                   description="remove a directory of the microcontroller's file system",
                                   add_help=False)
ap_rmdir.add_argument('directory', type=str)

ap_fetch_file = argparse.ArgumentParser(prog="%fetchfile",
                                        description="fetch a file from the microcontroller's file system",
                                        add_help=False)
ap_fetch_file.add_argument('--binary', '-b', action='store_true')
ap_fetch_file.add_argument('--print', '-p', action="store_true")
ap_fetch_file.add_argument('--load', '-l', action="store_true")
ap_fetch_file.add_argument('--quiet', '-q', action='store_true')
ap_fetch_file.add_argument('--QUIET', '-Q', action='store_true')
ap_fetch_file.add_argument('sourcefilename', type=str)
ap_fetch_file.add_argument('destinationfilename', type=str, nargs="?")

ap_mpycross = argparse.ArgumentParser(prog="%mpy-cross", add_help=False)
ap_mpycross.add_argument('--set-exe', type=str)
ap_mpycross.add_argument('pyfile', type=str, nargs="?")

ap_esptool = argparse.ArgumentParser(prog="%esptool", add_help=False)
ap_esptool.add_argument('--port', type=str, default=0)
ap_esptool.add_argument('espcommand', choices=['erase', 'esp32', 'esp8266'])
ap_esptool.add_argument('binfile', type=str, nargs="?")

ap_capture = argparse.ArgumentParser(prog="%capture", description="capture output printed by device and save to a file",
                                     add_help=False)
ap_capture.add_argument('--quiet', '-q', action='store_true')
ap_capture.add_argument('--QUIET', '-Q', action='store_true')
ap_capture.add_argument('outputfilename', type=str)

ap_write_file_pc = argparse.ArgumentParser(prog="%%writefile", description="write contents of cell to file on PC",
                                           add_help=False)
ap_write_file_pc.add_argument('--append', '-a', action='store_true')
ap_write_file_pc.add_argument('--execute', '-x', action='store_true')
ap_write_file_pc.add_argument('destinationfilename', type=str)

def parse_ap(ap, args1):
    try:
        return ap.parse_known_args(args1)[0]
    except SystemExit:
        # argparse throws these because it assumes you only want to do the command line
        # should be a default one
        return None

class MicroPythonKernel(Kernel):
    def do_apply(self, content, bufs, msg_id, reply_metadata):
        pass

    def do_clear(self):
        pass

    implementation = 'micropython_kernel'
    implementation_version = "v3"

    banner = "MicroPython Serializer"

    language_info = {'name': 'micropython',
                     'codemirror_mode': 'python',
                     'mimetype': 'text/python',
                     'file_extension': '.py'}

    def __init__(self, **kwargs):
        Kernel.__init__(self, **kwargs)
        self.silent = False
        self.dc = deviceconnector.DeviceConnector(self.sres, self.sres_system)
        self.mpycrossexe = None

        self.srescapturemode = 0
        # 0 none, 1 print lines, 2 print on-going line count (--quiet), 3 print only final line count (--QUIET)
        self.srescapturedoutputfile = None  # used by %capture command
        self.srescapturedlinecount = 0
        self.srescapturedlasttime = 0  # to control the frequency of capturing reported

    def interpret_percent_line(self, percent_line, cell_contents):
        try:
            percentstringargs = shlex.split(percent_line)
        except ValueError as e:
            self.sres("\n\n***Bad percentcommand [%s]\n" % str(e), 31)
            self.sres(percent_line)
            return None

        percentcommand = percentstringargs[0]

        if percentcommand == ap_serial_connect.prog:
            apargs = parse_ap(ap_serial_connect, percentstringargs[1:])

            self.dc.disconnect(apargs.verbose)
            self.dc.serial_connect(apargs.port, apargs.baud, apargs.verbose)
            if self.dc.working_serial:
                if not apargs.raw:
                    self.sres_system("\nKill off running programs...\n")
                    self.dc.write_bytes(b'\x03')
                    self.dc.write_bytes(b'\x03')
                    self.dc.write_bytes(b'\x03')
                    if self.dc.enter_paste_mode(verbose=apargs.verbose):
                        if apargs.clear:
                            self.dc.send_reboot_message()
                            self.dc.enter_paste_mode()
                        self.sres_system("\nReady.\n")
                    else:
                        self.sres("Disconnecting [paste mode not working]\n", 31)
                        self.dc.disconnect(verbose=apargs.verbose)
                        self.sres_system("  (You may need to reset the device)")
                        cell_contents = ""
            else:
                cell_contents = ""
            return cell_contents.strip() and cell_contents or None

        if percentcommand == ap_websocket_connect.prog:
            apargs = parse_ap(ap_websocket_connect, percentstringargs[1:])
            if apargs.password is None and not apargs.raw:
                self.sres(ap_websocket_connect.format_help())
                return None
            self.dc.websocket_connect(apargs.websocketurl)
            if self.dc.working_websocket:
                self.sres_system("** WebSocket connected **\n")
                if not apargs.raw:
                    pline = self.dc.working_websocket.recv()
                    self.sres(pline)
                    if pline == 'Password: ' and apargs.password is not None:
                        self.dc.working_websocket.send(apargs.password)
                        self.dc.working_websocket.send("\r\n")
                        res = self.dc.working_serial_readall()
                        self.sres(res)  # '\r\nWebREPL connected\r\n>>> '
                        if not apargs.raw:
                            if self.dc.enter_paste_mode(apargs.verbose):
                                self.sres_system("Ready.\n")
                            else:
                                self.sres("Disconnecting [paste mode not working]\n", 31)
                                self.dc.disconnect(verbose=apargs.verbose)
                                self.sres("  (You may need to reset the device)")
                                cell_contents = ""
            else:
                cell_contents = ""
            return cell_contents.strip() and cell_contents or None

        # this is the direct socket kind, not attached to a webrepl
        if percentcommand == ap_socket_connect.prog:
            apargs = parse_ap(ap_socket_connect, percentstringargs[1:])
            self.dc.socket_connect(apargs.ipnumber, apargs.portnumber)
            if self.dc.working_socket:
                self.sres("\n ** Socket connected **\n\n", 32)
                if apargs.verbose:
                    self.sres(str(self.dc.working_socket))
                self.sres("\n")
                # if not apargs.raw:
                #    self.dc.enterpastemode()
            return cell_contents.strip() and cell_contents or None

        if percentcommand == ap_esptool.prog:
            apargs = parse_ap(ap_esptool, percentstringargs[1:])
            if apargs and (apargs.espcommand == "erase" or apargs.binfile):
                self.dc.esptool(apargs.espcommand, apargs.port, apargs.binfile)
            else:
                self.sres(ap_esptool.format_help())
                self.sres("Please download the bin file from https://micropython.org/download/#{}".format(
                    apargs.espcommand if apargs else ""))
            return cell_contents.strip() and cell_contents or None

        if percentcommand == ap_write_file_pc.prog:
            apargs = parse_ap(ap_write_file_pc, percentstringargs[1:])
            if apargs:
                if apargs.append:
                    self.sres("Appending to {}\n\n".format(apargs.destinationfilename), asciigraphicscode=32)
                    file_out = open(apargs.destinationfilename, "a")
                    file_out.write("\n")
                else:
                    self.sres("Writing {}\n\n".format(apargs.destinationfilename), asciigraphicscode=32)
                    file_out = open(apargs.destinationfilename, "w")

                file_out.write(cell_contents)
                file_out.close()
            else:
                self.sres(ap_write_file_pc.format_help())
            if not apargs.execute:
                return None
            return cell_contents  # should add in some blank lines at top to get errors right

        if percentcommand == "%mpy-cross":
            apargs = parse_ap(ap_mpycross, percentstringargs[1:])
            if apargs and apargs.set_exe:
                self.mpycrossexe = apargs.set_exe
            elif apargs.pyfile:
                if self.mpycrossexe:
                    self.dc.mpycross(self.mpycrossexe, apargs.pyfile)
                else:
                    self.sres("Cross compiler executable not yet set\n", 31)
                    self.sres(
                        "try: %mpy-cross --set-exe ~/extrepositories/micropython/mpy-cross/mpy-cross\n")
                if self.mpycrossexe:
                    self.mpycrossexe = "~/extrepositories/micropython/mpy-cross/mpy-cross"
            else:
                self.sres(ap_mpycross.format_help())
            return cell_contents.strip() and cell_contents or None

        if percentcommand == "%comment":
            self.sres(" ".join(percentstringargs[1:]), asciigraphicscode=32)
            return cell_contents.strip() and cell_contents or None

        if percentcommand == "%lsmagic":
            self.sres(re.sub("usage: ", "", ap_serial_connect.format_usage()))
            self.sres("    connects to a device over USB wire\n\n")
            self.sres(re.sub("usage: ", "", ap_socket_connect.format_usage()))
            self.sres("    connects to a socket of a device over wifi\n\n")
            self.sres("%suppressendcode\n    doesn't send x04 or wait to read after sending the contents of the cell\n")
            self.sres("  (assists for debugging using %writebytes and %readbytes)\n\n")
            self.sres(re.sub("usage: ", "", ap_websocket_connect.format_usage()))
            self.sres("    connects to the webREPL websocket of an ESP8266 over wifi\n")
            self.sres("    websocketurl defaults to ws://192.168.4.1:8266 but be sure to be connected\n\n")
            self.sres(re.sub("usage: ", "", ap_disconnect.format_usage()))
            self.sres("    disconnects from web/serial connection\n\n")

            self.sres("%rebootdevice\n    reboots device\n\n")
            self.sres("%hardreset\n    A hard reset is the same as performing a power cycle to the board. \n\n")
            self.sres(re.sub("usage: ", "", ap_ls.format_usage()))
            self.sres("    list files on the device\n\n")
            self.sres(re.sub("usage: ", "", ap_meminfo.format_usage()))
            self.sres("    show RAM size/used/free/use% info\n\n")
            self.sres(re.sub("usage: ", "", ap_read_bytes.format_usage()))
            self.sres("    does serial.read_all()\n\n")
            self.sres(re.sub("usage: ", "", ap_write_bytes.format_usage()))
            self.sres("    does serial.write() of the python quoted string given\n\n")

            self.sres(re.sub("usage: ", "", ap_write_file_pc.format_usage()))
            self.sres("    write contents of cell to a file\n\n")
            self.sres(re.sub("usage: ", "", ap_remove.format_usage()))
            self.sres("    remove file on the device\n\n")
            self.sres(re.sub("usage: ", "", ap_rmdir.format_usage()))
            self.sres("    remove directory on the device\n\n")
            self.sres(re.sub("usage: ", "", ap_send_to_file.format_usage()))
            self.sres("    send cell contents or file/direcectory to the device\n\n")
            self.sres(re.sub("usage: ", "", ap_upload_main.format_usage()))
            self.sres("    convert a .py or .ipynb file to a main.py and upload it\n\n")
            self.sres((re.sub("usage:", "", ap_upload_project.format_usage())))
            self.sres("    upload all files in the specified folder to the microcontroller's file system"
                      " while convert all .ipynb files to .py files\n\n")
            self.sres(re.sub("usage: ", "", ap_fetch_file.format_usage()))
            self.sres("    fetch and save a file from the device\n\n")

            self.sres(re.sub("usage: ", "", ap_esptool.format_usage()))
            self.sres("    commands for flashing your esp-device\n\n")
            self.sres(re.sub("usage: ", "", ap_capture.format_usage()))
            self.sres("    records output to a file\n\n")
            self.sres("%comment\n    print this into output\n\n")
            self.sres(re.sub("usage: ", "", ap_mpycross.format_usage()))
            self.sres("    cross-compile a .py file to a .mpy file\n\n")
            self.sres("%lsmagic\n    list magic commands\n\n")

            return None

        if percentcommand == ap_disconnect.prog:
            apargs = parse_ap(ap_disconnect, percentstringargs[1:])
            self.dc.disconnect(raw=apargs.raw, verbose=True)
            return None

        # remaining commands require a connection
        if not self.dc.serial_exists():
            return cell_contents

        if percentcommand == ap_capture.prog:
            apargs = parse_ap(ap_capture, percentstringargs[1:])
            if apargs:
                self.sres("Writing output to file {}\n\n".format(apargs.outputfilename), asciigraphicscode=32)
                self.srescapturedoutputfile = open(apargs.outputfilename, "w")
                self.srescapturemode = (3 if apargs.QUIET else (2 if apargs.quiet else 1))
                self.srescapturedlinecount = 0
            else:
                self.sres(ap_capture.format_help())
            return cell_contents

        if percentcommand == ap_write_bytes.prog:
            # (not effectively using the --binary setting)
            apargs = parse_ap(ap_write_bytes, percentstringargs[1:])
            if apargs:
                byte_sto_send = apargs.stringtosend.encode().decode("unicode_escape").encode()
                res = self.dc.write_bytes(byte_sto_send)
                if apargs.verbose:
                    self.sres(res, asciigraphicscode=34)
            else:
                self.sres(ap_write_bytes.format_help())
            return cell_contents.strip() and cell_contents or None

        if percentcommand == ap_read_bytes.prog:
            # (not effectively using the --binary setting)
            apargs = parse_ap(ap_read_bytes, percentstringargs[1:])
            time.sleep(
                0.1)  # just give it a moment if running on from a series of values (could use an --expect keyword)
            msg = self.dc.working_serial_readall()
            if apargs.binary:
                self.sres(repr(msg))
            elif type(msg) == bytes:
                self.sres(msg.decode(errors="ignore"))
            else:
                self.sres(msg)  # strings come back from webrepl
            return cell_contents.strip() and cell_contents or None

        if percentcommand == "%rebootdevice":
            self.dc.send_reboot_message()
            self.dc.enter_paste_mode()
            return cell_contents.strip() and cell_contents or None

        if percentcommand == "%hardreset":
            self.dc.send_hard_reset_message()
            self.dc.enter_paste_mode()
            return cell_contents.strip() and cell_contents or None

        if percentcommand == "%reboot":
            self.sres("Did you mean %rebootdevice or %hardreset?\n", 31)
            return None

        if percentcommand == "%%writetofile" or percentcommand == "%writefile":
            self.sres("Did you mean %%writefile?\n", 31)
            return None

        if percentcommand == "%serialdisconnect":
            self.sres("Did you mean %disconnect?\n", 31)
            return None

        if percentcommand == "%sendbytes":
            self.sres("Did you mean %writebytes?\n", 31)
            return None

        if percentcommand == "%reboot":
            self.sres("Did you mean %rebootdevice?\n", 31)
            return None

        if percentcommand in ("%savetofile", "%savefile", "%send_file"):
            self.sres("Did you mean to write %sendfile?\n", 31)
            return None

        if percentcommand in ("%readfile", "%fetchfromfile"):
            self.sres("Did you mean to write %fetchfile?\n", 31)
            return None

        if percentcommand == ap_fetch_file.prog:
            apargs = parse_ap(ap_fetch_file, percentstringargs[1:])
            if apargs:
                fetchedcontents = self.dc.fetch_file(apargs.sourcefilename, apargs.binary, apargs.quiet)
                if apargs.print:
                    self.sres(fetchedcontents.decode() if type(fetchedcontents) == bytes else fetchedcontents,
                              clear_output=True)

                if (apargs.destinationfilename or (not apargs.print and not apargs.load)) and fetchedcontents:
                    dst_file = apargs.destinationfilename or os.path.basename(apargs.sourcefilename)
                    self.sres("Saving file to {}".format(repr(dst_file)))
                    file_out = open(dst_file, "wb" if apargs.binary else "w")
                    file_out.write(fetchedcontents)
                    file_out.close()

                if apargs.load:
                    file_contents = fetchedcontents.decode() if type(fetchedcontents) == bytes else fetchedcontents
                    if not apargs.quiet:
                        file_contents = "#%s\n\n%s" % (" ".join(percentstringargs), file_contents)
                    set_next_input_payload = {"source": "set_next_input", "text": file_contents, "replace": True}
                    return set_next_input_payload

            else:
                self.sres(ap_fetch_file.format_help())
            return None

        if percentcommand == ap_ls.prog:
            apargs = parse_ap(ap_ls, percentstringargs[1:])
            if apargs:
                self.dc.listdir(apargs.dirname or "", apargs.recurse)
            else:
                self.sres(ap_ls.format_help())
            return None

        if percentcommand == ap_meminfo.prog:
            apargs = parse_ap(ap_meminfo, percentstringargs[1:])
            if apargs:
                self.dc.mem_info()
            else:
                self.sres(ap_meminfo.format_help())
            return None

        if percentcommand == ap_remove.prog:
            apargs = parse_ap(ap_remove, percentstringargs[1:])
            if apargs:
                self.dc.remove_file(apargs.filename or "")
            else:
                self.sres(ap_remove.format_help())
            return None

        if percentcommand == ap_rmdir.prog:
            apargs = parse_ap(ap_rmdir, percentstringargs[1:])
            if apargs:
                self.dc.remove_dir(apargs.directory or "main.py")
            else:
                self.sres(ap_rmdir.format_help())
            return None

        if percentcommand == ap_send_to_file.prog:
            apargs = parse_ap(ap_send_to_file, percentstringargs[1:])
            if apargs and not (apargs.source == "<<cellcontents>>" and not apargs.destinationfilename) and (
                    apargs.source is not None):

                dest_file_name = apargs.destinationfilename

                def send_to_file(filename, contents):
                    self.dc.send_to_file(filename, apargs.mkdir, apargs.append, apargs.binary, apargs.quiet, contents)
                    self.dc.send_hard_reset_message()

                if apargs.source == "<<cellcontents>>":
                    file_contents = cell_contents
                    if not apargs.execute:
                        cell_contents = None
                    send_to_file(dest_file_name, file_contents)

                else:
                    mode = "rb" if apargs.binary else "r"
                    if not dest_file_name:
                        dest_file_name = os.path.basename(apargs.source)
                    elif dest_file_name[-1] == "/":
                        dest_file_name += os.path.basename(apargs.source)

                    if os.path.isfile(apargs.source):
                        file_contents = open(apargs.source, mode).read()
                        if apargs.execute:
                            self.sres("Cannot execute sourced file\n", 31)
                        send_to_file(dest_file_name, file_contents)

                    elif os.path.isdir(apargs.source):
                        if apargs.execute:
                            self.sres("Cannot execute folder\n", 31)
                        for root, dirs, files in os.walk(apargs.source):
                            for fn in files:
                                skip = False
                                fp = os.path.join(root, fn)
                                real_path = os.path.relpath(fp, apargs.source)
                                if real_path.endswith('.py'):
                                    # Check for compiled copy, skip py if exists
                                    if os.path.exists(fp[:-3] + '.mpy'):
                                        skip = True
                                if not skip:
                                    dest_path = os.path.join(dest_file_name, real_path).replace('\\', '/')
                                    file_contents = open(os.path.join(root, fn), mode).read()
                                    send_to_file(dest_path, file_contents)
            else:
                self.sres(ap_send_to_file.format_help())
            return cell_contents  # allows for repeat %send_to_file in same cell
        if percentcommand == ap_upload_main.prog:
            apargs = parse_ap(ap_upload_main, percentstringargs[1:])
            if apargs:
                source = apargs.source if apargs.source else "main.ipynb"
                if (not source.endswith(".py")) and (not source.endswith(".ipynb")):
                    self.sres("you must choose a .py or .ipynb file")
                    return None
                self.upload_file(source)
                if apargs.reboot:
                    self.dc.send_hard_reset_message()
                    self.dc.enter_paste_mode()
                    return cell_contents.strip() and cell_contents or None
            else:
                self.sres(ap_upload_main.format_help())
            return None
        if percentcommand == ap_upload_project.prog:
            apargs = parse_ap(ap_upload_project, percentstringargs[1:])
            if apargs:
                if not os.path.isdir(apargs.source):
                    self.sres("source: {0} is not a directory, you can upload main.py by '%upload main --source '"
                              .format(apargs.source))
                    self.sres(ap_upload_project.format_help())
                    return None
                if apargs.emptydevice:
                    self.dc.remove_dir(".")
                self.upload_dir(apargs.source, apargs.onlypy)
                if apargs.reboot:
                    self.dc.send_hard_reset_message()
                    self.dc.enter_paste_mode()
                    return cell_contents.strip() and cell_contents or None
            else:
                self.sres(ap_upload_project.format_help())
                self.sres(ap_upload_project.format_help())
            return None
        self.sres("Unrecognized percentline {}\n".format([percent_line]), 31)
        return cell_contents

    def upload_file(self, source, mkdir=False, append=False, binary=False, quiet=True, root=""):
        if os.path.exists(source) and os.path.isfile(source):
            destination = source
            root_len = len(root)
            if root != "" and root_len+1 < len(source) and source[:root_len] == root:
                destination = destination[root_len+1:]
            if destination.endswith(".ipynb"):
                notebook = nbformat.read(source, as_version=4)
                py_exporter = nbconvert.PythonExporter()

                output, resources = py_exporter.from_notebook_node(notebook)
                file_contents = output.replace("get_ipython().run_line_magic", "# %")
                destination = destination.replace(".ipynb", ".py")
            else:
                file_contents = open(source, "r").read()
            self.sres("\n\nuploading '{0}'\n".format(destination))
            self.dc.send_to_file(destination, mkdir=mkdir, append=append, binary=binary,
                                 quiet=quiet, file_contents=file_contents)
        else:
            self.sres("'{0}' is not a file\n\n".format(source))

    def upload_dir(self, source, onlypy):
        if os.path.exists(source) and os.path.isdir(source):
            for root, dirs, files in os.walk(source, topdown=False):
                if onlypy:
                    files = [f for f in files if (not f[0] == '.' ) and ("/." not in root) and (f.endswith('.py') or f.endswith('.ipynb'))]
                else:
                    files = [f for f in files if (not f[0] == '.') and ("/." not in root)]
                for f in files:
                    self.upload_file(os.path.join(root, f), mkdir=True, binary=f.endswith(".mpy"), root=source)
        else:
            self.sres("'{0}' is not a directory\n\n".format(source))

    def run_normal_cell(self, cell_contents, bsuppressendcode):
        cmd_lines = cell_contents.splitlines(True)
        r = self.dc.working_serial_readall()
        if r:
            self.sres('[priorstuff] ')
            self.sres(str(r))

        for line in cmd_lines:
            if line:
                if line[-2:] == '\r\n':
                    line = line[:-2]
                elif line[-1] == '\n':
                    line = line[:-1]
                self.dc.write_line(line)
                r = self.dc.working_serial_readall()
                if r:
                    self.sres('[duringwriting] ')
                    self.sres(str(r))

        if not bsuppressendcode:
            self.dc.write_bytes(b'\r\x04')
            self.dc.receive_stream(seek_okay=True)

    def send_command(self, cell_contents):
        bsuppressendcode = False  # can't yet see how to get this signal through

        if self.srescapturedoutputfile:
            self.srescapturedoutputfile.close()  # shouldn't normally get here
            self.sres("closing stuck open srescapturedoutputfile\n")
            self.srescapturedoutputfile = None

        # extract any %-commands we have here at the start (or ending?),
        # tolerating pure comment lines and white space before the first %
        # (if there's no %-command in there, then no lines at the front get dropped due to being comments)

        while True:
            match_per_cent_line = re.match("(?:(?:\s*|(?:\s*#.*\n))*)(%.*)\n?(?:[ \r]*\n)?", cell_contents)
            if not match_per_cent_line:
                break
            cell_contents = self.interpret_percent_line(match_per_cent_line.group(1),
                                                        cell_contents[match_per_cent_line.end():])
            # discards the %command and a single blank line (if there is one) from the cell contents
            if isinstance(cell_contents, dict) and cell_contents.get("source") == "set_next_input":
                return cell_contents  # set_next_input_payload:
            if cell_contents is None:
                return None

        if not self.dc.serial_exists():
            self.sres("No serial connected\n", 31)
            self.sres("  %serialconnect to connect\n")
            self.sres("  %esptool to flash the device\n")
            self.sres("  %lsmagic to list commands")
            return None

        # run the cell contents as normal
        if cell_contents:
            self.run_normal_cell(cell_contents, bsuppressendcode)
        return None

    def sres_system(self, output, clear_output=False):  # system call
        self.sres(output, asciigraphicscode=34, clear_output=clear_output)

    # 1=bold, 31=red, 32=green, 34=blue; from http://ascii-table.com/ansi-escape-sequences.php
    def sres(self, output, asciigraphicscode=None, n04count=0, clear_output=False):
        if self.silent:
            return

        if self.srescapturedoutputfile and (n04count == 0) and not asciigraphicscode:
            self.srescapturedoutputfile.write(output)
            self.srescapturedlinecount += len(output.split("\n")) - 1
            if self.srescapturemode == 3:
                # 0 none, 1 print lines, 2 print on-going line count (--quiet), 3 print only final line count (--QUIET)
                return

            # changes the printing out to a lines captured statement every 1second.
            if self.srescapturemode == 2:  # (allow stderrors to drop through to normal printing
                srescapturedtime = time.time()
                if srescapturedtime < self.srescapturedlasttime + 1:  # update no more frequently than once a second
                    return
                self.srescapturedlasttime = srescapturedtime
                clear_output = True
                output = "{} lines captured".format(self.srescapturedlinecount)

        if clear_output:  # used when updating lines printed
            self.send_response(self.iopub_socket, 'clear_output', {"wait": True})
        if asciigraphicscode:
            output = "\x1b[{}m{}\x1b[0m".format(asciigraphicscode, output)
        stream_content = {'name': ("stdout" if n04count == 0 else "stderr"), 'text': output}
        self.send_response(self.iopub_socket, 'stream', stream_content)

    def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False):
        self.silent = silent
        if not code.strip():
            return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}}

        interrupted = False

        # clear buffer out before executing any commands (except the readbytes one)
        if self.dc.serial_exists() and not re.match(
                "\s*%readbytes|\s*%disconnect|\s*%serialconnect|\s*websocketconnect",
                code):
            prior_buffer = None
            try:
                prior_buffer = self.dc.working_serial_readall()
            except KeyboardInterrupt:
                interrupted = True
            except OSError as e:
                prior_buffer = []
                self.sres("\n\n***Connection broken [%s]\n" % str(e.strerror), 31)
                self.sres("You may need to reconnect")
                self.dc.disconnect(raw=True, verbose=True)

            except websocket.WebSocketConnectionClosedException as e:
                prior_buffer = []
                self.sres("\n\n***Websocket connection broken [%s]\n" % str(e), 31)
                self.sres("You may need to reconnect")
                self.dc.disconnect(raw=True, verbose=True)

            if prior_buffer:
                if type(prior_buffer) == bytes:
                    try:
                        prior_buffer = prior_buffer.decode()
                    except UnicodeDecodeError:
                        prior_buffer = str(prior_buffer)

                for pbline in prior_buffer.splitlines():
                    if deviceconnector.wifiMessageIgnore.match(pbline):
                        continue  # filter out boring wifi status messages
                    if pbline:
                        self.sres('[leftinbuffer] ')
                        self.sres(str([pbline]))
                        self.sres('\n')

        set_next_input_payload = None
        try:
            if not interrupted:
                set_next_input_payload = self.send_command(code)
        except KeyboardInterrupt:
            interrupted = True
        except OSError as e:
            self.sres("\n\n***OSError [%s]\n\n" % str(e.strerror))
        # except pexpect.EOF:
        #    self.sres(self.asyncmodule.before + 'Restarting Bash')
        #    self.startasyncmodule()

        if self.srescapturedoutputfile:
            if self.srescapturemode == 2:
                self.send_response(self.iopub_socket, 'clear_output', {"wait": True})
            if self.srescapturemode == 2 or self.srescapturemode == 3:
                output = "{} lines captured.".format(
                    self.srescapturedlinecount)  # finish off by updating with the correct number captured
                stream_content = {'name': "stdout", 'text': output}
                self.send_response(self.iopub_socket, 'stream', stream_content)

            self.srescapturedoutputfile.close()
            self.srescapturedoutputfile = None
            self.srescapturemode = 0

        if interrupted:
            self.sres_system("\n\n*** Sending Ctrl-C\n\n")
            if self.dc.serial_exists():
                self.dc.write_bytes(b'\r\x03')
                # interrupted = True
                try:
                    self.dc.receive_stream(seek_okay=False, warn_okay_priors=True)
                except KeyboardInterrupt:
                    self.sres("\n\nKeyboard interrupt while waiting response on Ctrl-C\n\n")
                except OSError as e:
                    self.sres("\n\n***OSError while issuing a Ctrl-C [%s]\n\n" % str(e.strerror))
            return {'status': 'abort', 'execution_count': self.execution_count}

        # everything already gone out with send_response(), but could detect errors (text between the two \x04s

        payload = [set_next_input_payload] if set_next_input_payload else []
        # {"source": "set_next_input", "text": "some cell content", "replace": False}
        return {'status': 'ok', 'execution_count': self.execution_count, 'payload': payload, 'user_expressions': {}}