import os
from math import sin, cos, radians

try:
    import matplotlib.pyplot as plt
except:
    print("Matplot lib isn\'t installed - cannot plot")
    from unittest.mock import MagicMock
    plt = MagicMock()

# Test data, U,W is not phi/U0
Mm = dict()   #[Position][Y U W]
Yu = dict()   #[Position][Y U]
Yekp = dict() #[position][Y epsilon k p]

cs = {'-25' : [0,  1.3e-1],
      '025' : [10, 1.34341204e-01],
      '060' : [10, 1.40418891e-01],
      '100' : [10, 1.47364818e-01],
      '175' : [10, 1.60388431e-01],
      '250' : [10, 1.73412044e-01],
      '330' : [10, 1.87303899e-01],
      '405' : [10, 2.00327512e-01]}

def getExperimentalData():
    for filename in os.listdir('test60'):
        if 'Mm' in filename and '.dat' in filename:
            position = filename[2:5]
            print('Parsing:', filename, 'which has info for location', position, 'mm')
            file_name = os.path.join('test60', filename)
            file = open(file_name)
            text = file.read().split('\n')
            file.close()

            Mm[position] = list()

            for line_no in range(10, len(text)):
                line = text[line_no]
                line = line.split(' ')
                out_line = list()

                for item in line:
                    if 0 < len(item):
                        out_line.append(float(item))

                if len(out_line) == 3:
                    Mm[position].append(out_line)

def getCFDData():
    # Get latest folder from CFD output
    latest_folder = 0
    for folder in os.listdir('sets'):
        if latest_folder < int(folder):
            latest_folder = int(folder)
            print('Latest folder from CFD is for iteration', latest_folder)

    # Parse CFD output files
    for filename in os.listdir(os.path.join('sets', str(latest_folder))):
        position = filename[4:7]
        print('Parsing:', filename, 'which has info for location', position, 'mm')
        file = open(os.path.join('sets', str(latest_folder), filename))
        text = file.read().split('\n')
        file.close();

        if 'epsilon' in filename:
            destination = Yekp  # works as a pointer!
        else:
            destination = Yu;

        destination[position] = list()

        for line in text:
            this_line = line.split("\t")

            out_line = list()
            for item in this_line:
                try:
                    value = float(item)
                    out_line.append(value)
                except:
                    pass
            try:
                out_line[0] = out_line[0]*1000 #convert Y to meters
            except:
                next
            destination[position].append(out_line)
    print()

def calc_y(position, y):
    #($2-$$1)*1000/cos(angle)
    radius = cs[position][1]
    angle  = radians(cs[position][0])
    y_actual = (radius - y/1000)/cos(angle) * 1000
    return y_actual

def interpolate_U(position, Y_lower, Y_upper, Y_actual, Y_lower_index, Y_upper_index):
    deviation_percent = (Y_actual - Y_lower) / (Y_upper - Y_lower)
    data_lower = Yu[position][Y_lower_index]
    data_upper = Yu[position][Y_upper_index]

    data_range = list([1]*len(data_upper))
    data_interpolated = list([1]*len(data_upper))

    for i in range(0, len(data_upper)):
        data_range[i] = data_upper[i] - data_lower[i]
        data_interpolated[i] = data_lower[i] + deviation_percent * data_range[i]

    return data_interpolated

def subtract(array1, array2):
    new_array = list()
    for i in range(0, min(len(array1), len(array2))):
        new_array.append(array1[i] - array2[i])

    return new_array

def getScoreUW (plotComparison=[]):
    score = 0
    num_tests = 0

    for position in Mm:
        # We can plot a comparison if desired,
        #  just give a position you want the plot to be
        #  made
        if position in plotComparison:
            comparison_Y = list()
            comparison_Uexp = list()
            comparison_Wexp = list()
            comparison_Ucfd = list()
            comparison_Wcfd = list()

        # For each position, and for each point in the experimental data
        for data_exp in Mm[position]:
            Y_exp = data_exp[0]
            U_exp = data_exp[1]
            W_exp = data_exp[2]

            # Search through the CFD data using upper and lower bounds
            #  for the corrosponding experimental Y value. As we search
            #  upwards we break as soon as we get a Y value above the
            #  experimental Y value
            for index_Yu in range(0, len(Yu[position])-1):
                data_cfd = Yu[position][index_Yu]
                Y_cfd = calc_y(position, data_cfd[0])
                if Y_exp <= Y_cfd:
                    Y_upper = Y_cfd
                    Y_upper_index = index_Yu
                    break
                else:
                    Y_lower = Y_cfd
                    Y_lower_index = index_Yu

            # Then we interpolate between the upper and lower bounds
            if Y_upper < Y_lower:    # Unless something went wrong
                continue
            elif Y_upper != Y_lower: # And given Upper is not equal to lower
                Y_cfd = interpolate_U(position,
                                      Y_lower, Y_upper, Y_exp,
                                      Y_lower_index, Y_upper_index)
            else:                    # otherwise if we have an exact number, use that
                Y_cfd = Yu[position][Y_upper_index]

            # Now we can calculate the U and W values
            angle = radians(cs[position][0])

            # Calculate the U value
            U_cfd = Y_cfd[3] * cos(angle) + Y_cfd[2] * sin(angle)
            U_error = (U_cfd-U_exp) / U_exp

            # Calculate the W value
            W_cfd = -Y_cfd[1]
            W_error = (W_cfd - W_exp)/W_exp

            if 0.5 < max(abs(U_error), abs(W_error)):
                print('At', position, 'between', round(Y_lower,2),
                      '<' , round(Y_exp,2), '<', round(Y_upper,2))

                if 0.5 < abs(U_error):
                    print('U is about', round(U_error*100,2), '% off the actual value')
                    print('U:', round(U_cfd/11.6,4), '!=', round(U_exp/11.6,4), '\n')

                if 0.5 < abs(W_error):
                    print('W is about', round(W_error*100,2), '% off the actual value')
                    print('W:', round(W_cfd/11.6,4), '!=', round(W_exp/11.6,4), '\n')

            if position in plotComparison:
                comparison_Ucfd.append(U_cfd)
                comparison_Wcfd.append(W_cfd)
                comparison_Uexp.append(U_exp)
                comparison_Wexp.append(W_exp)
                comparison_Y.append(Y_exp)

            score = score + (1 - abs(U_error)) + (1 - abs(W_error))
            num_tests = num_tests + 2 #Max score

        if position in plotComparison:
            plt.plot(comparison_Y, subtract(comparison_Ucfd, comparison_Uexp), 'b--',
                     comparison_Y, subtract(comparison_Wcfd, comparison_Wexp), 'b--',
                     comparison_Y, comparison_Ucfd, 'r-',
                     comparison_Y, comparison_Uexp, 'g^',
                     comparison_Y, comparison_Wcfd, 'r-',
                     comparison_Y, comparison_Wexp, 'g^')
            plt.show()

    return score, num_tests

if __name__ == '__main__':
    getExperimentalData()
    getCFDData()
    score, max_score = getScoreUW(['405'])
    print(round(score/max_score,4)*100, '%')


# Given a list of non negative integers, arrange them such that they form the largest number.
#
# For example, given [3, 30, 34, 5, 9], the largest formed number is 9534330.
#
# Note: The result may be very large, so you need to return a string instead of an integer.
#
# Credits:
# Special thanks to @ts for adding this problem and creating all test cases.
#
# Subscribe to see which companies asked this question
import  functools
class Solution:
    # @param {integer[]} nums
    # @return {string}

    def _cmp(self,x, y):
        a = int(str(x) + str(y))
        b = int(str(y) + str(x))
        if a > b:
            return 1
        elif b < a:
            return -1
        else:
            return 0

    def largestNumber(self, nums):
        result = [str(ele) for ele in nums]
        def _cmp( x, y):
            a = x + y
            b = y + x
            if a > b:
                return -1
            elif b < a:
                return 1
            else:
                return 0
        result.sort(key = functools.cmp_to_key(_cmp))
        index = len(result)-1
        for i in range(len(result)):
            if result[i]!= '0':
                index = i
                break
        result[:index] = []
        return ''.join(result)

hh = Solution()
nums = [0,0]
print(hh.largestNumber(nums))


# -----------------------------------------------------------
# Licensed under the terms of GNU GPL 2
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# -----------------------------------------------------------
# Tim Hancock/Matthias Kuhn 2017
# Oslandia 2022

import configparser
import os

from qgis.core import Qgis, QgsExpressionContextUtils, QgsProject
from qgis.PyQt.QtCore import NULL, QObject, pyqtSignal
from qgis.PyQt.QtWidgets import QMessageBox
from qgis.utils import iface

from .core.tomsMessageLog import TOMsMessageLog

class TOMsParams(QObject):

    tomsParamsNotFound = pyqtSignal()
    """ signal will be emitted if there is a problem with opening TOMs - typically a layer missing """
    tomsParamsSet = pyqtSignal()
    """ signal will be emitted if there is a problem with opening TOMs - typically a layer missing """

    def __init__(self):
        QObject.__init__(self)

        TOMsMessageLog.logMessage("In TOMSParams.init ...", level=TOMsMessageLog.DEBUG)
        self.tomsParamsList = [
            "BayWidth",
            "BayLength",
            "BayOffsetFromKerb",
            "LineOffsetFromKerb",
            "CrossoverShapeWidth",
            "PhotoPath",
            "MinimumTextDisplayScale",
            "TOMsDebugLevel",
            "AllowZoneEditing",
        ]

        self.tomsParamsDict = {}

    def getParams(self):

        TOMsMessageLog.logMessage("In TOMsParams.getParams ...", level=TOMsMessageLog.DEBUG)
        found = True

        # Check for project being open
        currProject = QgsProject.instance()

        if len(currProject.fileName()) == 0:
            QMessageBox.information(None, "ERROR", ("Project not yet open"))
            found = False

        else:

            # TOMsMessageLog.logMessage("In TOMSLayers.getParams ... starting to get", level=logging.DEBUG)

            for param in self.tomsParamsList:
                TOMsMessageLog.logMessage(
                    "In TOMsParams.getParams ... getting " + str(param),
                    level=TOMsMessageLog.DEBUG,
                )

                currParam = None
                try:
                    currParam = QgsExpressionContextUtils.projectScope(QgsProject.instance()).variable(param)
                except Exception:
                    QMessageBox.information(
                        None,
                        "ERROR",
                        ("Property " + param + " is not present"),
                    )

                if len(str(currParam)) > 0:
                    self.tomsParamsDict[param] = currParam
                    TOMsMessageLog.logMessage(
                        "In TOMsParams.getParams ... set " + str(param) + " as " + str(currParam),
                        level=TOMsMessageLog.DEBUG,
                    )
                else:
                    QMessageBox.information(
                        None,
                        "ERROR",
                        ("Property " + param + " is not present"),
                    )
                    found = False
                    break

        if not found:
            self.tomsParamsNotFound.emit()
        else:
            self.tomsParamsSet.emit()

            # TOMsMessageLog.logMessage("In TOMSLayers.getParams ... finished ", level=logging.DEBUG)

        return found

    def setParam(self, param):
        return self.tomsParamsDict.get(param)

class TOMsConfigFile(QObject):

    tomsConfigFileNotFound = pyqtSignal()
    """ signal will be emitted if TOMs config file is not found """

    def __init__(self):
        super().__init__()

        TOMsMessageLog.logMessage("In TOMsConfigFile.init ...", level=TOMsMessageLog.DEBUG)

        self.config = configparser.ConfigParser()

    def initialiseTOMsConfigFile(self):

        # function to open file "toms.conf". Assume path is same as project file - unless environ variable is set

        # check for environ variable
        configPath = None
        try:
            configPath = os.environ.get("TOMs_CONFIG_PATH")
        except Exception:
            TOMsMessageLog.logMessage("In getTOMsConfigFile. TOMs_CONFIG_PATH not found ...", level=Qgis.Info)

        if configPath is None:
            configPath = QgsExpressionContextUtils.projectScope(QgsProject.instance()).variable("project_home")

        if configPath == NULL:
            QMessageBox.information(
                None,
                "Information",
                "Project probably not opened",
                QMessageBox.Ok,
            )
            self.tomsConfigFileNotFound.emit()
            return

        TOMsMessageLog.logMessage(
            "In getTOMsConfigFile. config_path: {}".format(configPath),
            level=TOMsMessageLog.DEBUG,
        )

        configFile = os.path.abspath(os.path.join(configPath, "TOMs.conf"))
        TOMsMessageLog.logMessage(
            "In getTOMsConfigFile. TOMs_CONFIG_PATH: {}".format(configFile),
            level=TOMsMessageLog.DEBUG,
        )

        if not os.path.isfile(configFile):
            QMessageBox.information(
                None,
                "Information",
                "TOMs configuration file not found. Stopping ...",
                QMessageBox.Ok,
            )
            self.tomsConfigFileNotFound.emit()

        # now read file
        self.readTOMsConfigFile(configFile)

    def readTOMsConfigFile(self, configFile):

        try:
            self.config.read(configFile)
        except Exception as e:
            TOMsMessageLog.logMessage(
                "In TOMsConfigFile.init. Error reading config file ... {}".format(e),
                level=Qgis.Warning,
            )
            self.tomsConfigFileNotFound.emit()

    def getTOMsConfigElement(self, section, value):
        item = None
        try:
            item = self.config[section][value]
        except KeyError:
            TOMsMessageLog.logMessage(
                "In getTOMsConfigElement. not able to find: {}:{}".format(section, value),
                level=Qgis.Info,
            )

        return item

class TOMsLayers(QObject):
    tomsLayersNotFound = pyqtSignal()
    """ signal will be emitted if there is a problem with opening TOMs - typically a layer missing """
    tomsLayersSet = pyqtSignal()
    """ signal will be emitted if everything is OK with opening TOMs """

    def __init__(self):
        QObject.__init__(self)

        TOMsMessageLog.logMessage("In TOMSLayers.init ...", level=Qgis.Info)

        self.tomsLayerDict = {}
        self.formPath = ""
        self.tomsLayerList = None
        self.configFileObject = None

    def getTOMsLayerListFromConfigFile(self, configFileObject: "TOMsConfigFile"):
        self.configFileObject = configFileObject
        layers = configFileObject.getTOMsConfigElement("TOMsLayers", "layers")
        if layers:
            self.tomsLayerList = layers.split("\n")
            return True

        self.tomsLayersNotFound.emit()
        return False

    def getTOMsFormPathFromConfigFile(self, configFileObject):
        formPath = configFileObject.getTOMsConfigElement("TOMsLayers", "form_path")
        return os.path.expandvars(formPath)  # expand env var like USERPROFILE

    def setLayers(self, configFileObject):

        TOMsMessageLog.logMessage("In TOMSLayers.getLayers ...", level=Qgis.Info)
        found = True

        # Check for project being open
        project = QgsProject.instance()

        if len(project.fileName()) == 0:
            QMessageBox.information(iface.mainWindow(), "ERROR", "Project not yet open")
            found = False

        else:

            if not self.getTOMsLayerListFromConfigFile(configFileObject):
                QMessageBox.information(
                    iface.mainWindow(),
                    "ERROR",
                    "Problem with TOMs config file ...",
                )
                self.tomsLayersNotFound.emit()
                found = False

            self.formPath = self.getTOMsFormPathFromConfigFile(configFileObject)
            TOMsMessageLog.logMessage(
                "In TOMsLayers:setLayers. formPath is {} ...".format(self.formPath),
                level=Qgis.Info,
            )

            # check that path exists
            if not os.path.isdir(self.formPath):
                QMessageBox.information(
                    iface.mainWindow(),
                    "ERROR",
                    "Form path in config file was not found ...",
                )
                self.tomsLayersNotFound.emit()
                found = False

        if found:
            for layer in self.tomsLayerList:
                if QgsProject.instance().mapLayersByName(layer):
                    self.tomsLayerDict[layer] = QgsProject.instance().mapLayersByName(layer)[0]
                    # set paths for forms
                    layerEditFormConfig = self.tomsLayerDict[layer].editFormConfig()

                    if len(layerEditFormConfig.initFilePath()) > 0:
                        TOMsMessageLog.logMessage(
                            "In TOMsLayers:setLayers. cleaning useless initFilePath for layer {}...".format(layer),
                            level=Qgis.Info,
                        )
                        layerEditFormConfig.setInitFilePath("")
                        self.tomsLayerDict[layer].setEditFormConfig(layerEditFormConfig)

                    uiPath = layerEditFormConfig.uiForm()
                    if len(uiPath) > 0:
                        TOMsMessageLog.logMessage(
                            "In TOMsLayers:setLayers. current ui_path for layer {} is {} ...".format(layer, uiPath),
                            level=Qgis.Info,
                        )
                        # try to get basename - doesn't seem to work on Linux
                        # base_ui_path = os.path.basename(ui_path)
                        pathAbsolute = os.path.abspath(os.path.join(self.formPath, os.path.basename(uiPath)))
                        if not os.path.isfile(pathAbsolute):
                            TOMsMessageLog.logMessage(
                                "In TOMsLayers:setLayers. form path not found for layer {} ...".format(layer),
                                level=Qgis.Warning,
                            )
                        else:
                            TOMsMessageLog.logMessage(
                                "In TOMsLayers:setLayers. setting new path for form {} ...".format(pathAbsolute),
                                level=Qgis.Info,
                            )
                            layerEditFormConfig.setUiForm(pathAbsolute)
                            self.tomsLayerDict[layer].setEditFormConfig(layerEditFormConfig)

                            # TODO: may need to reinstate original values here - so save them somewhere useful
                    else:
                        TOMsMessageLog.logMessage(
                            "In TOMsLayers:setLayers. no ui_path for layer {}".format(layer),
                            level=Qgis.Info,
                        )

                else:
                    QMessageBox.information(
                        iface.mainWindow(),
                        "ERROR",
                        "Table " + layer + " is not present",
                    )
                    found = False
                    break

        # TODO: need to deal with any errors arising ...

        if not found:
            self.tomsLayersNotFound.emit()

    def removePathFromLayerForms(self):

        TOMsMessageLog.logMessage("In TOMSLayers.removePathFromLayerForms ...", level=Qgis.Info)

        # Check for project being open
        project = QgsProject.instance()

        if len(project.fileName()) == 0:
            QMessageBox.information(iface.mainWindow(), "ERROR", ("Project not yet open"))

        else:

            for layer in self.tomsLayerList:
                if QgsProject.instance().mapLayersByName(layer):
                    self.tomsLayerDict[layer] = QgsProject.instance().mapLayersByName(layer)[0]
                    # set paths for forms
                    layerEditFormConfig = self.tomsLayerDict[layer].editFormConfig()
                    uiPath = layerEditFormConfig.uiForm()
                    TOMsMessageLog.logMessage(
                        "In TOMsLayers:removePathFromLayerForms. ui_path for layer {} is {} ...".format(layer, uiPath),
                        level=Qgis.Info,
                    )
                    if len(self.formPath) > 0 and len(uiPath) > 0:
                        # try to get basename - doesn't seem to work on Linux
                        # base_ui_path = os.path.basename(ui_path)
                        formName = os.path.basename(uiPath)
                        layerEditFormConfig.setUiForm(formName)
                        self.tomsLayerDict[layer].setEditFormConfig(layerEditFormConfig)

                else:
                    QMessageBox.information(
                        iface.mainWindow(),
                        "ERROR",
                        ("Table " + layer + " is not present"),
                    )
                    break

    def getLayer(self, layer):
        return self.tomsLayerDict.get(layer)

class TOMsLabelLayerNames(QObject):
    def __init__(self, currRestrictionLayer):
        QObject.__init__(self)

        TOMsMessageLog.logMessage(
            "In TOMsLabelLayerName:initialising .... {}".format(currRestrictionLayer),
            level=Qgis.Warning,
        )

        self.labelLayerName = self.setCurrLabelLayerNames(currRestrictionLayer)
        self.labelLeaderLayersNames = self.setCurrLabelLeaderLayerNames(currRestrictionLayer)

    def setCurrLabelLayerNames(self, currRestrictionLayer):
        # given a layer return the associated layer with label geometry
        # get the corresponding label layer

        if currRestrictionLayer.name() == "Bays":
            labelLayerName = ["Bays.label_pos"]
        if currRestrictionLayer.name() == "Lines":
            labelLayerName = ["Lines.label_pos", "Lines.label_loading_pos"]
        if currRestrictionLayer.name() == "Signs":
            labelLayerName = []
        if currRestrictionLayer.name() == "RestrictionPolygons":
            labelLayerName = ["RestrictionPolygons.label_pos"]
        if currRestrictionLayer.name() == "CPZs":
            labelLayerName = ["CPZs.label_pos"]
        if currRestrictionLayer.name() == "ParkingTariffAreas":
            labelLayerName = ["ParkingTariffAreas.label_pos"]

        if len(labelLayerName) == 0:
            return [""]

        return labelLayerName

    def getCurrLabelLayerNames(self):
        # given a layer return the associated layer with label geometry
        # get the corresponding label layer

        return self.labelLayerName

    def setCurrLabelLeaderLayerNames(self, currRestrictionLayer):
        # given a layer return the associated layer with label geometry
        # get the corresponding label layer

        if currRestrictionLayer.name() == "Bays":
            labelLeaderLayersNames = ["Bays.label_ldr"]
        if currRestrictionLayer.name() == "Lines":
            labelLeaderLayersNames = ["Lines.label_ldr", "Lines.label_loading_ldr"]
        if currRestrictionLayer.name() == "Signs":
            labelLeaderLayersNames = []
        if currRestrictionLayer.name() == "RestrictionPolygons":
            labelLeaderLayersNames = ["RestrictionPolygons.label_ldr"]
        if currRestrictionLayer.name() == "CPZs":
            labelLeaderLayersNames = ["CPZs.label_ldr"]
        if currRestrictionLayer.name() == "ParkingTariffAreas":
            labelLeaderLayersNames = ["ParkingTariffAreas.label_ldr"]

        if len(labelLeaderLayersNames) == 0:
            return [""]

        return labelLeaderLayersNames

    def getCurrLabelLeaderLayerNames(self):
        return self.labelLeaderLayersNames


import subprocess
import os
import os.path
import shlex
import sys
from subprocess import *
import shutil
import uuid
import glob
from common import DeviceItem
import Utils.HandlerUtil
import traceback
try:
        import ConfigParser as ConfigParsers
except ImportError:
        import configparser as ConfigParsers

class DiskUtil(object):
    __instance__ = None
    patching = None
    logger = None
    mount_output = None

    def __init__(self, patching, logger):

        if DiskUtil.__instance__ is None:
            self.patching = patching
            self.logger = logger
            self.mount_output = None
            DiskUtil.__instance__ = self
        else:
            return DiskUtil.__instance__

    @staticmethod
    def get_instance(patching, logger):
        if not DiskUtil.__instance__:
            DiskUtil(patching, logger)

        return DiskUtil.__instance__

    def get_device_items_property(self, lsblk_path, dev_name, property_name):
        get_property_cmd = lsblk_path + " /dev/" + dev_name + " -b -nl -o NAME," + property_name
        get_property_cmd_args =Utils.HandlerUtil.HandlerUtility.split(self.logger, get_property_cmd)
        get_property_cmd_p = Popen(get_property_cmd_args,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
        output,err = get_property_cmd_p.communicate()
        output= str(output)
        lines = output.splitlines()
        for i in range(0,len(lines)):
            item_value_str = lines[i].strip()
            if(item_value_str != ""):
                disk_info_item_array =Utils.HandlerUtil.HandlerUtility.split(self.logger, item_value_str)
                if(dev_name == disk_info_item_array[0]):
                    if(len(disk_info_item_array) > 1):
                        return disk_info_item_array[1]
        return None

    def get_device_items_sles(self,dev_path):
        self.logger.log("get_device_items_sles : getting the blk info from " + str(dev_path), True)
        device_items = []
        #first get all the device names
        if(dev_path is None):
            get_device_cmd = self.patching.lsblk_path + " -b -nl -o NAME"
        else:
            get_device_cmd = self.patching.lsblk_path + " -b -nl -o NAME " + dev_path
        get_device_cmd_args =Utils.HandlerUtil.HandlerUtility.split(self.logger, get_device_cmd)
        p = Popen(get_device_cmd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out_lsblk_output, err = p.communicate()
        out_lsblk_output = str(out_lsblk_output)
        lines = out_lsblk_output.splitlines()
        for i in range(0,len(lines)):
            item_value_str = lines[i].strip()
            if(item_value_str != ""):
                disk_info_item_array =Utils.HandlerUtil.HandlerUtility.split(self.logger, item_value_str)
                device_item = DeviceItem()
                device_item.name = disk_info_item_array[0]
                device_items.append(device_item)

        for i in range(0,len(device_items)):
            device_item = device_items[i]
            device_item.file_system = self.get_device_items_property(lsblk_path=self.patching.lsblk_path,dev_name=device_item.name,property_name='FSTYPE')
            device_item.mount_point = self.get_device_items_property(lsblk_path=self.patching.lsblk_path,dev_name=device_item.name,property_name='MOUNTPOINT')
            device_item.label = self.get_device_items_property(lsblk_path=self.patching.lsblk_path,dev_name=device_item.name,property_name='LABEL')
            device_item.uuid = self.get_device_items_property(lsblk_path=self.patching.lsblk_path,dev_name=device_item.name,property_name='UUID')
            #get the type of device
            model_file_path = '/sys/block/' + device_item.name + '/device/model'
            if(os.path.exists(model_file_path)):
                with open(model_file_path,'r') as f:
                    device_item.model = f.read().strip()
            if(device_item.model == 'Virtual Disk'):
                self.logger.log("model is virtual disk", True)
                device_item.type = 'disk'
            if(device_item.type != 'disk'):
                partition_files = glob.glob('/sys/block/*/' + device_item.name + '/partition')
                if(partition_files is not None and len(partition_files) > 0):
                    self.logger.log("partition files exists", True)
                    device_item.type = 'part'
        return device_items

    def get_device_items_from_lsblk_list(self, lsblk_path, dev_path):
        self.logger.log("get_device_items_from_lsblk_list : getting the blk info from " + str(dev_path), True)
        device_items = []
        #first get all the device names
        if(dev_path is None):
            get_device_cmd = lsblk_path + " -b -nl -o NAME"
        else:
            get_device_cmd = lsblk_path + " -b -nl -o NAME " + dev_path
        get_device_cmd_args =Utils.HandlerUtil.HandlerUtility.split(self.logger, get_device_cmd)
        p = Popen(get_device_cmd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out_lsblk_output, err = p.communicate()
        if sys.version_info > (3,):
            out_lsblk_output =str(out_lsblk_output, encoding='utf-8', errors="backslashreplace")
        else:
            out_lsblk_output =str(out_lsblk_output)
        lines = out_lsblk_output.splitlines()
        device_items_temp = []
        for i in range(0,len(lines)):
            item_value_str = lines[i].strip()
            if(item_value_str != ""):
                disk_info_item_array =Utils.HandlerUtil.HandlerUtility.split(self.logger, item_value_str)
                device_item = DeviceItem()
                device_item.name = disk_info_item_array[0]
                device_items_temp.append(device_item)

        for i in range(0,len(device_items_temp)):
            device_item = device_items_temp[i]
            device_item.mount_point = self.get_device_items_property(lsblk_path=lsblk_path,dev_name=device_item.name,property_name='MOUNTPOINT')
            if (device_item.mount_point is not None):
                device_item.file_system = self.get_device_items_property(lsblk_path=lsblk_path,dev_name=device_item.name,property_name='FSTYPE')
                device_item.label = self.get_device_items_property(lsblk_path=lsblk_path,dev_name=device_item.name,property_name='LABEL')
                device_item.uuid = self.get_device_items_property(lsblk_path=lsblk_path,dev_name=device_item.name,property_name='UUID')
                device_item.type = self.get_device_items_property(lsblk_path=lsblk_path,dev_name=device_item.name,property_name='TYPE')
                device_items.append(device_item)
                self.logger.log("lsblk MOUNTPOINT=" + str(device_item.mount_point) + ", NAME=" + str(device_item.name) + ", TYPE=" + str(device_item.type) + ", FSTYPE=" + str(device_item.file_system) + ", LABEL=" + str(device_item.label) + ", UUID=" + str(device_item.uuid) + ", MODEL=" + str(device_item.model), True)
        return device_items

    def get_lsblk_pairs_output(self, lsblk_path, dev_path):
        self.logger.log("get_lsblk_pairs_output : getting the blk info from " + str(dev_path) + " using lsblk_path " + str(lsblk_path), True)

        # If an alternate user is specified  in vmbackup.conf, run lsblk command through that user, not with root access.
        # Fixes issues found in some SUSE-related distros where lsblk command gets stuck with root access
        # Sample vmbackup.conf file with such alternate user setting:
        # [lsblkUser]
        # username: vmadmin

        configfile = '/etc/azure/vmbackup.conf'
        command_user = ''
        alternate_user = False

        try :
            if os.path.exists(configfile):
                config = ConfigParsers.ConfigParser()
                config.read(configfile)
                if config.has_option('lsblkUser','username'):
                    lsblk_user = config.get('lsblkUser','username')
                    command_user = "su - " + lsblk_user + " -c"
                    if (dev_path is None):
                        command_user = command_user + ' \'' + 'lsblk -b -n -P -o NAME,TYPE,FSTYPE,MOUNTPOINT,LABEL,UUID,MODEL,SIZE' + '\''
                    else:
                        command_user = command_user + ' \'' + 'lsblk -b -n -P -o NAME,TYPE,FSTYPE,MOUNTPOINT,LABEL,UUID,MODEL,SIZE' + ' ' + dev_path + '\''
                    alternate_user = True
        except Exception as e:
            pass

        out_lsblk_output = None
        error_msg = None
        is_lsblk_path_wrong = False
        try:
            if (alternate_user):
                self.logger.log("Switching to alternate user to run this lsblk command: " + str(command_user), True)
                p = Popen(command_user, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
            elif(dev_path is None):
                p = Popen([str(lsblk_path), '-b', '-n','-P','-o','NAME,TYPE,FSTYPE,MOUNTPOINT,LABEL,UUID,MODEL,SIZE'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            else:
                p = Popen([str(lsblk_path), '-b', '-n','-P','-o','NAME,TYPE,FSTYPE,MOUNTPOINT,LABEL,UUID,MODEL,SIZE',dev_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        except Exception as e:
            errMsg = 'Exception in lsblk command, error: %s, stack trace: %s' % (str(e), traceback.format_exc())
            self.logger.log(errMsg, True, 'Error')
            is_lsblk_path_wrong = True
        if is_lsblk_path_wrong == False :
            out_lsblk_output, err = p.communicate()
            if sys.version_info > (3,):
                out_lsblk_output = str(out_lsblk_output, encoding='utf-8', errors="backslashreplace")
            else:
                out_lsblk_output = str(out_lsblk_output)
            error_msg = str(err)
            if(error_msg is not None and error_msg.strip() != ""):
                self.logger.log(str(err), True)
        return is_lsblk_path_wrong, out_lsblk_output, error_msg

    def get_which_command_result(self, program_to_locate):
        self.logger.log("getting the which info for  " + str(program_to_locate), True)
        out_which_output = None
        error_msg = None
        try:
            p = Popen(['which', str(program_to_locate)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out_which_output, err = p.communicate()
            if sys.version_info > (3,):
                out_which_output = str(out_which_output, encoding='utf-8', errors="backslashreplace")
            else:
                out_which_output = str(out_which_output)
            error_msg = str(err)
            if(error_msg is not None and error_msg.strip() != ""):
                self.logger.log(str(err), True)
            self.logger.log("which command result :" + str(out_which_output), True)
            if (out_which_output is not None):
                out_which_output = out_which_output.splitlines()[0]
        except Exception as e:
            errMsg = 'Exception in which command, error: %s, stack trace: %s' % (str(e), traceback.format_exc())
            self.logger.log(errMsg, True, 'Error')
        return out_which_output, error_msg

    def get_device_items(self, dev_path):
        if(self.patching.distro_info[0].lower() == 'suse' and self.patching.distro_info[1] == '11'):
            return self.get_device_items_sles(dev_path)
        else:
            self.logger.log("getting the blk info from " + str(dev_path), True)
            device_items = []
            lsblk_path = self.patching.lsblk_path
            # Get lsblk command output using lsblk_path as self.patching.lsblk_path
            is_lsblk_path_wrong, out_lsblk_output, error_msg = self.get_lsblk_pairs_output(lsblk_path, dev_path)
            # if lsblk_path was wrong, use /bin/lsblk or usr/bin/lsblk based on self.patching.usr_flag to get lsblk command output again for centos/redhat distros
            if (is_lsblk_path_wrong == True) and (self.patching.distro_info[0].lower() == 'centos' or self.patching.distro_info[0].lower() == 'redhat'):
                if self.patching.usr_flag == 1:
                    self.logger.log("lsblk path is wrong, removing /usr prefix", True, 'Warning')
                    lsblk_path = "/bin/lsblk"
                else:
                    self.logger.log("lsblk path is wrong, adding /usr prefix", True, 'Warning')
                    lsblk_path = "/usr/bin/lsblk"
                is_lsblk_path_wrong, out_lsblk_output, error_msg = self.get_lsblk_pairs_output(lsblk_path, dev_path)
            # if lsblk_path was still wrong, lsblk_path using "which" command
            if (is_lsblk_path_wrong == True):
                self.logger.log("lsblk path is wrong. finding path using which command", True, 'Warning')
                out_which_output, which_error_msg = self.get_which_command_result('lsblk')
                # get lsblk command output
                if (out_which_output is not None):
                     lsblk_path = str(out_which_output)
                     is_lsblk_path_wrong, out_lsblk_output, error_msg = self.get_lsblk_pairs_output(lsblk_path, dev_path)
            # if error_msg contains "invalid option" or "P" (rely on only "-P" optiont in error to handle non-English locales), then get device_items using method get_device_items_from_lsblk_list
            if (error_msg is not None and error_msg.strip() != "" and ('invalid option' in error_msg or 'P' in error_msg)):
                device_items = self.get_device_items_from_lsblk_list(lsblk_path, dev_path)
            # else get device_items from parsing the lsblk command output
            elif (out_lsblk_output is not None):
                lines = out_lsblk_output.splitlines()
                for i in range(0,len(lines)):
                    item_value_str = lines[i].strip()
                    if(item_value_str != ""):
                        disk_info_item_array =Utils.HandlerUtil.HandlerUtility.split(self.logger, item_value_str)
                        device_item = DeviceItem()
                        disk_info_item_array_length = len(disk_info_item_array)
                        for j in range(0, disk_info_item_array_length):
                            disk_info_property = disk_info_item_array[j]
                            property_item_pair = disk_info_property.split('=')

                            if(property_item_pair[0] == 'NAME'):
                                device_item.name = property_item_pair[1].strip('"')

                            if(property_item_pair[0] == 'TYPE'):
                                device_item.type = property_item_pair[1].strip('"')

                            if(property_item_pair[0] == 'FSTYPE'):
                                device_item.file_system = property_item_pair[1].strip('"')

                            if(property_item_pair[0] == 'MOUNTPOINT'):
                                device_item.mount_point = property_item_pair[1].strip('"')

                            if(property_item_pair[0] == 'LABEL'):
                                device_item.label = property_item_pair[1].strip('"')

                            if(property_item_pair[0] == 'UUID'):
                                device_item.uuid = property_item_pair[1].strip('"')

                            if(property_item_pair[0] == 'MODEL'):
                                device_item.model = property_item_pair[1].strip('"')

                        self.logger.log("lsblk MOUNTPOINT=" + str(device_item.mount_point) + ", NAME=" + str(device_item.name) + ", TYPE=" + str(device_item.type) + ", FSTYPE=" + str(device_item.file_system) + ", LABEL=" + str(device_item.label) + ", UUID=" + str(device_item.uuid) + ", MODEL=" + str(device_item.model), True)

                        if(device_item.mount_point is not None and device_item.mount_point != "" and device_item.mount_point != " "):
                            device_items.append(device_item)
            return device_items

    def get_mount_command_output(self, mount_path):
        self.logger.log("getting the mount info using mount_path " + str(mount_path), True)
        out_mount_output = None
        error_msg = None
        is_mount_path_wrong = False
        try:
            p = Popen([str(mount_path)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        except Exception as e:
            errMsg = 'Exception in mount command, error: %s, stack trace: %s' % (str(e), traceback.format_exc())
            self.logger.log(errMsg, True, 'Error')
            is_mount_path_wrong = True
        if is_mount_path_wrong == False :
            out_mount_output, err = p.communicate()
            if sys.version_info > (3,):
                out_mount_output = str(out_mount_output, encoding='utf-8', errors="backslashreplace")
            else:
                out_mount_output = str(out_mount_output)
            self.logger.log("getting the mount info using mount_path " + out_mount_output, True)
            error_msg = str(err)
            if(error_msg is not None and error_msg.strip() != ""):
                self.logger.log(str(err), True)
        return is_mount_path_wrong, out_mount_output, error_msg

    def get_mount_points(self):
        mount_points_info = []
        mount_points = []
        fs_types = []
        out_mount_output = self.get_mount_output()
        if (out_mount_output is not None):
            #Extract the list of mnt_point in order
            lines = out_mount_output.splitlines()
            #Reverse the mount command output to go-through from last-to-first mounts in output
            lines.reverse()
            for line in lines:
                line = line.strip()
                if(line != ""):
                    deviceName =Utils.HandlerUtil.HandlerUtility.split(self.logger, line)[0]
                    mountPrefixStr = " on /"
                    prefixIndex = line.find(mountPrefixStr)
                    if(prefixIndex >= 0):
                        mountpointStart = prefixIndex + len(mountPrefixStr) - 1
                        fstypePrefixStr = " type "
                        mountpointEnd = line.find(fstypePrefixStr, mountpointStart)
                        if(mountpointEnd >= 0):
                            mount_point = line[mountpointStart:mountpointEnd]
                            fs_type = ""
                            fstypeStart = line.find(fstypePrefixStr) + len(fstypePrefixStr) - 1
                            if(line.find(fstypePrefixStr) >= 0):
                                fstypeEnd = line.find(" ", fstypeStart+1)
                                if(fstypeEnd >=0):
                                    fs_type = line[fstypeStart+1:fstypeEnd]
                            # If there is a duplicate, keep only the first instance
                            if (mount_point not in mount_points):
                                self.logger.log("mount command, adding mount :" + str(mount_point) + ":  device :" + str(deviceName) + ": fstype :"+ str(fs_type) + ":", True)
                                fs_types.append(fs_type)
                                mount_points.append(mount_point)
                                mount_points_info.append((mount_point,deviceName,fs_type))
                            else:
                                self.logger.log("####### mount command, not adding duplicate mount :" + str(mount_point) + ":  device :" + str(deviceName) + ": fstype :"+ str(fs_type) + ":", True)
        #Now reverse the mount_points & fs_types lists to make them in the same order as mount command output order
        mount_points_info.reverse()
        mount_points.reverse()
        for fstype in fs_types:
            if ("fuse" in fstype.lower() or "nfs" in fstype.lower() or "cifs" in fstype.lower()):
                Utils.HandlerUtil.HandlerUtility.add_to_telemetery_data("networkFSTypePresentInMount","True")
                break
        return mount_points,mount_points_info

    def get_mount_file_systems(self):
        out_mount_output = self.get_mount_output()
        file_systems_info = []
        mount_points = []
        if (out_mount_output is not None):
            lines = out_mount_output.splitlines()
            #Reverse the mount command output to go-through from last-to-first mounts in output
            lines.reverse()
            for line in lines:
                self.logger.log("print line by line :" + line , True)
                line = line.strip()
                if(line != ""):
                    file_system =Utils.HandlerUtil.HandlerUtility.split(self.logger, line)[0]
                    mountPrefixStr = " on /"
                    prefixIndex = line.find(mountPrefixStr)
                    if(prefixIndex >= 0):
                        mountpointStart = prefixIndex + len(mountPrefixStr) - 1
                        fstypePrefixStr = " type "
                        mountpointEnd = line.find(fstypePrefixStr, mountpointStart)
                        if(mountpointEnd >= 0):
                            mount_point = line[mountpointStart:mountpointEnd]
                            fs_type = ""
                            fstypeStart = line.find(fstypePrefixStr) + len(fstypePrefixStr) - 1
                            if(line.find(fstypePrefixStr) >= 0):
                                fstypeEnd = line.find(" ", fstypeStart+1)
                                if(fstypeEnd >=0):
                                    fs_type = line[fstypeStart+1:fstypeEnd]
                    # If there is a duplicate, keep only the first instance
                    if (mount_point not in mount_points):
                        file_systems_info.append((file_system,fs_type,mount_point))
                        mount_points.append(mount_point)
        #Now reverse the file_systems_info list to make them in the same order as mount command output order
        file_systems_info.reverse()
        return file_systems_info

    def get_mount_output(self):
        if self.mount_output is not None:
            return self.mount_output
        else :
            # Get the output on the mount command
            self.logger.log("getting the mount-points info using mount command ", True)
            mount_path = self.patching.mount_path
            is_mount_path_wrong, out_mount_output, error_msg = self.get_mount_command_output(mount_path)
            if (is_mount_path_wrong == True):
                if self.patching.usr_flag == 1:
                    self.logger.log("mount path is wrong.removing /usr prefix", True, 'Warning')
                    mount_path = "/bin/mount"
                else:
                    self.logger.log("mount path is wrong.Adding /usr prefix", True, 'Warning')
                    mount_path = "/usr/bin/mount"
                is_mount_path_wrong, out_mount_output, error_msg = self.get_mount_command_output(mount_path)
            # if mount_path was still wrong, mount_path using "which" command
            if (is_mount_path_wrong == True):
                self.logger.log("mount path is wrong. finding path using which command", True, 'Warning')
                out_which_output, which_error_msg = self.get_which_command_result('mount')
                # get mount command output
                if (out_which_output is not None):
                     mount_path = str(out_which_output)
                     is_mount_path_wrong, out_mount_output, error_msg = self.get_mount_command_output(mount_path)
            self.mount_output = out_mount_output
            return out_mount_output


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import json
import transformers
import textwrap
from transformers import LlamaTokenizer, LlamaForCausalLM
import os
import sys
from typing import List

from peft import (
    LoraConfig,
    get_peft_model,
    get_peft_model_state_dict,
    prepare_model_for_int8_training,
)

import fire
import torch
from datasets import load_dataset
import pandas as pd

import matplotlib.pyplot as plt
import matplotlib as mpl

import seaborn as sns
from pylab import rcParams

device = "cuda" if torch.cuda.is_available() else "cpu"

CUTOFF_LEN = 50

df = pd.read_csv("./data/BTC_Tweets_Updated.csv")

# df.New_Sentiment_State.value_counts().plot(kind="bar")

def sentiment_score_to_name(score: float):
    if score > 0:
        return "Positive"
    elif score <0:
        return "Negative"
    return "Neutral"

dataset_data = [
    {
        "instruction": "Detect the sentiment of the tweet.",
        "input": row_dict["Tweet"],
        "output": sentiment_score_to_name(row_dict["New_Sentiment_State"])
    }
    for row_dict in df.to_dict(orient="records")
]

with open("./data/alpaca-bitcoin-sentiment-dataset.json", "w") as f:
    json.dump(dataset_data, f)

data = load_dataset("json", data_files="./data/alpaca-bitcoin-sentiment-dataset.json")
print(data["train"])

BASE_MODEL = "../llama-7b-hf"
model = LlamaForCausalLM.from_pretrained(
    BASE_MODEL,
    torch_dtype=torch.float16,
    device_map="auto",
    offload_folder="./data/llama"
)

tokenizer = LlamaTokenizer.from_pretrained(BASE_MODEL)
tokenizer.pad_token_id = (0)
tokenizer.padding_side = "left"

def generate_prompt(data_point):
    return f"""Below is an instruction that describes a task, paired with an input that provide future context.
      Write a responce that appropriately completes the request. #noqa:
      ### Instruct:
      {data_point["instruction"]}
      ### Input
      {data_point["input"]}
      ### Response
      {data_point["output"]}
    """

def tokenize(prompt, add_eos_token=True):
    result = tokenizer(
        prompt,
        truncation=True,
        max_length=CUTOFF_LEN,
        padding=False,
        return_tensors=None,
    )
    if (result["input_ids"][-1] != tokenizer.eos_token_id and len(result["input_ids"]) < CUTOFF_LEN and add_eos_token):
        result["input_ids"].append(tokenizer.eos_token_id)
        result["attention_mask"].append(1)

    result["labels"] = result["input_ids"].copy()

    return result

def generate_and_tokenize_prompt(data_point):
    full_prompt = generate_prompt(data_point)
    tokenized_full_prompt = tokenize(full_prompt)
    return tokenized_full_prompt

train_val = data["train"].train_test_split(
    test_size=200, shuffle=True, seed=42
)

train_data = (
    train_val["train"].map(generate_and_tokenize_prompt)
)
val_data = (
    train_val["test"].map(generate_and_tokenize_prompt)
)

# Training
LORA_R = 8
LORA_ALPHA = 16
LORA_DROPOUT = 0.05
LORA_TARGET_MODULES = [
    "q_proj",
    "v_proj",
]

BATCH_SIZE = 128
MICRO_BATCH_SIZE = 4
GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE
LEARNING_RATE = 3e-4
TRAIN_STEPS = 300
OUTPUT_DIR = "experiments"

# We can now prepare model for training
model = prepare_model_for_int8_training(model)
config = LoraConfig(
    r = LORA_R,
    lora_alpha=LORA_ALPHA,
    target_modules=LORA_TARGET_MODULES,
    lora_dropout=LORA_DROPOUT,
    bias="none",
    task_type="CAUSAL_LM",
)

model = get_peft_model(model, config)
model.print_trainable_parameters()

training_arguments = transformers.TrainingArguments(
    per_device_train_batch_size=MICRO_BATCH_SIZE,
    gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS,
    warmup_steps=100,
    max_steps=TRAIN_STEPS,
    no_cuda=True,
    learning_rate=LEARNING_RATE,
    logging_steps=10,
    optim="adamw_torch",
    evaluation_strategy="steps",
    save_strategy="steps",
    eval_steps=50,
    save_steps=50,
    output_dir=OUTPUT_DIR,
    save_total_limit=3,
    load_best_model_at_end=True,
    report_to="tensorboard"
)

data_collector = transformers.DataCollatorForSeq2Seq(
    tokenizer, pad_to_multiple_of=8, return_tensors="pt", padding=True
)

trainer = transformers.Trainer(
    model=model,
    train_dataset=train_data,
    eval_dataset=val_data,
    args=training_arguments,
    data_collator=data_collector
)

model.config.use_cache = False
old_state_dict = model.state_dict
model.state_dict = (
    lambda self, *_, **__: get_peft_model_state_dict(
        self, old_state_dict()
    )
).__get__(model, type(model))

trainer.train()
model.save_pretrained(OUTPUT_DIR)


# -*- coding: utf-8 -*-
import cPickle as pickle

from stanza.research.instance import Instance

from utils import urepr  # NOQA: for doctest

class HanjaTranslator(object):
    '''
    A wrapper class for (by default) the logistic regression model in classifier.py.
    '''
    def __init__(self, picklefile=None):
        '''
        :param file picklefile: An open file-like object from which to
            load the model. Can be produced either from a normal experiment
            run or a quickpickle.py run. If `None`, try to load the default
            quickpickle file (this is less future-proof than the normal
            experiment-produced pickle files).
        '''
        if picklefile is None:
            with open('models/lr_prevnext/model.pkl', 'rb') as infile:
                self.model = pickle.load(infile)
        else:
            self.model = pickle.load(picklefile)
        self.model.options.verbosity = 0

    def translate(self, hangul_sent):
        u'''
        Infer the most likely hanja for each syllable in `hangul_sent` (where
        "leave the hangul unchanged" is usually one option), and return a string
        with the predicted hanja substituted for the hangul.

        >>> tr = HanjaTranslator()
        >>> print urepr(tr.translate(u'2011년 12월 26일'))
        u'2011年 12月 26日'
        '''
        return self.translate_all([hangul_sent])[0]

    def translate_all(self, hangul_sents):
        u'''
        Infer the most likely hanja for each syllable of each sentence in
        `hangul` (where "leave the hangul unchanged" is usually one options),
        and return a list of strings with the predicted hanja substituted
        for the hangul.

        >>> tr = HanjaTranslator()
        >>> print urepr(tr.translate_all([u'2011년 12월 26일', u'중화인민공화국']))
        [u'2011年 12月 26日', u'中華人民共和國']
        '''
        insts = [Instance(g) for g in hangul_sents]
        return self.model.predict(insts, verbosity=0)


import os
import joblib
import time
import argparse
from PIL import Image
from config import Config
from torchvision import datasets, transforms
from attack.mask_attack import shaped_mask_attack
from yolov3.detect import load_model, detect
# from yolov7.detect_attack import load_model, detect
# from yolov5.detect_attack import load_model, detect

# import os
# os.environ['CUDA_LAUNCH_BLOCKING'] = '1'

loader = transforms.Compose([
    transforms.ToTensor()
])
conf_thre = 0.5

def attack_process(H, W, img, threat_model, device, emp_iterations, max_pertubation_mask, content, folder_path, name, grad_avg):
    input = loader(img)
    bbox, prob = detect(threat_model, input) # 在攻击前检测原目标的置信度
    begin = time.time()
    adv_img_ts, adv_img, mask = shaped_mask_attack(H, W, bbox, threat_model, img, device, emp_iterations, max_pertubation_mask, content, grad_avg) # 调用攻击函数进行攻击
    _, prob = detect(threat_model,adv_img_ts) # 在攻击后检测目标的置信度
    end = time.time()
    print("optimization time: {}".format(end - begin))
    print("obj score after attack: ",prob.item())
    if prob.item() < conf_thre:
        imgs_dir = os.path.join(folder_path, "adv_imgs")
        msks_dir = os.path.join(folder_path, "infrared_masks")
        img_path = os.path.join(imgs_dir, name)
        adv_img.save(img_path,quality=99)
        msk_path = os.path.join(msks_dir, name)
        mask.save(msk_path,quality=99)
        return True
        # joblib.dump(adv_img_ts,folder_path+"/res/adv_ts" + str(args.number) + "+" + str(args.max_pertubation_mask) + "/{}_pedestrian&params.pkl".format(name))
        # joblib.dump(mask,folder_path+"/res/mask" + str(args.number) + "+" + str(args.max_pertubation_mask) + "/{}_mask&params.pkl".format(name))
    else: return False

if __name__=="__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument('--victim_imgs',type=str,default="", help='the folder with victim images to conduct attacks')
    args = parser.parse_args()
    ## 加载攻击参数 ##
    opt = Config()
    if args.victim_imgs == "":
        folder_path = opt.attack_dataset_folder
        if not os.path.join(folder_path):
            print("please prepare the dataset correctly")
            assert False
    else: folder_path = args.victim_imgs

    ## 加载待攻击模型 ##
    threat_model = load_model()
    threat_model.eval()

    suc = 0
    sum = 0
    for i,name in enumerate(os.listdir(folder_path)):
        file_path = os.path.join(folder_path, name)
        img = Image.open(file_path)
        input = loader(img)
        bbox, prob = detect(threat_model, input) # 在攻击前检测原目标的置信度
        print("{}th image".format(i))
        if prob.item()<0.5: # 本身检测分数较低的跳过
            print("detector cannot detect any pedestrian in the image")
            continue
        else:
            print("obj score before attack: ", prob.item())
        sum += 1
        for k in range(opt.iterations):
            print("{}th attack".format(k))
            flag = attack_process(opt.height, opt.width, img, threat_model, opt.device, opt.emp_iterations, opt.max_pertubation_mask, opt.content, opt.save_folder, name, opt.grad_avg)
            if flag:
                suc += 1
                break
    print("ASR: ", float(suc)/sum)


#! /usr/bin/env python3

import argparse
import csv
import glob
import json
import os

def parseArguments(args):
    parser = argparse.ArgumentParser(description="Gathers stats after a run")
    parser.add_argument(
        "-o",
        "--outdir",
        default="cases",
        help="Directory to generate the test suite in.",
    )
    parser.add_argument(
        "-f",
        "--file",
        type=argparse.FileType("w"),
        default="stats.csv",
        help="The resulting CSV file containing all stats.",
    )
    return parser.parse_args(args)

def statsFromTimings(dir):
    stats = {}
    assert os.path.isdir(dir)
    assert (
        os.system("command -v precice-profiling > /dev/null") == 0
    ), 'Could not find the profiling tool "precice-profiling", which is part of the preCICE installation.'
    event_dir = os.path.join(dir, "precice-profiling")
    json_file = os.path.join(dir, "profiling.json")
    timings_file = os.path.join(dir, "timings.csv")
    os.system("precice-profiling merge --output {} {}".format(json_file, event_dir))
    os.system(
        "precice-profiling analyze --output {} B {}".format(timings_file, json_file)
    )
    file = timings_file
    if os.path.isfile(file):
        try:
            timings = {}
            with open(file, "r") as csvfile:
                timings = csv.reader(csvfile)
                for row in timings:
                    if row[0] == "_GLOBAL":
                        stats["globalTime"] = row[-1]
                    if row[0] == "initialize":
                        stats["initializeTime"] = row[-1]
                    if row[0].startswith("initialize/map") and row[0].endswith(
                        "computeMapping.FromA-MeshToB-Mesh"
                    ):
                        computeMappingName = row[0]
                        stats["computeMappingTime"] = row[-1]
                    if row[0].startswith("advance/map") and row[0].endswith(
                        "mapData.FromA-MeshToB-Mesh"
                    ):
                        mapDataName = row[0]
                        stats["mapDataTime"] = row[-1]
        except BaseException:
            pass
    return stats

def memoryStats(dir):
    stats = {}
    assert os.path.isdir(dir)
    for P in "A", "B":
        memfile = os.path.join(dir, f"memory-{P}.log")
        total = 0
        if os.path.isfile(memfile):
            try:
                with open(memfile, "r") as file:
                    total = sum([float(e) / 1024.0 for e in file.readlines()])
            except BaseException:
                pass
        stats[f"peakMem{P}"] = total

    return stats

def main(argv):
    args = parseArguments(argv[1:])

    globber = os.path.join(args.outdir, "**", "*.stats.json")
    statFiles = [
        os.path.relpath(path, args.outdir)
        for path in glob.iglob(globber, recursive=True)
    ]
    allstats = []
    fields = []
    for file in statFiles:
        print("Found: " + file)
        casedir = os.path.join(args.outdir, os.path.dirname(file))
        parts = os.path.normpath(file).split(os.sep)
        assert len(parts) >= 5
        mapping, constraint, meshes, ranks, _ = parts[-5:]
        meshA, meshB = meshes.split("-")
        ranksA, ranksB = ranks.split("-")

        with open(os.path.join(args.outdir, file), "r") as jsonfile:
            stats = json.load(jsonfile)
            stats["mapping"] = mapping
            stats["constraint"] = constraint
            stats["mesh A"] = meshA
            stats["mesh B"] = meshB
            stats["ranks A"] = ranksA
            stats["ranks B"] = ranksB
            stats.update(statsFromTimings(casedir))
            stats.update(memoryStats(casedir))
            allstats.append(stats)
            if not fields:
                fields += stats.keys()

    assert fields
    writer = csv.DictWriter(args.file, fieldnames=fields)
    writer.writeheader()
    writer.writerows(allstats)
    return 0

if __name__ == "__main__":
    import sys

    sys.exit(main(sys.argv))


import os
import openai
import sys
import dotenv

dotenv_file = dotenv.find_dotenv()
dotenv.load_dotenv(dotenv_file)

job_arr = sys.argv[1]

job_info_arr = [{"role": "system", "content": "너는 일자리를 알려주는 봇이야"},
                {"role": "user","content":"예시질문 : 직종을 분류해서 5개의 카테고리로 (운송, 서비스, 제조, 복지, 기타) 나눠보여줘."     },
      {"role": "assistant","content":"예시대답 : 네, 알겠습니다. ### 운송 - [운전원](url) ### 서비스 - [조리사](url) ### 제조 - [직종](url) ### 복지 - [직종](url) ### 기타 - [직종](url)"     },
      {"role":"system","content":"한가지 직종에 해당하는 내용을 출력시에는 다음과 같은 예시로 대답. 예시대답 : ### 카테고리 - 직종 : (직종)  - 위치 : (위치) - 급여 : (급여) - 인원 : (인원) - 지원방법 - (지원방법) --------(가로줄넣기) "},
      {"role": "system", "content": "일자리에 대한 모든 데이터 및 정보는 아래와 같다. 제공된 일자리 데이터만 이용해 대답한다."},
      ]

job_info_arr.append({"role":"system","content":job_arr})

def chatGPT(prompt):
  openai.api_key = os.environ["OPENAI_API_KEY"]
  try:
        job_info_arr.append({"role":"user","content":prompt})

        completion = openai.ChatCompletion.create(
            model="gpt-3.5-turbo",
            messages=job_info_arr,
        )

        print(completion.choices[0].message.content)
  except Exception as e:
        print("An error occurred:", e)

chatGPT(sys.argv[2])
#chatGPT("")


import appdirs
import os
import hashlib
import configparser
from pathlib import Path

def get_configuration():
    config_dir = appdirs.user_config_dir("LibertyAI")
    filename = os.path.join(config_dir, 'config.ini')
    # Load from file
    config = configparser.ConfigParser()

    if os.path.exists(filename):
        config.read(filename)
    else:
        # General stuff for getting it to start up
        config.set('DEFAULT', 'APIServicePort', value='5001')
        config.set('DEFAULT', 'LLAMA_CPP_MODEL', '')
        # API
        config.add_section('API')
        config.set('API', 'KEY', value=hashlib.sha256(os.urandom(32)).hexdigest())
        config.set('API', 'GENERATION_ENDPOINT', "https://libergpt.univ.social/api/generation")
        config.set('API', 'EMBEDDING_ENDPOINT', "https://libergpt.univ.social/api/embedding")
        # Data base settings
        config.add_section('DATABASE')
        config.set('DATABASE', 'PGSQL_SERVER', value='localhost')
        config.set('DATABASE', 'PGSQL_SERVER_PORT', value='5432')
        config.set('DATABASE', 'PGSQL_DATABASE', value='libertyai')
        config.set('DATABASE', 'PGSQL_USER', value='libertyai')
        config.set('DATABASE', 'PGSQL_PASSWORD', value='libertyai')

        Path(os.path.dirname(os.path.abspath(filename))).mkdir( parents=True, exist_ok=True )
        with open(filename, 'w') as cf:
            config.write(cf)
            cf.close()

    return config


#!/usr/bin/env python3
# vim: fenc=utf-8:ts=4:sw=4:sta:et:sts=4:ai
import os
import datetime
import glob
import logging
import nbtlib
import bedrock.leveldb as leveldb
from PIL import ImageFont, Image, ImageDraw
import math
import operator
from collections.abc import Callable
from collections import defaultdict, OrderedDict, namedtuple
from tqdm import tqdm
import argparse
import gzip
import json
import distutils.dir_util
import re
from io import BytesIO
import sys
import hashlib
import time
import struct

__author__ = "Jason Green"
__copyright__ = "Copyright 2020, Tesseract Designs"
__credits__ = ["Jason Green"]
__license__ = "MIT"
__version__ = "2.0.6"
__maintainer__ = "Jason Green"
__email__ = "jason@green.io"
__status__ = "release"

dir_path = os.path.dirname(os.path.realpath(__file__))

filenameSeparator = "."

mapPngFilenameFormat = filenameSeparator.join(["{mapId}", "{mapHash}", "{epoch}", "{dimension}", "{x}", "{z}", "{scale}.png"])

# now in epoch
now = int(time.time())

# stuff to convert the map color data to RGB values
multipliers = [180, 220, 255, 135]

# this is base pallette Minecraft uses, used with the multipliers
basecolors = [(0, 0, 0, 0),
              (127, 178, 56),
              (247, 233, 163),
              (199, 199, 199),
              (255, 0, 0),
              (160, 160, 255),
              (167, 167, 167),
              (0, 124, 0),
              (255, 255, 255),
              (164, 168, 184),
              (151, 109, 77),
              (112, 112, 112),
              (64, 64, 255),
              (143, 119, 72),
              (255, 252, 245),
              (216, 127, 51),
              (178, 76, 216),
              (102, 153, 216),
              (229, 229, 51),
              (127, 204, 25),
              (242, 127, 165),
              (76, 76, 76),
              (153, 153, 153),
              (76, 127, 153),
              (127, 63, 178),
              (51, 76, 178),
              (102, 76, 51),
              (102, 127, 51),
              (153, 51, 51),
              (25, 25, 25),
              (250, 238, 77),
              (92, 219, 213),
              (74, 128, 255),
              (0, 217, 58),
              (129, 86, 49),
              (112, 2, 0),
              (209, 177, 161),
              (159, 82, 36),
              (149, 87, 108),
              (112, 108, 138),
              (186, 133, 36),
              (103, 117, 53),
              (160, 77, 78),
              (57, 41, 35),
              (135, 107, 98),
              (87, 92, 92),
              (122, 73, 88),
              (76, 62, 92),
              (76, 50, 35),
              (76, 82, 42),
              (142, 60, 46),
              (37, 22, 16),
              (189, 48, 49),
              (148, 63, 97),
              (92, 25, 29),
              (22, 126, 134),
              (58, 142, 140),
              (86, 44, 62),
              (20, 180, 133),
              (100, 100, 100),
              (216, 175, 147),
              (127, 167, 150)]

def multiplyColor(colorTuple, multiplier):
    "calculates final color values using multiplier"
    return tuple([math.floor(a * multiplier / 255.0) for a in colorTuple])

# this a list of all possible colors a pixel can be on a map
allColors = [multiplyColor(color, multiplier)
             for color in basecolors for multiplier in multipliers]

# convert dimension names to/from human readable
dimDict = {-1: "minecraft:the_nether",
           0: "minecraft:overworld",
           1: "minecraft:the_end",
           'minecraft:overworld': 0,
           'minecraft:the_end': 1,
           'minecraft:the_nether': -1,
           'minecraft@overworld': 0,
           'minecraft@the_end': 1,
           'minecraft@the_nether': -1}

def findMapFiles(inputFolder):
    mapFiles = []

    folderTree = list(os.walk(inputFolder))

    dataFolders = [f for f in folderTree if f[0].endswith(os.sep + "data")]

    for folder in dataFolders:
        maybeMapFiles = [os.path.join(folder[0], f) for f in folder[2] if f.startswith("map_") and f.endswith(".dat")]
        if "idcounts.dat" in folder[2]:
            logging.info("Found %s maps in %s", len(maybeMapFiles), folder[0])
            mapFiles = maybeMapFiles

    if not mapFiles:
        logging.info("Didn't find any maps, did you specify the correct world location?")
        sys.exit(1)

    return mapFiles

class LastUpdatedOrderedDict(OrderedDict):
    'Store items in the order the keys were last added'
    def __setitem__(self, key, value):
        if key in self:
            del self[key]
        OrderedDict.__setitem__(self, key, value)

class DefaultOrderedDict(OrderedDict):
    "Source: http://stackoverflow.com/a/6190500/562769"
    def __init__(self, default_factory=None, *a, **kw):
        if (default_factory is not None and
           not isinstance(default_factory, Callable)):
            raise TypeError('first argument must be callable')
        OrderedDict.__init__(self, *a, **kw)
        self.default_factory = default_factory

    def __getitem__(self, key):
        try:
            return OrderedDict.__getitem__(self, key)
        except KeyError:
            return self.__missing__(key)

    def __missing__(self, key):
        if self.default_factory is None:
            raise KeyError(key)
        self[key] = value = self.default_factory()
        return value

    def __reduce__(self):
        if self.default_factory is None:
            args = tuple()
        else:
            args = self.default_factory,
        return type(self), args, None, None, self.items()

    def copy(self):
        return self.__copy__()

    def __copy__(self):
        return type(self)(self.default_factory, self)

    def __deepcopy__(self, memo):
        import copy
        return type(self)(self.default_factory,
                          copy.deepcopy(self.items()))

    def __repr__(self):
        return 'OrderedDefaultDict(%s, %s)' % (self.default_factory,
                                               OrderedDict.__repr__(self))

# couple of structures to keep stuff
BannerTuple = namedtuple("BannerTuple", ["X", "Y", "Z", "name", "color", "dimension"])
MapTuple = namedtuple("MapTuple", ["mapData", "bannerData", "frameData"])
MapPngTuple = namedtuple("MapPngTuple", ["mapId", "mapHash", "epoch", "x", "z", "dimension", "scale"])

def mapPngsSortedByEpoch(mapPngs):
    """Returns a list of latest map png files by their center"""

    # get all generated maps

    centerEpochs = []

    for mapPng in mapPngs:
        centerEpochs.append((mapPng.epoch, mapPng))

    # sort the whole thing by epoch
    centerEpochs.sort(key=operator.itemgetter(0))

    # for centerEpoch in centerEpochs:
    #     # this will only keep the latest map ids around for rendering
    #     [centerEpoch[0]] = centerEpoch[2]

    # latestMapPngs = list(filterDict.values())

    return [m[1] for m in centerEpochs]

def filterLatestMapPngsById(mapPngs):
    """Returns a list of latest map png files by Id"""

    # get all generated maps

    idEpochs = []
    filterDict = {}

    for mapPng in mapPngs:
        idEpochs.append((mapPng.mapId, mapPng.epoch, mapPng))

    # sort the whole thing by epoch
    idEpochs.sort(key=operator.itemgetter(1))

    for idEpoch in idEpochs:
        # this will only keep the latest map ids around for rendering
        filterDict[idEpoch[0]] = idEpoch[2]

    latestMapPngs = list(filterDict.values())

    return latestMapPngs

def makeMaps(worldFolder, outputFolder, serverType, unlimitedTracking=False):
    nbtMapData = []
    if serverType == "bds":
        # open leveldb
        db = leveldb.open(os.path.join(worldFolder, "db"))

        # iterate over all the maps
        for a in tqdm(leveldb.iterate(db), "leveldb map keys -> nbt".ljust(24), bar_format="{l_bar}{bar}"):
            key = bytearray(a[0])
            if b"map" in key:
                # get extract an nbt object
                mapNbtIo = BytesIO(a[1])
                mapNbtFile = nbtlib.File.parse(mapNbtIo, byteorder="little")
                mapNbt = mapNbtFile
                mapId = int(mapNbt["mapId"])
                epoch = 0
                nbtMapData.append({"epoch": epoch, "id": mapId, "nbt": mapNbt})

    elif serverType == "java":
        mapDatFiles = findMapFiles(worldFolder)
        for mapDatFile in tqdm(mapDatFiles, "map_*.dat -> nbt".ljust(24), bar_format="{l_bar}{bar}"):
            mapNbtFile = nbtlib.load(mapDatFile)
            mapNbt = mapNbtFile["data"]
            mapId = int(os.path.basename(mapDatFile)[4:-4])
            epoch = int(os.path.getmtime(mapDatFile))
            nbtMapData.append({"epoch": epoch, "id": mapId, "nbt": mapNbt})

    maps = []
    os.makedirs(outputFolder, exist_ok=True)
    mapPngs = getMapPngs(outputFolder)

    currentIds = {x.mapId: x for x in mapPngs}

    for nbtMap in tqdm(nbtMapData, "nbt -> png".ljust(24), bar_format="{l_bar}{bar}"):
        mapId = nbtMap["id"]
        mapNbt = nbtMap["nbt"]
        mapEpoch = nbtMap["epoch"]
        try:
            mapUnlimitedTracking = mapNbt["unlimitedTracking"]
        except KeyError:
            mapUnlimitedTracking = False

        if mapUnlimitedTracking and not unlimitedTracking:
            continue
        scale = int(mapNbt["scale"])
        x = int(mapNbt["xCenter"])
        z = int(mapNbt["zCenter"])

        dimension = mapNbt["dimension"]
        mapColors = mapNbt["colors"]

        if type(dimension) == nbtlib.tag.Int:
            dimension = dimDict[mapNbt["dimension"]]
        elif type(dimension) == nbtlib.tag.Byte:
            dimension = dimDict[mapNbt["dimension"]]
        else:
            dimension = dimension.strip('"')
        dimension = dimension.replace(":", "@")

        try:
            mapBanners = mapNbt["banners"]
        except KeyError:
            mapBanners = []

        try:
            mapFrames = mapNbt["frames"]
        except KeyError:
            mapFrames = []

        banners = set()
        for banner in mapBanners:
            X = int(banner["Pos"]["X"])
            Y = int(banner["Pos"]["Y"])
            Z = int(banner["Pos"]["Z"])
            color = banner["Color"]

            try:
                name = json.loads(banner["Name"])["text"]

            except KeyError:
                name = ""

            bannerDict = {"X": X,
                          "Y": Y,
                          "Z": Z,
                          "color": color,
                          "name": name,
                          "dimension": dimension}
            bannerTuple = BannerTuple(**bannerDict)
            banners.add(bannerTuple)
        frames = []
        for frame in mapFrames:
            X = int(frame["Pos"]["X"])
            Y = int(frame["Pos"]["Y"])
            Z = int(frame["Pos"]["Z"])
            rotation = int(frame["Rotation"])

            frameDict = {"X": X,
                        "Y": Y,
                        "Z": Z,
                        "rotation": rotation}
            frames.append(frameDict)
        # logging.debug(mapColors)

        if serverType == "bds":
            mapImage = Image.frombytes("RGBA", (128, 128),
                                       bytes([x % 256 for x in mapColors]),
                                       'raw')
        elif serverType == "java":
            colorTuples = [allColors[x % 256] for x in mapColors]
            mapImage = Image.new("RGBA", (128, 128))
            mapImage.putdata(colorTuples)

        mapHash = hashlib.md5(mapImage.tobytes()).hexdigest()

        # empty map
        if mapHash == "fcd6bcb56c1689fcef28b57c22475bad":
            continue

        if mapId not in currentIds:
            # brand new image
            logging.debug("%s is a new map", mapId)
            epoch = mapEpoch
        else:
            # changed image
            logging.debug("%s is already known", mapId)
            if mapHash != currentIds.get(mapId).mapHash:
                # map has changed based on the hash

                logging.debug("%s changed and will get an updated epoch", mapId)
                epoch = now if not mapEpoch else mapEpoch

            elif mapEpoch > currentIds.get(mapId).epoch:
                logging.debug("%s has a more recent epoch from it's dat file, updating", mapId)
                epoch = mapEpoch

            else:
                logging.debug("%s has not changed and will keep it's epoch", mapId)
                epoch = currentIds.get(mapId).epoch

        mapPng = MapPngTuple(mapId=mapId,
                             mapHash=mapHash,
                             epoch=epoch,
                             dimension=dimension,
                             x=x,
                             z=z,
                             scale=scale)

        mapImage = mapImage.resize((128 * 2 ** scale,) * 2, Image.NEAREST)
        filename = mapPngFilenameFormat.format(**mapPng._asdict())

        try:
            oldFilename = mapPngFilenameFormat.format(**currentIds.get(mapId)._asdict())
            os.remove(os.path.join(outputFolder, oldFilename))
        except:
            logging.debug("%s isn't there, didn't delete", mapId)

        mapImage.save(os.path.join(outputFolder, filename))

        mapData = MapTuple(mapData=mapPng,
                           bannerData=banners,
                           frameData=frames)
        maps.append(mapData)

    logging.debug(maps)
    logging.info("Processed %s maps", len(maps))

    return maps

def getMapPngs(mapPngFolder):
    mapPngList = []

    globString = filenameSeparator.join(6 * ["*"] + ["*.png"])
    # get all the maps
    mapPngs = glob.glob(os.path.join(mapPngFolder, globString))

    # iterate over all the maps
    for mapPng in mapPngs:
        filename = os.path.basename(mapPng)
        (mapId,
         mapHash,
         epoch,
         dimension,
         x,
         z,
         scale,
         _) = filename.split(filenameSeparator)
        # change some types
        x = int(x)
        z = int(z)
        scale = int(scale)
        mapId = int(mapId)
        if not dimension in dimDict:
            logging.info("Skipped map %s with invalid dimension.", mapId)
            continue
        epoch = int(epoch)

        mapPngList.append(MapPngTuple(mapId=mapId,
                                      mapHash=mapHash,
                                      epoch=epoch,
                                      dimension=dimension,
                                      x=x,
                                      z=z,
                                      scale=scale))

    return mapPngList

def mergeToLevel4(mapPngFolder, outputFolder, disablezoomsort):
    """pastes all maps to render onto a intermediate zoom level 4 map"""
    # what are we calling these crazy things

    filenameFormat = filenameSeparator.join(["{dimension}", "{x}", "{z}.png"])

    # make sure the output exsists
    os.makedirs(outputFolder, exist_ok=True)

    # this will hold all the maps, by dimension and associated zoom level 4 map
    level4Dict = defaultdict(lambda: defaultdict(list))

    # get all the maps
    mapPngs = getMapPngs(mapPngFolder)
    latestMapPngs = mapPngsSortedByEpoch(mapPngs)

    # iterate over all the maps
    for mapPng in latestMapPngs:
        # convert the center of the map to the top left corner
        mapTopLeft = (mapPng.x - 128 * 2 ** mapPng.scale // 2 + 64,
                      mapPng.z - 128 * 2 ** mapPng.scale // 2 + 64)

        # figure out which level 4 map it belongs to
        level4Coords = (mapTopLeft[0] // 2048 * 2048,
                        mapTopLeft[1] // 2048 * 2048)

        # throw it into a "dict"
        level4Dict[mapPng.dimension][level4Coords].append(mapPng)

    logging.debug(level4Dict)

    # iterate over the level 4 buckets
    for dim in level4Dict.items():
        d = dim[0]
        for coords in tqdm(dim[1].items(), "level 4 of dim: {}".format(d).ljust(24), bar_format="{l_bar}{bar}"):
            c = coords[0]

            mapTuples = coords[1]

            if not disablezoomsort:
            # sort them, import for the rendering order
                mapTuples.sort(key=lambda x: x.scale, reverse=True)

            # create the level 4 images
            level4MapPng = Image.new("RGBA", (2048, 2048))

            # iterate over the maps in each bucket
            for mapTuple in mapTuples:
                # get the map details
                mapPngCoords = (divmod(mapTuple.x - 128 * 2 ** mapTuple.scale // 2 + 64, 2048)[1],
                                divmod(mapTuple.z - 128 * 2 ** mapTuple.scale // 2 + 64, 2048)[1])
                mapPngFilename = mapPngFilenameFormat.format(**mapTuple._asdict())
                # paste the image into the level 4 map
                with Image.open(os.path.join(mapPngFolder, mapPngFilename)) as mapPng:
                    level4MapPng.paste(mapPng, mapPngCoords, mapPng)
            # figure out the name of the file, and save it
            fileName = filenameFormat.format(dimension=d, x=c[0], z=c[1]*-1)

            filePath = os.path.join(outputFolder, fileName)
            level4MapPng.save(filePath)
            level4MapPng.close()

def genZoom17Tiles(level4MapFolder, outputFolder):
    """generates lowest zoom level tiles from combined zoom level 4 maps"""

    # get all the level 4 maps
    globString = filenameSeparator.join(["*", "*", "*.png"])
    level4MapFilenames = glob.glob(os.path.join(level4MapFolder, globString))
    # iterate over level4 maps
    for level4MapFilename in tqdm(level4MapFilenames, "level 4 -> zoom 17 tiles", bar_format="{l_bar}{bar}"):
        # get some details
        name = os.path.basename(level4MapFilename)
        dim, x, z, _ = name.split(filenameSeparator)
        level4x = int(x)
        level4z = int(z)
        tilex = level4x // 2048
        tilez = level4z // 2048 * -1
        # open the level 4 map
        with Image.open(level4MapFilename) as level4MapPng:
            # this is pretty useless
            for zoom in range(17, 18):
                numTiles = 2 ** (zoom - 13)
                imageWidth = 2048 // numTiles
                # do math
                for numx in range(numTiles):
                    levelNumx = tilex * numTiles + numx
                    foldername = os.path.join(outputFolder, dim, str(zoom), str(levelNumx))
                    os.makedirs(foldername, exist_ok=True)
                    for numz in range(numTiles):
                        levelNumz = tilez * numTiles + numz
                        cropBox = (numx * imageWidth,
                                numz * imageWidth,
                                numx * imageWidth + imageWidth,
                                numz * imageWidth + imageWidth)
                        filename = os.path.join(foldername, str(levelNumz) + ".png")
                        tilePng = level4MapPng.crop(cropBox)
                        tilePng = tilePng.resize((256, 256), Image.NEAREST)
                        tilePng.save(filename)

def extrapolateZoom(tileFolder, level):
    zoom17Filenames = glob.glob(os.path.join(tileFolder, "*", str(level + 1), "*", "*.png"))
    newZoomDict = defaultdict(list)
    for filename in zoom17Filenames:
        tilePath = os.path.relpath(filename, tileFolder)
        dim, zoom, x, y = tilePath.strip(".png").split(os.sep)
        x = int(x)
        y = int(y)
        xnew, xq = divmod(x, 2)
        ynew, yq = divmod(y, 2)
        newZoomDict[(dim, xnew, ynew)].append((xq, yq, filename))
    for newTile in tqdm(newZoomDict.items(), "zoom {} tiles".format(level).ljust(24), bar_format="{l_bar}{bar}"):
        foldername = os.path.join(tileFolder, str(newTile[0][0]), str(level), str(newTile[0][1]))
        tilePng = Image.new("RGBA", (512,512))
        for previousTile in newTile[1]:
            topLeft = (previousTile[0] * 256, previousTile[1] * 256)
            previousTilePng = Image.open(previousTile[2])
            tilePng.paste(previousTilePng, topLeft, previousTilePng)
        tilePng = tilePng.resize((256,256), Image.NEAREST)
        os.makedirs(foldername, exist_ok=True)
        tilePng.save(os.path.join(foldername, "{}.png".format(newTile[0][2])))

def genBannerMarkers(maps, outputFolder):
    """generate the banner.json file from maps list"""
    logging.debug(maps)

    with open(os.path.join(outputFolder, "banners.json"), "+w", encoding="utf-8") as f:
        # this will also remove deplicates
        bannerList = [a._asdict() for a in {a for amap in maps for a in amap.bannerData}]
        f.write(json.dumps(list(bannerList)))

def genMapIdMarkers(maps, outputFolder):
    mapIdMarkers = []
    dimCenterScaleDict = defaultdict(list)
    for amap in maps:
        mapData = amap.mapData
        dimCenterScaleDict[(mapData.dimension, mapData.x, mapData.z, mapData.scale)].append(amap)

    for dimCenterScale in dimCenterScaleDict.items():
        logging.debug("DimCenterScale %s", dimCenterScale[0])
        dimension, x, z, scale = dimCenterScale[0]

        maps = []
        for amap in dimCenterScale[1]:
            maps.append({"id": amap.mapData.mapId,
                         "scale" : amap.mapData.scale,
                         "filename": mapPngFilenameFormat.format(**amap.mapData._asdict()),
                         "banners": list(amap.bannerData),
                         "frames": amap.frameData})

        X = x - 64 * 2 ** scale
        Z = z - 64 * 2 ** scale

        width = 128 * 2 ** scale

        TL = [X, Z]
        TR = [X, Z + width]
        BL = [X + width, Z + width]
        BR = [X + width, Z]

        coordinates = [[TL, TR, BL, BR, TL]]
        properties = {"scale": scale,
                      "dimension": dimension,
                      "maps": maps }

        geometry = {"type": "Polygon",
                    "coordinates": coordinates}

        feature = {"type": "Feature",
                   "properties": properties,
                   "geometry": geometry}

        mapIdMarkers.append(feature)

    with open(os.path.join(outputFolder, "maps.json"), "+w", encoding="utf-8") as f:
        f.write(json.dumps(mapIdMarkers))

def copyTemplate(outputFolder, copytemplate):
    if not os.path.isdir(os.path.join(outputFolder, "assets")) or copytemplate:
        logging.info("Copying template to %s", outputFolder)
        distutils.dir_util.copy_tree(os.path.join(dir_path, "template"), outputFolder)

    else:
        logging.info("Assets folder found, not copying template")

def main():
    parser = argparse.ArgumentParser(description='convert minecraft maps to the web')
    parser.add_argument('--world', help="location of your world folder or save folder", required=True)
    parser.add_argument('--type', help="server type, bedrock or java", choices=["java", "bds"], required=True)
    parser.add_argument('--includeunlimitedtracking', help="include maps that have unlimited tracking on, this includes older maps from previous Minecraft versions and treasure maps in +1.13", action="store_true")
    parser.add_argument('--disablezoomsort', help="don't sort maps by zoom level before rendering, newer maps of higher zoom level will cover lower level maps", action="store_true")
    #parser.add_argument('--overlaymca', help="generate the regionfile overlay (Java only)", action="store_true")
    parser.add_argument('--output', help="output path for web stuff", required=True)
    parser.add_argument('--copytemplate', help="copy default index.html and assets (do this if a new release changes the tempalte)", action="store_true")
    parser.add_argument('--debug', help="show debug logging", action="store_true")

    # get the args
    args = parser.parse_args()

    # setup the logger
    if args.debug:
        level = logging.DEBUG
    else:
        level = logging.INFO

    logging.basicConfig(format='%(asctime)s %(message)s', level=level)
    # where to the maps go?
    mapsOutput = os.path.join(args.output, "maps")

    # where to the tiles go?
    tileOutput = os.path.join(args.output, "tiles")

    # where to the merged zoom level 4 maps go?
    mergedMapsOutput = os.path.join(args.output, "merged-maps")

    # figure out if the input folder is java or bedrock
    latestMaps = makeMaps(args.world, mapsOutput, serverType=args.type, unlimitedTracking=args.includeunlimitedtracking)

    # make the level 4 maps
    mergeToLevel4(mapsOutput, mergedMapsOutput, disablezoomsort=args.disablezoomsort)

    # create the tiles for the lowest zoom level
    genZoom17Tiles(mergedMapsOutput, tileOutput)

    # generate the rest of the zoom levels from level 17
    for zoom in range(16, -1, -1):
        extrapolateZoom(tileOutput, zoom)

    # make the banner markers
    genBannerMarkers(latestMaps, args.output)

    # make the maps info markers
    genMapIdMarkers(latestMaps, args.output)

    # make sure the html and assets are present and copied
    copyTemplate(args.output, args.copytemplate)

    logging.info("Done")

if __name__ == "__main__":
    main()