"""
二次封装selenium的工具
"""

from selenium import webdriver
# 使用显示等待查找元素更好控制时间以及报错
from selenium.webdriver.support.wait import WebDriverWait
# 使用 EC模块对网页的元素是否存在进行判定
from selenium.webdriver.support import expected_conditions as EC

class Webdriver:
    __instance = None

    def __new__(cls, *args, **kwargs):
        if not cls.__instance:
            options = webdriver.ChromeOptions()
            # 忽略https证书错误
            options.add_argument('--ignore-certificate-errors')
            # 忽略其他无关日志错误
            options.add_experimental_option("excludeSwitches", ['enable-automation', 'enable-logging'])
            cls.__instance = webdriver.Chrome(options=options)
        return cls.__instance

class Tool:
    __instance = None       # 这个属性用来判断程序全局是否已经存在这个 class 的实例

    def __new__(cls, *args, **kwargs):
        if cls.__instance is None:
            cls.__instance = super().__new__(cls)
        return cls.__instance

    def __init__(self):
        """
        初始化浏览器
        """
        # if Tool.__init_flag:
        #     return
        # print("start initial the class obj")

        self.driver = Webdriver()
        # self.driver.set_window_size(1080,720)
        self.driver.maximize_window()

        # Tool.__init_flag = True

    def open_url(self, url):
        """
        打开网页
        """
        url = "http://" + url
        self.driver.maximize_window
        self.driver.get(url)

    def element_find(self, locator, timeout=5):
        """
        查找网页上单个元素
        """
        try:
            element = WebDriverWait(self.driver, timeout).until(EC.presence_of_element_located(locator))
            return element
        except Exception as msg:
            print(f'该{locator}没有找到\n{msg}')

    def elements_find_list(self, locator, timeout=10):
        """寻找网络上同属性值的所有元素"""
        try:
            elements_list = WebDriverWait(self.driver, timeout).until(EC.presence_of_all_elements_located(locator))
            return elements_list
        except Exception as msg:
            print(f'该{locator}list没有找到\n{msg}')

    def send_key(self, locator=None, text=None):
        """
        输入字符
        """
        element = self.element_find(locator)
        # 输入之前清除输入框
        element.clear()
        element.send_keys(text)

    def clicks(self, locator, timeout=10):
        """点击元素"""
        element = WebDriverWait(self.driver, timeout).until(EC.element_to_be_clickable(locator))
        element.click()

    def quit_browser(self):
        """
        关闭浏览器
        """
        self.driver.quit()

    def refresh_windows(self):
        """
        刷新当前网页窗口
        """
        self.driver.refresh()

    def judge_element_view(self, locator, time=5):
        """
        判断元素的文本值是否可见
        可见返回元素本身
        不可见返回False
        """
        try:
            return WebDriverWait(self.driver, time).until(EC.visibility_of_element_located(locator))
        except Exception as msg:
            print(f'{locator}该元素找不到:\n{msg}')
            return False

    def judge_element_text(self, locator, text, time=10):
        """
        获得元素的文本信息，判断是否相等
        相等就返回True
        """
        try:
            result = WebDriverWait(self.driver, time).until(EC.text_to_be_present_in_element(locator, text))
            return result
        except Exception as msg:
            result = f'该{locator}元素文本错误：\n{msg}'
            return result

    def get_web_text(self, locator, time=10):
        """ 得到网页上可见元素的文本信息 """
        try:
            text = WebDriverWait(self.driver, time).until(EC.visibility_of_element_located(locator))
            return text.text
        except Exception as msg:
            print(f'{locator}文本不可见\n{msg}')
            return False

    def judge_element_value(self, locator=None, timeout=10, text=None):
        """判断元素的value值"""
        try:
            result = WebDriverWait(self.driver, timeout).until(EC.text_to_be_present_in_element_value(locator, text))
            return result
        except Exception as msg:
            print(f'该元素找不到,\n{msg}')

    def wait_element_vanish(self, locator=None, time=10):
        """
        等待元素消失
        元素消失返回false
        未消失，退出当前循环，继续循环
#### #设置时间的更改
        """
        while True:
            try:
                v = WebDriverWait(self.driver, time).until_not(EC.visibility_of_element_located(locator))
                print(f'可见元素已消失:{v}')
                break
            except:
                print('可见元素未消失')
                continue

    def get_input_text(self, locator):
        """获取输入框的文本信息"""
        text = self.judge_element_view(locator)
        return text.get_attribute('value')

    def judge_element_exist(self, element_list):
        """判断一系列元素是否存在"""
        for x in element_list:
            result = self.judge_element_view(x)
            # print(f'这是：{result}\n')
            if result is False:
                print(f'该{x}不可见')
                break
            else:
                return True

    def set_net_conditions(self, offline=False, latency=5, throughput=500 * 1024):
        """
        设置网络环境
        offline:网络状态设置，默认为Fales(不断网），True(断网）
        latency:网络延迟设置，默认5ms
        throughput:可以直接设置上下行网络速率
        """
        self.driver.set_network_conditions(offline=offline, latency=latency, throughput=throughput)

    def new_window_open(self, new_web):
        """打开新窗口"""
        new_web = "http://" + new_web
        # 创建js语句
        js = "window.open"+f'("{new_web}")'+";"
        # 执行js 语句
        self.driver.execute_script(js)

    def save_current_window(self):
        """保存当前窗口句柄"""
        # mainWindow变量保存当前窗口的句柄
        mainWindow = self.driver.current_window_handle
        return mainWindow

    def save_all_window(self):
        """储存所有窗口的句柄"""
        allHandles = self.driver.window_handles
        return allHandles

    def click_other_window(self, handle):
        """进入其他窗口"""
        self.driver.switch_to.window(handle)

    def switch_frame(self, frame_name=None):
        self.driver.switch_to.frame(frame_name)

    """综合公有方法"""
    def judge_account_state(self, account_type='Sign In / Sign up'):
        """
        判断账号是否登录
        以及账户类型
        MainElement.sidebar = 账户状态位置元组
        SideBarElement.account_state = 侧边栏位置元组
        account_type = 账户类型字符串

        """
        self.refresh_windows()
        self.clicks(MainElement.sidebar.value)
        result = self.judge_element_text(SideBarElement.account_state.value, account_type)
        if result is True:
            return True
        else:
            return False

    def sign_out(self, account_type='Sign In / Sign up'):
        """
        退出操作从主界面开始
        MainElement.sidebar: 侧边栏位置元组
        SideBarElement.account_state: 账号模块元组
        account_type = 账户类型字符串
        AccountElement.sign_out_but: 退出按钮位置元组
        AccountElement.out_ok_but: 退出确认OK弹窗位置元组
        """
        while True:
            state = self.judge_account_state(account_type)
            if state is False:
                print('执行分支')
                try:
                    self.refresh_windows()
                    print(1)
                    self.clicks(MainElement.sidebar.value)
                    print(2)
                    self.clicks(SideBarElement.account_state.value)
                    print(3)
                    self.clicks(AccountElement.sign_out_but.value)
                    print(4)
                    self.clicks(AccountElement.out_ok_but.value)
                    print(5)
                except:
                    continue
            else:
                break


import os
import shutil
from openpyxl import load_workbook
from django.http import HttpResponse, JsonResponse
from .models import *
from win32com import client
from django.template.loader import render_to_string
from django.core.cache import cache

def convert_to_pdf(input_path, output_path):
    excel = client.Dispatch(
        "Excel.Application"
    )  # 利用dispatch連接 microsoft 應用程式  這裡是連接到excel
    doc = excel.Workbooks.Open(input_path)
    doc.SaveAs(output_path, FileFormat=57)  # FileFormat = 57 是 PDF 格式
    doc.Close()
    excel.Quit()

def IDgenerator(year, Max_id, prefix):
    if year == int(Max_id[:4]):
        num = int(Max_id[-4:]) + 1
        ID = f"{year}{prefix}{str(num).zfill(4)}"
    else:
        num = 1
        ID = f"{year}{prefix}{str(num).zfill(4)}"
    return ID

def delete_action(type, id):
    object_data = type.objects.get(id=id)
    print(object_data)

    dest_folder = 'C://Users//user//Desktop//newpath'
    try:
        excel_file_path = os.path.abspath(object_data.excel_file)
        try:
            new_path = os.path.join(dest_folder, os.path.basename(excel_file_path))
            shutil.move(excel_file_path, new_path)
            object_data.excel_file = ""
            object_data.save()
        except:
            return None
    except:
        pass

    try:
        pdf_file_path = os.path.abspath(object_data.pdf_file)
        print(pdf_file_path)
        try:
            new_path = os.path.join(dest_folder, os.path.basename(pdf_file_path))
            shutil.move(pdf_file_path, new_path)
            object_data.pdf_file_path = ""
            object_data.save()
        except:
            return None
    except:
        pass
    object_data.delete()
    cache.delete(f"{type.__name__}:{object_data.id}")
    rest_form = type.objects.all().order_by("-date")
    prefix = str(type.__name__)[:2] + "s"
    prefix_lower = (str(type.__name__)[:2]).lower()
    data = render_to_string(f"back/async/{prefix_lower}list.html", {prefix: rest_form,"pc_type":type})
    print(data)
    return data

def pay_status_change(type, id):
    object_data = type.objects.get(id=id)
    if object_data.pay == False:
        object_data.pay = True
        object_data.save()
        all_form = type.objects.all().order_by("-date")
        prefix = str(type.__name__)[:2]
        prefix = prefix + "s"
        print(prefix)
        prefix_lower = str(type.__name__).lower()
        prefix_lower = prefix_lower[:2]
        print(prefix_lower)
        data = render_to_string(
            f"back/async/{prefix_lower}list.html", {prefix: all_form}
        )
        return data
    else:
        object_data.pay = False
        object_data.save()
        all_form = type.objects.all().order_by("-date")
        prefix = str(type.__name__)[:2]
        prefix = prefix + "s"
        print(prefix)
        prefix_lower = str(type.__name__).lower()
        prefix_lower = prefix_lower[:2]
        data = render_to_string(
            f"back/async/{prefix_lower}list.html", {prefix: all_form}
        )
        return data


import pickle
from LearningFunctions import learn
from tools import createNewPickleFile
from tools import openPickleFile
from tools import savePickleFile
import time

def sendNewData(inputData, sendData):
    print()
    if sendData == False:
        answer = "0"
        while answer != "1" and answer != "2":
            print("Teach Data?")
            print("1. Yes")
            print("2. No")
            answer = input()
        if answer == "1":
            sendData = True
        else:
            sendData = False

    if sendData == True:
        print()
        print("sending new data....")
        print()

        sendData = True

        learn(inputData)
        diseases_in = open("diseases.pickle", "rb")
        diseases = pickle.load(diseases_in)
    else:
        dataDumps_in = open("dataDumps.pickle", "rb")
        dataDumps = pickle.load(dataDumps_in)

        dataDumps = {"Lung Cancer": {"coughing": 0}}

        dataDumps_out = open("dataDumps.pickle", "wb")
        pickle.dump(dataDumps, dataDumps_out)
        dataDumps_out.close()

        originalDiseaseList_in = open("originalDiseaseList.pickle", "rb")
        originalDiseaseList = pickle.load(originalDiseaseList_in)

        originalDiseaseList = {"Lung Cancer": {"coughing": 0}}

        originalDiseaseList_out = open("originalDiseaseList.pickle", "wb")
        pickle.dump(originalDiseaseList, originalDiseaseList_out)
        originalDiseaseList_out.close()

        return "NO LOG"

def learnExtraData(diseases, disease, originalDiseaseList, dataDumps, symptom, sendData):
    print()
    if sendData == False:
        answer = "0"
        while answer != "1" and answer != "2":
            print("Teach Data?")
            print("1. Yes")
            print("2. No")
            answer = input()
        if answer == "1":
            sendData = True
        else:
            sendData = False

    if sendData == True:
        print()
        print("sending extra data....")
        print()

        sendData = True

        diseases[disease][symptom] = int((originalDiseaseList[disease][symptom] / dataDumps[disease][symptom]) * 1000)

        cases_in = open("cases.pickle", "rb")
        cases = pickle.load(cases_in)
        for disease in diseases:
            cases[disease] = 0
            for symptom in diseases[disease]:
                cases[disease] += diseases[disease][symptom]

        diseases_out = open("diseases.pickle", "wb")
        pickle.dump(diseases, diseases_out)
        diseases_out.close()

        cases_out = open("cases.pickle", "wb")
        pickle.dump(cases, cases_out)
        cases_out.close()

    else:
        dataDumps_in = open("dataDumps.pickle", "rb")
        dataDumps = pickle.load(dataDumps_in)

        dataDumps = {"Lung Cancer":{"coughing":0}}

        dataDumps_out = open("dataDumps.pickle", "wb")
        pickle.dump(dataDumps, dataDumps_out)
        dataDumps_out.close()

        originalDiseaseList_in = open("originalDiseaseList.pickle", "rb")
        originalDiseaseList = pickle.load(originalDiseaseList_in)

        originalDiseaseList = {"Lung Cancer":{"coughing":0}}

        originalDiseaseList_out = open("originalDiseaseList.pickle", "wb")
        pickle.dump(originalDiseaseList, originalDiseaseList_out)
        originalDiseaseList_out.close()

        return "NO LOG"

def findDisease(input):
    disease = ""
    lastIndex = input.index("[")
    disease = input[0:lastIndex]
    return disease

def totalCases(input):
    numOfCases = ""
    index1 = input.index("[")
    index2 = input.index("]")
    numOfCases = input[index1+1:index2]
    numOfCases = int(numOfCases)
    return numOfCases

def separateSymptoms(input):
    symptoms = []
    commaCount = 0
    hasCommas = False
    index1 = input.index(":")

    for char in input:
        if char == ",":
            hasCommas = True

    if hasCommas == True:
        index2 = input.index(",")
    else:
        index2 = input.index(".")

    for char in input:
        if char == "," or char == ".":
            commaCount += 1

    for num in range(0, commaCount):
        firstNew = True
        symptoms.append(input[index1+2:index2])
        for char in input[index2+1:len(input)]:
            if char == "," and firstNew == True or char == "." and firstNew == True :
                index1 = index2
                index2 = input[index2+1:len(input)].index(char) + len(input[0:index2+1])
                firstNew = False

    return symptoms

def formatSymptoms(roughSymptoms):
    symptoms = {}
    for symptom in roughSymptoms:
        index1 = symptom.index("(")
        index2 = symptom.index(")")
        name = symptom[0:index1]
        cases = symptom[index1+1:index2]
        symptoms[name] = cases

    return symptoms

def getKeys(dictionary):
    keys = {}
    num = 1
    for key in dictionary:
        keys[num] = key
        num += 1

    return keys

def sentenceFormatting():
    print()
    print("Format: ")
    print("Disease[total#ofcases]: symptom1(#ofcaseswithsymptom), symptom2(#ofcaseswithsymptom).")
    print()
    dataInput = input("Input: ")

    disease = findDisease(dataInput)
    numOfCases = totalCases(dataInput)
    roughSymptoms = separateSymptoms(dataInput)
    symptoms = formatSymptoms(roughSymptoms)

    return disease, numOfCases, symptoms

def seperateDiseases(dataInput):
    sentences = []
    startIndex = 0
    endIndex = -1
    for character in dataInput:
        endIndex += 1
        if character == '.':
            sentences.append(dataInput[startIndex:endIndex+1])
            startIndex = endIndex + 2

    return sentences

def formattedPrint():
    symptomTotals = openPickleFile('trainingSymptomTotals')
    diseaseTotals = openPickleFile('trainingDiseaseTotals')
    frequencyScores = openPickleFile('frequencyScores')
    weightScores = openPickleFile('weightScores')

    DISEASES = []
    for disease in diseaseTotals:
        formattedString = ''
        formattedString += disease + '[' + str(diseaseTotals[disease]) + ']: '
        count = 0
        for symptom in symptomTotals[disease]:
            count += 1
            formattedString += symptom + '[' + str(symptomTotals[disease][symptom]) + ']'
            formattedString += '(' + str(weightScores[disease][symptom]) + ')'
            formattedString += '(' + str(frequencyScores[disease][symptom]) + ')'
            if count == len(symptomTotals[disease]):
                formattedString += '. '
            else:
                formattedString += ', '

        DISEASES.append(formattedString)

    print('Diseases Data:')
    COUNT = 0
    for disease in DISEASES:
        COUNT += 1
        print(f'{COUNT}. {disease}')
        print()
    print()
    print()
    print()

    # print('Diseases Data:')
    # for disease in DISEASES:
    #     print(disease)
    # print()

def updateFrequencyScores(disease):
    symptomTotals = openPickleFile('trainingSymptomTotals')[disease]
    diseaseTotals = openPickleFile('trainingDiseaseTotals')[disease]
    frequencyScores = openPickleFile('frequencyScores')

    try:
        SCORES = frequencyScores[disease]
    except:
        SCORES = {}
    for symptom in symptomTotals:
        SCORES[symptom] = round(((symptomTotals[symptom] / diseaseTotals) * 100), 2)

    frequencyScores[disease] = SCORES
    savePickleFile('frequencyScores', frequencyScores)

def updateWeightScores(disease):
    symptomTotals = openPickleFile('trainingSymptomTotals')[disease]
    diseaseTotals = openPickleFile('trainingDiseaseTotals')[disease]
    weightScores = openPickleFile('weightScores')
    symptomTotalCount = 0

    for symptom in symptomTotals:
        symptomTotalCount += symptomTotals[symptom]

    try:
        SCORES = weightScores[disease]
    except:
        SCORES = {}
    for symptom in symptomTotals:
        SCORES[symptom] = round(((symptomTotals[symptom] / symptomTotalCount) * 100), 2)

    weightScores[disease] = SCORES
    savePickleFile('weightScores', weightScores)

def updateTotals(disease, symptoms, symptomNames, numOfCases):
    trainingSymptomTotals = openPickleFile('trainingSymptomTotals')
    try:
        SYMPTOMS = trainingSymptomTotals[disease]
    except:
        SYMPTOMS = {}
    for symptom in symptomNames:
        Symptom = symptomNames[symptom]
        try:
            SYMPTOMS[Symptom] += int(symptoms[Symptom])
            trainingSymptomTotals[disease] = SYMPTOMS
            savePickleFile('trainingSymptomTotals', trainingSymptomTotals)
        except:
            SYMPTOMS[Symptom] = int(symptoms[Symptom])
            trainingSymptomTotals[disease] = SYMPTOMS
            savePickleFile('trainingSymptomTotals', trainingSymptomTotals)

    trainingDiseaseTotals = openPickleFile('trainingDiseaseTotals')
    try:
        trainingDiseaseTotals[disease] += numOfCases
    except:
        trainingDiseaseTotals[disease] = numOfCases

    savePickleFile('trainingDiseaseTotals', trainingDiseaseTotals)

def fullDataFormatting():
    print()
    print("Format: ")
    print("Disease1[total#ofcases]: symptom1(#ofcaseswithsymptom), symptom2(#ofcaseswithsymptom). Disease2[total#ofcases]: symptom1(#ofcaseswithsymptom), symptom2(#ofcaseswithsymptom).")
    print()

    dataInput = input("Input: ")

    answer = "0"
    while answer != "1" and answer != "2":
        print("Would you like to train the program?")
        print("1. Yes")
        print("2. No")
        answer = input()
    if answer == "1":

        sentences = seperateDiseases(dataInput)
        data = {}
        for sentence in sentences:
            disease = findDisease(sentence)
            numOfCases = totalCases(sentence)
            roughSymptoms = separateSymptoms(sentence)
            symptoms = formatSymptoms(roughSymptoms)
            symptomNames = getKeys(symptoms)
            updateTotals(disease, symptoms, symptomNames, numOfCases)
            updateFrequencyScores(disease)
            updateWeightScores(disease)
            formattedPrint()
            time.sleep(0.1)
    else:
        print()

def formatData():
    ans = "4"
    fullData = False
    APPROVED = False
    while ans != "1" and ans != "2" and ans != "3" and APPROVED == False:
        print("1. Manual Input")
        print("2. Rapid Input")
        print("3. Full Data Input")
        ans = input()
        if ans == "2":
            print()
            print("Rapid Input")
            print()
            diseaseR, numOfCasesR, symptomsR = sentenceFormatting()
            symptomNamesR = getKeys(symptomsR)
            rapid = True
            fullData = False
            APPROVED = True
        elif ans == "1":
            print()
            print("Manual Input")
            print()
            rapid = False
            fullData = False
            APPROVED = True
        elif ans == '3':
            print()
            print("Full Data Input")
            print()
            rapid = False
            fullData = True
            diseases = openPickleFile('diseases')
            if diseases == {"Lung Cancer":{"coughing":0}}:
                fullDataFormatting()
                APPROVED = True
            else:
                APPROVED = False
            break

    data = []
    individualData = {}

    print()
    approved = False
    while approved == False:
        if rapid:
            disease = diseaseR
        else:
            disease = input("Disease: ")
        try:
            disease = int(disease)
            print()
            print("ERROR: Input must be a string.")
            print()
        except:
            if len(disease) > 0:
                approved = True
            else:
                print()
                print("ERROR: Input must be greater than 0 characters long.")
                print()

    totalCases = "one"
    while type(totalCases) == str or totalCases < 1:
        if rapid:
            totalCases = numOfCasesR
        else:
            totalCases = input("Number of Cases: ")
        try:
            totalCases = int(totalCases)
            if totalCases < 1:
                print()
                print("ERROR: Input must be greater than 0.")
                print()
        except:
            print()
            print("ERROR: Input must be an integer greater than 0.")
            print()

    totalSymptoms = "one"
    while type(totalSymptoms) == str or totalSymptoms < 1:
        if rapid:
            totalSymptoms = len(symptomsR)
        else:
            totalSymptoms = input("Number of Symptoms: ")
        try:
            totalSymptoms = int(totalSymptoms)
            if totalSymptoms < 1:
                print()
                print("ERROR: Input must be greater than 0.")
                print()
        except:
            print()
            print("ERROR: Input must be an integer greater than 0.")
            print()

    symptoms = {}
    dumps = {}
    originalDiseaseList = {}
    dataDumps = {}

    dataDumps_in = open("dataDumps.pickle", "rb")
    dataDumps = pickle.load(dataDumps_in)

    originalDiseaseList_in = open("originalDiseaseList.pickle", "rb")
    originalDiseaseList = pickle.load(originalDiseaseList_in)

    for numm in range(0, 1000):
        data.append(disease + ": ")

    for num in range(1, totalSymptoms + 1):

        approved = False
        while approved == False:
            if rapid:
                symptom = symptomNamesR[num]
            else:
                symptom = input("Symptom " + str(num) + ": ")
            try:
                symptom = int(symptom)
                print()
                print("ERROR: Input must be a string.")
                print()
            except:
                if len(symptom) > 0:
                    approved = True
                else:
                    print()
                    print("ERROR: Input must be greater than 0 characters long.")
                    print()

        symptomCases = "one"
        while type(symptomCases) == str or symptomCases < 1:
            if rapid:
                symptomCases = symptomsR[symptom]
            else:
                symptomCases = input("Number of Cases with " + symptom + ": ")
            try:
                symptomCases = int(symptomCases)
                if symptomCases < 1:
                    print()
                    print("ERROR: Input must be greater than 0.")
                    print()
                if symptomCases > totalCases:
                    symptomCases = 0
                    print()
                    print("ERROR: Cases with this symptom must be less than the total number of cases.")
                    print()
            except:
                print()
                print("ERROR: Input must be an integer greater than 0.")
                print()

        SymptomCases = int((symptomCases/totalCases) * 1000)
        print(symptom + ":" + str(SymptomCases))

        symptoms[symptom] = symptomCases
        dumps[symptom] = totalCases

        for number in range(0, SymptomCases):
            if num == totalSymptoms:
                data[number] += symptom + ". "
            else:
                data[number] += symptom + ", "

        goToexcept = {}
        GOTOEXCEPT = {}

        try:
            if originalDiseaseList[disease][symptom] > 0:
                originalDiseaseList[disease][symptom] = originalDiseaseList[disease][symptom]
                originalDiseaseList[disease][symptom] += symptomCases
                dataDumps[disease][symptom] = dataDumps[disease][symptom]
                dataDumps[disease][symptom] += totalCases
            else:
                print(goToexcept[GOTOEXCEPT][44])

        except:
            symptomData = {}
            for number in range(1, SymptomCases + 1):
                symptomData[number] = disease + ": "
                symptomData[number] += symptom + ". "
            individualData[symptom] = symptomData

    allFailed = True
    goToexcept = {}
    GOTOEXCEPT = {}

    for symptom in symptoms:
        try:
            if originalDiseaseList[disease][symptom] > 0:
                allFailed = False
            else:
                print(goToexcept[GOTOEXCEPT][44])
        except:
            print()
    if allFailed == True:
        originalDiseaseList[disease] = symptoms
        dataDumps[disease] = dumps
    else:
        for symptom in symptoms:
            try:
                if originalDiseaseList[disease][symptom] > 0:
                    print()
                else:
                    print()
            except:
                originalDiseaseList[disease][symptom] = symptoms[symptom]
                dataDumps[disease][symptom] = dumps[symptom]

    newData = []
    for numberr in range(0, 1000):
        if len(data[numberr]) > 7: #This prevents the data that doesn't have symptoms from being counted(good and bad) total symptoms = total cases of most common symptom
            if data[numberr][len(data[numberr]) - 2] == ",":
                data[numberr] = data[numberr][0: len(data[numberr]) - 2] + ". "
            newData.append(data[numberr])

    formattedData = ""
    for case in newData:
        formattedData += case

    individualFormatedData = {}
    for symptom in individualData:
        individualFormatedData[symptom] = ""
        for number in individualData[symptom]:
            individualFormatedData[symptom] += individualData[symptom][number]
        individualFormatedData[symptom] = individualFormatedData[symptom][0:len(individualFormatedData[symptom])-2]

    # print(originalDiseaseList[disease])
    # print(dataDumps[disease])

    goToexcept = {}
    GOTOEXCEPT = {}

    diseases_in = open("diseases.pickle", "rb")
    diseases = pickle.load(diseases_in)

    rapidInput = "Copy: "+ disease + "[" + str(totalCases) + "]: "
    for symptom in symptoms:
        rapidInput += symptom + "(" + str(symptoms[symptom]) + "), "
    rapidInput = rapidInput[0:len(rapidInput)-2]
    rapidInput += "."
    print(rapidInput)

    alreadyLearnedNew = False
    numOfSymptoms = len(symptoms)
    symptomNumber = 0
    sendData = False
    for symptom in symptoms:

        diseases_in = open("diseases.pickle", "rb")
        diseases = pickle.load(diseases_in)

        if symptomNumber >= 1:
            sendData = True
        symptomNumber += 1
        try:
            if diseases[disease][symptom] > 0:
                learnExtraData(diseases, disease, originalDiseaseList, dataDumps, symptom, sendData)

                diseases_out = open('disease.picke','wb')
                pickle.dump(diseases, diseases_out)
                diseases_out.close()
            else:
                print(goToexcept[GOTOEXCEPT][44])

            if symptomNumber == numOfSymptoms:
                return formattedData

        except:
            dataInput = individualFormatedData[symptom]
            sendNewData(dataInput, sendData)
            diseases_in = open("diseases.pickle", "rb")
            diseases = pickle.load(diseases_in)

            originalDiseaseList_out = open("originalDiseaseList.pickle", "wb")
            pickle.dump(originalDiseaseList, originalDiseaseList_out)
            originalDiseaseList_out.close()

            dataDumps_out = open("dataDumps.pickle", "wb")
            pickle.dump(dataDumps, dataDumps_out)
            dataDumps_out.close()

            diseases_out = open("diseases.pickle", "wb")
            pickle.dump(diseases, diseases_out)
            diseases_out.close()

            if symptomNumber == numOfSymptoms:
                return "NO LOG"

# trainingSymptomTotals = {"Lung Cancer": {"coughing": 0}}
# savePickleFile('trainingSymptomTotals', trainingSymptomTotals)
# trainingDiseaseTotals = {}
# savePickleFile('trainingDiseaseTotals', trainingDiseaseTotals)
# frequencyScores = {"Lung Cancer": {"coughing": 0}}
# savePickleFile('frequencyScores', frequencyScores)
# weightScores = {"Lung Cancer": {"coughing": 0}}
# savePickleFile('weightScores', weightScores)
# fullDataFormatting()


"""
module for making curve controls

TODO: fix set_shape to orient correctly - fix cycles
"""

import pymel.core as pm
import journey.lib.utils.tools as tools
import journey.lib.utils.shapes as shapes
import journey.lib.utils.color as color
from maya.cmds import DeleteHistory
reload(tools)
reload(shapes)

class Control:
    control_list = []

    def __init__(self,
                 prefix='new',
                 scale=1.0,
                 trans_to='',
                 rot_to='',
                 rot_shape=True,
                 parent='',
                 shape='circle',
                 channels=['s', 'v']):
        """Short text

        Desc

        Args:
            prefix (str): name for the controller
            scale (float, int): initial scale of the controller
            trans_to (str): object to translate the controller to
            rot_to (str): object to rotate the controller to
            rot_shape (bool): whether the controller shape should match the top transform node translation
            parent (str): object to parent the controller under
            shape (str): visual shape to give the controller. Usable names can be found under journey.lib.utils.shapes
        """
        Control.control_list.append(self)

        self.ctrl_object = None
        self.ctrl_offset = None
        self.prefix = prefix
        self.scale = scale
        self.trans_to = trans_to
        self.rot_to = rot_to
        self.rot_shape = rot_shape
        self.parent = parent
        self.shape = shape
        self.channels = channels

    def set_shape(self, shape):
        if self.get_ctrl():
            exec ("return_shape = shapes.{0}(self.scale, self.prefix + '1_ctrl')".format(shape))
            pm.delete(pm.parentConstraint(self.get_ctrl(), return_shape))
            for shape in pm.PyNode(self.get_ctrl()).getShapes():
                pm.delete(shape)
            for shape in pm.PyNode(return_shape).getShapes():
                pm.parent(shape, self.get_ctrl(), s=True, r=True)
            pm.delete(return_shape)
            self.set_color()
            self.set_rotation()
        else:
            # exec("return_shape = shapes.{0}({1}, {2})".format(self.shape, self.scale, self.prefix + '_ctrl'))
            exec("return_shape = shapes.{0}(self.scale, self.prefix + '_ctrl')".format(shape))
            self.ctrl_object = return_shape

        return self.ctrl_object

    def get_shapes(self):
        return pm.PyNode(self.get_ctrl()).getShapes()

    def set_parent(self):
        pass

    def set_color(self, ctrl_color=''):
        ctrl_color = ctrl_color.upper()
        [pm.setAttr(s + '.ove', 1) for s in self.ctrl_object.getShapes()]

        if self.prefix.startswith('l_'):
            [pm.setAttr(s + '.ovc', color.BLUE) for s in self.ctrl_object.getShapes()]
        elif self.prefix.startswith('r_'):
            [pm.setAttr(s + '.ovc', color.RED) for s in self.ctrl_object.getShapes()]
        else:
            [pm.setAttr(s + '.ovc', color.YELLOW) for s in self.ctrl_object.getShapes()]

        if ctrl_color:
            exec('clr_key = color.{0}'.format(ctrl_color))
            [pm.setAttr(s + '.ovc', clr_key) for s in self.ctrl_object.getShapes()]

    def set_channels(self, channels):
        attr = tools.lock_channels(self.ctrl_object, channels)
        return attr

    def set_translation(self, *args):
        # translate control
        if pm.objExists(self.trans_to):
            pm.delete(pm.pointConstraint(self.trans_to, self.ctrl_offset))

    def set_rotation(self, *args):
        # rotate control
        if pm.objExists(self.rot_to):
            if not self.rot_shape:
                pm.cycleCheck(e=0)
                ctrl_shapes = self.ctrl_object.getShapes()
                loc = pm.spaceLocator()
                for shape in ctrl_shapes:
                    pm.parent(shape, loc, relative=1, shape=1)
                pm.delete(pm.orientConstraint(self.rot_to, self.ctrl_offset))
                pm.parent(loc, self.rot_to)
                pm.setAttr(loc + ".tx", 0)
                pm.setAttr(loc + ".ty", 0)
                pm.setAttr(loc + ".tz", 0)
                pm.makeIdentity(loc, apply=True, r=True, t=True)
                for shape in ctrl_shapes:
                    pm.parent(shape, self.ctrl_object, relative=1, shape=1)
                pm.delete(loc)
                pm.cycleCheck(e=1)
            else:
                pm.delete(pm.orientConstraint(self.rot_to, self.ctrl_offset))

    def create(self):
        # set controller shape
        self.set_shape(self.shape)

        # set offset group for controller and re-parent
        self.ctrl_offset = pm.group(n=self.prefix + '_offset_grp', em=1)
        pm.parent(self.ctrl_object, self.ctrl_offset)

        # parent offset group to parent if set
        if pm.objExists(self.parent):
            pm.parent(self.ctrl_offset, self.parent)

        self.set_color()
        self.set_translation()
        self.set_rotation()

        self.set_channels(self.channels)

        return self

    def movable_pivot(self, *args):
        piv_ctrl = pm.spaceLocator(n=self.prefix + '_piv_ctrl')

        pm.scale(self.scale, self.scale, self.scale,)
        pm.makeIdentity(piv_ctrl, apply=True, t=1, r=1, s=1, n=0)
        DeleteHistory(piv_ctrl)

        pm.delete(pm.parentConstraint(self.get_ctrl(), piv_ctrl))
        pm.parent(piv_ctrl, self.get_ctrl())

        tools.lock_channels(piv_ctrl, channels=['r', 's'])

        pm.addAttr(self.get_ctrl(), longName='MOVABLEPIVOT', nn='HAS MOVABLE PIVOT',
                   at="enum", en='=======')
        pm.setAttr(self.get_ctrl() + '.MOVABLEPIVOT', e=True, channelBox=True)

        pm.connectAttr(piv_ctrl + '.t', self.get_ctrl() + '.rotatePivot')
        pm.connectAttr(piv_ctrl + '.t', self.get_ctrl() + '.scalePivot')

    def set_pivot(self, node):
        get_piv = pm.xform(node, piv=True, q=True, ws=True)
        pm.xform(self.get_ctrl(), ws=True, piv=(get_piv[0], get_piv[1], get_piv[2]))
        pm.xform(self.get_offset(), ws=True, piv=(get_piv[0], get_piv[1], get_piv[2]))

    def set_shape_scale(self, scale):
        """Multiplies current scale value
        """
        scale = tools.convert_scale(scale)
        for shape in self.get_shapes():
            pm.scale(pm.select(shape + '.cv[:]'), scale[0], scale[1], scale[2], relative=True)

    def set_constraint(self, driven, mo=True, channels=['t', 'r', 's']):
        tools.matrix_constraint(self.get_ctrl(), driven, mo=mo, channels=channels)

    def freeze_transforms(self, channels=['s']):
        tools.unlock_channels(self.get_offset(), channels=channels)
        tools.unlock_channels(self.get_ctrl(), channels=channels)
        pm.makeIdentity(self.get_offset(), apply=True)
        DeleteHistory()
        tools.lock_channels(self.get_offset(), channels=channels)
        tools.lock_channels(self.get_ctrl(), channels=channels)

    def get_ctrl(self, *args):
        return self.ctrl_object

    def get_offset(self, *args):
        return self.ctrl_offset


import os
import numpy as np
import pandas as pd
import SimpleITK as sitk
from scipy.stats import linregress
import matplotlib.pyplot as plt

# 00 Initialization
desired_width = 320
pd.set_option('display.width', desired_width)
np.set_printoptions(linewidth=desired_width,suppress=True,formatter={'float_kind':'{:f}'.format})

## Define functions
def DyadicProduct(A,B):

    if A.size == 3:

        if isinstance(A[0], np.float):
            C = np.zeros((3,3))
        else:
            C = sp.zeros(3)

        for i in range(3):
            for j in range(3):
                C[i,j] = A[i]*B[j]

        if not isinstance(A[0], np.float):
            C = np.array(C)

    elif A.size == 9:

        if isinstance(A[0,0], np.float):
            C = np.zeros((3,3,3,3))
        else:
            C = sp.zeros(9)

        for i in range(3):
            for j in range(3):
                for k in range(3):
                    for l in range(3):
                        if isinstance(A[0, 0], np.float):
                            C[i,j,k,l] = A[i,j] * B[k,l]
                        else:
                            C[3*i+j,3*k+l] = A[i,j] * B[k,l]

        if not isinstance(A[0,0], np.float):
            C = IsoMorphism99_3333(C)

    else:
        print('Matrices sizes mismatch')

    return C
def SymmetricProduct(A,B):

    if isinstance(A[0, 0], np.float):
        C = np.zeros((3, 3, 3, 3))
    else:
        C = sp.zeros(9)

    for i in range(3):
        for j in range(3):
            for k in range(3):
                for l in range(3):
                    if isinstance(A[0, 0], np.float):
                        C[i,j,k,l] = (1/2)*(A[i,k]*B[j,l]+A[i,l]*B[j,k])
                    else:
                        C[3*i+j,3*k+l] = (1/2)*(A[i,k]*B[j,l]+A[i,l]*B[j,k])

    if not isinstance(A[0,0], np.float):
        C = IsoMorphism99_3333(C)

    return C
def CheckMinorSymmetry(A):
    MinorSymmetry = True
    for i in range(3):
        for j in range(3):
            PartialTensor = A[:,:, i, j]
            if PartialTensor[1, 0] == PartialTensor[0, 1] and PartialTensor[2, 0] == PartialTensor[0, 2] and PartialTensor[1, 2] == PartialTensor[2, 1]:
                MinorSymmetry = True
            else:
                MinorSymmetry = False
                break

    if MinorSymmetry == True:
        for i in range(3):
            for j in range(3):
                PartialTensor = np.squeeze(A[i, j,:,:])
                if PartialTensor[1, 0] == PartialTensor[0, 1] and PartialTensor[2, 0] == PartialTensor[0, 2] and PartialTensor[1, 2] == PartialTensor[2, 1]:
                    MinorSymmetry = True
                else:
                    MinorSymmetry = False
                    break

    return MinorSymmetry
def IsoMorphism3333_66(A):

    if CheckMinorSymmetry == False:
        print('Tensor does not present minor symmetry')
    else:

        if isinstance(A[0, 0, 0, 0], np.float):
            B = np.zeros((6,6))
        else:
            B = sp.zeros(6)

        B[0, 0] = A[0, 0, 0, 0]
        B[0, 1] = A[0, 0, 1, 1]
        B[0, 2] = A[0, 0, 2, 2]
        B[0, 3] = np.sqrt(2) * A[0, 0, 1, 2]
        B[0, 4] = np.sqrt(2) * A[0, 0, 2, 0]
        B[0, 5] = np.sqrt(2) * A[0, 0, 0, 1]

        B[1, 0] = A[1, 1, 0, 0]
        B[1, 1] = A[1, 1, 1, 1]
        B[1, 2] = A[1, 1, 2, 2]
        B[1, 3] = np.sqrt(2) * A[1, 1, 1, 2]
        B[1, 4] = np.sqrt(2) * A[1, 1, 2, 0]
        B[1, 5] = np.sqrt(2) * A[1, 1, 0, 1]

        B[2, 0] = A[2, 2, 0, 0]
        B[2, 1] = A[2, 2, 1, 1]
        B[2, 2] = A[2, 2, 2, 2]
        B[2, 3] = np.sqrt(2) * A[2, 2, 1, 2]
        B[2, 4] = np.sqrt(2) * A[2, 2, 2, 0]
        B[2, 5] = np.sqrt(2) * A[2, 2, 0, 1]

        B[3, 0] = np.sqrt(2) * A[1, 2, 0, 0]
        B[3, 1] = np.sqrt(2) * A[1, 2, 1, 1]
        B[3, 2] = np.sqrt(2) * A[1, 2, 2, 2]
        B[3, 3] = 2 * A[1, 2, 1, 2]
        B[3, 4] = 2 * A[1, 2, 2, 0]
        B[3, 5] = 2 * A[1, 2, 0, 1]

        B[4, 0] = np.sqrt(2) * A[2, 0, 0, 0]
        B[4, 1] = np.sqrt(2) * A[2, 0, 1, 1]
        B[4, 2] = np.sqrt(2) * A[2, 0, 2, 2]
        B[4, 3] = 2 * A[2, 0, 1, 2]
        B[4, 4] = 2 * A[2, 0, 2, 0]
        B[4, 5] = 2 * A[2, 0, 0, 1]

        B[5, 0] = np.sqrt(2) * A[0, 1, 0, 0]
        B[5, 1] = np.sqrt(2) * A[0, 1, 1, 1]
        B[5, 2] = np.sqrt(2) * A[0, 1, 2, 2]
        B[5, 3] = 2 * A[0, 1, 1, 2]
        B[5, 4] = 2 * A[0, 1, 2, 0]
        B[5, 5] = 2 * A[0, 1, 0, 1]

        return B
def ComplianceTensor(EngineeringConstants, EigenValues=[1,1,1], Rho=0, EigenVectors=np.zeros(3)):

    if np.sum(EigenVectors).all() == 0:
        I = np.eye(3)
        EigenVectors = [I[:, 0], I[:, 1], I[:, 2]]

    ComplianceTensor = np.zeros((3, 3, 3, 3))

    if len(EngineeringConstants) == 9:

        E1, E2, E3 = EngineeringConstants[0:3]
        Mu23, Mu31, Mu12 = EngineeringConstants[3:6]
        Mu32, Mu13, Mu21 = Mu23, Mu31, Mu12
        Nu12, Nu13, Nu23 = EngineeringConstants[6:]
        Nu21 = Nu12 * E2 / E1
        Nu31 = Nu13 * E3 / E1
        Nu32 = Nu23 * E3 / E2

        E = E1, E2, E3
        Nu = np.array([[Nu13, Nu12], [Nu21, Nu23], [Nu32, Nu31]])
        Mu = np.array([[Mu13, Mu12], [Mu21, Mu23], [Mu32, Mu31]])

        for i in range(3):
            Mi = DyadicProduct(EigenVectors[i], EigenVectors[i])

            Part1 = 1 / E[i] * DyadicProduct(Mi, Mi)

            ComplianceTensor += Part1

            for ii in range(3 - 1):
                j = i - ii - 1
                Mj = DyadicProduct(EigenVectors[j], EigenVectors[j])
                Part2 = -Nu[i, ii] / E[i] * DyadicProduct(Mi, Mj)
                Part3 = 1 / (2 * Mu[i, ii]) * SymmetricProduct(Mi, Mj)

                ComplianceTensor += Part2 + Part3

    elif len(EngineeringConstants) == 5:

        if Rho == 0:
            print('BV/TV is missing!')

        else:

            E0, Nu0, Mu0, k, l = EngineeringConstants

            for i in range(3):
                Mi = DyadicProduct(EigenVectors[i], EigenVectors[i])

                Part1 = 1 / (E0 * Rho ** k * EigenValues[i] ** (2 * l)) * DyadicProduct(Mi, Mi)

                ComplianceTensor += Part1

                for ii in range(3 - 1):
                    j = i - ii - 1
                    Mj = DyadicProduct(EigenVectors[j], EigenVectors[j])
                    Part2 = -Nu0 / (E0 * Rho ** k * EigenValues[i] ** l * EigenValues[j] ** l) * DyadicProduct(Mi, Mj)
                    Part3 = 1 / (2 * Mu0 * Rho ** k * EigenValues[i] ** l * EigenValues[j] ** l) * SymmetricProduct(Mi,
                                                                                Mj)

                    ComplianceTensor += Part2 + Part3

    ComplianceTensor = IsoMorphism3333_66(ComplianceTensor)

    return ComplianceTensor
def StiffnessTensor(LaméConstants, EigenValues, Rho, EigenVectors=0):

    if EigenVectors == 0:
        I = np.eye(3)
        EigenVectors = [I[:, 0], I[:, 1], I[:, 2]]

    StiffnessTensor = np.zeros((3, 3, 3, 3))

    if len(LaméConstants) == 5:

        if Rho == 0:
            print('BV/TV is missing!')

        else:

            Lambda0, Lambda0p, Mu0, k, l = LaméConstants

            for i in range(3):
                Mi = DyadicProduct(EigenVectors[i], EigenVectors[i])

                Part1 = (Lambda0 + 2 * Mu0) * Rho ** k * EigenValues[i] ** (2 * l) * DyadicProduct(Mi, Mi)

                StiffnessTensor += Part1

                for ii in range(3 - 1):
                    j = i - ii - 1
                    Mj = DyadicProduct(EigenVectors[j], EigenVectors[j])
                    Part2 = Lambda0p * Rho ** k * EigenValues[i] ** l * EigenValues[j] ** l * DyadicProduct(Mi, Mj)
                    Part3 = 2 * Mu0 * Rho ** k * EigenValues[i] ** l * EigenValues[j] ** l * SymmetricProduct(Mi, Mj)

                    StiffnessTensor += Part2 + Part3

    else:
        print('Wrong number of parameters: Lambda0, Lambdaop, Mu0, k, l')

    StiffnessTensor = IsoMorphism3333_66(StiffnessTensor)

    return StiffnessTensor
def LinearRegression(Data,Exponents=[0,0]):

    if Exponents[0] == 0 and Exponents[1] == 0:

        for Index in Data.index:

            SampleData = Data.loc[Index]
            EngineeringConstants = SampleData[['E1','E2','E3','Mu23','Mu31','Mu12','Nu12','Nu13','Nu23']].values
            Rho = SampleData['BVTV']
            m1, m2, m3 = SampleData[['m1', 'm2', 'm3']].values

            # Build Compliance Tensors
            E = ComplianceTensor(EngineeringConstants)
            S = np.linalg.inv(E)

            y = np.array(np.log([S[0, 0], S[1, 1], S[2, 2],
                                 S[2, 1], S[0, 2], S[1, 0],
                                 S[3, 3] / 2, S[4, 4] / 2, S[5, 5] / 2,
                                 S[1, 2], S[2, 0], S[0, 1]]))

            X = np.array([[1, 0, 0, np.log(Rho), np.log(m1 ** 2)],
                          [1, 0, 0, np.log(Rho), np.log(m2 ** 2)],
                          [1, 0, 0, np.log(Rho), np.log(m3 ** 2)],
                          [0, 1, 0, np.log(Rho), np.log(m3 * m2)],
                          [0, 1, 0, np.log(Rho), np.log(m1 * m3)],
                          [0, 1, 0, np.log(Rho), np.log(m2 * m1)],
                          [0, 0, 1, np.log(Rho), np.log(m2 * m3)],
                          [0, 0, 1, np.log(Rho), np.log(m3 * m1)],
                          [0, 0, 1, np.log(Rho), np.log(m1 * m2)],
                          [0, 1, 0, np.log(Rho), np.log(m2 * m3)],
                          [0, 1, 0, np.log(Rho), np.log(m3 * m1)],
                          [0, 1, 0, np.log(Rho), np.log(m1 * m2)]])

            y = np.matrix(y).T
            X = np.matrix(X)

            if Index == Data.index[0]:
                ytot = y
                Xtot = X
            else:
                ytot = np.append(ytot, y, 0)
                Xtot = np.append(Xtot, X, 0)

        LHS = Xtot.T * Xtot
        RHS = Xtot.T * ytot
        Solution = np.linalg.solve(LHS, RHS)

        l = Solution[4, 0]
        k = Solution[3, 0]
        Mu0 = np.exp(Solution[2, 0])
        Lambda0p = np.exp(Solution[1, 0])
        Lambda0 = np.exp(Solution[0, 0]) - 2 * Mu0

    else:

        k, l = Exponents

        for Index in Data.index:

            SampleData = Data.loc[Index]
            EngineeringConstants = SampleData[['E1','E2','E3','Mu23','Mu31','Mu12','Nu12','Nu13','Nu23']].values
            Rho = SampleData['BVTV']
            m1, m2, m3 = SampleData[['m1', 'm2', 'm3']].values

            # Build Compliance Tensors
            E = ComplianceTensor(EngineeringConstants)
            S = np.linalg.inv(E)

            y = np.array(np.log([S[0, 0], S[1, 1], S[2, 2],
                                 S[2, 1], S[0, 2], S[1, 0],
                                 S[3, 3] / 2, S[4, 4] / 2, S[5, 5] / 2,
                                 S[1, 2], S[2, 0], S[0, 1]]))

            X1 = np.array([[1, 0, 0],
                           [1, 0, 0],
                           [1, 0, 0],
                           [0, 1, 0],
                           [0, 1, 0],
                           [0, 1, 0],
                           [0, 0, 1],
                           [0, 0, 1],
                           [0, 0, 1],
                           [0, 1, 0],
                           [0, 1, 0],
                           [0, 1, 0]])

            X2 = np.array([[np.log(Rho), np.log(m1 ** 2)],
                           [np.log(Rho), np.log(m2 ** 2)],
                           [np.log(Rho), np.log(m3 ** 2)],
                           [np.log(Rho), np.log(m3 * m2)],
                           [np.log(Rho), np.log(m1 * m3)],
                           [np.log(Rho), np.log(m2 * m1)],
                           [np.log(Rho), np.log(m2 * m3)],
                           [np.log(Rho), np.log(m3 * m1)],
                           [np.log(Rho), np.log(m1 * m2)],
                           [np.log(Rho), np.log(m2 * m3)],
                           [np.log(Rho), np.log(m3 * m1)],
                           [np.log(Rho), np.log(m1 * m2)]])

            y = np.matrix(y).T
            X1 = np.matrix(X1)
            X2 = np.matrix(X2)

            if Index == Data.index[0]:
                ytot = y
                X1tot = X1
                X2tot = X2

            else:
                ytot = np.append(ytot, y, 0)
                X1tot = np.append(X1tot, X1, 0)
                X2tot = np.append(X2tot, X2, 0)

        LHS = X1tot.T * X1tot
        RHS = X1tot.T * (ytot - X2tot * np.matrix([k, l]).T)
        Solution = np.linalg.solve(LHS, RHS)

        Mu0 = np.exp(Solution[2, 0])
        Lambda0p = np.exp(Solution[1, 0])
        Lambda0 = np.exp(Solution[0, 0]) - 2 * Mu0

    return Lambda0, Lambda0p, Mu0, k, l
def ComputeRelativeNormError(XData, YData):

    # Compute Relative Error (Euclidian distance) in log space
    TensorDiff = (np.log(XData) - np.log(YData))**2
    Error = TensorDiff.sum(axis=1)

    # Compute denominator to normalize the error
    LogTensor = np.log(XData)**2
    Denominator = LogTensor.sum(axis=1)

    RelativeNormError = np.sqrt(Error/Denominator)

    return RelativeNormError
def ComplianceRegression(XData, YData, DataFolder, DataType, FitDOF):

    ComplianceFile = os.path.join(DataFolder, '06_Compliance_' + DataType + '_' + str(FitDOF) + 'DOFs.png')

    Slope, Intercept, RValue, PValue, StdErr = linregress(np.log(XData), np.log(YData))
    XArray = np.array([XData.min(), XData.max()])
    YArray = XArray * Slope

    Figure, Axes = plt.subplots(1, 1, figsize=(5.5, 4.5),dpi=100)
    Axes.plot([1e1, 1e-5], [1e1, 1e-5], linestyle='--', linewidth=2, color=(0, 0, 0), label='Diagonal')
    Axes.plot(Eii123Data[['E11', 'E22', 'E33']],
              Eii123Fit[['E11', 'E22', 'E33']],
              linestyle='none', marker='o', color=(0, 0, 1))
    Axes.plot([], linestyle='none', marker='o', color=(0, 0, 1), label='$\epsilon_{ii}^{-1}$')
    Axes.plot(-Eij123Data[['E12', 'E13', 'E21', 'E23', 'E31', 'E32']],
              -Eij123Fit[['E12', 'E13', 'E21', 'E23', 'E31', 'E32']],
              linestyle='none', marker='o', color=(0, 1, 0))
    Axes.plot([], linestyle='none', marker='o', color=(0, 1, 0), label=r'$-\nu_{ij}/\epsilon_{ij}$')
    Axes.plot(Eii456Data[['E44', 'E55', 'E66']],
              Eii456Fit[['E44', 'E55', 'E66']],
              linestyle='none', marker='o', color=(1, 0, 0))
    Axes.plot([], linestyle='none', marker='o', color=(1, 0, 0), label='$\mu_{ij}^{-1}$')
    Axes.plot(XArray, YArray, color=(0, 0, 0), linewidth=0.75, label='Fit results')
    Axes.set_xlim([1e1, 1e-5])
    Axes.set_ylim([1e1, 1e-5])
    Axes.annotate('Number of sample : %i' % NSamples, (1e-2, 3e0))
    Axes.annotate('R$^{2}$ : %.2f' % (RValue ** 2), (1e-2, 7e0))
    Axes.set_xlabel('Observed compliance (MPa$^{-1}$)')
    Axes.set_ylabel('Fitted compliance (MPa$^{-1}$)')
    plt.legend(loc='upper left')
    plt.xscale('log')
    plt.yscale('log')
    plt.savefig(ComplianceFile,dpi=1000)
    plt.show()
    plt.close(Figure)

    return Slope, Intercept, RValue, PValue, StdErr
def StiffnessRegression(XData, YData, DataFolder, DataType, FitDOF):

    StiffnessFile = os.path.join(DataFolder, '07_Stiffness_' + DataType + '_' + str(FitDOF) + 'DOFs.png')

    Slope, Intercept, RValue, PValue, StdErr = linregress(np.log(XData), np.log(YData))
    XArray = np.array([XData.min(), XData.max()])
    YArray = XArray * Slope + Intercept

    Figure, Axes = plt.subplots(1, 1, figsize=(5.5, 4.5),dpi=100)
    Axes.plot([1e-2, 1e4], [1e-2, 1e4], linestyle='--', linewidth=2, color=(0, 0, 0), label='Diagonal')
    Axes.plot(Sii123Data[['S11', 'S22', 'S33']],
              Sii123Fit[['S11', 'S22', 'S33']],
              linestyle='none', marker='o', color=(0, 0, 1))
    Axes.plot([], linestyle='none', marker='o', color=(0, 0, 1), label='$\lambda_{ii}$')
    Axes.plot(Sij123Data[['S12', 'S13', 'S21', 'S23', 'S31', 'S32']],
              Sij123Fit[['S12', 'S13', 'S21', 'S23', 'S31', 'S32']],
              linestyle='none', marker='o', color=(0, 1, 0))
    Axes.plot([], linestyle='none', marker='o', color=(0, 1, 0), label=r'$\lambda_{ij}$')
    Axes.plot(Sii456Data[['S44', 'S55', 'S66']],
              Sii456Fit[['S44', 'S55', 'S66']],
              linestyle='none', marker='o', color=(1, 0, 0))
    Axes.plot([], linestyle='none', marker='o', color=(1, 0, 0), label='$\mu_{ij}$')
    Axes.plot(XArray, YArray, color=(0, 0, 0), linewidth=0.75, label='Fit results')
    Axes.set_xlim([1e-2, 1e4])
    Axes.set_ylim([1e-2, 1e4])
    Axes.annotate('Number of sample : %i' % NSamples, (1e1, 4e-2))
    Axes.annotate('R$^{2}$ : %.2f' % (RValue ** 2), (1e1, 1.5e-2))
    Axes.set_xlabel('Observed stiffness (MPa)')
    Axes.set_ylabel('Fitted stiffness (MPa)')
    plt.legend(loc='upper left')
    plt.yscale('log')
    plt.xscale('log')
    plt.savefig(StiffnessFile,dpi=1000)
    plt.show()
    plt.close(Figure)

    return Slope, Intercept, RValue, PValue, StdErr
def DeterminationCoefficients(XData, YData, NParameters, NSamples):

    # Sum of the square of the residuals
    Residuals = (np.log(YData) - np.log(XData))**2
    SCE = Residuals.sum()

    # Sum of the square of the observation
    Mean = np.mean(np.log(XData))
    TotalSum = np.sum(np.log(XData) ** 2)
    SCT = TotalSum - 12 * NSamples * Mean ** 2

    R2    = 1 - SCE / SCT
    R2adj = 1 - SCE / SCT * (12 * NSamples - 1) / (12 * NSamples - NParameters - 1)

    return R2, R2adj

## Set variables
WorkingDirectory = os.getcwd()
MainFolder = os.path.join(WorkingDirectory,'04_Results')
SubFolders = [File for File in os.listdir(MainFolder) if os.path.isdir(os.path.join(MainFolder,File))]
SubFolders.sort()

DataGroup = 0   # 0 = Healthy sub group, 1 = OI sub group

DataFolder = os.path.join(MainFolder,SubFolders[DataGroup],'03_LinearRegression')
GroupFolder = os.path.join(MainFolder,SubFolders[DataGroup+2])

# 01 Load tensor data
SubGroup = pd.read_csv(GroupFolder + '/02_SubGroup.csv')
if DataGroup == 1:
    SubGroup.columns = ['Folder', 'Patient_ID', 'Sample Number', 'Age', 'Sex', 'OI_Type']

Data = pd.read_csv(os.path.join(DataFolder,'00_Data.csv'))

SubData = pd.DataFrame()
for Sample in SubGroup['Sample Number']:

    for Scan in Data['Scan']:

        if DataGroup == 0:
            ScanNumber = Scan[4:8]
            SampleNumber = Sample
        elif DataGroup == 1:
            ScanNumber = Scan[11:15]
            SampleNumber = Sample[-4:]

        if int(ScanNumber) == int(SampleNumber):

            SampleData = Data[Data['Scan']==Scan]
            SubData = SubData.append(SampleData,ignore_index=True)
            break
SubData.to_csv(GroupFolder + '/02_SubGroupData.csv',index=False)

# 02 Define filter parameter
Threshold = (Data.quantile(0.75)['Variation Coefficient']
             - Data.quantile(0.25)['Variation Coefficient']) * 1.5 \
             + Data.quantile(0.75)['Variation Coefficient']
MinBVTV = Data['BVTV'].min()
MaxBVTV = Data['BVTV'].max()

## Compare to actual filter parameter
Filter = pd.read_csv(os.path.join(MainFolder,SubFolders[DataGroup+2],'03_Filter.csv'))

if MinBVTV < Filter['Min BVTV'].values[0]:
    MinBVTV = Filter['Min BVTV'].values[0]
if MaxBVTV > Filter['Max BVTV'].values[0]:
    MaxBVTV = Filter['Max BVTV'].values[0]

if DataGroup == 1:
    Threshold = Filter['Threshold'].values[0]

FilterData = pd.DataFrame({'Threshold':Threshold,'Min BVTV':MinBVTV,'Max BVTV':MaxBVTV},index=[0])

## Save filter parameters for both group
for Group in range(2):
    FilterData.to_csv(os.path.join(MainFolder,SubFolders[Group+2],'03_Filter.csv'),index=False)

# 03 Filter data
Threshold, MinBVTV, MaxBVTV = Filter.values[0]

CVFilter = SubData['Variation Coefficient'] > Threshold
BFMaxFilter = SubData['BVTV'] > MaxBVTV
BFMinFilter = SubData['BVTV'] < MinBVTV

ExcludedData = SubData[BFMaxFilter|BFMinFilter]
# ExcludedData = Data[CVFilter|BFMaxFilter|BFMinFilter]

CVFilter = SubData['Variation Coefficient'] <= Threshold
BFMaxFilter = SubData['BVTV'] <= MaxBVTV
BFMinFilter = SubData['BVTV'] >= MinBVTV

WindowedData = SubData[BFMaxFilter&BFMinFilter]
FilteredData = SubData[CVFilter&BFMaxFilter&BFMinFilter]

ThresholdFilter = WindowedData['Variation Coefficient'] > Threshold
ThresholdData = WindowedData[ThresholdFilter]

## Plot BVTV vs CV
plt.rc('font', size=12)
Figure, Axes = plt.subplots(1, 1, figsize=(5.5, 4.5),dpi=100)
Axes.plot(SubData['BVTV'],SubData['Variation Coefficient'],linestyle='none',marker='o',color=(0,0,0),fillstyle='none',label='Data')
Axes.plot([MinBVTV,MinBVTV],[0,2],color=(0,0,1),linestyle='--',label='BV/TV window')
Axes.plot([MaxBVTV,MaxBVTV],[0,2],color=(0,0,1),linestyle='--')
Axes.plot(ExcludedData['BVTV'],ExcludedData['Variation Coefficient'],linestyle='none',marker='x',color=(0,0,1),label='BV/TV Windowing')
Axes.plot([0,0.7],[Threshold,Threshold],color=(1,0,0),linestyle='--',label='Threshold')
Axes.plot([0,0.7],[Threshold,Threshold],color=(1,0,0),linestyle='--')
Axes.plot(ThresholdData['BVTV'],ThresholdData['Variation Coefficient'],linestyle='none',marker='x',color=(1,0,0),label='CV Thresholding')
Axes.set_xlim([0,0.7])
Axes.set_ylim([0,1.85])
Axes.set_xlabel('BV/TV (-)')
Axes.set_ylabel('Coefficient of Variation (-)')
plt.legend(loc='upper right',framealpha=1)
plt.savefig(os.path.join(MainFolder,SubFolders[DataGroup+2],'04_CVVsBVTV.png'),dpi=1000)
plt.show()
plt.close(Figure)

# 04 Perform the linear regression
FitResults = pd.DataFrame()
for Subset in range(3):

    if Subset == 0:
        Data2Fit = SubData
        DataType = 'Complete'
    if Subset == 1:
        Data2Fit = WindowedData
        DataType = 'Windowed'
    if Subset == 2:
        Data2Fit = FilteredData
        DataType = 'Filtered'

    DOFs = 5
    Lambda0, Lambda0p, Mu0, k, l = LinearRegression(Data2Fit)
    FitResults = FitResults.append({'Data type':DataType,
                                    'NSamples':len(Data2Fit),
                                    'DOFs':DOFs,
                                    'Lambda0':Lambda0,
                                    'Lambda0p':Lambda0p,
                                    'Mu0':Mu0,
                                    'k':k,
                                    'l':l},
                                   ignore_index=True)
    if Subset == 0 or Subset == 1:
        DOFs = 3
        Lambda0, Lambda0p, Mu0, k, l = LinearRegression(Data2Fit,Exponents=[1.55,0.84])
        FitResults = FitResults.append({'Data type':DataType,
                                        'NSamples':len(Data2Fit),
                                        'DOFs':DOFs,
                                        'Lambda0':Lambda0,
                                        'Lambda0p':Lambda0p,
                                        'Mu0':Mu0,
                                        'k':k,
                                        'l':l},
                                       ignore_index=True)

    if Subset == 2:
        DOFs = 3
        Lambda0, Lambda0p, Mu0, k, l = LinearRegression(Data2Fit,Exponents=[1.55,0.82])
        FitResults = FitResults.append({'Data type':DataType,
                                        'NSamples':len(Data2Fit),
                                        'DOFs':DOFs,
                                        'Lambda0':Lambda0,
                                        'Lambda0p':Lambda0p,
                                        'Mu0':Mu0,
                                        'k':k,
                                        'l':l},
                                       ignore_index=True)

## Store results
FitResults = FitResults.round(3)
ColsToInt = ['Lambda0', 'Lambda0p', 'Mu0', 'NSamples']
FitResults[ColsToInt] = FitResults[ColsToInt].astype(int)
FitResults.to_csv(os.path.join(GroupFolder,'05_FitResults.csv'),index=False)

LatexTable = FitResults.to_latex(index=False,caption='Fit Results')
LatexFile = open(os.path.join(GroupFolder,'05_FitResults.tex'), 'w')
LatexFile.writelines(LatexTable)
LatexFile.close()

# 05 Evaluate fit
DataTypes = ['Complete', 'Windowed', 'Filtered']
DataType = DataTypes[0]
FitDOFs = [3, 5]
FitDOF = FitDOFs[0]
plt.rc('font', size=12)

Eii123Data = pd.DataFrame()
Eij123Data = pd.DataFrame()
Eii456Data = pd.DataFrame()

Sii123Data = pd.DataFrame()
Sij123Data = pd.DataFrame()
Sii456Data = pd.DataFrame()

## If necessary filter data (variation coefficient)
if DataType == DataTypes[0]:
    Data2Eval = SubData
elif DataType == DataTypes[1]:
    Data2Eval = WindowedData
elif DataType == DataTypes[2]:
    Data2Eval = FilteredData

## Collect tensors data
print('Collect tensors data...')
for SampleNumber in Data2Eval.index:

    SampleData = Data2Eval.loc[SampleNumber]
    EngineeringConstants = SampleData[['E1', 'E2', 'E3', 'Mu23', 'Mu31', 'Mu12', 'Nu12', 'Nu13', 'Nu23']].values
    Rho = SampleData['BVTV']
    m1, m2, m3 = SampleData[['m1', 'm2', 'm3']].values

    # Build tensors
    E = ComplianceTensor(EngineeringConstants, EigenValues=[m1,m2,m3],Rho=Rho)
    S = np.linalg.inv(E)

    # Get nonzero elements

    Eii123 = np.zeros(3)
    Eij123 = np.zeros(6)
    Eii456 = np.zeros(3)

    Sii123 = np.zeros(3)
    Sij123 = np.zeros(6)
    Sii456 = np.zeros(3)

    ii123 = 0
    ij123 = 0
    ii456 = 0

    for i in range(6):
        for j in range(6):
            if i < 3 and j < 3:
                if i == j:
                    Eii123[ii123] = E[i,j]
                    Sii123[ii123] = S[i,j]
                    ii123 += 1
                else:
                    Eij123[ij123] = E[i,j]
                    Sij123[ij123] = S[i,j]
                    ij123 += 1
            else:
                if i == j:
                    Eii456[ii456] = E[i,j]
                    Sii456[ii456] = S[i,j]
                    ii456 += 1

    Eii123Data = Eii123Data.append({'Sample':SampleData['Scan'],
                                    'ROI Number': SampleData['ROI Number'],
                                    'E11':Eii123[0],
                                    'E22':Eii123[1],
                                    'E33':Eii123[2]},ignore_index=True)
    Eij123Data = Eij123Data.append({'Sample':SampleData['Scan'],
                                    'ROI Number': SampleData['ROI Number'],
                                    'E12':Eij123[0],
                                    'E13':Eij123[1],
                                    'E21':Eij123[2],
                                    'E23':Eij123[3],
                                    'E31':Eij123[4],
                                    'E32':Eij123[5]},ignore_index=True)
    Eii456Data = Eii456Data.append({'Sample':SampleData['Scan'],
                                    'ROI Number': SampleData['ROI Number'],
                                    'E44':Eii456[0],
                                    'E55':Eii456[1],
                                    'E66':Eii456[2]},ignore_index=True)

    Sii123Data = Sii123Data.append({'Sample':SampleData['Scan'],
                                    'ROI Number': SampleData['ROI Number'],
                                    'S11':Sii123[0],
                                    'S22':Sii123[1],
                                    'S33':Sii123[2]},ignore_index=True)
    Sij123Data = Sij123Data.append({'Sample':SampleData['Scan'],
                                    'ROI Number': SampleData['ROI Number'],
                                    'S12':Sij123[0],
                                    'S13':Sij123[1],
                                    'S21':Sij123[2],
                                    'S23':Sij123[3],
                                    'S31':Sij123[4],
                                    'S32':Sij123[5]},ignore_index=True)
    Sii456Data = Sii456Data.append({'Sample':SampleData['Scan'],
                                    'ROI Number': SampleData['ROI Number'],
                                    'S44':Sii456[0],
                                    'S55':Sii456[1],
                                    'S66':Sii456[2]},ignore_index=True)
print('Done')

## Build data with fitted parameter
print('Build data with fitted parameter...')
DOFFilter = FitResults['DOFs'] == FitDOF
DataFilter = FitResults['Data type'] == DataType
FitResult = FitResults[DOFFilter&DataFilter]

NSamples = FitResult['NSamples']
Lambda0, Lambda0p, Mu0, k, l = FitResult[['Lambda0', 'Lambda0p', 'Mu0', 'k', 'l']].values[0]

Eii123Fit = pd.DataFrame()
Eij123Fit = pd.DataFrame()
Eii456Fit = pd.DataFrame()

Sii123Fit = pd.DataFrame()
Sij123Fit = pd.DataFrame()
Sii456Fit = pd.DataFrame()

for SampleNumber in Data2Eval.index:

    SampleData = Data2Eval.loc[SampleNumber]
    Rho = SampleData['BVTV']
    m1, m2, m3 = SampleData[['m1', 'm2', 'm3']].values

    # Build tensors
    S = StiffnessTensor([Lambda0, Lambda0p, Mu0, k, l], EigenValues=[m1, m2, m3], Rho=Rho)
    E = np.linalg.inv(S)

    # Get nonzero elements

    Eii123 = np.zeros(3)
    Eij123 = np.zeros(6)
    Eii456 = np.zeros(3)

    Sii123 = np.zeros(3)
    Sij123 = np.zeros(6)
    Sii456 = np.zeros(3)

    ii123 = 0
    ij123 = 0
    ii456 = 0

    for i in range(6):
        for j in range(6):
            if i < 3 and j < 3:
                if i == j:
                    Eii123[ii123] = E[i, j]
                    Sii123[ii123] = S[i, j]
                    ii123 += 1
                else:
                    Eij123[ij123] = E[i, j]
                    Sij123[ij123] = S[i, j]
                    ij123 += 1
            else:
                if i == j:
                    Eii456[ii456] = E[i, j]
                    Sii456[ii456] = S[i, j]
                    ii456 += 1

    Eii123Fit = Eii123Fit.append({'Sample': SampleData['Scan'],
                                    'ROI Number': SampleData['ROI Number'],
                                    'E11': Eii123[0],
                                    'E22': Eii123[1],
                                    'E33': Eii123[2]}, ignore_index=True)
    Eij123Fit = Eij123Fit.append({'Sample': SampleData['Scan'],
                                    'ROI Number': SampleData['ROI Number'],
                                    'E12': Eij123[0],
                                    'E13': Eij123[1],
                                    'E21': Eij123[2],
                                    'E23': Eij123[3],
                                    'E31': Eij123[4],
                                    'E32': Eij123[5]}, ignore_index=True)
    Eii456Fit = Eii456Fit.append({'Sample': SampleData['Scan'],
                                    'ROI Number': SampleData['ROI Number'],
                                    'E44': Eii456[0],
                                    'E55': Eii456[1],
                                    'E66': Eii456[2]}, ignore_index=True)

    Sii123Fit = Sii123Fit.append({'Sample': SampleData['Scan'],
                                    'ROI Number': SampleData['ROI Number'],
                                    'S11': Sii123[0],
                                    'S22': Sii123[1],
                                    'S33': Sii123[2]}, ignore_index=True)
    Sij123Fit = Sij123Fit.append({'Sample': SampleData['Scan'],
                                    'ROI Number': SampleData['ROI Number'],
                                    'S12': Sij123[0],
                                    'S13': Sij123[1],
                                    'S21': Sij123[2],
                                    'S23': Sij123[3],
                                    'S31': Sij123[4],
                                    'S32': Sij123[5]}, ignore_index=True)
    Sii456Fit = Sii456Fit.append({'Sample': SampleData['Scan'],
                                    'ROI Number': SampleData['ROI Number'],
                                    'S44': Sii456[0],
                                    'S55': Sii456[1],
                                    'S66': Sii456[2]}, ignore_index=True)
print('Done')

# 05 Perform linear regression of results
print('Perform linear regression of results')
XData = pd.concat([Eii123Data[['E11','E22','E33']],
                   -Eij123Data[['E12','E13','E21','E23','E31','E32']],
                   Eii456Data[['E44', 'E55', 'E66']]],axis=1)
YData = pd.concat([Eii123Fit[['E11','E22','E33']],
                   -Eij123Fit[['E12','E13','E21','E23','E31','E32']],
                   Eii456Fit[['E44', 'E55', 'E66']]],axis=1)

RelativeNormError = ComputeRelativeNormError(XData, YData)
Data['Compliance Norm Error'] = RelativeNormError

XData = XData.values.reshape(XData.values.size)
YData = YData.values.reshape(YData.values.size)

Slope, Intercept, RValue, PValue, StdErr = ComplianceRegression(XData, YData, GroupFolder, DataType, FitDOF)
R2, R2adj = DeterminationCoefficients(XData, YData, NParameters=FitDOF, NSamples=NSamples)

ComplianceRegressionResults = pd.DataFrame({'Slope':Slope,
                                            'Interception':Intercept,
                                            'RValue':RValue,
                                            'PValue':PValue,
                                            'Standard Error':StdErr,
                                            'RSquare':R2.values[0],
                                            'RSquareAdj':R2adj.values[0],
                                            'Relative Norm Error':RelativeNormError.mean(),
                                            'Rel Norm Error Std':RelativeNormError.std()},index={0})
ComplianceFile = os.path.join(GroupFolder,'04_Compliance_' + DataType + '_' + str(FitDOF) + 'DOFs.csv')
ComplianceRegressionResults.to_csv(ComplianceFile,index=False)

XData = pd.concat([Sii123Data[['S11', 'S22', 'S33']],
                   Sij123Data[['S12', 'S13', 'S21', 'S23', 'S31', 'S32']],
                   Sii456Data[['S44', 'S55', 'S66']]], axis=1)
YData = pd.concat([Sii123Fit[['S11', 'S22', 'S33']],
                   Sij123Fit[['S12', 'S13', 'S21', 'S23', 'S31', 'S32']],
                   Sii456Fit[['S44', 'S55', 'S66']]], axis=1)

RelativeNormError = ComputeRelativeNormError(XData, YData)
Data['Stiffness Norm Error'] = RelativeNormError

XData = XData.values.reshape(XData.values.size)
YData = YData.values.reshape(YData.values.size)

Slope, Intercept, RValue, PValue, StdErr = StiffnessRegression(XData,YData, GroupFolder, DataType, FitDOF)
R2, R2adj = DeterminationCoefficients(XData, YData, NParameters=FitDOF, NSamples=NSamples)

StiffnessRegressionResults = pd.DataFrame({'Slope':Slope,
                                            'Interception':Intercept,
                                            'RValue':RValue,
                                            'PValue':PValue,
                                            'Standard Error':StdErr,
                                            'RSquare':R2.values[0],
                                            'RSquareAdj':R2adj.values[0],
                                            'Relative Norm Error':RelativeNormError.mean(),
                                            'Rel Norm Error Std':RelativeNormError.std()},index={0})
StiffnessFile = os.path.join(GroupFolder,'05_Stiffness_' + DataType + '_' + str(FitDOF) + 'DOFs.csv')
StiffnessRegressionResults.to_csv(StiffnessFile,index=False)
print('Done')


import gc
import os
import random
from typing import Dict, List, Tuple, Set, Union, Any, Optional, Callable

import numpy as np
import torch
import torch.fx as fx
import torch.nn as nn

def torch_device(default_device: str = None, **kwargs) -> str:
    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    if default_device is not None:
        device = default_device if default_device in ['cuda', 'cpu'] else device
    else:
        device = kwargs['device'] if 'device' in kwargs.keys() else device
    return device

def get_one_hot(target, num_class):
    one_hot = torch.zeros(target.shape[0], num_class).to(target.device)
    one_hot = one_hot.scatter(dim=1, index=target.long().view(-1, 1), value=1.)
    return one_hot

def tensor_reverse_permute(tensor: torch.Tensor) -> torch.Tensor:
    if tensor is None:
        return None
    dims = list(range(len(tensor.shape)))
    dims.reverse()
    return torch.permute(tensor, dims)

def extract_kwargs(kwargs: Dict, key: str, default_value=None):
    return kwargs[key] if key in kwargs else default_value

def params_state_size(params_state) -> int:
    if isinstance(params_state, (int, float, bool, complex, str)):
        return 1
    if isinstance(params_state, torch.Tensor):
        return params_state.numel()
    if isinstance(params_state, Dict):
        return sum([params_state_size(v) for k, v in params_state.items()])
    if isinstance(params_state, (List, Tuple, Set)):
        return sum([params_state_size(v) for v in params_state])
    raise Exception(f'unrecognized state type {type(params_state)} to calculate parameters size')

def extract_losses(losses) -> Union[torch.Tensor, float]:
    if isinstance(losses, torch.Tensor):
        return losses if losses.requires_grad and len(losses.shape) == 0 else 0.0
    if isinstance(losses, Dict):
        return sum([extract_losses(_loss) for _loss in losses.values()])
    if isinstance(losses, (List, Tuple, Set)):
        return sum([extract_losses(_loss) for _loss in losses])
    return 0.0

def random_shuffle(seed, _list):
    random.seed(seed)
    random.shuffle(_list)

def random_sample(seed, _list, num_pick):
    random.seed(seed)
    return random.sample(_list, num_pick)

def random_int(seed, start, end):
    random.seed(seed)
    random.randint(start, end)

def normalize(x, ord: int = None, axis: int = 0, keepdims=True):
    x_norm = np.linalg.norm(x, ord=ord, axis=axis, keepdims=keepdims)
    x = x / x_norm
    return x

def np_save(base_dir, filename, data):
    if not os.path.isdir(base_dir):
        os.makedirs(base_dir)
    np.save(os.path.join(base_dir, filename), data)

def load_task(base_dir, task):
    return np.load(os.path.join(base_dir, task), allow_pickle=True)

def same_seeds(seed=42069):
    random.seed(seed)
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

def tensor_value(*tensors: torch.Tensor):
    if len(tensors) == 1:
        return tensors[0].cpu().detach().item()
    else:
        return (tensor_value(tensor) for tensor in tensors)

class model_on_device(object):

    def __init__(self, model: nn.Module, device: str = 'cpu') -> None:
        self.model = model
        self.device = device
        super(model_on_device, self).__init__()

    def __enter__(self):
        self.model.to(self.device)

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.model.cpu()

def clear_cache(func):
    def _clear_cache():
        gc.collect()
        if torch.cuda.is_available():
            torch.cuda.empty_cache()

    def _func(*args, **kwargs):
        _clear_cache()
        res = func(*args, **kwargs)
        _clear_cache()
        return res

    return _func

class ModulePathTracer(torch.fx.Tracer):
    """
    ModulePathTracer is an FX tracer that--for each operation--also records
    the qualified name of the Module from which the operation originated.
    """

    # The current qualified name of the Module being traced. The top-level
    # module is signified by empty string. This is updated when entering
    # call_module and restored when exiting call_module
    current_module_qualified_name: str = ''
    # A map from FX Node to the qualname of the Module from which it
    # originated. This is recorded by `create_proxy` when recording an
    # operation
    node_to_originating_module: Dict[torch.fx.Node, str] = {}

    def call_module(self, m: torch.nn.Module, forward: Callable[..., Any],
                    args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> Any:
        """
        Override of Tracer.call_module (see
        https://pytorch.org/docs/stable/fx.html#torch.fx.Tracer.call_module).
        This override:
        1) Stores away the qualified name of the caller for restoration later
        2) Installs the qualified name of the caller in `current_module_qualified_name`
           for retrieval by `create_proxy`
        3) Delegates into the normal Tracer.call_module method
        4) Restores the caller's qualified name into current_module_qualified_name
        """
        old_qualname = self.current_module_qualified_name
        try:
            self.current_module_qualified_name = self.path_of_module(m)
            return super().call_module(m, forward, args, kwargs)
        finally:
            self.current_module_qualified_name = old_qualname

    def create_proxy(self, kind: str, target: torch.fx.node.Target, args: Tuple[Any, ...],
                     kwargs: Dict[str, Any], name: Optional[str] = None, type_expr: Optional[Any] = None):
        """
        Override of `Tracer.create_proxy`. This override intercepts the recording
        of every operation and stores away the current traced module's qualified
        name in `node_to_originating_module`
        """
        proxy = super().create_proxy(kind, target, args, kwargs, name, type_expr)
        self.node_to_originating_module[proxy.node] = self.current_module_qualified_name
        return proxy


from django.urls import path,include
from watchlist_app.api.views import WatchListAV,WatchListDetailAV,StreamPlatformListAV,StreamPlatformDetailAV,ReviewList,ReviewDetail,ReviewCreate

urlpatterns = [
	path('list/',WatchListAV.as_view(),name="movie-list"),
    path('<int:pk>',WatchListDetailAV.as_view(),name='movie-detail'),

    path('stream/',StreamPlatformListAV.as_view(),name="stream-platofrmlist"),
    path('stream/<int:pk>',StreamPlatformDetailAV.as_view(),name="stream-platofrmdetail"),

    # path('stream/<int:pk>/review',StreamPlatformDetailAV.as_view(),name="stream-platofrmdetail"),
    # path('stream/review/<int:pk>',ReviewDetail.as_view(),name='review-detail'),
    path('review',ReviewList.as_view(),name='review-list'),
    path('review/<int:pk>/review-create',ReviewCreate.as_view(),name='review-create'),
    path('review/<int:pk>',ReviewDetail.as_view(),name='review-detail'),

]


import logging

from loader import dp
from aiogram.types import Message
from data.config import admins

@dp.message_handler(chat_id=admins, commands=['help_admin'])
async def admin_help(message: Message):
    logging.info(f'from: {message.chat.full_name}, text: {message.text}')
    text = [
        'Список команд администратора: ',
        '/add_meme - добавление нового мема в БД',
        '/cancel_meme - отмена добавления нового мема '
        '\n(можно импользовать на любом шаге добавления)',
        '/del_meme - удаление мема из БД.',
        '/cancel_del_meme - отмена удаления мема.',
        '/mail - рассылка всем пользователям бота.',
        '/cancel_mail - отмена рассылки.',
        '/update_memes - обновление мемов с сайта memepedia.ru'
    ]
    await message.answer('\n'.join(text))


import torch
import time
from utils.time_log import time_since
import os
import sys
from utils.string_helper import *

def preprocess_beam_search_result(beam_search_result, idx2word, vocab_size, oov_lists, eos_idx, unk_idx, replace_unk, src_str_list):
    batch_size = beam_search_result['batch_size']
    predictions = beam_search_result['predictions']
    scores = beam_search_result['scores']
    attention = beam_search_result['attention']
    assert len(predictions) == batch_size
    pred_list = []  # a list of dict, with len = batch_size
    for pred_n_best, score_n_best, attn_n_best, oov, src_word_list in zip(predictions, scores, attention, oov_lists, src_str_list):
        # attn_n_best: list of tensor with size [trg_len, src_len], len=n_best
        pred_dict = {}
        sentences_n_best = []
        for pred, attn in zip(pred_n_best, attn_n_best):
            sentence = prediction_to_sentence(pred, idx2word, vocab_size, oov, eos_idx, unk_idx, replace_unk, src_word_list, attn)
            sentences_n_best.append(sentence)
        pred_dict['sentences'] = sentences_n_best  # a list of list of word, with len [n_best, out_seq_len], does not include tbe final <EOS>
        pred_dict['scores'] = score_n_best  # a list of zero dim tensor, with len [n_best]
        pred_dict['attention'] = attn_n_best  # a list of FloatTensor[output sequence length, src_len], with len = [n_best]
        pred_list.append(pred_dict)
    return pred_list

def predict_with_beam_search(generator, one2many_data_loader, opt, delimiter_word='<sep>'):
    # file for storing the predicted keyphrases
    if opt.pred_file_prefix == "":
        pred_output_file = open(os.path.join(opt.pred_path, "predictions.txt"), "w")
    else:
        pred_output_file = open(os.path.join(opt.pred_path, "%s_predictions.txt" % opt.pred_file_prefix), "w")
    # debug
    interval = 1000
    start_time_prediction = time.time()
    with torch.no_grad():
        start_time = time.time()
        for batch_i, batch in enumerate(one2many_data_loader):
            if (batch_i + 1) % interval == 0:
                print("Batch %d: Time for running beam search on %d batches : %.1f" % (batch_i+1, interval, time_since(start_time)))
                sys.stdout.flush()
                start_time = time.time()
            src, src_lens, src_mask, src_oov, oov_lists, src_str_list, trg_str_2dlist, _, _, _, _, original_idx_list = batch
            """
            src: [batch_size, max_src_len], a LongTensor containing the word indices of source sentences, , with oov words replaced by unk idx
            src_lens: [batch_size], a list containing the length of src sequences for each batch
            src_mask: [batch, max_src_len], a FloatTensor
            src_oov: [batch, max_src_len], a LongTensor containing the word indices of source sentences, contains the index of oov words (used by copy)
            oov_lists: [batch_size], a list of oov words for each src, 2dlist
            """
            src = src.to(opt.device)
            src_mask = src_mask.to(opt.device)
            src_oov = src_oov.to(opt.device)

            beam_search_result = generator.beam_search(src, src_lens, src_oov, src_mask, oov_lists, opt.word2idx, opt.max_eos_per_output_seq)
            pred_list = preprocess_beam_search_result(beam_search_result, opt.idx2word, opt.vocab_size, oov_lists, opt.word2idx[pykp.io.EOS_WORD], opt.word2idx[pykp.io.UNK_WORD], opt.replace_unk, src_str_list)
            # list of {"sentences": [], "scores": [], "attention": []}

            # recover the original order in the dataset
            seq_pairs = sorted(zip(original_idx_list, src_str_list, trg_str_2dlist, pred_list, oov_lists),
                               key=lambda p: p[0])
            original_idx_list, src_str_list, trg_str_2dlist, pred_list, oov_lists = zip(*seq_pairs)

            # Process every src in the batch
            for src_str, trg_str_list, pred, oov in zip(src_str_list, trg_str_2dlist, pred_list, oov_lists):
                # src_str: a list of words; trg_str: a list of keyphrases, each keyphrase is a list of words
                # pred_seq_list: a list of sequence objects, sorted by scores
                # oov: a list of oov words
                pred_str_list = pred['sentences']  # predicted sentences from a single src, a list of list of word, with len=[beam_size, out_seq_len], does not include the final <EOS>

                all_keyphrase_list = []  # a list of word list contains all the keyphrases in the top max_n sequences decoded by beam search
                for word_list in pred_str_list:
                    all_keyphrase_list += split_word_list_by_delimiter(word_list, delimiter_word)
                #pred_str_list = [word_list for word_list, is_keep in zip(all_keyphrase_list, not_duplicate_mask) if is_keep]
                pred_str_list = all_keyphrase_list

                # output the predicted keyphrases to a file
                pred_print_out = ''
                for word_list_i, word_list in enumerate(pred_str_list):
                    if word_list_i < len(pred_str_list) - 1:
                        pred_print_out += '%s;' % ' '.join(word_list)
                    else:
                        pred_print_out += '%s' % ' '.join(word_list)
                pred_print_out += '\n'
                pred_output_file.write(pred_print_out)

    pred_output_file.close()
    total_time_prediction = time_since(start_time_prediction)
    print("done!")
    print(total_time_prediction)


import csv
import matplotlib.pyplot as plt
'''
next()는 두 가지 포맷으로 사용된다.
function 구조로 사용되면 header만 리턴한다.
consumer 구조로 사용되면 data에서 header를 제거한다.

data: [] = list()는 list 타입의 data를 list()로 초기화 시키는 것이다.
단,한 메소드 내에서만 사용하면 로컬에서 초기화한다. 예제는 다음과 같다.
data: [] = None
def save_highest_temperature(self):
    data = list()
그러나 여러메소드에서 사용하면 필드에서 초기화한다. 예제는 다음과 같다.
data:[] = list()

row[날짜,지점,평균기온(℃),최저기온(℃),최고기온(℃)] 최고기온은 -1(뒤에서 첫번째, =4)이다.
'''

class ChangeTemperatureOnMyBirthday():
    data: [] = list()
    highest_temperature: [] = list()
    lowest_temperature: [] = list()

    def processing(self):
        self.read_data()
        self.highest_temperature()
        self.visualize_data()
        self.extract_date_data()

    def read_data(self):
        data = csv.reader(open('data/seoul.csv', 'rt', encoding='UTF-8'))
        next(data)
        # print([i for i in data])
        self.data = data

    def show_highest_temperature(self):
        # print([i[-1] for i in self.data])
        return [i[-1] for i in self.data]

    def save_highest_temperature(self):
        [(self.highest_temperature.append(float(i[-1]))) for i in self.data if i[-1] != '']
        print(f' 총 {len(self.highest_temperature)}개')

    def highest_temperature_my_birthday(self):
        high = []  # 최고기온
        low = []
        for i in self.data:
            if i[-1] != '' and i[-2] != '':
                if 1995 <= int(i[0].split('-')[0]):
                    if i[0].split('-')[1] == '03' and i[0].split('-')[2] == '10':
                        high.append(float(i[-1]))
                        low.append(float(i[-2]))

        plt.rc('font', family='Malgun Gothic')
        plt.rcParams['axes.unicode_minus'] = False
        plt.title("내 생일의 기온 변화 그래프")
        plt.plot(high, 'hotpink',label='high')
        plt.plot(low, 'skyblue', label='low')
        plt.legend()
        plt.show()

    def visualize_data(self):
        plt.plot(self.high, 'hotpink')
        plt.plot(self.low, 'skyblue')
        plt.show()

    def extract_date_data(self):
        '''
        for i in self.data:
            if i[-1] != '':
                if i[0].split('-')[1] == '03' and i[0].split('-')[2] == '10':
                    self.highest_temperature.append(float(i[-1]))
        '''

        [(self.highest_temperature.append(float(i[-1]))) for i in self.data if i[-1] != ''
         if i[0].split('-')[1] == '03' and i[0].split('-')[2] == '10']
        [(self.lowest_temperature.append(float(i[-2]))) for i in self.data if i[-2] != ''
         if i[0].split('-')[1] == '03' and i[0].split('-')[2] == '10']
        print(self.highest_temperature)

if __name__ == '__main__':
    this = ChangeTemperatureOnMyBirthday()
    this.read_data()
    # print([i for i in this.data])
    # this.show_highest_temperature()
    # this.save_highest_temperature()
    this.highest_temperature_my_birthday()
    # this.visualize_data()
    # this.extract_date_data()
    # this.visualize_highest_data()
    # this.visualize_lowest_data()


import pathlib

from bang.UnpackParser import UnpackParser, check_condition
from bang.UnpackParserException import UnpackParserException
from kaitaistruct import ValidationFailedError
from . import android_sparse

class AndroidSparseUnpackParser(UnpackParser):
    extensions = []
    signatures = [
        (0, b'\x3a\xff\x26\xed')
    ]
    pretty_name = 'androidsparse'

    def parse(self):
        try:
            self.data = android_sparse.AndroidSparse.from_io(self.infile)
            self.unpacked_size = self.data.header.len_header
            for chunk in self.data.chunks:
                check_condition(chunk.header.chunk_type in android_sparse.AndroidSparse.ChunkTypes,
                                "invalid chunk type")
                if chunk.header.chunk_type == android_sparse.AndroidSparse.ChunkTypes.raw:
                    check_condition(chunk.header.num_body_blocks * self.data.header.block_size == len(chunk.body),
                                    "not enough data in body")
                elif chunk.header.chunk_type == android_sparse.AndroidSparse.ChunkTypes.fill:
                    check_condition(len(chunk.body) == 4, "wrong body length")
                elif chunk.header.chunk_type == android_sparse.AndroidSparse.ChunkTypes.dont_care:
                    check_condition(len(chunk.body) == 0, "wrong body length")
                self.unpacked_size += chunk.header.len_chunk
        except (Exception, ValidationFailedError) as e:
            raise UnpackParserException(e.args)
        check_condition(self.infile.size >= self.unpacked_size, "not enough data")
        check_condition(self.data.header.version.major == 1, "unsupported major version")
        check_condition(self.data.header.block_size % 4 == 0, "unsupported block size")

    def unpack(self, meta_directory):
        # there is only one file that needs to be unpacked/created

        # this is a temporary name. In case there is an ext2/3/4
        # file system (a typical use case on Android) it might be
        # that there is a volume name embedded in the file.
        file_path = pathlib.Path("sparse.out")
        with meta_directory.unpack_regular_file(file_path) as (unpacked_md, outfile):
            for chunk in self.data.chunks:
                if chunk.header.chunk_type == android_sparse.AndroidSparse.ChunkTypes.raw:
                    outfile.write(chunk.body)
                elif chunk.header.chunk_type == android_sparse.AndroidSparse.ChunkTypes.fill:
                    # Fill data, always length 4
                    for c in range(0, chunk.header.num_body_blocks):
                        # It has already been checked that blk_sz
                        # is divisible by 4.
                       outfile.write(chunk.body*(self.data.header.block_size//4))
                elif chunk.header.chunk_type == android_sparse.AndroidSparse.ChunkTypes.dont_care:
                    for c in range(0, chunk.header.num_body_blocks):
                       outfile.write(b'\x00' * self.data.header.block_size)

            yield unpacked_md

    # make sure that self.unpacked_size is not overwritten
    def calculate_unpacked_size(self):
        pass

    labels = ['android', 'androidsparse']
    metadata = {}