import sys
import types
import inspect
import datetime
import pandas as pd
import time
import shutil
import os

import _utils

from openpyxl import Workbook
from openpyxl import load_workbook

from PyQt5.QtWidgets import QApplication, QWidget, QInputDialog, QLineEdit,\
            QFileDialog, QMessageBox,QPushButton, QComboBox,\
            QLabel, QVBoxLayout, QHBoxLayout, QInputDialog,\
            QGridLayout, QSpacerItem, QSizePolicy, QScrollArea
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtGui import QIcon, QGuiApplication, QColor

path = "./test_data.xlsx"
shName1 = "患者情報入力シート"
shName2 = "患者プルタブシート"

rowValue       = "行番号"
columnValue    = "列番号"
columnItemName = "項目名"
preValue   = "変更前の値"
postValue  = "変更後の値"

class Ui_scrollArea(_utils.basicUtils):
    def __init__(self):
        super().__init__()
        self.row = 3
        self.baseRow = 2
        self.baseCol = 1
        self.colTypeRefRow = 1
        self.colTypeRefLis = [self.tpNumeric, self.tpDate , self.tpDropDown]
        self.filter = []
        self.questionTitle = "confirmation"
        self.errorOccurFlag = False
        now = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")

        self.path = path
        self.pathName = path.split("/")[-1]
        self.pathDir = "./楽々インプット/"
        self.path2Diff = self.pathDir + self.pathName[:-5] + f"_{now}_差分_未.xlsx"
        self.path2DiffFin = self.pathDir + self.pathName[:-5] + f"_{now}_差分_済.xlsx"

        self.index = 1

    def initialize(self, scrollArea):

        self.setupGUI(scrollArea)
        self.readAllValuesFromSheet()

    def readData(self, path):
        self.wb = load_workbook(path)
        self.ws1 = self.wb[shName1]
        self.ws2 = self.wb[shName2]

        self.maxRow = self.ws1.max_row
        self.maxRow2 = self.ws2.max_row
        self.maxBaseRow = self.baseRow
        self.maxColumn = self.ws1.max_column
        self.maxColumn2 = self.ws2.max_column

        # set colNameDic
        self.colNameDic = {}
        self.colNameDic2 = {}
        self.indNameDic = {}
        self.colTypeDic = {}
        for col in range(1,self.maxColumn + 1 ):
            colName = self.readOneValue(self.baseRow, col)
            if colName == self.nonValue or col in self.filter:
                continue
            if colName in self.colNameDic.keys():
                raise Exception(f"{shName1} には、複数の同じ名称のカラムがあります。")
            self.colNameDic[colName] = col

        self.maxBaseRow = self.getMaxRow()

        for k,v in self.colNameDic.items():
            self.indNameDic[v] = k

        for col in range(1, self.maxColumn2 + 1) :
            v = self.ws2.cell(self.baseRow, col).value
            v = self.convert2Str(v)
            if v in self.colNameDic.keys() :
                if v in self.colNameDic2.keys():
                    raise Exception(f"{shName2} には、複数の同じ名称のカラムがあります。")
                self.colNameDic2[v] = col

                tp = self.ws2.cell(self.colTypeRefRow, col).value
                tp = self.convert2Str(tp)
                if tp in self.colTypeRefLis:
                    self.colTypeDic[v] = tp

        self.baseColValue= self.ws1.cell(self.baseRow, self.baseCol).value
        self.diffDic = {rowValue:[], columnValue:[], self.baseColValue:[],
                columnItemName:[], preValue:[], postValue:[] }

    def setupGUI(self,scrollArea):

        scrollArea.setObjectName("scrollArea")
        mainWidth = 800
        mainHeight = 600
        scrollArea.resize(mainWidth, mainHeight)
        scrollArea.setWidgetResizable(True)

        self.scrollAreaWidgetContents = QWidget()
        self.scrollAreaWidgetContents.setObjectName("scrollAreaWidgetContents")

        self.scrollVerticalLayout = QVBoxLayout(self.scrollAreaWidgetContents)
        self.scrollVerticalLayout.setObjectName("scrollVerticalLayout")
        self.verticalLayoutScroll = QVBoxLayout()
        self.verticalLayoutScroll.setObjectName("verticalLayoutScroll")

        self.setupGridLayoutTop()
        self.setupScrollMiddle()
        self.setupGridLayoutBottom()
        self.setupSize()
        scrollArea.setWidget(self.scrollAreaWidgetContents)
        self.retranslateUi(scrollArea)

        QtCore.QMetaObject.connectSlotsByName(scrollArea)

    def changeRowRelated(self):
        reply = self.checkDiffMessage()
        if reply == QMessageBox.No:
            self.spinRow.setValue(self.row)
            return()
        self.readAllValuesFromSheet()

    def readAllValuesFromSheet(self):
        print( inspect.currentframe().f_code.co_name)

        self.row = self.spinRow.value()
        print(f"   row ref number : {self.row} , max row : {self.ws1.max_row}" )

        # change values
        self.changeLineRef1( self.comboRef1.currentText() )
        self.changeLineRef2( self.comboRef2.currentText() )
        self.changeLineSearch( self.comboSearch.currentText() )
        self.changeMacroValues()

        self.compareAllValues()

    def setupGridLayoutTop(self):
        self.gridLayoutTop = QGridLayout()
        self.gridLayoutTop.setObjectName("gridLayoutTop")

        horizontalSpacerTop= QSpacerItem(50, 50, QSizePolicy.MinimumExpanding, QSizePolicy.Minimum)

        # comboRef1
        self.comboRef1 = QComboBox(self.scrollAreaWidgetContents)
        self.comboRef1.setObjectName("comboRef1")
        self.comboRef1.addItems( self.colNameDic.keys() )
        self.comboRef1.activated[str].connect(self.changeLineRef1)

        # comboRef2
        self.comboRef2 = QComboBox(self.scrollAreaWidgetContents)
        self.comboRef2.setObjectName("comboRef2")
        self.comboRef2.addItems( self.colNameDic.keys() )
        self.comboRef2.setCurrentIndex(1)
        self.comboRef2.activated[str].connect(self.changeLineRef2)

        # lineRef1
        self.lineRef1 = QLineEdit(self.scrollAreaWidgetContents)
        self.lineRef1.setObjectName("lineRef1")
        val =  self.getValueFromColumn( self.comboRef1.currentText() )
        self.lineRef1.setText(val)
        self.lineRef1.setEnabled(False)

        # lineRef2
        self.lineRef2 = QLineEdit(self.scrollAreaWidgetContents)
        self.lineRef2.setObjectName("lineRef2")
        val =  self.getValueFromColumn( self.comboRef2.currentText() )
        self.lineRef2.setText(val)
        self.lineRef2.setEnabled(False)

        # comboSearch
        self.comboSearch = QComboBox(self.scrollAreaWidgetContents)
        self.comboSearch.addItems( self.colNameDic.keys() )
        self.comboSearch.setObjectName("comboSearch")
        self.comboSearch.activated[str].connect(self.changeLineSearch)

        # lineSearch
        self.lineSearch = QLineEdit(self.scrollAreaWidgetContents)
        self.lineSearch.setObjectName("lineSearch")
        val =  self.getValueFromColumn( self.comboSearch.currentText() )
        self.lineSearch.setText(val)

        # pushNewRow
        self.pushNewRow = QPushButton(self.scrollAreaWidgetContents)
        self.pushNewRow.setObjectName("pushNewRow")
        self.pushNewRow.clicked.connect(self.addNewLine)
        # pushSearch
        self.pushSearch = QPushButton(self.scrollAreaWidgetContents)
        self.pushSearch.clicked.connect(self.findValue)
        self.pushSearch.setObjectName("pushSearch")

        # labelRow
        self.labelRow = QtWidgets.QLabel(self.scrollAreaWidgetContents)
        self.labelRow.setObjectName("labelRow")

        # spinRow
        self.spinRow = QtWidgets.QSpinBox(self.scrollAreaWidgetContents)
        self.spinRow.setObjectName("spinRow")
        self.spinRow.setValue(self.row)
        self.spinRow.setMinimum(1)
        self.spinRow.setMaximum(100000)
        self.spinRow.valueChanged.connect(self.changeRowRelated)

        # gridLayoutTop
        self.gridLayoutTop.addWidget(self.lineRef1, 0, 1, 1, 1)
        self.gridLayoutTop.addWidget(self.lineRef2, 1, 1, 1, 1)
        self.gridLayoutTop.addWidget(self.comboRef1, 0, 0, 1, 1)
        self.gridLayoutTop.addWidget(self.comboRef2, 1, 0, 1, 1)
        self.gridLayoutTop.addWidget(self.pushSearch, 1, 2, 1, 2)
        self.gridLayoutTop.addWidget(self.pushNewRow, 1, 5, 1, 2)
        self.gridLayoutTop.addWidget(self.labelRow, 0, 5, 1, 1)
        self.gridLayoutTop.addWidget(self.spinRow, 0, 6, 1, 1)
        self.gridLayoutTop.addWidget(self.comboSearch, 0, 2, 1, 1)
        self.gridLayoutTop.addWidget(self.lineSearch, 0, 3, 1, 1)
        self.gridLayoutTop.addItem(horizontalSpacerTop, 0, 4, 1, 1)

        self.verticalLayoutScroll.addLayout(self.gridLayoutTop)

    def setupScrollMiddle(self):
        self.scrollMiddle = QScrollArea(self.scrollAreaWidgetContents)
        self.scrollMiddle.setWidgetResizable(True)
        self.scrollMiddle.setObjectName("scrollMiddle")
        self.scrollMiddle.setMinimumSize(QtCore.QSize(500,300))
        self.scrollAreaWidgetContents_2 = QWidget()
        self.scrollAreaWidgetContents_2.setObjectName("scrollAreaWidgetContents_2")
        self.verticalLayout = QVBoxLayout(self.scrollAreaWidgetContents_2)
        self.verticalLayout.setObjectName("verticalLayout")

        self._horizontals = []
        self._labels = []
        self._comboBoxes = []
        self._lines = []
        self._horizontalSpacers = []

        for colName in self.colNameDic.keys():
            # check set new horizontal layout or not
            self.setupMacros(colName)

        self.scrollMiddle.setWidget(self.scrollAreaWidgetContents_2)
        self.verticalLayoutScroll.addWidget(self.scrollMiddle)

    def setupMacros(self,colName):
        self.horizontalLayoutMacro = QHBoxLayout()
        self.horizontalLayoutMacro.setObjectName("horizontalLayoutMacro")

        # labelMacro
        self.labelMacro = QLabel(self.scrollAreaWidgetContents_2)
        self.labelMacro.setObjectName("labelMacro")
        self.labelMacro.setText( colName )
        self.horizontalLayoutMacro.addWidget(self.labelMacro)

        # lineEdit setting.
        self.lineMacro = QLineEdit(self.scrollAreaWidgetContents_2)
        self.lineMacro.setObjectName("lineMacro")
        self.lineMacro.setStyleSheet(
                "QLineEdit { background-color : white; color:rgb(0,60,60)}")
        self.lineMacro._same = False
        self.lineMacro.colName = colName
        def changeBackLine(obj:object):
            v = obj.text()
            colInd = self.colNameDic[obj.colName]
            origV  = self.readOneValue(self.row, colInd)
            tp = self.colTypeDic.get(obj.colName,None)
            flag   = self.checkType( v,tp )
            if (flag == False) and (self.errorOccurFlag == True):
                if v != origV:
                    obj.setText(origV)
                    v = origV
                    self.typeErrorMessage(tp)

            b = self.compareValue( v , obj.colName )
            obj._same = b
            self.changeColor(obj, b, v)

        self.lineMacro.changeBackLine = types.MethodType(changeBackLine,self.lineMacro)
        self.lineMacro.editingFinished.connect( self.lineMacro.changeBackLine )

        v = self.getValueFromColumn( colName )
        self.lineMacro.setText( v )

        self.horizontalLayoutMacro.addWidget(self.lineMacro)

        # comboBox setting.
        self.comboBoxMacro = QComboBox(self.scrollAreaWidgetContents_2)
        self.lineMacro.combo = self.comboBoxMacro

        self.comboBoxMacro.setObjectName("comboBoxMacro")

        lis_ = self.getItemsFromSheet2(colName)
        self.comboBoxMacro.addItems( lis_ )
        self.comboBoxMacro.colName = colName
        self.comboBoxMacro.defaultItems = lis_
        self.comboBoxMacro.line = self.lineMacro
        # initial color
        self.comboBoxMacro.setStyleSheet("background-color: rgb(255,255,255);color:rgb(0,0,0);")

        def changeBackCombo(obj):
            v = obj.currentText()
            colInd = self.colNameDic[obj.colName]
            origV  = self.readOneValue(self.row, colInd)
            tp = self.colTypeDic.get(obj.colName,None)
            flag   = self.checkType(v, tp)
            if flag == False and self.errorOccurFlag:
                self.setComboIndex(obj, origV)
                v= origV
                self.typeErrorMessage(tp)

            obj.line.setText(v)
            b = self.compareValue( v, obj.colName)
            obj.line._same = b
            self.changeColor(obj, b, v)

        self.comboBoxMacro.changeBackCombo = types.MethodType(changeBackCombo,self.comboBoxMacro)
        self.comboBoxMacro.activated[str].connect(self.comboBoxMacro.changeBackCombo)

        self.setComboIndex(self.comboBoxMacro,v)

        self.horizontalLayoutMacro.addWidget(self.comboBoxMacro)

        # spacer  setting
        horizontalSpacerMacro = QSpacerItem(40, 20, QSizePolicy.Expanding, QSizePolicy.Minimum)
        self.horizontalLayoutMacro.addItem(horizontalSpacerMacro)
        self.verticalLayout.addLayout(self.horizontalLayoutMacro)

        # visible setting
        tp = self.colTypeDic.get(colName, None)
        if tp == self.tpDropDown:
            self.lineMacro.setVisible(False)
        else:
            self.comboBoxMacro.setVisible(False)

        # add each objects.
        self._horizontals.append(self.horizontalLayoutMacro)
        self._labels.append(self.labelMacro)
        self._comboBoxes.append(self.comboBoxMacro)
        self._lines.append(self.lineMacro)
        self._horizontalSpacers.append(self.horizontalLayoutMacro)

    def setupGridLayoutBottom(self):
        self.gridLayoutBottom = QGridLayout()
        self.gridLayoutBottom.setObjectName("gridLayoutBottom")
        horizontalSpacerBottom = QSpacerItem(40, 20, QSizePolicy.Expanding, QSizePolicy.Minimum)

        # label1
        self.labelSh1 = QLabel(self.scrollAreaWidgetContents)
        self.labelSh1.setObjectName("labelSh1")

        # label2
        self.labelSh2 = QLabel(self.scrollAreaWidgetContents)
        self.labelSh2.setObjectName("labelSh2")

        # lineSh1
        self.lineSh1 = QLineEdit(self.scrollAreaWidgetContents)
        self.lineSh1.setObjectName("lineSh1")
        self.lineSh1.setText(shName1)
        self.lineSh1.setEnabled(False)

        # lineSh2
        self.lineSh2 = QLineEdit(self.scrollAreaWidgetContents)
        self.lineSh2.setObjectName("lineSh2")
        self.lineSh2.setText(shName2)
        self.lineSh2.setEnabled(False)

        # pushWrite
        self.pushWrite = QPushButton(self.scrollAreaWidgetContents)
        self.pushWrite.setObjectName("pushWrite")

        self.pushWrite.clicked.connect(self.writeValues)

        # pushCancel
        self.pushCancel = QPushButton(self.scrollAreaWidgetContents)
        self.pushCancel.setObjectName("pushCancel")

        self.pushCancel.clicked.connect(self.changeRowRelated)

        # pushSave
        self.pushSave= QPushButton(self.scrollAreaWidgetContents)
        self.pushSave.setObjectName("pushSave")
        self.pushSave.clicked.connect(self.saveWorkBook)

        # pushQuit
        self.pushQuit = QPushButton(self.scrollAreaWidgetContents)
        self.pushQuit.setObjectName("pushQuit")
        self.pushQuit.clicked.connect(self.quitProcess)

        # pushHelp
        self.pushHelp = QtWidgets.QPushButton(self.scrollAreaWidgetContents)
        self.pushHelp.setObjectName("pushHelp")
        self.pushHelp.clicked.connect(self.showHelp)

        # gridLayoutBottom
        self.gridLayoutBottom.addWidget(self.lineSh1, 0, 1, 1, 1)
        self.gridLayoutBottom.addWidget(self.lineSh2, 1, 1, 1, 1)
        self.gridLayoutBottom.addWidget(self.labelSh1, 0, 0, 1, 1)
        self.gridLayoutBottom.addWidget(self.labelSh2, 1, 0, 1, 1)
        self.gridLayoutBottom.addItem(horizontalSpacerBottom, 0, 2, 1, 1)
        self.gridLayoutBottom.addWidget(self.pushCancel, 1, 4, 1, 1)
        self.gridLayoutBottom.addWidget(self.pushQuit, 1, 5, 1, 1)
        self.gridLayoutBottom.addWidget(self.pushWrite, 0, 4, 1, 1)
        self.gridLayoutBottom.addWidget(self.pushHelp, 1, 3, 1, 1)
        self.gridLayoutBottom.addWidget(self.pushSave, 0, 5, 1, 1)

        self.verticalLayoutScroll.addLayout(self.gridLayoutBottom)
        self.scrollVerticalLayout.addLayout(self.verticalLayoutScroll)

    def setupSize(self):
        sizePolicy1 = QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed)
        sizePolicy1.setHorizontalStretch(0)
        sizePolicy1.setVerticalStretch(0)
        sizePolicy2 = QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Fixed)
        sizePolicy2.setHorizontalStretch(0)
        sizePolicy2.setVerticalStretch(0)
        defaultSize1 = QtCore.QSize(130, 25)
        defaultSize2 = QtCore.QSize(100, 25)
        defaultSize3 = QtCore.QSize(200, 25)

        objectLis1 = [self.comboRef1, self.comboRef2, self.spinRow,
                self.labelSh1, self.labelSh2, self.lineSh1, self.lineSh2]

        objectLis2 = [ self.pushWrite, self.pushCancel, self.pushHelp,
                self.pushSave, self.pushQuit,
                self.lineRef1, self.lineRef2,
                self.comboSearch,self.lineSearch]

        for obj in objectLis1:
            obj.setSizePolicy(sizePolicy1)
            obj.setMinimumSize(defaultSize1)

        for obj in objectLis2:
            obj.setSizePolicy(sizePolicy1)
            obj.setMinimumSize(defaultSize2)

        self.pushNewRow.setSizePolicy(sizePolicy2)

        for i in range( len(self._lines) ) :
            objectLis3 = [self._lines[i], self._labels[i], self._comboBoxes[i]]
            for obj in objectLis3:
                obj.setSizePolicy(sizePolicy2)
                obj.setMinimumSize(defaultSize3)

    def retranslateUi(self, scrollArea):
        _translate = QtCore.QCoreApplication.translate
        scrollArea.setWindowTitle(_translate("scrollArea", "楽々インプット"))
        self.pushNewRow.setText(_translate("scrollArea", "新規追加"))
        self.pushSearch.setText(_translate("scrollArea", "検索"))
        self.pushWrite.setText(_translate("scrollArea", "変更を追加"))
        self.labelRow.setText(_translate("scrollArea", " 行 "))
        self.labelSh1.setText(_translate("scrollArea", "入力先シート"))
        self.labelSh2.setText(_translate("scrollArea", "プルタブ参照シート"))
        self.pushSave.setText(_translate("scrollArea", "変更を保存"))
        self.pushCancel.setText(_translate("scrollArea", "キャンセル"))
        self.pushQuit.setText(_translate("scrollArea", "終了"))
        self.pushHelp.setText(_translate("scrollArea", "ヘルプ"))

    def changeMacroValues(self):
        self.errorOccurFlag = False
        for i in range(len(self._lines) ) :
            colName = self._labels[i].text()
            v = self.getValueFromColumn( colName )

            # line setting
            self._lines[i].setText(v)
            # comboBox setting
            combo = self._comboBoxes[i]
            combo.clear()
            combo.addItems( combo.defaultItems )
            self.setComboIndex(combo, v)

        self.errorOccurFlag = True

    def setComboIndex(self, combo, v:str):
        index = combo.findText(v, QtCore.Qt.MatchFixedString)
        if index == - 1 :
            combo.insertItem(0,v)
            index = 0
        combo.setCurrentIndex(index)

    def checkDiffExist(self):
        flag = False
        for i in range( len(self._lines) ) :
            if not self._lines[i]._same:
                flag = True
                return(flag)
        return(flag)

    def checkDiffMessage(self,exp=None):
        if exp == None:
            exp = "Yes を押すとこのページの変更が破棄されます。"

        flag = self.checkDiffExist()
        reply = QMessageBox.Yes
        if flag :
            print(exp)
            reply = QMessageBox.question(self.scrollAreaWidgetContents, self.questionTitle,exp,
                            QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
        return( reply )

    def typeErrorMessage(self,tp):
        exp = "想定外のエラーです。"
        if tp == self.tpNumeric:
            exp = "整数を入力してください。"
        elif tp == self.tpDate:
            exp = "日付を入力してください。 \nex. 2020/04/01"

        QMessageBox.question(self.scrollAreaWidgetContents, self.questionTitle,exp, QMessageBox.Yes )

    def changeLineRef1(self, s:str) :
        v = self.getValueFromColumn(s)
        self.lineRef1.setText(v)

    def changeLineRef2(self, s:str) :
        v = self.getValueFromColumn(s)
        self.lineRef2.setText(v)

    def changeLineSearch(self, s:str):
        v = self.getValueFromColumn(s)
        self.lineSearch.setText(v)

    def changeColor(self,obj:object, compBool:bool, value:str):
        if compBool:
            if value == self.nonValue or value == self.empty:
                obj.setStyleSheet(
                "background-color : rgb(230,230,255) ; color:rgb(0,60,60)")
            else:
                obj.setStyleSheet(
                "background-color : rgb(255,255,255); color:rgb(0,60,60)")
        else:
            obj.setStyleSheet(
            "background-color : yellow ; color:rgb(0,60,60)")

    def compareValue(self, v:str, colName):
        colInd = self.colNameDic[colName]
        origV = self.readOneValue(self.row, colInd)
        if v == origV:
            return(True)
        else:
            return(False)

    def compareAllValues(self):
        self.errorOccurFlag = False
        for i in range(len(self._lines) ) :
            colName = self._lines[i].colName
            tp = self.colTypeDic.get(colName, None)
            if tp == self.tpDropDown:
                combo = self._comboBoxes[i]
                combo.changeBackCombo()
            else:
                line = self._lines[i]
                line.changeBackLine()
        self.errorOccurFlag = True

    def findValue(self) :
        reply = self.checkDiffMessage()
        if reply == QMessageBox.No:
            return()
        self.readAllValuesFromSheet()

        # find part.
        searchV = self.lineSearch.text()
        colName = self.comboSearch.currentText()
        colInd  = self.colNameDic[colName]

        colNameRef1 = self.comboRef1.currentText()
        colNameRef2 = self.comboRef2.currentText()
        colIndRef1 = self.colNameDic[ colNameRef1 ]
        colIndRef2 = self.colNameDic[ colNameRef2 ]

        findN = 0
        findRow = 1
        findExp = f" 行 ,  {colNameRef1} ,  {colNameRef2} \n"
        for row in range(self.baseRow + 1 ,self.maxRow):
            v = self.readOneValue(row, colInd)
            if v == searchV:
                findN += 1
                findRow = row
                valRef1 = self.readOneValue(row, colIndRef1)
                valRef2 = self.readOneValue(row, colIndRef2)
                findExp += f" {row} ,  {valRef1} ,  {valRef2} \n"

        if findN == 0:
            exp1 = "検索した値は見つかりませんでした。\n"
        elif findN == 1:
            exp1 = "検索した値は見つかりました。\n"
            exp1 += findExp
            self.spinRow.setValue(findRow)
        else:
            exp1 = "検索した値は、複数見つかりました。\n"
            exp1 += findExp
            self.spinRow.setValue(findRow)

        QMessageBox.question(self.scrollAreaWidgetContents,
                        self.questionTitle,exp1, QMessageBox.Yes )
        self.readAllValuesFromSheet()

    def readOneValue(self, row, col):
        v = self.ws1.cell(row,col).value
        v = self.convert2Str(v)
        return(v)

    def getItemsFromSheet2(self, colName):
        index = self.colNameDic2.get(colName, None)
        if index == None:
            return([])
        lis_ = []
        for row in range(self.baseRow + 1 , self.maxRow + 1 ):
            v = self.ws2.cell( row, index).value
            v = self.convert2Str(v)
            if v == self.nonValue:
                return(lis_)
            lis_.append(v)
        return(lis_)

    def getMaxRow(self):
        for row in range(self.maxRow, self.baseRow,-1) :
            v = self.readOneValue(row, self.baseCol)
            if v == self.nonValue or v == self.empty:
                pass
            else:
                return(row)
        return(self.baseRow)

    def getMaxIndex(self):
        max_ = 1
        for row in range(self.baseRow + 1, self.maxBaseRow + 1 ):
            v = self.readOneValue(row, self.baseCol)
            v = self.convertFromStr(v, tp=self.tpNumeric)
            if isinstance(v, int):
                if max_ < v:
                    max_ = v
        return(max_)

    def addNewLine(self):
        reply = self.checkDiffMessage()
        if reply == QMessageBox.No:
            return()

        # add new data
        self.maxBaseRow = self.getMaxRow() + 1
        nextIndex = self.getMaxIndex() + 1

        self.spinRow.setValue( self.maxBaseRow )
        self.row = self.maxBaseRow

        self.changeLineRef1( self.comboRef1.currentText() )
        self.changeLineRef2( self.comboRef2.currentText() )
        self.changeMacroValues()
        _translate = QtCore.QCoreApplication.translate
        # TO DO : If self.baseCol is changed,
        #         this part does not correctly catch the behavior.
        self._lines[0].setText(str(nextIndex) )
        self.compareAllValues()
        self._lines[0].setText(str(nextIndex))

    def getAllItemsCombo(self,combo):
        lis_ = []
        for i in range(combo.count()):
            lis_.append( combo.itemText(i) )
        return( lis_)

    def getValueFromColumn(self,colName):
        colInd = self.colNameDic.get(colName ,None)
        if colInd == None:
            raise Exception("指定されたカラム名はありません。")
            v = "eeeeerrrr"
        else:
            v = self.readOneValue( self.row, colInd)
        return(v)

    def writeValues(self):
        flag = self.checkDiffExist()
        if flag:
            print( inspect.currentframe().f_code.co_name)
            self.addDiff2Dic()

        for i in range(len(self._lines) ) :
            colName = self._labels[i].text()
            colInd  = self.colNameDic[colName]
            tp      = self.colTypeDic.get(colName, None)
            v       = self._lines[i].text()
            v       = self.convertFromStr(v, tp)
            self.ws1.cell(self.row, colInd).value = v
        self.readAllValuesFromSheet()
        self.compareAllValues()

    def addDiff2Dic(self):
        for i in range( len(self._lines) ) :
            line = self._lines[i]
            if not line._same:
                colInd = self.colNameDic[line.colName]
                baseV = self.readOneValue(self.row, self.baseCol)
                origV = self.readOneValue(self.row, colInd)

                tp = self.colTypeDic.get(line.colName, None)
                baseV = self.convertFromStr( baseV , tp)
                origV = self.convertFromStr( origV , tp)
                newV  = self.convertFromStr( line.text(), tp)

                self.diffDic[rowValue].append(self.row)
                self.diffDic[columnValue].append(colInd)
                self.diffDic[self.baseColValue].append(baseV)
                self.diffDic[columnItemName].append(line.colName)
                self.diffDic[preValue].append(origV)
                self.diffDic[postValue].append(newV)

        #print(self.diffDic)
        if not os.path.exists(self.pathDir):
            os.mkdir(self.pathDir)
        with pd.ExcelWriter( self.path2Diff, engine="openpyxl", mode="wa",
                datetime_format='yyyy/mm/dd') as writer:
            df = pd.DataFrame(self.diffDic)
            df.to_excel(writer, sheet_name="差分", index=True)

    def showHelp(self):
        exp = "まだ、ヘルプは実装されていません。"

        reply = QMessageBox.question(self.scrollAreaWidgetContents, self.questionTitle,exp,
                        QMessageBox.Yes )

    def quitProcess(self):
        print( inspect.currentframe().f_code.co_name)
        reply = self.preFinishProcess()
        if reply == QMessageBox.No:
            return()

        if os.path.exists(self.path2Diff):
            exp = "今までの差分は全てファイルに反映されません。\n本当によろしいですか？"

            reply = QMessageBox.question(self.scrollAreaWidgetContents, self.questionTitle,exp,
                            QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
            if reply == QMessageBox.No:
                return()
        QApplication.instance().quit()

    def closeEvent(self, event):
        # dose not work well.
        self.preFinishProcess()
        event.accept()

    def preFinishProcess(self):
        print( inspect.currentframe().f_code.co_name)
        reply = self.checkDiffMessage()
        return(reply)

    def saveWorkBook(self):
        reply = self.checkDiffMessage()
        if reply == QMessageBox.No:
            return()

        if os.path.exists(self.path2Diff):
            st = time.perf_counter()

            pathOutput = path[:-5] + "_temp.xlsx"
            self.wb.save(pathOutput)

            en = time.perf_counter()
            t = en- st
            shutil.move(self.path2Diff, self.path2DiffFin)
            exp = "データを保存して終了します。\n"
            p1 = self.pathName
            p2 = self.path2DiffFin.split("/")[-1]
            exp += f"以下のデータを更新しました。\n{p1}\n"
            exp += f"以下のデータを新規作成しました。\n{p2}\n"
        else:
            exp = "変更はないため、終了します。"

        QMessageBox.question(self.scrollAreaWidgetContents,
                        self.questionTitle,exp, QMessageBox.Yes )
        QApplication.instance().quit()
        print()

def main():
    app = QApplication(sys.argv)

    scrollArea = QScrollArea()
    ui = Ui_scrollArea()
    ui.readData(path)
    ui.initialize(scrollArea)
    scrollArea.show()

    sys.exit(app.exec_())

if __name__ == "__main__":
    main()


#!/usr/bin/env python3

import argparse
import os
import shutil

def add_version(image, path):
    if not os.path.isfile('.env'):
        if os.path.isfile('.env.example'):
            shutil.copyfile('.env.example', '.env')
        else:
            with open('.env', 'w') as f:
                f.write('')

    with open('.env', 'r') as f:
        lines = f.read().split('\n')

    out = []

    found = False

    for line in lines:
        if (not found) and line.startswith(image+'='):
            line = image + '=' + path
            found = True

        if line.strip():
            out.append(line)

    if not found:
        line = image + '=' + path
        out.append(line)

    shutil.copyfile('.env', '.env.old')
    with open('.env', 'w') as f:
        f.write('\n'.join(out))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Add image path to .env')
    parser.add_argument("--file", required=True, type=str, help="Path to txt file")
    parser.add_argument("--name", required=True, type=str, help="Environment variable name")
    args = parser.parse_args()

    add_version(args.name, args.file)


import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns

from sklearn.manifold import TSNE

from tfvae.data_readers import read_encoded_hdf5

nfloats = 49
prefix = 'stargalaxy_sim_20190214'

datafile = prefix + 'encoded_test.hdf5'
images, labels = read_encoded_hdf5(datafile, nfloats)

feat_cols = ['element' + str(i) for i in range(images.shape[1])]
df = pd.DataFrame(images, columns=feat_cols)
df['labels'] = labels
# df['labels'] = df['labels'].apply(lambda i: str(i))

np.random.seed(42)
rndperm = np.random.permutation(df.shape[0])

plt.gray()
fig = plt.figure(figsize=(16, 7))
for i in range(0, 15):
    ax = fig.add_subplot(3, 5, i + 1,
                         title='{}'.format(str(df.loc[rndperm[i], 'labels'])))
    ax.matshow(
        df.loc[rndperm[i], feat_cols].values.reshape((7, 7)).astype(float)
    )
    ax.axis('off')
# plt.show()
plt.savefig('vae_encodeds.pdf', bbox_inches='tight')

N = 10000
df_subset = df.loc[rndperm[:N], :].copy()
data_subset = df_subset[feat_cols].values

tsne = TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300)
tsne_results = tsne.fit_transform(data_subset)

df_subset['tsne-2d-one'] = tsne_results[:, 0]
df_subset['tsne-2d-two'] = tsne_results[:, 1]
plt.figure(figsize=(16, 10))
sns.scatterplot(
    x="tsne-2d-one", y="tsne-2d-two",
    hue="labels",
    palette=sns.color_palette("hls", 2),
    data=df_subset,
    legend="full",
    alpha=0.3
)
# plt.show()
plt.axis('off')
plt.savefig('vae_tsne.pdf', bbox_inches='tight')


from flask import Flask, render_template
from datasource.mysql import MySQLDataSource
import seaborn as sns
import matplotlib.pyplot as plt

app = Flask(__name__)

@app.route('/cohort-analysis')
def show_analysis():
    query1 = """
        SELECT DATE_FORMAT(sign_period,'%Y-%m') AS Period,
            FORMAT((MONTH0/qtd_organizations),2) AS MONTH0,
            FORMAT((MONTH1/qtd_organizations),2) AS MONTH1,
            FORMAT((MONTH2/qtd_organizations),2) AS MONTH2,
            FORMAT((MONTH3/qtd_organizations),2) AS MONTH3,
            FORMAT((MONTH4/qtd_organizations),2) AS MONTH4,
            FORMAT((MONTH5/qtd_organizations),2) AS MONTH5,
            FORMAT((MONTH6/qtd_organizations),2) AS MONTH6,
            FORMAT((MONTH7/qtd_organizations),2) AS MONTH7,
            FORMAT((MONTH8/qtd_organizations),2) AS MONTH8,
            FORMAT((MONTH9/qtd_organizations),2) AS MONTH9
            FROM WzOn7Gbgmv.cohort;
            """

    cohort_table = MySQLDataSource(query1)
    df1 = cohort_table.extract_data_from_database()
    df2 = df1.set_index('Period')
    df2 = df2.astype(float)

    # plot heatmap
    sns.set(style='white')
    plt.figure(figsize=(12, 8))
    plt.title('Cohorts: Organizations Creation')
    ax1 = sns.heatmap(df2, mask=df2.isnull(), annot=True, cmap='Blues',\
                     linewidths=1, fmt='.0%')

    figure1 = ax1.get_figure()
    figure1.savefig('static/images/heatmap.png', dpi = 400)

#-----------------------------------------------
    query2 = """
            SELECT DATE_FORMAT(signDate,'%Y-%m') AS Period,
            		SUM(signedOrganizationsCount) AS TotalSignOrganizations
            FROM accountants AC
            GROUP BY 1
            ORDER BY 1;
            """
    qtd_organization_by_accountants1 = MySQLDataSource(query2)
    qtd_organization_by_accountants2 = qtd_organization_by_accountants1.extract_data_from_database()
    qtd_organization_by_accountants3 = qtd_organization_by_accountants2.set_index('Period')

    # plot heatmap
    sns.set(style='white')
    plt.figure(figsize=(10, 8))
    plt.title('Quantity Signed Organizations/Period')
    ax2 = sns.heatmap(qtd_organization_by_accountants3, \
                      mask=qtd_organization_by_accountants3.isnull(), annot=True, cmap='Blues',\
                      linewidths=1, fmt='.0f')
    figure2 = ax2.get_figure()
    figure2.savefig('static/images/table.png', dpi = 400)

    return render_template('simple.html', message="Hello Flask!")

if __name__ == "__main__":
    app.run(debug = True, host = '0.0.0.0', port = 8080)


import dateutil.parser
import pytz

def normalise_dict(data, defaults):
    """
    Normalise the given data dictionary to have the same structure as defaults dict.
    Logic is;
        a. only include keys in defaults
        b. if key is not set in data, set it to the value in defaults
    :param data:
    :param defaults:
    """

    normalised_data = {}

    if isinstance(data, dict) and isinstance(defaults, dict):

        for key in defaults:

            if key in data:
                normalised_data[key] = data[key]
            else:
                normalised_data[key] = defaults[key]

    return normalised_data

def parse_datetime(datetime_string):
    """
    Parse a date time string, in the format used by ASX.com.au
    :return: Parsed datetime object if valid, None if invalid date
    """

    datetime_parsed = None

    try:

        if datetime_string is not None and len(datetime_string):

            datetime_parsed = dateutil.parser.parse(
                str(datetime_string)
            )

    except ValueError:
        # date parse failed

        datetime_parsed = None

    return datetime_parsed

def format_mysql_datetime(datetime_obj):
    """
    Format datetime to MySQL date & time format YYYY-MM-DD HH:MM;SS
    """

    datetime_string = ''

    if datetime_obj is not None:
        datetime_string = datetime_obj.strftime('%Y-%m-%d %H:%M:%S')

    return datetime_string

def format_mysql_date(datetime_obj):
    """
    Format datetime to MySQL format YYYY-MM-DD
    """

    datetime_string = ''

    if datetime_obj is not None:
        datetime_string = datetime_obj.strftime('%Y-%m-%d')

    return datetime_string


'''
Programme that creates a financial profiles for a group of projects i.e. can produce the portfolio profile or a chosen
set of projects profile.

Output document is an excel wb with four tabs containing the aggregate financial profile for 'latest quarter',
'last quarter', 'baseline 1' (i.e. nearest baseline of approvals) 'baseline 2' (second nearest baselines).

Just data handling for now. No graphics produced. Graph needs to be done manually.

See instructions below.

'''

from openpyxl import Workbook
from analysis.data import cost_list, income_list, year_list, latest_cost_profiles, last_cost_profiles, \
    baseline_1_cost_profiles, baseline_2_cost_profiles, latest_income_profiles, list_of_masters_all, \
    last_income_profiles, baseline_1_income_profiles, baseline_2_income_profiles, dont_double_count, root_path
from analysis.engine_functions import filter_project_group, calculate_group_project_total

def place_in_excel(project_name_list):
    wb = Workbook()

    financial_profile_list = ['Latest Profile', 'Last quarter profile', 'Baseline 1 Profile', 'Baseline 2 Profile']
    cost_profile_data_list = [latest_cost_profiles, last_cost_profiles, baseline_1_cost_profiles, \
                              baseline_2_cost_profiles]
    income_profile_data_list = [latest_income_profiles, last_income_profiles, baseline_1_income_profiles, \
                                baseline_2_income_profiles]

    for p, profile in enumerate(financial_profile_list):
        '''worksheet is created for each project'''
        ws = wb.create_sheet(profile, p)  # creating worksheets
        ws.title = profile  # title of worksheet

        '''place information in each sheet'''
        ws.cell(row=1, column=1).value = 'Project'
        for i, project_name in enumerate(project_name_list):
            '''lists project names in row one'''
            ws.cell(row=1, column=i + 2).value = project_name

            '''iterates through financial dictionary - placing financial data in ws'''
            row_number = 1
            for cost in cost_list:
                for year in year_list:
                    try:
                        ws.cell(row=row_number + 1, column=i+2).value = \
                            cost_profile_data_list[p][project_name][year + cost]
                    except KeyError:
                        pass
                    row_number += 1

        '''places totals in final column'''

        cost_totals = calculate_group_project_total(project_name_list, cost_profile_data_list[p], dont_double_count,
                                                    cost_list, year_list)

        ws.cell(row=1, column=len(project_name_list) + 2).value = 'Total'
        for i, value in enumerate(cost_totals.values()):
            ws.cell(row=i + 2, column=len(project_name_list)+2).value = value

        '''places keys into the chart in the first column'''
        for i, key in enumerate(cost_totals.keys()):
            ws.cell(row=i+2, column=1).value = key

        '''information on which projects are not included in totals'''
        ws.cell(row=1, column=len(project_name_list) + 4).value = 'Projects that have been removed to avoid double counting'
        for i, project in enumerate(dont_double_count):
            ws.cell(row=i + 2, column=len(project_name_list) + 4).value = project

        '''total cost data for output chart'''
        for z, year in enumerate(year_list):
            for x, type in enumerate(cost_list):
                ws.cell(row=z + 41, column=x + 2, value=cost_totals[year + type])

        '''labeling for total figure table '''
        ws.cell(row=40, column=1, value='Year')
        labeling_list_type = ['RDEL', 'CDEL', 'Non-Gov', 'Total']
        for i, label in enumerate(labeling_list_type):
            ws.cell(row=40, column=2 + i, value=label)

        '''labeling of years down the side'''
        for i, label in enumerate(year_list):
            ws.cell(row=41 + i, column=1, value=label)

        '''income total data'''
        income_totals = calculate_group_project_total(project_name_list, income_profile_data_list[p], dont_double_count,
                                                    income_list, year_list)

        for y, year in enumerate(year_list):
            for t, type in enumerate(income_list):
                ws.cell(row=y + 55, column=t + 2, value=income_totals[year + type])

        '''labeling for total figure table '''
        ws.cell(row=54, column=1, value='Year')
        ws.cell(row=54, column=2, value='Income Totals')

        '''labeling of years down the side'''
        for i, label in enumerate(year_list):
            ws.cell(row=55 + i, column=1, value=label)

    return wb

''' RUNNING PROGRAMME'''

'''To run the programme place list of project names into function.
NOTE: Default option is list of all current projects in portfolio. In majority of cases user should just run the
default programme'''
output = place_in_excel(list_of_masters_all[0].projects)
output.save(root_path/'output/portfolio_financial_profile_q4_1920.xlsx')


from net.model import AlexNet
import numpy as np
import torch
import torch.nn.functional as F
import copy
import time

class HCQ(AlexNet):

    def __init__(self, num_class, dropout=0.5):
        super(HCQ, self).__init__(num_class, dropout=dropout)
        self.linkage = {}
        def single_linkage(a, b, enlarge):
            if enlarge:
                a = np.exp(a)
                b = np.exp(b)
            return np.min(b) - np.max(a)
        def complete_linkage(a, b, enlarge):
            if enlarge:
                a = np.exp(a)
                b = np.exp(b)
            return np.max(b) - np.min(a)
        def avg_linkage(a, b, enlarge):
            if enlarge:
                a = np.exp(a)
                b = np.exp(b)
            dist = 0
            for i in a:
                for j in b:
                    dist += abs(j - i)
            return dist / (len(a) * len(b))
        # Use distance of clusters' mean value
        def designed_linkage(a, b, enlarge):
            if enlarge:
                a = np.exp(a)
                b = np.exp(b)
            return np.mean(b) - np.mean(a)
        self.linkage['single'] = single_linkage
        self.linkage['complete'] = complete_linkage
        self.linkage['avg'] = avg_linkage
        self.linkage['designed'] = designed_linkage

    def cal_group_dist(self, clusters, linkage_name, enlarge=False):
        dist_list = []
        for i in range(len(clusters) - 1):
            dist_list.append(self.linkage[linkage_name](clusters[i], clusters[i+1], enlarge))
        return np.array(dist_list)

    def hcq_initialization(self, layer_name, linkage_function, enlarge=False):
        '''
        Initialize HCQ to 8 bits status
        :param layer_name: conv/fc_weight/bias_num
        :param linkage_function: How to compute distance between clusters
        :return:
        '''
        original_weights = self.layers[layer_name].data.numpy().reshape(-1)
        sorted_data = sorted([(idx, value) for idx, value in enumerate(original_weights)], key=lambda i:i[1])
        weights = [[i[1]] for i in sorted_data]
        index = [[i[0]] for i in sorted_data]
        del sorted_data
        total_num = len(index)
        while (len(weights) >= 256):
            weights_dist = self.cal_group_dist(weights, linkage_function, enlarge=enlarge)
            clustering_index = []
            jump = False
            for i in range(len(weights)-1):
                if jump:
                    jump = False
                    continue
                if i == 0:
                    if weights_dist[0] < weights_dist[1]:
                        clustering_index.append([0, 1])
                        jump = True
                elif i == len(weights) - 2:
                    if weights_dist[i] < weights_dist[i-1]:
                        clustering_index.append([i, i+1])
                        jump = True
                elif weights_dist[i] <= weights_dist[i-1] and weights_dist [i] <= weights_dist[i+1]:
                    clustering_index.append([i, i+1])
                    jump = True
            for idx in clustering_index:
                weights[idx[0]] = list(np.concatenate((weights[idx[0]], weights[idx[1]]), axis=0))
                # print(weights[idx[0]], weights[idx[1]])
                index[idx[0]] = list(np.concatenate((index[idx[0]], index[idx[1]]), axis=0))
            weights = np.delete(weights, [i[1] for i in clustering_index])
            index = np.delete(index, [i[1] for i in clustering_index])
        # Generate Code Book according to hcq
        code_book = np.zeros(total_num, dtype=np.uint8)
        changed_weights = np.zeros(total_num)
        for i in range(len(index)):
            code_book[index[i]] = i
        code_book = code_book.reshape(self.layers[layer_name].data.numpy().shape)
        for i in range(len(index)):
            changed_weights[index[i]] = np.mean(weights[i])
        changed_weights = changed_weights.reshape(self.layers[layer_name].data.numpy().shape)
        self.layers[layer_name].data = torch.from_numpy(changed_weights.astype(np.float32))
        return code_book, original_weights.reshape(self.layers[layer_name].data.numpy().shape)

    def set_layers_status(self, status=False):
        '''
        Fix all layers, which immune from gradients update
        :return:
        '''
        for layer in self.layers:
            if 'weight' in layer:
                self.layers[layer].requires_grad=status

    def fine_tune(self, layer_name, code_book, retrain_epoch, learning_rate, train_loader,
                  valid_loader, test_loader, show_interval = 50):
        optimizer = torch.optim.Adam(self.parameters(), lr=learning_rate)
        baseline_valid_acc = self.compute_acc(valid_loader)
        self.set_layers_status(False)
        self.layers[layer_name].requires_grad = True
        start_time = time.time()
        for epoch in range(retrain_epoch):
            for batch_idx, (images, labels) in enumerate(train_loader):
                logits, probas = self.forward(images)
                cost = F.cross_entropy(logits, labels)
                optimizer.zero_grad()
                cost.backward()
                optimizer.step()
                weight = self.layers[layer_name].data.numpy()
                for idx in range(np.max(code_book)+1):
                    centroid = np.sum((code_book == idx) * weight) / np.sum(code_book == idx)
                    # grad = np.sum(self.layers[layer_name].grad.numpy() * (code_book == idx)) / np.sum(code_book == idx)
                    # centroid = centroid - grad * learning_rate
                    weight[code_book == idx] = centroid
                self.layers[layer_name].data = torch.from_numpy(weight.astype(np.float32))
                if not (batch_idx+1) % show_interval:
                    with torch.no_grad():
                        test_acc = self.compute_acc(test_loader)
                        valid_acc = self.compute_acc(valid_loader)
                        print('Epoch: {:0>3d}/{:0>3d} | '
                              'Batch {:0>3d}/{:0>3d} | '
                              'Test Accuracy: {:.2f}% | '
                              'Validation Accuracy: {:.2f}% | '
                              'Accumulated time Consumption: {:.2f} mins'.format(epoch+1, retrain_epoch,
                              batch_idx+1, len(train_loader), test_acc, valid_acc, (time.time() - start_time) / 60))
                        if valid_acc > baseline_valid_acc + 0.5:
                            break
        self.set_layers_status(True)
        # centroids = np.zeros(np.max(code_book) + 1, dtype=np.float32)
        # for idx in range(np.max(code_book) + 1):
        #     centroids[idx] = np.sum((code_book == idx) * self.layers[layer_name].data.numpy()) / \
        #                      np.sum((code_book == idx))
        return code_book

    def quantize_layer_under_acc_loss(self, layer_name, code_book, linkage_name, baseline_acc, valid_loader, enlarge=False):
        '''
        QUantize Layer with target accuracy loss
        :param layer_name:
        :param target_acc:
        :param train_loader:
        :param valid_loader:
        :param test_loader:
        :return:
        '''
        self.layers[layer_name].requires_grad = True
        valid_acc = original_acc = self.compute_acc(valid_loader)
        clusters = []
        cluster_num_list = []
        acc_list = []
        for idx in range(np.max(code_book)+1):
            clusters.append(self.layers[layer_name].data.numpy()[code_book == idx])
        start_time = time.time()
        while valid_acc >= baseline_acc:
            original_code_book = copy.deepcopy(code_book)
            weight_dist = self.cal_group_dist(clusters, linkage_name=linkage_name, enlarge=enlarge)
            idx = int((np.where(weight_dist==np.min(weight_dist)))[0])
            clusters[idx] = list(np.concatenate((clusters[idx], clusters[idx+1]), axis=0))
            code_book[code_book == idx+1] = idx
            clusters = np.delete(clusters, idx+1)
            bias = 0
            # modify code book
            for idx in range(np.max(code_book)+1):
                if np.sum(code_book == idx) == 0:
                    bias += 1
                else:
                    code_book[code_book == idx] = idx - bias
            # update weights
            original_weights = self.layers[layer_name].data.numpy()
            update_weights = np.zeros(code_book.shape)
            for idx in range(np.max(code_book)+1):
                update_weights[code_book == idx] = np.mean(clusters[idx])
            self.layers[layer_name].data = torch.from_numpy(update_weights.astype(np.float32))
            valid_acc = self.compute_acc(valid_loader)
            if valid_acc < baseline_acc:
                self.layers[layer_name].data = torch.from_numpy(original_weights.astype(np.float32))
                code_book = original_code_book
                print('End quantization')
                break
            print('Clusters:{:>3d} | Validation Accuracy:{:.2f}% | '
                  'Accuracy change:{:.2f}% | Accumulated time Consumption: {:.2f} mins'.format(np.max(code_book) + 1, valid_acc.item(),
                                                   (valid_acc - original_acc).item(), (time.time() - start_time) / 60))
            cluster_num_list.append(np.max(code_book))
            acc_list.append(valid_acc.item())
        centroids = np.zeros(np.max(code_book)+1, dtype=np.float32)
        for idx in range(np.max(code_book)+1):
            centroids[idx] = np.sum((code_book == idx) * self.layers[layer_name].data.numpy()) / \
                             np.sum((code_book == idx))
        return code_book, centroids, cluster_num_list, acc_list

    def reduce(self, layer_name, code_book, centroids):
        reduce_weight = np.zeros(code_book.shape, dtype=np.float32)
        for idx in range(len(centroids)):
            reduce_weight[code_book == idx] = centroids[idx]
        self.layers[layer_name].data = torch.from_numpy(reduce_weight.astype(np.float32))

    # def get_centroids(self):
    #


import os
import json
import datetime
import time
import subprocess

ts = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')

p = subprocess.Popen(['netstat','-i'],stdout=subprocess.PIPE)
out,err = p.communicate()
p = subprocess.Popen(['grep','wlan0'],stdout=subprocess.PIPE,stdin=subprocess.PIPE)
out,err = p.communicate(out)
p = subprocess.Popen(['sed','-r','s/wlan0[\\t ]+1500[\\t ]+([0-9]+).+/\\1/'],stdout=subprocess.PIPE,stdin=subprocess.PIPE)
rx,err = p.communicate(out)
p = subprocess.Popen(['sed','-r','s/wlan0[\\t ]+1500[\\t ]+[0-9]+[\\t ]+[0-9]+[\\t ]+[0-9]+[\\t ]+[0-9]+[\\t ]+([0-9]+).+/\\1/'],stdout=subprocess.PIPE,stdin=subprocess.PIPE)
tx,err = p.communicate(out)

rx = int(rx.decode('utf-8'))
tx = int(tx.decode('utf-8'))

rx_json_string = '{"id": "comms.recd","timestamp": "'+ts+'","value":'+str(rx)+'}'
tx_json_string = '{"id": "comms.sent","timestamp": "'+ts+'","value":'+str(tx)+'}'

print(rx_json_string)
print(tx_json_string)


from django.test import TestCase

import httpretty
import json

usage_statistics = [
    {
        "target": "core_count",
        "datapoints": [
            [20.0, 1324216800],
        ]
    },
    {
        "target": "instance_count",
        "datapoints": [
            [10.0, 1324303200],
        ]
    }
]

class AllocationViewTest(TestCase):

    fixtures = ['allocation_requests']
    multi_db = True

    # Web pages

    def test_page_index(self):
        response = self.client.get("/allocations/")
        self.assertEqual(200, response.status_code)

    def test_page_visualisation(self):
        response = self.client.get(
            "/allocations/applications/approved/visualisation")
        self.assertEqual(200, response.status_code)

    def test_page_project(self):
        response = self.client.get(
            "/allocations/applications/1654/approved")
        self.assertEqual(200, response.status_code)

    def test_page_project_allocations(self):
        response = self.client.get(
            "/allocations/applications/1654/approved")
        self.assertEqual(200, response.status_code)

    # Web services with JSON pay loads.

    def test_rest_allocation_tree(self):
        response = self.client.get(
            "/allocations/rest/applications/approved/tree")
        self.assertEqual(200, response.status_code)

    def test_rest_for_code(self):
        response = self.client.get("/allocations/rest/for_codes")
        self.assertEqual(200, response.status_code)

    @httpretty.activate
    def test_rest_for_project_from_allocation_request_id(self):
        httpretty.register_uri(
            httpretty.GET,
            'http://graphite.dev.rc.nectar.org.au/render/',
            body=json.dumps(usage_statistics),
            content_type="application/json")

        response = self.client.get(
            "/allocations/rest/applications/1654/approved")
        self.assertEqual(200, response.status_code)

    @httpretty.activate
    def test_rest_project_summary_response(self):
        httpretty.register_uri(
            httpretty.GET,
            'http://graphite.dev.rc.nectar.org.au/render/',
            body=json.dumps(usage_statistics),
            content_type="application/json")

        response = self.client.get(
            "/allocations/rest/applications/1654/approved")
        result = json.loads(response.content)

        assert result['used_cores'] == 20
        assert result['used_instances'] == 10

    def test_rest_for_project_allocations_from_allocation_request_id(self):
        response = self.client.get(
            "/allocations/rest/applications/1654/history")
        self.assertEqual(200, response.status_code)

    # Site map.

    def test_sitemap(self):
        response = self.client.get(
            "/sitemap.xml")
        self.assertContains(
            response,
            'http://example.com/allocations/applications', 2, 200)


# -*- coding: utf-8 -*-
#
# authors: Vipin
#
# Created on Fri June 7 2019

# -- External Libraries
from django.test import TestCase
from django.contrib.auth.models import User

# -- Local Libraries
from comments.models import Comment

class CommentCreateViewTest(TestCase):

	@classmethod
	def setUpTestData(cls):
		# Set up non-modified objects used by all test methods
		user = User.objects.create(username='testuser')
		user.set_password('testpass')
		user.save()

	def test_response_status_on_post_data(self):
		post_data = {
			"comment_text": "This is comment 1",
			"user": 1,
			"parent": None
		}
		response = self.client.post('/comments/create', data=post_data, content_type='application/json')
		self.assertEqual(response.status_code, 201)

class CommentDetailViewTest(TestCase):

	@classmethod
	def setUpTestData(cls):
		# Set up non-modified objects used by all test methods
		user_two = User.objects.create(username='testuser')
		user_two.set_password('testpass')
		user_two.save()

		# Crating another user
		user_three = User.objects.create(username='anotheruser')
		user_three.set_password('testpass')
		user_three.save()

		# Creating a comment to test GET operation
		comment = Comment.objects.create(comment_text="This is a test comment", user=user_two)
		comment.save()

	def test_response_status_on_get_request(self):
		response = self.client.get('/comments/2')
		self.assertEqual(response.status_code, 200)

	def test_response_data_received_on_get_request(self):
		response = self.client.get('/comments/2')
		data = {
			"id": 2,
			"comment_text": "This is a test comment",
			"user": 2,
			"parent": None,
			"child": []
		}
		self.assertEqual(response.data, data)

	def test_response_status_on_patch_request(self):
		post_data = {
			"comment_text": "This comment is patched",
			"user": 2,
			"parent": None,
		}
		response = self.client.patch('/comments/2', data=post_data, content_type='application/json')
		self.assertEqual(response.status_code, 200)

	def test_response_status_on_patch_request_if_user_does_not_have_permission(self):
		post_data = {
			"comment_text": "This comment is patched",
			"user": 3,
			"parent": None
		}
		response = self.client.patch('/comments/2', data=post_data, content_type='application/json')
		self.assertEqual(response.status_code, 403)

	def test_response_status_on_delete_request_if_user_does_not_have_permission(self):
		post_data = {
			"user": 3,
		}
		response = self.client.delete('/comments/2', data=post_data, content_type='application/json')
		self.assertEqual(response.status_code, 403)

	def test_response_status_on_delete_request(self):
		post_data = {
			"user": 2,
		}
		response = self.client.delete('/comments/2', data=post_data, content_type='application/json')
		self.assertEqual(response.status_code, 204)


#!/usr/bin/env python3
import os
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), 'trim-psbt'))
from bitcoin import psbt
from bitcoin.networks import NETWORKS
# base64 encoding
from binascii import a2b_base64, b2a_base64

def main():
    # parse psbt transaction
    # From sign-psbt.p2wsh.psbt:
    orig_psbt = "cHNidP8BAFICAAAAATrA+nAjorbvWhg+VE1ql6DnmRMibJRuJjHVq1I1b72fAQAAAAD/////AQxKTAAAAAAAFgAUpBDHZKyNoStXigYCfxHgLwe34dMAAAAAAAEAfQIAAAABEw04mQpL5Ny5AZwHaJeEoqjk51OwcCOHc7VnnAD9294AAAAAAP////8C2goOAAAAAAAWABRPf0Rx/SsT/Uez2y+ve3H6+M4a/UBLTAAAAAAAIgAgZchjx1AzHMA++02sohfnklz/h/XpypknVkI0vg1ieOgAAAAAAQErQEtMAAAAAAAiACBlyGPHUDMcwD77TayiF+eSXP+H9enKmSdWQjS+DWJ46AEFi1IhA9FN3PtoF/VXlpW7s+s+GFiHvylCsDHm9xY0O4/n6ejiIQKPzUb4YUssvzGAlpaCQqHiK8+22PK23JOcjCfDR7KTeyEDFay1UBIPTNy0YNXEkICrUIykzNje3srCL+6sjNAX1MEhAisGPuLyL54ZgsFA/neGcsaDERp5bczNvUe7ZePWCKmDVK4AAA=="

    # first convert it to binary
    raw = a2b_base64(orig_psbt)
    # then parse
    tx = psbt.PSBT.parse(raw)

    # Since PSBT has both witness and non-witness input descriptions:
    if not all(x.non_witness_utxo for x in tx.inputs):
        raise ValueError("Expected every input to have non_witness_utxo")
    if not all(x.witness_utxo for x in tx.inputs):
        raise ValueError("Expected every input to have witness_utxo")

    def amount_for(idx):
        """Return satoshis on input {idx}. Uses non-witness inputs."""
        vout = tx.tx.vin[idx].vout
        outp = tx.inputs[idx].non_witness_utxo.vout[vout]
        return outp.value

    # Calculate and display transaction fee.
    input_total = sum(amount_for(idx) for idx in range(len(tx.tx.vin)))
    output_total = sum(out.value for out in tx.tx.vout)

    print("Inputs total", input_total, "sats from", len(tx.tx.vin), "inputs")
    print("Outputs total", output_total, "sats")
    print("Fee equals", input_total - output_total, "sats")

    # print how much we are spending and where
    for out in tx.tx.vout:
        print(out.value,"to",out.script_pubkey.address(NETWORKS["test"]))

    # Corrupt the PSBT by modifying the amount on the first input
    tx.inputs[0].witness_utxo.value = 1

    save_to_file(tx, 'sign-psbt.corrupted-witness-utxo.psbt')

def save_to_file(psbt, filename):
    """Given a PSBT object, write it in base64 to filename."""
    raw = psbt.serialize()
    # convert to base64
    b64_psbt = b2a_base64(raw)
    # somehow b2a ends with \n...
    if b64_psbt[-1:] == b"\n":
        b64_psbt = b64_psbt[:-1]
    # print
    new_psbt = b64_psbt.decode('utf-8')
    print("Creating", filename)
    with open(filename, 'wt') as outfile:
        print(new_psbt, file=outfile)

if __name__ == '__main__':
    main()


'''
This code is based on the one available at https://pypi.org/project/pytorch-categorical/.
We make few change to make it works with torch>1.3.
'''
import torch
import pdb

class Categorical:
    def __init__(
        self, probs, device=None, normalized=False
    ):

        self.validate_probs_type_and_shape(probs)

        # Copy probs tensor; Enforce float32 dtype.  Preserve device.
        self.device = device or probs.device
        self.probs = probs.clone().detach().to(self.device, dtype=torch.float32)

        if not normalized:
            self.probs /= self.probs.sum()

        # Setup the alias outcomes and probabilities.
        self.alias_probs = None
        self.alias_outcomes = None
        self.setup()

    def validate_probs_type_and_shape(self, probs):
        if not isinstance(probs, torch.Tensor):
            raise ValueError("``probs`` should be a tensor.  Got {}.".format(
                type(probs).__name__
            ))
        if len(probs.shape) != 1:
            raise ValueError("``probs`` should be a 1D-tensor. Got {}.".format(
                probs.shape
            ))

    def __len__(self):
        return self.probs.shape[0]

    def setup(self):
        self.alias_probs = self.probs * len(self)
        self.alias_outcomes = torch.zeros(
            (len(self), 2), dtype=torch.long, device=self.device)
        self.alias_outcomes[:,0] = torch.arange(len(self), device=self.device)
        small_idxs, large_idxs = self.split_idxs(
            self.alias_probs, torch.arange(len(self), device=self.device))
        while small_idxs.shape[0] > 0 and large_idxs.shape[0] > 0:

            overlap_size = min(small_idxs.shape[0], large_idxs.shape[0])
            # Available large_idxs get allocated to the secondary
            # outcomes for aliases holding small outcomes
            covered_small = small_idxs[:overlap_size]
            used_large = large_idxs[:overlap_size]
            self.alias_outcomes[covered_small, 1] = used_large

            remaining_small = small_idxs[overlap_size:]
            self.alias_probs[used_large] -= (1-self.alias_probs[covered_small])

            new_small, large_idxs = self.split_idxs(
                self.alias_probs[large_idxs], large_idxs)

            small_idxs = torch.cat((remaining_small, new_small))

    def split_idxs(self, vector, idxs=None):
        is_large = vector > 1
        num_large = is_large.sum()
        sorted_positions = torch.sort(is_large.byte())[1]
        sorted_idxs = idxs[sorted_positions]
        large_idxs = sorted_idxs[-num_large:]
        small_idxs = sorted_idxs[:-num_large]

        return small_idxs, large_idxs

    def slow_setup(self):
        """
        For reference, this is an approach to assembling the alias probabilities
        and alias outcome map without taking advantage of easy parallelization
        using tensor-based operations.
        """
        self.alias_probs = torch.zeros(
            len(self), dtype=self.dtype, device=self.device)
        self.alias_outcomes = torch.zeros(
            (len(self), 2), dtype=torch.long, device=self.device)

        # Sort the data into the outcomes with probabilities
        # that are larger and smaller than 1/K.
        smaller = []
        larger  = []
        for outcome in range(len(self)):
            self.alias_probs[outcome] = len(self) * self.probs[outcome]
            self.alias_outcomes[outcome,0] = outcome
            if self.alias_probs[outcome] < 1.0:
                smaller.append(outcome)
            else:
                larger.append(outcome)
        while len(smaller) > 0 and len(larger) > 0:
            small_outcome = smaller.pop()
            large_outcome = larger.pop()
            self.alias_outcomes[small_outcome, 1] = large_outcome
            self.alias_probs[large_outcome] = (
                self.alias_probs[large_outcome] -
                (1.0 - self.alias_probs[small_outcome])
            )
            if self.alias_probs[large_outcome] < 1.0:
                smaller.append(large_outcome)
            else:
                larger.append(large_outcome)

    def sample(self, sample_shape=(1,), dtype='int64'):
        primary_outcomes = torch.randint(
            0, len(self), sample_shape, dtype=torch.long)
        secondary_outcomes = (
            torch.rand(sample_shape, device=self.device) >
            self.alias_probs[primary_outcomes]
        ).long()
        return self.alias_outcomes[primary_outcomes, secondary_outcomes]


# -*- coding: utf-8 -*-
"""

 Stat 202A 2019 Fall - Homework 02
 Author:
 Date :

 INSTRUCTIONS: Please fill in the corresponding function. Do not change function names,
 function inputs or outputs. Do not write anything outside the function.

"""

import numpy as np

### Part 1 : Metropolis Algorithm

def sample_uniform(size=10000, low=0, high=1):

	"""
	Function 1A: Uniform (Same as HW1)
	Detail : 	This function return a np.ndarray with given size.
	"""
	import random
	X = np.zeros(size)

	m = 2**31 - 1
	a = 7**5
	c = 12345

	# Set the seed using the current system time in microseconds
	import time
	d = int(time.time() * random.random())
	for i in range (0,size):
		d = (a * d + c) % m
		X[i] = d / m * (high - low) + low

	return X

def sample_normal_chain(x0, c, chain_length=100, mean=0, var=1):
	import math
	"""
	Function 1B: Normal by Metropolis
	Detail : 	This function return multiple chains by Metropolis sampling from N(mean, var).
				For every train, the proposal distribution at x is y ~ Uniform[x-c, x+c].
				The input x0 is a 1 dimension np.ndarray.
				Return a np.ndarray X with size x0.shape[0] * chain_length. In X, each row X[i]
				should be a chain and t-th column X[:, t] corresponding to all different X_t.
	# """
	X = np.zeros((x0.shape[0], chain_length))
	eps = np.zeros((x0.shape[0], chain_length))
	probs = np.zeros((x0.shape[0], chain_length))
	X[:, 0] = x0
	for i in range (1, chain_length):
		eps[:,i] = sample_uniform(x0.shape[0], -c, c)
		probs[:,i] = sample_uniform(x0.shape[0], 0, 1)
		y = X[:,i-1] + eps[:,i]
		for j in range (0, x0.shape[0]):
			acceptance = min(1, math.exp(((X[j][i-1]-mean)**2-(y[j]-mean)**2)/(2*var)))
			if probs[j][i] < acceptance:
				X[j][i] = y[j]
			else:
				X[j][i] = X[j][i-1]
	return X

def metropolis_simulation(num_chain=1000, chain_length=100, mean=0, var=1):
	import matplotlib.pyplot as plt
	import matplotlib.animation as animation
	"""
	Function 1C: Simulate metropolis with different setting.
	Detail : 	Try different setting and output movie of histgrams.
	"""

	list_a = [0, -0.1, -1, -10] # Add other value as you want.
	list_b = [1, 0.1, 1, 10]
	list_c = [1, 2]

	for a, b, c in [(a, b, c) for a in list_a for b in list_b for c in list_c]:

		# Sample num_chain x0 from uniform[a,b]
		x0 = sample_uniform(num_chain, a, b)

		# Run Metropolis
		X_normal = sample_normal_chain(x0, c, chain_length, mean, var)

		# Define the graph for i-th frame
		def animate(i):
		    plt.cla()
		    plt.title("Metropolis Sampling (a=%s_b=%s_c=%s) Time: %d" % (a,b,c,i))
		    plt.hist(X_normal[:,i], bins=30)

		# Define the graph for first frame
		fig = plt.figure()
		hist = plt.hist(X_normal[:,0], bins=30)

		# Define animation class and save
		ani = animation.FuncAnimation(fig, animate, frames=chain_length, repeat_delay=3000, repeat=True)
		ani.save('Python_Metropolis_a=%s_b=%s_c=%s.gif' % (a, b, c), writer='imagemagick')

		# Plot movie and save
		# Here plot chain_length graphs, each of them is a histogram of num_chain point.
		# You may use matplotlib.animation and matplotlib.rc to save graphs into gif movies.

# ### Part 2 : Gibbs Sampling

def gibbs_sample(x0, y0, rho, num_chain=1000, chain_length=100, mean=0, var=1):
	"""
	Function 2A: Bivariate normal with correlation rho
	Detail : 	This function return multiple chains by Gibbs sampling
				The input x0, y0, rho, num_chain is a number. This time, we use same starting point.
				Return a np.ndarray X with size num_chain * chain_length * 2. In X, each row X[i]
				should be a chain and t-th column X[:, t] corresponding to all different pair (X_t, Y_t).
	"""
	X = np.zeros(((num_chain, chain_length, 2)))
	random_normal = np.random.normal(mean, var, (num_chain, chain_length, 2))
	X[:,0,0] = x0
	X[:,0,1] = y0
	for i in range (0, num_chain):
		for j in range (1, chain_length):
			X[i][j][0] = rho * X[i][j-1][1] + (1-rho**2)**(1/2)*random_normal[i][j][0]
			X[i][j][1] = rho * X[i][j][0] + (1-rho**2)**(1/2)*random_normal[i][j][1]
	return X

def gibbs_simulation():
	import matplotlib.pyplot as plt
	import matplotlib.animation as animation
	"""
	Function 2B: Simulate Gibbs with different rho and plot
	Detail : 	Try different setting and output movie of histgrams.
				Discard first 50 steps and output 50~100 steps only.
	"""
	list_rho = [0, -1, 1, -0.5, 0.5, -0.25, 0.25, -0.75, 0.75] # Add other value as you want.
	for rho in list_rho:
		X_gibbs = gibbs_sample(1, 1, rho)
		fig = plt.figure('Gibbs_Sample_Sccaterplot_rho=%s' % rho, figsize = (6,6)).add_subplot(111)
		fig.scatter(X_gibbs[1,50:,0],X_gibbs[1,50:,1])
		plt.savefig ("Python_Gibbs Single Chain Scatterplot(rho=%s).png" % (rho))

		def animate(i):
		    plt.cla()
		    plt.title("Gibbs 1000 Chain (rho=%s) Time: %d" % (rho,i))
		    plt.scatter(X_gibbs[:,i,0],X_gibbs[:,i,1])

		# Define the graph for first frame
		fig = plt.figure()
		hist = plt.scatter(X_gibbs[:,0,0],X_gibbs[:,0,1])

		# Define animation class and save
		ani = animation.FuncAnimation(fig, animate, frames=100, repeat_delay=3000, repeat=True)
		ani.save('Python_Gibbs_rho=%s.gif' % (rho), writer='imagemagick')
		# Run Gibbs Sampling

# def main():
# 	metropolis_simulation()
# 	gibbs_simulation()
# main()


# -*- coding: utf-8 -*-
"""
Created on Sat Mar 04 20:38:51 2017

@author: Pericle
"""

import binascii
import os
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.primitives import hashes, hmac
from cryptography.hazmat.backends import default_backend

# Cipher block size, expressed in bytes.
block = algorithms.AES.block_size/8

# Read the plaintext file.
filename = raw_input('Type the file to encrypt-then-MAC: ')
with open(filename, 'rb') as f:
    plaintext = f.read()

# Read the keys in hexadecimal digits from keyboard.
key_hex = raw_input('Type the encryption key in ' + str(2*block) + ' hexadecimal digits: ')
cryptkey = binascii.unhexlify(key_hex)
key_hex = raw_input('Type the authentication key in ' + str(2*block) + ' hexadecimal digits: ')
authkey = binascii.unhexlify(key_hex)

# Pad the plaintext to make it multiple of the block size.
ctx = padding.PKCS7(8*block).padder()
padded_plaintext = ctx.update(plaintext) + ctx.finalize()

# Encrypt the plaintext with a random IV.
iv = os.urandom(block)
cipher = Cipher(algorithms.AES(cryptkey), modes.CBC(iv), default_backend())
ctx = cipher.encryptor()
ciphertext = ctx.update(padded_plaintext) + ctx.finalize()

# Authenticate the ciphertext and the IV.
ctx = hmac.HMAC(authkey, hashes.SHA256(), default_backend())
ctx.update(iv)
ctx.update(ciphertext)
digest = ctx.finalize()

# Write the MAC, the IV, and the ciphertext in the output file.
with open(filename + '.enc', 'wb') as f:
    f.write(digest)
    f.write(iv)
    f.write(ciphertext)
print('Encrypt-then-MACed file: ' + filename + '.enc')


from django.conf.urls import patterns, url
from django.views.decorators.csrf import csrf_exempt
from . import views

urlpatterns = patterns('',
    url(r'^redirect/$', views.RedirectView.as_view(), name='sofort-redirect'),
    url(
        r'^return/$',
        views.SuccessResponseView.as_view(preview=True),
        name='sofort-success-response',
    ),
    url(
        r'^place-order/(?P<basket_id>\d+)/$',
        views.SuccessResponseView.as_view(),
        name='sofort-place-order',
    ),
    url(
        r'^cancel/(?P<basket_id>\d+)/$',
        views.CancelResponseView.as_view(),
        name='sofort-abort-response',
    ),
    url(
        r'^notify/$',
        csrf_exempt(views.NotificationView.as_view(preview=True)),
        name='sofort-notify'
    ),
)


from itertools import combinations
from queue import deque

AMOUNT_LINES = 18 # количесвто линий, подверженных КЗ

 # получение всевозможных позиций растановки КА в зависимочти от их количесвта
def combinationLines(amount_KA, amountLines = AMOUNT_LINES):
    positKA = [i for i in range(2, amountLines + 1)]  #
    combPosit = list(combinations(positKA, amount_KA))
    combinat_list = [list(item) for item in combPosit]
    return combinat_list

# -------------------- данные для схемы (линии электропередачи) ----------------
 # граф для линий (пока не нужен)
graph = {1: [2, 3],          2: [1],           3: [1, 4, 5],  4: [3],           5: [3, 6, 7],    6: [5],
         7: [5, 8, 11, 14],  8: [7, 9, 10],    9: [8],       10: [8],          11: [7, 12, 13], 12: [11],
        13: [11],            14: [7, 15, 16], 15: [14],      16: [14, 17, 18], 17: [16],        18: [16]}

 # матрица сопряженности линий (1 - если расположения КА на данных позициях
 #  будут влиять на функционирование друг друга
              # 1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18
matrixGraph = [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],  # 1
               [0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],  # 2
               [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 3
               [0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],  # 4
               [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 5
               [0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],  # 6
               [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 7
               [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1],  # 8
               [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0],  # 9
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0],  # 10
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1],  # 11
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0],  # 12
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],  # 13
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0],  # 14
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0],  # 15
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0],  # 16
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],  # 17
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]]  # 18

 # для каждой позиции опаисано количество следующих за ней линий
dictProtectLines = {1: 18, 2: 16, 3: 1, 4: 14, 5: 1, 6: 12,
                    7: 1, 8: 5, 9: 3, 10: 3, 11: 3, 12: 1,
                    13: 1, 14: 1, 15: 1, 16: 1, 17: 1, 18: 1}

# ------------------------------------------------------------------------------

# количесвто защищаемых потребителей для каждой позиции КА
def functProtectConsumer(dictAmountConsumer):
    dictPositConsumer = {1: 0, 2: 0,
                         3: dictAmountConsumer[0], 4: 0,
                         5: dictAmountConsumer[2], 6: 0,
                         7: dictAmountConsumer[1], 8: 0,
                         9: 0, 10: 0,
                         11: 0, 12: dictAmountConsumer[7],
                         13: dictAmountConsumer[6], 14: dictAmountConsumer[5],
                         15: dictAmountConsumer[4], 16: dictAmountConsumer[3],
                         17: dictAmountConsumer[9], 18: dictAmountConsumer[8]
                         }

    arrayProtectConsumer = []
    for posKA1 in range(0, AMOUNT_LINES):
        protectConsumer = 0
        for posKA2 in range(0, AMOUNT_LINES):
            if matrixGraph[posKA1][posKA2] == 1:
                protectConsumer += dictPositConsumer[posKA2 + 1]
        arrayProtectConsumer.append(protectConsumer)
    return arrayProtectConsumer

# Функция нахождения зависимых линий. На выходе - зависмые линии для конкретного расположения
def lineDepend(arrayPos, matrixLine=matrixGraph):
    lineDependReturn = []
    for posJ in range(len(arrayPos)):
        sliceArray = arrayPos[posJ : len(arrayPos)]
        workArray = []
        if len(sliceArray) > 1:
            for i in range(1, len(sliceArray)):
                row = sliceArray[0] - 1
                col = sliceArray[i] - 1
                if matrixLine[row][col] == 1: # если линии зависимы, то оставляем
                    workArray.append(sliceArray[i])

            if len(workArray) > 1:
                delIt = 1
                array1 = workArray
                array3 = []
                while(delIt < len(array1)):
                    array2 = []
                    array3.append(array1[delIt-1])
                    for j in range(delIt, len(array1)):
                        row = array1[delIt-1]-1
                        col = array1[j] - 1
                        if matrixLine[row][col] == 0:  # если линии независимы, то оставляем
                            array2.append(array1[j])
                    array1 = array3+array2
                    delIt += 1
                workArray = array1
        lineDependReturn.append([sliceArray[0]]+workArray)

    return lineDependReturn

# функция подсчета защищаемых КА линий в зависимости от расположения
def lineProtect(dependLines, dictProtectLines):
    lineProtectReturn = []
    for arrPos in dependLines:
        protectLinesPosit = dictProtectLines[arrPos[0]]
        if len(arrPos) > 1:
            for i in range(1, len(arrPos)):
                protectLinesPosit -= dictProtectLines[arrPos[i]]
        lineProtectReturn.append(protectLinesPosit)

    if len(lineProtectReturn) != len(dependLines):
        print("lineProtect(): Количесвто значений в входном массиве не равно в выходном")
    else: return lineProtectReturn

 # рассчитывается количество линий, влияющих на функционирование КА на соответсвующей позиции
def lineInfluence(positions, protectLines, matrixLine=matrixGraph):
    #arrayInfl = []
    #arrayInfl.append(protectLines[0])
    #for i in range(1, len(positions)):
    #    for j in range(i):
    #        row = positions[i-j-1] - 1
    #        col = positions[i] - 1
    #        if matrixLine[row][col] == 1:  # если линии зависимы, то
    #            arrayInfl.append(protectLines[i] + arrayInfl[i-j-1])
    #            break
    #arrayInfl[0] = AMOUNT_LINES
    arrayInfl = [AMOUNT_LINES] * len(positions)
    return arrayInfl

 # рассчитывается количество защищаемых потребителей при установке КА на соответсвующей позиции
def consumerProtect(positions, arrayProtectConsumer):
    returnConsumerProtect = []
    for pos in positions:
        returnConsumerProtect.append(arrayProtectConsumer[pos-1])
    return returnConsumerProtect

 # рассчитывается математическое ожидание
def mathExpect(protectLines, influenceLines, consumers):
    me = 0
    for i in range(len(protectLines)):
        me += (protectLines[i] / influenceLines[i]) * consumers[i]
    return me

 # математическая модель для рассчета при установке N КА
def modelNka(nKA, arrayProtectConsumer, dictProtectLines=dictProtectLines):
     # формирование всевозможных позиций расположения КА (без повторений)
    combinatPositions = combinationLines(nKA)
    meMin = 1000000000
    optimalPositions = []
    for positions in combinatPositions:
        positions.append(1) # для расчета линий, незащищаемых ни одной КА
        positions.sort()    # сортировка по уровням графа линий
        #print(positions)
         # для каждой позиции находятся другие позиции КА, которые будут влиять на количесвто защищаемых ей линий
        dependLines = lineDepend(positions)
         # для каждой позиции рассчитываются соответсвующее количесвто защищаемых линий
        protectLines = lineProtect(dependLines, dictProtectLines)
        #print(protectLines)
         # для каждой позиции рассчитывается количесвто линий, влияющих на работу КА
        influenceLines = lineInfluence(positions, protectLines)
        #print(influenceLines)
         # для каждой позиции рассчитывается соответсвующее количесвто защищаемых потребителей
        consumers = consumerProtect(positions, arrayProtectConsumer)
        #print(consumers)
         # расчет мат ожидания
        me = mathExpect(protectLines, influenceLines, consumers)
        if me < meMin:
            optimalPositions.clear()
            meMin = me
            optimalPositions.append(positions[1:])
        elif me == meMin: optimalPositions.append(positions[1:]) # если оптимальных позиций окажется несколько

    return (meMin, nKA, optimalPositions)

 # рассчет для всех 1:N установленных КА
def calculationFrom1toNka(amountKA, numberCustomersTP):
    arrayCalculatData = []
    for i in range(1, amountKA+1):
        arrayCalculatData.append(modelNka(i, functProtectConsumer(numberCustomersTP)))
    return arrayCalculatData

#print(modelNka(17, functProtectConsumer([30, 20, 10, 5, 15, 40, 10, 30, 5, 10])))


#Smayan Daruka
#CSEC-380
#Professor Chaim Sanders
#ACT4 Script 1 - act4ste1.py

import requests

def main():
    req = requests.get('https://csec.rit.edu')
    print(req.text)

if __name__ == '__main__':
    main()


import math
import scipy.stats as stats
import pandas as pd
import numpy as np

df = pd.read_csv('data/house_pricing.csv')
sample = df['GrLivArea']
z_critical = stats.norm.ppf(q = 0.95)

def confidence_interval(sample):
    sample_mean = sample.mean()
    samplestd = sample.std()
    n = np.size(sample)
    stderror = samplestd/(n**(1/2))
    marginerror = z_critical*stderror
    lower = sample_mean - marginerror
    upper = sample_mean + marginerror

    return lower, upper

confidence_interval(sample)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
 This script evaluates all embeddings available in the package
 and saves .csv results

 Usage:

 ./evaluate_all_embeddings -o <output_dir>
"""

# external imports
# ---

from six import iteritems
from multiprocessing import Pool
from os import path
import logging
import optparse
import multiprocessing

# internal imports
# ---
from web.evaluate import evaluate_on_all
from web import embeddings

parser = optparse.OptionParser()

parser.add_option("-j", "--n_jobs", type="int", default=4)

parser.add_option("-o", "--output_dir", type="str", default="")

(opts, args) = parser.parse_args()

# Configure logging

logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%I:%M:%S')

logger = logging.getLogger(__name__)

job_selection = []

job_selection.append('conceptnet')
# job_selection.append('FastText')
# job_selection.append('PDC')
# job_selection.append('HDC')
# job_selection.append('GloVe')
# job_selection.append('SG_GoogleNews')
# job_selection.append('LexVec')

# WARNING
# NMT embeddings are no longer available through the provided url
# job_selection.append('NMT')

jobs = []

# ConceptNet Numberbatch
# ---

if 'conceptnet' in job_selection:

    jobs.append(["fetch_conceptnet_numberbatch", {}])

# FastText
# ---

if 'FastText' in job_selection:

    jobs.append(["fetch_FastText", {}])

# PDC and HDC
# ---

for dim in [50, 100, 300]:

    if 'PDC' in job_selection:

        jobs.append(["fetch_PDC", {"dim": dim}])

    if 'HDC' in job_selection:

        jobs.append(["fetch_HDC", {"dim": dim}])

# GloVe
# ---

if 'GloVe' in job_selection:

    for dim in [50, 100, 200, 300]:

        jobs.append(["fetch_GloVe", {"dim": dim, "corpus": "wiki-6B"}])

    for dim in [25, 50, 100, 200]:

        jobs.append(["fetch_GloVe", {"dim": dim, "corpus": "twitter-27B"}])

    for corpus in ["common-crawl-42B", "common-crawl-840B"]:

        jobs.append(["fetch_GloVe", {"dim": 300, "corpus": corpus}])

# SG
# ---
if 'SG_GoogleNews' in job_selection:

    jobs.append(["fetch_SG_GoogleNews", {}])

# LexVec
# ---

if 'LexVec' in job_selection:

    jobs.append(["fetch_LexVec", {}])

# NMT
# ---

if 'NMT' in job_selection:

    # WARNING
    # NMT embeddings are no longer available through the provided url

    jobs.append(["fetch_NMT", {"which": "FR"}])
    jobs.append(["fetch_NMT", {"which": "DE"}])

def run_job(job):

    fetch_function_name, kwargs = job

    embedding_name = fetch_function_name.split("_")[1]

    filename = embedding_name

    if kwargs:

        job_description = "_" + "_".join(str(k) + "=" + str(v)
                                         for k, v in iteritems(kwargs))

        filename += job_description

    filename += ".csv"

    outf = path.join(opts.output_dir, filename)

    logger.info("Processing " + outf)

    if not path.exists(outf):

        w = getattr(embeddings, fetch_function_name)(**kwargs)

        res = evaluate_on_all_fast(w)

        res.to_csv(outf)

if __name__ == "__main__":

    Pool(opts.n_jobs).map(run_job, jobs)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.mixins import PermissionRequiredMixin
from django.core.exceptions import ImproperlyConfigured
from django.shortcuts import redirect
from django.utils.decorators import method_decorator
from .decorators import verified_email_required
from .models import OrganizingHubFeature
import logging
logger = logging.getLogger(__name__)

@method_decorator(verified_email_required, name='dispatch')
class LocalGroupPermissionRequiredMixin(PermissionRequiredMixin):
    """
    Local Group based Permissions check with Email verification
    """
    local_group = None
    organizing_hub_feature = None
    skip_feature_check = False

    def get_local_group(self):
        """
        Override this method to override the local_group attribute.
        """
        if self.local_group is None:
            raise ImproperlyConfigured(
                '{0} is missing the local_group attribute. Define {0}.local_group, or override '
                '{0}.get_local_group().'.format(self.__class__.__name__)
            )
        return self.local_group

    def handle_no_permission(self):
        """
        Override default logic and redirect to Dashboard instead
        """
        messages.error(
            self.request,
            "You do not have permission to access this page."
        )
        return redirect(settings.ORGANIZING_HUB_DASHBOARD_URL)

    def has_feature_access(self):
        """Return True if we are skipping feature check"""
        if self.skip_feature_check:
            return True

        """Throw error if feature is missing"""
        if self.organizing_hub_feature is None:
            raise ImproperlyConfigured(
                '{0} is missing the organizing_hub_feature attribute. Define {0}.organizing_hub_feature, or override '
                '{0}.skip_feature_check.'.format(self.__class__.__name__)
            )

        """Check if Local Group has access to Feature"""
        local_group = self.get_local_group()
        if local_group is not None and hasattr(
            local_group,
            'organizinghubaccess',
        ):
            access = local_group.organizinghubaccess
            if access.has_feature_access(self.organizing_hub_feature):
                return True

        """Otherwise return False"""
        return False

    def has_permission(self):
        """
        Override this method to customize the way permissions are checked.
        """
        local_group = self.get_local_group()
        permissions = self.get_permission_required()

        """Return False for missing or non-approved local group"""
        if local_group is None or local_group.status != 'approved':
            return False

        """Return False for missing feature access"""
        if not self.has_feature_access():
            return False

        """Return True for empty permission list"""
        if not permissions:
            return True

        """Otherwise check permissions against Local Group Profile"""
        user = self.request.user
        if not hasattr(user, 'localgroupprofile'):
            return False
        else:
            profile = user.localgroupprofile
            has_permissions = profile.has_permissions_for_local_group(
                local_group,
                permissions
            )
            return has_permissions


from scrapy.spiders import Rule
from scrapy.linkextractors import LinkExtractor
from tutorial.items import LiePinItem3
from scrapy_redis.spiders import RedisCrawlSpider

class MyCrawler(RedisCrawlSpider):
    """Spider that reads urls from redis queue (myspider:start_urls)."""
    name = 'mycrawler_redis'
    redis_key = 'mycrawler:start_urls'

    rules = (
        Rule(LinkExtractor(allow=r'/zhaopin/\?init=-1.*?&curPage=\d+')),
        Rule(LinkExtractor(allow=r'https://www.liepin.com/job/\d+.shtml'), callback='parse_item'),
        Rule(LinkExtractor(allow=r'/a/\d+.shtml'), callback='parse_item2'),
        # 配置url提取的规律，注意不是数据的提取规律  要求所有的列表页以及所有的列表页上职位的详情页
        # 问题：它是怎么来实现你给了一个初始的url地址，怎么就找到了其它的列表页的url地址以及职位详情页的url地址
    )

    # 适合：网页的数据就在源代码里面，而且还要有下一页。
    # 怎么爬
    # 给一个起始的url地址
    # 再配置提取的url规则

    def parse_item(self, response):
        item = LiePinItem3()
        item["job_name"] = response.xpath("//div[@class='title-info']/h1/@title").extract()[0]
        item["com_name"] = response.xpath("//div[@class='title-info']/h3/a/@title").extract()[0]
        item["salary"] = response.xpath("//p[@class='job-item-title']/text()").extract()[0].replace("\r", "").replace(
            "\n", "")
        addr = response.xpath("//p[@class='basic-infor']/span/a/text()").extract()
        item["address"] = addr[0] if len(addr) > 0 else "没有公司地址，是一家黑公司"
        item["xueli"] = response.xpath("//div[@class='job-qualifications']/span[1]/text()").extract()[0]
        item["work_year"] = response.xpath("//div[@class='job-qualifications']/span[2]/text()").extract()[0]
        miaoshu = ""
        miaoshu_data = response.xpath("//div[@class='content content-word']/text()").extract()
        for ii in miaoshu_data:
            miaoshu += ii.replace("\r", "").replace("\n", "").strip()
        item["miaoshu"] = miaoshu

        return item

    def parse_item2(self, response):
        item = LiePinItem3()
        item["job_name"] = response.xpath("//div[@class='title-info ']/h1/text()").extract()[0]
        item["com_name"] = response.xpath("//div[@class='title-info ']/h3/text()").extract()[0]
        item["salary"] = response.xpath("//p[@class='job-main-title']/text()").extract()[0].replace("\r", "").replace(
            "\n", "")
        item["address"] = response.xpath("//p[@class='basic-infor']/span/text()").extract()[0]
        item["xueli"] = response.xpath("//div[@class='resume clearfix']/span[1]/text()").extract()[0]
        item["work_year"] = response.xpath("//div[@class='resume clearfix']/span[2]/text()").extract()[0]
        miaoshu = ""
        miaoshu_data = response.xpath("//div[@class='content content-word']/text()").extract()
        for ii in miaoshu_data:
            miaoshu += ii.replace("\r", "").replace("\n", "").strip()
        item["miaoshu"] = miaoshu

        return item


import json
import urllib.request
import pymongo

# connect to mongo
connection = pymongo.MongoClient("mongodb://localhost")

# get a handle to jobs db
db = connection.jobs

# get a handle to dice collection
pythonJobsWarren = db.pythonJobsWarren

# drop existing collection
pythonJobsWarren.drop()

# get the dice jobs
dice_jobs = urllib.request.urlopen("http://service.dice.com/api/rest/jobsearch/v1/simple.json?text=python&city=07059")
parsed = json.loads(dice_jobs.read())

# iterate and insert jobs
for item in parsed['resultItemList']:
    pythonJobsWarren.insert_one(item)

# for paginated results
while parsed['count'] != parsed['lastDocument']:

    # get nextUrl for paginated results
    nextURL = parsed['nextUrl']
    dice_jobs = urllib.request.urlopen("http://service.dice.com" + nextURL)

    parsed = json.loads(dice_jobs.read())
    for item in parsed['resultItemList']:
        pythonJobsWarren.insert_one(item)


from seekr.fasta_reader import Reader
import numpy as np
import corefunctions
import argparse
import pickle
from tqdm import tqdm as tqdm
from scipy.special import logsumexp
from collections import defaultdict
import os

'''
Author: Daniel Sprague
Calabrese Lab, Department of Pharmacology UNC Chapel Hill
June 24, 2020

Purpose: Run the Baum-Welch MLE algorithm to identify putative transition parameters for mSEEKR given a set of training sequences.

Input:

-k : K-mer value to use
--db : Training sequences
--prior : Path to binary file output by train.py in the mSEEKR pipeline, essentially, initial guesses for the transition parameters,
    as well as trained values for the k-mer frequencies for query and null states

    e.g. markovModels/D_null/2/hmm.mkv

--its : How many iterations of the BW algorithm to run (default=100)

Output:

    A new .mkv python pickle file (binary) that contains the original k-mer frequencies and updated transition matrix

'''

# Initiate command line arguments

parser = argparse.ArgumentParser()
parser.add_argument("-k",type=int)
parser.add_argument('--db',type=str,help='Path to fasta file containing training sequences')
parser.add_argument('--prior',type=str,help='Path to binary .mkv file output from train.py (e.g. markovModels/D_null/2/hmm.mkv')
parser.add_argument('-cf','--createfile',action='store_true',help='Create new file rather than overwrite')
parser.add_argument('--its',type=int,help='Iterations to do, default=100',default=20)

args = parser.parse_args()

assert args.its > 0, 'Please provide an integer greater than or equal to 1'
assert args.k > 0, 'Please provide an integer greater than or equal to 1'

fa = Reader(args.db)
seqs = '$'.join(fa.get_seqs())
model = args.prior

k = args.k

# Identify the location of any ambiguous nucleotides (N)
O,oIdx,nBP = corefunctions.kmersWithAmbigIndex(seqs,k)
# Load in train.py output
hmm = pickle.load(open(args.prior,'rb'))

'''
A - transition matrix (dictionary)
E - emission matrix (dictionary)
pi - initial state probabilities (always 50/50)
states - list of states (query,null)
'''
A,E,pi,states = hmm['A'],hmm['E'],hmm['pi'],hmm['states']

data = defaultdict(list)
data['alpha'].append(A['+']['+'])
data['beta'].append(A['-']['-'])
# Perform --its number of baum-welch iterations
# BW converges to a LOCAL MINIMUM -- best to perform a grid search (run this script on a variety of different --priors)
for i in tqdm(range(args.its)):
    a = corefunctions.fwd(O,A,pi,states,E) # forward probabilities
    b = corefunctions.bkw(O,A,pi,states,E) # backwards probabilities
    A = corefunctions.update(a,b,O,states,A,E) # update parameter values

    # The above step yields probabilities that are not yet normalized (do not sum to 1)
    # That is, for prior state "i" (can be either query or null) transitioning to either query (+) or null (-): p(-|i) + p(+|j) != 1
    # Here, we calculate the marginal probability for transitioning from state i to either state i or j (normalizing constant)
    # Then, divide each probability p(-|i) and p(+|i) by the marginal probability
    # Now, p(-|i) + p(+|i) = 1
    for i in states:
        # marginal probability (normalizing constant)
        marg = logsumexp(list(A[i].values())) # cannot simply sum in log-space, logsumexp to sum log-space numbers without over/under-flow (see wiki page)
        A[i]['+']-=marg # log-space, so this represents the division by the normalizing constant
        A[i]['-']-=marg # as above
    data['alpha'].append(A['+']['+'])
    data['beta'].append(A['-']['-'])

# Data dictionary tracks the iterations of BW algorithm for manual inspection if necessary

arr = np.array(list(data.values()))
arr = 2**arr
arr = arr.T
np.savetxt(os.path.dirname(args.prior) +'/hmm_BWparameters.txt',arr,fmt='%.9f')
if args.createfile:
    bn = os.path.basename(args.prior)
    bn = bn.split('.')[0]
    bn+='_MLE'
    bn = os.path.dirname(args.prior) +'/'+ bn
    pickle.dump({'A':A,'E':E,'pi':pi,'states':states},open(f'{bn}.mkv','wb'))
elif not args.createfile:
    pickle.dump({'A':A,'E':E,'pi':pi,'states':states},open(f'{args.prior}','wb'))