# Student ID: 1814226        Name: Hua Phuoc Thuan
"""
 * @author nhphung
"""
from abc import ABC, abstractmethod, ABCMeta
from dataclasses import dataclass
from typing import List, Tuple
from AST import *
from Visitor import *
from StaticError import *
from functools import *

class Type(ABC):
    __metaclass__ = ABCMeta
    pass

class Prim(Type):
    __metaclass__ = ABCMeta
    pass

class IntType(Prim):
    pass

class FloatType(Prim):
    pass

class StringType(Prim):
    pass

class BoolType(Prim):
    pass

class VoidType(Type):
    pass

class Unknown(Type):
    pass

@dataclass
class ArrayType(Type):
    dimen: List[int]
    eletype: Type

@dataclass
class MType(Type):
    intype: List[Type]
    restype: Type

@dataclass
class Symbol:
    name: str
    mtype: MType
    kind:Kind

class Utils():
    def lookup(self, name, lst: List[List[Symbol]]):
        for scope in lst:
            for x in scope:
                if name == x.name:
                    return x
        return None

class StaticChecker(BaseVisitor, Utils):
    def __init__(self, ast):
        self.ast = ast
        self.built_in = [
            Symbol("int_of_float", MType([FloatType()], IntType()), Function()),
            Symbol("float_to_int", MType([IntType()], FloatType()), Function()),
            Symbol("int_of_string", MType([StringType()], IntType()), Function()),
            Symbol("string_of_int", MType([IntType()], StringType()), Function()),
            Symbol("float_of_string", MType([StringType()], FloatType()), Function()),
            Symbol("string_of_float", MType([FloatType()], StringType()), Function()),
            Symbol("bool_of_string", MType([StringType()], BoolType()), Function()),
            Symbol("string_of_bool", MType([BoolType()], StringType()), Function()),
            Symbol("read", MType([], StringType()), Function()),
            Symbol("printLn", MType([], VoidType()), Function()),
            Symbol("print", MType([StringType()], VoidType()), Function()),
            Symbol("printStrLn", MType([StringType()], VoidType()), Function())
        ]
        self.global_envi = [self.built_in[:]]

    def check(self):
        return self.visit(self.ast, self.global_envi)

    def visitProgram(self, ast: Program, c):
        for declare in ast.decl:
            if isinstance(declare, VarDecl):
                self.visit(declare, c)

        for declare in ast.decl:
            if isinstance(declare,FuncDecl):
                var = self.lookup(declare.name.name, [c[0]])
                if var is not None:
                    raise Redeclared(Function(), declare.name.name)

                c[0].append(Symbol(declare.name.name, MType([], Unknown()),Function()))
                [c[0][-1].mtype.intype.append(Unknown()) for parameter in declare.param]

        for declare in ast.decl:
            if isinstance(declare, FuncDecl):
                self.visit(declare, c)

        entry = self.lookup("main", c)
        if entry is None:
            raise NoEntryPoint()

    def visitVarDecl(self, ast: VarDecl, param):
        var = self.lookup(ast.variable.name, [param[0]])
        if var is not None:
            raise Redeclared(Variable(), ast.variable.name)

        if ast.varDimen:
            if ast.varInit:
                if type(ast.varInit) is ArrayLiteral:
                    param[0].append(
                        Symbol(ast.variable.name, MType([], self.visit(ast.varInit, param)), Variable()))
                    #print(param[0][-1])
                else:
                    pass
            else:
                arr_type = Unknown()
                for i in range(len(ast.varDimen)):
                    arr_type = ArrayType(ast.varDimen[(len(ast.varDimen) - 1 - i):], arr_type)

                param[0].append(Symbol(ast.variable.name, MType([], arr_type), Variable()))
                #print(param[0][-1])
        else:
            if ast.varInit:
                param[0].append(Symbol(ast.variable.name, MType([], self.visit(ast.varInit, param)), Variable()))
            else:
                param[0].append(Symbol(ast.variable.name, MType([], Unknown()), Variable()))

    def ReturnStatement(self, Stmts, func: Symbol, param):
        if isinstance(Stmts, Return):
            stmt_type = self.visit(Stmts, param)
            if isinstance(func.mtype.restype, Unknown):
                if isinstance(stmt_type, Unknown):
                    raise TypeCannotBeInferred(Stmts)
                if isinstance(stmt_type, ArrayType):
                    while True:
                        if isinstance(stmt_type, Unknown):
                            raise TypeCannotBeInferred(Stmts)
                        if not isinstance(stmt_type, ArrayType):
                            break
                        stmt_type = stmt_type.eletype

            stmt_type = self.visit(Stmts, param)
            if isinstance(func.mtype.restype, Unknown):
                func.mtype.restype = stmt_type

            if not isinstance(self.visit(Stmts, param), ArrayType) or not isinstance(func.mtype.restype, ArrayType):
                if type(self.visit(Stmts, param)) != type(func.mtype.restype):
                    raise TypeMismatchInStatement(Stmts)
            else:
                stmt_type = self.visit(Stmts, param)
                ret_type = func.mtype.restype
                while True:
                    if type(ret_type) != type(stmt_type):
                        raise TypeMismatchInStatement(Stmts)
                    if not isinstance(ret_type, ArrayType) and not isinstance(stmt_type, ArrayType):
                        break
                    ret_type = ret_type.eletype
                    stmt_type = stmt_type.eletype

    def visitFuncDecl(self, ast: FuncDecl, param):
        local = [[]]
        func = self.lookup(ast.name.name, param)
        for i in range(len(ast.param)):
            parameter = self.lookup(ast.param[i].variable.name, [local[0]])
            if parameter is not None:
                raise Redeclared(Parameter(), parameter.name)
            self.visit(ast.param[i], local)
            if isinstance(local[0][-1].mtype.restype, Unknown):
                local[0][-1].mtype.restype = func.mtype.intype[i]

        total = local + param + [[func]]
        [self.visit(varDecl, total) for varDecl in ast.body[0]]
        for Stmts in ast.body[1]:
            self.visit(Stmts, total)
            self.ReturnStatement(Stmts, func, total)

        if isinstance(func.mtype.restype, Unknown):
            func.mtype.restype = VoidType()

        for i in range(len(ast.param)):
            parameter = self.lookup(ast.param[i].variable.name, total)
            if not isinstance(parameter.mtype.restype,Unknown):
                func = self.lookup(ast.name.name,param)
                func.mtype.intype[i] = parameter.mtype.restype
        #print(var.name,var.mtype.intype)

    def visitBinaryOp(self, ast: BinaryOp, param):
        op = ast.op
        left = self.visit(ast.left,param)
        right = self.visit(ast.right,param)

        if "TypeCannotBeInferred" in [left, right]:
            return "TypeCannotBeInferred"
        #print(left,right)
        if op in ['+','-','*',"\\","%"]:
            if isinstance(left,(Unknown,IntType)) and isinstance(right,(Unknown,IntType)):
                if isinstance(left,Unknown):
                    var = self.lookup(ast.left.name,param) if isinstance(ast.left,Id) \
                        else self.lookup(ast.left.method.name,param)
                    var.mtype.restype = IntType()

                if isinstance(right,Unknown):
                    var = self.lookup(ast.right.name,param) if isinstance(ast.right,Id) \
                        else self.lookup(ast.right.method.name,param)
                    var.mtype.restype = IntType()
                return IntType()
            else:
                raise TypeMismatchInExpression(ast)

        if op in ['+.','-.','*.','\\.']:
            if isinstance(left,(Unknown,FloatType)) and isinstance(right,(Unknown,FloatType)):
                if isinstance(left,Unknown):
                    var = self.lookup(ast.left.name,param) if isinstance(ast.left,Id) \
                        else self.lookup(ast.left.method.name,param)
                    var.mtype.restype = FloatType()

                if isinstance(right,Unknown):
                    var = self.lookup(ast.right.name,param) if isinstance(ast.right,Id) \
                        else self.lookup(ast.right.method.name,param)
                    var.mtype.restype = FloatType()

                return FloatType()
            else:
                raise TypeMismatchInExpression(ast)

        if op in ["==","!=","<",">","<=",">="]:
            if isinstance(left,(Unknown,IntType)) and isinstance(right,(Unknown,IntType)):
                if isinstance(left,Unknown):
                    var = self.lookup(ast.left.name,param) if isinstance(ast.left,Id) \
                        else self.lookup(ast.left.method.name,param)
                    var.mtype.restype = IntType()

                if isinstance(right,Unknown):
                    var = self.lookup(ast.right.name,param) if isinstance(ast.right,Id) \
                        else self.lookup(ast.right.method.name,param)
                    var.mtype.restype = IntType()
                return BoolType()
            else:
                raise TypeMismatchInExpression(ast)

        if op in ["=/=","<.",">.","<=.",">=."]:
            if isinstance(left,(Unknown,FloatType)) and isinstance(right,(Unknown,FloatType)):
                if isinstance(left,Unknown):
                    var = self.lookup(ast.left.name,param) if isinstance(ast.left,Id) \
                        else self.lookup(ast.left.method.name,param)
                    var.mtype.restype = FloatType()

                if isinstance(right,Unknown):
                    var = self.lookup(ast.right.name,param) if isinstance(ast.right,Id) \
                        else self.lookup(ast.right.method.name,param)
                    var.mtype.restype = FloatType()
                return BoolType()
            else:
                raise TypeMismatchInExpression(ast)

        if op in ["||","&&"]:
            if isinstance(left,(Unknown,BoolType)) and isinstance(right,(Unknown,BoolType)):
                if isinstance(left,Unknown):
                    var = self.lookup(ast.left.name,param) if isinstance(ast.left,Id) \
                        else self.lookup(ast.left.method.name,param)
                    var.mtype.restype = BoolType()

                if isinstance(right,Unknown):
                    var = self.lookup(ast.right.name,param) if isinstance(ast.right,Id) \
                        else self.lookup(ast.right.method.name,param)
                    var.mtype.restype = BoolType()
                return BoolType()
            else:
                raise TypeMismatchInExpression(ast)

    def visitUnaryOp(self, ast: UnaryOp, param):
        op = ast.op
        operand = self.visit(ast.body,param)

        if operand == "TypeCannotBeInferred":
            return "TypeCannotBeInferred"
        if op == "-":
            if isinstance(operand,(Unknown,IntType)):
                if isinstance(operand,Unknown):
                    var = self.lookup(ast.body.name,param) if isinstance(ast.body,Id) \
                        else self.lookup(ast.body.method.name,param)
                    var.mtype.restype = IntType()

                return IntType()
            else:
                raise TypeMismatchInExpression(ast)
        if op == "-.":
            if isinstance(operand,(Unknown,FloatType)):
                if isinstance(operand,Unknown):
                    var = self.lookup(ast.body.name,param) if isinstance(ast.body,Id) \
                        else self.lookup(ast.body.method.name,param)
                    var.mtype.restype = FloatType()

                return FloatType()
            else:
                raise TypeMismatchInExpression(ast)

        if op == "!":
            if isinstance(operand,(Unknown,BoolType)):
                if isinstance(operand,Unknown):
                    var = self.lookup(ast.body.name,param) if isinstance(ast.body,Id) \
                        else self.lookup(ast.body.method.name,param)
                    var.mtype.restype = BoolType()

                return BoolType()
            else:
                raise TypeMismatchInExpression(ast)

    def visitCallExpr(self, ast: CallExpr, param):
        var = self.lookup(ast.method.name, param)
        if var is not None:
            if not isinstance(var.kind, Function):
                raise Undeclared(Function(),ast.method.name)
            if len(var.mtype.intype) != len(ast.param):
                raise TypeMismatchInExpression(ast)

            if var.mtype.intype:
                for i in range(len(ast.param)):
                    parameter = self.lookup(ast.param[i].name, param) if isinstance(ast.param[i], Id) \
                        else self.lookup(ast.param[i].method.name, param) if isinstance(ast.param[i], CallExpr) else ""
                    param_type = parameter.mtype.restype if isinstance(parameter, Symbol) else self.visit(ast.param[i], param)

                    if (isinstance(param_type, Unknown) and isinstance(var.mtype.intype[i], Unknown)) or param_type == "TypeCannotBeInferred":
                        return "TypeCannotBeInferred"
                    if isinstance(param_type, Unknown):
                        parameter.mtype.restype = var.mtype.intype[i]

                    if isinstance(var.mtype.intype[i], Unknown):
                        var.mtype.intype[i] = param_type

                    param_type = parameter.mtype.restype if isinstance(parameter, Symbol) else self.visit(ast.param[i], param)
                    if type(var.mtype.intype[i]) != type(param_type):
                        raise TypeMismatchInExpression(ast)
                    self.visit(ast.param[i],param)
        else:
            raise Undeclared(Function(), ast.method.name)
        #print(var.mtype.restype)
        return var.mtype.restype

    def visitId(self, ast: Id, param):
        var = self.lookup(ast.name, param)
        if var is not None:
            if not isinstance(var.kind, Variable):
                raise Undeclared(Identifier(),ast.name)
            return var.mtype.restype
        raise Undeclared(Identifier(), ast.name)

    def visitArrayCell(self, ast: ArrayCell, param):
        id_type = self.visit(ast.arr, param)
        ACell = self.lookup(ast.arr.name, param) if isinstance(ast.arr, Id) else self.lookup(ast.arr.method.name, param)
        #print(ast.arr, id_type)
        if id_type == "TypeCannotBeInferred" or (isinstance(id_type,Unknown) and isinstance(ast.arr,CallExpr)):  # id is a CallExpr but cannot inferred the parameter type
            return "TypeCannotBeInferred"
        #print(ast.arr)

        if not isinstance(id_type, ArrayType):
            raise TypeMismatchInExpression(ast)
        else:
            if len(ACell.mtype.restype.dimen) != len(ast.idx):
                raise TypeMismatchInExpression(ast)

        for expr in ast.idx:
            expr_type = self.visit(expr, param)
            if expr_type == "TypeCannotBeInferred":
                return "TypeCannotBeInferred"
            if isinstance(expr_type, Unknown):
                var = self.lookup(expr.name, param) if isinstance(expr, Id) else self.lookup(expr.method.name, param)
                var.mtype.restype = IntType()

            expr_type = self.visit(expr, param)
            if not isinstance(expr_type, IntType):
                raise TypeMismatchInExpression(ast)

        for x in range(len(ast.idx)):
            if not isinstance(id_type,ArrayType):
                raise TypeMismatchInExpression(ast)
            id_type = id_type.eletype
        return id_type

    def visitAssign(self, ast: Assign, param):
        lhs = self.visit(ast.lhs, param)
        rhs = self.visit(ast.rhs, param)
        #print(ast.lhs,lhs,rhs)
        if (isinstance(lhs, Unknown) and isinstance(rhs, Unknown)) or ("TypeCannotBeInferred" in [lhs, rhs]):
            # both lhs and rhs is Unknown Type or (rhs or lhs is CallExpr and can't be inffered)
            raise TypeCannotBeInferred(ast)
        if isinstance(lhs, VoidType) or isinstance(rhs, VoidType):
            raise TypeMismatchInStatement(ast)
        # print(self.visit(ast.lhs,param),self.visit(ast.rhs,param))
        if isinstance(lhs, Unknown):
            if isinstance(ast.lhs, Id):
                var = self.lookup(ast.lhs.name, param)
                if isinstance(self.visit(ast.rhs, param), ArrayType):
                    raise TypeMismatchInStatement(ast)
                var.mtype.restype = self.visit(ast.rhs, param)
            elif isinstance(ast.lhs, CallExpr):
                var = self.lookup(ast.lhs.method.name,param)
                var.mtype.restype = self.visit(ast.rhs, param)
            else:
                if isinstance(ast.lhs.arr, Id):
                    var = self.lookup(ast.lhs.arr.name,param)
                else:
                    var = self.lookup(ast.lhs.arr.method.name, param)
                eletype = var.mtype.restype
                while True:
                    if isinstance(eletype.eletype, Unknown):
                        eletype.eletype = self.visit(ast.rhs, param)
                        break
                    eletype = eletype.eletype

        if isinstance(rhs, Unknown):
            if isinstance(ast.rhs, Id):
                var = self.lookup(ast.rhs.name, param)
                if isinstance(self.visit(ast.rhs, param), ArrayType):
                    raise TypeMismatchInStatement(ast)

                var.mtype.restype = self.visit(ast.lhs, param)
            elif isinstance(ast.rhs, CallExpr):
                var = self.lookup(ast.rhs.method.name,param)
                var.mtype.restype = self.visit(ast.lhs, param)
            else:
                if isinstance(ast.rhs.arr, Id):
                    var = self.lookup(ast.rhs.arr.name,param)
                else:
                    var = self.lookup(ast.rhs.arr.method.name, param)
                eletype = var.mtype.restype
                while True:
                    if isinstance(eletype.eletype, Unknown):
                        eletype.eletype = self.visit(ast.lhs, param)
                        break
                    eletype = eletype.eletype
        #print(self.visit(ast.lhs,param),self.visit(ast.rhs,param))
        lhs = self.visit(ast.lhs, param)
        rhs = self.visit(ast.rhs, param)
        #print(type(lhs),type(rhs))

        if isinstance(lhs,ArrayType) and isinstance(rhs,ArrayType):
            if lhs.dimen != rhs.dimen:
                raise TypeMismatchInStatement(ast)
            lhs_eletype = lhs.eletype
            rhs_eletype = rhs.eletype
            while True:
                if isinstance(lhs_eletype, Unknown):
                    if isinstance(ast.lhs, Id):
                        var = self.lookup(ast.lhs.name, param)
                    elif isinstance(ast.lhs, CallExpr):
                        var = self.lookup(ast.lhs.method.name, param)
                    else:
                        if isinstance(ast.lhs.arr, Id):
                            var = self.lookup(ast.lhs.arr.name, param)
                        else:
                            var = self.lookup(ast.lhs.arr.method.name, param)

                    eletype = var.mtype.restype
                    while True:
                        if isinstance(eletype.eletype,Unknown):
                            eletype.eletype = rhs_eletype
                            break
                        eletype = eletype.eletype
                    break

                if isinstance(rhs_eletype, Unknown):
                    if isinstance(ast.rhs, Id):
                        var = self.lookup(ast.rhs.name, param)
                    elif isinstance(ast.rhs, CallExpr):
                        var = self.lookup(ast.rhs.method.name, param)
                    else:
                        if isinstance(ast.rhs.arr, Id):
                            var = self.lookup(ast.rhs.arr.name, param)
                        else:
                            var = self.lookup(ast.rhs.arr.method.name, param)

                    eletype = var.mtype.restype
                    while True:
                        if isinstance(eletype.eletype,Unknown):
                            eletype.eletype = lhs_eletype
                            break
                        eletype = eletype.eletype
                    break

                if not isinstance(lhs_eletype,ArrayType) and not isinstance(rhs_eletype,ArrayType):
                    break

                if isinstance(lhs_eletype,ArrayType):
                    lhs_eletype = lhs_eletype.eletype

                if isinstance(rhs_eletype,ArrayType):
                    rhs_eletype = rhs_eletype.eletype

            if isinstance(lhs_eletype,Unknown) and isinstance(rhs_eletype,Unknown):
                raise TypeCannotBeInferred(ast)

            lhs_eletype = self.visit(ast.lhs, param)
            rhs_eletype = self.visit(ast.rhs, param)
            while True:
                if type(lhs_eletype) != type(rhs_eletype):
                    raise TypeMismatchInStatement(ast)

                if isinstance(lhs_eletype, ArrayType):
                    lhs_eletype = lhs_eletype.eletype
                else:
                    break

                if isinstance(rhs_eletype, ArrayType):
                    rhs_eletype = rhs_eletype.eletype
                else:
                    break

        if type(self.visit(ast.lhs,param)) is not type(self.visit(ast.rhs,param)):
            raise TypeMismatchInStatement(ast)

    def visitIf(self, ast: If, param):
        func = param[-1][0]
        for ifthenStmt in ast.ifthenStmt:
            var = self.lookup(ifthenStmt[0].name, param) if isinstance(ifthenStmt[0], Id) \
                else self.lookup(ifthenStmt[0].method.name, param) if isinstance(ifthenStmt[0],CallExpr) else ""
            exp_type = var.mtype.restype if isinstance(var,Symbol) else self.visit(ifthenStmt[0], param)
            if exp_type == "TypeCannotBeInferred":
                raise TypeCannotBeInferred(ast)

            if isinstance(exp_type, Unknown):
                var.mtype.restype = BoolType()

            exp_type = self.visit(ifthenStmt[0], param)
            if exp_type == "TypeCannotBeInferred":
                raise TypeCannotBeInferred(ast)

            if not isinstance(exp_type, BoolType):
                raise TypeMismatchInStatement(ast)

            local = [[]]
            for VarDecl in ifthenStmt[1]:
                self.visit(VarDecl, local)
            total = local + param
            for Stmts in ifthenStmt[2]:
                self.visit(Stmts, total)
                self.ReturnStatement(Stmts, func, param)

        else_stmt = ast.elseStmt
        local = [[]]
        for VarDecl in else_stmt[0]:
            self.visit(VarDecl, local)
        total = local + param
        for Stmts in else_stmt[1]:
            self.visit(Stmts, total)
            self.ReturnStatement(Stmts, func, param)

    def visitFor(self, ast: For, param):
        func = param[-1][0]
        idx1 = self.lookup(ast.idx1.name, param)

        if isinstance(idx1.mtype.restype, Unknown):
            idx1.mtype.restype = IntType()

        var = self.lookup(ast.expr1.name, param) if isinstance(ast.expr1, Id) \
                else self.lookup(ast.expr1.method.name, param) if isinstance(ast.expr1, CallExpr) else ""
        expr1 = var.mtype.restype if isinstance(var, Symbol) else self.visit(ast.expr1, param)
        if isinstance(expr1, Unknown):
            var.mtype.restype = IntType()

        var = self.lookup(ast.expr2.name, param) if isinstance(ast.expr2, Id) \
                else self.lookup(ast.expr2.method.name, param) if isinstance(ast.expr2, CallExpr) else ""
        expr2 = var.mtype.restype if isinstance(var, Symbol) else self.visit(ast.expr2, param)
        if isinstance(expr2, Unknown):
            var.mtype.restype = BoolType()

        var = self.lookup(ast.expr3.name, param) if isinstance(ast.expr3, Id) \
                else self.lookup(ast.expr3.method.name, param) if isinstance(ast.expr3, CallExpr) else ""
        expr3 = var.mtype.restype if isinstance(var, Symbol) else self.visit(ast.expr3, param)
        if isinstance(expr3, Unknown):
            var.mtype.restype = IntType()

        idx1 = self.visit(ast.idx1, param)
        expr1 = self.visit(ast.expr1, param)
        expr2 = self.visit(ast.expr2, param)
        expr3 = self.visit(ast.expr3, param)
        if "TypeCannotBeInferred" in [expr1, expr2, expr3]:
            raise TypeCannotBeInferred(ast)

        iType = [isinstance(idx1, IntType), isinstance(expr1, IntType), isinstance(expr3, IntType)]
        # print(iType)
        if False in iType:
            raise TypeMismatchInStatement(ast)

        # if type(self.visit(ast.expr2,param)) is not type(BoolType()):
        if not isinstance(expr2, BoolType):
            raise TypeMismatchInStatement(ast)

        local = [[]]
        for VarDecl in ast.loop[0]:
            self.visit(VarDecl, local)
        total = local + param
        for Stmts in ast.loop[1]:
            self.visit(Stmts, total)
            self.ReturnStatement(Stmts, func, param)

    def visitContinue(self, ast: Continue, param):
        pass

    def visitBreak(self, ast: Break, param):
        pass

    def visitReturn(self, ast: Return, param):
        if ast.expr:
            expr_type = self.visit(ast.expr, param)
            if expr_type == "TypeCannotBeInferred" or isinstance(expr_type, Unknown):
                raise TypeCannotBeInferred(ast)
            return self.visit(ast.expr, param)
        else:
            return VoidType()

    def visitDowhile(self, ast: Dowhile, param):
        local = [[]]
        func = param[-1][0]
        for VarDecl in ast.sl[0]:
            self.visit(VarDecl, local)
        total = local + param
        for Stmts in ast.sl[1]:
            self.visit(Stmts, total)
            self.ReturnStatement(Stmts, func, param)

        var = self.lookup(ast.exp.name, param) if isinstance(ast.exp, Id) \
                else self.lookup(ast.exp.method.name, param) if isinstance(ast.exp, CallExpr) else ""
        exp_type = var.mtype.restype if isinstance(var, Symbol) else self.visit(ast.exp, param)

        if isinstance(exp_type, Unknown):
            var.mtype.restype = BoolType()

        exp_type = self.visit(ast.exp, param)
        if exp_type == "TypeCannotBeInferred":
            raise TypeCannotBeInferred(ast)

        if not isinstance(exp_type, BoolType):
            raise TypeMismatchInStatement(ast)

    def visitWhile(self, ast: While, param):
        func = param[-1][0]

        var = self.lookup(ast.exp.name, param) if isinstance(ast.exp, Id) \
                else self.lookup(ast.exp.method.name, param) if isinstance(ast.exp, CallExpr) else ""
        exp_type = var.mtype.restype if isinstance(var, Symbol) else self.visit(ast.exp, param)
        if isinstance(exp_type, Unknown):
            var.mtype.restype = BoolType()

        exp_type = self.visit(ast.exp, param)
        if exp_type == "TypeCannotBeInferred":
            raise TypeCannotBeInferred(ast)

        if not isinstance(exp_type, BoolType):
            raise TypeMismatchInStatement(ast)
        local = [[]]
        for VarDecl in ast.sl[0]:
            self.visit(VarDecl, local)
        total = local + param
        for Stmts in ast.sl[1]:
            self.visit(Stmts, total)
            self.ReturnStatement(Stmts, func, param)

    def visitCallStmt(self, ast: CallStmt, param):
        var = self.lookup(ast.method.name, param)
        if var is not None:
            if not isinstance(var.kind, Function):
                raise Undeclared(Function(),ast.method.name)

            if len(var.mtype.intype) != len(ast.param):
                raise TypeMismatchInStatement(ast)

            if isinstance(var.mtype.restype,Unknown):
                var.mtype.restype = VoidType()

            if not isinstance(var.mtype.restype, VoidType):
                raise TypeMismatchInStatement(ast)

            if var.mtype.intype:
                for i in range(len(ast.param)):
                    param_type = self.visit(ast.param[i], param)
                    if param_type == "TypeCannotBeInferred":
                        raise TypeCannotBeInferred(ast)
                    if isinstance(param_type, Unknown) and isinstance(var.mtype.intype[i], Unknown):
                        raise TypeCannotBeInferred(ast)

                    if isinstance(param_type, Unknown):
                        parameter = self.lookup(ast.param[i].name, param) if isinstance(ast.param[i], Id) \
                            else self.lookup(ast.param[i].method.name, param)
                        parameter.mtype.restype = var.mtype.intype[i]

                    if isinstance(var.mtype.intype[i], Unknown):
                        var.mtype.intype[i] = param_type

                    param_type = self.visit(ast.param[i], param)
                    #print(param_type, var.mtype.intype[i])
                    if type(var.mtype.intype[i]) != type(param_type):
                        raise TypeMismatchInStatement(ast)
        else:
            raise Undeclared(Function(), ast.method.name)

    def visitIntLiteral(self, ast: IntLiteral, param):
        return IntType()

    def visitFloatLiteral(self, ast: FloatLiteral, param):
        return FloatType()

    def visitBooleanLiteral(self, ast: BooleanLiteral, param):
        return BoolType()

    def visitStringLiteral(self, ast: StringLiteral, param):
        return StringType()

    def visitArrayLiteral(self, ast: ArrayLiteral, param):
        listdimen = [len(ast.value)]
        for value in ast.value:
            x = self.visit(value,param)
        t = self.visit(ast.value[0],param)
        if isinstance(t,ArrayType):
            listdimen.extend(t.dimen)

        return ArrayType(listdimen,self.visit(ast.value[0],param))


from gevent import monkey
monkey.patch_all()

import hashlib
import os
import random
import redis
import time

from collections import defaultdict
from functools import wraps

from flask import Flask, request, render_template, render_template_string, make_response, json, jsonify, abort

app = Flask('jokes')
r = redis.StrictRedis.from_url(os.environ.get("REDIS_URL", "redis://localhost:6379/0"))
jokes = json.load(open('jokes.json'))['value']
jokes = {joke['id']: joke for joke in jokes if 'explicit' not in joke['categories']}

APPNAME = "chitter"

THROTTLE_HNAME           = "{}_thr".format(APPNAME)
THROTTLE_OPTION_REQUESTS = "{}_throttle_requests".format(APPNAME)
THROTTLE_OPTION_INTERVAL = "{}_throttle_interval".format(APPNAME)
THROTTLE_EXEMPT_HEADER   = "x-full-throttle"

CRASH_OPTION        = "{}_crash_thresh".format(APPNAME)
CRASH_EXEMPT_HEADER = "x-wedding-crashers"

OVERLOAD_OPTION        = "{}_overload_thresh".format(APPNAME)
OVERLOAD_EXEMPT_HEADER = "x-overload"

SLOWDOWN_OPTION        = "{}_slowdown_thresh".format(APPNAME)
SLOWDOWN_EXEMPT_HEADER = "x-ludicrous-speed"
SLOWDOWN_TIME          = "{}_slowdown_max_time".format(APPNAME)
SLOWDOWN_BASE_TIME     = "{}_slowdown_base_time".format(APPNAME)

@app.route('/')
@app.route('/api/help')
def help():
    """ Print available api functions """
    func_list = defaultdict(dict)
    for rule in app.url_map.iter_rules():
        if rule.endpoint != 'static':
            func_list[rule.endpoint][rule.rule] = {
                    'doc': app.view_functions[rule.endpoint].__doc__.strip(),
                    'methods': list(rule.methods - {'HEAD', 'OPTIONS'}),
                    }
    return render_template("help.html", data=func_list)

@app.route('/api/joke/<id>')
@app.route('/api/joke')
def joke(id=None):
    """
    Get a joke from the db with the given id. If no id is given, get a random joke.
    The returned json will be of the format:
        {"message":"", "status": 200, value: {"categories": categories, "id": id, "author": id, "joke": "text", "pic": "pic_url"}}
    """
    if not id:
        id = random.choice(jokes.keys())
    try:
        id = int(id)
    except ValueError:
        abort(400)
    if id not in jokes:
        abort(404)
    return jsonify(message="",
            status=200,
            value={'categories': jokes[id]['categories'], 'joke': jokes[id]['joke'], 'id': id, 'author': 'Chuck Norris'})

@app.after_request
def add_header(resp):
    resp.headers['Access-Control-Allow-Origin'] = resp.headers.get('Access-Control-Allow-Origin', '*')
    return resp

@app.errorhandler(429)
def too_many_requests(err):
    resp = jsonify(
            message="Slow your roll, homie",
            status=429,
            data={}
            )
    resp.headers['Retry-After'] = r.get(THROTTLE_OPTION_INTERVAL) or 30
    return resp, 429

@app.errorhandler(400)
def bad_request(err):
    return jsonify(
            message=str(err),
            status=400,
            data={}
            ), 400

@app.before_request
def before_request():
    def exempt():
        return request.endpoint in {'help'}

    @unless_header(THROTTLE_EXEMPT_HEADER)
    def throttle():
        forwarded_for = request.headers.getlist("X-Forwarded-For")
        remote_addr = forwarded_for[-1] if len(forwarded_for) > 0 else request.remote_addr
        throttle_reqs = float(r.get(THROTTLE_OPTION_REQUESTS) or 5)
        throttle_interval = int(r.get(THROTTLE_OPTION_INTERVAL) or 30)
        throttle_key = "{}_{}_{}".format(remote_addr,
                throttle_interval,
                int(time.time() / throttle_interval))
        if r.hincrby(THROTTLE_HNAME, throttle_key, 1) > throttle_reqs:
            abort(429)

    @unless_header(SLOWDOWN_EXEMPT_HEADER)
    def random_slowdown():
        base_wait = float(r.get(SLOWDOWN_BASE_TIME) or 0.25)
        time.sleep(base_wait)

        thresh = float(r.get(SLOWDOWN_OPTION) or 0.4)
        if random.random() < thresh:
            max_wait = float(r.get(SLOWDOWN_TIME) or 2.0)
            time.sleep(random.random() * max_wait - base_wait)

    @unless_header(OVERLOAD_EXEMPT_HEADER)
    def random_overload():
        thresh = float(r.get(OVERLOAD_OPTION) or 0.1)
        if random.random() < thresh:
            abort(503)

    @unless_header(CRASH_EXEMPT_HEADER)
    def random_crash():
        thresh = float(r.get(CRASH_OPTION) or 0.1)
        if random.random() < float(thresh):
            abort(500)

    if not exempt():
        random_crash()
        random_overload()
        throttle()
        random_slowdown()

def unless_header(*header_names):
    def decorator(f):
        @wraps(f)
        def decorated_func(*args, **kwargs):
            if any(request.headers.get(header_name) for header_name in header_names):
                return
            return f(*args, **kwargs)
        return decorated_func
    return decorator

def main():
    app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 5000)), debug=True)
    return 0

if __name__ == '__main__':
    sys.exit(main())


import logging as log

import numpy as np
import pandas as pd
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.neighbors import KNeighborsRegressor
from sklearn.preprocessing import *

class KNNRegression:
    def __init__(self, trainfile):
        log.getLogger().setLevel(log.INFO)
        log.info('KNN Regressor')

        # Load set
        self.trainFile = trainfile
        trainDataFrame = pd.read_csv(self.trainFile, sep='\t', header=None)

        # Mapping string and bool values to numeric
        self.mapping_string = self.map_columns(trainDataFrame, 4)
        self.mapping_bool = self.map_columns(trainDataFrame, 1)
        trainDataFrame = trainDataFrame.applymap(
            lambda x: self.mapping_string.get(x) if x in self.mapping_string else x)
        trainDataFrame = trainDataFrame.applymap(
            lambda x: self.mapping_bool.get(x) if x in self.mapping_bool else x)
        trainArray = trainDataFrame.values

        # Shuffle Data
        np.random.shuffle(trainArray)

        # Extract values to numpy.Arrays
        self.X = trainArray[:, 1:]
        self.Y = trainArray[:, 0]

        self.grided_params = []
        self.knnr = None

        # Split to train-test sets
        self.X_train, self.X_test, self.Y_train, self.Y_test = train_test_split(self.X, self.Y, test_size=0.3,
                                                                                random_state=0)

    def __str__(self):
        """
        Printing data
        :return: None
        """
        print("Features: {}, Labels: {}".format(self.X, self.Y))

    @staticmethod
    def map_columns(df, colnumber: int):
        """
        Mapping non numeric values to numeric
        :param df: pandas dataframe that contain dataset
        :param colnumber: number collumn to map
        :return: dictionary with mapped values
        """
        return dict([(y, x + 1) for x, y in enumerate(sorted(set(df[colnumber].unique())))])

    def rescale(self):
        """
        Rescaling data in dataset to [0,1]
        :return: None
        """
        scaler = MinMaxScaler(feature_range=(0, 1))
        self.X_train = scaler.fit_transform(self.X_train)
        self.X_test = scaler.fit_transform(self.X_test)

    def normalize(self):
        """
        Normalizing data in dataset
        :return: None
        """
        scaler = Normalizer()
        self.X_train = scaler.fit_transform(self.X_train)
        self.X_test = scaler.fit_transform(self.X_test)

    def standalizer(self):
        """
        Standardlizing data in dataset
        :return: None
        """
        scaler = StandardScaler()
        self.X_train = scaler.fit_transform(self.X_train)
        self.X_test = scaler.fit_transform(self.X_test)

    def output(self):
        """
        Predicting and log values
        :return: None
        """
        y_pred = self.knnr.predict(self.X_test)
        log.info(f"MSE: {mean_squared_error(self.Y_test, y_pred)}")
        for x, y in zip(y_pred, self.Y_test):
            log.info(f"Predicted: {x}| Actual: {y}")

    def train_model(self):
        """
        Fiting model with grid search hyper-parameters
        :return: None
        """
        self.knnr = KNeighborsRegressor(n_neighbors=self.grided_params[0], weights=self.grided_params[1],
                                        algorithm=self.grided_params[2]).fit(self.X_train, self.Y_train)

    def grid_search(self):
        """
        Sklearn hyper-parameters grid search
        :return: None
        """
        hyperparam_grid = {
            'n_neighbors': [5, 10, 15, 20, 25, 30],
            'weights': ('uniform', 'distance'),
            'algorithm': ('ball_tree', 'kd_tree', 'brute')

        }
        classifier = GridSearchCV(KNeighborsRegressor(), hyperparam_grid, cv=5, iid=False)
        classifier.fit(self.X_train, self.Y_train)
        self.grided_params = [classifier.best_estimator_.n_neighbors, classifier.best_estimator_.weights,
                              classifier.best_estimator_.algorithm]


import json

'''
序列化数据
json本质就是一个字符串,里面是双引号的字符串，转json，元素类型对应js类型
作用：
    1、序列化数据（dump,dumps），使数据转字符串，可以在文件操作是写入文件中，
    2、反序列化数据（load,loads），使字符串转数据

'''
list = ["name", 'age']

file = open('jsondump.txt', 'w', encoding='utf8')

liststr = json.dumps(list)  # 转字符串,不会保留数据到文件里
liststr2 = json.dump(list, file)  # 转字符串,并且保留数据到指定文件里

file.close()

# =====================================================

file1 = open('jsondump.txt', 'r', encoding='utf8')

strs = '["aaa","bbb"]'
strsloads = json.loads(strs)
strsload = json.load(file1)  # 读出文件中的json数据并反序列化为可用数据
print(type(strsloads))
print(strsload)

file1.close()


"""View module for handling requests about game types"""
from django.http import HttpResponseServerError
from django.core.exceptions import ValidationError
from rest_framework.viewsets import ViewSet
from rest_framework.response import Response
from rest_framework import serializers, status
from levelupapi.models import Game
from levelupapi.models.game_type import GameType
from levelupapi.models.gamer import Gamer

class GameView(ViewSet):
    """Level up game types view"""

    def retrieve(self, request, pk):
        """Handle GET requests for single game type
        Returns:
            Response -- JSON serialized game type
        """
        try:
            game = Game.objects.get(pk=pk)
            serializer = GameSerializer(game)
            return Response(serializer.data)
        except Game.DoesNotExist as ex:
            return Response({'message': ex.args[0]}, status=status.HTTP_404_NOT_FOUND)

    def list(self, request):
        """Handle GET requests to get all game types
        Returns:
            Response -- JSON serialized list of game types
        """
        games = Game.objects.all()
        # What if we wanted to pass in a query string parameter?
        # The request from the method parameters holds all the information for the request from the client. The request.query_params is a dictionary of any query parameters that were in the url. Using the .get method on a dictionary is a safe way to find if a key is present on the dictionary. If the 'type' key is not present on the dictionary it will return None.
        game_type = request.query_params.get('type', None)
        if game_type is not None:
            games = games.filter(game_type_id=game_type)
        serializer = GameSerializer(games, many=True)
        return Response(serializer.data)

    '''
    def create(self, request):
        """Handle POST operations

        Returns
            Response -- JSON serialized game instance
        """
        gamer = Gamer.objects.get(user=request.auth.user)
        game_type = GameType.objects.get(pk=request.data["game_type"])

        game = Game.objects.create(
            title=request.data["title"],
            maker=request.data["maker"],
            number_of_players=request.data["number_of_players"],
            skill_level=request.data["skill_level"],
            gamer=gamer,
            game_type=game_type
        )
        serializer = GameSerializer(game)
        return Response(serializer.data)
    '''

    # Inside the method, the first line of code is getting the game that is logged in. Since all of our postman or fetch requests have the user’s auth token in the headers, the request will get the user object based on that token. From there, we use the request.auth.user to get the Gamer object based on the user. Here’s the equivalent sql:
        # db_cursor.execute("""
        # select *
        # from levelupapi_gamer
        # where user = ?
        # """, (user,))

    def create(self, request):
        """Handle POST operations

        Returns:
            Response -- JSON serialized game instance
        """
        gamer = Gamer.objects.get(user=request.auth.user)
        serializer = CreateGameSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        serializer.save(gamer=gamer)
        return Response(serializer.data, status=status.HTTP_201_CREATED)

    #To add the game to the database, we call the create ORM method and pass the fields as parameters to the function. Here’s the sql that will run:
        # db_cursor.execute("""
        # Insert into levelupapi_game (title, maker, number_of_players, skill_level, gamer_id, game_type_id)
        # values (?, ?, ?, ?, ?, ?)
        # """, (request.data["title"], request.data["maker"], request.data["numberOfPlayers"], request.data["skillLevel"], gamer, game_type))

    # This time, when using the CreateGameSerializer, the original game object is passed to the serializer, along with the request.data. This will make any updates on the game object. Then, just like in the create, check for validity and save the updated object.

    def update(self, request, pk):
        """Handle PUT requests for a game
        Returns:
            Response -- Empty body with 204 status code
        """
        game = Game.objects.get(pk=pk)
        serializer = CreateGameSerializer(game, data=request.data)
        serializer.is_valid(raise_exception=True)
        serializer.save()
        return Response(None, status=status.HTTP_204_NO_CONTENT)

    def destroy(self, request, pk):
        game = Game.objects.get(pk=pk)
        game.delete()
        return Response(None, status=status.HTTP_204_NO_CONTENT)

# Right now the GET methods do not include any nested data, only the foreign key. Embedding that data is only 1 line of code! Take a look at the response for getting all the games. Notice that game_type is just the id of the type. Back in the GameSerializer add this to the end of Meta class tabbed to the same level as the fields property
class GameSerializer(serializers.ModelSerializer):
    """JSON serializer for game types
    """
    class Meta:
        model = Game
        fields = ('id', 'title', 'maker', 'gamer', 'number_of_players', 'skill_level', 'game_type')
        depth = 1

# Instead of making a new instance of the Game model, the request.data dictionary is passed to the new serializer as the data. The keys on the dictionary must match what is in the fields on the serializer. After creating the serializer instance, call is_valid to make sure the client sent valid data. If the code passes validation, then the save method will add the game to the database and add an id to the serializer.

class CreateGameSerializer(serializers.ModelSerializer):
    class Meta:
        model = Game
        fields = ['id', 'title', 'maker', 'number_of_players', 'skill_level', 'game_type']


#!/usr/bin/env python3
"""

"""
from argparse import ArgumentParser, BooleanOptionalAction
import pathlib
import random
from rc import pmap, run
import requests
import sys
import time

sys.path.append(str(pathlib.Path(__file__).resolve().parents[2] / 'lib'))

import mocknet

from configured_logger import logger

def get_nodes(args):
    pattern = args.chain_id + '-' + str(
        args.start_height) + '-' + args.unique_id
    all_nodes = mocknet.get_nodes(pattern=pattern)
    if len(all_nodes) < 1:
        sys.exit(f'no known nodes matching {pattern}')

    traffic_generator = None
    nodes = []
    for n in all_nodes:
        if n.instance_name.endswith('traffic'):
            if traffic_generator is not None:
                sys.exit(
                    f'more than one traffic generator instance found. {traffic_generator.instance_name} and {n.instance_name}'
                )
            traffic_generator = n
        else:
            nodes.append(n)

    if traffic_generator is None:
        sys.exit(f'no traffic generator instance found')
    return traffic_generator, nodes

def run_cmd(node, cmd):
    r = node.machine.run(cmd)
    if r.exitcode != 0:
        sys.exit(
            f'failed running {cmd} on {node.instance_name}:\nstdout: {r.stdout}\nstderr: {r.stderr}'
        )
    return r

LOG_DIR = '/home/ubuntu/logs'
STATUS_DIR = '/home/ubuntu/logs/status'

def run_in_background(node, cmd, log_filename, env=''):
    setup_cmd = f'truncate --size 0 {STATUS_DIR}/{log_filename} '
    setup_cmd += f'&& for i in {{8..0}}; do if [ -f {LOG_DIR}/{log_filename}.$i ]; then mv {LOG_DIR}/{log_filename}.$i {LOG_DIR}/{log_filename}.$((i+1)); fi done'
    run_cmd(
        node,
        f'( {setup_cmd} && {env} nohup {cmd} > {LOG_DIR}/{log_filename}.0 2>&1; nohup echo "$?" ) > {STATUS_DIR}/{log_filename} 2>&1 &'
    )

def wait_node_up(node):
    while True:
        try:
            res = node.get_validators()
            if 'error' not in res:
                assert 'result' in res
                logger.info(f'Node {node.instance_name} is up')
                return
        except (ConnectionRefusedError,
                requests.exceptions.ConnectionError) as e:
            pass
        time.sleep(10)

def prompt_setup_flags(args):
    if not args.yes:
        print(
            'this will reset all nodes\' home dirs and initialize them with new state. continue? [yes/no]'
        )
        if sys.stdin.readline().strip() != 'yes':
            sys.exit()

    if args.epoch_length is None:
        print('epoch length for the initialized genesis file?: ')
        args.epoch_length = int(sys.stdin.readline().strip())

    if args.num_validators is None:
        print('number of validators?: ')
        args.num_validators = int(sys.stdin.readline().strip())

    if args.num_seats is None:
        print('number of block producer seats?: ')
        args.num_seats = int(sys.stdin.readline().strip())

    if args.genesis_protocol_version is None:
        print('genesis protocol version?: ')
        args.genesis_protocol_version = int(sys.stdin.readline().strip())

def start_neard_runner(node):
    run_in_background(node, f'/home/ubuntu/neard-runner/venv/bin/python /home/ubuntu/neard-runner/neard_runner.py ' \
        '--home /home/ubuntu/neard-runner --neard-home /home/ubuntu/.near ' \
        '--neard-logs /home/ubuntu/neard-logs --port 3000', 'neard-runner.txt')

def upload_neard_runner(node):
    node.machine.upload('tests/mocknet/helpers/neard_runner.py',
                        '/home/ubuntu/neard-runner',
                        switch_user='ubuntu')
    node.machine.upload('tests/mocknet/helpers/requirements.txt',
                        '/home/ubuntu/neard-runner',
                        switch_user='ubuntu')

def init_neard_runner(node, config, remove_home_dir=False):
    stop_neard_runner(node)
    rm_cmd = 'rm -rf /home/ubuntu/neard-runner && ' if remove_home_dir else ''
    run_cmd(
        node,
        f'{rm_cmd}mkdir -p {LOG_DIR} && mkdir -p {STATUS_DIR} && mkdir -p /home/ubuntu/neard-runner'
    )
    upload_neard_runner(node)
    mocknet.upload_json(node, '/home/ubuntu/neard-runner/config.json', config)
    cmd = 'cd /home/ubuntu/neard-runner && python3 -m virtualenv venv -p $(which python3)' \
    ' && ./venv/bin/pip install -r requirements.txt'
    run_cmd(node, cmd)
    start_neard_runner(node)

def stop_neard_runner(node):
    # it's probably fine for now, but this is very heavy handed/not precise
    node.machine.run('kill $(ps -C python -o pid=)')

def prompt_init_flags(args):
    if args.neard_binary_url is None:
        print('neard binary URL?: ')
        args.neard_binary_url = sys.stdin.readline().strip()
        assert len(args.neard_binary_url) > 0

    if args.neard_upgrade_binary_url is None:
        print(
            'add a second neard binary URL to upgrade to mid-test? enter nothing here to skip: '
        )
        url = sys.stdin.readline().strip()
        if len(url) > 0:
            args.neard_upgrade_binary_url = url

def init_neard_runners(args, traffic_generator, nodes, remove_home_dir=False):
    prompt_init_flags(args)
    if args.neard_upgrade_binary_url is None:
        configs = [{
            "is_traffic_generator": False,
            "binaries": [{
                "url": args.neard_binary_url,
                "epoch_height": 0
            }]
        }] * len(nodes)
        traffic_generator_config = {
            "is_traffic_generator": True,
            "binaries": [{
                "url": args.neard_binary_url,
                "epoch_height": 0
            }]
        }
    else:
        # for now this test starts all validators with the same stake, so just make the upgrade
        # epoch random. If we change the stakes, we should change this to choose how much stake
        # we want to upgrade during each epoch
        configs = []
        for i in range(len(nodes)):
            configs.append({
                "is_traffic_generator":
                    False,
                "binaries": [{
                    "url": args.neard_binary_url,
                    "epoch_height": 0
                }, {
                    "url": args.neard_upgrade_binary_url,
                    "epoch_height": random.randint(1, 4)
                }]
            })
        traffic_generator_config = {
            "is_traffic_generator":
                True,
            "binaries": [{
                "url": args.neard_upgrade_binary_url,
                "epoch_height": 0
            }]
        }

    init_neard_runner(traffic_generator, traffic_generator_config,
                      remove_home_dir)
    pmap(lambda x: init_neard_runner(x[0], x[1], remove_home_dir),
         zip(nodes, configs))

def init_cmd(args, traffic_generator, nodes):
    init_neard_runners(args, traffic_generator, nodes, remove_home_dir=False)

def hard_reset_cmd(args, traffic_generator, nodes):
    print("""
        WARNING!!!!
        WARNING!!!!
        This will undo all chain state, which will force a restart from the beginning,
        icluding the genesis state computation which takes several hours.
        Continue? [yes/no]""")
    if sys.stdin.readline().strip() != 'yes':
        return
    all_nodes = nodes + [traffic_generator]
    pmap(stop_neard_runner, all_nodes)
    mocknet.stop_nodes(all_nodes)
    init_neard_runners(args, traffic_generator, nodes, remove_home_dir=True)

def restart_cmd(args, traffic_generator, nodes):
    all_nodes = nodes + [traffic_generator]
    pmap(stop_neard_runner, all_nodes)
    if args.upload_program:
        pmap(upload_neard_runner, all_nodes)
    pmap(start_neard_runner, all_nodes)

# returns boot nodes and validators we want for the new test network
def get_network_nodes(new_test_rpc_responses, num_validators):
    validators = []
    boot_nodes = []
    for ip_addr, response in new_test_rpc_responses:
        if len(validators) < num_validators:
            if response['validator_account_id'] is not None:
                # we assume here that validator_account_id is not null, validator_public_key
                # better not be null either
                validators.append({
                    'account_id': response['validator_account_id'],
                    'public_key': response['validator_public_key'],
                    'amount': str(10**33),
                })
        if len(boot_nodes) < 20:
            boot_nodes.append(
                f'{response["node_key"]}@{ip_addr}:{response["listen_port"]}')

        if len(validators) >= num_validators and len(boot_nodes) >= 20:
            break
    # neither of these should happen, since we check the number of available nodes in new_test(), and
    # only the traffic generator will respond with null validator_account_id and validator_public_key
    if len(validators) == 0:
        sys.exit('no validators available after new_test RPCs')
    if len(validators) < num_validators:
        logger.warning(
            f'wanted {num_validators} validators, but only {len(validators)} available'
        )
    return validators, boot_nodes

def new_test(args, traffic_generator, nodes):
    prompt_setup_flags(args)

    if args.epoch_length <= 0:
        sys.exit(f'--epoch-length should be positive')
    if args.num_validators <= 0:
        sys.exit(f'--num-validators should be positive')
    if len(nodes) < args.num_validators:
        sys.exit(
            f'--num-validators is {args.num_validators} but only found {len(nodes)} under test'
        )

    all_nodes = nodes + [traffic_generator]

    logger.info(f'resetting/initializing home dirs')
    test_keys = pmap(neard_runner_new_test, all_nodes)

    validators, boot_nodes = get_network_nodes(
        zip([n.machine.ip for n in all_nodes], test_keys), args.num_validators)

    logger.info("""setting validators: {0}
Then running neard amend-genesis on all nodes, and starting neard to compute genesis \
state roots. This will take a few hours. Run `status` to check if the nodes are \
ready. After they're ready, you can run `start-traffic`""".format(validators))
    pmap(
        lambda node: neard_runner_network_init(
            node, validators, boot_nodes, args.epoch_length, args.num_seats,
            args.genesis_protocol_version), all_nodes)

def status_cmd(args, traffic_generator, nodes):
    all_nodes = nodes + [traffic_generator]
    statuses = pmap(neard_runner_ready, all_nodes)
    num_ready = 0
    not_ready = []
    for ready, node in zip(statuses, all_nodes):
        if not ready:
            not_ready.append(node.instance_name)

    if len(not_ready) == 0:
        print(f'all {len(all_nodes)} nodes ready')
    else:
        print(
            f'{len(all_nodes)-len(not_ready)}/{len(all_nodes)} ready. Nodes not ready: {not_ready[:3]}'
        )

def reset_cmd(args, traffic_generator, nodes):
    print(
        'this will reset all nodes\' home dirs to their initial states right after test initialization finished. continue? [yes/no]'
    )
    if sys.stdin.readline().strip() != 'yes':
        sys.exit()
    all_nodes = nodes + [traffic_generator]
    pmap(neard_runner_reset, all_nodes)
    logger.info(
        'Data dir reset in progress. Run the `status` command to see when this is finished. Until it is finished, neard runners may not respond to HTTP requests.'
    )

def stop_nodes_cmd(args, traffic_generator, nodes):
    pmap(neard_runner_stop, nodes + [traffic_generator])

def stop_traffic_cmd(args, traffic_generator, nodes):
    neard_runner_stop(traffic_generator)

def neard_runner_jsonrpc(node, method, params=[]):
    j = {'method': method, 'params': params, 'id': 'dontcare', 'jsonrpc': '2.0'}
    r = requests.post(f'http://{node.machine.ip}:3000', json=j, timeout=30)
    if r.status_code != 200:
        logger.warning(
            f'bad response {r.status_code} trying to send {method} JSON RPC to neard runner on {node.instance_name}:\n{r.content}'
        )
    r.raise_for_status()
    return r.json()['result']

def neard_runner_start(node):
    neard_runner_jsonrpc(node, 'start')

def neard_runner_stop(node):
    neard_runner_jsonrpc(node, 'stop')

def neard_runner_new_test(node):
    return neard_runner_jsonrpc(node, 'new_test')

def neard_runner_network_init(node, validators, boot_nodes, epoch_length,
                              num_seats, protocol_version):
    return neard_runner_jsonrpc(node,
                                'network_init',
                                params={
                                    'validators': validators,
                                    'boot_nodes': boot_nodes,
                                    'epoch_length': epoch_length,
                                    'num_seats': num_seats,
                                    'protocol_version': protocol_version,
                                })

def neard_update_config(node, state_cache_size_mb, state_snapshot_enabled):
    return neard_runner_jsonrpc(
        node,
        'update_config',
        params={
            'state_cache_size_mb': state_cache_size_mb,
            'state_snapshot_enabled': state_snapshot_enabled,
        },
    )

def update_config_cmd(args, traffic_generator, nodes):
    nodes = nodes + [traffic_generator]
    results = pmap(
        lambda node: neard_update_config(
            node,
            args.state_cache_size_mb,
            args.state_snapshot_enabled,
        ),
        nodes,
    )
    if not all(results):
        logger.warn('failed to update configs for some nodes')
        return

def neard_runner_ready(node):
    return neard_runner_jsonrpc(node, 'ready')

def neard_runner_reset(node):
    return neard_runner_jsonrpc(node, 'reset')

def start_nodes_cmd(args, traffic_generator, nodes):
    if not all(pmap(neard_runner_ready, nodes)):
        logger.warn(
            'not all nodes are ready to start yet. Run the `status` command to check their statuses'
        )
        return
    pmap(neard_runner_start, nodes)
    pmap(wait_node_up, nodes)

def start_traffic_cmd(args, traffic_generator, nodes):
    if not all(pmap(neard_runner_ready, nodes + [traffic_generator])):
        logger.warn(
            'not all nodes are ready to start yet. Run the `status` command to check their statuses'
        )
        return
    pmap(neard_runner_start, nodes)
    logger.info("waiting for validators to be up")
    pmap(wait_node_up, nodes)
    logger.info(
        "waiting a bit after validators started before starting traffic")
    time.sleep(10)
    neard_runner_start(traffic_generator)
    logger.info(
        f'test running. to check the traffic sent, try running "curl http://{traffic_generator.machine.ip}:3030/metrics | grep mirror"'
    )

if __name__ == '__main__':
    parser = ArgumentParser(description='Run a load test')
    parser.add_argument('--chain-id', type=str, required=True)
    parser.add_argument('--start-height', type=int, required=True)
    parser.add_argument('--unique-id', type=str, required=True)

    subparsers = parser.add_subparsers(title='subcommands',
                                       description='valid subcommands',
                                       help='additional help')

    init_parser = subparsers.add_parser('init-neard-runner',
                                        help='''
    Sets up the helper servers on each of the nodes. Doesn't start initializing the test
    state, which is done with the `new-test` command.
    ''')
    init_parser.add_argument('--neard-binary-url', type=str)
    init_parser.add_argument('--neard-upgrade-binary-url', type=str)
    init_parser.set_defaults(func=init_cmd)

    update_config_parser = subparsers.add_parser(
        'update-config',
        help='''Update config.json with given flags for all nodes.''')
    update_config_parser.add_argument('--state-cache-size-mb', type=int)
    update_config_parser.add_argument('--state-snapshot-enabled',
                                      action=BooleanOptionalAction)
    update_config_parser.set_defaults(func=update_config_cmd)

    restart_parser = subparsers.add_parser(
        'restart-neard-runner',
        help='''Restarts the neard runner on all nodes.''')
    restart_parser.add_argument('--upload-program', action='store_true')
    restart_parser.set_defaults(func=restart_cmd, upload_program=False)

    hard_reset_parser = subparsers.add_parser(
        'hard-reset',
        help='''Stops neard and clears all test state on all nodes.''')
    hard_reset_parser.add_argument('--neard-binary-url', type=str)
    hard_reset_parser.add_argument('--neard-upgrade-binary-url', type=str)
    hard_reset_parser.set_defaults(func=hard_reset_cmd)

    new_test_parser = subparsers.add_parser('new-test',
                                            help='''
    Sets up new state from the prepared records and genesis files with the number
    of validators specified. This calls neard amend-genesis to create the new genesis
    and records files, and then starts the neard nodes and waits for them to be online
    after computing the genesis state roots. This step takes a long time (a few hours).
    ''')
    new_test_parser.add_argument('--epoch-length', type=int)
    new_test_parser.add_argument('--num-validators', type=int)
    new_test_parser.add_argument('--num-seats', type=int)
    new_test_parser.add_argument('--genesis-protocol-version', type=int)
    new_test_parser.add_argument('--yes', action='store_true')
    new_test_parser.set_defaults(func=new_test)

    status_parser = subparsers.add_parser(
        'status',
        help='''Checks the status of test initialization on each node''')
    status_parser.set_defaults(func=status_cmd)

    start_traffic_parser = subparsers.add_parser(
        'start-traffic',
        help=
        'Starts all nodes and starts neard mirror run on the traffic generator.'
    )
    start_traffic_parser.set_defaults(func=start_traffic_cmd)

    start_nodes_parser = subparsers.add_parser(
        'start-nodes',
        help='Starts all nodes, but does not start the traffic generator.')
    start_nodes_parser.set_defaults(func=start_nodes_cmd)

    stop_parser = subparsers.add_parser('stop-nodes',
                                        help='kill all neard processes')
    stop_parser.set_defaults(func=stop_nodes_cmd)

    stop_parser = subparsers.add_parser(
        'stop-traffic',
        help='stop the traffic generator, but leave the other nodes running')
    stop_parser.set_defaults(func=stop_traffic_cmd)

    reset_parser = subparsers.add_parser('reset',
                                         help='''
    The new_test command saves the data directory after the genesis state roots are computed so that
    the test can be reset from the start without having to do that again. This command resets all nodes'
    data dirs to what was saved then, so that start-traffic will start the test all over again.
    ''')
    reset_parser.set_defaults(func=reset_cmd)

    args = parser.parse_args()

    traffic_generator, nodes = get_nodes(args)
    args.func(args, traffic_generator, nodes)


from ShippingCost.logger import logging
from ShippingCost.exception import ShippingException
from ShippingCost.entity import config_entity
from ShippingCost.entity import artifact_entity
import sys, os
import pandas as pd
import numpy as np
from sklearn.preprocessing import OneHotEncoder
from sklearn.impute import KNNImputer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import RobustScaler
from ShippingCost.utils import save_numpy_array_data, load_numpy_array_data , save_object, load_object

class DataTransformation:
    def __init__(self,
                 data_validation_artifact: artifact_entity.DataValidationArtifact,
                 data_transformation_config: config_entity.DataTransformationConfig) -> None:

        try:
            self.data_transformation_config = data_transformation_config
            self.data_validation_artifact = data_validation_artifact

        except Exception as e:
            raise ShippingException(e, sys)

    def random_imputation(self,df, cols):
        try:
            for col in cols:
                imputed_col_values = np.random.choice(df[~df[col].isna()][col].values, size = df[col].isna().sum())
                col_null_indices = df[df[col].isna()].index
                df.loc[col_null_indices, col] = imputed_col_values
            return df

        except Exception as e:
            raise ShippingException(e, sys)

    def remove_outliers_iqr(self,dataframe, cols):
        """
        Remove outliers from a pandas DataFrame using the IQR method.

        Parameters:
        dataframe (pandas.DataFrame): Input DataFrame.
        cols (str): List of columns to remove outliers from.

        Returns:
        pandas.DataFrame: DataFrame with outliers removed.

        """
        for col in cols:
            data = dataframe[col]
            Q1 = data.quantile(0.25)
            Q3 = data.quantile(0.75)
            IQR = Q3 - Q1
            lower_bound = Q1 - 1.5 * IQR
            upper_bound = Q3 + 1.5 * IQR
            cleaned_dataframe = dataframe[(data > lower_bound) & (data < upper_bound)]
        return cleaned_dataframe

    def initiate_data_transformation(self):
        try:
            logging.info(f"{'<<'*20} Data Transformation {'>>' * 20}")
            logging.info('Loading validated dataset')
            df = pd.read_csv(self.data_validation_artifact.validated_dataset_path)
            #Randomly imputing missing values of Height, Width and Artist Reputation
            df['Cost'] = df['Cost'].abs()

            logging.info(f"Null values before imputation in Height {df['Height'].isna().sum()}")
            logging.info('Randomly Imputing missing values of Height')
            df = self.random_imputation(df , ['Height','Width', 'Artist Reputation', 'Remote Location','Transport','Material'])

            #Finding Numerical and Categorical Columns
            cat_cols = [col for col in df.columns if df[col].dtype == 'object']
            num_cols = ['Artist Reputation','Height','Width','Weight','Price Of Sculpture','Base Shipping Price']

            logging.info(f'Categorical Columns {cat_cols}')
            logging.info(f'Numerical Columns {num_cols}')

            #One-hot Encoding

            logging.info('One-hot Encoding the Categorical Columns')
            logging.info(f'Shape of Dataframe before one hot encoding {df.shape}')
            logging.info('Creating one-hot Encoder object')
            encoder = OneHotEncoder(sparse_output= False, handle_unknown = 'ignore')

            logging.info('One hot encoding "Material, International, Express Shipment, Installation Included, Transport, Fragile, Customer Information, Remote Location')
            df_encoded = encoder.fit_transform(df[cat_cols])
            df_encoded = pd.DataFrame(df_encoded,  columns = encoder.get_feature_names_out(cat_cols))

            logging.info('OneHotEncoding completed Successfully')

            X = pd.concat([df[num_cols] , df_encoded], axis= 1)
            logging.info(f'Shape of Dataframe after one hot encoding {X.shape}')

            #Performing KNN Imputation on dataset to impute values of weight column
            logging.info('Imputing null values of Weight with KNN imputer')
            logging.info(f"Finding null values in weight {X['Weight'].isna().sum()}")
            imputer = KNNImputer(n_neighbors=5)
            imputed_df = imputer.fit_transform(X)
            X = pd.DataFrame(imputed_df, columns=X.columns)
            logging.info(f"Finding null values in weight {X['Weight'].isna().sum()}")

            df_new = pd.concat([X , df['Cost']], axis =1)

            #Removing outliers

            logging.info("Removing Outliers from  'Height','Width','Weight','Price Of Sculpture'")
            logging.info(f"Shape of the dataframe before removing outliers{df_new.shape}")
            df_new = self.remove_outliers_iqr(df_new, [ 'Height','Width','Weight','Price Of Sculpture'])

            logging.info(f"Shape of the dataframe after removing outliers{df_new.shape}")

            logging.info('Separating target column from dataframe')
            X = df_new.drop('Cost', axis = 1)
            y = df_new['Cost']

            #Splitting training and testing data
            logging.info(f'X_train feature names {X.columns} ')
            logging.info('Splitting training and testing data')
            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = .3, random_state = 42)

            #Performing Feature Scaling
            logging.info('Feature Scaling')
            scaler = RobustScaler()
            X_train = scaler.fit_transform(X_train)
            X_test = scaler.transform(X_test)

            logging.info('Creating transformation path directory')
            transformation_file_dir = os.path.dirname(self.data_transformation_config.data_transformation_dir)
            os.makedirs(transformation_file_dir, exist_ok= True)

            #Saving X_train, X_test, y_train, y_test in data transformation path
            logging.info('Saving X_train, X_test, y_train, y_test in data transformation path')
            save_numpy_array_data(file_path=self.data_transformation_config.Xtrain_dataset_path, array=X_train)
            save_numpy_array_data(file_path=self.data_transformation_config.Ytrain_dataset_path, array=y_train)
            save_numpy_array_data(file_path=self.data_transformation_config.Xtest_dataset_path, array=X_test)
            save_numpy_array_data(file_path=self.data_transformation_config.Ytest_dataset_path, array=y_test)
            logging.info('Saving One Hot Encoder Object')
            save_object(file_path= self.data_transformation_config.ohe_object_path, obj=encoder)
            save_object(file_path=self.data_transformation_config.scaler_object_path, obj = scaler)

            logging.info('Successfully saved Train, test dataset, one hot encoder object and scaler object')

            #Creating Data Transformation Artifact
            logging.info('Creating data transformation Artifact')
            data_transformation_artifact = artifact_entity.DataTransformationArtifact(Xtrain_dataset_path =self.data_transformation_config.Xtrain_dataset_path,
                                                                                Xtest_dataset_path= self.data_transformation_config.Xtest_dataset_path,
                                                                                Ytrain_dataset_path=self.data_transformation_config.Ytrain_dataset_path,
                                                                                Ytest_dataset_path=self.data_transformation_config.Ytest_dataset_path,
                                                                                ohe_object_path= self.data_transformation_config.ohe_object_path,
                                                                                scaler_object_path=self.data_transformation_config.scaler_object_path
                                                                                )

            logging.info(f'Data Transformation Artifact {data_transformation_artifact}')

            return data_transformation_artifact

        except Exception as e:
            raise ShippingException(e, sys)


import shutil
import streamlit as st
from streamlit_option_menu import option_menu
#---------------------------------------------------------------------------------------------
st.set_page_config(page_title="Natural Neural Vision Algorithm", page_icon=":house:", layout="wide")

with st.sidebar:
    choose = option_menu("Natural Neural Vision Algorithms", ["About", "DL Model","Data Analysis", "Contact Us"],
                         icons=['house-fill', 'camera-reels-fill','bar-chart-line-fill','person-fill'],

                         menu_icon="app-indicator", default_index=0,
                         styles={
        "container": {"padding": "5!important", "background-color": "#fafafa"},
        "icon": {"color": "orange", "font-size": "25px"},
        "nav-link": {"font-size": "16px", "text-align": "left", "margin":"0px", "--hover-color": "#eee"},
        "nav-link-selected": {"background-color": "#02ab21"},

    },
    )

#---------------------------------------------------------------------------------------------------
from about_page import about_page
from contact_page import contact_page
from edit_page import edit_page
from dividing import analyse

if choose == "About":
    about_page()
    try:
        shutil.rmtree("images")
    except FileNotFoundError:
        print("")
elif choose == "DL Model":
    edit_page()

elif choose=="Data Analysis":
    analyse()
    try:
        shutil.rmtree("images")
    except FileNotFoundError:
        print("")
elif choose == "Contact Us":
    contact_page()
    try:
        shutil.rmtree("images")
    except FileNotFoundError:
        print("")


from hactivate.model.declarative_objects import *
from hactivate.model.meta import Session

import logging
log = logging.getLogger(__name__)

def init_base_data():
        log.info("Populating tables with base test data")

        u1 = User()
        u1.username      = u"test"
        u1.name          = "TES MONKEY MAN"

        u2 = User()
        u2.username      = u"elroid"
        u2.name          = "Elliot Test"

        Session.add_all([u1,u2])
        Session.commit()

        uc1 = UserContact()
        uc1.contact_type = "sms"
        uc1.data = "447928065717"
        uc1.contact_direction_type = "out"
        u2.contacts.append(uc1)

        # -- Items ---

        i1 = Item()
        #i1.user_id              = 1
        i1.title                = "Bike Seat"
        i1.description          = "You sit on it"
        i1.item_type            = "item"
        i1.direction_type       = "offer"
        i1.lon                  = 0.0
        i1.lat                  = 0.0
        u1.items.append(i1)

        i0 = Item()
        #i0.user_id              = 1
        i0.title                = "Necromicon"
        i0.description          = "Don't read it"
        i0.item_type            = "item"
        i0.direction_type       = "offer"
        i0.lon                  = 1.0
        i0.lat                  = 1.0
        u1.items.append(i0)

        # -- Searchs ---

        s1 = UserSearch()
        s1.radius = 1
        s1.lon =  -0.120335
        s1.lat =  51.532255
        s1.keywords = "bike"
        u1.searchs.append(s1)

        s1 = UserSearch()
        s1.radius = 1
        s1.lon =  -0.133188
        s1.lat =  51.539662
        s1.keywords = "ladder"
        u1.searchs.append(s1)

        s1 = UserSearch()
        s1.radius = 1
        s1.lon = -2.357254
        s1.lat =  51.386352
        s1.keywords = "mattress"
        u2.searchs.append(s1)

        s1 = UserSearch()
        s1.radius = 1
        s1.lon = -2.357254
        s1.lat =  51.386352
        s1.keywords = "camera"
        u2.searchs.append(s1)

        #Session.add_all([i1, i0])
        Session.commit()


import matplotlib.pyplot as plt
import numpy as np
import matplotlib
plt.rcParams.update({
    "font.family": "serif",
    "font.serif": [],
    "font.sans-serif": [],
})

f = ['FX_','FY_','FZ_']

plt.figure(1)
a = open('OMADA 12/'+f[0]+'1.lvm','r')
l = readlines(1)
l = l[-2:]
plt.plot(l)


import shlex, subprocess
from StringIO import StringIO
from django.db import models
from django.db.models.fields import IPAddressField
import paramiko
from paramiko.rsakey import RSAKey
from expedient.clearinghouse.aggregate.models import Aggregate
from expedient.clearinghouse.resources.models import Resource, Sliver
from expedient.common.utils.modelfields import LimitedIntegerField
from expedient.common.middleware import threadlocals
from expedient.clearinghouse.utils import post_message_to_current_user
from expedient.common.messaging.models import DatedMessage
from expedient.clearinghouse.slice.models import Slice

# SSHServer class
class SSHServer(Resource):
    # SSHServer fields
    ip_address = IPAddressField(
        "IP address",
        help_text="Specify the server's IP address.",
    )
    ssh_port = LimitedIntegerField(
        "SSH port number",
        min_value=1,
        max_value=2**16-1,
        default=22,
        help_text="Specify the SSH port number to use."
    )
    # end

    def is_alive(self):
        """Ping the server and check if it's alive.

        @return: True if ping succeeds, False otherwise.
        """
        ret = subprocess.call(
            shlex.split("ping -c 1 -W 2 %s" % self.ip_address),
            stdout=open('/dev/null', 'w'),
            stderr=subprocess.STDOUT,
        )

        if ret == 0:
            return True
        else:
            return False

    def exec_command(self, command, **connection_info):
        """Connect to the server using an SSH session and execute a command.

        @param command: The command to execute
        @type command: C{str}
        @param username: The username to use to connect to the server.
        @type username: C{str}
        @keyword connection_info: A dict of other info to pass to
            C{paramiko.SSHClient.exec_command}.
        @return: A (out, err) tuple that is the output read on the
            stdout and stderr channels.
        @rtype: C{tuple(str, str)}
        """

        client = paramiko.SSHClient()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        client.connect(
            str(self.ip_address),
            port=int(self.ssh_port),
            **connection_info
        )
        _, sout, serr = client.exec_command(command)
        o = sout.read()
        e = serr.read()
        client.close()
        return o, e

    def __unicode__(self):
        return u"SSH server at IP %s" % self.ip_address

class SSHServerSliver(Sliver): pass

class SSHSliceInfo(models.Model):
    slice = models.OneToOneField(Slice)
    public_key = models.TextField()

# SSHAggregate class
class SSHAggregate(Aggregate):
    # SSHAggregate information field
    information = "An aggregate of SSH servers that are controlled" \
        " by a single administrator, to which users can request" \
        " access. Once approved, users get SSH access to all" \
        " machines using a public key they provide."
    # SSHAggregate end information field

    # SSHAggregate meta
    class Meta:
        verbose_name = "SSH Aggregate"
    # SSHAggregate end meta

    # SSHAggregate required fields
    admin_username = models.CharField(max_length=255)
    private_key = models.TextField()
    # SSHAggregate end required fields

    # SSHAggregate optional fields
    add_user_command = models.TextField(
        default="sh -c 'sudo useradd -m %(username)s'",
        help_text="Specify the command to create a new user. " \
            "'%(username)s' will be replaced by the user's " \
            " username. The command should return non-zero on failure " \
            " and 0 on success.",
    )
    del_user_command = models.TextField(
        default="sh -c 'sudo userdel -r -f %(username)s'",
        help_text="Specify the command to delete an existing user. " \
            "'%(username)s' will be replaced by the user's " \
            " username. The command should return non-zero on failure " \
            " and 0 on success.",
    )
    add_pubkey_user_command = models.TextField(
        default="sudo -u %(username)s mkdir /home/%(username)s/.ssh; "
            "sudo -u %(username)s chmod 700 /home/%(username)s/.ssh; "
            "sh -c 'sudo -u %(username)s echo %(pubkey)s >> "
            "/home/%(username)s/.ssh/authorized_keys'",
        help_text="Specify the command to add a public key to a user's " \
            "account. '%(username)s' will be replaced by the user's " \
            " username and '%(pubkey)s' will be replaced by the public key." \
            " The command should return non-zero on failure " \
            " and 0 on success.",
    )
    # SSHAggregate end optional fields

    def _op_user(self, op, server, cmd_subs, quiet=False):
        """common code for adding/removing users."""

        pkey_f = StringIO(self.private_key)
        pkey = RSAKey.from_private_key(pkey_f)
        pkey_f.close()

        cmd = getattr(self, "%s_user_command" % op) % cmd_subs
        cmd = cmd + "; echo $?"
        out, err = server.exec_command(
            cmd,
            username=str(self.admin_username),
            pkey=pkey,
        )

        lines = out.strip().split("\n")
        ret = int(lines[-1])

        if ret != 0:
            error = "".join(lines[:-1])
            if not quiet:
                # msg example
                msg = "Failed to %s user on %s. Output was:\n%s" \
                    % (op, server, error),
                post_message_to_current_user(
                    msg,
                    msg_type=DatedMessage.TYPE_ERROR,
                )
                # end msg example
            raise Exception(msg)

    def add_user(self, server, username, pubkey, quiet=False):
        """Add a user to a server.

        Add a user with username C{username} with public key C{pubkey} to
        server C{server}.

        @param server: The server to add the user to.
        @type server: L{SSHServer}
        @param username: the new user's username
        @type username: C{str}
        @param pubkey: The public key to add to the user's account.
        @type pubkey: the public key's value a C{str}
        @keyword quiet: If True, no messages will be sent on failure.
            Defaults to False.
        @type quiet: C{boolean}
        """
        self._op_user("add", server, {"username": username}, quiet)
        self._op_user(
            "add_pubkey",
            server,
            {"username": username, "pubkey": pubkey},
            quiet,
        )

    def del_user(self, server, username, quiet=False):
        """Remove user from a server.

        Remove user with username C{username} from server C{server}.

        @param server: The server to remove the user from.
        @type server: L{SSHServer}
        @param username: the user's username
        @type username: C{str}
        @keyword quiet: If True, no messages will be sent on failure.
            Defaults to False.
        @type quiet: C{boolean}
        """
        self._op_user("del", server, {"username": username}, quiet)

    def check_status(self):
        return self.available and reduce(
            lambda x, y: x and y.is_alive(),
            SSHServer.objects.filter(aggregate__id=self.id),
    )

    # start_slice func
    def start_slice(self, slice):
        # start_slice call super
        super(SSHAggregate, self).start_slice(slice)
        # start_slice end call super

        # start_slice get info
        slice_info = SSHSliceInfo.objects.get(slice=slice)
        # start_slice get user
        user = slice.owner
        # start_slice get slivers
        slivers = SSHServerSliver.objects.filter(
            slice=slice, resource__aggregate__id=self.id)
        # start_slice end info

        # start_slice loop
        succeeded = []
        for sliver in slivers:
            # Execute the command on the server and get status
            server = sliver.resource.as_leaf_class()
            # start_slice add user
            try:
                self.add_user(server, user.username, slice_info.public_key)
            except:
                for s in succeeded:
                    try:
                        self.del_user(s, user.username)
                    except:
                        pass
                raise

            succeeded.append(server)
            # start_slice end loop

    def stop_slice(self, slice):
        super(SSHAggregate, self).start_slice(slice)
        user = threadlocals.get_thread_locals()["user"]
        for sliver in SSHServerSliver.objects.filter(slice=slice):
            server = sliver.resource.as_leaf_class()
            try:
                self.del_user(server, user.username)
            except:
                pass


import matplotlib.pyplot as plt

def save(show_plot=0, save=None, dpi=300, fmt='.png', verbose=True, **kwargs):
    """

    Optional:
        show_plot : Show plot.
        save : Save figure to file.

    """
    if save:
        filename = save + fmt
        if verbose:
            print("Writing:", filename)
        plt.savefig(save, dpi=300, **kwargs)
    if show_plot:
        plt.show()