# byteplay: CPython assembler/disassembler
# Copyright (C) 2006 Noam Raphael | Version: http://code.google.com/p/byteplay
# Rewritten 2009 Demur Rumed | Version: http://github.com/serprex/byteplay
#                            Screwed the style over, modified stack logic to be more flexible, updated to Python 3
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

__version__ = '0.4'
__all__ = [
    'opmap',
    'opname',
    'opcodes',
    'hasflow',
    'stack_effect',
    'cmp_op',
    'hasarg',
    'hasname',
    'hasjrel',
    'hasjabs',
    'hasjump',
    'haslocal',
    'hascompare',
    'hasfree',
    'hascode',
    'hasconst',
    'Opcode',
    'SetLineno',
    'Label',
    'isopcode',
    'Code']

from sys import version_info
if version_info < (3, 4):
    raise NotImplementedError("Currently only Python versions 3.4 and 3.5 are supported!")

import opcode
from dis import findlabels
from types import CodeType
from enum import Enum

class Opcode(int):
    __str__ = __repr__ = lambda s: opname[s]

opmap = {name.replace('+', '_'): Opcode(code) for name, code in opcode.opmap.items()}
opname = {code: name for name, code in opmap.items()}
opcodes = set(opname)
for cmp_op, hasarg in opmap.items():
    globals()[cmp_op] = hasarg
    __all__.append(cmp_op)
cmp_op = opcode.cmp_op

hasarg = {x for x in opcodes if x >= opcode.HAVE_ARGUMENT}
hasconst = {Opcode(x) for x in opcode.hasconst}
hasname = {Opcode(x) for x in opcode.hasname}
hasjrel = {Opcode(x) for x in opcode.hasjrel}
hasjabs = {Opcode(x) for x in opcode.hasjabs}
hasjump = hasjabs | hasjrel
haslocal = {Opcode(x) for x in opcode.haslocal}
hascompare = {Opcode(x) for x in opcode.hascompare}
hasfree = {Opcode(x) for x in opcode.hasfree}
hascode = {MAKE_FUNCTION, MAKE_CLOSURE}

if version_info.minor > 2:
    STOP_CODE = -1
if version_info.minor < 4:
    _se = {
        IMPORT_FROM: 1,
        DUP_TOP: 1,
        LOAD_CONST: 1,
        LOAD_NAME: 1,
        LOAD_GLOBAL: 1,
        LOAD_FAST: 1,
        LOAD_CLOSURE: 1,
        LOAD_DEREF: 1,
        BUILD_MAP: 1,
        LOAD_BUILD_CLASS: 1,
        YIELD_VALUE: 0,
        UNARY_POSITIVE: 0,
        UNARY_NEGATIVE: 0,
        UNARY_NOT: 0,
        UNARY_INVERT: 0,
        GET_ITER: 0,
        LOAD_ATTR: 0,
        IMPORT_NAME: 0,
        ROT_TWO: 0,
        ROT_THREE: 0,
        NOP: 0,
        DELETE_GLOBAL: 0,
        DELETE_NAME: 0,
        DELETE_FAST: 0,
        STORE_LOCALS: 0,
        IMPORT_NAME: -1,
        POP_TOP: -1,
        PRINT_EXPR: -1,
        IMPORT_STAR: -1,
        DELETE_ATTR: -1,
        STORE_DEREF: -1,
        STORE_NAME: -1,
        STORE_GLOBAL: -1,
        STORE_FAST: -1,
        BINARY_POWER: -1,
        BINARY_MULTIPLY: -1,
        BINARY_FLOOR_DIVIDE: -1,
        BINARY_TRUE_DIVIDE: -1,
        BINARY_MODULO: -1,
        BINARY_ADD: -1,
        BINARY_SUBTRACT: -1,
        BINARY_SUBSCR: -1,
        BINARY_LSHIFT: -1,
        BINARY_RSHIFT: -1,
        BINARY_AND: -1,
        BINARY_XOR: -1,
        BINARY_OR: -1,
        COMPARE_OP: -1,
        INPLACE_POWER: -1,
        INPLACE_MULTIPLY: -1,
        INPLACE_FLOOR_DIVIDE: -1,
        INPLACE_TRUE_DIVIDE: -1,
        INPLACE_MODULO: -1,
        INPLACE_ADD: -1,
        INPLACE_SUBTRACT: -1,
        INPLACE_LSHIFT: -1,
        INPLACE_RSHIFT: -1,
        INPLACE_AND: -1,
        INPLACE_XOR: -1,
        INPLACE_OR: -1,
        LIST_APPEND: -1,
        SET_ADD: -1,
        DELETE_SUBSCR: -2,
        STORE_ATTR: -2,
        STORE_MAP: -2,
        MAP_ADD: -2,
        STORE_SUBSCR: -3}
    _rf = {CALL_FUNCTION: lambda x: -((x & 0xFF00) >> 7) - (x & 0xFF),
           CALL_FUNCTION_VAR_KW: lambda x: -((x & 0xFF00) >> 7) - (x & 0xFF) - 2,
           CALL_FUNCTION_VAR: lambda x: -((x & 0xFF00) >> 7 | 1) - (x & 0xFF),
           CALL_FUNCTION_KW: lambda x: -((x & 0xFF00) >> 7 | 1) - (x & 0xFF),
           RAISE_VARARGS: lambda x: x,
           MAKE_FUNCTION: lambda x: x,
           UNPACK_EX: lambda x: (x & 0xFF) + (x >> 8),
           UNPACK_SEQUENCE: lambda x: x - 1,
           MAKE_CLOSURE: lambda x: x - 1,
           BUILD_TUPLE: lambda x: 1 - x,
           BUILD_LIST: lambda x: 1 - x,
           BUILD_SET: lambda x: 1 - x,
           BUILD_SLICE: lambda x: 1 - x}
    if version_info.minor > 1:
        _se[DUP_TOP_TWO] = 2
        _se[DELETE_DEREF] = 0
    else:
        _se[ROT_FOUR] = 0
        _rf[DUP_TOPX] = lambda x: x

    def stack_effect(op, arg=None):
        if op in _se:
            return _se[op]
        if arg is None:
            raise ValueError("%s requires arg" % op)
        if op in _rf:
            return _rf[op](arg)
        raise ValueError("Unknown %s %s" % (op, arg))
else:
    from dis import stack_effect

hasflow = hasjump | {
    POP_BLOCK,
    END_FINALLY,
    BREAK_LOOP,
    RETURN_VALUE,
    RAISE_VARARGS,
    STOP_CODE,
    POP_EXCEPT}

if version_info < (3, 5):
    hasflow |= {WITH_CLEANUP}
else:
    hasflow |= {WITH_CLEANUP_START, WITH_CLEANUP_FINISH, SETUP_ASYNC_WITH}
    coroutine_opcodes = {GET_AWAITABLE, GET_AITER, GET_ANEXT, BEFORE_ASYNC_WITH, SETUP_ASYNC_WITH}

class Label:
    pass

class SetLinenoType:
    def __repr__(self):
        return 'SetLineno'
SetLineno = SetLinenoType()

def isopcode(x):
    return x is not SetLineno and not isinstance(x, Label)

# Flags for codeobject.co_flags, taken from Include/code.h, other flags are no longer used
CO_OPTIMIZED   = 0x0001
CO_NEWLOCALS   = 0x0002
CO_VARARGS     = 0x0004
CO_VARKEYWORDS = 0x0008
CO_NESTED      = 0x0010
CO_GENERATOR   = 0x0020
CO_NOFREE      = 0x0040

if version_info >= (3, 5):
    CO_COROUTINE          = 0x0080
    CO_ITERABLE_COROUTINE = 0x0100

CO_FUTURE_BARRY_AS_BDFL = 0x40000

if version_info >= (3, 5):
    CO_FUTURE_GENERATOR_STOP = 0x80000

class Code(object):
    """An object which holds all the information which a Python code object
    holds, but in an easy-to-play-with representation

    The attributes are:

    Affecting action
    code - list of 2-tuples: the code
    freevars - list of strings: the free vars of the code (those are names
               of variables created in outer functions and used in the function)
    args - list of strings: the arguments of the code
    kwonly - number of keyword only arguments
    varargs - boolean: Does args end with a '*args' argument
    varkwargs - boolean: Does args end with a '**kwargs' argument
    newlocals - boolean: Should a new local namespace be created
                (True in functions, False for module and exec code)

    force_generator - set CO_GENERATOR in co_flags for generator Code objects without generator-specific code
    Python 3.5:
        force_coroutine - set CO_COROUTINE in co_flags for coroutine Code objects (native coroutines) without coroutine-specific code
        force_iterable_coroutine - set CO_ITERABLE_COROUTINE in co_flags for generator-based coroutine Code objects
        future_generator_stop - set CO_FUTURE_GENERATOR_STOP flag (see PEP-479)

    Not affecting action
    name - string: the name of the code (co_name)
    filename - string: the file name of the code (co_filename)
    firstlineno - int: the first line number (co_firstlineno)
    docstring - string or None: the docstring (the first item of co_consts,
                if it's str)

    code is a list of 2-tuples. The first item is an opcode, or SetLineno, or a
    Label instance. The second item is the argument, if applicable, or None"""

    def __init__(self, code, freevars, args, kwonly, varargs, varkwargs, newlocals,
                 name, filename, firstlineno, docstring,
                 force_generator=False,
                 *, force_coroutine=None, force_iterable_coroutine=None, future_generator_stop=None):
        self.code = code
        self.freevars = freevars
        self.args = args
        self.kwonly = kwonly
        self.varargs = varargs
        self.varkwargs = varkwargs
        self.newlocals = newlocals
        self.name = name
        self.filename = filename
        self.firstlineno = firstlineno
        self.docstring = docstring
        self.force_generator = force_generator
        if version_info < (3, 5):
            # Flags unsupported in earlier versions
            assert force_coroutine is None and force_iterable_coroutine is None and future_generator_stop is None
        else:
            self.force_coroutine = force_coroutine
            self.force_iterable_coroutine = force_iterable_coroutine
            self.future_generator_stop = future_generator_stop

    @staticmethod
    def _findlinestarts(code):
        """Find the offsets in a byte code which are start of lines in the source
        Generate pairs offset,lineno as described in Python/compile.c
        This is a modified version of dis.findlinestarts, which allows multiplelinestarts
        with the same line number"""
        lineno = code.co_firstlineno
        addr = 0
        for byte_incr, line_incr in zip(code.co_lnotab[0::2], code.co_lnotab[1::2]):
            if byte_incr:
                yield addr, lineno
                addr += byte_incr
            lineno += line_incr
        yield addr, lineno

    @classmethod
    def from_code(cls, co):
        """Disassemble a Python code object into a Code object"""
        free_cell_isection = set(co.co_cellvars) & set(co.co_freevars)
        if free_cell_isection:
            print(co.co_name + ': has non-empty co.co_cellvars & co.co_freevars', free_cell_isection)
            return None

        co_code = co.co_code
        labels = {addr: Label() for addr in findlabels(co_code)}
        linestarts = dict(cls._findlinestarts(co))
        cellfree = co.co_cellvars + co.co_freevars
        code = []
        n = len(co_code)
        i = extended_arg = 0
        is_generator = False
        if version_info >= (3, 5):
            is_coroutine = False

        while i < n:
            op = Opcode(co_code[i])
            if i in labels:
                code.append((labels[i], None))
            if i in linestarts:
                code.append((SetLineno, linestarts[i]))
            i += 1
            if op in hascode:
                lastop, lastarg = code[-2]
                if lastop != LOAD_CONST:
                    raise ValueError("%s should be preceded by LOAD_CONST" % op)

                sub_code = Code.from_code(lastarg)
                if sub_code is None:
                    print(co.co_name + ': has unexpected subcode block')
                    return None

                code[-2] = (LOAD_CONST, sub_code)
            if op not in hasarg:
                code.append((op, None))
            else:
                arg = co_code[i] | co_code[i + 1] << 8 | extended_arg
                extended_arg = 0
                i += 2
                if op == opcode.EXTENDED_ARG:
                    extended_arg = arg << 16
                else:
                    byteplay_arg = co.co_consts[arg] if op in hasconst else \
                                   co.co_names[arg] if op in hasname else \
                                   labels[arg] if op in hasjabs else \
                                   labels[i + arg] if op in hasjrel else \
                                   co.co_varnames[arg] if op in haslocal else \
                                   cmp_op[arg] if op in hascompare else \
                                   cellfree[arg] if op in hasfree else \
                                   arg
                    code.append((op, byteplay_arg))

            if op == YIELD_VALUE or op == YIELD_FROM:
                is_generator = True

            if version_info >= (3, 5) and op in coroutine_opcodes:
                is_coroutine = True

        varargs = not not co.co_flags & CO_VARARGS
        varkwargs = not not co.co_flags & CO_VARKEYWORDS
        force_generator = not is_generator and (co.co_flags & CO_GENERATOR)

        if version_info >= (3, 5):
            force_coroutine = not is_coroutine and (co.co_flags & CO_COROUTINE)
            force_iterable_coroutine = co.co_flags & CO_ITERABLE_COROUTINE
            assert not (force_coroutine and force_iterable_coroutine)
            future_generator_stop = co.co_flags & CO_FUTURE_GENERATOR_STOP
        else:
            force_coroutine = None
            force_iterable_coroutine =None
            future_generator_stop = None

        return cls(code=code,
                   freevars=co.co_freevars,
                   args=co.co_varnames[:co.co_argcount + varargs + varkwargs + co.co_kwonlyargcount],
                   kwonly=co.co_kwonlyargcount,
                   varargs=varargs,
                   varkwargs=varkwargs,
                   newlocals=not not co.co_flags & CO_NEWLOCALS,
                   name=co.co_name,
                   filename=co.co_filename,
                   firstlineno=co.co_firstlineno,
                   docstring=co.co_consts[0] if co.co_consts and isinstance(co.co_consts[0], str) else None,
                   force_generator=force_generator,
                   force_coroutine=force_coroutine,
                   force_iterable_coroutine=force_iterable_coroutine,
                   future_generator_stop=future_generator_stop)

    def __eq__(self, other):
        try:
            if (self.freevars != other.freevars or
                    self.args != other.args or
                    self.kwonly != other.kwonly or
                    self.varargs != other.varargs or
                    self.varkwargs != other.varkwargs or
                    self.newlocals != other.newlocals or
                    self.name != other.name or
                    self.filename != other.filename or
                    self.firstlineno != other.firstlineno or
                    self.docstring != other.docstring or
                    self.force_generator != other.force_generator or
                    len(self.code) != len(other.code)):
                return False
            elif version_info >= (3, 5):
                if (self.force_coroutine != other.force_coroutine or
                        self.force_iterable_coroutine != other.force_iterable_coroutine or
                        self.future_generator_stop != other.future_generator_stop):
                    return False

            # This isn't trivial due to labels
            lmap = {}
            for (op1, arg1), (op2, arg2) in zip(self.code, other.code):
                if isinstance(op1, Label):
                    if lmap.setdefault(arg1, arg2) is not arg2:
                        return False
                else:
                    if op1 != op2:
                        return False
                    if op1 in hasjump:
                        if lmap.setdefault(arg1, arg2) is not arg2:
                            return False
                    elif arg1 != arg2:
                        return False

            return True
        except:
            return False

    def _compute_stacksize(self, logging=False):
        code = self.code
        label_pos = {op[0]: pos for pos, op in enumerate(code) if isinstance(op[0], Label)}
        # sf_targets are the targets of SETUP_FINALLY opcodes. They are recorded
        # because they have special stack behaviour. If an exception was raised
        # in the block pushed by a SETUP_FINALLY opcode, the block is popped
        # and 3 objects are pushed. On return or continue, the block is popped
        # and 2 objects are pushed. If nothing happened, the block is popped by
        # a POP_BLOCK opcode and 1 object is pushed by a (LOAD_CONST, None)
        # operation
        # Our solution is to record the stack state of SETUP_FINALLY targets
        # as having 3 objects pushed, which is the maximum. However, to make
        # stack recording consistent, the get_next_stacks function will always
        # yield the stack state of the target as if 1 object was pushed, but
        # this will be corrected in the actual stack recording
        if version_info < (3, 5):
            sf_targets = {label_pos[arg] for op, arg in code
                          if (op == SETUP_FINALLY or op == SETUP_WITH)}
        else:
            sf_targets = {label_pos[arg] for op, arg in code
                          if (op == SETUP_FINALLY or op == SETUP_WITH or op == SETUP_ASYNC_WITH)}

        states = [None] * len(code)
        maxsize = 0

        class BlockType(Enum):
            DEFAULT = 0,
            TRY_FINALLY = 1,
            TRY_EXCEPT = 2,
            LOOP_BODY = 3,
            WITH_BLOCK = 4,
            EXCEPTION = 5,
            SILENCED_EXCEPTION_BLOCK = 6,

        class State:

            def __init__(self, pos=0, stack=(0,), block_stack=(BlockType.DEFAULT,), log=[]):
                self._pos = pos
                self._stack = stack
                self._block_stack = block_stack
                self._log = log

            @property
            def pos(self):
                return self._pos

            @property
            def stack(self):
                return self._stack

            @stack.setter
            def stack(self, val):
                self._stack = val

            def newstack(self, n):
                if self._stack[-1] < -n:
                    raise ValueError("Popped a non-existing element at %s %s" %
                                     (self._pos, code[self._pos - 4: self._pos + 3]))
                return self._stack[:-1] + (self._stack[-1] + n,)

            @property
            def block_stack(self):
                return self._block_stack

            @property
            def log(self):
                return self._log

            def newlog(self, msg):
                if not logging:
                    return None

                log_msg = str(self._pos) + ": " + msg
                if self._stack:
                    log_msg += " (on stack: "
                    log_depth = 2
                    log_depth = min(log_depth, len(self._stack))
                    for pos in range(-1, -log_depth, -1):
                        log_msg += str(self._stack[pos]) + ", "
                    log_msg += str(self._stack[-log_depth])
                    log_msg += ")"
                else:
                    log_msg += " (empty stack)"
                return [log_msg] + self._log

        op = [State()]

        while op:
            cur_state = op.pop()
            o = sum(cur_state.stack)
            if o > maxsize:
                maxsize = o

            o, arg = code[cur_state.pos]

            if isinstance(o, Label):
                if cur_state.pos in sf_targets:
                    cur_state.stack = cur_state.newstack(5)
                if states[cur_state.pos] is None:
                    states[cur_state.pos] = cur_state
                elif states[cur_state.pos].stack != cur_state.stack:
                    check_pos = cur_state.pos + 1
                    while code[check_pos][0] not in hasflow:
                        check_pos += 1
                    if code[check_pos][0] not in (RETURN_VALUE, RAISE_VARARGS, STOP_CODE):
                        if cur_state.pos not in sf_targets:
                            raise ValueError("Inconsistent code at %s %s %s\n%s" %
                                             (cur_state.pos, cur_state.stack, states[cur_state.pos].stack,
                                              code[cur_state.pos - 5:cur_state.pos + 4]))
                        else:
                            # SETUP_FINALLY target inconsistent code!
                            #
                            # Since Python 3.2 assigned exception is cleared at the end of
                            # the except clause (named exception handler).
                            # To perform this CPython (checked in version 3.4.3) adds special
                            # bytecode in exception handler which currently breaks 'regularity' of bytecode.
                            # Exception handler is wrapped in try/finally block and POP_EXCEPT opcode
                            # is inserted before END_FINALLY, as a result cleanup-finally block is executed outside
                            # except handler. It's not a bug, as it doesn't cause any problems during execution, but
                            # it breaks 'regularity' and we can't check inconsistency here. Maybe issue should be
                            # posted to Python bug tracker.
                            pass
                    continue
                else:
                    continue

            if o not in (BREAK_LOOP, RETURN_VALUE, RAISE_VARARGS, STOP_CODE):
                next_pos = cur_state.pos + 1

                if not isopcode(o):
                    op += State(next_pos, cur_state.stack, cur_state.block_stack, cur_state.log),

                elif o not in hasflow:
                    if o in (LOAD_GLOBAL, LOAD_CONST, LOAD_NAME, LOAD_FAST, LOAD_ATTR, LOAD_DEREF,
                             LOAD_CLASSDEREF, LOAD_CLOSURE,
                             STORE_GLOBAL, STORE_NAME, STORE_FAST, STORE_ATTR, STORE_DEREF,
                             DELETE_GLOBAL, DELETE_NAME, DELETE_FAST, DELETE_ATTR, DELETE_DEREF,
                             IMPORT_NAME, IMPORT_FROM, COMPARE_OP):
                        se = stack_effect(o, 0)
                    else:
                        se = stack_effect(o, arg)

                    log = cur_state.newlog("non-flow command (" + str(o) + ", se = " + str(se) + ")")
                    op += State(next_pos, cur_state.newstack(se), cur_state.block_stack, log),

                elif o == FOR_ITER:
                    inside_for_log = cur_state.newlog("FOR_ITER (+1)")
                    op += State(label_pos[arg], cur_state.newstack(-1), cur_state.block_stack, cur_state.log),\
                          State(next_pos, cur_state.newstack(1), cur_state.block_stack, inside_for_log)

                elif o in (JUMP_FORWARD, JUMP_ABSOLUTE):
                    after_jump_log = cur_state.newlog(str(o))
                    op += State(label_pos[arg], cur_state.stack, cur_state.block_stack, after_jump_log),

                elif o in (JUMP_IF_FALSE_OR_POP, JUMP_IF_TRUE_OR_POP):
                    after_jump_log = cur_state.newlog(str(o) + ", jumped")
                    log = cur_state.newlog(str(o) + ", not jumped (-1)")
                    op += State(label_pos[arg], cur_state.stack, cur_state.block_stack, after_jump_log),\
                          State(next_pos, cur_state.newstack(-1), cur_state.block_stack, log)

                elif o in (POP_JUMP_IF_TRUE, POP_JUMP_IF_FALSE):
                    after_jump_log = cur_state.newlog(str(o) + ", jumped (-1)")
                    log = cur_state.newlog(str(o) + ", not jumped (-1)")
                    op += State(label_pos[arg], cur_state.newstack(-1), cur_state.block_stack, after_jump_log),\
                          State(next_pos, cur_state.newstack(-1), cur_state.block_stack, log)

                elif o == CONTINUE_LOOP:
                    next_stack, next_block_stack = cur_state.stack, cur_state.block_stack
                    last_popped_block = None
                    while next_block_stack[-1] != BlockType.LOOP_BODY:
                        last_popped_block = next_block_stack[-1]
                        next_stack, next_block_stack = next_stack[:-1], next_block_stack[:-1]

                    if next_stack != cur_state.stack:
                        log = cur_state.newlog("CONTINUE_LOOP, from non-loop block")
                    else:
                        log = cur_state.newlog("CONTINUE_LOOP")

                    jump_to_pos = label_pos[arg]
                    if last_popped_block == BlockType.WITH_BLOCK:
                        next_stack = next_stack[:-1] + (next_stack[-1] - 1,)
                    op += State(jump_to_pos, next_stack, next_block_stack, log),

                elif o == SETUP_LOOP:
                    inside_loop_log = cur_state.newlog("SETUP_LOOP (+block)")
                    op += State(label_pos[arg], cur_state.stack, cur_state.block_stack, cur_state.log),\
                          State(next_pos, cur_state.stack + (0,), cur_state.block_stack + (BlockType.LOOP_BODY,), inside_loop_log)

                elif o == SETUP_EXCEPT:
                    inside_except_log = cur_state.newlog("SETUP_EXCEPT, exception (+6, +block)")
                    inside_try_log = cur_state.newlog("SETUP_EXCEPT, try-block (+block)")
                    op += State(label_pos[arg], cur_state.stack + (6,), cur_state.block_stack + (BlockType.EXCEPTION,), inside_except_log),\
                          State(next_pos, cur_state.stack + (0,), cur_state.block_stack + (BlockType.TRY_EXCEPT,), inside_try_log)

                elif o == SETUP_FINALLY:
                    inside_finally_block = cur_state.newlog("SETUP_FINALLY (+1)")
                    inside_try_log = cur_state.newlog("SETUP_FINALLY try-block (+block)")
                    op += State(label_pos[arg], cur_state.newstack(1), cur_state.block_stack, inside_finally_block),\
                          State(next_pos, cur_state.stack + (0,), cur_state.block_stack + (BlockType.TRY_FINALLY,), inside_try_log)

                elif o == POP_BLOCK:
                    log = cur_state.newlog("POP_BLOCK (-block)")
                    op += State(next_pos, cur_state.stack[:-1], cur_state.block_stack[:-1], log),

                elif o == POP_EXCEPT:
                    log = cur_state.newlog("POP_EXCEPT (-block)")
                    op += State(next_pos, cur_state.stack[:-1], cur_state.block_stack[:-1], log),

                elif o == END_FINALLY:
                    if cur_state.block_stack[-1] == BlockType.SILENCED_EXCEPTION_BLOCK:
                        log = cur_state.newlog("END_FINALLY pop silenced exception block (-block)")
                        op += State(next_pos, cur_state.stack[:-1], cur_state.block_stack[:-1], log),
                    elif cur_state.block_stack[-1] == BlockType.EXCEPTION:
                        # Reraise exception
                        pass
                    else:
                        log = cur_state.newlog("END_FINALLY (-6)")
                        op += State(next_pos, cur_state.newstack(-6), cur_state.block_stack, log),

                elif o == SETUP_WITH or (version_info >= (3, 5) and o == SETUP_ASYNC_WITH):
                    inside_with_block = cur_state.newlog("SETUP_WITH, with-block (+1, +block)")
                    inside_finally_block = cur_state.newlog("SETUP_WITH, finally (+1)")
                    op += State(label_pos[arg], cur_state.newstack(1), cur_state.block_stack, inside_finally_block),\
                          State(next_pos, cur_state.stack + (1,), cur_state.block_stack + (BlockType.WITH_BLOCK,), inside_with_block)

                elif version_info < (3, 5) and o == WITH_CLEANUP:
                    # There is special case when 'with' __exit__ function returns True,
                    # that's the signal to silence exception, in this case additional element is pushed
                    # and next END_FINALLY command won't reraise exception.
                    log = cur_state.newlog("WITH_CLEANUP (-1)")
                    silenced_exception_log = cur_state.newlog("WITH_CLEANUP silenced_exception (+1, +block)")
                    op += State(next_pos, cur_state.newstack(-1), cur_state.block_stack, log),\
                          State(next_pos, cur_state.newstack(-7) + (8,), cur_state.block_stack + (BlockType.SILENCED_EXCEPTION_BLOCK,), silenced_exception_log)

                elif version_info >= (3, 5) and o == WITH_CLEANUP_START:
                    # There is special case when 'with' __exit__ function returns True,
                    # that's the signal to silence exception, in this case additional element is pushed
                    # and next END_FINALLY command won't reraise exception.
                    # Emulate this situation on WITH_CLEANUP_START with creating special block which will be
                    # handled differently by WITH_CLEANUP_FINISH and will cause END_FINALLY not to reraise exception.
                    log = cur_state.newlog("WITH_CLEANUP_START (+1)")
                    silenced_exception_log = cur_state.newlog("WITH_CLEANUP_START silenced_exception (+block)")
                    op += State(next_pos, cur_state.newstack(1), cur_state.block_stack, log),\
                          State(next_pos, cur_state.newstack(-7) + (9,), cur_state.block_stack + (BlockType.SILENCED_EXCEPTION_BLOCK,), silenced_exception_log)

                elif version_info >= (3, 5) and o == WITH_CLEANUP_FINISH:
                    if cur_state.block_stack[-1] == BlockType.SILENCED_EXCEPTION_BLOCK:
                        # See comment in WITH_CLEANUP_START handler
                        log = cur_state.newlog("WITH_CLEANUP_FINISH silenced_exception (-1)")
                        op += State(next_pos, cur_state.newstack(-1), cur_state.block_stack, log),
                    else:
                        log = cur_state.newlog("WITH_CLEANUP_FINISH (-2)")
                        op += State(next_pos, cur_state.newstack(-2), cur_state.block_stack, log),

                else:
                    raise ValueError("Unhandled opcode %s" % o)

        return maxsize + 6  # for exception raise in deepest place

    def to_code(self, from_function=False):
        """Assemble a Python code object from a Code object"""

        num_fastnames = sum(1 for op, arg in self.code if isopcode(op) and op in haslocal)
        is_function = self.newlocals or num_fastnames > 0 or len(self.args) > 0
        nested = is_function and from_function

        co_flags = {op[0] for op in self.code}

        is_generator = self.force_generator or (YIELD_VALUE in co_flags or YIELD_FROM in co_flags)
        no_free = (not self.freevars) and (not co_flags & hasfree)

        if version_info >= (3, 5):
            is_native_coroutine = bool(self.force_coroutine or (co_flags & coroutine_opcodes))
            assert not (is_native_coroutine and self.force_iterable_coroutine)

        co_flags =\
            (not(STORE_NAME in co_flags or LOAD_NAME in co_flags or DELETE_NAME in co_flags)) |\
            (self.newlocals and CO_NEWLOCALS) |\
            (self.varargs and CO_VARARGS) |\
            (self.varkwargs and CO_VARKEYWORDS) |\
            (is_generator and CO_GENERATOR) |\
            (no_free and CO_NOFREE) |\
            (nested and CO_NESTED)

        if version_info >= (3, 5):
            co_flags |= (is_native_coroutine and CO_COROUTINE) |\
                        (self.force_iterable_coroutine and CO_ITERABLE_COROUTINE) |\
                        (self.future_generator_stop and CO_FUTURE_GENERATOR_STOP)

        co_consts = [self.docstring]
        co_names = []
        co_varnames = list(self.args)
        co_freevars = tuple(self.freevars)

        # Find all cellvars beforehand for two reasons
        # Need the number of them to construct the numeric arg for ops in hasfree
        # Need to put args which are cells in the beginning of co_cellvars
        cellvars = {arg for op, arg in self.code
                    if isopcode(op) and op in hasfree
                    and arg not in co_freevars}
        co_cellvars = [jumps for jumps in self.args if jumps in cellvars]

        def index(seq, item, eq=True, can_append=True):
            for i, x in enumerate(seq):
                if x == item if eq else x is item:
                    return i
            if can_append:
                seq.append(item)
                return len(seq) - 1
            else:
                raise IndexError("Item not found")

        jumps = []
        label_pos = {}
        lastlineno = self.firstlineno
        lastlinepos = 0
        co_code = bytearray()
        co_lnotab = bytearray()

        for i, (op, arg) in enumerate(self.code):
            if isinstance(op, Label):
                label_pos[op] = len(co_code)
            elif op is SetLineno:
                incr_lineno = arg - lastlineno
                incr_pos = len(co_code) - lastlinepos
                lastlineno = arg
                lastlinepos += incr_pos
                if incr_lineno != 0 or incr_pos != 0:
                    while incr_pos > 255:
                        co_lnotab += b"\xFF\0"
                        incr_pos -= 255
                    while incr_lineno > 255:
                        co_lnotab += (0xFF00|incr_pos).to_bytes(2, "little")
                        incr_pos = 0
                        incr_lineno -= 255
                    if incr_pos or incr_lineno:
                        co_lnotab += bytes((incr_pos, incr_lineno))
            elif op == opcode.EXTENDED_ARG:
                self.code[i + 1][1] |= 1 << 32
            elif op not in hasarg:
                co_code.append(op)
            else:
                if op in hasconst:
                    if isinstance(arg, Code) and\
                            i + 2 < len(self.code) and self.code[i + 2][0] in hascode:
                        arg = arg.to_code(from_function=is_function)
                        assert arg is not None
                    arg = index(co_consts, arg, 0)
                elif op in hasname:
                    arg = index(co_names, arg)
                elif op in hasjump:
                    jumps.append((len(co_code), arg))
                    co_code += op.to_bytes(3, "little")
                    continue
                elif op in haslocal:
                    arg = index(co_varnames, arg)
                elif op in hascompare:
                    arg = index(cmp_op, arg, can_append=False)
                elif op in hasfree:
                    try:
                        arg = index(co_freevars, arg, can_append=False) + len(cellvars)
                    except IndexError:
                        arg = index(co_cellvars, arg)
                if arg > 0xFFFF:
                    co_code += (opcode.EXTENDED_ARG | ((arg & 0xFFFF0000) >> 8)).to_bytes(3, "little")
                co_code += (op | (arg << 8)).to_bytes(3, "little")

        for pos, label in jumps:
            jump = label_pos[label]
            if co_code[pos] in hasjrel:
                jump -= pos + 3
            if jump > 0xFFFF:
                raise NotImplementedError("Extended jumps not implemented")
            co_code[pos + 1] = jump & 0xFF
            co_code[pos + 2] = jump >> 8

        co_argcount = len(self.args) - self.varargs - self.varkwargs - self.kwonly
        co_stacksize = self._compute_stacksize()

        return CodeType(co_argcount, self.kwonly, len(co_varnames), co_stacksize, co_flags,
                        bytes(co_code), tuple(co_consts), tuple(co_names), tuple(co_varnames),
                        self.filename, self.name, self.firstlineno, bytes(co_lnotab), co_freevars,
                        tuple(co_cellvars))


from __future__ import print_function
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Conv2D, Dense, Reshape, Conv2DTranspose, UpSampling2D, Input, Reshape, Concatenate, MaxPooling2D, ZeroPadding2D, BatchNormalization
import tensorflow as tf; tf.image # There's a bug in tensorflow that requires us to import tf.image like this
from tensorflow_addons.layers import InstanceNormalization
from StyleGAN_utils import fade_in, wasserstein_loss, pixel_norm, minibatch_std, EqualizedConv, EqualizedDense, AddNoise, AdaIN

# Need to make this output (51,51,3) for working with ANIMEFACES. Note that 51 = 17*3
def simple_generator(digit_dimension):
    inputs = Input(shape=(digit_dimension,))
    coarse_1 = Reshape((1, 1, digit_dimension,))(inputs)
    coarse_1 = ZeroPadding2D(padding=((8,8),(8,8)))(coarse_1)
    coarse_1 = Conv2D(digit_dimension, (7, 7), padding='same', activation='elu')(coarse_1)
    coarse_1 = BatchNormalization()(coarse_1)
    coarse_1 = Conv2D(128, (5, 5), padding='same', activation='elu')(coarse_1)
    coarse_1 = BatchNormalization()(coarse_1)
    coarse_3 = UpSampling2D(size=(3, 3), interpolation='bilinear')(coarse_1)
    mid_2 = Conv2D(128, (5, 5), padding='same', activation='elu')(coarse_1)
    mid_2 = BatchNormalization()(mid_2)
    mid_2 = Conv2D(128, (3, 3), padding='same', activation='elu')(mid_2)
    mid_2 = BatchNormalization()(mid_2)
    mid_3 = UpSampling2D(size=(3, 3), interpolation='bilinear')(mid_2)
    fine_3 = Conv2D(64, (5, 5), padding='same', activation='elu')(mid_3)
    fine_3 = BatchNormalization()(fine_3)
    fine_3 = Conv2D(64, (3, 3), padding='same', activation='elu')(fine_3)
    fine_3 = BatchNormalization()(fine_3)
    composition = Concatenate(axis=3)([coarse_3, mid_3, fine_3])
    outputs = Conv2D(3, (3, 3), padding='same', activation='sigmoid')(composition)
    model = Model(inputs=inputs, outputs=outputs)

    return model

def simple_discriminator():
    inputs = Input(shape=(51, 51, 3,))
    fine = Conv2D(64,(5,5), padding='same', activation='elu')(inputs)
    fine = Conv2D(64, (5, 5), padding='same', activation='elu')(fine)
    fine = Conv2D(64,(5,5), padding='same', activation='elu')(fine)
    mid = Conv2D(32,(5,5), padding='same', activation='elu', strides=(2,2))(fine)
    mid = Conv2D(32,(5,5), padding='same', activation='elu')(mid)
    mid = Conv2D(32,(5,5), padding='same', activation='elu')(mid)
    coarse = Conv2D(16,(3,3), padding='same', activation='elu', strides=(2,2))(mid)
    coarse = Conv2D(16,(3,3), padding='same', activation='elu')(coarse)
    coarse = Conv2D(16, (3, 3), padding='same', activation='elu')(coarse)
    fine_scores = MaxPooling2D(pool_size=(28,28))(fine)
    mid_scores = MaxPooling2D(pool_size=(14,14))(mid)
    coarse_scores= MaxPooling2D(pool_size=(7,7))(coarse)
    composition = Concatenate(axis=3)([fine_scores,mid_scores,coarse_scores])
    composition = Reshape((64+32+16,))(composition)
    outputs = Dense(1, activation='sigmoid')(composition)
    model = Model(inputs=inputs, outputs=outputs)
    return model

#Much of the below is for the StyleGAN. This is mostly ripped from https://keras.io/examples/generative/stylegan/

def Mapping(num_stages, input_shape=512):
    z = layers.Input(shape=(input_shape))
    w = pixel_norm(z)
    for i in range(8):
        w = EqualizedDense(512, learning_rate_multiplier=0.01)(w)
        w = layers.LeakyReLU(0.2)(w)
    w = tf.tile(tf.expand_dims(w, 1), (1, num_stages, 1))
    return keras.Model(z, w, name="mapping")

class Generator:
    def __init__(self, start_res_log2, target_res_log2):
        self.start_res_log2 = start_res_log2
        self.target_res_log2 = target_res_log2
        self.num_stages = target_res_log2 - start_res_log2 + 1
        # list of generator blocks at increasing resolution
        self.g_blocks = []
        # list of layers to convert g_block activation to RGB
        self.to_rgb = []
        # list of noise input of different resolutions into g_blocks
        self.noise_inputs = []
        # filter size to use at each stage, keys are log2(resolution)
        self.filter_nums = {
            0: 512,
            1: 512,
            2: 512,  # 4x4
            3: 512,  # 8x8
            4: 512,  # 16x16
            5: 512,  # 32x32
            6: 256,  # 64x64
            7: 128,  # 128x128
            8: 64,  # 256x256
            9: 32,  # 512x512
            10: 16,
        }  # 1024x1024

        start_res = 2 ** start_res_log2
        self.input_shape = (start_res, start_res, self.filter_nums[start_res_log2])
        self.g_input = layers.Input(self.input_shape, name="generator_input")

        for i in range(start_res_log2, target_res_log2 + 1):
            filter_num = self.filter_nums[i]
            res = 2 ** i
            self.noise_inputs.append(
                layers.Input(shape=(res, res, 1), name=f"noise_{res}x{res}")
            )
            to_rgb = Sequential(
                [
                    layers.InputLayer(input_shape=(res, res, filter_num)),
                    EqualizedConv(3, 1, gain=1),
                ],
                name=f"to_rgb_{res}x{res}",
            )
            self.to_rgb.append(to_rgb)
            is_base = i == self.start_res_log2
            if is_base:
                input_shape = (res, res, self.filter_nums[i - 1])
            else:
                input_shape = (2 ** (i - 1), 2 ** (i - 1), self.filter_nums[i - 1])
            g_block = self.build_block(
                filter_num, res=res, input_shape=input_shape, is_base=is_base
            )
            self.g_blocks.append(g_block)

    def build_block(self, filter_num, res, input_shape, is_base):
        input_tensor = layers.Input(shape=input_shape, name=f"g_{res}")
        noise = layers.Input(shape=(res, res, 1), name=f"noise_{res}")
        w = layers.Input(shape=512)
        x = input_tensor

        if not is_base:
            x = layers.UpSampling2D((2, 2))(x)
            x = EqualizedConv(filter_num, 3)(x)

        x = AddNoise()([x, noise])
        x = layers.LeakyReLU(0.2)(x)
        x = InstanceNormalization()(x)
        x = AdaIN()([x, w])

        x = EqualizedConv(filter_num, 3)(x)
        x = AddNoise()([x, noise])
        x = layers.LeakyReLU(0.2)(x)
        x = InstanceNormalization()(x)
        x = AdaIN()([x, w])
        return keras.Model([input_tensor, w, noise], x, name=f"genblock_{res}x{res}")

    def grow(self, res_log2):
        res = 2 ** res_log2

        num_stages = res_log2 - self.start_res_log2 + 1
        w = layers.Input(shape=(self.num_stages, 512), name="w")

        alpha = layers.Input(shape=(1), name="g_alpha")
        x = self.g_blocks[0]([self.g_input, w[:, 0], self.noise_inputs[0]])

        if num_stages == 1:
            rgb = self.to_rgb[0](x)
        else:
            for i in range(1, num_stages - 1):

                x = self.g_blocks[i]([x, w[:, i], self.noise_inputs[i]])

            old_rgb = self.to_rgb[num_stages - 2](x)
            old_rgb = layers.UpSampling2D((2, 2))(old_rgb)

            i = num_stages - 1
            x = self.g_blocks[i]([x, w[:, i], self.noise_inputs[i]])

            new_rgb = self.to_rgb[i](x)

            rgb = fade_in(alpha[0], new_rgb, old_rgb)

        return keras.Model(
            [self.g_input, w, self.noise_inputs, alpha],
            rgb,
            name=f"generator_{res}_x_{res}",
        )

class Discriminator:
    def __init__(self, start_res_log2, target_res_log2):
        self.start_res_log2 = start_res_log2
        self.target_res_log2 = target_res_log2
        self.num_stages = target_res_log2 - start_res_log2 + 1
        # filter size to use at each stage, keys are log2(resolution)
        self.filter_nums = {
            0: 512,
            1: 512,
            2: 512,  # 4x4
            3: 512,  # 8x8
            4: 512,  # 16x16
            5: 512,  # 32x32
            6: 256,  # 64x64
            7: 128,  # 128x128
            8: 64,  # 256x256
            9: 32,  # 512x512
            10: 16,
        }  # 1024x1024
        # list of discriminator blocks at increasing resolution
        self.d_blocks = []
        # list of layers to convert RGB into activation for d_blocks inputs
        self.from_rgb = []

        for res_log2 in range(self.start_res_log2, self.target_res_log2 + 1):
            res = 2 ** res_log2
            filter_num = self.filter_nums[res_log2]
            from_rgb = Sequential(
                [
                    layers.InputLayer(
                        input_shape=(res, res, 3), name=f"from_rgb_input_{res}"
                    ),
                    EqualizedConv(filter_num, 1),
                    layers.LeakyReLU(0.2),
                ],
                name=f"from_rgb_{res}",
            )

            self.from_rgb.append(from_rgb)

            input_shape = (res, res, filter_num)
            if len(self.d_blocks) == 0:
                d_block = self.build_base(filter_num, res)
            else:
                d_block = self.build_block(
                    filter_num, self.filter_nums[res_log2 - 1], res
                )

            self.d_blocks.append(d_block)

    def build_base(self, filter_num, res):
        input_tensor = layers.Input(shape=(res, res, filter_num), name=f"d_{res}")
        x = minibatch_std(input_tensor)
        x = EqualizedConv(filter_num, 3)(x)
        x = layers.LeakyReLU(0.2)(x)
        x = layers.Flatten()(x)
        x = EqualizedDense(filter_num)(x)
        x = layers.LeakyReLU(0.2)(x)
        x = EqualizedDense(1)(x)
        return keras.Model(input_tensor, x, name=f"d_{res}")

    def build_block(self, filter_num_1, filter_num_2, res):
        input_tensor = layers.Input(shape=(res, res, filter_num_1), name=f"d_{res}")
        x = EqualizedConv(filter_num_1, 3)(input_tensor)
        x = layers.LeakyReLU(0.2)(x)
        x = EqualizedConv(filter_num_2)(x)
        x = layers.LeakyReLU(0.2)(x)
        x = layers.AveragePooling2D((2, 2))(x)
        return keras.Model(input_tensor, x, name=f"d_{res}")

    def grow(self, res_log2):
        res = 2 ** res_log2
        idx = res_log2 - self.start_res_log2
        alpha = layers.Input(shape=(1), name="d_alpha")
        input_image = layers.Input(shape=(res, res, 3), name="input_image")
        x = self.from_rgb[idx](input_image)
        x = self.d_blocks[idx](x)
        if idx > 0:
            idx -= 1
            downsized_image = layers.AveragePooling2D((2, 2))(input_image)
            y = self.from_rgb[idx](downsized_image)
            x = fade_in(alpha[0], x, y)

            for i in range(idx, -1, -1):
                x = self.d_blocks[i](x)
        return keras.Model([input_image, alpha], x, name=f"discriminator_{res}_x_{res}")


import numpy as np
import matplotlib.pyplot as plt
import sys
sys.path.append('../../../../scripts/')
from fig_settings import configure_fig_settings
sys.path.append('../../modules_gammak24/')
from plotpsi import PlotPsi
import seaborn as sns

colors = sns.color_palette()

configure_fig_settings()

# see user_inputs.md for details on what typically goes in these inputs.
user_input = input("input string of k24,Lambda,omega values, "
                   "using comma as delimiter: ")

gammas = np.array([0.025,0.05,0.075,0.1,0.125,0.15,0.175],float)

k24,Lambda,omega = np.array(user_input.split(','),float)

fig,ax = plt.subplots()
width  = 3.487
height = width
fig.set_size_inches(width,height)

for i,gamma in enumerate(gammas):
    if i == 0:
        label = fr'$\gamma=\num{{{gamma:.2e}}}$'
    elif i == len(gammas)-1:
        label = fr'$\gamma=\num{{{gamma:.2e}}}$'
    else:
        label = None
    pl = PlotPsi(gamma,k24,fig,ax)
    pl.plot_psivsr(omega,Lambda,label=label,color=colors[i])

ax.set_xlabel(r'$r$')
ax.set_ylabel(r'$\psi(r)$')
ax.legend(frameon=False)
fig.tight_layout()
fig.savefig(pl.psivsr_sname(omega,Lambda))


from boto3.session import Session

import json
import urllib.parse
import boto3
import tempfile
import traceback
import botocore
import zipfile
import os

ECS_REGION = os.environ.get("ECS_REGION")
ECR_REGION = os.environ.get("ECR_REGION")
ECS_CLUSTER = os.environ.get("ECS_CLUSTER")
SERVICE_NAME = os.environ.get("SERVICE_NAME")
FAMILY_NAME = os.environ.get("FAMILY_NAME")
#ECS_TASK_CPU = int(os.environ.get("ECS_TASK_CPU"))
#ECS_TASK_MEMORY = int(os.environ.get("ECS_TASK_MEMORY"))
#ECS_TASK_PORT = int(os.environ.get("ECS_TASK_PORT"))
DESIRED_COUNT = int(os.environ.get("DESIRED_COUNT"))
MAX_HEALTHY_PERCENT = int(os.environ.get("MAX_HEALTHY_PERCENT"))
MIN_HEALTH_PERCENT = int(os.environ.get("MIN_HEALTH_PERCENT"))

ecs_client = boto3.client('ecs',region_name=ECS_REGION)
code_pipeline = boto3.client('codepipeline')

def setup_s3_client(job_data):
    """Creates an S3 client

    Uses the credentials passed in the event by CodePipeline. These
    credentials can be used to access the artifact bucket.

    Args:
        job_data: The job data structure

    Returns:
        An S3 client with the appropriate credentials

    """
    key_id = job_data['artifactCredentials']['accessKeyId']
    key_secret = job_data['artifactCredentials']['secretAccessKey']
    session_token = job_data['artifactCredentials']['sessionToken']

    session = Session(aws_access_key_id=key_id,
        aws_secret_access_key=key_secret,
        aws_session_token=session_token)
    return session.client('s3', config=botocore.client.Config(signature_version='s3v4'))

def updateECService(imageTag):
    # Get the latest task definition
    response = ecs_client.describe_task_definition(
        taskDefinition=FAMILY_NAME
    )
    familyName = response['taskDefinition']['family']
    revision = response['taskDefinition']['revision']
    task_def = response['taskDefinition']['containerDefinitions'][0]
    preImage = task_def['image']
    tagStart = preImage.find(':')+1
    tagEnd = len(preImage)
    tag = preImage[tagStart:tagEnd]
    imageRepo = preImage[:tagStart]
    print("Image Old Tag:"+tag+",Image New Tag:"+imageTag)
    print("imageRepo:"+imageRepo)
    task_def['image']=imageRepo+imageTag

    response = ecs_client.register_task_definition(
       family=familyName,
       containerDefinitions=[
          task_def
       ]
    )
    print("Register Task Response: " + json.dumps(response, indent=2))
    revision = response['taskDefinition']['revision']
    srvUpdatResponse = ecs_client.update_service(
        cluster=ECS_CLUSTER,
        service=SERVICE_NAME,
        desiredCount=DESIRED_COUNT,
        taskDefinition=FAMILY_NAME+':'+str(revision),
        deploymentConfiguration={
          'maximumPercent': MAX_HEALTHY_PERCENT,
          'minimumHealthyPercent': MIN_HEALTH_PERCENT
        }
    )

def put_job_success(job, message):
    """Notify CodePipeline of a successful job

    Args:
        job: The CodePipeline job ID
        message: A message to be logged relating to the job status

    Raises:
        Exception: Any exception thrown by .put_job_success_result()

    """
    print('Putting job success')
    print(message)
    code_pipeline.put_job_success_result(jobId=job)

def put_job_failure(job, message):
    """Notify CodePipeline of a failed job

    Args:
        job: The CodePipeline job ID
        message: A message to be logged relating to the job status

    Raises:
        Exception: Any exception thrown by .put_job_failure_result()

    """
    print('Putting job failure')
    print(message)
    code_pipeline.put_job_failure_result(jobId=job, failureDetails={'message': message, 'type': 'JobFailed'})

def lambda_handler(event, context):
    """
     Args:
        event: The event passed from code pipeline
    """
    print("Received event: " + json.dumps(event, indent=2))
    ACCOUNT_ID = context.invoked_function_arn.split(":")[4]

    try:
        # Extract the Job ID
        job_id = event['CodePipeline.job']['id']
        # Extract the Job Data
        job_data = event['CodePipeline.job']['data']
         # Get the list of artifacts passed to the function
        artifacts = job_data['inputArtifacts']
        imageTabArtifact = artifacts[0]
        imageTagData = {}

        if 'continuationToken' in job_data:
            # If we're continuing then the update ecs service has already been triggered
            # we will do nothing.
            put_job_success(job_id, 'There were ongoing ecs service updates')
        else:
            s3 = setup_s3_client(job_data)
            tmp_file = tempfile.NamedTemporaryFile()
            bucket = imageTabArtifact['location']['s3Location']['bucketName']
            key = imageTabArtifact['location']['s3Location']['objectKey']
            with tempfile.NamedTemporaryFile() as tmp_file:
                s3.download_file(bucket, key, tmp_file.name)
                with zipfile.ZipFile(tmp_file.name, 'r') as zip:
                    imageTagData = json.loads(zip.read('build.json').decode())

            print("Downloaded Image Tab  : " + json.dumps(imageTagData, indent=2))
            imageTag = imageTagData['tag']
            updateECService(imageTag)
            put_job_success(job_id, 'Update ecs serivce using the new docker image tag'+imageTag)

    except Exception as e:
        # If any other exceptions which we didn't expect are raised
        # then fail the job and log the exception message.
        print('Function failed due to exception.')
        print(e)
        traceback.print_exc()
        put_job_failure(job_id, 'Function exception: ' + str(e))

    print('Function complete.')
    return "Complete."


"""yuanquan URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include
from . import home
import user, article, comment
from django.conf import settings
from django.conf.urls.static import static
import notifications.urls

urlpatterns = [
    path('admin/', admin.site.urls),
    path('', home.home, name='home'),
    path('article/', include('article.urls')),
    path('user/', include('user.urls')),
    path('like/', include('comment.urls')),
    path('login/', user.views.login, name='login'),
    path('logout/', user.views.logout, name='logout'),
    path('register/', user.views.register, name='register'),
    path('error404/', home.error404, name='error404'),
    path('search/', home.search, name='search'),
    path('test/', home.test, name='test'),
    path('classify/<int:classify_id>', article.views.article_type, name='classify_type'),
    path('add_comment/<int:article_id>', comment.views.add_comment, name='add_comment'),
    path('info/', home.info, name='info'),
    path('record/', home.record, name='record'),
    path('load_file/', home.load_file, name='load_file'),
    path('notifications/', include(notifications.urls, namespace='notifications')),
    path('delete_all_notification', home.delete_all_notification, name='delete_all_notification'),
    path('delete_notification', home.delete_notification, name='delete_notification'),
    path('mdeditor/', include('mdeditor.urls')),

] + static('/pic', document_root=settings.MEDIA_ROOT)


# XML (Extensible Markup Language) Parser

"""
    XML Parser
    Corresponding xml document 'doc.xml'
    Problem:
        Extract the RefText values for the following RefCodes: ‘MWB’, ‘TRV’ and ‘CAR’
"""

import xml.etree.ElementTree as Et

class XmlParser:

    def __init__(self, filename):
        """
        initialise xml parser
        :param filename: full path to xml document
        """
        self.dom = self.read_xml(filename)
        self.root = self.dom.getroot()
        self.tags_with_attrib = []
        self.filtered_tags = []
        self.extacted_vals = []
        self.traced_vals = {}

    @staticmethod
    def read_xml(filename):
        """
        read xml document using Element Tree module
        :param filename: full path to file containing message
        :return xml element tree object
        """
        return Et.parse(filename)

    def get_tags_with_attrib(self, element, attrib):
        """
        get all tags that have the specified attribute
        :param element: xml doc element to check
        :param attrib: attribute we are looking for
        """
        for child in element:
            if attrib in child.attrib:
                self.tags_with_attrib.append(child)
            self.get_tags_with_attrib(child, attrib)

    def filter_tags(self, attrib, attrib_values):
        """
        filter tags based on a particular attribute and list of possible values
        :param attrib: xml attribute name
        :param attrib_values: list of accepted attributed values
        """
        for tag in self.tags_with_attrib:
            if tag.attrib[attrib] in attrib_values:
                self.filtered_tags.append(tag)

    def extract_vals(self, tag, attrib):
        """
        extract the values from the tags within the filtered tags
        :param attrib: attribute name
        :param tag: inner tag from which the value is extracted
        """
        for element in self.filtered_tags:
            for child in element:
                if child.tag == tag:
                    self.extacted_vals.append(child.text)
                    self.traced_vals.setdefault(element.attrib[attrib], []).append(child.text)

    ######################################################################################

    # simple solution if path through dom is known

    def get_reftext(self, path, attrib, attrib_values, tag):
        """
        directly get values from RefText if dom structure is known
        :param path: path through dom to Reference tags
        :param attrib: outer tag filtered attribute name
        :param attrib_values: attribute values to filter tag by
        :param tag: return text from this tag
        """
        self.traced_vals = {}
        references = self.root.findall(path)
        for ref in references:
            if ref.attrib[attrib] in attrib_values:
                reftext = ref.findall(tag)
                for reft in reftext:
                    self.traced_vals.setdefault(ref.attrib[attrib], []).append(reft.text)

    ######################################################################################

    # debugging. see above for solution

    def print_tags_and_attribs(self, ele):
        """
        print all element tags and their attributes
        :param ele: display info for element
        """
        for el in ele:
            print(el, ' ==> tag:', el.tag, ', attrib:', el.attrib)
            self.print_tags_and_attribs(el)

# Program driver
if __name__ == '__main__':

    _filename = 'doc.xml'  # xml document path
    _attrib = 'RefCode'  # search for tags that have this attribute
    _attrib_values = ['MWB', 'TRV', 'CAR']  # filter tags based on _attrib that matching one of the following
    _tag = 'RefText'  # get the value from this tag from within the filtered tags

    parser = XmlParser(_filename)  # create new xml parser object
    parser.get_tags_with_attrib(parser.root, _attrib)  # get all tags which have a specified attrib i.e all tags with attribute 'RefCode'
    parser.filter_tags(_attrib, _attrib_values)  # filter tags for tags that have 'RefCode' matching one of the given
    parser.extract_vals(_tag, _attrib)  # extract the values from the 'RefText' tags if is inside the filtered tags

    # solution
    print('corresponding values: ', parser.traced_vals)
    print('extracted values are: ', parser.extacted_vals)

    # simple solution if dom structure is known
    _path = 'DeclarationList/DeclarationHeader/Reference'
    parser.get_reftext(_path, _attrib, _attrib_values, _tag)
    print('corresponding values: ', parser.traced_vals)


from django.conf.urls import url
from . import views
from django.views.generic.base import TemplateView

app_name = 'blogg'

urlpatterns = [
	url(r'^signup$', views.signup, name='signup'),
	url(r'^profile/$', views.profile, name='profile'),
	url(r'^$', views.post_list, name = 'post_list'),
	url(r'^post/(?P<pk>\d+)/$', views.post_detail, name = 'post_detail'),
	url(r'^post/new/$', views.post_new, name = 'post_new'),
	url(r'^post/(?P<pk>\d+)/edit/$', views.post_edit, name='post_edit'),
	url(r'^post/(?P<pk>\d+)/delete/$', views.post_delete, name='post_delete'),
]


#!/usr/bin/env python
"""A smoke test for containers"""

import json
import logging
import os
import pytest
import requests
import time

from kubetest_stdlib import testutils
from requests.packages.urllib3.exceptions import InsecureRequestWarning

# Supress warnings
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

logger = logging.getLogger(__name__)

class TestSmoke():
    """ Use pytest to execute rest api operations against a container. """

    @classmethod
    def setup_class(cls):
        """
        Setup environment for the upcoming tests. This is basically the constructor for
        this test class. Its used to create the KubeUtils API instance used by some
        tests, retrieve environment variables from the testconfig.yaml and set up any
        structures and variables that might be used by All of the test methods below.

        """
        # Initial setup
        test_target = "configuratn"
        test_vars = testutils.k8s_test_setup(test_target)
        # Expose variables to test functions
        cls.TARGET_BASE_NAME = test_target
        cls.TARGET_POD_NAME = test_vars.get("target_pod_name")
        cls.NAMESPACE = test_vars.get("namespace")
        cls.kubeutilobj = test_vars.get("kubeutilobj")

    def test_01_pod_user_not_root(self):
        """
        Check user not root
        """
        time.sleep(1)
        cmd = "exec {} whoami".format(self.TARGET_POD_NAME)
        status,result = self.kubeutilobj.kubectl(cmd, self.NAMESPACE)
        error_flag = False
        if "failure" in status.lower() or b"root" in result["output"]:
            error_flag = True
        assert error_flag == False

    def test_02_hostname(self):
        """
        Check consul in hostname
        """
        cmd = "exec {} hostname".format(self.TARGET_POD_NAME)
        status, result = self.kubeutilobj.kubectl(cmd, self.NAMESPACE)
        assert self.TARGET_BASE_NAME in result["output"].decode("utf-8")

    def test_03_agent_self_no_tls(self):
        '''
        Ensure agent member name contains pod name
        http://consul.kubeworker1.sas.com/v1/agent/self

        '''
        url = "http://localhost:8500/v1/agent/self"
        cmd = 'exec {} -- curl -s -o /dev/null -w '.format(self.TARGET_POD_NAME) + \
               str('"%{http_code}"') + ' {}'.format(url)
        status, result = self.kubeutilobj.kubectl(cmd, self.NAMESPACE)
        print(result)
        assert b"200" in result["output"]

    def test_04_agent_member_no_tls(self):
        '''
        http://consul.kubeworker1.sas.com/v1/agent/peers
        '''
        url = "http://localhost:8500/v1/agent/members"
        cmd = 'exec {} -- curl -s -o /dev/null -w '.format(self.TARGET_POD_NAME) + \
               str('"%{http_code}"') + ' {}'.format(url)
        status, result = self.kubeutilobj.kubectl(cmd, self.NAMESPACE)
        print(result)
        assert b"200" in result["output"]


import re
from eunjeon import Mecab
from wordcloud import WordCloud
from collections import Counter
import matplotlib.pyplot as plt

def read_txt(fname):
    f = open(f'./{fname}', 'r', encoding='UTF8')
    lines = f.readlines()
    f.close()

    p = re.compile(r'(\d+[:]\d+)')
    time = 0
    talk_dic = {}
    for line in lines:
        m = p.search(line)
        if line.startswith('---------------'):
            time = line[16:line.find('---------------', 1)]
        elif m is not None and line.find('[') == 0:
            talker = line[1:line.find(']')]
            talk = line[m.end() + 1:].strip()
            hour = line[line.find('[', line.find('[')+1)+1 : line.find(']', line.find(']')+1)]
            if talker not in talk_dic:
                talk_dic[talker] = [(time+hour, talk)]
            else:
                talk_dic[talker].append((time+hour, talk))
    return talk_dic

def message_cleaning(docs):
    '''
    1. Photo, Emoticon은 내용 알 수 없으므로 제거
    2. 자음/모음 표현 처리
    3. http://로 시작하는 하이퍼링크 제거
    4. 특수문자 제거
    '''

    pattern1 = re.compile("이모티콘")
    docs = pattern1.sub("", docs)

    pattern2 = re.compile("[ㄱ-ㅎ]*|[ㅏ-ㅢ]*")
    docs = pattern2.sub("", docs)

    pattern3 = re.compile(r"\b(https?:\/\/)?([\w.]+){1,2}(\.[\w]{2,4}){1,2}(.*)")
    docs = pattern3.sub("", docs)

    pattern4 = re.compile("[\{\}\[\]\/?.,;:|\)*-`!^\-_+<>@\#$&\\\=\(\'\"]")
    docs = pattern4.sub("", docs)

    return docs

def define_stopwords(path):
    SW = set()
    # 불용어 추가 방법 Sw.add("있다") or stopwords-ko.txt에 직접 추가
    with open(path, encoding='utf-8') as f:
        for word in f:
            SW.add(word)
    words = ['이랑', '웅웅', '아서', '시반']
    for word in words:
        SW.add(word)
    return SW

def tokenize(filename):
    talk_dic = read_txt(filename)
    SW = define_stopwords("./stopwords-ko.txt")
    mecab= Mecab()
    total = {}
    for k, v in talk_dic.items():
        total_sub = []
        for idx, talk_set in enumerate(v):
            time, talk = talk_set
            clean_talk = message_cleaning(talk)
            tokenized_talk = [word for word in mecab.morphs(clean_talk) if word not in SW and len(word) > 1]
            talk_dic[k][idx] = (time, talk, tokenized_talk)
            total_sub.extend(tokenized_talk)
        total[k] = total_sub
    return talk_dic, total

def get_wordcloud(filename):
    talk_dic, total = tokenize(filename)
    word_counts = []
    for k, v in total.items():
        count = Counter(v)
        wc = count.most_common(100)
        wordcloud = WordCloud(font_path = './static/malgun.ttf',
                        relative_scaling=0.2,
                        background_color = 'white',
                        ).generate_from_frequencies(dict(wc))
        wc = [(idx, w) for idx, w in enumerate(wc)]
        word_counts.append(wc)
        fig = plt.figure(figsize=(16,8))
        plt.imshow(wordcloud)
        plt.axis("off")
        plt.show()
        fig.savefig(f'./static/wordcloud_{k}.png')
    return talk_dic, word_counts

#talk_dic, total = tokenize('kakao.txt')
#print(total['조성빈'][:3])


# coding: utf-8
import os
import shutil
import sqlite3
import pathlib
from jinjaql.cache import test as test_cache
from jinjaql.factory import JinJAQL
from jinjaql.engine import sqlite_test

try:
    import unittest2 as unittest
except ImportError:
    import unittest

class TestIntegration(unittest.TestCase):

    def setUp(self):
        self.sql_root = os.path.abspath(os.path.dirname(__file__))
        self.jinjaql_engine = JinJAQL(pathlib.Path(self.sql_root, 'queries'), engine=sqlite_test)
        self.jinjaql_engine_cache = JinJAQL(pathlib.Path(self.sql_root, 'queries'), engine=sqlite_test, cache=test_cache)
        self.sqlite_folder = os.path.join(self.sql_root, 'data')
        self.sqlite_path = os.path.join(self.sqlite_folder, 'test_db.sqlite')
        if not os.path.exists(self.sqlite_folder):
            os.makedirs(self.sqlite_folder)
        if not os.path.exists(self.sqlite_path):
            open(self.sqlite_path, 'a').close()

    def tearDown(self):
        shutil.rmtree(self.sqlite_folder)

    def test_engine(self):
        queries = self.jinjaql_engine.load_queries('integration.sql')
        try:
            queries.create_artists()
        except:
            pass
        query = queries.insert_artist(**{
            'id': 1,
            'name': 'Lana Del Rey',
            'age': 30,
            'instrument': 'voice',
            'creation_date': '2015-10-13',
        })
        result = queries.get_artists(id=1)

        self.assertEqual(result, [('Lana Del Rey', 30, 'voice')])

    def test_engine_cache(self):
        queries = self.jinjaql_engine_cache.load_queries('integration.sql')
        try:
            queries.create_artists()
        except:
            pass
        query = queries.insert_artist(**{
            'id': 1,
            'name': 'Lana Del Rey',
            'age': 30,
            'instrument': 'voice',
            'creation_date': '2015-10-13',
        })
        result = queries.get_artists(id=1)

        self.assertEqual(result, [('Lana Del Rey', 30, 'voice'), ('Lana Del Rey', 32, 'voice')])


#from flask import Flask, render_template, request
from chatterbot import ChatBot
from chatterbot.response_selection import get_random_response
from chatterbot.trainers import ChatterBotCorpusTrainer

#For getting the spreadsheets
import requests
import gspread
from oauth2client.service_account import ServiceAccountCredentials

import logging
logging.basicConfig(level=logging.INFO)

##Try to download the spreadsheet from Google to retrain from
scope = ['https://spreadsheets.google.com/feeds',
         'https://www.googleapis.com/auth/drive']

credentials = ServiceAccountCredentials.from_json_keyfile_name('googleapi.json', scope)

gc = gspread.authorize(credentials)

worksheet = gc.open("ChatBot Knowledge Base").sheet1

all_cells = worksheet.get_all_values()

with open('data/trainingdata.yml', 'w') as f:
    f.write("categories:\r\n")
    f.write("- Conversations")
    f.write("\r\nconversations:")
    for i in range(len(all_cells)):
        if i != 0:
            f.write("\r\n- - " + all_cells[i][0])
            f.write("\r\n  - " + all_cells[i][1])
            #print("- - " + all_cells[i][0])
            #print("  - " + all_cells[i][1])

print("I have successfully imported " + str(len(all_cells)) + " rows of data and will now retrain...")

bot = ChatBot(
    "Chatbot",
    logic_adapters=[
        {
            'import_path': 'chatterbot.logic.BestMatch'
        },
        {
            'import_path': 'chatterbot.logic.LowConfidenceAdapter',
            'threshold': 0.65,
            'default_response': 'I don\'t have a response for that. What else can we talk about?'
        }
    ],
    response_selection_method=get_random_response, #Comment this out if you want best response
    input_adapter="chatterbot.input.VariableInputTypeAdapter",
    output_adapter="chatterbot.output.OutputAdapter",
    storage_adapter="chatterbot.storage.SQLStorageAdapter",
    database="botdata.sqlite3"
)

bot.read_only=True #Comment this out if you want the bot to learn based on experience
print("Bot Learn Read Only:" + str(bot.read_only))

#You can comment these out for production later since you won't be training everytime:
bot.set_trainer(ChatterBotCorpusTrainer)
bot.train("data/trainingdata.yml")

print("I am all trained up and ready to chat!")


'''
This file implements a multi layer neural network for a multiclass classifier

Hemanth Venkateswara
hkdv1@asu.edu
Oct 2018
'''
import numpy as np
from mod_load_mnist import mnist, one_hot
import matplotlib.pyplot as plt
from cycler import cycler
import pdb
import sys, ast
import os

def tanh(Z):
    '''
    computes tanh activation of Z

    Inputs:
        Z is a numpy.ndarray (n, m)

    Returns:
        A is activation. numpy.ndarray (n, m)
        cache is a dictionary with {"Z", Z}
    '''
    A = np.tanh(Z)
    cache = {}
    cache["Z"] = Z
    return A, cache

def tanh_der(dA, cache):
    '''
    computes derivative of tanh activation

    Inputs:
        dA is the derivative from subsequent layer. numpy.ndarray (n, m)
        cache is a dictionary with {"Z", Z}, where Z was the input
        to the activation layer during forward propagation

    Returns:
        dZ is the derivative. numpy.ndarray (n,m)
    '''
    ### CODE HERE
    tan_h = tanh(cache["Z"])[0]
    dZ = dA * (1-tan_h)**2
    return dZ

def sigmoid(Z):
    '''
    computes sigmoid activation of Z

    Inputs:
        Z is a numpy.ndarray (n, m)

    Returns:
        A is activation. numpy.ndarray (n, m)
        cache is a dictionary with {"Z", Z}
    '''
    A = 1/(1+np.exp(-Z))
    cache = {}
    cache["Z"] = Z
    return A, cache

def sigmoid_der(dA, cache):
    '''
    computes derivative of sigmoid activation

    Inputs:
        dA is the derivative from subsequent layer. numpy.ndarray (n, m)
        cache is a dictionary with {"Z", Z}, where Z was the input
        to the activation layer during forward propagation

    Returns:
        dZ is the derivative. numpy.ndarray (n,m)
    '''
    ### CODE HERE
    sig = sigmoid(cache["Z"])[0]
    dZ = dA*sig*(1-sig)
    return dZ

def relu(Z):
    '''
    computes relu activation of Z

    Inputs:
        Z is a numpy.ndarray (n, m)

    Returns:
        A is activation. numpy.ndarray (n, m)
        cache is a dictionary with {"Z", Z}
    '''
    A = np.maximum(0,Z)
    cache = {}
    cache["Z"] = Z
    return A, cache

def relu_der(dA, cache):
    '''
    computes derivative of relu activation

    Inputs:
        dA is the derivative from subsequent layer. numpy.ndarray (n, m)
        cache is a dictionary with {"Z", Z}, where Z was the input
        to the activation layer during forward propagation

    Returns:
        dZ is the derivative. numpy.ndarray (n,m)
    '''
    dZ = np.array(dA, copy=True)
    Z = cache["Z"]
    dZ[Z<0] = 0
    return dZ

def linear(Z):
    '''
    computes linear activation of Z
    This function is implemented for completeness

    Inputs:
        Z is a numpy.ndarray (n, m)

    Returns:
        A is activation. numpy.ndarray (n, m)
        cache is a dictionary with {"Z", Z}
    '''
    A = Z
    cache = {}
    return A, cache

def linear_der(dA, cache):
    '''
    computes derivative of linear activation
    This function is implemented for completeness

    Inputs:
        dA is the derivative from subsequent layer. numpy.ndarray (n, m)
        cache is a dictionary with {"Z", Z}, where Z was the input
        to the activation layer during forward propagation

    Returns:
        dZ is the derivative. numpy.ndarray (n,m)
    '''
    dZ = np.array(dA, copy=True)
    return dZ

def softmax_cross_entropy_loss(Z, Y=np.array([])):
    '''
    Computes the softmax activation of the inputs Z
    Estimates the cross entropy loss

    Inputs:
        Z - numpy.ndarray (n, m)
        Y - numpy.ndarray (1, m) of labels
            when y=[] loss is set to []

    Returns:
        A - numpy.ndarray (n, m) of softmax activations
        cache -  a dictionary to store the activations later used to estimate derivatives
        loss - cost of prediction
    '''
    ### CODE HERE
    #softmax
    max_z = np.max(Z, axis=0, keepdims=True)

    # add e^-10 to prevent divide by zero error while taking log A?
    z_exp = np.exp(Z-max_z) #+ np.exp(-10)

    denominator = np.sum(z_exp, axis=0, keepdims=True)
    A = z_exp/denominator
    cache = {'A':A}

    #crossentropy
    if Y.size != 0:
        m = Y.shape[1]
        Y_ = one_hot(Y.astype(int),10).T
        # loss = -(1/Y.shape[1]) * np.sum( np.log( np.sum(A * Y_, axis = 0) ) )
        loss = -(1/Y.shape[1]) * np.sum( np.log( A[Y.astype(int),np.arange(m)] ) )

    else:
        loss = None

    return A, cache, loss

def softmax_cross_entropy_loss_der(Y, cache):
    '''
    Computes the derivative of softmax activation and cross entropy loss

    Inputs:
        Y - numpy.ndarray (1, m) of labels
        cache -  a dictionary with cached activations A of size (n,m)

    Returns:
        dZ - numpy.ndarray (n, m) derivative for the previous layer
    '''
    ### CODE HERE
    Y_ = one_hot(Y.astype(int),10).T
    dZ = (cache['A'] - Y_)/Y.shape[1]

    return dZ

def initialize_multilayer_weights(net_dims):
    '''
    Initializes the weights of the multilayer network

    Inputs:
        net_dims - tuple of network dimensions

    Returns:
        dictionary of parameters
    '''
    np.random.seed(0)
    numLayers = len(net_dims)
    parameters = {}
    for l in range(numLayers-1):
        parameters["W"+str(l+1)] = np.random.randn(net_dims[l+1], net_dims[l]) * 0.01
        parameters["b"+str(l+1)] = np.zeros((net_dims[l+1], 1)) #np.random.randn(net_dims[l+1], 1) * 0.01
    return parameters

def linear_forward(A, W, b):
    '''
    Input A propagates through the layer
    Z = WA + b is the output of this layer.

    Inputs:
        A - numpy.ndarray (n,m) the input to the layer
        W - numpy.ndarray (n_out, n) the weights of the layer
        b - numpy.ndarray (n_out, 1) the bias of the layer

    Returns:
        Z = WA + b, where Z is the numpy.ndarray (n_out, m) dimensions
        cache - a dictionary containing the inputs A
    '''
    ### CODE HERE
    Z = W.dot(A) + b
    cache = {}
    cache["A"] = A
    return Z, cache

def layer_forward(A_prev, W, b, activation):
    '''
    Input A_prev propagates through the layer and the activation

    Inputs:
        A_prev - numpy.ndarray (n,m) the input to the layer
        W - numpy.ndarray (n_out, n) the weights of the layer
        b - numpy.ndarray (n_out, 1) the bias of the layer
        activation - is the string that specifies the activation function

    Returns:
        A = g(Z), where Z = WA + b, where Z is the numpy.ndarray (n_out, m) dimensions
        g is the activation function
        cache - a dictionary containing the cache from the linear and the nonlinear propagation
        to be used for derivative
    '''
    Z, lin_cache = linear_forward(A_prev, W, b)
    if activation == "relu":
        A, act_cache = relu(Z)
    elif activation == "linear":
        A, act_cache = linear(Z)
    elif activation == "tanh":
        A, act_cache = tanh(Z)
    elif activation == "sigmoid":
        A, act_cache = sigmoid(Z)

    cache = {}
    cache["lin_cache"] = lin_cache
    cache["act_cache"] = act_cache
    return A, cache

def multi_layer_forward(X, parameters):
    '''
    Forward propgation through the layers of the network

    Inputs:
        X - numpy.ndarray (n,m) with n features and m samples
        parameters - dictionary of network parameters {"W1":[..],"b1":[..],"W2":[..],"b2":[..]...}
    Returns:
        AL - numpy.ndarray (c,m)  - outputs of the last fully connected layer before softmax
            where c is number of categories and m is number of samples in the batch
        caches - a dictionary of associated caches of parameters and network inputs
    '''
    L = len(parameters)//2
    A = X
    caches = []
    for l in range(1,L):  # since there is no W0 and b0
        A, cache = layer_forward(A, parameters["W"+str(l)], parameters["b"+str(l)], "relu")
        caches.append(cache)

    AL, cache = layer_forward(A, parameters["W"+str(L)], parameters["b"+str(L)], "linear")
    caches.append(cache)
    return AL, caches

def linear_backward(dZ, cache, W, b):
    '''
    Backward prpagation through the linear layer

    Inputs:
        dZ - numpy.ndarray (n,m) derivative dL/dz
        cache - a dictionary containing the inputs A, for the linear layer
            where Z = WA + b,
            Z is (n,m); W is (n,p); A is (p,m); b is (n,1)
        W - numpy.ndarray (n,p)
        b - numpy.ndarray (n, 1)

    Returns:
        dA_prev - numpy.ndarray (p,m) the derivative to the previous layer
        dW - numpy.ndarray (n,p) the gradient of W
        db - numpy.ndarray (n, 1) the gradient of b
    '''
    A_prev = cache["A"]
    ## CODE HERE
    dA_prev = W.T.dot(dZ)
    dW = np.dot(dZ, A_prev.T)
    db = np.sum(dZ, axis=1, keepdims=True)
    return dA_prev, dW, db

def layer_backward(dA, cache, W, b, activation):
    '''
    Backward propagation through the activation and linear layer

    Inputs:
        dA - numpy.ndarray (n,m) the derivative to the previous layer
        cache - dictionary containing the linear_cache and the activation_cache
        activation - activation of the layer
        W - numpy.ndarray (n,p)
        b - numpy.ndarray (n, 1)

    Returns:
        dA_prev - numpy.ndarray (p,m) the derivative to the previous layer
        dW - numpy.ndarray (n,p) the gradient of W
        db - numpy.ndarray (n, 1) the gradient of b
    '''
    lin_cache = cache["lin_cache"]
    act_cache = cache["act_cache"]

    if activation == "sigmoid":
        dZ = sigmoid_der(dA, act_cache)
    elif activation == "tanh":
        dZ = tanh_der(dA, act_cache)
    elif activation == "relu":
        dZ = relu_der(dA, act_cache)
    elif activation == "linear":
        dZ = linear_der(dA, act_cache)
    dA_prev, dW, db = linear_backward(dZ, lin_cache, W, b)
    return dA_prev, dW, db

def multi_layer_backward(dAL, caches, parameters):
    '''
    Back propgation through the layers of the network (except softmax cross entropy)
    softmax_cross_entropy can be handled separately

    Inputs:
        dAL - numpy.ndarray (n,m) derivatives from the softmax_cross_entropy layer
        caches - a dictionary of associated caches of parameters and network inputs
        parameters - dictionary of network parameters {"W1":[..],"b1":[..],"W2":[..],"b2":[..]...}

    Returns:
        gradients - dictionary of gradient of network parameters
            {"dW1":[..],"db1":[..],"dW2":[..],"db2":[..],...}
    '''
    L = len(caches)  # with one hidden layer, L = 2
    gradients = {}
    dA = dAL
    activation = "linear"
    for l in reversed(range(1,L+1)):
        dA, gradients["dW"+str(l)], gradients["db"+str(l)] = \
                    layer_backward(dA, caches[l-1], \
                    parameters["W"+str(l)],parameters["b"+str(l)],\
                    activation)
        activation = "relu"
    return gradients

def classify(X, parameters):
    '''
    Network prediction for inputs X

    Inputs:
        X - numpy.ndarray (n,m) with n features and m samples
        parameters - dictionary of network parameters
            {"W1":[..],"b1":[..],"W2":[..],"b2":[..],...}
    Returns:
        YPred - numpy.ndarray (1,m) of predictions
    '''
    ### CODE HERE
    # Forward propagate X using multi_layer_forward
    # Get predictions using softmax_cross_entropy_loss
    # Estimate the class labels using predictions
    AL, caches = multi_layer_forward(X, parameters)
    Af, cache, cost = softmax_cross_entropy_loss(AL)

    # maxp = np.max(Af, axis=0, keepdims=True)
    # Af = Af - maxp
    # Af[Af == 0] = 1
    # Af[Af < 0] = 0
    YPred = np.zeros(Af.shape)
    maxix = np.argmax(Af, axis=0)
    YPred[maxix, np.arange(len(maxix))] = 1

    return YPred

def update_parameters(parameters, gradients, epoch, learning_rate, decay_rate=0.0):
    '''
    Updates the network parameters with gradient descent

    Inputs:
        parameters - dictionary of network parameters
            {"W1":[..],"b1":[..],"W2":[..],"b2":[..],...}
        gradients - dictionary of gradient of network parameters
            {"dW1":[..],"db1":[..],"dW2":[..],"db2":[..],...}
        epoch - epoch number
        learning_rate - step size for learning
        decay_rate - rate of decay of step size - not necessary - in case you want to use
    '''
    alpha = learning_rate*(1/(1+decay_rate*epoch))
    L = len(parameters)//2
    for i in range(1,L+1):
        parameters['W'+str(i)] = parameters['W'+str(i)] - alpha*gradients['dW'+str(i)]
        parameters['b'+str(i)] = parameters['b'+str(i)] - alpha*gradients['db'+str(i)]
    return parameters, alpha

def multi_layer_network(X, Y, vX, vY, net_dims, num_iterations=500, learning_rate=0.2, decay_rate=0.00):
    '''
    Creates the multilayer network and trains the network

    Inputs:
        X - numpy.ndarray (n,m) of training data
        Y - numpy.ndarray (1,m) of training data labelsf
        net_dims - tuple of layer dimensions
        num_iterations - num of epochs to train
        learning_rate - step size for gradient descent

    Returns:
        costs - list of costs over training
        parameters - dictionary of trained network parameters
    '''
    parameters = initialize_multilayer_weights(net_dims)
    A0 = X
    costs = []
    val_costs = []
    for ii in range(num_iterations):
        ### CODE HERE
        # Forward Prop

        ## call to multi_layer_forward to get activations
        ## call to softmax cross entropy loss
        AL, caches = multi_layer_forward(A0, parameters)
        Af, cache, cost = softmax_cross_entropy_loss(AL, Y)

        vAL, vcaches = multi_layer_forward(vX, parameters)
        vAf, vcache, vcost = softmax_cross_entropy_loss(vAL, vY)

        # Backward Prop
        ## call to softmax cross entropy loss der
        ## call to multi_layer_backward to get gradients
        ## call to update the parameters
        dZ = softmax_cross_entropy_loss_der(Y, cache)
        grads = multi_layer_backward(dZ, caches, parameters)
        parameters, alpha = update_parameters(parameters, grads, ii, learning_rate, decay_rate)

        costs.append(cost)
        val_costs.append(vcost)
        if ii % 10 == 0:
            print("Cost at iteration %i is: %.05f, learning rate: %.05f" %(ii, cost, alpha))

    return costs, val_costs, parameters

def accuracy(YPred, Y):

    Y_ = one_hot(Y.astype(int),10).T
    x = np.abs(YPred-Y_)
    x = np.sum(x)/(2*Y.shape[1])
    return (1-x)

def train(train_data, train_label, val_data, val_label, test_data, test_label, net_dims, learning_rate=0.1, decay_rate=0.00, iters = 501):
    np.random.seed(1)
    # initialize learning rate and num_iterations
    num_iterations = iters

    costs, val_costs, parameters = multi_layer_network(train_data, train_label, val_data, val_label, net_dims, \
            num_iterations, learning_rate, decay_rate)

    # compute the accuracy for training set and testing set
    train_Pred = classify(train_data, parameters)
    val_Pred = classify(val_data, parameters)
    test_Pred = classify(test_data, parameters)

    #test loss
    # AL, caches = multi_layer_forward(test_data, parameters)
    # Af, cache, test_error = softmax_cross_entropy_loss(AL, test_label)

    trAcc = accuracy(train_Pred, train_label)
    valAcc = accuracy(val_Pred, val_label)
    teAcc = accuracy(test_Pred, test_label)
    print("\nAccuracy after",num_iterations,"decay_rate=",decay_rate)
    print("Training: {0:0.3f}%".format(trAcc*100), "Validation: {0:0.3f}%".format(valAcc*100), "Testing: {0:0.3f}%".format(teAcc*100))

    return costs, val_costs, 1-teAcc

def main():
    '''
    Trains a multilayer network for MNIST digit classification (all 10 digits)
    To create a network with 1 hidden layer of dimensions 800
    Run the progam as:
        python deepMultiClassNetwork_starter.py "[784,800]"
    The network will have the dimensions [784,800,10]
    784 is the input size of digit images (28pix x 28pix = 784)
    10 is the number of digits

    To create a network with 2 hidden layers of dimensions 800 and 500
    Run the progam as:
        python deepMultiClassNetwork_starter.py "[784,800,500]"
    The network will have the dimensions [784,800,500,10]
    784 is the input size of digit images (28pix x 28pix = 784)
    10 is the number of digits
    '''
    os.system("mkdir graphs_2")

    net_dims = ast.literal_eval( sys.argv[1] )
    net_dims.append(10) # Adding the digits layer with dimensionality = 10
    print("Network dimensions are:" + str(net_dims))

    # getting the subset dataset from MNIST
    train_data, train_label, val_data, val_label, test_data, test_label = \
            mnist(noTrSamples=5000, noValSamples=1000, noTsSamples=1000,\
            digit_range=[0,1,2,3,4,5,6,7,8,9],\
            noTrPerClass=500, noValPerClass=100, noTsPerClass=100)

    # initialize learning rate and num_iterations
    learning_rate = 0.2
    num_iterations = 1001
    decay_rate = 0.01

    costs, val_costs, test_error = train(train_data, train_label, val_data, val_label, test_data, test_label, net_dims, learning_rate, decay_rate, num_iterations)

    ### CODE HERE to plot costs
    #### Q2 1)
    x_points = [i for i in range(0,len(costs))]

    plt.clf()
    plt.title("Error v/s Iterations")
    cost_line, = plt.plot(x_points, costs, 'b-', label="Training")
    val_line, = plt.plot(x_points, val_costs, 'r-', label="Validation")
    plt.legend(handles=[cost_line, val_line])
    plt.ylabel('Error')
    plt.xlabel('Iterations')
    plt.savefig('graphs_2/Error_vs_Iterations.png')
    # plt.show()
    ####

    #### Q2 2)
    print("Test Error or Misclassification %: "+str(test_error*100))
    print("\n")
    ####

    #### Q2 3)
    c ,vc = [], []
    all_costs = []
    all_v_costs = []
    test_errors = []
    lrs = []
    for i in [0.1,0.5,1,10]:
        print("###")
        print("Learning Rate: ",i)

        tc, tvc, t_e = train(train_data, train_label, val_data, val_label, test_data, test_label, net_dims, i, 0.01, iters=1001)
        x_points = [i for i in range(0,len(tc))]

        plt.clf()
        plt.title('Learning Rate:'+str(i))
        plt.ylabel('Error')
        plt.xlabel('Iterations')
        cost_line, = plt.plot(x_points, tc, 'b-', label="Training")
        val_line, = plt.plot(x_points, tvc, 'r-', label="Validation")
        plt.legend(handles=[cost_line, val_line])
        plt.savefig('graphs_2/Error_Graph_Learning_Rate_'+str(i)+'.png')
        # plt.show()

        lrs.append(i)
        all_costs.append(tc)
        all_v_costs.append(tvc)
        test_errors.append(t_e)
        c.append(tc[-1])
        vc.append(tvc[-1])
        print("###")
        print("\n")

    #All lines together
    handles = []
    plt.clf()
    plt.title('All Learning Rates')
    plt.ylabel('Error')
    plt.xlabel('Iterations')
    plt.rc('axes', prop_cycle=(cycler('color', ['r', 'g', 'b', 'y'])))
    for i in range(len(all_costs)):
        x_points = [i for i in range(0,len(all_costs[i]))]
        cost_line, = plt.plot(x_points, all_costs[i], label=str(lrs[i]))
        handles.append(cost_line)
    plt.legend(handles=handles)
    plt.savefig('graphs_2/Error for all Learning Rates'+str(i*0.01)+'.png')

    # Error (After x iterations) v/s Learning Rate graph
    # x_points = list(range(1, 21, 1))
    # x_points = [i*0.01 for i in x_points]
    # plt.clf()
    # plt.plot(x_points, c, 'b-', x_points, vc, 'r-')
    # plt.ylabel('Error')
    # plt.xlabel('Learning Rate')
    # plt.savefig('graphs_2/Error_vs_Learning_Rate.png')
    # plt.show()
    ####

    #### Q2 4)
    ix = np.argmin(vc)
    lr = lrs[ix]
    print("Minimum validation error found when learning_rate="+str(lr)+", Test accuracy for that run has been printed above")
    ####

if __name__ == "__main__":
    main()


import sendgrid
from app import sg

def send_email(to, subject, body):
    message = sendgrid.Mail()
    message.add_to(to)
    message.set_subject(subject)
    message.set_html(body)
    message.set_text(body)
    message.set_from('admin@budokai-arena.com')
    status, msg = sg.send(message)


#encoding=utf-8
import sys
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.externals import joblib
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
from sklearn.metrics import accuracy_score
import re

if __name__ == '__main__':

   # vocabulary = ['word', 'hello', 'apple', 'milk']
   # tv = TfidfVectorizer(analyzer=u'word', vocabulary = vocabulary)
    from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer,TfidfVectorizer
#为了解决测试集vocabulary维度不同，先加载训练集的vocabulary
    #参考文章http://blog.csdn.net/u013083549/article/details/51262721?locationNum=2
    pattern = re.compile(r'(\w|\d)*\.(com|net|cn|org|cc|gov|edu|cc|co|\w{1,10}$)')
    tv = joblib.load('mix360.m')
    decisionTree = joblib.load('mix360.pkl')

    print('*************')
    fp = open('dnsunique.txt',  'r')
    uris = fp.readlines()
    for uri in uris:
        uriPrint = uri
        #print(uri)
        m = re.search(pattern, uri, flags=0)
        if m:
          #  print(m.group(0))
            uri = m.group(0)
        uri = re.sub("(\.com)|(\.cc)|(\.us)|(\.org)|(\.cn)|(\.\w{1,10})", "", uri)
       # print(uri)
        df = []
        df.append(uri)
        n_grams_tfidf = tv.transform(df)

        predicted = decisionTree.predict(n_grams_tfidf)
        if (predicted):
            print(uriPrint)
           # print(uri)
            print('devil url')
        else:
            pass
            #print('safe url')
    '''
    print('start')
    for i in range(0,len(predicted)):
        if(predicted[i]!= df['label'][i] ):
            print((str(df['uri'][i]))+'预测的'+ str(predicted[i]) +'实际的'+str(df['label'][i] ))
    print('end')
    print("Classifier accuracy:", accuracy_score(df['label'], predicted))

'''

   # n_grams_train = count_vectorizer.fit_transform(x_train['uri'])
   # n_grams_dev = count_vectorizer.transform(x_dev['uri'])

   # n_grams_dev = count_vectorizer.transform(x_dev['uri'])
  #  print('Number of features:', len(count_v1.vocabulary_))


"""

Exercise: 12.3

Description:

	- Rocket: Make a game that begins with a rocket in the center of the screen. Allow the player to move the rocket up, down, left, or right using the four arrow keys. Make sure the rocket never moves beyond any edge of the screen

"""

import pygame

from settings import Settings
from rocket import Rocket
import game_functions as gf

def run_game():
	#Initializes pygame, settings, and screen object
	pygame.init()
	ai_settings = Settings()
	screen = pygame.display.set_mode((ai_settings.screen_width, ai_settings.screen_height))
	pygame.display.set_caption("Rocket Launcher")

	#Make a rocket
	rocket = Rocket(ai_settings, screen)

	#Start the main loop for the game.
	while True:

		gf.check_events(rocket)
		rocket.update()
		gf.update_screen(ai_settings, screen, rocket)

run_game()


import json
from ej15_escribir_venta import inp

if __name__ == "__main__":

    data = {}
    for y, line in enumerate(open("venta")):
        d_obj = {}
        L = line.split()
        for x in range(5):
            d_obj[inp(x)[:-2]] = L[x]
        data[y] = d_obj
    with open("venta.json", "w") as v:
        json.dump(data, v, indent=4)


import inspect

from django.contrib import admin
from django.contrib.admin import helpers
from django.http import HttpResponseRedirect
from django.http.response import HttpResponseBase

class AbstractBaseAdmin(admin.ModelAdmin):
    default_list_display = ('created_at',)
    list_display = ('id', '__str__') + default_list_display
    default_list_filter = ('created_at', 'updated_at')
    list_filter = default_list_filter
    default_search_fields = ('id',)
    search_fields = default_search_fields
    readonly_fields = ('created_at', 'updated_at')
    default_fieldset = (None, {'fields': (('created_at', 'updated_at'),)})

    def changelist_view(self, request, extra_context=None):
        actions = self.get_actions(request)
        if actions and request.method == 'POST' and 'index' in request.POST and '_save' not in request.POST:
            response = self.response_action(request, queryset=self.get_queryset(request))
            if response:
                return response
        return super().changelist_view(request, extra_context)

    def response_action(self, request, queryset):
        try:
            action_index = int(request.POST.get('index', 0))
        except ValueError:
            action_index = 0

        data = request.POST.copy()
        data.pop(helpers.ACTION_CHECKBOX_NAME, None)
        data.pop("index", None)

        try:
            data.update({'action': data.getlist('action')[action_index]})
        except IndexError:
            pass

        action_form = self.action_form(data, auto_id=None)
        action_form.fields['action'].choices = self.get_action_choices(request)

        if action_form.is_valid():
            action = action_form.cleaned_data['action']
            select_across = action_form.cleaned_data['select_across']
            func = self.get_actions(request)[action][0]
            has_queryset = 'queryset' in inspect.signature(func).parameters
            selected = request.POST.getlist(helpers.ACTION_CHECKBOX_NAME)
            if not selected and not select_across and has_queryset:
                return None

            if not select_across:
                queryset = queryset.filter(pk__in=selected)

            if has_queryset:
                response = func(self, request, queryset)
            else:
                response = func(self, request)

            if isinstance(response, HttpResponseBase):
                return response
            else:
                return HttpResponseRedirect(request.get_full_path())
        else:
            return None