# Imports
import matplotlib.pyplot as plt
import autograd
from autograd import numpy as np
# Amorce

def find_seed(g, c=0, eps=2**(-26)) :
    if (c >= g(0,0) and c <= g(0,1)) or (c <= g(0,0) and c >= g(0,1)) :
        #-----------------------------#
        def f(y) :
            return(g(0,y) - c)

        def dichotomie(f,a,b) :
            while abs(a - b) > eps :
                c = (a+b)/2
                if f(c)*f(a) > 0 :
                    a = c
                else :
                    b = c
            return(c)
        #------------------------------#
        t = dichotomie(f,0,1)
        return(t)
    else :
        return(None)

def norme(X):
    return (X[0]**2 + X[1]**2)**(1/2)

def simple_contour(f, c=0.0, delta=0.01) :
    X = [] ; Y = []
    # Définition du gradient
    def grad_f(x,y) :
        g = autograd.grad
        grad = np.array([g(f,0)(x,y),g(f,1)(x,y)])
        if norme(grad) == 0:
            return grad
        else:
            return(grad/norme(grad))  # On retourne un gradient normé
    # Recherche de t sur la frontière
    y = find_seed(f,c) ; x = 0.0
    print(y)
    if y != None :
        X.append(0) ; Y.append(y)
    else :
        return(X,Y)     # Cas où c n'est pas sur la frontière
    gf = grad_f(x,y)
    if norme(gf) == 0:    #si le gradient s'annule c'est qu'on est sur un extremum donc on s'arrête là
        return (X,Y)
    elif gf[1] >= 0 :   #on choisit le bon vecteur orthogonal au gradient pour rester dans la cellule
        E = 1
    else :
        E = -1
    # Recherche de la ligne de niveau
    while x >= 0 and x <= 1 and y >= 0 and y <= 1 :
        gf = grad_f(x,y)
        if norme(gf) == 0:
            return X,Y
        gf = gf/norme(gf)
        x += E*gf[1]*delta ; y -= E*gf[0]*delta
        X.append(x) ; Y.append(y)

    X.pop() ; Y.pop() #le dernier point est hors du cadre
    return(X,Y)
#--------------------Contour complexe--------------------#
#----------Seed pour le contour complexe----------#
def seed_complexe(g, x0, x1, y0, y1, c=0.0, eps=2**(-26)) :
    T = []
    #---------Fonction pour la recherche de t-----------#
    def dichotomie(f,a,b) :
            while abs(a - b) > eps :
                c = (a+b)/2
                if f(c)*f(a) > 0 :
                    a = c
                else :
                    b = c
            return(c)
    #---------On cherche le seed sur les 4 côtés du carré-----------#
    if (c >= g(x0,y0) and c <= g(x0,y1)) or (c <= g(x0,y0) and c >= g(x0,y1)) :
        def f1(y) :
            return(g(x0,y) - c)
        t = dichotomie(f1,y0,y1)
        T.append([x0,t])
    #--------------------#
    if (c >= g(x0,y0) and c <= g(x1,y0)) or (c <= g(x0,y0) and c >= g(x1,y0)) :
        def f2(x) :
            return(g(x,y0) - c)
        t = dichotomie(f2,x0,x1)
        T.append([t,y0])
    #--------------------#
    if (c >= g(x1,y1) and c <= g(x1,y0)) or (c <= g(x1,y1) and c >= g(x1,y0)) :
        def f3(y) :
            return(g(x1,y) - c)
        t = dichotomie(f3,y0,y1)
        T.append([x1,t])
    #--------------------#
    if (c >= g(x1,y1) and c <= g(x0,y1)) or (c <= g(x1,y1) and c >= g(x0,y1)) :
        def f4(x) :
            return(g(x,y1) - c)
        t = dichotomie(f4,x0,x1)
        T.append([t,y1])
    return(T)

###------------------------------###
def dichotomie2(f, c, x, y, gf, n, eps=2**(-26), delta=0.01) :
    if n == 0 :
        a = [x,y] ; b = [x+gf[0]*delta, y+gf[1]*delta]
    elif n == 1 :
        a = [x,y] ; b = [x-gf[0]*delta,y-gf[1]*delta]
    while abs(a[0] - b[0]) > eps and abs(a[1] - b[1]) > eps :
        m = [(a[0]+b[0])/2, (a[1]+b[1])/2]
        if (f(m[0],m[1]) - c)*(f(a[0],a[1]) - c) > 0 :
            a = m[:]
        else :
            b = m[:]
    return(m)

#-------------------------------#
def cplx_contour(f, x0, x1, y0, y1, c=0.0, delta=0.01) : # Fonction modifiée
    X = [] ; Y = []
    # Définition du gradient
    def grad_f(x,y) :
        g = autograd.grad
        grad = np.array([g(f,0)(x,y),g(f,1)(x,y)])
        if norme(grad) == 0:
            return grad
        else:
            return(grad/norme(grad))  # On retourne un gradient normé
    # Recherche de t sur la frontière
    T = seed_complexe(f, x0, x1, y0, y1, c)
    for x,y in T :
        X.append(x) ; Y.append(y)
        gf = grad_f(x,y)
        if norme(gf) == 0:
            return X,Y
        # Il y a différentes conditions pour avoir un vecteur
        # orthogonal au gradient dirigé vers l'intérieur du cadre
        if x == x0 :
            if gf[1] >= 0 :
                E = 1
            else :
                E = -1
        elif x == x1 :
            if gf[1] >= 0 :
                E = -1
            else :
                E = 1
        elif y == y0 :
            if gf[0] >= 0 :
                E = 1
            else :
                E = -1
        else :
            if gf[0] >= 0 :
                E = -1
            else :
                E = 1
        # Recherche de la ligne de niveau
        while x >= x0 and x <= x1 and y >= y0 and y <= y1 :
            gf = grad_f(x,y)
            x += E*gf[1]*delta ; y -= E*gf[0]*delta
            # On se rapproche alors de la ligne de niveau
            if f(x,y) < c :
                x,y = dichotomie2(f,c,x,y,gf,0)
            elif f(x,y) > c :
                x,y = dichotomie2(f,c,x,y,gf,1)

            X.append(x) ; Y.append(y)
        X.pop() ; Y.pop() #le dernier point est hors du cadre
    return(X,Y)

def contour(f, c=0.0, xc=[0.0,1.0], yc=[0.0,1.0], delta=0.01) :
    xs = [] ; ys = []
    for i in range(len(xc)-1) :
        for j in range(len(yc)-1) :
            x,y = cplx_contour(f, xc[i], xc[i+1],yc[j],yc[j+1],c,delta)
            xs.append(x) ; ys.append(y)
    return(xs,ys)

#-----------------------------------------#
def f(x,y) :
    return(2*(np.exp(-x**2-y**2) - np.exp(-(x-1)**2-(y-1)**2)))

def g(x,y) :
    return(x**2 + y**2)

C = [0.0,0.5,1.0,1.5,2.0]
xc = np.linspace(-1,3,40)
yc = np.linspace(-1,2,30)

for c in C :
    xs,ys = contour(f,c,xc,yc)
    for x,y in zip(xs,ys) :
        plt.plot(x,y,'b')

plt.xlim(-1,3)
plt.ylim(-1,2)
plt.show()


# -*- coding: utf-8 -*-
import pytest

from ethereum import slogging

from raiden.mtree import check_proof
from raiden.tests.utils.blockchain import wait_until_block
from raiden.tests.utils.messages import setup_messages_cb
from raiden.tests.utils.network import CHAIN
from raiden.tests.utils.transfer import (
    assert_synched_channels,
    channel,
    direct_transfer,
    get_received_transfer,
    get_sent_transfer,
    pending_mediated_transfer,
    claim_lock,
)
from raiden.transfer.mediated_transfer.state_change import (
    ContractReceiveClosed,
    ContractReceiveWithdraw,
    ContractReceiveSettled,
    ReceiveSecretReveal,
)
from raiden.transfer.state_change import Block
from raiden.utils import sha3, privatekey_to_address

# pylint: disable=too-many-locals,too-many-statements
slogging.configure(':DEBUG')

def assert_secretreveal_or_withdraw(state_change, secret, channel_address, raiden_address):
    if isinstance(state_change, ReceiveSecretReveal):
        assert state_change.secret == secret
        assert state_change.sender == raiden_address
    elif isinstance(state_change, ContractReceiveWithdraw):
        assert state_change.channel_address == channel_address
        assert state_change.secret == secret
        assert state_change.receiver == raiden_address
    else:
        raise ValueError(
            '{} is neither ReceiveSecretReveal or ContractReceiveWithdraw'.format(state_change)
        )

@pytest.mark.parametrize('privatekey_seed', ['settlement:{}'])
@pytest.mark.parametrize('number_of_nodes', [2])
def test_settlement(raiden_network, settle_timeout, reveal_timeout):
    alice_app, bob_app = raiden_network  # pylint: disable=unbalanced-tuple-unpacking

    setup_messages_cb()

    alice_graph = alice_app.raiden.channelgraphs.values()[0]
    bob_graph = bob_app.raiden.channelgraphs.values()[0]
    assert alice_graph.token_address == bob_graph.token_address

    alice_bob_channel = alice_graph.partneraddress_channel[bob_app.raiden.address]
    bob_alice_channel = bob_graph.partneraddress_channel[alice_app.raiden.address]

    alice_deposit = alice_bob_channel.balance
    bob_deposit = bob_alice_channel.balance

    token = alice_app.raiden.chain.token(alice_bob_channel.token_address)

    alice_balance = token.balance_of(alice_app.raiden.address)
    bob_balance = token.balance_of(bob_app.raiden.address)

    alice_chain = alice_app.raiden.chain

    alice_to_bob_amount = 10
    expiration = alice_app.raiden.chain.block_number() + reveal_timeout + 1
    secret = 'secretsecretsecretsecretsecretse'
    hashlock = sha3(secret)

    assert bob_app.raiden.address in alice_graph.partneraddress_channel

    nettingaddress0 = alice_bob_channel.external_state.netting_channel.address
    nettingaddress1 = bob_alice_channel.external_state.netting_channel.address
    assert nettingaddress0 == nettingaddress1

    identifier = 1
    fee = 0
    transfermessage = alice_bob_channel.create_mediatedtransfer(
        alice_app.raiden.address,
        bob_app.raiden.address,
        fee,
        alice_to_bob_amount,
        identifier,
        expiration,
        hashlock,
    )
    alice_app.raiden.sign(transfermessage)
    alice_bob_channel.register_transfer(transfermessage)
    bob_alice_channel.register_transfer(transfermessage)

    assert_synched_channels(
        alice_bob_channel, alice_deposit, [],
        bob_alice_channel, bob_deposit, [transfermessage.lock],
    )

    # At this point we are assuming the following:
    #
    #    A -> B MediatedTransfer
    #    B -> A SecretRequest
    #    A -> B RevealSecret
    #    - protocol didn't continue
    #
    # B knowns the secret but doesn't have an updated balance proof, B needs to
    # call settle.

    # get proof, that locked transfermessage was in merkle tree, with locked.root
    lock = bob_alice_channel.our_state.balance_proof.get_lock_by_hashlock(hashlock)
    assert sha3(secret) == hashlock
    unlock_proof = bob_alice_channel.our_state.balance_proof.compute_proof_for_lock(secret, lock)

    root = bob_alice_channel.our_state.balance_proof.merkleroot_for_unclaimed()

    assert check_proof(
        unlock_proof.merkle_proof,
        root,
        sha3(transfermessage.lock.as_bytes),
    )
    assert unlock_proof.lock_encoded == transfermessage.lock.as_bytes
    assert unlock_proof.secret == secret

    # a ChannelClose event will be generated, this will be polled by both apps
    # and each must start a task for calling settle
    bob_alice_channel.external_state.netting_channel.close(
        bob_app.raiden.address,
        transfermessage,
    )
    wait_until_block(alice_chain, alice_chain.block_number() + 1)

    assert alice_bob_channel.close_event.wait(timeout=15)
    assert bob_alice_channel.close_event.wait(timeout=15)

    assert alice_bob_channel.external_state.closed_block != 0
    assert bob_alice_channel.external_state.closed_block != 0
    assert alice_bob_channel.external_state.settled_block == 0
    assert bob_alice_channel.external_state.settled_block == 0

    # unlock will not be called by Channel.channel_closed because we did not
    # register the secret
    assert lock.expiration > alice_app.raiden.chain.block_number()
    assert lock.hashlock == sha3(secret)

    bob_alice_channel.external_state.netting_channel.withdraw(
        bob_app.raiden.address,
        [unlock_proof],
    )

    settle_expiration = alice_chain.block_number() + settle_timeout + 2
    wait_until_block(alice_chain, settle_expiration)

    assert alice_bob_channel.settle_event.wait(timeout=15)
    assert bob_alice_channel.settle_event.wait(timeout=15)
    # settle must be called by the apps triggered by the ChannelClose event,
    # and the channels must update it's state based on the ChannelSettled event
    assert alice_bob_channel.external_state.settled_block != 0
    assert bob_alice_channel.external_state.settled_block != 0

    address0 = alice_app.raiden.address
    address1 = bob_app.raiden.address

    alice_netted_balance = alice_balance + alice_deposit - alice_to_bob_amount
    bob_netted_balance = bob_balance + bob_deposit + alice_to_bob_amount

    assert token.balance_of(address0) == alice_netted_balance
    assert token.balance_of(address1) == bob_netted_balance

    # Now let's query the WAL to see if the state changes were logged as expected
    state_changes = [
        change[1] for change in alice_app.raiden.transaction_log.get_all_state_changes()
        if not isinstance(change[1], Block)
    ]

    state_change1 = state_changes[0]
    state_change2 = state_changes[1]
    state_change3 = state_changes[2]
    state_change4 = state_changes[3]

    assert(isinstance(state_change1, ContractReceiveClosed))
    assert state_change1.channel_address == nettingaddress0
    assert state_change1.closing_address == bob_app.raiden.address
    assert state_change1.block_number == alice_bob_channel.external_state.closed_block

    # Can't be sure of the order in which we encounter the SecretReveal and the withdraw
    assert_secretreveal_or_withdraw(state_change2, secret, nettingaddress0, bob_app.raiden.address)
    assert_secretreveal_or_withdraw(state_change3, secret, nettingaddress0, bob_app.raiden.address)

    assert(isinstance(state_change4, ContractReceiveSettled))
    assert state_change4.channel_address == nettingaddress0
    assert state_change4.block_number == bob_alice_channel.external_state.settled_block

@pytest.mark.parametrize('privatekey_seed', ['settled_lock:{}'])
@pytest.mark.parametrize('number_of_nodes', [4])
@pytest.mark.parametrize('channels_per_node', [CHAIN])
# TODO: Need to expose the netted value to use a different blockchain_type
@pytest.mark.parametrize('blockchain_type', ['mock'])
def test_settled_lock(token_addresses, raiden_network, settle_timeout, reveal_timeout):
    """ Any transfer following a secret revealed must update the locksroot, so
    that an attacker cannot reuse a secret to double claim a lock.
    """
    token = token_addresses[0]
    amount = 30

    app0, app1, app2, _ = raiden_network  # pylint: disable=unbalanced-tuple-unpacking
    address0 = app0.raiden.address
    address1 = app1.raiden.address

    # mediated transfer
    identifier = 1
    expiration = app0.raiden.chain.block_number() + settle_timeout - reveal_timeout
    secret = pending_mediated_transfer(
        raiden_network,
        token,
        amount,
        identifier,
        expiration,
    )
    hashlock = sha3(secret)

    # get a proof for the pending transfer
    back_channel = channel(app1, app0, token)
    secret_transfer = get_received_transfer(back_channel, 0)
    lock = back_channel.our_state.balance_proof.get_lock_by_hashlock(hashlock)
    unlock_proof = back_channel.our_state.balance_proof.compute_proof_for_lock(secret, lock)

    # reveal the secret
    claim_lock(raiden_network, token, secret)

    # a new transfer to update the hashlock
    direct_transfer(app0, app1, token, amount)

    forward_channel = channel(app0, app1, token)
    last_transfer = get_sent_transfer(forward_channel, 1)

    # call close giving the secret for a transfer that has being revealed
    back_channel.external_state.netting_channel.close(
        app1.raiden.address,
        last_transfer,
    )

    # check that the double unlock will fail
    with pytest.raises(Exception):
        back_channel.external_state.netting_channel.withdraw(
            app1.raiden.address,
            [(unlock_proof, secret_transfer.lock.as_bytes, secret)],
        )

    # forward the block number to allow settle
    settle_expiration = app2.raiden.chain.block_number() + settle_timeout
    wait_until_block(app2.raiden.chain, settle_expiration)

    back_channel.external_state.netting_channel.settle()

    participant0 = back_channel.external_state.netting_channel.contract.participants[address0]
    participant1 = back_channel.external_state.netting_channel.contract.participants[address1]

    assert participant0.netted == participant0.deposit - amount * 2
    assert participant1.netted == participant1.deposit + amount * 2

@pytest.mark.xfail(reason="test incomplete")
@pytest.mark.parametrize('privatekey_seed', ['start_end_attack:{}'])
@pytest.mark.parametrize('number_of_nodes', [3])
def test_start_end_attack(token_addresses, raiden_chain, deposit, reveal_timeout):
    """ An attacker can try to steal tokens from a hub or the last node in a
    path.

    The attacker needs to use two addresses (A1 and A2) and connect both to the
    hub H, once connected a mediated transfer is initialized from A1 to A2
    through H, once the node A2 receives the mediated transfer the attacker
    uses the known secret and reveal to close and settles the channel H-A2,
    without revealing the secret to H's raiden node.

    The intention is to make the hub transfer the token but for him to be
    unable to require the token A1.
    """
    amount = 30

    token = token_addresses[0]
    app0, app1, app2 = raiden_chain  # pylint: disable=unbalanced-tuple-unpacking

    # the attacker owns app0 and app2 and creates a transfer through app1
    identifier = 1
    expiration = reveal_timeout + 5
    secret = pending_mediated_transfer(
        raiden_chain,
        token,
        amount,
        identifier,
        expiration
    )
    hashlock = sha3(secret)

    attack_channel = channel(app2, app1, token)
    attack_transfer = get_received_transfer(attack_channel, 0)
    attack_contract = attack_channel.external_state.netting_channel.address
    hub_contract = channel(app1, app0, token).external_state.netting_channel.address

    # the attacker can create a merkle proof of the locked transfer
    lock = attack_channel.our_state.balance_proof.get_lock_by_hashlock(hashlock)
    unlock_proof = attack_channel.our_state.balance_proof.compute_proof_for_lock(secret, lock)

    # start the settle counter
    attack_channel.netting_channel.close(
        app2.raiden.address,
        attack_transfer,
        None
    )

    # wait until the last block to reveal the secret, hopefully we are not
    # missing a block during the test
    wait_until_block(app2.raiden.chain, attack_transfer.lock.expiration - 1)

    # since the attacker knows the secret he can net the lock
    attack_channel.netting_channel.withdraw(
        [(unlock_proof, attack_transfer.lock, secret)],
    )
    # XXX: verify that the secret was publicized

    # at this point the hub might not know yet the secret, and won't be able to
    # claim the token from the channel A1 - H

    # the attacker settle the contract
    app2.raiden.chain.next_block()

    attack_channel.netting_channel.settle(token, attack_contract)

    # at this point the attack has the "stolen" funds
    attack_contract = app2.raiden.chain.token_hashchannel[token][attack_contract]
    assert attack_contract.participants[app2.raiden.address]['netted'] == deposit + amount
    assert attack_contract.participants[app1.raiden.address]['netted'] == deposit - amount

    # and the hub's channel A1-H doesn't
    hub_contract = app1.raiden.chain.token_hashchannel[token][hub_contract]
    assert hub_contract.participants[app0.raiden.address]['netted'] == deposit
    assert hub_contract.participants[app1.raiden.address]['netted'] == deposit

    # to mitigate the attack the Hub _needs_ to use a lower expiration for the
    # locked transfer between H-A2 than A1-H, since for A2 to acquire the token
    # it needs to make the secret public in the block chain we publish the
    # secret through an event and the Hub will be able to require it's funds
    app1.raiden.chain.next_block()

    # XXX: verify that the Hub has found the secret, close and settle the channel

    # the hub has acquired its token
    hub_contract = app1.raiden.chain.token_hashchannel[token][hub_contract]
    assert hub_contract.participants[app0.raiden.address]['netted'] == deposit + amount
    assert hub_contract.participants[app1.raiden.address]['netted'] == deposit - amount

@pytest.mark.parametrize('blockchain_type', ['geth'])
@pytest.mark.parametrize('number_of_nodes', [2])
def test_automatic_dispute(raiden_network, deposit, settle_timeout, reveal_timeout):
    app0, app1 = raiden_network
    channel0 = app0.raiden.channelgraphs.values()[0].partneraddress_channel.values()[0]
    channel1 = app1.raiden.channelgraphs.values()[0].partneraddress_channel.values()[0]
    privatekey0 = app0.raiden.private_key
    privatekey1 = app1.raiden.private_key
    address0 = privatekey_to_address(privatekey0.secret)
    address1 = privatekey_to_address(privatekey1.secret)
    token = app0.raiden.chain.token(channel0.token_address)
    initial_balance0 = token.balance_of(address0)
    initial_balance1 = token.balance_of(address1)

    # Alice sends Bob 10 tokens
    amount_alice1 = 10
    alice_first_transfer = channel0.create_directtransfer(
        amount_alice1,
        1  # TODO: fill in identifier
    )
    alice_first_transfer.sign(privatekey0, address0)
    channel0.register_transfer(alice_first_transfer)
    channel1.register_transfer(alice_first_transfer)

    # Bob sends Alice 50 tokens
    amount_bob1 = 50
    bob_first_transfer = channel1.create_directtransfer(
        amount_bob1,
        1  # TODO: fill in identifier
    )
    bob_first_transfer.sign(privatekey1, address1)
    channel0.register_transfer(bob_first_transfer)
    channel1.register_transfer(bob_first_transfer)

    # Finally Alice sends Bob 60 tokens
    amount_alice2 = 60
    alice_second_transfer = channel0.create_directtransfer(
        amount_alice2,
        1  # TODO: fill in identifier
    )
    alice_second_transfer.sign(privatekey0, address0)
    channel0.register_transfer(alice_second_transfer)
    channel1.register_transfer(alice_second_transfer)

    bob_last_transaction = bob_first_transfer

    # Alice can only provide one of Bob's transfer, so she is incetivized to
    # use the one with the largest transferred_amount.
    channel0.external_state.close(
        None,
        bob_last_transaction,
    )
    chain0 = app0.raiden.chain
    wait_until_block(chain0, chain0.block_number() + 1)

    assert channel0.close_event.wait(timeout=25)
    assert channel1.close_event.wait(timeout=25)

    assert channel0.external_state.closed_block != 0
    assert channel1.external_state.closed_block != 0

    # Bob needs to provide a transfer otherwise it's netted balance will be
    # wrong, so he is incetivized to use Alice's transfer with the largest
    # transferred_amount.
    channel1.external_state.update_transfer(
        None,
        alice_second_transfer,
    )

    # wait until the settle timeout has passed
    settle_expiration = chain0.block_number() + settle_timeout
    wait_until_block(chain0, settle_expiration)

    # the settle event must be set
    assert channel0.settle_event.wait(timeout=60)
    assert channel1.settle_event.wait(timeout=60)

    # check that the channel is properly settled and that Bob's client
    # automatically called updateTransfer() to reflect the actual transactions
    assert channel0.external_state.settled_block != 0
    assert channel1.external_state.settled_block != 0
    assert token.balance_of(channel0.external_state.netting_channel.address) == 0
    total_alice = amount_alice1 + amount_alice2
    total_bob = amount_bob1
    assert token.balance_of(address0) == initial_balance0 + deposit - total_alice + total_bob
    assert token.balance_of(address1) == initial_balance1 + deposit + total_alice - total_bob


'''
In /home/cyrf0006/research/PeopleStuff/BelangerStuff
'''

import pandas as pd
import matplotlib.pyplot as plt
import os

# Time period
YEAR_MIN = 1998
YEAR_MAX = 2020

## Load bloom param and compute metrics
df_bloom = pd.read_csv('BloomFitParams_NAFO_15Jun2021.csv')
df_bloom = df_bloom[['Region', 'Sensor', 'Year', 'Mean', 'Median', 't[start]', 't[max]', 't[end]', 't[duration]', 'Magnitude[real]', 'Magnitude[fit]', 'Amplitude[real]', 'Amplitude[fit]']]
df_2J = df_bloom[df_bloom.Region=='2J'].groupby('Year').mean()
df_3K = df_bloom[df_bloom.Region=='3K'].groupby('Year').mean()
df_3L = df_bloom[df_bloom.Region=='3L'].groupby('Year').mean()
df_3NO = df_bloom[df_bloom.Region=='3NO'].groupby('Year').mean()
df_3LNO = df_bloom[df_bloom.Region=='3LNO'].groupby('Year').mean()
# Initiation
df_init = pd.concat([df_2J['t[start]'], df_3K['t[start]'], df_3LNO['t[start]']], keys=['2J', '3K','3LNO'], axis=1)
anom_init = (df_init-df_init.mean()) / df_init.std()
anom_init.to_csv('bloom_init_anomaly_for_adamack.csv')
anom_init = anom_init.mean(axis=1)
# Peak
df_max = pd.concat([df_2J['t[max]'], df_3K['t[max]'], df_3LNO['t[max]']], keys=['2J', '3K','3LNO'], axis=1)
anom_max = (df_max-df_max.mean()) / df_max.std()
anom_max = anom_max.mean(axis=1)
anom_max.to_csv('bloom_max_timing_anom.csv')
# End bloom
df_end = pd.concat([df_2J['t[end]'], df_3K['t[end]'], df_3LNO['t[end]']], keys=['2J', '3K','3LNO'], axis=1)
anom_end = (df_end-df_end.mean()) / df_end.std()
anom_end = anom_end.mean(axis=1)
# All together
df_anom = pd.concat([anom_init, anom_max, anom_end], axis=1).mean(axis=1)
# restrict period
df_init = df_init[(df_init.index>=YEAR_MIN) & (df_init.index<=YEAR_MAX)]
df_max = df_max[(df_max.index>=YEAR_MIN) & (df_max.index<=YEAR_MAX)]
df_end = df_end[(df_end.index>=YEAR_MIN) & (df_end.index<=YEAR_MAX)]
df_anom = df_anom[(df_anom.index>=YEAR_MIN) & (df_anom.index<=YEAR_MAX)]

## Load climate index
nlci = pd.read_csv('/home/cyrf0006/AZMP/state_reports/climate_index/NL_climate_index.csv')
nlci.set_index('Year', inplace=True)
nlci = nlci[(nlci.index>=YEAR_MIN) & (nlci.index<=YEAR_MAX)]

## Load calfin stuff (need to be updated)
df0 = pd.read_csv('/home/cyrf0006/research/PeopleStuff/BelangerStuff/CorrelationData.csv', index_col='Year')
df = pd.read_csv('Zooplankton_zonal_mean_anomalies.csv', index_col='year')
df = df[(df.index>=YEAR_MIN) & (df.index<=YEAR_MAX)]

## Correlation
anom_init.name='init'
anom_max.name='max'
anom_end.name='end'
matrix = pd.concat([nlci, anom_init, anom_max, anom_end, df0['calfin'], df['meanAnomaly_Biomass'], df['meanAnomaly_calfin']], axis=1)
from scipy.stats import pearsonr
def calculate_pvalues(df):
    df = df.dropna()._get_numeric_data()
    dfcols = pd.DataFrame(columns=df.columns)
    pvalues = dfcols.transpose().join(dfcols, how='outer')
    for r in df.columns:
        for c in df.columns:
            pvalues[r][c] = round(pearsonr(df[r], df[c])[1], 4)
    return pvalues

corrMatrix = matrix.corr().round(2)
pvalues = calculate_pvalues(matrix)

## plot
fig, ax = plt.subplots(nrows=1, ncols=1)
plt.plot(nlci, linewidth=2)
plt.plot(anom_init, linewidth=2)
plt.grid()
plt.ylabel('Standardized anomaly')
plt.legend(['NLCI', 'Bloom initiation'])
plt.text(2000, 1.25, ' r = -0.59', fontsize=14, fontweight='bold')
fig_name = 'correlation_NLCI_bloom-init.png'
fig.savefig(fig_name, dpi=200)
os.system('convert -trim correlation_NLCI_bloom-init.png correlation_NLCI_bloom-init.png')

fig, ax = plt.subplots(nrows=1, ncols=1)
plt.plot(nlci, linewidth=2)
plt.plot(anom_max, linewidth=2)
plt.grid()
plt.ylabel('Standardized anomaly')
plt.legend(['NLCI', 'Bloom max timing'])
plt.text(2015, -1.15, ' r = -0.73', fontsize=14, fontweight='bold')
plt.text(1997, 1.3, 'a)', fontsize=14, fontweight='bold')
# No tick label
plt.gca().axes.get_xaxis().set_ticklabels([])
fig.set_size_inches(w=7, h=4)
fig_name = 'correlation_NLCI_bloom-max.png'
fig.savefig(fig_name, dpi=200)
os.system('convert -trim correlation_NLCI_bloom-max.png correlation_NLCI_bloom-max.png')

fig, ax = plt.subplots(nrows=1, ncols=1)
plt.plot(nlci, linewidth=2)
plt.plot(anom_end, linewidth=2)
plt.grid()
plt.ylabel('Standardized anomaly')
plt.legend(['NLCI', 'Bloom end timing'])
plt.text(2000, 1.25, ' r = -0.63', fontsize=14, fontweight='bold')
fig.set_size_inches(w=7, h=4)
fig_name = 'correlation_NLCI_bloom-end.png'
fig.savefig(fig_name, dpi=200)
os.system('convert -trim correlation_NLCI_bloom-end.png correlation_NLCI_bloom-end.png')

fig, ax = plt.subplots(nrows=1, ncols=1)
plt.plot(nlci, linewidth=2)
plt.plot(df['meanAnomaly_calfin'], linewidth=2)
plt.grid()
plt.ylabel('Standardized anomaly')
plt.legend(['NLCI', 'Cal. finmarchicus'], loc='upper right')
plt.text(2015, 0.85, ' r = 0.53', fontsize=14, fontweight='bold')
plt.text(1997, 1.25, 'b)', fontsize=14, fontweight='bold')
fig_name = 'correlation_NLCI_calfin.png'
fig.set_size_inches(w=7, h=4)
fig.savefig(fig_name, dpi=200)
os.system('convert -trim correlation_NLCI_calfin.png correlation_NLCI_calfin.png')

fig, ax = plt.subplots(nrows=1, ncols=1)
plt.plot(nlci, linewidth=2)
plt.plot(df['meanAnomaly_Biomass'], linewidth=2)
plt.grid()
plt.ylabel('Standardized anomaly')
plt.legend(['NLCI', 'Zooplankton'], loc='upper right')
plt.text(2015, 0.85, ' r = 0.59', fontsize=14, fontweight='bold')
plt.text(1997, 1.25, 'b)', fontsize=14, fontweight='bold')
fig_name = 'correlation_NLCI_zooplankton.png'
fig.set_size_inches(w=7, h=4)
fig.savefig(fig_name, dpi=200)
os.system('convert -trim correlation_NLCI_zooplankton.png correlation_NLCI_zooplankton.png')

## fig, ax = plt.subplots(nrows=1, ncols=1)
## plt.plot(df_anom, linewidth=2)
## plt.plot(df['calfin'], linewidth=2)
## plt.grid()
## plt.ylabel('Standardized anomaly')
## plt.legend(['bloom metrics', 'Cal. finmarchicus'])
## plt.text(2015, 0.75, ' r = -0.33', fontsize=14, fontweight='bold')
## fig.set_size_inches(w=7, h=4)
## fig_name = 'correlation_bloom_calfin.png'
## fig.savefig(fig_name, dpi=200)
## os.system('convert -trim correlation_bloom_calfin.png correlation_bloom_calfin.png')

os.system('montage correlation_NLCI_bloom-max.png correlation_NLCI_calfin.png -tile 1x2 -geometry +10+10  -background white  montage_nlci_bloom_calfin.png')


"""Main entry point of program. Here, a configs file is parsed and the trainer is instantiated and run."""

import argparse

from senn.trainer import init_trainer

def main():
    """
    Entry point to the trainer.
    Binds together the config and the Trainer class
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('--config', default="configs/compas_lambda1e-4_seed111111.json", help='experiment config file')
    args = parser.parse_args()

    trainer = init_trainer(args.config)
    trainer.run()
    trainer.finalize()

if __name__ == "__main__":
    main()


#!/usr/bin/env python

import curses
import curses.textpad
import time

stdscr = curses.initscr()

curses.noecho()
stdscr.nodelay(1)

curses.textpad.rectangle(stdscr,0,0,10,10)
curses.textpad.rectangle(stdscr,0,11,10,21)
curses.textpad.rectangle(stdscr,0,22,10,33)
stdscr.refresh()
curses.curs_set(1)
win = curses.newwin(9,9,1,1)
updatewin = curses.newwin(9,9,1,12)
numberbox = curses.newwin(9,9,1,23)

rate = .1
number = 0
x = -1
string = ''

def number_print(number,win):
	win.erase()
	win.addstr(str(number))
	numberbox.move(1,1)
	win.noutrefresh()
back = False
while x != 27:
	if x == 127:
		string = string[:-1]
		win.erase()
	if x != -1 and x != 127:
		string+=chr(x)

	if x == 10:
		if string == "hello\n":
			updatewin.erase()
			updatewin.addstr(0,0,"and hello to you!")
			updatewin.noutrefresh()
			win.erase()
			string=''
		if string == "slow\n":
			rate = .001
			win.erase()
			string=''
		if string == "back\n":
			back = True
			win.erase()
			string=''

	number_print(number,numberbox)
	win.addstr(0,0,string)
	win.noutrefresh()
	x = stdscr.getch()
	if back == False:
		number+=rate
	else:
		number-=rate
	curses.doupdate()

curses.endwin()


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.axis as axes
import seaborn as sns
import gmsxfr
import os
import style_parameters as sp
import argparse

pd.plotting.register_matplotlib_converters()

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument("--gams_sysdir", dest="gams_sysdir", default=None, type=str)
    parser.add_argument("--data_repo", dest="data_repo", default=None, type=str)
    parser.add_argument("--output", dest="output", default=None, type=str)

    # parser.set_defaults(gams_sysdir="some path here")
    # parser.set_defaults(data_repo="some path here")

    args = parser.parse_args()

    #
    #
    # get model results
    gdx = gmsxfr.GdxContainer(
        args.gams_sysdir, os.path.join(args.data_repo, "solve_mode.gdx")
    )
    gdx.rgdx(["x_title"])
    x_title = gdx.to_dict("x_title")["text"]

    gdx = gmsxfr.GdxContainer(
        args.gams_sysdir, os.path.join(args.data_repo, "final_results.gdx")
    )
    gdx.rgdx(["frac_r", "import_ibr", "import_ir", "export_ibr", "export_ir"])

    carbon = {
        i: gdx.to_dict("frac_r")["elements"][i]
        if gdx.to_dict("frac_r")["elements"][i] > np.finfo(float).tiny
        else 0
        for i in gdx.to_dict("frac_r")["elements"]
    }

    import_ibr = gdx.to_dataframe("import_ibr")
    import_ir = gdx.to_dataframe("import_ir")
    export_ibr = gdx.to_dataframe("export_ibr")
    export_ir = gdx.to_dataframe("export_ir")

    imports = import_ir["elements"].copy()
    imports.loc[imports[imports["L"] <= np.finfo(float).tiny].index, "L"] = 0
    imports["r"] = imports["r"].map(carbon)

    exports = export_ir["elements"].copy()
    exports.loc[exports[exports["L"] <= np.finfo(float).tiny].index, "L"] = 0
    exports["r"] = exports["r"].map(carbon)

    exim = imports.set_index(["i", "r"]).copy()
    exim.rename(columns={"L": "imports"}, inplace=True)
    exim["exports"] = exports.set_index(["i", "r"]).L
    exim["net"] = exim["imports"] - exim["exports"]
    exim.reset_index(drop=False, inplace=True)

    exim = exim[exim.net != 0].copy()
    exim["i"] = exim["i"].map(sp.region_map)

    if sum(exim["i"].isnull()) != 0:
        raise Exception("incomplete region mapping from style_parameters")

    #
    #
    # plot ex/imports by region for policy scenario
    n_plt = list(set(exim.i))

    plt.style.use(["seaborn-white", "werewolf_style.mplstyle"])

    # plot for cntlreg
    fig, ax = plt.subplots()
    y_div = 5000
    for n, i in enumerate(n_plt):
        df = exim[exim.i == i]

        ax.plot(
            df["r"],
            df["net"],
            visible=True,
            color=sp.cm_region[i],
            linewidth=1,
            label=i,
        )

        ax.plot(
            df["r"],
            [0 for _ in range(len(carbon))],
            visible=True,
            color="black",
            linewidth=2,
            linestyle="--",
        )

        # plt.suptitle('super title here')
        plt.xlabel(x_title)
        plt.ylabel("Net Imports to Cntl Region (MW)")
        plt.tight_layout()
        plt.ylim(
            -y_div * (-min(exim["net"]) // y_div + 1),
            y_div * (max(exim["net"]) // y_div + 1),
        )
        ax.grid(which="major", axis="both", linestyle="--")
        ax.legend(loc="upper right", frameon=True, prop={"size": 6})

    plt.savefig(
        os.path.join(args.output, "agg_exim.png"), dpi=600, format="png",
    )


"""
Flask App for LHS
"""

from flask import Flask, request, jsonify
from flask_pymongo import PyMongo
from utils import AppUtils

# Create an instance of Flask for our app
app = Flask(__name__)

# Connect to our mongo database
app.config['MONGO_URI'] = "mongodb://localhost:27017/test"
mongo = PyMongo(app)

@app.route('/', methods=['GET'])
def query_records():
    """
    Query records in database
    :return: JSON of the records returned
    """
    # Get details from request
    name = request.args.get('name')
    email = request.args.get('email')

    # Form search query
    query = AppUtils.make_search_query(email, name)

    # Search database
    users = mongo.db.users.find(query)

    # Get output into form we want
    output = [AppUtils.object_id_to_string(user) for user in users]

    # Return JSON of users
    return jsonify(output)

# This is the bit that runs:
if __name__ == "__main__":
    # Start the flask app
    app.run()


#!/usr/bin/env python
#

import pygame
from pygame.locals import *
from sys import exit
import random

pygame.init()

screen=pygame.display.set_mode((1024,480),0,32)
pygame.display.set_caption("Cards by Rich Schumacher")

#Create a  background.
background = pygame.Surface((1024,480))
background.fill((0,255,0))

#Load cards image
card_image = pygame.image.load('cards.gif')
CARD_WIDTH=71
CARD_HEIGHT=96

#Display A Card Function
#cardnum : 0:12 = Ace to King, Hearts, 13:25 = Hearts, 25:38 = Clubs, 39:51 = Spades
#loc_x, loc_y : x,y coordinate on screen to place image
def display_a_card(cardnum, loc_x, loc_y):
    cardnum = cardnum % 52
    suit = int(cardnum/13)
    cardnum = cardnum % 13
    card_to_show = pygame.Rect((cardnum * CARD_WIDTH, suit * CARD_HEIGHT),
                               (CARD_WIDTH, CARD_HEIGHT))
    screen.blit(card_image,(loc_x,loc_y),card_to_show)

#Initialize locals
number = 0
suit = 0

#clock and font objects
clock = pygame.time.Clock()
font = pygame.font.SysFont("calibri",40)

#Create a Deck 0..51
myDeck = list(range(52))
spacing = CARD_WIDTH

#Shuffle Deck
def shuffle():
    effect = pygame.mixer.Sound('shuffling-cards-1.wav')
    effect.play()

def display_deck(deck):
    xpos = 0
    ypos = 0

    for j in range(4):
        for i in range(13):
            display_a_card(deck[i+j+j*12], xpos, ypos)
            xpos += spacing
        xpos = 0
        ypos += CARD_HEIGHT

while True:
    for event in pygame.event.get():
        if event.type == QUIT:
            exit()
        if event.type == KEYDOWN:
            flag = True
        if event.type == KEYUP:
            if (event.key == K_s) and (flag == True):
                random.shuffle(myDeck)
                shuffle()
                flag = False
            if (event.key == K_d) and (flag == True):
                spacing /= 2
                flag = False
            if (event.key == K_i) and (flag == True):
                spacing *= 2
                flag = False

    screen.blit(background,(0,0))
    display_deck(myDeck)
    text = font.render('Hit S to Shuffle', True,(0,0,0))
    screen.blit(text,(250,410))
    text2 = font.render('Hit i or d to change spacing', True,(0,0,0))
    screen.blit(text2,(250,440))

    pygame.display.update()


"""
Requests in these tests should not be handled by the middleware.
"""

from django.conf import settings as django_settings

from tests.utils import patch_response_validation_middleware_settings

from django_swagger_tester.configuration import SwaggerTesterSettings

def test_exempt_url(client, caplog, monkeypatch):
    monkeypatch.setattr(
        django_settings,
        'SWAGGER_TESTER',
        patch_response_validation_middleware_settings(
            'VALIDATION_EXEMPT_URLS', [{'url': '^api/v1/cars/correct$', 'status_codes': ['*']}]
        ),
    )
    settings = SwaggerTesterSettings()
    monkeypatch.setattr('django_swagger_tester.middleware.settings', settings)
    client.get('/api/v1/cars/correct')
    assert (
        'Validation skipped: GET request to `/api/v1/cars/correct` with status code 200 is in VALIDATION_EXEMPT_URLS'
        in caplog.messages
    )

def test_non_endpoint_options_request(client, caplog):
    """
    Makes sure these types of requests are *not* handled by the middleware.
    """
    for path in ['', 'test']:
        client.options(path)
        assert 'Validation skipped: `/%s` is not a relevant endpoint' % path in caplog.messages


# Author:Cecilia
import socket
import struct
import json
from concurrent.futures import ThreadPoolExecutor
pool = ThreadPoolExecutor(100)

from db import user_data
from interface import common_interface
from interface import admin_interface
from interface import user_interface
from lib import common

# 做任务分发的功能函数字典
func_dic = {
    'register':common_interface.register_interface,
    'login':common_interface.login_interface,
    'check_movie':common_interface.check_movie_interface,
    'upload_movie':admin_interface.upload_movie_interface,
    'get_movie_list':common_interface.get_movie_list_interface,
    'delete_movie':admin_interface.delete_movie_interface,
    'send_notice':admin_interface.send_notice_interface,
    'buy_vip':user_interface.buy_vip_interface,
    'download_free_movie':user_interface.download_free_movie_interface,
    'download_nofree_movie':user_interface.download_nofree_movie_interface,
    'check_movie_record':user_interface.check_movie_record_interface,
    'check_notice':user_interface.check_notice_interface,

}

# 服务端入口
def run():
    server = socket.socket()
    server.bind(('127.0.0.1',8007))
    server.listen(100)

    while True:
        conn,addr = server.accept()
        print(f'{addr}已经连接服务器')

        # 由线程池对象提交异步任务
        pool.submit(recv_work,conn,addr)

# 执行每个连接对象的请求操作
def recv_work(conn,addr):
    while True:
        try:
            # 接收数据头
            headers = conn.recv(4)
            data_len = struct.unpack('i',headers)[0]# 获取数据的总长度
            json_data = conn.recv(data_len) # 接收数据部分
            back_dic = json.loads(json_data.decode('utf8')) # 将数据反序列化
            back_dic['addr'] = addr

            # func是做任务分发工作的，将每个客户端的任务接收到，转去另一个地方执行
            func(back_dic,conn)
        except Exception as e:
            print(e)
            print(f'{addr}已经断开服务器连接了。。。。。。。')
            user_data.metux.acquire()
            user_data.online_user.pop(addr)
            user_data.metux.release()
            conn.close()
            break

# 任务分发函数
def func(back_dic,conn):
    if back_dic.get('type') not in func_dic:
        send_dic = {'flag':False,'msg':'请求错误！'}
        common.send_msg(send_dic,conn)

    else:
        print('进入',back_dic.get('type'))
        func_dic[back_dic.get('type')](back_dic,conn)
        print('结束',back_dic.get('type'))


from django.contrib import admin
from .models import Department_Home, Subjects

class SubjectInline(admin.TabularInline):
	model = Subjects
	extra = 1

class Department_Admin(admin.ModelAdmin):
	inlines = [SubjectInline]
	model = Department_Home

admin.site.register(Department_Home, Department_Admin)


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import math
import pandas as pd
import numpy as np
import ast
from numpy import zeros
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Flatten
from keras.layers.embeddings import Embedding
from keras.preprocessing.text import Tokenizer
import collections
from keras.layers import LSTM
from sklearn.metrics import mean_squared_error
from sklearn.metrics import accuracy_score
from datetime import datetime
# In[ ]:

df1=pd.read_csv('complete_data9thOct.csv')

# In[ ]:

print("Length:",len(df1.index))

# In[ ]:

print("Grades:",set(df1['grade']))

# In[ ]:

df1['parsed_cleaned_answers2']=[" ".join(ast.literal_eval(a)) for a in list(df1['parsed_cleaned_answers'])]
df1['parsed_cleaned_answers2'] = df1['parsed_cleaned_answers2'].str.replace(',','')
df1['parsed_cleaned_answers2'] = df1['parsed_cleaned_answers2'].str.replace(';','')
df1['parsed_cleaned_answers2'] = df1['parsed_cleaned_answers2'].str.replace('!','')
df1['parsed_cleaned_answers2'] =  df1['parsed_cleaned_answers2'].str.replace('','none')

# In[ ]:

df1=df1[['problem_id','folds','parsed_cleaned_answers','parsed_cleaned_answers2','grade','grade_0','grade_1','grade_2','grade_3','grade_4']]
#df1[0:5]

# In[ ]:

df1=df1[['problem_id','folds','parsed_cleaned_answers','parsed_cleaned_answers2','grade']]
df1 = pd.concat([df1, pd.get_dummies(df1['grade'],prefix='grade')], axis=1)
# df1[0:5]

# In[ ]:

#Tokenizer convert it to numbers
t = Tokenizer()
docs=df1['parsed_cleaned_answers2']
t.fit_on_texts(docs)
vocab_size = len(t.word_index) + 1
print("Vocab size",vocab_size)
# # integer encode the documents
#encoded_docs = t.texts_to_sequences(docs)

# #Converting to numpy array
#for i in range(0,len(encoded_docs )):
#     encoded_docs[i]=np.asarray(encoded_docs[i])
# encoded_docs=np.array(encoded_docs)

#Loading Glove
from numpy import asarray
embeddings_index = dict()
f = open('./glove.6B/glove.6B.100d.txt', 'rb')
for line in f:
    values = line.split()
    word = values[0]
    coefs = asarray(values[1:], dtype='float32')
    embeddings_index[word] = coefs
f.close()
print('Loaded %s word vectors.' % len(embeddings_index))

# # create a weight matrix for words in training docs
embedding_matrix = zeros((vocab_size, 100))
for word, i in t.word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
       embedding_matrix[i] = embedding_vector

# In[ ]:
#lens=[len(x) for x in docs]
#e = Embedding(vocab_size, 100, weights=[embedding_matrix], input_length=None, trainable=False)

e=Embedding(vocab_size, 100, weights=[embedding_matrix], input_length=None, trainable=False,embeddings_initializer='uniform', embeddings_regularizer=None, activity_regularizer=None, embeddings_constraint=None, mask_zero=False)
model = Sequential()
model.add(e)
model.add(LSTM(10,return_sequences=True))
model.add(LSTM(5, activation='softmax',return_sequences=False))
model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])
model.summary()

model.save("InitialModel.h5")

# In[ ]:

problem_rmse=[]
problem_acc=[]
def model_for_each_problem(pid, count, total_probs):
    problem_df=df1.loc[df1['problem_id']==pid]
    folds=list(set(problem_df['folds']))
    validate = False
    fold_rmse=[]
    fold_acc=[]
    if(len(folds)>=3):
        validate=True
        val_fold=folds[-1]
        print("More than 3 folds")

        folds.remove(val_fold)
    print("Folds:",folds)

    results=[]
    val_results=[]

    for f in folds:
        print("Problem:", count, "/", total_probs)
        # current date and time
        now = datetime.now()
        timestamp = datetime.timestamp(now)
        print("timestamp =", timestamp)
        actual=[]
        pred=[]
        train_df=problem_df[problem_df['folds']!=f]
        #print("Length of train:",len(train_df.index))
        #Tokenizer convert it to numbers
        #t = Tokenizer()
        docs=train_df['parsed_cleaned_answers2']
        t.fit_on_texts(docs)
        #vocab_size = len(t.word_index) + 1
        # integer encode the documents
        train_encoded_docs = t.texts_to_sequences(docs)

        for i in range(0,len(train_encoded_docs )):
            train_encoded_docs[i]=np.asarray(train_encoded_docs[i])
        train_encoded_docs=np.array(train_encoded_docs)

        # create a weight matrix for words in training docs
        #embedding_matrix = zeros((vocab_size, 100))
        #for word, i in t.word_index.items():
        #    embedding_vector = embeddings_index.get(word)
        #    if embedding_vector is not None:
        #        embedding_matrix[i] = embedding_vector

        y_train=train_df[['grade_1','grade_2','grade_3','grade_4','grade_5']]
        y_train=np.asarray(y_train)

        # print("Train shape:",train_encoded_docs.shape)

        #TEST DATA:

        test_df=problem_df[problem_df['folds']==f]
        print("Test Fold:",f)
        #print("Length of test df:",len(test_df.index))
        print(test_df)

        docs=test_df['parsed_cleaned_answers2']

        #use same tokenizer
        test_encoded_docs=t.texts_to_sequences(docs)

        for i in range(0,len(test_encoded_docs )):
            test_encoded_docs[i]=np.asarray(test_encoded_docs[i])
        test_encoded_docs=np.array(test_encoded_docs)

        #print("Test shape:",test_encoded_docs.shape)

        y_test=test_df[['grade_1','grade_2','grade_3','grade_4','grade_5']]
        y_test=np.asarray(y_test)

        #VALIDATION DATA:
        if validate==True:

            val_df=problem_df[problem_df['folds']==val_fold]
            docs=val_df['parsed_cleaned_answers2']

            #use same tokenizer
            val_encoded_docs=t.texts_to_sequences(docs)

            for i in range(0,len(val_encoded_docs )):
                val_encoded_docs[i]=np.asarray(val_encoded_docs[i])
            val_x=np.array(val_encoded_docs)
           # print("Validate shape:",val_x.shape)

            y_val=val_df[['grade_1','grade_2','grade_3','grade_4','grade_5']]
            val_y=np.asarray(y_val)

        #define model
        model.load_weights("InitialModel.h5")
        train_x,test_x = train_encoded_docs, test_encoded_docs
        train_y,test_y = y_train, y_test
        prev_a=1000000
        for index,val in enumerate(train_x):
            #print("train X:",train_x[index],"train y:",train_y[index])
            #print("Train index shape:",train_x[index].shape,"Train y shape:",train_y[index].shape)
            model.fit(train_x[index].reshape(1,len(val)),train_y[index].reshape(1,5),epochs = 10 ,batch_size=1)
            mean_a=[]
            if validate==True:

                for index,val in enumerate(val_x):
                    #print("VAL:",val)
                    print("VAL:",val_x.shape,val_y.shape)
                    a=model.evaluate(val_x[index].reshape(1,len(val)),val_y[index].reshape(1,5))
                    mean_a.append(a[0])
        #Checking if you should save these weights

                mean_a=np.mean(mean_a)
                print("Previous loss:",prev_a,"Current loss:",mean_a)
                if(mean_a<prev_a):
                    model.save_weights('./weights/val_weights_lstm.h5')
                    #print("Previous loss:",prev_a,"Current loss:",mean_a)

                    print("saving")
                    prev_a=mean_a

        #If validate set exists, then use saved weights, else just use model. predict()
        if(validate == True):
            model.load_weights('./weights/val_weights_lstm.h5')
        #model.fit(train_x,train_y,epochs = 100 ,batch_size=1)
        print("Test:",test_x)
        for index,val in enumerate(test_x):
            a=model.evaluate(test_x[index].reshape(1,len(val)),test_y[index].reshape(1,5))
            predictions=model.predict(test_x[index].reshape(1,len(val)))
            print("Pred:",predictions,"Argmax:",np.argmax(predictions,axis=1))
            actual.append(np.argmax(test_y[index].reshape(1,5)))
            pred.append(np.argmax(predictions,axis=1))

        actual=np.array(actual).flatten()
        pred=np.array(pred).flatten()
        print("Actual :",actual)
        print("Predictions:",pred)
        print("Fold:",f,"RMSE:",math.sqrt(mean_squared_error(actual,pred)))
        print("Fold:",f,"Acc:",accuracy_score(actual,pred))

        results.append(a)
        fold_rmse.append(math.sqrt(mean_squared_error(actual,pred)))
        fold_acc.append(accuracy_score(actual,pred))
        # print("VALIDATION:")

        # for index,val in enumerate(val_x):
        #     a=model.evaluate(val_x[index].reshape(1,len(val)),val_y[index].reshape(1,5))
        #Checking if you should save these weights
        # prev_a=val_results[-1][0]
        # if(a[0]<prev_a):
        #     model.save_weights('./weights/val_weights_lstm.h5')
        #     print("saving")
        # val_results.append(a)

    results=np.asarray(results)
    fold_rmse=np.asarray(fold_rmse)
    fold_acc=np.asarray(fold_acc)
    print("problem id:",pid)
    print("Result:",results)
    print("RMSE:",fold_rmse)
    print("Mean RMSE:",np.mean(fold_rmse))
    print("Accuracy:",fold_acc)
    print("Mean Accuracy:",np.mean(fold_acc))
    problem_rmse.append(np.mean(fold_rmse))
    problem_acc.append(np.mean(fold_acc))
    print("RMSE so far:",np.mean(problem_rmse))
    print("Acc so far:",np.mean(problem_acc))

problem_ids=list(set(df1['problem_id']))
#model_for_each_problem(1228874)
count = 0
for p in problem_ids:
    count+=1
    print("Problem:", count, "/", len(problem_ids))
    model_for_each_problem(p, count, len(problem_ids))
    #model_for_each_problem(p)

print("Overall RMSE:",np.mean(problem_rmse))
print("Overall Acc:",np.mean(problem_acc))


#automatic.pyを実行し、search.pyを呼び出す
from selenium import webdriver
import search

#chromedriverのpathを指定する
browser = webdriver.Chrome(executable_path='/Users/MatsumotoNasa/Desktop/step/hw1/chromedriver')
browser.get('https://icanhazwordz.appspot.com/')

total_score = 0
#search.pyのdictionary関数を呼び出す
dic = search.dictionary()

#単語入力を10回繰り返す
for i in range(10):
	character = ""
	letters = ""

	#ゲームのWebサイトから16文字を読み込む
	for i in range(1,5):
		for j in range(1,5):
			character = browser.find_element_by_xpath('//table[1][@border="2"]/tbody/tr/td[1]/table/tbody/tr[' + str(j) + ']/td[' + str(i) + ']/div')
			letters = letters + character.text

	#search.pyのmain関数を呼び出す
	high = search.main(letters, dic)
	high_words = high[0]
	total_score = total_score + high[1]

	#scoreをSubmitする。16文字で１つも単語が作れない場合はPASSする。
	if (high[1] > 0):
		browser.find_element_by_id("MoveField").send_keys(high_words)
		browser.find_element_by_xpath('//input[@type="submit"]').click()
	else:
		browser.find_element_by_name('pass').click()

#total score を出力
print('total_score = ' + str(total_score))


from keras.layers import Input, Dense
from keras.models import Model
import math
# =============================================================================
# Dataset Initialization
dataset_main_path = "C:\\Music Technology Master\\7100 - Master Project\\Dataset - Spanish"
vocal_types_train = {"Normal" : 0, "Pathol" : 0}
vocal_types_test = {"Normal" : 0, "Pathol" : 0}
# =============================================================================
# =============================================================================
# Dsp Initialization
snippet_length = 500  #in milliseconds
fs = 16000
block_size = 512
hop_size = 256
mel_length = 128
# =============================================================================

# =============================================================================
# Autoencoder Initialization
encoding_dimension = 64
num_epochs = 50
batch = 256
shuffle_choice = True
# =============================================================================

input_vector_length = mel_length * math.ceil(snippet_length / 1000 * fs / hop_size)

input_mel_spectrogram = Input(shape = (input_vector_length, ))

encoded = Dense(encoding_dimension, activation = 'relu')(input_mel_spectrogram)

decoded = Dense(input_vector_length, activation = 'sigmoid')(encoded)

autoencoder = Model(input_mel_spectrogram, decoded)
encoder = Model(input_mel_spectrogram, encoded)

encoded_input = Input(shape = (encoding_dimension, ) )

decoder_layer = autoencoder.layers[-1]

decoder = Model(encoded_input, decoder_layer(encoded_input))

autoencoder.compile(optimizer = 'adadelta', loss = 'mean_squared_error')

import numpy as np
from os import walk
import librosa

def load_data(vocal_types, dataset_main_path, train_or_test, snippet_length):
    print('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    file_names = []
    for vocal_type in list(vocal_types.keys()):
        print("Now searching " + vocal_type + " recordings for " + train_or_test + "ing.")
        input_path = dataset_main_path + "\\" + vocal_type + "_Augmented_" + str(snippet_length) + "ms_" + train_or_test

        for (dirpath, dirnames, filenames) in walk(input_path):
            file_names.extend(filenames)
            vocal_types[vocal_type] = len(filenames)
            print('We found ' + str(len(filenames)) + ' files')
            print('----------------------------------------------------')
            break

    x_loaded = np.zeros( (len(file_names), mel_length, int(input_vector_length / mel_length)))

    for vocal_type in list(vocal_types.keys()):
        print("Now getting mel spectrogrm from " + vocal_type + " recordings for " + train_or_test + "ing.")
        input_path = dataset_main_path + "\\" + vocal_type + "_Augmented_" + str(snippet_length) + "ms_" + train_or_test

        i = 0
        for filename in file_names:
            x, _ = librosa.load(input_path + "\\" + filename, sr = fs)
            S = librosa.feature.melspectrogram(y = x, sr = fs, n_fft = block_size, hop_length = hop_size)
            x_loaded[i] = S / S.max()
            i = i + 1
            if i == vocal_types[vocal_type] - 1:
               file_names = file_names[ vocal_types[vocal_type] : ]
               break

    return x_loaded

x_train = load_data(vocal_types_train, dataset_main_path, 'train', snippet_length)
x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
print('Training Data is Loaded!')

x_test  = load_data(vocal_types_test, dataset_main_path, 'test',  snippet_length)
x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:])))
print('Testing Data is Loaded!')

print('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
print('Autoencoder Starts')
autoencoder.fit(x_train, x_train,
                epochs = num_epochs,
                batch_size = batch,
                shuffle = shuffle_choice,
                validation_data = (x_test, x_test)
                )


"""Test ETag feature"""

from collections import OrderedDict
import json
import hashlib
from unittest import mock

import pytest

from flask import jsonify, Response
from flask.views import MethodView

from flask_rest_api import Api, Blueprint, abort
from flask_rest_api.etag import _get_etag_ctx
from flask_rest_api.exceptions import (
    CheckEtagNotCalledError,
    NotModified, PreconditionRequired, PreconditionFailed)
from flask_rest_api.utils import get_appcontext
from flask_rest_api.compat import MARSHMALLOW_VERSION_MAJOR

from .mocks import ItemNotFound

HTTP_METHODS = ['OPTIONS', 'HEAD', 'GET', 'POST', 'PUT', 'PATCH', 'DELETE']

@pytest.fixture(params=[True, False])
def app_with_etag(request, collection, schemas, app):
    """Return a basic API sample with ETag"""

    as_method_view = request.param
    DocSchema = schemas.DocSchema
    DocEtagSchema = schemas.DocEtagSchema
    blp = Blueprint('test', __name__, url_prefix='/test')

    if as_method_view:
        @blp.route('/')
        class Resource(MethodView):

            @blp.etag(DocEtagSchema(many=True))
            @blp.response(
                DocSchema(many=True))
            def get(self):
                return collection.items

            @blp.etag(DocEtagSchema)
            @blp.arguments(DocSchema)
            @blp.response(DocSchema, code=201)
            def post(self, new_item):
                return collection.post(new_item)

        @blp.route('/<int:item_id>')
        class ResourceById(MethodView):

            def _get_item(self, item_id):
                try:
                    return collection.get_by_id(item_id)
                except ItemNotFound:
                    abort(404)

            @blp.etag(DocEtagSchema)
            @blp.response(DocSchema)
            def get(self, item_id):
                return self._get_item(item_id)

            @blp.etag(DocEtagSchema)
            @blp.arguments(DocSchema)
            @blp.response(DocSchema)
            def put(self, new_item, item_id):
                item = self._get_item(item_id)
                blp.check_etag(item, DocEtagSchema)
                return collection.put(item_id, new_item)

            @blp.etag(DocEtagSchema)
            @blp.response(code=204)
            def delete(self, item_id):
                item = self._get_item(item_id)
                blp.check_etag(item, DocEtagSchema)
                del collection.items[collection.items.index(item)]

    else:
        @blp.route('/')
        @blp.etag(DocEtagSchema(many=True))
        @blp.response(DocSchema(many=True))
        def get_resources():
            return collection.items

        @blp.route('/', methods=('POST',))
        @blp.etag(DocEtagSchema)
        @blp.arguments(DocSchema)
        @blp.response(DocSchema, code=201)
        def post_resource(new_item):
            return collection.post(new_item)

        def _get_item(item_id):
            try:
                return collection.get_by_id(item_id)
            except ItemNotFound:
                abort(404)

        @blp.route('/<int:item_id>')
        @blp.etag(DocEtagSchema)
        @blp.response(DocSchema)
        def get_resource(item_id):
            return _get_item(item_id)

        @blp.route('/<int:item_id>', methods=('PUT',))
        @blp.etag(DocEtagSchema)
        @blp.arguments(DocSchema)
        @blp.response(DocSchema)
        def put_resource(new_item, item_id):
            item = _get_item(item_id)
            blp.check_etag(item)
            return collection.put(item_id, new_item)

        @blp.route('/<int:item_id>', methods=('DELETE',))
        @blp.etag(DocEtagSchema)
        @blp.response(code=204)
        def delete_resource(item_id):
            item = _get_item(item_id)
            blp.check_etag(item)
            del collection.items[collection.items.index(item)]

    api = Api(app)
    api.register_blueprint(blp)

    return app

class TestEtag():

    def test_etag_is_deterministic(self):
        """Check etag computation is deterministic

           _generate_etag should return the same value everytime the same
           dictionary is passed. This is not obvious since dictionaries
           are unordered by design. We check this by feeding it different
           OrderedDict instances that are equivalent to the same dictionary.
        """

        blp = Blueprint('test', __name__)

        data = OrderedDict([
            ('a', 1),
            ('b', 2),
            ('c', OrderedDict([('a', 1), ('b', 2)]))
        ])
        etag = blp._generate_etag(data)

        data_copies = [
            OrderedDict([
                ('b', 2),
                ('a', 1),
                ('c', OrderedDict([('a', 1), ('b', 2)])),
            ]),
            OrderedDict([
                ('a', 1),
                ('b', 2),
                ('c', OrderedDict([('b', 2), ('a', 1)])),
            ]),
            OrderedDict([
                ('a', 1),
                ('c', OrderedDict([('a', 1), ('b', 2)])),
                ('b', 2),
            ]),
            OrderedDict([
                ('c', OrderedDict([('a', 1), ('b', 2)])),
                ('b', 2),
                ('a', 1),
            ]),
        ]

        data_copies_etag = [blp._generate_etag(d) for d in data_copies]
        assert all(e == etag for e in data_copies_etag)

    @pytest.mark.parametrize('extra_data', [None, {}, {'answer': 42}])
    def test_etag_generate_etag(self, schemas, extra_data):
        blp = Blueprint('test', __name__)
        etag_schema = schemas.DocEtagSchema
        item = {'item_id': 1, 'db_field': 0}
        item_schema_dump = etag_schema().dump(item)
        if MARSHMALLOW_VERSION_MAJOR < 3:
            item_schema_dump = item_schema_dump[0]
        if extra_data is None or extra_data == {}:
            data = item
            data_dump = item_schema_dump
        else:
            data = (item, extra_data)
            data_dump = (item_schema_dump, extra_data)

        etag = blp._generate_etag(item, extra_data=extra_data)
        assert etag == hashlib.sha1(
            bytes(json.dumps(data, sort_keys=True), 'utf-8')
            ).hexdigest()
        etag = blp._generate_etag(item, etag_schema, extra_data=extra_data)
        assert etag == hashlib.sha1(
            bytes(json.dumps(data_dump, sort_keys=True), 'utf-8')
            ).hexdigest()
        etag = blp._generate_etag(item, etag_schema(), extra_data=extra_data)
        assert etag == hashlib.sha1(
            bytes(json.dumps(data_dump, sort_keys=True), 'utf-8')
            ).hexdigest()

    @pytest.mark.parametrize('method', HTTP_METHODS)
    def test_etag_check_precondition(self, app, method):
        blp = Blueprint('test', __name__)

        with app.test_request_context('/', method=method):
            if method in ['PUT', 'PATCH', 'DELETE']:
                with pytest.raises(PreconditionRequired):
                    blp._check_precondition()
            else:
                blp._check_precondition()

    @pytest.mark.parametrize('etag_disabled', (True, False))
    def test_etag_check_etag(self, app, schemas, etag_disabled):
        app.config['ETAG_DISABLED'] = etag_disabled
        blp = Blueprint('test', __name__)
        etag_schema = schemas.DocEtagSchema
        old_item = {'item_id': 1, 'db_field': 0}
        new_item = {'item_id': 1, 'db_field': 1}
        old_etag = blp._generate_etag(old_item)
        old_etag_with_schema = blp._generate_etag(old_item, etag_schema)

        with app.test_request_context('/', headers={'If-Match': old_etag}):
            blp.check_etag(old_item)
            if not etag_disabled:
                with pytest.raises(PreconditionFailed):
                    blp.check_etag(new_item)
            else:
                blp.check_etag(new_item)
        with app.test_request_context(
                '/', headers={'If-Match': old_etag_with_schema}):
            blp.check_etag(old_item, etag_schema)
            if not etag_disabled:
                with pytest.raises(PreconditionFailed):
                    blp.check_etag(new_item, etag_schema)
            else:
                blp.check_etag(new_item)

    @pytest.mark.parametrize('method', HTTP_METHODS)
    def test_etag_verify_check_etag_warning(self, app, method):
        blp = Blueprint('test', __name__)
        old_item = {'item_id': 1, 'db_field': 0}
        old_etag = blp._generate_etag(old_item)

        with mock.patch.object(app.logger, 'warning') as mock_warning:
            with app.test_request_context('/', method=method,
                                          headers={'If-Match': old_etag}):
                blp._verify_check_etag()
                if method in ['PUT', 'PATCH', 'DELETE']:
                    assert mock_warning.called
                    mock_warning.reset_mock()
                else:
                    assert not mock_warning.called
                blp.check_etag(old_item)
                blp._verify_check_etag()
                assert not mock_warning.called

    @pytest.mark.parametrize('method', HTTP_METHODS)
    @pytest.mark.parametrize('debug', (True, False))
    @pytest.mark.parametrize('testing', (True, False))
    def test_etag_verify_check_etag_exception(
            self, app, method, debug, testing):
        app.config['DEBUG'] = debug
        app.config['TESTING'] = testing
        blp = Blueprint('test', __name__)

        with app.test_request_context('/', method=method):
            if (debug or testing) and method in ['PUT', 'PATCH', 'DELETE']:
                with pytest.raises(
                        CheckEtagNotCalledError,
                        match='ETag not checked in endpoint'
                ):
                    blp._verify_check_etag()
            else:
                blp._verify_check_etag()

    @pytest.mark.parametrize('etag_disabled', (True, False))
    def test_etag_set_etag(self, app, schemas, etag_disabled):
        app.config['ETAG_DISABLED'] = etag_disabled
        blp = Blueprint('test', __name__)
        etag_schema = schemas.DocEtagSchema
        item = {'item_id': 1, 'db_field': 0}
        etag = blp._generate_etag(item)
        etag_with_schema = blp._generate_etag(item, etag_schema)

        with app.test_request_context('/'):
            blp.set_etag(item)
            if not etag_disabled:
                assert _get_etag_ctx()['etag'] == etag
                del _get_etag_ctx()['etag']
            else:
                assert 'etag' not in _get_etag_ctx()
        with app.test_request_context(
                '/', headers={'If-None-Match': etag}):
            if not etag_disabled:
                with pytest.raises(NotModified):
                    blp.set_etag(item)
            else:
                blp.set_etag(item)
                assert 'etag' not in _get_etag_ctx()
        with app.test_request_context(
                '/', headers={'If-None-Match': etag_with_schema}):
            if not etag_disabled:
                with pytest.raises(NotModified):
                    blp.set_etag(item, etag_schema)
            else:
                blp.set_etag(item, etag_schema)
                assert 'etag' not in _get_etag_ctx()
        with app.test_request_context(
                '/', headers={'If-None-Match': 'dummy'}):
            if not etag_disabled:
                blp.set_etag(item)
                assert _get_etag_ctx()['etag'] == etag
                del _get_etag_ctx()['etag']
                blp.set_etag(item, etag_schema)
                assert _get_etag_ctx()['etag'] == etag_with_schema
                del _get_etag_ctx()['etag']
            else:
                blp.set_etag(item)
                assert 'etag' not in _get_etag_ctx()
                blp.set_etag(item, etag_schema)
                assert 'etag' not in _get_etag_ctx()

    @pytest.mark.parametrize('etag_disabled', (True, False))
    @pytest.mark.parametrize('method', HTTP_METHODS)
    def test_set_etag_method_not_allowed_warning(
            self, app, method, etag_disabled):
        app.config['ETAG_DISABLED'] = etag_disabled
        blp = Blueprint('test', __name__)

        with mock.patch.object(app.logger, 'warning') as mock_warning:
            with app.test_request_context('/', method=method):
                blp.set_etag(None)
            if method in ['GET', 'HEAD', 'POST', 'PUT', 'PATCH']:
                assert not mock_warning.called
            else:
                assert mock_warning.called

    @pytest.mark.parametrize('paginate', (True, False))
    def test_etag_set_etag_in_response(self, app, schemas, paginate):
        blp = Blueprint('test', __name__)
        etag_schema = schemas.DocEtagSchema
        item = {'item_id': 1, 'db_field': 0}
        if paginate:
            extra_data = (('X-Pagination', 'Dummy pagination header'),)
        else:
            extra_data = tuple()
        etag = blp._generate_etag(item, extra_data=extra_data)
        etag_with_schema = blp._generate_etag(
            item, etag_schema, extra_data=extra_data)

        with app.test_request_context('/'):
            resp = Response()
            if extra_data:
                resp.headers['X-Pagination'] = 'Dummy pagination header'
            get_appcontext()['result_dump'] = item
            blp._set_etag_in_response(resp, None)
            assert resp.get_etag() == (etag, False)

        with app.test_request_context('/'):
            resp = Response()
            if extra_data:
                resp.headers['X-Pagination'] = 'Dummy pagination header'
            get_appcontext()['result_raw'] = item
            blp._set_etag_in_response(resp, etag_schema)
            assert resp.get_etag() == (etag_with_schema, False)

    def test_etag_duplicate_header(self, app):
        """Check duplicate header results in a different ETag"""

        class CustomBlueprint(Blueprint):
            ETAG_INCLUDE_HEADERS = Blueprint.ETAG_INCLUDE_HEADERS + ['X-test']

        blp = CustomBlueprint('test', __name__, url_prefix='/test')

        with app.test_request_context('/'):
            resp = Response()
            resp.headers.add('X-test', 'Test')
            get_appcontext()['result_dump'] = {}
            blp._set_etag_in_response(resp, None)
            etag_1 = resp.get_etag()

        with app.test_request_context('/'):
            resp = Response()
            resp.headers.add('X-test', 'Test')
            resp.headers.add('X-test', 'Test')
            get_appcontext()['result_dump'] = {}
            blp._set_etag_in_response(resp, None)
            etag_2 = resp.get_etag()

        assert etag_1 != etag_2

    def test_etag_response_object(self, app):
        api = Api(app)
        blp = Blueprint('test', __name__, url_prefix='/test')
        client = app.test_client()

        @blp.route('/')
        @blp.etag
        @blp.response()
        def func_response_etag():
            # When the view function returns a Response object,
            # the ETag must be specified manually
            blp.set_etag('test')
            return jsonify({})

        api.register_blueprint(blp)

        response = client.get('/test/')
        assert response.json == {}
        assert response.get_etag() == (blp._generate_etag('test'), False)

    def test_etag_operations_etag_enabled(self, app_with_etag):

        client = app_with_etag.test_client()

        # GET without ETag: OK
        response = client.get('/test/')
        assert response.status_code == 200
        list_etag = response.headers['ETag']

        # GET with correct ETag: Not modified
        response = client.get(
            '/test/',
            headers={'If-None-Match': list_etag}
        )
        assert response.status_code == 304

        # POST item_1
        item_1_data = {'field': 0}
        response = client.post(
            '/test/',
            data=json.dumps(item_1_data),
            content_type='application/json'
        )
        assert response.status_code == 201
        item_1_id = response.json['item_id']

        # GET with wrong/outdated ETag: OK
        response = client.get(
            '/test/',
            headers={'If-None-Match': list_etag}
        )
        assert response.status_code == 200

        # GET by ID without ETag: OK
        response = client.get('/test/{}'.format(item_1_id))
        assert response.status_code == 200
        item_etag = response.headers['ETag']

        # GET by ID with correct ETag: Not modified
        response = client.get(
            '/test/{}'.format(item_1_id),
            headers={'If-None-Match': item_etag}
        )
        assert response.status_code == 304

        # PUT without ETag: Precondition required error
        item_1_data['field'] = 1
        response = client.put(
            '/test/{}'.format(item_1_id),
            data=json.dumps(item_1_data),
            content_type='application/json'
        )
        assert response.status_code == 428

        # PUT with correct ETag: OK
        response = client.put(
            '/test/{}'.format(item_1_id),
            data=json.dumps(item_1_data),
            content_type='application/json',
            headers={'If-Match': item_etag}
        )
        assert response.status_code == 200
        new_item_etag = response.headers['ETag']

        # PUT with wrong/outdated ETag: Precondition failed error
        item_1_data['field'] = 2
        response = client.put(
            '/test/{}'.format(item_1_id),
            data=json.dumps(item_1_data),
            content_type='application/json',
            headers={'If-Match': item_etag}
        )
        assert response.status_code == 412

        # GET by ID with wrong/outdated ETag: OK
        response = client.get(
            '/test/{}'.format(item_1_id),
            headers={'If-None-Match': item_etag}
        )
        assert response.status_code == 200

        # DELETE without ETag: Precondition required error
        response = client.delete('/test/{}'.format(item_1_id))
        assert response.status_code == 428

        # DELETE with wrong/outdated ETag: Precondition failed error
        response = client.delete(
            '/test/{}'.format(item_1_id),
            headers={'If-Match': item_etag}
        )
        assert response.status_code == 412

        # DELETE with correct ETag: No Content
        response = client.delete(
            '/test/{}'.format(item_1_id),
            headers={'If-Match': new_item_etag}
        )
        assert response.status_code == 204

    def test_etag_operations_etag_disabled(self, app_with_etag):

        app_with_etag.config['ETAG_DISABLED'] = True
        client = app_with_etag.test_client()

        # GET without ETag: OK
        response = client.get('/test/')
        assert response.status_code == 200

        # GET with whatever ETag: OK (dummy ETag ignored)
        response = client.get(
            '/test/',
            headers={'If-None-Match': 'dummy_etag'}
        )
        assert response.status_code == 200

        # POST item_1
        item_1_data = {'field': 0}
        response = client.post(
            '/test/',
            data=json.dumps(item_1_data),
            content_type='application/json'
        )
        assert response.status_code == 201
        item_1_id = response.json['item_id']

        # GET by ID: OK
        response = client.get('/test/{}'.format(item_1_id))
        assert response.status_code == 200

        # GET by ID with whatever ETag: OK (dummy ETag ignored)
        response = client.get(
            '/test/{}'.format(item_1_id),
            headers={'If-None-Match': 'dummy_etag'}
        )
        assert response.status_code == 200

        # PUT without ETag: OK
        item_1_data['field'] = 1
        response = client.put(
            '/test/{}'.format(item_1_id),
            data=json.dumps(item_1_data),
            content_type='application/json'
        )
        assert response.status_code == 200

        # PUT with whatever ETag: OK (dummy ETag ignored)
        item_1_data['field'] = 2
        response = client.put(
            '/test/{}'.format(item_1_id),
            data=json.dumps(item_1_data),
            content_type='application/json'
        )
        assert response.status_code == 200

        # POST item_2
        item_2_data = {'field': 9}
        response = client.post(
            '/test/',
            data=json.dumps(item_2_data),
            content_type='application/json'
        )
        assert response.status_code == 201
        item_2_id = response.json['item_id']

        # DELETE without ETag: No Content (dummy ETag ignored)
        response = client.delete('/test/{}'.format(item_1_id))
        assert response.status_code == 204

        # DELETE with whatever ETag: No Content (dummy ETag ignored)
        response = client.delete(
            '/test/{}'.format(item_2_id),
            headers={'If-Match': 'dummy_etag'}
        )
        assert response.status_code == 204


"""
Week 2: Assignment - Question 1

Loads citation data and maps it as a graph(physics citation graph).
Plots the log/log distribution of the normalized distribution of the graph.

Desktop Python version.
Windows cmd call: py - 3.8

Online version: http://www.codeskulptor.org/#user47_EfijJINTuy_16.py

"""
#!/bin/python3

# import math
import urllib.request
import matplotlib.pyplot as plt
import module1_graph as module_1

# Citation url
CITATION_URL = "http://storage.googleapis.com/codeskulptor-alg/alg_phys-cite.txt"

def load_graph(graph_url):
    """ Function that loads a graph given the URL for a text representation of the graph.
    Returns a dictionary that models a graph.
    """
    graph_lines = []
    # Open and process the url
    with urllib.request.urlopen(graph_url) as response:
        graph_text = response.read().decode()
        graph_lines = graph_text.split('\n')
        graph_lines = graph_lines[:-1]

    # Create the graph.
    answer_graph = {}
    for line in graph_lines:
        neighbors = line.split(" ")
        node = int(neighbors[0])
        answer_graph[node] = set([])
        for neighbor in neighbors[1:-1]:
            answer_graph[node].add(int(neighbor))
    return answer_graph

def normalize_distribution(graph, nodes):
    """ Function that computes the normalized distribution of a graph given a
    graph of in-degrees distribution.
    """
    result_graph = {}
    for key, value in graph.items():
        result_graph[key] = value / nodes
    return result_graph

def get_xy_values(graph_dict):
    """ Function that computes the x and y values to be plotted.
    """
    plot_x, plot_y = [], []
    for degree, value in graph_dict.items():
        if degree != 0:
            plot_x.append(degree)
            plot_y.append(value)
    return plot_x, plot_y

def draw_plot(plot_details, plot_type="log", point_color="#8116de", x_limit="", y_limit=""):
    """ Draws the plot figure given the x and y values, and the x and y
    axis labels and the plot title.
    """
    plt.figure(figsize=[7.4, 5.8], facecolor="#e8e8e8")
    plt.figtext(0.2, 0.945, s=plot_details["title"], fontsize='large')
    plt.figtext(0.4, 0.9, s=plot_details["subtitle"], fontsize='large')
    plt.xscale(plot_type)
    plt.yscale(plot_type)
    if x_limit != "":
        plt.xlim(x_limit[0], x_limit[1])
    if y_limit != "":
        plt.ylim(y_limit[0], y_limit[1])
    plt.xlabel(plot_details["label_x"], fontsize="medium")
    plt.ylabel(plot_details["label_y"], fontsize="medium")
    plt.scatter(plot_details["x"], plot_details["y"], c=point_color)
    plt.savefig(plot_details["plot_name"] + ".png")
    plt.close()

if __name__ == "__main__":
    # Loads and creates the citation graph.
    citation_graph = load_graph(CITATION_URL)

    # Compute the unnormalized_distribution
    unnormalized_distr = module_1.in_degree_distribution(citation_graph)

    # Compute the normalized distribution - values normalized to sum of 1.
    num_nodes = len(citation_graph)
    normalized_distr = normalize_distribution(unnormalized_distr, num_nodes)

    # Compute the values and then plot the log/log of the normalized distribution.
    VALUES = sorted(normalized_distr.items())
    X, Y = zip(*VALUES)

    # Draw the plot.
    PLOT_NAME = "Physics_Citation_Graph_Distribution"
    TITLE = "Normalized Distribution of a Physics Citation graph"
    SUBTITLE = "Nodes=" + str(num_nodes)
    LABEL_Y = "In-degrees Distribution (log)"
    LABEL_X = "Degrees (log)"
    PLOT_INFO = {
        "plot_name": PLOT_NAME,
        "title" : TITLE,
        "subtitle" : SUBTITLE,
        "label_x" : LABEL_X,
        "label_y" : LABEL_Y,
        "x" : X,
        "y" : Y
    }
    draw_plot(PLOT_INFO, "log", "#d92b54", (1e0, 1e4), (1e-5, 1e0))
    # draw_plot(X, Y, LABEL_X, LABEL_Y, PLOT_NAME, TITLE, SUBTITLE, "log", "#d92b54",
    #           (1e0, 1e4), (1e-5, 1e0))


# -*- coding: utf-8 -*-

import telegram
import info

class Telegram:

    __bot = None
    user_id = info.Telegram.user_id

    @staticmethod
    def init():
        my_token = info.Telegram.token
        Telegram.__bot = telegram.Bot(token=my_token)

    @staticmethod
    def print_update():
        updates = Telegram.__bot.getUpdates()

        for u in updates:
            print(u.message)

    @staticmethod
    def send_message(text):
        Telegram.__bot.sendMessage(chat_id=Telegram.user_id,
                                   text=text)

if __name__ == '__main__':

    Telegram.init()
    Telegram.send_message('???')


#!/usr/bin/python
# -*- codding: utf-8 -*-
import os
import sys
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
from common.execute_command import write_one_parameter

# url : https://awscli.amazonaws.com/v2/documentation/api/latest/reference/autoscaling/create-or-update-tags.html
if __name__ == '__main__':
    """

    """

    parameter_display_string = """
    # tags : One or more tags.
(structure)

Describes a tag for an Auto Scaling group.
ResourceId -> (string)

The name of the group.

ResourceType -> (string)

The type of resource. The only supported value is auto-scaling-group .

Key -> (string)

The tag key.

Value -> (string)

The tag value.

PropagateAtLaunch -> (boolean)

Determines whether the tag is added to new instances as they are launched in the group.
    """
    add_option_dict = {}

    #######################################################################
    # parameter display string
    add_option_dict["parameter_display_string"] = parameter_display_string
    # ex: add_option_dict["no_value_parameter_list"] = "--single-parameter"
    write_one_parameter("autoscaling", "create-or-update-tags", "tags", add_option_dict)


#!/usr/bin/python3

'''
Date: 17/Nov/2021
Author name: Luis Felipe Cedeño Pérez (pipecedeno@gmail.com)
version: 1.0

Program Description:
This program is used to delete non A,T,C,G or N letters from the genome were the kmers are going to be aligned,
this is important because razers marks an error is any other letter is found in the sequence. The non ATCGN
letters are passed to N, and a new file is created with the modified genome so the original file isn't modified.

Inputs:
-f genome file to be processed
-o name of the output file (the processed genome)
'''

import argparse

parser = argparse.ArgumentParser(description="")
parser.add_argument("-f", "--file",dest="file",required=True) #genome file
parser.add_argument("-o", "--output", dest="output", required=True) #output file
args = parser.parse_args()

genome=open(args.file,"r")
resp=open(args.output,"w")

for line in genome:
	if(">" in line):
		resp.write(line)
	else:
		for base in list(line.rstrip("\n")):
			if((set(base) <= set('ATCG'))):
				resp.write(base)
			else:
				resp.write("N")
		resp.write("\n")
genome.close()
resp.close()


from typing import Set
import random
import pickle
from collections import defaultdict
import nltk

from qanta.datasets.abstract import AbstractDataset, TrainingData
from qanta.wikipedia.cached_wikipedia import Wikipedia, extract_wiki_sentences
from qanta.tagme import TagmeClient
from qanta.util.io import safe_open

class WikipediaDataset(AbstractDataset):
    def __init__(self, answers: Set[str], n_sentences=5, replace_title_mentions=''):
        super().__init__()
        self.answers = answers
        self.n_sentences = n_sentences
        self.replace_title_mentions = replace_title_mentions

    def training_data(self) -> TrainingData:
        wiki_lookup = Wikipedia()
        wiki_content = []
        wiki_answers = []
        for ans in self.answers:
            if ans not in wiki_lookup:
                continue
            wiki_page = wiki_lookup[ans]
            if len(wiki_page.text) != 0:
                sentences = extract_wiki_sentences(
                    ans, wiki_page.text, self.n_sentences,
                    replace_title_mentions=self.replace_title_mentions
                )
                for sent in sentences:
                    wiki_content.append([sent])
                    wiki_answers.append(ans)

        return wiki_content, wiki_answers, None

class TagmeWikipediaDataset(AbstractDataset):
    def __init__(self, location='output/tagme/tagme-wikipedia.pickle', n_examples=20):
        """
        :param answers: Answer set to use in QB normalized format
        :param n_examples: Number of examples per answer to add
        """
        super().__init__()
        self.n_examples = n_examples
        self.location = location

    def build(self, answers: Set[str], save=True):
        client = TagmeClient()
        wiki_lookup = Wikipedia()

        page_sentences = defaultdict(list)
        for ans in answers:
            if ans not in wiki_lookup:
                continue
            wiki_page = wiki_lookup[ans]
            if len(wiki_page.text) != 0:
                sentences = nltk.sent_tokenize(wiki_page.text)
                random.shuffle(sentences)
                clean_sentences, all_mentions = client.tag_mentions(sentences)
                for sent, mentions in zip(clean_sentences, all_mentions):
                    page_mentions = {m.page for m in mentions}
                    n_mentions = len(page_mentions)
                    for page in page_mentions.intersection(answers):
                        raise NotImplementedError('Need to fix this to use extract_wiki_sentences')
                        stripped_sent = strip_title_references(page, sent)
                        page_sentences[page].append((n_mentions, stripped_sent))

        if save:
            with safe_open(self.location, 'wb') as f:
                pickle.dump(page_sentences, f)

        return page_sentences

    def training_data(self):
        with open(self.location, 'rb') as f:
            page_sentences = pickle.load(f)

        tagme_content = []
        tagme_answers = []

        for ans in page_sentences:
            sorted_sentences = sorted(page_sentences[ans], reverse=True, key=lambda x: x[0])
            sent_list = [t[1] for t in sorted_sentences]

            for sentence in sent_list[:self.n_examples]:
                tagme_content.append([sentence])
                tagme_answers.append(ans)

        return tagme_content, tagme_answers, None


import json
from .basemessage import BaseMessage
from .message import MessageType

class CallMessage(BaseMessage):
    def __init__(self,
                 destination: str,
                 origin: str,
                 parameters: list[str | dict | list | int | float | bool | None] | None,
                 reply_to: str,
                 id: str | None = None,
                 latest_hop: str | None = None,
                 type: MessageType = MessageType.CALL
        ):
        super().__init__(origin=origin, destination=destination, type=type, id=id, latest_hop=latest_hop)
        self.parameters = parameters
        self.reply_to = reply_to

    def __repr__(self) -> str:
        attributes = vars(self)
        formatted_attributes = [f'{var}={value}' for var, value in attributes.items()]
        return f"{type(self).__name__}({', '.join(formatted_attributes)})"

    @classmethod
    def from_json(cls, json_message: str):
        #message_dict = json.loads(json_message, object_hook=cls.flatten_hook)
        message_dict = json.loads(json_message)
        ATTR_MAP_INV = {v: k for k, v in cls.ATTR_MAP.items()}
        message_dict = {ATTR_MAP_INV[attr]: value for attr, value in message_dict.items()}
        return cls(message_dict['destination'], message_dict['origin'], message_dict['parameters'], message_dict['reply_to'], message_dict['id'], message_dict['latest_hop'])


# -*- coding: utf-8 -*-

from django.conf.urls.i18n import i18n_patterns
from django.template.loader import render_to_string
from django.http import HttpResponse, HttpResponseNotFound, HttpResponseServerError
from django.conf.urls import patterns, include, url
from views.index    import IndexView, RegisterView
from views.captcha  import CaptchaView
from views.action   import ActionView
from views.addedit  import AddView, EditView
from views.news     import NewsView
from views.article  import ArticleView, ArticleEditView, CategoryView, ArticleReviewEditsView
from views.search   import SearchView
from views.base import menu as m

from django.contrib import admin
admin.autodiscover()

def err404(request):
    menu = m()
    if request.method == 'GET':
        return HttpResponseNotFound(render_to_string('404.html', locals()))
    else:
        return HttpResponseNotFound('404')

def err500(request):
    menu = m()
    if request.method == 'GET':
        return HttpResponseServerError(render_to_string('500.html', locals()))
    else:
        return HttpResponseServerError('500')

handler404 = 'urls.err404'
handler500 = 'urls.err500'

urlpatterns = i18n_patterns(
    '',
    url(r'^$', IndexView.as_view()),
    url(r'^news/(?P<id>\d+)$',       NewsView.as_view()),

    url(r'^news$',      IndexView.as_view()),
    url(r'^news/$',     IndexView.as_view()),
    url(r'^news/add$',               AddView.as_view()),
    url(r'^news/(?P<id>\d+)/edit$',  EditView.as_view()),
    url(r'^news/(?P<id>\d+)$',       NewsView.as_view()),
    url(r'^news/page(?P<page>\d+)$', IndexView.as_view()),

    url(r'^login$', RegisterView.as_view()),

    url(r'^article/cat/(?P<title>.+)$',          CategoryView.as_view()),
    url(r'^article/(?P<title>.+)/edit$',         ArticleEditView.as_view()),
    #url(r'^article/(?P<title>.+)/review/edits$', ArticleReviewEditsView.as_view()),
    url(r'^article/(?P<title>.+)$',              ArticleView.as_view()),

    url(r'^search$',     SearchView.as_view()),
)

urlpatterns += patterns(
    '',
    url(r'^$',          IndexView.as_view()),
    url(r'^act$',     ActionView.as_view()),
    url(r'^news/action$',            ActionView.as_view()),

    url(r'^captcha/1\.jpg$',      CaptchaView.as_view()),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
     url(r'^admin/', include(admin.site.urls)),
)


#!/usr/bin/env python
# -*- coding:utf-8 -*-

#!/usr/bin/env python
# -*- coding:utf-8 -*-
from config.random_date import *
import uiautomator2 as u2
import unittest
import time
from utx import *
import uiautomator2.ext.ocr as ocr

class TestCase(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.d = u2.connect()
        ocr.API = "http://ocr.open.netease.com/api/ocr"
        u2.plugin_register("ocr", ocr.OCR)
        cls.d.set_orientation('natural')
        cls.d.healthcheck()
        cls.d.implicitly_wait(10)
        cls.d.app_clear("com.xiaoxiao.ludan")
        cls.d.app_stop_all()
        cls.NowTime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        cls.test_random = test_random_date()
        Case_DIR = os.path.abspath(os.path.dirname(__file__))
        cls.loanID_PATH = os.path.join(Case_DIR + "\\config\\loan_info.txt")

    def setUp(self):
        self.d.set_fastinput_ime(True)
        self.sess = self.d.session("com.xiaoxiao.ludan")
        self.name = self.test_random.get_name()
        self.phone = self.test_random.createPhone()
        self.idcard = self.test_random.idcard_generator()

    def tearDown(self):
        self.d.app_stop_all()
        self.d.set_fastinput_ime(False)

    def login(self,username,password):
        d = self.sess
        d.watchers.remove()
        d.watchers.watched = False
        log.info('开始登录>>>>>>>>>>')
        d.watcher("获取app权限").when(resourceId="android:id/button1").when(text="允许").click(text="允许")
        d.watchers.run()
        d(resourceId="com.xiaoxiao.ludan:id/et_account").set_text(username,timeout=10)
        d(resourceId="com.xiaoxiao.ludan:id/et_password").set_text(password,timeout=5)
        d(resourceId="com.xiaoxiao.ludan:id/bt_login").click_exists(timeout=5)
        if d(resourceId="com.xiaoxiao.ludan:id/title").exists(timeout=2) == True:
            d(resourceId="com.xiaoxiao.ludan:id/ed_vc").set_text('8888',timeout=5)
            d(resourceId="com.xiaoxiao.ludan:id/tv_sign").click(timeout=5)
        else:
            pass
        self.assertTrue(d(text=u"首页").exists(timeout=3),msg=d.toast.get_message(10, 5))
        log.info('服务器返回：%s' % d.toast.get_message(10, 10))
        print('服务器返回：%s' % d.toast.get_message(10, 10))

    @tag(Tag.Encoding)
    def test_update_loan(self):
        """ 按揭员更新贷款

        :return:
        """
        d = self.sess
        self.login(69,12345678)
        log.info('开始更新贷款>>>>>>>>>>')
        d(resourceId="com.xiaoxiao.ludan:id/tv_menu", text=u"贷款跟进").click(timeout=10)
        self.assertTrue(d(resourceId="com.xiaoxiao.ludan:id/et_pact_search").exists(timeout=3),msg=d.toast.get_message(10, 5))
        if d(resourceId="com.xiaoxiao.ludan:id/tv_search_type",text='客户名').exists() == True:
            d(resourceId="com.xiaoxiao.ludan:id/tv_search_type", text='客户名').click()
        else:
            self.d.ext_ocr.all()
            self.d.ext_ocr("客户名").click(timeout=3)
        d(text=u"贷款号").click(timeout=2)
        with open(self.loanID_PATH, 'r') as f:
            loan_id = f.read()
            log.info('本次待更新的贷款编号为：%s' % loan_id)
            print('本次待更新的贷款编号为：%s' % loan_id)
        d(resourceId="com.xiaoxiao.ludan:id/et_pact_search").clear_text()
        d(resourceId="com.xiaoxiao.ludan:id/et_pact_search").set_text(loan_id)
        self.d.set_fastinput_ime(False)
        d.send_action('search')
        self.d.set_fastinput_ime(True)
        self.assertTrue(d(resourceId="com.xiaoxiao.ludan:id/tv_loan_numb",text='%s' % loan_id).exists(timeout=5),msg=d.toast.get_message(10, 5))
        d(resourceId="com.xiaoxiao.ludan:id/tv_loan_numb", text='%s' % loan_id).click()
        d(resourceId="com.xiaoxiao.ludan:id/title",text='贷款详情').exists(timeout=3)
        d(resourceId="com.xiaoxiao.ludan:id/iv_add").click()
        d(resourceId="com.xiaoxiao.ludan:id/tv_bt_content",text='跟进').click(timeout=5)
        self.assertTrue(d(resourceId="com.xiaoxiao.ludan:id/title",text='已签约待面签').exists(timeout=5),msg=d.toast.get_message(10, 5))
        d(resourceId="com.xiaoxiao.ludan:id/tv_content_name",text='银行经理').sibling(resourceId="com.xiaoxiao.ludan:id/tv_content").click()
        d(text='猴赛雷').click(timeout=3)
        d(resourceId="com.xiaoxiao.ludan:id/tv_edit_title",text='备注').sibling('com.xiaoxiao.ludan:id/et_content').set_text(u"测试数据 - %s" % self.NowTime,timeout=2)
        d(resourceId="com.xiaoxiao.ludan:id/tv_add_manager").click()
        self.assertTrue(d(resourceId="com.xiaoxiao.ludan:id/title", text='贷款详情').exists(timeout=2),msg=d.toast.get_message(10, 5))

if __name__ == '__main__':
    unittest.main()


import base64

with open("favicon.ico","rb") as f:
    strr = base64.b64encode(f.read())
    f.close()
    data = "img = %s" % strr
with open("picture.py","w+") as g:
    g.write(data)
    g.close()

将图片转成py文件.py