# -*- coding: utf-8 -*-
"""
Created on Tue Jun  9 09:19:35 2020

@author: vanderwal1

summary

description

:REQUIRES:

:TODO: Add support for non-sequential models and multiple inputs and outputs.
   1. Add support for more model types. (VAE,DJINN)
   2. Give model types their own construction logic.
   3. Add support for non-SGD-based optimizers if possible (X-BFGS)
   4. Make functions that reduce the number of commands needed by user to build
        a model.

:AUTHOR: Michael Vander Wal ( vanderwal1 )
:ORGANIZATION: Lawrence-Livermore National Laboratory (LLNL)
:CONTACT:
:SINCE: Tue Jun  9 09:19:35 2020
:VERSION: 0.1
"""
#===============================================================================
# PROGRAM METADATA
#===============================================================================
__author__ = 'Michael Vander Wal ( vanderwal1 )'
__contact__ = ''
__copyright__ = ''
__license__ = ''
__date__ = 'Tue Jun  9 09:19:35 2020'
__version__ = '0.2.0'

import tensorflow as tf
import tensorflow.keras as krs
# import numpy as np
# import os
from sys import exit
import argparse

try:
    from archparse_parameter_check import parameter_check
    from archparse_parser_constructor import construct_parsers
    import archparse_parameter_check as arparc
    from archparse_tf_funcs import convert_args
    from archparse_custom_tf import MeanInfNormMSE, custom_objects
except:
    raise

TYPE_LINE = 0
NAME_LINE = TYPE_LINE + 1
INPUT_DIM_LINE = NAME_LINE + 1
LATENT_DIM_LINE = INPUT_DIM_LINE + 1
END_OF_HEADER = LATENT_DIM_LINE + 3

supportedLayers = ['FC','CONV','CONVT','CONV2','CONV2T','FLAT','RSHP','MXP1'
                   'AVP1','UPS1','POLREG']
regChoices = ['L1','L2','L1L2','l1','l2','l1l2']
regLayerChoices = ['CONV','CONVT','CONV2','CONV2T','FC']
typeChoices = ['FC','CONV']
actChoices = ['relu','sig','elu','exp','selu','spls','ssgn','None',
              'lin','smax','swsh','hsig','tanh','slisin','slicos',
              'repolu','relpolu','nlrelu','lognrelu']
initChoices = ['xavu','xavn','glu','gln','orth']

actMap = {
            'elu':  'elu',
            'exp':  'exponential',
            'hsig': 'hard_sigmoid',
            'lin':  'linear',
            'relu': 'relu',
            'selu': 'selu',
            'sig':  'sigmoid',
            'smax': 'softmax',
            'spls': 'softplus',
            'ssgn': 'softsign',
            'swsh': 'swish',
            'tanh': 'tanh',
            'None': None
            }
initMap = {
            'xavu': 'GlorotUniform',
            'xavn': 'GlorotNormal',
            'glu':  'GlorotUniform',
            'gln':  'GlorotNormal',
            'orth': 'Orthogonal',
            'heu':  'he_uniform',
            'hen':  'he_normal',
            'rndn': 'RandomNormal',
            'rndu': 'RandomUniform',
            'vsc':  'VarianceScaling'
            }
padMap = {
            'v': 'valid',
            's': 'same',
            'c': 'causal'
            }
lossMap = {
            'BINCR': krs.losses.BinaryCrossentropy,
            'CATCR': krs.losses.CategoricalCrossentropy,
            'CATH':  krs.losses.CategoricalHinge,
            'COSIM': krs.losses.CosineSimilarity,
            'HINGE': krs.losses.Hinge,
            'HUBER': krs.losses.Huber,
            'KLD':   krs.losses.KLDivergence,
            'LOGC':  krs.losses.LogCosh,
            'MAE':   krs.losses.MeanAbsoluteError,
            'MAPE':  krs.losses.MeanAbsolutePercentageError,
            'MSE':   krs.losses.MeanSquaredError,
            'MSLE':  krs.losses.MeanSquaredLogarithmicError,
            'POIS':  krs.losses.Poisson,
            'SCATC': krs.losses.SparseCategoricalCrossentropy,
            'SQHIN': krs.losses.SquaredHinge,
            'MIMSE' : MeanInfNormMSE
            }
regMap = {
            'L1': krs.regularizers.L1,
            'L2': krs.regularizers.L2,
            'L1L2': krs.regularizers.L1L2
            }

class InputSyntaxError(Exception):
    pass

def read_lines(filePath,fileName):
    """
    Read lines of input file.

    Parameters
    ----------
    filePath : str
        Path to directory of input file.
    fileName : str
        Name of input file.

    Returns
    -------
    lines : list
        List of lines from input file.

    """
    file = open("%s%s"%(filePath,fileName),"r")
    lines = file.readlines()
    for i in range(len(lines)):
        lines[i] = lines[i].partition('#')[0].strip()
    return lines

def parse_loss(line):
    """
    Parse input arguments of a line and return a loss function.

    Builds parser for loss function inputs and then parses the line containing
    the loss funciton.

    Parameters
    ----------
    line : str
        The line from the input file that specifies the loss functio to be used.

    Returns
    -------
    loss :
        The loss function to be used.

    TODO
    ----
    1. Instead of using large if-else if tree, utilize a dict to functions that
        use **kwargs

    """
    parser = argparse.ArgumentParser()
    parser.add_argument('-reduct',metavar='reduction',dest='reduct',type=str,choices = ['none','sum','sum_over_batch','sum_over_batch_size','auto'],default='auto', help = "Reduction method, Default is AUTO")
    subparser = parser.add_subparsers(title = "LOSS FUNCS",dest = 'loss')

    binCross = subparser.add_parser('BINCR',add_help=True, help = "Binary Cross-Entropy")
    binCross.add_argument('-logits',action='store_true', help="Boolean - whether to interpret as logits")
    binCross.add_argument('-smooth',metavar='smooth',dest='smooth',type=float,default=0.0, help="label smoothing - in range [0,1]. 0 is not smoothing, 1 is heaviest smothing.")

    catCross = subparser.add_parser('CATCR',add_help=True, help = "Categorical Cross-Entropy")
    catCross.add_argument('-logits',action='store_true', help="Boolean - whether to interpret as logits")
    catCross.add_argument('-smooth',metavar='smooth',dest='smooth',type=float,default=0.0, help="label smoothing - in range [0,1]. 0 is not smoothing, 1 is heaviest smothing.")

    catHinge = subparser.add_parser('CATH',add_help=True, help = "Categorical Hinge")

    cosSim = subparser.add_parser('COSIM',add_help=True, help = "Cosine Similarity")
    cosSim.add_argument('-axis',metavar='axis',dest='axis',type=int,default=-1, help = "Axis along which cosine similarity is computed")

    hinge = subparser.add_parser('HINGE',add_help=True, help = "Hinge")

    huber = subparser.add_parser('HUBER',add_help=True, help = "Huber")
    huber.add_argument('-delta',metavar='delta',dest='delta',type=float,default=1.0, help = "a float that is the point where the Huber loss function changes from quadratic to linear.")

    kld = subparser.add_parser('KLD',add_help=True, help="Kulback-Liebler Divergence")
    logc = subparser.add_parser('LOGC',add_help=True, help = "LogCosh")
    mae = subparser.add_parser('MAE',add_help=True, help = "Mean Absolute Error")
    mape = subparser.add_parser('MAPE',add_help=True, help = "Mean Absolute Percent Error")
    mse = subparser.add_parser('MSE',add_help=True, help = "Mean Square Error")
    msle = subparser.add_parser('MSLE',add_help=True, help = "Mean Square Logarithmic Error")
    poisson = subparser.add_parser('POIS',add_help=True, help = "Poisson")

    mimse = subparser.add_parser('MIMSE',add_help=True, help = "Mean-Infinity Norm + Mean Square Error")
    mimse.add_argument('-mse','--mse_weight',metavar="mse_weight",dest="mse_weight",type=float,default=.5, help = "MSE weight")
    mimse.add_argument('-inf','--inf_weight',metavar="inf_weight",dest="inf_weight",type=float,default=.5, help = "Mean-Inf Norm weight")

    sparseCatCross = subparser.add_parser('SCATC',add_help=True, help = "Sparse Categorical Cross-Entropy")
    sparseCatCross.add_argument('-logits',action='store_true', help="Boolean - whether to interpret as logits")

    sqHinge = subparser.add_parser('SQHIN',add_help=True, help = "Square Hinge")

    args = vars(parser.parse_args(line))

    if args['reduct'] == 'auto':
        reduct = krs.losses.Reduction.AUTO
    elif args['reduct'] == 'none':
        reduct = krs.losses.Reduction.NONE
    elif args['reduct'] == 'sum':
        reduct = krs.losses.Reduction.SUM
    else:
        reduct = krs.losses.Reduction.SUM_OVER_BATCH_SIZE

    lossFunc = args['loss']
    if lossFunc == 'BINCR':
        loss = lossMap[lossFunc](from_logits=args['logits'],label_smoothing=args['smooth'],reduction=reduct)
    elif lossFunc == 'CATCR':
        loss = lossMap[lossFunc](from_logits=args['logits'],label_smoothing=args['smooth'],reduction=reduct)
    elif lossFunc == 'COSIM':
        loss = lossMap[lossFunc](axis=args['axis'],reduction=reduct)
    elif lossFunc == 'HUBER':
        loss = lossMap[lossFunc](delta=args['delta'],reduction=reduct)
    elif lossFunc == 'SCATC':
        loss = lossMap[lossFunc](from_logits=args['logits'],reduction=reduct)
    elif lossFunc == 'MIMSE':
        loss = lossMap[lossFunc](mse_weight=args['mse_weight'],infNorm_weight=args['inf_weight'])
    else:
        loss = lossMap[lossFunc](reduction=reduct)

    return loss

def parse_body(lines):
    """
    Extract the various sections of the input file.

    This extracts the header information and some of the training information
    as well as the architecture information in blocks/sections. These blocks
    undergo further processing either in this function or another function.

    Parameters
    ----------
    lines : list
        The lines from the input file.

    Raises
    ------
    InputSyntaxError
        If there is a line with unknown structure/syntax or there is one or
        more blocks missing or in a an unexpected order, an error is raised.
    ValueError
        If the first lines is not recognized abbreviation for a supported
        network, an error is raised.

    Returns
    -------
    param_dict : dict
        A dictionary of the network parameters/blocks.

    TODO
    ----
    Break the model structure parsing into different functions for different
    network types.

    Use a dict to addressed by the abbreviation in the first line of input
    which declares the network type to access those functions.

    """
    if 'AE' in lines and 'NN' in lines:
        raise InputSyntaxError("AE and NN cannot both be present in the same file.")

    networkType = lines[TYPE_LINE]
    name = lines[NAME_LINE]

    inputDim = lines[INPUT_DIM_LINE].split()[1:]
    for i in range(len(inputDim)):
        inputDim[i] = int(inputDim[i])

    latentDim = lines[LATENT_DIM_LINE].split()[1:]
    for i in range(len(latentDim)):
        latentDim[i] = int(latentDim[i])

    if 'LOSS' in lines[LATENT_DIM_LINE+1].upper():
        global END_OF_HEADER
        END_OF_HEADER = LATENT_DIM_LINE+4
        # print("END_OF_HEADER",END_OF_HEADER)
        sep_loss = lines[LATENT_DIM_LINE+1].split()
        lossLine = [sep_loss[1].upper()]
        if len(sep_loss) > 2:
            lossLine += sep_loss[2:]

        # print("-----------LossLine----------",lossLine)
        loss = parse_loss(lossLine)
    else:
        lossLine = ['MSE']
        loss = parse_loss(lossLine)

    inputDim = tuple(inputDim)
    latentDim = tuple(latentDim)
    networkSets = []

    if networkType == 'AE':
        if not "ENCOD" in lines or not "DECOD" in lines:
            raise InputSyntaxError("ENCOD and DECOD are both expected in the input file when AE input provided. One or both were not provided")
        encoderStart=lines.index('ENCOD')
        decoderStart=lines.index('DECOD')
        encoderLines = []
        decoderLines = []

        for i in range(encoderStart+1,decoderStart-1):
            encoderLines.append(lines[i])
        for i in range(decoderStart+1,len(lines)):
            decoderLines.append(lines[i])
        networkSets = (encoderLines,decoderLines)

    elif networkType == 'NN':
        if "ENCOD" in lines or "DECOD" in lines:
            raise InputSyntaxError("ENCOD and DECOD cannot be present when NN is provided.")
        if not "NTWRK" in lines:
            raise InputSyntaxError("NTWRK block must be present when NN is provided")
        networkStart = lines.index('NTWRK')
        networkLines = []
        for i in range(networkStart+1,len(lines)):
            networkLines.append(lines[i])
        networkSets = (networkLines,)

    else:
        raise ValueError("The first line of input file must be either NN or AE, but %s was given"%(networkType))

    custom_model_components = {}
    if isinstance(loss,custom_objects):
        for i in range(len(custom_objects)):
            if isinstance(loss,custom_objects[i]):
                key = str(type(loss))
                key = key.split('\'')[1]
                key = key.rpartition('.')[2]
                custom_model_components[key] = custom_objects[i]
                # custom_model_components[key] = loss
        # print(custom_model_components)

    # USE FOR OPTIMIZER LATER
    # if isinstance(opt,custom_objects):
    #     key = str(loss)
    #     key = key.split('\'')[1]
    #     key = key.rpartition('.')[2]
    #     custom_model_components[key] = opt

    param_dict = {    'type': networkType,
                      'name': name,
                      'inputDim': inputDim,
                      'latentDim': latentDim,
                      'unparsed_model': networkSets,
                      'loss': loss,
                      'optimizer': None,
                      'callbacks': None,
                      'algorithms': None,
                      'nInputs': 1,
                      'learningRate': .001,
                      'custom': custom_model_components}

    return param_dict

def parse_line(line,lineNumber=-1):
    """
    Parse input file line corresponding to a network layer type.

    Parameters
    ----------
    line : list
        List of strings that were the substrings in the input file separated
        by whitespace.
    lineNumber : int
        The line number of the file from which the command came from.
        Default is -1.

    Raises
    ------
    ValueError
        If the input for an activation function is not recognized,
        an error is raised.

    Returns
    -------
    args : dict
        A dictionary with arguments as keys which point to their
        associated values.

    TODO
    ----
    Pull contruct_parsers() out of this function and either pass the parser as
    a function argument or as a global. Possibly make it an overridable global.

    """
    if '-a' in line and not line[line.index('-a')+1] in actChoices:
        raise ValueError('The choice given for an activation function on line %i is not supported.'%(lineNumber))
    parser = construct_parsers()

    args = parser.parse_args(line)
    # print(line)
    # print(args)
    args = vars(args)

    parameter_check(args,lineNumber)

    # print(args['act'],args['init'])
    # print(args['actOptions'])
    # parser.print_help()
    # exit()
    # for item in args.items():
    #     print(item)
    return args

def link_layers(argList,input_shape):
    """
    Link the layers of the model/sub-model together.

    Parameters
    ----------
    argList : list
        List of argument dictionaries for layers.
    input_shape : tuple
        Shape of input for the model.

    Returns
    -------
    tensorflow.keras.Model
        A linked model.

    Notes
    -----
    This is currently only capable of supporting single inputs and
    sequential models.

    TODO
    ----
    Add multiple input support

    """
    layers = []
    inputs = krs.Input(shape=input_shape)
    x = inputs
    for args in argList:
        layers.append(convert_args(args))

    for layer in layers:
        x = layer(x)

    return krs.Model(inputs,x)

# def process_arch_file(filePath,fileName,debug=False):
#     """
#     Process the architecture input file.

#     This function handles the passing of input to the various stages of
#     parsing and translation.

#     The model isn't directly built with this function. Instead a list of the
#     translated inputs is returned to be done with as the user wishes.

#     Parameters
#     ----------
#     filePath : str
#         The path to the directory of the input file.
#     fileName : str
#         The name of the input file.
#     debug : bool, optional
#         Used to set whether debugging messages will be printed.
#         The default is False.

#     Raises
#     ------
#     Exception
#         If the debug=True and an error occurs during parsing due to an
#         invalid option.

#     Returns
#     -------
#     layers : list of layers
#         DESCRIPTION.

#     """
#     lines = read_lines(filePath,fileName)
#     lineNumber = 1
#     layers = []
#     if not debug:
#         try:
#             for line in lines:
#                 print(line.partition('\n')[0])
#                 args = parse_line(line.split(),lineNumber)
#                 layers.append(convert_args(args))
#                 print(type(layers[-1]))
#                 lineNumber += 1
#             return layers
#         except ValueError:
#             raise
#         except:
#             raise Exception("Invalid option(s) on line %i of %s%s."%(lineNumber,filePath,fileName))
#     else:
#         for line in lines:
#             print(line.partition('\n')[0])
#             args = parse_line(line.split(),lineNumber)
#             layers.append(convert_args(args))
#             print(type(layers[-1]))
#             lineNumber += 1
#         return layers

def retrieve_args_parsedBody(parsedBody):
    """
    Retrieve args from parsedBody.

    This function extracts the arguments by calling `parse_line` on the lines
    stored in parsedBody dict. The arguments are grouped by the sub-model to
    which they belong.

    Parameters
    ----------
    parsedBody : dict
        Dictionary containing the model information..

    Returns
    -------
    argSet : tuple
        The tuple contains the arguments for all of the sub-models of the model.

    """
    netType = parsedBody['type']
    networkSets = parsedBody['unparsed_model']
    if netType == 'AE':
        encArgs = []
        decArgs = []
        lineNumber = END_OF_HEADER+1
        print("line number",lineNumber, "end of header",END_OF_HEADER+1)
        for line in networkSets[0]:
            print(line.partition('\n')[0])
            encArgs.append(parse_line(line.split(),lineNumber))
            lineNumber += 1
        lineNumber += 3
        for line in networkSets[1]:
            print(line.partition('\n')[0])
            decArgs.append(parse_line(line.split(),lineNumber))
            lineNumber += 1
        argSet = (encArgs,decArgs)

    elif netType == 'NN':
        args = []
        lineNumber = END_OF_HEADER
        for line in networkSets[0]:
            print(line.partition('\n')[0])
            args.append(parse_line(line.split(),lineNumber))
            lineNumber += 1
        argSet = (args,)
    return argSet

def retrieve_args(parsedBody = None, lines = None, filePath=None, fileName=None, debug=False):
    """
    Extract layer construction arguments from parsedBody.

    Retrieves the layer construction arguments from one of many different input
    methods. Only of of parsedBody, lines, or filePath & fileName should be
    passed. If more than one is is passed priority is as follows

    parsedBody > lines > filePath & fileName

    parsedBody should be used as the other input options currently do nothing.
    The other options may be removed in the future.

    Parameters
    ----------
    parsedBody : dict, optional
        Dictionary containing the model information. The default is None.
    lines : list, optional
        List of strings that are the lines of input file. The default is None.
    filePath : str, optional
        Path to the directory of the input file. The default is None.
    fileName : str, optional
        Name of the input file. The default is None.
    debug : bool, optional
        Used to set whether debug statements are printed. The default is False.

    Raises
    ------
    Exception
        If an invalid option is encountered during parsing, and error is raised.

    Returns
    -------
    tuple
        The tuple contains the arguments for all of the sub-models of the model.

    TODO
    ----
    Either remove lines, filePath, and fileName, or properly re-implement them.

    """
    # Priority is parsedBody > lines > filePath & fileName
    if not parsedBody is None:
        if not debug:
            try:
                return retrieve_args_parsedBody(parsedBody)
            except:
                raise
        else:
            return retrieve_args_parsedBody(parsedBody)

    # else:
    #     if lines is None:
    #         lines = read_lines(filePath,fileName)

    #     lineNumber = 1
    #     args = []
    #     if not debug:
    #         try:
    #             for line in lines:
    #                 print(line.partition('\n')[0])
    #                 args.append(parse_line(line.split(),lineNumber))
    #                 lineNumber += 1
    #             return args
    #         except ValueError:
    #             raise
    #         except:
    #             raise Exception("Invalid option(s) on line %i of %s%s."%(lineNumber,filePath,fileName))
    #     else:
    #         for line in lines:
    #             print(line.partition('\n')[0])
    #             args.append(parse_line(line.split(),lineNumber))
    #             lineNumber += 1
    #         return args

# def generate_layers(args,debug=False):
#     lineNumber = 1
#     layers = []
#     if not debug:
#         try:
#             for line in args:
#                 layers.append(convert_args(line))
#                 print(type(layers[-1]))
#                 lineNumber += 1
#             return layers
#         except:
#             print("\033[0;31;48m Attention: \033[0;37;48m Invalid option(s) on line %i of input"%(lineNumber))
#             raise
#     else:
#         for line in args:
#             layers.append(convert_args(line))
#             print(type(layers[-1]))
#             lineNumber += 1
#         return layers

def generate_model( parsedBody, argSet):
    """
    Translate arguments to layers and link layers.

    The arguments in argSet are converted into tensorflow.keras layers.
    The layers are then linked together to make the full model. The models and
    its sub-models, if there are sub-models, are returned in a tuple.

    Parameters
    ----------
    parsedBody : dict
        Dictionary containing the model information.
    argSet : tuple
        The arguments of the sub-models.

    Returns
    -------
    modelSet : tuple
        The full model and its sub-models. The sub-models are in order of
        appearance in the full model. The last item in the tuple is the
        full model.

    TODO
    ----
    Possibly add non-sequential model support.

    """
    # modelSet = (0,)
    netType = parsedBody['type']
    name = parsedBody['name']
    inDim = parsedBody['inputDim']
    latDim = parsedBody['latentDim']
    loss = parsedBody['loss']
    # networkSets = parsedBody[NETWORK_SET_INDEX]

    print(f"inDim :  {inDim}   latDim : {latDim}")
    if netType == 'AE':
        encoder = link_layers(argSet[0],inDim)
        encoder._name = "encoder"
        # encoder.summary()

        decoder = link_layers(argSet[1],latDim)
        decoder._name = "decoder"
        # decoder.summary()

        inputs = krs.Input(shape=inDim)
        autoencoder = krs.Model(inputs,decoder(encoder(inputs)),name="%s_%s"%(name,'AE'))
        autoencoder.run_eagerly=True
        extract_custom_layers(parsedBody['custom'],autoencoder)

        modelSet = (encoder,decoder,autoencoder)
    elif netType == 'NN':
        neuralNet = link_layers(argSet[0],inDim)
        neuralNet._name = "%s_%s"%(name,'NN')
        modelSet = (neuralNet,)

    return modelSet

def extract_custom_layers(custom_model_components,layer):
    """
    Extract custom layers types from the model.

    This function searches for any custom layers in the model so that they can
    be passed as the custom_objects argument in `tf.keras.models.load_model`.

    Parameters
    ----------
    custom_model_components : dict
        This is dict where the handles to custom_objects will be stored.
    layer : tensorflow.keras.layers.Layer
        The current layers being checked to see if it is a custom object.

    Returns
    -------
    None.

    """
    try:
        for layer in layer.layers:
            if isinstance(layer,custom_objects):
                key = str(type(layer))
                key = key.split('\'')[1]
                key = key.rpartition('.')[2]
                custom_model_components[key] = layer
            extract_custom_layers(custom_model_components, layer)
    except:
        return

# def extract_shape(args):
#     shape = []
#     for line in args:
#         if line['type'] == 'FC':
#             shape.append(line['size'])
#         elif line['type'] == 'CONV':
#             shape.append(-1)
#     return shape

# def extract_activations(args):
#     acts = []
#     for line in args:
#         acts.append(line['act'])
#     return acts

def main():
    """
    Use for testing.

    Returns
    -------
    None.

    """
    krs.backend.clear_session()
    # filePath = "./"
    # fileName = "test.arch"
    # lines = read_lines(filePath,fileName)
    # parsedBody = parse_body(lines)

    # combArgs = retrieve_args(parsedBody = parsedBody, debug = True)
    # modelSet = generate_model(parsedBody, combArgs)

if __name__ == "__main__":
    main()


# Python 3
# Uses PCS exports to produce a traveling salesman solution to ordering the papers for discussion
# Created by Michael Bernstein in June 2019. MIT License.

import csv
from collections import defaultdict
import pprint
from numpy import median
import pandas as pd
import sys
import math
import tsp
import numpy as np

# Group papers into batches of this score range (e.g., .25 = 3.25-3.50) for discussion
# This defines the minimum and maximum score range to be discussed
SCORE_STEPS = [3.01, 3.25, 3.5, 3.75, 4.0, 4.25, 4.5, 4.75]

# Where the output should be written to.
OUTPUT_CSV = 'uist19a_travelingsalesman_order.csv'
# Bidding CSVs from PCS: this is the only CSV that contains conflict information in PCS, so we use it.
# This may be a list of CSVs if needed, since one of the ACs may be conflicted with a paper
# and that one won't show up in their CSV. This just merges all the CSVs in the list.
BIDDING_CSVS = ['uist19a_committee_bidding.csv']
# Submission CSV: we use this as an index of all the papers and their current overall scores
SUBMISSION_CSV = 'uist19a_submission.csv'

# get the set of papers that will be discussed
def load_eligible_papers(filename, min_score, max_score):
    with open(filename) as reviewfile:
        df = pd.read_csv(reviewfile)
        df['Overall Score'] = pd.to_numeric(df['Overall Score'])
        df = df[pd.notnull(df['Overall Score'])]
        df = df[df['Overall Score'] >= min_score]
        df = df[df['Overall Score'] < max_score]
        df = df.sort_values(by=['Overall Score'], ascending=False)
        papers = df['Paper ID'].tolist()
        print(papers)
        return papers

# load the conflicts from the PCS file
def load_conflicts(file_list, eligible_papers):
    conflicts = dict()
    ac_columns = dict()

    for filename in file_list: #will do some redundant work, but that's ok
        with open(filename) as conflictfile:
            reader = csv.reader(conflictfile)
            row_num = 0
            for row in reader:
                row_num += 1
                if row_num == 1 or row_num == 2 or row_num == 4 or row_num == 5: # blank or not useful  lines
                    continue
                elif row_num == 3:
                    for i in range(2, len(row)): # skip the first couple of columns because they just give the paper number and title
                        ac_columns[i] = row[i].split('\n')[0]
                else:
                    sub_id = row[0]
                    if sub_id in eligible_papers:
                        conflicts[sub_id] = list()
                        for i in range(2, len(row)): # skip the first couple of columns because they just give the paper number and title
                            if (i-2) % 3 == 0: # bid column
                                if row[i] == "C": # conflict
                                    conflicts[sub_id].append(ac_columns[i])
    return ac_columns, conflicts

# load the ACs from a PCS file
def load_ACs(filename, eligible_papers):
    acs = defaultdict(list)
    with open(filename) as reviewfile:
        reader = csv.DictReader(reviewfile)
        for row in reader:
            role = row['Role']
            submission = row['\ufeffSub ID']#row['\xef\xbb\xbfSub ID']
            if submission in eligible_papers:
                if role is not None and 'AC' in role:
                    acs[submission].append(row['Reviewer'])
    return acs

# Return pairs of papers that cannot be in the same group: an AC from one paper is conflicted the other paper
def create_costs(conflicts):
    costs = list()
    for paper1 in conflicts.keys():
        paper1_distances = list()
        for paper2 in conflicts.keys():
            paper1_conflicts = set(conflicts[paper1])
            paper2_conflicts = set(conflicts[paper2])
            difference = paper1_conflicts.symmetric_difference(paper2_conflicts)
            cost = 0
            if len(difference) > 0:
                cost = len(difference)#1

            paper1_distances.append(cost)
        costs.append(paper1_distances)
    return costs

# Create a graph coloring, to identify sets of nodes (papers) that are safe to discuss together
def construct_tsp(costs):
    print("Solving traveling salesman problem. Did you know this is NP-hard? COME ON, GIVE ME A SEC!")
    r = range(len(costs))
    dist = {(i, j): costs[i][j] for i in r for j in r}
    tsp_result = tsp.tsp(r, dist)
    return tsp_result[1]

# Split the papers into groups with additional metadata
def split_into_groups(tsp_result, conflicts):
    groups = list()
    step = 0
    papers_per_group = 1 # increase this if you want to batch papers out and send conflicts out for multiple papers at once (e.g., "next two papers, get out") to reduce cross traffic. We ultimately chose not to do this.
    while step < len(tsp_result):
        indices = tsp_result[step:step+1]
        subgroup = [list(conflicts.keys())[paper] for paper in indices]
        groups.append(create_subgroup(subgroup, conflicts))
        step += len(subgroup)

    return groups

# Returns a structured subgroup for a set of papers
def create_subgroup(subgroup, conflicts):
    group_conflicts = set()
    for subgroup_paper in subgroup:
        if subgroup_paper in conflicts:
            group_conflicts.update(conflicts[subgroup_paper])
    return {
        'papers': subgroup,
        'conflicts': group_conflicts
    }

def print_moves(title, groups, ac_columns):
    num_papers = 0
    is_in_room = defaultdict(lambda: True)
    total_movements = 0
    at_least_one_movements = 0
    for i, group in enumerate(groups):
        num_papers += len(group['papers'])

        # calculate symmetric difference
        paper1_conflicts = group['conflicts']
        if i > 0:
            paper2_conflicts = groups[i-1]['conflicts']
        else:
            paper2_conflicts = set()
        difference = paper1_conflicts.symmetric_difference(paper2_conflicts)

        print("%s: %d movements, now out of room: %s" % (group['papers'], len(difference), group['conflicts']))
        has_at_least_one_move = False
        for ac in set(ac_columns.values()):
            is_conflicted = ac in group['conflicts']

            if is_in_room[ac] and is_conflicted:
                total_movements += 1
                is_in_room[ac] = False
                has_at_least_one_move = True
            elif not is_in_room[ac] and not is_conflicted:
                total_movements += 1
                is_in_room[ac] = True
                has_at_least_one_move = True
        if has_at_least_one_move:
            at_least_one_movements += 1
    print("%s: total person movements in and out of the room: %d" % (title, total_movements))
    print("%s: total papers requiring at least one move in or out: %d" % (title, at_least_one_movements))
    print("%s: %d total papers" % (title, num_papers))

# how much time do ACs spend out of the room?
def print_AC_stats(ac_columns, all_conflicts, groups, all_eligible_papers):

    print_moves("If we go by score descending order", [create_subgroup([paper], all_conflicts) for paper in all_eligible_papers], ac_columns)
    print("\n\n\n-----\n\n\n")
    print_moves("If we go by TSP order", groups, ac_columns)

# output the CSV with the papers, the order to run them in, and the decision label to give them
def append_CSV(writer, groups, cur_min_score):
    for i, group in enumerate(groups):
        for paper in group['papers']:
            # in order for the descending sort to work in PCS, we need to actually apply a label that is the reverse of the intended traversal order. In other words, TS3.00-1 will be the last one in the sort, and TS3.00-9 (assuming nine entries) will be the first one. So, we reverse the order of the labels, which keeps us with the original order. Technically this isn't necessary because the traveling salesman solution is reversible, but this way the output from the program in the terminal matches the traversal order in PCS.
            d = {
                'paper': paper,
                'group': cur_min_score,
                'order': i+1,
                'label': 'TS%.2f-%d' % (cur_min_score, len(groups) - i),
                'conflicts': group['conflicts'] if len(group['conflicts']) > 0 else '{}'
            }
            writer.writerow(d)

if __name__ == "__main__":
    all_groups = list()
    all_conflicts = dict()
    all_ac_columns = dict()
    all_eligible_papers = set()

    with open(OUTPUT_CSV, 'w') as output_csv:
        writer = csv.DictWriter(output_csv, fieldnames=['paper', 'group', 'order', 'label', 'conflicts'])
        writer.writeheader()

        # We will iterate by score range (e.g., 3.5-3.25, then 3.25-3.0, then...)
        for index in range(len(SCORE_STEPS)-1):
            cur_min_score = SCORE_STEPS[index]
            cur_max_score = SCORE_STEPS[index+1]
            # Submissions CSV from PCS
            eligible_papers = load_eligible_papers(SUBMISSION_CSV, cur_min_score, cur_max_score)
            print("Range [%.2f, %.2f): %d papers" % (cur_min_score, cur_max_score, len(eligible_papers)))
            all_eligible_papers.update(eligible_papers)

            ac_columns, conflicts = load_conflicts(BIDDING_CSVS, eligible_papers)
            print("Conflicts loaded for %d ACs on %d papers." % (len(ac_columns.keys()), len(conflicts.keys())))
            all_conflicts.update(conflicts)

            # Now, create the distances of traversing from each paper to each other paper
            costs = create_costs(conflicts)
            print("Cost structure:")
            print(costs)

            # Solve the traveling salesman problem
            tsp_result = construct_tsp(costs)
            print('Recommended path: %s' % tsp_result)

            # Slim the coloring down into groups of reasonable size
            #inverted_coloring = descriptive_stats(coloring, conflicts)
            groups = split_into_groups(tsp_result, conflicts)
            all_groups.extend(groups)
            all_ac_columns = ac_columns

            append_CSV(writer, groups, cur_min_score)

    # Print some checking information about how much the ACs are out of the room
    print_AC_stats(all_ac_columns, all_conflicts, all_groups, all_eligible_papers)


#!/usr/bin/env python3
#  -*- coding: utf-8 -*-

import matplotlib.pyplot as plt
import os
import pandas as pd
import numpy as np
import random

def pareto_frontier(df, xname, yname):
    df = df.sort_values(yname)
    df_out = df.iloc[[0]]
    max_rec = df_out[xname]

    for i in range(1,len(df)):
        row = df.iloc[i,:]
        if (row[xname] > max_rec).any():
            df_out = df_out.append(row)
            max_rec = row[xname]

    return df_out

def plot_recalls(df, marker, color, algo):
    print(algo, ':')
    df = pareto_frontier(df, 'recall', 'query_time')
    df = df[df['recall'] <= 0.99]
    del df['var_recall']
    del df['k']
    if 'val_recall' in df.columns:
        del df['val_recall']
        del df['val_var_recall']
        del df['val_query_time']
    if 'r0' in df.columns:
        del df['r0']
    if 'r1' in df.columns:
        del df['r1']
    if 'r2' in df.columns:
        del df['r2']
    if 'r3' in df.columns:
        del df['r3']
    if 'r4' in df.columns:
        del df['r4']
    if 'r5' in df.columns:
        del df['r5']
    if 'r6' in df.columns:
        del df['r6']
    if 'r7' in df.columns:
        del df['r7']
    if 'r8' in df.columns:
        del df['r8']
    if 'r9' in df.columns:
        del df['r9']
    if 'r10' in df.columns:
        del df['r10']
    df.insert(0, 'recall', df.pop('recall'))
    df.insert(1, 'query_time', df.pop('query_time'))
    df.index = [''] * len(df)
    print(df)
    print()
    x = df['recall']
    y = df['query_time']
    plt.plot(x, y, linewidth=2.0, label=algo, linestyle='-', marker=marker, markersize=6, color=color)
    return x, y

def plot_scatter(ax, datapath, colors, data = 2, n_corpus = 1000, n_train = 1000, s_corpus = 10, s_train = 15, alpha = 0.8):
    corpus_file = os.path.join(datapath, 'corpus.csv')
    train_file = os.path.join(datapath, 'train.csv')
    df_corpus = pd.read_csv(corpus_file, header=None)
    df_train = pd.read_csv(train_file, header=None)
    ax.scatter(df_corpus.iloc[0:n_corpus,0], df_corpus.iloc[0:n_corpus,1], s=s_corpus, c=colors[-1], alpha=alpha)
    if data == 2:
        ax.scatter(df_train.iloc[0:n_train,0], df_train.iloc[0:n_train,1], s=s_train, alpha=alpha, c=colors[0])
    else:
        n_clusters = 5
        n = int(n_train / n_clusters)
        nx = int(len(df_train) / n_clusters)

        for i in [4, 0, 1, 2, 3]:
            idx = np.arange(i * nx, i * nx + n)
            ax.scatter(df_train.iloc[idx,0], df_train.iloc[idx,1], s=s_train, c=colors[i], alpha=alpha)

def precision(df, n_trees, depth, k=10, k_build=50):
    if 'k_build' in df.columns:
        df = df[df['k_build'] == k_build]
    df2 = df[df['n_trees'] == n_trees]
    df3 = df2[df2['depth'] == depth]
    df4 = df3[df3['k'] == k]
    df6 = df4[['v', 'k', 'recall', 'cs_size', 'query_time']]

#    df0 = pd.DataFrame({'v' : [0],
#                        'k' : [k],
#                        'recall' : [1],
#                        'cs_size' : [n]})
#    df6 = df0.append(df5)

    df6['k_found'] = df6['recall'] * df6['k']
    df6 = df6[df6['k_found'] > 0]
    df6['precision'] = df6['k_found'] / df6['cs_size']
    return df6

def sample(images, labels, n_sample):
    n = images.shape[0]
    idx = np.arange(n)
    idx_corpus = idx[labels == 0]
    idx_train = idx[labels == 1]
    pos_corpus = random.sample(set(idx_corpus), n_sample)
    pos_train = random.sample(set(idx_train), n_sample)

    out_images = np.concatenate((images[pos_corpus,:], images[pos_train,:]))
    out_labels = np.concatenate((np.zeros(n_sample), np.ones(n_sample)))
    return out_images, out_labels

def scatter_csv(fname_tsne, n_sample=1000, s_corpus=10, s_train=15, alpha=0.8,
                scatter_colors = ['#45a2a8', '#cf7e15']):
    df_tsne = pd.read_csv(fname_tsne, sep = '\t')
    images, labels = sample(df_tsne.iloc[:,0:2].values, df_tsne.iloc[:,2], n_sample)
    labels = np.flip(labels)
    images = np.flip(images, axis=0)
    col = np.where(labels == 1, scatter_colors[-1], scatter_colors[0])
    s = np.where(labels == 1, s_train, s_corpus)
    plt.scatter(images[:,0], images[:,1], c=col, s=s, alpha=alpha)

def generate_lookup_data(filepath, treetype):
    infile = os.path.join(filepath, treetype + '.txt')
    outfile = os.path.join(filepath, treetype + '-lookup.txt')
    df = pd.read_csv(infile, delim_whitespace=True)
    df[df['v'] == 1].to_csv(outfile, sep='\t')

def generate_knn_data(filepath, treetype):
    infile = os.path.join(filepath, 'rf-' + treetype + '.txt')
    outfile = os.path.join(filepath, treetype + '-knn.txt')
    df = pd.read_csv(infile, delim_whitespace=True)
    df[df['v'] == 1].to_csv(outfile, sep='\t')


import argparse
import json
import random
import os.path as osp

import torch
import torch.nn.functional as F

from utils import ensure_path, set_gpu, l2_loss
from models.gcn_dense_att import GCN_Dense_Att

def save_checkpoint(name):
    torch.save(gcn.state_dict(), osp.join(save_path, name + '.pth'))
    torch.save(pred_obj, osp.join(save_path, name + '.pred'))

def mask_l2_loss(a, b, mask):
    return l2_loss(a[mask], b[mask])

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    #创建对象
    parser.add_argument('--max-epoch', type=int, default=3000)#添加参数，这些调用指定 ArgumentParser 如何获取命令行字符串并将其转换为对象。这些信息在 parse_args() 调用时被存储和使用
    parser.add_argument('--trainval', default='10,0')#训练集和测试集占得比例
    parser.add_argument('--lr', type=float, default=0.001)
    parser.add_argument('--weight-decay', type=float, default=0.0005)
    #weight-decay权值衰减，在损失函数中，他是放在正则项前面的系数，为了调节模型复杂度对损失函数的影响
    parser.add_argument('--save-epoch', type=int, default=300)
    parser.add_argument('--save-path', default='save/gcn-dense-att')

    parser.add_argument('--gpu', default='0')

    parser.add_argument('--no-pred', action='store_true')#action命令行遇到参数时默认store
    args = parser.parse_args()
    #解析添加的参数，把每个参数转换为适当的类型然后调用相应的操作
    set_gpu(args.gpu)

    save_path = args.save_path
    ensure_path(save_path)

    graph = json.load(open('materials/imagenet-dense-grouped-graph.json', 'r'))#json是用来存储简单的数据结构和对象的文件
    wnids = graph['wnids']
    n = len(wnids)

    edges_set = graph['edges_set']
    print('edges_set', [len(l) for l in edges_set])

    lim = 4#每个阶段k值的数量设置为4
    for i in range(lim + 1, len(edges_set)):
        edges_set[lim].extend(edges_set[i])
    edges_set = edges_set[:lim + 1]
    print('edges_set', [len(l) for l in edges_set])#这里将14层减少到了4层+上自环，将后面的层全部加到第4层。后面计算权重的时候将他们算作一层

    word_vectors = torch.tensor(graph['vectors']).cuda()
    word_vectors = F.normalize(word_vectors)#对输入数据进行标准化使得输入数据满足正态分布，使得网络更容易收敛
    #fc为全连接层
    fcfile = json.load(open('materials/fc-weights.json', 'r'))
    train_wnids = [x[0] for x in fcfile]
    fc_vectors = [x[1] for x in fcfile]
    assert train_wnids == wnids[:len(train_wnids)]
    fc_vectors = torch.tensor(fc_vectors).cuda()#cnn训练的权重
    fc_vectors = F.normalize(fc_vectors)#F.normalize将某一个维度除以那个维度对应的范数(默认是2范数)。

    hidden_layers = 'd2048,d'
    gcn = GCN_Dense_Att(n, edges_set,
                        word_vectors.shape[1], fc_vectors.shape[1], hidden_layers).cuda()#训练dgp来预测最后一层的cnn权值，最终输出维数对应于resnrt-50架构的最后一层
    #对几个邻接矩阵进行处理

    print('word vectors:', word_vectors.shape)
    print('fc vectors:', fc_vectors.shape)
    print('hidden layers:', hidden_layers)

    optimizer = torch.optim.Adam(gcn.parameters(), lr=args.lr, weight_decay=args.weight_decay)

    v_train, v_val = map(float, args.trainval.split(','))
    n_trainval = len(fc_vectors)
    n_train = round(n_trainval * (v_train / (v_train + v_val)))#round四舍五入函数
    print('num train: {}, num val: {}'.format(n_train, n_trainval - n_train))
    tlist = list(range(len(fc_vectors)))
    random.shuffle(tlist)#将原列表打乱顺序,这里是将1000打乱顺序
    min_loss = 1e18

    trlog = {}
    trlog['train_loss'] = []
    trlog['val_loss'] = []
    trlog['min_loss'] = 0

    for epoch in range(1, args.max_epoch + 1):
        gcn.train()
        output_vectors = gcn(word_vectors)
        # z=output_vectors[tlist[0]]
        # o=fc_vectors[tlist[0]]
        loss = mask_l2_loss(output_vectors, fc_vectors, tlist[:n_train])#对输出进行l2正则化
        #训练DGP来训练每个节点的权值，训练类的权重从一个预先训练好的resnet网络优化提取，训练模型，通过损失来预测所见类的分类器权重
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        #上面的loss是为了进行反向传播进行参数的更新，而下面的train_loss是为了观察训练时损失的下降情况

        gcn.eval()
        output_vectors = gcn(word_vectors)#这里进行了测试是为了记录训练时候的每一步的损失么？？
        train_loss = mask_l2_loss(output_vectors, fc_vectors, tlist[:n_train]).item()
        if v_val > 0:
            val_loss = mask_l2_loss(output_vectors, fc_vectors, tlist[n_train:]).item()#这里是为了干什么的，与上面的train_loss不是一样的么
            loss = val_loss
        else:
            val_loss = 0
            loss = train_loss
        print('epoch {}, train_loss={:.4f}, val_loss={:.4f}'
              .format(epoch, train_loss, val_loss))

        trlog['train_loss'].append(train_loss)
        trlog['val_loss'].append(val_loss)
        trlog['min_loss'] = min_loss
        torch.save(trlog, osp.join(save_path, 'trlog'))

        if (epoch % args.save_epoch == 0):
            if args.no_pred:
                pred_obj = None
            else:
                pred_obj = {
                    'wnids': wnids,
                    'pred': output_vectors#个人理解：通过gcn传出来的一个分类器
                }

        if epoch % args.save_epoch == 0:
            save_checkpoint('epoch-{}'.format(epoch))

        pred_obj = None
    #一共运行3000次，也就是进行3000次参数优化，每次进行一次loss,
    #这里有一个疑问，就是我每次输出的output_vectors跟fc_vector进行损失计算都是取前1000个,那么后面的就不会参与更新
    #那我最后取的pred就必须也取前1000个参数才可以


import pytest

import clique.descriptor

class Mock(object):
    '''Mock class for test.'''

    x = clique.descriptor.Unsettable('x')

def test_unsettable():
    '''Unsettable descriptor prevent standard setting of attribute.'''
    instance = Mock()
    assert instance.x is None

    instance.__dict__['x'] = True
    assert instance.x is True

    with pytest.raises(AttributeError):
        instance.x = False


import random
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
from collections import deque

from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.initializers import RandomUniform

class NumberCube:
    def __init__(self, goal):
        self.goal = goal.flatten()
        self.board = self.goal.copy()
        self.row, self.col = np.shape(goal)
        self.state_size = self.row * self.col

    def left(self, idx):
        if idx % self.col != 0:
            self.board[idx], self.board[idx-1] = self.board[idx-1], self.board[idx]

    def up(self, idx):
        if idx >= self.row:
            self.board[idx], self.board[idx-3] = self.board[idx-3], self.board[idx]

    def right(self, idx):
        if idx % self.col != (self.row - 1):
            self.board[idx], self.board[idx+1] = self.board[idx+1], self.board[idx]

    def down(self, idx):
        if idx <= (self.state_size - self.row - 1) :
            self.board[idx], self.board[idx+self.row] = self.board[idx+self.row], self.board[idx]

    def take_action(self,action):
        idx = np.where(self.board == 0)[0][0]
        if action == 0: self.left(idx)
        elif action == 1: self.up(idx)
        elif action == 2: self.right(idx)
        else: self.down(idx)

    def step(self,action):
        self.take_action(action)
        reward = (np.sum(self.board == self.goal.flatten()) - 8) / 9
        done = np.array_equal(self.board, self.goal.flatten())
        return self.board, reward, done

    def reset(self):
        self.board = self.goal.copy()
        [self.take_action(i) for i in [np.random.randint(4) for i in range(np.random.randint(10)+1)]]
        return self.board

# 상태가 입력, 큐함수가 출력인 인공신경망 생성
def build_net(state_size, action_size):
    model = tf.keras.models.Sequential()
    model.add(tf.keras.layers.Dense(590, input_dim=state_size, activation='relu',
                        kernel_initializer='he_uniform'))
    model.add(tf.keras.layers.Dense(590, input_dim=state_size, activation='relu',
                        kernel_initializer='he_uniform'))
    model.add(tf.keras.layers.Dense(590, input_dim=state_size, activation='relu',
                        kernel_initializer='he_uniform'))
    model.add(tf.keras.layers.Dense(590, input_dim=state_size, activation='relu',
                        kernel_initializer='he_uniform'))
    model.add(tf.keras.layers.Dense(action_size))
    model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(lr=0.001), metrics=['acc'])
    print(model.summary())
    return model

def Replay_Bellman(main_model, target_model, batch, discnt_rate):

    states = np.vstack([x[0] for x in batch])
    actions = np.array([x[1] for x in batch])
    rewards = np.array([x[2] for x in batch])
    states2 = np.vstack([x[3] for x in batch])
    done = np.array([x[4] for x in batch])

    # 벨만 최적 방정식을 이용한 예측 Q 값
    Q = rewards + discnt_rate * np.max(target_model.predict(states2), axis=1) * ~done

    target = main_model.predict(states)
    target[np.arange(len(states)), actions] = Q

    return states, target

goal = np.array([[1, 2, 3],[4, 0, 6],[7, 8, 9]])
env = NumberCube(goal)
state_size = env.state_size
action_size = 4

Episodes = 10000
discnt_rate = 0.99
batch_size = 64
Re_m = 20000
Target_update_frequency = 5
counters = []
loss,accurancy = [],[]
e_min = 0.01

# 리플레이 메모리, 최대 크기 Re_m = 20000
Re_buffer = deque(maxlen=Re_m)

G = tf.Graph()
with G.as_default():
    # 메인 모델과 타깃 모델 생성
    main_model = build_net(state_size, action_size)
    target_model = build_net(state_size, action_size)

    # 타깃 모델 초기화
    target_model.set_weights(main_model.get_weights())

    for episode in range(Episodes):
        e = 1. / ((episode / 10) + 1)  # 입실론
        if e < e_min:
            e = e_min
        done = False

        count = 0
        # env 초기화
        state = env.reset()
        state = np.reshape(state,[1,state_size]) / 9

        while not done:
            # env.render()  # 행동 모니터링
            count += 1

            # Q-net에서 입실론 탐욕 정책으로 현재 상태에 따른 행동 선택

            if np.random.rand() <= e:
                action = random.randrange(action_size)
            else:
                action = np.argmax(main_model.predict(state))

            # 선택한 행동으로 환경에서 한 타임스텝 진행
            state2, reward, done = env.step(action)
            state2 = np.reshape(state2,[1,state_size]) / 9

            # 리플레이 메모리에 경험 샘플 [s, a, r, s', d] 저장
            Re_buffer.append((state, action, reward, state2, done))
            # 리플레이 메모리가 배치 크기만큼 쌓이면, 무작위로 추출한 배치로 학습
            if len(Re_buffer) > batch_size:
                batch = random.sample(Re_buffer, batch_size)
                states, target = Replay_Bellman(main_model, target_model, batch, discnt_rate)
                hist = main_model.fit(states, target, batch_size=batch_size, epochs=1, verbose=0)

            # 일정 주기마다 타겟 모델 초기화
            if episode % Target_update_frequency == 0:
                target_model.set_weights(main_model.get_weights())

            if count == 15: break

            state = state2
        loss.append(hist.history['loss'][0])
        accurancy.append(hist.history['acc'][0])
        counters.append(count)
        #print("Episode: {:>4}, count: {:>4}, Re_len: {:>4}, e: {:>4.2f}".format(episode, count, len(Re_buffer), e))
        print('episode:',episode,'memory:',len(Re_buffer),'count:',count)

fig, loss_ax = plt.subplots()

acc_ax = loss_ax.twinx()

loss_ax.plot(loss, 'y', label='train loss')

acc_ax.plot(accurancy, 'b', label='train acc')

loss_ax.set_xlabel('epoch')
loss_ax.set_ylabel('loss')
acc_ax.set_ylabel('accuray')

loss_ax.legend(loc='upper left')
acc_ax.legend(loc='lower left')

plt.show()
#plt.plot(counters)


import urllib
from bs4 import BeautifulSoup
import csv

def find_sales(url, filename):
    print("Scraping... {}".format(url))
    header = {
        'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36"}

    soup = BeautifulSoup(urllib.request.urlopen(
        urllib.request.Request(url=url, headers=header)
    ), "html.parser")

    print("Find sales data")
    table = soup.find_all("table")[0]
    rows = table.find_all("tr")

    print("Write CSV file as {}".format(filename))
    with open(filename, "w", encoding="utf-8") as file:
        writer = csv.writer(file)
        for row in rows:
            csv_row = []
            for cell in row.findAll(['td', 'th']):
                text = cell.get_text()
                text = text.replace("／", " / ").replace("~", "〜")
                csv_row.append(text)
            print(csv_row)
            writer.writerow(csv_row)

if __name__ == "__main__":
    find_sales("https://w.atwiki.jp/orideta/pages/70.html", "../data/Single sales.csv")
    find_sales("https://w.atwiki.jp/orideta/pages/182.html", "../data/Album sales.csv")


# coding=utf-8
__author__ = 'chenlei'

from django import template
from django.utils.translation import ugettext as _

register = template.Library()

@register.filter
def bool_map(value):
    if value not in [True, False]:
        raise Exception("Bool Map Value should be a bool value")
    if value:
        return _(u"是")
    return _(u"否")


from __future__ import print_function
import boto3
import json
import os
from time import sleep
from pprint import pprint

RISKY_PORTS = [20, 21, 22, 1433, 1434, 3306, 3389, 4333, 5432, 5500]

template = """
Security Group Alert !
Alert Level : {0}
Account: {1}
Region: {2}
Group Id: {3}
Detail: {4}

"""

def aws_get_regions(aws_key, aws_secret):
    aws_client = boto3.client(
        "ec2",
        aws_access_key_id=aws_key,
        aws_secret_access_key=aws_secret,
        region_name='eu-west-1'
    )
    response = aws_client.describe_regions()
    result = []
    for region in response['Regions']:
        result.append(region['RegionName'])
    return result

def aws_get_security_groups(aws_key, aws_secret, test_sg=""):
    # TODO: Requires Unit Testing
    global message
    for region in aws_get_regions(aws_key, aws_secret):
        aws_client = boto3.client(
            "ec2",
            aws_access_key_id=aws_key,
            aws_secret_access_key=aws_secret,
            region_name=region
        )
        message = ""

        response = aws_client.describe_security_groups()
        account = response['SecurityGroups'][0]['OwnerId']
        for security_group in response['SecurityGroups']:
            # message = ""
            for ip_permissions in security_group['IpPermissions']:
                pprint(security_group)
                affected_ports = []
                if ip_permissions['IpProtocol'] == '-1' and '0.0.0.0/0' in str(ip_permissions['IpRanges']) and \
                        (test_sg == '' or security_group['GroupId'] == test_sg):
                    message += template.format('Critical', account, region, security_group['GroupId'],
                                               'All ports open to the world !')
                if ip_permissions['IpProtocol'] == 'tcp' and '0.0.0.0/0' in str(ip_permissions['IpRanges']) and \
                        (test_sg == '' or security_group['GroupId'] == test_sg):
                    from_port = ip_permissions['FromPort']
                    to_port = ip_permissions['ToPort']
                    for port in RISKY_PORTS:
                        if from_port <= port <= to_port:
                            affected_ports.append(port)
                    if len(affected_ports) > 0:
                        message += template.format('Important', account, region, security_group['GroupId'],
                                                   'TCP port(s) ' + ",".join(
                                                       str(x) for x in affected_ports) + ' open to the world !')
                    else:
                        message += template.format('Warning', account, region, security_group['GroupId'],
                                                   'TCP port' + (
                                                       ' ' + str(to_port) if to_port == from_port else 's from ' + str(
                                                           from_port) + ' to ' + str(to_port)) + ' open to the world !')
                if message != "":
                    print(message)
                    aws_send_sns_message(message, os.environ.get('SNS_ARN'), aws_key, aws_secret)

                if security_group['GroupId'] == test_sg:
                    return message

        sleep(1)  # Not to overload AWS API
    return message

def aws_send_sns_message(message, topic_arn, aws_key, aws_secret):
    # type: (string, string, string, string) -> string
    # TODO: Requires Unit Testing
    topic_keys = topic_arn.split(':', 5)
    region = topic_keys[3]
    aws_client = boto3.client(
        "sns",
        aws_access_key_id=aws_key,
        aws_secret_access_key=aws_secret,
        region_name=region
    )
    response = aws_client.publish(
        TopicArn=os.environ.get('SNS_ARN'),
        Message=message
    )
    return response


import torchvision.transforms as transforms
import matplotlib.pyplot as plt
from PIL import Image
import torch

def get_loader(imsize):
    loader = transforms.Compose([
        transforms.Resize(imsize),
        transforms.CenterCrop(imsize),
        transforms.ToTensor()])
    return loader

unloader = transforms.ToPILImage()

def image_loader(image_name, loader, device):
    image = Image.open(image_name)
    image = loader(image).unsqueeze(0)
    return image.to(device, torch.float)

def imshow(tensor, title=None):
    """

    :param tensor:
    :param title:
    :return:
    """
    image = tensor.cpu().clone()
    image = image.squeeze(0)
    image = unloader(image)
    plt.imshow(image)
    if title is not None:
        plt.title(title)
    plt.pause(0.001)


import copy
from datetime import datetime

from .db import db
from .helpers import pretty_date

class Toggle(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(128), unique=True)
    description = db.Column(db.String(256))
    enabled = db.Column(db.Boolean, nullable=False, default=False)
    last_changed_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)

    def __repr__(self) -> str:
        return f"<Toggle: {self.name},{self.enabled}>"

    def __eq__(self, other):
        if not isinstance(other, Toggle):
            return NotImplemented

        return self.name == other.name and self.enabled == other.enabled

    def serialize(self):
        return {self.name: self.enabled}

    @staticmethod
    def insert_all():
        for toggle in toggles:
            if Toggle.query.get(toggle.id) is None:
                print(f"Creating toggle: {toggle.name}")
                db.session.add(toggle)

        db.session.commit()

    def last_changed(self) -> str:
        return pretty_date(self.last_changed_at)

    def toggle(self):
        self.enabled = not self.enabled
        self.last_changed_at = datetime.utcnow()

toggles = [
    Toggle(
        id=1,
        name="game.tutorial.enable",
        description="Aktiviert das Tutorial",
    ),
    Toggle(
        id=2,
        name="game.announcements.enable",
        description="Aktiviert 'Ansagen'",
        enabled=True,
    ),
    Toggle(
        id=3,
        name="game.multiplayer.enable",
        description="Aktiviert den Multiplayer-Modus",
    ),
]


# !/usr/bin/env python
# Name:     standardValues.py
# By:       LA-Shill
# Date:     30.01.2022
# Version   0.3.1
# -----------------------------------------------

import pymongo
from os import environ, path, getcwd
from dotenv import load_dotenv
from datetime import timedelta

"""Load in .env file"""
basedir = path.abspath(path.dirname(__file__))
load_dotenv(path.join(basedir, '.env'))

host = str(environ.get('DB_HOST'))
CORE_MONGO_DB = str(environ.get('CORE_MONGO_DB'))
VUL_MONGO_DB = str(environ.get('VUL_MONGO_DB'))
port = int(environ.get('DB_PORT'))

client = pymongo.MongoClient(host, port)
mainDB = client[CORE_MONGO_DB.rsplit('/', 1)[-1]]

try:
    if mainDB['settings'].count_documents({}) > 0:
        print("[INFORMANT] Settings DB file detected")
    else:
        settings = {'CENSYS_API_ID' : '', 'CENSYS_API_SECRET' : '', 'SHODAN_API_KEY' : '', 'BINARY_EDGE_API_KEY' : '', 'ONYPHE_API_KEY' : '', 'FARSIGHT_API_KEY' : '', 'HIGH_RISK_PORTS' : [], 'GEO_LOCATION' : ''}
        mainDB['settings'].insert(settings)
except Exception as e:
    print("Error on first run, check you have all DB dependencies installed. Error: " + str(e))

class StandardValues:
    """
    Default vaules used across program,
    can be stored in volatile memory if required
    """
    db_r = mainDB['settings'].find({})

    CENSYS_API_ID: str = db_r[0]['CENSYS_API_ID']
    CENSYS_API_SECRET: str =  db_r[0]['CENSYS_API_SECRET']
    SHODAN_API_KEY: str =  db_r[0]['SHODAN_API_KEY']
    BINARY_EDGE_API_KEY: str = db_r[0]['BINARY_EDGE_API_KEY']
    ONYPHE_API_KEY: str =  db_r[0]['ONYPHE_API_KEY']
    FARSIGHT_API_KEY: str = db_r[0]['FARSIGHT_API_KEY']
    HIGH_RISK_PORTS: list = db_r[0]['HIGH_RISK_PORTS']
    GEO_LOCATION: str = db_r[0]['GEO_LOCATION']

    CENSYS_DEFAULT_RESULTS_QUANTITY: int = 10000
    SHODAN_DEFAULT_RESULTS_QUANTITY: int = 10
    BINARYEDGE_DEFAULT_RESULTS_PAGE: int = 1

    THREATMINER_API_DELAY: int = 6
    ROBTEX_API_DELAY: int = 20
    THREATCROWD_API_DELAY: int = 10
    DALOO_API_DELAY: int = 5
    FARSIGHT_API_DELAY: int = 1

    DB_HOST: str = host
    DB_PORT: int = port
    MAIN_DB_NAME: str = CORE_MONGO_DB.rsplit('/', 1)[-1]
    VUL_DB_NAME: str = VUL_MONGO_DB.rsplit('/', 1)[-1]


import numpy
import PIL
import math
import time
import picamera
import numpy as np
import cv2
import matplotlib as plt
import io

stream=io.BytesIO()
with picamera.PiCamera() as camera:
	camera.resolution=(320,240)
	camera.framerate=24
	time.sleep(1)
	camera.capture(stream,format='jpeg')
data=np.fromstring(stream.getvalue(),dtype=np.uint8)
image=cv2.imdecode(data,1)

# with picamera.PiCamera() as camera:
#     camera.resolution=(320,240)
#     time.sleep(3)
#     camera.capture('assets/pi_photo')
#     image = cv2.imread('assets/pi_photo')

window_name = '“lab 5.2'

cv2.imshow(window_name, image)
raw_key = cv2.waitKey(1000)
rgb=cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
height,width,channels=image.shape #Add this line here

for x in range (0,height):
	for y in range (0,width):
		r,g,b=rgb[x,y]
		if r>g and r>b and r>128:
			image[x,y]=0,0,255

window_name2 = 'lab 5.2 red filter'
cv2.imshow(window_name2, image)
raw_key = cv2.waitKey(20000)


import numpy as np
from scipy.signal import convolve2d

def markers_evaluator(game, is_max, *, markers=8):
    """ Evaluation based on the number of markers already placed """
    me = game.current_player if is_max else game.previous_player
    opponent = game.previous_player if is_max else game.current_player

    my_placed = np.count_nonzero(game.board == me)
    opponent_placed = np.count_nonzero(game.board == opponent)

    def f(p, m):
        return (2 ^ 63 - 1) if m - p == 0 else -m / (m - p)

    return f(my_placed, markers) - f(opponent_placed, markers)

def combination_evaluator(game, is_max, weight=10.0):
    """ Evaluation of consecutive markers of the same player """
    me = game.current_player if is_max else game.previous_player
    opponent = game.previous_player if is_max else game.current_player

    score = 0
    kernels = [np.ones(shape=(1, 2)), np.ones(shape=(2, 1)), np.eye(2), np.fliplr(np.eye(2))]
    for kernel in kernels:
        score += np.count_nonzero(convolve2d(game.board == me, kernel, mode='valid') == 2) * weight
        score -= np.count_nonzero(convolve2d(game.board == opponent, kernel, mode='valid') == 2) * weight

    return score

def mix_evaluator(game, is_max, *, markers=8, weight=10.0):
    """ Better evaluation function that combines the markers already placed and their combinations """
    markers_score = markers_evaluator(game, is_max, markers=markers)
    combination_score = combination_evaluator(game, is_max, weight=weight)

    return markers_score + combination_score


import torch
import torch.nn as nn
import pandas as pd
import numpy as np
from .useless_ids import useless_ids
from collections import Iterable

class FBaselineConfig:
    def __init__(self,
            anova_info_path="../data/anova_info.csv",
            f_cut = 150,
            out_dim = 3,
            dropout = 0.1
        ):
        self.anova_info_path = anova_info_path
        self.f_cut = f_cut
        self.out_dim = out_dim
        self.dropout = dropout

    def initialize(self):
        self.x_ids = self.cal_x_ids()
        self.in_dim = len(self.x_ids)
        self.n_hidden = min(self.in_dim // 6,512)

    def cal_x_ids(self):
        anova_info = pd.read_csv(self.anova_info_path)
        anova_info = anova_info[anova_info["f"] > self.f_cut]
        dims = []
        fs = []
        for i in range(len(anova_info)):
            f = "%.7f"%(anova_info["f"][i])
            if f not in fs and anova_info["dim"][i] not in useless_ids:
                fs.append(f)
                dims.append(anova_info["dim"][i])
        return np.array(dims)

class FullResLayer(nn.Module):
    def __init__(self,in_dim,dropout):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(in_dim,in_dim),
            nn.ReLU(),
            nn.Dropout(dropout),
            nn.Linear(in_dim,in_dim)
        )
        self.relu = nn.ReLU()

    def forward(self,x):
        return self.relu(self.net(x) + x)

class FBaseline(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.x_ids = config.x_ids

        self.net = nn.Sequential(
            nn.Dropout(0.65),
            nn.Linear(self.config.in_dim, self.config.n_hidden),
            nn.ReLU(),
            nn.Dropout(self.config.dropout),
            FullResLayer(self.config.n_hidden,self.config.dropout),
            nn.Dropout(self.config.dropout),
            FullResLayer(self.config.n_hidden,self.config.dropout),
            nn.Dropout(self.config.dropout),
            FullResLayer(self.config.n_hidden,self.config.dropout),
            nn.Dropout(self.config.dropout),
            nn.Linear(self.config.n_hidden, self.config.out_dim),
            nn.Softmax(dim=-1)
        )

    def _preprocess(self,x):
        if len(x.shape) == 1:
            return x[self.x_ids]
        if len(x.shape) == 2:
            return x[:,self.x_ids]

    def forward(self, x):
        """
        args:
            x : input of feaure
        return:
            x : possibility of three class
        Shape:
            input : [batch_size, in_dim]
            output : [batch_size,out_dim]
        """
        x = self._preprocess(x)
        x = self.net(x)
        return x

def test():
    config = FBaselineConfig()
    config.initialize()
    print(config.__dict__)
    model = FBaseline(config)
    x = torch.randn((32,28169))
    y = model(x)
    print(y)

if __name__ == "__main__":
    test()


import json
from pathlib import Path

from dotenv import dotenv_values

from aem_utils.aem_constants import DIR_CONFIG_OSGICONFIG_CONFIG, OSGI_CONFIG_EXTENSION
from aem_utils.aem_constants import JCR
from constants import DOT, LESS, UTF_8, XML_EXTENSION
from osgi_xml_to_json_converter import OsgiXmlToJsonConverter
from utils.py_utils_file import read_files_in_folder_filter_by_extension, read_file_to_list_of_string
from utils.py_utils_string import substring_before_last

def main():
    config = dotenv_values(".env")
    aem_source_code_base_path = config["aem_source_code_base_path"]

    path_to_osgiconfigs = Path(aem_source_code_base_path + DIR_CONFIG_OSGICONFIG_CONFIG)
    print(f"\nRead XML files from: {path_to_osgiconfigs}\n")

    xml_files_in_osgiconfig_folder = []
    read_files_in_folder_filter_by_extension(xml_files_in_osgiconfig_folder, path_to_osgiconfigs, XML_EXTENSION)
    for xml_file_name in xml_files_in_osgiconfig_folder:
        print(f"Converting file: {xml_file_name}")

        # create new object to store the key/values
        converter = OsgiXmlToJsonConverter()

        # processing file content
        file_content = read_file_to_list_of_string(xml_file_name)
        for line in file_content:
            if not line.lstrip().rstrip().startswith((LESS, JCR)):
                converter.add_line(line.lstrip().rstrip())

        # write json file
        json_file_name = substring_before_last(xml_file_name, DOT) + DOT + OSGI_CONFIG_EXTENSION
        print(f"Writing new file: {json_file_name}")
        with open(json_file_name, 'w', encoding=UTF_8) as f:
            json.dump(converter.get_json_object(), f, ensure_ascii=False, indent=4)

    exit(0)

if __name__ == "__main__":
    main()


import numpy as np
import cv2

"""
Variable setup-
1 - Canny thresholds
2 - Mask polygon co-ords
3 - Hough transformation vars
"""
# 1 - Lower/Upper Canny thresholds
# Depreciated - Variables have to manually be set in the canny function otherwise the code breaks.

#2 - Points 1-4 co-ordinates for mask
x1 = 277 * 2.02
y1 = 351 * 2.12
x2 = 0
y2 = 535 * 2.12
x3 = 680 * 2.02
y3 = 535 * 2.12
x4 = 454 * 2.02
y4 = 306 * 2.12

#3 - Hough transformation variables
rho = 2
theta = 2 * np.pi/180
threshold = 15
minLength = 15
maxGap = 15

# Importing the test video
cap = cv2.VideoCapture("test_videos/Example002.mp4")
if not cap.isOpened():
    raise BrokenPipeError("Video not initializing")

# Defining codecs and videoWriter for creating output video
fourcc = cv2.VideoWriter_fourcc(*"DIVX")
output = cv2.VideoWriter("test_videos_output/2_final.avi", fourcc, 20.0, (int(cap.get(3)), int(cap.get(4))))

while True:
    ret, frame = cap.read()

    if not ret: # If frame is not read, video has ended, exit
        break

    # Image converted to grayscale for processing
    img = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Canny edge detection run on image
    canny = cv2.Canny(img, 173, 106)

    # Creating polygon mask
    mask = np.zeros_like(canny)
    imshape = img.shape
    vertices = np.array([[(x1, y1), (x2, y2), (x3, y3), (x4, y4)]], dtype=np.int32)
    cv2.fillPoly(mask, vertices, 255)

    # Applying mask and hough transform
    masked_edges = cv2.bitwise_and(canny, mask)
    linesP = cv2.HoughLinesP(masked_edges, rho, theta, threshold, np.array([]), minLength, maxGap)

    # Drawing hough lines onto original frame
    if linesP is not None:
        for i in range(0, len(linesP)):
            l = linesP[i][0]
            cv2.line(frame, (l[0], l[1]), (l[2], l[3]), (0, 0, 255), 2, cv2.LINE_AA)

    # Writing the modified frame to the output video
    output.write(frame)

    cv2.imshow("frame", frame)
    if cv2.waitKey(19) == ord('q'):
        break

cap.release()
output.release()
cv2.destroyAllWindows()


import sqlite3

#connect to SQLite, connects to datebase file or creates it.
connection = sqlite3.connect('emaildb.sqlite')
#Send and receive SQL commands through cursor
cursor= connection.cursor()
#Sends command removes the table only if the table exists (to start fresh)
#otherwise, it just ignores the statement and does nothing.
cursor.execute('DROP TABLE IF EXISTS Counts');
#Sends command to create table, pass fields and type in parathenses
cursor.execute('CREATE TABLE Counts (email TEXT, count INTEGER)')
#Prompts user to input a file name
filename = input('Enter file name:')
#Error handling
if (len(filename) < 1): filename = 'nbox-short.txt'
#File handler, allows us to read the content of the File
filehandler = open(filename)
#Loop across every line in the file
for line in filehandler:
    #if line doesnt start by From: skip line and continue Loop
    if not line.startswith('From: '): continue
    #Split the line into a list of strings to retrieve email
    list = line.split()
    email = list[1]
    # This line is not retrieving data. It's 'connecting' to the database row
    # It is dangerous to put strings into SQL.
    # the ? is a placeholder, it's meant to not allow SQL injections
    cursor.execute('SELECT count FROM Counts WHERE email = ?', (email,))
    #row is the information we get from the database. If no records, row = None
    row = cursor.fetchone()
    if row is None:
        #if email isnt in the database, insert email and add 1 to count.
        cursor.execute('INSERT INTO Counts (email, count) VALUES (?, 1)', (email,))
    else:
        #if email is in the database, add +1 to count.
        cursor.execute('UPDATE Counts SET count = count + 1 WHERE email = ?', (email,))
    #commit is the command to "commit" the changes. Forces to disk, can be slow, avoid committing in loops.
    connection.commit()

#command to select the emails and counts order them by descing order, only retrieve 10.
sqlstr = 'SELECT email, count FROM Counts ORDER BY count DESC LIMIT 10'

#execute command and loop across all the rows
for row in cursor.execute(sqlstra):
    #str() fonction converts the specified value into a string
    print(str(row[0]), row[1])

cursor.close()


# -*- coding: utf-8 -*-
"""
Created on Wed Mar 21 16:28:46 2018

@author: Daniel Becerra Pedraza

Exercise 3 of first homework from Scientific Computing with High Added Value
----------------------------------
Solve the following equation:

$
\dfrac{d \phi}{dt} + \dfrac{d}{dx} \left( \rho u \phi) = \dfrac{d}{dx} \left( \Gamma \dfrac{d T}{dx} \right)
$

with boundary conditions:

$
\phi(x,0) = 0\ para\ 0 \leq x \leq \infty
\phi(0,t) = 1\ para\ t > 0
\phi(L,t) = 0\ para\ t > 0,\ L \rightarrow \infty
$

And the following values:

$
u = 1,
\rho = 1.0,
\Delta t = 0.002,
t_max = 1.0,
\Gamma = 0.001,
N = 50 y N = 350,
L = 2.5
$
___________________________________________________________

 |<-------------- 2.5 m ------------->|
           --- u = 1 m/S --->
 A                                    B
 |====================================|
\phi_A = 1                          \phi_B = 0
___________________________________________________________
Figure 5.2

Analytical solution:

\phi(x,t) = 0.5 \left[erfc\left(\frac{x-ut}{2 \sqrt{\Gamma t}}\right) + exp\left(\frac{ux}{D}\right)erfc\left(\frac{x+ut}{2 \sqrt{\Gamma t}}\right)\right]
"""

import FiniteVolumeMethod as fvm
import numpy as np
import scipy as sp
import viscoflow as vfl

def analyticSol(x, u, t, Gamma):
 	sol = 0.5 * (sp.special.erfc((x - u * t)/(2 * np.sqrt(Gamma * t))) + np.exp(u * x) * np.exp(-Gamma) * sp.special.erfc((x + u * t)/(2 * np.sqrt(Gamma * t))))
 	return sol

longitud = 2.5 # metros
PhiA = 1
PhiB = 0.0001
gamma  = 0.001 # Gamma
rho = 1.0 #Kg/m^3
u = 1.0 #m/s
Ti = 0.0
Tf = 1.0
dt = 0.002
N  = 350 # Número de nodos
Esquema = "QUICK" #CD, UpW, UpW2 y QUICK, si no se especifica se usará diferencias centradas.
#
# Creamos la malla y obtenemos datos importantes
#
malla = fvm.Mesh(nodes = N, length = longitud)
nx    = malla.nodes()     # Número de nodos
nvx   = malla.volumes()   # Número de volúmenes
dx = malla.delta()     # Tamaño de los volúmenes
x = malla.createMesh()
#
# Imprimimos los datos del problema (nicely)
#
fvm.printData(Longitud = longitud,
              Propiedad_A = PhiA,
              Propiedad_B = PhiB,
              Conductividad = gamma,
              Nodos = nx,
              Volúmenes = nvx,
              Delta = dx)

#
# Se construye el arreglo donde se guardará la solución
#
Phi = np.zeros(nvx) # El arreglo contiene ceros
Phi[0]  = PhiA        # Condición de frontera izquierda
Phi[-1] = PhiB        # Condición de frontera derecha

#
#  Creamos los coeficientes de FVM
#
coef = fvm.Coefficients()
coef.alloc(nvx)

df1 = fvm.Diffusion1D(nvx, Gamma = gamma, dx = dx)

adv1 = fvm.Advection1D(nvx, rho = rho, dx = dx)
adv1.setU(u)

D = gamma/dx

tem = fvm.Temporal1D(nvx, rho = rho, dx = dx, dt = dt)

# Calculamos la solución analítica
#
Phiaf = analyticSol(x, u, Tf-Ti, gamma)
vfl.grafica(x,Phiaf,kind='-',label='solución analítica')

for i in range(1,int((Tf-Ti)/dt)+1):
#
# Iniciamos con coeficientes cero
#
    print('Time step = {}'.format(i * dt), sep = '\t')
    coef.cleanCoefficients()
#
#  Calculamos los coeficientes de FVM de la Difusión
#
    df1.calcCoef()
#
#  Calculamos los coeficientes de FVM de la Advección
#
    adv1.setU(u)
    if Esquema == "UpW":
        adv1.calcCoef('UpW')
    elif Esquema == "UpW2":
        adv1.calcCoef('UpW2', PhiA, PhiB)
    elif Esquema == "QUICK":
        adv1.calcCoef('QUICK', PhiA, PhiB, D)
    else:
        adv1.calcCoef()
#
#  Calculamos los coeficientes de FVM de la parte temporal
#
    tem.calcCoef(Phi)

#
# Se aplican las condiciones de frontera dependiendo del esquema a usar
#
    if Esquema != "UpW2" and Esquema != "QUICK":
        coef.bcDirichlet('LEFT_WALL', PhiA)   # Se actualizan los coeficientes
        coef.bcDirichlet('RIGHT_WALL', PhiB) # de acuerdo a las cond. de frontera

#
# Se construye el sistema lineal de ecuaciones a partir de los coef. de FVM
#
    Su = coef.Su()  # Vector del lado derecho
    A = fvm.Matrix(malla.volumes())  # Matriz del sistema
    A.build(coef) # Construcción de la matriz en la memoria
#
# Se resuelve el sistema usando un algoritmo del módulo linalg
#
    Phi[1:-1] = np.linalg.solve(A.mat(),Su[1:-1])

#
# Usamos Viscoflow para graficar
# VISCOFLOW : Visualization of Complex Flows
#
    if (i % 100 == 0):
        etiqueta = 'Step = {}'.format(i*dt)
        vfl.grafica(x,Phi,title='Solución de $\partial phi /\partial t + \partial ( rho*u*phi)/\partial x = \partial ( Gamma (\partial T /\partial x ))/\partial x$ con FVM', label=etiqueta)

vfl.show('Problema6-ADT_' + Esquema + '_' + str(N) + 'nodos.png')


#!/usr/bin/python
# -*- coding: utf-8 -*-
import io

class FilteredTextFile:

    def __init__(self, file_name, encoding = 'utf-8'):
        self._file_name = file_name
        self._encoding = encoding
        self._contents = []

    @property
    def contents(self):
        return self._contents

    @property
    def file_name(self):
        return self._file_name

    @file_name.setter
    def file_name(self, file_name):
        self._file_name = file_name

    @property
    def encoding(self):
        return self._encoding

    @encoding.setter
    def set_encoding(self, encoding):
        self._encoding = encoding

    def filter_line(self, line):
        return line

    def append(self, line):
        self._contents.append(line)

    def start_load(self):
        pass

    def load(self, file_name = None):

        if file_name == None:
            file_name = self.file_name

        self.start_load()
        self._contents = []
        with io.open(file_name, mode ='r', encoding = self.encoding) as inp:
            for line in inp:
                line = self.filter_line(line)
                if line != None:
                    self.append(line)

    def save(self, file_name = None):
        if file_name == None:
            file_name = self.file_name

        with io.open(file_name, mode = 'w', encoding = self.encoding) as outp:
            for line in self.contents:
                outp.write(line)

class FilteredJavaFile(FilteredTextFile):
    ST_START = 0
    ST_INITIAL_COMMENT = 1
    ST_INSIDE_CODE = 2

    def __init__(self, file_name, encoding = 'utf-8'):
        super().__init__(file_name, encoding)
        self._state = FilteredJavaFile.ST_START

    def _is_package_declaration(self, line):
        return line.lstrip().startswith('package')

    def start_load(self):
         self._state = FilteredJavaFile.ST_START

    def is_multi_line_comment_start(self, line):
        line = line.strip()
        if line.find('*/') != -1:
            # The block starts and ends in the same line
            return False
        elif line.startswith('/**') or line.startswith('/*!'):
            # Doxygen block
            return False
        elif line.startswith('/*'):
            return True
        else:
            return False

    def filter_line(self, line):
        if self._state == FilteredJavaFile.ST_INSIDE_CODE:
            return line
        elif self._state == FilteredJavaFile.ST_START:
            if self.is_multi_line_comment_start(line):
                self._state = FilteredJavaFile.ST_INITIAL_COMMENT
                return None
            else:
                self._state = FilteredJavaFile.ST_INSIDE_CODE
                return line
        elif self._state == FilteredJavaFile.ST_INITIAL_COMMENT:
            idx = line.find('*/')
            if idx == -1:
                return None
            else:
                self._state = FilteredJavaFile.ST_INSIDE_CODE
                line = line[(idx + 2):]
                if len(line.strip()) == 0:
                    return None
                else:
                    return line
        else:
            raise Exception('Invalid internal state {}.'.format(self._state))

    def add_license(self, license):
        for i in range(0, len(license)):
            self.contents.insert(i, license[i])