import time

import mindspore as ms
import mindspore.nn as nn
import mindspore.communication as comm
from config import cfg
import datetime
import logging
import shutil
import warnings
from model_utils.config import config,default_setup
# from utils.backup_files import sync_root
from src.kitti_dataset import create_kitti_dataset
from model_utils.utils import *
from src.monodde import *
from src.optimizer import *

ms.set_seed(1)
# numpy.random.seed(1)
# random.seed(1)

def init_distribute():
    if cfg.is_distributed:
        comm.init()
        config.rank = comm.get_rank()   #获取当前进程的排名
        config.group_size = comm.get_group_size()  #获取当前通信组大小
        config.local_rank=comm.get_local_rank()
        ms.set_auto_parallel_context(parallel_mode=ms.ParallelMode.DATA_PARALLEL, gradients_mean=True,
                                     device_num=cfg.group_size)   #配置自动并行计算
    else:
        cfg.MODEL.USE_SYNC_BN = False

def setup(args):
    '''load default config from config\defaults'''
    cfg.merge_from_file(args.config_path)
    # cfg.merge_from_list(args.opts)

    cfg.SEED = args.seed
    cfg.DATASETS.DATASET = args.dataset
    cfg.SOLVER.IMS_PER_BATCH = args.batch_size
    cfg.DATALOADER.NUM_WORKERS = args.num_work
    cfg.TEST.EVAL_DIS_IOUS = args.eval_iou
    cfg.TEST.EVAL_DEPTH = args.eval_depth
    cfg.TEST.SURVEY_DEPTH = args.survey_depth

    cfg.MODEL.COOR_ATTRIBUTE = args.Coor_Attribute
    cfg.MODEL.COOR_UNCERN = args.Coor_Uncern
    cfg.MODEL.GRM_ATTRIBUTE = args.GRM_Attribute
    cfg.MODEL.GRM_UNCERN = args.GRM_Uncern
    cfg.MODEL.BACKBONE.CONV_BODY = args.backbone
    cfg.MODEL.PRETRAIN=args.pretrained

    if args.vis_thre > 0:
        cfg.TEST.VISUALIZE_THRESHOLD = args.vis_thre

    if args.output is not None:
        cfg.OUTPUT_DIR = args.output

    if args.test:
        cfg.DATASETS.TEST_SPLIT = 'test'
        cfg.DATASETS.TEST = ("kitti_test",)
    cfg.is_training=args.is_training

    if args.demo:
        cfg.DATASETS.TRAIN = ("kitti_demo",)
        cfg.DATASETS.TEST = ("kitti_demo",)

    if args.data_root is not None:
        cfg.DATASETS.DATA_ROOT = args.data_root

    if args.debug:
        cfg.DATALOADER.NUM_WORKERS = 0
        cfg.TEST.DEBUG = args.debug

    cfg.START_TIME = datetime.datetime.strftime(datetime.datetime.now(), '%m-%d %H:%M:%S')
    default_setup(cfg, args)
    return cfg

def train_preprocess():
    cfg=setup(config)
    if cfg.MODEL.DEVICE=='Ascend':
        device_id = get_device_id()
        ms.set_context(mode=ms.GRAPH_MODE, device_target=cfg.MODEL.DEVICE, device_id=device_id)
    else:
        ms.context.set_context(mode=ms.PYNATIVE_MODE, device_target=cfg.MODEL.DEVICE, device_id=0)
    device=ms.get_context("device_target")
    init_distribute()  # init distributed

def load_parameters(val_network, train_network):
    logging.info("Load parameters of train network")
    param_dict_new = {}
    for key, values in train_network.parameters_and_names():
        if key.startswith('moments.'):
            continue
        elif key.startswith('yolo_network.'):
            param_dict_new[key[13:]] = values
        else:
            param_dict_new[key] = values
    ms.load_param_into_net(val_network, param_dict_new)
    logging.info('Load train network success')

def get_val_dataset(cfg,is_train=False):
    cfg.is_training=is_train
    datasets=create_kitti_dataset(cfg)
    return datasets

# @moxing_wrapper(pre_process=modelarts_pre_process, post_process=modelarts_post_process, pre_args=[config])
def train():
    train_preprocess()
    dataset=create_kitti_dataset(cfg)
    data_loader = dataset.create_tuple_iterator(do_copy=False)
    meters = MetricLogger(delimiter=" ",)

    network = Mono_net(cfg)
    val_network = Mono_net(cfg)
    network = MonoddeWithLossCell(network,cfg)
    opt=get_optim(cfg,network)
    network = nn.TrainOneStepCell(network, opt)

    network.set_train()
    logger = logging.getLogger("monoflex.trainer")
    logger.info("Start training")
    max_iter = cfg.SOLVER.MAX_ITERATION
    start_training_time = time.time()
    end = time.time()
    ckpt_queue = deque()

    ds_val = get_val_dataset(cfg)
    eval_wrapper = EvalWrapper(cfg, val_network,ds_val)

    default_depth_method = cfg.MODEL.HEAD.OUTPUT_DEPTH
    if cfg.local_rank == 0:
        best_mAP = 0
        best_result_str = None
        best_iteration = 0
        eval_iteration = 0
        record_metrics = ['Car_bev_', 'Car_3d_']

    iter_per_epoch=cfg.SOLVER.IMS_PER_BATCH

    # for iteration in range(0, max_iter):
    for data, iteration in zip(data_loader,range(0, max_iter)):
        data_time = time.time() - end
        loss=network(data,iteration)
        # loss=ms.Tensor(0)
        meters.update(loss=loss.asnumpy())
        batch_time = time.time() - end
        end = time.time()
        meters.update(time=batch_time, data=data_time)
        print(loss)

        eta_seconds = meters.time.global_avg * (max_iter - iteration)
        eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
        if iteration % 10 == 0 or iteration == max_iter:
            logger.info(
                meters.delimiter.join(
                    [
                        "eta: {eta}",
                        "iter: {iter}",
                        "{meters}",
                        "lr: {lr:.8f} \n",
                    ]
                ).format(
                    eta=eta_string,
                    iter=iteration,
                    meters=str(meters),
                    lr=cfg.SOLVER.BASE_LR,
                )
            )

        if cfg.rank == 0 and (iteration % cfg.SOLVER.SAVE_CHECKPOINT_INTERVAL == 0):
            logger.info('iteration = {}, saving checkpoint ...'.format(iteration))
            ckpt_name = os.path.join(cfg.OUTPUT_DIR, "MonoDDE_{}_{}.ckpt".format(iteration, cfg.SOLVER.IMS_PER_BATCH))
            ms.save_checkpoint(network, ckpt_name)
            if len(ckpt_queue) == cfg.SOLVER.SAVE_CHECKPOINT_MAX_NUM:
                ckpt_to_remove = ckpt_queue.popleft()
                # shutil.rmtree(ckpt_to_remove)
            ckpt_queue.append(ckpt_name)
        if iteration == max_iter and cfg.rank == 0:
            ckpt_name = os.path.join(cfg.OUTPUT_DIR,
                                     "MonoDDE_{}_{}.ckpt".format(iteration + 1, iter_per_epoch))
            ms.save_checkpoint(network, ckpt_name)

        if iteration % cfg.SOLVER.EVAL_INTERVAL == 0:
            if cfg.SOLVER.EVAL_AND_SAVE_EPOCH:
                cur_epoch = iteration // iter_per_epoch
                logger.info('epoch = {}, evaluate model on validation set with depth {}'.format(cur_epoch,
                                                                                default_depth_method))
            else:
                logger.info('iteration = {}, evaluate model on validation set with depth {}'.format(iteration,
                                                                                default_depth_method))

            val_types = ("detection",)
            dataset_name = cfg.DATASETS.TEST[0]

            if cfg.OUTPUT_DIR:
                output_folder = os.path.join(cfg.OUTPUT_DIR, dataset_name, "inference_{}".format(iteration))
                os.makedirs(output_folder, exist_ok=True)
            load_parameters(val_network, train_network=network)
            # result_dict, result_str, dis_ious  = eval_wrapper.inference(cur_epoch=iteration + 1, cur_step=1)
            result_dict, dis_ious = eval_wrapper.inference(iteration)

            if comm.get_local_rank() == 0:
                # only record more accurate R40 results
                result_dict = result_dict[0]

                # record the best model according to the AP_3D, Car, Moderate, IoU=0.7
                important_key = '{}_3d_{:.2f}/moderate'.format('Car', 0.7)
                eval_mAP = float(result_dict[important_key])
                if eval_mAP >= best_mAP:
                    # save best mAP and corresponding iterations
                    best_mAP = eval_mAP
                    best_iteration = iteration
                    # best_result_str = result_str
                    ckpt_name = os.path.join(cfg.OUTPUT_DIR,
                                             "model_moderate_best_{}.ckpt".format(default_depth_method))
                    ms.save_checkpoint(network, ckpt_name)
            #
                    if cfg.SOLVER.EVAL_AND_SAVE_EPOCH:
                        logger.info(
                            'epoch = {}, best_mAP = {:.2f}, updating best checkpoint for depth {} \n'.format(
                                cur_epoch, eval_mAP, default_depth_method))
                    else:
                        logger.info(
                            'iteration = {}, best_mAP = {:.2f}, updating best checkpoint for depth {} \n'.format(
                                iteration, eval_mAP, default_depth_method))

                eval_iteration += 1
    total_training_time = time.time() - start_training_time
    total_time_str = str(datetime.timedelta(seconds=total_training_time))
    if cfg.rank == 0:
        logger.info(
            "Total training time: {} ({:.4f} s / it), best model is achieved at iteration = {}".format(
                total_time_str, total_training_time / (max_iter), best_iteration,
            )
        )

        logger.info('The best performance is as follows')
        logger.info('\n' + best_result_str)

if __name__ == '__main__':
    # ms.set_context(save_graphs=True, save_graphs_path="src")
    train()


import discord
import asyncio
import re
from discord.utils import get
from discord.ext import commands
from datetime import datetime, timezone,timedelta
from bfunc import callAPI, db, traceBack, timeConversion, calculateTreasure, roleArray,timeConversion
from pymongo import UpdateOne
from pymongo.errors import BulkWriteError

logChannel = 73707667723806312 # 728456783466725427 73707667723806312

async def generateLog(self,  num : int, sessionInfo=None, guildDBEntriesDic=None, characterDBentries=None, userDBEntriesDic=None, ):
    logData =db.logdata
    if sessionInfo == None:
        sessionInfo = logData.find_one({"Log ID": int(num)})

    channel = self.bot.get_channel(logChannel) # 728456783466725427 737076677238063125
    editMessage = await channel.fetch_message(num)

    if not editMessage or editMessage.author != self.bot.user:
        print("Invalid Message")
        return None

    # get the collections of characters
    playersCollection = db.players
    guildCollection = db.guilds
    statsCollection = db.stats
    usersCollection = db.users

    sessionLogEmbed = editMessage.embeds[0]
    summaryIndex = sessionLogEmbed.description.find('Summary:')
    description = sessionLogEmbed.description[summaryIndex:]+"\n"

    role = sessionInfo["Role"]
    game = sessionInfo["Game"]
    start = sessionInfo["Start"]
    end = sessionInfo["End"]
    tierNum = sessionInfo["Tier"]

    guilds = sessionInfo["Guilds"]

    players = sessionInfo["Players"]
    #dictionary indexed by user id
    # {cp, magic items, consumables, inventory, partial, status, user id, character id, character name, character level, character cp, double rewards, guild, }

    dm = sessionInfo["DM"]
    # {cp, magic items, consumables, inventory, partial, status, user id, character id, character name, character level, character cp, double rewards, guild, noodles}

    maximumCP = dm["CP"]

    deathChars = []
    allRewardStrings = {}

    characterIDs = [p["Character ID"] for p in players.values()]
    userIDs = list(players.keys())
    guildIDs = list(guilds.keys())
    for gt in guildIDs:
        if guilds[gt]["Status"] == False:
            del guilds[gt]

    guildIDs = list(guilds.keys())

    userIDs.append(str(dm["ID"]))

    print(userIDs)
    # the db entry of every character
    if characterDBentries == None:
        characterDBentries = playersCollection.find({"_id": {"$in": characterIDs}})

    # the db entry of every user
    if userDBEntriesDic == None:
        userDBentries = usersCollection.find({"User ID": {"$in": userIDs}})
        userDBEntriesDic = {}
        for u in userDBentries:
            userDBEntriesDic[u["User ID"]] = u

    # the db entry of every guild
    if guildDBEntriesDic == None:
        guildDBEntriesDic = {}
        guildDBentries = guildCollection.find({"Name": {"$in": guildIDs}})
        for g in guildDBentries:
            guildDBEntriesDic[g["Name"]] = g

    print(userDBEntriesDic)

    for k, player in players.items():
        # this indicates that the character had died
        if player["Status"] == "Dead":
            deathChars.append(player)

        duration = player["CP"] * 3600

        guildDouble = False
        playerDouble = False
        dmDouble = False

        if role != "":
            guild_valid =("Guild" in player and
                            player["Guild"] in guilds and
                            guilds[player["Guild"]]["Status"])
            guildDouble = (guild_valid and
                            guilds[player["Guild"]]["Rewards"] and
                            guildDBEntriesDic[player["Guild"]]["Reputation"]>20)
            if((guild_valid and
                guilds[player["Guild"]]["Items"] and
                guildDBEntriesDic[player["Guild"]]["Reputation"]>15+20*guildDouble)):
                print("A", player["Double Items"])
                if player["Double Items"][0] == "Magic Items":
                    player["Magic Items"].append(player["Double Items"][1])
                else:
                    player[player["Double Items"][0]]["Add"].append(player["Double Items"][1])

            player["Double"] = "Double" in userDBEntriesDic[k] and userDBEntriesDic[k]["Double"] >0
            playerDouble = player["Double"]

            dmDouble = False

            treasureArray  = calculateTreasure(player["Level"], player["Character CP"], tierNum, duration, (player in deathChars), num, guildDouble, playerDouble, dmDouble)
            treasureString = f"{treasureArray[0]} CP, {sum(treasureArray[1].values())} TP, {treasureArray[2]} GP"

        else:
            # if there were no rewards we only care about the time
            treasureString = timeConversion(duration)
        groupString = ""
        groupString += guildDouble * "Guild "
        groupString += playerDouble * "Fanatic "

        groupString += f'{role} Friend {"Full"*(player["CP"]==dm["CP"])+"Partial"*(not player["CP"]==dm["CP"])} Rewards:\n{treasureString}'

        # if the player was not present the whole game and it was a no rewards game
        if role == "" and not (player["CP"]==dm["CP"]):
            # check if the reward has already been added to the reward dictionary
            if treasureString not in allRewardStrings:
                # if not, create the entry
                allRewardStrings[treasureString] = [player]
            else:
                # otherwise add the new players
                allRewardStrings[treasureString] += [player]
        else:
            print(groupString, "\n", allRewardStrings)
            # check if the full rewards have already been added, if yes create it and add the players
            if groupString in allRewardStrings:
                allRewardStrings[groupString] += [player]
            else:
                allRewardStrings[groupString] = [player]

    datestart = datetime.fromtimestamp(start).strftime("%b-%d-%y %I:%M %p")
    dateend = datetime.fromtimestamp(end).strftime("%b-%d-%y %I:%M %p")
    totalDuration = timeConversion(end - start)
    sessionLogEmbed.title = f"Timer: {game} [END] - {totalDuration}"
    sessionLogEmbed.description = f"Start: {datestart} CDT\nEnd: {dateend} CDT\n"

    dm_double_string = ""
    dmRewardsList = []
    #DM REWARD MATH STARTS HERE
    if("Character ID" in dm):
        charLevel = int(dm['Level'])
        player = dm
        # calculate the tier of the DM character
        if charLevel < 5:
            dmRole = 'Junior'
        elif charLevel < 11:
            dmRole = 'Journey'
        elif charLevel < 17:
            dmRole = 'Elite'
        elif charLevel < 20:
            dmRole = 'True'
        else:
            dmRole = 'Ascended'
        duration = player["CP"]*3600
        print(duration)
        if role != "":
            guild_valid =("Guild" in player and
                            player["Guild"] in guilds and
                            guilds[player["Guild"]]["Status"])
            guildDouble = (guild_valid and
                            guilds[player["Guild"]]["Rewards"] and
                            guildDBEntriesDic[player["Guild"]]["Reputation"]>20)
            player["Double"] = "Double" in userDBEntriesDic[k] and userDBEntriesDic[k]["Double"] >0
            playerDouble = player["Double"]
            if((guild_valid and
                guilds[player["Guild"]]["Items"] and
                guildDBEntriesDic[player["Guild"]]["Reputation"]>15+20*guildDouble)):
                print("A", player["Double Items"][0])
                if player["Double Items"][0] == "Magic Items":
                    player["Magic Items"].append(player["Double Items"][1])
                else:
                    player[player["Double Items"][0]]["Add"].append(player["Double Items"][1])

            dmDouble = player["DM Double"]

            dm_double_string += guildDouble * "Guild "
            dm_double_string += playerDouble * "Fanatic "
            dm_double_string += dmDouble * "DDMRW "

            dmtreasureArray  = calculateTreasure(player["Level"], player["CP"], tierNum, duration, (player in deathChars), num, guildDouble, playerDouble, dmDouble)

        # add the items that the DM awarded themselves to the items list
        for d in dm["Magic Items"]+dm["Consumables"]["Add"]+dm["Inventory"]["Add"]:
            dmRewardsList.append("+"+d)

    # get the collections of characters
    playersCollection = db.players

    noodles = dm["Noodles"]
    # Noodles Math

    # calculate the hour duration and calculate how many 3h segements were played
    hoursPlayed = maximumCP
    # that is the base line of sparkles and noodles gained
    noodlesGained = sparklesGained = int(hoursPlayed) // 3
    print("Noodles:", noodlesGained)
    # add the noodles to the record or start a record if needed

    #update noodle role if dm
    noodleString = "Current :star:: " + str(noodles)

    # if the game received rewards
    if role != "":
        # clear the embed message to repopulate it
        sessionLogEmbed.clear_fields()
        # for every unique set of TP rewards
        for key, value in allRewardStrings.items():
            temp = ""
            # for every player of this reward
            for v in value:
                vRewardList = []
                # for every brough consumable for the character
                for r in  v["Magic Items"]+ v["Consumables"]["Add"]+ v["Inventory"]["Add"]:
                    # add the awarded items to the list of rewards
                    vRewardList.append("+"+r)
                # if the character was not dead at the end of the game
                if v not in deathChars:
                    temp += f"{v['Mention']} | {v['Character Name']} {', '.join(vRewardList).strip()}\n"
                else:
                    # if they were dead, include that death
                    temp += f"~~{v['Mention']} | {v['Character Name']}~~ **DEATH** {', '.join(vRewardList).strip()}\n"

            # since if there is a player for this reward, temp will have text since every case above results in changes to temp
            # this informs us that there were no players
            if temp == "":
                temp = 'None'
            # add the information about the reward to the embed object
            sessionLogEmbed.add_field(name=f"**{key}**\n", value=temp, inline=False)
            # add temp to the total output string

        # list of all guilds
        guildsListStr = ""

        guildRewardsStr = ""
        players[dm["ID"]] = dm
        # passed in parameter, check if there were guilds involved
        if guilds != dict():
            guildsListStr = "Guilds: "
            # for every guild in the game
            for name, g in guilds.items():

                guildsListStr += "\n"+g["Mention"]
                # get the DB record of the guild
                #filter player list by guild
                print(players.values())
                gPlayers = [p for p in players.values() if "Guild" in p and p["Guild"]==name]
                if(len(gPlayers)>0):
                    gain = sparklesGained*len(gPlayers) + int("Guild" in dm and dm["Guild"]==name)
                    guildRewardsStr += f"{g['Name']}: +{gain} :sparkles:\n"

        sessionLogEmbed.title = f"\n**{game}**\n*Tier {tierNum} Quest* \n{sessionInfo['Channel']}"
        sessionLogEmbed.description = f"{guildsListStr}\nStart: {datestart} CDT\nEnd: {dateend} CDT\nRuntime: {totalDuration}\n"+description
        status_text = "Log is being processed! Characters are currently on hold."
        if sessionInfo["Status"] == "Approved":
            status_text = "✅ Log approved! Players have received their rewards."
        elif sessionInfo["Status"] == "Denied":
            status_text = "❌ Log Denied! Characters have been cleared"
        sessionLogEmbed.set_footer(text=f"Game ID: {num}\n{status_text}")

        # add the field for the DM's player rewards
        dm_text = "**No Character**"
        dm_name_text = "**No Rewards**"
        # if no character signed up then the character parts are excluded
        if("Character ID" in dm):
            dm_text = f"{dm['Character Name']} {', '.join(dmRewardsList)}"
            dm_name_text = f"DM {dm_double_string}Rewards (Tier {roleArray.index(dmRole) + 1}):\n**{dmtreasureArray[0]} CP, {sum(dmtreasureArray[1].values())} TP, {dmtreasureArray[2]} GP**\n"
        sessionLogEmbed.add_field(value=f"{dm['Mention']} | {dm_text}\n{'Gained :star:: ' + str(noodlesGained)} \n{noodleString}", name=dm_name_text)

        # if there are guild rewards then add a field with relevant information
        if guildRewardsStr != "":
            sessionLogEmbed.add_field(value=guildRewardsStr, name=f"Guild Rewards", inline=False)
        await editMessage.edit(embed=sessionLogEmbed)

    pass

class Log(commands.Cog):
    def __init__ (self, bot):
        self.bot = bot
        self.logChannel = 73707667723806312 # 728456783466725427 73707667723806312

    @commands.group()
    async def session(self, ctx):
        pass

    async def cog_command_error(self, ctx, error):
        msg = None

        if isinstance(error, commands.MissingAnyRole):
            await ctx.channel.send("You do not have the required permissions for this command.")
            bot.get_command(ctx.invoked_with).reset_cooldown(ctx)
            return
        else:
            if isinstance(error, commands.MissingRequiredArgument):
                msg = "Your command was missing an argument! "
            elif isinstance(error, commands.UnexpectedQuoteError) or isinstance(error, commands.ExpectedClosingQuoteError) or isinstance(error, commands.InvalidEndOfQuotedStringError):
              msg = ""

            if msg:
                ctx.command.reset_cooldown(ctx)
                await ctx.channel.send(msg)
                await traceBack(ctx,error)
            else:
                ctx.command.reset_cooldown(ctx)
                await traceBack(ctx,error)

    @commands.Cog.listener()
    async def on_raw_reaction_remove(self,payload):

        pass

    @commands.Cog.listener()
    async def on_raw_reaction_add(self,payload):
        pass

    @commands.has_any_role('Mod Friend', 'Admins')
    @session.command()
    async def approve(self,ctx,  num : int):
        channel = self.bot.get_channel
        logData = db.logdata
        sessionInfo = logData.find_one({"Log ID": int(num)})
        channel = self.bot.get_channel(self.logChannel) # 728456783466725427 737076677238063125

        editMessage = await channel.fetch_message(num)

        if not editMessage or editMessage.author != self.bot.user:
            return ctx.channel.send("Session has no corresponding message in the log channel.")

        guilds = sessionInfo["Guilds"]
        tierNum = sessionInfo["Tier"]
        role = sessionInfo["Role"]

        #dictionary indexed by user id
        players = sessionInfo["Players"]
        # {cp, magic items, consumables, inventory, status, character id, character name, character level, character cp, double rewards, guild, }

        dm = sessionInfo["DM"]
        # {cp, magic items, consumables, inventory, character id, character name, character level, character cp, double rewards, guild, noodles, dm double}

        maximumCP = dm["CP"]
        deathChars = []
        playerUpdates = []

        # get the collections of characters
        playersCollection = db.players
        guildCollection = db.guilds
        statsCollection = db.stats
        usersCollection = db.users

        characterIDs = [p["Character ID"] for p in players.values()]
        userIDs = list(players.keys())
        guildIDs = list(guilds.keys())

        userIDs.append(str(dm["ID"]))

        # the db entry of every character
        characterDBentries = playersCollection.find({"_id": {"$in": characterIDs}})

        # the db entry of every user
        userDBentries = usersCollection.find({"User ID": {"$in": userIDs}})

        # the db entry of every guild
        guildDBentries = guildCollection.find({"Name": {"$in": guildIDs}})
        print("GUILD", guildDBentries)
        guildDBEntriesDic = {}
        for g in guildDBentries:
            guildDBEntriesDic[g["Name"]] = g

        userDBEntriesDic = {}
        for u in userDBentries:
            userDBEntriesDic[u["User ID"]] = u

        #print(characterDBentries)
        print(players)

        for character in characterDBentries:
            player = players[str(character["User ID"])]
            # this indicates that the character had died
            if player["Status"] == "Dead":
                deathChars.append(player)

            duration = player["CP"] * 3600
            guildDouble = False
            playerDouble = False
            dmDouble = False

            if role != "":
                guild_valid =("Guild" in player and
                                player["Guild"] in guilds and
                                guilds[player["Guild"]]["Status"])
                guildDouble = (guild_valid and
                                guilds[player["Guild"]]["Rewards"] and
                                guildDBEntriesDic[player["Guild"]]["Reputation"]>20)

                player["Double"] = "Double" in userDBEntriesDic[str(character["User ID"])] and userDBEntriesDic[str(character["User ID"])]["Double"] >0
                playerDouble = player["Double"]
                dmDouble = False

                treasureArray  = calculateTreasure(player["Level"], player["Character CP"] , tierNum, duration, (player in deathChars), num, guildDouble, playerDouble, dmDouble)

                if((guild_valid and
                        guilds[player["Guild"]]["Items"] and
                        guildDBEntriesDic[player["Guild"]]["Reputation"]>15+20*guildDouble)):
                    if player["Double Items"][0] == "Inventory":
                        player["Inventory"].append(player["Double Items"][1])
                    else:
                        player[player["Double Items"][0]]["Add"].append(player["Double Items"][1])

                # create a list of all items the character has
                consumableList = character["Consumables"].split(", ")
                consumablesString = ""

                #if we know they didnt have any items, we know that changes could only be additions
                if(character["Consumables"]=="None"):
                    # turn the list of added items into the new string
                    consumablesString = ", ".join(player["Consumables"]["Add"])
                else:
                    #remove the removed items from the list of original items and then combine the remaining and the new items
                    for i in player["Consumables"]["Remove"]:
                        consumableList.remove(i)
                    consumablesString = ", ".join(player["Consumables"]["Add"]+consumableList)

                # if the string is empty, turn it into none
                consumablesString += "None"*(consumablesString=="")

                # magic items cannot be removed so we only care about addtions
                # if we have no items and no additions, string is None

                magicItemString = "None"
                if(character["Magic Items"]=="None"):
                    if(len(player["Magic Items"])==0):
                        pass
                    else:
                        # otherwise it is just the combination of new items
                        magicItemString = ", ".join(player["Magic Items"])
                else:
                    # otherwise connect up the old and new items
                    magicItemString = character["Magic Items"]+(", "+ ", ".join(player["Magic Items"]))*(len(player["Magic Items"])>0)

                # increase the relevant inventory entries and create them if necessary
                for i in player["Inventory"]["Add"]:
                    if i in character["Inventory"]:
                        character["Inventory"][i] += 1
                    else:
                        character["Inventory"][i] = 1

                # decrement the relevant items and delete the entry when necessary
                for i in player["Inventory"]["Remove"]:
                    character["Inventory"][i] -= 1
                    if int(character["Inventory"][i]) <= 0:
                        del character["Inventory"][i]

                # set up all db values that need to be incremented
                increment = {"CP":  treasureArray[0], "GP":  treasureArray[2],"Games": 1}
                # for every TP tier value that was gained create the increment field
                for k,v in treasureArray[1].items():
                    increment[k] = v
                print("IIIIIIIIIIIIIIIIIIIIII", increment)
                charRewards = {'_id': player["Character ID"],
                                    "fields": {"$unset": {f"GID": 1} ,"$inc": increment,
                                    "$set": {"Consumables": consumablesString, "Magic Items": magicItemString, "Inventory" : character["Inventory"]}}}
                if(player["Status"] == "Dead"):
                    del charRewards["fields"]["$inc"]["Games"]
                    print("charRewards 1", charRewards)
                    deathDic = {"inc": increment.copy(), "set": charRewards["fields"]["$set"]}
                    charRewards["fields"]["$inc"] = {"Games": 1}
                    charRewards["fields"]["$set"] = {"Death": deathDic}
                    print("death", deathDic)
                    print("charRewards 2", charRewards)
                playerUpdates.append(charRewards)

            else:
                # if there were no rewards we only care about the time
                pass

        dmRewardsList = []
        dm["Double"] = False
        #DM REWARD MATH STARTS HERE
        if("Character ID" in dm):

            player = dm

            duration = player["CP"] * 3600
            # the db entry of every character
            print(dm)
            character = playersCollection.find_one({"_id": dm["Character ID"]})
            charLevel = int(dm['Level'])
            # calculate the tier of the DM character
            dmTierNum= 5
            dmRole = "Ascended"
            if charLevel < 5:
                dmRole = 'Junior'
                dmTierNum = 1
            elif charLevel < 11:
                dmRole = 'Journey'
                dmTierNum = 2
            elif charLevel < 17:
                dmRole = 'Elite'
                dmTierNum = 3
            elif charLevel < 20:
                dmRole = 'True'
                dmTierNum = 4

            guild_valid =("Guild" in player and
                                player["Guild"] in guilds and
                                guilds[player["Guild"]]["Status"])
            guildDouble = (guild_valid and
                            guilds[player["Guild"]]["Rewards"] and
                            guildDBEntriesDic[player["Guild"]]["Reputation"]>20)

            player["Double"] = "Double" in userDBEntriesDic[dm["ID"]] and userDBEntriesDic[dm["ID"]]["Double"] >0
            playerDouble = player["Double"]

            dmDouble = player["DM Double"]

            treasureArray  = calculateTreasure(charLevel, player["Character CP"] , dmTierNum, duration, (player in deathChars), num, guildDouble, playerDouble, dmDouble)

            # create a list of all items the character has
            consumableList = character["Consumables"].split(", ")
            consumablesString = ""

            #if we know they didnt have any items, we know that changes could only be additions
            if(character["Consumables"]=="None"):
                # turn the list of added items into the new string
                consumablesString = ", ".join(player["Consumables"]["Add"])
            else:
                #remove the removed items from the list of original items and then combine the remaining and the new items
                for i in player["Consumables"]["Remove"]:
                    consumableList.remove(i)
                consumablesString = ", ".join(player["Consumables"]["Add"]+consumableList)

            # if the string is empty, turn it into none
            consumablesString += "None"*(consumablesString=="")

            # magic items cannot be removed so we only care about addtions
            # if we have no items and no additions, string is None
            magicItemString = "None"
            if(character["Magic Items"]=="None"):
                if(len(player["Magic Items"])==0):
                    pass
                else:
                    # otherwise it is just the combination of new items
                    magicItemString = ", ".join(player["Magic Items"])
            else:
                # otherwise connect up the old and new items
                magicItemString = character["Magic Items"]+(", "+ ", ".join(player["Magic Items"]))*(len(player["Magic Items"])>0)

            # increase the relevant inventory entries and create them if necessary
            for i in player["Inventory"]["Add"]:
                if i in character["Inventory"]:
                    character["Inventory"][i] += 1
                else:
                    character["Inventory"][i] = 1

            # decrement the relevant items and delete the entry when necessary
            for i in player["Inventory"]["Remove"]:
                character["Inventory"][i] -= 1
                if int(character["Inventory"][i]) <= 0:
                    del character["Inventory"][i]

            # set up all db values that need to be incremented
            increment = {"CP":  treasureArray[0], "GP":  treasureArray[2],"Games": 1}
            # for every TP tier value that was gained create the increment field
            for k,v in treasureArray[1].items():
                increment[k] = v

            charRewards = {'_id': player["Character ID"],  "fields": {"$unset": {f"GID": 1} ,
                            "$inc": increment,
                             "$set": {"Consumables": consumablesString, "Magic Items": magicItemString, "Inventory" : character["Inventory"]}}}
            playerUpdates.append(charRewards)

        noodles = dm["Noodles"]
        # Noodles Math
        hoursPlayed = maximumCP
        # that is the base line of sparkles and noodles gained
        noodlesGained = sparklesGained = int(hoursPlayed) // 3

        timerData = list(map(lambda item: UpdateOne({'_id': item['_id']}, item['fields']), playerUpdates))

        players[dm["ID"]] = dm
        guildsData = []
        # if the game received rewards
        if role != "":
            guildsRecordsList = []
            # passed in parameter, check if there were guilds involved
            if guilds != list():
                # for every guild in the game
                for g in guildDBentries:
                    name = g["Name"]
                    # get the DB record of the guild
                    #filter player list by guild
                    gPlayers = [p for p in players.values() if "Guild" in p and p["Guild"] == name]
                    p_count = len(gPlayers) - int("Guild" in dm and dm["Guild"] == name)
                    guilds[g["Name"]]["Player Sparkles"] = sparklesGained*p_count
                    guilds[g["Name"]]["DM Sparkles"] = sparklesGained*int("Guild" in dm and dm["Guild"] == name)
                    if(len(gPlayers)>0):
                        reputationCost = (20*guilds[g["Name"]]["Rewards"]+15*guilds[g["Name"]]["Items"])*guilds[g["Name"]]["Status"]
                        gain = sparklesGained*len(gPlayers) + int("Guild" in dm and dm["Guild"] == name)
                        guildsData.append(UpdateOne({"Name": name},
                                                   {"$inc": {"Games": 1, "Reputation": gain- reputationCost, "Total Reputation": gain}}))

        try:
            end = sessionInfo["End"]
            start = sessionInfo["Start"]

            totalDuration = end - start

            # get the stats for the month and create an entry if it doesnt exist yet
            statsIncrement ={"Games": 1, "Playtime": totalDuration, 'Players': len(players)}
            statsAddToSet = {}
            for name in [g["Name"] for g in guilds.values() if g["Status"]]:
                statsIncrement["Guilds."+name] = 1# Track how many guild quests there were
                statsIncrement["Guilds."+name+".GQ"] = 1
                statsIncrement["Guilds."+name+".Player Sparkles"] = guilds[g["Name"]]["Player Sparkles"]
                statsIncrement["Guilds."+name+".DM Sparkles"] = guilds[g["Name"]]["DM Sparkles"]
                # track how many games had a guild memeber with rewards and how many have not
                if guilds[g["Name"]]["Player Sparkles"] > 0:
                    statsIncrement["Guilds."+name+".GQM"] = 1
                elif guilds[g["Name"]]["DM Sparkles"] > 0:
                    statsIncrement["Guilds."+name+".GQDM"] = 1
                else:
                    statsIncrement["Guilds."+name+".GQNM"] = 1
            # Total Number of Games for the Month / Life

            # increment or create the stat entry for the DM of the game
            statsIncrement[f"DM.{dm['ID']}.T{tierNum}"] = 1
            statsIncrement[f"T{tierNum}"] = 1

            # track a list of unique players
            statsAddToSet = { 'Unique Players': { "$each":  list([p for p in players.keys()])} }

            # track playtime and players

            dateyear = datetime.fromtimestamp(start).strftime("%b-%y")
            # update all the other data entries
            # update the DB stats
            statsCollection.update_one({'Date': dateyear}, {"$set": {'Date': dateyear}, "$inc": statsIncrement, "$addToSet": statsAddToSet}, upsert=True)
            statsCollection.update_one({'Life': 1}, {"$inc": statsIncrement, "$addToSet": statsAddToSet}, upsert=True)
            # update the DM' stats

            usersCollection.update_one({'User ID': str(dm["ID"])}, {"$set": {'User ID':str(dm["ID"])}, "$inc": {'Games': 1, 'Noodles': noodlesGained, 'Double': -1*dm["Double"]}}, upsert=True)
            playersCollection.bulk_write(timerData)

            usersData = list([UpdateOne({'User ID': key}, {'$inc': {'Games': 1, 'Double': -1*item["Double"] }}, upsert=True) for key, item in players.items()])
            usersCollection.bulk_write(usersData)

            logData.update_one({"_id": sessionInfo["_id"]}, {"$set" : {"Status": "Approved"}})
            sessionInfo["Status"]="Approved"

            if guildsRecordsList != list():
                # do a bulk write for the guild data
                guildsCollection.bulk_write(guildsData)
            del players[dm["ID"]]

            await  generateLog(ctx, num, sessionInfo=sessionInfo, userDBEntriesDic=userDBEntriesDic, guildDBEntriesDic=guildDBEntriesDic, characterDBentries=characterDBentries)

            del players[dm["ID"]]
            game = sessionInfo["Game"]
            for k,p in players.items():
                c = ctx.guild.get_member(int(k))
                if(c):
                    await c.send(f"The session log for **{game}** has been approved. **{p['Character Name']}** has received their rewards.")
            dm_text = ""
            if("Character ID" in dm):
                dm_text += f" **{dm['Character Name']}** has received their rewards."

            c = ctx.guild.get_member(int(dm["ID"]))
            if(c):
                await c.send(f"Your session log for **{game}** has been approved."+dm_text)
            #logData.delete_one({"Log ID": num})

        except BulkWriteError as bwe:
            print(bwe.details)
            charEmbedmsg = await ctx.channel.send(embed=None, content="Uh oh, looks like something went wrong. Please try the timer again.")
        #except Exception as e:
        #    print ('MONGO ERROR: ' + str(e))
        #    charEmbedmsg = await ctx.channel.send(embed=None, content="Uh oh, looks like something went wrong. Please try the timer again.")
        else:
            print('Success')
        guild = ctx.guild
        dmUser = ctx.guild.get_member(int(dm["ID"]))
        if dmUser:
            dmRoleNames = [r.name for r in dmUser.roles]
            # for each noodle roll cut-off check if the user would now qualify for the roll and if they do not have it and remove the old roll
            if noodles >= 150:
                if 'Immortal Noodle' not in dmRoleNames:
                    noodleRole = get(guild.roles, name = 'Immortal Noodle')
                    await dmUser.add_roles(noodleRole, reason=f"Hosted 150 sessions. This user has 150+ Noodles.")
                    if 'Ascended Noodle' in dmRoleNames:
                        await dmUser.remove_roles(get(guild.roles, name = 'Ascended Noodle'))
                    noodleString += "\n**Immortal Noodle** role received! :tada:"
            elif noodles >= 100:
                if 'Ascended Noodle' not in dmRoleNames:
                    noodleRole = get(guild.roles, name = 'Ascended Noodle')
                    await dmUser.add_roles(noodleRole, reason=f"Hosted 100 sessions. This user has 100+ Noodles.")
                    if 'True Noodle' in dmRoleNames:
                        await dmUser.remove_roles(get(guild.roles, name = 'True Noodle'))
                    noodleString += "\n**Ascended Noodle** role received! :tada:"

            elif noodles >= 60:
                if 'True Noodle' not in dmRoleNames:
                    noodleRole = get(guild.roles, name = 'True Noodle')
                    await dmUser.add_roles(noodleRole, reason=f"Hosted 60 sessions. This user has 60+ Noodles.")
                    if 'Elite Noodle' in dmRoleNames:
                        await dmUser.remove_roles(get(guild.roles, name = 'Elite Noodle'))
                    noodleString += "\n**True Noodle** role received! :tada:"

            elif noodles >= 30:
                if 'Elite Noodle' not in dmRoleNames:
                    noodleRole = get(guild.roles, name = 'Elite Noodle')
                    await dmUser.add_roles(noodleRole, reason=f"Hosted 30 sessions. This user has 30+ Noodles.")
                    if 'Good Noodle' in dmRoleNames:
                        await dmUser.remove_roles(get(guild.roles, name = 'Good Noodle'))
                    noodleString += "\n**Elite Noodle** role received! :tada:"

            elif noodles >= 10:
                if 'Good Noodle' not in dmRoleNames:
                    noodleRole = get(guild.roles, name = 'Good Noodle')
                    await dmUser.add_roles(noodleRole, reason=f"Hosted 10 sessions. This user has 10+ Noodles.")
                    noodleString += "\n**Good Noodle** role received! :tada:"

    @commands.has_any_role('Mod Friend', 'Admins')
    @session.command()
    async def deny(self,ctx,  num : int):
        channel = self.bot.get_channel
        logData = db.logdata
        sessionInfo = logData.find_one({"Log ID": int(num)})
        channel = self.bot.get_channel(self.logChannel) # 728456783466725427 737076677238063125

        editMessage = await channel.fetch_message(num)

        if not editMessage or editMessage.author != self.bot.user:
            return ctx.channel.send("Session has no corresponding message in the log channel.")

        sessionLogEmbed = editMessage.embeds[0]
        #dictionary indexed by user id
        players = sessionInfo["Players"]
        # {cp, magic items, consumables, inventory, status, character id, character name, character level, character cp, double rewards, guild, }

        playerUpdates = []
        dm = sessionInfo["DM"]
        # {cp, magic items, consumables, inventory, character id, character name, character level, character cp, double rewards, guild, noodles, dm double}
        if "Character ID" in dm:
            charRewards = {'_id': dm["Character ID"],
                                "fields": {"$unset": {f"GID": 1} }}
            playerUpdates.append(charRewards)

        role = sessionInfo["Role"]

        # get the collections of characters
        playersCollection = db.players

        for player in players.values():
            if role != "":
                charRewards = {'_id': player["Character ID"],
                                    "fields": {"$unset": {f"GID": 1} }}
                playerUpdates.append(charRewards)

            else:
                # if there were no rewards we only care about the time
                pass

        timerData = list(map(lambda item: UpdateOne({'_id': item['_id']}, item['fields']), playerUpdates))

        try:
            playersCollection.bulk_write(timerData)
            logData.update_one({"_id": sessionInfo["_id"]}, {"$set" : {"Status": "Denied"}})

            game = sessionInfo["Game"]
            for k,p in players.items():
                c = ctx.guild.get_member(int(k))
                if(c):
                    await c.send(f"The session log for **{game}** has been denied. **{p['Character Name']}** has been cleared.")
            dm_text = ""
            if("Character ID" in dm):
                dm_text += f" **{dm['Character Name']}** has been cleared."

            c = ctx.guild.get_member(int(dm["ID"]))
            if(c):
                await c.send(f"Your session log for **{game}** has been denied."+dm_text)
            #logData.delete_one({"Log ID": num})
            sessionLogEmbed.set_footer(text=f"Game ID: {num}\n❌ Log Denied! Characters have been cleared.")
            await editMessage.edit(embed=sessionLogEmbed)
        except BulkWriteError as bwe:
            print(bwe.details)
            charEmbedmsg = await ctx.channel.send(embed=None, content="Uh oh, looks like something went wrong. Please try the timer again.")
        #except Exception as e:
        #    print ('MONGO ERROR: ' + str(e))
        #    charEmbedmsg = await ctx.channel.send(embed=None, content="Uh oh, looks like something went wrong. Please try the timer again.")
        else:
            print('Success')

    @session.command()
    async def denyGuild(self, ctx,  num : int, *, guilds):
        await self.guildPermission(ctx, num, "Status", False, 0)
    @session.command()
    async def permitGuild(self, ctx,  num : int, *, guilds):
        await self.guildPermission(ctx, num, "Status", True, 0)

    @session.command()
    async def denyRewards(self, ctx,  num : int, *, guilds):
        await self.guildPermission(ctx, num, "Rewards", False, 0)
    @session.command()
    async def permitRewards(self, ctx,  num : int, *, guilds):
        await self.guildPermission(ctx, num, "Rewards", True, 3)

    @session.command()
    async def denyItems(self, ctx,  num : int, *, guilds):
        await self.guildPermission(ctx, num, "Items", False, 0)
    @session.command()
    async def permitItems(self, ctx,  num : int, *, guilds):
        await self.guildPermission(ctx, num, "Items", True, 3)

    async def guildPermission(self, ctx, num : int, target, goal, min_members):
        logData = db.logdata
        sessionInfo = logData.find_one({"Log ID": int(num)})
        if( sessionInfo):
            if( (str(ctx.author.id) == sessionInfo["DM"]["ID"]) or "Mod Friend" in [r.name for r in ctx.author.roles]):
            # if the game received rewards
                if sessionInfo["Role"] != "":
                    players = sessionInfo["Players"]
                    players[sessionInfo["DM"]["ID"]] = sessionInfo["DM"]
                    guilds = sessionInfo["Guilds"]
                    guild_dic = {}
                    for g in guilds.values():
                        guild_dic[g["Mention"]] = g
                    err_message = ""
                    for guildChannel in ctx.message.channel_mentions:
                        m = guildChannel.mention
                        # filter player list by guild
                        gPlayers = [p for p in players.values() if "Guild" in p and guilds[p["Guild"]]["Mention"] == m]
                        if(len(gPlayers) >= min_members):
                            print(guilds)
                            if guildChannel.mention in guild_dic:
                                try:
                                    db.logdata.update_one({"_id": sessionInfo["_id"]}, {"$set": {"Guilds."+guild_dic[m]["Name"]+"."+target: goal}})
                                except BulkWriteError as bwe:
                                    print(e)
                            else:
                                err_message += m +" not found in game.\n"
                        else:
                            err_message += "Not enough members to apply this change for "+ m +"\n"
                    if err_message != "":
                        await ctx.channel.send(err_message)
                    else:
                        await ctx.channel.send("Session updated.")
                        await generateLog(self, num)
            else:
                await ctx.channel.send("You do not have the permissions to perform this change to the session.")
        else:
            await ctx.channel.send("The session could not be found, please double check your number or if the session has already been approved.")

    @commands.has_any_role('Mod Friend', 'Admins')
    @session.command()
    async def denyDDMRW(self, ctx,  num : int):
        await self.session_set(ctx, num, "DDMRW", False)

    @commands.has_any_role('Mod Friend', 'Admins')
    @session.command()
    async def permitDDMRW(self, ctx,  num : int):
        await self.session_set(ctx, num, "DDMRW", True)

    async def session_set(self, ctx, num : int, target, goal):
        logData = db.logdata
        sessionInfo = logData.find_one({"Log ID": int(num)})
        if( sessionInfo):
            try:
                db.logdata.update_one({"_id": sessionInfo["_id"]}, {"$set": {target: goal}})
            except BulkWriteError as bwe:
                print(e)
            await ctx.channel.send("Session updated.")
            await generateLog(self, num)
        else:
            await ctx.channel.send("The session could not be found, please double check your number or if the session has already been approved.")

    @session.command()
    async def optout(self, ctx, num):
        await self.opt(ctx, num, False)
    @session.command()
    async def optin(self, ctx, num):
        await self.opt(ctx, num, True)

    # allows DMs to opt in or out of DDMRW
    async def opt(self, ctx, num : int, goal):
        logData =db.logdata
        sessionInfo = logData.find_one({"Log ID": int(num)})
        if( sessionInfo):
            if (str(ctx.author.id) == sessionInfo["DM"]["ID"] or "Mod Friend" in [r.name for r in ctx.author.roles] and sessionInfo["DDMRW"]):

                try:
                    db.logdata.update_one({"_id": sessionInfo["_id"]}, {"$set": {"DM.DM Double": goal}})
                    await generateLog(self, num)
                except BulkWriteError as bwe:
                    print(e)
            else:
                await ctx.channel.send("You were not the DM of that session or it was not DDMRW.")
        else:
            await ctx.channel.send("The session could not be found, please double check your number or if the session has already been approved.")

    @session.command()
    async def log(self, ctx,  num : int, *, editString=""):
        # The real Bot
        botUser = self.bot.user

        # Logs channel
        # channel = self.bot.get_channel(577227687962214406)
        channel = self.bot.get_channel(self.logChannel) # 728456783466725427 737076677238063125

        limit = 100
        msgFound = False
        async with channel.typing():
            async for message in channel.history(oldest_first=False, limit=limit):
                if int(num) == message.id and message.author == botUser:
                    editMessage = message
                    msgFound = True
                    break

        if not msgFound:
            delMessage = await ctx.channel.send(content=f"I couldn't find your game with ID - `{num}` in the last {limit} games. Please try again, I will delete your message and this message in 10 seconds.")
            await asyncio.sleep(10)
            await delMessage.delete()
            await ctx.message.delete()
            return

        logData =db.logdata
        sessionInfo = logData.find_one({"Log ID": int(num)})
        if( sessionInfo):
            if (str(ctx.author.id) == sessionInfo["DM"]["ID"] or "Mod Friend" in [r.name for r in ctx.author.roles] and sessionInfo["DDMRW"]):
                pass

            else:
                await ctx.channel.send("You were not the DM of that session.")
                return
        else:
            await ctx.channel.send("The session could not be found, please double check your number or if the session has already been approved.")
            return
        sessionLogEmbed = editMessage.embeds[0]

        summaryIndex = sessionLogEmbed.description.find('Summary:')
        sessionLogEmbed.description = sessionLogEmbed.description[:summaryIndex] + "Summary: " + editString+"\n"

        await editMessage.edit(embed=sessionLogEmbed)
        delMessage = await ctx.channel.send(content=f"I've edited the summary for quest #{num}.\n```{editString}```\nPlease double-check that the edit is correct. I will now delete your message and this one in 30 seconds.")

        await editMessage.edit(embed=sessionLogEmbed)
        await asyncio.sleep(30)
        await delMessage.delete()
        await ctx.message.delete()

def setup(bot):
    bot.add_cog(Log(bot))


from autograd import numpy as np

def ionstr_DOE94(Sal):
    """Ionic strength following DOE94."""
    # === CO2SYS.m comments: =======
    # This is from the DOE handbook, Chapter 5, p. 13/22, eq. 7.2.4.
    return 19.924 * Sal / (1000 - 1.005 * Sal)

def borate_C65(Sal):
    """Total borate in mol/kg-sw following C65."""
    # === CO2SYS.m comments: =======
    # This is .00001173*Sali, about 1% lower than Uppstrom's value
    # Culkin, F., in Chemical Oceanography, ed. Riley and Skirrow, 1965:
    # GEOSECS references this, but this value is not explicitly given here
    # Output in mol/kg-SW
    return 0.0004106 * Sal / 35

def borate_U74(Sal):
    """Total borate in mol/kg-sw following U74."""
    # === CO2SYS.m comments: =======
    # Uppstrom, L., Deep-Sea Research 21:161-162, 1974:
    # this is .000416*Sali/35. = .0000119*Sali
    # TB[FF] = (0.000232/10.811)*(Sal[FF]/1.80655); in mol/kg-SW.
    return 0.0004157 * Sal / 35

def borate_LKB10(Sal):
    """Total borate in mol/kg-sw following LKB10."""
    # === CO2SYS.m comments: =======
    # Lee, Kim, Byrne, Millero, Feely, Yong-Ming Liu. 2010.
    # Geochimica Et Cosmochimica Acta 74 (6): 1801-1811.
    # Output in mol/kg-SW.
    return 0.0004326 * Sal / 35

def calcium_C65(Sal):
    """Calcium in mol/kg-sw following C65."""
    # === CO2SYS.m comments: =======
    # *** CalculateCaforGEOSECS:
    # Culkin, F, in Chemical Oceanography, ed. Riley and Skirrow, 1965:
    # (quoted in Takahashi et al, GEOSECS Pacific Expedition v. 3, 1982)
    # Culkin gives Ca = (.0213/40.078)*(Sal/1.80655) in mol/kg-SW
    # which corresponds to Ca = .01030*Sal/35.
    return 0.01026 * Sal / 35

def calcium_RT67(Sal):
    """Calcium in mol/kg-sw following RT67."""
    # === CO2SYS.m comments: =======
    # This is .010285*Sal/35
    return 0.02128 / 40.087 * Sal / 1.80655

def fluoride_R65(Sal):
    """Total fluoride in mol/kg-sw following R65."""
    # === CO2SYS.m comments: =======
    # Riley, J. P., Deep-Sea Research 12:219-220, 1965:
    # this is .000068*Sali/35. = .00000195*Sali; in mol/kg-SW.
    return (0.000067 / 18.998) * Sal / 1.80655

def sulfate_MR66(Sal):
    """Total sulfate in mol/kg-sw following MR66."""
    # === CO2SYS.m comments: =======
    # Morris, A. W., and Riley, J. P., Deep-Sea Research 13:699-705, 1966:
    # this is .02824*Sali/35. = .0008067*Sali; in mol/kg-SW.
    return (0.14 / 96.062) * Sal / 1.80655

def _co2sys_TB(Sal, WhichKs, WhoseTB):
    """Calculate total borate from salinity for the given options."""
    TB = np.where(WhichKs == 8, 0.0, np.nan)  # pure water
    TB = np.where((WhichKs == 6) | (WhichKs == 7), borate_C65(Sal), TB)
    F = (WhichKs != 6) & (WhichKs != 7) & (WhichKs != 8)
    TB = np.where(F & (WhoseTB == 1), borate_U74(Sal), TB)
    TB = np.where(F & (WhoseTB == 2), borate_LKB10(Sal), TB)
    return TB

def _co2sys_TCa(Sal, WhichKs):
    """Calculate total calcium from salinity for the given options."""
    F = (WhichKs == 6) | (WhichKs == 7)  # GEOSECS values
    TCa = np.where(F, calcium_C65(Sal), calcium_RT67(Sal))
    return TCa

def fromSal(Sal, WhichKs, WhoseTB, totals=None):
    """Estimate total molinities of calcium, borate, fluoride and sulfate from salinity.

    Subfunctions based on Constants, version 04.01, 10-13-97, by Ernie Lewis.
    """
    if totals is None:
        totals = {}
    if "TB" not in totals:
        TB = _co2sys_TB(Sal, WhichKs, WhoseTB)
        totals["TB"] = TB
    if "TF" not in totals:
        TF = fluoride_R65(Sal)
        totals["TF"] = TF
    if "TSO4" not in totals:
        TSO4 = sulfate_MR66(Sal)
        totals["TSO4"] = TSO4
    if "TCa" not in totals:
        TCa = _co2sys_TCa(Sal, WhichKs)
        totals["TCa"] = TCa
    if "alpha" not in totals:
        totals["alpha"] = 0.0
    if "beta" not in totals:
        totals["beta"] = 0.0
    return totals

def assemble(Sal, TSi, TPO4, TNH3, TH2S, WhichKs, WhoseTB, totals=None):
    """Estimate total molinities from salinity and assemble along with other salts and
    related variables.
    """
    # Pure Water case:
    Sal = np.where(WhichKs == 8, 0.0, Sal)
    # GEOSECS and Pure Water:
    F = (WhichKs == 6) | (WhichKs == 8)
    TPO4 = np.where(F, 0.0, TPO4)
    TSi = np.where(F, 0.0, TSi)
    TNH3 = np.where(F, 0.0, TNH3)
    TH2S = np.where(F, 0.0, TH2S)
    # Convert micromol to mol
    TPO4 = TPO4 * 1e-6
    TSi = TSi * 1e-6
    TNH3 = TNH3 * 1e-6
    TH2S = TH2S * 1e-6
    totals = fromSal(Sal, WhichKs, WhoseTB, totals=totals)
    # The vector `PengCorrection` is used to modify the value of TA, for those
    # cases where WhichKs==7, since PAlk(Peng) = PAlk(Dickson) + TP.
    # Thus, PengCorrection is 0 for all cases where WhichKs is not 7.
    PengCorrection = np.where(WhichKs == 7, TPO4, 0.0)
    # Add everything else to `totals` dict
    totals["TPO4"] = TPO4
    totals["TSi"] = TSi
    totals["TNH3"] = TNH3
    totals["TH2S"] = TH2S
    totals["Sal"] = Sal  # this is input `Sal` but with pure water case forced to 0.0
    totals["PengCorrection"] = PengCorrection
    return totals


import aiohttp
import datetime
import json
import random

import common.constants as const

from common.uhtml import ItemInfo
from common.utils import gen_uhtml_img_code

async def anilist_num_entries(query, query_vars, session):
    '''
    Oneshot query to determine the total number of entries
    in an anilist query. Requires the query to have pageInfo.

    Does not have the anilist context manager lock.
    '''
    async with session.post(const.ANILIST_API, json={'query': query, 'variables': query_vars}) as r:
        resp = await r.json()

        if r.status != 200:
            return

    return resp['data']['Page']['pageInfo']['total']

async def anilist_search(medium, search, anilist_man):
    query = '''
    query ($search: String) {
        Page (page: 1, perPage: 1) {
            media (MEDIUM_PLACEHOLDER, search: $search, isAdult: false) {
                id
                idMal
                title {
                    english
                    romaji
                }
                coverImage {
                    extraLarge
                }
                status
                episodes
                chapters
                description
                averageScore
            }
        }
    }
    '''

    mq = 'type: ANIME' if medium == 'anime' else 'type: MANGA'
    query = query.replace('MEDIUM_PLACEHOLDER', mq)
    query_vars = {
        'search': search
    }

    series_data = {}

    async with anilist_man.lock():
        async with aiohttp.ClientSession() as session:
            async with session.post(const.ANILIST_API, json={'query': query, 'variables': query_vars}) as r:
                resp = await r.json()

                if r.status != 200:
                    err_msg = resp['errors'][0]['message']
                    return f'hippoerror, Status code {r.status} when fetching {search}: {err_msg}'

                series_list = resp['data']['Page']['media']
                if not series_list:
                    return f'hippoerror, No {medium} found for {search}.'

                series_data = series_list[0]

    mal_url = f'https://myanimelist.net/{medium}/{series_data["idMal"]}'
    img_uhtml = gen_uhtml_img_code(series_data['coverImage']['extraLarge'], dims=(65, 100))
    title = series_data['title']['english']
    if not title:
        title = series_data['title']['romaji']

    parts = 'Episodes' if medium == 'anime' else 'Chapters'
    score = f'{series_data["averageScore"]}%' if series_data['averageScore'] else 'N/A'

    item_info = ItemInfo(title, mal_url, 'mal')

    kwargs = {'img_uhtml': img_uhtml,
              'al_link': f'https://anilist.co/{medium}/{series_data["id"]}',
              'ongoing': series_data['status'],
              'parts': f'{parts}: {series_data[parts.lower()]}',
              'score': score,
              'synopsis': series_data['description']}

    uhtml = item_info.animanga(**kwargs)

    return f'hippo{medium}{series_data["idMal"]}, {uhtml}'

async def anilist_rand_series(medium, anilist_man, genres=[], tags=[]):
    query = '''
    query ($page: Int) {
        Page (page: $page, perPage: 1) {
            pageInfo {
                total
            }
            media (CATEGORIES_PLACEHOLDER minimumTagRank: 50, type: TYPE_PLACEHOLDER, isAdult: false) {
                id
                idMal
                title {
                    english
                    userPreferred
                }
                coverImage {
                    extraLarge
                }
                status
                episodes
                chapters
                description
                averageScore
            }
        }
    }
    '''

    category_params = []
    if genres:
        category_params.append(f'genre_in: {json.dumps(genres)}')
    if tags:
        category_params.append(f'tag_in: {json.dumps(tags)}')

    category_params_str = ','.join(category_params)
    if category_params_str:
        category_params_str += ','

    query = query.replace('CATEGORIES_PLACEHOLDER', category_params_str)
    query = query.replace('TYPE_PLACEHOLDER', medium.upper())
    query_vars = {'page': 1}

    num_entries = 0
    async with aiohttp.ClientSession() as session:
        async with anilist_man.lock():
            num_entries = await anilist_num_entries(query, query_vars, session)

        if not num_entries:
            return 'No series found with the given specifications.'

        query_vars['page'] = random.randint(1, num_entries)

        series_data = {}
        async with anilist_man.lock():
            async with session.post(const.ANILIST_API, json={'query': query, 'variables': query_vars}) as r:
                resp = await r.json()

                if r.status != 200:
                    err_msg = resp['errors'][0]['message']
                    return f'Status code {r.status} when fetching {query}: {err_msg}'

                series_data = resp['data']['Page']['media'][0]

    mal_url = f'https://myanimelist.net/{medium}/{series_data["idMal"]}'
    img_uhtml = gen_uhtml_img_code(series_data['coverImage']['extraLarge'], dims=(65, 100))
    title = series_data['title']['english']
    if not title:
        title = series_data['title']['userPreferred']

    parts = 'Episodes' if medium == 'anime' else 'Chapters'
    score = f'{series_data["averageScore"]}%' if series_data['averageScore'] else 'N/A'

    item_info = ItemInfo(title, mal_url, 'mal')

    kwargs = {'img_uhtml': img_uhtml,
              'al_link': f'https://anilist.co/{medium}/{series_data["id"]}',
              'ongoing': series_data['status'],
              'parts': f'{parts}: {series_data[parts.lower()]}',
              'score': score,
              'synopsis': series_data['description']}

    uhtml = item_info.animanga(**kwargs)

    return f'hippo{medium}{series_data["idMal"]}, {uhtml}'

async def get_related_series(medium, series, anilist_man):
    related_series = [(medium, series)]
    query = '''
    query ($mal_id: Int) {
        Page (page: 1, perPage: 1) {
            media (type: TYPE_PLACEHOLDER, idMal: $mal_id, isAdult: false) {
                title {
                    userPreferred
                }
                relations {
                    edges {
                        relationType
                        node {
                            idMal
                            type
                        }
                    }
                }
            }
        }
    }
    '''
    title = ''

    for s in related_series:
        async with anilist_man.lock():
            async with aiohttp.ClientSession() as session:
                series_query = query.replace('TYPE_PLACEHOLDER', s[0].upper())
                query_vars = {'mal_id': s[1]}
                async with session.post(const.ANILIST_API, json={'query': series_query, 'variables': query_vars}) as r:
                    resp = await r.json()

                    if r.status != 200:
                        continue

                    if s == (medium, series):
                        title = resp['data']['Page']['media'][0]['title']['userPreferred']

                    for rs in resp['data']['Page']['media'][0]['relations']['edges']:
                        rs_info = (rs['node']['type'].lower(), rs['node']['idMal'])
                        if rs_info not in related_series and rs['relationType'] != 'CHARACTER':
                            related_series.append(rs_info)

    return (related_series, title)

async def add_anotd_bl(medium, series, anilist_man, db_man):
    related_series, title = await get_related_series(medium, series, anilist_man)

    expiration = str(datetime.datetime.now() + datetime.timedelta(days=365))

    await db_man.execute("INSERT INTO anotd_banlist (medium, mal_id, name, expiration) "
                            f"VALUES ('{medium}', {series}, '{title}', '{expiration}')")

    for s in related_series:
        exists = await db_man.execute(f"SELECT * FROM mal_banlist WHERE medium='{s[0]}' AND mal_id={s[1]}")

        if not exists:
            await db_man.execute("INSERT INTO mal_banlist (medium, mal_id, anotd_source) "
                                    f"VALUES ('{s[0]}', {s[1]}, 1)")

async def rm_anotd_bl(medium, series, anilist_man, db_man):
    related_series, _ = await get_related_series(medium, series, anilist_man)

    for s in related_series:
        await db_man.execute(f"DELETE FROM mal_banlist WHERE medium='{s[0]}' AND mal_id={s[1]} AND anotd_source=1")

    await db_man.execute(f"DELETE FROM anotd_banlist WHERE medium='{medium}' AND mal_id={series}")


from discord.ext import commands
from collections import defaultdict
import random
random.seed()
from discord.ext import commands

class Cog(commands.Cog, name="noplead"):
    def __init__(self ,bot):
        self.bot = bot
        self.times = defaultdict(int)

    @commands.command()
    async def command(self ,ctx):
        self.permload()
        if self.permcheck(ctx ,self.commandid):
            pass

    @commands.Cog.listener()
    async def on_message(self, message):
        content = message.content
        if "ðŸ¥º" in content or ":pleading:" in content or "ðŸ¥¹" in content or ":face_holding_back_tears:" in content:
          await message.channel.send("gtfo pleading user")
          await message.delete()

async def setup(bot):
    await bot.add_cog(Cog(bot))


#jiayizha
#Tutorial from GitHub: https://github.com/MiniGirlGeek/pygame-paintme

import pygame
from colorSheet import *
from pygame.locals import *
from dialogueBox import *
from sys import exit
from pygamegame import PygameGame
from dialogueBox import dialogue

class writeScreen(object):
    def __init__(self, width=300, height=200):
        self.width = width
        self.height = height

class drawWindow(PygameGame):
    def __init__(self, x=620, y=410, width=300, height=200):
        self.x = x
        self.y = y
        self.width = width
        self.height = height
        self.charaList = [hito, ki, hana, sora, kawa]
        self.mouse_pos = None
        self.isDrawing = False
        self.step = 0
        self.brushColor = darkMagenta
        self.writing = pygame.Surface((370,200))
        self.writing.fill(white)

    def drawBox(self,screen):
        chara = self.charaList[self.step]
        chara.image = pygame.image.load(chara.path).convert()
        screen.blit(chara.image,(self.x+90,self.y))
        screen.blit(self.writing,(self.x,self.y))

    def fillColor(self):
        self.writing.fill(white)

    def transparent(self):
        self.writing.set_alpha(150) #transparency

    def redrawAll(self,screen):
        self.drawWindow.drawBox(screen)


# -*- coding = utf-8 -*-
# @time:2020/9/22 22:55
# Author:TC
# @File:xgb_test.py
# @Software:PyCharm

import numpy as np
import pandas as pd
import xgboost as xgb
import matplotlib.pyplot as plt
plt.style.use('ggplot')

from sklearn.datasets import load_iris
import xgboost as xgb
from xgboost import plot_importance
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

iris=load_iris()
X,y=iris.data,iris.target
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=123456)

# # 定义模型训练参数
# params = {
#     "objective": "binary:logistic",
#     "booster": "gbtree",
#     "max_depth": 3
#          }
# # 训练轮数
# num_round = 5
#
# # 训练过程中实时输出评估结果
# watchlist = [(xgb_train, 'train'), (xgb_test, 'test')]
#
# # 模型训练
# model = xgb.train(params, xgb_train, num_round, watchlist)

# 算法参数
params = {
    'booster': 'gbtree',
    'objective': 'multi:softmax', #多分类问题
    'num_class': 3,               #类别数，与 multisoftmax 并用
    'gamma': 0.1,                 #用于控制是否后剪枝的参数,越大越保守，一般0.1、0.2这样子。
    'max_depth': 6,               #构建树的深度，越大越容易过拟合
    'lambda': 2,                  #控制模型复杂度的权重值的L2正则化项参数，参数越大，模型越不容易过拟合。
    'subsample': 0.7,             #随机采样训练样本
    'colsample_bytree': 0.75,     #生成树时进行的列采样
    'min_child_weight': 3,        #min_child_weight就是叶子上的最小样本数
    'silent': 1,                  #设置成1则没有运行信息输出，最好是设置为0.
    'eta': 0.1,                   #如同学习率
    'seed': 1,
    'nthread': 4,                 #cpu 线程数
}

plst=list(params.items())

#先保存到XGBoost二进制文件中将使加载速度更快，然后再加载进来
dtrain=xgb.DMatrix(X_train,y_train)
num_rounds=500
model=xgb.train(plst,dtrain,num_rounds)

dtest=xgb.DMatrix(X_test)
y_pred=model.predict(dtest)

accuracy=accuracy_score(y_test,y_pred)
print('accuracy:%.2f%%'%(accuracy*100.0))

#显示重要特征
plot_importance(model)
plt.show()


from django.urls import path
from MainApp import views

urlpatterns = [
    path('', views.index,),
    path('about', views.about),
    path('item/<int:id>', views.get_item),
    path('items', views.items_list),
    path('countries', views.countrie_list),
    path('Australia', views.Australia),
    path('Austria', views.Austria),
    path('Bahamas', views.Bahamas),
]


from __future__ import annotations

from typing import (
    TYPE_CHECKING,
    Callable,
)

import numpy as np

from pandas._typing import Scalar
from pandas.compat._optional import import_optional_dependency

from pandas.core.util.numba_ import (
    NUMBA_FUNC_CACHE,
    get_jit_arguments,
)

def generate_shared_aggregator(
    func: Callable[..., Scalar],
    engine_kwargs: dict[str, bool] | None,
    cache_key_str: str,
):
    """
    Generate a Numba function that loops over the columns 2D object and applies
    a 1D numba kernel over each column.

    Parameters
    ----------
    func : function
        aggregation function to be applied to each column
    engine_kwargs : dict
        dictionary of arguments to be passed into numba.jit
    cache_key_str: str
        string to access the compiled function of the form
        <caller_type>_<aggregation_type> e.g. rolling_mean, groupby_mean

    Returns
    -------
    Numba function
    """
    nopython, nogil, parallel = get_jit_arguments(engine_kwargs, None)

    cache_key = (func, cache_key_str)
    if cache_key in NUMBA_FUNC_CACHE:
        return NUMBA_FUNC_CACHE[cache_key]

    if TYPE_CHECKING:
        import numba
    else:
        numba = import_optional_dependency("numba")

    @numba.jit(nopython=nopython, nogil=nogil, parallel=parallel)
    def column_looper(
        values: np.ndarray,
        start: np.ndarray,
        end: np.ndarray,
        min_periods: int,
        *args,
    ):
        result = np.empty((len(start), values.shape[1]), dtype=np.float64)
        for i in numba.prange(values.shape[1]):
            result[:, i] = func(values[:, i], start, end, min_periods, *args)
        return result

    return column_looper


from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required
from django.contrib import messages

from users.models import Influencer
from campaign.models import Contract
from invitation.models import Invitation
from message.models import Message
# Create your views here.
@login_required(login_url='/login')
def dashboard(request):
    if request.user.is_influencer:
        influencer_profiles = Influencer.objects.filter(user=request.user)
        if influencer_profiles.count() == 0:
            messages.warning(request, 'You must fill out the profile form to see home page.')
            return redirect('influencer_profile')
        else:
            if not influencer_profiles.first().instagram_account and not influencer_profiles.first().facebook_account and not influencer_profiles.first().twitter_account:
                return redirect('influencer_profile')

    context = {
        'menu': 'home'
    }

    if request.user.is_influencer:
        context.update(get_num_notification(request))
        return render(request, 'home/home_influencer.html', context)
    elif request.user.is_client:
        context.update(get_num_notification(request))
        return render(request, 'home/home_client.html', context)
    elif request.user.is_staff:
        context.update(get_num_notification(request))
        return render(request, 'home/home_agent.html', context)

def get_num_notification(request):
    context = dict()
    if not request.user.is_client:
        context = {
            'numOfInfluencerMessage': Message.objects.filter(sent_to=request.user,read_status=False,campaign__isnull=True).count()
        }
    if not request.user.is_influencer:
        context.update({'numOfClientMessage': Message.objects.filter(sent_to=request.user,read_status=False,discussion__isnull=True).count()})
    return context


from collections import OrderedDict

import numpy

from theano import config

from fuel.transformers import Transformer
from picklable_itertools.extras import equizip

class MaximumFrameCache(Transformer):
    """Cache examples, and create batches of maximum number of frames.

    Given a data stream which reads large chunks of data, this data
    stream caches these chunks and returns batches with a maximum number
    of acoustic frames.

    Parameters
    ----------
    max_frames : int
        maximum number of frames per batch

    Attributes
    ----------
    cache : list of lists of objects
        This attribute holds the cache at any given point. It is a list of
        the same size as the :attr:`sources` attribute. Each element in
        this list is a deque of examples that are currently in the
        cache. The cache gets emptied at the start of each epoch, and gets
        refilled when needed through the :meth:`get_data` method.

    """
    def __init__(self, data_stream, max_frames, rng):
        super(MaximumFrameCache, self).__init__(
            data_stream)
        self.max_frames = max_frames
        self.cache = OrderedDict([(name, []) for name in self.sources])
        self.num_frames = []
        self.rng = rng

    def next_request(self):
        curr_max = 0
        for i, n_frames in enumerate(self.num_frames):
            # Select max number of frames because of future padding
            curr_max = max(n_frames, curr_max)
            total = curr_max * (i + 1)
            if total >= self.max_frames:
                return i + 1
        return len(self.num_frames)

    def get_data(self, request=None):
        if not self.cache[self.cache.keys()[0]]:
            self._cache()
        data = []
        request = self.next_request()
        for source_name in self.cache:
            data.append(numpy.asarray(self.cache[source_name][:request]))
        self.cache = OrderedDict([(name, dt[request:]) for name, dt
                                  in self.cache.iteritems()])
        self.num_frames = self.num_frames[request:]

        return tuple(data)

    def get_epoch_iterator(self, **kwargs):
        self.cache = OrderedDict([(name, []) for name in self.sources])
        self.num_frames = []
        return super(MaximumFrameCache, self).get_epoch_iterator(**kwargs)

    def _cache(self):
        data = next(self.child_epoch_iterator)
        indexes = range(len(data[0]))
        self.rng.shuffle(indexes)
        data = [[dt[i] for i in indexes] for dt in data]
        self.cache = OrderedDict([(name, self.cache[name] + dt) for name, dt
                                  in equizip(self.data_stream.sources, data)])
        self.num_frames.extend([x.shape[0] for x in data[0]])

class Transpose(Transformer):
    """Transpose axes of datastream.
    """
    def __init__(self, datastream, axes_list):
        super(Transpose, self).__init__(datastream)
        self.axes_list = axes_list

    def get_data(self, request=None):
        data = next(self.child_epoch_iterator)
        transposed_data = []
        for axes, data in zip(self.axes_list, data):
            transposed_data.append(numpy.transpose(data, axes))
        return transposed_data

class AddUniformAlignmentMask(Transformer):
    """Adds an uniform alignment mask to the incoming batch.

    Parameters
    ----------

    """
    def __init__(self, data_stream):
        super(AddUniformAlignmentMask, self).__init__(data_stream)
        self.sources = self.data_stream.sources + ('alignment',)

    def get_data(self, request=None):
        data = next(self.child_epoch_iterator)
        sources = self.data_stream.sources

        x_idx = sources.index('x')
        y_idx = sources.index('y')
        x_mask_idx = sources.index('x_mask')
        y_mask_idx = sources.index('y_mask')

        batch_size = data[x_idx].shape[1]
        max_len_output = data[y_idx].shape[0]
        max_len_input = data[x_idx].shape[0]
        mask_shape = (max_len_output, batch_size, max_len_input)
        alignment = numpy.zeros(mask_shape, dtype=config.floatX)

        for k in xrange(batch_size):
            in_size = numpy.count_nonzero(data[x_mask_idx][:,k])
            out_size = numpy.count_nonzero(data[y_mask_idx][:,k])
            n = int(in_size/out_size) # Maybe clever way than int to do this
            v = numpy.hstack([numpy.ones(n, dtype=config.floatX),
                              numpy.zeros(max_len_input - n,
                                          dtype=config.floatX)])
            alignment[0,k] = v
            for i in xrange(1, out_size):
                alignment[i,k] = numpy.roll(v, i*n)

            # DEBUG
            #plt.figure()
            #plt.imshow(alignment[:,k,:], cmap='gray', interpolation='none')
            #plt.show()
        data = data + (alignment,)

        return data

class AlignmentPadding(Transformer):
    def __init__(self, data_stream, alignment_source):
        super(AlignmentPadding, self).__init__(data_stream)
        self.alignment_source = alignment_source

    def get_data(self, request=None):
        data = next(self.child_epoch_iterator)
        data = OrderedDict(equizip(self.sources, data))

        alignments = data[self.alignment_source]

        input_lengths = [alignment.shape[1] for alignment in alignments]
        output_lengths = [alignment.shape[0] for alignment in alignments]
        max_input_length = max(input_lengths)
        max_output_length = max(output_lengths)

        batch_size = len(alignments)

        padded_alignments = numpy.zeros((max_output_length, batch_size,
                                         max_input_length))

        for i, alignment in enumerate(alignments):
            out_size, inp_size = alignment.shape
            padded_alignments[:out_size, i, :inp_size] = alignment

        data[self.alignment_source] = padded_alignments

        return data.values()

class Reshape(Transformer):
    """Reshapes data in the stream according to shape source."""
    def __init__(self, data_source, shape_source, **kwargs):
        super(Reshape, self).__init__(**kwargs)
        self.data_source = data_source
        self.shape_source = shape_source
        self.sources = tuple(source for source in self.data_stream.sources
                             if source != shape_source)

    def get_data(self, request=None):
        data = next(self.child_epoch_iterator)
        data = OrderedDict(zip(self.data_stream.sources, data))
        shapes = data.pop(self.shape_source)
        reshaped_data = []
        for dt, shape in zip(data[self.data_source], shapes):
            reshaped_data.append(dt.reshape(shape))
        data[self.data_source] = reshaped_data
        return data.values()

class Subsample(Transformer):
    def __init__(self, data_stream, source, step):
        super(Subsample, self).__init__(data_stream)
        self.source = source
        self.step = step

    def get_data(self, request=None):
        data = next(self.child_epoch_iterator)
        data = OrderedDict(equizip(self.sources, data))
        dt = data[self.source]

        indexes = ((slice(None, None, self.step),) +
                   (slice(None),) * (len(dt.shape) - 1))
        subsampled = dt[indexes]
        data[self.source] = subsampled
        return data.values()

class WindowFeatures(Transformer):
    def __init__(self, data_stream, source, window_size):
        super(WindowFeatures, self).__init__(data_stream)
        self.source = source
        self.window_size = window_size

    def get_data(self, request=None):
        data = next(self.child_epoch_iterator)
        data = OrderedDict(equizip(self.sources, data))
        feature_batch = data[self.source]

        windowed_features = []
        for features in feature_batch:
            features_padded = features.copy()

            features_shifted = [features]
            # shift forward
            for i in xrange(self.window_size / 2):
                feats = numpy.roll(features_padded, i + 1, axis=0)
                feats[:i + 1, :] = 0
                features_shifted.append(feats)
            features_padded = features.copy()

            # shift backward
            for i in xrange(self.window_size / 2):
                feats = numpy.roll(features_padded, -i - 1, axis=0)
                feats[-i - 1:, :] = 0
                features_shifted.append(numpy.roll(features_padded, -i - 1,
                                                   axis=0))
            windowed_features.append(numpy.concatenate(
                features_shifted, axis=1))
        data[self.source] = windowed_features
        return data.values()

class Normalize(Transformer):
    """Normalizes each features : x = (x - means)/stds"""
    def __init__(self, data_stream, means, stds):
        super(Normalize, self).__init__(data_stream)
        self.means = means
        self.stds = stds

    def get_data(self, request=None):
        data = next(self.child_epoch_iterator)
        data = OrderedDict(zip(self.data_stream.sources, data))
        for i in range(len(data['features'])):
            data['features'][i] -= self.means
            data['features'][i] /= self.stds
        return data.values()

def length_getter(dt):
    def get_length(k):
        return dt[k].shape[0]
    return get_length

class SortByLegth(Transformer):
    def __init__(self, data_stream, source='features'):
        super(SortByLegth, self).__init__(data_stream)
        self.source = source

    def get_data(self, request=None):
        data = next(self.child_epoch_iterator)
        data = OrderedDict(zip(self.data_stream.sources, data))
        dt = data[self.source]
        indexes = sorted(range(len(dt)), key=length_getter(dt))
        for source in self.sources:
            data[source] = [data[source][k] for k in indexes]
        return data.values()

class Cut(Transformer):
    """Cuts input to given length, and prepare targets, and cut the features
    as well.

    """
    def __init__(self, data_stream, seq_len):
        super(Cut, self).__init__(data_stream)
        self.seq_len = seq_len
        self.sources = ('features', 'targets')

    def get_data(self, request=None):
        data = next(self.child_epoch_iterator)
        data = OrderedDict(zip(self.data_stream.sources, data))
        for i in range(len(data['features'])):
            data['targets'][i] = data['features'][i, 1:seq_len+1, :]
            data['features'][i] = data['features'][i, :seq_len, :]
        return data.values()


import numpy as np
import torch
import torchvision.transforms as transforms
from torch.utils.data import Dataset

class RTNetDataset(Dataset):
    def __init__(self, face_data, mask_data, target):
        self.transform = transforms.Compose([transforms.ToTensor()])
        self.face = face_data
        self.mask = mask_data
        self.label = target.reshape(-1, 1)

    def __getitem__(self, index):
        if torch.is_tensor(index):
            index = index.tolist()

        appearance_data = torch.tensor(np.transpose(self.face[index], (2, 0, 1)), dtype=torch.float32)
        motion_data = torch.tensor(np.transpose(self.mask[index], (2, 0, 1)), dtype=torch.float32)
        target = torch.tensor(self.label[index], dtype=torch.float32)

        inputs = torch.stack([appearance_data,motion_data],dim=0)

        if torch.cuda.is_available():
            inputs = inputs.to('cuda')
            target = target.to('cuda')

        return inputs, target

    def __len__(self):
        return len(self.label)