from sympy import Mul, Pow, Integer
from compartor.compartments import Moment, Expectation
import itertools

# -------------------------------------------------
def __getMomentPowers(expr):
    """
    Decompose a product of powers of moments to list [(Moment, power)]

    :param expr: product of powers of Moments
    :return: list of tuples (Moment, power)
    """
    if expr.func == Mul:
        moments = [__getMomentPowers(arg) for arg in expr.args]
        return list(itertools.chain(*moments))
    elif expr.func == Pow and expr.args[0].func == Moment and issubclass(expr.args[1].func, Integer):
        return [(expr.args[0], expr.args[1])]
    elif expr.func == Moment:
        return [(expr, 1)]
    elif expr.is_number: #test
        return [] #test
    else:
        raise TypeError("Unexpected expression " + str(expr))

# -------------------------------------------------
def gamma_closure(expr):
    """
    Try to compute gamma closure for expectation of expr.

    :param expr: product of powers of moments
    :return: gamma closure of <expr>
    """

    def __closure1(moment_powers):
        moment = moment_powers[0][0]
        if 3 in moment.args:
            i = moment.args.index(3)
            j = i
        else:
            i = moment.args.index(2)
            j = moment.args.index(1)

        def m(a, b):
            gamma = [0] * len(moment.args)
            gamma[i] += a
            gamma[j] += b
            return Expectation(Moment(*gamma))

        expr = 2 * m(2, 0) * m(1, 1) / m(1, 0) - m(2, 0) * m(0, 1) / m(0, 0)
        return expr

    def __closure2(moment_powers):
        m_xi = moment_powers[0][0]
        if len(moment_powers) == 1:
            m_gamma = m_xi
        else:
            m_gamma = moment_powers[1][0]
        expr = 2 * (Expectation(m_gamma ** 2) * Expectation(m_gamma * m_xi)) / Expectation(m_gamma) - Expectation(
            m_gamma ** 2) * Expectation(m_xi)
        return expr

    # get tuples(M, pow) in expr
    moment_powers = __getMomentPowers(expr)
    # sort by ascending pow
    moment_powers = sorted(moment_powers, key=lambda x: x[1])

    if len(moment_powers) == 1 and moment_powers[0][1] == 1 and moment_powers[0][0].order() == 3:
        return __closure1(moment_powers)
    elif len(moment_powers) == 1 and moment_powers[0][1] == 3:
        return __closure2(moment_powers)
    elif len(moment_powers) == 2 and moment_powers[0][1] == 1 and moment_powers[1][1] == 2:
        return __closure2(moment_powers)
    else:
        raise ValueError("Could not derive Gamma closure for " + str(expr) + " (Not implemented yet).")

# -------------------------------------------------
def gamma_closures(expressions, could_not_close=None):
    """
    Try to compute gamma closure for each of expressions.

    :param expressions: list of moment expressions, where each expression is a product of
            powers of moments
    :param could_not_close: a list that will be filled with expressions that could not be closed.
            If this optional argument is not specified, then a ValueError will be raised if an
            expression cannot be closed.
    :return: list of tuples of (expr, gamma_closure(expr))
    """
    closures = []
    for expr in expressions:
        try:
            closure = gamma_closure(expr)
            closures.append((expr, closure))
        except ValueError:
            if could_not_close is not None:
                could_not_close.append(expr)
            else:
                raise
    return closures

# -------------------------------------------------
def meanfield_closure(expr):
    """
    Compute meanfield closure for expectation of expr.

    :param expr: product of powers of moments
    :return: meanfield closure of <expr>
    """

    def __product(list):
        r = 1
        for x in list:
            r *= x
        return r

    # get tuples(M, pow) in expr
    moment_powers = __getMomentPowers(expr)
    # sort by ascending pow
    moment_powers = sorted(moment_powers, key=lambda x: x[1])

    return  __product([Expectation(moment_power[0])**moment_power[1] for moment_power in moment_powers])

# -------------------------------------------------
def meanfield_closures(expressions):
    """
    Compute meanfield closure for each of expressions.

    :param expressions: list of moment expressions, where each expression is a product of powers of moments
    :return: list of tuples of (expr, meanfield_closure(expr))
    """
    return [(expr, meanfield_closure(expr)) for expr in expressions]

# -------------------------------------------------
def hybrid_closures(expressions):
    """
    Attempt gamma closure for each of expressions.
    For the expressions that could not be closed by gamma closure, do mean-field closure
    """
    could_not_close = []
    gamma = gamma_closures(expressions, could_not_close)
    meanfield = meanfield_closures(could_not_close)
    return gamma+meanfield

# -------------------------------------------------
def substitute_closures(equations, closures):
    """
    Take expectation of rhs in evolutions, and substitute closures.

    :param equations: list of pairs (fM, dfMdt). (as returned by compute_moment_equations)
    :param closures: list of tuples of (expr, closure(expr))
    :return: list of pairs (fM, dfMdt'), where dfMdt' is obtained by substituting closed moments into dfMdt
    """
    substitutions = {Expectation(m): c for m, c in closures}
    # print("Closures: %s" %(substitutions)) #debug
    closed_equations = [(fM, dfMdt.subs(substitutions)) for fM, dfMdt in equations]
    return closed_equations

def apply_substitutions(equations, substitutions):
    """
    Perform raw term substitutions.

    :param equations: list of pairs (fM, dfMdt). (as returned by compute_moment_equations)
    :param substitution: list of tuples of (expr, substitution(expr))
    :return: list of pairs (fM, dfMdt'), where dfMdt' is obtained by substituting closed moments into dfMdt
    """
    substitutions = {m: c for m, c in substitutions}
    # print("Substitutions: %s" %(substitutions)) #debug
    subs_equations = [(fM, dfMdt.subs(substitutions)) for fM, dfMdt in equations]
    return subs_equations

#eof


from django.urls import path
from .views import Homepage, About, Products, Services, Contact

urlpatterns = [
    path("", Homepage, name='home'), # localhost:8000 -> Homepage
    path("about/", About, name='about'),
    path("product/", Products, name='product'),
    path("service/", Services, name='service'),
    path("contact/", Contact, name='contact'),
]


"""
集計特徴量

featuretoolsを利用して作成した集計特徴量を用いてLightGBMで予測を行う。
"""

import argparse
import joblib

from util import run_lgb

def parse_args():
    parser = argparse.ArgumentParser(description='featuretools')
    parser.add_argument('--seed', action='store', type=int, default=1234)
    parser.add_argument('--objective', action='store', type=str, default='binary')
    parser.add_argument('--n_estimators', action='store', type=int, default=10000)
    parser.add_argument('--early_stopping', action='store', type=int, default=100)
    parser.add_argument('--device', action='store', type=str, default='gpu')
    return parser.parse_args()

def main():
    args = parse_args()
    feature_matrix = joblib.load('../data/02_featuretools/feature_matrix.joblib')
    app_train = feature_matrix.dropna(subset=['TARGET'])
    app_test = feature_matrix[feature_matrix['TARGET'].isnull()].drop('TARGET', axis=1)
    run_lgb(args, app_train, app_test, '12_featuretools')

if __name__ == "__main__":
    main()


from typing import Tuple
import csv
import regex

def read_ngs_references(path: str) -> dict:
    """Function reads the csv file containing the reference sequences with their
    respective names and returns a dictionary with the sequence coupled to the name."""
    ngs_references = {}
    with open(path, newline='') as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:
            ngs_references[row["sequence"]] = row["name"]
    return ngs_references

def complement_reverse_sequence(sequence: str) -> str:
    """Function creates a complementary sequence and reverses the sequence."""
    complement = {"A": "T", "T": "A", "C": "G", "G": "C"}
    # Create a list containing the complement bases
    complement_seq = [complement[base] for base in sequence]
    # Reverse the list
    complement_seq = complement_seq[::-1]
    complement_seq = "".join(complement_seq)
    return complement_seq

def clean_ngs_reference_sequences(ngs_references_dict: dict, prefix_info: dict, suffix_info: dict) -> dict:
    """Function goes over every reference sequence in a dict. Finds the
    complement for that sequence and reverses it. Then it cleans this sequences
    by removing the prefix and suffix. The cleaned sequence is then stored in a dictionary,
    where the sequence is the key and the value is the name of the reference sequence.\n
    \n
    args:\n
    ngs_references_dict: (dict) a dictionary where the key is a reference sequence and the value is the name of this sequence.\n
    suffix: (str) sequence of the suffix which should be removed.\n
    \n
    return values:\n
    cleaned_ngs_references: (dict) a dictionary where the key is a cleaned reference sequence and the value is the name of this sequence.
    """
    cleaned_ngs_references = {}
    for reference_seq in ngs_references_dict:
        reversed_seq = complement_reverse_sequence(reference_seq)
        prefix_seq, _ = determine_pattern(reversed_seq, prefix_info)
        suffix_seq, _ = determine_pattern(reversed_seq, suffix_info)
        cleaned_seq = clean_sequence(reversed_seq, prefix_seq, suffix_seq)
        cleaned_ngs_references[cleaned_seq] = ngs_references_dict[reference_seq]
    return cleaned_ngs_references

def create_ngs_references_patterns(ngs_references_dict: dict) -> dict:
    """Function compiles every sequence in the dictionary into a regex object.
    This is done so that during the database insertion, no compilation has to been done
    every time we want to check for a reference sequence."""
    patterns_dict = {}
    for seq in ngs_references_dict:
        patterns_dict[regex.compile(
            fr"(?e)({seq}){{e<=1}}")] = ngs_references_dict[seq]
    return patterns_dict

def reference_seq(sequence: str, ngs_references_dict: dict):
    """Function returns the name of the reference sequence or the string 'NULL'"""
    for reference_seq in ngs_references_dict.keys():
        match = reference_seq.search(sequence)
        if match:
            # True
            return ngs_references_dict[reference_seq]
    # False
    return None

def determine_pattern(sequence: str, patterns_info: dict) -> Tuple[str, str, int]:
    """Function loops over all the patterns in patterns_info. The patterns_info should either contain the sequences of the prefixes
    or suffixes. While looping over the patterns it checks whether there is a match of this pattern in the sequence.
    If this is the case, it returns the matched sequence and the name of the pattern that matched.\n
    args:\n
    sequence: (str) DNA sequence in which you want to check for a pattern.\n
    patterns_info: (dict) key should be a compiled regex pattern and the value should be the name of the regex pattern. Example:\n
    {regex.compile(r"(?e)(ACAAAACAAAAC){e<=1}"): "Z", regex.compile(r"(?e)(AAACAAACAAA){e<=1}"): "W", regex.compile(r"(?e)(CTTTTCCGTATATCTCGCCAG){e<=1}"): "A"}\n
    \n
    return values:\n
    pattern_seq: (str) the matched sequence to the pattern, the regex pattern can allow for errors, so this is the actual matched sequence.\n
    pattern_name: (str) name of the pattern.
    """
    for pattern in patterns_info:
        pattern_match = pattern.search(sequence)
        if bool(pattern_match):
            pattern_name = patterns_info[pattern]
            pattern_seq = pattern_match.group()
            mutated_pattern = 0
            # # If there is a substitution/insertion/deletion, the pattern is "mutated"
            # if sum(pattern_match.fuzzy_counts) > 0:
            #     mutated_pattern = 1
            return pattern_seq, pattern_name  # , mutated_pattern
    return None, None  # , None

def determine_clvd_prefix(prefix_name, clvd_prefix_name="Z", unclvd_prefix_name="W"):
    """Function returns whether a sequence matches with a cleaved or uncleaved prefix sequences.\n
    args:\n
    prefix_name: (str) name of the prefix.\n
    clvd_prefix_name: (str) name of the prefix that indicates cleavage.\n
    unclvd_prefix_name: (str) name of the prefix that indicates no cleavage took place.\n
    \n
    return values:\n
    clvd_prefix: (int) indicates whether the prefix indicates cleavage. Yes(1)/No(0)/Don't know(2)"""
    if prefix_name is None:
        clvd_prefix = 2
    elif prefix_name.lower() == clvd_prefix_name.lower():
        clvd_prefix = 1
    elif prefix_name.lower() == unclvd_prefix_name.lower():
        clvd_prefix = 0
    else:
        clvd_prefix = 2
    return clvd_prefix

def retrieve_barcode(sequence: str, prefix: str) -> str:
    if prefix is not None:
        prefix_match = regex.search(prefix, sequence)
        return sequence[:prefix_match.start()]
    else:
        return None

def clean_sequence(sequence: str, prefix: str, suffix: str) -> str:
    if prefix is not None and suffix is not None:
        prefix_match = regex.search(prefix, sequence)
        suffix_match = regex.search(suffix, sequence)
        if not bool(suffix_match):
            return None
        return sequence[prefix_match.end():suffix_match.start()]
    return None


import json

def configure():
    try:
        f = open("cemubot/misc/config.cfg", "x+", encoding="utf-8")
    except FileExistsError:
        f = open("cemubot/misc/config.cfg", "w", encoding="utf-8")
    config = {}
    config["bot_token"] = None
    while not config["bot_token"]:
        config["bot_token"] = input("Please provide a bot token: ")
    config["command_prefix"] = input("Please provide a command prefix (e.g. ^ for ^help) ")
    if not config["command_prefix"]:
        print("No prefix provided; defaulting to ^.")
        config["command_prefix"] = "^"
    print("Please provide the channel ID of the channel(s) you want the bot to parse logs in.")
    print("NOTE: A channel ID is something like 123456789123456789, not #channel-name.")
    config["parsing_channel"] = {}
    config["parsing_channel"]["preferred"] = int(input("Preferred channel ID goes here (0 to parse in all channels): "))
    config["parsing_channel"]["alternates"] = list(map(int, input("Comma-separated list of IDs goes here (leave blank to skip): ").split(',')))
    config["compatibility_colors"] = {
        "perfect": 0x3380CC,
        "playable": 0x16A689,
        "runs": 0xD9D936,
        "loads": 0xDF8D12,
        "unplayable": 0xBF3E32,
        "unknown": 0x858585
    }
    print(f"Your config file now looks like this:\n{json.dumps(config, indent=2)}")
    if input("Is this correct? (y/n) ").lower() == 'n':
        f.close()
        configure()
    else:
        json.dump(config, f, indent=4)
        print("Configuration complete. If the bot still doesn't work, try running setup.py again.")
        f.close()

try:
    open("cemubot/misc/config.cfg", "r", encoding="utf-8").close()
    if input("config.cfg already exists; would you like to re-configure the bot? (y/n) ").lower() == 'y':
        configure()
except FileNotFoundError:
    if input("config.cfg not found; would you like to configure the bot? (y/n) ").lower() == 'y':
        configure()


"""
multi_run does multiple runs with the models
OBSELETE 3/2020
"""
from __future__ import print_function

import os.path
import model_run as MR
import pprint
import json

pp = pprint.PrettyPrinter(indent=4, width=60)

# create an instance of the modelrunner for each cell configuration

# population the configuration parameters

# run the model with the configuration. Data is automatically stored
# set the "auto initialize" flag.

"""

Define run parameters with a dictionary:
common parameters: all keys correspond to the .Params of the model instance.
common = {'FO': F0, 'fmod': fmod, 'dmod': dmod, 'nrep': nrep, 'SRType': SR,
    'run_duration': dur, 'dB': level, 'type': celltype}

For each run, the following values are changed:
{'runID': {'cellID': name, 'inputPattern': inputpattername}}

Parameters:
        self.cellChoices = ['Bushy', 'TStellate', 'DStellate']
        self.modelChoices = ['XM13', 'RM03', 'XM13PasDend', 'Calyx', 'MNTB', 'L23Pyr']
        self.cmmrModeChoices = ['CM', 'CD', 'REF']  # comodulated, codeviant, reference
        self.SRChoices = ['LS', 'MS', 'HS', 'fromcell']  # AN SR groups (assigned across all inputs)
        self.protocolChoices = ['initIV', 'testIV', 'runIV', 'initAN', 'runANPSTH', 'runANSingles']
        self.soundChoices = ['tone', 'noise', 'stationaryNoise', 'SAM', 'CMMR']
        self.speciesChoices = ['mouse', 'guineapig']

        self.srname = ['**', 'LS', 'MS', 'HS']  # runs 1-3, not starting at 0
        self.cellID = None  # ID of cell (string, corresponds to directory name under VCN_Cells)
        self.Params = OrderedDict()

        self.Params['initIVStateFile'] = 'IVneuronState.dat'
        self.Params['initANStateFile'] = 'ANneuronState.dat'
        self.Params['infile ']= None

        self.Params['cellType'] = self.cellChoices[0]
        self.Params['model_type'] = self.modelChoices[0]
        self.Params['species'] = self.speciesChoices[0]
        self.Params['SRType'] = self.SRChoices[2]
        self.Params['SR'] = self.Params['SRType']  # actually used SR this might be cell-defined, rather than command defined
        self.Params['inputPattern'] = None # ID of cellinput pattern (same as cellID): for substitute input patterns.

        self.Params['runProtocol'] = self.protocolChoices[2]  # testIV is default because it is fast and should be run often

        self.Params['run_duration'] = 0.8 # in sec
        self.Params['soundtype'] = 'SAM'  # or 'tonepip'
        self.Params['pip_duration'] = 0.1
        self.Params['pip_start'] = [0.1]
        self.Params['Fs'] = 100e3
        self.Params['F0'] = 4000.
        self.Params['dB'] = 40.
        self.Params['RF'] = 2.5e-3
        self.Params['fmod'] = 20 # hz, modulation if SAM
        self.Params['dmod'] = 0 # percent if SAM
        # spontaneous rate (in spikes/s) of the fiber BEFORE refractory effects; "1" = Low; "2" = Medium; "3" = High
        self.Params['threshold'] = -20
        self.Params['plotFlag'] = False
        self.Params['auto_initialize'] = False

"""
common = {'F0': 4000., 'fmod': 20., 'dmod': 0., 'nReps': 2, 'SRType': 'MS',
    'run_duration': 2.25, 'dB': 40., 'cellType': 'Bushy', 'pip_duration': 2.0}

# For each run, the following values are changed:

runs = {'selfinputs': {'cellID': 'VCN_c18', 'inputPattern': 'VCN_c18'}}

for runid in runs.keys():
    i1 = MR.ModelRun()
    parkeys = i1.Params.keys()
    for p in common.keys():
        if p not in parkeys:
            print('parameter %s not in parkeys list' % p)
            exit()
        else:
            i1.Params[p] = common[p] # set the parameters
        # force a couple of things
    i1.Params['initANStateFile'] = 'initAN_hoc_' + runs[runid]['cellID']+'_input_' + runs[runid]['inputPattern']
    i1.Params['inputPattern'] = runs[runid]['inputPattern']
    i1.Params['cell'] = runs[runid]['cellID']
    i1.Params['infile'] = runs[runid]['cellID'] + '.hoc'

    print(json.dumps(i1.Params, indent=4))

    stateFileExists = i1.check_for_an_statefile()

    if not stateFileExists:
        i1.Params['runProtocol'] = 'initAN'
        i1.run_model()
    i1.Params['runProtocol'] = 'runANPSTH'
    i1.run_model()


from copy import deepcopy

import numpy as np
import torch
import torch.autograd

from ppo_pytorch.algs.impala import IMPALA, AttrDict, copy_state_dict

class GuidedES(IMPALA):
    def __init__(self, *args,
                 grad_buffer_len=32,
                 steps_per_update=1,
                 es_lr=0.01,
                 es_std=0.01,
                 es_blend=0.0,
                 **kwargs):
        super().__init__(*args, **kwargs)
        assert self.num_actors == 1
        assert steps_per_update == 1
        self.grad_buffer_len = grad_buffer_len
        self.steps_per_update = steps_per_update
        self.es_lr = es_lr
        self.es_std = es_std
        self.es_blend = es_blend

        # (k, n) grad history matrix
        self._grad_buffer = None
        self._grad_buffer_index = 0
        # up to 2 * P weight sets
        self._weights_to_test = []
        # up to 2 * P rewards
        self._es_rewards = []
        # (P, n) noise matrix
        self._noise = None
        self._orig_model_weights = deepcopy(list(self._train_model.state_dict().values()))

        # self.value_loss_scale = 0

    def _step(self, rewards, dones, states) -> torch.Tensor:
        reward = rewards[0] if rewards is not None else 0
        done = dones[0] if dones is not None else False

        actions = super()._step(rewards, dones, states)

        if len(self._es_rewards) != 0:
            # ent = self._eval_model.heads.logits.pd.entropy(torch.tensor(self._steps_processor.data.logits[-1])).item() if len(self._steps_processor.data.logits) != 0 else 0
            self._es_rewards[-1] += reward * self.reward_scale #+ self.entropy_reward_scale * ent

        if self._grad_buffer is not None and done:
            if len(self._weights_to_test) == 0:
                if len(self._es_rewards) != 0:
                    self._es_update()
                    self._es_rewards.clear()
                self._generate_weights_to_test()
            self._es_rewards.append(0)
            self._set_next_model_weights()

        return actions

    def _es_update(self):
        fitness = torch.tensor(self._es_rewards).view(self.steps_per_update, 2)
        fitness = fitness[:, 0] - fitness[:, 1]
        # (n) vector
        grad = self.es_lr / (2 * self.es_std * self.steps_per_update) * (fitness @ self._noise)
        # print('es grad', grad.pow(2).mean().sqrt(), fitness)
        weights = list(self._train_model.state_dict().values())
        w_lens = [w.numel() for w in weights]
        for curw, g, origw in zip(weights, grad.split(w_lens, 0), self._orig_model_weights):
            origw += g.view_as(origw)
            curw.copy_(origw)
        copy_state_dict(self._train_model, self._eval_model)

        if self._do_log:
            self.logger.add_scalar('es grad rms', grad.pow(2).mean().sqrt(), self.frame_train)
            self.logger.add_scalar('es buffer rms', self._grad_buffer.pow(2).mean().sqrt(), self.frame_train)
            self.logger.add_scalar('es fitness rms', fitness.pow(2).mean().sqrt(), self.frame_train)

    def _create_data(self):
        data = super()._create_data()
        data.state_values.fill_(0)
        return data

    def _impala_update(self, data: AttrDict):
        cur_weights = deepcopy(list(self._train_model.state_dict().values()))
        for src, dst in zip(self._orig_model_weights, self._train_model.state_dict().values()):
            dst.copy_(src)
        super()._impala_update(data)
        grads = []
        for prev, new in zip(self._orig_model_weights, self._train_model.state_dict().values()):
            grads.append(new - prev)
            new.copy_(prev)
        for src, dst in zip(cur_weights, self._train_model.state_dict().values()):
            dst.copy_(src)
        self._update_grad_buffer(grads)

    def _generate_weights_to_test(self):
        assert len(self._weights_to_test) == 0

        weights = list(self._orig_model_weights)
        w_sizes = [w.numel() for w in weights]

        self._noise = self._get_noise()

        for noises in zip(*self._noise.split(w_sizes, dim=1)):
            self._weights_to_test.append([w + e.view_as(w) for (e, w) in zip(noises, weights)])
            self._weights_to_test.append([w - e.view_as(w) for (e, w) in zip(noises, weights)])

    def _get_noise(self):
        """Generates (P, n) noise matrix"""
        buf_len, grad_len = self._grad_buffer.shape
        full_space_noise = self._grad_buffer.new_zeros(self.steps_per_update, grad_len).normal_()
        subspace_noise = self._grad_buffer.new_zeros(self.steps_per_update, buf_len).normal_() @ self._grad_buffer
        subspace_noise /= subspace_noise.pow(2).mean().sqrt()
        full_space_noise *= self.es_std * self.es_blend
        subspace_noise *= self.es_std * (1 - self.es_blend)
        # print('noises full', full_space_noise.pow(2).mean().sqrt(), 'ss', subspace_noise.pow(2).mean().sqrt())
        noise = full_space_noise + subspace_noise
        return noise

    def _set_next_model_weights(self):
        weights = self._weights_to_test.pop(0)
        for src, dst in zip(weights, self._train_model.state_dict().values()):
            dst.copy_(src)
        copy_state_dict(self._train_model, self._eval_model)

    def _update_grad_buffer(self, new_grads):
        new_grads = torch.cat([g.view(-1) for g in new_grads], dim=0)
        # print('algs grad', new_grads.pow(2).mean().sqrt())
        if self._grad_buffer is None:
            self._grad_buffer = new_grads.new_zeros((self.grad_buffer_len, new_grads.numel()))
        self._grad_buffer[self._grad_buffer_index] = new_grads
        self._grad_buffer_index += 1
        if self._grad_buffer_index >= self.grad_buffer_len:
            self._grad_buffer_index = 0


import discord
import utils as ut
from random import choices

initial_betting = 1

class Hand():
    def __init__(self, player: discord.Member | discord.User | str):
        self.player = player
        self.hand = ut.Alist()
        self.bet = initial_betting  # The amount that the player betted in this hand
        self.total = 0

    def eval_hand(self):
        self.total = 0

        for card in self.hand[:]:
            if card.value is None:
                continue
            if card.value == 11:
                if self.total + 11 > 21:
                    self.total += 1
                    continue
            self.total += card.value

    def canDouble(self):
        return len(self.hand) <= 2

    def canSplit(self):
        return len(self.hand) == 2 and self.hand[0].value == self.hand[1].value

    def draw_cards(self, deck: list, amount=1):
        cards = choices(deck, k=amount)
        self.hand.extend(cards)
        for card in cards:
            deck.remove(card)
        self.eval_hand()

    async def show_hand(self, channel: discord.TextChannel, show_options=False):
        name = self.player if isinstance(
            self.player, str) else self.player.name
        cards_img = ut.generateCards(self.hand)

        msg = discord.Embed(title=f'**{name}\'s Hand:**',  colour=discord.Colour.dark_green(),
                            description=f'``Total = {self.total}``')
        msg.set_footer(text='Made By: Victório Maculan')

        await channel.send(embed=msg)
        await channel.send(file=ut.pil2discord(cards_img))

    async def show_turn(self, channel: discord.TextChannel, hand_num=1):
        name = self.player if isinstance(
            self.player, str) else self.player.name
        cards_img = ut.generateCards(self.hand)
        msg = discord.Embed(title=f'**{name}\'s Hand N°{hand_num}:**',  colour=discord.Colour.dark_green(),
                            description=f'''``Total = {self.total}``

                                            **Your options:**
                                            🎯 -> Hit
                                            🛑 -> Stand''')

        if self.canDouble():
            msg.description += '\n 💸 -> Double Down'
        if self.canSplit():
            msg.description += '\n ✂ -> Split'
        msg.set_footer(text='You have 30 seconds to choose what to do!')

        menu = await channel.send(embed=msg)
        await channel.send(file=ut.pil2discord(cards_img))

        await menu.add_reaction('🎯')
        await menu.add_reaction('🛑')
        if self.canDouble():
            await menu.add_reaction('💸')
        if self.canSplit():
            await menu.add_reaction('✂')


import numpy as np
import torch
from torch.utils.data.sampler import Sampler
import copy

def ActualSequentialLoader(subsetRandomLoader, indices=None, transform=None, batch_size=None):
    indices = indices if indices is not None else subsetRandomLoader.sampler.indices
    train_sampler = ActualSequentialSampler(indices)
    dataset = copy.deepcopy(subsetRandomLoader.dataset)
    if transform is not None:
        dataset.transform = transform

    batch_size = batch_size if batch_size is not None else subsetRandomLoader.batch_size
    actualSequentialLoader = torch.utils.data.DataLoader(dataset, sampler=train_sampler,
                                                         num_workers=subsetRandomLoader.num_workers,
                                                         batch_size=batch_size, drop_last=False)
    return actualSequentialLoader

class ActualSequentialSampler(Sampler):
    r"""Samples elements sequentially, always in the same order.

    Arguments:
        data_source (Dataset): dataset to sample from
    """

    def __init__(self, data_source):
        self.data_source = data_source

    def __iter__(self):
        return iter(self.data_source)

    def __len__(self):
        return len(self.data_source)

def row_norms(X, squared=False):
    """Row-wise (squared) Euclidean norm of X.
    Equivalent to np.sqrt((X * X).sum(axis=1)), but also supports sparse
    matrices and does not create an X.shape-sized temporary.
    Performs no input validation.
    Parameters
    ----------
    X : array_like
        The input array
    squared : bool, optional (default = False)
        If True, return squared norms.
    Returns
    -------
    array_like
        The row-wise (squared) Euclidean norm of X.
    """
    norms = np.einsum('ij,ij->i', X, X)

    if not squared:
        np.sqrt(norms, norms)
    return norms

def outer_product_opt(c1, d1, c2, d2):
    """Computes euclidean distance between a1xb1 and a2xb2 without evaluating / storing cross products
    """
    B1, B2 = c1.shape[0], c2.shape[0]
    t1 = np.matmul(np.matmul(c1[:, None, :], c1[:, None, :].swapaxes(2, 1)), np.matmul(d1[:, None, :], d1[:, None, :].swapaxes(2, 1)))
    t2 = np.matmul(np.matmul(c2[:, None, :], c2[:, None, :].swapaxes(2, 1)), np.matmul(d2[:, None, :], d2[:, None, :].swapaxes(2, 1)))
    t3 = np.matmul(c1, c2.T) * np.matmul(d1, d2.T)
    t1 = t1.reshape(B1, 1).repeat(B2, axis=1)
    t2 = t2.reshape(1, B2).repeat(B1, axis=0)
    return t1 + t2 - 2*t3

def kmeans_plus_plus_opt(X1, X2, n_clusters, init=[0], random_state=np.random.RandomState(1234), n_local_trials=None):
    """Init n_clusters seeds according to k-means++ (adapted from scikit-learn source code)
    Parameters
    ----------
    X1, X2 : array or sparse matrix
        The data to pick seeds for. To avoid memory copy, the input data
        should be double precision (dtype=np.float64).
    n_clusters : integer
        The number of seeds to choose
    init : list
        List of points already picked
    random_state : int, RandomState instance
        The generator used to initialize the centers. Use an int to make the
        randomness deterministic.
        See :term:`Glossary <random_state>`.
    n_local_trials : integer, optional
        The number of seeding trials for each center (except the first),
        of which the one reducing inertia the most is greedily chosen.
        Set to None to make the number of trials depend logarithmically
        on the number of seeds (2+log(k)); this is the default.
    Notes
    -----
    Selects initial cluster centers for k-mean clustering in a smart way
    to speed up convergence. see: Arthur, D. and Vassilvitskii, S.
    "k-means++: the advantages of careful seeding". ACM-SIAM symposium
    on Discrete algorithms. 2007
    Version ported from http://www.stanford.edu/~darthur/kMeansppTest.zip,
    which is the implementation used in the aforementioned paper.
    """

    n_samples, n_feat1 = X1.shape
    _, n_feat2 = X2.shape
    # x_squared_norms = row_norms(X, squared=True)
    centers1 = np.empty((n_clusters+len(init)-1, n_feat1), dtype=X1.dtype)
    centers2 = np.empty((n_clusters+len(init)-1, n_feat2), dtype=X1.dtype)

    idxs = np.empty((n_clusters+len(init)-1,), dtype=np.long)

    # Set the number of local seeding trials if none is given
    if n_local_trials is None:
        # This is what Arthur/Vassilvitskii tried, but did not report
        # specific results for other than mentioning in the conclusion
        # that it helped.
        n_local_trials = 2 + int(np.log(n_clusters))

    # Pick first center randomly
    center_id = init

    centers1[:len(init)] = X1[center_id]
    centers2[:len(init)] = X2[center_id]
    idxs[:len(init)] = center_id

    # Initialize list of closest distances and calculate current potential
    distance_to_candidates = outer_product_opt(centers1[:len(init)], centers2[:len(init)], X1, X2).reshape(len(init), -1)

    candidates_pot = distance_to_candidates.sum(axis=1)
    best_candidate = np.argmin(candidates_pot)
    current_pot = candidates_pot[best_candidate]
    closest_dist_sq = distance_to_candidates[best_candidate]

    # Pick the remaining n_clusters-1 points
    for c in range(len(init), len(init)+n_clusters-1):
        # Choose center candidates by sampling with probability proportional
        # to the squared distance to the closest existing center
        rand_vals = random_state.random_sample(n_local_trials) * current_pot
        candidate_ids = np.searchsorted(closest_dist_sq.cumsum(),
                                        rand_vals)
        # XXX: numerical imprecision can result in a candidate_id out of range
        np.clip(candidate_ids, None, closest_dist_sq.size - 1,
                out=candidate_ids)

        # Compute distances to center candidates
        distance_to_candidates = outer_product_opt(X1[candidate_ids], X2[candidate_ids], X1, X2).reshape(len(candidate_ids), -1)

        # update closest distances squared and potential for each candidate
        np.minimum(closest_dist_sq, distance_to_candidates,
                   out=distance_to_candidates)
        candidates_pot = distance_to_candidates.sum(axis=1)

        # Decide which candidate is the best
        best_candidate = np.argmin(candidates_pot)
        current_pot = candidates_pot[best_candidate]
        closest_dist_sq = distance_to_candidates[best_candidate]
        best_candidate = candidate_ids[best_candidate]

        idxs[c] = best_candidate

    return None, idxs[len(init)-1:]

def get_embedding(model, loader, device, num_classes, cfg, with_emb=False, emb_dim=512):
    model.eval()
    embedding = torch.zeros([len(loader.sampler), num_classes])
    embedding_pen = torch.zeros([len(loader.sampler), emb_dim])
    labels = torch.zeros(len(loader.sampler))
    preds = torch.zeros(len(loader.sampler))
    batch_sz = cfg.DATALOADER.BATCH_SIZE
    with torch.no_grad():
        for batch_idx, (data, target, _) in enumerate(loader):
            data, target = data.to(device), target.to(device)
            if with_emb:
                e1, e2 = model(data, with_emb=True)
                embedding_pen[batch_idx*batch_sz:batch_idx*batch_sz + min(batch_sz, e2.shape[0]), :] = e2.cpu()
            else:
                e1 = model(data, with_emb=False)

            embedding[batch_idx*batch_sz:batch_idx*batch_sz + min(batch_sz, e1.shape[0]), :] = e1.cpu()
            labels[batch_idx*batch_sz:batch_idx*batch_sz + min(batch_sz, e1.shape[0])] = target
            preds[batch_idx*batch_sz:batch_idx*batch_sz + min(batch_sz, e1.shape[0])] = e1.argmax(dim=1, keepdim=True).squeeze()

    return embedding, labels, preds, embedding_pen


"""
Requerimientos:

Crea un programa que lea una ip y determine su clase según su rango, la información que debe mostrar debe ser la siguiente:

°Clase de ip (clase a, b o c)
°Cuantos host pueden haber en esa subred

Ej: en una clase c pueden haber 254 host por cada red, por lo tanto imprimiría 254 por la red actual)

Ejemplo de la salida del programa:

"Tu ip es 192.168.0.1 es de clase C y pueden haber 254 host en tu red actual"

code:prog_b7_tuuser.py

Datos para elegir el tipo de IP

Class        Networks                     mask          CDIR       leading bits
A       0.0.0.0   - 127.255.255.255     255.0.0.0         /8        0
B       128.0.0.0 - 191.255.255.255     255.255.0.0       /16       10
C       192.0.0.0 - 223.255.255.255     255.255.0.0       /24       110
D       224.0.0.0 - 239.255.255.255         not defined             1110        Multicast
E       240.0.0.0 - 247.255.255.255         not defined             1111        Reserved

información procedente de:
https://en.wikipedia.org/wiki/Classful_network

LIBRERIAS ADICIONALES

"""
import ipaddress
import sys

# Interactua con el usuario para pedir la IP
def pedir_ip ():
    ip_s = input  ("Introduce una IP en notación clásica o CIDR: ")
    return ip_s

def comprobar_tipos (ipt):
    if ipt.is_multicast:
        print (f'La ip {ipt} es multicast.')
    if ipt.is_private:
        print (f'La ip {ipt} es private.')
    if ipt.is_global:
        print (f'La ip {ipt} es global.')
    if ipt.is_unspecified:
        print (f'La ip {ipt} es sin especificar.')
    if ipt.is_reserved:
        print (f'La ip {ipt} es reservada.')
    if ipt.is_loopback:
        print (f'La ip {ipt} es loopback.')
    if ipt.is_link_local:
        print (f'La ip {ipt} es link local.')

def es_ip (ip_s):
    """

    """
    ip      = None
    network = None
    clase   = None
    tipo    = None
    # tipo de ip 1-> notación clásica, 2-> CIDR
    dir_ok  = False

    try:
        ip = ipaddress.IPv4Address(ip_s)
        # si pasa la IP con notación clásica es valida')
        dir_ok = True
        tipo = 1
    except:
        # si no pasa la IP no es valida
        dir_ok = False

    if not dir_ok:
        try:
            network = ipaddress.IPv4Network(ip_s, strict=False)
            # si pasa es una IP con notación CIDR
            ip, _ = ip_s.split('/')
            ip = ipaddress.IPv4Address(ip)
            dir_ok = True
            tipo = 2    # Es notación CIDR, ya hemos guardo IP y network
        except:
            # si no pasa no es ningún
            dir_ok = False

    if tipo == 1:   # es notación clasica, vemos de que clase sería.
        a, b, c, d = ip_s.split('.')
        a, b, c, d = int(a), int (b), int (c), int (d)

        if a < 128:
            clase = "A"
            full_ip = ip_s +'/8'
        elif a < 192:
            clase = 'B'
            full_ip = ip_s +'/16'
        elif a < 224:
            clase = 'C'
            full_ip = ip_s +'/24'
        elif a < 240:
            clase = 'D'
            full_ip = None
        else:
            clase = 'E'
            full_ip = None

        try:
            # generamos la variable network para los calculos de prefijo, mascara ...
            network = ipaddress.IPv4Network(full_ip, strict=False)
        except:
            network = None

    # devolvera ('None', 'None', 'None', 'None') si lo introducido no es una IP valida
    # en otro caso devolera los valores correctos, de IP, network, clase y tipo
    return (ip, network, clase, tipo)

def imprimir (data):

    print (f"Dirección ip: {data[0]}")
    if data[2]:
        print (f'Es una red de clase {data[2]}.')
    else:
        # data[1] -> network, sacamos el prefijo de la red, para luego calcular los hosts
        prefijo = data[1].prefixlen
        print ('Has introducido la dirección IP en formato CIDR.')
        # Aunque se ha usado notación CIDR, usamos el prefijo para saber que clase de red sería
        if prefijo == 24:
            print ('Que por la máscara de red podría ser de clase C')
        elif prefijo == 16:
            print ('Que por la máscara de red podría ser de clase B')
        elif prefijo == 8:
            print ('Que por la máscara de red podría ser de clase A')

    if data[1]:
        print (f'La máscara de red es: {str(data[1].netmask)}')
        # para calcular los host (2^(32-prefijo))-2
        # 32-prefijo -> número de 0 que hay en la máscara
        # -2, 1 por la IP de net y 1 por la IP de broadcast
        prefijo = data[1].prefixlen
        hosts = (2**(32-prefijo))-2 # 1 por broadcast y otro por dirección de red
        print (f'En esta red puede haber {hosts} hosts')
    else:
        print ('Es una red reservada, por tanto el numero de hosts es "not defined"')
        # Esto es por definición de las redes de clase D y E

    comprobar_tipos(data[0])

def main_chequear_IP ():
    """

    """

    data = (None, None, None, None)

    while data[0] == None:
        ip = pedir_ip ()
        data = es_ip (ip)

    imprimir (data)

# programa PRINCIPAL
if __name__ == "__main__":
    arguments_count=len(sys.argv)

    print ("________________________________________________________________________________")
    print ('Ejercicio b7. Para una dirección IP mostrar clase y número de hosts que admite.')
    print ('@jabaselga')
    print (f'Para ejecutar el comando: {sys.argv[0]} [IP|IP/mask|IP/prefix]')
    print ('Ejemplo de notación clásica: 192.168.4.2')
    print ('Ejemplo de notación CIDR: 192.168.4.2/23 o 192.168.5.2/255.255.240.0')
    print ("________________________________________________________________________________")

    if arguments_count > 2:
        print ('Para ejecutar el comando:')
        print (f'{sys.argv[0]} [IP|IP/mask|IP/prefix]')

    elif arguments_count == 2:
        data = es_ip (sys.argv[1])
        if data[0] != None:
            imprimir(data)
        else:
            print (f'El argumento "{sys.argv[1]}" introducido no es una dirección IP válida')
    else:
        main_chequear_IP()


import os
import sys
import time
import re
import json
from urllib.parse import urlparse
from collections import OrderedDict
import requests
from PyQt5.QtWidgets import QApplication,QWidget,QVBoxLayout,QHBoxLayout,QTabWidget,QPushButton,QTextEdit,QPlainTextEdit,QLineEdit,QLabel,QComboBox,QListWidget,QListWidgetItem
from PyQt5.QtWebEngineWidgets import QWebEngineView
from PyQt5.QtCore import QUrl,QThread,pyqtSignal
from PyQt5.QtGui import QIcon,QFont,QFontDatabase

# 全局异常捕获
def printErrors(exc_type, exc_value, traceback):
    sys.stderr.write(traceback)
sys.excepthook = printErrors

from lib.sqlite import ReqLogDB
reqLogDB = ReqLogDB()

# 超时
# 请求详细信息
# 支持多方法
# 支持动态body/query
# TODO 支持Headers
# 编码问题
# 美化、字体
# TODO 增加记录
# TODO 增加侧边栏
# TODO debug 报错
# 成功打包icon和字体
# TODO 增加菜单
# 增加查看快捷键
# url和query联动
# TODO 尾部状态栏
# TODO Request标签页 自动切换

# 全局变量

dataPath = 'data.pkl'

logs = reqLogDB.selectAll()

def jsonPretty(jstr):
    return json.dump(json.loads(jstr), indent=2)

class RequestThread(QThread):
    finishSignal = pyqtSignal(dict)

    def __init__(self, window):
        super().__init__()
        self._window = window
        self.headers = {
            'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
        }
    def __request(self, options):
        response = requests.request(options['method'], options['url'], headers=options['headers'], data=window.body, params=window.query)
        return response
    def run(self):
        start = time.time()
        url = self._window.reqUrlInput.text()
        text = ''
        try:
            print('Request Sending:', url)
            method = window.reqMethodCombo.currentText()
            print(method)
            response = self.__request({
                'url': url,
                'method': method,
                'headers': self.headers
            })
            charsetPatt = re.compile('charset=["\']{0,1}([A-Za-z0-9\-]+)["\']', re.IGNORECASE)
            matches = charsetPatt.search(str(response.text))
            if matches :
                response.encoding = matches.group(1)
            text = response.text
            print(response.__getattribute__('encoding'))
            print(response.status_code)
            print(response.headers)
            print(response.cookies)
            stats = 'Status: Success \n'
            stats += '{}: {}\n'.format('Code', response.status_code)
            stats += '{}: {:.3f}s\n'.format('ResponseTime', time.time() - start)
            stats += '{}: {}\n'.format('Encoding', response.encoding)
            stats += '{}: {}\n'.format('Headers', json.dumps(dict(response.headers), indent=2))
            print('Request Success:', response.url)
        except Exception as e:
            #print('Request Failed:', e)
            stats = 'Status: Failed \n' + 'Error: ' + str(e)
        print('请求耗时：', time.time() - start)
        try :
            text = json.dumps(json.loads(text), ensure_ascii=False)
        except:
            pass
        sigData = {
            'url': url,
            'text': text,
            'stats': stats
        }
        self.finishSignal.emit(sigData)

def formatParamParse(paramText):
    param = OrderedDict()
    paramLines = str.split(paramText, '\n')
    for line in paramLines:
        items = str.split(line)
        if len(items) == 2:
            param[str(items[0])] = str(items[1])
    return param

def formatParamStringify(param):
    paramFormat = ''
    for k in param:
        paramFormat += '{} {}\n'.format(k, param[k])
    return paramFormat

def paramParse(paramStr):
    paramPats = str.split(paramStr, '&')
    param = OrderedDict()
    for paramPat in paramPats:
        equalIndex = paramPat.find('=')
        if equalIndex > 0:
            param[paramPat[:equalIndex]] = paramPat[equalIndex+1:]
    return param

def urlencodeFromMap(m):
    us = ''
    for k in m:
        us += k + '=' + str(m[k]) + '&'
    return us[:-1]

class Window(QWidget):
    query = {}
    reqStatsObj = {}
    zoom = 1.2

    def __init__(self):
        super().__init__()
        self.__render()
        self.show()
        self.reqUrlInput.setFocus()
        self.requestThread = RequestThread(self)
        self.requestThread.finishSignal.connect(self.__setRes)

    # 渲染组件
    def __render(self):
        self.setWindowTitle('PyRequest')
        self.__renderSelf()
        self.__renderMain()
        self.__renderLeft()
        self.reqUrlInput.textEdited.emit(self.reqUrlInput.text())
        self.queryEdit.textChanged.emit()
        self.bodyEdit.textChanged.emit()

    def __renderSelf(self):
        self.leftLayout = QVBoxLayout()
        self.mainLayout = QVBoxLayout()
        layout = QHBoxLayout(self)
        layout.addLayout(self.leftLayout)
        layout.addLayout(self.mainLayout)
        self.setWindowIcon(QIcon('assets/icon.ico'))
        self.resize(900 * self.zoom, 600 * self.zoom)

    def __renderLeft(self):
        self.reqList = QListWidget()
        self.reqList.itemClicked.connect(self.__logItemClicked)
        self.reqList.setMaximumWidth(300)
        for log in logs:
            logItem = QListWidgetItem(log['method'] + ' ' + log['url'])
            logItem.setData(99, log['id'])
            self.reqList.addItem(logItem)
        self.leftLayout.addWidget(self.reqList)

    def __logItemClicked(self, item):
        id = item.data(99)
        log = reqLogDB.selectOne(id)
        print(log)
        self.reqMethodCombo.setCurrentText(log['method'])
        self.reqUrlInput.setText(log['url'])
        self.queryEdit.setPlainText(log['query'])
        self.bodyEdit.setPlainText(log['body'])
        self.__clearRes()

    def __appendLog(self, log):
        logItem = QListWidgetItem(log['method'] + ' ' + log['url'])
        logItem.setData(99, log['id'])
        self.reqList.insertItem(0, logItem)

    def __renderMain(self):
        # input
        self.reqMethodCombo = QComboBox()
        self.reqMethodCombo.addItems(['GET', 'POST'])
        self.reqMethodCombo.currentTextChanged.connect(self.__methodChange)
        self.reqUrlInput = QLineEdit()
        self.reqUrlInput.setText('http://ip.taobao.com/service/getIpInfo.php?ip=59.41.95.234')
        self.reqUrlInput.textEdited.connect(self.__urlChanged)
        self.reqButton = QPushButton()
        self.reqButton.setText('SEND')
        self.reqButton.clicked.connect(self.__request)
        inputLayout = QHBoxLayout()
        inputLayout.addWidget(self.reqMethodCombo)
        inputLayout.addWidget(self.reqUrlInput)
        inputLayout.addWidget(self.reqButton)
        # body&query
        self.queryLabel = QLabel('Query')
        self.queryEdit = QPlainTextEdit()
        self.queryEdit.textChanged.connect(self.__queryEditChanged)
        self.bodyLabel = QLabel('Body')
        self.bodyEdit = QPlainTextEdit()
        self.bodyEdit.textChanged.connect(self.__bodyEditChanged)
        queryLayout = QVBoxLayout()
        queryLayout.addWidget(self.queryLabel)
        queryLayout.addWidget(self.queryEdit)
        bodyLayout = QVBoxLayout()
        bodyLayout.addWidget(self.bodyLabel)
        bodyLayout.addWidget(self.bodyEdit)
        self.bodyEdit.hide()
        self.bodyLabel.hide()
        paramLayout = QHBoxLayout()
        paramLayout.addLayout(queryLayout)
        paramLayout.addLayout(bodyLayout)
        self.mainLayout.addLayout(inputLayout)
        self.mainLayout.addLayout(paramLayout)
        # response
        self.resTab = self.__createResTab()
        self.mainLayout.addWidget(self.resTab)

    def __createResTab(self):
        resTab = QTabWidget()
        self.reqStats = QTextEdit()
        self.resStats = QTextEdit()
        self.resText = QTextEdit()
        self.resJSON = QTextEdit()
        self.resView = QWebEngineView()
        resTab.addTab(self.reqStats, 'req')
        resTab.addTab(self.resStats, 'res')
        resTab.addTab(self.resText, 'text')
        resTab.addTab(self.resJSON, 'json')
        resTab.addTab(self.resView, 'view')
        return resTab

    # 发起请求
    def __request(self):
        self.__clearRes()
        bodyRaw = self.bodyEdit.toPlainText()
        self.body = formatParamParse(bodyRaw)
        self.resView.setHtml('')
        # self.resView.setUrl(QUrl(self.reqUrlInput.text()))
        self.requestThread.start()

    # 处理返回
    def __setRes(self, res):
        if (self.resTab.currentIndex() == 0):
            self.resTab.setCurrentIndex(1)
        self.resStats.setPlainText(res['stats'])
        self.resText.setPlainText(res['text'])
        try :
            jsonstr = json.dumps(json.loads(res['text']), indent=2, ensure_ascii=False)
            self.resJSON.setPlainText(jsonstr)
        except Exception as e:
            print(e)
            self.resJSON.setPlainText('Not a JSON string')
        self.resView.setHtml(res['text'])
        self.__log()

    # 请求记录
    def __log(self):
        log = {}
        log['method'] = self.reqMethodCombo.currentText()
        log['url'] = self.reqUrlInput.text()
        log['query'] = self.queryEdit.toPlainText()
        log['body'] = self.bodyEdit.toPlainText()
        id = reqLogDB.insert(log)
        log['id'] = id
        self.__appendLog(log)

    # 清空返回栏
    def __clearRes(self):
        self.resStats.setText('')
        self.resText.setText('')
        self.resView.setHtml('')
        self.resJSON.setText('')

    # 方法切换
    def __methodChange(self, text):
        if text == 'GET':
            self.bodyEdit.hide()
            self.bodyLabel.hide()
        else:
            self.bodyEdit.show()
            self.bodyLabel.show()

    # 快捷键
    def keyPressEvent(self, event):
        print(event.key())
        key = event.key()
        if key in (16777268, 16777220, 16777221):
            self.__request()
        if key >= 49 and key <= 53:
            self.resTab.setCurrentIndex(key - 49)
        if key == 71:
            self.reqMethodCombo.setCurrentText('GET')
        if key == 80:
            self.reqMethodCombo.setCurrentText('POST')
        if key == 87:
            if (self.reqMethodCombo.currentText() == 'GET'):
                self.reqMethodCombo.setCurrentText('POST')
            elif (self.reqMethodCombo.currentText() == 'POST'):
                self.reqMethodCombo.setCurrentText('GET')

    # query/body/reqStats 联动
    def __queryEditChanged(self):
        queryRaw = self.queryEdit.toPlainText()
        self.__querySetFromFormat(queryRaw)

    def __bodyEditChanged(self):
        bodyRaw = self.bodyEdit.toPlainText()
        self.__reqStatsChanged({'body': formatParamParse(bodyRaw)})

    def __urlChanged(self, url):
        self.__querySetFromUrl(url)

    def __querySetFromFormat(self, queryRaw):
        query = formatParamParse(queryRaw)
        self.query = query
        self.__reqStatsChanged({'query': query})
        queryStr = urlencodeFromMap(query)
        if queryStr:
            url = self.reqUrlInput.text()
            urlParts = urlparse(url)
            url = '{}://{}{}?{}'.format(urlParts.scheme, urlParts.netloc, urlParts.path, queryStr)
            self.reqUrlInput.setText(url)
            self.__reqStatsChanged({'url': url})

    def __querySetFromUrl(self, url):
        queryStr = urlparse(url).query
        query = paramParse(queryStr)
        self.__reqStatsChanged({'url': url})
        self.__reqStatsChanged({'query': query})
        queryFormat = formatParamStringify(query)
        self.queryEdit.blockSignals(True)
        self.queryEdit.setPlainText(queryFormat)
        self.queryEdit.blockSignals(False)

    def __reqStatsChanged(self, things):
        for k in things:
            thing = things[k]
            self.reqStatsObj[k] = thing
        reqStatsStr = json.dumps(self.reqStatsObj, indent=2, ensure_ascii=False)
        self.reqStats.setPlainText(reqStatsStr)

def getfont():
    fontId = QFontDatabase.addApplicationFont('assets/MSYHMONO.ttf')
    if fontId != -1:
        fontFamilies = QFontDatabase.applicationFontFamilies(fontId)
        font = QFont()
        font.setFamily(fontFamilies[0])
        font.setPixelSize(12)
        return font

app = QApplication(sys.argv)

font = getfont()
if font:
    app.setFont(font)

window = Window()

sys.exit(app.exec_())


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
 @Author :yicg
 @Time : 2021/3/1 下午10:27
 @Version : 1.0
 @Description :   测试script
"""
from time import sleep

from selenium import webdriver

class TestJs():
    driver_path = '/Users/yicg/Downloads/chromedrivers/chromedriver'

    def setup(self):
        self.driver = webdriver.Chrome(executable_path=self.driver_path)
        self.driver.maximize_window()
        self.driver.implicitly_wait(5)  # 隐式等待

    def teardown(self):
        sleep(3)
        self.driver.quit()

    def test_js_1(self):
        """
        案例1：滑动到浏览器底部或者顶部
        1.打开百度首页
        2.输入搜索关键字
        3.点击搜索后，跳转到搜索结果页
        4.滑动到底部点击下一页

        :return:
        """
        self.driver.get("http://www.baidu.com")
        self.driver.find_element_by_id("kw").send_keys("selenium测试")
        ele=self.driver.execute_script("return document.getElementById('su')")  #js
        ele.click()
        sleep(3)
        #滑动到最低端
        self.driver.execute_script("document.documentElement.scrollTop=10000")
        #点击下一页
        self.driver.find_element_by_xpath('//*[@class="n"]').click()
        sleep(3)
        #在滑一次
        self.driver.execute_script("document.documentElement.scrollTop=10000")

    def test_js_2(self):
        """
        js案例2
        1.打开12306网站
        2.移除时间的只读模式
        3.输入新的时间
        :return:
        """
        self.driver.get("https://www.12306.cn/index/")
        #移除只读模式
        self.driver.execute_script(" a=document.getElementById('train_date');a.removeAttribute('readonly');")
        #输入新的时间
        self.driver.execute_script("a=document.getElementById('train_date');a.value='2020-3-20'")


import json
from roamon_verify import roamon_verify_checker
from roamon_verify import roamon_verify_getter
import roamon_alert_watcher
import os
import signal
import daemon
import daemon.pidfile
import sys
import logging
import time

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

class RoamonAlertDaemon():
    def __init__(self, file_path_pid_lockfile, log_path, checker, watch_interval):
        self.path_pid_lockfile = file_path_pid_lockfile
        self.log_file_path = log_path
        self.checker = checker
        self.watch_interval = watch_interval  # 分指定

    @classmethod
    def __start_daemon_with_func(cls, func, *args, logpath, pidpath, **kws):
        logger.debug("DAEMON_INVOKER: invoke daemon")
        # [ここ](https://qiita.com/KosukeJin/items/e626ddda850aa8407cee)からコピペ気味
        # 多重起動防止
        if pidpath:
            pid = daemon.pidfile.PIDLockFile(pidpath)
            logger.debug("DAEMON_INVOKER: pid: {}".format(pid))
            if pid.is_locked():
                raise Exception('Process is already started.')

        # PID指定がなければフォアグラウンド
        else:
            pid = None
        # ログ
        if logpath:
            std_out = open(logpath, mode='a+', encoding='utf-8')
            std_err = std_out

        # 指定がなければ標準出力
        else:
            std_out = sys.stdout
            std_err = sys.stderr

        dc = daemon.DaemonContext(
            umask=0o002,

            pidfile=pid,
            stdout=std_out,
            stderr=std_err,
        )

        # 内部で実行される
        def forever():
            # while 1は実行失敗時にまた行うためのもの(渡されるfuncは中でループするので基本的にこっちではループしないはず)
            while 1:
                try:
                    func(*args, **kws)

                # kill -SIGTERM {pid} で停止する
                except SystemExit as e:
                    print('Killed by SIGTERM.')
                    raise

                # それ以外のエラーは無視して動き続ける
                except Exception as e:
                    print('Uncaught exception was raised, but process continue.', e)

        with dc:
            forever()

    # デーモンが実行するループ
    def main_loop(self, kws=None):
        logger.debug("enter main_loop")

        while 1:
            logger.debug("start loop")
            # TODO: やっつけ実装なので、一定間隔でこれらを実行するいい案を考える (非同期にするとそれはそれで面倒で、ダウンロード中や処理中のファイルを開こうとするときはやめてリトライとかする必要がありそう。でもそうすべきかなー)

            # BGP経路情報とVRPのフェッチ
            self.checker.fetch_rib_data()
            self.checker.fetch_vrps_data()

            # BGP経路情報とVRP,および連絡先情報をロードし直す
            self.checker.load_all_data()
            # 異常がないかチェック
            self.checker.check_roa_with_all_watched_asn()

            logger.debug("end working in loop")

            # 　規定時間まつ
            time.sleep(60 * self.watch_interval)
            logger.debug("end loop")

            # 以下はもしもっと正確にやるならこんな感じというやつ...(未完成)
            # interval_min_fetch_rib = self.watch_interval
            # interval_min_fetch_vrps = self.watch_interval
            # interval_min_check_roa_with_wtached_asn = self.watch_interval
            # # 現在時刻と次回起動時刻
            # t = datetime.now().replace(second=0, microsecond=0)
            # next_up_time_fetch_rib  = t + timedelta(minutes=interval_min_fetch_rib)
            # next_up_time_fetch_vrps = t + timedelta(minutes=interval_min_fetch_vrps)
            # next_up_time_check_roa_with_wtached_asn = t + timedelta(minutes=interval_min_check_roa_with_wtached_asn)
            #
            # # intervalの倍数ジャストに起動 (非同期でやらないのは、書き込み中のファイルを読み込んだりしないように)
            # if t.minute % interval_min_fetch_rib == 0:
            #     self.checker.fetch_rib_data()
            # # 1周目 次回実行時間まで調整
            # else:
            #     stop = stop = interval_min_fetch_rib - t.minute % interval_min_fetch_rib
            #     n = t + timedelta(minutes=stop)
            #
            # if t.minute % interval_min_fetch_vrps == 0:
            #     self.checker.fetch_vrps_data()
            #
            # if t.minute % interval_min_check_roa_with_wtached_asn == 0:
            #     self.checker.check_roa_with_all_watched_asn()
            #
            # w = (n - datetime.now()).total_seconds()
            # log_to_stdout('[INFO]', 'Sleep', w, 'seconds.')
            # time.sleep(w)

    # デーモン起動
    def start(self):
        logger.debug("call start daemon!")
        self.__start_daemon_with_func(self.main_loop, logpath=self.log_file_path, pidpath=self.path_pid_lockfile,
                                      kws=None)

    # デーモン停止
    def stop(self):
        # PIDファイルがあるかどうか見る。あるなら既に起動中のはず
        if not os.path.exists(self.path_pid_lockfile):
            logger.error("PID file for the daemon is not found at {}".format(self.path_pid_lockfile))

        pid = daemon.pidfile.PIDLockFile(self.path_pid_lockfile)

        # デーモンのプロセスを終了させる
        daemon_pid = pid.read_pid()
        logger.debug("killing daemon process with PID:{}...".format(daemon_pid))
        try:
            os.kill(daemon_pid, signal.SIGTERM)
        except ProcessLookupError:
            # プロセスが動いてなかったらここでおしまい
            logger.error("Process PID:{} is not running.".format(daemon_pid))
            return

        # PIDファイルを始末
        logger.debug("releasing daemon process with PID file:{}...".format(self.path_pid_lockfile))
        pid.break_lock()


""" Build Cleaned CSV files -- Logic to:
    - create cleaned season inventories based on multiple data sources
        * LILA/Dryad data (S1-S6)
        * MSI SQL DB data
        * MSI SX_cleaned.csv files
    - specifically:
        * ensure valid/invalid flags are consistent
        * ensure timestamps are consistent and as correct as possible
        * generally, ensure consistency with LILA/Dryad
    - general procedure:
        1. Base files are SX_cleaned.csv for S9-S10, df_db for S1-S8
        2. Ensure everything S1-S6 in LILA dataset is contained
        (3. Add additional captures from S1-S6 if 'include'
            status but not in LILA)
        (4. verify with DB export valid flags)
        (5. verify timestamps LILA and DB export / SX_cleaned)
    - required files:
        1. LILA export (SnapshotSerengeti.json)
        2. DB export (timestamps_db.csv)
        3. SX_cleaned.csv files
"""
import os
import pandas as pd

from utils.utils import sort_df, sort_df_by_capture_id
from zooniverse_exports.legacy.legacy_utils import (
    read_lila, read_db, read_cleaned, print_stats, create_capture_id,
    create_date, create_time, _get_datetime_obj
)

#################################
# Parameters
#################################

root_data_path = '/home/packerc/shared/season_captures/SER/legacy_S1_S10_data/'

path_cleaned = root_data_path
lila_path = os.path.join(root_data_path, 'lila.json')
path_images_db = os.path.join(root_data_path, 'df_db_S1_S8.csv')
path_output = os.path.join(root_data_path, 'cleaned/')

col_mapper = {
    'ZoonID': 'subject_id',
    'idCaptureEvent': 'CaptureEventId',
    'Zooniverse_ID': 'subject_id',
    'ZooniverseIdentifier': 'subject_id',
    'Roll': 'roll_db_id',
    'Season': 'season',
    'Site': 'site_db_id',
    'Capture': 'capture_db_id',
    'GridCell': 'site',
    'RollNumber': 'roll',
    'CaptureEventNum': 'capture'
    }

#################################
# Read Data
#################################

# Read Base Data -- LILA / DB Exports / CLEANED CSV Files
df_lila = read_lila(lila_path)
df_db = read_db(path_images_db, col_mapper)
df_cleaned = read_cleaned(path_cleaned)

#################################
# Look at Stats
#################################

# investigate data
df_db.columns
print_stats(df_db['season'].value_counts())
print_stats(df_db['site'].value_counts())
print_stats(df_db['roll'].value_counts())
print_stats(df_db['SequenceNum'].value_counts())
print_stats(df_db['ZooniverseStatus'].value_counts())
print_stats(df_db['idTimestampStatuses'].value_counts())
print_stats(df_db['StatusDescription'].value_counts())
print_stats(df_db['Invalid'].value_counts())
df_db.groupby(['Invalid', 'season']).size()
df_cleaned.groupby(['include', 'season']).size()
df_cleaned.groupby(['include', 'invalid']).size()
df_db.groupby(['idTimestampStatuses', 'StatusDescription']).size()

# idTimestampStatuses  StatusDescription
# 0                    OK                                        5011159
# 1                    Not recoverable                             25836
# 2                    Fix is hard                                 36392
# 3                    Fix is hard but timestamp likely close      14555

# -> include only for invalid=0

#################################
# Select relevant columns
#################################

df_lila = df_lila[[
    'image_path', 'subject_id', 'season',
    'site', 'datetime', 'species']]

df_db = df_db[[
    'subject_id', 'image_path', 'season',
    'site', 'roll', 'capture', 'SequenceNum',
    'TimestampJPG', 'TimestampFile', 'TimestampAccepted',
    'Invalid', 'idTimestampStatuses', 'StatusDescription']]

df_cleaned = df_cleaned[[
    'newtime', 'oldtime', 'PathFilename', 'roll', 'season', 'site',
    'capture', 'image', 'include', 'invalid']]

# Export to Disk
df_lila.to_csv(os.path.join(root_data_path, 'df_lila.csv'), index=False)
df_db.to_csv(os.path.join(root_data_path, 'df_db.csv'), index=False)
df_cleaned.to_csv(os.path.join(root_data_path, 'df_cleaned.csv'), index=False)

# Read from disk
# df_lila = pd.read_csv(os.path.join(root_data_path, 'df_lila.csv'), index_col=False, dtype=str)
# df_db = pd.read_csv(os.path.join(root_data_path, 'df_db.csv'), index_col=False, dtype=str)
# df_cleaned = pd.read_csv(os.path.join(root_data_path, 'df_cleaned.csv'), index_col=False, dtype=str)
# df_lila.fillna('', inplace=True)
# df_db.fillna('', inplace=True)
# df_cleaned.fillna('', inplace=True)

# create flags
df_cleaned['in_clean'] = '1'
df_db['in_db'] = '1'
df_lila['in_lila'] = '1'

# remove specific images that don't actually exist
def remove_non_existing(df):
    """ Remove images that dont exist """
    i_subset = ~df['image_path'].isin([
        'S1/L10/L10_R1/S1_L10_R1_PICT0004 (2).JPG',
        'S1/L10/L10_R1/S1_L10_R1_PICT0023 (2).JPG',
        'S1/L10/L10_R1/S1_L10_R1_PICT0024 (2).JPG',
        'S1/L10/L10_R1/S1_L10_R1_PICT0025 (2).JPG',
        'S7/G09/G09_R2/S7_G09_R2_P1060627.JPG'])
    return df.loc[i_subset]

df_lila = remove_non_existing(df_lila)
df_db = remove_non_existing(df_db)

#################################
# Merge Data
#################################

# build full data frame
df_merged = pd.merge(
    left=df_db, how='outer',
    right=df_cleaned,
    left_on=['image_path', 'season', 'site', 'roll'],
    right_on=['PathFilename', 'season', 'site', 'roll'],
    suffixes=('_db', '_cleaned'))
df_merged['image_path'] = df_merged['image_path'].fillna(df_merged['PathFilename'])

# create 'capture' column, take 'capture_db', substitute with 'capture_cleaned' if empty
df_merged['capture'] = df_merged['capture_db']
i_replace = (df_merged['capture_db'] == '') | (df_merged['capture_db'].isna())
df_merged['capture'].loc[i_replace] = df_merged['capture_cleaned'].loc[i_replace]

# merge lila data
df_merged2 = pd.merge(
    left=df_merged, how='outer',
    right=df_lila,
    left_on=['image_path', 'season', 'site'],
    right_on=['image_path', 'season', 'site'],
    suffixes=('', '_lila'))
df_merged2.fillna('', inplace=True)

#################################
# Clean Invalid Image Names
#################################

# remove images with either '(' or ')' in image name
i_special_char = df_merged2['image_path'].apply(lambda x: (')' in x) | ('(' in x))

bad_names = list(df_merged2['image_path'].loc[i_special_char])
good_names = [x.replace(' (2)', '') for x in bad_names]
df_merged2['image_path'].loc[i_special_char] = good_names

# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
# RENAMING DONE!
# for old, new in zip(bad_names, good_names):
#     root_path = '/home/packerc/shared/albums/SER/'
#     old_path = os.path.join(root_path, old)
#     new_path = os.path.join(root_path, new)
#     try:
#         os.rename(old_path, new_path)
#         print("mv {} to {}".format(old_path, new_path))
#     except:
#         raise

#################################
# Some checks
#################################

df_db.groupby(['season', 'Invalid']).size().to_frame('count').reset_index()
df_merged2.groupby(['season', 'Invalid']).size().to_frame('count').reset_index()
df_merged2.groupby(['season', 'in_clean', 'in_db', 'in_lila']).size().to_frame('count').reset_index()
df_merged2.iloc[0]

# some random checks / comments

# # check investigate discrepancies
# img = 'S1/E02/E02_R5/S1_E02_R5_PICT0148.JPG'
# img = 'S4/B12/B12_R2/S4_B12_R2_IMAG0100.JPG'
# img = 'S5/G07/G07_R4/S5_G07_R4_IMAG0219.JPG'
# df_merged2[df_merged2['image_path'] == img]

# # complete roll missing
# # S4/B12/B12_R2/S4_B12_R2_
# # ASG000euyt, https://static.zooniverse.org/www.snapshotserengeti.org/subjects/standard/50dcd6e6a2fc8e37891057fa_0.jpg
# # SER_S4#B12#2#1,S4,B12,R2,1,
# # 4,B12,2,1
# # is in S4_cleaned.csv with correct time...
# # has invalid=3 flag, therefore, makes sense to be missing in LILA
#
#
# missing_in_cleaned.shape
# missing_in_cleaned.iloc[1]
#
# # 164 captures from S5 missing, S4/S6 are good
# # 464 images missing from S5
# missing_in_cleaned['season_y'].value_counts()
#
# # ASG000pf4u
# # S5/G07/G07_R4/S5_G07_R4_IMAG0219.JPG
# # entire roll S5/G07/G07_R4 missing in S5_cleaned.csv
# # however, is in LILA and DB

# df_merged2[df_merged2['image_rank_in_capture'] == '6']
# df_merged2[df_merged2['subject_id'] == 'ASG000twd2']

#################################
# Handle Different Seasons
#################################

# Build DFs for each season
col_mapper_final = {
    'SequenceNum': 'image_rank_in_capture',
    'image': 'image_rank_in_capture',
    'TimestampJPG': 'datetime_exif',
    'oldtime': 'datetime_exif',
    'TimestampFile': 'datetime_file_creation',
    'TimestampAccepted': 'datetime',
    'newtime': 'datetime',
    'Invalid': 'invalid',
    'image_path': 'image_path_rel',
    'datetime': 'datetime_lila'
    }

season_dfs = list()
for i in range(0, 10):
    season_num = i+1
    df_season = df_merged2[df_merged2['season'] == 'S{}'.format(season_num)].copy()
    # LILA and DB
    if season_num <= 6:
        df_season = df_season[[
            'image_path', 'season', 'site', 'roll', 'capture',
            'SequenceNum', 'TimestampJPG', 'TimestampFile',
            'TimestampAccepted', 'Invalid', 'datetime']]
    # DB
    elif season_num <= 8:
        df_season = df_season[[
            'image_path', 'season', 'site', 'roll', 'capture',
            'SequenceNum', 'TimestampJPG', 'TimestampFile',
            'TimestampAccepted', 'Invalid']]
    # SEASON
    else:
        df_season = df_season[[
            'image_path', 'season', 'site', 'roll', 'capture',
            'image', 'oldtime', 'newtime',
            'include', 'invalid']]
    df_season.rename(columns=col_mapper_final, inplace=True)
    season_dfs.append(df_season)

df_final = pd.concat(season_dfs)
df_final.fillna('', inplace=True)

# remove any images for S1-S8 that are not in either the db export or LILA
# it turned out that some of these are faulty captures, for security reasons
# we just remove all that are not in the DB and/or LILA
i_only_in_cleaned_s1_s8 = \
    (df_final['season'].isin(['S1', 'S2', 'S3', 'S4', 'S5', 'S6', 'S7', 'S8'])) & \
    (df_final['invalid'] == '')

before = df_final.shape[0]
df_final = df_final.loc[~i_only_in_cleaned_s1_s8]
print("Removed {} only in cleaned for S1-S8".format(before - df_final.shape[0]))

# final checks
print_stats(df_final['season'].value_counts())
df_final.groupby(['season', 'include', 'invalid']).size().to_frame('count').reset_index()
df_final.groupby(['season', 'invalid']).size().to_frame('count').reset_index()

# 'SER_S5#G07#3#44', 'SER_S5#G07#3#129'
should_be_excluded = df_final[
    (df_final['season'] == 'S5') &
    (df_final['site'] == 'G07') &
    (df_final['roll'] == '3') &
    (df_final['capture'].isin(['44', '129']))]
assert len(should_be_excluded) == 0

##########################
# check with LILA data
##########################

tt = df_final[(df_final['datetime_lila'] != df_final['datetime']) & (df_final['datetime_lila'] != '') & (df_final['datetime'] != '') & (df_final['image_rank_in_capture'] == '1')]
tt.shape
#(62, 12) are different

# check LILA invalid flags
tt = pd.merge(
    left=df_final,
    right=df_lila,
    left_on='image_path_rel',
    right_on='image_path',
    how='left',
    suffixes=('', '_lila'))
tt = tt.loc[(tt['season'].isin(['S1', 'S2', 'S3', 'S4', 'S5', 'S6']))]
tt.groupby(['season_lila', 'include', 'invalid']).size().to_frame('count').reset_index()
tt.groupby(['season', 'include', 'invalid']).size().to_frame('count').reset_index()
# -> LILA data had only 'invalid' == 0 records

# -> newtime is mostly 'TimestampAccepted' (small discrepancies for S4, S5 and S6)
print_stats(tt['season'].value_counts())

##################################
# Create Captures DF
##################################

# Problems
# 1) some captures have no time
# 2) some captures have timestampaccepted not on first image in sequence
# 3) some captures have correct timestamp only on first image

# Logic
# 1) take timestamp of first non-empty ordered by sequence num
# 2) leave blank if no timestamp found

# Create capture to timestamp df
# take lila datetime and substitute with 'datetime' if not available
print_stats(df_final['season'].value_counts())
df_capture = df_final.copy()
df_capture['datetime_clean'] = df_capture['datetime_lila']
i_no_lila = (df_capture['datetime_clean'] == '') | (df_capture['datetime_clean'].isna())
df_capture['datetime_clean'].loc[i_no_lila] = df_capture['datetime'].loc[i_no_lila]

# check
# tt = df_capture.drop_duplicates(subset=['season', 'site', 'roll', 'capture'])
# tt[tt['image_rank_in_capture'] != '1']
# df_merged2[(df_merged2['image_path'] == 'S4/B12/B12_R2/S4_B12_R2_IMAG0070.JPG') & (df_merged2['invalid'] == '0')]
#df_captures_first_image[df_captures_first_image['datetime_clean'] == ''][['image_path','SequenceNum']]

# move forward with only datetime_clean (mix of LILA and DB timestamp)
df_capture = df_capture[['season', 'site', 'roll', 'capture', 'datetime_clean']]

# get captures that have at least one non-empty datetime
df_capture_first_with_datetime = df_capture.loc[df_capture['datetime_clean'] != '']
df_capture_first_with_datetime.drop_duplicates(subset=['season', 'site', 'roll', 'capture'], inplace=True)

# get the rest of the captures
df_captures_first_image = df_capture.drop_duplicates(subset=['season', 'site', 'roll', 'capture'])

# df_captures_first_image.iloc[0]
# df_merged2[(df_merged2['season'] == 'S1') &
#  (df_merged2['site'] == 'O05') &
#  (df_merged2['roll'] == '1') &
#  (df_merged2['capture'] == '1171')]
# df_merged2[(df_merged2['season'] == 'S1') &
#  (df_merged2['site'] == 'C06') &
#  (df_merged2['roll'] == '1') &
#  (df_merged2['capture'] == '14')]

# merge both together to find captures without timestamps
df_captures_without_timestamps = pd.merge(
    left=df_captures_first_image,
    right=df_capture_first_with_datetime,
    on=['season', 'site', 'roll', 'capture'],
    how='left',
    suffixes=('', '_f'))
df_captures_without_timestamps = df_captures_without_timestamps[df_captures_without_timestamps['datetime_clean_f'].isna()]

df_captures_without_timestamps.shape[0]
# 6 captures without timestamps

# combine
df_captures_all = df_capture_first_with_datetime.append(df_captures_without_timestamps[['season', 'site', 'roll', 'capture', 'datetime_clean']])
df_captures_all.shape
df_captures_all.columns

# check
tt = df_captures_all.drop_duplicates(subset=['season', 'site', 'roll', 'capture'])
assert tt.shape[0] == df_captures_all.shape[0]

df_captures_all['capture_id'] = df_captures_all[['season', 'site', 'roll', 'capture']].apply(
    lambda x: create_capture_id(*x), axis=1)

# sort by capture id
sort_df_by_capture_id(df_captures_all)

###########
# checks
###########

# wrong capture from S5 should not be included
assert sum(df_captures_all['capture_id'].isin(['SER_S5#G07#3#44', 'SER_S5#G07#3#129'])) == 0

tt = pd.merge(left=df_captures_all, right=df_final[['season', 'site', 'roll', 'capture', 'datetime_lila']],
 on=['season', 'site', 'roll', 'capture'],
 how='outer')

# no diffs to LILA datetimes
tt_lila = tt[(~tt['datetime_lila'].isna()) & (tt['datetime_lila'] != '')]
tt_diff = tt_lila[tt_lila['datetime_lila'] != tt_lila['datetime_clean']]
assert tt_diff.shape[0] == 0

# no empty capture ids
assert df_captures_all[df_captures_all['capture_id'] == ''].shape[0] == 0
assert df_captures_all[df_captures_all['capture_id'].isna()].shape[0] == 0

# 6 captures without datetime_clean
no_datetime = df_captures_all[df_captures_all['datetime_clean'] == '']
no_datetime.shape

# Create capture date/time
# df_captures_all['capture_date_local'] = df_captures_all['datetime_clean'].apply(
#     lambda x: create_date(x))
#
# df_captures_all['capture_time_local'] = df_captures_all['datetime_clean'].apply(
#     lambda x: create_time(x))

df_captures_all.columns

df_captures_all.rename(columns={'datetime_clean': 'datetime'}, inplace=True)
df_captures_all = df_captures_all[[
    'capture_id', 'season', 'site', 'roll', 'capture', 'datetime']]

df_captures_all.to_csv(os.path.join(path_output, 'capture_events.csv'), index=False)

#df_captures_all = pd.read_csv(os.path.join(path_output, 'capture_events.csv'), index_col=False, dtype=str)

##################################
# Create Subject to Cap Mapper
##################################

df_base = df_merged2[['subject_id_lila', 'subject_id', 'season', 'site', 'roll', 'capture']].copy()

df_base['capture_id'] = df_base[['season', 'site', 'roll', 'capture']].apply(
    lambda x: create_capture_id(*x), axis=1)

df_subject_to_capture = pd.merge(
    left=df_captures_all,
    right=df_base,
    on=['capture_id', 'season', 'site', 'roll', 'capture'],
    how='left'
)

i_sub_missing = (df_subject_to_capture['subject_id'] == '')
df_subject_to_capture['subject_id'].loc[i_sub_missing] = df_subject_to_capture['subject_id_lila'].loc[i_sub_missing]
df_subject_to_capture = df_subject_to_capture.loc[df_subject_to_capture['subject_id'] != '']

tt = df_subject_to_capture.groupby('capture_id').subject_id.nunique()
multi_sub = tt[tt>1]
assert multi_sub.shape[0] == 0

df_subject_to_capture = df_subject_to_capture[['capture_id', 'subject_id']]
df_subject_to_capture.to_csv(os.path.join(path_output, 'capture_id_to_subject_id.csv'), index=False)

# tt = df_subject_to_capture.loc[(df_subject_to_capture['subject_id'] == '')]
# print_stats(tt['season'].value_counts())
#
# tt = df_subject_to_capture.loc[(df_subject_to_capture['subject_id'] == '') | (df_subject_to_capture['subject_id_lila'] == '')]
# print_stats(tt['season'].value_counts())
#
# tt = df_subject_to_capture.loc[(df_subject_to_capture['subject_id'] == '') & (df_subject_to_capture['subject_id_lila'] == '')]
# print_stats(tt['season'].value_counts())
#
# tt = df_subject_to_capture.loc[
#     (df_subject_to_capture['subject_id'] != df_subject_to_capture['subject_id_lila']) &
#     (df_subject_to_capture['subject_id'] != '') &
#     (df_subject_to_capture['subject_id_lila'] != '')]
# tt.shape
#
#
# tt.groupby(['season', 'include', 'invalid']).size().to_frame('count').reset_index()
# # -> LILA data had only 'invalid' == 0 records
#
# # -> newtime is mostly 'TimestampAccepted' (small discrepancies for S4, S5 and S6)
# print_stats(tt['season'].value_counts())

##################################
# Create Final DF
##################################

###########################
# Current / New Attributes
# capture_id,
# season,
# site,
# roll,
# capture,
# image_rank_in_capture,
# image_name,
# image_path,
# image_path_rel,
# datetime,
# image_rank_in_roll,
# seconds_to_next_image_taken,seconds_to_last_image_taken,
# days_to_last_image_taken,days_to_next_image_taken,
# image_name_original,image_path_original,image_path_original_rel,
# image_check__all_black,image_check__all_white,image_check__corrupt_file,image_check__corrupt_exif
# ,image_check__empty_exif,image_check__time_lapse,image_check__time_too_new,
# image_check__time_too_old,image_check__captures_with_too_many_images,
# datetime_file_creation,
# datetime_exif,
# exif__EXIF:Make,exif__EXIF:Model,exif__EXIF:Software,exif__EXIF:ModifyDate,exif__EXIF:Copyright,exif__EXIF:Orientation,exif__EXIF:XResolution,exif__EXIF:YResolution,exif__EXIF:ResolutionUnit,exif__EXIF:YCbCrPo
# sitioning,exif__EXIF:ExifVersion,exif__EXIF:DateTimeOriginal,exif__EXIF:CreateDate,exif__EXIF:ExposureTime,exif__EXIF:FNumber,exif__EXIF:MeteringMode,exif__EXIF:WhiteBalance,exif__EXIF:ColorSpace
# ,exif__EXIF:Flash,exif__EXIF:ExposureProgram,exif__EXIF:ISO,exif__EXIF:ExposureCompensation,exif__EXIF:ExifImageWidth,exif__EXIF:ExifImageHeight,exif__EXIF:UserComment,exif__Composite:Aperture,ex
# if__Composite:ImageSize,exif__Composite:Megapixels,
# exif__Composite:ShutterSpeed,exif__Composite:LightValue,
# exif__EXIF:Rating,exif__EXIF:RatingPercent,
# exif__EXIF:Padding,
# image_was_deleted,
# image_is_invalid,
# image_no_upload,
# image_uncertain_datetime,
# action_taken, action_taken_reason

# from captures_all
# 'capture_id', 'season', 'site', 'roll', 'capture', 'datetime'
# from df_final
# 'image_rank_in_capture', 'image_path_rel'
# datetime_file_creation
# datetime_exif
# image_was_deleted ? no
# image_is_invalid ? if 'invalid' = 0 then 1 else 0
# image_uncertain_datetime ? if 'invalid' 1,2,3 then 1 else 0

# final record
# 'capture_id', 'season', 'site', 'roll', 'capture', 'image_rank_in_capture',
# 'image_path_rel',
# 'datetime', 'datetime_file_creation', 'datetime_exif',
# 'invalid', 'include', 'image_is_invalid', 'image_uncertain_datetime'

df_final_cols = df_final[['image_path_rel', 'season', 'site', 'roll',
 'capture', 'image_rank_in_capture', 'datetime_exif',
 'datetime_file_creation', 'invalid', 'include']].copy()

df_final_cols.fillna('', inplace=True)
print_stats(df_final_cols['include'].value_counts())
df_final_cols.groupby(['invalid', 'include']).size().to_frame('count').reset_index()
df_final_cols.groupby(['season']).size().to_frame('count').reset_index()

# re-create 'include' flag
i_to_include = df_final_cols['invalid'].isin(['0'])
i_to_exclude = df_final_cols['invalid'].isin(['1', '2', '3'])
i_to_unknown = df_final_cols['invalid'].isin([''])
df_final_cols['include'].loc[i_to_include] = '1'
df_final_cols['include'].loc[i_to_exclude] = '0'
df_final_cols['include'].loc[i_to_unknown] = ''

# re-create 'image_is_valid' / 'image_uncertain_datetime' flags
df_final_cols['image_is_invalid'] = ''
df_final_cols['image_datetime_uncertain'] = ''
df_final_cols['image_is_invalid'].loc[i_to_exclude] = '1'
df_final_cols['image_datetime_uncertain'].loc[i_to_exclude] = '1'
df_final_cols['image_is_invalid'].loc[i_to_include] = '0'
df_final_cols['image_datetime_uncertain'].loc[i_to_include] = '0'

# re-create dummy columns
df_final_cols['image_was_deleted'] = ''
df_final_cols['image_no_upload'] = ''
df_final_cols['action_taken'] = ''
df_final_cols['action_taken_reason'] = ''

# re-create image_name  column
df_final_cols['image_name'] = [os.path.basename(x) for x in df_final_cols['image_path_rel']]

df_final_cols = pd.merge(
    left=df_final_cols, right=df_captures_all,
    how='left', on=['season', 'site', 'roll', 'capture'])

# checks
assert sum(df_final_cols['capture_id'].isna()) == 0
assert sum(df_final_cols['capture_id'] == '') == 0
assert sum(df_final_cols['capture'] == '') == 0
assert sum(df_final_cols['image_path_rel'] == '') == 0

# some missing values expected
sum(df_final_cols['datetime'].isna())
sum(df_final_cols['datetime'] == '')
sum(df_final_cols['datetime_file_creation'] == '')
sum(df_final_cols['datetime_file_creation'] != '')
sum(df_final_cols['datetime_exif'] == '')

df_final_cols.groupby(['invalid', 'image_is_invalid']).size().to_frame('count').reset_index()
df_final_cols.groupby(['invalid', 'image_datetime_uncertain']).size().to_frame('count').reset_index()

# final record
# 'capture_id', 'season', 'site', 'roll', 'capture', 'image_rank_in_capture',
# 'image_path_rel',
# 'datetime', 'datetime_exif',
# 'invalid', 'include', 'image_is_invalid', 'image_uncertain_datetime'

final_cols = [
 'capture_id',
 'season',
 'site',
 'roll',
 'capture',
 'image_rank_in_capture',
 'image_name',
 'image_path_rel',
 'datetime',
 'datetime_exif',
 'datetime_file_creation',
 'image_is_invalid',
 'image_datetime_uncertain',
 'image_no_upload',
 'image_was_deleted',
 'action_taken',
 'action_taken_reason',
 'invalid',
 'include']

df_final_cols = df_final_cols[final_cols]

df_final_cols.to_csv(os.path.join(path_output, 'captures_cleaned.csv'), index=False)

##################################
# Create Filtered / Valid DF
##################################

df_final_include = df_final_cols.loc[df_final_cols['include'].isin(['1'])]

df_final_include.to_csv(os.path.join(path_output, 'captures_include.csv'), index=False)

##################################
# Create Season Cleaned
##################################

all_seasons = df_final_cols['season'].value_counts()

# sum(df_final_cols['image_rank_in_capture'] == '')
# tt = df_final_cols[df_final_cols['image_rank_in_capture'] == '']
# tt.groupby(['invalid', 'include']).size().to_frame('count').reset_index()

for season in all_seasons.index:
    print("Starting with Season {}".format(season))
    output_path_season = os.path.join(path_output, 'SER_{}_cleaned.csv'.format(season))
    i_season = df_final_cols['season'].isin([season])
    df_season = df_final_cols.copy()
    df_season = df_season.loc[i_season]
    print("Sorting Season {}".format(season))
    sort_df(df_season)
    print("Writing Season {}".format(season))
    df_season.to_csv(output_path_season, index=False)
    print("Wrote Season {}".format(season))

##################################
# Checks
##################################

# check for duplicates
tt = df_final_include.groupby('image_path_rel').size()
assert tt[tt>1].shape[0] == 0

# check datetime available
assert sum(df_final_include['datetime'].isna()) == 0
assert sum(df_final_include['datetime'] == '') == 0
assert sum(df_final_include['datetime_exif'] == '') == 0

# some missing values expected
sum(df_final_include['datetime_file_creation'] == '')

# check lila overlap
lila_check = pd.merge(left=df_lila, right=df_final_include, how='left',
 left_on='image_path', right_on='image_path_rel')

assert sum(lila_check['capture_id'].isna()) == 0
assert sum(lila_check['capture_id'] == '') == 0
assert sum((lila_check['datetime_x'] != lila_check['datetime_y']) & (lila_check['datetime_x'] != '')) == 0

print_stats(df_final_include['season'].value_counts())
print_stats(df_lila['season'].value_counts())

# check / investigate captures previously not in LILA but now are
tt = pd.merge(
    left=df_final_include,
    right=df_lila,
    left_on='image_path_rel',
    right_on='image_path',
    how='left',
    suffixes=('', '_lila'))

tt = tt.loc[(tt['season'].isin(['S1', 'S2', 'S3', 'S4', 'S5', 'S6'])) & (tt['season_lila'].isna())]
tt.shape[0]
print_stats(tt['season'].value_counts())
print_stats(tt['invalid'].value_counts())
tt.iloc[0]
tt
# 12 new images (4 captures)

# check status of lila data
tt = pd.merge(
    left=df_final_include,
    right=df_lila,
    left_on='image_path_rel',
    right_on='image_path',
    how='left',
    suffixes=('', '_lila'))
tt = tt.loc[(tt['season'].isin(['S1', 'S2', 'S3', 'S4', 'S5', 'S6'])) & (tt['season_lila'].isna())]

# not in LILA but look ok
# 1908680  S5/B03/B03_R2/S5_B03_R2_IMAG0015.JPG  2012-08-05 07:37:08
# 1908681  S5/B03/B03_R2/S5_B03_R2_IMAG0016.JPG  2012-08-05 07:37:08
# 1908682  S5/B03/B03_R2/S5_B03_R2_IMAG0017.JPG  2012-08-05 07:37:08
# 2040775  S5/D06/D06_R1/S5_D06_R1_IMAG1601.JPG  2012-06-10 11:15:55
# 2040776  S5/D06/D06_R1/S5_D06_R1_IMAG1602.JPG  2012-06-10 11:15:55
# 2040777  S5/D06/D06_R1/S5_D06_R1_IMAG1603.JPG  2012-06-10 11:15:55
# 3157969  S6/S13/S13_R1/S6_S13_R1_IMAG5890.JPG  2013-01-17 08:08:47
# 3157970  S6/S13/S13_R1/S6_S13_R1_IMAG5891.JPG  2013-01-17 08:08:47
# 3157971  S6/S13/S13_R1/S6_S13_R1_IMAG5892.JPG  2013-01-17 08:08:47
# 3182402  S6/U10/U10_R2/S6_U10_R2_IMAG0448.JPG  2013-03-08 17:31:15
# 3182403  S6/U10/U10_R2/S6_U10_R2_IMAG0449.JPG  2013-03-08 17:31:15
# 3182404  S6/U10/U10_R2/S6_U10_R2_IMAG0450.JPG  2013-03-08 17:31:15

# check file existence
# no files missing
# all_files = set()
# for dirpath, dirnames, filenames in os.walk('/home/packerc/shared/albums/SER/'):
#     for filename in [f for f in filenames if f.endswith(".JPG")]:
#         all_files.add(os.path.join(dirpath, filename))
#
# root_path = '/home/packerc/shared/albums/SER/'
# missing = list()
# for _i, row in df_final_include.iterrows():
#     if (_i % 100000) == 0:
#         print("Processed {}".format(_i))
#     img_path = os.path.join(root_path, row.image_path_rel)
#     if not img_path in all_files:
#         missing.append(row.image_path_rel)
#         print("Did not find: {}".format(row.image_path_rel))

##################################
# Observations
##################################

"""
Rational:

To ensure consistency between how we have selected Serengeti S1-S6 for public release on LILA and
the upcoming release of newly created exports for S1-S11 I have investigated in detail how
previous seasons were selected. Main questions were to figure out which images were selected and how
the timestamps were defined. Along the way I have found some smaller inconsistencies. I think I have now found
a good way to create a consistent LILA delivery for public release. To ensure whatever I did makes sense to you,
I'd like you to validate the following points/observations/decisions:

1. Most images have an include and/or an invalid flag (some have neither)
2. The invalid flag indicates images with correct/confirmed timestamps.
3. All images released on LILA were flagged with invalid = '0'. You only want to release images with invalid = '0'.
4. 12 images with invalid = '0' were not released on LILA (unclear why -- the images look fine)
    -> I will include these images.
4. Some (~40) images from S7 have a faulty image name: * (2).JPG
    -> I will correct these image names.
5. Some (~6) images with faulty names in the db don't exist on MSI
    -> I will remove these non-existing images from the cleaned files.
9. 'newtime' corresponds to 'TimestampAccepted', 'oldtime' corresponds to 'TimestampJPG', this column name
    was changed over time.
6. 'TimestampAccepted' is the 'correct' time but is only available on 'capture' level. The LILA dateset, therefore,
   has the same time for each image of the same capture.
7. TimestampAccepted is usually defined for the first image of a capture but not always.
8. Some captures don't have a 1st, 2nd and 3rd image but a 2nd, 4th, 6th as defined by their 'SequenceNum'
    -> I will NOT correct these.
9. Empty/blanks released on LILA do not have a timestamp.
    -> I will add timestamps for blanks.

I can guarantee:

1. All images released on LILA (S1-S6) will be in the newly created reports
2. All timestamps released on LILA (S1-S6) will match with the newly created reports
3. The labels for S1-S6 may have changed because:
    1) they can not be reproduced b/c of random effects,
    2) we have gathered more annotations,
    3) small changes in how 'blanks' are defined
4. All images in the reports / cleaned.csv files actually exist on MSI

Does the following distribution per season make sense (number of images):
S9  -- counts:     983081 / 6679922 (14.72 %)
S8  -- counts:     980256 / 6679922 (14.67 %)
S7  -- counts:     832154 / 6679922 (12.46 %)
S5  -- counts:     827224 / 6679922 (12.38 %)
S10 -- counts:     685686 / 6679922 (10.26 %)
S2  -- counts:     573200 / 6679922 (8.58 %)
S4  -- counts:     531554 / 6679922 (7.96 %)
S6  -- counts:     462846 / 6679922 (6.93 %)
S1  -- counts:     411414 / 6679922 (6.16 %)
S3  -- counts:     392507 / 6679922 (5.88 %)
"""

#df_merged2.groupby(['season', 'Invalid', 'include']).size().to_frame('count').reset_index()
#   invalid include    count
# 0                    27072
# 1       0       1  6679922
# 2       1       0    32572
# 3       2       0    36773
# 4       3       0    14555

##################################
# OLD CHECKS
##################################

# # check for duplicates
# tt = df_final_include.groupby('image_path_rel').size()
# tt[tt>1]
#
# # check lila content
# lila_check = pd.merge(left=df_lila, right=df_final_include, how='left',
#  left_on='image_path', right_on='image_path_rel')
#
# tt = lila_check[lila_check['capture_id'].isna()]
#
# df_lila[df_lila['image_path'] == 'S1/L10/L10_R1/S1_L10_R1_PICT0001.JPG']
# df_export[df_export['image_path'] == 'S1/L10/L10_R1/S1_L10_R1_PICT0001.JPG']
# # 'SequenceNum' wrong, capture starts with 2, 4 6
#
#
# lila_check.shape
# df_lila.shape
#
#
# # duplicates
# # df_export[df_export['image_path']== 'S1/I07/I07_R1/S1_I07_R1_PICT0007.JPG']
# # df_merged2[df_merged2['image_path'] == 'S1/I07/I07_R1/S1_I07_R1_PICT0007.JPG']
# # df_merged2[df_merged2['image_path'] == 'S10/H06/H06_R2/S10_H06_R2_IMAG0062.JPG']
# # df_merged2[(df_merged2['season'] == 'S10') & (df_merged2['site'] == 'H06') & (df_merged2['roll'] == '2') & (df_merged2['capture_clean'] == '23')]
#
# # test timestamps
# tt = df_merged2[(df_merged2['oldtime'] != df_merged2['TimestampJPG']) & (df_merged2['oldtime'] != '') & (df_merged2['TimestampJPG'] != '')]
# tt.shape
# # -> oldtime is TimestampJPG
#
# tt = df_merged2[(df_merged2['newtime'] != df_merged2['TimestampAccepted']) & (df_merged2['newtime'] != '') & (df_merged2['TimestampAccepted'] != '')]
# tt.shape
# # -> newtime is mostly 'TimestampAccepted' (small discrepancies for S4, S5 and S6)
# print_stats(tt['season'].value_counts())
#
# tt = df_merged2[(df_merged2['oldtime'] != df_merged2['newtime']) & (df_merged2['oldtime'] != '') & (df_merged2['newtime'] != '')]
# tt.shape
# # -> oldtime is not newtime
#
# # compare lila
# tt = df_merged2[(df_merged2['datetime'] != df_merged2['TimestampAccepted']) & (df_merged2['datetime'] != '') & (df_merged2['TimestampAccepted'] != '')]
# tt.shape
# print_stats(tt['SequenceNum'].value_counts())
# tt.iloc[10]
# # wrong in db (but only for sequecenum2)
# df_merged2[df_merged2['image_path'] == 'S1/D09/D09_R2/S1_D09_R2_PICT0014.JPG']
#
# tt = df_merged2[(df_merged2['datetime'] != df_merged2['TimestampAccepted']) &
#     (df_merged2['datetime'] != '') &
#     (df_merged2['TimestampAccepted'] != '') &
#     (df_merged2['SequenceNum'] == '1')]
# tt.shape
# # 65 first images with different timestamp
# for _i, row in tt.iterrows():
#     print("cleaned: {} lila: {}".format(row.TimestampAccepted, row.datetime))
# # some significant differences - db is wrong
# tt.iloc[4]
# df_merged2[df_merged2['image_path'] == 'S1/E02/E02_R5/S1_E02_R5_PICT0274.JPG']
# # -> generally accept LILA timestamp
#
#
# df_merged2.groupby(['season', 'invalid', 'include']).size().to_frame('count').reset_index()
#
#
# # TESTS
# df_merged[(df_merged['season'] == 'S4') &
# (df_merged['site'] == 'C07') &
#  (df_merged['roll'] == '2') &
#   (df_merged['capture'] == '1234')]
#
#
# #df_capture = df_capture[df_capture['datetime_clean'] != '']
# df_capture3 = df_capture.groupby(['season', 'site', 'roll', 'capture']).first()
#
#
# # try to get datetime
# def create_datetime(date_str):
#     try:
#         tm = _get_datetime_obj(date_str)
#     except:
#         tm = None
#     return tm
# df_capture['datetime_obj'] = df_capture['datetime_clean'].apply(
#     lambda x: create_datetime(x))
# df_capture2 = df_capture.groupby(['season', 'site', 'roll', 'capture']).datetime_obj.agg(['min', 'max'])
#
# df_capture2[df_capture2['min'] !=  df_capture2['max']]
# # TESTS
# df_final[(df_final['season'] == 'S1') &
# (df_final['site'] == 'C07') &
#  (df_final['roll'] == '2') &
#   (df_final['capture'] == '1042')]
#
# df_capture.drop_duplicates(subset=['season', 'site', 'roll', 'capture'], inplace=True)
# df_capture.shape
# # (2513837, 6) without excluding 'datetime_clean' != ''
# # (2503309, 5) with excluding 'datetime_clean' != ''
#
# # TESTS
# no_datetime = df_capture[df_capture['datetime_clean'] == '']
# df_final[(df_final['season'] == 'S2') &
# (df_final['site'] == 'H07') &
#  (df_final['roll'] == '2') &
#   (df_final['capture'] == '242')]['datetime']
# df_capture[(df_capture['season'] == 'S2') &
# (df_capture['site'] == 'H07') &
#  (df_capture['roll'] == '2') &
#   (df_capture['capture'] == '242')]['datetime']
#
# tt = df_capture[(df_capture['season'] == 'S2') &
# (df_capture['site'] == 'H07') &
#  (df_capture['roll'] == '2') &
#   (df_capture['capture'] == '242')]
# tt['datetime_obj'] = tt['datetime_clean'].apply(
#     lambda x: create_datetime(x))
# tt2 = tt.groupby(['season', 'site', 'roll', 'capture']).datetime_obj.agg(['min', 'max'])
#
# # SER_S8#L11#3#1224
# # has no datetime
# df_final[(df_final['season'] == 'S8') &
# (df_final['site'] == 'L11') &
#  (df_final['roll'] == '3') &
#   (df_final['capture'] == '1224')]['datetime']
#
#
#
# def create_capture_id(season, site, roll, capture):
#     return 'SER_{}#{}#{}#{}'.format(season, site, roll, capture)
#
# df_capture['capture_id'] = df_capture[['season', 'site', 'roll', 'capture']].apply(
#     lambda x: create_capture_id(*x), axis=1)
#
# # investigate
# df_capture.shape
#
#
# # SER_S1#L10#1#9
# df_merged2[(df_merged2['season'] == 'S1') & (df_merged2['site'] == 'L10') & (df_merged2['roll'] == '1') & (df_merged2['capture_export'] == '9')]
# # remove:
# # S1/L10/L10_R1/S1_L10_R1_PICT0023 (2).JPG
# # S1/L10/L10_R1/S1_L10_R1_PICT0024 (2).JPG
# # S1/L10/L10_R1/S1_L10_R1_PICT0025 (2).JPG
#
# # SER_S1#O05#1#1171
# df_merged2[(df_merged2['season'] == 'S1') &
#     (df_merged2['site'] == 'O05') &
#     (df_merged2['roll'] == '1') &
#     (df_merged2['capture_export'] == '1171')]
# # if no lila timestamp and if no timestamp accepted, take timestamp jpg
#
# # SER_S8#L11#3#
# df_merged2[(df_merged2['season'] == 'S8') &
#  (df_merged2['site'] == 'L11') &
#   (df_merged2['roll'] == '3') &
#   (df_merged2['capture_export'] == '1225')]
# # substitute capture export with capture clean if capture export is empty
# # some captures not in db but in cleaned?
#
#
#
# S1/O05/O05_R1/S1_O05_R1_PICT3650.JPG
# S1/O05/O05_R1/S1_O05_R1_PICT3711.JPG
# df_merged2[df_merged2['image_path'] == 'S1/O05/O05_R1/S1_O05_R1_PICT3650.JPG']
# df_merged2[df_merged2['image_path'] == 'S1/O05/O05_R1/S1_O05_R1_PICT3711.JPG']


import cv2
import matplotlib
import matplotlib.pylab as plt
import numpy as np

def cv_show(name, img):
    cv2.imshow(name, img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

img = cv2.imread('06_pie.png')

sobelx = cv2.Sobel(img, cv2.CV_64F, 1, 0, ksize=3)
# cv2.CV_64F是为了在np数值上保留负数值，但对于RGB图仍需作如下转换
sobelx = cv2.convertScaleAbs(sobelx)
# cv_show('sobel', sobelx)

sobely = cv2.Sobel(img, cv2.CV_64F, 0, 1, ksize=3)
sobely = cv2.convertScaleAbs(sobely)

res = cv2.addWeighted(sobelx, 0.5, sobely, 0.5, 0)
cv_show('edge', res)


# 규칙 찾아내서 풀기 (통과)
N = int(input())
if N == 1:
    print(N)
else:
    card_list = list(range(0, N+1, 2))
    for i in range(1, 20):
        if N <= 2**i:
            print(card_list[N - 2**(i-1)])
            break

# 큐로 풀기 (시간초과)
N = int(input())
card_list = list(range(1, N+1))

while len(card_list) > 1:
    card_list.pop(0)
    card_list.append(card_list.pop(0))
print(card_list[0])

# 덱으로 풀기 (통과)
from collections import deque

n = int(input())
queue = deque(range(1, n + 1))

while len(queue) > 1:
    queue.popleft()
    queue.append(queue.popleft())

print(queue[0])

# 신기한 풀이 1번이랑 비슷한데 더 스마트함
n = int(input())

a = 1
while n > a * 2:
    a *= 2

print((n - a) * 2 or n)