#!/usr/bin/python
""" Parse XML documents returned by Eutilities

Convert XML file to a tabular formatted file, e.g. Efetch (data record
downloads), or ESummary (document summary downloads).

The program/function produces rows by iterating over elements in the XML
file while collecting attributes and text material. Relations between
sub-sections are lost while migrating away from the XML structure. To try
to retain that information in the new document, this one contains a
field named "stack" that for each row shows stacked indices. The length
of the stack corresponds to the depth in the XML data structure at which
we are parsing, while the value corresponds to the index of the element
being parsed at that very level. In addition there is a field named "lvl"
that shows the level, or depth that were parsing, and a field named "idx",
showing the index of the child of the root element beeing worked at. This
value is the same as that at index position 1 in the stack. Indexing is
zero-based by the way. The advantage of using a tabular format is that
it might be easier to extract information, for instance using (Microsoft)
Excel or LibreOffice Calc. Other than those fields, the new document contains
a "text" field for texts and keywords from the attributes in each element.

From the Command-line: Use "-h/--help" for info on execution

"""

__author__ = "Johansson, O."
__email__ = "oscarpeterjohansson@outlook.com"
__contributors__ = ""
__version__ = "2.0"
__licence__ = "GPL-3"

import argparse
import sys
import traceback
import xml.etree.ElementTree as ET

DICT = {}
FIELDS = []
DTA = []

def tracker():
    """ traceback """
    t,v,tb = sys.exc_info()
    traceback.print_exception(t,v,tb,file=sys.stdout)

def file_parser(inputfile):
    """ Open file, parse xml string, return root """
    s_xml = None
    root = None
    try:
        with open(inputfile, 'r') as fd:
            s_xml = fd.read()
    except (IOError,) as e:
        tracker()
        return None
    try:
        root = ET.fromstring(s_xml)
    except (ET.ParseError,) as e:
        tracker()
        return None
    return root

def set_fields(root):
    """ Find and set fields """
    global DTA, FIELDS
    sf = set()
    sf.update(root.keys())
    # immediate children or root are identical, as child == ID.
    for elt in root:
        for elt in elt.iter():
            sf.update(elt.keys())
        if "ERROR" not in sf:
            break
    sf.update(["tag","stack","lvl","text","idx"])
    lf = list(sf)
    lf.sort()
    FIELDS = lf
    DTA.append(FIELDS)

def set_dict():
    """ Create dictionary that maps field to index"""
    global DICT, FIELDS
    DICT = dict([(FIELDS[i],i) for i in xrange(len(FIELDS))])

def iterate(elt, stack, lvl):
    """ Recursion - keep track of idx and level with stack """
    global DICT, DTA
    row = ["" for i in xrange(len(FIELDS))]
    for k in elt.keys():
        if k in DICT.keys():
            row[DICT.get(k)] = elt.get(k)
    if elt.text != None:
        row[DICT.get("text")] = elt.text
    row[DICT.get("tag")] = elt.tag
    row[DICT.get("lvl")] = lvl
    try:
        row[DICT.get("idx")] = stack[1] # child of root element
    except (IndexError,) as e:
        row[DICT.get("idx")] = "root"
    row[DICT.get("stack")] = ';'.join([str(i) for i in stack])
    row = [str(i) for i in row]
    DTA.append(row)
    idx = -1  # cheating a little: idx = 0
    lvl += 1
    for elt in elt:
        idx += 1
        iterate(elt, stack+[idx], lvl)

def write_file(l_dta, outputfile):
    """ Write dta to file """
    l_dta2 = []
    for row in l_dta:
        s = '\t'.join(row)
        l_dta2.append(s)
    s_dta = "\r\n".join(l_dta2)
    try:
        with open(outputfile, 'w') as fd:
            fd.write(s_dta)
    except (IOError,) as e:
        tracker()
    return None

parser = argparse.ArgumentParser(
    prog = sys.argv[0],
    description = """
    Parse XML documents returned by Eutilities

    Convert XML file to a tabular formatted file, e.g. Efetch (data record
    downloads), or ESummary (document summary downloads).

    The program/function produces rows by iterating over elements in the XML
    file while collecting attributes and text material. Relations between
    sub-sections are lost while migrating away from the XML structure. To try
    to retain that information in the new document, this one contains a
    field named "stack" that for each row shows stacked indices. The length
    of the stack corresponds to the depth in the XML data structure at which
    we are parsing, while the value corresponds to the index of the element
    being parsed at that very level. In addition there is a field named "lvl"
    that shows the level, or depth that were parsing at, and a field named
    "idx", showing the index of the child of the root element beeing worked
    at. This value is the same as that at index position 1 in the stack.
    Indexing is zero-based by the way. The advantage of using a tabular format
    is that  it might be easier to extract information, for instance using
    (Microsoft) Excel or LibreOffice Calc. Other than those fields, the new
    document contains a "text" field for texts and keywords from the attributes
    in each element.
    """,
    conflict_handler = "resolve",
    add_help = True
)

parser.add_argument(
    "--input",
    dest = "input",
    required = True,
    help = """
    Name of/Path to inputfile. It's expected that the inputfile to this
    program is the xml-formatted output from the efetch utility program.
    """
)

parser.add_argument(
    "--output",
    dest = "output",
    required = True,
    help = """
    Name of/Path to outputfile. This file will be tabulated.
    """
)

def main(parser, argv):
    """ for Command-line use
    """
    global DTA
    n_argv = parser.parse_args(argv)
    d_argv = vars(n_argv)
    root = file_parser(d_argv.get("input"))
    set_fields(root)
    set_dict()
    iterate(root, [0], 0)
    write_file(DTA, d_argv.get("output"))
    parser.exit(status=0, message=None)

if __name__ == "__main__":
    main(parser, sys.argv[1:])


import pandas
from sklearn.metrics import confusion_matrix, classification_report
import re
import numpy as np
import tensorflow as tf
from keras.models import Sequential
from keras.layers import Dense, Activation, Embedding
from keras.layers import LSTM
from keras.optimizers import RMSprop, SGD
from keras.utils import np_utils
from collections import Counter
import argparse
import time
import pickle

print("TF version: ", tf.__version__)
FILEPATH = "../data/amazon_reviews.csv"
RANDOM_SEED = 1
SAMPLE_FRAC = 0.1

def add_binary_label(df):
    """
    Converts the score attributes to a binary good/bad value. 1 and 2 is mapped to bad (0),
    4 and 5 is mapped to good (1) and 3 is removed
    :param df:
    :return:
    """
    new_array = []
    for e in df["Score"]:
        if e == 1 or e == 2:
            new_array.append(0)
        elif e == 3:
            new_array.append(-1)
        elif e == 4 or e == 5:
            new_array.append(1)
    new_df = df
    drop_indexes = [i for i, e in enumerate(new_array) if e == -1]
    new_df["label"] = new_array
    new_df = new_df.drop(drop_indexes)
    return new_df

def text_to_words(raw_tweet):
    """
    Only keeps ascii characters in the tweet and discards @words

    :param raw_tweet:
    :return:
    """
    letters_only = re.sub("[^a-zA-Z@]", " ", raw_tweet)
    words = letters_only.lower().split()
    meaningful_words = [w for w in words if not re.match("^[@]", w)]
    return " ".join(meaningful_words)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--verbose', help="Verbose output (enables Keras verbose output)", action='store_true', default=False)
    parser.add_argument('--gpu', help="Use LSTM/GRU gpu implementation", action='store_true', default=False)
    args = parser.parse_args()

    verbose = 1 if args.verbose else 2
    impl = 2 if args.gpu else 0

    start_time = time.time()
    print("Starting:", time.ctime())

    ############################################
    # Data

    # Load the data and select
    df = pandas.read_csv(FILEPATH)
    df = df[["Score", "Text"]]
    df = add_binary_label(df)
    df = df.sample(frac=SAMPLE_FRAC, random_state=RANDOM_SEED)

    # Make weighting adjustments for the solver to counter unbalanced data
    num_of_class_0 = len([i for i in df["label"] if i == 0])
    num_of_class_1 = len([i for i in df["label"] if i == 1])
    print(f"#Class 0: {num_of_class_0} \n#Class 1: {num_of_class_1}")
    scale = max(num_of_class_1, num_of_class_0)
    CLASS_WEIGHTS = {0 : scale/num_of_class_0, 1 : scale/num_of_class_1}

    # Pre-process the tweet and store in a separate column
    df['clean_text'] = df['Text'].apply(lambda x: text_to_words(x))

    # Join all the words in review to build a corpus
    all_text = ' '.join(df['clean_text'])
    words = all_text.split()

    # Convert words to integers
    counts = Counter(words)

    numwords = 5000  # Limit the number of words to use
    vocab = sorted(counts, key=counts.get, reverse=True)[:numwords]
    vocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}

    text_to_ints = []
    for each in df['clean_text']:
        text_to_ints.append([vocab_to_int[word] for word in each.split() if word in vocab_to_int])

    # Create a list of labels
    labels = np.array(df['label'])

    # Find the number of tweets with zero length after the data pre-processing
    text_len = Counter([len(x) for x in text_to_ints])
    print("Zero-length reviews: {}".format(text_len[0]))
    print("Maximum text length: {}".format(max(text_len)))

    # Remove those tweets with zero length and its corresponding label
    maximum_allowed_words = 500
    text_idx = [idx for idx, text in enumerate(text_to_ints) if len(text) > 0 and len(text) < maximum_allowed_words]
    labels = labels[text_idx]
    df = df.iloc[text_idx]
    text_to_ints = [text for text in text_to_ints if len(text) > 0 and len(text) < maximum_allowed_words]

    seq_len = min(max(text_len), maximum_allowed_words)
    features = np.zeros((len(text_to_ints), seq_len), dtype=int)
    for i, row in enumerate(text_to_ints):
        features[i, -len(row):] = np.array(row)[:seq_len]

    split_frac = 0.8
    split_idx = int(len(features) * 0.8)
    train_x, val_x = features[:split_idx], features[split_idx:]
    train_y, val_y = labels[:split_idx], labels[split_idx:]

    test_idx = int(len(val_x) * 0.5)
    val_x, test_x = val_x[:test_idx], val_x[test_idx:]
    val_y, test_y = val_y[:test_idx], val_y[test_idx:]

    print("\t\t\tFeature Shapes:")
    print("Train set: \t\t{}".format(train_x.shape),
          "\nValidation set: \t{}".format(val_x.shape),
          "\nTest set: \t\t{}".format(test_x.shape))

    print("Train set: \t\t{}".format(train_y.shape),
          "\nValidation set: \t{}".format(val_y.shape),
          "\nTest set: \t\t{}".format(test_y.shape))

    ############################################
    # Model
    drop = 0.0
    nlayers = 2  # >= 1
    RNN = LSTM  # GRU

    neurons = 128
    embedding = 40

    model = Sequential()
    model.add(Embedding(numwords + 1, embedding, input_length=seq_len, mask_zero=True))

    if nlayers == 1:
        model.add(RNN(neurons))
    else:
        model.add(RNN(neurons, return_sequences=True))
        for i in range(1, nlayers - 1):
            model.add(RNN(neurons, return_sequences=True))
        model.add(RNN(neurons))

    model.add(Dense(len(set(labels)), activation='softmax'))

    ############################################
    # Training

    learning_rate = 0.01
    optimizer = SGD(lr=learning_rate, momentum=0.95)
    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
    print(model.summary())

    epochs = 100
    batch_size = 256

    train_y_c = np_utils.to_categorical(train_y, len(set(labels)))
    val_y_c = np_utils.to_categorical(val_y, len(set(labels)))

    history = model.fit(train_x, train_y_c,
              batch_size=batch_size,
              epochs=epochs,
              validation_data=(val_x, val_y_c),
              verbose=verbose,
              class_weight=CLASS_WEIGHTS)
    file = open(f"""../output/history_{time.ctime().replace(" ", "_").replace(":", "~")}""", "wb")
    pickle.dump(history.history, file)
    file.close()

    model.save(f"""../output/model_{time.ctime().replace(" ", "_").replace(":", "~")}.h5""")

    ############################################
    # Results

    test_y_c = np_utils.to_categorical(test_y, len(set(labels)))
    score, acc = model.evaluate(test_x, test_y_c,
                                batch_size=batch_size,
                                verbose=verbose)
    print("Score:", score)
    print('Test ACC=', acc)

    test_pred = model.predict_classes(test_x, verbose=verbose)

    print()
    print('Confusion Matrix')
    print('-'*20)
    conf_matrix = confusion_matrix(test_y, test_pred)
    print(conf_matrix)
    print()
    print('Classification Report')
    print('-'*40)
    print(classification_report(test_y, test_pred))
    print()
    print("Ending:", time.ctime())
    print(f"Duration: {time.time()-start_time}")


'''

# DATA FROM CAMB

we use the _lenspotentialcls so

 l CTT CEE CBB CTE Cdd CdT CdE
 0  1  2    3   4   5   6  7

  CX are l(l+1)Cl/2pi and Cdd=[l(l+1)]^2 Clphi/2pi, CdT=[l(l+1)]^(3/2) ClphiT/2pi, CdE=[l(l+1)]^(3/2)ClphiE/2pi

'''

import numpy as np
import pickle as pickle
#import sys
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from scipy import integrate
import scipy.special
import scipy.interpolate
from itertools import cycle
#from matplotlib.colors import colorConverter
from matplotlib.ticker import MaxNLocator  # needed for tick
import n_priors_code.utils as utils

plot_type ='lmax'

no_lcdm_parameters = ['massless_neutrinos', 'w', 'omnuh2', 'helium_fraction']
plot_now = ['scalar_spectral_index(1)','omnuh2']
excluded_parameters = list(set(no_lcdm_parameters) - set(plot_now))
# omnuh2

# READ DATA
# DEFINE YOUR FOLDER HERE
base_dir = '/home/manzotti/n_eff-dependence-on-prior/n_priors_code/'
data_type = 'varying+Yp'
run_idx = 1
lmax = 3000
lmin = 50
N_det = 10 ** 6
# N_phi_l = np.loadtxt('data/noise/wu_cdd_noise_5.txt')
fsky = 0.75
# ======
fid = pickle.load(open(base_dir + 'data/{}/run{}/fid_values.p'.format(data_type, str(run_idx)), "rb"))
values = pickle.load(open(base_dir + 'data/{}/run{}/grid_values.p'.format(data_type, str(run_idx)), "rb"))
par_gaps = pickle.load(open(base_dir + 'data/{}/run{}/par_gaps.p'.format(data_type, str(run_idx)), "rb"))
fisher_mat = np.loadtxt(
    base_dir + 'data/{}/run{}/output/fisher_mat_joint_lmin={}_lmax={}_ndet={}_fsky={}.txt'.format(data_type, str(run_idx), lmin, lmax, N_det, fsky))

par_gaps, values, fid, fisher_mat = utils.exclude_parameters_from_fisher(
    excluded_parameters, par_gaps, values, fid, fisher_mat)
plot_param = list(set(fid.keys()) - set(excluded_parameters))
sigma1 = np.loadtxt(base_dir + 'data/{}/run{}/output/marginalized_ell_joint_lmin={}_lmax={}_ndet={}_fsky={}.txt'.format(
    data_type, str(run_idx), lmin, lmax, N_det, fsky))
sigma_no = np.loadtxt(base_dir + 'data/{}/run{}/output/no_marginalized_ell_joint_lmin={}_lmax={}_ndet={}_fsky={}.txt'.format(
    data_type, str(run_idx), lmin, lmax, N_det, fsky))
full_fisher = np.load(base_dir+ 'data/{}/run{}/output/full_fisher_mat_joint_lmin={}_lmax={}_ndet={}_fsky={}.npy'.format(data_type, str(run_idx),lmin,lmax,N_det,fsky))

# ============================================
# ============================================
# ============================================

# ============================================
# SIZE OF THE PICTURE
# ============================================
WIDTH = 246.0  # the number latex spits out
# the fraction of the width you'd like the figure to occupy, 0.5 if double
# column cause
FACTOR = 0.9
fig_width_pt = WIDTH * FACTOR

inches_per_pt = 1.0 / 72.27
# golden_ratio  = (np.sqrt(5) - 1.0) / 2.0  # because it looks good
ratio = 0.8
fig_width_in = fig_width_pt * inches_per_pt  # figure width in inches
fig_height_in = fig_width_in * ratio   # figure height in inches
fig_dims = [fig_width_in, fig_height_in]  # fig dims as a list
# ============================================
# ============================================

# ============================================
# SET LATEX
plt.rc('text', usetex=True)
plt.rc('font', **{'family': 'serif', 'serif': ['Computer Modern Roman']})
plt.rcParams['font.weight'] = 400

# ============================================

# ============================================
# FONT SIZES
font_size = 25
plt.rcParams['font.size'] = font_size
plt.rcParams['axes.labelsize'] = font_size / 1.2
plt.rcParams['axes.linewidth'] = font_size / 22.
plt.rcParams['axes.titlesize'] = font_size / 1.1
plt.rcParams['legend.fontsize'] = font_size / 1.7
plt.rcParams['xtick.labelsize'] = font_size / 2
plt.rcParams['ytick.labelsize'] = font_size / 2

plt.rcParams['lines.linewidth'] = font_size / 1000

# ============================================
# ============================================
# TICKS

plt.rcParams['xtick.major.width'] = 0.13
plt.rcParams['xtick.major.size'] = 5
plt.rcParams['xtick.minor.width'] = 0.13
plt.rcParams['xtick.minor.size'] = 2.8
plt.rcParams['ytick.major.width'] = 0.13
plt.rcParams['ytick.major.size'] = 5
plt.rcParams['ytick.minor.width'] = 0.13
plt.rcParams['ytick.minor.size'] = 2.8

# fg = plt.figure(figsize=fig_dims)
fg = plt.figure(figsize=(8, 8))

ax1 = plt.subplot2grid((1, 1), (0, 0))  # , rowspan=2)
#ax2 = plt.subplot2grid((1,2), (0, 1))
#ax3 = plt.subplot2grid((2,2), (1, 1))

# ============================================
# set colors and lines
lines = ["-", "--", "-.", ":"]
linecycler = cycle(lines)
ax1.set_color_cycle(['#38197A', '#FC7D00', '#318B10', '#B61911'])
colorcycler = cycle(['#38197A', '#FC7D00', '#318B10', '#B61911'])

# ============================================

# ============================================

# Simplify paths by removing "invisible" points, useful for reducing
# file size when plotting a large number of points
plt.rcParams['path.simplify'] = False
# ============================================

# ============================================

# Have the legend only plot one point instead of two, turn off the
# frame, and reduce the space between the point and the label

plt.rcParams['axes.linewidth'] = 1.0
#plt.rc("lines", markeredgewidth=10)
# ============================================

# ============================================
# LEGEND
# ============================================
# OPTION
plt.rcParams['legend.frameon'] = False
plt.rcParams['legend.numpoints'] = 3
plt.rcParams['legend.handletextpad'] = 0.3

# ============================================
# ============================================
# ============================================
# ============================================
# ============================================
# REAL PLOT HERE
# ============================================
# data5 is> data6 so he probaly does + bedfore -
# plot1 = ax1.semilogx(data1[:,0], (data5[:,1]-data6[:,1])/pargaps[3]/data0[:,1],
# next(linecycler), linewidth=1, color='r',label=r'$C^{T}$')

# plot2 = plt.semilogx(data_fid[:,0] , 10.*np.nan_to_num((dats[:,1,]- dats[:,1,] )/data_fid[:,1]),linewidth=1, color='b',label=r'$C^{T}$')
key = 'scalar_spectral_index(1)'

if plot_type=='lmin':
    sigma_test = [ np.nan_to_num(np.sqrt((np.diag(np.linalg.inv(np.sum(full_fisher[:,:,i:], axis =2)))[fid.keys().index(key)]))) for i,j in enumerate(sigma1[:, 0])]

elif plot_type=='lmax':
    sigma_test = [ np.nan_to_num(np.sqrt((np.diag(np.linalg.inv(np.sum(full_fisher[:,:,:i], axis =2)))[fid.keys().index(key)]))) for i,j in enumerate(sigma1[:, 0])]
else:
    sys.exit('plot type wrong')

label = {}

label['massless_neutrinos'] = 'N_{eff}'
label['hubble'] = 'H_{0}'
label['mnu'] = 'M'
label['scalar_amp(1)'] = 'A_{s}'
label['scalar_spectral_index(1)'] = 'n_{s}'
label['omnuh2'] = r'\Omega_{\nu}h^{2}'
label['re_optical_depth'] = r'\tau'
label['ombh2'] = '\Omega_{b}h^{2}'
label['ombch2'] = '\Omega_{m}h^{2}'
label['omch2'] = '\Omega_{c}h^{2}'
label['helium_fraction'] = 'Y_{p}'
label['w'] = 'w'

plot = plt.plot
# plot2 = plot_type(sigma1[:, 0], sigma1[:, fid.keys().index(key) + 1], linewidth=1, label='Degeneracies')
plot2 = plot(sigma1[:80, 0], sigma_test[:80], linewidth=1)

# plot2 = plot_type(sigma_no[:, 0], sigma_no[:, fid.keys().index(key) + 1],
#                   linewidth=1, label='Perfect priors')

# plot2=plt.axvline(0.01,linewidth=1, ls= '--')

legend = ax1.legend()
ax1.legend(loc=0)

# ============================================
# FINALLY SAVE
ax1.set_ylabel(r'$ \sigma(' + label[key]+ ')$')

if plot_type =='lmax': ax1.set_xlabel(r'$\ell_{\mathrm{max}}$')

if plot_type =='lmin': ax1.set_xlabel(r'$\ell_{\mathrm{min}}$')

# ax1.set_xlim((0, 2000))
# ax1.set_ylim((10 ** -9, 10 ** -3))
ax1.minorticks_on()
# ax1.set_xscale('log')
# ax1.set_yscale('log')
# ============================================
# ============================================

# ============================================

plt.savefig(base_dir + 'data/{}/run{}/output/sigma_ell_{}_snow_mass_lmin={}_lmax={}_ndet={}_fsky={}.png'.format(data_type, str(run_idx), str(key), lmin, lmax, N_det, fsky), dpi=400, papertype='Letter',
            format='png', bbox_inches='tight')

plt.close()


# Standard imports
import unittest
import datetime as pydt
import logging
import uuid
import json

# Our imports
import emission.storage.decorations.trip_queries as esdt
import emission.storage.decorations.section_queries as esds
import emission.storage.decorations.stop_queries as esdst
import emission.net.usercache.abstract_usercache as enua
import emission.core.get_database as edb

class TestTripQueries(unittest.TestCase):
    def setUp(self):
        self.testUserId = uuid.uuid4()
        edb.get_trip_new_db().remove()

    def create_fake_trip(self):
        new_trip = esdt.create_new_trip(self.testUserId)
        new_trip.start_ts = 5
        new_trip.end_ts = 6
        esdt.save_trip(new_trip)
        return new_trip

    def testCreateNew(self):
        new_trip = esdt.create_new_trip(self.testUserId)
        self.assertIsNotNone(new_trip.get_id())
        self.assertEqual(new_trip.user_id, self.testUserId)

    def testSaveTrip(self):
        new_trip = self.create_fake_trip()
        self.assertEqual(edb.get_trip_new_db().find({"end_ts": 6}).count(), 1)
        self.assertEqual(edb.get_trip_new_db().find_one({"end_ts": 6})["_id"], new_trip.get_id())
        self.assertEqual(edb.get_trip_new_db().find_one({"end_ts": 6})["user_id"], self.testUserId)

    def testQueryTrips(self):
        new_trip = self.create_fake_trip()
        ret_arr_time = esdt.get_trips(self.testUserId, enua.UserCache.TimeQuery("start_ts", 4, 6))
        self.assertEqual(ret_arr_time, [new_trip])

    def testGetTrip(self):
        new_trip = self.create_fake_trip()
        ret_trip = esdt.get_trip(new_trip.get_id())
        self.assertEqual(ret_trip, new_trip)

    def testGetTimeRangeForTrip(self):
        new_trip = self.create_fake_trip()
        ret_tq = esdt.get_time_query_for_trip(new_trip.get_id())
        self.assertEqual(ret_tq.timeType, "write_ts")
        self.assertEqual(ret_tq.startTs, 5)
        self.assertEqual(ret_tq.endTs, 6)

    def testQuerySectionsForTrip(self):
        new_trip = self.create_fake_trip()
        new_section = esds.create_new_section(self.testUserId, new_trip.get_id())
        new_section.start_ts = 5
        new_section.end_ts = 6
        esds.save_section(new_section)
        ret_sections = esdt.get_sections_for_trip(self.testUserId, new_trip.get_id())
        self.assertEqual(ret_sections, [new_section])

    def testQueryStopsForTrip(self):
        new_trip = self.create_fake_trip()
        new_stop = esdst.create_new_stop(self.testUserId, new_trip.get_id())
        new_stop.enter_ts = 5
        new_stop.exit_ts = 6
        esdst.save_stop(new_stop)
        ret_stops = esdt.get_stops_for_trip(self.testUserId, new_trip.get_id())
        self.assertEqual(ret_stops, [new_stop])

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    unittest.main()


from time import sleep
from datetime import datetime
from sh import gphoto2 as gp
import signal, os, subprocess
import RPi.GPIO as GPIO

#setup GPIO, change pin as required.
GPIO.setmode(GPIO.BOARD)
btn_pin = 33
GPIO.setup(btn_pin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
buttonPressedTime = None

#setup gphoto2

#kill gphoto2 process that starts whenever we connect the camera
def killgphoto2Process ():
    p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE)
    out, err = p.communicate()

# Search for the line that has the process we want to kill
    for line in out.splitlines():
        if b'gvfsd-gphoto2' in line:
            #kill the process!
            pid = int(line.split(None,1) [0])
            os.kill(pid, signal.SIGKILL)

shot_date = datetime.now().strftime ("%Y-%m-%d")
shot_time = datetime.now().strftime ("%Y-%m-%d %H:%M:%S")

triggerCommand = ["--capture-image"]

def captureImages():
    gp(triggerCommand)
    sleep(3)

#countdown
def countdown(n):
    if n == 0:
        print ("Smile!!")
    else:
        sleep(2)
        print(n)
        countdown(n-1)

#detect the button being pressed
def buttonStateChanged(pin):
    global buttonPressedTime

    if not (GPIO.input(pin)):
        if buttonPressedTime is None:
            buttonPressedTime = datetime.now()
            print ('Button Pressed')

    else:
        if buttonPressedTime is not None:
            countdown (3)
            killgphoto2Process()
            captureImages()
            sleep(1)
            killgphoto2Process()
            captureImages()
            buttonPressedTime = None

#read button
GPIO.add_event_detect(btn_pin, GPIO.BOTH, callback=buttonStateChanged)

while True:
    sleep(5)


from django.urls import reverse_lazy, reverse
from core.views.mydatatableview import MyDatatable, columns, helpers
from core.views import crud
from core.models import Vacance
from parametrage.forms.vacances import Formulaire

class Page(crud.Page):
    model = Vacance
    url_liste = "vacances_liste"
    url_ajouter = "vacances_ajouter"
    url_modifier = "vacances_modifier"
    url_supprimer = "vacances_supprimer"
    description_liste = "Voici ci-dessous la liste des périodes de vacances."
    description_saisie = "Saisissez toutes les informations concernant la période de vacances à saisir et cliquez sur le bouton Enregistrer."
    objet_singulier = "une période de vacances"
    objet_pluriel = "des périodes de vacances"
    boutons_liste = [
        {"label": "Ajouter", "classe": "btn btn-success", "href": reverse_lazy(url_ajouter), "icone": "fa fa-plus"},
        {"label": "Importer depuis internet", "classe": "btn btn-default", "href": reverse_lazy("vacances_importation", args="a"), "icone": "fa fa-download"}
    ]

class Liste(Page, crud.Liste):
    model = Vacance

    def get_queryset(self):
        return Vacance.objects.filter(self.Get_filtres("Q"))

    def get_context_data(self, **kwargs):
        context = super(Liste, self).get_context_data(**kwargs)
        context['impression_introduction'] = ""
        context['impression_conclusion'] = ""
        context['afficher_menu_brothers'] = True
        return context

    class datatable_class(MyDatatable):
        filtres = ['idvacance', 'nom', 'annee', 'date_debut', 'date_fin']

        actions = columns.TextColumn("Actions", sources=None, processor='Get_actions_standard')

        class Meta:
            structure_template = MyDatatable.structure_template
            columns = ['idvacance', 'nom', 'annee', 'date_debut', 'date_fin', 'actions']
            processors = {
                'date_debut': helpers.format_date('%d/%m/%Y'),
                'date_fin': helpers.format_date('%d/%m/%Y'),
            }
            ordering = ['date_debut']

class Ajouter(Page, crud.Ajouter):
    form_class = Formulaire

class Modifier(Page, crud.Modifier):
    form_class = Formulaire

class Supprimer(Page, crud.Supprimer):
    pass


"""Basic Message Tests"""
import asyncio
import json
import pytest

from acapy_client import Client
from acapy_client.api.basicmessage import send_basicmessage
from acapy_client.models.send_message import SendMessage
from acapy_client.models import ConnectionStaticResult
from echo_agent.client import EchoClient
from echo_agent.models import ConnectionInfo
from redis.asyncio import RedisCluster
from redis.exceptions import RedisClusterException, RedisError

PAYLOAD_B64 = """
    eyJwcm90ZWN0ZWQiOiAiZXlKbGJtTWlPaUFpZUdOb1lXTm9ZVEl3Y0c5c2VURXpNRFZmYVdWM
    FppSXNJQ0owZVhBaU9pQWlTbGROTHpFdU1DSXNJQ0poYkdjaU9pQWlRWFYwYUdOeWVYQjBJaX
    dnSW5KbFkybHdhV1Z1ZEhNaU9pQmJleUpsYm1OeWVYQjBaV1JmYTJWNUlqb2dJakZqWjNsMFF
    tMTNNM0V4YUdkaVZ6Qkpiak50U0c4MldXaExUMnRwUnpWRWVUaHJSakpJV2pZeGNUSnZXV00z
    Ym10dVN6bE9TVWMyU0VobFUyTm9lV0VpTENBaWFHVmhaR1Z5SWpvZ2V5SnJhV1FpT2lBaU5FU
    kNTalJhY0RnMU1XZHFlazUwU20xdGIwVTVOMWR4Vm5KWFRqTTJlVnBTWVVkcFpqUkJSM0o0ZD
    FFaUxDQWljMlZ1WkdWeUlqb2dJak5XY0hsU2NVRlpUV3N5Tms1RmMwUXpObU5mWjJnMFZIazB
    aamd3TUd4RFJHRXdNMWxsUlc1bVJYQm1WMmhKTFdkelpFY3RWR1JrTVdWTmFEbFpTWG8zTkhS
    RlN6SnNSMVZhVFhwZk5HdDFkMEpUVWtvMFRGOWhkMVJLUVZWVmQydFRWbmhyTXpSblVWVmZOV
    2RyZDFSa09FWTFUa0ZsU1U1UVZTSXNJQ0pwZGlJNklDSnFWVkpDUW1OaVQzZzNOa05zVmw4eG
    F6aFJNMjlyVW5KdFJHUTFhM0JwUWlKOWZWMTkiLCAiaXYiOiAiTVdnR3VRNF9ab2dxVVJUbiI
    sICJjaXBoZXJ0ZXh0IjogIlVNTGFQOU13ZF9wOFR1bWdwcVZWQWZTSWZXc1g3a0lWLUR4Rndf
    VHRTQ2pWdTVTbG5RYmtkTVJLd3VyZGI1dmd6Q0tUNUFybFV0WEFMMm1sSUlpUGpSYzVmSzhLc
    013S0dFemkycEtrdmxDN1EzUXRKWTE5WmVTSjlYMGlUOWxOamNEM25KS0o1bzlkSjhVWGZpNU
    80ZEtaLWxlVy1qOHlzTEFTSTh1eEZYVVNoUmxlNy03bm5HZkZnRlZBRjNaWVpqNlRXUUJrdkd
    SUk96TzMwTHNEWHBzalNqMWZfd056RWdxTmpPMERZemRKa0lBNm1BQ1AiLCAidGFnIjogImVB
    ZVFiakktVmpkN21hcWdTNElGTlEifQ==
"""

@pytest.mark.asyncio
async def test_event_pushed_to_redis(
    agent_connection: ConnectionStaticResult, echo: EchoClient, suite_seed: str, redis
):
    conn = await echo.new_connection(
        seed=suite_seed,
        endpoint=agent_connection.my_endpoint,
        their_vk=agent_connection.my_verkey,
    )
    assert conn
    assert await redis.blpop("acapy-record-base", 10)
    assert await redis.blpop("acapy-record-with-state-base", 10)
    # await echo.send_message(
    #     conn,
    #     {
    #         "@type": "https://didcomm.org/basicmessage/1.0/message",
    #         "content": "Your hovercraft is full of eels.",
    #     },
    # )
    # assert await redis.blpop("acapy-basicmessage-received", 10)

@pytest.mark.asyncio
async def test_outbound_queue(backchannel: Client, connection_id: str, redis):
    await send_basicmessage.asyncio(
        client=backchannel,
        conn_id=connection_id,
        json_body=SendMessage(content="test"),
    )
    msg_received = False
    retry_pop_count = 0
    while not msg_received:
        msg = await redis.blpop("acapy_outbound", 10)
        if not msg:
            if retry_pop_count > 3:
                raise Exception("blpop call failed to retrieve message")
            retry_pop_count = retry_pop_count + 1
            await asyncio.sleep(1)
        msg_received = True

@pytest.mark.asyncio
async def test_deliverer(
    backchannel: Client,
    connection_id: str,
    echo: EchoClient,
    connection: ConnectionInfo,
):
    await send_basicmessage.asyncio(
        client=backchannel,
        conn_id=connection_id,
        json_body=SendMessage(content="test"),
    )
    await asyncio.sleep(5)
    message = await echo.get_message(connection)
    assert message["content"] == "test"

@pytest.mark.asyncio
async def test_deliverer_retry_on_failure(
    backchannel: Client,
    connection_id: str,
    redis,
):
    outbound_msg = {
        "service": {"url": "http://echo:3002/fake/"},
        "payload": PAYLOAD_B64,
    }
    # produce a outbound message with bad enpoint
    await redis.rpush(
        "acapy_outbound",
        str.encode(json.dumps(outbound_msg)),
    )
    # assume failure code 400, delay queue, and failure code 400 ...
    await asyncio.sleep(15)
    msg = await redis.blpop("acapy_outbound", 15)
    assert msg
    # check for manual commit of previous message by handling a new message
    await send_basicmessage.asyncio(
        client=backchannel,
        conn_id=connection_id,
        json_body=SendMessage(content="test2"),
    )
    msg_received = False
    retry_pop_count = 0
    while not msg_received:
        msg = await redis.blpop("acapy_outbound", 10)
        if not msg:
            if retry_pop_count > 3:
                raise Exception("blpop call failed to retrieve message")
            retry_pop_count = retry_pop_count + 1
            await asyncio.sleep(1)
        msg_received = True


import datetime, uuid
from django.urls import reverse_lazy, reverse
from django.http import JsonResponse, HttpResponseRedirect
from django.db.models import Q
from core.views.mydatatableview import MyDatatable, columns, helpers
from core.views import crud
from core.models import Location, Vacance, Ferie, Produit, TarifProduit, Prestation
from core.utils import utils_dates
from fiche_famille.forms.famille_locations import Formulaire, FORMSET_PRESTATIONS
from fiche_famille.views.famille import Onglet

def Get_tarif_location(request):
    # Récupération des variables
    idproduit = request.POST.get("idproduit")
    if not idproduit:
        return JsonResponse({"erreur": "Vous devez sélectionner un produit"}, status=401)
    quantite = int(request.POST.get("quantite"))
    try:
        date_debut = datetime.datetime.strptime(request.POST.get("date_debut"), "%d/%m/%Y %H:%M").date()
    except:
        return JsonResponse({"erreur": "La date de début semble erronée"}, status=401)

    # Importation du produit
    produit = Produit.objects.get(pk=int(idproduit))
    if produit.montant:
        # Si tarif simple
        return JsonResponse({
            "date": utils_dates.ConvertDateToFR(date_debut),
            "label": produit.nom,
            "montant": produit.montant,
            "tva": 0.0,
        })
    else:
        # Si tarifs avancés
        tarifs = TarifProduit.objects.filter(produit=produit).order_by("date_debut")
        tarif_trouve = None
        for tarif in tarifs:
            if tarif.date_debut <= date_debut and (not tarif.date_fin or tarif.date_fin >= date_debut):
                montant = tarif.montant if tarif.methode == "produit_montant_unique" else tarif.montant * quantite
                tarif_trouve = {
                    "date": utils_dates.ConvertDateToFR(date_debut),
                    "label": produit.nom,
                    "montant": montant,
                    "tva": tarif.tva or 0.0,
                }
        if tarif_trouve:
            return JsonResponse(tarif_trouve)

class Page(Onglet):
    model = Location
    url_liste = "famille_locations_liste"
    url_ajouter = "famille_locations_ajouter"
    url_modifier = "famille_locations_modifier"
    url_supprimer = "famille_locations_supprimer"
    url_supprimer_plusieurs = "famille_locations_supprimer_plusieurs"
    description_liste = "Saisissez ici les locations de la famille."
    description_saisie = "Saisissez toutes les informations concernant la location et cliquez sur le bouton Enregistrer."
    objet_singulier = "une location"
    objet_pluriel = "des locations"

    def get_context_data(self, **kwargs):
        """ Context data spécial pour onglet """
        context = super(Page, self).get_context_data(**kwargs)
        if not hasattr(self, "verbe_action"):
            context['box_titre'] = "Locations"
        context['onglet_actif'] = "locations"
        context['boutons_liste'] = [
            {"label": "Ajouter", "classe": "btn btn-success", "href": reverse_lazy(self.url_ajouter, kwargs={'idfamille': self.kwargs.get('idfamille', None)}), "icone": "fa fa-plus"},
        ]
        # Ajout l'idfamille à l'URL de suppression groupée
        context['url_supprimer_plusieurs'] = reverse_lazy(self.url_supprimer_plusieurs, kwargs={'idfamille': self.kwargs.get('idfamille', None), "listepk": "xxx"})
        return context

    def get_form_kwargs(self, **kwargs):
        """ Envoie l'idfamille au formulaire """
        form_kwargs = super(Page, self).get_form_kwargs(**kwargs)
        form_kwargs["idfamille"] = self.Get_idfamille()
        return form_kwargs

    def get_success_url(self):
        url = self.url_liste
        if "SaveAndNew" in self.request.POST:
            url = self.url_ajouter
        return reverse_lazy(url, kwargs={'idfamille': self.kwargs.get('idfamille', None)})

class Liste(Page, crud.Liste):
    model = Location
    template_name = "fiche_famille/famille_liste.html"

    def get_queryset(self):
        return Location.objects.select_related("produit").filter(Q(famille=self.Get_idfamille()) & self.Get_filtres("Q"))

    def get_context_data(self, **kwargs):
        context = super(Liste, self).get_context_data(**kwargs)
        context['impression_introduction'] = ""
        context['impression_conclusion'] = ""
        context['active_checkbox'] = True
        return context

    class datatable_class(MyDatatable):
        filtres = ["idlocation", "date_debut", "date_fin"]
        check = columns.CheckBoxSelectColumn(label="")
        actions = columns.TextColumn("Actions", sources=None, processor='Get_actions_speciales')
        nom_produit = columns.TextColumn("Nom", sources=["produit__nom"], processor='Get_nom_produit')

        class Meta:
            structure_template = MyDatatable.structure_template
            columns = ["check", "idlocation", "nom_produit", "date_debut", "date_fin", "actions"]
            processors = {
                'date_debut': helpers.format_date('%d/%m/%Y %H:%M'),
                'date_fin': helpers.format_date('%d/%m/%Y %H:%M'),
            }
            ordering = ['date_debut']

        def Get_nom_produit(self, instance, *args, **kwargs):
            return instance.produit.nom

        def Get_actions_speciales(self, instance, *args, **kwargs):
            view = kwargs["view"]
            kwargs = view.kwargs
            kwargs["pk"] = instance.pk
            html = [
                self.Create_bouton_modifier(url=reverse(view.url_modifier, kwargs=kwargs)),
                self.Create_bouton_supprimer(url=reverse(view.url_supprimer, kwargs=kwargs)),
                self.Create_bouton_imprimer(url=reverse("famille_voir_location", kwargs={"idfamille": kwargs["idfamille"], "idlocation": instance.pk}), title="Imprimer ou envoyer par email la location"),
            ]
            return self.Create_boutons_actions(html)

class Ajouter(Page, crud.Ajouter):
    form_class = Formulaire
    template_name = "fiche_famille/famille_edit.html"

    def get_context_data(self, **kwargs):
        """ Context data spécial pour onglet """
        context = super(Page, self).get_context_data(**kwargs)
        if self.request.POST:
            context["formset_prestations"] = FORMSET_PRESTATIONS(self.request.POST, instance=self.object)
        else:
            context["formset_prestations"] = FORMSET_PRESTATIONS(instance=self.object)
        return context

    def form_valid(self, form):
        resultat = Form_valid_ajouter(form=form, request=self.request, object=self.object)
        if resultat == True:
            return HttpResponseRedirect(self.get_success_url())
        else:
            return self.render_to_response(self.get_context_data(form=resultat))

class Modifier(Page, crud.Modifier):
    form_class = Formulaire
    template_name = "fiche_famille/famille_edit.html"

    def get_context_data(self, **kwargs):
        """ Context data spécial pour onglet """
        context = super(Page, self).get_context_data(**kwargs)
        if self.request.POST:
            context["formset_prestations"] = FORMSET_PRESTATIONS(self.request.POST, instance=self.object)
        else:
            context["formset_prestations"] = FORMSET_PRESTATIONS(instance=self.object)
        return context

    def form_valid(self, form):
        resultat = Form_valid_modifier(form=form, request=self.request, object=self.object)
        if isinstance(resultat, Location):
            self.object = resultat
            return HttpResponseRedirect(self.get_success_url())
        else:
            return self.render_to_response(self.get_context_data(form=resultat))

class Supprimer(Page, crud.Supprimer):
    template_name = "fiche_famille/famille_delete.html"

    def Check_protections(self, objet=None):
        protections = []
        if Prestation.objects.filter(location=objet, facture__isnull=False):
            protections.append("Vous ne pouvez pas supprimer cette location car au moins une prestation facturée y est associée")
        return protections

class Supprimer_plusieurs(Page, crud.Supprimer_plusieurs):
    template_name = "fiche_famille/famille_delete.html"

    def Check_protections(self, objets=[]):
        protections = []
        prestations_facturees = {}
        for prestation in Prestation.objects.filter(location__in=objets, facture__isnull=False):
            prestations_facturees[prestation.location_id] = True
        for location in objets:
            if location.pk in prestations_facturees:
                protections.append("La location ID%d est déjà facturée" % location.pk)
        return protections

def Form_valid_ajouter(form=None, request=None, object=None):
    # Vérification du formulaire
    if not form.is_valid():
        return form

    # Vérification du formset des prestations
    formset_prestations = FORMSET_PRESTATIONS(request.POST, instance=object)
    if not formset_prestations.is_valid():
        return form

    # Sauvegarde de la location
    liste_locations = []
    if form.cleaned_data["selection_periode"] == "UNIQUE":
        object = form.save()
        liste_locations.append(object)

    if form.cleaned_data["selection_periode"] == "RECURRENCE":
        occurences = Calcule_occurences(form.cleaned_data)
        serie = str(uuid.uuid4())
        for occurence in occurences:
            location = Location.objects.create(
                famille=form.cleaned_data["famille"],
                produit=form.cleaned_data["produit"],
                observations=form.cleaned_data["observations"],
                date_debut=occurence["date_debut"],
                date_fin=occurence["date_fin"],
                quantite=form.cleaned_data["quantite"],
                serie=serie,
            )
            liste_locations.append(location)

    # Sauvegarde des prestations
    if formset_prestations.is_valid():
        for formline in formset_prestations.forms:
            if formline.cleaned_data.get('DELETE') and form.instance.pk and formline.instance.pk:
                formline.instance.delete()
            if formline.cleaned_data and not formline.cleaned_data.get('DELETE'):
                for location in liste_locations:
                    prestation = formline.save(commit=False)
                    prestation.pk = None
                    prestation.location = location
                    prestation.date = location.date_debut.date()
                    prestation.famille = form.cleaned_data["famille"]
                    prestation.montant_initial = prestation.montant
                    prestation.categorie = "location"
                    prestation.save()

    return True

def Form_valid_modifier(form=None, request=None, object=None):
    # Vérification du formulaire
    if not form.is_valid():
        return form

    # Vérification du formset des prestations
    formset_prestations = FORMSET_PRESTATIONS(request.POST, instance=object)
    if not formset_prestations.is_valid():
        return form

    # Sauvegarde de la location
    object = form.save()

    # Sauvegarde des prestations
    if formset_prestations.is_valid():
        for formline in formset_prestations.forms:
            if formline.cleaned_data.get('DELETE') and form.instance.pk and formline.instance.pk:
                formline.instance.delete()
            if formline.cleaned_data and not formline.cleaned_data.get('DELETE'):
                prestation = formline.save(commit=False)
                prestation.location = object
                prestation.date = object.date_debut.date()
                prestation.famille = form.cleaned_data["famille"]
                prestation.montant_initial = prestation.montant
                prestation.categorie = "location"
                prestation.save()

    return object

def Calcule_occurences(cleaned_data={}):
    """ Calcule les occurences """
    liste_resultats = []

    liste_vacances = Vacance.objects.all()
    liste_feries = Ferie.objects.all()

    cleaned_data["recurrence_jours_vacances"] = [int(x) for x in cleaned_data["recurrence_jours_vacances"]]
    cleaned_data["recurrence_jours_scolaires"] = [int(x) for x in cleaned_data["recurrence_jours_scolaires"]]

    # Liste dates
    listeDates = [cleaned_data["recurrence_date_debut"],]
    tmp = cleaned_data["recurrence_date_debut"]
    while tmp < cleaned_data["recurrence_date_fin"]:
        tmp += datetime.timedelta(days=1)
        listeDates.append(tmp)

    date = cleaned_data["recurrence_date_debut"]
    numSemaine = int(cleaned_data["recurrence_frequence_type"])
    dateTemp = date
    for date in listeDates:

        # Vérifie période et jour
        valide = False
        if utils_dates.EstEnVacances(date=date, liste_vacances=liste_vacances):
            if date.weekday() in cleaned_data["recurrence_jours_vacances"]:
                valide = True
        else:
            if date.weekday() in cleaned_data["recurrence_jours_scolaires"]:
                valide = True

        # Calcul le numéro de semaine
        if len(listeDates) > 0:
            if date.weekday() < dateTemp.weekday():
                numSemaine += 1

        # Fréquence semaines
        if cleaned_data["recurrence_frequence_type"] in (2, 3, 4):
            if numSemaine % cleaned_data["recurrence_frequence_type"] != 0:
                valide = False

        # Semaines paires et impaires
        if valide == True and cleaned_data["recurrence_frequence_type"] in (5, 6):
            numSemaineAnnee = date.isocalendar()[1]
            if numSemaineAnnee % 2 == 0 and cleaned_data["recurrence_frequence_type"] == 6:
                valide = False
            if numSemaineAnnee % 2 != 0 and cleaned_data["recurrence_frequence_type"] == 5:
                valide = False

        # Vérifie si férié
        if cleaned_data["recurrence_feries"] and utils_dates.EstFerie(date, liste_feries):
            valide = False

        # Application
        if valide:
            date_debut_final = datetime.datetime(year=date.year, month=date.month, day=date.day, hour=cleaned_data["recurrence_heure_debut"].hour, minute=cleaned_data["recurrence_heure_debut"].minute)
            date_fin_final = datetime.datetime(year=date.year, month=date.month, day=date.day, hour=cleaned_data["recurrence_heure_fin"].hour, minute=cleaned_data["recurrence_heure_fin"].minute)
            liste_resultats.append({"date_debut": date_debut_final, "date_fin": date_fin_final})

        dateTemp = date
    return liste_resultats


from app import app
from apis import APIValueError, APIResourceNotFoundError, APIPermissionError, Page
from func import check_admin, datetime_filter, get_page_index, get_user
from flask import request
from models import Comment, db, Blog
from log import logging
from sqlalchemy.sql import func as sqlfunc

logging.info('api/comment.py started.')

@app.route('/api/comments/<id>/delete', methods=['POST'])
def delete_comment(id):
    if check_admin():
        return check_admin()
    Comment.query.filter(Comment.id == id).delete()
    db.session.commit()
    return 'delete ok'

@app.route('/api/comments')
def api_comments():
    page = request.form.get('page', 1)
    page_index = get_page_index(page)
    num = db.session.query(sqlfunc.count(Blog.id)).scalar()
    p = Page(num, page_index)
    if num == 0:
        return dict(page=p.__dict__, comments=())
    comments = Comment.query.offset(
        p.offset).limit(
        p.limit).from_self().order_by(
        Comment.created_at.desc()).all()
    for b in range(len(comments)):
        comments[b] = comments[b].to_dict()
        comments[b]['created_at'] = [
            comments[b]['created_at'], datetime_filter(
                comments[b]['created_at'])]
    return dict(page=p.__dict__, comments=comments)

@app.route('/api/blogs/<id>/comments', methods=['POST'])
def api_create_comment(id):
    user = get_user()
    content = request.form.get('content', None)
    if user is None:
        raise APIPermissionError('Please sign in first.')
    if not content or not content.strip():
        raise APIValueError('content')
    blog = Blog.query.filter(Blog.id == id).one()
    if blog is None:
        raise APIResourceNotFoundError('Blog')
    comment = Comment(blog_id=blog.id, user_id=user.id,
    user_name=user.name, user_image=user.image,
                      content=content.strip())
    db.session.add(comment)
    db.session.commit()
    return comment.to_dict()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# Imports
import numpy as np
import tensorflow as tf
import pandas as pd
from sklearn.model_selection import train_test_split

tf.logging.set_verbosity(tf.logging.INFO)

# global variable denoting the length of each episode
eeg_length = -1

def expand_dataset(df, depth, crop_stride, orig_len, labels):
    # calculate new length of eeg readings
    new_len = orig_len - depth

    df_ = pd.DataFrame()
    for i in range(0, depth, crop_stride):
        # create a list of the readings to keep in this iteration
        eeg_keep = list(range(i, new_len + i))

        # add in the labels to that list
        columns_keep = eeg_keep + labels

        # keep the resulting dataframe
        df_i = df[columns_keep]

        # rename the columns in the resulting dataframe so that they all line up, index-wise
        df_i.columns = list(range(new_len)) + labels

        # append the iteration dataframe to the placeholder
        df_ = df_.append(df_i)
    return df_

#make hyperparameter string
def hyper_param_string(learn_rate, use_two_cv, use_two_fc):
    name = "lr_"+str(learn_rate)+"_use2cv_"+str(use_two_cv)+"_use2fc_"+str(use_two_fc)
    print(name)
    return name

# shallow convnet model
#params is a dict of the hyperparameters for the model
def cnn(features, labels, mode, params):
    global eeg_length

    # reshape input to account for minibatch size. Also, the number of channels (1 in this case) is specified.
    x = tf.reshape(features["x"], [-1, eeg_length, 1])

    # temporal convolution
    with tf.name_scope("conv1_pool1"):
        x = tf.layers.conv1d(
            inputs=x,
            filters=params["conv1_kernel_num"],
            kernel_size=params["conv1_filter_size"],
            strides=params["conv1_stride"],
            padding='valid',
            use_bias=True,
            activation=None,
            name="conv1")
        # with some luck, conv1 will have shape [-1,289,24] ie [batch_size, readings, num_kernels]
        # pooling layer
        x = tf.layers.max_pooling1d(
            inputs=x,
            pool_size=params["pool1_size"],
            strides=params["pool1_stride"],
            name="pool1")

    #only perform these ops if two layers of convolutions are specified
    if(params["use_two_cv"]):#60/100   .4(.4) + .6(.6) =
        with tf.name_scope("conv2_pool2"):
            x = tf.layers.conv1d(
                inputs=x,
                filters=params["conv2_kernel_num"],
                kernel_size=params["conv2_filter_size"],
                strides=params["conv2_stride"],
                padding='valid',
                use_bias=True,
                activation=None,
                name="conv2")
            # pooling layer
            x = tf.layers.max_pooling1d(
                inputs=x,
                pool_size=params["pool2_size"],
                strides=params["pool2_stride"],
                name="pool2")

    #flatten the data to pass through the dense layers
    x_shape = x.shape.as_list()
    x = tf.reshape(x, [-1, x_shape[1] * x_shape[2]])

    if(params["num_fc"] >= 1):
        with tf.name_scope("dense1"):
            x = tf.layers.dense(
                inputs=x,
                units=params["dense1_neurons"],
                activation=params["activations"],
                name="dense1")

    #only perform this if two fully connected layers are specified
    if(params["num_fc"] >= 2):
        with tf.name_scope("dense2"):
            x = tf.layers.dense(
                inputs=x,
                units=params["dense2_neurons"],
                activation=params["activations"],
                name="dense2")

    # rate = .4 => 40% of the output from the dense layers will be randomly dropped during training
    # training checks to see if the CNN is in training mode. if so, the dropout proceeds.
    with tf.name_scope("dropout"):
        x = tf.layers.dropout(
            inputs=x,
            rate=params['dropout_rate'],
            training=mode == tf.estimator.ModeKeys.TRAIN,
            name="dropout")

    # logits output has shape [batch_size, 2]
    with tf.name_scope("logits"):
        logits = tf.layers.dense(
            inputs=x,
            units=2,
            name="logits")

    predictions = {
        "classes": tf.argmax(input=logits, axis=1),
        "probabilities": tf.nn.softmax(logits, name="softmax_tensor")}

    if mode == tf.estimator.ModeKeys.PREDICT:
        return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
    # if in TRAIN or EVAL mode, calculate loss and backpropagate
    onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=2)

    loss = tf.losses.softmax_cross_entropy(onehot_labels=onehot_labels, logits=logits)

    # record metrics for display in tensorboard
    with tf.name_scope("metrics"):
        precision = tf.metrics.precision(
            labels=labels,
            predictions=predictions["classes"])

        recall = tf.metrics.recall(
            labels=labels,
            predictions=predictions["classes"])
        tf.summary.scalar('train_cross_entropy', loss)

        accuracy = tf.metrics.accuracy(
            labels=labels,
            predictions=predictions["classes"])
        tf.summary.scalar('train_accuracy', accuracy[1])

        tf.summary.scalar('train_precision', precision[1])

        tf.summary.scalar('train_recall', recall[1])

        predictions_float = tf.cast(predictions['classes'], dtype=tf.float32)
        win_prediction_freq = tf.reshape(tf.reduce_mean(predictions_float), shape=()) #convert to scalar
        tf.summary.scalar('win_prediction_freq', win_prediction_freq)

        tf.summary.histogram('train_prediction_distributions', predictions['probabilities'])

        # prediction_distribution = np.mean(tf.Session().run(predictions['probabilities']), axis=0)
        # print(predictions['probabilities'])
        # predict_win = prediction_distribution[1]
        # predict_loss = prediction_distribution[0]

        # tf.summary.scalar('predict_win', predict_win)
        # tf.summary.scalar('predict_loss', predict_loss)

    # if in training mode, calculate loss and step to take
    if mode == tf.estimator.ModeKeys.TRAIN:
        if(params["optimizer"] == "GD"):
            optimizer = tf.train.GradientDescentOptimizer(learning_rate=params["learn_rate"])
        else:
            optimizer = tf.train.AdamOptimizer(learning_rate=params["learn_rate"])
        #i think tf.train.get_global_step() is self explanatory here
        train_op = optimizer.minimize(
            loss=loss,
            global_step=tf.train.get_global_step())
        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)

    # if in eval mode, eval!
    eval_metric_ops = {"eval_accuracy": accuracy,
                       "eval_precision": precision,
                       "eval_recall": recall}
                       # "predict_win": predict_win,
                       # "predict_loss": predict_loss}

    return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)

def main(unused_argv):
    # make eeg_length global
    global eeg_length
    # training only within a subject?
    # intra_sbj = True
    sbj_num = 6

    labels = ["episode_type", "trial_type", "trial_num",
              "episode_num", "is_corrupt", "action_index",
              "trimmed_data", "left", "subject_num", "win"]

    # load data
    data_full = pd.read_pickle("data/data_100hz_intrasubject_scaled.pkl")

    # #different hyperparameters I'd like to test
    learning_rates = [.0001, .00005, .00001]
    # two_conv_layers = [True]
    # two_dense_layers = [True]
    #
    conv1_filter_sizes = [30, 15]
    # conv2_filter_sizes = [30, 60]
    #
    conv1_kernel_nums = [30, 10]
    # conv2_kernel_nums = [75, 100]
    #
    conv1_strides = [3, 6]

    pool1__sizes = [10, 5]

    dropout_rates = [.4, .6]

    # conv2_strides = [5, 10]
    #
    # dense1_layer_sizes = [100, 200]
    # dense2_layer_sizes = [50, 100]

    batch_sizes = [50, 25]

    # learn_rate = .0001
    use_two_cv = False
    num_fcs = [0,1,2]
    conv1_stride = 3
    pool1_size = 20
    batch_size = 25

    conv1_filter_size = 25
    dropout_rate = .4
    learn_rate = .00001
    conv1_kernel_num = 25

    # for conv1_filter_size in conv1_filter_sizes:
    #     for conv1_kernel_num in conv1_kernel_nums:
    #         for conv1_stride in conv1_strides:
    #             for fc1_size in dense1_layer_sizes:
    # for learn_rate in learning_rates:
    #     for use_two_fc in two_dense_layers:
    #         for use_two_cv in two_conv_layers:
                #create the log name for this model
    for sbj_num in range(1,11):
        # remove data from other subjects if toggled
        data_sbj = data_full[data_full.subject_num == sbj_num].reset_index(drop=True)

        train, test = train_test_split(data_sbj, test_size=.25)

        expand = False

        if(expand):
            train = expand_dataset(train, 40, 5, 375, labels)
            test = expand_dataset(test, 40, 5, 375, labels)

        x_train = np.array(train.drop(labels, axis=1), dtype='float32')
        y_train = np.array(train['win'], dtype='float32')

        x_test = np.array(test.drop(labels, axis=1), dtype='float32')
        y_test = np.array(test['win'], dtype='float32')

        eeg_length = x_train[0].size
        # x = data_sbj.drop(labels, axis=1)
        # # fill in NaN values
        # x = x.fillna(x.mean())
        # # convert to numpy array rank 2
        # x = np.array(x, dtype="float32")
        # y = np.array(data_sbj['win'], dtype='float32')
        #
        # # get the number of readings per episode

        #
        # x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state=27)

        # for conv1_filter_size in conv1_filter_sizes:
        #     for conv1_kernel_num in conv1_kernel_nums:
        #         for dropout_rate in dropout_rates:
        #             for learn_rate in learning_rates:
        model_name = "sbj_num_"+str(sbj_num)+"_cvf_"+str(conv1_filter_size)+\
                     "_dr_"+str(dropout_rate)+"_lr_"+str(learn_rate)+\
                     "_cvk_" +str(conv1_kernel_num)+"_cvs_"+str(conv1_stride)+"_pos_"+str(pool1_size)+\
                     "_bs_"+str(batch_size)

        hyp_params = {"optimizer":"Adam",
                      "use_f1_as_loss":False,
                      "learn_rate":learn_rate,
                      "batch_size":batch_size,
                      "steps":100,
                      "use_two_cv":use_two_cv,
                      "num_fc":num_fcs[1],
                      "conv1_filter_size":conv1_filter_size,
                      "conv1_kernel_num":conv1_kernel_num,
                      "conv1_stride":conv1_stride,
                      "pool1_size":pool1_size,
                      "pool1_stride":3,
                      "conv2_filter_size":15,
                      "conv2_kernel_num":80,
                      "conv2_stride":3,
                      "pool2_size":5,
                      "pool2_stride":2,
                      "dense1_neurons":50,
                      "dense2_neurons":200,
                      "activations":tf.nn.relu,
                      "dropout_rate":dropout_rate}

        # Create the Estimator and the model directory is where the model stats are saved
        #the log_name is dynamically decided based on the hyperparams above
        model_dir = "models/intrasubject_final_4/"+model_name
        classifier = tf.estimator.Estimator(
            params = hyp_params,
            model_fn = cnn,
            model_dir = model_dir)

        # Set up logging for predictions
        # these tensors are printed to the output every 50 iterations
        tensors_to_log = {"probabilities": "softmax_tensor"}
        logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log, every_n_iter=50)

        # def predict_input_fn():
        #     rand = np.random.randint(data_sbj.shape[0])
        #     rand_samp = data_sbj[data_sbj.index == rand]
        #     eeg = rand_samp.drop(labels, axis=1)
        #     eeg = np.array(eeg)
        #     print('TRUE:', rand_samp['win'].values[0])
        #     return eeg
        #
        # predict_results = classifier.predict(input_fn=predict_input_fn)
        # print("PREDICT:", list(predict_results))

        for i in range(15):
            # Train the model
            #this function returns a dictionary of features -> targets
            train_input_fn = tf.estimator.inputs.numpy_input_fn(
                x={"x": x_train},
                y=y_train,
                batch_size=hyp_params["batch_size"],
                num_epochs=None,
                shuffle=True)
            classifier.train(
                input_fn=train_input_fn,
                steps=hyp_params["steps"],
                hooks=[logging_hook])

            # Evaluate the model and print results
            eval_input_fn = tf.estimator.inputs.numpy_input_fn(
                x={"x": x_test},
                y=y_test,
                num_epochs=1,
                shuffle=False)
            eval_results = classifier.evaluate(input_fn=eval_input_fn)
            print(eval_results)

        saver = tf.train.Saver()

if __name__ == "__main__":
    tf.app.run()


#!/usr/bin/env python

import os
from glob import glob
import json
import codecs
import pandas as pd
import numpy as np
import datetime
from tqdm import tqdm

here = os.path.dirname(os.path.abspath(__file__))
folder = os.path.basename(here)
latest = '%s/latest' % here
year = datetime.datetime.today().year

output_data = os.path.join(here, 'data-latest.tsv')
output_year = os.path.join(here, 'data-%s.tsv' % year)

# Don't continue if we don't have latest folder
if not os.path.exists(latest):
    print('%s does not have parsed data.' % folder)
    sys.exit(0)

# Don't continue if we don't have results.json
results_json = os.path.join(latest, 'records.json')
if not os.path.exists(results_json):
    print('%s does not have results.json' % folder)
    sys.exit(1)

with open(results_json, 'r') as filey:
    results = json.loads(filey.read())

columns = ['charge_code',
           'price',
           'description',
           'hospital_id',
           'filename',
           'charge_type']

df = pd.DataFrame(columns=columns)

# First parse standard charges (doesn't have DRG header)
for result in tqdm(results):
    filename = os.path.join(latest, result['filename'])
    if not os.path.exists(filename):
        print('%s is not found in latest folder.' % filename)
        continue

    if os.stat(filename).st_size == 0:
        print('%s is empty, skipping.' % filename)
        continue

    charge_type = 'standard'

    print("Parsing %s" % filename)

    if filename.endswith('json'):

         with codecs.open(filename, "r",
            encoding='utf-8-sig', errors='ignore') as filey:
             content = json.load(filey)

         charge_types = {'DRG': 'drg',
                         'IP': 'inpatient',
                         'OP': 'outpatient',
                         'RX': 'pharmacy',
                         'SUP': 'supply'}

         # Create dataframe and make empty strings null
         temp = pd.DataFrame.from_records(content['CDM']).replace({'': np.nan})

         # For any null hospital names,
         # replace with hospital_id from records.json
         temp['HOSPITAL_NAME'].fillna(result["hospital_id"], inplace=True)

         # Rename columns accordingly
         # Note that we'll need to fill in the filename column manually
         temp.rename(columns={
            'CDM': 'charge_code',
            'CHARGE': 'price',
            'DESCRIPION': 'description',
            'HOSPITAL_NAME': 'hospital_id',
            'SERVICE_SETTING': 'charge_type'
            }, inplace=True)

         # Reformat hospital_id to be lowercase with hyphens instead of spaces
         temp['hospital_id'] = \
         temp['hospital_id'].astype(str).str.lower().str.replace(" ", "-")

         # Make sure we map the values of charge_type properly
         temp['charge_type'] = temp['charge_type'].map(charge_types)

         # Add filename column
         temp['filename'] = result['filename']

         # Remove extraneous columns
         extra_cols = list(temp.columns[~temp.columns.isin(columns)])
         temp.drop(columns=extra_cols, inplace=True)

         df = df.append(temp, ignore_index=True)

    else:
        break

# Remove empty rows
df = df.dropna(how='all')
print(df.head())

# Save data! ...and make sure columns are ordered in standard way
print(df.shape)
df.to_csv(output_data, columns=columns, sep='\t', index=False)
df.to_csv(output_year, columns=columns, sep='\t', index=False)


import argparse
import os
from collections import Sequence
from pathlib import Path

import mmcv
import numpy as np
from mmcv import Config, DictAction

from mmdet.core.utils import mask2ndarray
from mmdet.core.visualization import imshow_det_bboxes
from mmdet.datasets.builder import build_dataset
from mmdet.utils import replace_cfg_vals, update_data_root

def parse_args():
    parser = argparse.ArgumentParser(description='Browse a dataset')
    parser.add_argument('config', help='train config file path')
    parser.add_argument(
        '--skip-type',
        type=str,
        nargs='+',
        default=['DefaultFormatBundle', 'Normalize', 'Collect'],
        help='skip some useless pipeline')
    parser.add_argument(
        '--output-dir',
        default=None,
        type=str,
        help='If there is no display interface, you can save it')
    parser.add_argument('--not-show', default=False, action='store_true')
    parser.add_argument(
        '--show-interval',
        type=float,
        default=2,
        help='the interval of show (s)')
    parser.add_argument(
        '--cfg-options',
        nargs='+',
        action=DictAction,
        help='override some settings in the used config, the key-value pair '
        'in xxx=yyy format will be merged into config file. If the value to '
        'be overwritten is a list, it should be like key="[a,b]" or key=a,b '
        'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" '
        'Note that the quotation marks are necessary and that no white space '
        'is allowed.')
    args = parser.parse_args()
    return args

def retrieve_data_cfg(config_path, skip_type, cfg_options):

    def skip_pipeline_steps(config):
        config['pipeline'] = [
            x for x in config.pipeline if x['type'] not in skip_type
        ]

    cfg = Config.fromfile(config_path)

    # replace the ${key} with the value of cfg.key
    cfg = replace_cfg_vals(cfg)

    # update data root according to MMDET_DATASETS
    update_data_root(cfg)

    if cfg_options is not None:
        cfg.merge_from_dict(cfg_options)
    train_data_cfg = cfg.data.train
    while 'dataset' in train_data_cfg and train_data_cfg[
            'type'] != 'MultiImageMixDataset':
        train_data_cfg = train_data_cfg['dataset']

    if isinstance(train_data_cfg, Sequence):
        [skip_pipeline_steps(c) for c in train_data_cfg]
    else:
        skip_pipeline_steps(train_data_cfg)

    return cfg

def main():
    args = parse_args()
    cfg = retrieve_data_cfg(args.config, args.skip_type, args.cfg_options)

    if 'gt_semantic_seg' in cfg.train_pipeline[-1]['keys']:
        cfg.data.train.pipeline = [
            p for p in cfg.data.train.pipeline if p['type'] != 'SegRescale'
        ]
    dataset = build_dataset(cfg.data.train)

    progress_bar = mmcv.ProgressBar(len(dataset))

    for item in dataset:
        filename = os.path.join(args.output_dir,
                                Path(item['filename']).name
                                ) if args.output_dir is not None else None

        gt_bboxes = item['gt_bboxes']
        gt_labels = item['gt_labels']
        gt_masks = item.get('gt_masks', None)
        if gt_masks is not None:
            gt_masks = mask2ndarray(gt_masks)

        gt_seg = item.get('gt_semantic_seg', None)
        if gt_seg is not None:
            pad_value = 255  # the padding value of gt_seg
            sem_labels = np.unique(gt_seg)
            all_labels = np.concatenate((gt_labels, sem_labels), axis=0)
            all_labels, counts = np.unique(all_labels, return_counts=True)
            stuff_labels = all_labels[np.logical_and(counts < 2,
                                                     all_labels != pad_value)]
            stuff_masks = gt_seg[None] == stuff_labels[:, None, None]
            gt_labels = np.concatenate((gt_labels, stuff_labels), axis=0)
            gt_masks = np.concatenate((gt_masks, stuff_masks.astype(np.uint8)),
                                      axis=0)
            # If you need to show the bounding boxes,
            # please comment the following line
            gt_bboxes = None

        imshow_det_bboxes(
            item['img'],
            gt_bboxes,
            gt_labels,
            gt_masks,
            class_names=dataset.CLASSES,
            show=not args.not_show,
            wait_time=args.show_interval,
            out_file=filename,
            bbox_color=dataset.PALETTE,
            text_color=(200, 200, 200),
            mask_color=dataset.PALETTE)

        progress_bar.update()

if __name__ == '__main__':
    main()


import collections

def dict_find_recursive(d, target, self):

    if not isinstance(d, collections.Mapping):
        return d == target
    else:
        for k in d:
            if dict_find_recursive(d[k], target) != False:
                    return k
        return False

def dict_flatten(d):
    if not isinstance(d, collections.Mapping):
        yield d
    else:
        for value in d:
            for item in dict_flatten(d[value]):
                yield item


import random
from random import randint, sample, uniform
from acme import Product

ADJECTIVES = ['Awesome', 'Shiny', 'Impressive', 'Portable', 'Improved']
NOUNS = ['Anvil', 'Catapult', 'Disguise', 'Mousetrap', '???']

def generate_products(num_products=30):
    """Randomly generatle list of products

    Args:
        num_products (int, optional): [description]. Defaults to 30.
    """

    products = []

    for _ in range(num_products):
        name = ADJECTIVES[randint(0, 4)] + \
               ' ' + \
               NOUNS[randint(0, 4)]

        price = randint(5, 100)
        weight = randint(5, 100)
        flammability = uniform(0., 2.5)

        product = Product(name=name,
                       price=price,
                       weight=weight,
                       flammability=flammability)
        products.append(product)
    return products

def inventory_report(products):
    """Print a summary of the list of products
    """

    product_count = len(products)
    if product_count <= 0:
        return "No products!"

    total_price, total_weight, total_flam = 0, 0, 0
    for prod in products:
        total_price += prod.price
        total_weight += prod.weight
        total_flam += prod.flammability

    avg_price = total_price / product_count
    avg_weight = total_weight / product_count
    avg_flam = total_flam / product_count

    print("ACME CORPORATION OFFICIAL INVENTORY REPORT")
    print("Unique product names:", len(set(products)))
    print("Average price:", avg_price)
    print("Average weight:", avg_weight)
    print("Average flammability:", avg_flam)

if __name__ == '__main__':
    inventory_report(generate_products())


"""
9.18. Defining classes programmatically

Problem: You're writing code that ultimately needs to create a new class object. You've
         thought about emitting emit class source code to a string and using a function
         such as exec() to evaluate it, but you'd prefer a more elegant solution.

Solution: You can use the function types.new_class() to instantiate new class objects. all
          you need to de is provide the name of the class, tuple of parents classes, keyword
          arguments, and a callback that populates the class dictionary with members.
"""

def __init__(self, name, shares, price):
    self.name = name
    self.shares = shares
    self.price = price

def cost(self):
    return self.shares * self.price

cls_dict = {
    '__init__': __init__,
    'cost': cost,
}

import types
Stock = types.new_class("Stock", (), {}, lambda ns: ns.update(cls_dict))
Stock.__module__ = __name__

print(Stock) # it will work like a normal class.
# ========================================================================

import abc
Stock = types.new_class("Stock", (), {"metaclass": abc.ABCMeta}, lambda ns: ns.update(cls_dict))
Stock.__module__ = __name__
print(Stock)

# Being able to manifacture new class objects can be useful in certain contexts. One of
# the more familiar examples involves the collections.namedtuple() function.
from collections import namedtuple

Stock = namedtuple("Stock", ["name", "shares", "price"])
print(Stock)

# namedtuple() uses exec() instead of the technique shown here. However, here is a simple
# variant that creates a class directly
import operator
import types
import sys

def named_typle(classname, fieldnames):
    # populate a dictionary of field property accessors
    cls_dict = { name: property(operator.itemgetter(n)) for n, name in enumerate(fieldnames) }

    # make a __new__ function and add to the class dict
    def __new__(cls, *args):
        if len(args) != len(fieldnames):
            raise TypeError("Expected {} arguments".format(len(fieldnames)))
        return tuple.__new__(cls, args)

    cls_dict['__new__'] = __new__

    # make the class
    cls = types.new_class(classname, (tuple,), {}, lambda ns: ns.update(cls_dict))

    # set the module to that of the caller
    cls.__module__ = sys._getframe(1).f_globals['__name__']
    return cls

Point = named_typle("Point", ["x", "y"])
print(Point)


import pandas as pd
import re
import argparse

def argparser():
	parser = argparse.ArgumentParser(description = "Merge htseq like output")
	parser.add_argument("ensg_gene_file", type = str, help = "Path to ref file that matches ENSG IDs with gene names"),
	parser.add_argument("file_list", type = str, help = "Comma sep string containing all file names to merge"),
	parser.add_argument("sample_regex", type = str, help = "Regex to ID sample name from file name"),
	parser.add_argument("index_cols", type = str, help = "Comma sep string containing integer locations of index columns"),
	parser.add_argument("--ensg_exon_file", type = str, default = "", help = "Path to file containing exon ensg id mapping"),
	parser.add_argument("--colnames", "-cnames", type = str, default = "", help = "Comma sep string containing column names"),
	parser.add_argument("--colselect", "-csel", type = str, default = "", help = "Comma sep string contianing column names to select"),
	parser.add_argument("out_path", type = str, help = "Path to save outputted csv file"),
	parser.add_argument("--reads_to_genes_path", type = str, help = "Path to save outputted txt file for reads to genes counts")
	return parser.parse_args()

def get_format_data(file, index_cols, regex_sample, colnames, col_to_select):
	sample = re.search(regex_sample, file).group()
	data = pd.read_csv(file, sep = "\s+", index_col = index_cols, names = colnames + [sample]) if colnames != [""] else pd.read_csv(file, sep = "\s+", index_col = index_cols)

	if col_to_select != "":
		data = pd.DataFrame(data.loc[:, col_to_select])
		data.rename(columns = {col_to_select : sample}, inplace = True)
	return data

def merge_data(full_df, to_merge):
	return full_df.merge(to_merge, left_index=True, right_index=True, how='outer')

def add_gene_num(ensg_exon_file, full_df):
	exon_table = pd.read_csv(ensg_exon_file, sep="\s+", index_col=1)
	exon_table = exon_table.loc[~exon_table.index.duplicated(keep="first")]
	full_df.reset_index(inplace=True)
	full_df.set_index("exon_id", inplace=True)
	full_df.insert(0, "gene_num", exon_table)
	return full_df

def add_gene_name(ensg_gene_file, full_df, ensg_col_name):
	gene_names = pd.read_csv(ensg_gene_file, sep="\s+", index_col=0)
	full_df.reset_index(inplace=True)
	full_df.set_index(ensg_col_name, inplace=True)
	full_df.insert(0, "gene_name", gene_names)
	return full_df.reset_index()

def main():
	args = argparser()
	file_list = sorted(args.file_list.split(" "))
	icols = [int(col) for col in args.index_cols.split(",")]
	colnames = args.colnames.split(",")
	colsel = args.colselect.split(",")[0]

	first_one = True
	for file in file_list:
		tmp = get_format_data(file, icols, args.sample_regex, colnames, colsel)
		full_df = tmp if first_one else merge_data(full_df, tmp)
		first_one = False

	full_df.fillna(value = 0, inplace = True)
	full_df = full_df.astype(int)

	#Ann file
	full_df = add_gene_num(args.ensg_exon_file, full_df) if args.ensg_exon_file != "" else full_df
	ensg_col_name = "gene_num" if "gene_num" in full_df.reset_index().columns.tolist() else "ensg" #To accnt for dif colnames for gene_num
	full_df = add_gene_name(args.ensg_gene_file, full_df, ensg_col_name = ensg_col_name)

	#Reorg columns
	cols = [col for col in full_df.columns.tolist() if col not in ["gene_name", ensg_col_name]]
	cols = ["gene_name", ensg_col_name] + cols

	full_df = full_df[cols]

	full_df.to_csv(args.out_path, index=False)

	if args.ensg_exon_file != "":
		return

	suffix_idx = args.out_path.find('.csv')
	addtional_out_path = args.out_path[:suffix_idx] + '_genes_only' + args.out_path[suffix_idx:]
	full_df.iloc[:-6].to_csv(addtional_out_path, index = False)

	r2g = full_df.iloc[:-6, 2:].sum(axis = 0)
	r2g.index.rename('sample', inplace = True)
	r2g.rename('reads_to_genes', inplace = True)
	r2g.to_frame().to_csv(args.reads_to_genes_path, sep = '\t')

if __name__ == "__main__":
	main()


from flask import Flask
from flask import request
from flask import render_template
import json
from src.functions import extraction, data_match

app = Flask(__name__)

@app.after_request
def cors(environ):
    environ.headers['Access-Control-Allow-Origin'] = '*'
    # environ.headers['Access-Control-Allow-Method']='*'
    # environ.headers['Access-Control-Allow-Headers']='x-requested-with,content-type'
    return environ

personal_data = {}

@app.route('/home')
def render_home():
    return render_template('home.html')

@app.route('/query', methods=['POST'])
def query():
    url = request.form["url"]
    global personal_data
    personal_data = extraction(url)
    personal_data = data_match(personal_data)
    return render_template('show.html')

@app.route('/data')
def get_data():
    return personal_data

def main():
    app.run()

if __name__ == '__main__':
    main()


from __future__ import absolute_import, division, print_function

import operator
from toolz import first
import numpy as np
import pandas as pd
from datashape import (
    DataShape,
    DateTime,
    Option,
    String,
    TimeDelta,
    coretypes as ct,
    datetime_,
    discover,
    dshape,
    optionify,
    promote,
    timedelta_,
    unsigned,
)
from datashape.predicates import isscalar, isboolean, isnumeric, isdatelike
from datashape.typesets import integral
from dateutil.parser import parse as dt_parse

from .core import parenthesize, eval_str
from .expressions import Expr, shape, ElemWise, binop_inputs, binop_name
from .utils import maxshape
from ..dispatch import dispatch
from ..compatibility import _strtypes

__all__ = '''
BinOp
UnaryOp
Arithmetic
Add
Mult
Repeat
Sub
Div
FloorDiv
Pow
Mod
Interp
USub
Relational
Eq
Ne
Ge
Lt
Le
Gt
Gt
And
Or
Not
'''.split()

class BinOp(ElemWise):
    _arguments = 'lhs', 'rhs'
    _input_attributes = 'lhs', 'rhs'

    def __str__(self):
        lhs = parenthesize(eval_str(self.lhs))
        rhs = parenthesize(eval_str(self.rhs))
        return '%s %s %s' % (lhs, self.symbol, rhs)

    def _dshape(self):
        # TODO: better inference.  e.g. int + int -> int
        return DataShape(*(maxshape([shape(self.lhs), shape(self.rhs)]) +
                           (self._dtype,)))

    _name = property(binop_name)

    @property
    def _inputs(self):
        return tuple(binop_inputs(self))

class UnaryOp(ElemWise):
    _arguments = '_child',

    def __str__(self):
        return '%s(%s)' % (self.symbol, eval_str(self._child))

    @property
    def symbol(self):
        return type(self).__name__

    def _dshape(self):
        return DataShape(*(shape(self._child) + (self._dtype,)))

    @property
    def _name(self):
        return self._child._name

class Arithmetic(BinOp):
    """ Super class for arithmetic operators like add or mul """

    @property
    def _dtype(self):
        # we can't simply use .schema or .datashape because we may have a bare
        # integer, for example
        lhs, rhs = discover(self.lhs).measure, discover(self.rhs).measure
        return promote(lhs, rhs)

class Add(Arithmetic):
    symbol = '+'
    op = operator.add

    @property
    def _dtype(self):
        lmeasure = discover(self.lhs).measure
        lty = getattr(lmeasure, 'ty', lmeasure)
        rmeasure = discover(self.rhs).measure
        rty = getattr(rmeasure, 'ty', rmeasure)
        if lmeasure == datetime_ and rmeasure == datetime_:
            raise TypeError('cannot add datetime to datetime')

        l_is_datetime = lty == datetime_
        if l_is_datetime or rty == datetime_:
            if l_is_datetime:
                other = rty
            else:
                other = lty
            if isinstance(other, TimeDelta):
                return optionify(lmeasure, rmeasure, datetime_)
            else:
                raise TypeError(
                    'can only add timedeltas to datetimes',
                )

        return super(Add, self)._dtype

class Mult(Arithmetic):
    symbol = '*'
    op = operator.mul

class Repeat(Arithmetic):
    # Sequence repeat
    symbol = '*'
    op = operator.mul

    @property
    def _dtype(self):
        lmeasure = discover(self.lhs).measure
        rmeasure = discover(self.rhs).measure
        if not (isinstance(getattr(lmeasure, 'ty', lmeasure), String) and
                getattr(rmeasure, 'ty', rmeasure) in integral):
            raise TypeError(
                'can only repeat strings by an integer amount, got: %s * %s' %
                (lmeasure, rmeasure),
            )

        return optionify(lmeasure, rmeasure, lmeasure)

class Sub(Arithmetic):
    symbol = '-'
    op = operator.sub

    @property
    def _dtype(self):
        lmeasure = discover(self.lhs).measure
        lty = getattr(lmeasure, 'ty', lmeasure)
        rmeasure = discover(self.rhs).measure
        rty = getattr(rmeasure, 'ty', rmeasure)
        if lty == datetime_:
            if isinstance(rty, DateTime):
                return optionify(lmeasure, rmeasure, timedelta_)
            if isinstance(rty, TimeDelta):
                return optionify(lmeasure, rmeasure, datetime_)
            else:
                raise TypeError(
                    'can only subtract timedelta or datetime from datetime',
                )

        return super(Sub, self)._dtype

class Div(Arithmetic):
    symbol = '/'
    op = operator.truediv

    @property
    def _dtype(self):
        lhs, rhs = discover(self.lhs).measure, discover(self.rhs).measure
        return optionify(lhs, rhs, ct.float64)

class FloorDiv(Arithmetic):
    symbol = '//'
    op = operator.floordiv

    @property
    def _dtype(self):
        lhs, rhs = discover(self.lhs).measure, discover(self.rhs).measure
        is_unsigned = lhs in unsigned and rhs in unsigned
        max_width = max(lhs.itemsize, rhs.itemsize)
        prefix = 'u' if is_unsigned else ''
        measure = getattr(ct, '%sint%d' % (prefix, max_width * 8))
        return optionify(lhs, rhs, measure)

class Pow(Arithmetic):
    symbol = '**'
    op = operator.pow

class Mod(Arithmetic):
    symbol = '%'
    op = operator.mod

class Interp(Arithmetic):
    # String interpolation
    symbol = '%'
    op = operator.mod

    @property
    def _dtype(self):
        lmeasure = discover(self.lhs).measure
        rmeasure = discover(self.rhs).measure
        if not (isinstance(getattr(lmeasure, 'ty', lmeasure), String)):
            raise TypeError('can only interp strings got: %s' % lmeasure)

        return optionify(lmeasure, rmeasure, lmeasure)

class USub(UnaryOp):
    op = operator.neg
    symbol = '-'

    def __str__(self):
        return '-%s' % parenthesize(eval_str(self._child))

    @property
    def _dtype(self):
        # TODO: better inference.  -uint -> int
        return self._child.schema

@dispatch(ct.Option, object)
def scalar_coerce(ds, val):
    return scalar_coerce(ds.ty, val) if val is not None else None

@dispatch((ct.Record, ct.Mono, ct.Option, DataShape), Expr)
def scalar_coerce(ds, val):
    return val

@dispatch(ct.Date, _strtypes)
def scalar_coerce(_, val):
    if val == '':
        raise TypeError('%r is not a valid date' % val)
    dt = dt_parse(val)
    if any(x > 0 for x in (dt.hour, dt.minute, dt.second, dt.microsecond)):
        msg = "Can not coerce %r to type Date, contains time information"
        raise TypeError(msg % val)
    return dt.date()

@dispatch(ct.DateTime, _strtypes)
def scalar_coerce(_, val):
    if val == '':
        raise TypeError('%r is not a valid datetime' % val)
    return pd.Timestamp(val)

@dispatch(ct.CType, _strtypes)
def scalar_coerce(dt, val):
    return np.asscalar(np.asarray(val, dtype=dt.to_numpy_dtype()))

@dispatch(ct.Record, object)
def scalar_coerce(rec, val):
    if len(rec.fields) == 1:
        return scalar_coerce(first(rec.types), val)
    else:
        raise TypeError("Trying to coerce complex datashape\n"
                "got dshape: %s\n"
                "scalar_coerce only intended for scalar values" % rec)

@dispatch(ct.DataShape, object)
def scalar_coerce(ds, val):
    return scalar_coerce(ds.measure, val)

@dispatch(object, object)
def scalar_coerce(dtype, val):
    return val

@dispatch(_strtypes, object)
def scalar_coerce(ds, val):
    return scalar_coerce(dshape(ds), val)

def _neg(self):
    return USub(self)

def _mkbin(name, cons, private=True, reflected=True):
    prefix = '_' if private else ''

    def _bin(self, other):
        result = cons(self, scalar_coerce(self.dshape, other))
        result.dshape  # Check that shapes and dtypes match up
        return result
    _bin.__name__ = prefix + name

    if reflected:
        def _rbin(self, other):
            result = cons(scalar_coerce(self.dshape, other), self)
            result.dshape  # Check that shapes and dtypes match up
            return result
        _rbin.__name__ = prefix + 'r' + name

        return _bin, _rbin

    return _bin

_add, _radd = _mkbin('add', Add)
_div, _rdiv = _mkbin('div', Div)
_floordiv, _rfloordiv = _mkbin('floordiv', FloorDiv)
_mod, _rmod = _mkbin('mod', Mod)
_mul, _rmul = _mkbin('mul', Mult)
_pow, _rpow = _mkbin('pow', Pow)
repeat = _mkbin('repeat', Repeat, reflected=False, private=False)
_sub, _rsub = _mkbin('sub', Sub)
interp = _mkbin('interp', Interp, reflected=False, private=False)

class _Optional(Arithmetic):
    @property
    def _dtype(self):
        # we can't simply use .schema or .datashape because we may have a bare
        # integer, for example
        lhs, rhs = discover(self.lhs).measure, discover(self.rhs).measure
        if isinstance(lhs, Option) or isinstance(rhs, Option):
            return Option(ct.bool_)
        return ct.bool_

class Relational(_Optional):
    # Leave this to separate relationals from other types of optionals.
    pass

class Eq(Relational):
    symbol = '=='
    op = operator.eq

class Ne(Relational):
    symbol = '!='
    op = operator.ne

class Ge(Relational):
    symbol = '>='
    op = operator.ge

class Le(Relational):
    symbol = '<='
    op = operator.le

class Gt(Relational):
    symbol = '>'
    op = operator.gt

class Lt(Relational):
    symbol = '<'
    op = operator.lt

class And(_Optional):
    symbol = '&'
    op = operator.and_

class Or(_Optional):
    symbol = '|'
    op = operator.or_

class Not(UnaryOp):
    symbol = '~'
    op = operator.invert

    @property
    def _dtype(self):
        return self._child.schema

    def __str__(self):
        return '~%s' % parenthesize(eval_str(self._child))

_and, _rand = _mkbin('and', And)
_eq = _mkbin('eq', Eq, reflected=False)
_ge = _mkbin('ge', Ge, reflected=False)
_gt = _mkbin('gt', Gt, reflected=False)
_le = _mkbin('le', Le, reflected=False)
_lt = _mkbin('lt', Lt, reflected=False)
_ne = _mkbin('ne', Ne, reflected=False)
_or, _ror = _mkbin('or', Or)

def _invert(self):
    result = Invert(self)
    result.dshape  # Check that shapes and dtypes match up
    return result

Invert = Not
BitAnd = And
BitOr = Or

from .expressions import schema_method_list

schema_method_list.extend([
    (isnumeric,
     set([_add, _radd, _mul, _rmul, _div, _rdiv, _floordiv, _rfloordiv, _sub,
          _rsub, _pow, _rpow, _mod, _rmod,  _neg])),
    (isscalar, set([_eq, _ne, _lt, _le, _gt, _ge])),
    (isboolean, set([_or, _ror, _and, _rand, _invert])),
    (isdatelike, set([_add, _radd, _sub, _rsub])),
    ])


from enum import IntEnum, IntFlag
from typing import Generator, Optional
from contextlib import contextmanager

from ragger.backend.interface import BackendInterface, RAPDU
from ragger.bip import pack_derivation_path

from .ton_utils import split_message

MAX_APDU_LEN: int = 255

CLA: int = 0xE0

class P1(IntEnum):
    P1_NONE = 0x00

    P1_CONFIRM = 0x01

    P1_NON_CONFIRM = 0x00

class P2(IntFlag):
    P2_NONE = 0x00

    P2_FIRST = 0x01

    P2_MORE = 0x02

class InsType(IntEnum):
    GET_VERSION       = 0x03
    GET_APP_NAME      = 0x04
    GET_PUBLIC_KEY    = 0x05
    SIGN_TX           = 0x06
    GET_ADDRESS_PROOF = 0x08
    SIGN_DATA         = 0x09

class Errors(IntEnum):
    SW_DENY                    = 0x6985
    SW_WRONG_P1P2              = 0x6A86
    SW_WRONG_DATA_LENGTH       = 0x6A87
    SW_INS_NOT_SUPPORTED       = 0x6D00
    SW_CLA_NOT_SUPPORTED       = 0x6E00
    SW_WRONG_RESPONSE_LENGTH   = 0xB000
    SW_DISPLAY_ADDRESS_FAIL    = 0xB002
    SW_DISPLAY_AMOUNT_FAIL     = 0xB003
    SW_WRONG_TX_LENGTH         = 0xB004
    SW_TX_PARSING_FAIL         = 0xB010
    SW_WRONG_SIGN_DATA_LENGTH  = 0xB005
    SW_SIGN_DATA_PARSING_FAIL  = 0xB011
    SW_BAD_STATE               = 0xB007
    SW_SIGNATURE_FAIL          = 0xB008
    SW_REQUEST_TOO_LONG        = 0xB00B
    SW_BAD_BIP32_PATH          = 0XB0BD

class AddressDisplayFlags(IntFlag):
    NONE = 0
    TESTNET = 1
    MASTERCHAIN = 2

class BoilerplateCommandSender:
    def __init__(self, backend: BackendInterface) -> None:
        self.backend = backend

    def get_app_and_version(self) -> RAPDU:
        return self.backend.exchange(cla=0xB0,  # specific CLA for BOLOS
                                     ins=0x01,  # specific INS for get_app_and_version
                                     p1=P1.P1_NONE,
                                     p2=P2.P2_NONE,
                                     data=b"")

    def get_version(self) -> RAPDU:
        return self.backend.exchange(cla=CLA,
                                     ins=InsType.GET_VERSION,
                                     p1=P1.P1_NONE,
                                     p2=P2.P2_NONE,
                                     data=b"")

    def get_app_name(self) -> RAPDU:
        return self.backend.exchange(cla=CLA,
                                     ins=InsType.GET_APP_NAME,
                                     p1=P1.P1_NONE,
                                     p2=P2.P2_NONE,
                                     data=b"")

    def get_public_key(self, path: str) -> RAPDU:
        return self.backend.exchange(cla=CLA,
                                     ins=InsType.GET_PUBLIC_KEY,
                                     p1=P1.P1_NON_CONFIRM,
                                     p2=AddressDisplayFlags.NONE,
                                     data=pack_derivation_path(path))

    @contextmanager
    def get_public_key_with_confirmation(self,
                                         path: str,
                                         display_flags: AddressDisplayFlags
                                        ) -> Generator[None, None, None]:
        with self.backend.exchange_async(cla=CLA,
                                         ins=InsType.GET_PUBLIC_KEY,
                                         p1=P1.P1_CONFIRM,
                                         p2=display_flags,
                                         data=pack_derivation_path(path)) as response:
            yield response

    @contextmanager
    def get_address_proof(self,
                          path: str,
                          display_flags: AddressDisplayFlags,
                          domain: str,
                          timestamp: int,
                          payload: bytes) -> Generator[None, None, None]:
        domain_b = bytes(domain, "utf8")
        req_bytes = b"".join([
            pack_derivation_path(path),
            bytes([len(domain_b)]),
            domain_b,
            timestamp.to_bytes(8, byteorder="big"),
            payload
        ])
        with self.backend.exchange_async(cla=CLA,
                                         ins=InsType.GET_ADDRESS_PROOF,
                                         p1=P1.P1_CONFIRM,
                                         p2=display_flags,
                                         data=req_bytes) as response:
            yield response

    @contextmanager
    def sign_tx(self, path: str, transaction: bytes) -> Generator[None, None, None]:
        self.backend.exchange(cla=CLA,
                              ins=InsType.SIGN_TX,
                              p1=P1.P1_NONE,
                              p2=(P2.P2_FIRST | P2.P2_MORE),
                              data=pack_derivation_path(path))
        messages = split_message(transaction, MAX_APDU_LEN)

        for msg in messages[:-1]:
            self.backend.exchange(cla=CLA,
                                  ins=InsType.SIGN_TX,
                                  p1=P1.P1_NONE,
                                  p2=P2.P2_MORE,
                                  data=msg)

        with self.backend.exchange_async(cla=CLA,
                                         ins=InsType.SIGN_TX,
                                         p1=P1.P1_NONE,
                                         p2=P2.P2_NONE,
                                         data=messages[-1]) as response:
            yield response

    @contextmanager
    def sign_data(self, path: str, data: bytes) -> Generator[None, None, None]:
        self.backend.exchange(cla=CLA,
                              ins=InsType.SIGN_DATA,
                              p1=P1.P1_NONE,
                              p2=(P2.P2_FIRST | P2.P2_MORE),
                              data=pack_derivation_path(path))
        messages = split_message(data, MAX_APDU_LEN)

        for msg in messages[:-1]:
            self.backend.exchange(cla=CLA,
                                  ins=InsType.SIGN_DATA,
                                  p1=P1.P1_NONE,
                                  p2=P2.P2_MORE,
                                  data=msg)

        with self.backend.exchange_async(cla=CLA,
                                         ins=InsType.SIGN_DATA,
                                         p1=P1.P1_NONE,
                                         p2=P2.P2_NONE,
                                         data=messages[-1]) as response:
            yield response

    def get_async_response(self) -> Optional[RAPDU]:
        return self.backend.last_async_response