import torch
if torch.__version__ >= "1.8":
    import torch_npu
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.backends.cudnn as cudnn
import torchvision
import torchvision.transforms as transforms
import models
import os
import time
import argparse
import numpy as np

from torch.autograd import Variable

import imdbfolder_coco as imdbfolder
import config_task
import utils_pytorch
import sgd
try:
    from apex import amp
except ImportError:
    apex = None

parser = argparse.ArgumentParser(description='PyTorch Residual Adapters training')
parser.add_argument('--dataset', default='cifar100', nargs='+', help='Task(s) to be trained')
parser.add_argument('--lr', default=0.1, type=float, help='initial learning rate')
parser.add_argument('--wd', default=1., type=float, help='weight decay for the classification layer')
parser.add_argument('--wd3x3', default=1., type=float, nargs='+', help='weight decay for the 3x3')
parser.add_argument('--wd1x1', default=1., type=float, nargs='+', help='weight decay for the 1x1')
parser.add_argument('--nb_epochs', default=120, type=int, help='nb epochs')
parser.add_argument('--step1', default=80, type=int, help='nb epochs before first lr decrease')
parser.add_argument('--step2', default=100, type=int, help='nb epochs before second lr decrease')
parser.add_argument('--mode', default='parallel_adapters', type=str, help='Task adaptation mode')
parser.add_argument('--proj', default='11', type=str, help='Position of the adaptation module')
parser.add_argument('--dropout', default='00', type=str, help='Position of dropouts')
parser.add_argument('--expdir', default='/scratch/shared/nfs1/srebuffi/exp/dem_learning/tmp/', help='Save folder')
parser.add_argument('--datadir', default='/scratch/local/ramdisk/srebuffi/decathlon/', help='folder containing data folder')
parser.add_argument('--imdbdir', default='/scratch/local/ramdisk/srebuffi/decathlon/annotations/', help='annotation folder')
parser.add_argument('--source', default='/scratch/shared/nfs1/srebuffi/exp/dem_learning/C100_alone/checkpoint/ckptpost11bnresidual11cifar1000.000180607060.t7', type=str, help='Network source')
parser.add_argument('--seed', default=0, type=int, help='seed')
parser.add_argument('--factor', default='1.', type=float, help='Width factor of the network')
# Mixed precision training parameters
parser.add_argument('--apex', action='store_true', help='Use apex for mixed precision training')
parser.add_argument('--apex-opt-level', default='O1', type=str,help='For apex mixed precision training'
                        'O0 for FP32 training, O1 for mixed precision training.'
                        'For further detail, see https://github.com/NVIDIA/apex/tree/master/examples/imagenet')
parser.add_argument('--loss-scale-value', default=1024., type=float,
                    help='loss scale using in amp, default -1 means dynamic')
# for ascend 910
parser.add_argument('--device_id', default=5, type=int, help='device id')
args = parser.parse_args()
args.archi ='default'
config_task.mode = args.mode
config_task.proj = args.proj
config_task.factor = args.factor
args.use_npu = torch.npu.is_available()

device = torch.device(f'npu:{args.device_id}')
torch.npu.set_device(device)
print("Use NPU: {} for training".format(args.device_id))

if type(args.dataset) is str:
    args.dataset = [args.dataset]

if type(args.wd3x3) is float:
    args.wd3x3 = [args.wd3x3]

if type(args.wd1x1) is float:
    args.wd1x1 = [args.wd1x1]

if not os.path.isdir(args.expdir):
    os.mkdir(args.expdir)

config_task.decay3x3 = np.array(args.wd3x3) * 0.0001
config_task.decay1x1 = np.array(args.wd1x1) * 0.0001
args.wd = args.wd *  0.0001

args.ckpdir = args.expdir + '/checkpoint/'
args.svdir  = args.expdir + '/results/'

if not os.path.isdir(args.ckpdir):
    os.mkdir(args.ckpdir)

if not os.path.isdir(args.svdir):
    os.mkdir(args.svdir)

config_task.isdropout1 = (args.dropout[0] == '1')
config_task.isdropout2 = (args.dropout[1] == '1')

#####################################

# Prepare data loaders
train_loaders, val_loaders, num_classes = imdbfolder.prepare_data_loaders(args.dataset,args.datadir,args.imdbdir,True)
args.num_classes = num_classes

# Create the network
net = models.resnet26(num_classes)

start_epoch = 0
best_acc = 0  # best test accuracy
results = np.zeros((4,start_epoch+args.nb_epochs,len(args.num_classes)))
all_tasks = range(len(args.dataset))
np.random.seed(1993)

if args.use_npu:
    net.npu()
    cudnn.benchmark = True

args.criterion = nn.CrossEntropyLoss()
optimizer = sgd.SGD(filter(lambda p: p.requires_grad, net.parameters()), lr=args.lr, momentum=0.9, weight_decay=args.wd)

if args.apex:
    net, optimizer = amp.initialize(net, optimizer,
                                      opt_level=args.apex_opt_level,
                                      loss_scale=args.loss_scale_value,
                                      combine_grad=True)

print("Start training")
for epoch in range(start_epoch, start_epoch+args.nb_epochs):
    training_tasks = utils_pytorch.adjust_learning_rate_and_learning_taks(optimizer, epoch, args)
    st_time = time.time()

    # Training and validation
    train_acc, train_loss = utils_pytorch.train(epoch, train_loaders, training_tasks, net, args, optimizer)
    test_acc, test_loss, best_acc = utils_pytorch.test(epoch,val_loaders, all_tasks, net, best_acc, args, optimizer)

    # Record statistics
    for i in range(len(training_tasks)):
        current_task = training_tasks[i]
        results[0:2,epoch,current_task] = [train_loss[i].cpu(),train_acc[i].cpu()]
    for i in all_tasks:
        results[2:4,epoch,i] = [test_loss[i].cpu(),test_acc[i].cpu()]
    np.save(args.svdir+'/results_'+'adapt'+str(args.seed)+args.dropout+args.mode+args.proj+''.join(args.dataset)+'wd3x3_'+str(args.wd3x3)+'_wd1x1_'+str(args.wd1x1)+str(args.wd)+str(args.nb_epochs)+str(args.step1)+str(args.step2),results)
    print('Epoch lasted {0}'.format(time.time()-st_time))


from typing import TYPE_CHECKING, Optional, Tuple, Union

from .base import MATCH_WINDOW_OPTION, ArgsType, Boss, PayloadGetType, PayloadType, RCOptions, RemoteCommand, ResponseType, Window

if TYPE_CHECKING:
    from kitty.cli_stub import ScrollWindowRCOptions as CLIOptions

class ScrollWindow(RemoteCommand):

    protocol_spec = __doc__ = '''
    amount+/list.scroll_amount: The amount to scroll, a two item list with the first item being \
             either a number or the keywords, start and end. \
             And the second item being either 'p' for pages or 'l' for lines or 'u'
             for unscrolling by lines.
    match/str: The window to scroll
    '''

    short_desc = 'Scroll the specified windows'
    desc = (
        'Scroll the specified windows, if no window is specified, scroll the window this command is run inside.'
        ' :italic:`SCROLL_AMOUNT` can be either the keywords :code:`start` or :code:`end` or an'
        ' argument of the form :italic:`<number>[unit][+-]`. For example, :code:`30` will scroll down 30 lines, :code:`2p-`'
        ' will scroll up 2 pages and :code:`0.5p`will scroll down half page. :code:`3u` will *unscroll* by 3 lines, which means that 3 lines will move from the'
        ' scrollback buffer onto the top of the screen.'
    )
    options_spec = MATCH_WINDOW_OPTION + '''\n
--no-response
type=bool-set
default=false
Don't wait for a response indicating the success of the action. Note that
using this option means that you will not be notified of failures.
'''
    args = RemoteCommand.Args(spec='SCROLL_AMOUNT', count=1, special_parse='parse_scroll_amount(args[0])', json_field='amount')

    def message_to_kitty(self, global_opts: RCOptions, opts: 'CLIOptions', args: ArgsType) -> PayloadType:
        if len(args) < 1:
            self.fatal('Scroll amount must be specified')
        amt = args[0]
        amount: Tuple[Union[str, float], Optional[str]] = (amt, None)
        if amt not in ('start', 'end'):
            pages = 'p' in amt
            unscroll = 'u' in amt
            mult = -1 if amt.endswith('-') and not unscroll else 1
            q = float(amt.rstrip('+-plu'))
            if not pages and not q.is_integer():
                self.fatal('The number must be an integer')
            amount = q * mult, 'p' if pages else ('u' if unscroll else 'l')

        # defaults to scroll the window this command is run in
        return {'match': opts.match, 'amount': amount, 'self': True}

    def response_from_kitty(self, boss: Boss, window: Optional[Window], payload_get: PayloadGetType) -> ResponseType:
        amt = payload_get('amount')
        for window in self.windows_for_match_payload(boss, window, payload_get):
            if window:
                if amt[0] in ('start', 'end'):
                    getattr(window, {'start': 'scroll_home'}.get(amt[0], 'scroll_end'))()
                else:
                    amt, unit = amt
                    if unit == 'u':
                        window.screen.reverse_scroll(int(abs(amt)), True)
                    else:
                        unit = 'page' if unit == 'p' else 'line'
                        if unit == 'page' and not isinstance(amt, int) and not amt.is_integer():
                            amt = round(window.screen.lines * amt)
                            unit = 'line'
                        direction = 'up' if amt < 0 else 'down'
                        func = getattr(window, f'scroll_{unit}_{direction}')
                        for i in range(int(abs(amt))):
                            func()
        return None

scroll_window = ScrollWindow()


import os
from sys import argv
from json import dump
from skimage import io, transform

from find import *

def angles_poll(angles):
    real_angles = [a for a in angles if a is not None]
    if len(real_angles) > 0:
        return np.mean(real_angles)
    return None

def main():
    # bamboo.py <data_path> <out_path> <show>
    data = argv[1]  # path to the data directory
    out = argv[2]   # path to the output directory
    show = len(argv) == 4 and argv[3] == "-s"  # plots the found bamboo lines
    group_dirs = [gd for gd in os.listdir(data) if gd.startswith("group")]
    for group in group_dirs:
        print("---" + group + "---")
        group_data = os.path.join(data, group)
        group_out = os.path.join(out, group)
        if not os.path.exists(group_out):
            os.mkdir(group_out)
        file_names = [fn for fn in os.listdir(group_data) if fn.endswith(OBJ)]
        for fn in file_names:
            print(fn)
            data_file_path = os.path.join(group_data, fn)
            fragment = Render(data_file_path, pose='backside')
            bamboo_angles = [find_bamboo_lines_angle(fragment, show) for i in range(10)]
            chosen_angle = angles_poll(bamboo_angles)
            out_file_path = os.path.join(group_out, fn.removesuffix(OBJ) + JSON)
            with open(out_file_path, mode='w') as out_file:
                dump(chosen_angle, out_file)

def my_main():
    # ...09-...11 contains bamboo lines
    # ...12 doesn't contain bamboo lines
    path = "data/RPf_00311.obj"
    frag = Render(path, pose='backside')
    frag.plot()
    angle = find_bamboo_lines_angle(frag, show=True)
    if angle is not None:
        print(np.rad2deg(angle))
        frag.planar_rotation(angle)
        frag.plot()

    # Previous attempt:
    # real_angles = [a for a in angles if a is not None]
    # if len(real_angles) == 0:
    #     return None
    # angle_votes = np.zeros_like(real_angles)
    # for index_1, angle_1 in enumerate(real_angles):
    #     for index2, angle_2 in enumerate(real_angles):
    #         if index_1 != index2 and abs(angle_1 - angle_2) < epsilon:
    #             angle_votes[index_1] += 1
    # print(angle_votes)
    # return real_angles[np.argmax(angle_votes)]

def align_intact(frag_path, intact_path, show=False):
    frag = Render(frag_path, pose='backside')
    if show:
        frag.plot()
    angles = [find_bamboo_lines_angle(frag, show=show) for i in range(10)]
    angle = angles_poll(angles)
    if angle is not None:
        intact = io.imread(intact_path)
        rotated = transform.rotate(intact, np.rad2deg(angle))
        if show:
            io.imshow(rotated)
        return rotated
    return None

def my_main_2():
    for frag_id in ["09"]+([str(i) for i in range(10, 21)]):
        frag_path = "data/group39/RPf_003"+frag_id+".obj"
        intact_path = "data/group39/RPf_003"+frag_id+"_intact_mesh.png"
        aligned = align_intact(frag_path, intact_path)
        if aligned is not None:
            aligned = np.array(aligned * 255, dtype="uint8")
            io.imsave("out/RPf_003"+frag_id+"_aligned.png", aligned)

if __name__ == '__main__':
    main()


from rest_framework import status
from rest_framework.decorators import api_view, renderer_classes
from rest_framework.response import Response
from .models import User
from .services import UserService, JwtService
from .serializers import UserSerializer
from rest_framework.renderers import JSONRenderer

service = UserService()
jwt = JwtService()

@api_view(['GET', 'POST'])
@renderer_classes((JSONRenderer,))
def users(request, version):
    if request.method == 'GET':
        serializer = UserSerializer(User.objects.all(), many=True)
        return Response(serializer.data)

    elif request.method == 'POST':
        serializer = UserSerializer(data=request.data)
        if not serializer.is_valid():
            return Response(status=status.HTTP_400_BAD_REQUEST)

        serializer.save()
        return Response(serializer.data, status=status.HTTP_201_CREATED)

    else:
        return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)

@api_view(['GET', 'PUT', 'DELETE'])
@renderer_classes((JSONRenderer,))
def user(request, id, version):
    try:
        user = User.objects.get(user_id=id)
    except User.DoesNotExist:
        return Response(status=status.HTTP_404_NOT_FOUND)

    if request.method == 'GET':
        serializer = UserSerializer(user)
        return Response(serializer.data)

    elif request.method == 'PUT':
        serializer = UserSerializer(user, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        user.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

    else:
        return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)

@api_view(['POST'])
@renderer_classes((JSONRenderer,))
def authenticate(request, version):
    if request.method != 'POST':
        return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)

    if not request.data['email'] and request.data['password']:
        return Response(status=status.HTTP_400_BAD_REQUEST)
    response = dict()
    response['token'] = service.authenticate(request.data['email'], request.data['password'])

    if not response['token']:
        return Response(status=status.HTTP_401_UNAUTHORIZED)

    return Response(response)

@api_view(['GET'])
@renderer_classes((JSONRenderer,))
def authorize(request, version):
    token = request.META.get('HTTP_ACCESS_TOKEN')
    if not (token and jwt.validate(token)):
        return Response(status=status.HTTP_401_UNAUTHORIZED)

    return Response(status=status.HTTP_200_OK)


from canvas import Canvas
from shapes import Rectangle, Square

if __name__ == "__main__":

    print("Time to draw some shapes, my dudes...")
    print("First let's set up the canvas to draw on")

    user_height = int(input("How tall, in pixels, should the canvas be??  >"))
    user_width = int(input("How wide, in pixels, should the canvas be??  >"))
    canvas_color = None
    while canvas_color is None:
        print("Would you like to draw on a white or black canvas?")
        print("\t1. Black")
        print("\t2. White")
        color_choice = int(input("Please choose with number keys:  >"))
        if color_choice == 1:
            canvas_color = "black"
        if color_choice == 2:
            canvas_color = "white"

    user_canvas = Canvas(height = user_height, width = user_width,
        color = canvas_color)
    print(f"OK, we just set up a {user_canvas} Time to draw!\n")
    is_drawing = True
    while is_drawing:
        print("What type of shape would you like to draw??")
        print("\t1. A Rectangle")
        print("\t2. A Square")
        print("\t3. I'm done drawing")
        draw_choice = int(input("Please choose with number keys:  >"))
        coord_hint = "[type two non-negative integers separated by a comma and a space]"
        color_hint = "[type three integers 0 to 255 separated by a comma and a space]"
        if draw_choice == 1:
            rect_coord = input(f"Upper-left coordinate of the Rectangle? {coord_hint}: >")
            rect_x = int(rect_coord.split(", ")[0])
            rect_y = int(rect_coord.split(", ")[1])
            rect_height = int(input(f"How tall is the rectangle?  >"))
            rect_width = int(input(f"How wide is the rectangle?  >"))
            rect_rgb = input(f"What color is the Rectangle? {color_hint}: >")
            rect_color = [int(v) for v in rect_rgb.split(", ")]
            user_rect = Rectangle(x = rect_x, y = rect_y, height = rect_height,
                width = rect_width, color = rect_color)
            print(f"Drawing {user_rect}\n")
            user_rect.draw(canvas = user_canvas)
        if draw_choice == 2:
            square_coord = input(f"Upper-left coordinate of the Square? {coord_hint}: >")
            square_x = int(square_coord.split(", ")[0])
            square_y = int(square_coord.split(", ")[1])
            square_side = int(input(f"How long is each sides?  >"))
            square_rgb = input(f"What color is the Square? {color_hint}: >")
            square_color = [int(v) for v in square_rgb.split(", ")]
            user_square = Square(x = square_x, y = square_y,
                side = square_side, color = square_color)
            print(f"Drawing {user_square}\n")
            user_square.draw(canvas = user_canvas)
        if draw_choice == 3:
            is_drawing = False

    print("Thanks for drawing! Please type a name to save your canvas to PNG.")
    canvas_file = input("Save my sketch in file named:  >")
    user_canvas.save(canvas_file)


import torch
from torch import nn

class PositionalEmbedding(nn.Module):
    def __init__(self, demb):
        super(PositionalEmbedding, self).__init__()

        self.demb = demb

        inv_freq = 1 / (10000 ** (torch.arange(0.0, demb, 2.0) / demb))
        self.register_buffer('inv_freq', inv_freq)

    def forward(self, pos_seq, bsz=None):
        sinusoid_inp = torch.ger(pos_seq, self.inv_freq)
        pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=-1)

        if bsz is not None:
            return pos_emb[:,None,:].expand(-1, bsz, -1)
        else:
            return pos_emb[:,None,:]

def calc_fourier_position_embeddings(pos, config):
    dmodel = config["n_embd"]
    # pos.shape is (1, context_length)
    pe = torch.zeros(*pos.shape, dmodel, device=pos.device)
    pos = pos[..., None]
    i2divdmodel = torch.arange(0, dmodel, 2, device=pos.device) / dmodel
    pe[..., ::2] = torch.sin(pos / 10000**i2divdmodel)
    pe[..., 1::2] = torch.cos(pos / 10000**i2divdmodel)
    return pe


# Required packages
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import scipy.stats as stats
import math

# Change directory
import os
#%%
file_path = "C://Users//vojha//Dropbox//0_Programming//ESUM_Experiments//Participant_Data//gps_sensor_data//5s//smoot_signals//complied_data_filtered_with_ID.csv"
# Read in the data: Participants Biofeedback
#xl = pd.ExcelFile("Participants_Bio_feedback_data_dynamics.xlsx")
#xl.sheet_names for calling the name of the sheet. "Sheet1" is it in this excel sheet.
#bio = xl.parse("Sheet1")
file_path_demograph = "C://Users//vojha//Dropbox//0_Programming//ESUM_Experiments//Participant_Data//demography"

data = pd.read_csv(file_path)
#del data["Unnamed: 0"]#deleting unneccasry column
#histogram
#data = data["Area"].tolist()
#fig = plt.figure(figsize=(3.5, 3.8))
#hist, bins = np.histogram(data)
#width = 0.9 * (bins[1] - bins[0])
#center = (bins[:-1] + bins[1:]) / 2
#plt.bar(center, hist, align='center', color = "royalblue", width=width)
##plt.title("Light value distribution")
##plt.xlabel("Sound")
##plt.xlabel("Dust")
##plt.xlabel("Temperature")
##plt.xlabel("Relative Humidity")
#plt.xlabel("Area")
##plt.xlabel("Area")
##plt.ylabel("Frequency")
#fig.savefig('hist_area.pdf')
#fig.savefig('hist_area.jpeg')
#plt.show()

#result = data.sort(["participant"], ascending=[1,1])
#grouped_sum = data.groupby(['ID']).sum()
grouped_mean = data.groupby(['ID']).mean()
x_y_phase_peak = grouped_mean[['X','Y','Sound','Dust','TempEN','RH','Light','EDA_phase','EDA_peaks']]
x_y_phase_peak.to_csv("per_xy_pahse_mean_all_var.csv")
#normalized_grouped_mean =(x_y_phase_peak-x_y_phase_peak.min())/(x_y_phase_peak.max()-x_y_phase_peak.min())
cols_to_norm = ['Sound','Dust','TempEN','RH','Light','EDA_phase','EDA_peaks']
x_y_phase_peak[cols_to_norm] = x_y_phase_peak[cols_to_norm].apply(lambda x: (x - x.min()) / (x.max() - x.min()))
normalized_grouped_mean = x_y_phase_peak
normalized_grouped_mean.to_csv("per_xy_pahse_mean_norm_all_var.csv")

#slicing the data frame participants wise
#[6,7,9,10,13,24,27,28,34,35]#native
#[8,11,16,23,25,26,29,30,31,32]#non-native
#[6,8,9,10,11,16,23,25,27,28,29,30,31]#20-30
#[7,13,23,24,26,32,34,35]#30 and above

#%%
cols_to_norm = ['EDA_phase','EDA_peaks']
data[cols_to_norm] = data[cols_to_norm].apply(lambda x: (x - x.min()) / (x.max() - x.min()))

#%%all participants
grouped_mean = data.groupby(['X', 'Y']).mean()
grouped_mean = grouped_mean[['EDA_phase','EDA_peaks']]
#grouped_mean.to_csv("per_xy_pahse_norm_mean.csv")

#%%
native = data[(data.participant == 6) |
        (data.participant == 7)|
        (data.participant == 9)|
        (data.participant == 10)|
        (data.participant == 13)|
        (data.participant == 24)|
        (data.participant == 27)|
        (data.participant == 28)|
        (data.participant == 34)|
        (data.participant == 35)]

native_grouped_mean = native.groupby(['ID']).mean()
#native_grouped_mean = native_grouped_mean[['EDA_phase','EDA_peaks']]
native_grouped_mean = native_grouped_mean[['X','Y','EDA_phase','EDA_peaks']]

non_native = data[(data.participant == 8)|
        (data.participant == 11)|
        (data.participant == 16)|
        (data.participant == 23)|
        (data.participant == 25)|
        (data.participant == 26)|
        (data.participant == 29)|
        (data.participant == 30)|
        (data.participant == 31)|
        (data.participant == 32)]

non_native_grouped_mean = non_native.groupby(['ID']).mean()
#non_native_grouped_mean = non_native_grouped_mean[['EDA_phase','EDA_peaks']]
non_native_grouped_mean = non_native_grouped_mean[['X','Y','EDA_phase','EDA_peaks']]

min_a = native_grouped_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()
min_b = non_native_grouped_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()

max_a = native_grouped_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()
max_b = non_native_grouped_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()

min_phase = min([min_a[0],min_b[0]])
min_peak = min([min_a[1],min_b[1]])
max_phase = max([max_a[0],max_b[0]])
max_peak = max([max_a[1],max_b[1]])

cols_to_norm = ['EDA_phase']
native_grouped_mean["EDA_phase_norm"] = native_grouped_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))
non_native_grouped_mean["EDA_phase_norm"] = non_native_grouped_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))

cols_to_norm = ['EDA_peaks']
native_grouped_mean["EDA_peaks_norm"] = native_grouped_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))
non_native_grouped_mean["EDA_peaks_norm"] = non_native_grouped_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))

native_grouped_mean.to_csv("per_xy_pahse_norm_mean_native.csv")
non_native_grouped_mean.to_csv("per_xy_pahse_norm_mean_non_native.csv")

#%%age
age_20_30 = data[(data.participant == 6)|
        (data.participant == 8)|
        (data.participant == 9)|
        (data.participant == 10)|
        (data.participant == 11)|
        (data.participant == 16)|
        (data.participant == 23)|
        (data.participant == 25)|
        (data.participant == 27)|
        (data.participant == 28)|
        (data.participant == 29)|
        (data.participant == 30)|
        (data.participant == 31)]

age_20_30_grouped_mean = age_20_30.groupby(['ID']).mean()
#age_20_30_grouped_mean = age_20_30_grouped_mean[['EDA_phase','EDA_peaks']]
age_20_30_grouped_mean = age_20_30_grouped_mean[['X','Y','EDA_phase','EDA_peaks']]

age_30_above = data[(data.participant == 7)|
        (data.participant == 13)|
        (data.participant == 23)|
        (data.participant == 24)|
        (data.participant == 26)|
        (data.participant == 32)|
        (data.participant == 34)|
        (data.participant == 35)]

age_30_above_grouped_mean = age_30_above.groupby(['ID']).mean()
#age_30_above_grouped_mean = age_30_above_grouped_mean[['EDA_phase','EDA_peaks']]
age_30_above_grouped_mean = age_30_above_grouped_mean[['X','Y','EDA_phase','EDA_peaks']]

min_a = age_20_30_grouped_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()
min_b = age_30_above_grouped_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()

max_a = age_20_30_grouped_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()
max_b = age_30_above_grouped_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()

min_phase = min([min_a[0],min_b[0]])
min_peak = min([min_a[1],min_b[1]])
max_phase = max([max_a[0],max_b[0]])
max_peak = max([max_a[1],max_b[1]])

cols_to_norm = ['EDA_phase']
age_20_30_grouped_mean["EDA_phase_norm"] = age_20_30_grouped_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))
age_30_above_grouped_mean["EDA_phase_norm"] = age_30_above_grouped_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))

cols_to_norm = ['EDA_peaks']
age_20_30_grouped_mean["EDA_peaks_norm"] = age_20_30_grouped_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))
age_30_above_grouped_mean["EDA_peaks_norm"] = age_30_above_grouped_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))

age_20_30_grouped_mean.to_csv("per_xy_pahse_norm_mean_age_20_30.csv")
age_30_above_grouped_mean.to_csv("per_xy_pahse_norm_mean_age_30_above.csv")

#%%place
village_town = data[(data.participant == 6)|
        (data.participant == 7)|
        (data.participant == 24)|
        (data.participant == 27)|
        (data.participant == 28)|
        (data.participant == 32)|
        (data.participant == 34)|
        (data.participant == 35)]

village_town_mean = village_town.groupby(['ID']).mean()
#village_town_mean = village_town_mean[['EDA_phase','EDA_peaks']]
village_town_mean = village_town_mean[['X','Y','EDA_phase','EDA_peaks']]

city = data[(data.participant == 9)|
        (data.participant == 10)|
        (data.participant == 11)|
        (data.participant == 13)|
        (data.participant == 26)|
        (data.participant == 30)|
        (data.participant == 29)|
        (data.participant == 31)]

city_mean = city.groupby(['ID']).mean()
#city_mean = city_mean[['EDA_phase','EDA_peaks']]
city_mean = city_mean[['X','Y','EDA_phase','EDA_peaks']]

metropolis = data[(data.participant == 8)|
        (data.participant == 16)|
        (data.participant == 23)|
        (data.participant == 25)]

metropolis_mean = metropolis.groupby(['ID']).mean()
#metropolis_mean = metropolis_mean[['EDA_phase','EDA_peaks']]
metropolis_mean = metropolis_mean[['X','Y','EDA_phase','EDA_peaks']]

min_a = village_town_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()
min_b = city_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()
min_c = metropolis_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()

max_a = village_town_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()
max_b = city_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()
max_c = metropolis_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()

min_phase = min([min_a[0],min_b[0],min_c[0]])
min_peak = min([min_a[1],min_b[1],min_c[1]])
max_phase = max([max_a[0],max_b[0],max_c[0]])
max_peak = max([max_a[1],max_b[1],max_c[1]])

cols_to_norm = ['EDA_phase']
village_town_mean["EDA_phase_norm"] = village_town_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))
city_mean["EDA_phase_norm"] = city_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))
metropolis_mean["EDA_phase_norm"] = metropolis_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))

cols_to_norm = ['EDA_peaks']
village_town_mean["EDA_peaks_norm"] = village_town_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))
city_mean["EDA_peaks_norm"] = city_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))
metropolis_mean["EDA_peaks_norm"] = metropolis_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))

village_town_mean.to_csv("per_xy_pahse_norm_mean_village.csv")
city_mean.to_csv("per_xy_pahse_norm_mean_city.csv")
metropolis_mean.to_csv("per_xy_pahse_norm_mean_metro.csv")

#%% Stat
list_stat = []
A = native_grouped_mean['EDA_phase'].tolist()
B = non_native_grouped_mean['EDA_phase'].tolist()
stats.ttest_ind(a= A,b= B, equal_var=False)    # Assume samples have equal variance?
#stats.ks_2samp(A, B)

A = age_20_30_grouped_mean['EDA_phase'].tolist()
B = age_30_above_grouped_mean['EDA_phase'].tolist()
stats.ttest_ind(a= A,b= B, equal_var=False)    # Assume samples have equal variance?
stats.ks_2samp(A, B)

A = village_town_mean['EDA_phase'].tolist()
B = city_mean['EDA_phase'].tolist()
stats.ttest_ind(a= A,b= B, equal_var=False)    # Assume samples have equal variance?
#stats.ks_2samp(A, B)

A = village_town_mean['EDA_phase'].tolist()
B = metropolis_mean['EDA_phase'].tolist()
stats.ttest_ind(a= A,b= B, equal_var=False)    # Assume samples have equal variance?
#stats.ks_2samp(A, B)

A = city_mean['EDA_phase'].tolist()
B = metropolis_mean['EDA_phase'].tolist()
stats.ttest_ind(a= A,b= B, equal_var=False)    # Assume samples have equal variance?
#stats.ks_2samp(A, B)
#%%mean

list_sums = []
colum_to_sum = "EDA_peaks_norm"
list_sums.append(native_grouped_mean[colum_to_sum].mean())
list_sums.append(non_native_grouped_mean[colum_to_sum].mean())
list_sums.append(age_20_30_grouped_mean[colum_to_sum].mean())
list_sums.append(age_30_above_grouped_mean[colum_to_sum].mean())
list_sums.append(village_town_mean[colum_to_sum].mean())
list_sums.append(city_mean[colum_to_sum].mean())
list_sums.append(metropolis_mean[colum_to_sum].mean())
list_sums
#%%Combination point to point
result_ethenic =  pd.concat([native_grouped_mean, non_native_grouped_mean], axis=1, join_axes=[native_grouped_mean.index])
result_ethenic.columns = ["native_phase","native_peak","non_native_phase","non_native_peak"]

result_age =  pd.concat([age_20_30_grouped_mean, age_30_above_grouped_mean], axis=1, join_axes=[age_20_30_grouped_mean.index])
result_age.columns = ["age_tt_phase","age_tt_peak","age_ta_phase","age_ta_peak"]

result_place =  pd.concat([village_town_mean,city_mean, metropolis_mean], axis=1, join_axes=[village_town_mean.index])
result_place.columns = ["vilage_phase","vilage_peak","city_phase","city_peak","metro_phase","metro_peak"]

#%%line plot

plt.style.use("default")
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=[5,2])
plt.plot(grouped_mean['EDA_phase'].tolist())
#plt.plot(result_ethenic['native_phase'].tolist(), label = 'native (0.13)', color = 'seagreen', linewidth=1)
#plt.plot(result_ethenic['non_native_phase'].tolist(), label = 'non-native (0.15)', color = 'red',linewidth=1)
#plt.plot(result_age['age_tt_phase'].tolist(), label = 'age 20-29 (0.12)', color = 'dimgray',linewidth=1)
#plt.plot(result_age['age_ta_phase'].tolist(), label = 'age 30+ (0.18)', color = 'orange',linewidth=1)
#plt.plot(result_place['vilage_phase'].tolist(), label = 'village (0.13)', color = 'olivedrab',linewidth=1)
#plt.plot(result_place['city_phase'].tolist(), label = 'city (0.17)', color = 'orange',linewidth=1)
#plt.plot(result_place['metro_phase'].tolist(), label = 'metropolis (0.11)', color = 'royalblue',linewidth=1)

#ax.set_ylim([0,1])
plt.xticks([])
plt.ylabel('Normalized mean')
#plt.xlabel('Locations')
#plt.title('All variable normalized mean signal analysis')
plt.legend()
fig.savefig('demograpgy_all.pdf')
fig.savefig('demograpgy_all.jpeg')
plt.show()
plt.close(fig)

#%%bar plot data

list1 = result_ethenic["native_peak"].tolist()
list2 = result_ethenic["non_native_peak"].tolist()

list1 = result_age["age_tt_phase"].tolist()
list2 = result_age["age_ta_phase"].tolist()

#%% create bar plot
length = len(list1)

fig, ax = plt.subplots()
index = np.arange(length)
bar_width = 1
opacity = 0.8

rects1 = plt.bar(index,list1 , bar_width,
                 alpha=opacity,
                 color='b',
                 label='age 20-29')

rects2 = plt.bar(index , list2, bar_width,
                 alpha=opacity,
                 color='g',
                 label='age 30+')

#plt.xlabel('Person')
#plt.ylabel('Scores')
#plt.title('Scores by person')
#plt.xticks(index + bar_width, ('A', 'B', 'C', 'D'))
plt.xticks([])
plt.legend()

plt.tight_layout()
plt.show()


#!/usr/bin/env python3

import os
import requests
import re

directory = "/home/student-03-b60bd46ecaa6/supplier-data/descriptions/"
fruits = []
for file in os.listdir(directory):
        content = []
        f = open("/home/student-03-b60bd46ecaa6/supplier-data/descriptions/{}".format(file))
        lines = f.readlines()
        for line in lines:
                line = line.rstrip()
                content.append(line)
        weight = re.sub('[^0-9]','',content[1])
        fruit = {"name":content[0], "weight":int(weight), "description":content[2], "image_name":"{}.jpeg".format(file[0:3])}
        fruits.append(fruit)
for fruit in fruits:
        response = requests.post("http://34.69.148.85/fruits/", json=fruit)
        print(fruit)
        response.raise_for_status()


#!/usr/bin/env python3
# -*- coding: utf8 -*-

#YakDB commandline tool
import argparse
import sys
import code
import os
import os.path
import YakDB
import YakDB.Dump
from YakDB.Iterators import KeyIterator, KeyValueIterator

def info(db, args):
    print((db.serverInfo()).decode("utf-8"))

def tableInfo(db, args):
    print(db.tableInfo(args.table))

def stop(db, args):
    db.stopServer()

def read(db, args):
    tableNo = args.tableNo
    keys = args.keys
    output = db.read(tableNo, keys) #Always use custom mapping
    #Convert value-only to key-->value map if not --print-raw is set
    if args.printRaw:
        for line in output:
            print(line)
    else:
        outMap = {}
        for index, inval in enumerate(keys):
            outMap[inval] = output[index]
        print(outMap)

def put(db, args):
    tableNo = args.tableNo
    keysValues = args.keyValuePairs
    #Build the key/value dict
    if len(keysValues) % 2 != 0:
        print(("Didn't find a value for key %s, ignoring that key" % keysValues[-1]))
        keysValues = keysValues[0:-1]
    keys = keysValues[0::2]
    values = keysValues[1::2]
    putDict = dict(list(zip(keys, values)))
    db.put(tableNo, putDict)
    #Convert value-only to key-->value map
    if not args.quiet:
        for key, value in putDict.items():
            print(("Successfully put '%s' --> '%s'" % (key, value)))

def exists(db, args):
    tableNo = args.tableNo
    keys = args.keys
    result = db.exists(tableNo, keys)
    if args.printNumeric:
        result = [1 if x else 0 for x in result]
    #Convert value-only to key-->value map
    if args.printRaw:
        for value in result:
            print(value)
    else: #Print dictionary
        outMap = {}
        for index, inval in enumerate(keys):
            outMap[inval] = result[index]
        print(outMap)

def delete(db, args):
    tableNo = args.tableNo
    keys = args.keys
    db.delete(tableNo, keys)
    if not args.quiet:
        print(("Deleted [%s]" % ", ".join(keys)))

def deleteRange(db, args):
    tableNo = args.tableNo
    fromKey = args.fromKey
    toKey = args.toKey
    limit = args.scanLimit
    #Data is remapped into dictionary-form in connection class
    db.deleteRange(tableNo, fromKey, toKey, limit)

def scan(db, args):
    tableNo = args.tableNo
    #Override -t with positional argument, if any
    if args.table is not None:
        tableNo = args.table
    fromKey = args.fromKey
    toKey = args.toKey
    limit = args.scanLimit
    keyFilter = args.keyFilter
    valueFilter = args.valueFilter
    invert = args.invertDirection
    skip = args.skip
    #Lazily iterate over key value tuples
    it = KeyValueIterator(db, tableNo, fromKey, toKey, limit, keyFilter=keyFilter, valueFilter=valueFilter, skip=skip, invert=invert)
    for key, value in it:
        if args.onlyKeys:
            #Print key but strip "b'" and "'"
            print(str(key)[2:-1])
        else: #Print k,v
            print("{0},{1}".format(key, value))

def listKeysInRange(db, args):
    tableNo = args.tableNo
    #Override -t with positional argument, if any
    if args.table is not None:
        tableNo = args.table
    fromKey = args.fromKey
    toKey = args.toKey
    limit = args.scanLimit
    keyFilter = args.keyFilter
    valueFilter = args.valueFilter
    invert = args.invertDirection
    skip = args.skip
    codec = args.codec
    #Lazily iterate over the key list
    it = KeyIterator(db, tableNo, fromKey, toKey, limit, keyFilter=keyFilter, valueFilter=valueFilter, skip=skip, invert=invert)
    if codec == "binary":
        for key in it:
            print(key)
    else:
        for key in it:
            print(key.decode(codec))

def dump(db, args):
    tableNo = args.tableNo
    #Override -t with positional argument, if any
    if args.table is not None:
        tableNo = args.table
    fromKey = args.fromKey
    toKey = args.toKey
    limit = args.scanLimit
    outputFile = args.outputFile
    YakDB.Dump.dumpYDF(db, outputFile, tableNo, fromKey, toKey, limit)

def importDump(db, args):
    tableNo = args.tableNo
    #Override -t with positional argument, if any
    if args.table is not None:
        tableNo = args.table
    inputFile = args.inputFile
    if not os.path.exists(inputFile):
        print("Error: Input file '%s' does not exist!" % inputFile, file=sys.stderr)
        sys.exit(1)
    if not os.path.isfile(inputFile):
        print("Error: Input file '%s' is not a file!" % inputFile, file=sys.stderr)
        sys.exit(1)
    YakDB.Dump.importYDFDump(db, inputFile, tableNo)

def count(db, args):
    tableNo = args.tableNo
    #Override -t with positional argument, if any
    if args.table is not None:
        tableNo = args.table
    fromKey = args.fromKey
    toKey = args.toKey
    print((db.count(tableNo, fromKey, toKey)))

def compact(db, args):
    tables = [args.tableNo]
    fromKey = args.fromKey
    toKey = args.toKey
    if len(args.tables) > 0:
        tables = args.tables
    for table in tables:
        db.compactRange(table, fromKey, toKey)
        if not args.quiet:
            print("Compaction finished")

def openTable(db, args):
    compression = args.compression
    mergeOperator = args.mergeOperator
    lruCacheSize = args.lruCacheSize
    writeBufferSize = args.writeBufferSize
    blocksize = args.blocksize
    bloomFilterBitsPerKey = args.bloomFilterBitsPerKey
    for table in args.tables:
        db.openTable(table, lruCacheSize=lruCacheSize, writeBufferSize=writeBufferSize,
                    tableBlocksize=blocksize, bloomFilterBitsPerKey=bloomFilterBitsPerKey,
                    compression=compression, mergeOperator=mergeOperator)
        if not args.quiet:
            print("Opening table #%d finished" % table)

def closeTable(db, args):
    tables = [args.tableNo]
    if len(args.tables) > 0:
        tables = args.tables
    for table in tables:
        db.closeTable(table)
        if not args.quiet:
            print("Closing table #%d finished" % table)

def truncateTable(db, args):
    tables = [args.tableNo]
    if args.tables:
        tables = args.tables
    for table in tables:
        db.truncate(table)
        if not args.quiet:
            print("Truncating table #%d finished" % table)

def repl(db, args):
    code.InteractiveConsole(locals={"db":db}).interact("YakDB REPL -- Connection is available in 'db' variable")

def yakCLI():
    """
    Call this function to use the yak CLI on sys.argv.
    """
    if sys.version_info.major < 3:
        print("YakDB requires Python 3 to run. Please run yak using a python3k interpreter!")

    parser = argparse.ArgumentParser(description="YakDB client tool")
    #Server optionstype=int
    serverArgsGroup = parser.add_argument_group(parser, "Server options")
    serverArgsGroup.add_argument("-s","--server",
            help="Specifies server URL to connect to",
            default="tcp://localhost:7100",
            action='store',
            dest="serverURL")
    serverArgsGroup.add_argument("-g","--server-group",
            help="Specifies the server group to connect to for PUB/SUB connections",
            action='store',
            dest="serverGroup")
    serverArgsGroup.add_argument("-c","--connection-mode",
            choices=["PUB","PUSH","REQ"],
            default="REQ",
            action="store",
            dest="connectMode",
            help="The connection mode to use for requests. Most requests only work in REQ mode.")
    #Database options
    dbOptsGroup = parser.add_argument_group(parser, "Database options")
    dbOptsGroup.add_argument("-t","--table",
            help="The table number / ID to use",
            action="store",
            default=1,
            type=int,
            dest="tableNo")
    #CLI options
    cliOptsGroup = parser.add_argument_group(parser, "CLI options")
    #Data is remapped in connection class
    cliOptsGroup.add_argument("-q","--quiet",
            help="Don't print connection info",
            action="store_true",
            dest="quiet",
            default=False)
    cliOptsGroup.add_argument("-r","--repl",
            help="Start into a REPL where db is the database connection",
            action="store_true",
            default=False,
            dest="repl")
    ###
    #Create parsers for the individual commands
    ###
    subparsers = parser.add_subparsers(title="Commands")
    #Info
    parserInfo = subparsers.add_parser("info", description="Request server information (features and version number)")
    parserInfo.set_defaults(func=info)
    #Info
    parserTableInfo = subparsers.add_parser("table-info", description="Request table information")
    parserTableInfo.add_argument('table',
        type=int,
        nargs='?',
        action="store",
        help="The tables to get info from. Overrides -t option.")
    parserTableInfo.set_defaults(func=tableInfo)
    #Stop
    parserStop = subparsers.add_parser("stop", description="Stop the YakDB server")
    parserStop.set_defaults(func=stop)
    #Read
    parserRead = subparsers.add_parser("read", description="Read the values for one or more keys")
    parserRead.add_argument('keys',
            nargs='+',
            action="store",
            help="The keys to read")
    parserRead.add_argument('-p','--print-raw',
            dest="printRaw",
            action="store_true",
            default=False,
            help="Print only the values read (separated by newline), not a dictionary")
    parserRead.set_defaults(func=read)
    #Put
    parserPut = subparsers.add_parser("put", description="Write a single key-value pair")
    parserPut.add_argument('keyValuePairs',
            action="store",
            nargs="+",
            help="The keys / values to write, in alternating order (key value key2 value2 ...)")
    parserPut.set_defaults(func=put)
    #Exists
    parserExists = subparsers.add_parser("exists", description="Check if one or more keys exist in the table")
    parserExists.add_argument('keys',
            nargs='+',
            action="store",
            help="The keys to check for existence")
    parserExists.add_argument('-n','--numeric-boolean',
            dest="printNumeric",
            action="store_true",
            default=False,
            help="Instead of printing True or False, print 1 or 0")
    parserExists.add_argument('-p','--print-raw',
            dest="printRaw",
            action="store_true",
            default=False,
            help="Do not print a map from key to its existence value, but only the existence values in the same order as the keys")
    parserExists.set_defaults(func=exists)
    #Delete
    parserDelete = subparsers.add_parser("delete", description="Delete one or more keys")
    parserDelete.add_argument('keys',
            nargs='+',
            action="store",
            help="The keys to delete (keys that don't exist will be ignored)")
    parserDelete.set_defaults(func=delete)
    #Delete range
    parserDeleteRange = subparsers.add_parser("delete-range", description="Delete a range of keys")
    parserDeleteRange.add_argument('--from',
            action="store",
            dest="fromKey",
            default=None,
            help="The key to start deleting at (inclusive), default: start of table")
    parserDeleteRange.add_argument('--to',
            action="store",
            dest="toKey",
            default=None,
            help="The key to stop deleting at (exclusive), default: end of table")
    parserDeleteRange.add_argument('-l','--limit',
            action="store",
            dest="scanLimit",
            type=int,
            default=None,
            help="The maximum number of keys to delete")
    parserDeleteRange.set_defaults(func=deleteRange)
    #Scan
    parserScan = subparsers.add_parser("scan", description="Scan over a specified range in the table and return all key-value-pairs in that range")
    parserScan.add_argument('--from',
            action="store",
            dest="fromKey",
            default=None,
            help="The key to start scanning at (inclusive), default: Start of table")
    parserScan.add_argument('--to',
            action="store",
            dest="toKey",
            default=None,
            help="The key to stop scanning at (exclusive), default: end of table")
    parserScan.add_argument('-k','--key-filter',
            action="store",
            dest="keyFilter",
            default=None,
            help="The server-side key filter. Ignored KV pairs don't count when calculating the limit.")
    parserScan.add_argument('-v','--value-filter',
            action="store",
            dest="valueFilter",
            default=None,
            help="The server-side value filter. Ignored KV pairs don't count when calculating the limit.")
    parserScan.add_argument('-l','--limit',
            action="store",
            dest="scanLimit",
            type=int,
            default=None,
            help="The maximum number of keys to scan")
    parserScan.add_argument('-s','--skip',
            type=int,
            nargs='?',
            default=0,
            action="store",
            help="How many records to skip. Only filter-passing records count as skipped.")
    parserScan.add_argument('-i','--invert-direction',
            action="store_true",
            dest="invertDirection",
            default=False,
            help="Invert the scan direction")
    parserScan.add_argument('--only-keys',
            action="store_true",
            dest="onlyKeys",
            default=False,
            help="Only print keys, omit values completely")
    parserScan.add_argument('table',
            type=int,
            nargs='?',
            action="store",
            help="The tables to scan. Overrides -t option.")
    parserScan.set_defaults(func=scan)
    #List
    parserList = subparsers.add_parser("list", description="List keys in a specified range in the table and return all key-value-pairs in that range")
    parserList.add_argument('--from',
            action="store",
            dest="fromKey",
            default=None,
            help="The key to start listing at (inclusive), default: Start of table")
    parserList.add_argument('--to',
            action="store",
            dest="toKey",
            default=None,
            help="The key to stop listing at (exclusive), default: end of table")
    parserList.add_argument('-k','--key-filter',
            action="store",
            dest="keyFilter",
            default=None,
            help="The server-side key filter. Ignored KV pairs don't count when calculating the limit.")
    parserList.add_argument('-v','--value-filter',
            action="store",
            dest="valueFilter",
            default=None,
            help="The server-side value filter. Ignored KV pairs don't count when calculating the limit.")
    parserList.add_argument('-n','--limit',
            action="store",
            dest="scanLimit",
            type=int,
            default=None,
            help="The maximum number of keys to scan")
    parserList.add_argument('-s','--skip',
            type=int,
            nargs='?',
            default=0,
            action="store",
            help="How many records to skip. Only filter-passing records count as skipped.")
    parserList.add_argument('-i','--invert-direction',
            action="store_true",
            dest="invertDirection",
            default=False,
            help="Invert the scan direction")
    parserList.add_argument('-c','--codec',
            action="store_true",
            dest="codec",
            default="utf-8",
            help="Which codec to use for printing (or 'binary')")
    parserList.add_argument('table',
            type=int,
            nargs='?',
            action="store",
            help="The table to list. Overrides -t option.")
    parserList.set_defaults(func=listKeysInRange)
    #Count
    parserCount = subparsers.add_parser("count", description="Count how many keys exist in a specified range of the table")
    parserCount.add_argument('--from',
            action="store",
            dest="fromKey",
            default=None,
            help="The key to start scanning at (inclusive)")
    parserCount.add_argument('--to',
            action="store",
            dest="toKey",
            default=None,
            help="The key to stop scanning at (exclusive)")
    parserCount.add_argument('table',
            type=int,
            nargs='?',
            action="store",
            help="The tables to count in. Overrides -t option.")
    parserCount.set_defaults(func=count)
    #Compact
    parserCompact = subparsers.add_parser("compact", description="Compact a specified range of the table\nThis operation may take a long time, especially for large tables.")
    parserCompact.add_argument('tables',
            nargs='*',
            action="store",
            type=int,
            help="The tables to compact. Overrides -t option.")
    parserCompact.add_argument('--from',
            action="store",
            dest="fromKey",
            default=None,
            help="The key to start compacting at (inclusive), default: start of table")
    parserCompact.add_argument('--to',
            action="store",
            dest="toKey",
            default=None,
            help="The key to stop compacting at (exclusive), default: end of table")
    parserCompact.set_defaults(func=compact)
    #Open table
    parserOpenTable = subparsers.add_parser("open", description="Open a table.\nThis is only neccessary if you intend to use nonstandard open options.")
    parserOpenTable.add_argument('tables',
            nargs='+',
            type=int,
            action="store",
            help="The table ID to open")
    parserOpenTable.add_argument('-c','--compression',
            dest="compression",
            default="SNAPPY",
            help="Blocklevel compression code (NONE, SNAPPY, ZLIB, BZIP2, LZ4 or LZ4HC)")
    parserOpenTable.add_argument('-m','--merge-operator',
            dest="mergeOperator",
            default="REPLACE",
            help="Merge operator (e.g. REPLACE, INT64ADD, DMUL or APPEND)")
    parserOpenTable.add_argument('-l','--lru-cache-size',
            action="store",
            dest="lruCacheSize",
            type=int,
            default=None,
            help="The size of the LRU cache where uncompressed block data will be stored. Increasing this yields better performance for random-read-access-heavy workloads.")
    parserOpenTable.add_argument('-w','--write-buffer-size',
            action="store",
            dest="writeBufferSize",
            type=int,
            default=None,
            help="The size of the write buffer. Increasing this yields better write performance")
    parserOpenTable.add_argument('-b','--blocksize',
            action="store",
            dest="blocksize",
            type=int,
            default=None,
            help="The table blocksize. Larger blocks yield better compression and scan performance, but usually worse random-access performance. See LevelDB docs for details")
    parserOpenTable.add_argument('-f','--bloom-filter-bits-per-key',
            action="store",
            dest="bloomFilterBitsPerKey",
            type=int,
            default=None,
            help="The number of bloom filter bits per key. Increasing this improves read & exist performance for keys that don't exist, but increases memory usage.")
    parserOpenTable.set_defaults(func=openTable)
    #Close table
    parserCloseTable = subparsers.add_parser("close", description="Close a table\nThis is usually not neccessary, unless you want to save memory. ")
    parserCloseTable.add_argument('tables',
            nargs='*',
            type=int,
            action="store",
            help="The tables to compact. Overrides -t option.")
    parserCloseTable.set_defaults(func=closeTable)
    #Close table
    parserTruncateTable = subparsers.add_parser("truncate", description="Close and truncate a table. Deletes the table's files on filesystem-level.")
    parserTruncateTable.add_argument('tables',
            nargs='*',
            type=int,
            action="store",
            help="The tables to compact. Overrides -t option.")
    parserTruncateTable.add_argument('-y','--yes',
            dest="confirmed",
            action="store_true",
            default=False,
            help="Skip the 'Do you really want to truncate?' question")
    parserTruncateTable.set_defaults(func=truncateTable)
    #Dump
    parserDump = subparsers.add_parser("dump", description="Dump a specified range of a table (default: entire table) into a YDF format. Automatically creates a table snapshot for the dump.")
    parserDump.add_argument('-o','--output',
            action="store",
            dest="outputFile",
            required=True,
            help="The file to write the YDF dump to. Append .gz to use gzipped output (recommended)")
    parserDump.add_argument('--from',
            action="store",
            dest="fromKey",
            default=None,
            help="The key to start dumping at (inclusive), default: Start of table")
    parserDump.add_argument('--to',
            action="store",
            dest="toKey",
            default=None,
            help="The key to stop dumping at (exclusive), default: end of table")
    parserDump.add_argument('-l','--limit',
            action="store",
            dest="scanLimit",
            type=int,
            default=None,
            help="The maximum number of keys to scan")
    parserDump.add_argument('table',
            type=int,
            nargs='?',
            action="store",
            help="The table to dump. Overrides -t option.")
    parserDump.set_defaults(func=dump)
    #Import
    parserImport = subparsers.add_parser("import", description="Import a YDF dump")
    parserImport.add_argument('-i','--input',
            action="store",
            dest="inputFile",
            required=True,
            help="The file to read the YDF dump from. Append .gz to use transparent decompression.")
    parserImport.add_argument('table',
            type=int,
            nargs='?',
            action="store",
            help="The table to import to. Overrides -t option.")
    parserImport.set_defaults(func=importDump)
    #REPL
    parserREPL = subparsers.add_parser("repl", description="Start a Read-eval-print loop (REPL) for interactive DB usage")
    parserREPL.set_defaults(func=repl)
    ###
    #Parse and call the function
    ###
    args = parser.parse_args()
    db = YakDB.Connection()
    if not args.quiet:
        sys.stderr.write("Connecting to %s\n" % args.serverURL)
    if args.connectMode == "PUSH":
        db.usePushMode()
    elif args.connectMode == "PUB":
        db.usePubMode()
    else:
        db.useRequestReplyMode()
    db.connect(args.serverURL)
    #For some reason, the default=info setting only works with Python2
    try:
        args.func(db, args)
    except AttributeError as e:
        print(e)
        info(db, args)


from __future__ import annotations

import logging
from abc import abstractmethod
from typing import Any, Dict, List, Optional

import requests
from langchain_core.callbacks import CallbackManagerForToolRun
from langchain_core.pydantic_v1 import root_validator
from langchain_core.tools import BaseTool
from langchain_core.utils import get_from_dict_or_env

logger = logging.getLogger(__name__)

class EdenaiTool(BaseTool):

    """
    the base tool for all the EdenAI Tools .
    you should have
    the environment variable ``EDENAI_API_KEY`` set with your API token.
    You can find your token here: https://app.edenai.run/admin/account/settings
    """

    feature: str
    subfeature: str
    edenai_api_key: Optional[str] = None
    is_async: bool = False

    providers: List[str]
    """provider to use for the API call."""

    @root_validator(allow_reuse=True)
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key exists in environment."""
        values["edenai_api_key"] = get_from_dict_or_env(
            values, "edenai_api_key", "EDENAI_API_KEY"
        )
        return values

    @staticmethod
    def get_user_agent() -> str:
        from langchain_community import __version__

        return f"langchain/{__version__}"

    def _call_eden_ai(self, query_params: Dict[str, Any]) -> str:
        """
        Make an API call to the EdenAI service with the specified query parameters.

        Args:
            query_params (dict): The parameters to include in the API call.

        Returns:
            requests.Response: The response from the EdenAI API call.

        """

        # faire l'API call

        headers = {
            "Authorization": f"Bearer {self.edenai_api_key}",
            "User-Agent": self.get_user_agent(),
        }

        url = f"https://api.edenai.run/v2/{self.feature}/{self.subfeature}"

        payload = {
            "providers": str(self.providers),
            "response_as_dict": False,
            "attributes_as_list": True,
            "show_original_response": False,
        }

        payload.update(query_params)

        response = requests.post(url, json=payload, headers=headers)

        self._raise_on_error(response)

        try:
            return self._parse_response(response.json())
        except Exception as e:
            raise RuntimeError(f"An error occurred while running tool: {e}")

    def _raise_on_error(self, response: requests.Response) -> None:
        if response.status_code >= 500:
            raise Exception(f"EdenAI Server: Error {response.status_code}")
        elif response.status_code >= 400:
            raise ValueError(f"EdenAI received an invalid payload: {response.text}")
        elif response.status_code != 200:
            raise Exception(
                f"EdenAI returned an unexpected response with status "
                f"{response.status_code}: {response.text}"
            )

        # case where edenai call succeeded but provider returned an error
        # (eg: rate limit, server error, etc.)
        if self.is_async is False:
            # async call are different and only return a job_id,
            # not the provider response directly
            provider_response = response.json()[0]
            if provider_response.get("status") == "fail":
                err_msg = provider_response["error"]["message"]
                raise ValueError(err_msg)

    @abstractmethod
    def _run(
        self, query: str, run_manager: Optional[CallbackManagerForToolRun] = None
    ) -> str:
        pass

    @abstractmethod
    def _parse_response(self, response: Any) -> str:
        """Take a dict response and condense it's data in a human readable string"""
        pass

    def _get_edenai(self, url: str) -> requests.Response:
        headers = {
            "accept": "application/json",
            "authorization": f"Bearer {self.edenai_api_key}",
            "User-Agent": self.get_user_agent(),
        }

        response = requests.get(url, headers=headers)

        self._raise_on_error(response)

        return response

    def _parse_json_multilevel(
        self, extracted_data: dict, formatted_list: list, level: int = 0
    ) -> None:
        for section, subsections in extracted_data.items():
            indentation = "  " * level
            if isinstance(subsections, str):
                subsections = subsections.replace("\n", ",")
                formatted_list.append(f"{indentation}{section} : {subsections}")

            elif isinstance(subsections, list):
                formatted_list.append(f"{indentation}{section} : ")
                self._list_handling(subsections, formatted_list, level + 1)

            elif isinstance(subsections, dict):
                formatted_list.append(f"{indentation}{section} : ")
                self._parse_json_multilevel(subsections, formatted_list, level + 1)

    def _list_handling(
        self, subsection_list: list, formatted_list: list, level: int
    ) -> None:
        for list_item in subsection_list:
            if isinstance(list_item, dict):
                self._parse_json_multilevel(list_item, formatted_list, level)

            elif isinstance(list_item, list):
                self._list_handling(list_item, formatted_list, level + 1)

            else:
                formatted_list.append(f"{'  ' * level}{list_item}")


import time
import os
import owncloud

from fastapi import APIRouter, Request, Depends
from fastapi.responses import JSONResponse
from pydantic import AnyHttpUrl

from src.models.responses.image import ImagesFolderModel
from src.models.responses.error import ErrorModel

import src.utils.constants as constants
from src.utils.authorization import authorization

import structlog

logger = structlog.getLogger("json_logger")

router = APIRouter()

@router.get(
    "/experiments/{experiment_id}/images/folder-name",
    tags=["image"],
    summary="Get public images folder name",
    response_model=ImagesFolderModel,
    responses={
        404: {
            "model": ErrorModel
        }
    }
)
def get_images_folder_name(request: Request, experiment_id: str, user_id: dict = Depends(authorization)):
    total_time = time.time()
    response = {}
    storage = request.state.storage

    if experiment_id.startswith('demo'):
        images_dir = os.path.join(constants.DEMO_DIR, experiment_id, constants.IMAGES_DIR)
    else:
        data_dir = '{}{}'.format(constants.NEXTCLOUD_PREFIX_USER_DIR, user_id)
        images_dir = os.path.join(data_dir, experiment_id, constants.IMAGES_DIR)

    try:
        public_name = storage.get_link(images_dir)
        response['images_folder_name'] = os.path.split(public_name)[-1]

    except:
        logger.error(message='Experiment images folder name not exist', action='get_images_folder_name', status='FAILED', resource='lse-service', userid=user_id)
        return JSONResponse(
            status_code=404,
            content={"message": "Experiment images folder name not exist"}
        )
    elapsed = time.time() - total_time
    logger.info(message='Folder link retrieved', duration=elapsed, action='get_images_folder_name', status='SUCCED', resource='lse-service', userid=user_id)
    return response

@router.get(
    "/experiments/{experiment_id}/images/{image_name}",
    tags=["image"],
    summary="Get single image",
    response_model=AnyHttpUrl,
    responses={
        404: {
            "model": ErrorModel
        }
    }
)
def get_single_image(request: Request, image_name: str, experiment_id: str, user_id: dict = Depends(authorization)):
    storage = request.state.storage

    data_dir = '{}{}'.format(constants.NEXTCLOUD_PREFIX_USER_DIR, user_id)
    image_path = os.path.join(data_dir, experiment_id,
                              constants.IMAGES_DIR, image_name)

    try:
        link = storage.get_link(image_path)

    except owncloud.owncloud.HTTPResponseError:
        return JSONResponse(
            status_code=404,
            content={"message": "The image doesn't exist"}
        )

    return '{}{}'.format(link, "/preview")


import os
import cv2
import argparse
import numpy as np

def convert_video_to_frames(video_path, outfolder, sample_rate = 1.0):
    if not os.path.exists(outfolder):
        os.makedirs(outfolder, exist_ok=True)
    capture = cv2.VideoCapture(video_path)
    video_name = video_path.split("/")[-1]
    frameNr = 0
    while True:
        success, frame = capture.read()
        if success:
            seed = np.random.random()
            if seed >= (1 - sample_rate):
                img_path = os.path.join(outfolder, f"{video_name}#frame={frameNr}.jpg")
                cv2.imwrite(img_path, frame)
                print(f"Successfully print frame {frameNr} at path {img_path}")
        else:
            break
        frameNr+=1
    capture.release()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--input", "-i", type=str, help="Input Video Path")
    parser.add_argument("--output", "-o", type=str, help="Output Folder")
    parser.add_argument("--sample_rate", "-r", type=float, default=1.0, help="Sampling Rate for frames")
    args = parser.parse_args()

    convert_video_to_frames(args.input, args.output, args.sample_rate)


import logging

import numpy as np
import scipy.ndimage as nd
import scipy.ndimage.morphology as morph
import scipy.stats as ss
from scipy.signal import savgol_coeffs
import astropy.wcs as wcs
import astropy.units as u
from astropy.convolution import convolve, Gaussian2DKernel
from astropy.io import fits
from astropy.stats import mad_std
from spectral_cube import SpectralCube

from .pipelineVersion import tableversion, version

np.seterr(divide='ignore', invalid='ignore')

mad_to_std_fac = 1.482602218505602

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

def mad_zero_centered(data, mask=None):
    """
    Estimates the noise in a data set using the median absolute
    deviation. Assumes a normal distribution and that the data are
    centered on zero. Excludes any masked regions.

    Parameters:
    -----------

    data : np.array

        Array of data (floats)

    Keywords:
    ---------

    mask : bool

        Boolean array with True indicating where data can be used in
        the noise estimate. (i.e., True is noise). If none is supplied
        a trivial mask is estimated.

    """

    # TBD: Add input error checking

    # Select finite data and exclude not-a-number
    where_data_valid = np.logical_and(~np.isnan(data), np.isfinite(data))

    # Make a trivial mask (to simplify the code) if none has been supplied
    if mask is None:
        mask = np.isfinite(data)

    # Check that we have enough data to estimate the noise
    nData = mask.sum()
    if nData == 0:
        logger.info('No data in mask. Returning NaN.')
        return(np.nan)

    # Estimate the significance threshold likely associated with
    # false positives based on the size of the data.

    sig_false = ss.norm.isf(0.5 / nData)

    # Make a first estimate of the noise based on the negatives.

    data_lt_zero = np.logical_and(
        np.less(data, 0,
                where=where_data_valid,
                out=np.full(where_data_valid.shape,
                            False, dtype=bool)), mask)

    mad1 = mad_to_std_fac * np.abs(np.median(data[data_lt_zero]))

    # Make a second estimate now including the positives less than
    # the false-positive threshold.

    data_lt_mad1 = np.logical_and(
        np.less(data, (sig_false * mad1),
                where=where_data_valid,
                out=np.full(where_data_valid.shape,
                            False, dtype=bool)), mask)
    mad2 = mad_to_std_fac * np.abs(np.median(np.abs(data[data_lt_mad1])))

    # Return this second estimate

    return(mad2)

def noise_cube(data, mask=None,
               nThresh=30, iterations=1,
               do_map=True, do_spec=True,
               box=None, spec_box=None,
               bandpass_smooth_window=None,
               bandpass_smooth_order=3,
               oversample_boundary=False):

    """

    Makes an empirical estimate of the noise in a cube assuming that
    it is normally distributed about zero. Treats the spatial and
    spectral dimensions as separable.

    Parameters:
    -----------

    data : np.array
        Array of data (floats)

    Keywords:
    ---------

    mask : bool

        Boolean array with False indicating where data can be
        used in the noise estimate. (i.e., True is signal)

    do_map : bool

        Estimate spatial variations in the noise. Default is True. If
        set to False, all locations in a plane have the same noise
        estimate.

    do_spec : bool

        Estimate spectral variations in the noise. Default is True. If
        set to False, all channels in a spectrum have the same noise
        estimate.

    box : int

        Spatial size of the box over which noise is calculated in
        pixels.  Default: no box, every pixel gets its own noise
        estimte.

    spec_box : int

        Spectral size of the box overwhich the noise is calculated.
        Default: no box, each channel gets its own noise estimate.

    nThresh : int
        Minimum number of data to be used in an individual noise estimate.

    iterations : int
        Number of times to iterate the noise solution to force Gaussian
        statistics.  Default: no iterations.

    bandpass_smooth_window : int
        Number of channels used in bandpass smoothing kernel.  Defaults to
        nChan / 4 where nChan number of channels.  Set to zero to suppress
        smoothing. Uses Savitzky-Golay smoothing

    bandpass_smooth_order : int
        Polynomial order used in smoothing kernel.  Defaults to 3.

    """

    # TBD: add error checking

    # Create a mask that identifies voxels to be fitting the noise

    noisemask = np.isfinite(data)
    if mask is not None:
        noisemask[mask] = False

    # Default the spatial step size to individual pixels

    step = 1
    halfbox = step // 2

    # If the user has supplied a spatial box size, recast this into a
    # step size that critically samples the box and a halfbox size
    # used for convenience.

    if box is not None:
        step = np.floor(box/2.5).astype(int)
        halfbox = int(box // 2)

    # Include all pixels adjacent to the spatial
    # boundary of the data as set by NaNs
    boundary = np.all(np.isnan(data), axis=0)

    if oversample_boundary:
        struct = nd.generate_binary_structure(2, 1)
        struct = nd.iterate_structure(struct, halfbox)
        rind = np.logical_xor(nd.binary_dilation(boundary, struct),
                              boundary)
        extray, extrax = np.where(rind)
    else:
        extray, extrax = None, None

    # If the user has supplied a spectral box size, use this to
    # calculate a spectral step size.

    if spec_box is not None:
        spec_step = np.floor(spec_box / 2).astype(int)
        boxv = int(spec_box // 2)
    else:
        boxv = 0

    # Default the bandpass smoothing window

    if bandpass_smooth_window is None:
        bandpass_smooth_window = 2 * (data.shape[0] // 8) + 1

    # Initialize output to be used in the case of iterative
    # estimation.

    noise_cube_out = np.ones_like(data)

    # Iterate

    for ii in np.arange(iterations):

        if not do_map:

            # If spatial variations are turned off then estimate a
            # single value and fill the noise map with this value.

            noise_value = mad_zero_centered(data, mask=noisemask)
            noise_map = np.zeros(data.shape[1:]) + noise_value

        else:

            # Initialize map to be full of not-a-numbers
            noise_map = np.zeros(data.shape[1:]) + np.nan

            # Make a noise map

            xx = np.arange(data.shape[2])
            yy = np.arange(data.shape[1])

            # Sample starting at halfbox and stepping by step. In the
            # individual pixel limit this just samples every spectrum.

            xsamps = xx[halfbox::step]
            ysamps = yy[halfbox::step]
            xsampsf = (xsamps[np.newaxis,:]
                      * (np.ones_like(ysamps))[:,np.newaxis]).flatten()
            ysampsf = (ysamps[:,np.newaxis] * np.ones_like(xsamps)).flatten()

            for x, y in zip(xsampsf, ysampsf):
                # Extract a minicube and associated mask from the cube

                minicube = data[:, (y-halfbox):(y+halfbox+1),
                               (x-halfbox):(x+halfbox+1)]
                minicube_mask = noisemask[:, (y-halfbox):(y+halfbox+1),
                                             (x-halfbox):(x+halfbox+1)]

                # If we have enough data, fit a noise value for this entry

                if np.sum(minicube_mask) > nThresh:
                    noise_map[y, x] = mad_zero_centered(minicube,
                                                        mask=minicube_mask)

            if extrax is not None and extray is not None:
                for x, y in zip(extrax, extray):

                    minicube = data[:, (y-halfbox):(y+halfbox+1),
                                    (x-halfbox):(x+halfbox+1)]
                    minicube_mask = noisemask[:, (y-halfbox):(y+halfbox+1),
                                              (x-halfbox):(x+halfbox+1)]

                    if np.sum(minicube_mask) > nThresh:
                        noise_map[y, x] = mad_zero_centered(minicube,
                                                            mask=minicube_mask)

            noise_map[boundary] = np.nan

            # If we are using a box size greater than an individual pixel
            # interpolate to fill in the noise map.

            if halfbox > 0:

                # Note the location of data, this is the location
                # where we want to fill in noise values.
                data_footprint = np.any(np.isfinite(data), axis=0)

                # Generate a smoothing kernel based on the box size.
                kernel = Gaussian2DKernel(box / np.sqrt(8 * np.log(2)))

                # Make a weight map to be used in the convolution, in
                # this weight map locations with measured values have
                # unity weight. This without measured values have zero
                # weight.

                # wt_map = np.isfinite(noise_map).astype(np.float)
                # wt_map[boundary] = np.nan
                # Take an average weighted by the kernel at each
                # location.
                # noise_map[np.isnan(noise_map)] = 0.0
                # y, x = np.where(np.isfinite(noise_map))
                # import scipy.interpolate as interp
                # func = interp.interp2d(x, y, noise_map[y, x], kind='cubic')

                noise_map = convolve(noise_map, kernel, boundary='extend')

                # yy, xx = np.indices(noise_map.shape)
                # noise_map_beta = interp.griddata((y, x), noise_map[y,x],
                #                                  (yy, xx), method='cubic')
                # noise_map_beta = func(yy, xx)
                # noise_map_beta[boundary] = np.nan
                # noise_map = noise_map_beta
                # wt_map = convolve(wt_map, kernel, boundary='extend')

                # noise_map /= wt_map

                # Set the noise map to not-a-number outside the data
                # footprint.

                noise_map[~data_footprint] = np.nan
        # Initialize spectrum

        noise_spec = np.zeros(data.shape[0]) + np.nan

        if not do_spec:

            # If spectral variations are turned off then assume that
            # the noise_map describes all channels of the cube. In
            # that case the spectrum is just a series of ones.

            noise_spec = np.ones(data.shape[0])

        else:

            # Loop over channels

            zz = np.arange(data.shape[0])
            for z in zz:

                # Identify the range of channels to be considered
                # in this estimate.

                lowz = np.clip(z - boxv, 0, data.shape[0])
                hiz = np.clip(z + boxv + 1, 0, data.shape[0])

                # Extract a slab from the cube and normalize it by the
                # noise map. Now any measured noise variations are
                # relative to those in the noise map.

                slab = data[lowz:hiz, :, :] / noise_map[np.newaxis, :, :]
                slab_mask = noisemask[lowz:hiz, :, :]
                noise_spec[z] = mad_zero_centered(slab, mask=slab_mask)

            # Smooth the spectral variations in the noise.

            if bandpass_smooth_window > 0:

                # Initialize a Savitzky-Golay filter then run it over
                # the noise spectrum.

                kernel = savgol_coeffs(int(bandpass_smooth_window),
                                       int(bandpass_smooth_order))

                baddata = np.isnan(noise_spec)
                noise_spec = convolve(noise_spec, kernel,
                                      nan_treatment='interpolate',
                                      boundary='extend')
                noise_spec[baddata] = np.nan

                # Make sure that the noise spectrum is normalized by
                # setting the median to one.

                noise_spec /= np.nanmedian(noise_spec)

        # Combine the spatial and spectral variations into a
        # three-dimensional noise estimate.

        noise_cube = np.ones_like(data)
        noise_cube *= (noise_map[np.newaxis, :]
                       * noise_spec[:, np.newaxis, np.newaxis])

        if iterations == 1:
            return(noise_cube)
        else:

            # If iterating, normalize the data by the current noise
            # estimate and scale the current noise cube by the new
            # estimate.

            data = data / noise_cube
            noise_cube_out *= noise_cube

    # If iterating return the iterated noise cube.

    return(noise_cube_out)

def recipe_phangs_noise(
    incube=None,
    outfile=None,
    mask=None,
    noise_kwargs=None,
    return_spectral_cube=False,
    overwrite=False):
    """

    Wrap noise_cube with a set of preferred parameters for the
    PHANGS-ALMA CO work.

    Parameters:
    -----------

    cube : np.array

        Array of data (floats)

    Keywords:
    ---------

    mask : bool

        Boolean array with False indicating where data can be used in
        the noise estimate. (i.e., True is signal).

    """

    # &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%
    # Error checking and work out inputs
    # &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%

    if type(incube) is SpectralCube:
        cube = incube
    elif type(incube) == str:
        cube = SpectralCube.read(incube)
    else:
        logger.error("Input must be a SpectralCube object or a filename.")

    # Initialize an empty kwargs dictionary
    if noise_kwargs is None:
        noise_kwargs = {}

    # If no box is specified, default to one about two beams across
    if 'box' not in noise_kwargs:
        pixels_per_beam = cube.pixels_per_beam
        box = np.ceil(2.5 * pixels_per_beam**0.5)
        noise_kwargs['box'] = box

    # Default to an odd bandpass smothing window
    if 'bandpass_smooth_window' not in noise_kwargs:
        spectral_smooth = np.ceil(cube.shape[0] / 5) // 2 * 2 + 1
        noise_kwargs['bandpass_smooth_window'] = spectral_smooth

    if 'spec_box' not in noise_kwargs:
        noise_kwargs['spec_box'] = 5

    if 'iterations' not in noise_kwargs:
        noise_kwargs['iterations'] = 4

    # Require a valid cube input as a
    if mask is not None:
        if type(mask) is SpectralCube:
            noise_kwargs['mask'] = mask
        elif type(mask) == type("hello"):
            noise_kwargs['mask'] = SpectralCube.read(mask)
        else:
            logger.error("Mask must be a SpectralCube object or a filename or None.")

    # Fill in the mask if it hasn't already been filled in.
    if 'mask' not in noise_kwargs:

        # Check if a non-trivial signal mask is attached to the cube
        if (np.sum(cube.mask.include())
            < np.sum(np.isfinite(cube.filled_data[:].value))):
            noise_kwargs['mask'] = cube.mask.include()
        else:
            noise_kwargs['mask'] = None

    # &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%
    # Run the noise estimate
    # &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%

    data = cube.filled_data[:].value
    badmask = np.isnan(data)
    badmask = nd.binary_dilation(badmask,
                                 structure=nd.generate_binary_structure(3, 2))
    data[badmask] = np.nan
    rms = noise_cube(data,
                     **noise_kwargs)

    # &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%
    # Write or return as requested
    # &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%

    # In this case can avoid a recast
    if not return_spectral_cube and (outfile is None):
        return(rms)

    # Recast from numpy array to spectral cube
    header = cube.header
    datamin = np.nanmin(rms)
    datamax = np.nanmax(rms)
    if np.isfinite(datamin):
        header['DATAMIN'] = datamin
    if np.isfinite(datamax):
        header['DATAMAX'] = datamax
    header['COMMENT'] = 'Produced with PHANGS-ALMA pipeline version ' + version
    if tableversion:
        header['COMMENT'] = 'Galaxy properties from PHANGS sample table version ' + tableversion
    rms = SpectralCube(rms, wcs=cube.wcs, header=header,
                       meta={'BUNIT':cube.header['BUNIT']})

    # Write to disk, if desired
    if outfile is not None:
        rms.write(outfile, overwrite=overwrite)

    if return_spectral_cube:
        return(rms)
    else:
        return(rms.filled_data[:].value)


from gtts import gTTS
from playsound import playsound

# 영어 문장
text = 'Can I help you?'
file_name = 'sample.mp3'

# 문장을 영어로 인식해서 음성으로 바꿈
# tts_en = gTTS(text=text, lang='en')
# tts_en.save(file_name)
# playsound(file_name)

# 한글 문장
text = '모델에 대한 하이퍼파라미터는 모델 학습을 위한 설정과 모델 레이어의 차원 수 설정으로 나뉜다.'
tts_kr = gTTS(text=text, lang='ko')
tts_kr.save(file_name)
# playsound(file_name)

with open('sample.txt', 'r', encoding='utf8') as f:
    text = f.read()

tts_en = gTTS(text=text, lang='ko')
tts_en.save(file_name)
playsound(file_name)


import tensorflow as tf
import time
from datetime import timedelta
import numpy as np
import random
from tensorflow.python.training import training_util
from tensorflow.contrib import layers
from .. import trainer, metrics
from ..tf_dataset_generator import TFDataSetGenerator
from ..configuration import *
from ..rnn.text_classification_train import _load_embeddings

class DocPredictionDataset(TFDataSetGenerator):
    """
    Custom dataset. We use it to feed the session.run method directly.
    """

    def __init__(self, type='train',
                 vocabulary_size=VOCABULARY_SIZE,
                 embedding_size=EMBEDDINGS_SIZE,
                 balance_classes=False):
        self.type = type
        if type != 'train' and type != 'test' and type != 'stage2_test' and type != 'val':
            raise ValueError('Type must be train, test, stage2_test or val')
        docs_filename = '{}_set'.format(type)
        if type == 'train':
            embeds_filename = 'doc_embeddings_{}_{}'.format(vocabulary_size, embedding_size)
        else:
            embeds_filename = 'doc_eval_embeddings_{}_{}_{}'.format(type, vocabulary_size,
                                                                    embedding_size)
        word_embeds = _load_embeddings(vocabulary_size, embedding_size,
                                       filename_prefix='word_embeddings',
                                       from_dir=DIR_DATA_DOC2VEC)

        self.docs_file = os.path.join(DIR_DATA_DOC2VEC, docs_filename)
        self.embeds_file = os.path.join(DIR_DATA_DOC2VEC, embeds_filename)
        self.doc_genes = []
        self.doc_variants = []
        if type == 'train' or type == 'val':
            self.doc_labels = []
        else:
            self.doc_labels = None
        # pre load data in memory for the generator
        with open(self.docs_file) as f:
            for line in f.readlines():
                sp = line.split('||')
                if type == 'train' or type == 'val':
                    self.doc_labels.append(int(sp[0].strip()))
                gene = word_embeds[int(sp[1].strip())]
                self.doc_genes.append(gene)
                variant = np.zeros(embedding_size, dtype=np.float32)
                v = [int(x) for x in sp[2].split()]
                for x in v:
                    variant += np.asarray(word_embeds[x], dtype=np.float32)
                variant /= len(v)
                self.doc_variants.append(variant)

        output_types = (tf.float32, tf.float32, tf.float32, tf.int32)
        with open(self.embeds_file) as f:
            self.embeds = [l.split(',') for l in f.readlines()]
            for i, line in enumerate(self.embeds):
                self.embeds[i] = [float(e) for e in line]
        if type == 'train' and balance_classes:
            self._balance_classes()

        super(DocPredictionDataset, self).__init__(name=type,
                                                   generator=self._generator,
                                                   output_types=output_types,
                                                   min_queue_examples=1000,
                                                   shuffle_size=2000)

    def _balance_classes(self):
        dataset = zip(self.doc_labels, self.embeds)
        classes_group = {}
        for d in dataset:
            if d[0] not in classes_group:
                classes_group[d[0]] = []
            classes_group[d[0]].append(d)
        max_in_class = np.max([len(v) for v in classes_group.values()]) * 2
        new_dataset = []
        for key, class_list in classes_group.iteritems():
            random.shuffle(class_list)
            for index in range(max_in_class - len(class_list)):
                class_list.append(class_list[index])
            new_dataset.extend(class_list)

        random.shuffle(new_dataset)
        self.doc_labels, self.embeds = zip(*new_dataset)

    def _generator(self):
        for i in range(len(self.embeds)):
            embeds = np.asarray(self.embeds[i], dtype=np.float32)
            gene = np.asarray(self.doc_genes[i], dtype=np.float32)
            variant = np.asarray(self.doc_variants[i], dtype=np.float32)
            if self.doc_labels is None:
                # evaluation set:
                yield embeds, gene, variant, np.int32(-1)
            else:
                # train or validation set:
                # subtract 1 to class as classes goes from 1 to 9 (both inclusive)
                label = np.int32(int(self.doc_labels[i]) - 1)
                yield embeds, gene, variant, label

def doc2vec_prediction_model(input_vectors, input_gene, input_variation, output_label, batch_size,
                             is_training, embedding_size, output_classes):
    # inputs/outputs
    input_vectors = tf.reshape(input_vectors, [batch_size, embedding_size])
    input_gene = tf.reshape(input_gene, [batch_size, embedding_size])
    input_variation = tf.reshape(input_variation, [batch_size, embedding_size])
    targets = None
    if output_label is not None:
        output_label = tf.reshape(output_label, [batch_size, 1])
        targets = tf.one_hot(output_label, axis=-1, depth=output_classes, on_value=1.0,
                             off_value=0.0)
        targets = tf.squeeze(targets, axis=1)

    net = tf.concat([input_vectors, input_gene, input_variation], axis=1)
    net = layers.fully_connected(net, embedding_size * 2, activation_fn=tf.nn.relu)
    net = layers.dropout(net, keep_prob=0.85, is_training=is_training)
    net = layers.fully_connected(net, embedding_size, activation_fn=tf.nn.relu)
    net = layers.dropout(net, keep_prob=0.85, is_training=is_training)
    net = layers.fully_connected(net, embedding_size // 4, activation_fn=tf.nn.relu)
    logits = layers.fully_connected(net, output_classes, activation_fn=None)

    return logits, targets

class DocPredictionTrainer(trainer.Trainer):
    """
    Helper class to run the training and create the model for the training. See trainer.Trainer for
    more details.
    """

    def __init__(self, dataset):
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        super(DocPredictionTrainer, self).__init__(DIR_D2V_DOC_LOGDIR, dataset=dataset,
                                                   monitored_training_session_config=config,
                                                   save_checkpoint_secs=60,
                                                   log_step_count_steps=1000,
                                                   save_summaries_steps=1000)

    def model(self,
              input_vectors, input_gene, input_variation, output_label, batch_size,
              embedding_size=EMBEDDINGS_SIZE,
              output_classes=9,
              learning_rate_initial=D2V_DOC_LEARNING_RATE_INITIAL,
              learning_rate_decay=D2V_DOC_LEARNING_RATE_DECAY,
              learning_rate_decay_steps=D2V_DOC_LEARNING_RATE_DECAY_STEPS):
        self.global_step = training_util.get_or_create_global_step()

        logits, targets = doc2vec_prediction_model(input_vectors, input_gene, input_variation,
                                                   output_label, batch_size,
                                                   is_training=True, embedding_size=embedding_size,
                                                   output_classes=output_classes)

        self.prediction = tf.nn.softmax(logits)

        self.loss = tf.nn.softmax_cross_entropy_with_logits(labels=targets, logits=logits)
        self.loss = tf.reduce_mean(self.loss)
        tf.summary.scalar('loss', self.loss)

        # learning rate & optimizer
        self.learning_rate = tf.train.exponential_decay(learning_rate_initial, self.global_step,
                                                        learning_rate_decay_steps,
                                                        learning_rate_decay,
                                                        staircase=True, name='learning_rate')
        tf.summary.scalar('learning_rate', self.learning_rate)
        sgd = tf.train.GradientDescentOptimizer(self.learning_rate)
        self.optimizer = sgd.minimize(self.loss, global_step=self.global_step)

        # metrics
        self.metrics = metrics.single_label(self.prediction, targets)

        # saver to save the model
        self.saver = tf.train.Saver()
        # check a nan value in the loss
        self.loss = tf.check_numerics(self.loss, 'loss is nan')

        return None

    def create_graph(self, dataset_tensor, batch_size):
        input_vectors, input_gene, input_variation, output_label = dataset_tensor
        return self.model(input_vectors, input_gene, input_variation, output_label, batch_size)

    def step(self, session, graph_data):
        lr, _, loss, step, metrics = \
            session.run([self.learning_rate, self.optimizer, self.loss,
                         self.global_step, self.metrics])
        if self.is_chief and time.time() > self.print_timestamp + 1 * 60:
            self.print_timestamp = time.time()
            elapsed_time = str(timedelta(seconds=time.time() - self.init_time))
            m = 'step: {}  loss: {:0.4f}  learning_rate = {:0.6f}  elapsed seconds: {}  ' \
                'precision: {}  recall: {}  accuracy: {}'
            logging.info(m.format(step, loss, lr, elapsed_time,
                                  metrics['precision'], metrics['recall'], metrics['accuracy']))

    def after_create_session(self, session, coord):
        self.init_time = time.time()
        self.print_timestamp = time.time()

if __name__ == '__main__':
    import logging

    logging.getLogger().setLevel(logging.INFO)
    # start the training
    trainer = DocPredictionTrainer(dataset=DocPredictionDataset())
    trainer.run(epochs=D2V_DOC_EPOCHS, batch_size=D2V_DOC_BATCH_SIZE)