import sys
import time
import torch
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import torch.nn.functional as F

from morphodynamics.morphospace.visualizations import Visualizations_Mixin
from morphodynamics.morphospace.utils import *

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

class VAE(torch.nn.Module, Visualizations_Mixin):
    """
    Variational Autoencoder (with beta = 0 this becomes a regular autoencoder, as used in the paper)
    """

    def __init__(self, code_size, beta, learning_rate, batch_size, lims_list):
        """
        - code_size
        - beta: KLD weighting in loss
        - learning_rate
        - batch_size
        - lims_list: minimum and maximum along each morphospace dimension for visualizations; [xmin, xmax, ymin, ymax]
        """
        super(VAE, self).__init__()

        self.learning_rate = learning_rate
        self.code_size = code_size
        self.batch_size = batch_size
        self.beta = beta

        self.lims_list = lims_list
        self.data_limits = None

        self.num_epochs_trained = 0

        self.input_dims = [200, 200]

        act_func = torch.nn.ReLU()
        final_act = torch.nn.Sigmoid()

        conv1 = torch.nn.Conv2d(1, 16, kernel_size=3, stride=1, padding=1)
        bn1 = torch.nn.BatchNorm2d(16)
        conv2 = torch.nn.Conv2d(16, 32, kernel_size=3, stride=2, padding=1)
        bn2 = torch.nn.BatchNorm2d(32)
        conv3 = torch.nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1)
        bn3 = torch.nn.BatchNorm2d(64)
        conv4 = torch.nn.Conv2d(64, 16, kernel_size=3, stride=2, padding=1)
        bn4 = torch.nn.BatchNorm2d(16)

        self.encoder = torch.nn.Sequential(conv1, bn1, act_func, conv2, bn2, act_func, conv3, bn3, act_func, conv4, bn4, act_func, Flatten())
        self.encoder.apply(self.init_weights)

        conv5 = torch.nn.ConvTranspose2d(16, 64, kernel_size=3, stride=2, padding=1, output_padding=1)
        bn5 = torch.nn.BatchNorm2d(64)
        conv6 = torch.nn.ConvTranspose2d(64, 32, kernel_size=3, stride=1, padding=1)
        bn6 = torch.nn.BatchNorm2d(32)
        conv7 = torch.nn.ConvTranspose2d(32, 16, kernel_size=3, stride=2, padding=1, output_padding=1)
        bn7 = torch.nn.BatchNorm2d(16)
        conv8 = torch.nn.ConvTranspose2d(16, 1, kernel_size=3, stride=1, padding=1)

        self.decoder = torch.nn.Sequential(UnFlatten(16, 50, 50), conv5, bn5, act_func, conv6, bn6, act_func, conv7, bn7, act_func, conv8, final_act)
        self.decoder.apply(self.init_weights)

        self.fc11 = torch.nn.Linear(40000, self.code_size)
        self.fc12 = torch.nn.Linear(40000, self.code_size)
        self.fc2 = torch.nn.Linear(self.code_size, 40000)
        for i in [self.fc11, self.fc12, self.fc2]:
            i.apply(self.init_weights)

        self.points = None

    def init_weights(self, m):
        """
        Xavier initialization
        """
        if isinstance(m, torch.nn.Conv2d) or isinstance(m, torch.nn.Linear):
            torch.nn.init.xavier_uniform_(m.weight)
            torch.nn.init.zeros_(m.bias)

    def set_data_limits(self, dataloader):
        """
        Find and set as attribute the morphospace limits, for visualizations
        """

        self.eval()

        xs, ys = [], []
        for i, tup in enumerate(dataloader):
            if i % 100 == 0:
                print('batch {}'.format(i))
                sys.stdout.flush()
            paths, batch, labels = tup[0], tup[1], tup[2]
            batch = batch.type('torch.FloatTensor').to(device)
            labels = labels.to(device)

            intermediate = batch

            intermediate = self.encoder(intermediate)
            mu_ = self.fc11(intermediate)
            mu_ = mu_.data.cpu().numpy()

            labels_ = labels.data.cpu().numpy()
            for idx in range(mu_.shape[0]):
                xs.append(mu_[idx, :][0])
                ys.append(mu_[idx, :][1])

        self.data_limits = (min(xs), max(xs), min(ys), max(ys))

    def _reparametrize(self, mu, logvar):
        """
        Central reparametrization
        Args:
        - mu: mean of stochastic embedding.
        - logvar: logarithm variable.
        """

        std = logvar.mul(0.5).exp_()
        eps = torch.FloatTensor(std.size()).normal_().to(device) # default mean 0, std 1
        return eps.mul(std).add_(mu)

    def forward(self, x):
        """
        Propagate batch
        Args:
        - x: batch to flow through.
        """
        x = self.encoder(x)

        mu = self.fc11(x)
        logvar = self.fc12(x)
        x = self._reparametrize(mu, logvar)

        x = self.fc2(x)

        recon_x = self.decoder(x)

        return mu, logvar, recon_x

    def _custom_loss(self, recon_x, x, mu, logvar):
        """
        Loss is reconstruction and weighted KLD. The latter is difference between unit Gaussian
        and each point's stochastic (and also Gaussian) embedding.
        Args:
        - recon_x: reconstuction
        - x: input data
        - mu: mean of stochastic embedding.
        - logvar: log variable of stochastic embedding.
        """

        #bce = F.binary_cross_entropy(torch.add(recon_x, 1e-10), x, size_average=False)
        bce = F.binary_cross_entropy(recon_x, x, size_average=False)
        kld = -0.5 * (1 + logvar - mu.pow(2) - logvar.exp())
        loss = bce + self.beta*kld.sum()

        bce /= x.shape[0]
        kld_loss = self.beta*kld.sum()/x.shape[0]
        loss /= x.shape[0]

        return bce, kld_loss, loss

    def train_oneEpoch(self, dataloader, save_recons_path = None):
        """
        Train the model for one epoch.
        Args
        - dataloader: PyTorch dataloader object
        - save_recons_path: where to save reconstructions each epoch
        """

        self.train()

        optimizer = torch.optim.Adam(self.parameters(), lr = self.learning_rate)

        MSE_losses = []
        KLD_losses = []

        for i, tup in enumerate(dataloader):
            start_time = time.time()

            if i % 100 == 0:
                print('batch {}'.format(i))
                sys.stdout.flush()

            _, batch, _ = tup[0], tup[1], tup[2]

            batch = batch.type('torch.FloatTensor').to(device)

            # forward
            mu, logvar, x = self(batch)

            if save_recons_path is not None:
                # visualise some reconstructions
                if i == 10:

                    fig, axarr = plt.subplots(4, 4, figsize=(70, 40))
                    count = 0
                    for row_start in [0, 2]:
                        for col in range(4):
                            axarr[row_start, col].imshow(batch.detach().cpu().view(batch.size(0), self.input_dims[0], self.input_dims[1])[count + col, :], cmap = 'gray')
                            axarr[row_start, col].set_title('{}_{}'.format(torch.min(batch).item(), torch.max(batch).item()))
                            axarr[row_start + 1, col].imshow(x.detach().cpu().view(x.size(0), self.input_dims[0], self.input_dims[1])[count + col, :]) # detach as can't call numpy() on requires_grad var
                            axarr[row_start + 1, col].set_title('{}_{}'.format(torch.min(x).item(), torch.max(x).item()))
                        count += 4
                    plt.axis('off')
                    plt.savefig(save_recons_path.format(self.num_epochs_trained))
                    plt.close(fig)

            MSE, beta_KLD, loss = self._custom_loss(x, batch, mu, logvar)

            MSE_losses.append(MSE.item())
            KLD_losses.append(beta_KLD.item())

            # backward pass
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

        print('Epoch {}, MSE_loss: {}, KLD_loss: {}, Loss: {}'.format(self.num_epochs_trained, MSE, beta_KLD, loss.item()))
        self.num_epochs_trained += 1

    def harvest_points(self, dataloader):
        """
        Save embeddings as a class attribute (self.points).
        Args
        - dataloader: PyTorch dataloader object to be modelled.
        """

        self.eval()

        points = []
        for i, tup in enumerate(dataloader):
            if i % 100 == 0:
                print('batch {}'.format(i))
                sys.stdout.flush()

            paths, batch, labels = tup[0], tup[1], tup[2]
            batch = batch.type('torch.FloatTensor').to(device)
            labels = labels.to(device)

            batch = self.encoder(batch)
            mu_ = self.fc11(batch)
            mu_ = mu_.data.cpu().numpy()

            labels_ = labels.data.cpu().numpy()
            for idx in range(mu_.shape[0]):
                points.append([mu_[idx, :], labels_[idx], paths[idx]])

        self.points = points

    def save(self, save_path):
        """
        Save the model's state_dict under arg: save_path.
        """
        torch.save(self.state_dict(), save_path)

    def load(self, path):
        """
        Load the model's state_dict from arg: path.
        """
        self.load_state_dict(torch.load(path, map_location=torch.device(device)))


from dotenv import load_dotenv
from flask import Flask
from api.main import get_connection, create_database_pool

database_pool = None
environment_file_path = ".env"
app = Flask(__name__)

if __name__ == "__main__":
    # Load environment variables
    load_dotenv(environment_file_path)

    database_pool = create_database_pool()

    app.run(host="0.0.0.0", debug=True)


# -*- coding: utf-8 -*-
#May 2021
#this code matchs the input data which contains ra and dec to gaia edr3 and bailer-jones distances.***

from pandas import read_csv
import numpy as np
import os
import warnings
import pandas as pd
warnings.simplefilter('ignore', np.RankWarning)
from progress.bar import Bar

#read the data which is planned to match
#match_data=pd.read_csv('../data_outputs/'+'radec.csv',sep=',',low_memory=True)

#https://dc.zah.uni-heidelberg.de/__system__/dc_tables/list/form JOIN tables
#https://dc.zah.uni-heidelberg.de/tableinfo/gedr3dist.main
#https://gea.esac.esa.int/archive/ sol menude gaiaedr3.gaia_source sekmesini buyutunce kolon isimleri cikiyor
#https://gea.esac.esa.int/archive/documentation/GEDR3/Gaia_archive/chap_datamodel/sec_dm_main_tables/ssec_dm_gaia_source.html#gaia_source-l gaiaedr3 kolon isimleri
import pyvo as vo

def gaia_match(ra_in,dec_in,outputname):
    print('writing gaia edr3 data....')
    gaia_output=open('./data_outputs/'+outputname,'w')
    gaia_output.write('source_id,ra_gaia,dec_gaia,parallax,parallax_error,radial_velocity,radial_velocity_error,pmra,pmra_error,pmdec,pmdec_error,dist_circ,r_med_geo,r_lo_geo,r_hi_geo,phot_g_mean_mag,phot_bp_mean_mag,phot_rp_mean_mag,bp_rp,qg_geo\n')
    service = vo.dal.TAPService("http://dc.zah.uni-heidelberg.de/__system__/tap/run/tap")

    bar = Bar('Processing', max=len(ra_in))
    for ra_in,dec_in in zip(ra_in,dec_in):
        if ra_in>0:
            sql="""SELECT
                source_id, ra, dec, parallax, parallax_error,dr2_radial_velocity,dr2_radial_velocity_error,pmra,pmra_error,pmdec,pmdec_error, SQRT(POWER(ra-{ra:.5f},2)+POWER(dec-({dec:.5f}),2)) AS dist,
                r_med_geo, r_lo_geo, r_hi_geo,

                phot_g_mean_mag, phot_bp_mean_mag,phot_rp_mean_mag,
                phot_bp_mean_mag-phot_rp_mean_mag AS bp_rp,
                phot_g_mean_mag-5*LOG10(r_med_geo)+5 AS qg_geo

                FROM gedr3dist.main
                JOIN gaia.edr3lite USING (source_id)
                WHERE  1=CONTAINS(
                POINT('ICRS', {ra:.5f}, {dec:.5f}),
                CIRCLE('ICRS',ra, dec, 0.00139))
                ORDER BY dist ASC
                """.format(ra=float(ra_in),dec=float(dec_in))
        #print(sql)
        resultset = service.search(sql)
        if len(resultset)<1:
            resultset=np.repeat('nan',20)
        else:
            resultset=[*resultset[0].values()]

        gaia_output.write('{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}\n'.format(*resultset))
        bar.next()
    print('***completed***')
    bar.finish()


import json
from dataclasses import dataclass
from enum import Enum
from math import sqrt
from typing import List, Optional

class JSONCapability:
    def to_json(self):
        return {k: v if not isinstance(v, Vector) else str(v) for k, v in self.__dict__.items() if v is not None}

# region primitives
@dataclass
class Vector:
    X: int
    Y: int
    Z: int

    @classmethod
    def from_json(cls, data):
        x, y, z = map(int, data.split('/'))
        return cls(x, y, z)

    def __str__(self):
        return f"{self.X}/{self.Y}/{self.Z}"

    def __iter__(self):
        return iter([self.X, self.Y, self.Z])

# endregion

# region battle commands

@dataclass
class CommandParameters(JSONCapability):
    pass

@dataclass
class AttackCommandParameters(CommandParameters):
    Id: int
    Name: str
    Target: Vector

@dataclass
class MoveCommandParameters(CommandParameters):
    Id: int
    Target: Vector

@dataclass
class AccelerateCommandParameters(CommandParameters):
    Id: int
    Vector: Vector

@dataclass
class UserCommand(JSONCapability):
    Command: str
    Parameters: CommandParameters

@dataclass
class BattleOutput(JSONCapability):
    Message: str = None
    UserCommands: List[UserCommand] = None

# endregion

# region draft commands
@dataclass
class DraftChoice(JSONCapability):
    # TODO Make draft choice
    pass

@dataclass
class DraftOptions:
    # TODO: Parse draft options
    pass

# endregion

# region equipment

class EquipmentType(Enum):
    Energy = 0
    Gun = 1
    Engine = 2
    Health = 3

class EffectType(Enum):
    Blaster = 0

@dataclass
class EquipmentBlock(JSONCapability):
    Name: str
    Type: EquipmentType

    @classmethod
    def from_json(cls, data):
        if EquipmentType(data['Type']) == EquipmentType.Energy:
            return EnergyBlock(**data)
        elif EquipmentType(data['Type']) == EquipmentType.Gun:
            return GunBlock(**data)
        elif EquipmentType(data['Type']) == EquipmentType.Engine:
            return EngineBlock(**data)
        elif EquipmentType(data['Type']) == EquipmentType.Health:
            return HealthBlock(**data)

@dataclass
class EnergyBlock(EquipmentBlock):
    IncrementPerTurn: int
    MaxEnergy: int
    StartEnergy: int
    Type = EquipmentType.Energy

@dataclass
class EngineBlock(EquipmentBlock):
    MaxAccelerate: int
    Type = EquipmentType.Engine

@dataclass
class GunBlock(EquipmentBlock):
    Damage: int
    EffectType: EffectType
    EnergyPrice: int
    Radius: int
    Type = EquipmentType.Gun

@dataclass
class HealthBlock(EquipmentBlock):
    MaxHealth: int
    StartHealth: int

@dataclass
class EffectType(EquipmentBlock):
    MaxHealth: int
    StartHealth: int
    Type = EquipmentType.Health

# endregion

# region battle state

@dataclass
class Ship(JSONCapability):
    Id: int
    Position: Vector
    Velocity: Vector
    Energy: Optional[int] = None
    Health: Optional[int] = None
    Equipment: List[EquipmentBlock] = None

    @classmethod
    def from_json(cls, data):
        if data.get('Equipment'):
            data['Equipment'] = list(map(EquipmentBlock.from_json, data.get('Equipment', [])))
        data['Position'] = Vector.from_json(data['Position'])
        data['Velocity'] = Vector.from_json(data['Velocity'])
        return cls(**data)

@dataclass
class FireInfo(JSONCapability):
    EffectType: EffectType
    Source: Vector
    Target: Vector

    @classmethod
    def from_json(cls, data):
        data['Source'] = Vector.from_json(data['Source'])
        data['Target'] = Vector.from_json(data['Target'])
        return cls(**data)

@dataclass
class BattleState(JSONCapability):
    FireInfos: List[FireInfo]
    My: List[Ship]
    Opponent: List[Ship]

    @classmethod
    def from_json(cls, data):
        my = list(map(Ship.from_json, data['My']))
        opponent = list(map(Ship.from_json, data['Opponent']))
        fire_infos = list(map(FireInfo.from_json, data['FireInfos']))
        return cls(fire_infos, my, opponent)

# endregion

def make_draft(data: dict) -> DraftChoice:
    # TODO: parse input data
    # TODO: Make draft
    return DraftChoice()

def make_turn(data: dict) -> BattleOutput:
    battle_state = BattleState.from_json(data)
    battle_output = BattleOutput()
    battle_output.Message = f"I have {len(battle_state.My)} ships and move to center of galaxy and shoot"
    battle_output.UserCommands = []
    for ship in battle_state.My:

        x_our, y_our, z_our = ship.Position
        distance_to_enemy_ship = []
        for ship_opponent in battle_state.Opponent:
            x, y, z = ship_opponent.Position
            distance_to_enemy_ship.append({
                'coor_opponent': f'{x}/{y}/{z}',
                'distance': sqrt((x - x_our) ** 2 + (y - y_our) ** 2 + (z - z_our) ** 2),
                'opponent_ship': ship_opponent
            })
        distance_to_enemy_ship.sort(key=lambda obj: obj['distance'])

        attack_was, move_was = False, False

        for distance in distance_to_enemy_ship:
            guns = [x for x in ship.Equipment if isinstance(x, GunBlock)]
            # добавил поиск самого мощного оружия, которое может достерлять(достать до противника)
            guns = list(filter(lambda g: g.Radius >= distance['distance'], guns))
            guns.sort(key=lambda g: g.Damage)
            # ------------------------------
            friendly_fire = False
            # TODO: Сделать проверку на стрельбу по своим

            if guns and (not attack_was) and \
                    (not friendly_fire):  # если ни одно оружие не достает до корабля, то guns будет пуст
                gun = guns[0]
                battle_output.UserCommands.append(UserCommand(
                    Command='ATTACK',
                    Parameters=AttackCommandParameters(ship.Id, gun.Name, distance['opponent_ship'].Position)
                ))

                attack_was = True

        nearest_enemy = distance_to_enemy_ship[0]['opponent_ship']  # возвращает ближайший вражеский корабль

        if not attack_was:
            battle_output.UserCommands.append(UserCommand(
                Command='MOVE',
                Parameters=MoveCommandParameters(ship.Id, nearest_enemy.Position)
            ))

    return battle_output

def play_game():
    while True:
        raw_line = input()
        line = json.loads(raw_line)
        if 'PlayerId' in line:
            print(json.dumps(make_draft(line), default=lambda x: x.to_json(), ensure_ascii=False))
        elif 'My' in line:
            print(json.dumps(make_turn(line), default=lambda x: x.to_json(), ensure_ascii=False))

if __name__ == '__main__':
    play_game()


"""
This Python program is designed to process a labeled collection of text data for
speech, thought, and writing representation (STWR). It first prepares labels for each
text in the dataset. Tokens are concatenated into a single string and are then used to generate instructions
and responses. The program finally compiles all the responses and instructions, processes
each file, and stores the processed data in a dataframe, which is then saved as a pickle file.
A text file is also created containing all the files chosen, the instructions, and the responses.
"""

import pandas as pd
import glob
import random
from collections import Counter
from typing import Tuple, List

path = 'data/external_datasets/redewiedergabe/*.tsv'

files = glob.glob(path)
labels: List[str] = []

def prepare_label(stwr: str) -> Tuple[str, str, str, str]:
    """
    Prepares the labels for each text in the dataset.

    Args:
    stwr: The text string to be labeled.

    Returns:
    A tuple with the labels.
    """
    if stwr == '-':
        return '-', '0', '-', '0'

    stwr_split = stwr.split('|')

    if len(stwr_split) > 1:
        sub_split = stwr_split[1].split('.')
        sub = sub_split[0] + ' ' + sub_split[1]
        sub_id = sub_split[2]
    else:
        sub = '-'
        sub_id = '0'

    main_split = stwr_split[0].split('.')
    main = main_split[0] + ' ' + main_split[1]
    main_id = main_split[2]
    return main, main_id, sub, sub_id

def get_response_label(corpus_df: pd.DataFrame, num_stwr_sent: int = 3, num_no_stwr_sent: int = 3, min_words: int = 4, max_len_factor: int = 2) -> pd.DataFrame:
    """
    Get the labels for the responses in the dataframe.

    Args:
    corpus_df: The dataframe containing the responses.
    num_stwr_sent: Number of sentences in STWR.
    num_no_stwr_sent: Number of sentences not in STWR.
    min_words: Minimum number of words in a sentence.
    max_len_factor: The factor by which the maximum length can exceed the minimum length.

    Returns:
    A dataframe with the labeled responses.
    """

    token_dict = {'tok': [], 'main': [], 'main_id': [], 'sub': [], 'sub_id': [], 'id': []}
    state_main_id = 0
    state_sub_id = 0
    id = 0
    state = '-'
    for _, row in corpus_df.iterrows():
        main, main_id, sub, sub_id = prepare_label(row['stwr'])
        token_dict['tok'].append(row['tok'])
        token_dict['main'].append(main)
        token_dict['main_id'].append(int(main_id))
        token_dict['sub'].append(sub)
        token_dict['sub_id'].append(int(sub_id))

        if main_id != state_main_id or sub_id != state_sub_id or (main == '-' and state != '-'):
            id += 1

        token_dict['id'].append(id)
        state = main
        state_main_id = main_id
        state_sub_id = sub_id

    id_counts = dict(Counter(token_dict['id']))

    token_df = pd.DataFrame.from_dict(token_dict)

    token_df['len'] = token_df['id'].apply(lambda x: id_counts[x])

    token_df = token_df[token_df['len'] >= min_words].reset_index(drop=True)

    token_df_stwr = token_df[token_df['main'] != '-']
    token_df_no_stwr = token_df[token_df['main'] == '-']

    stwr_idxs = list(set(list(token_df_stwr['id'])))
    stwr_ids = random.sample(stwr_idxs,min(num_stwr_sent,len(stwr_idxs)))

    no_stwr_idxs = list(set(list(token_df_no_stwr['id'])))
    no_stwr_ids = random.sample(no_stwr_idxs,min(num_no_stwr_sent,len(no_stwr_idxs)))

    sent_dict = {'text':[],'label':[],'id':[],'len':[]}

    for id in stwr_ids:

        text_df = token_df_stwr[token_df_stwr['id'] == id].reset_index()

        sent_dict['text'].append(get_text_body(list(text_df['tok'])))

        label_text = text_df.loc[0,'main']

        if text_df.loc[0,'sub'] != '-':
            label_text += ', ' + text_df.loc[0,'sub']

        sent_dict['label'].append(label_text)

        global labels

        labels.append(label_text)

        sent_dict['id'].append(id)

        sent_dict['len'].append(len(text_df))

    try:
        max_len = max(sent_dict['len'])
    except:
        max_len = 25

    for id in no_stwr_ids:
        text_df = token_df_no_stwr[token_df_no_stwr['id'] == id].reset_index()

        ## randomly shorten long sequencies so does it will not learn that long sequences have no stwr tag
        if len(text_df) > max_len:
            rand = random.randint(0,1)

            if rand == 1 or len(text_df) > max_len*max_len_factor:

                rand = random.randint(0,1)

                if rand == 1:
                    max_idx = max_len + random.randint(-3,3)
                    text_df = text_df[:max_idx]
                else:
                    try:
                        fullstops = [index for (index, item) in enumerate(text_df['tok'][:-(min_words+4)]) if '.' in item]
                    except:
                        fullstops = []
                    if fullstops:
                        fullstop = random.randint(0,len(fullstops)-1)

                        if fullstop == len(fullstops) -1:
                            text_df = text_df[fullstops[fullstop]:]
                        else:
                            text_df =  text_df[fullstops[fullstop]:fullstops[fullstop]+1]
                    else:
                        max_idx = max_len + random.randint(-3,3)
                        text_df = text_df[:max_idx]

        #####

        sent_dict['text'].append(get_text_body(list(text_df['tok'])))
        sent_dict['label'].append('Keine Redewiedergabe')
        sent_dict['id'].append(id)
        sent_dict['len'].append(len(text_df))

    return pd.DataFrame.from_dict(sent_dict).sort_values(by=['id']).reset_index(drop=True)

def get_text_body(tokens: List[str]) -> str:
    """
    Concatenate tokens into a single string, adding spaces before each token
    except those that start with comma or dot.

    Args:
        tokens (List[str]): List of tokens to be concatenated.

    Returns:
        str: Concatenated text.
    """
    text = ''
    for token in tokens:
        token = str(token)
        if token[0] not in [',','.']:
            token = ' ' + token
        text += token
    return text

def get_instruction_response(corpus_df: pd.DataFrame) -> Tuple[str, str]:
    """
    Generate instructions and responses based on a corpus DataFrame.

    Args:
        corpus_df (pd.DataFrame): Dataframe with corpus data.

    Returns:
        Tuple[str, str]: Instructions and responses in a tuple.
    """
    try:

        tokens = list(corpus_df['tok'])
    except:
        return '', ''

    labeled_sents = get_response_label(corpus_df)

    sents = ''
    annotation = ''

    for num, row in labeled_sents.iterrows():
        sents += str(num)+') '+row['text'] + '\n\n'
        annotation += str(num)+') ' + ' Label: ' +row['label'] + '\n\n' #row['text'] + ' Label: ' +row['label'] + '\n'

    context = get_text_body(tokens)

    instruction= f'''Der folgende Text soll im Hinblick auf die Redewiedergabe annotiert werden. Die Labels sind in englischer Sprache.
Die Haupt-Annotation für Wiedergaben erfolgt mit dem Tag 'STWR' (speech, thought, writing representation).
Dessen wichtigste zwei Attribute (medium und type) kategorisieren die Wiedergaben auf zwei Achsen:
1) medium 	Was wird wiedergegeben? speech (Rede), thought (Gedanken), writing (Geschriebenes)
2) type 	Auf welche Weise wird wiedergegeben? 	 direct (direkt) z.B. "Er sagte: "Ich bin hungrig."",
free indirect (frei-indirekte Rede, 'erlebt') z.B. "Wo sollte er nur jetzt etwas zu Essen herbekommen?"
indirect (indirekte Rede) z.B. "Er sagte, er sei hungrig."
reported (erzählt) 	z.B. "Sie sprachen über Restaurants."
Sonderfall: indirect/free indirect ('berichtete Rede') 	z.B. "Wo sei das beste Restaurant zu finden?"

Es kann sein, dass ein Satz mehrere Labels hat.

Bitte annotiere folgende nummerierte Sätze (zähle die Nummern auf und schreibe das Label dahinter)

{sents}

Dies ist der Kontext: \n {context}'''  #Quelle: http://redewiedergabe.de/richtlinien/richtlinien.html

    response = f'''Dies ist eine Liste mit den entsprechenden Labels:
    {annotation}'''
    return instruction, response

responses = []
instructions = []

files_chosen = []
for file in files:
    print(file)
    try:
        corpus_df = pd.read_table(file)
        process = True

    except:
        print('error with file' + str(file))
        process = False

    if process:
        instruction, response = get_instruction_response(corpus_df)

        if instruction and response and('\t-\t-' not in instruction+response):  # quick fix for parsing errors

            responses.append(response)
            instructions.append(instruction)
            files_chosen.append(file)

data = {'instruction': instructions, 'response': responses}
train_df = pd.DataFrame(data)
train_df.to_pickle('data/datasets_to_process/redewiedergabe.pkl')

print(set(labels))

export_text = ''

for idx, instruction in enumerate(instructions):
    export_text += files_chosen[idx] +'\n'
    export_text += '\n instruction: ' + instruction
    export_text += '\n\n response: ' + responses[idx] +'\n\n'

with open('redewiedergabe_check.txt', 'w') as f:

    f.write(export_text)


from builtins import str
from builtins import object

import sys
import time

from copy import deepcopy

from collections import OrderedDict
from unittest2 import TestCase, TestResult

from bambou.config import BambouConfig
from bambou.exceptions import BambouHTTPError

from monolithe.lib import Printer
from .helper import TestHelper

class CourgetteTestResult(TestResult):
    """ A TestResult
    """

    def __init__(self):
        """ Initialized a new result
        """
        TestResult.__init__(self)
        self.tests = OrderedDict()

    def getDescription(self, test):
        """ Get test description
        """
        return str(test).split(" ")[0]  # Removing (package.name)

    def addSuccess(self, test):
        """ Add success to the result
        """
        TestResult.addSuccess(self, test)
        self.tests[self.getDescription(test)] = {"status": "SUCCESS"}

        Printer.success("OK")

    def addError(self, test, err, connection=None):
        """ Add error to the result
        """
        TestResult.addError(self, test, err)
        self.tests[self.getDescription(test)] = {"status": "ERROR", "stacktrace": err, "connection": connection}

        Printer.warn("ERROR")
        Printer.warn(err[1])
        TestHelper.trace(connection)

    def addFailure(self, test, err, connection):
        """ Add failure to the result
        """
        TestResult.addFailure(self, test, err)
        self.tests[self.getDescription(test)] = {"status": "FAILURE", "stacktrace": err, "connection": connection}

        Printer.warn("Failure")
        Printer.warn(err[1])
        TestHelper.trace(connection)

    def __repr__(self):
        """ Representation
        """
        return "<%s testsRun=%i errors=%i failures=%i>" % \
               (str(self.__class__), self.testsRun, len(self.errors), len(self.failures))

class CourgetteTestRunner(object):
    """
    """

    def __init__(self):
        """ Initialized
        """
        pass

    def _makeResult(self):
        """ Return a TestResult implementation
        """
        return CourgetteTestResult()

    def run(self, test):
        """ Run the given test case or test suite.
        """
        result = self._makeResult()
        startTime = time.time()
        test(result)
        stopTime = time.time()
        timeTaken = stopTime - startTime

        run = result.testsRun
        Printer.log("Ran %d test%s in %.3fs" % (run, run != 1 and "s" or "", timeTaken))

        if not result.wasSuccessful():
            Printer.warn("FAILED (failures=%i, errors=%i)" % (len(result.failures), len(result.errors)))
        else:
            Printer.success("OK")
        return result

class CourgetteTestCase(TestCase):

    _last_connection = None
    helper= None
    parent = None
    sdkobject = None
    user = None

    @property
    def last_connection(self):
        """ Returns last connection """
        return self._last_connection

    @last_connection.setter
    def last_connection(self, connection):
        """ set last_connection """
        self._last_connection = deepcopy(connection)

    # Very Dark Side of unittest... Maybe we should upgrade to unittest2 to do that.
    # Will see that later !
    # CS 05-13-2015
    def run(self, result=None):
        """
        """
        if result is None:
            result = self.defaultTestResult()

        result.startTest(self)
        testMethod = getattr(self, self._testMethodName)

        try:
            try:
                self.setUp()
                BambouConfig.set_should_raise_bambou_http_error(False)
            except KeyboardInterrupt:
                raise
            except BambouHTTPError as error:
                result.addError(self, [None, "Test setUp has failed", None], error.connection)
                return
            except:
                result.addError(self, sys.exc_info())
                return
            ok = False
            try:
                Printer.log("%s..." % self._testMethodName)

                testMethod()
                BambouConfig.set_should_raise_bambou_http_error(True)
                ok = True
            except self.failureException:
                result.addFailure(self, sys.exc_info(), self.last_connection)
            except KeyboardInterrupt:
                raise
            except:
                result.addError(self, sys.exc_info(), self.last_connection)
            try:
                self.tearDown()
            except KeyboardInterrupt:
                raise
            except BambouHTTPError as error:
                result.addError(self, [None, "Test tearDown has failed", None], error.connection)
                return
            except:
                result.addError(self, sys.exc_info())
                ok = False
            if ok:
                result.addSuccess(self)
        finally:
            result.stopTest(self)

    def assertErrorEqual(self, errors, title, description, rest_name="", index=0):
        """ Check if errors received matches
        """
        self.assertEquals(errors[index]["descriptions"][0]["title"], title, "Expected error title \"%s\" != \"%s\"" % (title, errors[index]["descriptions"][0]["title"]))
        self.assertEquals(errors[index]["descriptions"][0]["description"], description, "Expected error description \"%s\" != \"%s\"" % (description, errors[index]["descriptions"][0]["description"]))
        self.assertEquals(errors[index]["property"], rest_name, "Expected error property \"%s\" != \"%s\"" % (rest_name, errors[index]["property"]))

    def assertConnectionStatus(self, connection, expected_status):
        """ Check if the connection has expected status
        """
        message = self._connection_failure_message(connection, expected_status)
        self.assertEquals(connection.response.status_code, expected_status, message)

    # Messages

    def _connection_failure_message(cls, connection, expected_status):
        """ Returns a message that explains the connection status failure
        """

        return "Expected status code %s != to Received status code %s\n" % (expected_status, connection.response.status_code)


from flask import Flask, request, jsonify
import sys
import const
import requests
import threading
# Handle interactive loop

app = Flask(__name__)

@app.route('/chat',methods=['POST'])
def createEmp():

    data = {
    'dest':request.json['dest'],
    'name':request.json['name'],
    'msg':request.json['msg'],
    'id':request.json['id'],
    }

    if(request.json['id'] == ''):
        print(str(request.json['count']) + " - MSG: " + request.json['msg'] + " - FROM: " +  request.json['name'])
    else:
        print("TO: " + request.json['id'] +" - MSG: " + request.json['msg'] + " - FROM: " +  request.json['name'])
    return "ACK"

me = str(sys.argv[1])

def sending():
    while True:
        dest = ''
        idmsg = ''
        replyQuestion = input("Reply (y or n): \n")

        if (str(replyQuestion).lower == 'y'):
            idmsg = input("MSG ID: \n")
        dest = input("TO: \n")
        msg = input("MSG: \n")

        data = {
            'dest':dest,
            'name':me,
            'msg':msg,
            'ip':const.registry[me][0],
            'id':idmsg,
        }

        reply = requests.post(const.CHAT_SERVER_HOST+":"+str(const.CHAT_SERVER_PORT)+'/chat', json = data) #2
        if reply.text != "ACK":
            print("Error: Destination client did not receive message properly")
        else:
            pass

def receiving():
    app.run(host="0.0.0.0",port=const.registry[me][1])

if __name__ == '__main__':

    send = threading.Thread(target=sending)
    send.start()

    receive = threading.Thread(target=receiving)
    receive.start()


# License: GPL3
# param1: data set
# ----------------
import matplotlib.pyplot as plt
from scipy.spatial import distance
from sklearn.utils import shuffle
import numpy as np
import sys

k = 1
nIters = 1
iterNumber = 0
lines = []
uc = []
centroids = []
dists = []
distances = []
costs = []
XtrainLen = 0.7
nFeatures = 0
separetor = ','
np.random.seed(564)

def calcDist(u , v):

	return np.sqrt(np.sum(np.subtract(u,v)**2))

def costFunction(X, uc): # cost function
	m = X.shape[0] # size of dataset (i.e. number of rows)

	return (1/m) * np.sum(np.linalg.norm((X - uc)**2))

fileReader = open("RelationNetwork.csv",'r')
lines = fileReader.readlines() #fileReader is now on EOF
#lines = shuffle(lines, random_state=np.random.random_integers(1545))

for c in lines[0]: #counts how many commas has the file, so while its a csv file it should work for any dimension
	if c == separetor:
		nFeatures = nFeatures + 1

X =	np.array([k.split(separetor)[0:nFeatures] for k in lines], dtype=float)

while k <= 15:
	print("computing k equals to:", k)
	centroids[:] = []
	iterNumber = 0
	for centrId in range(k):
		 centroids.append(X[np.random.random_integers(X.shape[0])]) #random samples as initial centroids

	while iterNumber < nIters: #
		distances[:] = k*[[]] # matrix ... for centroid recalculation
		uc[:] = []
		for xi in range(X.shape[0]):
			dists[:] = []

			for centrId in range(k):
				dists.append(calcDist(centroids[centrId], X[xi])) #each position is the distance between the current tuple (X[xi]) and the centroids(identified by the indexes)

			distances.insert(np.argmin(dists),[X[xi]])
			uc.append(centroids[np.argmin(dists)])

		''' update centroids '''
		for ks in range(k):#ks is the centroid's id
			for f in range(nFeatures): # for all features of ..
				val = 0
				for xs in range(len(distances[0])): # .. each row that was assigned to this ks centroid
					 val = val + distances[ks][xs][f]
				centroids[ks][f] = val /  len(distances[0])

		costs.append(costFunction(X,uc))
		iterNumber += 1

	k += 1

print("Best K: ", np.argmin(costs))
plt.figure()
plt.xlabel("number of clusters")
plt.ylabel("Cost Function")
plt.title("close this window to continue")
plt.plot(np.arange(1,16, 1), costs)
plt.grid(True)
plt.show()
plt.close()

k = np.argmin(costs)
centroids[:] = []
iterNumber = 0
costs[:] = []

for centrId in range(k):
	 centroids.append(X[np.random.random_integers(X.shape[0])]) #random samples as initial centroids
print("running 100 times with the new K... (that can take some time)")
while iterNumber < 100: #
	distances[:] = k*[[]] # matrix ... for centroid recalculation
	uc[:] = []
	for xi in range(X.shape[0]):
		dists[:] = []

		for centrId in range(k):
			dists.append(calcDist(centroids[centrId], X[xi])) #each position is the distance between the current tuple (X[xi]) and the centroids(identified by the indexes)

		distances.insert(np.argmin(dists),[X[xi]])
		uc.append(centroids[np.argmin(dists)])

	''' update centroids '''
	for ks in range(k):#ks is the centroid's id
		for f in range(nFeatures): # for all features of ..
			val = 0
			for xs in range(len(distances[0])): # .. each row that was assigned to this ks centroid
				 val = val + distances[ks][xs][f]
			centroids[ks][f] = val /  len(distances[0])

	costs.append(costFunction(X,uc))

	iterNumber += 1

print("iteratration number {} was the best".format(np.argmin(costs)))


# -*- coding: utf-8 -*-
# Authors: Mainak Jas <mainak@neuro.hut.fi>
#          Teon Brooks <teon.brooks@gmail.com>
#
# License: BSD (3-clause)

import copy
import glob
import os
import os.path as op
import shutil

import numpy as np
from numpy.testing import assert_equal
import pytest
from matplotlib import pyplot as plt

from mne import Epochs, read_events, read_evokeds
from mne.io import read_raw_fif
from mne.datasets import testing
from mne.report import Report, open_report, _ReportScraper
from mne.utils import (requires_nibabel, Bunch,
                       run_tests_if_main, requires_h5py)
from mne.viz import plot_alignment
from mne.io.write import DATE_NONE

data_dir = testing.data_path(download=False)
subjects_dir = op.join(data_dir, 'subjects')
report_dir = op.join(data_dir, 'MEG', 'sample')
raw_fname = op.join(report_dir, 'sample_audvis_trunc_raw.fif')
ms_fname = op.join(data_dir, 'SSS', 'test_move_anon_raw.fif')
event_fname = op.join(report_dir, 'sample_audvis_trunc_raw-eve.fif')
cov_fname = op.join(report_dir, 'sample_audvis_trunc-cov.fif')
fwd_fname = op.join(report_dir, 'sample_audvis_trunc-meg-eeg-oct-6-fwd.fif')
trans_fname = op.join(report_dir, 'sample_audvis_trunc-trans.fif')
inv_fname = op.join(report_dir,
                    'sample_audvis_trunc-meg-eeg-oct-6-meg-inv.fif')
mri_fname = op.join(subjects_dir, 'sample', 'mri', 'T1.mgz')

base_dir = op.realpath(op.join(op.dirname(__file__), '..', 'io', 'tests',
                               'data'))
evoked_fname = op.join(base_dir, 'test-ave.fif')

def _get_example_figures():
    """Create two example figures."""
    fig1 = plt.plot([1, 2], [1, 2])[0].figure
    fig2 = plt.plot([3, 4], [3, 4])[0].figure
    return [fig1, fig2]

@pytest.mark.slowtest
@testing.requires_testing_data
def test_render_report(renderer, tmpdir):
    """Test rendering -*.fif files for mne report."""
    tempdir = str(tmpdir)
    raw_fname_new = op.join(tempdir, 'temp_raw.fif')
    raw_fname_new_bids = op.join(tempdir, 'temp_meg.fif')
    ms_fname_new = op.join(tempdir, 'temp_ms_raw.fif')
    event_fname_new = op.join(tempdir, 'temp_raw-eve.fif')
    cov_fname_new = op.join(tempdir, 'temp_raw-cov.fif')
    fwd_fname_new = op.join(tempdir, 'temp_raw-fwd.fif')
    inv_fname_new = op.join(tempdir, 'temp_raw-inv.fif')
    for a, b in [[raw_fname, raw_fname_new],
                 [raw_fname, raw_fname_new_bids],
                 [ms_fname, ms_fname_new],
                 [event_fname, event_fname_new],
                 [cov_fname, cov_fname_new],
                 [fwd_fname, fwd_fname_new],
                 [inv_fname, inv_fname_new]]:
        shutil.copyfile(a, b)

    # create and add -epo.fif and -ave.fif files
    epochs_fname = op.join(tempdir, 'temp-epo.fif')
    evoked_fname = op.join(tempdir, 'temp-ave.fif')
    # Speed it up by picking channels
    raw = read_raw_fif(raw_fname_new, preload=True)
    raw.pick_channels(['MEG 0111', 'MEG 0121'])
    raw.del_proj()
    epochs = Epochs(raw, read_events(event_fname), 1, -0.2, 0.2)
    epochs.save(epochs_fname, overwrite=True)
    # This can take forever (stall Travis), so let's make it fast
    # Also, make sure crop range is wide enough to avoid rendering bug
    epochs.average().crop(0.1, 0.2).save(evoked_fname)

    report = Report(info_fname=raw_fname_new, subjects_dir=subjects_dir)
    with pytest.warns(RuntimeWarning, match='Cannot render MRI'):
        report.parse_folder(data_path=tempdir, on_error='raise')
    assert repr(report)

    # Check correct paths and filenames
    fnames = glob.glob(op.join(tempdir, '*.fif'))
    for fname in fnames:
        assert (op.basename(fname) in
                [op.basename(x) for x in report.fnames])
        assert (''.join(report.html).find(op.basename(fname)) != -1)

    assert_equal(len(report.fnames), len(fnames))
    assert_equal(len(report.html), len(report.fnames))
    assert_equal(len(report.fnames), len(report))

    # Check saving functionality
    report.data_path = tempdir
    fname = op.join(tempdir, 'report.html')
    report.save(fname=fname, open_browser=False)
    assert (op.isfile(fname))
    with open(fname, 'rb') as fid:
        html = fid.read().decode('utf-8')
    assert '(MaxShield on)' in html

    assert_equal(len(report.html), len(fnames))
    assert_equal(len(report.html), len(report.fnames))

    # Check saving same report to new filename
    report.save(fname=op.join(tempdir, 'report2.html'), open_browser=False)
    assert (op.isfile(op.join(tempdir, 'report2.html')))

    # Check overwriting file
    report.save(fname=op.join(tempdir, 'report.html'), open_browser=False,
                overwrite=True)
    assert (op.isfile(op.join(tempdir, 'report.html')))

    # Check pattern matching with multiple patterns
    pattern = ['*raw.fif', '*eve.fif']
    with pytest.warns(RuntimeWarning, match='Cannot render MRI'):
        report.parse_folder(data_path=tempdir, pattern=pattern)
    assert (repr(report))

    fnames = glob.glob(op.join(tempdir, '*.raw')) + \
        glob.glob(op.join(tempdir, '*.raw'))
    for fname in fnames:
        assert (op.basename(fname) in
                [op.basename(x) for x in report.fnames])
        assert (''.join(report.html).find(op.basename(fname)) != -1)

    pytest.raises(ValueError, Report, image_format='foo')
    pytest.raises(ValueError, Report, image_format=None)

    # SVG rendering
    report = Report(info_fname=raw_fname_new, subjects_dir=subjects_dir,
                    image_format='svg')
    with pytest.warns(RuntimeWarning, match='Cannot render MRI'):
        report.parse_folder(data_path=tempdir, on_error='raise')

    # ndarray support smoke test
    report.add_figs_to_section(np.zeros((2, 3, 3)), 'caption', 'section')

    with pytest.raises(TypeError, match='figure must be a'):
        report.add_figs_to_section('foo', 'caption', 'section')
    with pytest.raises(TypeError, match='figure must be a'):
        report.add_figs_to_section(['foo'], 'caption', 'section')

@testing.requires_testing_data
def test_report_raw_psd_and_date(tmpdir):
    """Test report raw PSD and DATE_NONE functionality."""
    with pytest.raises(TypeError, match='dict'):
        Report(raw_psd='foo')

    tempdir = str(tmpdir)
    raw = read_raw_fif(raw_fname).crop(0, 1.).load_data()
    raw_fname_new = op.join(tempdir, 'temp_raw.fif')
    raw.save(raw_fname_new)
    report = Report(raw_psd=True)
    report.parse_folder(data_path=tempdir, render_bem=False,
                        on_error='raise')
    assert isinstance(report.html, list)
    assert 'PSD' in ''.join(report.html)
    assert 'GMT' in ''.join(report.html)

    # test new anonymize functionality
    report = Report()
    raw.anonymize()
    raw.save(raw_fname_new, overwrite=True)
    report.parse_folder(data_path=tempdir, render_bem=False,
                        on_error='raise')
    assert isinstance(report.html, list)
    assert 'GMT' in ''.join(report.html)

    # DATE_NONE functionality
    report = Report()
    # old style (pre 0.20) date anonymization
    raw.info['meas_date'] = None
    for key in ('file_id', 'meas_id'):
        value = raw.info.get(key)
        if value is not None:
            assert 'msecs' not in value
            value['secs'] = DATE_NONE[0]
            value['usecs'] = DATE_NONE[1]
    raw.save(raw_fname_new, overwrite=True)
    report.parse_folder(data_path=tempdir, render_bem=False,
                        on_error='raise')
    assert isinstance(report.html, list)
    assert 'GMT' not in ''.join(report.html)

@testing.requires_testing_data
def test_render_add_sections(renderer, tmpdir):
    """Test adding figures/images to section."""
    tempdir = str(tmpdir)
    report = Report(subjects_dir=subjects_dir)
    # Check add_figs_to_section functionality
    fig = plt.plot([1, 2], [1, 2])[0].figure
    report.add_figs_to_section(figs=fig,  # test non-list input
                               captions=['evoked response'], scale=1.2,
                               image_format='svg')
    pytest.raises(ValueError, report.add_figs_to_section, figs=[fig, fig],
                  captions='H')
    pytest.raises(ValueError, report.add_figs_to_section, figs=fig,
                  captions=['foo'], scale=0, image_format='svg')
    pytest.raises(ValueError, report.add_figs_to_section, figs=fig,
                  captions=['foo'], scale=1e-10, image_format='svg')
    # need to recreate because calls above change size
    fig = plt.plot([1, 2], [1, 2])[0].figure

    # Check add_images_to_section with png
    img_fname = op.join(tempdir, 'testimage.png')
    fig.savefig(img_fname)
    report.add_images_to_section(fnames=[img_fname],
                                 captions=['evoked response'])

    report.add_images_to_section(fnames=[img_fname],
                                 captions=['evoked response'])

    pytest.raises(ValueError, report.add_images_to_section,
                  fnames=[img_fname, img_fname], captions='H')

    pytest.raises(ValueError, report.add_images_to_section,
                  fnames=['foobar.xxx'], captions='H')

    evoked = read_evokeds(evoked_fname, condition='Left Auditory',
                          baseline=(-0.2, 0.0))
    fig = plot_alignment(evoked.info, trans_fname, subject='sample',
                         subjects_dir=subjects_dir)

    report.add_figs_to_section(figs=fig,  # test non-list input
                               captions='random image', scale=1.2)
    assert (repr(report))

@pytest.mark.slowtest
@testing.requires_testing_data
@requires_nibabel()
def test_render_mri(renderer, tmpdir):
    """Test rendering MRI for mne report."""
    tempdir = str(tmpdir)
    trans_fname_new = op.join(tempdir, 'temp-trans.fif')
    for a, b in [[trans_fname, trans_fname_new]]:
        shutil.copyfile(a, b)
    report = Report(info_fname=raw_fname,
                    subject='sample', subjects_dir=subjects_dir)
    report.parse_folder(data_path=tempdir, mri_decim=30, pattern='*')
    report.save(op.join(tempdir, 'report.html'), open_browser=False)
    assert repr(report)
    report.add_bem_to_section('sample', caption='extra', section='foo',
                              subjects_dir=subjects_dir, decim=30)
    report.save(op.join(tempdir, 'report.html'), open_browser=False,
                overwrite=True)

@testing.requires_testing_data
@requires_nibabel()
def test_render_mri_without_bem(tmpdir):
    """Test rendering MRI without BEM for mne report."""
    tempdir = str(tmpdir)
    os.mkdir(op.join(tempdir, 'sample'))
    os.mkdir(op.join(tempdir, 'sample', 'mri'))
    shutil.copyfile(mri_fname, op.join(tempdir, 'sample', 'mri', 'T1.mgz'))
    report = Report(info_fname=raw_fname,
                    subject='sample', subjects_dir=tempdir)
    report.parse_folder(tempdir, render_bem=False)
    report.save(op.join(tempdir, 'report.html'), open_browser=False)

@testing.requires_testing_data
@requires_nibabel()
def test_add_htmls_to_section():
    """Test adding html str to mne report."""
    report = Report(info_fname=raw_fname,
                    subject='sample', subjects_dir=subjects_dir)
    html = '<b>MNE-Python is AWESOME</b>'
    caption, section = 'html', 'html_section'
    report.add_htmls_to_section(html, caption, section)
    idx = report._sectionlabels.index('report_' + section)
    html_compare = report.html[idx]
    assert (html in html_compare)
    assert (repr(report))

def test_add_slider_to_section(tmpdir):
    """Test adding a slider with a series of images to mne report."""
    tempdir = str(tmpdir)
    report = Report(info_fname=raw_fname,
                    subject='sample', subjects_dir=subjects_dir)
    section = 'slider_section'
    figs = _get_example_figures()
    report.add_slider_to_section(figs, section=section, title='my title')
    assert report.fnames[0] == 'my title-#-report_slider_section-#-custom'
    report.save(op.join(tempdir, 'report.html'), open_browser=False)

    pytest.raises(NotImplementedError, report.add_slider_to_section,
                  [figs, figs])
    pytest.raises(ValueError, report.add_slider_to_section, figs, ['wug'])
    pytest.raises(TypeError, report.add_slider_to_section, figs, 'wug')
    # need at least 2
    pytest.raises(ValueError, report.add_slider_to_section, figs[:1], 'wug')

    # Smoke test that SVG w/unicode can be added
    report = Report()
    fig, ax = plt.subplots()
    ax.set_xlabel('µ')
    report.add_slider_to_section([fig] * 2, image_format='svg')

def test_validate_input():
    """Test Report input validation."""
    report = Report()
    items = ['a', 'b', 'c']
    captions = ['Letter A', 'Letter B', 'Letter C']
    section = 'ABCs'
    comments = ['First letter of the alphabet.',
                'Second letter of the alphabet',
                'Third letter of the alphabet']
    pytest.raises(ValueError, report._validate_input, items, captions[:-1],
                  section, comments=None)
    pytest.raises(ValueError, report._validate_input, items, captions, section,
                  comments=comments[:-1])
    values = report._validate_input(items, captions, section, comments=None)
    items_new, captions_new, comments_new = values
    assert_equal(len(comments_new), len(items))

@requires_h5py
def test_open_report(tmpdir):
    """Test the open_report function."""
    tempdir = str(tmpdir)
    hdf5 = op.join(tempdir, 'report.h5')

    # Test creating a new report through the open_report function
    fig1 = _get_example_figures()[0]
    with open_report(hdf5, subjects_dir=subjects_dir) as report:
        assert report.subjects_dir == subjects_dir
        assert report._fname == hdf5
        report.add_figs_to_section(figs=fig1, captions=['evoked response'])
    # Exiting the context block should have triggered saving to HDF5
    assert op.exists(hdf5)

    # Load the HDF5 version of the report and check equivalence
    report2 = open_report(hdf5)
    assert report2._fname == hdf5
    assert report2.subjects_dir == report.subjects_dir
    assert report2.html == report.html
    assert report2.__getstate__() == report.__getstate__()
    assert '_fname' not in report2.__getstate__()

    # Check parameters when loading a report
    pytest.raises(ValueError, open_report, hdf5, foo='bar')  # non-existing
    pytest.raises(ValueError, open_report, hdf5, subjects_dir='foo')
    open_report(hdf5, subjects_dir=subjects_dir)  # This should work

    # Check that the context manager doesn't swallow exceptions
    with pytest.raises(ZeroDivisionError):
        with open_report(hdf5, subjects_dir=subjects_dir) as report:
            1 / 0

def test_remove():
    """Test removing figures from a report."""
    r = Report()
    fig1, fig2 = _get_example_figures()
    r.add_figs_to_section(fig1, 'figure1', 'mysection')
    r.add_slider_to_section([fig1, fig2], title='figure1',
                            section='othersection')
    r.add_figs_to_section(fig2, 'figure1', 'mysection')
    r.add_figs_to_section(fig2, 'figure2', 'mysection')

    # Test removal by caption
    r2 = copy.deepcopy(r)
    removed_index = r2.remove(caption='figure1')
    assert removed_index == 2
    assert len(r2.html) == 3
    assert r2.html[0] == r.html[0]
    assert r2.html[1] == r.html[1]
    assert r2.html[2] == r.html[3]

    # Test restricting to section
    r2 = copy.deepcopy(r)
    removed_index = r2.remove(caption='figure1', section='othersection')
    assert removed_index == 1
    assert len(r2.html) == 3
    assert r2.html[0] == r.html[0]
    assert r2.html[1] == r.html[2]
    assert r2.html[2] == r.html[3]

    # Test removal of empty sections
    r2 = copy.deepcopy(r)
    r2.remove(caption='figure1', section='othersection')
    assert r2.sections == ['mysection']
    assert r2._sectionvars == {'mysection': 'report_mysection'}

def test_add_or_replace():
    """Test replacing existing figures in a report."""
    r = Report()
    fig1, fig2 = _get_example_figures()
    r.add_figs_to_section(fig1, 'duplicate', 'mysection')
    r.add_figs_to_section(fig1, 'duplicate', 'mysection')
    r.add_figs_to_section(fig1, 'duplicate', 'othersection')
    r.add_figs_to_section(fig2, 'nonduplicate', 'mysection')
    # By default, replace=False, so all figures should be there
    assert len(r.html) == 4

    old_r = copy.deepcopy(r)

    # Re-add fig1 with replace=True, it should overwrite the last occurrence of
    # fig1 in section 'mysection'.
    r.add_figs_to_section(fig2, 'duplicate', 'mysection', replace=True)
    assert len(r.html) == 4
    assert r.html[1] != old_r.html[1]  # This figure should have changed
    # All other figures should be the same
    assert r.html[0] == old_r.html[0]
    assert r.html[2] == old_r.html[2]
    assert r.html[3] == old_r.html[3]

def test_scraper(tmpdir):
    """Test report scraping."""
    r = Report()
    fig1, fig2 = _get_example_figures()
    r.add_figs_to_section(fig1, 'a', 'mysection')
    r.add_figs_to_section(fig2, 'b', 'mysection')
    # Mock a Sphinx + sphinx_gallery config
    app = Bunch(builder=Bunch(srcdir=str(tmpdir),
                              outdir=op.join(str(tmpdir), '_build', 'html')))
    scraper = _ReportScraper()
    scraper.app = app
    gallery_conf = dict(src_dir=app.builder.srcdir, builder_name='html')
    img_fname = op.join(app.builder.srcdir, 'auto_examples', 'images',
                        'sg_img.png')
    target_file = op.join(app.builder.srcdir, 'auto_examples', 'sg.py')
    os.makedirs(op.dirname(img_fname))
    os.makedirs(app.builder.outdir)
    block_vars = dict(image_path_iterator=(img for img in [img_fname]),
                      example_globals=dict(a=1), target_file=target_file)
    # Nothing yet
    block = None
    rst = scraper(block, block_vars, gallery_conf)
    assert rst == ''
    # Still nothing
    block_vars['example_globals']['r'] = r
    rst = scraper(block, block_vars, gallery_conf)
    # Once it's saved, add it
    assert rst == ''
    fname = op.join(str(tmpdir), 'my_html.html')
    r.save(fname, open_browser=False)
    rst = scraper(block, block_vars, gallery_conf)
    out_html = op.join(app.builder.outdir, 'auto_examples', 'my_html.html')
    assert not op.isfile(out_html)
    os.makedirs(op.join(app.builder.outdir, 'auto_examples'))
    scraper.copyfiles()
    assert op.isfile(out_html)
    assert rst.count('"') == 6
    assert "<iframe" in rst
    assert op.isfile(img_fname.replace('png', 'svg'))

run_tests_if_main()


# https://leetcode.com/problems/find-all-possible-recipes-from-given-supplies/
# tags: #google, #graph, #hash_table, #topological_sort
#
# Solution: Topological sort
# For each recipe, count its dependent ingredients as in degree,
# store each ingredient dependent to a recipe in a dictionary.
# Use the supplies as the starting points of topological sort, one option is to convert supplies list to a queue
# The topological sort will be based on the supplies first given, we will decrease the in degree of recipes
# and whenever in-degree reaches zero it means we can create the recipe, so we add that recipe to the supplies_left.
# Time complexity : O(m*n) m=recipes n=ingredients, Space complexity: O(n)
from collections import defaultdict, deque

from typing import List

class Solution:
    def findAllRecipes(self, recipes: List[str], ingredients: List[List[str]], supplies: List[str]) -> List[str]:
        ingredients_to_recipe = defaultdict(list)  # graph
        in_degree = dict()

        for i, recipe in enumerate(recipes):
            n = len(ingredients[i])
            in_degree[recipe] = n
            for j in range(n):
                ingredients_to_recipe[ingredients[i][j]].append(recipe)

        # Topological sort
        allowed_recipes = []
        supplies_left = deque(supplies)
        while supplies_left:
            ing_supplied = supplies_left.popleft()

            for recipe in ingredients_to_recipe.pop(ing_supplied, []):
                in_degree[recipe] -= 1
                if in_degree[recipe] == 0:
                    supplies_left.append(recipe)
                    allowed_recipes.append(recipe)

        return allowed_recipes

if __name__ == '__main__':
    sol = Solution()
    print(sol.findAllRecipes(
        recipes=["bread", "sandwich", "burger"],
        ingredients=[["yeast", "flour"], ["bread", "meat"], ["sandwich", "meat", "bread"]],
        supplies=["yeast", "flour", "meat"]
    ))  # ["bread","sandwich","burger"]

    print(sol.findAllRecipes(
        recipes=["ju", "fzjnm", "x", "e", "zpmcz", "h", "q"],
        ingredients=[
            ["d"], ["hveml", "f", "cpivl"], ["cpivl", "zpmcz", "h", "e", "fzjnm", "ju"],
            ["cpivl", "hveml", "zpmcz", "ju", "h"],
            ["h", "fzjnm", "e", "q", "x"],
            ["d", "hveml", "cpivl", "q", "zpmcz", "ju", "e", "x"],
            ["f", "hveml", "cpivl"]],
        supplies=["f", "hveml", "cpivl", "d"],
    ))  # ['fzjnm', 'q', 'ju']


from data import load_gmd_hvo_sequences, load_down_sampled_gmd_hvo_sequences
from eval.GrooveEvaluator import Evaluator
from eval.GrooveEvaluator import load_evaluator
import os

import logging
logger = logging.getLogger("eval.GrooveEvaluator.templates.main")
logger.setLevel("DEBUG")

def create_template(dataset_setting_json_path, subset_name, down_sampled_ratio=None,
                    cached_folder="eval/GrooveEvaluator/templates/", divide_by_genre=True):
    """
    Create a template for the given dataset and subset. The template will ALWAYS be saved in the cached_folder.
    :param dataset_setting_json_path:   The path to the json file that contains the dataset settings.
                                        (e.g. "data/dataset_json_settings/4_4_Beats_gmd.json")
    :param subset_name:                 The name of the subset to be used. (e.g. "train", "test", "validation")
    :param down_sampled_ratio:          The ratio of the down-sampled set. (e.g. 0.02)
    :param cache_down_sampled_set:      Whether to cache the down-sampled set.
    :param cached_folder:               The folder to save the template.
    :param divide_by_genre:             Whether to divide the dataset by genre.
    :return:
    """
    # load data using the settings specified in the dataset_setting_json_path
    if down_sampled_ratio is None:
        hvo_seq_set = load_gmd_hvo_sequences(dataset_setting_json_path, subset_name)
    else:
        hvo_seq_set = load_down_sampled_gmd_hvo_sequences(dataset_setting_json_path, subset_name,
                                                          down_sampled_ratio=down_sampled_ratio,
                                                          cache_down_sampled_set=False)

    # create a list of filter dictionaries for each genre if divide_by_genre is True
    if divide_by_genre:
        list_of_filter_dicts_for_subsets = []
        styles = [
            "afrobeat", "afrocuban", "blues", "country", "dance", "funk", "gospel", "highlife", "hiphop", "jazz",
            "latin", "middleeastern", "neworleans", "pop", "punk", "reggae", "rock", "soul"]
        for style in styles:
            list_of_filter_dicts_for_subsets.append(
                {"style_primary": [style]}  # , "beat_type": ["beat"], "time_signature": ["4-4"]}
            )
    else:
        list_of_filter_dicts_for_subsets = None

    dataset_name = dataset_setting_json_path.split("/")[-1].split(".")[0]

    _identifier = f"_{down_sampled_ratio}_ratio_of_{dataset_name}_{subset_name}" \
        if down_sampled_ratio is not None else f"complete_set_of_{dataset_name}_{subset_name}"

    # create the evaluator
    eval = Evaluator(
        hvo_sequences_list_=hvo_seq_set,
        list_of_filter_dicts_for_subsets=list_of_filter_dicts_for_subsets,
        _identifier=_identifier,
        n_samples_to_use=-1,
        max_hvo_shape=(32, 27),
        need_hit_scores=False,
        need_velocity_distributions=False,
        need_offset_distributions=False,
        need_rhythmic_distances=False,
        need_heatmap=False,
        need_global_features=False,
        need_audio=False,
        need_piano_roll=False,
        need_kl_oa=False,
        n_samples_to_synthesize_and_draw="all",
        disable_tqdm=False)

    eval.dump(path=cached_folder)

    logger.info(f"Template created and cached at {cached_folder}")
    return eval

def load_evaluator_template(dataset_setting_json_path, subset_name,
                            down_sampled_ratio, cached_folder="eval/GrooveEvaluator/templates/",
                            divide_by_genre=True, disable_logging=True):
    """
    Load a template for the given dataset and subset. If the template does not exist, it will be created and
    automatically saved in the cached_folder.

    :param dataset_setting_json_path:  The path to the json file that contains the dataset settings.

    :param subset_name:             The name of the subset to be used. (e.g. "train", "test", "validation")
    :param down_sampled_ratio:      The ratio of the down-sampled set. (e.g. 0.02)
    :param cached_folder:           The folder to save the template.
    :return:                        The evaluator template.
    """
    dataset_name = dataset_setting_json_path.split("/")[-1].split(".")[0]

    _identifier = f"_{down_sampled_ratio}_ratio_of_{dataset_name}_{subset_name}" \
        if down_sampled_ratio is not None else f"complete_set_of_{dataset_name}_{subset_name}"
    path = os.path.join(cached_folder, f"{_identifier}_evaluator.Eval.bz2")

    if os.path.exists(path):
        if not disable_logging:
            logger.info(f"Loading template from {path}")
        return load_evaluator(path)
    else:

        return create_template(
            dataset_setting_json_path=dataset_setting_json_path,
            subset_name=subset_name,
            down_sampled_ratio=down_sampled_ratio,
            cached_folder=cached_folder,
            divide_by_genre=divide_by_genre)


from datetime import datetime
from dateutil import parser
from tracardi.service.plugin.domain.register import Plugin, Spec, MetaData, \
    Documentation, PortDoc
from tracardi.service.plugin.runner import ActionRunner
from tracardi.service.plugin.domain.result import Result

class LastVisitAction(ActionRunner):

    async def set_up(self, init):
        pass

    async def run(self, payload: dict, in_edge=None) -> Result:
        if self.profile is None:
            self.console.error("Can not get last profile visit for without a profile.")
        else:
            last_visit = None
            if self.profile.metadata.time.visit.current:
                last_visit = self.profile.metadata.time.visit.current
            elif self.profile.metadata.time.insert:
                last_visit = self.profile.metadata.time.insert

            if last_visit:

                if isinstance(last_visit, str):
                    last_visit = parser.parse(last_visit)

                if isinstance(last_visit, datetime):
                    diff_secs = (datetime.utcnow() - last_visit).total_seconds()

                    return Result(port="visit", value={
                        'last': last_visit,
                        'difference': {
                            "seconds": diff_secs,
                            "minutes": diff_secs / 60,
                            "hours": diff_secs / (60 * 60),
                            "days": diff_secs / (60 * 60 * 24),
                            "weeks": diff_secs / (60 * 60 * 24 * 7)
                        }
                    })
                else:
                    self.console.error(f"Last visit is not a date. Expected datetime object got {type(last_visit)}.")

        return Result(port="error", value=payload)

def register() -> Plugin:
    return Plugin(
        start=False,
        spec=Spec(
            module=__name__,
            className=LastVisitAction.__name__,
            inputs=["payload"],
            outputs=["visit", "error"],
            version='0.7.3',
            license="MIT",
            author="Risto Kowaczewski"
        ),
        metadata=MetaData(
            name='Last profile visit time',
            desc='Returns last profile visit time difference till now.',
            icon='time',
            group=["Time"],
            tags=['previous', 'last'],
            purpose=['collection', 'segmentation'],
            documentation=Documentation(
                inputs={
                    "payload": PortDoc(desc="This port takes any JSON-like object.")
                },
                outputs={
                    "visit": PortDoc(desc="Returns last visit."),
                    "payload": PortDoc(desc="This port returns exactly same payload as given one.")
                }
            )
        )
    )


import json
from collections import Counter
import numpy as np

from sentence_retrieval.sent_tfidf import OnlineTfidfDocRanker
from utils import fever_db, check_sentences, text_clean
import config
import drqa_yixin.tokenizers
from drqa_yixin.tokenizers import CoreNLPTokenizer
from tqdm import tqdm
from utils import c_scorer
import math
import utils
from collections import namedtuple
from pathlib import Path

from utils import common

path_stanford_corenlp_full_2017_06_09 = str(config.PRO_ROOT / 'dep_packages/stanford-corenlp-full-2017-06-09/*')
print(path_stanford_corenlp_full_2017_06_09)

drqa_yixin.tokenizers.set_default('corenlp_classpath', path_stanford_corenlp_full_2017_06_09)
tok = CoreNLPTokenizer(annotators=['pos', 'lemma'])

def easy_tokenize(text):
    return tok.tokenize(text_clean.normalize(text)).words()

def load_data(file):
    d_list = []
    with open(file, encoding='utf-8', mode='r') as in_f:
        for line in in_f:
            item = json.loads(line.strip())
            d_list.append(item)

    return d_list

def utest_for_ground_truth(d_list):
    nei_c = 0
    support_c = 0
    refute_c = 0
    for item in tqdm(d_list):
        e_list = check_sentences.check_and_clean_evidence(item)
        evidence_sent_id = []
        gt_evidence = []
        if item['verifiable'] == "VERIFIABLE":
            for doc_id, ln in list(e_list)[0]:
                evidence_sent_id.append(doc_id + c_scorer.SENT_LINE + str(ln))
                gt_evidence.append([doc_id, ln])
        elif item['verifiable'] == "NOT VERIFIABLE":
            evidence_sent_id = []

        item["predicted_sentids"] = evidence_sent_id

        # item['predicted_evidence'] = []
        item['predicted_evidence'] = gt_evidence
        item['predicted_label'] = item["label"]

        if item["label"] == 'NOT ENOUGH INFO':
            nei_c += 1
        elif item["label"] == 'SUPPORTS':
            support_c += 1
        elif item["label"] == 'REFUTES':
            refute_c += 1

    print(support_c, refute_c, nei_c)
        # if len(evidence_sent_id) >= 2:
        #     print(evidence_sent_id)

def utest_score_ground_truth():
    d_list = load_data(config.FEVER_DEV_JSONL)
    utest_for_ground_truth(d_list)

    eval_mode = {'check_sent_id_correct': True, 'standard': True}
    print(c_scorer.fever_score(d_list, d_list, mode=eval_mode, verbose=False))

def utest_check_sentence_lines():
    sent_number_coutner = Counter()
    number_list = []
    db_cursor = fever_db.get_cursor()
    # d_list = load_data("/Users/Eason/RA/FunEver/results/doc_retri/2018_07_04_21:56:49_r/dev.jsonl")
    d_list = load_data("/Users/Eason/RA/FunEver/results/doc_retri/2018_07_04_21:56:49_r/train.jsonl")
    for item in tqdm(d_list):
        p_docids = item['predicted_docids']
        current_sent_list = []
        for doc_id in p_docids:
            r_list = fever_db.get_all_sent_by_doc_id(db_cursor, doc_id)
            current_sent_list.extend(r_list)

        sent_number_coutner.update([len(current_sent_list)])
        number_list.append(len(current_sent_list))
        # print(current_sent_list)

    print(len(number_list))
    print('Mean:', np.mean(number_list))
    print('Max:', np.max(number_list))
    print('Min:', np.min(number_list))
    print('Std:', np.std(number_list))
    print(sent_number_coutner)

        # print(p_docids)

def save_intermidiate_results(d_list, out_filename: Path, last_loaded_path=None):
    if not out_filename.parent.exists():
        out_filename.parent.mkdir(exist_ok=False)

    with open(out_filename, encoding='utf-8', mode='w') as out_f:
        for item in d_list:
            out_f.write(json.dumps(item) + '\n')

    if last_loaded_path is not None:
        with open(out_filename.parent / "log_info.txt", encoding='utf-8', mode='a') as out_f:
            out_f.write(last_loaded_path)

def if_idf_select_sentence():
    db_cursor = fever_db.get_cursor()
    loaded_path = "/Users/Eason/RA/FunEver/results/doc_retri/2018_07_04_21:56:49_r/dev.jsonl"
    d_list = load_data(loaded_path)
    # d_list = load_data("/Users/Eason/RA/FunEver/results/doc_retri/2018_07_04_21:56:49_r/train.jsonl")

    for item in tqdm(d_list):
        # print()
        p_docids = item['predicted_docids']
        cleaned_claim = ' '.join(easy_tokenize(item['claim']))
        # print(cleaned_claim)

        current_sent_list = []
        current_id_list = []
        for doc_id in p_docids:
            r_list, id_list = fever_db.get_all_sent_by_doc_id(db_cursor, doc_id)
            current_sent_list.extend(r_list)
            current_id_list.extend(id_list)

        Args = namedtuple('Args', 'ngram hash_size num_workers')

        args = Args(2, int(8192), 4)

        ranker = OnlineTfidfDocRanker(args, args.hash_size, args.ngram,
                                      current_sent_list)

        selected_index, selected_score = ranker.closest_docs(cleaned_claim, k=5)

        selected_sent_id = []
        for ind in selected_index:
            curent_selected = current_id_list[ind]
            doc_id, ln = curent_selected.split('(-.-)')
            # ln = int(ln)
            # selected_sent_id.append([doc_id, ln])
            selected_sent_id.append(doc_id + c_scorer.SENT_LINE + ln)

        item['predicted_sentids'] = selected_sent_id

    eval_mode = {'check_sent_id_correct': True, 'standard': False}
    print(c_scorer.fever_score(d_list, d_list, mode=eval_mode, verbose=False))

    out_fname = config.RESULT_PATH / "sent_retri" / f"{utils.get_current_time_str()}_r" / "dev.jsonl"
    save_intermidiate_results(d_list, out_filename=out_fname, last_loaded_path=loaded_path)

def script(in_path, out_path):
    print(in_path)
    print(out_path)
    db_cursor = fever_db.get_cursor()

    if isinstance(in_path, str):
        loaded_path = in_path
        d_list = load_data(loaded_path)
    elif isinstance(in_path, list):
        d_list = in_path
    else:
        print("Error input format")
        d_list = None
        exit(-1)

    # d_list = load_data("/Users/Eason/RA/FunEver/results/doc_retri/2018_07_04_21:56:49_r/train.jsonl")
    res_list = []

    for item in tqdm(d_list):
        # print()
        p_docids = item['predicted_docids']
        # cleaned_claim = ' '.join(easy_tokenize(item['claim']))
        # print(cleaned_claim)
        cleaned_claim = ' '.join(item['claim'].split(' '))

        current_sent_list = []
        current_id_list = []
        for doc_id in p_docids:
            r_list, id_list = fever_db.get_all_sent_by_doc_id(db_cursor, doc_id)
            current_sent_list.extend(r_list)
            current_id_list.extend(id_list)

        Args = namedtuple('Args', 'ngram hash_size num_workers')

        args = Args(2, int(8192), 4)

        ranker = OnlineTfidfDocRanker(args, args.hash_size, args.ngram,
                                      current_sent_list)

        selected_index, selected_score = ranker.closest_docs(cleaned_claim, k=5)

        selected_sent_id = []
        for ind in selected_index:
            curent_selected = current_id_list[ind]
            doc_id, ln = curent_selected.split('(-.-)')
            # ln = int(ln)
            # selected_sent_id.append([doc_id, ln])
            selected_sent_id.append(doc_id + c_scorer.SENT_LINE + ln)

        item['predicted_sentids'] = selected_sent_id

        res_list.append(item)
        # break

    eval_mode = {'check_sent_id_correct': True, 'standard': False}
    print(c_scorer.fever_score(res_list, res_list, mode=eval_mode, verbose=False))

    # out_fname = config.RESULT_PATH / "sent_retri" / f"{utils.get_current_time_str()}_r" / "dev.jsonl"
    out_fname = Path(out_path)
    save_intermidiate_results(res_list, out_filename=out_fname, last_loaded_path=None)

def check_acc(in_path):
    d_list = load_data(in_path)
    eval_mode = {'check_sent_id_correct': True, 'standard': False}
    print(c_scorer.fever_score(d_list, d_list, mode=eval_mode, verbose=False))

if __name__ == '__main__':
    # check_acc("/Users/Eason/RA/FunEver/results/sent_retri/2018_07_05_17:17:50_r/train.jsonl")

    # utest_check_sentence_lines()
    # check_acc(in_path)
    # d_list = load_data(in_path)
    # if_idf_select_sentence()

    # script(
    #     in_path='/Users/Eason/RA/FunEver/results/doc_retri/docretri.pageview/dev.jsonl',
    #     out_path='/Users/Eason/RA/FunEver/results/sent_retri/docretri.pageview.tf_idf/dev.jsonl'
    # )

    # script(
    #     in_path=common.load_jsonl('/Users/Eason/RA/FunEver/results/doc_retri/docretri.pageview/dev.jsonl'),
    #     out_path='/Users/Eason/RA/FunEver/results/sent_retri/docretri.pageview.tf_idf/dev_0.jsonl'
    # )

    doc_num = 10

    dev_doc_file = config.SRC_ROOT / "analysis/analysis_data/sentence_selection/upstream_nn_doc_retri.jsonl"
    dev_doc_list = common.load_jsonl(dev_doc_file)

    for item in dev_doc_list:
        item['predicted_docids'] = item['predicted_docids'][:doc_num]

    script(
        in_path=dev_doc_list,
        out_path=str(config.SRC_ROOT / f"analysis/analysis_data/sentence_selection/tf_idf_sent_selection_docn({doc_num}).jsonl")
    )

    # 16783 0.8392339233923393 document = 5 upstream
    # 16752 0.8376837683768377 document = 10 upstream

    # utest_score_ground_truth()
    # d_list = load_data(config.DATA_ROOT / "fever/shared_task_dev.jsonl")
    # db_cursor = fever_db.get_cursor()
    #
    # contain_head = True
    #
    # for item in tqdm(d_list):
    #     e_list = check_sentences.check_and_clean_evidence(item)
    #     evidence_text_list = []
    #     if item['verifiable'] == "VERIFIABLE":
    #         evidence_text_list = sample_for_verifiable(db_cursor, e_list, contain_head=contain_head)
    #         print(evidence_text_list)
    #     elif item['verifiable'] == "NOT VERIFIABLE":
    #         pass

        # if len(evidence_text_list) >= 2:
        #     print("Claim:", item['claim'])
        #     for text in evidence_text_list:
                # print(text)


import tkinter as tk
from tkinter import ttk

from matplotlib import pyplot as plt
from matplotlib.figure import Figure
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2Tk

class Graphs:
    step = 0

    def step_init(self, value) : self.step = value

    def __init__(self, drawFrame, impulse):
        self.drawFrame = drawFrame
        self.impulse = impulse

        self.tab_control = ttk.Notebook(self.drawFrame)

        self.tab1 = ttk.Frame(self.tab_control)
        self.tab_control.add(self.tab1, text = "Sygnał wejściowy")
        self.add_signal(self.impulse.time, self.impulse.input_singal, self.tab1, "x(t)")

        self.tab2 = ttk.Frame(self.tab_control)
        self.tab_control.add(self.tab2, text = "Sygnał wyjściowy")

        time_base = self.impulse.time[0::self.step]
        size_differ = (len(time_base) - len(impulse.output_signal))
        if size_differ == 0: self.add_signal(time_base, self.impulse.output_signal, self.tab2, "y(t)")
        else:  self.add_signal(time_base[0: -size_differ], self.impulse.output_signal, self.tab2, "y(t)")

        self.tab3 = ttk.Frame(self.tab_control)
        self.tab_control.add(self.tab3, text = "Wykres amplitudowy")
        self.add_bode_amp()

        self.tab4 = ttk.Frame(self.tab_control)
        self.tab_control.add(self.tab4, text = "Wykres fazowy")
        self.add_bode_phase()

        self.tab_control.pack(expan = 1, fill = "both")

    def add_signal(self, s_x, s_y, frame, y):
        f = Figure(figsize=(4,4), dpi=100)

        a = f.add_subplot(1,1,1)
        a.plot(s_x, s_y)

        a.set_ylabel(y)
        a.set_xlabel("time [s]")
        a.grid(True, linestyle='-.')

        canvas = FigureCanvasTkAgg(f, frame)
        canvas.get_tk_widget().pack(fill = "both", side = "bottom", expand = True)
        canvas._tkcanvas.pack(side = "top", fill = "both", expand = True)

    def add_bode_amp(self):
        fa = Figure(figsize=(4,4), dpi=100)

        aa = fa.add_subplot(1,1,1)
        aa.semilogx(self.impulse.w, self.impulse.mag, color="blue", linewidth='1')

        aa.set_xlabel("Czestotliwosc")
        aa.set_ylabel("Wzmocnienie")
        aa.grid(True, linestyle='-.')

        canvas = FigureCanvasTkAgg(fa, self.tab3)
        canvas.get_tk_widget().pack(fill = "both", side = "bottom", expand = True)
        canvas._tkcanvas.pack(side = "top", fill = "both", expand = True)

    def add_bode_phase(self):
        fp = Figure(figsize=(4,4), dpi=100)

        ap = fp.add_subplot(1,1,1)
        ap.semilogx(self.impulse.w, self.impulse.phase, color="blue", linewidth='1')

        ap.set_xlabel("Czestotliwosc")
        ap.set_ylabel("Faza")
        ap.grid(True, linestyle='-.')

        canvas = FigureCanvasTkAgg(fp, self.tab4)
        canvas.get_tk_widget().pack(fill = "both", side = "bottom", expand = True)
        canvas._tkcanvas.pack(side = "top", fill = "both", expand = True)


import numpy as np
import torch
from collections import OrderedDict
import os

def save_model(model, optimizer, path = "/", filename = 'check_point.pth'):
	torch.save({'model':model.state_dict(), 'optimizer':optimizer.state_dict()}, os.path.join(path, filename))
	print("model saved as" + os.path.join(path, filename))

def load_model(model, path, model_name, mode = "parallel"):
	state_dict = torch.load(os.path.join(path, model_name))["model"]
	new_state_dict = OrderedDict()

	for k, v in state_dict.items():
		name = ""
		if mode == "single" and k.startswith("module."):
			name = k[7:]

		elif mode == "parallel" and not k.startswith("module."):
			name = "module."+k

		else:
			name = k

		new_state_dict[name] = v

	model.load_state_dict(new_state_dict)
	print("loaded " + os.path.join(path, model_name))


from typing import List, Tuple
from glob import glob
import random

from librosa.display import specshow, waveshow
from librosa import amplitude_to_db
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

def enumerate_recordings(pathname: str, shuffle=False) -> List[Tuple[int, str]]:
    recordings = glob(f"{pathname}/*/*.wav")
    if shuffle:
        random.shuffle(recordings)
    return list(enumerate(recordings))

def plot_model_metrics(epoch, metrics):
    plt.figure(figsize=(10, 5))

    plt.subplot(1, 2, 1)
    plt.plot(epoch, metrics["loss"], label="Training Loss")
    plt.plot(epoch, metrics["val_loss"], label="Validation Loss")
    plt.legend()
    plt.xlabel("Epoch", fontsize=14, fontweight="bold", labelpad=10)
    plt.ylabel("Loss [CrossEntropy]", fontsize=14, fontweight="bold", labelpad=10)
    plt.title("Loss Evolution", fontsize=16, fontweight="bold")

    plt.subplot(1, 2, 2)
    plt.plot(epoch, 100 * np.array(metrics["accuracy"]), label="Training Accuracy")
    plt.plot(
        epoch, 100 * np.array(metrics["val_accuracy"]), label="Validation Accuracy"
    )
    plt.legend()
    plt.ylim(0, 100)
    plt.xlabel("Epoch", fontsize=14, fontweight="bold", labelpad=10)
    plt.ylabel("Accuracy [%]", fontsize=14, fontweight="bold", labelpad=10)
    plt.title("Accuracy Evolution", fontsize=16, fontweight="bold")

    plt.tight_layout()
    plt.show()

def plot_confusion_matrix(confusion_mtx, classes):
    plt.figure(figsize=(10, 5))
    sns.heatmap(
        confusion_mtx,
        xticklabels=classes,
        yticklabels=classes,
        annot=True,
        fmt="g",
    )
    plt.xlabel("Prediction", fontsize=14, fontweight="bold", labelpad=10)
    plt.ylabel("Expected", fontsize=14, fontweight="bold", labelpad=10)
    plt.title("Confusion Matrix", fontsize=16, fontweight="bold")
    plt.show()

def plot_waveform(waveform: np.ndarray, sample_rate: int):
    """
    Plots the waveform of an audio signal.
    """
    plt.figure(figsize=(8, 4))
    waveshow(waveform, sr=sample_rate)
    plt.xlabel("Time (s)")
    plt.ylabel("Amplitude")
    plt.title("Waveform")
    plt.show()

def plot_spectrogram(spectrogram: np.ndarray, sample_rate: int, hop_length: int):
    """
    Plots the spectrogram of an audio signal.
    """
    plt.figure(figsize=(8, 4))
    db = amplitude_to_db(spectrogram[:, :, 0].T, ref=np.max)
    specshow(
        db,
        sr=sample_rate,
        hop_length=hop_length,
        x_axis="ms",
        y_axis="linear",
        cmap="viridis",
    )
    plt.colorbar(format="%+2.0f dB")
    plt.xlabel("Time (ms)")
    plt.ylabel("Frequency")
    plt.title("Spectrogram")
    plt.show()

def get_pred_percentage(logits: np.ndarray) -> np.float32:
    # Subtract the max logit to prevent overflow
    logits_exp = np.exp(logits - np.max(logits))
    probabilities = logits_exp / np.sum(logits_exp, axis=-1, keepdims=True)
    max_probability = np.max(probabilities, axis=-1)
    return round(max_probability * 100, 1)


from z3 import Int, Implies, solve

# the set of initial subterms of a set of monoid terms
def initial_subterms(terms):
    subterms = set()
    for t in terms:
        for i in range(len(t) + 1):
            subterms.add(t[:i])
    return subterms

# the set of implications u <= v ==> ux <= vx
# for ux != vx in terms.
# because there is no patterm matching,
# we do slicing, and what is `u` in the instructions
# is now `u[:-1]`, and what is `ux` is now `u`
def sigma(terms):
    sig = set()
    # this iteration could be optimised
    for u in terms:
        for v in terms:
            if u == '' or v == '' or u == v:
                pass
            elif u[-1] == v[-1]:
                sig.add(Implies(Int(u[:-1]) <= Int(v[:-1]),
                                Int(u) <= Int(v)))
    return sig

# the set of inequations s > t for
# s in terms1 and t in terms2.
# if this is satisfiable together with sigma,
# then the equation Meet(terms1) <= Join(terms2)
# is not valid.
def fail(terms1, terms2):
    fail = set()
    for s in terms1:
        for t in terms2:
            fail.add(Int(s) > Int(t))
    return fail

def check(terms1, terms2):
    return solve(sigma(initial_subterms(terms1.union(terms2)))
                    .union(fail(terms1, terms2))
    )


from datasets import load_dataset

train_dataset = load_dataset('json', 'train.json')
valid_dataset = load_dataset('json', 'validation.json')

from transformers import AutoTokenizer

checkpoint = "facebook/blenderbot_small-90M"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

source_lang = "input"
target_lang = "output"
prefix = "translate input to output: "

def preprocess_function(examples):
    inputs = [prefix + example[source_lang] for example in examples["translation"]]
    targets = [example[target_lang] for example in examples["translation"]]
    model_inputs = tokenizer(inputs, text_target=targets, max_length=128, truncation=True)
    return model_inputs

tokenized_train = train_dataset.map(preprocess_function, batched=True)
tokenized_valid = valid_dataset.map(preprocess_function, batched=True)

from transformers import DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=checkpoint)

import evaluate

metric = evaluate.load("sacrebleu")

import numpy as np

def postprocess_text(preds, labels):
    preds = [pred.strip() for pred in preds]
    labels = [[label.strip()] for label in labels]

    return preds, labels

def compute_metrics(eval_preds):
    preds, labels = eval_preds
    if isinstance(preds, tuple):
        preds = preds[0]
    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)

    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)

    decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)

    result = metric.compute(predictions=decoded_preds, references=decoded_labels)
    result = {"bleu": result["score"]}

    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]
    result["gen_len"] = np.mean(prediction_lens)
    result = {k: round(v, 4) for k, v in result.items()}
    return result

from transformers import AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer

model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)

training_args = Seq2SeqTrainingArguments(
    output_dir="models",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=32,
    per_device_eval_batch_size=32,
    weight_decay=0.01,
    save_total_limit=3,
    num_train_epochs=2,
    predict_with_generate=True,
    fp16=False,  # Set true only for CUDA
    push_to_hub=False,
)

trainer = Seq2SeqTrainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_train["train"],
    eval_dataset=tokenized_valid["validation"],
    tokenizer=tokenizer,
    data_collator=data_collator,
    compute_metrics=compute_metrics,
)

trainer.train()


import numpy as np
import torch

from torch.utils.data import Dataset
from torch.utils.data import DataLoader

class RecordDataset:
    """Return signal data from a specific record as a batch of continuous
    windows. Overlap in seconds allows overlapping among windows in the
    batch. The last data points will be ignored if their length is
    inferior to window_size.
    """

    def get_record_dataset(self, record, batch_size, stride=None):
        class PredictionDataset(Dataset):
            def __init__(
                self, record, batch_size, window, fs, window_size, channels_start, channels_stop, stride=None
            ):
                super().__init__()
                self.record = record
                self.channels_start = channels_start
                self.channels_stop = channels_stop
                self.signal_size = record["size"]
                self.window_size = window_size

                self.stride = int((stride if stride is not None else window) * fs)
                self.t = np.arange(self.signal_size, dtype=np.int32)
                self.n_windows = int((self.signal_size - self.stride) / (self.window_size - self.stride))

            def __len__(self):
                return self.n_windows

            def __getitem__(self, idx):

                start = idx * self.stride
                stop = start + self.window_size

                return (
                    torch.from_numpy(self.record["data"][self.channels_start : self.channels_stop, start:stop]),
                    self.t[start:stop],
                )

            @staticmethod
            def collate_fn(batch):
                x = torch.stack([b[0] for b in batch])
                t = np.stack([b[1] for b in batch])
                return x, t

        prediction_dataset = PredictionDataset(
            self.signals[record],
            batch_size,
            self.window,
            self.fs,
            self.window_size,
            self.channels_start,
            self.channels_end,
            stride,
        )
        return DataLoader(
            prediction_dataset,
            batch_size=64,
            collate_fn=PredictionDataset.collate_fn,
            num_workers=12,
            pin_memory=True,
            shuffle=False,
            drop_last=False,
        )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
filename: beta-calculator
"""
import pandas as pd
import pandas_datareader.data as web
import datetime as dt
import matplotlib.pyplot as plt

def main():
    start = dt.datetime(2007,1,1)
    end = dt.datetime(2008,1,1)
    ticker = 'GS'

    beta = getBeta(ticker,start,end)
    print('Beta:', beta)

def getBeta(ticker,start,end):
#    returns beta value
    index = '^GSPC'

#    gets stock data for ticker and S&P500 as benchmark
    df = web.DataReader(ticker,'yahoo',start,end)
    bench = web.DataReader(index,'yahoo',start,end)

    total = pd.DataFrame()
    total[ticker]=df['Adj Close']
    total[ticker + ' Return']= total[ticker].pct_change()

    total[index]=bench['Adj Close']
    total[index+' Return']= total[index].pct_change()

    beta = (total[index+' Return'].cov(total[ticker + ' Return']))/total[index+' Return'].var()

    plot(total,index,ticker)
    return beta

def plot(df,index,ticker):
    fig,ax1 = plt.subplots()
    fig.set_figheight(4.5)
    fig.set_figwidth(6)

    plt.scatter(df[index+' Return'],df[ticker + ' Return'],c='k',alpha=0.7,marker=".")

    plt.grid()
    plt.xlabel(index+' Returns')
    plt.ylabel(ticker + ' Returns')

    plt.show()

if  __name__ =='__main__':main()