import paho.mqtt.client as mqtt
import stmpy
import logging
from threading import Thread
import json

MQTT_BROKER = 'mqtt20.iik.ntnu.no'
MQTT_PORT = 1883
MQTT_TOPIC_INPUT = 'ttm4115/team_5/#'
MQTT_TOPIC_OUTPUT = 'ttm4115/team_5/command'
MQTT_TOPIC_STUDENT = 'ttm4115/team_5/student'
MQTT_TOPIC_TEACHER = 'ttm4115/team_5/teacher'

TOTAL_TEAMS = 3
TEAM_ONE = 3
TEAM_TWO = 3
TEAM_THREE = 4

TEAM_ONE_C = 3
TEAM_TWO_C = 3
TEAM_THREE_C= 4

teams_done_with_irat = []
TEAM_IDS = [1,2,3]
irat_finished_counter = 0
trat_finished_counter = 0

class TimerLogic:
    def __init__(self, name, duration, component):
        self._logger = logging.getLogger(__name__)
        self.name = name
        self.duration = duration
        self.component = component
        self.mqtt_client = mqtt.Client()
        self.mqtt_client.connect(MQTT_BROKER, MQTT_PORT)
        self.mqtt_client.subscribe(MQTT_TOPIC_INPUT)
        self.mqtt_client.loop_start()

    def started(self):
        self.stm.start_timer('t', self.duration)
        self._logger.debug('New timer {} with duration {} started.'
                           .format(self.name, self.duration))

    def timer_completed(self):
        self._logger.debug('Timer {} expired.'.format(self.name))
        t_name = self.name
        string = str('{{"command":"{}_expired"}}'.format(t_name))
        self.mqtt_client.publish(MQTT_TOPIC_STUDENT, string)
        self.stm.terminate()

    def create_machine(timer_name, duration, component):
        timer_logic = TimerLogic(name=timer_name, duration=duration, component=component)
        t0 = {'source': 'initial',
              'target': 'active',
              'effect': 'started'}
        t1 = {
            'source': 'active',
            'target': 'completed',
            'trigger': 't',
            'effect': 'timer_completed'}
        t2 = {
            'source': 'active',
            'target': 'completed',
            'trigger': 'rat_completed',
            'effect': 'timer_completed'}
        timer_stm = stmpy.Machine(name=timer_name, transitions=[t0, t1, t2],
                                  obj=timer_logic)
        timer_logic.stm = timer_stm
        return timer_stm

class TeamManagerComponent:
    def on_connect(self, client, userdata, flags, rc):
        self._logger.debug('MQTT connected to {}'.format(client))

    def on_message(self, client, userdata, msg):
        global TEAM_ONE, TEAM_TWO, TEAM_THREE, TEAM_ONE_C, TEAM_TWO_C, TEAM_THREE_C, teams_done_with_irat, TEAM_IDS

        self._logger.debug('Incoming message to topic {}'.format(msg.topic))

        try:
            payload = json.loads(msg.payload.decode("utf-8"))
        except Exception as err:
            self._logger.error('Message sent to topic {} had no valid JSON. Message ignored. {}'.format(msg.topic, err))
            return
        command = payload.get('command')
        self._logger.debug('Command in message is {}'.format(command))

        if command == 'student_iRAT_done':

            try:
                print(type(self))
                student_id = payload.get('student_id')
                team_id = payload.get('team_id')
                RAT_id = payload.get("RAT_id")
                answers = payload.get('answers')
                print(f"answers from student {student_id}: {answers}")

                score = answers.count("correct")
                print(f"The score for student id {student_id}, team {team_id}, RAT {RAT_id}: {score}")

                """
                #Åpner studentscores og legger til iRAT scoren på riktig sted
                with open('Server/resources/ratDB2.json', 'r') as f:
                    data = json.load(f)
                data[student_id][RAT_id]['iRAT_score'] = str(score)

                with open('Server/resources/ratDB2.json', 'w') as f:
                    json.dump(data, f)
                """

                message_non_parsed = {"command":"iRAT_done"}
                message_start_tRAT_non_parsed = {"command":"start_tRAT", "team_id":team_id, "leader_id": "1"}
                message = json.dumps(message_non_parsed)
                message_start_tRAT = json.dumps(message_start_tRAT_non_parsed)
                if team_id == "1":
                    TEAM_ONE +=1
                    if TEAM_ONE == TEAM_ONE_C:
                        self._logger.debug('everyone in TEAM_ONE is done with iRAT, starting tRAT')
                        self.mqtt_client.publish(MQTT_TOPIC_STUDENT, message)
                        self.mqtt_client.publish(MQTT_TOPIC_STUDENT, message_start_tRAT)
                        timer_stm2 = TimerLogic.create_machine("timer_team1", 200000, self)
                        teams_done_with_irat.append(team_id)
                        self.stm_driver.add_machine(timer_stm2)

                if team_id == "2":
                    TEAM_TWO +=1
                    if TEAM_TWO == TEAM_TWO_C:
                        self._logger.debug('everyone in TEAM_TWO is done with iRAT, starting tRAT')
                        self.mqtt_client.publish(MQTT_TOPIC_STUDENT, message)
                        self.mqtt_client.publish(MQTT_TOPIC_STUDENT, message_start_tRAT)
                        timer_stm3 = TimerLogic.create_machine("timer_team2", 200000, self)
                        teams_done_with_irat.append(team_id)
                        self.stm_driver.add_machine(timer_stm3)

                if team_id == "3":
                    TEAM_THREE +=1
                    if TEAM_THREE == TEAM_THREE_C:
                        self._logger.debug('everyone in TEAM_THREE is done with iRAT, starting tRAT')
                        self.mqtt_client.publish(MQTT_TOPIC_STUDENT, message)
                        self.mqtt_client.publish(MQTT_TOPIC_STUDENT, message_start_tRAT)
                        timer_stm4 = TimerLogic.create_machine("timer_team3", 200000, self)
                        teams_done_with_irat.append(team_id)
                        self.stm_driver.add_machine(timer_stm4)

            except Exception as err:
                print("An error occured in TeamManagerComponent")

        if command == "tRAT_answers":
            try:
                print(type(self))
                team_id = payload.get('team_id')
                RAT_id = payload.get("RAT_id")
                answers = payload.get('answers')
                print(f"answers from team {team_id}: {answers}")

                score = answers.count("correct")
                print(f"Score for team {team_id}: {score}")

                """
              #Updates scores
                with open('Server/resources/ratDB2.json') as f:
                    data = json.load(f)
                    print("opened file")
                for student in data.values():
                    print("Parsing file")
                    if student["team"] == team_id:
                        print(f"team id: {team_id}")
                        student[RAT_id]["tRAT_score"] = str(score)
                with open('Server/resources/ratDB2.json', 'w') as f:
                    print("Writing to file")
                    json.dump(data, f)

                """

                if team_id == "1":
                    self.stm_driver.send('rat_completed','timer_team1')
                elif team_id == "2":
                    self.stm_driver.send('rat_completed','timer_team2')
                elif team_id == "3":
                    self.stm_driver.send('rat_completed','timer_team3')

                trat_message_non_parsed = {"command":"tRAT_done"}
                tRAT_message = json.dumps(trat_message_non_parsed)
                self.mqtt_client.publish(MQTT_TOPIC_INPUT, tRAT_message)

            except Exception as err:
                print("Error occured in Team manager component adter tRAT_done")

    def __init__(self):
        self._logger = logging.getLogger(__name__)
        print('logging under name {}.'.format(__name__))
        self._logger.info('Starting Component')
        self._logger.debug('Connecting to MQTT broker {} at port {}'.format(MQTT_BROKER, MQTT_PORT))
        self.mqtt_client = mqtt.Client()
        self.mqtt_client.on_connect = self.on_connect
        self.mqtt_client.on_message = self.on_message
        self.mqtt_client.connect(MQTT_BROKER, MQTT_PORT)
        self.mqtt_client.subscribe(MQTT_TOPIC_INPUT)
        self.mqtt_client.loop_start()
        self.stm_driver = stmpy.Driver()
        self.stm_driver.start(keep_active=True)
        self._logger.debug('Component initialization finished')

    def stop(self):
        self.mqtt_client.loop_stop()
        self.stm_driver.stop()

class ManagerComponent:
    def on_connect(self, client, userdata, flags, rc):
        self._logger.debug('MQTT connected to {}'.format(client))

    def on_message(self, client, userdata, msg):
        global irat_finished_counter, TEAM_ONE, TEAM_TWO, TEAM_THREE, trat_finished_counter
        self._logger.debug('Incoming message to topic {}'.format(msg.topic))

        try:
            payload = json.loads(msg.payload.decode("utf-8"))
        except Exception as err:
            self._logger.error('Message sent to topic {} had no valid JSON. Message ignored. {}'.format(msg.topic, err))
            return
        command = payload.get('command')
        self._logger.debug('Command in message is {}'.format(command))

        if command == 'start_RAT':
            irat_finished_counter = 0
            trat_finished_counter = 0
            TEAM_ONE = 0
            TEAM_TWO = 0
            TEAM_THREE = 0
            try:
                print(type(self))
                rat_id = payload.get('RAT_ID')
                print(f"RAT_ID is {rat_id}")
                timer_stm1 = TimerLogic.create_machine("t1", 100000, self)

                try:
                    with open('Server/resources/ratDB2.json', 'r') as f:
                        print(f"Opened ratDB2.json")
                        data = json.load(f)
                    question = data[str(rat_id)]
                    print(type(question))
                    print(question)

                    new_data = {
                        "command": "start_iRAT",
                        **question
                        }

                    self.mqtt_client.publish(MQTT_TOPIC_STUDENT, json.dumps({"command" : "start_iRAT", "RAT" : question}))
                except Exception as err:
                    print("Error occured in Manager component after start_RAT error: {}".format(err))

                self.stm_driver.add_machine(timer_stm1)
            except Exception as err:
                self._logger.error('Invalid arguments to command. {}'.format(err))

        elif command == 'iRAT_done':
            irat_finished_counter += 1
            self._logger.debug('{} teams are done with iRAT'.format(irat_finished_counter))
            if irat_finished_counter == TOTAL_TEAMS:
                self.stm_driver.send('rat_completed','t1')

        elif command == 'tRAT_done':
            print(f"trad_finished_counter before: {trat_finished_counter}")
            trat_finished_counter += 1
            self._logger.debug('{} teams are done with tRAT'.format(trat_finished_counter))
            if trat_finished_counter == TOTAL_TEAMS:
                self.mqtt_client.publish(MQTT_TOPIC_TEACHER, "RAT SESSION DONE")

        elif command == 'fetch_RATs':
            print("Fetching RATs")
            def read_rat_names(file_path):
                with open(file_path, 'r') as f:
                    data = json.load(f)
                rat_names = {}
                for rat_id, rat_data in data.items():
                    rat_names[rat_id] = rat_data['name']
                return rat_names

            file_path = 'Server/resources/ratDB2.json'
            rat_names = read_rat_names(file_path)
            string = json.dumps({"command": "available_RATs", "rat_info": rat_names})
            self.mqtt_client.publish(MQTT_TOPIC_TEACHER, string)

        elif command == 'create_RAT':
            print("Creating RAT")
            message = payload
            message.pop('command', None)
            with open("Server/resources/ratDB2.json", "r") as f:
                data = json.load(f)
                data.update(message)
            with open("Server/resources/ratDB2.json", "w") as f:
                json.dump(data, f, indent=4)

    def __init__(self):
        self._logger = logging.getLogger(__name__)
        print('logging under name {}.'.format(__name__))
        self._logger.info('Starting Component')
        self._logger.debug('Connecting to MQTT broker {} at port {}'.format(MQTT_BROKER, MQTT_PORT))
        self.mqtt_client = mqtt.Client()
        self.mqtt_client.on_connect = self.on_connect
        self.mqtt_client.on_message = self.on_message
        self.mqtt_client.connect(MQTT_BROKER, MQTT_PORT)
        self.mqtt_client.subscribe(MQTT_TOPIC_INPUT)
        self.mqtt_client.loop_start()
        self.stm_driver = stmpy.Driver()
        self.stm_driver.start(keep_active=True)
        self._logger.debug('Component initialization finished')

    def stop(self):
        self.mqtt_client.loop_stop()
        self.stm_driver.stop()

debug_level = logging.DEBUG
logger = logging.getLogger(__name__)
logger.setLevel(debug_level)
ch = logging.StreamHandler()
ch.setLevel(debug_level)
formatter = logging.Formatter('%(asctime)s - %(name)-12s - %(levelname)-8s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

t = ManagerComponent()
t2= TeamManagerComponent()


import jieba
from openpyxl import load_workbook

def read_train_data(file_path, data_number):
    src_list, tgt_list = [], []
    wb = load_workbook(file_path)
    ws = wb.active
    rows = ws.rows
    n = 0
    for row in rows:
        n += 1
        if n <= data_number:
            line = [col.value for col in row]
            src_sen = line[0]
            tgt_sen = line[1]
            src_list.append(src_sen)
            tgt_list.append(tgt_sen)
    return src_list, tgt_list

def read_dev_data(file_path, data_number):
    src_list, endings_list, answer_list = [], [], []
    wb = load_workbook(file_path)
    ws = wb.active
    rows = ws.rows
    n = 0
    for row in rows:
        n += 1
        if n <= data_number:
            line = [col.value for col in row]
            src_sen = line[0]
            endings = line[1:5]
            answer = line[5]
            src_list.append(src_sen)
            endings_list += endings
            answer_list.append(answer)
    return src_list, endings_list, answer_list

def remove_blank(text):
    after = []
    for i in text:
        if i != ' ':
            after.append(i)
    return after

def build_vocab(sentences, vocab_type):
    vocab_list = []
    for sen in sentences:
        word_list = jieba.lcut(sen)
        word_list = remove_blank(word_list)
        vocab_list += word_list
    vocab_list = list(set(vocab_list))

    vocab_dict = dict()
    # Padding Should be Zero index
    vocab_dict['P'] = 0
    n = 1
    for word in vocab_list:
        vocab_dict[word] = n
        n += 1
    # source句子的词典
    if vocab_type == 'src':
        return vocab_dict
    # target句子的词典，需要加上S、E
    else:
        vocab_dict['S'] = n
        n += 1
        vocab_dict['E'] = n
        return vocab_dict

def padding_list(words, max_seq_len):
    if len(words) >= max_seq_len:
        return words[:max_seq_len]
    else:
        words += [0 for _ in range(max_seq_len - len(words))]
        return words

def coding_sentences(max_seq_len, train_data_number, dev_data_number):
    pre_sentences, ending = read_train_data(file_path='data/train.xlsx', data_number=train_data_number)
    pres, ends, answers = read_dev_data(file_path='data/dev.xlsx', data_number=dev_data_number)

    src_vocab = build_vocab(pre_sentences + pres, vocab_type='src')
    tgt_vocab = build_vocab(ending + ends, vocab_type='tgt')

    src_int = []
    for src_sen in pre_sentences:
        word_list = jieba.lcut(src_sen)
        word_list = remove_blank(word_list)
        src_sen_int = [src_vocab[i] for i in word_list]
        src_sen_int = padding_list(src_sen_int, max_seq_len=max_seq_len)
        src_int.append(src_sen_int)

    tgt_int, output_int = [], []
    for tgt_sen in ending:
        word_list = jieba.lcut(tgt_sen)
        word_list = remove_blank(word_list)
        tgt_sen_int = [tgt_vocab[i] for i in word_list]
        tgt_sen_int1 = [tgt_vocab['S']] + tgt_sen_int
        tgt_sen_int2 = tgt_sen_int + [tgt_vocab['E']]
        tgt_sen_int1 = padding_list(tgt_sen_int1, max_seq_len=max_seq_len)
        tgt_sen_int2 = padding_list(tgt_sen_int2, max_seq_len=max_seq_len)
        # 防止padding时把句子的结束标识符删除
        if tgt_sen_int2[-1] != 0 and tgt_sen_int2[-1] != tgt_vocab['E']:
            tgt_sen_int2[-1] = tgt_vocab['E']
        tgt_int.append(tgt_sen_int1)
        output_int.append(tgt_sen_int2)

    dev_src_int = []
    for dev_src_sen in pres:
        word_list = jieba.lcut(dev_src_sen)
        word_list = remove_blank(word_list)
        src_sen_int = [src_vocab[i] for i in word_list]
        src_sen_int = padding_list(src_sen_int, max_seq_len=max_seq_len)
        dev_src_int.append(src_sen_int)

    dev_endings_int, dev_output_int = [], []
    for dev_end in ends:
        word_list = jieba.lcut(dev_end)
        word_list = remove_blank(word_list)
        tgt_sen_int = [tgt_vocab[i] for i in word_list]
        tgt_sen_int1 = [tgt_vocab['S']] + tgt_sen_int
        tgt_sen_int2 = tgt_sen_int + [tgt_vocab['E']]
        tgt_sen_int1 = padding_list(tgt_sen_int1, max_seq_len=max_seq_len)
        tgt_sen_int2 = padding_list(tgt_sen_int2, max_seq_len=max_seq_len)
        # 防止padding时把句子的结束标识符删除
        if tgt_sen_int2[-1] != 0 and tgt_sen_int2[-1] != tgt_vocab['E']:
            tgt_sen_int2[-1] = tgt_vocab['E']
        dev_endings_int.append(tgt_sen_int2)
        dev_output_int.append(tgt_sen_int1)
    return src_vocab, tgt_vocab, src_int, tgt_int, output_int, dev_src_int, dev_endings_int, dev_output_int, answers


"""
Adapted from https://github.com/rwightman/pytorch-image-models/blob/master/timm/utils/agc.py
"""
import torch
from torch.nn.utils import clip_grad_norm_, clip_grad_value_

def unitwise_norm(x: torch.Tensor, p=2.0):
  if x.ndim <= 1:
    return x.norm(p)
  else:
    return x.norm(p, dim=tuple(range(1, x.ndim)), keepdim=True)

def clip_grad_adaptive_(parameters, value=.01, norm_type=2.0, eps=1e-3):
  """
  Adaptive grad clipping
  """
  if isinstance(parameters, torch.Tensor):
    parameters = [parameters]
  parameters = [p for p in parameters if p.grad is not None]
  if len(parameters) == 0:
    return torch.tensor(0.)
  for param in parameters:
    weights, grads = param.detach(), param.grad.detach()
    max_norm = (
      unitwise_norm(weights, p=norm_type)
        .clamp_(min=eps)
        .mul_(value)
    )
    grad_norm = unitwise_norm(grads, p=norm_type)
    clipped_grad = grads * (max_norm / grad_norm.clamp(min=1e-6))
    new_grads = torch.where(grad_norm < max_norm, grads, clipped_grad)
    param.grad.detach().copy_(new_grads)

def clip_grad_(parameters, value, norm_type=2.0, mode='adaptive'):
  if mode == 'adaptive':
    return clip_grad_adaptive_(
      parameters=parameters,
      value=value,
      norm_type=norm_type
    )
  elif mode == 'norm':
    return clip_grad_norm_(
      parameters=parameters,
      max_norm=value,
      norm_type=norm_type
    )
  elif mode == 'value':
    return clip_grad_value_(
      parameters=parameters,
      clip_value=value,
    )
  else:
    raise ValueError(f'Unsupported mode={mode}, must be adaptive, norm or value')

class GradClipper:
  def __init__(self, value, norm_type=2.0, mode='adaptive'):
    """

    Args:
      value:
      norm_type:
      mode: 'adaptive' | 'norm' | 'value'
    """
    self.value = value
    self.norm_type = norm_type
    self.mode = mode

  def __call__(self, parameters):
    return clip_grad_(
      parameters=parameters,
      value=self.value,
      norm_type=self.norm_type,
      mode=self.mode
    )

  def state_dict(self):
    return self.__dict__

  def load_state_dict(self, dict):
    self.__dict__.update(dict)


import json
from pymongo import ReturnDocument
from database import database

def find_many(ids):
    return [
        storage for storage in database.storage.find(
            {
                '_id': {'$in': ids}
            }
        )
    ]

def save(storage_id, new_value):
    database.storage.find_one_and_update(
        {'_id': storage_id},
        {'$set': {'value': json.dumps(new_value)}},
        return_document=ReturnDocument.AFTER
    )


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# Author: qiujiarong
# Date: 08/03/2018

from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

import tensorflow as tf
import numpy as np
from PIL import Image

from video.inception_resnet_v2 import inception_resnet_v2_arg_scope, inception_resnet_v2
from audio import vggish_postprocess, vggish_slim, vggish_params, vggish_input
from video.flow_test import dense_optical_flow

slim = tf.contrib.slim

DIR_NAME = "../data/"

class RGBFeature(object):
    """"""

    def __init__(self):
        """Constructor for RGBFeature"""
        checkpoint_file = DIR_NAME + 'checkpoints/inception_resnet_v2_2016_08_30.ckpt'
        self.input_tensor = tf.placeholder(tf.float32, [None, 299, 299, 3])
        sess = tf.Session()
        arg_scope = inception_resnet_v2_arg_scope()
        with slim.arg_scope(arg_scope):
            _, end_points = inception_resnet_v2(self.input_tensor, is_training=False)
        saver = tf.train.Saver()
        saver.restore(sess, checkpoint_file)
        self.end_points = end_points
        self.sess = sess
        print("RGB Model Load In Finished")

    def images2feature(self, images):
        ret = []
        for image in images:
            im = Image.open(image).resize((299, 299))
            im = np.array(im)
            im = im.reshape(-1, 299, 299, 3)
            im = 2 * (im / 255.0) - 1.0
            drop_out_values = self.sess.run(self.end_points['PreLogitsFlatten'], feed_dict={self.input_tensor: im})
            # print(drop_out_values.shape)
            ret.append(drop_out_values.squeeze())
        return np.array(ret)

    def test(self):
        sample_images = [DIR_NAME + 'dog.jpg']
        print(self.images2feature(sample_images).shape)

class FlowFeature(object):
    """"""

    def __init__(self ):
        """Constructor for FlowFeature"""

    def images2feature(self,images):
        return dense_optical_flow(images)

    def test(self):
        sample_images = []
        for i in range(1,6):
            sample_images.append(DIR_NAME+"flow_test/"+str(i)+".jpg")
        print(sample_images)
        print(self.images2feature(images=sample_images).shape)

class AudioFeature(object):
    """"""

    def __init__(self):
        """Constructor for AudioFeature"""
        checkpoint_path = DIR_NAME + 'checkpoints/vggish_model.ckpt'
        pca_params_path = DIR_NAME + 'checkpoints/vggish_pca_params.npz'
        pproc = vggish_postprocess.Postprocessor(pca_params_path)
        sess = tf.Session()
        vggish_slim.define_vggish_slim(training=False)
        vggish_slim.load_vggish_slim_checkpoint(sess, checkpoint_path)
        features_tensor = sess.graph.get_tensor_by_name(
            vggish_params.INPUT_TENSOR_NAME)
        embedding_tensor = sess.graph.get_tensor_by_name(
            vggish_params.OUTPUT_TENSOR_NAME)
        self.embedding_tensor = embedding_tensor
        self.features_tensor = features_tensor
        self.sess = sess
        self.pproc = pproc
        print("Audio Model Load In Finished")

    def wav2feature(self, wav_file):
        examples_batch = vggish_input.wavfile_to_examples(wav_file)
        [embedding_batch] = self.sess.run([self.embedding_tensor], feed_dict={self.features_tensor: examples_batch})
        postprocessed_batch = self.pproc.postprocess(embedding_batch)
        return postprocessed_batch

    def test(self):
        wav_file = DIR_NAME + "audio_test.wav"
        print(self.wav2feature(wav_file).shape)

if __name__ == '__main__':
    # rgb=RGBFeature()
    # rgb.test()
    audio = AudioFeature()
    audio.test()
    # flow=FlowFeature()
    # flow.test()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
@author: trivizakis

@github: github.com/trivizakis
"""
import os
import sys
import random
import pandas as pd
import numpy as np
import pickle as pkl

from sklearn.linear_model import LogisticRegression
from sklearn.feature_selection import SelectFromModel

from imblearn.over_sampling import SMOTE

from nsclc_objectives import objective_maker

from sklearn.preprocessing import robust_scale

from sklearn.model_selection import StratifiedKFold

from matplotlib import pyplot as plt

from sklearn.feature_selection import f_classif as fc
from sklearn.feature_selection import SelectKBest as kbest
from sklearn.feature_selection import VarianceThreshold as va

from numpy import interp
import math
import scipy.stats as st

from sklearn.metrics import roc_auc_score,roc_curve,confusion_matrix
from sklearn import svm, tree
from sklearn.neighbors import KNeighborsClassifier
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF, DotProduct, ExpSineSquared, Matern

from sksurv.ensemble import RandomSurvivalForest, ExtraSurvivalTrees
from sksurv.linear_model import CoxnetSurvivalAnalysis, CoxPHSurvivalAnalysis
from sksurv.tree import SurvivalTree
from sksurv.svm import NaiveSurvivalSVM, HingeLossSurvivalSVM, FastKernelSurvivalSVM, FastSurvivalSVM, MinlipSurvivalAnalysis

from sksurv.metrics import concordance_index_censored

def get_features(selected, df, selected_features):
    for key in list(df.index):
        selected[key] = df[selected_features].loc[key].to_numpy()
    return selected

def get_ml_model(classifier):
    if classifier == "Forest-surv":
        ml_model = RandomSurvivalForest(n_estimators=1000,
                                                       min_samples_split=6,
                                                       min_samples_leaf=3,
                                                       max_features="sqrt",
                                                       n_jobs=5)
    if classifier == "Surv-Tree":
        ml_model = SurvivalTree()
    elif classifier == "Extra-Tree":
        ml_model = ExtraSurvivalTrees()
    elif classifier == "Cox":
        ml_model = CoxnetSurvivalAnalysis(l1_ratio=0.99, fit_baseline_model=True)
    elif classifier == "CoxPH":
        ml_model = CoxnetSurvivalAnalysis(l1_ratio=0.99, fit_baseline_model=True)
    elif classifier == "Hinge-SVM":
        ml_model = HingeLossSurvivalSVM()
    elif classifier == "FastKernel-SVM":
        ml_model = FastKernelSurvivalSVM()
    elif classifier == "Fast-SVM":
        ml_model = FastSurvivalSVM()
    elif classifier == "Min-SVM":
        ml_model = MinlipSurvivalAnalysis()
    elif classifier == "Naive-SVM":
        ml_model = NaiveSurvivalSVM()
    return ml_model

def get_normalized_value(value,mean,std):
    value = (value-mean)/std
    if value>1:
        value=1
    elif value<-1:
        value=-1
    return value

def get_normalized_value_categorical(value):
    if value==0:
        return -1
    else:
        return value

def get_composite_score(feature_vector,coefs):
    score=0
    for index, feature in enumerate(feature_vector):
        score+=feature*coefs[index]
    return score

def get_composite_dict(dicts, coefs):
    #dicts: list of dictionaries (patterns)
    #coefs: list of coefs for each pattern
    score_dict={}

    for key in list(dicts[0].keys()):
        combined_score=0
        scores = []
        for index in range(0,len(dicts)):
            score = get_composite_score(dicts[index][key],coefs[index])
            scores.append(score)
            combined_score += score
        scores.append(combined_score)
        score_dict[key]= np.array(scores).reshape(-1,)
    return score_dict

def plot_survival(model, CI, path, X, y, clf):

    if clf == "Hinge-SVM" or "FastKernel-SVM" or "Fast-SVM" or "Min-SVM" or "Naive-SVM":
        prediction = model.predict(X)
        results = concordance_index_censored(y['Status'].astype(np.bool), y['Survival_in_days'].values, -prediction)
        print("CI: "+str(round(results[0], 3)))
        print("Concordant: "+str(results[1]))
        print("Discordant: "+str(results[2]))
        print("Tied_risk: "+str(results[3]))
        print("Tied_time: "+str(results[4]))

        file2write=open(path+"performance_CI",'w')
        file2write.write("CI: "+str(round(results[0], 3))+str("\n"))
        file2write.write("Concordant: "+str(results[1])+str("\n"))
        file2write.write("Discordant: "+str(results[2])+str("\n"))
        file2write.write("Tied_risk: "+str(results[3])+str("\n"))
        file2write.write("Tied_time: "+str(results[4])+str("\n"))
        file2write.close()
    else:
        if clf == "Forest-surv":
            surv = model.predict_survival_function(X, return_array=True)
            for i, s in enumerate(surv):
                plt.step(model.event_times_, s, where="post", label=str(i))
        elif clf == "Extra-Tree":
            surv = model.predict_survival_function(X, return_array=True)
            for i, s in enumerate(surv):
                plt.step(model.event_times_, s, where="post", label=str(i))
        elif clf == "Cox":
            surv = model.predict_survival_function(X)
            surv_funcs = {}
            for alpha in model.alphas_[:5]:
                surv_funcs[alpha] = model.predict_survival_function(X, alpha=alpha)#X.iloc[:1]
            for alpha, surv_alpha in surv_funcs.items():
                for fn in surv_alpha:
                    plt.step(fn.x, fn(fn.x), where="post",
                         label="alpha = {:.3f}".format(alpha))
        elif clf =="CoxPH":
            surv = model.predict_survival_function(X)
            surv_funcs = {}
            for alpha in model.alphas_[:5]:
                surv_funcs[alpha] = model.predict_survival_function(X, alpha=alpha)#X.iloc[:1]
            for alpha, surv_alpha in surv_funcs.items():
                for fn in surv_alpha:
                    plt.step(fn.x, fn(fn.x), where="post",
                         label="alpha = {:.3f}".format(alpha))
        elif clf =="Surv-Tree":
            surv = model.predict_survival_function(X, return_array=True)
            for i, s in enumerate(surv):
                plt.step(model.event_times_, s, where="post", label=str(i))
        plt.ylabel("Survival probability")
        plt.xlabel("Time in days")
        plt.legend(X.index)
        plt.title("CI %.4f" % CI)
        plt.grid(True)
        plt.savefig(path+"survival analysis.png", dpi=300, bbox_inches="tight")
        plt.clf()

def apply_feature_selection(df, labels, cutoff_pvalue=0.05, c_value=1):
    X=[]

    for key in list(df.index):
        X.append(df.loc[key])
    X = np.array(X)
    y = np.hstack(labels)

    remover = va(threshold=0.2)
    X = remover.fit_transform(X)
    # variance_mask = remover.get_support()
    variance_indices = remover.get_support(indices=True)
    # print(variance_indices)
    # print(np.unique(variance_mask,return_counts=True)) #indices=True

    f_scores, p_values = fc(X, y)
    critical_value = st.f.ppf(q=1-cutoff_pvalue, dfn=len(np.unique(y))-1, dfd=len(y)-len(np.unique(y)))

    best_indices=[]
    for index, p_value in enumerate(p_values):
        if f_scores[index]>=critical_value and p_value<cutoff_pvalue:
            best_indices.append(index)
    print("Best ANOVA features: "+str(len(best_indices)))

    df = df.iloc[:,variance_indices]
    # print(df.shape)
    best_columns = np.array(list(df.columns))[best_indices]
    best_features = np.array(list(df[best_columns].values))

    sel_ = SelectFromModel(LogisticRegression(C=c_value, penalty='l1', solver="liblinear"))
    # sel_ = SelectFromModel(LinearSVR(C=0.1))
    sel_.fit(best_features, y)
    selected_features_bool = sel_.get_support()
    coef = sel_.estimator_.coef_.reshape(-1,)

    final_selected=[]
    coefs = []
    if len(selected_features_bool)<1:
        final_selected.append(np.array(list(df.columns))[best_indices[0]])
        coefs.append(1)
    else:
        for index,feat_id in enumerate(best_columns):
            if selected_features_bool[index]:
                final_selected.append(feat_id)
                coefs.append(coef[index])
        print("Best l1 features: "+str(len(final_selected)))

    return np.array(final_selected),np.array(coefs)

#Parameters
hypes={}
hypes["dataset_dir"] = "path/to/dataset/"

#load transcriptomics
transcriptomics = pd.read_excel(hypes["dataset_dir"]+"RNAseq.xls")

#load radiomics
radiomics = pd.read_excel(hypes["dataset_dir"]+"radiomics.xls")

#load deep feratures
deep = pd.read_excel(hypes["dataset_dir"]+"deep_features_raw.xls")

#load clinical data
clinical=pd.read_excel(hypes["dataset_dir"]+"clinical_data.xls")
clinical_data=pd.read_excel(hypes["dataset_dir"]+"clinical_data_days.xls")

#load labels
labels=pd.read_excel(hypes["dataset_dir"]+"labels.xls")

# Analysis
exps = ["SURVIVAL-MEDIAN-DAYS"]#"SURVIVAL-OVERALL-DAYS"

results={}
r_results={}
d_results={}
t_results={}
c_results={}
tcs_results={}
rs_results={}
rts_results={}
ds_results={}
mvs_results={}
failed=[]
for classifier in ["Surv-Tree","Extra-Tree","CoxPH","Cox","Forest-surv","Hinge-SVM","FastKernel-SVM","Fast-SVM","Min-SVM","Naive-SVM"]:
    results_path="results_"+classifier
    os.mkdir("chkp/"+results_path)
    for model_name in list(deep.keys()):
        for experiment in exps:#survival type
            for expindex in range(0,1):
                splits={}
                pids = np.array(list(labels[experiment].keys()),dtype=str)
                # set_ = np.array(list(dataset_.values()))
                f_labels = np.array(list(labels[experiment].values()))
                sss = StratifiedKFold(n_splits=4, shuffle=True)
                kfolds=[]
                for train_index, test_index in sss.split(pids, f_labels):
                    kfolds.append([pids[train_index], pids[test_index]])
                splits[experiment] = kfolds

                Concordance_index = []
                CI_deep = []
                CI_radiomics = []
                CI_transcriptomics = []
                CI_clinical = []

                CI_transcriptomics_score=[]
                CI_radiomics_score=[]
                CI_radiotranscriptomics_score=[]
                CI_deep_score=[]
                CI_multiview_score=[]
                for index,split in enumerate(splits[experiment]):
                    # print("feature selection, deep: "+model_name+" n_feat: "+str(n_feat)+" clf: "+classifier+" exp: "+experiment)
                    g_tr_split=[]
                    g_tst_split=[]
                    for key in list(transcriptomics.index):
                        if key in list(split[0]):
                            g_tr_split.append(key)
                        elif key in list(split[1]):
                            g_tst_split.append(key)

                    r_tr_split=[]
                    r_tst_split=[]
                    for key in list(radiomics.index):
                        if key in list(split[0]):
                            r_tr_split.append(key)
                        elif key in list(split[1]):
                            r_tst_split.append(key)

                    #standardize training set
                    radiomics_tr = robust_scale(radiomics.loc[r_tr_split],unit_variance=True,quantile_range=(10.0, 90.0))
                    radiomics_tr[radiomics_tr>1]=1
                    radiomics_tr[radiomics_tr<-1]=-1
                    radiomics_tr = pd.DataFrame(data=radiomics_tr,index=radiomics.loc[r_tr_split].index, columns=radiomics.loc[r_tr_split].columns)
                    transcriptomics_tr = robust_scale(transcriptomics.loc[r_tr_split],unit_variance=True,quantile_range=(10.0, 90.0))
                    transcriptomics_tr[transcriptomics_tr>1]=1
                    transcriptomics_tr[transcriptomics_tr<-1]=-1
                    transcriptomics_tr = pd.DataFrame(data=transcriptomics_tr,index=transcriptomics.loc[r_tr_split].index, columns=transcriptomics.loc[r_tr_split].columns)
                    deep_tr = robust_scale(deep[model_name].loc[r_tr_split],unit_variance=True,quantile_range=(10.0, 90.0))
                    deep_tr[deep_tr>1]=1
                    deep_tr[deep_tr<-1]=-1

                    #standardize testing set
                    radiomics_tst = robust_scale(radiomics.loc[r_tst_split],unit_variance=True,quantile_range=(10.0, 90.0))
                    radiomics_tst[radiomics_tst>1]=1
                    radiomics_tst[radiomics_tst<-1]=-1
                    radiomics_tst = pd.DataFrame(data=radiomics_tst,index=radiomics.loc[r_tst_split].index, columns=radiomics.loc[r_tst_split].columns)
                    transcriptomics_tst = robust_scale(transcriptomics.loc[r_tst_split],unit_variance=True,quantile_range=(10.0, 90.0))
                    transcriptomics_tst[transcriptomics_tst>1]=1
                    transcriptomics_tst[transcriptomics_tst<-1]=-1
                    transcriptomics_tst = pd.DataFrame(data=transcriptomics_tst,index=transcriptomics.loc[r_tst_split].index, columns=transcriptomics.loc[r_tst_split].columns)
                    deep_tst = robust_scale(deep[model_name].loc[r_tst_split],unit_variance=True,quantile_range=(10.0, 90.0))
                    deep_tst[deep_tst>1]=1
                    deep_tst[deep_tst<-1]=-1

                    columns=["f"+str(s) for s in deep[model_name].columns]
                    deep_tr = pd.DataFrame(data=deep_tr,index=deep[model_name].loc[r_tr_split].index, columns=columns)
                    deep_tst = pd.DataFrame(data=deep_tst,index=deep[model_name].loc[r_tst_split].index, columns=columns)

                    #labels for single source experiments
                    g_labels=[]
                    for pid in list(transcriptomics.loc[g_tr_split].index):
                        try:
                            g_labels.append(labels[experiment][pid])
                        except:
                            continue

                    r_labels=[]
                    for pid in list(radiomics.loc[r_tr_split].index):
                        try:
                            r_labels.append(labels[experiment][pid])
                        except:
                            continue

                    #feature selection
                    print("-feature selection, deep: "+model_name+" clf: "+classifier+" exp: "+experiment)
                    try:
                        print("---Transcriptomics")
                        transcriptomics_feat, transcriptomics_coefs = apply_feature_selection(transcriptomics_tr, g_labels, cutoff_pvalue=0.005, c_value=0.6)
                        print("---Radiomics")
                        radiomic_feat, radiomic_coefs = apply_feature_selection(radiomics_tr, r_labels, cutoff_pvalue=0.05, c_value=0.8)
                        print("---Deep")
                        deep_feat, deep_coefs = apply_feature_selection(deep_tr, r_labels, cutoff_pvalue=0.05, c_value=0.6)
                        print("---Clinical")
                        clinic_feat, clinic_coefs = apply_feature_selection(clinical.loc[r_tr_split], r_labels, cutoff_pvalue=0.05, c_value=1)
                    except:
                        failed = experiment+"_"+model_name+"_"+"_"+classifier
                        continue

                    path = "chkp/"+results_path+"/"+experiment+"_"+model_name+"_"+classifier+"_nsp"+str(index+1)+"_expindex"+str(expindex)
                    os.mkdir(path)
                    np.save(path+"/selected_gen",np.array(transcriptomics_feat,dtype=str))
                    np.save(path+"/selected_rad",np.array(radiomic_feat,dtype=str))
                    np.save(path+"/selected_deep",np.array(deep_feat,dtype=str))
                    np.save(path+"/selected_clinical",np.array(clinic_feat,dtype=str))
                    np.save(path+"/selected_gen_coefs",np.array(transcriptomics_coefs,dtype=str))
                    np.save(path+"/selected_rad_coefs",np.array(radiomic_coefs,dtype=str))
                    np.save(path+"/selected_deep_coefs",np.array(deep_coefs,dtype=str))
                    np.save(path+"/selected_clinical_coefs",np.array(clinic_coefs,dtype=str))
                    np.save(path+"/tr_split",np.array(r_tr_split,dtype=str))
                    np.save(path+"/tst_split",np.array(r_tst_split,dtype=str))

                    selected_transcriptomics={}
                    selected_transcriptomics = get_features(selected_transcriptomics, r_tr_split, transcriptomics_feat)
                    selected_transcriptomics = get_features(selected_transcriptomics, r_tst_split, transcriptomics_feat)

                    selected_radiomics={}
                    selected_radiomics = get_features(selected_radiomics, r_tr_split, radiomic_feat)
                    selected_radiomics = get_features(selected_radiomics, r_tst_split, radiomic_feat)

                    selected_deep={}
                    selected_deep = get_features(selected_deep, r_tr_split, deep_feat)
                    selected_deep = get_features(selected_deep, r_tst_split, deep_feat)

                    selected_clinical={}
                    selected_clinical = get_features(selected_clinical, clinical, clinic_feat)

                    # align multi-view labels
                    rg_labels={}
                    rg_patterns={}
                    for key in sorted(selected_radiomics.keys()):
                        try:
                            rg_labels[key] = labels[experiment][key]
                            # rg_patterns[key]=combined_patterns[key]
                            rg_patterns[key] = np.concatenate((clinical[key],selected_transcriptomics[key],selected_radiomics[key],selected_deep[key]))
                        except:
                            # print(key+" not labeled!")
                            continue

                    r_labels={}
                    r_patterns={}
                    d_patterns={}
                    c_patterns={}
                    g_patterns={}
                    for key in sorted(rg_labels.keys()):
                        try:
                            r_labels[key]=labels[experiment][key]

                            g_patterns[key]=selected_transcriptomics[key]
                            r_patterns[key]=selected_radiomics[key]
                            d_patterns[key]=selected_deep[key]
                            c_patterns[key]=clinical[key]
                        except:
                            #print(key+" not labeled!")
                            continue

                    transcriptomics_score = get_composite_dict([g_patterns], [transcriptomics_coefs])
                    radiomics_score = get_composite_dict([r_patterns], [radiomic_coefs])
                    deep_score = get_composite_dict([d_patterns], [deep_coefs])
                    radiomics_scores = get_composite_dict([r_patterns,d_patterns], [radiomic_coefs,deep_coefs])
                    radiotranscriptomics_score = get_composite_dict([g_patterns,r_patterns,d_patterns], [transcriptomics_coefs, radiomic_coefs, deep_coefs])
                    multiview_score = get_composite_dict([g_patterns, r_patterns, d_patterns, c_patterns], [transcriptomics_coefs, radiomic_coefs, deep_coefs, clinic_coefs])

                    x_pd = pd.DataFrame.from_dict(rg_patterns, orient='index')
                    x_pd_deep = pd.DataFrame.from_dict(d_patterns, orient='index')
                    x_pd_clinical = pd.DataFrame.from_dict(c_patterns, orient='index')
                    x_pd_radiomics = pd.DataFrame.from_dict(r_patterns, orient='index')
                    x_pd_transcriptomics = pd.DataFrame.from_dict(g_patterns, orient='index')

                    x_pd_transcriptomics_score = pd.DataFrame.from_dict(transcriptomics_score, orient='index')
                    x_pd_radiomics_score = pd.DataFrame.from_dict(radiomics_score, orient='index')
                    x_pd_deep_score = pd.DataFrame.from_dict(deep_score, orient='index')
                    x_pd_radiotranscriptomics_score = pd.DataFrame.from_dict(radiotranscriptomics_score, orient='index')
                    x_pd_multiview_score = pd.DataFrame.from_dict(multiview_score, orient='index')

                    days_pd = pd.DataFrame.from_dict(clinical_data['SURVIVAL-MEDIAN-DAYS'], orient='index', columns=["Survival_in_days"])
                    status_pd = pd.DataFrame.from_dict(clinical_data['SURVIVAL-MEDIAN'], orient='index', columns=["Status"])

                    y_pd = status_pd.join(days_pd, sort=True)#, on="Patient", how="left")

                    X_train = x_pd.loc[r_tr_split]
                    X_test = x_pd.loc[r_tst_split]
                    X_train_deep = x_pd_deep.loc[r_tr_split]
                    X_test_deep = x_pd_deep.loc[r_tst_split]
                    X_train_clinical = x_pd_clinical.loc[r_tr_split]
                    X_test_clinical = x_pd_clinical.loc[r_tst_split]
                    X_train_radiomics = x_pd_radiomics.loc[r_tr_split]
                    X_test_radiomics = x_pd_radiomics.loc[r_tst_split]
                    X_train_transcriptomics = x_pd_transcriptomics.loc[r_tr_split]
                    X_test_transcriptomics = x_pd_transcriptomics.loc[r_tst_split]

                    X_train_transcriptomics_score = x_pd_transcriptomics_score.loc[r_tr_split]
                    X_test_transcriptomics_score = x_pd_transcriptomics_score.loc[r_tst_split]
                    X_train_radiomics_score = x_pd_radiomics_score.loc[r_tr_split]
                    X_test_radiomics_score = x_pd_radiomics_score.loc[r_tst_split]
                    X_train_radiotranscriptomics_score = x_pd_radiotranscriptomics_score.loc[r_tr_split]
                    X_test_radiotranscriptomics_score = x_pd_radiotranscriptomics_score.loc[r_tst_split]
                    X_train_deep_score = x_pd_deep_score.loc[r_tr_split]
                    X_test_deep_score = x_pd_deep_score.loc[r_tst_split]
                    X_train_multiview_score = x_pd_multiview_score.loc[r_tr_split]
                    X_test_multiview_score = x_pd_multiview_score.loc[r_tst_split]

                    y_train = y_pd.loc[r_tr_split]
                    y_test = y_pd.loc[r_tst_split]

                    y_train_np = y_train.to_numpy(dtype=np.float)
                    y_test_np = y_test.to_numpy(dtype=np.float)

                    print("classification: "+classifier)
                    struct_arr_train = y_train.astype({'Status':'?','Survival_in_days':'<f8'}).dtypes
                    y_train_np = np.array([tuple(x) for x in y_train.values], dtype=list(zip(struct_arr_train.index,struct_arr_train)))
                    struct_arr_test = y_test.astype({'Status':'?','Survival_in_days':'<f8'}).dtypes
                    y_test_np = np.array([tuple(x) for x in y_test.values], dtype=list(zip(struct_arr_test.index,struct_arr_test)))
                    # print(y_test_np)

                    ml_model = get_ml_model(classifier)
                    ml_model.fit(X_train, y_train_np)
                    ci = ml_model.score(X_test, y_test_np)
                    Concordance_index.append(ci)
                    # plot_survival(ml_model, ci, path+"/multi-view_", X_test, y_test, classifier)
                    pkl.dump(ml_model, open(path+"/multi-view_clr.pkl","wb"))

                    ml_model = get_ml_model(classifier)
                    ml_model.fit(X_train_deep, y_train_np)
                    ci = ml_model.score(X_test_deep, y_test_np)
                    CI_deep.append(ci)
                    # plot_survival(ml_model, ci, path+"/deep_", X_test_deep, y_test, classifier)
                    pkl.dump(ml_model, open(path+"/deep_clr.pkl","wb"))

                    ml_model = get_ml_model(classifier)
                    ml_model.fit(X_train_radiomics, y_train_np)
                    ci = ml_model.score(X_test_radiomics, y_test_np)
                    CI_radiomics.append(ci)
                    # plot_survival(ml_model, ci, path+"/radiomics_", X_test_radiomics, y_test, classifier)
                    pkl.dump(ml_model, open(path+"/radiomics_clr.pkl","wb"))

                    ml_model = get_ml_model(classifier)
                    ml_model.fit(X_train_transcriptomics, y_train_np)
                    ci = ml_model.score(X_test_transcriptomics, y_test_np)
                    CI_transcriptomics.append(ci)
                    # plot_survival(ml_model, ci, path+"/transcriptomics_", X_test_transcriptomics, y_test, classifier)
                    pkl.dump(ml_model, open(path+"/transcriptomics_clr.pkl","wb"))

                    ml_model = get_ml_model(classifier)
                    ml_model.fit(X_train_clinical, y_train_np)
                    ci =ml_model.score(X_test_clinical, y_test_np)
                    CI_clinical.append(ci)
                    # plot_survival(ml_model, ci, path+"/clinical_", X_test_clinical, y_test, classifier)
                    pkl.dump(ml_model, open(path+"/clinical_clr.pkl","wb"))

                    ml_model = get_ml_model(classifier)
                    ml_model.fit(X_train_transcriptomics_score, y_train_np)
                    ci =ml_model.score(X_test_transcriptomics_score, y_test_np)
                    CI_transcriptomics_score.append(ci)
                    # plot_survival(ml_model, ci, path+"/transcriptomics_score_", X_test_transcriptomics_score, y_test, classifier)
                    pkl.dump(ml_model, open(path+"/transcriptomics_score_clr.pkl","wb"))

                    ml_model = get_ml_model(classifier)
                    ml_model.fit(X_train_radiomics_score, y_train_np)
                    ci =ml_model.score(X_test_radiomics_score, y_test_np)
                    CI_radiomics_score.append(ci)
                    # plot_survival(ml_model, ci, path+"/radiomics_score_", X_test_radiomics_score, y_test, classifier)
                    pkl.dump(ml_model, open(path+"/radiomics_score_clr.pkl","wb"))

                    ml_model = get_ml_model(classifier)
                    ml_model.fit(X_train_radiotranscriptomics_score, y_train_np)
                    ci =ml_model.score(X_test_radiotranscriptomics_score, y_test_np)
                    CI_radiotranscriptomics_score.append(ci)
                    # plot_survival(ml_model, ci, path+"/radiotranscriptomics_score_", X_test_radiotranscriptomics_score, y_test, classifier)
                    pkl.dump(ml_model, open(path+"/radiotranscriptomics_score_clr.pkl","wb"))

                    ml_model = get_ml_model(classifier)
                    ml_model.fit(X_train_deep_score, y_train_np)
                    ci =ml_model.score(X_test_deep_score, y_test_np)
                    CI_deep_score.append(ci)
                    # plot_survival(ml_model, ci, path+"/deep_score_", X_test_deep_score, y_test, classifier)
                    pkl.dump(ml_model, open(path+"/deep_score_clr.pkl","wb"))

                    ml_model = get_ml_model(classifier)
                    ml_model.fit(X_train_multiview_score, y_train_np)
                    ci =ml_model.score(X_test_multiview_score, y_test_np)
                    CI_multiview_score.append(ci)
                    # plot_survival(ml_model, ci, path+"/multiview_score_", X_test_multiview_score, y_test, classifier)
                    pkl.dump(ml_model, open(path+"/multiview_score_clr.pkl","wb"))

                print("____Multi-view results____")
                print('List of possible CI:', Concordance_index)
                print('\nMaximum CI That can be obtained from this model is:',np.array(Concordance_index).max())
                print('\nMinimum CI:',np.array(Concordance_index).min())
                print('\nMean CI:',np.array(Concordance_index).mean())
                print('\nStandard Deviation is:', np.array(Concordance_index).std())

                results["multi-view: "+model_name+", clf:"+ classifier+" "+str(expindex)] = pd.Series({"Maximum":np.array(Concordance_index).max(),
                                                                          "Minimum CI":np.array(Concordance_index).min(),
                                                                          "Mean CI":np.array(Concordance_index).mean(),
                                                                          "Standard Deviation":np.array(Concordance_index).std()})

                d_results["deep: "+model_name+", clf:"+ classifier+" "+str(expindex)] = pd.Series({"Maximum":np.array(CI_deep).max(),
                                                                          "Minimum CI":np.array(CI_deep).min(),
                                                                          "Mean CI":np.array(CI_deep).mean(),
                                                                          "Standard Deviation":np.array(CI_deep).std()})

                r_results["radiomics: "+model_name+", clf:"+ classifier+" "+str(expindex)] = pd.Series({"Maximum":np.array(CI_radiomics).max(),
                                                                          "Minimum CI":np.array(CI_radiomics).min(),
                                                                          "Mean CI":np.array(CI_radiomics).mean(),
                                                                          "Standard Deviation":np.array(CI_radiomics).std()})

                t_results["transcriptomics: "+model_name+", clf:"+ classifier+" "+str(expindex)] = pd.Series({"Maximum":np.array(CI_transcriptomics).max(),
                                                                          "Minimum CI":np.array(CI_transcriptomics).min(),
                                                                          "Mean CI":np.array(CI_transcriptomics).mean(),
                                                                          "Standard Deviation":np.array(CI_transcriptomics).std()})

                c_results["clinical: "+model_name+", clf:"+ classifier+" "+str(expindex)] = pd.Series({"Maximum":np.array(CI_clinical).max(),
                                                                          "Minimum CI":np.array(CI_clinical).min(),
                                                                          "Mean CI":np.array(CI_clinical).mean(),
                                                                          "Standard Deviation":np.array(CI_clinical).std()})

                tcs_results["transcriptomics_score: "+model_name+", clf:"+ classifier+" "+str(expindex)] = pd.Series({"Maximum":np.array(CI_transcriptomics_score).max(),
                                                                          "Minimum CI":np.array(CI_transcriptomics_score).min(),
                                                                          "Mean CI":np.array(CI_transcriptomics_score).mean(),
                                                                          "Standard Deviation":np.array(CI_transcriptomics_score).std()})

                rs_results["radiomics_score: "+model_name+", clf:"+ classifier+" "+str(expindex)] = pd.Series({"Maximum":np.array(CI_radiomics_score).max(),
                                                                          "Minimum CI":np.array(CI_radiomics_score).min(),
                                                                          "Mean CI":np.array(CI_radiomics_score).mean(),
                                                                          "Standard Deviation":np.array(CI_radiomics_score).std()})

                rts_results["radiotranscriptomics_score: "+model_name+", clf:"+ classifier+" "+str(expindex)] = pd.Series({"Maximum":np.array(CI_radiotranscriptomics_score).max(),
                                                                          "Minimum CI":np.array(CI_radiotranscriptomics_score).min(),
                                                                          "Mean CI":np.array(CI_radiotranscriptomics_score).mean(),
                                                                          "Standard Deviation":np.array(CI_radiotranscriptomics_score).std()})

                ds_results["deep_score: "+model_name+", clf:"+ classifier+" "+str(expindex)] = pd.Series({"Maximum":np.array(CI_deep_score).max(),
                                                                          "Minimum CI":np.array(CI_deep_score).min(),
                                                                          "Mean CI":np.array(CI_deep_score).mean(),
                                                                          "Standard Deviation":np.array(CI_deep_score).std()})

                mvs_results["multi-view_score: "+model_name+", clf:"+ classifier+" "+str(expindex)] = pd.Series({"Maximum":np.array(CI_multiview_score).max(),
                                                                          "Minimum CI":np.array(CI_multiview_score).min(),
                                                                          "Mean CI":np.array(CI_multiview_score).mean(),
                                                                          "Standard Deviation":np.array(CI_multiview_score).std()})

multiview_results = pd.DataFrame.from_dict(results, orient="index")
r_results = pd.DataFrame.from_dict(r_results, orient="index")
d_results = pd.DataFrame.from_dict(d_results, orient="index")
t_results = pd.DataFrame.from_dict(t_results, orient="index")
c_results = pd.DataFrame.from_dict(c_results, orient="index")
tcs_results = pd.DataFrame.from_dict(tcs_results, orient="index")
rs_results = pd.DataFrame.from_dict(rs_results, orient="index")
rts_results = pd.DataFrame.from_dict(rts_results, orient="index")
ds_results = pd.DataFrame.from_dict(ds_results, orient="index")
mvs_results = pd.DataFrame.from_dict(mvs_results, orient="index")

multiview_results.to_csv("chkp/"+results_path+"/multi-view_results.csv")
r_results.to_csv("chkp/"+results_path+"/radiomics_results.csv")
d_results.to_csv("chkp/"+results_path+"/deep_results.csv")
t_results.to_csv("chkp/"+results_path+"/transcriptomics_results.csv")
c_results.to_csv("chkp/"+results_path+"/clinical_results.csv")
tcs_results.to_csv("chkp/"+results_path+"/tcs_results.csv")
rs_results.to_csv("chkp/"+results_path+"/rs_results.csv")
rts_results.to_csv("chkp/"+results_path+"/rts_results.csv")
ds_results.to_csv("chkp/"+results_path+"/ds_results.csv")
mvs_results.to_csv("chkp/"+results_path+"/mvs_results.csv")


from typing import List

class Solution:
    def kidsWithCandies(self, candies: List[int], extraCandies: int) -> List[bool]:
        max = -1
        isGreatest = []

        for each in candies:
            if each > max:
                max = each

        for each in candies:
            isGreater = False
            if (extraCandies + each >= max):
                isGreater = True
            isGreatest.append(isGreater)
        return isGreatest

if __name__ == "__main__":
    sol = Solution()
    print(f'The output is : {sol.kidsWithCandies([2,3,5,1,3], 3)}')

Data Structures and Algorithms/Python/LeetCode/Kids With the Greatest Number of Candies.py


try:
    from urllib.parse import urlencode
except ImportError:
    from urllib import urlencode

try:
    from urlparse import parse_qsl
except ImportError:
    from cgi import parse_qsl

from scheme.formats.structuredtext import StructuredText
from scheme.util import string

__all__ = ('UrlEncoded',)

class UrlEncoded(StructuredText):
    mimetype = 'application/x-www-form-urlencoded'
    name = 'urlencoded'

    @classmethod
    def serialize(cls, content):
        if not isinstance(content, dict):
            raise ValueError(content)

        data = []
        for name, value in content.items():
            data.append((name, cls._serialize_content(value)))

        return urlencode(data)

    @classmethod
    def unserialize(cls, content):
        if not isinstance(content, string):
            raise ValueError(content)

        data = {}
        for name, value in parse_qsl(content):
            if value[0] in ('{', '['):
                value = cls._unserialize_structured_value(value)
            else:
                value = cls._unserialize_simple_value(value)
            data[name] = value

        return data


from mappening.utils.secrets import FACEBOOK_SECRET_KEY
from mappening.api.utils import user_utils
from mappening.api.models.user import User
from mappening.api import users
from facebook import facebook_oauth

from flask import Flask, jsonify, redirect, url_for, session, request, Blueprint
from flask_login import UserMixin, LoginManager, login_required, current_user, login_user, logout_user
from flask_cors import CORS
from flask_oauth import OAuth
from datetime import datetime
import json

# Route Prefix: /auth
auth = Blueprint('auth', __name__)

# Got APP_ID and APP_SECRET from Mappening app with developers.facebook.com
DEBUG = True
auth.debug = DEBUG
auth.secret_key = FACEBOOK_SECRET_KEY

# Flask-Login - configure application for login
login_manager = LoginManager()

# Required to allow Blueprints to work with flask_login
# on_load is run when Blueprint is first registered to the app
# Enable session protection to prevent user sessions from being stolen
# By default is in "basic" mode. Can be set to "None or "strong"
@auth.record_once
def on_load(state):
    login_manager.init_app(state.app)
    # login_manager.session_protection = "strong"

    print("Login manager set up with auth Blueprint!")

# Can use user ID as remember token
# Must change user's ID to invalidate login sessions
# Can achieve this with session tokens rather than the user's ID
# Would modify get_id() method to return session token rather than user's ID
# Return None if ID is not valid, will remove ID from session and continue processing
@login_manager.user_loader
def user_loader(user_id):
    db_user = user_utils.get_user(user_id)
    if db_user:
        return User(db_user['account']['id'], db_user['account']['is_active'], db_user['account']['is_admin'])
    return None

# TODO: move this to users but don't require admin privileges?
@auth.route('/current', methods=['GET'])
def get_current_user():
    if not current_user.is_authenticated:
      return "No user is logged in!"

    user = user_utils.get_user(current_user.get_id())
    if user:
        return jsonify(user)
    return "Could not get current user!"

@auth.route('/')
def auth_redirect():
    if current_user.is_authenticated:
        # return redirect(url_for('main page route'))
        return "Already logged in!"
    return redirect(url_for('auth.login'))

@auth.route('/login')
def login():
    return facebook_oauth.authorize(
      callback=url_for('auth.facebook_authorized',
      next=request.args.get('next') or None, _external=True))

# Checks whether authentication works or access is denied
@auth.route('/login/authorized')
@facebook_oauth.authorized_handler
def facebook_authorized(resp):
    if resp is None:
        return "Access denied: reason=%s error=%s" % (
                request.args["error_reason"],
                request.args["error_description"]
            )
    session['oauth_token'] = (resp['access_token'], '')
    session['expires'] = resp['expires_in']
    print("Token expires in " + str(resp['expires_in']))

    # me = facebook_oauth.get("/me")
    # return str(me.data)

    me = facebook_oauth.get('/me?fields=id,name,first_name,last_name,email,picture')
    userID = me.data['id']
    userName = me.data['name']
    accessToken = resp['access_token']

    # If user exists in collection, logs them in
    # Otherwise, registers new user and logs them in
    # TODO get email if we can
    fb_user = user_utils.get_user(userID)
    if not fb_user:
        user_utils.add_user(userID, userName, me.data['first_name'], me.data['last_name'])
        user = User(userID)
        login_user(user)
        return "Successfully registered new user!"
    else:
        users.update_user(userID)
        user = User(userID, fb_user['account']['is_active'], fb_user['account']['is_admin'])
        login_user(user)
        return "Successfully logged in with Facebook!"

# Log out user.
# Will no longer be able to access any route decorated with @login_required
@auth.route('/logout')
@login_required
def logout():
    logout_user()
    return "Successfully logged out!"


from django.shortcuts import render
from .admin import Order
from.forms import OrderRegistrationForm

def register_orders(request):
    if request.method == "POST":
        form = OrderRegistrationForm(request.POST, request.FILES)
        if form.is_valid():
            form.save()
        else:
            print(form.errors)
    else:
        form = OrderRegistrationForm
    return render(request, "register_orders.html", {"form": form})

def order_list(request):
    orders = Order.objects.all()
    return render(request, "order_list.html", {"order": orders})
# Create your views here.

# Create your views here.


from __future__ import print_function
from PIL import Image, ImageGrab
import fitz
import time, sys
import win32com.client
"""
@created: 2017-06-05 18:00:00

@author: (c) 2017 Jorj X. McKie

Create a readable PDF copy of any (potentially DRM protected) pageable content
-------------------------------------------------------------------------------

Dependencies:
-------------
PyMuPDF v1.11.0+, PIL, SendKeys, AutoIt (external tool, separately installed),
operating system Windows (because of AutoIt).
Python version 2 (or 3 once the 'raw_input' function is replaced).

License:
--------
GNU GPL V3+

Description
------------
Page through the display of an e-book and store each page as a full page image
in a new PDF. The page image is created using ImageGrab of PIL.

The PIL module ImageGrab curently is only available on Mac and Windows.
However, any other product capable of scraping defined screen areas
would suffice as well (pyscreenshot on Linux? Not tested).

We also need a way to automatically page down our e-book reader.
This version uses AutoIt to do this in an elegant way, which also works
independantly of the used Python version. An alternative version of this script
uses the Python module SendKey instead.

"""
aut = win32com.client.Dispatch("AutoItX3.Control")    # AutoIt COM interface
# This is the rectangle copied. To be adjusted until it fits ...!
bbox = (70, 100, 850, 1008)            # the screen area to grab
width = (bbox[2] - bbox[0])/2.         # determines PDF page width
height = (bbox[3] - bbox[1])/2.        # ... and page height

# If you need to test your bbox image, answer "y"
answer = raw_input("need test picture? y/n ")
if answer == "y":
    print("you have 5 seconds to activate the e-book window ...")
    time.sleep(5)
    print("grabbing picture now")
    img = ImageGrab.grab(bbox = bbox)
    img.show()
    raise SystemExit

print("you have 5 seconds to activate the e-book window ...")
time.sleep(5)
print("starting process now")
print("Don't touch! The e-book window needs focus all the time!")

doc = fitz.open()                      # new empty PDF
old_samples = None                     # used to check end of e-book
i = 0
while 1:
    img = ImageGrab.grab(bbox = bbox)  # get the displayed e-book page
    samples = img.tobytes()            # copy for PyMuPDF
    if samples == old_samples:         # if no change: must be end of book
        print("end of book encountered ... finishing")
        break
    old_samples = samples              # store this page image
    pix = fitz.Pixmap(fitz.csRGB, img.size[0], img.size[1], samples, 0)
    doc.insertPage(-1, width = width, height = height)     # new PDF page
    page = doc[-1]                     # load it as a page object
    page.insertImage(page.rect, pixmap = pix)    # insert screen print as image
    aut.send("{PGDN}")                 # next page in e-book reader
    time.sleep(2)                      # allow for slow response time
    i += 1

# new PDF now contains page images as if scanned-in.
doc.save("ebook.pdf", deflate=True)     # save our PDF copy with compression
print(i, "pages saved.")