# Copyright 2020 - 2021 MONAI Consortium
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import itertools
import logging
import os
import platform
import shutil
import tempfile
import time
from datetime import timedelta
from distutils.util import strtobool
from typing import Callable, Dict, Optional, Sequence

import requests
import schedule
from dicomweb_client import DICOMwebClient
from dicomweb_client.session_utils import create_session_from_user_pass
from monai.apps import download_and_extract, download_url, load_from_mmar
from monai.data import partition_dataset
from timeloop import Timeloop

from monailabel.config import settings
from monailabel.datastore.dicom import DICOMWebDatastore
from monailabel.datastore.local import LocalDatastore
from monailabel.interfaces.datastore import Datastore, DefaultLabelTag
from monailabel.interfaces.exception import MONAILabelError, MONAILabelException
from monailabel.interfaces.tasks.batch_infer import BatchInferImageType, BatchInferTask
from monailabel.interfaces.tasks.infer import InferTask
from monailabel.interfaces.tasks.scoring import ScoringMethod
from monailabel.interfaces.tasks.strategy import Strategy
from monailabel.interfaces.tasks.train import TrainTask
from monailabel.tasks.activelearning.random import Random
from monailabel.tasks.infer.deepgrow_2d import InferDeepgrow2D
from monailabel.tasks.infer.deepgrow_3d import InferDeepgrow3D
from monailabel.tasks.infer.deepgrow_pipeline import InferDeepgrowPipeline
from monailabel.utils.async_tasks.task import AsyncTask
from monailabel.utils.sessions import Sessions

logger = logging.getLogger(__name__)

class MONAILabelApp:
    """
    Default Pre-trained Path for downloading models
    """

    PRE_TRAINED_PATH: str = "https://github.com/Project-MONAI/MONAILabel/releases/download/data/"

    def __init__(
        self,
        app_dir: str,
        studies: str,
        conf: Dict[str, str],
        name: str = "",
        description: str = "",
        version: str = "2.0",
        labels: Optional[Sequence[str]] = None,
    ):
        """
        Base Class for Any MONAI Label App

        :param app_dir: path for your App directory
        :param studies: path for studies/datalist
        :param conf: dictionary of key/value pairs provided by user while running the app

        """
        self.app_dir = app_dir
        self.studies = studies
        self.conf = conf if conf else {}

        self.name = name
        self.description = description
        self.version = version
        self.labels = labels

        self._datastore: Datastore = self.init_datastore()

        self._infers = self.init_infers()
        self._trainers = self.init_trainers()
        self._strategies = self.init_strategies()
        self._scoring_methods = self.init_scoring_methods()
        self._batch_infer = self.init_batch_infer()

        if strtobool(conf.get("download_tools", "true")):
            self._download_tools()
        self._server_mode = strtobool(conf.get("server_mode", "false"))
        self._auto_update_scoring = strtobool(conf.get("auto_update_scoring", "true"))
        self._sessions = self._load_sessions(strtobool(conf.get("sessions", "true")))

    def init_infers(self) -> Dict[str, InferTask]:
        return {}

    def init_trainers(self) -> Dict[str, TrainTask]:
        return {}

    def init_strategies(self) -> Dict[str, Strategy]:
        return {"random": Random()}

    def init_scoring_methods(self) -> Dict[str, ScoringMethod]:
        return {}

    def init_batch_infer(self) -> Callable:
        return BatchInferTask()

    def init_datastore(self) -> Datastore:
        logger.info(f"Init Datastore for: {self.studies}")
        if self.studies.startswith("http://") or self.studies.startswith("https://"):
            self.studies = self.studies.rstrip("/").strip()
            dw_session = None
            if settings.MONAI_LABEL_DICOMWEB_USERNAME and settings.MONAI_LABEL_DICOMWEB_PASSWORD:
                dw_session = create_session_from_user_pass(
                    settings.MONAI_LABEL_DICOMWEB_USERNAME, settings.MONAI_LABEL_DICOMWEB_PASSWORD
                )

            dw_client = DICOMwebClient(
                url=self.studies,
                session=dw_session,
                qido_url_prefix=settings.MONAI_LABEL_QIDO_PREFIX,
                wado_url_prefix=settings.MONAI_LABEL_WADO_PREFIX,
                stow_url_prefix=settings.MONAI_LABEL_STOW_PREFIX,
            )

            cache_path = settings.MONAI_LABEL_DICOMWEB_CACHE_PATH
            cache_path = cache_path.strip() if cache_path else ""
            return DICOMWebDatastore(dw_client, cache_path) if cache_path else DICOMWebDatastore(dw_client)

        return LocalDatastore(
            self.studies,
            extensions=settings.MONAI_LABEL_DATASTORE_FILE_EXT,
            auto_reload=settings.MONAI_LABEL_DATASTORE_AUTO_RELOAD,
        )

    def info(self):
        """
        Provide basic information about APP.  This information is passed to client.
        """
        meta = {
            "name": self.name,
            "description": self.description,
            "version": self.version,
            "labels": self.labels,
            "models": {k: v.info() for k, v in self._infers.items() if v.is_valid()},
            "trainers": {k: v.info() for k, v in self._trainers.items()},
            "strategies": {k: v.info() for k, v in self._strategies.items()},
            "scoring": {k: v.info() for k, v in self._scoring_methods.items()},
            "train_stats": {k: v.stats() for k, v in self._trainers.items()},
            "datastore": self._datastore.status(),
        }

        # If labels are not provided, aggregate from all individual infers
        if not self.labels:
            meta["labels"] = list(
                set(itertools.chain.from_iterable([v.get("labels", []) for v in meta["models"].values()]))
            )

        return meta

    def infer(self, request, datastore=None):
        """
        Run Inference for an exiting pre-trained model.

        Args:
            request: JSON object which contains `model`, `image`, `params` and `device`
            datastore: Datastore object.  If None then use default app level datastore to save labels if applicable

                For example::

                    {
                        "device": "cuda"
                        "model": "segmentation_spleen",
                        "image": "file://xyz",
                        "save_label": "true/false",
                        "label_tag": "original"
                    }

        Raises:
            MONAILabelException: When ``model`` is not found

        Returns:
            JSON containing `label` and `params`
        """
        model = request.get("model")
        if not model:
            raise MONAILabelException(
                MONAILabelError.INVALID_INPUT,
                "Model is not provided for Inference Task",
            )

        task = self._infers.get(model)
        if not task:
            raise MONAILabelException(
                MONAILabelError.INVALID_INPUT,
                f"Inference Task is not Initialized. There is no model '{model}' available",
            )

        request = copy.deepcopy(request)
        image_id = request["image"]
        datastore = datastore if datastore else self.datastore()
        if os.path.exists(image_id):
            request["save_label"] = False
        else:
            request["image"] = datastore.get_image_uri(request["image"])

        # TODO:: BUG In MONAI? Currently can not load DICOM through ITK Loader
        if os.path.isdir(request["image"]):
            logger.info("Input is a Directory; Consider it as DICOM")
            logger.info(os.listdir(request["image"]))
            request["image"] = [os.path.join(f, request["image"]) for f in os.listdir(request["image"])]

        logger.info(f"Image => {request['image']}")
        result_file_name, result_json = task(request)

        label_id = None
        if result_file_name and os.path.exists(result_file_name):
            tag = request.get("label_tag", DefaultLabelTag.ORIGINAL)
            save_label = request.get("save_label", True)
            if save_label:
                label_id = datastore.save_label(image_id, result_file_name, tag, result_json)
                if os.path.exists(result_file_name):
                    os.unlink(result_file_name)
            else:
                label_id = result_file_name

        return {"label": label_id, "tag": DefaultLabelTag.ORIGINAL, "params": result_json}

    def batch_infer(self, request, datastore=None):
        """
        Run batch inference for an existing pre-trained model.

        Args:
            request: JSON object which contains `model`, `params` and `device`
            datastore: Datastore object.  If None then use default app level datastore to fetch the images

                For example::

                    {
                        "device": "cuda"
                        "model": "segmentation_spleen",
                        "images": "unlabeled",
                        "label_tag": "original"
                    }

        Raises:
            MONAILabelException: When ``model`` is not found

        Returns:
            JSON containing `label` and `params`
        """
        return self._batch_infer(request, datastore if datastore else self.datastore(), self.infer)

    def scoring(self, request, datastore=None):
        """
        Run scoring task over labels.

        Args:
            request: JSON object which contains `model`, `params` and `device`
            datastore: Datastore object.  If None then use default app level datastore to fetch the images

                For example::

                    {
                        "device": "cuda"
                        "method": "dice",
                        "y": "final",
                        "y_pred": "original",
                    }

        Raises:
            MONAILabelException: When ``method`` is not found

        Returns:
            JSON containing result of scoring method
        """
        method = request.get("method")
        if not method:
            raise MONAILabelException(
                MONAILabelError.INVALID_INPUT,
                "Method is not provided for Scoring Task",
            )

        task = self._scoring_methods.get(method)
        if not task:
            raise MONAILabelException(
                MONAILabelError.INVALID_INPUT,
                f"Scoring Task is not Initialized. There is no such scoring method '{method}' available",
            )

        request = copy.deepcopy(request)
        return task(copy.deepcopy(request), datastore if datastore else self.datastore())

    def datastore(self) -> Datastore:
        return self._datastore

    @staticmethod
    def partition_datalist(datalist, val_split, shuffle=True):
        if val_split > 0.0:
            return partition_dataset(datalist, ratios=[(1 - val_split), val_split], shuffle=shuffle)
        return datalist, []

    def train(self, request):
        """
        Run Training.  User APP has to implement this method to run training

        Args:
            request: JSON object which contains train configs that are part APP info

                For example::

                    {
                        "mytrain": {
                            "device": "cuda"
                            "max_epochs": 1,
                            "amp": False,
                            "lr": 0.0001,
                        }
                    }

        Returns:
            JSON containing train stats
        """
        model = request.get("model")
        if not model:
            raise MONAILabelException(
                MONAILabelError.INVALID_INPUT,
                "Model is not provided for Training Task",
            )

        task = self._trainers.get(model)
        if not task:
            raise MONAILabelException(
                MONAILabelError.INVALID_INPUT,
                f"Train Task is not Initialized. There is no model '{model}' available",
            )

        request = copy.deepcopy(request)
        result = task(request, self.datastore())

        # Run all scoring methods
        if self._auto_update_scoring:
            self.async_scoring(None)
        return result

    def next_sample(self, request):
        """
        Run Active Learning selection.  User APP has to implement this method to provide next sample for labelling.

        Args:
            request: JSON object which contains active learning configs that are part APP info

                For example::

                    {
                        "strategy": "random"
                    }

        Returns:
            JSON containing next image info that is selected for labeling
        """
        strategy = request.get("strategy")
        strategy = strategy if strategy else "random"

        task = self._strategies.get(strategy)
        if task is None:
            raise MONAILabelException(
                MONAILabelError.APP_INIT_ERROR,
                f"ActiveLearning Task is not Initialized. There is no such strategy '{strategy}' available",
            )

        image_id = task(request, self.datastore())
        if not image_id:
            return {}

        image_path = self._datastore.get_image_uri(image_id)

        # Run all scoring methods
        if self._auto_update_scoring:
            self.async_scoring(None)

        return {
            "id": image_id,
            "path": image_path,
        }

    def on_init_complete(self):
        logger.info("App Init - completed")

        # Run all scoring methods
        if self._auto_update_scoring:
            self.async_scoring(None)

        # Run Cleanup Jobs
        def cleanup_sessions(instance):
            instance.cleanup_sessions()

        cleanup_sessions(self)
        time_loop = Timeloop()
        schedule.every(5).minutes.do(cleanup_sessions, self)

        @time_loop.job(interval=timedelta(seconds=30))
        def run_scheduler():
            schedule.run_pending()

        time_loop.start(block=False)

    def on_save_label(self, image_id, label_id):
        """
        Callback method when label is saved into datastore by a remote client
        """
        logger.info(f"New label saved for: {image_id} => {label_id}")

    # TODO :: Allow model files to be monitored and call this method when it is published (during training)
    # def on_model_published(self, model):
    #    pass

    def server_mode(self, mode: bool):
        self._server_mode = mode

    def async_scoring(self, method, params=None):
        if not method and not self._scoring_methods:
            return {}

        methods = [method] if method else list(self._scoring_methods.keys())
        result = {}
        for m in methods:
            if self._server_mode:
                request = {"method": m}
                request.update(params[m] if params and params.get(m) else {})
                res, _ = AsyncTask.run("scoring", request=request, params=params, enqueue=True)
                result[m] = res
            else:
                url = f"/scoring/{m}"
                p = params[m] if params and params.get(m) else None
                result[m] = self._local_request(url, p, "Scoring")
        return result[method] if method else result

    def async_training(self, model, params=None, enqueue=False):
        if not model and not self._trainers:
            return {}

        models = [model] if model else list(self._trainers.keys())
        enqueue = True if model > 1 else enqueue
        result = {}
        for m in models:
            if self._server_mode:
                request = {"model": m}
                request.update(params[m] if params and params.get(m) else {})
                res, _ = AsyncTask.run("train", request=request, params=params, enqueue=enqueue)
                result[m] = res
            else:
                url = f"/train/{model}?enqueue={enqueue}"
                p = params[m] if params and params.get(m) else None
                result[m] = self._local_request(url, p, "Training")
        return result[model] if model else result

    def async_batch_infer(self, model, images: BatchInferImageType, params=None):
        if self._server_mode:
            request = {"model": model, "images": images}
            res, _ = AsyncTask.run("batch_infer", request=request, params=params)
            return res

        url = f"/batch/infer/{model}?images={images}"
        return self._local_request(url, params, "Batch Infer")

    def _local_request(self, url, params, action):
        params = params if params else {}
        response = requests.post(f"http://127.0.0.1:{settings.MONAI_LABEL_SERVER_PORT}{url}", json=params)

        if response.status_code != 200:
            logger.error(f"Failed To Trigger {action}: {response.text}")
        return response.json() if response.status_code == 200 else None

    def _download_tools(self):
        target = os.path.join(self.app_dir, "bin")
        os.makedirs(target, exist_ok=True)

        dcmqi_tools = ["segimage2itkimage", "itkimage2segimage", "segimage2itkimage.exe", "itkimage2segimage.exe"]
        existing = [tool for tool in dcmqi_tools if shutil.which(tool) or os.path.exists(os.path.join(target, tool))]
        logger.debug(f"Existing Tools: {existing}")

        if len(existing) in [len(dcmqi_tools), len(dcmqi_tools) // 2]:
            logger.debug("No need to download dcmqi tools")
            return

        target_os = "win64.zip" if any(platform.win32_ver()) else "linux.tar.gz"
        with tempfile.TemporaryDirectory() as tmp:
            download_and_extract(
                url=f"https://github.com/QIICR/dcmqi/releases/download/v1.2.4/dcmqi-1.2.4-{target_os}", output_dir=tmp
            )
            for root, _, files in os.walk(tmp):
                for f in files:
                    if f in dcmqi_tools:
                        shutil.copy(os.path.join(root, f), target)

    def _load_sessions(self, load=False):
        if not load:
            return None
        return Sessions(settings.MONAI_LABEL_SESSION_PATH, settings.MONAI_LABEL_SESSION_EXPIRY)

    def cleanup_sessions(self):
        if not self._sessions:
            return
        count = self._sessions.remove_expired()
        logger.debug("Total sessions cleaned up: {}".format(count))

    def sessions(self):
        return self._sessions

    @staticmethod
    def download(resources):
        if not resources:
            return

        for resource in resources:
            if not os.path.exists(resource[0]):
                os.makedirs(os.path.dirname(resource[0]), exist_ok=True)
                logger.info(f"Downloading resource: {resource[0]} from {resource[1]}")
                download_url(resource[1], resource[0])
                time.sleep(1)

    @staticmethod
    def deepgrow_infer_tasks(model_dir, pipeline=True):
        """
        Dictionary of Default Infer Tasks for Deepgrow 2D/3D
        """
        deepgrow_2d = load_from_mmar("clara_pt_deepgrow_2d_annotation_1", model_dir)
        deepgrow_3d = load_from_mmar("clara_pt_deepgrow_3d_annotation_1", model_dir)

        infers = {
            "deepgrow_2d": InferDeepgrow2D(None, deepgrow_2d),
            "deepgrow_3d": InferDeepgrow3D(None, deepgrow_3d),
        }
        if pipeline:
            infers["deepgrow_pipeline"] = InferDeepgrowPipeline(
                path=None,
                network=deepgrow_2d,
                model_3d=infers["deepgrow_3d"],
                description="Combines Deepgrow 2D model and 3D deepgrow model",
            )
        return infers


##########
# This script is created by Huilei Xu, 4/2016
# Used to convert conventional SAM/BAM read alignment to Novoalign Native format for utilizing CIMS downstream analysis scripts
# Novoalign Native format:
#1. read header: >xx_xxx_xxx
#2. S/L/R: S means single ended, L means read from first file, R means read from second file
#3. read Sequence
#4. Base qualities
#5. U/R/QC/NM/QL: U means unique alignment, R means multiple alignments, QC means failure by QC, NM means no alignment, QL means failure by mapQ
#6. (assuming #5 is U/R) mapping score: Phred format
#7. (assuming #5 is U/R) mapping quality: Phred format
#8. aligned sequence: fasta header, truncated at first space
#9. aligned offset: 1-based position of the alignment in the sequence
#10. strand
#11. pair sequence
#12. pair offset
#13. pair strand
#14. mismatches
import pysam
import re
ifname = "/home/hxu/test/BrainD.Aligned.out.sam"
samfile = pysam.AlignmentFile(ifname,"r")
output = open(ifname.replace('sam','novoalign'),'w')
for read in samfile.fetch():
	if read.is_read1:
		pairFlag = 'L'
	elif read.is_read2:
		pairFlag = 'R'
	else:
		pairFlag = 'S'

	if read.is_reverse:
		strand = 'R'
	else:
		strand = 'F'

	if str(read.qual) == 'None':
		readqual = '.'
	else:
		readqual = read.qual

	mapn = read.get_tag('NH')
	mutations = []
	if mapn == 1:
		status = 'U'
		dels = []
		snps = []
		ins = []
		mutations = []
		### parsing CIGAR to get insertion base (MD not include insertion info)
		cigar = read.cigar
		idx_read = 0
		offset = 0
		read_clipped = ''
		for (code,num) in cigar:
			offset = idx_read + 1
			if code == 4:
				idx_read += num
				read_clipped += read.seq[idx_read:]
				continue
			elif code == 2: #del
				continue
			elif code == 1: #ins
				ins.append(str(offset)+"+"+read.seq[idx_read:idx_read+num])
				read_clipped += read.seq[idx_read:idx_read+num]
				idx_read += num
				continue
			read_clipped += read.seq[idx_read:idx_read+num]
			idx_read += num

		### parsing MD tag to get deletion and mismatch base (CIGAR not include these info)
		mdstr = read.get_tag('MD')
		mdSub = re.sub(r'([\\^]*[ACGT]+)[0]*', ' \\1 ', mdstr)
		mdSplit = re.split('[ ]+', mdSub)
		startpos = 0
		for md in mdSplit:
			if md.isdigit():
				startpos += int(md)
			elif md.startswith('^'):
				delpos = startpos
				for base in md[1:]:
					delpos +=1
					dels.append('%d-%s'%(delpos,base))
			elif md.isalpha():
				for base in md:
					startpos +=1
					snps.append('%d%s>%s'%(startpos,base,read_clipped[startpos-1]))

		mutations = dels + ins + snps

		if len(mutations)>0:
			output.write("\t".join(['@'+read.qname,pairFlag,read.seq,readqual,status,str(read.mapq),str(read.mapq),\
		'>'+str(read.reference_name),str(read.pos+1),strand,'.','.','.'," ".join(mutations)])+"\n")
		else:
			output.write("\t".join(['@'+read.qname,pairFlag,read.seq,readqual,status])+"\n")

	else:
		if  mapn > 1:
			status = 'R'
		elif mapn == 0:
			status = 'NM'
		output.write("\t".join(['@'+read.qname,pairFlag,read.seq,readqual,status])+"\n")

output.close()


import os
from os import path
import shutil

####### All you need is to create parent directory and put run the .sh file inside it.
# Parent_dir/
# ├── data
# ├── physionet-challenge-2020-12-lead-ecg-public
# │   ├── Training_2
# │   ├── Training_PTB
# │   ├── Training_StPetersburg
# │   ├── Training_WFDB
# │   └── WFDB

src = '/path/to/parent/dir/' # Change your directory from here.
dst = os.path.join(src,'data')
data_files = ['Training_StPetersburg', 'Training_WFDB', 'Training_PTB', 'Training_2', 'WFDB']

# create file if it doesn't exist.
if not os.path.exists(dst):
    os.makedirs(dst)

# move file from a dir to another
def move_files(list_f):
#### warning: I advise against using shutil.move, because in the case of failed run, you risk
#### to lose your subjects files, and you'll have to repeat the download process from the start.
    for f in list_f:
        print(f)
        shutil.copy(f, dst)

# want to move only the subject files and not any other files in the dir
def lambda_file(x):
    if x.endswith(tuple(['.mat', 'hea'])) :
        return os.path.join(src_, x)

##### this is a parallelized for loop. It will run this code on all your CPUs
##### independently for faster run time.

for data_file in data_files:
    # go to one file
    src_ = os.path.join(src, data_file)
    # get list of subjects' files
    list_of_files= list(map(lambda_file, os.listdir(src)))
# start the parallel process
    from multiprocessing import Pool, cpu_count
    n = round(len(list_of_files)/cpu_count())+1
    files= [list_of_files[i:i + n] for i in range(0, len(list_of_files), n)]

    with Pool(processes=cpu_count()) as pool:
        print('processing ..')
        res1 = pool.map(move_files, files)

    pool.close() # shut down the pool


from functools import lru_cache

class Solution:
    def change(self, amount: int, coins: List[int]) -> int:
        @lru_cache(maxsize=None)
        def count_change(n: int, i: int) -> int:
            nonlocal coins
            if n < 0: return 0
            if n == 0: return 1
            if i >= len(coins): return 0
            return count_change(n-coins[i], i) + \
                   count_change(n, i+1)
        return count_change(amount, 0)


"""

PyAlgoTrade
ib live broker

Requires:
- ibPy - https://github.com/blampe/IbPy
- trader work station or IB Gateway - https://www.interactivebrokers.com/en/?f=%2Fen%2Fsoftware%2Fibapi.php&ns=T

Disclaimer: No warranty express or implied is offered for this code

.. moduleauthor:: Kimble Young <kbcool@gmail.com>
"""

import threading
import time
import Queue
import datetime
import random

from pyalgotrade import broker
from pyalgotrade.strategy import position
from pyalgotrade.utils import dt

from ib.ext.Contract import Contract
from ib.ext.Order import Order
from ib.opt import ibConnection, message
from pdb import set_trace as bp

#build order object from IB API's definition of an open order which has a contract and order attribute
def build_order_from_open_order(openOrder, instrumentTraits):
    #order_id = openOrder.order.m_permId   #we use the TWS id for the order rather than our id - not sure this is a good idea but its apparently consistent across sessions - https://www.interactivebrokers.com/en/software/api/apiguide/java/order.htm
    order_id = openOrder.order.m_orderId
    #doesn't seem to be a useable date/time for orders so going to use current time
    order_time = dt.as_utc(datetime.datetime.now())

    order_type = openOrder.order.m_orderType     #stop, limit, stoplimit, market

    order_action = openOrder.order.m_action

    order_amount = openOrder.order.m_totalQuantity
    order_limprice = openOrder.order.m_lmtPrice
    order_auxprice = openOrder.order.m_auxPrice
    contract_symbol = openOrder.contract.m_symbol

    if order_action == 'BUY':
        action = broker.Order.Action.BUY
    elif order_action == 'SELL':
        action = broker.Order.Action.SELL
    elif order_action == 'SSHORT':
        action = broker.Order.Action.SELL_SHORT
    else:
        raise Exception("Invalid order action")

    if order_type == 'LMT':     #Limit
        ret = broker.LimitOrder(action, contract_symbol, order_limprice, order_amount, instrumentTraits)
    elif order_type == 'MKT':   #Market
        ret = broker.MarketOrder(action, contract_symbol, order_amount, False, instrumentTraits)
    elif order_type == 'MOC':   #Market On Close
        ret = broker.MarketOrder(action, contract_symbol, order_amount, True, instrumentTraits)
    elif order_type == 'STP':   #Stop order
        ret = broker.StopOrder(action, contract_symbol, order_auxprice, order_amount, instrumentTraits)
    elif order_type == 'STP LMT':
        ret = broker.StopLimitOrder(action, contract_symbol, order_auxprice, order_limprice, order_amount, instrumentTraits)
    else:
        #Totally possible if you use pyalgotrade and TWS to manage the same account which is not really a good idea
        raise Exception("Unsupported order type - %s" % order_type)

    ret.setSubmitted(order_id, order_time)
    ret.setState(broker.Order.State.ACCEPTED)
    return ret

#roundQuantity is the number of decimal places in an asset quantity - for stocks this can only be a whole number (at least in US/AU/UK etc) and IB also requires an integer so force it
class EquityTraits(broker.InstrumentTraits):
    def roundQuantity(self, quantity):
        return int(quantity)

    #price is to 2 decimal points  (US markets and ASX are two decimal places for stocks exceeding $1)
    def roundPrice(self, price):
        return round(price,2)

class LiveOrder(object):
    def __init__(self):
        self.__accepted = None

    def setAcceptedDateTime(self, dateTime):
        self.__accepted = dateTime

    def getAcceptedDateTime(self):
        return self.__accepted

    # Override to call the fill strategy using the concrete order type.
    # return FillInfo or None if the order should not be filled.
    def process(self, broker_, bar_):
        raise NotImplementedError()

class MarketOrder(broker.MarketOrder, LiveOrder):
    def __init__(self, action, instrument, quantity, onClose, instrumentTraits):
        broker.MarketOrder.__init__(self, action, instrument, quantity, onClose, instrumentTraits)
        LiveOrder.__init__(self)

    def process(self, broker_, bar_):
        return broker_.getFillStrategy().fillMarketOrder(broker_, self, bar_)

class LimitOrder(broker.LimitOrder, LiveOrder):
    def __init__(self, action, instrument, limitPrice, quantity, instrumentTraits):
        broker.LimitOrder.__init__(self, action, instrument, limitPrice, quantity, instrumentTraits)
        LiveOrder.__init__(self)

    def process(self, broker_, bar_):
        return broker_.getFillStrategy().fillLimitOrder(broker_, self, bar_)

class StopOrder(broker.StopOrder, LiveOrder):
    def __init__(self, action, instrument, stopPrice, quantity, instrumentTraits):
        broker.StopOrder.__init__(self, action, instrument, stopPrice, quantity, instrumentTraits)
        LiveOrder.__init__(self)
        self.__stopHit = False

    def process(self, broker_, bar_):
        return broker_.getFillStrategy().fillStopOrder(broker_, self, bar_)

    def setStopHit(self, stopHit):
        self.__stopHit = stopHit

    def getStopHit(self):
        return self.__stopHit

# http://www.sec.gov/answers/stoplim.htm
# http://www.interactivebrokers.com/en/trading/orders/stopLimit.php
class StopLimitOrder(broker.StopLimitOrder, LiveOrder):
    def __init__(self, action, instrument, stopPrice, limitPrice, quantity, instrumentTraits):
        broker.StopLimitOrder.__init__(self, action, instrument, stopPrice, limitPrice, quantity, instrumentTraits)
        LiveOrder.__init__(self)
        self.__stopHit = False  # Set to true when the limit order is activated (stop price is hit)

    def setStopHit(self, stopHit):
        self.__stopHit = stopHit

    def getStopHit(self):
        return self.__stopHit

    def isLimitOrderActive(self):
        # TODO: Deprecated since v0.15. Use getStopHit instead.
        return self.__stopHit

    def process(self, broker_, bar_):
        return broker_.getFillStrategy().fillStopLimitOrder(broker_, self, bar_)

class LiveBroker(broker.Broker):
    """An IB live broker.

    :param host: host to connect to default localhost
    :param host: hostname running your IB API - usually localhost
    :type host: string.
    :param port: port of server running your IB - usually 7496
    :type port: int.
    :param marketOptions: configure asset type, currency and routing - see https://www.interactivebrokers.com/en/software/api/apiguide/java/contract.htm
    :type marketOptions: dict.
    :param debug: have ibPy spit out all messages to screen (very noisy)
    :type debug: bool.
    :param clientId: client id to use - set this to an integer between 1 and 999 (reserved) if you want to be able modify an order that was submitted on a previous session
    :type clientId: int.

    .. note::
        * Must have read/write access - go into TWS and enable
        * No stop losses, hedging etc - very simple right now
    """

    def __init__(self, host="localhost", port=7496, marketOptions={'assetType':'STK', 'currency':'GBP','routing': 'SMART'}, debug=False, clientId = None):
        broker.Broker.__init__(self)

        if debug:
            self.__debug = True
        else:
            self.__debug = False

        self.__stop = False

        if clientId == None:
            clientId = random.randint(1000,10000)

        self.__ib = ibConnection(host=host,port=port,clientId=clientId)
        #self.__ib = ibConnection(host=host,port=port)

        #register all the callback handlers
        self.__ib.registerAll(self.__debugHandler)
        self.__ib.register(self.__accountHandler,'UpdateAccountValue')
        self.__ib.register(self.__portfolioHandler,'UpdatePortfolio')
        self.__ib.register(self.__openOrderHandler, 'OpenOrder')
        #self.__ib.register(self.__positionHandler, 'Position')
        self.__ib.register(self.__disconnectHandler,'ConnectionClosed')
        self.__ib.register(self.__nextIdHandler,'NextValidId')
        self.__ib.register(self.__orderStatusHandler,'OrderStatus')

        self.__ib.connect()

        self.__cash = 0
        self.__shares = {}
        self.__detailedShares = {}
        self.__activeOrders = {}
        self.__nextOrderId = 0
        self.__initialPositions = []

        #parse marketoptions and set defaults
        self.__marketOptions = {}

        if marketOptions.get('assetType') == None:
            self.__marketOptions['assetType'] = 'STK'
        else:
            self.__marketOptions['assetType'] = marketOptions['assetType']

        if marketOptions.get('currency') == None:
            self.__marketOptions['currency'] = 'GBP'
        else:
            self.__marketOptions['currency'] = marketOptions['currency']

        if marketOptions.get('routing') == None:
            self.__marketOptions['routing'] = 'SMART'
        else:
            self.__marketOptions['routing'] = marketOptions['routing']

        self.refreshAccountBalance()
        self.refreshOpenOrders()

        self.__ib.reqPositions()

        #give ib time to get back to us
        time.sleep(2)

    def __disconnectHandler(self,msg):
        self.__ib.reconnect()

    #prints all messages from IB API
    def __debugHandler(self,msg):
        if self.__debug: print(msg)

    def __nextIdHandler(self,msg):
        self.__nextOrderId = msg.orderId

    '''
    #build position array from ib object (NOTE: This isn't a pyalgotrade position it's an array with enough details hopefully to build one)
    def build_position_from_open_position(self,msg):
        #return pyalgotrade.strategy.position.LongPosition(self., instrument, stopPrice, None, quantity, goodTillCanceled, allOrNone)
        return {
            'stock': 'STW',
            'shortLong': 'long',
            'quantity': 500,
            'price': 31.63
        }
        pass

    #creates positions and hopefully tells the strategy on startup
    #great for error recovery
    def __positionHandler(self,msg):
        self.__initialPositions.append(self.build_position_from_open_position(msg))
        print "GOT POSITIONS"

    def getInitialPositions(self):
        return self.__initialPositions

    '''

    #listen for orders to be fulfilled or cancelled
    def __orderStatusHandler(self,msg):
        order = self.__activeOrders.get(msg.orderId)
        if order == None:
            return

        #watch out for dupes - don't submit state changes or events if they were already submitted

        eventType = None
        if msg.status == 'Filled' and order.getState() != broker.Order.State.FILLED:
            eventType = broker.OrderEvent.Type.FILLED
            self._unregisterOrder(order)
            #order.setState(broker.Order.State.FILLED)
        if msg.status == 'Submitted' and msg.filled > 0:
            eventType = broker.OrderEvent.Type.PARTIALLY_FILLED
            #may already be partially filled
            #if order.getState() != broker.Order.State.PARTIALLY_FILLED:
            #    order.setState(broker.Order.State.PARTIALLY_FILLED)
        if msg.status == 'Cancelled' and order.getState() != broker.Order.State.CANCELED:
            #self._unregisterOrder(order)
            eventType = broker.OrderEvent.Type.CANCELED
            #order.setState(broker.Order.State.CANCELED)
            self._unregisterOrder(order)
            order.switchState(broker.Order.State.CANCELED)

            # Notify that the order was canceled.
            self.notifyOrderEvent(broker.OrderEvent(order, broker.OrderEvent.Type.CANCELED, "User requested cancellation"))

        orderExecutionInfo = None
        if eventType == broker.OrderEvent.Type.FILLED or eventType == broker.OrderEvent.Type.PARTIALLY_FILLED:
            orderExecutionInfo = broker.OrderExecutionInfo(msg.avgFillPrice, abs(msg.filled), 0, datetime.datetime.now())

            order.addExecutionInfo(orderExecutionInfo)

            if order.isFilled():
                #self._unregisterOrder(order)
                self.notifyOrderEvent(broker.OrderEvent(order, broker.OrderEvent.Type.FILLED, orderExecutionInfo))
            elif order.isPartiallyFilled():
                self.notifyOrderEvent(
                    broker.OrderEvent(order, broker.OrderEvent.Type.PARTIALLY_FILLED, orderExecutionInfo)
                )

    #get account messages like cash value etc
    def __accountHandler(self,msg):
        #FYI this is not necessarily USD - probably AUD for me as it's the base currency so if you're buying international stocks need to keep this in mind
        #self.__cash = round(balance.getUSDAvailable(), 2)
        if msg.key == 'TotalCashBalance' and msg.currency == 'USD':
            self.__cash = round(float(msg.value))

    #get portfolio messages - stock, price, purchase price etc
    def __portfolioHandler(self,msg):
        self.__shares[msg.contract.m_symbol] = msg.position

        self.__detailedShares[msg.contract.m_symbol] = {    'shares': msg.position,             #number of units
                                                            'marketPrice': msg.marketPrice,     #current price on market
                                                            'entryPrice': msg.averageCost,      #cost per unit at acquistion (unfortunately minus commissions)
                                                            'PL': msg.unrealizedPNL             #unrealised profit and loss
                                                        }

    def __openOrderHandler(self,msg):
        #Do nothing now but might want to use this to pick up open orders at start (eg in case of shutdown or crash)
        #note if you want to test this make sure you actually have an open order otherwise it's never called
        #Remember this is called once per open order so if you have 3 open orders it's called 3 times

        self._registerOrder(build_order_from_open_order(msg, self.getInstrumentTraits(msg.contract.m_symbol)))

    def _registerOrder(self, order):

        assert(order.getId() is not None)

        #need to make sure order doesn't overwrite as we may lose information
        if order.getId() not in self.__activeOrders:
            self.__activeOrders[order.getId()] = order

    def _unregisterOrder(self, order):
        assert(order.getId() in self.__activeOrders)
        assert(order.getId() is not None)
        del self.__activeOrders[order.getId()]

    #subscribes for regular account balances which are sent to portfolio and account handlers
    def refreshAccountBalance(self):
        self.__ib.reqAccountUpdates(1,'')

    def refreshOpenOrders(self):
        self.__ib.reqAllOpenOrders()

    def _startTradeMonitor(self):
        return

    # BEGIN observer.Subject interface
    def start(self):
        return

    def stop(self):
        self.__stop = True
        self.__ib.disconnect()

    def join(self):
        pass

    def eof(self):
        return self.__stop

    def dispatch(self):
        # Switch orders from SUBMITTED to ACCEPTED.
        ordersToProcess = self.__activeOrders.values()
        for order in ordersToProcess:
            if order.isSubmitted():
                order.switchState(broker.Order.State.ACCEPTED)
                self.notifyOrderEvent(broker.OrderEvent(order, broker.OrderEvent.Type.ACCEPTED, None))

    def peekDateTime(self):
        # Return None since this is a realtime subject.
        return None

    # END observer.Subject interface

    # BEGIN broker.Broker interface

    def getCash(self, includeShort=True):
        return self.__cash

    def getInstrumentTraits(self, instrument):
        return EquityTraits()

    def getShares(self, instrument):
        return self.__shares.get(instrument, 0)

    def getPositions(self):
        return self.__shares

    #positions is just stock and number of shares - detailed positions includes cost and p/l info
    def getDetailedPositions(self):
        return self.__detailedShares

    def getActiveOrders(self, instrument=None):
        return self.__activeOrders.values()

    def submitOrder(self, order):
        if order.isInitial():

            ibContract = Contract()
            ibOrder = Order()

            ibContract.m_symbol = order.getInstrument()

            ibContract.m_secType = self.__marketOptions['assetType']
            ibContract.m_currency = self.__marketOptions['currency']
            ibContract.m_exchange = self.__marketOptions['routing']

            ibOrder.m_totalQuantity = order.getInstrumentTraits().roundQuantity(order.getQuantity())
            if order.getAction() == (broker.Order.Action.BUY or broker.Order.Action.BUY_TO_COVER):
                ibOrder.m_action = 'BUY'
            elif order.getAction() == broker.Order.Action.SELL:
                ibOrder.m_action = 'SELL'
            elif order.getAction() == broker.Order.Action.SELL_SHORT:
                ibOrder.m_action = 'SELL'

            if order.getType() == broker.Order.Type.MARKET:
                if order.getFillOnClose():
                    ibOrder.m_orderType = 'MOC'
                else:
                    ibOrder.m_orderType = 'MKT'
            elif order.getType() == broker.Order.Type.LIMIT:
                ibOrder.m_orderType = 'LMT'
                ibOrder.m_lmtPrice = order.getInstrumentTraits().roundPrice(order.getLimitPrice())
            elif order.getType() == broker.Order.Type.STOP:
                ibOrder.m_orderType = 'STP'
                ibOrder.m_auxPrice = order.getInstrumentTraits().roundPrice(order.getStopPrice())
            elif order.getType() == broker.Order.Type.STOP_LIMIT:
                ibOrder.m_orderType = 'STP LMT'
                ibOrder.m_lmtPrice = order.getInstrumentTraits().roundPrice(order.getLimitPrice())
                ibOrder.m_auxPrice = order.getInstrumentTraits().roundPrice(order.getStopPrice())

            if order.getAllOrNone() == True:
                ibOrder.m_allOrNone = 1
            else:
                ibOrder.m_allOrNone = 0

            if order.getGoodTillCanceled() == True:
                ibOrder.m_tif = 'GTC'
            else:
                ibOrder.m_tif = 'DAY'

            self.__ib.placeOrder(self.__nextOrderId, ibContract, ibOrder)

            order.setSubmitted(self.__nextOrderId, datetime.datetime.now())

            self.__nextOrderId += 1

            self._registerOrder(order)
            # Switch from INITIAL -> SUBMITTED
            # IMPORTANT: Do not emit an event for this switch because when using the position interface
            # the order is not yet mapped to the position and Position.onOrderUpdated will get called.
            order.switchState(broker.Order.State.SUBMITTED)
        else:
            raise Exception("The order was already processed")

    def createMarketOrder(self, action, instrument, quantity, onClose=False):
        #IB doesn't support buy to cover
        if action == broker.Order.Action.BUY_TO_COVER:
            action = broker.Order.Action.BUY

        instrumentTraits = self.getInstrumentTraits(instrument)

        return broker.MarketOrder(action, instrument, quantity, onClose, instrumentTraits)

    def createLimitOrder(self, action, instrument, limitPrice, quantity):
        #IB doesn't support buy to cover
        if action == broker.Order.Action.BUY_TO_COVER:
            action = broker.Order.Action.BUY

        instrumentTraits = self.getInstrumentTraits(instrument)

        return broker.LimitOrder(action, instrument, limitPrice, quantity, instrumentTraits)

    def createStopOrder(self, action, instrument, stopPrice, quantity):
        #IB doesn't support buy to cover
        if action == broker.Order.Action.BUY_TO_COVER:
            action = broker.Order.Action.BUY

        instrumentTraits = self.getInstrumentTraits(instrument)

        return broker.StopOrder(action, instrument, stopPrice, quantity, instrumentTraits)

    def createStopLimitOrder(self, action, instrument, stopPrice, limitPrice, quantity):
        #IB doesn't support buy to cover
        if action == broker.Order.Action.BUY_TO_COVER:
            action = broker.Order.Action.BUY

        instrumentTraits = self.getInstrumentTraits(instrument)

        return broker.StopLimitOrder(action, instrument, stopPrice,limitPrice, quantity, instrumentTraits)

    def cancelOrder(self, order):
        activeOrder = self.__activeOrders.get(order.getId())
        if activeOrder is None:
            raise Exception("The order is not active anymore")
        if activeOrder.isFilled():
            raise Exception("Can't cancel order that has already been filled")

        self.__ib.cancelOrder(order.getId())

        #DO NOT DO THE BELOW:
        '''
        self._unregisterOrder(order)
        order.switchState(broker.Order.State.CANCELED)

        # Update cash and shares. - might not be needed
        self.refreshAccountBalance()

        # Notify that the order was canceled.
        self.notifyOrderEvent(broker.OrderEvent(order, broker.OrderEvent.Type.CANCELED, "User requested cancellation"))
        '''

    # END broker.Broker interface


import matplotlib
matplotlib.pyplot.switch_backend('Agg')

import numpy as np
import pandas as pd
import pylab as plt
import pymzml
import math
import seaborn as sns

import sys
sys.path.append('../..')

from vimms.Roi import RoiToChemicalCreator, make_roi
from vimms.DataGenerator import DataSource, PeakSampler, get_spectral_feature_database
from vimms.MassSpec import IndependentMassSpectrometer
from vimms.Controller import TopNController
from vimms.TopNExperiment import get_params, run_serial_experiment, run_parallel_experiment
from vimms.PlotsForPaper import get_df, load_controller, compute_performance_scenario_2
from vimms.Common import *

def varying_topn_processor():
    pathlist = []
    base_dir = 'documents/simple_ms1/example_data'
    mzml_path = os.path.join(base_dir, 'beers', 'fragmentation', 'mzML')
    file_name = 'Beer_multibeers_1_T10_POS.mzML'

    experiment_name = 'beer1pos'
    url_experiment_out_dir = os.path.join(base_dir, 'results', experiment_name, 'mzML')
    experiment_out_dir = os.path.abspath(os.path.join(base_dir, 'results', experiment_name, 'mzML'))
    min_rt = 3*60 # start time when compounds begin to elute in the mzML file
    max_rt = 21*60
    kde_min_ms1_intensity = 0 # min intensity to be selected for kdes
    kde_min_ms2_intensity = 0

    roi_mz_tol = 10
    roi_min_length = 1
    roi_min_intensity = 0
    roi_start_rt = min_rt
    roi_stop_rt = max_rt

    isolation_window = 1   # the isolation window in Dalton around a selected precursor ion
    ionisation_mode = POSITIVE
    N = 10
    rt_tol = 15
    mz_tol = 10
    min_ms1_intensity = 1.75E5 # minimum ms1 intensity to fragment

    mzml_out = os.path.join(experiment_out_dir, 'simulated.mzML')
    print('#'*10, 'Train densities')
    ds = DataSource()
    ds.load_data(mzml_path, file_name=file_name)
    bandwidth_mz_intensity_rt=1.0
    bandwidth_n_peaks=1.0
    ps = get_spectral_feature_database(ds, file_name, kde_min_ms1_intensity, kde_min_ms2_intensity, min_rt, max_rt,
                   bandwidth_mz_intensity_rt, bandwidth_n_peaks)
    print('#'*10, 'Extract all ROIs')
    mzml_file = os.path.join(mzml_path, file_name)
    good_roi, junk = make_roi(mzml_file, mz_tol=roi_mz_tol, mz_units='ppm', min_length=roi_min_length,
                              min_intensity=roi_min_intensity, start_rt=roi_start_rt, stop_rt=roi_stop_rt)
    all_roi = good_roi + junk
    print('#'*10, 'How many singleton and non-singleton ROIs =>', len([roi for roi in all_roi if roi.n == 1]))

    keep = []
    for roi in all_roi:
        if np.count_nonzero(np.array(roi.intensity_list) > min_ms1_intensity) > 0:
            keep.append(roi)

    all_roi = keep
    set_log_level_debug()
    rtcc = RoiToChemicalCreator(ps, all_roi)
    data = rtcc.chemicals
    save_obj(data, os.path.join(experiment_out_dir, 'dataset.p'))
    print('#'*10, 'Run Top-N Controller')
    set_log_level_warning()
    pbar = False # turn off progress bar
    Ns = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100]
    rt_tols = [15]
    params = get_params(experiment_name, Ns, rt_tols, mz_tol, isolation_window, ionisation_mode, data, ps,
                        min_ms1_intensity, min_rt, max_rt, experiment_out_dir, pbar)
    run_serial_experiment(params)

    print('#'*10, 'Analyse Results')
    min_ms1_intensity = 0
    rt_range = [(min_rt, max_rt)]
    mz_range = [(0, math.inf)]
    results_dir = os.path.join(base_dir, 'results', 'ground_truth', 'mzML')
    csv_file = os.path.join(results_dir, 'extracted_peaks_ms1.csv')
    P_peaks_df = get_df(csv_file, min_ms1_intensity, rt_range, mz_range)

    csv_file = os.path.join(experiment_out_dir, 'extracted_peaks_ms1.csv')
    Q_peaks_df = get_df(csv_file, min_ms1_intensity, rt_range, mz_range)

    fullscan_filename = 'Beer_multibeers_1_fullscan1.mzML'
    matching_mz_tol = 10 # ppm
    matching_rt_tol = 30 # seconds

    results = []
    for N in Ns:
        for rt_tol in rt_tols:

            # load chemicals and check for matching
            chemicals = load_obj(os.path.join(experiment_out_dir, 'dataset.p'))
            fragfile_filename = 'experiment_%s_N_%d_rttol_%d.mzML' % (experiment_name, N, rt_tol)

            # load controller and compute performance
            controller = load_controller(experiment_out_dir, experiment_name, N, rt_tol)
            mytemp = os.path.join(url_experiment_out_dir, fragfile_filename)
            pathlist.append(mytemp)

            if controller is not None:
                tp, fp, fn, prec, rec, f1 = compute_performance_scenario_2(controller, chemicals, min_ms1_intensity,
                                                                           fullscan_filename, fragfile_filename,
                                                                           P_peaks_df, Q_peaks_df, matching_mz_tol, matching_rt_tol)
                print('%s N=%d rt_tol=%d tp=%d fp=%d fn=%d prec=%.3f rec=%.3f f1=%.3f' % (experiment_name,
                    N, rt_tol, tp, fp, fn, prec, rec, f1))
                res = (experiment_name, N, rt_tol, tp, fp, fn, prec, rec, f1)
                results.append(res)
    result_df = pd.DataFrame(results, columns=['experiment', 'N', 'rt_tol', 'TP', 'FP', 'FN', 'Prec', 'Rec', 'F1'])

    plt.figure(figsize=(12, 6))
    ax = sns.lineplot(x='N', y='Prec', hue='experiment', legend='brief', data=result_df)
    plt.title('Top-N Precision')
    for l in ax.lines:
        plt.setp(l, linewidth=5)
    plt.ylabel('Precision')
    plt.xlabel(r'Top-$N$')
    plt.legend(prop={'size': 20})
    plt.tight_layout()

    fig_out = os.path.join(experiment_out_dir, 'topN_precision.png')
    plt.savefig(fig_out, dpi=300)

    plt.figure(figsize=(12, 6))
    ax = sns.lineplot(x='N', y='Rec', hue='experiment', legend='brief', data=result_df)
    plt.title('Top-N Recall')
    for l in ax.lines:
        plt.setp(l, linewidth=5)
    plt.ylabel('Recall')
    plt.xlabel(r'Top-$N$')
    plt.legend(prop={'size': 20})
    plt.tight_layout()

    fig_out = os.path.join(experiment_out_dir, 'topN_recall.png')

    plt.figure(figsize=(12, 6))
    ax = sns.lineplot(x='N', y='F1', hue='experiment', legend='brief', data=result_df)
    plt.title('Top-N F1')
    for l in ax.lines:
        plt.setp(l, linewidth=5)
    plt.ylabel(r'$F_{1}\;score$')
    plt.xlabel(r'Top-$N$')
    plt.legend(prop={'size': 20})
    plt.tight_layout()

    fig_out = os.path.join(experiment_out_dir, 'topN_f1.png')
    plt.savefig(fig_out, dpi=300)

    return pathlist


from pymongo import MongoClient
import re
import matplotlib.pyplot as plt
import numpy as np

connection = MongoClient("mongodb://localhost:27017/")
db = connection.fake_news                     # name of db (it should be changed to real_news or fake_news
collections = db.collection_names()

# get controversial words from bias lexicon
file1 = open('input/bias-lexicon.txt', 'r')
text = file1.read()

words = text.split("\n")

bias_counter = 0
all_replies = 0

for collection in collections:
    if collection == "25073877":
        print("Skip realDonaldTrump")
        continue
    tweets = db[collection].find().sort("replies_count", -1).batch_size(10)
    for tweet in tweets:
        replies_count = db[collection].find_one({'id': tweet["id"]})["replies_count"]
        if replies_count > 20:
            replies = db[collection].find_one({'id': tweet["id"]})["replies"]
            # count replies with at least one controversy word
            for reply in replies:
                bias_word_exists = False
                reply_text = reply["text"].lower()

                all_replies += 1

                # check if at least one controversy word is included
                for word in words:
                    if re.search(r"\b" + re.escape(word) + r"\b", reply_text):
                        bias_word_exists = True
                if bias_word_exists:
                    bias_counter += 1

file1.close()
print("All replies: ", all_replies)
print("Bias word replies: ", bias_counter)

# get controversial words from Mejova controversy lexicon
file2 = open('input/controversial_words.txt', 'r')
text = file2.read()

words = text.split(", ")

contr_counter = 0

for collection in collections:
    if collection == "25073877":
        print("Skip realDonaldTrump")
        continue
    tweets = db[collection].find().sort("replies_count", -1).batch_size(10)
    for tweet in tweets:
        replies_count = db[collection].find_one({'id': tweet["id"]})["replies_count"]
        if replies_count > 20:
            replies = db[collection].find_one({'id': tweet["id"]})["replies"]
            # count replies with at least one controversy word
            for reply in replies:
                contr_word_exists = False
                reply_text = reply["text"].lower()

                # check if at least one controversy word is included
                for word in words:
                    if re.search(r"\b" + re.escape(word) + r"\b", reply_text):
                        contr_word_exists = True
                if contr_word_exists:
                    contr_counter += 1

file2.close()
print("All replies: ", all_replies)
print("Controversy word replies: ", contr_counter)

# get skepticism words from skepticism lexicon
file3 = open('input/skepticism_words.txt', 'r')
text = file3.read()

words = text.split(", ")

skepticism_counter = 0

for collection in collections:
    if collection == "25073877":
        print("Skip realDonaldTrump")
        continue
    tweets = db[collection].find().sort("replies_count", -1).batch_size(10)
    for tweet in tweets:
        replies_count = db[collection].find_one({'id': tweet["id"]})["replies_count"]
        if replies_count > 20:
            replies = db[collection].find_one({'id': tweet["id"]})["replies"]
            # count replies with at least one controversy word
            for reply in replies:
                skepticism_word_exists = False
                reply_text = reply["text"].lower()

                # check if at least one controversy word is included
                for word in words:
                    if re.search(r"\b" + re.escape(word) + r"\b", reply_text):
                        skepticism_word_exists = True
                if skepticism_word_exists:
                    skepticism_counter += 1

file3.close()
print("All replies: ", all_replies)
print("Controversy word replies: ", skepticism_counter)

# write results to file
file4 = open('input/Fake news - words_results.txt', 'w')
file4.write("%d replies with bias words out of %d replies \n" % (bias_counter,all_replies))
file4.write("%d replies with controversy words out of %d replies \n" % (contr_counter,all_replies))
file4.write("%d replies with skepticism words out of %d replies \n" % (skepticism_counter,all_replies))
file4.close()

# create plot
n_groups = 3
controversy_replies = (bias_counter, contr_counter, skepticism_counter)
no_controversy_replies = (all_replies - bias_counter, all_replies - contr_counter, all_replies - skepticism_counter)

# create plot
fig, ax = plt.subplots()
index = np.arange(n_groups)
bar_width = 0.25
opacity = 0.8

rects1 = plt.bar(index, controversy_replies, bar_width,
                 alpha=opacity,
                 color=(0.2, 0.4, 0.6, 0.9),
                 label='With',
                 zorder=3)

rects2 = plt.bar(index + bar_width, no_controversy_replies, bar_width,
                 alpha=opacity,
                 color=(0.6, 0.3, 0.5, 0.9),
                 label='Without',
                 zorder=3)

plt.legend(loc=4)
plt.ylabel('Number of replies')
plt.title('Fake news - percentage of replies with and without controversial words')
plt.xticks(index + bar_width/2, ('Bias lexicon', 'Controversy lexicon', 'Skepticism lexicon'),
           rotation=45, ha="right")

def autolabel(rects):
    """
    Attach a text label above each bar displaying its height
    """
    for rect in rects:
        height = rect.get_height()
        ax.text(rect.get_x() + rect.get_width()/2., 0.99*height,
                '%d %%' % int(height*100/all_replies),
                ha='center', va='bottom')

autolabel(rects1)
autolabel(rects2)

ax.yaxis.grid(linestyle=':', zorder=0)
plt.tight_layout()

plt.savefig("plots/Fake news - Replies with controversy words.svg")
plt.show()


import requests
import json
from todoist.api import TodoistAPI
from requests.auth import HTTPDigestAuth
from retrieve_canvas_course_ids import load_courses

with open("api_keys.txt") as api_file:
    keys = api_file.readlines()

#Initialize TodoistAPI
todoist_api_token = keys[0].strip()
todoist_api = TodoistAPI(todoist_api_token)
todoist_api.reset_state()
todoist_api.sync()

canvas_api_heading = 'https://canvas.instructure.com'
canvas_token = keys[1].strip()

courses_id_name_dict = load_courses(False)

course_ids = []
for course_id in keys[2:]:
    course_ids.append(int(course_id.strip()))

header = {"Authorization":"Bearer " + canvas_token}
param = {'per_page': '100', 'include':'submission'}

assignments = []
todoist_tasks = []
todoist_project_dict = {}

# Iterates over the course_name_id_dict and loads all of the users assignments
# for those classes. Appends assignment objects to assignments list
def load_assignments():
    for course_id in course_ids:
        response = requests.get(canvas_api_heading + '/api/v1/courses/' +
        str(course_id) + '/assignments', headers=header,
        params=param)

        for assignment in response.json():
            assignments.append(assignment)

# Loads all user tasks from Todoist
def load_todoist_tasks():
    tasks = todoist_api.state['items']
    for task in tasks:
        todoist_tasks.append(task)

# Loads all user projects from Todoist
def load_todoist_projects():
    projects = todoist_api.state['projects']
    for project in projects:
        todoist_project_dict[project['name']] = project['id']
    # print(todoist_project_dict)

# Checks to see if the user has a project matching their course names, if there
# isn't a new project will be created
def create_todoist_projects():
    for course_id in course_ids:
        if courses_id_name_dict[course_id] not in todoist_project_dict:
            project = todoist_api.projects.add(courses_id_name_dict[course_id])
            todoist_api.commit();
            todoist_api.sync()

            todoist_project_dict[project['name']] = project['id']
        else:
            print("the key was in dict, don't create project")

# Transfers over assignments from canvas over to Todoist, the method Checks
# to make sure the assignment has not already been trasnfered to prevent overlap
def transfer_assignments_to_todoist():
    for assignment in assignments:
        course_name = courses_id_name_dict[assignment['course_id']]
        assignment_name = assignment['name']
        project_id = todoist_project_dict[course_name]

        is_synced = False
        for task in todoist_tasks:
            if task['content'] == (assignment_name + ' Due') and \
            task['project_id'] == project_id:
                print("Assignment already synced: " + assignment['name'])
                is_synced = True

        if not is_synced:
            if assignment['submission']['submitted_at'] == None:
                print("Adding assignment " + assignment['name'])
                add_new_task(assignment, project_id)
            else:
                print("assignment already submitted " + assignment['name'])
        else:
            print("assignmentt already synced")
    todoist_api.commit()

# Adds a new task from a Canvas assignment object to Todoist under the
# project coreesponding to project_id
def add_new_task(assignment, project_id):
    test_task = todoist_api.items.add(assignment['name'] + ' Due',
            project_id=project_id,
            date_string=assignment['due_at'])

load_todoist_projects()
load_assignments()
load_todoist_tasks()

create_todoist_projects()

transfer_assignments_to_todoist()


from sqlalchemy import Table, Column
from sqlalchemy import String, DateTime
from sqlalchemy.dialects.mysql import BIGINT, TINYINT
from sqlalchemy import Index
from sqlalchemy import event, DDL
from . import metadata
from ...default_mysql_args import mysql_args

account = Table(
    'account',
    metadata,
    Column('accid', BIGINT(unsigned=True), primary_key=True),
    Column('username', String(length=200), unique=True),
    Column('password', String(length=1024), nullable=False),
    Column('date_sign_in', DateTime, nullable=False),
    Column('date_last_actived', DateTime, nullable=False),
    Column('disabled', TINYINT, nullable=False, server_default='0'),
    Column('date_disabled', DateTime, nullable=False,
           server_default='0000-00-00 00:00:00'),

    Index('accid_disabled', 'accid', 'disabled'),

    **mysql_args
)

event.listen(
    account,
    'after_create',
    DDL('ALTER TABLE %(table)s AUTO_INCREMENT=100000000')
)


from typing import List

class NodeInfo:
    def __init__(self, name, host, tcp_port, http_port):
        self.name = name
        self.host = host
        self.tcp_port = tcp_port
        self.http_port = http_port

    def get_str_info(self):
        ret = 'name: ' + self.name + ', '
        ret += 'host: ' + self.host + ', '
        ret += 'tcp_port: ' + str(self.tcp_port) + ', '
        ret += 'http_port: ' + str(self.http_port)
        return ret

class Network:
    def get_nodes() -> List[NodeInfo]:
        nodes : NodeInfo = []
        with open('network_info.csv') as file:
            for line in file:
                if line[0] != '#': # jump comments
                    (name, host, tcp_port, http_port) = line.split(',')
                    nodes.append(NodeInfo(name, host, int(tcp_port), int(http_port)))
        return nodes

    def get_node_info(ip) -> NodeInfo:
        with open('network_info.csv') as file:
            for line in file:
                if line[0] != '#': # jump comments
                    (name, host, tcp_port, http_port) = line.split(',')
                    if host == ip:
                        return NodeInfo(name, host, int(tcp_port), int(http_port))


import numpy as np
import matplotlib.pylab as plt
from sklearn import datasets
from sklearn.model_selection import train_test_split
from plotka import plot_decision_regions

class LogisticRegressionGD(object):
    def __init__(self, eta=0.05, n_iter=2000, random_state=1):
        self.eta = eta
        self.n_iter = n_iter
        self.random_state = random_state

    def fit(self, X, y):
        rgen = np.random.RandomState(self.random_state)
        self.w_ = rgen.normal(loc=0.0, scale=0.01, size=1 + X.shape[1])

        for i in range(self.n_iter):
            net_input = self.net_input(X)
            output = self.activation(net_input)
            errors = (y - output)
            self.w_[1:] += self.eta * X.T.dot(errors)
            self.w_[0] += self.eta * errors.sum()

        return self

    def net_input(self, X):
        return np.dot(X, self.w_[1:]) + self.w_[0]

    def activation(self, z):
        return 1. / (1. + np.exp(-np.clip(z, -250, 250)))

    def predict(self, X):
        return np.where(self.net_input(X) >= 0.0, 1, 0)

class Multiclass(object):
    def __init__(self):
        self.cls1 = LogisticRegressionGD()
        self.cls3 = LogisticRegressionGD()

    def fit(self, X, y):

        y1 = y.copy()
        y3 = y.copy()

        y1[(y1 != 0)] = -3
        y1[y1 == 0] = 1
        y1[y1 == -3] = 0

        y3[(y3 != 2)] = -3
        y3[y3 == 2] = 1
        y3[y3 == -3] = 0

        self.cls1.fit(X, y1)
        self.cls3.fit(X, y3)

    def predict(self, X):
        result = []
        for data in X:
            if self.cls1.predict(data) == 1:
                result.append(0)
            elif self.cls3.predict(data) == 1:
                result.append(2)
            else:
                result.append(1)

        return np.array(result)

def main():
    iris = datasets.load_iris()
    X = iris.data[:, [1, 3]]
    y = iris.target

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=1, stratify=y)

    multi = Multiclass()
    multi.fit(X_train, y_train)
    multi.predict(X_test)

    plot_decision_regions(X=X_test, y=y_test, classifier=multi)
    plt.xlabel(r'$x_1$')
    plt.ylabel(r'$x_2$')
    plt.legend(loc='upper left')
    plt.show()

if __name__ == '__main__':
    main()


"""
Jupyter Widgets

roger.bermudez@epfl.ch
CVLab EPFL 2018
"""

import os
from glob import glob
import ipywidgets as widgets
from IPython.display import Javascript, display

class FileBrowser():
    def __init__(self, initial_path=os.getcwd(), file_filter="*", ignore_hidden=True):
        self.path = initial_path
        self.filter = file_filter

        self.ignore_hidden = ignore_hidden
        self._update_files()
    def _update_files(self):
        self.files = list()
        self.dirs = list()
        if os.path.isdir(self.path):
            files = glob(os.path.join(self.path, self.filter))
            for f in os.listdir(self.path):
                ff = os.path.join(self.path, f)
                if os.path.isdir(ff):
                    if not (self.ignore_hidden and f.startswith(".")):
                        self.dirs.append(f)
                else:
                    #if f in files:
                    if os.path.join(self.path, f) in files:
                        self.files.append(f)

    def widget(self):
        box = widgets.VBox()
        self._update(box)
        return box

    def show(self):
        return self.widget()

    def _update(self, box):

        def on_click(b):
            if b.description == '..':
                self.path = os.path.split(self.path)[0]
            else:
                self.path = self.path + "/" + b.description
            self._update_files()
            self._update(box)

        buttons = []
        if self.files:
            button = widgets.Button(description='..', background_color='#d0d0ff', layout=widgets.Layout(width='50%'))
            button.on_click(on_click)
            buttons.append(button)
        for f in self.dirs:
            button = widgets.Button(description=f, background_color='#d0d0ff', layout=widgets.Layout(width='50%'))
            button.on_click(on_click)
            buttons.append(button)
        for f in self.files:
            button = widgets.Button(description=f, layout=widgets.Layout(width='50%'))
            button.on_click(on_click)
            buttons.append(button)
        box.children = tuple([widgets.HTML("<h3>%s</h3>" % (self.path,))] + buttons)
        # display(Javascript('IPython.notebook.execute_cells_below()'))


def get_list_top(string, symbol_count=5, quantity=3):
    result = ""
    list_big_word = {}
    for word in string.split(" "):
        if len(word) > symbol_count:
            list_big_word.update({word: list_big_word.get(word, 0) + 1})
    list_for_sort = list(list_big_word.items())
    list_for_sort.sort(key=lambda i: i[1], reverse=True)
    for item, _ in zip(list_for_sort, range(quantity)):
        result = result + " " + item[0]
    return result

import json

with open('data.json', 'r', encoding='utf-8') as fh:
    data = json.load(fh)
    words_array = ""
    for item in data["rss"]["channel"]["items"]:
        for word in item["description"].split(" "):
            words_array = words_array + " " + word
print(get_list_top(words_array, 6, 10))


#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import datetime
import operator

from oslo_log import log

import ceilometer
from ceilometer.alarm.storage import base
from ceilometer.alarm.storage import models
from ceilometer.storage.hbase import base as hbase_base
from ceilometer.storage.hbase import migration as hbase_migration
from ceilometer.storage.hbase import utils as hbase_utils
from ceilometer import utils

LOG = log.getLogger(__name__)

AVAILABLE_CAPABILITIES = {
    'alarms': {'query': {'simple': True,
                         'complex': False},
               'history': {'query': {'simple': True,
                                     'complex': False}}},
}

AVAILABLE_STORAGE_CAPABILITIES = {
    'storage': {'production_ready': True},
}

class Connection(hbase_base.Connection, base.Connection):
    """Put the alarm data into a HBase database

    Collections:

    - alarm:

      - row_key: uuid of alarm
      - Column Families:

        f: contains the raw incoming alarm data

    - alarm_h:

      - row_key: uuid of alarm + ":" + reversed timestamp
      - Column Families:

        f: raw incoming alarm_history data. Timestamp becomes now()
          if not determined
    """

    CAPABILITIES = utils.update_nested(base.Connection.CAPABILITIES,
                                       AVAILABLE_CAPABILITIES)
    STORAGE_CAPABILITIES = utils.update_nested(
        base.Connection.STORAGE_CAPABILITIES,
        AVAILABLE_STORAGE_CAPABILITIES,
    )
    _memory_instance = None

    ALARM_TABLE = "alarm"
    ALARM_HISTORY_TABLE = "alarm_h"

    def __init__(self, url):
        super(Connection, self).__init__(url)

    def upgrade(self):
        tables = [self.ALARM_HISTORY_TABLE, self.ALARM_TABLE]
        column_families = {'f': dict()}
        with self.conn_pool.connection() as conn:
            hbase_utils.create_tables(conn, tables, column_families)
            hbase_migration.migrate_tables(conn, tables)

    def clear(self):
        LOG.debug('Dropping HBase schema...')
        with self.conn_pool.connection() as conn:
            for table in [self.ALARM_TABLE,
                          self.ALARM_HISTORY_TABLE]:
                try:
                    conn.disable_table(table)
                except Exception:
                    LOG.debug('Cannot disable table but ignoring error')
                try:
                    conn.delete_table(table)
                except Exception:
                    LOG.debug('Cannot delete table but ignoring error')

    def update_alarm(self, alarm):
        """Create an alarm.

        :param alarm: The alarm to create. It is Alarm object, so we need to
          call as_dict()
        """
        _id = alarm.alarm_id
        alarm_to_store = hbase_utils.serialize_entry(alarm.as_dict())
        with self.conn_pool.connection() as conn:
            alarm_table = conn.table(self.ALARM_TABLE)
            alarm_table.put(_id, alarm_to_store)
            stored_alarm = hbase_utils.deserialize_entry(
                alarm_table.row(_id))[0]
        return models.Alarm(**stored_alarm)

    create_alarm = update_alarm

    def delete_alarm(self, alarm_id):
        with self.conn_pool.connection() as conn:
            alarm_table = conn.table(self.ALARM_TABLE)
            alarm_table.delete(alarm_id)

    def get_alarms(self, name=None, user=None, state=None, meter=None,
                   project=None, enabled=None, alarm_id=None,
                   alarm_type=None, severity=None):

        if meter:
            raise ceilometer.NotImplementedError(
                'Filter by meter not implemented')

        q = hbase_utils.make_query(alarm_id=alarm_id, name=name,
                                   enabled=enabled, user_id=user,
                                   project_id=project, state=state,
                                   type=alarm_type, severity=severity)

        with self.conn_pool.connection() as conn:
            alarm_table = conn.table(self.ALARM_TABLE)
            gen = alarm_table.scan(filter=q)
            alarms = [hbase_utils.deserialize_entry(data)[0]
                      for ignored, data in gen]
            for alarm in sorted(
                    alarms,
                    key=operator.itemgetter('timestamp'),
                    reverse=True):
                yield models.Alarm(**alarm)

    def get_alarm_changes(self, alarm_id, on_behalf_of,
                          user=None, project=None, alarm_type=None,
                          severity=None, start_timestamp=None,
                          start_timestamp_op=None, end_timestamp=None,
                          end_timestamp_op=None):
        q = hbase_utils.make_query(alarm_id=alarm_id,
                                   on_behalf_of=on_behalf_of, type=alarm_type,
                                   user_id=user, project_id=project,
                                   severity=severity)
        start_row, end_row = hbase_utils.make_timestamp_query(
            hbase_utils.make_general_rowkey_scan,
            start=start_timestamp, start_op=start_timestamp_op,
            end=end_timestamp, end_op=end_timestamp_op, bounds_only=True,
            some_id=alarm_id)
        with self.conn_pool.connection() as conn:
            alarm_history_table = conn.table(self.ALARM_HISTORY_TABLE)
            gen = alarm_history_table.scan(filter=q, row_start=start_row,
                                           row_stop=end_row)
            for ignored, data in gen:
                stored_entry = hbase_utils.deserialize_entry(data)[0]
                yield models.AlarmChange(**stored_entry)

    def record_alarm_change(self, alarm_change):
        """Record alarm change event."""
        alarm_change_dict = hbase_utils.serialize_entry(alarm_change)
        ts = alarm_change.get('timestamp') or datetime.datetime.now()
        rts = hbase_utils.timestamp(ts)
        with self.conn_pool.connection() as conn:
            alarm_history_table = conn.table(self.ALARM_HISTORY_TABLE)
            alarm_history_table.put(
                hbase_utils.prepare_key(alarm_change.get('alarm_id'), rts),
                alarm_change_dict)


"""""
Ex. 17

Create a linear regression model with the given points on a scatter plot. To do this get the data points slope, intercept, r, p,and std_err from stats (import scipy).linregress(x, y). Then graph it with list(map())
"""

import numpy as np
import random
import matplotlib.pyplot as plt
from scipy import stats

speed = np.array([])

for i in range(13):
  speed = np.append(speed,random.randint(60,120))

mean = np.mean(speed)
std = np.std(speed)
pct_75 = np.percentile(speed,75)
pct_99 = np.percentile(speed,99)

print(speed)
print(mean)
print(std)
print(pct_75)
print(pct_99)

y = np.random.triangular (1,50, 100, 250000)

x_val = [5,7,8,7,2,17,2,9,4,11,12,9,6]
y_val = [99,86,87,88,111,86,103,87,94,78,77,85,86]

slope, intercept, r, p, std_err = stats.linregress(x_val, y_val)

def myfunc(x):
  return slope * x + intercept

mymodel = list(map(myfunc, x_val))

plt.scatter(x_val, y_val)
plt.plot(x_val, mymodel)
plt.show()


import csv
import os
import requests
import sys
import re

def pripravi_imenik(ime_datoteke):
    '''Ce se ne obstaja, pripravi prazen imenik za dano datoteko.'''
    imenik = os.path.dirname(ime_datoteke)
    if imenik:
        os.makedirs(imenik, exist_ok=True)

def shrani(url, ime_datoteke, vsili_prenos=False):
    '''Vsebino strani na danem naslovu shrani v datoteko z danim imenom.'''
    try:
        print('Shranjujem {}...'.format(url))
        sys.stdout.flush()
        if os.path.isfile(ime_datoteke) and not vsili_prenos:
            print('shranjeno ze od prej!')
            return
        r = requests.get(url, headers={'Accept-Language': 'en'})
    except requests.exceptions.ConnectionError:
        print('stran ne obstaja!')
    else:
        pripravi_imenik(ime_datoteke)
        with open(ime_datoteke, 'w') as datoteka:
            datoteka.write(r.text)
            print('shranjeno!')

def vsebina_datoteke(ime_datoteke):
    '''Vrne niz z vsebino datoteke z danim imenom.'''
    with open(ime_datoteke) as datoteka:
        vsebina = datoteka.read()
    return vsebina

def datoteke(imenik):
    '''Vrne imena vseh datotek v danem imeniku skupaj z imenom imenika.'''
    return [os.path.join(imenik, datoteka) for datoteka in os.listdir(imenik)]

strani = ["1-100", "101-200", "201-300", "301-400", "401-500", "501-600", "601-700", "701-800", "801-900", "901-1000"]

for stran in strani:
    osnovni_naslov = 'http://www.atpworldtour.com/en/rankings/singles/'
    parametri = 'rankDate=2017-1-2&countryCode=all'
    naslov = '{}?{}&rankRange={}'.format(osnovni_naslov, parametri, stran)
    datoteka = 'tenis/{}.txt'.format(stran)
    shrani(naslov, datoteka)

moj_regex = r'td class="rank-cell">.*?(\d{1,4}).*?	.*?alt="(\w{1,3})".*?data-ga-label="(.*?)".*?<td class="age-cell".*?(\d{2}).*?player-activity.*?matchType=singles" data-ga-label="(\d{1,3})".*?'

def ustvari_slovarje (ime_datoteke,regex_izraz):
    igralci = []

    for mapa in datoteke(ime_datoteke):
        with open(mapa) as f:
            vsebina = f.read()
            for (i, j, k, l, m) in re.findall(regex_izraz, vsebina, flags=re.DOTALL):
                igralec = {"id":(int(i)), "Ranking": (int(i)), "Drzava": (j.strip()), "Ime": (k.strip()), "Starost": (int(l)),
                           "Turnirji": (int(m))}
                igralci.append(igralec)

    return igralci

def zapisi_tabelo(slovarji, imena_polj, ime_datoteke):
    pripravi_imenik(ime_datoteke)
    with open(ime_datoteke, 'w') as csv_dat:
        writer = csv.DictWriter(csv_dat, fieldnames=imena_polj)
        writer.writeheader()
        for slovar in slovarji:
            writer.writerow(slovar)

igralci = ustvari_slovarje("tenis", moj_regex)

zapisi_tabelo(igralci, ["id","Ranking", "Drzava", "Ime", "Starost", "Turnirji"], "igralci.csv")


from flask import Blueprint, request, redirect, abort

import json
from bson import json_util

from API.template.template import Template
from API.event.event import Event
import API.utils as utils

mod = Blueprint('template', __name__, url_prefix='/template')

@mod.route('/hi')
def hello_world():
	return "Hello World, this is template'"

@mod.route('/<status>/<params>', methods=['GET'])
def getTemplates(status=None, params=None):
	e = Event('web.newTemplate')
	template = Template()
	res = template.getActiveTemplates(status, params)
	e.save()
	return res

@mod.route('', methods=['POST'])
def newTemplate():
	e = Event('web.newTemplate')
	name = request.form['bname']
	desc = request.form['bdesc']
	key = request.form['bkey']
	status = request.form['bstatus']
	thumb = request.form['thumbnail']
	user = utils.getKey(key)
	t = Template()
	res = str(t.insert(name, desc, user, thumb, status))
	e.save()
	return res

@mod.route('/pub', methods=['POST'])
def publishTemplate():
	e = Event('web.publishTemplate')
	t_id = request.form['tid']
	view = request.form['view']
	key = request.form['k']
	user = utils.getKey(key)
	t = Template()
	if t.isOwner(t_id, user):
		t.load(t_id)
		t.createDefaultView()
		res = getMessage(t_id, True)
	else:
		res = getErrorMessage('User is not the owner of the Template')
	e.save()
	return res

@mod.route('/<template_id>', methods=['GET'])
def getTemplate(template_id=None):
	e = Event('web.getTemplate')
	t = Template()
	res = t.getById(template_id)
	e.save()
	if res != 'null':
		return res
	else:
		abort(404)

@mod.route('/add', methods=['POST'])
def addControl():
	e = Event('web.addControl')
	c_id = request.form['cid']
	t_id = request.form['tid']
	order = request.form['order']
	title = request.form['title']
	help = request.form['help']
	view = request.form['view']
	slug = request.form['slug']
	typex = request.form['typex']
	key = request.form['k']
	user = utils.getKey(key)
	t = Template()
	if t.isOwner(t_id, user):
		res = t.addControl(c_id, t_id, title, help, order, view, slug, typex)
	else:
		res = getErrorMessage('User is not the owner of the Template')
	e.save()
	return res

def getMessage(message, asJson=True, key='response'):
		res = dict()
		res[key] = message
		if asJson:
			return json.dumps(res)
		else:
			return res


# coding: utf-8
from flask.ext.mail import Mail
from flask.ext.security import Security as _Security
from flask.ext.security import MongoEngineUserDatastore

from dealer.contrib.flask import Dealer
from quokka.core.db import db
from quokka.core.cache import cache
from quokka.core.admin import configure_admin
from quokka.core.templates import render_template
from quokka.modules.accounts.models import Role, User

from . import (generic, babel, blueprints, error_handlers, context_processors,
               template_filters, before_request, views, themes, fixtures,
               oauthlib)

class Security(_Security):
    def render_template(self, *args, **kwargs):
        return render_template(*args, **kwargs)

def configure_extensions(app, admin):
    cache.init_app(app)
    babel.configure(app)
    generic.configure(app)
    Mail(app)
    Dealer(app)
    error_handlers.configure(app)
    db.init_app(app)
    fixtures.configure(app, db)
    themes.configure(app, db)  # Themes should be configured after db

    context_processors.configure(app)
    template_filters.configure(app)

    app.security = Security(app, MongoEngineUserDatastore(db, User, Role))

    blueprints.load_from_packages(app)
    blueprints.load_from_folder(app)

    configure_admin(app, admin) #配置管理参数

    if app.config.get('DEBUG_TOOLBAR_ENABLED'):
        try:
            from flask_debugtoolbar import DebugToolbarExtension
            DebugToolbarExtension(app)
        except:
            pass

    before_request.configure(app)
    views.configure(app)

    oauthlib.configure(app)

    if app.config.get('SENTRY_ENABLED', False):
        from .sentry import configure
        configure(app)

    return app


import csv
import pandas as pd
import numpy as np
import math
from sklearn.model_selection import KFold
from sklearn.linear_model import LinearRegression, Ridge, RidgeCV, Lasso, LassoCV
from sklearn import datasets, linear_model
from numpy.linalg import inv
from regressors import LinearRegressor                  # From demos by Andreas Krause
from regularizers import Regularizer, L2Regularizer     # From demos by Andreas Krause
from util import gradient_descent                       # From demos by Andreas Krause
import load_dataset
from sklearn.datasets import load_iris
from sklearn import preprocessing

# Data writing, reading
def write_csv(path, result_data):
    with open(path, 'w') as file:
        writer = csv.writer(file)
        writer.writerows(map(lambda x: [x], result_data))
        file.close()

def data_set(num_shuffle):
    i_data = load_dataset.ImportData("../../Task1b/data/train.csv", 0.1)
    i_data.read_csv()
    for i in range(num_shuffle):
        i_data.shuf_data()
    train = np.array(i_data.df_s)
    y = train[:,0]
    y = y.reshape(1000,1)
    x = train[:,1:]
    X = np.concatenate((x,sqr(x)),axis = 1)
    X = np.concatenate((X,expo(x)),axis = 1)
    X = np.concatenate((X,cos_mat(x)),axis = 1)
    const = np.ones((np.size(y),1))
    X = np.append(X,const,axis = 1)
    train2 = np.append(X, y, axis =1)
    return train2

# element-wise functions
def sqr(x):
    return x**2
def exp(a,x):
    return a**x
def expo(x):
    return [exp(np.e,a) for a in x]
def cos_scalar(x):
    return math.cos(x)
def cos_array(x):
    return [cos_scalar(a) for a in x]
def cos_mat(x):
    return [cos_array(a) for a in x]

# Different methods
def closed_form_sol(X,y):
    Mat = np.matmul(inv(np.matmul(np.transpose(X),X)),np.transpose(X))
    w = Mat.dot(y)
    return w

def lr(X,y):
    regr = LinearRegression(fit_intercept = False)
    regr.fit(X, y)
    w = regr.coef_
    return w

def rr(X,y,lamb):
    ridge_model = Ridge(alpha=lamb, fit_intercept = False)
    ridge_model.fit(X, y)
    w = ridge_model.coef_
    return w, ridge_model

def rrCV(X,y,lamb,k):
    ridge_model = RidgeCV(alphas=lamb, cv = k, fit_intercept = False)
    ridge_model.fit(X, y)
    w = ridge_model.coef_
    lamb = ridge_model.alpha_
    return w, lamb

def lasoCV(X,y,lamb,k):
    ridge_model = LassoCV(alphas=lamb, cv = k, fit_intercept = False)
    ridge_model.fit(X, y)
    w = ridge_model.coef_
    lamb = ridge_model.alpha_
    return w, lamb

def gd(X,y,compo):
    ## Initial guess with closed form solution
    # w0 = closed_form_sol(X,y)
    # w0 = w0.reshape(21)
    w0 = np.zeros(21)

    regularizer = L2Regularizer(compo)    # regularize component
    regressor = LinearRegressor(X,y)
    opts = {'eta0': 0.001,              # learning rate component
            'n_iter': 1000,            # number of iteration component
            'n_samples': X.shape[0],
            'algorithm': 'GD',
            'learning_rate_scheduling': 'Bold driver' # choose
            }
    trajectory, indexes = gradient_descent(w0, regressor, regularizer, opts)
    print(trajectory)
    w_ = trajectory[-1,:]
    return w_

def k_fold_with_lr(train, k):
    fold_size = k
    kf = KFold(n_splits = fold_size)
    kf.get_n_splits(train)
    counter = 1
    w_aver = np.zeros(21)

    for train_index, test_index in kf.split(train):
        print(counter,"th fold")
        counter += 1
        X_train, X_test = train[train_index, :-1], train[test_index, :-1]
        y_train, y_test = train[train_index, -1], train[test_index, -1]
        w_star = lr(X_train,y_train)
        w_aver = w_aver + w_star
    w_aver = w_aver*(k**(-1))
    return w_aver

def k_fold_with_rr(train, Params, k):
    fold_size = k
    kf = KFold(n_splits = fold_size)
    kf.get_n_splits(train)
    min = 100000000000000000000000

    for i, compo in enumerate(Params):
        print(i+1,"th component")
        sum = 0
        counter = 1
        for train_index, test_index in kf.split(train):
            print(counter,"th fold")
            counter += 1
            X_train, X_test = train[train_index, :-1], train[test_index, :-1]
            y_train, y_test = train[train_index, -1], train[test_index, -1]
            w_star, regr = rr(X_train,y_train, compo)
            result = regr.predict(X_test)
            sum = sum + np.sum(np.square(y_test-result))
        if (min > sum):
            min = sum
            print(min)
            opt_value = compo
            w_fin = w_star
            print("updated: ",opt_value)
    return opt_value, w_fin

def k_fold_with_gd(train, params, k):
    fold_size = k
    kf = KFold(n_splits = fold_size)
    kf.get_n_splits(train)
    min = 100000000000000000000000

    for i, compo in enumerate(params):
        print(i+1,"th component")
        sum = 0
        counter = 1
        for train_index, test_index in kf.split(train):
            print(counter,"th fold")
            counter += 1
            X_train, X_test = train[train_index, :-1], train[test_index, :-1]
            y_train, y_test = train[train_index, -1], train[test_index, -1]
            w_star = gd(X_train,y_train,compo)
            sum = sum + np.sum(np.square(y_test-X_test.dot(w_star)))
        if (min > sum):
            min = sum
            print(min)
            opt_value = compo
            print("updated: ",opt_value)
    return opt_value

# K_fold to find optimal parameters
# reg_params = [1e-1, 1, 1e1, 1e2, 1e3, 1e4]
reg_params = [1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4]
# reg_params = range(1375, 1385, 1)
num_iter_params = [10, 100, 200, 500]
learning_rate = [1, 1e-1, 1e-2, 1e-3, 1e-5, 1e-10, 1e-15, 1e-17, 1e-20]
k = 100

# Parameters optimization
# train = np.array(pd.read_csv("../../Task1b/data/train_nonlinear.csv"))
# X, y = train[:,2:], train[:,1]
# y = y.reshape(1000,1)
# train2 = np.append(X, y, axis =1)
# opt_reg1 = k_fold_with_gd(train2, reg_params, k)
# opt_num_iter = k_fold_with_gd(train, num_iter_params, k)
# opt_learn_rate = k_fold_with_gd(train, learning_rate, k)
# print("Optimal value with K_fold with gd: ",opt_reg1)

# opt_reg2, w_fin = k_fold_with_rr(train2, reg_params, k)
# print("Optimal value with K_fold with rr: ",opt_reg2)
## 256 for 500 folds, 274 for 500 folds

# epoch > 1
epoch = 10
w_average = np.zeros(21)

for i in range(epoch):
    train = data_set(i+1)
    X, y = train[:,:-1], train[:,-1]
    w_star, lamb = rrCV(X,y,reg_params,k)
    w_average = w_average + w_star
    print(i+1, "th epoch")
    print("w_star = ",w_star)
    print("lambda_star = ",lamb)
w_average = w_average*(epoch**(-1))

# # epoch = 1
# train = data_set(1)
# X, y = train[:,:-1], train[:,-1]
# w_star, lamb = rrCV(X,y,reg_params,k)
# print("w_star: ",w_star)
# print("Optimal lambda: ",lamb)

write_csv("../../Task1b/Hokwang/Result.csv", w_star)