#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jan  7 15:44:55 2021

@author: zachz
"""

#%% Imports

import numpy as np
import scipy.io as spio
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
import random
import warnings
warnings.filterwarnings("ignore")

#%% Load in data (trying amygdala here)

amy = spio.loadmat('/Users/zachz/Dropbox/Timescales across species/Spiketimes only/Faraut - Human MTL/faraut_amygdala.mat',simplify_cells=True)

spikes = amy['spikes'] / 10**6

try:
   cell_info = acc['cell_info']
except NameError:
   pass

#%% Choose a random start point and 1 sec after, estimate timescale from this
  # Then repeat 250x per unit to simulate 250 trials

  # Do it 100 times

repeated_taus = []

for repeat in range(100):

    print('now on repeat %i' %repeat)

    faraut_amyg_all_means = []
    faraut_amyg_taus = []
    faraut_amyg_failed_fits = []

    faraut_amyg_failed_autocorr = []
    faraut_amyg_no_spikes_in_a_bin = []
    faraut_amyg_low_fr = []

    faraut_amyg_avg_fr = []
    faraut_amyg_correlation_matrices = []

    for unit in range(len(spikes)):

        binned_unit_spikes = []

        for iteration in range(250):

        # Get spikes from random interval of 1s

            this_unit = spikes[unit]

            min_t = np.min(this_unit)
            max_t = np.max(this_unit)

            start_t = random.uniform(min_t,max_t)
            end_t = start_t + 1

            rand_spikes = []

            for spike in range(len(this_unit)):

                if start_t <= this_unit[spike] < end_t:

                    rand_spikes.append(this_unit[spike])

            # Bin spikes in 50ms bins

            bins = np.linspace(start_t,end_t,20)

            binned, bin_edges = np.histogram(rand_spikes,bins=bins)

            binned_spikes = binned

            binned_unit_spikes.append(binned_spikes)

        binned_unit_spikes = np.vstack(binned_unit_spikes)

        [trials,bins] = binned_unit_spikes.shape

        summed_spikes_per_bin = np.sum(binned_unit_spikes,axis=0)

        faraut_amyg_avg_fr.append(np.sum(summed_spikes_per_bin)/trials)

        #%% Do autocorrelation

        one_autocorrelation = []

        for i in range(bins):
            for j in range(bins):
                ref_window = binned_unit_spikes[:,i]
                test_window = binned_unit_spikes[:,j]

                correlation = np.corrcoef(ref_window,test_window)[0,1]

                one_autocorrelation.append(correlation)

        if np.isnan(one_autocorrelation).any() == True:

            faraut_amyg_failed_autocorr.append(unit) # skip this unit if any autocorrelation fails

        elif [summed_spikes_per_bin[bin] == 0 for bin in range(len(summed_spikes_per_bin))] == True:

            faraut_amyg_no_spikes_in_a_bin.append(unit) # skip this unit if any bin doesn't have spikes

        elif np.sum(summed_spikes_per_bin) < 1:

            faraut_amyg_low_fr.append(unit) # skip this unit if avg firing rate across all trials is < 1

        else:

            #%% Reshape list of autocorrelations into 19x19 matrix, plot it

            correlation_matrix = np.reshape(one_autocorrelation,(-1,19))

            faraut_amyg_correlation_matrices.append(correlation_matrix)

            # plt.imshow(correlation_matrix)
            # plt.title('Human amygdala unit %i' %unit)
            # plt.xlabel('lag')
            # plt.ylabel('lag')
            # plt.xticks(range(0,19))
            # plt.yticks(range(0,19))
            # plt.show()

            #%% Fit exponential decay

            # shift correlation matrix over so that all 1's are at x=0
            new_corr = []

            for i in range(18):
                new_corr.append(correlation_matrix[i,i:])

            new_corr = np.array(new_corr)

            new_lag = []

            for i in range(19,0,-1):
                new_lag.append(np.array(range(i)))

            new_lag = np.array(new_lag)

            new_corr = np.hstack(new_corr)
            new_lag = np.hstack(new_lag)
            new_lag = new_lag[:-1]

            # plt.scatter(new_lag,new_corr)
            # plt.ylabel('autocorrelation')
            # plt.xlabel('lag (ms)')

            #%% Remove 0 lag time and sort values for curve fitting

            no_1_corr = np.delete(new_corr,np.where(new_corr>=0.9))
            no_1_corr = no_1_corr[~ np.isnan(no_1_corr)]
            no_0_lag = np.delete(new_lag,np.where(new_lag==0))

            no_0_lag = no_0_lag * 50

            x = no_0_lag
            y = no_1_corr

            x = np.array(x, dtype=float)
            y = np.array(y, dtype=float)

            sorted_pairs = sorted((i,j) for i,j in zip(x,y))

            x_s = []
            y_s = []

            for q in range(len(sorted_pairs)):
                x_q = sorted_pairs[q][0]
                y_q = sorted_pairs[q][1]

                x_s.append(x_q)
                y_s.append(y_q)

            x_s = np.array(x_s)
            y_s = np.array(y_s)

            # plt.plot(x_s,y_s,'ro')
            # plt.title('Human amygdala unit %i' %unit)
            # plt.xlabel('lag (ms)')
            # plt.ylabel('autocorrelation')
            # plt.show()

            #%% get means and std

            from statistics import mean, stdev
            from itertools import groupby

            grouper = groupby(sorted_pairs, key=lambda x: x[0])
            #The next line is again more elegant, but slower:
            mean_pairs = [[x, mean(yi[1] for yi in y)] for x,y in grouper]
            std_pairs = [[x,stdev(yi[1] for yi in y)] for x,y in grouper]

            x_m = []
            y_m = []

            for w in range(len(mean_pairs)):
                x_w = mean_pairs[w][0]
                y_w = mean_pairs[w][1]

                x_m.append(x_w)
                y_m.append(y_w)

            x_m = np.array(x_m)
            y_m = np.array(y_m)

            # Only start fitting when slope is decreasing

            diff = np.diff(x_m)

            neg_diffs = []

            for dif in range(len(diff)):

                if diff[dif] >= 0:

                    neg_diffs.append(dif)

            first_neg_diff = np.min(neg_diffs)

            first_neg_diff = int(first_neg_diff)

            def func(x,a,tau,b):
                return a*((np.exp(-x/tau))+b)

            try:
                pars,cov = curve_fit(func,x_m[first_neg_diff:],y_m[first_neg_diff:],p0=[1,100,1],bounds=((0,np.inf)),maxfev=5000)

            except RuntimeError:
                print("Error - curve_fit failed")
                faraut_amyg_failed_fits.append(unit)

            faraut_amyg_taus.append(pars[1])

            faraut_amyg_all_means.append(y_m)

            # plt.plot(x_m,y_m,'ro',label='original data')
            # plt.plot(x_m[first_neg_diff:],func(x_m[first_neg_diff:],*pars),label='fit')
            # plt.xlabel('lag (ms)')
            # plt.ylabel('mean autocorrelation')
            # plt.title('Human amygdala %i' %unit)
            # plt.legend()
            # plt.show()

    #%% How many units got filtered?

    faraut_amyg_bad_units = len(faraut_amyg_failed_autocorr) + len(faraut_amyg_no_spikes_in_a_bin) + len(faraut_amyg_low_fr)

    print('%i units were filtered out' %faraut_amyg_bad_units)
    print('out of %i total units' %len(spikes))

    #%% Take mean of all units

    faraut_amyg_all_means = np.vstack(faraut_amyg_all_means)

    faraut_amyg_mean = np.mean(faraut_amyg_all_means,axis=0)
    faraut_amyg_sd = np.std(faraut_amyg_all_means,axis=0)
    faraut_amyg_se = faraut_amyg_sd/np.sqrt(len(faraut_amyg_mean))

    def func(x,a,tau,b):
        return a*((np.exp(-x/tau))+b)

    neg_mean_diffs = []

    mean_diff = np.diff(faraut_amyg_mean)

    for diff in range(len(mean_diff)):

        if mean_diff[diff] <= 0:

            neg_mean_diffs.append(diff)

    first_neg_mean_diff = np.min(neg_mean_diffs)

    faraut_amyg_pars,cov = curve_fit(func,x_m[first_neg_mean_diff:],faraut_amyg_mean[first_neg_mean_diff:],p0=[1,100,1],bounds=((0,np.inf)))

    repeated_taus.append(faraut_amyg_pars[1])

    # plt.plot(x_m,faraut_amyg_mean,label='original data')
    # plt.plot(x_m[first_neg_mean_diff:],func(x_m[first_neg_mean_diff:],*faraut_amyg_pars),label='fit curve')
    # plt.legend(loc='upper right')
    # plt.xlabel('lag (ms)')
    # plt.ylabel('mean autocorrelation')
    # plt.title('Mean of all human amygdala units \n Faraut (simulated trials)')
    # plt.text(710,0.11,'tau = %i' %faraut_amyg_pars[1])
    # plt.show()

    #%% Histogram of taus

    # plt.hist(np.log(faraut_amyg_taus))
    # plt.axvline(faraut_amyg_pars[1],color='r',linestyle='dashed',linewidth=1)
    # plt.xlabel('log(tau)')
    # plt.ylabel('count')
    # plt.title('%i human amygdala units \n Faraut (simulated trials)' %len(faraut_amyg_taus))
    # plt.show()

    #%% Correlation matrix

    # faraut_amyg_mean_matrix = np.mean(faraut_amyg_correlation_matrices,axis=0)

    # plt.imshow(faraut_amyg_mean_matrix)
    # plt.tight_layout()
    # plt.title('Faraut Amygdala \n  (simulated trials)')
    # plt.xlabel('lag (ms)')
    # plt.ylabel('lag (ms)')
    # plt.xticks(range(0,20,2),range(0,1000,100))
    # plt.yticks(range(0,20,2),range(0,1000,100))
    # plt.colorbar()
    # plt.show()

#%% Distribution of simulated taus

plt.hist(np.log(repeated_taus))
plt.xlabel('log(tau)')
plt.ylabel('count')
plt.title('100 repeats of 250 simulated trials per unit')
plt.show()


from math import log
import operator
import matplotlib.pyplot as plt

def create_data_set():
    data_set = [[1, 1, 'yes'],
                [1, 1, 'yes'],
                [1, 0, 'no'],
                [0, 1, 'no'],
                [0, 1, 'no']]
    labels = ['no surfacing', 'flippers']
    # change to discrete values
    return data_set, labels

# 定义文本框和箭头格式
decisionNode = dict(boxstyle="sawtooth", fc="0.8")
leafNode = dict(boxstyle="round4", fc="0.8")
arrow_args = dict(arrowstyle="<-")

# 绘制带箭头的注解
def plotNode(nodeTxt, centerPt, parentPt, nodeType):
    createPlot.ax1.annotate(nodeTxt, xy=parentPt, xycoords='axes fraction',
                            xytext=centerPt, textcoords='axes fraction',
                            va="center", ha="center", bbox=nodeType, arrowprops=arrow_args)

# 绘树主函数
def createPlot(inTree):
    fig = plt.figure(1, facecolor='white')
    fig.clf()
    # 设置坐标轴数据
    axprops = dict(xticks=[], yticks=[])
    # # 无坐标轴
    createPlot.ax1 = plt.subplot(111, frameon=False, **axprops)
    # 带坐标轴
    # createPlot.ax1=plt.subplot(111,frameon=False)
    plotTree.totalW = float(getNumLeafs(inTree))
    plotTree.totalD = float(getTreeDepth(inTree))
    # 两个全局变量plotTree.xOff和plotTree.yOff追踪已经绘制的节点位置，
    # 以及放置下一个节点的恰当位置
    plotTree.xOff = -0.5 / plotTree.totalW;
    plotTree.yOff = 1.0;
    plotTree(inTree, (0.5, 1.0), '')
    plt.show()

'''
获取叶节点的数目
'''

def getNumLeafs(myTree):
    numLeafs = 0
    firstSides = list(myTree.keys())
    firstStr = firstSides[0]
    secondDict = myTree[firstStr]
    for key in secondDict.keys():
        # 判断节点是否为字典来以此判断是否为叶子节点
        if type(secondDict[
                    key]).__name__ == 'dict':  # test to see if the nodes are dictonaires, if not they are leaf nodes
            numLeafs += getNumLeafs(secondDict[key])
        else:
            numLeafs += 1
    return numLeafs

'''
获取树的层数
'''

def getTreeDepth(myTree):
    maxDepth = 0
    firstSides = list(myTree.keys())
    firstStr = firstSides[0]
    secondDict = myTree[firstStr]
    for key in secondDict.keys():
        if type(secondDict[
                    key]).__name__ == 'dict':  # test to see if the nodes are dictonaires, if not they are leaf nodes
            thisDepth = 1 + getTreeDepth(secondDict[key])
        else:
            thisDepth = 1
        if thisDepth > maxDepth: maxDepth = thisDepth
    return maxDepth

def plotTree(myTree, parentPt, nodeTxt):  # if the first key tells you what feat was split on
    # 计算宽与高
    numLeafs = getNumLeafs(myTree)  # this determines the x width of this tree
    depth = getTreeDepth(myTree)
    firstStr = list(myTree.keys())[0]  # the text label for this node should be this
    cntrPt = (plotTree.xOff + (1.0 + float(numLeafs)) / 2.0 / plotTree.totalW, plotTree.yOff)
    # 标记子节点属性值
    plotMidText(cntrPt, parentPt, nodeTxt)
    plotNode(firstStr, cntrPt, parentPt, decisionNode)
    secondDict = myTree[firstStr]
    # 减少y偏移
    plotTree.yOff = plotTree.yOff - 1.0 / plotTree.totalD
    for key in secondDict.keys():
        if type(secondDict[
                    key]).__name__ == 'dict':  # test to see if the nodes are dictonaires, if not they are leaf nodes
            plotTree(secondDict[key], cntrPt, str(key))  # recursion
        else:  # it's a leaf node print the leaf node
            plotTree.xOff = plotTree.xOff + 1.0 / plotTree.totalW
            plotNode(secondDict[key], (plotTree.xOff, plotTree.yOff), cntrPt, leafNode)
            plotMidText((plotTree.xOff, plotTree.yOff), cntrPt, str(key))
    plotTree.yOff = plotTree.yOff + 1.0 / plotTree.totalD

# if you do get a dictonary you know it's a tree, and the first element will be another dict

'''
计算父节点和子节点的中间位置，并在此处添加简单的文本标签信息
'''

def plotMidText(cntrPt, parentPt, txtString):
    xMid = (parentPt[0] - cntrPt[0]) / 2.0 + cntrPt[0]
    yMid = (parentPt[1] - cntrPt[1]) / 2.0 + cntrPt[1]
    createPlot.ax1.text(xMid, yMid, txtString, va="center", ha="center", rotation=30)

def retrieveTree(i):
    listOfTrees = [{'no surfacing': {0: 'no', 1: {'flippers': {0: 'no', 1: 'yes'}}}},
                   {'no surfacing': {0: 'no', 1: {'flippers': {0: {'head': {0: 'no', 1: 'yes'}}, 1: 'no'}}}}
                   ]
    return listOfTrees[i]

'''
使用决策树的分类函数
'''

def classify(inputTree, featLabels, testVec):
    firstStr = inputTree.keys()[0]
    secondDict = inputTree[firstStr]
    featIndex = featLabels.index(firstStr)
    key = testVec[featIndex]
    valueOfFeat = secondDict[key]
    if isinstance(valueOfFeat, dict):
        classLabel = classify(valueOfFeat, featLabels, testVec)
    else:
        classLabel = valueOfFeat
    return classLabel

def storeTree(inputTree,filename):
    import pickle
    fw = open(filename,'w')
    pickle.dump(inputTree,fw)
    fw.close()

def grabTree(filename):
    import pickle
    fr = open(filename)
    return pickle.load(fr)

if __name__ == '__main__':
    dateSet, labes = create_data_set()
    createPlot(retrieveTree(1))
    print(retrieveTree(0), labes, [1, 0])
    print(retrieveTree(0), labes, [1, 1])
    # todo TypeError: file must have a 'write' attribute
    # 应该是编码的问题，但是目前不知道如何解决 time:2019年02月24日10:24:50
    print(storeTree(retrieveTree(0),'classifierStorage.txt'))
    # print(grabTree('classifierStorage.txt'))


#!/usr/bin/env python
#-*-coding:utf-8
from kazoo.client import KazooClient
from kazoo.security import *
from kazoo.exceptions import *
from flask import Flask
from flask_jsonrpc import JSONRPC,site
import sys
import os,time
import argparse
import logging
import configparser
from conn_db import *

logging.basicConfig(level=logging.DEBUG,
                format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s,PID:%(process)d',
                datefmt='%a, %d %b %Y %H:%M:%S',
                filename='/var/log/zk.log',
                filemode='a')

app = Flask(__name__)
jsonrpc = JSONRPC(app, '/api')

def CreateNode(path, value=b'None', acl=None,
               ephemeral=False,sequence=False,
               makepath=False,Recursively=False):

    if Recursively == True and value:
        raise  ValueError('Only Options Value or Recursively')
    else:
        zk.start()
        if Recursively == True and not value:
            zk.ensure_path(path, acl)
        else:
            zk.create(path, value, acl, ephemeral, sequence, makepath)
        zk.stop()

def DeleteNode(path=None, version=-1, recursive=False):
    zk.start()
    zk.delete(path, version, recursive)
    zk.stop()

def GetNode(path, watch=None, include_data=None):
    """

    :rtype : object
    """

    zk.start()
    if include_data:
        value, Stat=zk.get_children(path, watch, include_data)
    else:
        value, Stat=zk.get(path, watch)
    zk.stop()
    value.sort()

def SetNode(path, value, version=-1):
    zk.start()
    zk.set(path,value,version)
    zk.stop()

list=[]   #需要初始化的zookeeper结果列表

def deco(func):
    def _deco(*args, **kwargs):
        env=kwargs['env']
        obj=connect(env)
        url=obj.zk()
        #print('url',url)
        zk = KazooClient(url)
        if kwargs.get('path'):
            prefix='/platform'
            kwargs['path'] = prefix + kwargs['path']
        zk.start()
        ret = func(zk=zk,*args, **kwargs)
        zk.stop()
        return ret
    return _deco

@jsonrpc.method('load')
@deco
def load(filestream,env=None,zk=None):
    try:
        for line in filestream:
            line='/platform' + line
            list=line.split('=',maxsplit=1)

            if zk.exists(list[0]) != None:
                zk.set(list[0],list[1].encode())

            else:
                zk.ensure_path(list[0])
                zk.set(list[0],list[1].encode())
        return True
    except Exception as e:
        return e
        #print(e)

#

@jsonrpc.method('dump')
@deco
def dump(path,env=None,zk=None):
    #print('dumppath=',path)
    try:
        del list[:]
        subdump(path,zk)
        return list
    except Exception as e:
        return e

def subdump(path,zk):
    try:
        if zk.get_children(path):       #第一次判断必须以子节点为迭代条件
            nodes=zk.get_children(path)
            for node in nodes:
                new_path=path+'/'+node
                if len(zk.get_children(new_path)) >= 0:   #以路径是否有子节点为迭代条件
                #if value == None or len(value) == 0:   #以路径是否有值为迭代条件
                    subdump(new_path,zk)
        else:
            new_value,new_status=zk.get(path)
            if new_value == None:
                new_value=b""
            result=path+'='+new_value.decode()
            list.append(result[9:])
    except Exception as e:
        raise Exception(path,e)

@jsonrpc.method('exists')
@deco
def Exists(path, zk=None, watch=None):
    try:
        if zk.exists(path):
            return True
        else:
            return False
    except Exception as err:
        return

@jsonrpc.method('create')
@deco
def Create(path=None, value="", acl=None,sequence=False,Recursively=False,env=None,zk=None):
    if Recursively and value:
        raise ValueError("the recursively=True can't create when you put value='' args")
    elif Recursively:
        try:
            zk.ensure_path(path, acl)
            return True
        except Exception as err:
            return False
    else:
        try:
            if value =="":
                value=""
            if value is None:
                value=""
            if value is not None:
                value=str.encode(value)
            zk.create(path, value, acl, sequence)
            return True
        except Exception as err:
            return err

@jsonrpc.method('delete')
@deco
def Delete(path, version=-1, recursive=False,env=None,zk=None):
    try:
        zk.delete(path, version, recursive)
        return True
    except Exception as err:
        return False

@jsonrpc.method('get')
@deco
def Get(path, watch=None, include_data=None,env=None,zk=None):
    '''zookeeper获取数据与获取子节点'''
    if include_data:
        include_data=True
        value, Stat=zk.get_children(path, watch, include_data)
        value.sort()
        return value
    else:
        value, Stat=zk.get(path, watch)
        if not value:
            return None
        elif isinstance(value,bytes) or len(value) >= 0:
            return value.decode(encoding="utf-8")
        else:
            value.sort()
            return value

@jsonrpc.method('set')
@deco
def Set(path, value="", version=-1,env=None,zk=None):
    if value==None:
        try:
            zk.set(path,value,version)
            return True
        except Exception as err:
            return err
    else:
        value=str.encode(value)
        try:
            zk.set(path,value,version)
            return True
        except Exception as err:
            return err

if __name__ == '__main__':
    if len(sys.argv[1:]) > 0:
        parser = argparse.ArgumentParser(description='CREATE,DELETE,SET,GET method operate zookeeper')
        parser.add_argument('--host', default='127.0.0.1', help='connect host, --host "IP"')
        parser.add_argument('--port', default='2181', help='connect host port, --port "port"')
        parser.add_argument('--action', help='choices method, --action "create"',choices=['create', 'delete', 'set', 'get'])
        parser.add_argument('--path', help='path of znode')
        parser.add_argument('--data',default='', help='znode data')
        parser.add_argument('-r', '--recursively',action='store_true',help='whether recursively create')
        parser.add_argument('-e', '--ephemeral',action='store_true',help='create ephemeral znode')
        parser.add_argument('-s', '--sequence',action='store_true',help='create sequence znode')
        parser.add_argument('--acl',help='define acl and you must provide a dictionary,for example:{"perms":3,"scheme":"ip","id":"192.168.0.0"} ACL scheme chiose (world,auth,digest,ip,super,sasl)')
        parser.add_argument('--timeout',help='zookeeper connect timeout',type=int,default=10.0)
        parser.add_argument('--read_only',action='store_false',help='read only')
        parser.add_argument('-v', '--version',default=-1,type=int,help='znode version')
        args = parser.parse_args()
        zk = KazooClient(hosts="%s:%s" % (args.host,args.port),
                     timeout=args.timeout, client_id=None, handler=None,
                     default_acl=None, auth_data=None, read_only=None,
                     randomize_hosts=True, connection_retry=None,
                     command_retry=None, logger=None)
        args.datas=args.data.encode(encoding="utf-8")
#
        if args.action == 'create':
            if isinstance(args.acl,str):
                acl=eval(args.acl)
                ACLS=[ACL(perms=acl['perms'], id=Id(scheme=acl['scheme'], id=acl['id']))]
            else:
                ACLS=None
            CreateNode(path=args.path, value=args.datas, acl=ACLS,
                   ephemeral=args.ephemeral,sequence=args.sequence,
                   makepath=False,Recursively=args.recursively)
        elif args.action == 'set':
            SetNode(path=args.path,value=args.datas, version=-1)
        elif args.action == 'get':
            GetNode(path=args.path, watch=None, include_data=None)
        elif args.action == 'delete':
            DeleteNode(path=args.path, version=args.version, recursive=args.recursively)
        else:
            parser.print_usage()
    else:
        app.run(host='0.0.0.0',port=8081, processes=10,debug=True,use_reloader=True)


"""
Interpolated Antenna array gain
================================
"""
import time
import matplotlib.pyplot as plt

import pyant

beam = pyant.beam_of_radar("e3d_stage1", "array")

interp_beam = pyant.models.InterpolatedArray()
interp_beam.generate_interpolation(beam, resolution=(200, 200, None))

fig, axes = plt.subplots(1, 2)

start_time = time.time()
pyant.plotting.gain_heatmap(beam, ax=axes[0], resolution=100, min_elevation=80.0)
axes[0].set_title("Array")
array_time = time.time() - start_time

start_time = time.time()
pyant.plotting.gain_heatmap(interp_beam, ax=axes[1], resolution=100, min_elevation=80.0)
axes[1].set_title("Interpolated")
interp_time = time.time() - start_time

print(f"Heatmap plot antenna array: {array_time:.1f} seconds")
print(f"Heatmap plot interpolated array: {interp_time:.1f} seconds")
print(f"Speedup = factor of {array_time/interp_time:.2f}")

plt.show()


"""Utility functions for Markive."""

import datetime
import os

import yaml

def get_current_entry(markive_folder: str, date=None) -> str:
    """Get the current entry based on today's date.

    Arguments:
        markive_folder: String path of where entries are stored

    Returns:
        entry: Today's entry as a filepath.
    """
    date = date if date else datetime.datetime.now()
    month = os.path.join(markive_folder, date.strftime("%B-%Y"))
    entry = os.path.join(month, date.strftime("%b-%d.md"))
    return entry

def read_config(markive_folder: str) -> dict:
    """Read the configuration YAML file from the Markive folder.

    Arguments:
        markive_folder: String path of the markive folder.

    Returns:
        config: Dictionary of configuration parameters.
    """
    config = {
        "pre_write": '',
        "post_write": '',
        "template": ("---\n"
                     "date: %Y-%m-%d\n"
                     "---\n"),
    }
    config_file = os.path.join(markive_folder, "config.yml")
    if os.path.exists(config_file):
        with open(config_file, 'r') as file:
            content = yaml.safe_load(file.read())
            if content:
                config.update(content)
    return config


from django.shortcuts import render
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required, permission_required
from datetime import datetime
from travels.models import Travel
from clients.models import Client
from drivers.models import Driver
from django.core.exceptions import ObjectDoesNotExist

def sendTravelList(request, error=None):
    client = Client.objects.get(username=request.user)
    travels = Travel.objects.filter(client=client)
    context = {
        'travels': [t.getTravelDict() for t in travels],
        'error': error
    }
    return render(request, 'clients/clients.html', context=context)

@login_required
@permission_required("clients.client")
def travelsList(request):
    if 'refundReq' in request.GET:
        try:
            travel = Travel.objects.get(pk=request.GET['refundReq'])

            if travel.client.username != request.user.username:
                return sendTravelList(request, error='Permission Error')
            travel.refound_request = True
            travel.save()
        except ObjectDoesNotExist:
            return sendTravelList(request, error='Travel not exists')

    if 'reportDriver' in request.GET:
        try:
            travel = Travel.objects.get(pk=request.GET['reportDriver'])
            if travel.client.username != request.user.username:
                return sendTravelList(request, error='Permission Error')
            status = travel.reportDriver()
            if not status:
                return sendTravelList(request, error="Double report error")
        except ObjectDoesNotExist:
            return sendTravelList(request, error='Travel not exists')
    if 'removeTravel' in request.GET:
        try:
            travel = Travel.objects.get(pk=request.GET['removeTravel'])
            if not travel.isRemovable() or travel.client.username != request.user.username:
                return sendTravelList(request, error='Permission Error')
            travel.delete()

        except ObjectDoesNotExist:
            return sendTravelList(request, error='Travel not exists')

    return sendTravelList(request)


import logging
import superdesk
from superdesk.etree import etree
from superdesk.celery_app import celery
from superdesk.locators.locators import find_cities
from superdesk.metadata.item import Priority

parsers = []
providers = {}
allowed_providers = []
provider_errors = {}
publish_errors = []
logger = logging.getLogger(__name__)

from .commands.remove_expired_content import RemoveExpiredContent
from .commands.update_ingest import UpdateIngest

def init_app(app):
    from .ingest_provider_model import IngestProviderResource, IngestProviderService
    endpoint_name = 'ingest_providers'
    service = IngestProviderService(endpoint_name, backend=superdesk.get_backend())
    IngestProviderResource(endpoint_name, app=app, service=service)

    from .io_errors import IOErrorsService, IOErrorsResource
    endpoint_name = 'io_errors'
    service = IOErrorsService(endpoint_name, backend=superdesk.get_backend())
    IOErrorsResource(endpoint_name, app=app, service=service)

    from .ingest import IngestResource, IngestService
    endpoint_name = 'ingest'
    service = IngestService(endpoint_name, backend=superdesk.get_backend())
    IngestResource(endpoint_name, app=app, service=service)
    from .commands.add_provider import AddProvider  # NOQA

def register_provider(type, provider, errors):
    providers[type] = provider
    allowed_providers.append(type)
    provider_errors[type] = dict(errors)

@celery.task(soft_time_limit=15)
def update_ingest():
    UpdateIngest().run()

@celery.task
def gc_ingest():
    RemoveExpiredContent().run()

class ParserRegistry(type):
    """Registry metaclass for parsers."""

    def __init__(cls, name, bases, attrs):
        """Register sub-classes of Parser class when defined."""
        super(ParserRegistry, cls).__init__(name, bases, attrs)
        if name != 'Parser':
            parsers.append(cls())

class Parser(metaclass=ParserRegistry):
    """Base Parser class for all types of Parsers like News ML 1.2, News ML G2, NITF, etc."""

    def parse_message(self, xml, provider):
        """Parse the ingest XML and extracts the relevant elements/attributes values from the XML."""
        raise NotImplementedError()

    def can_parse(self, xml):
        """Test if parser can parse given xml."""
        raise NotImplementedError()

    def qname(self, tag, ns=None):
        if ns is None:
            ns = self.root.tag.rsplit('}')[0].lstrip('{')
        elif ns is not None and ns == 'xml':
            ns = 'http://www.w3.org/XML/1998/namespace'

        return str(etree.QName(ns, tag))

    def set_dateline(self, item, city=None, text=None):
        """
        Sets the 'dateline' to the article identified by item. If city is passed then the system checks if city is
        available in Cities collection. If city is not found in Cities collection then dateline's located is set with
        default values.

        :param item: article.
        :param city: Name of the city, if passed the system will search in Cities collection.
        :param text: dateline in full. For example, "STOCKHOLM, Aug 29, 2014"
        """

        item['dateline'] = {}

        if city:
            cities = find_cities()
            located = [c for c in cities if c['city'] == city]
            item['dateline']['located'] = located[0] if len(located) > 0 else {'city_code': city, 'city': city,
                                                                               'tz': 'UTC', 'dateline': 'city'}
        if text:
            item['dateline']['text'] = text

    def map_priority(self, source_priority):
        """
        Maps the source priority to superdesk priority
        :param str source_priority:
        :return int: priority of the item
        """
        if source_priority and source_priority.isdigit():
            if int(source_priority) in Priority.values():
                return int(source_priority)

        return Priority.Ordinary.value

def get_xml_parser(etree):
    """Get parser for given xml.

    :param etree: parsed xml
    """
    for parser in parsers:
        if parser.can_parse(etree):
            return parser

# must be imported for registration
import superdesk.io.nitf
import superdesk.io.newsml_2_0
import superdesk.io.newsml_1_2
import superdesk.io.wenn_parser
import superdesk.io.teletype
import superdesk.io.email
register_provider('search', None, [])

superdesk.privilege(name='ingest_providers', label='Ingest Channels', description='User can maintain Ingest Channels.')


from django.urls import path
from . import views
from .views import ProductList, ProductDetailView, ProductCreateView, ProductDeleteView, ProductUpdateView, login

urlpatterns = [
    # path — означает путь. В данном случае путь ко всем товарам у нас останется пустым, позже станет ясно почему
    path('', ProductList.as_view(), name='products'),
    path('<int:pk>', ProductDetailView.as_view(), name='product'),
    path('create/', ProductCreateView.as_view(), name='product_create'),
    path('update/<int:pk>', ProductUpdateView.as_view(), name='product_update'),
    path('delete/<int:pk>', ProductDeleteView.as_view(), name='product_delete'),

]


from __future__ import division
import os
import math
#import json
import random
#import pprint
#import scipy.misc
import numpy as np
from time import gmtime, strftime
#from osgeo import gdal
import glob
#from skimage.transform import resize
#from sklearn import preprocessing as pre
#import matplotlib.pyplot as plt
import cv2
import pathlib
#from sklearn.feature_extraction.image import extract_patches_2d
#from skimage.util import view_as_windows
import sys
import pickle
# Local
import deb
import argparse
from sklearn.preprocessing import StandardScaler
from skimage.util import view_as_windows

from PIL import Image
from osgeo import gdal

#Input configuration
parser = argparse.ArgumentParser(description='')
parser.add_argument('-pl','--patch_len', dest='patch_len', type=int, default=5, help='# timesteps used to train')
parser.add_argument('-ps','--patch_step', dest='patch_step', type=int, default=0, help='Debug')
parser.add_argument('--band_n', dest='band_n', type=int, default=6, help='Debug')
parser.add_argument('--path', dest='path', default="../../data/AP1_Para/", help='Data path')
parser.add_argument('--class_n', dest='class_n', type=int, default=2, help='Class number')
parser.add_argument('-bs','--balance_samples_per_class', dest='balance_samples_per_class',type=int,default=None, help="Class number. 'local' or 'remote'")
parser.add_argument('-ttmn','--train_test_mask_name', dest='train_test_mask_name',default="TrainTestMask.tif", help="Class number. 'local' or 'remote'")
parser.add_argument('-psv','--patches_save', dest='patches_save',default=True, help="Patches npy store")

a = parser.parse_args()
np.set_printoptions(suppress=True)

#class Patches(object):
#	def __init__(self,a):
#		self.a=a
#		deb.prints(self.a)
#	def load_image():
#	def train_test_split():
#	def normalize():

#def load_image():
def print_min_max_avg(in_):
	print(np.min(in_),np.max(in_),np.average(in_))

def normalize(im, mask):
	h,w,channels=im.shape
	im_flat=np.reshape(im,(h*w,channels))
	mask_flat=np.reshape(mask,-1)

	deb.prints(im_flat.shape)
	deb.prints(mask_flat.shape)
	train_flat=im_flat[mask_flat==255,:]

	deb.prints(train_flat.shape)

	print_min_max_avg(train_flat)

	scaler=StandardScaler()
	scaler.fit(train_flat)
	train_norm_flat=scaler.transform(train_flat)

	print_min_max_avg(train_norm_flat)

	im_norm_flat=scaler.transform(im_flat)
	im_norm=np.reshape(im_norm_flat,(h,w,channels))
	deb.prints(im_norm.shape)

	print("FINISHED NORMALIZING, RESULT:")
	print_min_max_avg(im_norm)
	return im_norm

def patches_extract(im,patch_len,band_n,step,debug=1):
	deb.prints(band_n)
	if band_n != -1:
		window_shape=(patch_len,patch_len,band_n)
	else:
		window_shape=(patch_len,patch_len)
	if debug:
		deb.prints(window_shape)
		deb.prints(im.shape)
	return np.squeeze(view_as_windows(im, window_shape, step=step))

def im_reconstruct_from_patches(patches,a):
	patches=patches.reshape((a.patches_info['rows'], \
		a.patches_info['cols']) + patches.shape[1:])
	out=np.zeros(a.im_shape[:-1])
	for row in range(a.patches_info['rows']):
		for col in range(a.patches_info['cols']):
			out[row*a.patch_step:(row+1)*a.patch_step, \
				col*a.patch_step:(col+1)*a.patch_step]= \
				patches[row,col]
	return out

if __name__ == '__main__':
	path={}
	path['raster']=a.path+'L8_224-66_ROI_clip.tif'
	path['label']=a.path+'labels.tif'
	path['train_test_mask']=a.path+'TrainTestMask.png'
	# Read image
	im = gdal.Open(path['raster'])
	im = np.array(im.ReadAsArray())
	im = np.transpose(im, (1, 2, 0))
	im = im[0:-1,0:-1,:] # Eliminate non used pixels
	im = im.astype(np.float32)
	deb.prints(im.shape)
	deb.prints(im.dtype)

	a.band_n=im.shape[2]
	deb.prints(a.band_n)
	masks={}
	masks['train_test']=cv2.imread(path['train_test_mask'],0).astype(np.uint8)
	masks['label']=cv2.imread(path['label'],-1).astype(np.uint8)
	masks['label'][masks['label']==2]=1 # Only use 2 classes
	cv2.imwrite("label_original.png",masks['label']*255)
	deb.prints(masks['train_test'].shape)
	deb.prints(masks['label'].shape)

	# ============ Normalize ========================
	im = normalize(im, masks['train_test'])
	deb.prints(im.shape)

	# =========== Extract patches  ===================
	patches={}
	patches['im'] = patches_extract(im,a.patch_len,a.band_n,a.patch_step)
	patches['label'] = patches_extract(masks['label'],a.patch_len,-1,a.patch_step)
	patches['train_test'] = patches_extract(masks['train_test'],a.patch_len,-1,a.patch_step)

	a.im_shape=im.shape
	a.patches_info={}
	a.patches_info['rows']=patches['label'].shape[0]
	a.patches_info['cols']=patches['label'].shape[1]

	deb.prints(patches['label'].shape)
	deb.prints(patches['im'].shape)

	#print()
	patches['im']=patches['im'].reshape((a.patches_info['rows']*a.patches_info['cols'],)+patches['im'].shape[2:])
	patches['label']=patches['label'].reshape((a.patches_info['rows']*a.patches_info['cols'],)+patches['label'].shape[2:])
	patches['train_test']=patches['train_test'].reshape((a.patches_info['rows']*a.patches_info['cols'],)+patches['train_test'].shape[2:])

	# ====== Reconstruct label. Just for assertion here. =====
	label_reconstruct = im_reconstruct_from_patches(patches['label'],a)
	deb.prints(patches['im'].shape)
	deb.prints(label_reconstruct.shape)
	deb.prints(np.unique(label_reconstruct))
	cv2.imwrite("label_reconstruct.png",label_reconstruct*255)

	id_train_test=np.zeros(patches['im'].shape[0])
	# ====== Train test split
	for idx in range(patches['im'].shape[0]):
		if np.all(patches['train_test'][idx]>=1):
			id_train_test[idx]=True
		else:
			id_train_test[idx]=False

	names={"train":{},"test":{}}
	names['train']['im']='train_ims.npy'

	patches_im_name="patches_im.npy"
	patches_label_name="patches_label.npy"
	id_train_test_name="id_train_test.npy"

	np.save(patches_im_name,patches['im'])
	np.save(patches_label_name,patches['label'])
	np.save(id_train_test_name,id_train_test)

	deb.prints(id_train_test.shape)
	deb.prints(patches['im'][id_train_test==True].shape)
	deb.prints(patches['im'][id_train_test==False].shape)

	# ============== Store patches into npy

	# Extract
	# Train test split
	#patches=Patches(a)
	#patches.load_image()
	#patches.train_test_split()
	#patches.normalize()
	#patches.extract()


from string import Template
from typing import Optional

from gql import Client, gql
from gql.transport.aiohttp import AIOHTTPTransport

MECH_SUBGRAPH_URL = "https://api.studio.thegraph.com/query/46780/mech/v0.0.1"
AGENT_QUERY_TEMPLATE = Template(
    """{
    createMeches(where:{agentId:$agent_id}) {
        mech
    }
}
"""
)

def query_agent_address(agent_id: int) -> Optional[str]:
    """
    Query agent address from subgraph.

    :param agent_id: The ID of the agent.
    :type agent_id: int
    :return: The agent address if found, None otherwise.
    :rtype: Optional[str]
    """
    client = Client(transport=AIOHTTPTransport(url=MECH_SUBGRAPH_URL))
    response = client.execute(
        document=gql(
            request_string=AGENT_QUERY_TEMPLATE.substitute({"agent_id": agent_id})
        )
    )
    mechs = response["createMeches"]  # pylint: disable=unsubscriptable-object
    if len(mechs) == 0:
        return None

    (record,) = mechs
    return record["mech"]


from urllib import response
from django.test import TestCase,Client
from django.urls import reverse
from home.models import Blog,Contact

class TestViews(TestCase):
    def setup(self):
        self.client = Client()

    def test_index_GET(self):
        response = self.client.get(reverse('index:index'))
        self.assertEqual(response.status_code,200)
        self.assertTemplateUsed(response,'home/index.html')

    def test_contact_POST(self):
        response = self.client.post(reverse('index:contact'),{
            'name':'automated-test',
            'email':'aryanjainak@gmail.com',
            'phone_no': '3434',
            'subject' : 'Subject',
            'message' : 'message hai'
        })
        self.assertEqual(response.status_code,302)
        query= Contact.objects.get(id=1)
        self.assertEqual(query.name,'automated-test')


# -*- coding: utf-8 -*-

"""
@Author: xiezizhe
@Date: 14/6/2020 下午6:48
"""
from typing import List

class Solution:

    def dfs(self, nums: List[int], idx: int) -> List[List[int]]:
        if len(nums) <= idx + 1:
            return [[nums[idx]]]

        f = nums[idx]
        res = []
        for p in self.dfs(nums, idx + 1):
            for i in range(1, len(p)):
                res.append(p[:i] + [f] + p[i:])
            res.append([f] + p)
            res.append(p + [f])

        return res

    def permute(self, nums: List[int]) -> List[List[int]]:
        if nums is None or len(nums) == 0:
            return [[]]

        if len(nums) == 1:
            return [nums]

        return self.dfs(nums, 0)

if __name__ == "__main__":
    s = Solution()
    print(s.permute([1]))


"""
The following script has 4 main sections.
We first calculate the optimal investment weights of a trading strategy
by optimization. Then, we calculate the in and out of sample returns
for the strategy and the chosen benchmark.
Ultimately, we calculate performance and risk ratios and plot portfolio
characteristics of the investment strategy in comparison to the benchmark.
 """

# load libraries
import yfinance as yf
import pandas as pd
import numpy as np
import math as m
import matplotlib.pyplot as plt
import matplotlib.ticker as mtick
import matplotlib.dates as mdates
from scipy.optimize import minimize
from scipy.optimize import Bounds
from sklearn.linear_model import LinearRegression
import dataframe_image as dfi

#############################################################################
#                                                                           #
#                                #SECTION 1                                 #
#       Define key parameters, import needed files and define investment    #
#       universe of stocks which the strategy invests in                    #
#                                                                           #
#############################################################################

# define parameters of investment period start (start_backtesting as maximum time period!)
# actual time could be shorter depending on availability of data!
start_backtesting = np.datetime64("2011-01-01")
end_backtesting = np.datetime64("2021-12-31")
end_out_sample = np.datetime64("2022-05-21")

# Import needed CSV files
stocks_index = pd.read_csv("files/index_constituents_data.csv", index_col=0)
benchmark = pd.read_csv("files/benchmark.csv", index_col=0)
Gross_Price = pd.read_csv("files/Gross_Prices_EUR.csv", index_col=0)
Net_Price = pd.read_csv("files/Net_Prices_EUR.csv", index_col=0)

# format index to date
Gross_Price.index = pd.to_datetime(Gross_Price.index)
Net_Price.index = pd.to_datetime(Net_Price.index)
benchmark.index = pd.to_datetime(benchmark.index)

"""
Select stocks which the strategy invests in
our strategy goes long in the top 10 highest dividend stocks and short in the
top 10 lowest dividend stocks --> We select these stocks as our investmnet stocks
Also: drop BHP stock for which dividends have not been calculated correctly!
"""

stocks_index = stocks_index.sort_values(by="Yield", ascending=False)
stocks_index = stocks_index.drop("BHP.L")
stocks_invest = stocks_index.iloc[np.r_[0:10, 39:49]]

# select prices of stocks which we invest in
Gross_Price_selected = Gross_Price[stocks_invest.index]
Net_Price_selected = Net_Price[stocks_invest.index]

#############################################################################
#                                                                           #
#                                #SECTION 2                                 #
#     Calculate Optimal Strategy stock weights with various constraints     #
#                       and visualize optimization                          #
#                                                                           #
#############################################################################

# estimate expected returns and var-cov matrix for optimization (with in sample data!)

rf_daily = 0
ER = Gross_Price_selected[Gross_Price_selected.index < end_backtesting].pct_change().mean()
S = Gross_Price_selected[Gross_Price_selected.index < end_backtesting].pct_change().cov()

# define functions for optimization (variance, return,  negative sharp ratio)
def pvar(w, S):
    return (w.T @ S @ w)

def pret(w, ER):
    return (w.T @ ER)

def sharpe(w, ER, S):
    return -(w.T @ (ER - rf_daily)) / ((w.T @ S @ w) ** 0.5)

# ---------------------------------------------------------------------------
# Calculate Optimized Portfolio
# ---------------------------------------------------------------------------
N = len(ER)
x0 = np.ones(N) / N

"""
#set up constraints
first constraint -> total investment = 100% long
second constraint- > Values smaller than - min_weight (shocks which are shorted)
third constraint -> Values larger than min_weight (stock which are longed)
bounds = maximum long and short % for stocks
"""

min_weight = 0.02

cons_multiple = ({"type": "eq", "fun": lambda x: np.sum(x) - 1},
                 {"type": "ineq", "fun": lambda x: -min_weight - x[10:20]},
                 {"type": "ineq", "fun": lambda x: x[0:10] - min_weight})

bounds = Bounds(-0.2, 0.2)

# only constraint for unconstrained portfolio --> Total stock weights == 100%
cons_simple = ({"type": "eq", "fun": lambda x: np.sum(x) - 1})

# calculate optimized values (GMVP and MSRP constrained and unconstrained)
GMVP_const = minimize(pvar, x0, method='SLSQP', args=S, constraints=cons_multiple,
                      options={'disp': True, 'ftol': 1e-9}, bounds=bounds)

GMVP_unconst = minimize(pvar, x0, method="SLSQP", args=S, constraints=cons_simple,
                        options={'disp': True, 'ftol': 1e-9})

MSRP_const = minimize(sharpe, x0, method='SLSQP', args=(ER, S), constraints=cons_multiple,
                      options={'disp': True, 'ftol': 1e-9}, bounds=bounds)

MSRP_unconst = minimize(sharpe, x0, method='SLSQP', args=(ER, S), constraints=cons_simple,
                        options={'disp': True, 'ftol': 1e-9})

# complete data set of selected stocks with calculated constrained MSRP values & export for presentation
stocks_invest = stocks_invest.assign(weights=MSRP_const.x)
stocks_invest = stocks_invest.assign(weights_unconst=MSRP_unconst.x)

stocks_invest_export = stocks_invest.copy()
stocks_invest_export["Yield"] = stocks_invest_export["Yield"].map('{:,.2%}'.format)
stocks_invest_export["weights"] = stocks_invest_export["weights"].map('{:,.2%}'.format)
stocks_invest_export["index_weights"] = stocks_invest_export["index_weights"].map('{:,.2%}'.format)
stocks_invest_export["weights_unconst"] = stocks_invest_export["weights_unconst"].map('{:,.2%}'.format)

dfi.export(stocks_invest_export, "plots/selected_portfolio_characteristics.png")

# ---------------------------------------------------------------------------
# Visualize efficient Frontier
# ---------------------------------------------------------------------------

"""
Generate Minimum Variance Frontier / Investment Frontier plot by doing the following:
1. Minimize the negative value of the expected portfolio return
given a deterministic variance and the same rules as above (eg. min 2% in each stock)
as restrictions in the optimization. --> We thus retrieve the maximum return given
all restrictions for a given variance.

2. We construct "random portfolios" which fulfill all the restrictions by minimizing a function
which consists of a random array and a modulus operation which should randomize the optimization
results. We can then calculate the expected return and variance of these portfolios and add them
to the plot.
"""

# calculate yearly volatility and expected return of GMVP and MSRP constrained for plotting further below
# assumption: year = 250 trading days

GMVP_const_ER = pret(w=GMVP_const.x, ER=ER) * 250
GMVP_const_VAR = m.sqrt(pvar(w=GMVP_const.x, S=S) * 250)
MSRP_const_ER = pret(w=MSRP_const.x, ER=ER) * 250
MSRP_const_VAR = m.sqrt(pvar(w=MSRP_const.x, S=S) * 250)

# define negative portfolio return function
def pret_sim(w, ER):
    return -(w.T @ ER)

# variances to loop over
var_list = np.arange(0.1, 0.6, 0.01)
bounds = Bounds(-0.2, 0.2)

# initialize lists to store results in
MVF_var_const = []
MVF_ret_const = []

# loop over variances and maximize expected return given that portfolio variance = given variance
for i in var_list:
    max_var = i
    cons = ({"type": "eq", "fun": lambda x: np.sum(x) - 1},
            {"type": "eq", "fun": lambda x: m.sqrt((x.T @ S @ x) * 250) - max_var},
            {"type": "ineq", "fun": lambda x: -min_weight - x[10:20]},
            {"type": "ineq", "fun": lambda x: x[0:10] - min_weight})

    maximized = minimize(pret_sim, x0, method='SLSQP', args=ER,
                         options={'disp': True, 'ftol': 1e-9},
                         constraints=cons, bounds=bounds)

    # only store result if optimization was successful!
    if maximized.success:
        MVF_var_const.append(i)
        MVF_ret_const.append(maximized.fun * -250)
    else:
        continue

# ---------------------------------------------------------------------------
# Visualize efficient Frontier
# ---------------------------------------------------------------------------

random_portfolio = []

# "random" function which when optimized gives "random" portfolios
def rand_funct(x, y):
    return ((x % y) / y).sum()

# weight constraints of individual stocks
cons = ({"type": "eq", "fun": lambda x: np.sum(x) - 1},
        {"type": "ineq", "fun": lambda x: -min_weight - x[10:20]},
        {"type": "ineq", "fun": lambda x: x[0:10] - min_weight})

# simulate 100 different portfolios which fulfill the constraints
i = 0
np.random.seed(0)

while i < 100:
    y = np.random.uniform(low=0, high=1, size=N)
    portfolio_sim = minimize(rand_funct, x0, method='SLSQP', args=y,
                             constraints=cons, options={'disp': True, 'ftol': 1e-9},
                             bounds=bounds)
    i = i + 1

    # only store successful optimizations
    if portfolio_sim.success:
        random_portfolio.append(portfolio_sim.x)
    else:
        continue

MVF_randvar_const = []
MVF_randret_const = []

# calculate return and variance (yearly) from calculated "random" portfolios
for i in range(len(random_portfolio)):
    exp_return = pret(w=random_portfolio[i], ER=ER)
    exp_var = pvar(w=random_portfolio[i], S=S)
    exp_return_year = exp_return * 250
    exp_var_year = m.sqrt(exp_var * 250)

    MVF_randvar_const.append(exp_var_year)
    MVF_randret_const.append(exp_return_year)

# plot results
MVF_var_const.insert(0, GMVP_const_VAR)
MVF_ret_const.insert(0, GMVP_const_ER)
CAL_x = np.linspace(0, MSRP_const_VAR + 0.2, 50, endpoint=True)
SR_MSRP = MSRP_const_ER / MSRP_const_VAR
CAL_y = 0 + SR_MSRP * CAL_x

frontier_points = pd.DataFrame({"variance": MVF_var_const, "return": MVF_ret_const})
max_ret_index = frontier_points["return"].idxmax()

fig, ax = plt.subplots(figsize=(15, 10))
plt.plot(frontier_points.variance.iloc[0:max_ret_index + 1], frontier_points["return"].iloc[0:max_ret_index + 1])
plt.plot(frontier_points.variance.iloc[max_ret_index: len(frontier_points)],
         frontier_points["return"].iloc[max_ret_index: len(frontier_points)], color="grey", ls="--")
plt.scatter(MVF_randvar_const, MVF_randret_const)
plt.scatter(GMVP_const_VAR, GMVP_const_ER, s=70)
plt.scatter(MSRP_const_VAR, MSRP_const_ER, s=70)
plt.plot(CAL_x, CAL_y)
plt.xlim([0, 0.4])
plt.ylim([0, 0.4])
plt.title("Investment Frontier Simulated", size=25)
plt.xlabel("Volatility", size=15)
plt.ylabel("Return", size=15)
plt.legend(["Minimum Variance Frontier",
            "Investment Frontier",
            "Capital Allocation Line",
            "random portfolios",
            "GMVP",
            "MSRP"],
           prop={'size': 15})
ax.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.xticks(size=15)
plt.yticks(size=15)
plt.savefig("plots/investmentfrontier.png")

#############################################################################
#                                                                           #
#                                #SECTION 2                                 #
#                           Calculate  returns of                           #
#                 strategy and benchmark in and out of sample               #
#                                                                           #
#############################################################################

# function which calculates the indexed performance (start = 1) of various stocks and start investment weights
def indexed_performance(start, prices, weights, end=None):
    # filter desired time range
    if end is None:
        temp = prices[prices.index >= start]
    else:
        temp = prices[(prices.index >= start) & (prices.index < end)]

    # define daily percentage changes + 1
    temp = temp.pct_change() + 1

    # replace first row with weights of each stock --> Initial Investment = 1
    temp.iloc[0] = weights

    # take cumulative return of each stock * initial investment (weight) and sum up horizontaly
    temp = temp.cumprod().sum(axis=1)
    return temp

# calculate weights of short stocks / long stocks  that they sum up to 1
weights_short = stocks_invest.weights[stocks_invest.weights < 0] / stocks_invest.weights[stocks_invest.weights < 0].sum()
weights_long = stocks_invest.weights[stocks_invest.weights > 0] / stocks_invest.weights[stocks_invest.weights > 0].sum()

# calculate the needed indexed performance series
in_sample_net = indexed_performance(start=start_backtesting, end=end_backtesting,
                                    weights=MSRP_const.x, prices=Net_Price_selected)

in_sample_gross = indexed_performance(start=start_backtesting, end=end_backtesting,
                                      weights=MSRP_const.x, prices=Gross_Price_selected)

out_sample_gross_short = indexed_performance(start=end_backtesting, weights=weights_short,
                                             prices=Gross_Price_selected[
                                                 stocks_invest[stocks_invest.weights < 0].index])

out_sample_gross_long = indexed_performance(start=end_backtesting, weights=weights_long,
                                            prices=Gross_Price_selected[stocks_invest[stocks_invest.weights > 0].index])

out_sample_gross = indexed_performance(start=end_backtesting, weights=MSRP_const.x,
                                       prices=Gross_Price_selected)

# combine all returns to data frame (one for in and one for out of sample performance)
in_sample = pd.DataFrame({"strategy_net": in_sample_net,
                          "strategy_gross": in_sample_gross},
                         index=in_sample_net.index)

out_sample = pd.DataFrame({"strategy_gross": out_sample_gross,
                           "long_gross": out_sample_gross_long,
                           "short_gross": out_sample_gross_short},
                          index=out_sample_gross.index)

# join benchmark data & calculate indexed performance of benchmark
in_sample = in_sample.join(benchmark)
out_sample = out_sample.join(benchmark)

in_sample[["benchmark_gross", "benchmark_net"]] = in_sample[["benchmark_gross", "benchmark_net"]].div(
    in_sample[["benchmark_gross", "benchmark_net"]].head(1).iloc[0])
out_sample[["benchmark_gross", "benchmark_net"]] = out_sample[["benchmark_gross", "benchmark_net"]].div(
    out_sample[["benchmark_gross", "benchmark_net"]].head(1).iloc[0])

# save in and out of sample returns
in_sample.to_csv("files/in_sample.csv")
out_sample.to_csv("files/out_sample.csv")

#############################################################################
#                                                                           #
#                            #SECTION 3                                     #
#        #Calculate in and out of sample risk & performance ratios          #
#                                                                           #
#############################################################################

# ---------------------------------------------------------------------------
# Annualized Sharp Ratio
# ---------------------------------------------------------------------------

# days = days per year (assumed 250 here)
# rf_rate = annualized risk free rate
# prices for which ratio should be alculated

def sharp_ratio(price, days, rf_rate):
    ER = price.pct_change().mean()
    SD = price.pct_change().std()
    SR = (ER - rf_rate / days) / SD * m.sqrt(days)

    return round(SR, 2)

# ---------------------------------------------------------------------------
# Yearly volatility
# ---------------------------------------------------------------------------

def yearly_vol(price, days, pct=True):
    vol = m.sqrt(price.pct_change().var() * days)

    if pct:
        return format(vol, ".2%")
    else:
        return vol

# ---------------------------------------------------------------------------
# alpha and beta
# ---------------------------------------------------------------------------

# period = period of returns for which regression should be run (eg. "1Y", "1M", "1d")
# kwargs = optional where only alpha, beta, x or y can be returned form function (keyword = param)
# rf_rate = risk free rate for the "period"

def alpha_beta(strategy, benchmark, period, rf_period, pct=True, **kwargs):
    daily_ret_strategy = strategy.pct_change().fillna(0) + 1
    daily_ret_BM = benchmark.pct_change().fillna(0) + 1

    # calculate excess returns for the period over the risk-free rate
    period_ret_strategy = daily_ret_strategy.groupby(pd.Grouper(freq=period)).prod() - 1 - rf_period
    period_ret_BM = daily_ret_BM.groupby(pd.Grouper(freq=period)).prod() - 1 - rf_period

    model = LinearRegression().fit(period_ret_BM.to_numpy().reshape((-1, 1)),
                                   period_ret_strategy.to_numpy().reshape((-1, 1)))

    beta = model.coef_[0][0]

    if pct:
        alpha = format(model.intercept_[0], ".2%")
    else:
        alpha = model.intercept_[0]

    results = {"alpha": alpha,
               "beta": beta,
               "x": period_ret_BM.to_list(),
               "y": period_ret_strategy.to_list()}

    if len(kwargs) == 0:
        return results
    else:
        return results.get(kwargs.get("param"))

# ---------------------------------------------------------------------------
# Maximum Drawdown
# ---------------------------------------------------------------------------

def maxdd(price, pct=True):
    # Value at time T divided by max value from time t = 0 to t = T (T> 0)
    diffmax = price / price.cummax()
    maxdd = (diffmax - 1).min()

    if pct:
        return format(maxdd, ".2%")
    else:
        return maxdd

# ---------------------------------------------------------------------------
# N Day unfiltered historical 1% VAR
# ---------------------------------------------------------------------------

# N as number of days (integer)
def NDAYVar(price, N, pct=True):
    daily_ret = price.pct_change().fillna(0) + 1
    daily_ret = daily_ret.reset_index(drop=True)
    nday_ret = daily_ret.groupby(daily_ret.index // N).prod() - 1
    VAR = nday_ret.quantile(0.01)

    if pct:
        return format(np.abs(VAR), ".2%")
    else:
        return np.abs(VAR)

# ---------------------------------------------------------------------------
# Expected N day return or total Return
# ---------------------------------------------------------------------------

def nday_ret(price, N=None, TR=False, pct=True):
    if TR:
        ret = price.tail(1)[0] / price.head(1)[0] - 1
    else:
        ret = price.pct_change().mean() * N
    if pct:
        return format(ret, ".2%")
    else:
        return ret

# define return series for which ratios should be calculated:
BM_in = in_sample["benchmark_gross"]
BM_out = out_sample["benchmark_gross"]
strategy_in = in_sample["strategy_gross"]
strategy_out = out_sample["strategy_gross"]

# assemble dicts with risk return metrics
ratios_BM_in = {"Avg. Yearly Return": nday_ret(BM_in, N=250),
                "Avg. Yearly Sharp Ratio": sharp_ratio(BM_in, days=250, rf_rate=0),
                "Max. Drawdown": maxdd(BM_in),
                "Alpha (monthly Returns)": "0%",
                "Beta (monthly Returns)": 1,
                "Avg. Ann. Vol": yearly_vol(BM_in, days=250),
                "5d 1% Hist. VAR": NDAYVar(BM_in, N=5)}

ratios_BM_out = {"Return YTD": nday_ret(BM_out, TR=True),
                 "Avg. Yearly Sharp Ratio": sharp_ratio(BM_out, days=250, rf_rate=0),
                 "Max. Drawdown": maxdd(BM_out),
                 "Alpha (weekly Returns)": "0%",
                 "Beta (weekly Returns)": 1,
                 "Avg. Ann. Vol": yearly_vol(BM_out, days=250),
                 "5d 1% Hist. VAR": NDAYVar(BM_out, N=5)}

ratios_strategy_in = {"Avg. Yearly Return": nday_ret(strategy_in, N=250),
                      "Avg. Yearly Sharp Ratio": sharp_ratio(strategy_in, days=250, rf_rate=0),
                      "Max. Drawdown": maxdd(strategy_in),
                      "Alpha (monthly Returns)": alpha_beta(strategy_in, BM_in, "1M", rf_period=0, param="alpha"),
                      "Beta (monthly Returns)": round(alpha_beta(strategy_in, BM_in, "1M", rf_period=0, param="beta"),
                                                      2),
                      "Avg. Ann. Vol": yearly_vol(strategy_in, days=250),
                      "5d 1% Hist. VAR": NDAYVar(strategy_in, N=5)}

ratios_strategy_out = {"Return YTD": nday_ret(strategy_out, TR=True),
                       "Avg. Yearly Sharp Ratio": sharp_ratio(strategy_out, days=250, rf_rate=0),
                       "Max. Drawdown": maxdd(strategy_out),
                       "Alpha (weekly Returns)": alpha_beta(strategy_out, BM_out, "1W", rf_period=0, param="alpha"),
                       "Beta (weekly Returns)": round(alpha_beta(strategy_out, BM_out, "1W", rf_period=0, param="beta"),
                                                      2),
                       "Avg. Ann. Vol": yearly_vol(strategy_out, days=250),
                       "5d 1% Hist. VAR": NDAYVar(strategy_out, N=5)}

risk_factors_in = pd.DataFrame({"Benchmark": ratios_BM_in, "Strategy": ratios_strategy_in})
risk_factors_out = pd.DataFrame({"Benchmark": ratios_BM_out, "Strategy": ratios_strategy_out})

dfi.export(risk_factors_out, "plots/risk_factors_out.png")
dfi.export(risk_factors_in, "plots/risk_factors_in.png")

# ---------------------------------------------------------------------------
# Plot Visual Example for in Sample Alpha/ Beta Calculation
# ---------------------------------------------------------------------------

# plot regression line
params = alpha_beta(strategy_in, BM_in, rf_period=0, period="1M", pct=False)
x = np.arange(min(params.get("x")) - 0.1, max(params.get("x")) + 0.1, 0.01)
fitted_y = params.get("alpha") + params.get("beta") * x

# assemble plot
fig, ax = plt.subplots(figsize=(15, 10))
plt.scatter(params.get("x"), params.get("y"))
plt.plot(x, fitted_y)
plt.plot(x, x)
plt.title("Monthly Excess Returns Strategy vs. Benchmark (in sample)", size=25)
ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
ax.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.ylabel("Monthly Excess Returns Strategy", size=15)
plt.xlabel("Monthly Excess Returns Benchmark", size=15)
plt.axhline(0, color="black", ls="--", lw=1)
plt.axvline(0, color="black", ls="--", lw=1)
plt.xticks(size=15)
plt.yticks(size=15)
plt.legend(["liner fit", "x = y"], prop={"size": 15})
plt.savefig("plots/in_sample_alphabetaplot.png")

#############################################################################
#                                                                           #
#                            #SECTION 4                                     #
#        #Calculate and plot portfolio characteristics vs. Benchmark        #
#                                                                           #
#############################################################################

# ---------------------------------------------------------------------------
# Excess Return Strategy vs. Benchmark Plot
# ---------------------------------------------------------------------------

# calculate return over out of sample period for all stocks in the benchmark index
out_sample_all = Gross_Price[Gross_Price.index > end_backtesting]
out_sample_all = out_sample_all.pct_change() + 1
out_sample_all = out_sample_all.fillna(1).cumprod()
out_return_all = out_sample_all.iloc[len(out_sample_all.index) - 1].div(out_sample_all.iloc[0]) - 1

# calculate excess return contribution as (Weight Strategy - Weigh Index) * Return
ret_contrib = stocks_index.join(stocks_invest["weights"]).fillna(0)
ret_contrib = ret_contrib[["Name", "weights", "index_weights"]]
ret_contrib["diff_weights"] = ret_contrib["weights"] - ret_contrib["index_weights"]
ret_contrib = ret_contrib.join(pd.DataFrame(out_return_all))
ret_contrib = ret_contrib.rename(columns={0: "return"})
ret_contrib["diff_return"] = ret_contrib["diff_weights"] * ret_contrib["return"]

# plot and save results
fig = plt.figure(figsize=(15, 10))
ax = fig.add_subplot(1, 1, 1)
plt.bar(ret_contrib.Name[0:10], ret_contrib.diff_return[0:10])
plt.bar(ret_contrib.Name[10:39], ret_contrib.diff_return[10:39])
plt.bar(ret_contrib.Name[39:49], ret_contrib.diff_return[39:49])
plt.legend(["Long", "Not Invested", "Short", "Currencies"])
ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.title("Decomposition of out of Sample excess Returns", size=22)
plt.ylabel("Excess Return Strategy vs. Benchmark", size=15)
plt.subplots_adjust(bottom=0.3)
plt.xticks(rotation=90)
plt.yticks(size=12)
plt.savefig("plots/excess_return_breakdown.png")

# ---------------------------------------------------------------------------
# Out of Sample Gross Strategy vs Benchmark Performance
# ---------------------------------------------------------------------------

fig = plt.figure(figsize=(15, 10))
plt.plot(out_sample[["strategy_gross", "benchmark_gross"]])
plt.legend(out_sample[["strategy_gross", "benchmark_gross"]].columns,
           prop={"size": 15})
plt.title("Out of Sample Strategy Performance (TR)", size=20)
plt.ylabel("Cumulative Return", size=15)
plt.yticks(size=15)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b-%Y'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=1))
plt.xticks(size=15)
plt.savefig("plots/outofsample_performance.png")

# ---------------------------------------------------------------------------
# In Sample Gross Strategy vs Benchmark Performance
# ---------------------------------------------------------------------------

fig = plt.figure(figsize=(15, 10))
plt.plot(in_sample[["strategy_gross", "benchmark_gross"]])
plt.legend(in_sample[["strategy_gross", "benchmark_gross"]].columns,
           prop={"size": 15})
plt.title("In Sample Strategy Performance", size=22)
plt.ylabel("Cumulative Return", size=15)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b-%Y'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=3))
plt.xticks(size=13)
plt.yticks(size=13)
plt.savefig("plots/insample_performance.png")

# ---------------------------------------------------------------------------
# In Sample Performance Net vs. Gross
# ---------------------------------------------------------------------------

fig = plt.figure(figsize=(15, 10))
plt.plot(in_sample[["strategy_gross", "strategy_net", "benchmark_gross", "benchmark_net"]])
plt.legend(in_sample[["strategy_gross", "strategy_net", "benchmark_gross", "benchmark_net"]].columns,
           prop={"size": 15})
plt.title("In Sample Strategy Performance (net and gross) Comparison", size=22)
plt.ylabel("Cumulative Return", size=15)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b-%Y'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=3))
plt.xticks(size=13)
plt.yticks(size=13)
plt.savefig("plots/insample_performance_netgross.png")

# ---------------------------------------------------------------------------
# Out of Sample Performance Decomposition Long/ Short Portfolio
# ---------------------------------------------------------------------------

fig = plt.figure(figsize=(15, 10))
plt.plot(out_sample[["strategy_gross", "benchmark_gross", "long_gross", "short_gross"]])
plt.legend(out_sample[["strategy_gross", "benchmark_gross", "long_gross", "short_gross"]].columns,
           prop={"size": 15})
plt.title("Decomposition of out of sample performance", size=22)
plt.ylabel("Cumulative Return", size=15)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b-%Y'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=1))
plt.xticks(size=13)
plt.yticks(size=13)
plt.savefig("plots/outofsamlpe_brekdown.png")

# ---------------------------------------------------------------------------
# Stock Weights Index vs. Constrained & Unconstrained Optimization
# ---------------------------------------------------------------------------

width = 0.3
fig, ax = plt.subplots(figsize=(15, 10))
ind = np.arange(len(stocks_invest))
ax.barh(ind, stocks_invest.weights, width, label="Strategy")
ax.barh(ind + width, stocks_invest.index_weights, width, label="Index")
ax.barh(ind + 2 * width, stocks_invest.weights_unconst, width, label="Unconstrained Strategy")
ax.set(yticks=ind + width, yticklabels=stocks_invest.Name)
ax.legend(prop={'size': 10})
ax.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.title("Stock Weights Strategy vs Index", size=25)
plt.yticks(size=10)
plt.xticks(size=15, ticks=np.arange(-0.4, 0.6, 0.1))
plt.savefig("plots/strategy_weights.png")

# ---------------------------------------------------------------------------
# Plot Country Weights Index vs. Strategy
# ---------------------------------------------------------------------------

country_weights_strategy = stocks_invest[["Country", "weights"]].groupby(by="Country").sum()
country_weights_index = stocks_index[["Country", "index_weights"]].groupby(by="Country").sum()
country_weights = country_weights_index.join(country_weights_strategy).fillna(0)
country_weights = pd.DataFrame(country_weights, index=country_weights_index.index)

# assemble plot
width = 0.3
fig, ax = plt.subplots(figsize=(15, 10))
ind = np.arange(len(country_weights))
ax.barh(ind, country_weights.weights, width, label="Strategy")
ax.barh(ind + width, country_weights.index_weights, width, label="Index")
ax.set(yticks=ind + width, yticklabels=country_weights.index)
ax.legend(prop={'size': 15})
ax.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.title("Country Weights Strategy vs Index", size=25)
plt.yticks(size=15)
plt.xticks(size=15)
plt.savefig("plots/country_weights.png")

# ---------------------------------------------------------------------------
# Plot Currency weights index vs. Strategy
# ---------------------------------------------------------------------------

values = stocks_index.join(stocks_invest["weights"]).fillna(0)
currency_weights_strategy = values[["Currency", "weights"]].groupby(by="Currency").sum()
currency_weights_index = values[["Currency", "index_weights"]].groupby(by="Currency").sum()

currency_weights = currency_weights_index.join(currency_weights_strategy).fillna(0)
currency_weights = currency_weights.rename(index={"GBp": "GBP"})

# assemble plot
width = 0.4
fig, ax = plt.subplots()
ind = np.arange(len(currency_weights_index))
ax.barh(ind, currency_weights.weights, width, label="Strategy")
ax.barh(ind + width, currency_weights.index_weights, width, label="Index")
ax.set(yticks=ind + width, yticklabels=currency_weights.index)
ax.legend(["Strategy", "Index"])
plt.title("Currency Weights Strategy vs Index", size=20)
ax.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.yticks(size=15)
plt.xticks(size=15)
plt.savefig("plots/curency_comparison.png")

# ---------------------------------------------------------------------------
# Plot sector weights strategy vs. index
# ---------------------------------------------------------------------------

sector_weights_strategy = stocks_invest[["Sector", "weights"]].groupby(by="Sector").sum()
sector_weights_index = stocks_index[["Sector", "index_weights"]].groupby(by="Sector").sum()
sector_weights = sector_weights_index.join(sector_weights_strategy)

# assemble plot
width = 0.4
fig, ax = plt.subplots(figsize=(15, 10))
ind = np.arange(len(sector_weights))
ax.barh(ind + width, sector_weights.weights, width, label="Strategy")
ax.barh(ind, sector_weights.index_weights, width, label="Index")
ax.set(yticks=ind + width, yticklabels=sector_weights.index)
ax.legend(prop={"size": 20})
plt.title("Sector Weights Strategy vs Index", size=25)
ax.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.yticks(size=15)
plt.xticks(size=15)
plt.subplots_adjust(left=0.2)
plt.savefig("plots/sector_weights.png")

# ---------------------------------------------------------------------------
# Plot performance Nasdaq100 vs. MSCI World vs. FTSE AW-HighDivYield
# ---------------------------------------------------------------------------

# download prices and clean data
prices = yf.download(tickers=["VHYL.AS", "EXXT.DE", "IWDA.AS"], start="2022-01-01", end=end_out_sample)
prices = prices["Adj Close"]
prices = prices.div(prices.iloc[0])
prices = prices.rename(
    columns={"EXXT.DE": "Nasdaq 100", "IWDA.AS": "MSCI World", "VHYL.AS": "FTSE All World High Dividend"})

# assemble plot
plt.figure(figsize=(15, 10))
plt.plot(prices)
plt.title("High Dividend Stocks vs. Technology Stocks vs. Total Market", size=25)
plt.legend(prices.columns, prop={'size': 15})
plt.ylabel("Cumulative Return", size=15)
plt.xticks(size=15)
plt.yticks(size=15)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b-%Y'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=1))
plt.savefig("plots/comparison.png")

# ---------------------------------------------------------------------------
# calculate change of correlation  matrix in vs out of sample
# ---------------------------------------------------------------------------

out_sample_cor = Gross_Price_selected[Gross_Price_selected.index > end_backtesting].pct_change().corr()
in_sample_cor = Gross_Price_selected[Gross_Price_selected.index < end_backtesting].pct_change().corr()

rel_cor = out_sample_cor / in_sample_cor
pd.DataFrame(rel_cor).to_csv("files/correlation_change.csv")

# ---------------------------------------------------------------------------
# calculate performance ratios for short and long portfolios & Benchmark
# ---------------------------------------------------------------------------

"""
we only include stocks which contain a forward,trailing PE ratio &
PB Ratio to calculate weighted values.
Further, as described by iShares,
we restrict the maximum value of PE ratios to 60 and of PB ratios to 25.
"""
stocks_short = stocks_invest[stocks_invest.weights < 0]
stocks_long = stocks_invest[stocks_invest.weights > 0]

# dividend yield
div_yield_long = (stocks_long.weights / stocks_long.weights.sum() * stocks_long.Yield).sum()
div_yield_short = (stocks_short.weights.abs() / stocks_short.weights.abs().sum() * stocks_short.Yield).sum()
div_yield_index = (stocks_index.index_weights / stocks_index.index_weights.sum() * stocks_index.Yield).sum()

# trailing PE Ratio, forward PE, PB Ratio

# get all stocks and join strategy weights
index = stocks_index.join(stocks_invest[["weights"]])

# 1. remove stocks where ratios are not given
index = index[index.Trailing_PE.notna()]
index = index[index.Forward_PE.notna()]

# 2. set PE ratios above 60 to 60, set PB ratios above 25 to 25
index.Trailing_PE = np.where(index.Trailing_PE > 60, 60, index.Trailing_PE)
index.Forward_PE = np.where(index.Forward_PE > 60, 60, index.Forward_PE)
index.PB_Ratio = np.where(index.PB_Ratio > 25, 25, index.PB_Ratio)

# 3. separate long and short stocks from strategy to calculate ratios separately
short = index.loc[stocks_short.index]
long = index.loc[stocks_long.index]

# calculate weighted Trailing PE
PE_long = (long.weights / long.weights.sum() * long.Trailing_PE).sum()
PE_short = (short.weights / short.weights.sum() * short.Trailing_PE).sum()
PE_index = (index.index_weights / index.index_weights.sum() * index.Trailing_PE).sum()

# calculate weighted forward PE
PE_fwd_long = (long.weights / long.weights.sum() * long.Forward_PE).sum()
PE_fwd_short = (short.weights / short.weights.sum() * short.Forward_PE).sum()
PE_fwd_index = (index.index_weights / index.index_weights.sum() * index.Forward_PE).sum()

# calculate PB Ratio
PB_long = (long.weights / long.weights.sum() * long.PB_Ratio).sum()
PB_short = (short.weights / short.weights.sum() * short.PB_Ratio).sum()
PB_index = (index.index_weights / index.index_weights.sum() * index.PB_Ratio).sum()

# assemble metrics dataframe
ratios_short = {"Yield": format(div_yield_short, ".2%"),
                "Price_Book": round(PB_short, 2),
                "Trailing_PE": round(PE_short, 2),
                "Forward_PE": round(PE_fwd_short, 2)}

ratios_long = {"Yield": format(div_yield_long, ".2%"),
               "Price_Book": round(PB_long, 2),
               "Trailing_PE": round(PE_long, 2),
               "Forward_PE": round(PE_fwd_long, 2)}

ratios_index = {"Yield": format(div_yield_index, ".2%"),
                "Price_Book": round(PB_index, 2),
                "Trailing_PE": round(PE_index, 2),
                "Forward_PE": round(PE_fwd_index, 2)}

ratios_table = pd.DataFrame({"Portfolio Short": ratios_short, "Portfolio Long": ratios_long, "Index": ratios_index})

# export table
dfi.export(ratios_table, "plots/portfolio_characteristics.png")


"""
----------------
accessChecker.py
----------------

To run:

Input csv file should have two columns
 - column one is the name of the vendor selected from the list below
 - column two is the url
 - column three is the title of journal

Execute the command: "python accessChecker.py {name_of_csv_file}"

Output will be written to accessCheckerResults.csv
"""

import sys
from bs4 import BeautifulSoup
import csv
import re
import requests
from selenium import webdriver
import yaml
import time

def url_check(url):
    if url == "title_url":
        return "Default URL"
    else:
        return url

def get_columns():
    yml = open('AccessCheckerSettings.yml', 'r')
    settings = yaml.load(yml)
    columns = []
    for setting in settings["Columns"]:
        if settings["Columns"][setting]:
            columns.append(setting)
    yml.close()
    return columns

def get_rows(filename):
    file_extension = filename.split('.')[-1]
    if file_extension == "csv":
        f = open(filename,'r')
        reader=csv.reader(f)
    if file_extension == "txt":
        f = open(filename,'rU')
        reader=csv.reader((line.replace('\0','') for line in f),delimiter='\t')
        for _ in range(28):
            header = next(reader) #skip (**) lines
            if (len(header) > 1) and ("itle" in header[0]): break
    if len(header) == 0: header = next(reader) # to account for weird case where an extra blank line exists in the txt file.
    header=list(map(url_check, header))
    rows = [dict(zip(header, map(str, row))) for row in reader]
    f.close()
    for index,row in enumerate(rows):
        if not bool(row):
            del rows[index]
    return rows

def main(filename, vendor):
    columns = get_columns()
    rows= get_rows(filename)
    output = csv.writer(open('accessCheckerResults.csv', "w", newline=''))
    num_lines = len(rows)
    output_header = ["Vendor"] + columns + ["Have Access?"]
    output.writerow(output_header)
    for index, row in enumerate(rows):
        message = globals().get(vendor)(row["Default URL"])
        print(str(index + 1) +  " of " + str(num_lines) + " | " + message + " | " + row["Default URL"])

        output_row = [vendor]
        for column in columns:
            if column in row:
                output_row.append(row[column])
            else:
                output_row.append('')
        output_row.append(message)
        output.writerow(output_row)
    #[vendor, row["Title"], row["Default URL"], message]

def adam(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all(text = re.compile("Download whole document")) or soup.find_all(text = re.compile("Download entire document")):
        return "Right On!"
    else:
        return "Look into this . . . "

def asp(url, count, num_lines):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all("div", id = "1"):
        return "Right On!"
    elif soup.find_all("title", text = re.compile("Trial login")):
        return "Nope!"
    else:
        return "Look into this . . . "

def curio(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all("div", class_="container content"):
        return "Right On!"
    elif soup.find_all("h2", class_ = "pull-right"):
        return "Nope!"
    else:
        return "Look into this . . . "

def ebookcentral(url): #updated Feb 2018 PP
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all(text = re.compile("Your institution has access")):
        return "Right On!"
    elif soup.find_all(text = re.compile("Sorry, this book is not available")):
        return "Nope!"
    else:
        return "Look into this . . . "
"""
    browser = webdriver.PhantomJS()
    browser.get(url)
    soup=BeautifulSoup(browser.page_source)
    if soup.find_all(text = re.compile("Your institution has access")):
        return "Right On!"
    elif soup.find_all(text = re.compile("Your institution has access to 1 copy of this book.")):
        return "Right On! [supo]"
    elif soup.find_all(text = re.compile("Sorry, this book is not available")):
        return "Nope!"
    else:
        return "Look into this . . . "
"""
def ebsco(url):
    browser = webdriver.PhantomJS()
    browser.get(url)
    soup=BeautifulSoup(browser.page_source)
    if soup.find_all("a", class_= "record-type pdf-ft") or soup.find_all("a", class_= "record-type epub"):
        return "Right on!"
    else:
        return "Nope!"

def fod(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all("i", class_= "fa fa-play") or soup.find_all(text = re.compile("Show Segments")):
        return "Right On!"
    elif soup.find_all("span", id = "MainContent_lblSearchTerm"):
        return "Nope!"
    else:
        return "Look into this . . . "

def gale(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all(text = re.compile("Table of Contents")):
        return "Right On!"
    else:
        return "Look into this . . . "

def harvard(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all("li", class_= "paidAccess"):
        return "Right On!"
    elif soup.find_all("a", class_= "get-access"):
        return "Nope!"
    else:
        return "Look into this . . . "

def ilib(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all(text = re.compile("Download Page Range")):
        return "Right On!"
    elif soup.find_all(text = re.compile("Access Error")):
        return "Nope!"
    else:
        return "Look into this . . . "

#igi updated Oct 2017/PP
def igiglobal(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all(text = re.compile("Full-Book Download")):
        return "Right On!"
    elif soup.find_all(text = re.compile("Purchase")):
        return "Nope!"
    else:
        return "Look into this . . . "

#ingenta created Feb 2018/PP
def ingenta(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all(src="/images/icon_s_square.gif"):
        return "Right On!"
    elif soup.find_all(src="/images/icon_f_square.gif"):
        return "Free content"
    elif soup.find_all(text = re.compile("Wiley-Blackwell")):
        return "has changed publishers to Wiley"
    elif soup.find_all(text = re.compile("Content Not Found")):
        return "Content Not Found"
    else:
        return "Look into this . . . "
def jstor(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all("a", class_= "pdfLink tt-track-nolink"):
        return "Right On!"
    elif soup.find_all("span", text = re.compile("Your institution has not purchased this book from JSTOR.")):
        return "Nope!"
    else:
        return "Look into this . . . "

def muse(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all("span", class_= "access_yes"):
        return "Right On!"
    elif soup.find_all("span", class_= "access_no"):
        return "Nope!"
    else:
        return "Look into this . . . "

def nfb(url):
    r = requests.get(url[1])
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all(text = re.compile("Access your rentals without")):
        return "Nope!"
    elif soup.find_all("div", class_= "embed-player-container"):
        return "Right On!"
    else:
        return "Look into this . . . "

#cannot get oxford to work? OCt 2017/PP
def oxford(url):
    time.sleep(5)
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all(text = re.compile("availabilityIcon unlocked")):
        return "Right On!"
    elif soup.find_all(text = re.compile("availabilityIcon locked")):
        return "Nope!"
    else:
        return "Look into this . . . "

#springer updated Oct 2017/PP
def springer(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all("a", class_= "webtrekk-track pdf-link"):
        return "Right On!"
    elif soup.find_all("a", class_= "test-bookpdf-link"):
        return "Right On!"
    elif soup.find_all("a", class_= "access-link"):
        return "Nope!"
    elif soup.find_all("title", text = re.compile("Deleted DOI")):
        return "Deleted DOI!"
    elif soup.find_all("div", id = "error"):
        return "Page not found"
    else:
        return "Look into this . . . "

def tandf(url):
    r = requests.get(url[1])
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all("a", text = re.compile("Download a copy")) or soup.find_all("a", text = re.compile("Quick access")):
        return "Right On!"
    elif soup.find_all("span", text = re.compile("Sorry, you do not have access to this book.")):
        return "Nope!"
    else:
        return "Look into this . . . "

 #wiley updated May 2018/PP
def wiley(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    if soup.find_all(text = re.compile("full access")):
        return "Right On!"
    elif soup.find_all(text = re.compile("Full access")):
        return "Right on!"
    elif soup.find_all(text = re.compile("We're sorry, the page you've requested does not exist at this address")):
        return "Page not found"
    elif soup.find_all(text = re.compile("Hindawi")):
        return "Hindawi open access"
    elif soup.find_all(text = re.compile("Open access")):
        return "Open access"
    elif soup.find_all(text = re.compile("Birth Defects Research")):
        return "Journal merge Birth Defects Research"

    elif soup.find_all(text = re.compile("You have free access to this content")):
        return "Free Access"
    elif soup.find_all(text = re.compile("This journal is now published as")):
        return "Change of title"
    elif soup.find_all(text = re.compile("Open Access")):
        return "Open access"
    elif soup.find_all(text = re.compile("free access")):
        return "Free Access"
    else:
        return "Look into this . . . "
# Run it!
if __name__ == "__main__":
    if len(sys.argv) > 1 and sys.argv[1] == "vendors":
        methods = list(globals())
        main_ind =  methods.index('main')
        for vendor in methods[main_ind+1:]:
            print(vendor)
    elif len(sys.argv) != 3:
        print("Correct Usage: "+sys.argv[0]+" file/path/and/name.ext vendor")
        print("To see a list of vendors run: "+sys.argv[0]+" vendors")
    else:
        main(sys.argv[1],sys.argv[2])