import datetime
import sqlalchemy as sa
import psycopg2

from . import DatabaseConnectionError, SQLError

from .models import InfernalJob, JOB_CHUNK_STATUS_CHOICES

async def save_infernal_job(engine, job_id, priority):
    """
    Create infernal job
    :param engine: params to connect to the db
    :param job_id: id of the job
    :param priority: priority of the job, high or low
    """
    try:
        async with engine.acquire() as connection:
            try:
                await connection.scalar(
                    InfernalJob.insert().values(
                        job_id=job_id,
                        submitted=datetime.datetime.now(),
                        priority=priority,
                        status=JOB_CHUNK_STATUS_CHOICES.pending)
                )
            except Exception as e:
                raise SQLError("Failed to save_infernal_job for job_id = %s" % job_id) from e
    except psycopg2.Error as e:
        raise DatabaseConnectionError("Failed to open database connection in save_infernal_job for "
                                      "job_id = %s" % job_id) from e

async def set_infernal_job_status(engine, job_id, status):
    """
    Update the status of the infernal job
    :param engine: params to connect to the db
    :param job_id: id of the job
    :param status: an option from consumer.JOB_CHUNK_STATUS
    :return: None
    """
    finished = None
    if status == JOB_CHUNK_STATUS_CHOICES.success or \
       status == JOB_CHUNK_STATUS_CHOICES.error or \
       status == JOB_CHUNK_STATUS_CHOICES.timeout:
        finished = datetime.datetime.now()

    submitted = None
    if status == JOB_CHUNK_STATUS_CHOICES.started:
        submitted = datetime.datetime.now()

    try:
        async with engine.acquire() as connection:
            try:
                if submitted:
                    query = sa.text('''
                        UPDATE infernal_job
                        SET status = :status, submitted = :submitted
                        WHERE job_id = :job_id
                        RETURNING *;
                    ''')

                    infernal_job = None  # if connection didn't return any rows, return None
                    async for row in connection.execute(query, job_id=job_id, status=status, submitted=submitted):
                        infernal_job = row.id
                    return infernal_job
                elif finished:
                    query = sa.text('''
                        UPDATE infernal_job
                        SET status = :status, finished = :finished
                        WHERE job_id = :job_id
                        RETURNING *;
                    ''')

                    infernal_job = None  # if connection didn't return any rows, return None
                    async for row in connection.execute(query, job_id=job_id, status=status, finished=finished):
                        infernal_job = row.id
                    return infernal_job
                else:
                    query = sa.text('''
                        UPDATE infernal_job
                        SET status = :status
                        WHERE job_id = :job_id
                        RETURNING *;
                    ''')

                    infernal_job = None  # if connection didn't return any rows, return None
                    async for row in connection.execute(query, job_id=job_id, status=status):
                        infernal_job = row.id
                    return infernal_job
            except Exception as e:
                raise SQLError("Failed to set_job_chunk_status in the database,"
                               " job_id = %s, status = %s" % (job_id, status)) from e
    except psycopg2.Error as e:
        raise DatabaseConnectionError("Failed to open connection to the database in set_job_chunk_status, "
                                      "job_id = %s" % job_id) from e

async def set_consumer_to_infernal_job(engine, job_id, consumer_ip):
    """
    Update the infernal_job table to register the consumer who will run the job
    :param engine: params to connect to the db
    :param job_id: id of the job
    :param consumer_ip: ip address of the consumer
    :return: id or none
    """
    try:
        async with engine.acquire() as connection:
            try:
                query = sa.text('''
                    UPDATE infernal_job
                    SET consumer = :consumer_ip
                    WHERE job_id=:job_id
                    RETURNING *;
                ''')
                infernal_job = None  # if connection didn't return any rows, return None
                async for row in connection.execute(query, job_id=job_id, consumer_ip=consumer_ip):
                    infernal_job = row.id
                return infernal_job
            except Exception as e:
                raise SQLError("Failed to set_consumer_to_infernal_job in the database, job_id = %s" % job_id) from e
    except psycopg2.Error as e:
        raise DatabaseConnectionError("Failed to open connection to the database in set_consumer_to_infernal_job, "
                                      "job_id = %s" % job_id) from e


from django.urls import path
from . import views

app_name = "user"

urlpatterns = [
    path("", views.home, name="home_page"),
    path("login/", views.login, name="login_page"),
    path("logout/", views.LogoutPage.as_view(), name="logout_page"),  # For logging out
    path("signup/", views.RegisterPage.as_view(), name="register_page"),  # For Sign Up
    path(
        "profile/", views.ProfilePage.as_view(), name="profile_page"
    ),  # For User Profile
    path(
        "profile/edit/", views.ProfileEdit.as_view(), name="profile_edit"
    ),  # For Edit User Profile
    path("company/", views.CompanyCreate.as_view(), name="company_name"),

    #     Verification URLS
    path("verify/email/", views.VerifyEmail.as_view(), name="verify_email"),
    path("verify/email/submit/", views.VerifyEmailCode.as_view(), name="submit_verify_email"),
    #     ======
    #     Social Authentication
    path('google/', views.GoogleSocialAuthView.as_view()),
    # path("login/google/", views.GoogleLogin.as_view(), name="google_login"),
    # path("login/facebook/", views.FacebookLogin.as_view(), name="facebook_login"),
    # path("login/instagram/", views.login, name="login_page"),
    # path("instagram-callback/", views.InstagramCallbackView.as_view(), name="instagram_callback"),

]


from scrapy.http import HtmlResponse
from selenium import webdriver

from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

from jobcrawler.settings import SELENIUM_HUB_EXECUTOR

class JSMiddleware(object):

    def __init__(self):

        options = Options()  # Set webdriver options
        options.add_argument('--headless')
        options.add_argument('--disable-gpu')

        self.driver = webdriver.Remote(
            command_executor=SELENIUM_HUB_EXECUTOR,
            desired_capabilities=DesiredCapabilities.FIREFOX
        )

        # self.driver = webdriver.Chrome('C:/webdrivers/chromedriver.exe', chrome_options=options)

    def process_request(self, request, spider):
        """This middleware is applied on skywalker's start urls that are protected with JS"""

        # check response url
        if (spider.allowed_domains == ['skywalker.gr'] and 'aggelia/ergasias' not in request.url) or (
                spider.allowed_domains == ["gr.indeed.com"] and 'viewjob' not in request.url):

            self.driver.get(request.url)
            body = self.driver.page_source  # get page source

            return HtmlResponse(self.driver.current_url, body=body, encoding='utf-8', request=request)

    def spider_closed(self, spider):
        self.driver.close()


"""DSO Protocol Message"""

from array import array
from dataclasses import dataclass

DEBUG_MESSAGE_MARKER = 0x43
NORMAL_MESSAGE_MARKER = 0x53

SETTINGS_RESPONSE_CMD = 0x81

SAMPLE_RESPONSE_CMD = 0x82
SAMPLE_LEN_SUBCMD   = 0x00
SAMPLE_DATA_SUBCMD  = 0x01
SAMPLE_SUM_SUBCMD   = 0x02
SAMPLE_STOP_SUBCMD  = 0x03  # Errors or STOP mode

NORMAL_SUBCOMMAND = [
    0x02,   # Read sample data
    SAMPLE_RESPONSE_CMD,
    0x10,   # Read file
    0x90,
    0x12,   # Lock/unlock control panel, start/stop acquisition
    0x92,
    0xA0,   # screenshot response
]   # The command with subcommand

@dataclass
class Message:
    """Protocol Message"""

    mark: int = NORMAL_MESSAGE_MARKER
    length: int = 0
    command: int = 0
    subcommand: int = -1
    data: array = None
    checksum: bool = False
    response: bool = False  # response from DSO

def build(pkt: array) -> Message:
    """Build a message from array"""

    if pkt[0] == 0:
        return None

    msg = Message(
            pkt[0],
            pkt[1] + (pkt[2] << 8),
            pkt[3])

    msg.checksum = _checksum(pkt)
    msg.response = msg.command >> 7 > 0

    if msg.length < 3:
        # no data, subcommand
        return msg

    data_idx = 4

    if msg.command in NORMAL_SUBCOMMAND:
        msg.subcommand = pkt[data_idx]
        data_idx += 1

    msg.data = pkt[data_idx: msg.length+2]

    return msg

def create_packet(msg: Message) -> array:
    """Create packet from a message"""

    pkt = array('B', [msg.mark, 0, 0, msg.command])
    length = 2

    if msg.subcommand > -1:
        pkt.append(msg.subcommand)
        length += 1
    if msg.data and len(msg.data) > 0:
        pkt.extend(msg.data)
        length += len(msg.data)

    pkt[1] = length & 0x00ff
    pkt[2] = length >> 8

    pkt.append(make_sum(pkt))

    return pkt

def make_sum(pkt: array) -> int:
    """Create check sum"""

    summary = 0
    for dat in pkt:
        summary += dat

    return summary & 0xFF

def _checksum(pkt: array) -> bool:

    pkt_len = pkt[1] + (pkt[2] << 8)
    chk_idx = pkt_len + 2
    if chk_idx >= len(pkt):
        return False

    return make_sum(pkt[:chk_idx]) == pkt[chk_idx]


import os
from flask import Flask, render_template,url_for, flash, redirect, request
from forms import RegistrationForm
from mangodb import get_result,insertCollection,dropCollection

#def create_app():
app = Flask(__name__)
app.config["SECRET_KEY"] = 'd2707fea9778e085491e2dbbc73ff30e'

@app.route('/')
def home():
    return  render_template('layout.html')

@app.route('/register', methods=["GET", "POST"])
def register():
    form = RegistrationForm()
    if form.validate_on_submit():
        # 알람 카테고리에 따라 부트스트랩에서 다른 스타일을 적용 (success, danger)
        flash(f'{form.username.data} 님 가입 완료!', 'success')
        return redirect(url_for('home'))
    return render_template('register.html', form=form)

@app.route('/search', methods=["GET", "POST"])
def search():
    flag="대기"
    pushpop=[]
    if request.method=='GET':
        keyword=request.args.get('keyword')
        try:
            Pnum=int(request.args.get('PNum'))
        except:
            Pnum=-1
        try:
            Psize=int(request.args.get('Psize'))
        except:
            Psize=-1
        sortCd=request.args.get('sortC')
        #clearing=request.args.get('clearing')
        if (keyword != ""):
            result=get_result(key=keyword,number=Pnum,pSize=Psize,SortCd=sortCd)
            pushpop.append(result[4])
            insertCollection(result)
            flag="완료"
            return render_template('search.html',dd=flag)
    return render_template('search.html',dd=flag)

if __name__ == "__main__":
    app.run(debug=True)


import json
import os
import platform
import plistlib
import subprocess
import uuid
from lento.config import Config
from lento.utils import is_url
from PIL import Image

def create_card():
    new_card = {
        "id": str(uuid.uuid4().hex),
        "name": "Untitled Card",
        "emoji": "ðŸ˜ƒ",
        "time": 0,
        "hard_blocked_sites": {},
        "soft_blocked_sites": {},
        "hard_blocked_apps": {},
        "soft_blocked_apps": {},
        "notifications": {},
        "goals": {}
    }

    path = os.path.join(os.path.expanduser("~"), "lentosettings.json")

    with open(path, "r", encoding="UTF-8") as settings_json:
        settings = json.load(settings_json)

    settings["cards"][new_card["name"]] = new_card

    with open(path, "w", encoding="UTF-8") as settings_json:
        json.dump(settings, settings_json)

def read_cards():
    path = os.path.join(os.path.expanduser("~"), "lentosettings.json")
    with open(path, "r", encoding="UTF-8") as settings_json:
        settings = json.load(settings_json)
    return settings["cards"]

def delete_card(card_to_delete):
    path = os.path.join(os.path.expanduser("~"), "lentosettings.json")
    with open(path, "r", encoding="UTF-8") as settings_json:
        settings = json.load(settings_json)
    del settings["cards"][card_to_delete]
    with open(path, "w", encoding="UTF-8") as settings_json:
        json.dump(settings, settings_json)

def update_metadata(card_to_modify, field_to_modify, new_value):
    path = os.path.join(os.path.expanduser("~"), "lentosettings.json")
    with open(path, "r", encoding="UTF-8") as settings_json:
        settings = json.load(settings_json)

    if settings["cards"][card_to_modify][field_to_modify] is None:
        raise Exception("Card field is nonexistent!")
    elif field_to_modify not in ["name", "emoji", "time"]:
        raise Exception("Card field is restricted!")
    elif field_to_modify == "time":
        new_value = float(new_value)

    settings["cards"][card_to_modify][field_to_modify] = new_value
    if field_to_modify == "name":
        new_cards = {
            new_value if k == card_to_modify
            else k: v for k, v in settings["cards"].items()
        }
        settings["cards"] = new_cards

    with open(path, "w", encoding="UTF-8") as settings_json:
        json.dump(settings, settings_json)

def add_to_site_blocklists(card_to_modify, list_to_modify, new_value):
    """Add to either the `hard_blocked_sites` or `soft_blocked_sites` lists."""
    path = os.path.join(os.path.expanduser("~"), "lentosettings.json")
    with open(path, "r", encoding="UTF-8") as settings_json:
        settings = json.load(settings_json)

    card_to_mod = settings["cards"][card_to_modify]
    if card_to_mod[list_to_modify] is None:
        raise Exception("List is nonexistent!")
    elif list_to_modify not in [
                "hard_blocked_sites",
                "soft_blocked_sites"
            ]:
        raise Exception("Card field is restricted!")
    elif is_url(new_value) is False:
        raise Exception("URL not valid!")
    elif list_to_modify == "hard_blocked_sites":
        if new_value in card_to_mod["soft_blocked_sites"]:
            raise Exception(f"'{new_value}' already soft blocked!")
    elif list_to_modify == "soft_blocked_sites":
        if new_value in card_to_mod["hard_blocked_sites"]:
            raise Exception(f"'{new_value}' already hard blocked!")

    settings["cards"][card_to_modify][list_to_modify][new_value] = True
    with open(path, "w", encoding="UTF-8") as settings_json:
        json.dump(settings, settings_json)

def update_site_blocklists(card_to_modify, list_to_modify, new_sites_dict):
    """Update one of the blocked_sites lists. Minimal validation."""
    path = os.path.join(os.path.expanduser("~"), "lentosettings.json")
    with open(path, "r", encoding="UTF-8") as settings_json:
        settings = json.load(settings_json)

    if settings["cards"][card_to_modify][list_to_modify] is None:
        raise Exception("List is nonexistent!")
    elif list_to_modify not in [
                "hard_blocked_sites",
                "soft_blocked_sites"
            ]:
        raise Exception("Card field is restricted!")
    elif not isinstance(new_sites_dict, dict):
        raise Exception("new_sites_dict is not a dict!")

    settings["cards"][card_to_modify][list_to_modify] = new_sites_dict
    with open(path, "w", encoding="UTF-8") as settings_json:
        json.dump(settings, settings_json)

def add_to_app_blocklists(card_to_modify, list_to_modify, apps_to_add):
    path = os.path.join(os.path.expanduser("~"), "lentosettings.json")
    with open(path, "r", encoding="UTF-8") as settings_json:
        settings = json.load(settings_json)

    card = settings["cards"][card_to_modify]

    current_os = platform.system()
    if current_os == "Darwin":
        for app in apps_to_add:
            app_name = os.path.basename(app).replace(".app", "")

            if list_to_modify == "hard_blocked_apps":
                if app_name in card["soft_blocked_apps"]:
                    raise Exception(f"'{app_name}' already soft blocked!")
            elif list_to_modify == "soft_blocked_apps":
                if app_name in card["hard_blocked_apps"]:
                    raise Exception(f"'{app_name}' already hard blocked!")

            bundle_id = subprocess.check_output([
                "mdls",
                "-name",
                "kMDItemCFBundleIdentifier",
                "-r",
                app
            ]).decode("utf-8")

            if app[:14] == "/Applications/":
                app = os.path.join(
                    Config.MACOS_APPLICATION_FOLDER,
                    app_name + ".app"
                )
                plist_path = os.path.join(
                    app,
                    "Contents",
                    "Info.plist"
                )
            else:
                plist_path = os.path.join(app, "Contents", "Info.plist")
            with open(plist_path, "rb") as fp:
                app_plist = plistlib.load(fp)
            icon_name = app_plist["CFBundleIconFile"]
            if icon_name[-5:] != ".icns":
                icon_name = app_plist["CFBundleIconFile"] + ".icns"
            original_icon_path = os.path.join(
                app,
                "Contents",
                "Resources",
                icon_name
            )
            im = Image.open(original_icon_path)
            rgb_im = im.convert("RGB")

            path_to_save_at = os.path.join(
                    os.path.expanduser("~"),
                    "Library/Application Support/Lento/" + app_name + ".jpg"
                )
            rgb_im.save(path_to_save_at)

            card[list_to_modify][app_name] = {
                "enabled": True,
                "bundle_id": bundle_id,
                "app_icon_path": path_to_save_at
            }
    elif current_os == "Windows":
        for app in apps_to_add:
            app_name = app["name"]
            if list_to_modify == "hard_blocked_apps":
                if app_name in card["soft_blocked_apps"]:
                    raise Exception(f"'{app_name}' already soft blocked!")
            elif list_to_modify == "soft_blocked_apps":
                if app_name in card["hard_blocked_apps"]:
                    raise Exception(f"'{app_name}' already hard blocked!")
            card[list_to_modify][app["name"]] = {
                "enabled": True,
                "path": app["path"],
                "app_icon_path": app["icon_path"]
            }
    else:
        raise Exception("OS name invalid or not found!")

    with open(path, "w", encoding="UTF-8") as settings_json:
        json.dump(settings, settings_json)

def update_app_blocklists(card_to_modify, list_to_modify, new_list_structure):
    path = os.path.join(os.path.expanduser("~"), "lentosettings.json")
    with open(path, "r", encoding="UTF-8") as settings_json:
        settings = json.load(settings_json)

    if list_to_modify not in [
                "hard_blocked_apps",
                "soft_blocked_apps"
            ]:
        raise Exception("Card field is restricted!")

    settings["cards"][card_to_modify][list_to_modify] = new_list_structure

    with open(path, "w", encoding="UTF-8") as settings_json:
        json.dump(settings, settings_json)

def add_notification(
            name,
            enabled,
            card_to_modify,
            type,
            blocked_visit_triggers,
            associated_goals,
            time_interval_trigger,
            title,
            body,
            audio_paths
        ):
    path = os.path.join(os.path.expanduser("~"), "lentosettings.json")
    with open(path, "r", encoding="UTF-8") as settings_json:
        settings = json.load(settings_json)

    if not isinstance(name, str):
        raise Exception("Name must be a string!")
    elif not isinstance(enabled, bool):
        raise Exception("Enabled must be a boolean!")
    elif type not in ["banner", "popup", "audio"]:
        raise Exception("Notification type not valid!")
    elif not isinstance(blocked_visit_triggers, list):
        raise Exception("Blocked visit triggers is not a list!")
    for item in blocked_visit_triggers:
        list_checks = (
            item in settings["cards"][card_to_modify]["hard_blocked_sites"],
            item in settings["cards"][card_to_modify]["soft_blocked_sites"],
            item in settings["cards"][card_to_modify]["hard_blocked_apps"],
            item in settings["cards"][card_to_modify]["soft_blocked_apps"],
        )
        if True not in list_checks:
            raise Exception(f"Blocked visit triggers '{item}' not found in blocklists!")  # noqa: E501
    if not isinstance(associated_goals, list):
        raise Exception("Associated goals is not a list!")
    for item in associated_goals:
        if item not in settings["cards"][card_to_modify]["goals"]:
            raise Exception("Associated goals not found in goal list!")
    if not isinstance(time_interval_trigger, int) and time_interval_trigger is not None:  # noqa: E501
        raise Exception("Timer interval trigger is not an integer or None!")
    elif not isinstance(title, str) and title is not None:
        raise Exception("Notification title is not a string!")
    elif not isinstance(body, str) and body is not None:
        raise Exception("Notification body is not a string!")

    new_notif = {
        "name": name,
        "enabled": enabled,
        "type": type,
        "blocked_visit_triggers": blocked_visit_triggers,
        "associated_goals": associated_goals,
        "time_interval_trigger": time_interval_trigger,
        "title": title,
        "body": body,
        "audio_paths": audio_paths
    }

    card = settings["cards"][card_to_modify]
    card["notifications"][str(uuid.uuid4().hex)] = new_notif

    with open(path, "w", encoding="UTF-8") as settings_json:
        json.dump(settings, settings_json)

def update_notification_list(card_to_modify, new_notifs_dict):
    """Update notifications for a card. Minimal validation."""
    path = os.path.join(os.path.expanduser("~"), "lentosettings.json")
    with open(path, "r", encoding="UTF-8") as settings_json:
        settings = json.load(settings_json)

    if not isinstance(new_notifs_dict, dict):
        raise Exception("new_notifs_dict is not a dict!")
    for item in list(new_notifs_dict.keys()):
        is_notif_dict_valid = set(new_notifs_dict[item].keys()) == set([
            "name",
            "enabled",
            "type",
            "blocked_visit_triggers",
            "associated_goals",
            "time_interval_trigger",
            "title",
            "body",
            "audio_paths"
        ])
        if not is_notif_dict_valid:
            raise Exception(f"Notif {item} had invalid structure!")

    settings["cards"][card_to_modify]["notifications"] = new_notifs_dict

    with open(path, "w", encoding="UTF-8") as settings_json:
        json.dump(settings, settings_json)

def add_goal(card_to_modify: str, goal_to_add: str) -> None:
    """Add a goal to a card. Automatically adds as disabled."""
    path = os.path.join(os.path.expanduser("~"), "lentosettings.json")

    with open(path, "r", encoding="UTF-8") as settings_json:
        settings = json.load(settings_json)

    if not isinstance(goal_to_add, str):
        raise Exception("Goal to add is not string!")

    settings["cards"][card_to_modify]["goals"][goal_to_add] = False

    with open(path, "w", encoding="UTF-8") as settings_json:
        json.dump(settings, settings_json)

def update_goal_list(card_to_modify, new_goals_dict):
    """Update goals for a card. Minimal validation."""
    path = os.path.join(os.path.expanduser("~"), "lentosettings.json")
    with open(path, "r", encoding="UTF-8") as settings_json:
        settings = json.load(settings_json)

    if not isinstance(new_goals_dict, dict):
        raise Exception("new_goals_dict is not a dict!")
    for item in list(new_goals_dict.keys()):
        if not isinstance(new_goals_dict[item], bool):
            raise Exception(f"Goal '{item}' has invalid structure!")

    settings["cards"][card_to_modify]["goals"] = new_goals_dict

    with open(path, "w", encoding="UTF-8") as settings_json:
        json.dump(settings, settings_json)


import random
import numpy as np
from tqdm import  tqdm
from tensorflow.keras.preprocessing.sequence import pad_sequences

def gen_data_set(data, negsample=0):

    data.sort_values("timestamp", inplace=True)
    item_ids = data['movie_id'].unique()

    train_set = []
    test_set = []
    for reviewerID, hist in tqdm(data.groupby('user_id')):
        pos_list = hist['movie_id'].tolist()
        rating_list = hist['rating'].tolist()

        if negsample > 0:
            candidate_set = list(set(item_ids) - set(pos_list))
            neg_list = np.random.choice(candidate_set,size=len(pos_list)*negsample,replace=True)
        for i in range(1, len(pos_list)):
            hist = pos_list[:i]
            if i != len(pos_list) - 1:
                train_set.append((reviewerID, hist[::-1], pos_list[i], 1, len(hist[::-1]),rating_list[i]))
                for negi in range(negsample):
                    train_set.append((reviewerID, hist[::-1], neg_list[i*negsample+negi], 0,len(hist[::-1])))
            else:
                test_set.append((reviewerID, hist[::-1], pos_list[i],1,len(hist[::-1]),rating_list[i]))

    random.shuffle(train_set)
    random.shuffle(test_set)

    print(len(train_set[0]),len(test_set[0]))

    return train_set,test_set

def gen_model_input(train_set,user_profile,seq_max_len):

    train_uid = np.array([line[0] for line in train_set])
    train_seq = [line[1] for line in train_set]
    train_iid = np.array([line[2] for line in train_set])
    train_label = np.array([line[3] for line in train_set])
    train_hist_len = np.array([line[4] for line in train_set])

    train_seq_pad = pad_sequences(train_seq, maxlen=seq_max_len, padding='post', truncating='post', value=0)
    train_model_input = {"user_id": train_uid, "movie_id": train_iid, "hist_movie_id": train_seq_pad,
                         "hist_len": train_hist_len}

    for key in ["gender", "age", "occupation", "zip"]:
        train_model_input[key] = user_profile.loc[train_model_input['user_id']][key].values

    return train_model_input, train_label


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import torch
import torchvision
from torchvision import datasets, transforms
import matplotlib.pyplot as plt
import numpy as np
import os
from PIL import Image

import argparse
import torch.nn as nn
import time

from cnn_model import CNN

# Data augmentation and normalization for training
# Just normalization for validation
data_transforms = {
    'train': transforms.Compose([
#         transforms.RandomSizedCrop(512), # 224 before，544 is good
        transforms.CenterCrop(512),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'val': transforms.Compose([
#         transforms.Scale(256),
        transforms.CenterCrop(512), # 224 before
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}

# Default constants
LEARNING_RATE_DEFAULT = 1e-4
BATCH_SIZE_DEFAULT = 10
MAX_EPOCHS_DEFAULT = 1
EVAL_FREQ_DEFAULT = 1
OPTIMIZER_DEFAULT = 'ADAM'

data_dir = './Data/'

train_sets = datasets.ImageFolder(os.path.join(data_dir, 'train'), data_transforms['train'])
train_loader = torch.utils.data.DataLoader(train_sets, batch_size=10, shuffle=True, num_workers=4)
train_size = len(train_sets)
train_classes = train_sets.classes

val_sets = datasets.ImageFolder(os.path.join(data_dir, 'val'), data_transforms['val'])
val_loader = torch.utils.data.DataLoader(val_sets, batch_size=10, shuffle=False, num_workers=4)
val_size = len(val_sets)

inputs, classes = next(iter(train_loader))

# Visualize a few images
def imshow(inp, title=None):
    """Imshow for Tensor."""
    inp = inp.numpy().transpose((1, 2, 0))
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    inp = std * inp + mean
    plt.imshow(inp)
    if title is not None:
        plt.title(title)

# Make a grid from batch
out = torchvision.utils.make_grid(inputs, nrow=5)

imshow(out, title=[train_classes[x] for x in classes])

inputs, classes = next(iter(val_loader))

# Make a grid from batch
out = torchvision.utils.make_grid(inputs, nrow=5)

imshow(out, title=[train_classes[x] for x in classes])

def accuracy(outputs, batch_y):
    """
    Computes the prediction accuracy, i.e., the average of correct predictions
    of the network.
    Args:
        predictions: 2D float array of size [number_of_data_samples, n_classes]
        labels: 2D int array of size [number_of_data_samples, n_classes] with one-hot encoding of ground-truth labels
    Returns:
        accuracy: scalar float, the accuracy of predictions.
    """

    _, predicted = torch.max(outputs.data, 1)
    total = batch_y.size(0)
    correct = (predicted == batch_y).sum().item()
    accuracy = 100.0 * correct / total

    return accuracy

def train(trainloader):
    """
    Performs training and evaluation of CNN model.
    NOTE: You should the model on the whole test set each eval_freq iterations.
    """
    # YOUR TRAINING CODE GOES HERE
    n_channels = 3
    n_classes = 5

    cnn = CNN(n_channels, n_classes)

    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    cnn.to(device)

    # Loss and Optimizer
    criterion = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(cnn.parameters(), lr=LEARNING_RATE_DEFAULT)

    losses = []
    accuracies = []

    for epoch in range(MAX_EPOCHS_DEFAULT):
        timestart = time.time()
        running_loss = 0.0
        for step, (batch_x, batch_y) in enumerate(trainloader):

            # zero the parameter gradients
            optimizer.zero_grad()

            # Forward + Backward + Optimize

            batch_x, batch_y = batch_x.to(device), batch_y.to(device)

            outputs = cnn(batch_x)
            loss = criterion(outputs, batch_y)
            loss.backward()
            optimizer.step()

            running_loss += loss.item()

            if step % EVAL_FREQ_DEFAULT == EVAL_FREQ_DEFAULT-1:
                print('[epoch: %d, step: %5d] loss: %.4f' %
                          (epoch, step, running_loss / EVAL_FREQ_DEFAULT))
                losses.append(running_loss / EVAL_FREQ_DEFAULT)
                running_loss = 0.0
                accu = accuracy(outputs, batch_y)
                accuracies.append(accu)
                print('Accuracy on the %d train images: %.3f %%' % (batch_y.size(0),
                            accu))

        break

        print('epoch %d cost %3f sec' %(epoch,time.time()-timestart))

    print('---Finished Training---')

    return cnn, losses, accuracies

cnn, losses, accuracies = train(train_loader)


import pandas as pd
import numpy as np
import matplotlib
from matplotlib import pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
import seaborn as sns
from itertools import combinations
from sklearn.preprocessing import StandardScaler

class EDA:
    def __init__(
        self, problem_type, data, label, numeric_features_names,
        category_features_names, datetime_features_names, scale=True):

        self.problem_type = problem_type
        self.data = data
        self.label = label
        self.numeric_features_names = numeric_features_names
        self.category_features_names = category_features_names
        self.datetime_features_names = datetime_features_names
        self.scale = scale
        self.num_plot_per_fig = 4
        self.scale_alg = None

        # Get numeric and category features
        self.numeric_features = self.data[self.numeric_features_names]
        self.category_features = self.data[self.category_features_names]

    def __general_analysis(self):
        numeric_features_count = len(self.numeric_features_names)
        category_features_count = len(self.category_features_names)
        self.data.replace("", np.nan, inplace=True)
        missing_value_percent = (self.data.isna().sum() / len(self.data) * 100).round(2)

        general_analysis = pd.DataFrame({
            "dtype": self.data.dtypes, "MissingV_Value_Percent": missing_value_percent
        })
        general_analysis["Features_Type"] = ""
        general_analysis.loc[general_analysis["Features_Type"].index.isin(self.numeric_features_names), "Feature_Type"] = "Numeric"
        general_analysis.loc[general_analysis["Features_Type"].index.isin(self.category_features_names), "Feature_Type"] = "Category"

        general_analysis_fig, general_analysis_ax = plt.subplots()
        general_analysis_ax.axis('tight')
        general_analysis_ax.axis('off')
        general_analysis_ax.table(
            cellText=general_analysis.values,
            colLabels=general_analysis.columns,
            rowLabels=general_analysis.index, loc='center', fontsize=200)

        return general_analysis_fig

    def __univariate_analysis(self, numeric_features_dropna, category_features_dropna):
        # numeric features
        numeric_features_stats = numeric_features_dropna.agg([
            "min", "max", "mean", "median", "std",
            "skew", "kurtosis"])
        numeric_features_quantile = numeric_features_dropna.quantile([0.25, 0.75])
        numeric_features_stats = pd.concat(
            [numeric_features_stats, numeric_features_quantile], 0)
        numeric_features_stats.sort_values(
            by="std", axis=1, ascending=True, inplace=True)
        numeric_features_stats = numeric_features_stats.round(4)
        numeric_features_stats_fig, numeric_features_stats_ax = plt.subplots()
        numeric_features_stats_ax.axis('tight')
        numeric_features_stats_ax.axis('off')
        numeric_features_stats_ax.table(
            cellText=numeric_features_stats.values,
            colLabels=numeric_features_stats.columns,
            rowLabels=numeric_features_stats.index, loc='center', fontsize=20)

        # Kde plot
        kdeplot_all = []
        for i, (name, column) in enumerate(numeric_features_dropna.iteritems()):
            if i % self.num_plot_per_fig == 0:
                kdeplot_fig, kdeplot_ax = plt.subplots(
                    int(self.num_plot_per_fig**0.5),
                    int(self.num_plot_per_fig**0.5))
            row_idx, col_idx = divmod(
                i % self.num_plot_per_fig, int(self.num_plot_per_fig**0.5))
            column.plot.kde(
                ax=kdeplot_ax[row_idx, col_idx], secondary_y=True, title=name)
            # ax_tmp.text(0.5, 0.5, "test", fontsize=22)
            if (i + 1) % self.num_plot_per_fig == 0 or i == len(numeric_features_dropna.columns)-1:
                kdeplot_all.append(kdeplot_fig)
                plt.tight_layout()

        # Boxplot
        boxplot_all = []
        fig, ax = plt.subplots()
        for i, (name, column) in enumerate(numeric_features_dropna.iteritems()):
            if i % self.num_plot_per_fig == 0:
                boxplot_fig, boxplot_ax = plt.subplots(
                    int(self.num_plot_per_fig**0.5),
                    int(self.num_plot_per_fig**0.5))
            row_idx, col_idx = divmod(
                i % self.num_plot_per_fig, int(self.num_plot_per_fig**0.5))
            boxplot_ax[row_idx, col_idx].boxplot(column, 0, 'gD')
            boxplot_ax[row_idx, col_idx].set_title(f"{name}")
            if (i + 1) % self.num_plot_per_fig == 0 or i == len(numeric_features_dropna.columns)-1:
                boxplot_all.append(boxplot_fig)
                plt.tight_layout()

        # Category features
        # Freq plot
        freqplot_all = []
        for i, (name, column) in enumerate(category_features_dropna.iteritems()):
            if i % self.num_plot_per_fig == 0:
                freqplot_fig, freqplot_ax = plt.subplots(
                    int(self.num_plot_per_fig**0.5),
                    int(self.num_plot_per_fig**0.5))
            row_idx, col_idx = divmod(
                i % self.num_plot_per_fig, int(self.num_plot_per_fig**0.5))
            bar = freqplot_ax[row_idx, col_idx].bar(
                column.value_counts().index, column.value_counts().values)
            freqplot_ax[row_idx, col_idx].bar_label(bar)
            freqplot_ax[row_idx, col_idx].set_title(f"{name}")
            if (i + 1) % self.num_plot_per_fig == 0 or i == len(category_features_dropna.columns)-1:
                freqplot_all.append(freqplot_fig)
                plt.tight_layout()

        return numeric_features_stats_fig, kdeplot_all, boxplot_all, freqplot_all

    def __bivariate_analysis(self, numeric_features_dropna, category_features_dropna):
        # Categorical vs Categorical
        cat_vs_cat_plot_all = []
        if self.problem_type == "classification":
            columns_comb = list(combinations(category_features_dropna.columns, 2))
            for i, (column_a, column_b) in enumerate(columns_comb):
                category_features_crosstab = pd.crosstab(
                    category_features_dropna[column_a],
                    category_features_dropna[column_b],
                    margins=True, values=category_features_dropna[self.label],
                    aggfunc=pd.Series.count, normalize="all")
                category_features_crosstab.fillna(0, inplace=True)
                if i % self.num_plot_per_fig == 0:
                    cat_vs_cat_plot_fig, cat_vs_cat_plot_ax = plt.subplots(
                        int(self.num_plot_per_fig**0.5),
                        int(self.num_plot_per_fig**0.5))
                row_idx, col_idx = divmod(
                    i % self.num_plot_per_fig, int(self.num_plot_per_fig**0.5))
                sns.heatmap(
                    category_features_crosstab,
                    ax=cat_vs_cat_plot_ax[row_idx, col_idx],
                    annot=True, cbar=True, fmt=".2%")
                cat_vs_cat_plot_ax[row_idx, col_idx].set(
                    title=f"{column_a} vs {column_b} with {self.label} as count")
                if (i + 1) % self.num_plot_per_fig == 0 or i == len(columns_comb)-1:
                    cat_vs_cat_plot_all.append(cat_vs_cat_plot_fig)
                    plt.tight_layout()

        # Numerical vs Categorical
        num_vs_cat_plot_all = []
        if self.problem_type == "classification":
            numeric_features_and_label = pd.concat(
                [numeric_features_dropna, category_features_dropna[self.label]], 1)
            for i, (name, column) in enumerate(numeric_features_dropna.iteritems()):
                if i % self.num_plot_per_fig == 0:
                    num_vs_cat_plot_fig, num_vs_cat_plot_ax = plt.subplots(
                        int(self.num_plot_per_fig**0.5),
                        int(self.num_plot_per_fig**0.5))
                row_idx, col_idx = divmod(
                    i % self.num_plot_per_fig, int(self.num_plot_per_fig**0.5))
                sns.boxplot(
                    x=self.label, y=column, data=numeric_features_and_label,
                    ax=num_vs_cat_plot_ax[row_idx, col_idx])
                if (i + 1) % self.num_plot_per_fig == 0 or i == len(columns_comb)-1:
                    num_vs_cat_plot_all.append(num_vs_cat_plot_fig)
                    plt.tight_layout()

        # Numerical vs Numerical
        # correlation plot
        corr_matrix = numeric_features_dropna.corr(method="pearson")
        corr_fig, corr_ax = plt.subplots()
        sns.heatmap(corr_matrix, ax=corr_ax, annot=True)
        corr_ax.set(title="Correlation Matrix (Pearson)")

        num_vs_num_plot_all = []
        columns_comb = list(combinations(numeric_features_dropna.columns, 2))
        for i, (column_a, column_b) in enumerate(columns_comb):
            if i % self.num_plot_per_fig == 0:
                num_vs_num_plot_fig, num_vs_num_plot_ax = plt.subplots(
                    int(self.num_plot_per_fig**0.5),
                    int(self.num_plot_per_fig**0.5))
            row_idx, col_idx = divmod(
                i % self.num_plot_per_fig, int(self.num_plot_per_fig**0.5))
            num_vs_num_plot_ax[row_idx, col_idx].scatter(
                x=numeric_features_dropna[column_a],
                y=numeric_features_dropna[column_b],
                c=category_features_dropna[self.label if self.problem_type == "classification" else None])
            num_vs_num_plot_ax[row_idx, col_idx].set(
                xlabel=column_a, ylabel=column_b,
                title=f"Plot of {column_b} vs {column_a}")
            if (i + 1) % self.num_plot_per_fig == 0 or i == len(columns_comb)-1:
                num_vs_num_plot_all.append(num_vs_num_plot_fig)
                plt.tight_layout()

        return cat_vs_cat_plot_all, num_vs_cat_plot_all, corr_fig, num_vs_num_plot_all

    def __save_to_pdf(self, *args):

        pp = PdfPages('eda.pdf')
        for arg in args:
            if isinstance(arg, matplotlib.figure.Figure):
                pp.savefig(arg)
            else:
                for fig in arg:
                    pp.savefig(fig)
        pp.close()

    def generate_report(self):
        plt.ioff()

        general_analysis_fig = self.__general_analysis()

        # Drop na
        data_dropna = self.data.dropna(axis=0)
        numeric_features_dropna = data_dropna[self.numeric_features_names]
        if self.scale:
            numeric_features_dropna[self.numeric_features_names] =\
                StandardScaler().fit_transform(numeric_features_dropna)
        category_features_dropna = data_dropna[self.category_features_names]

        numeric_features_stats_fig, kdeplot_all, boxplot_all, freqplot_all =\
            self.__univariate_analysis(
                numeric_features_dropna, category_features_dropna)
        cat_vs_cat_plot_all, num_vs_cat_plot_all, corr_fig, num_vs_num_plot_all =\
            self.__bivariate_analysis(
                numeric_features_dropna, category_features_dropna)

        plt.ion()

        self.__save_to_pdf(
            general_analysis_fig, numeric_features_stats_fig, kdeplot_all,
            boxplot_all, freqplot_all, cat_vs_cat_plot_all,
            num_vs_cat_plot_all, corr_fig, num_vs_num_plot_all
        )

        return None


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 20 15:47:03 2021

@author: rdchlmtr
"""

# -*- coding: utf-8 -*-

import yfinance as yf
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pandas_market_calendars as mcal
import matplotlib.dates as mdates
import seaborn as sns

sns.set_theme()
plt.close('all')
#%%

start_date = '2021-01-01'
end_date = '2022-01-01'

par1 = np.arange(2,5)
par2 = np.arange(15,30)
par3 = np.arange(2,5)
net = np.zeros(len(par1)*len(par2)*(len(par3)))
output = np.zeros((5,len(net)))
#%%

tqqq = yf.download(tickers='TQQQ', start=start_date, end=end_date)
vix = yf.download(tickers='^VXN', start=start_date, end=end_date)

nyse = mcal.get_calendar('NYSE')
nyse_schedule = nyse.schedule(start_date=tqqq.index[0], end_date=tqqq.index[-1])
nyse_dti = mcal.date_range(nyse_schedule, frequency='1D').tz_convert(nyse.tz.zone)

t = pd.DataFrame({'Close':tqqq.Close.values}, index=nyse_dti)
v = pd.DataFrame({'Close':vix.Close.values}, index=nyse_dti)

n = 0

for x in par1:
    for y in par2:
        for z in par3:

            exp1 = t.Close.ewm(span=x, adjust=False).mean()
            exp2 = t.Close.ewm(span=y, adjust=False).mean()
            t['macd'] = exp1-exp2
            t['exp3'] = t.macd.ewm(span=z, adjust=False).mean()
            t['good'] = t.macd > t.exp3
            t['zeros'] = np.zeros(len(t.macd))

            exp1 = v.Close.ewm(span=x, adjust=False).mean()
            exp2 = v.Close.ewm(span=y, adjust=False).mean()
            v['macd'] = exp1-exp2
            v['exp3'] = v.macd.ewm(span=z, adjust=False).mean()
            v['good'] = v.macd < v.exp3
            v['zeros'] = np.zeros(len(v.macd))

            buy = np.where(np.diff(t.good & v.good))[0]
            if len(buy) % 2 != 0:
                buy = np.append(buy, -1)
                action = 'BUY'
                condition= 'BULL'
            else:
                action = 'SELL'
                condition = 'BEAR'

            buy = buy.reshape((len(buy)//2,2))
            buy_times = np.array(t.index)[buy]

            price = tqqq.Open.values
            balance = np.zeros(len(t))
            shares = np.zeros(len(t))
            value = np.zeros(len(t))
            good = t.good & v.good
            balance[0] = 1000

            for i in range(1, len(t)):
                if good[i] > good[i-1]:
                    new_shares = balance[i-1] // price[i]
                    new_balance = balance[i-1] % price[i]
                    shares[i] = new_shares
                    balance[i] = new_balance
                elif good[i] == good[i-1]:
                    shares[i] = shares[i-1]
                    balance[i] = balance[i-1]
                elif good[i] < good[i-1]:
                    new_shares = 0
                    new_balance = shares[i-1] * price[i]
                    shares[i] = new_shares
                    balance[i] = balance[i-1] + new_balance
                value[i] = shares[i] * price[i]

            model = pd.DataFrame({'value':value,'balance':balance}, index=nyse_dti)
            model['worth'] = model.value + model.balance
            initial = 1000//price[0]
            model['hodl'] = price*initial + 1000 - price[0]*initial

            net[n] = model.worth[-1] - model.hodl[-1]
            output[0,n] = x
            output[1,n] = y
            output[2,n] = z
            output[3,n] = len(buy_times)
            output[4,n] = net[n]
            # print('{:.0f}'.format(net[n]))
            n = n + 1

dat = pd.DataFrame(output.T, columns=['P1', 'P2', 'P3', 'Buys', 'Net'])

#%%
fig1 = plt.figure()

ax1 = plt.subplot(4,1,1)
sns.boxplot(x=dat.P1, y=dat.Net, data=dat)

ax1 = plt.subplot(4,1,2)
sns.boxplot(x=dat.P2, y=dat.Net, data=dat)

ax1 = plt.subplot(4,1,3)
sns.boxplot(x=dat.P3, y=dat.Net, data=dat)

ax1 = plt.subplot(4,1,4)
sns.boxplot(x=dat.P1, y=dat.Buys, data=dat)


import albumentations as A
from albumentations import *
from albumentations.pytorch import ToTensorV2
import numpy as np
import cv2
import torch

class Augmentations:
    def __init__(self, train_rgb: bool = True):
        """Base class for creating Augmentations object"""
        self.train_rgb = train_rgb

    def _parse_cfg(
        self, image_size: list, augmentations: dict, keep_aspect_ratio: bool = True
    ):
        """Parses provided config and returns Albumentations Compose object"""
        image_size = image_size

        # Always perform Resize
        if keep_aspect_ratio:
            resize = A.Sequential(
                [
                    A.LongestMaxSize(max_size=max(image_size), interpolation=1),
                    A.PadIfNeeded(
                        min_height=image_size[0],
                        min_width=image_size[1],
                        border_mode=0,
                        value=(0, 0, 0),
                    ),
                ],
                p=1,
            )
        else:
            resize = A.Resize(image_size[0], image_size[1])

        all_augs = [resize]
        if augmentations:
            for aug in augmentations:
                all_augs.append(eval(aug["name"])(**aug.get("params", {})))
        all_augs.append(ToTensorV2())

        return A.Compose(
            all_augs,
            bbox_params=A.BboxParams(format="coco", label_fields=["bbox_classes"]),
            keypoint_params=A.KeypointParams(
                format="xy", label_fields=["keypoints_classes"], remove_invisible=False
            ),
        )

    def __call__(self, data: tuple):
        """Performs augmentations on provided data"""
        img, anno_dict = data
        present_annotations = anno_dict.keys()
        img_in = img.numpy()

        # albumentations expects with RGB image with HWC format
        img_in = np.transpose(img_in, (1, 2, 0))
        img_in = cv2.cvtColor(img_in, cv2.COLOR_BGR2RGB)

        classes = anno_dict.get("class", torch.zeros(1))

        seg = anno_dict.get("segmentation", torch.zeros((1, *img_in.shape[:-1])))
        masks = [m.numpy() for m in seg]

        # COCO format in albumentations is [x,y,w,h] non-normalized
        bboxes = anno_dict.get("bbox", torch.zeros((0, 5)))
        ih, iw, _ = img_in.shape
        bboxes_points = bboxes[:, 1:]
        bboxes_points[:, 0::2] *= iw
        bboxes_points[:, 1::2] *= ih
        bboxes_points = check_bboxes(bboxes_points)
        bbox_classes = bboxes[:, 0]

        # albumentations expects "list" of keypoints e.g. [(x,y),(x,y),(x,y),(x,y)]
        keypoints = anno_dict.get("keypoints", torch.zeros((1, 3 + 1)))
        keypoints_classes = keypoints[:, 0]
        keypoints_flat = torch.reshape(keypoints[:, 1:], (-1, 3))
        keypoints_points = keypoints_flat[:, :2]
        keypoints_points[:, 0] *= iw
        keypoints_points[:, 1] *= ih
        keypoints_visibility = keypoints_flat[:, 2]

        transformed = self.transform(
            image=img_in,
            masks=masks,
            bboxes=bboxes_points,
            bbox_classes=bbox_classes,
            keypoints=keypoints_points,
            keypoints_classes=keypoints_visibility,  # not object class, but per-kp class
        )

        (
            transformed_image,
            out_bboxes,
            transformed_mask,
            final_keypoints,
        ) = post_augment_process(
            transformed, keypoints, keypoints_classes, use_rgb=self.train_rgb
        )

        out_annotations = create_out_annotations(
            present_annotations,
            classes=classes,
            bboxes=out_bboxes,
            masks=transformed_mask,
            keypoints=final_keypoints,
        )

        return transformed_image, out_annotations

class TrainAugmentations(Augmentations):
    def __init__(
        self,
        image_size: list,
        augmentations: dict,
        train_rgb: bool = True,
        keep_aspect_ratio: bool = True,
    ):
        """Class for train augmentations"""
        super().__init__(train_rgb=train_rgb)
        self.transform = self._parse_cfg(
            image_size=image_size,
            augmentations=augmentations,
            keep_aspect_ratio=keep_aspect_ratio,
        )

class ValAugmentations(Augmentations):
    def __init__(
        self,
        image_size: list,
        augmentations: dict,
        train_rgb: bool = True,
        keep_aspect_ratio: bool = True,
    ):
        """Class for val augmentations"""
        super().__init__(train_rgb=train_rgb)
        self.transform = self._parse_cfg(
            image_size=image_size,
            augmentations=[a for a in augmentations if a["name"] == "Normalize"],
            keep_aspect_ratio=keep_aspect_ratio,
        )

def post_augment_process(
    transformed: dict,
    keypoints: torch.Tensor,
    keypoints_classes: np.array,
    use_rgb: bool = True,
):
    """Post process augmentation outputs to prepare for training"""
    transformed_image = transformed["image"]
    if not use_rgb:
        transformed_image = transformed_image.flip(-3)

    transformed_mask = torch.stack(transformed["masks"])  # stack list of masks

    transformed_bboxes = torch.tensor(transformed["bboxes"])
    transformed_bbox_classes = torch.tensor(transformed["bbox_classes"])
    # merge bboxes and classes back together
    transformed_bbox_classes = torch.unsqueeze(transformed_bbox_classes, dim=-1)
    out_bboxes = torch.cat((transformed_bbox_classes, transformed_bboxes), dim=1)

    out_bboxes[:, 1::2] /= transformed_image.shape[2]
    out_bboxes[:, 2::2] /= transformed_image.shape[1]

    transformed_keypoints = torch.tensor(transformed["keypoints"])
    transformed_keypoints_classes = torch.tensor(transformed["keypoints_classes"])
    # merge keypoints and classes back together
    transformed_keypoints_classes = torch.unsqueeze(
        transformed_keypoints_classes, dim=-1
    )
    out_keypoints = torch.cat(
        (transformed_keypoints, transformed_keypoints_classes), dim=1
    )

    out_keypoints = torch.reshape(out_keypoints, (-1, 3))
    out_keypoints = mark_invisible_keypoints(out_keypoints, transformed_image)
    out_keypoints[..., 0] /= transformed_image.shape[2]
    out_keypoints[..., 1] /= transformed_image.shape[1]
    out_keypoints = torch.reshape(
        out_keypoints, (keypoints.shape[0], keypoints.shape[1] - 1)
    )

    final_keypoints = torch.zeros_like(keypoints)
    final_keypoints[:, 1:] = out_keypoints
    final_keypoints[:, 0] = keypoints_classes

    return transformed_image, out_bboxes, transformed_mask, final_keypoints

def mark_invisible_keypoints(keypoints: torch.Tensor, image: np.array):
    """Mark invisible keypoints with label == 0"""
    _, h, w = image.shape
    for kp in keypoints:
        if not (0 <= kp[0] < w and 0 <= kp[1] < h):
            kp[2] = 0
    return keypoints

def check_bboxes(bboxes: torch.Tensor):
    """Check bbox annotations and correct those with width or height 0"""
    for i in range(bboxes.shape[0]):
        if bboxes[i, 2] == 0:
            bboxes[i, 2] = 1
        if bboxes[i, 3] == 0:
            bboxes[i, 3] = 1
    return bboxes

def create_out_annotations(
    present_annotations: list,
    classes: torch.Tensor,
    bboxes: torch.Tensor,
    masks: torch.Tensor,
    keypoints: torch.Tensor,
):
    """Create dictionary of output annotations"""
    out_annotations = {}
    if "class" in present_annotations:
        out_annotations["class"] = classes
    if "bbox" in present_annotations:
        out_annotations["bbox"] = bboxes
    if "segmentation" in present_annotations:
        out_annotations["segmentation"] = masks
    if "keypoints" in present_annotations:
        out_annotations["keypoints"] = keypoints
    return out_annotations


"""
Asynchronous API for data scraping
"""

from __future__ import annotations

import traceback
import warnings
from typing import Type, TypeVar, Union

from playwright.async_api import Page, Route, TimeoutError, async_playwright
from pydantic import ValidationError
from tiktokapipy import TikTokAPIError, TikTokAPIWarning
from tiktokapipy.api import TikTokAPI
from tiktokapipy.models.challenge import Challenge
from tiktokapipy.models.raw_data import (
    ChallengePage,
    PrimaryResponseType,
    SentToLoginResponse,
    UserResponse,
    VideoPage,
)
from tiktokapipy.models.user import User, user_link
from tiktokapipy.models.video import Video, is_mobile_share_link
from tiktokapipy.util.queries import get_challenge_detail_async, get_video_detail_async

_DataModelT = TypeVar("_DataModelT", bound=PrimaryResponseType, covariant=True)
"""
Generic used for data scraping.
"""

class AsyncTikTokAPI(TikTokAPI):
    """Asynchronous API used to scrape data from TikTok"""

    def __enter__(self):
        raise TikTokAPIError("Must use async context manager with AsyncTikTokAPI")

    async def __aenter__(self) -> AsyncTikTokAPI:
        self._playwright = await async_playwright().start()
        self._browser = await self.playwright.chromium.launch(
            headless=self.headless, **self.kwargs
        )

        context_kwargs = self.context_kwargs
        context_kwargs.update(self.playwright.devices["Desktop Edge"])

        self._context = await self.browser.new_context(**context_kwargs)
        self.context.set_default_navigation_timeout(self.navigation_timeout)

        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb):
        await self.context.close()
        await self.browser.close()
        await self.playwright.stop()

    async def _scrape_data(
        self,
        link: str,
        data_model: Type[_DataModelT],
    ) -> _DataModelT:
        for _ in range(self.navigation_retries + 1):
            await self.context.clear_cookies()
            page: Page = await self._context.new_page()
            await page.add_init_script(
                """
if (navigator.webdriver === false) {
    // Post Chrome 89.0.4339.0 and already good
} else if (navigator.webdriver === undefined) {
    // Pre Chrome 89.0.4339.0 and already good
} else {
    // Pre Chrome 88.0.4291.0 and needs patching
    delete Object.getPrototypeOf(navigator).webdriver
}
            """
            )

            async def ignore_scripts(route: Route):
                if route.request.resource_type == "script":
                    return await route.abort()
                return await route.continue_()

            await page.route("**/*", ignore_scripts)
            try:
                await page.goto(link, wait_until=None)
                await page.wait_for_selector("#SIGI_STATE", state="attached")
                content = await page.content()

                data = content.split(
                    '<script id="SIGI_STATE" type="application/json">'
                )[1].split("</script>")[0]

                if "LoginContextModule" in data:
                    warnings.warn(
                        "Redirected to a login page. Trying again...",
                        category=TikTokAPIWarning,
                        stacklevel=2,
                    )
                    sent_to_login = SentToLoginResponse.model_validate_json(data)
                    await page.goto(
                        sent_to_login.login_context_module.redirect_url, wait_until=None
                    )
                    await page.wait_for_selector("#SIGI_STATE", state="attached")
                    content = await page.content()
                    data = content.split(
                        '<script id="SIGI_STATE" type="application/json">'
                    )[1].split("</script>")[0]

                await page.close()

                extracted = self._extract_and_dump_data(data, data_model)
            except (ValidationError, IndexError) as e:
                traceback.print_exception(type(e), e, e.__traceback__)
                await page.close()
                continue
            except TimeoutError:
                warnings.warn(
                    "Reached navigation timeout. Retrying...",
                    category=TikTokAPIWarning,
                    stacklevel=2,
                )
                await page.close()
                continue
            break
        else:
            raise TikTokAPIError(
                f"Data scraping unable to complete in {self.navigation_timeout / 1000}s "
                f"(retries: {self.navigation_retries})"
            )

        return extracted

    async def challenge(
        self, challenge_name: str, *, video_limit: int = -1
    ) -> Challenge:
        response = ChallengePage.model_validate(
            await get_challenge_detail_async(challenge_name, self.context)
        )
        challenge = self._extract_challenge_from_response(response)
        challenge.videos.limit(video_limit)
        return challenge

    async def user(self, user: str, *, video_limit: int = -1) -> User:
        link = user_link(user)
        response = await self._scrape_data(
            link,
            UserResponse,
        )
        user = self._extract_user_from_response(response)
        user.videos.limit(video_limit)
        return user

    async def video(
        self,
        link_or_id: Union[int, str],
    ) -> Video:
        if isinstance(link_or_id, str):
            if is_mobile_share_link(link_or_id):
                await self.context.clear_cookies()
                page: Page = await self.context.new_page()
                await page.add_init_script(
                    """
    if (navigator.webdriver === false) {
        // Post Chrome 89.0.4339.0 and already good
    } else if (navigator.webdriver === undefined) {
        // Pre Chrome 89.0.4339.0 and already good
    } else {
        // Pre Chrome 88.0.4291.0 and needs patching
        delete Object.getPrototypeOf(navigator).webdriver
    }
                """
                )

                async def ignore_scripts(route: Route):
                    if route.request.resource_type == "script":
                        return await route.abort()
                    return await route.continue_()

                await page.route("**/*", ignore_scripts)
                await page.goto(link_or_id, wait_until=None)
                await page.wait_for_selector("#SIGI_STATE", state="attached")

                link_or_id = page.url

                await page.close()
            video_id = link_or_id.split("/")[-1].split("?")[0]
        else:
            video_id = link_or_id

        response = VideoPage.model_validate(
            await get_video_detail_async(video_id, self.context)
        )
        return self._extract_video_from_response(response)

__all__ = ["AsyncTikTokAPI"]


import logging

import torch
import numpy
from fedstellar.learning.aggregators.aggregator import Aggregator

class Krum(Aggregator):
    """
    Krum [Peva Blanchard et al., 2017]
    Paper: https://papers.nips.cc/paper/2017/hash/f4b9ec30ad9f68f89b29639786cb62ef-Abstract.html
    """

    def __init__(self, node_name="unknown", config=None):
        super().__init__(node_name, config)
        self.config = config
        self.role = self.config.participant["device_args"]["role"]
        logging.info("[Krum] My config is {}".format(self.config))

    def aggregate(self, models):
        """
        Krum selects one of the m local models that is similar to other models
        as the global model, the euclidean distance between two local models is used.

        Args:
            models: Dictionary with the models (node: model,num_samples).
        """
        # Check if there are models to aggregate
        if len(models) == 0:
            logging.error(
                "[Krum] Trying to aggregate models when there is no models"
            )
            return None

        models = list(models.values())

        # Total Samples
        total_samples = sum([y for _, y in models])

        # Create a Zero Model
        accum = (models[-1][0]).copy()
        for layer in accum:
            accum[layer] = torch.zeros_like(accum[layer])

        # Add weighteds models
        logging.info("[Krum.aggregate] Aggregating models: num={}".format(len(models)))

        # Create model distance list
        total_models = len(models)
        distance_list = [0 for i in range(0, total_models)]

        # Calculate the L2 Norm between xi and xj
        min_index = 0
        min_distance_sum = float('inf')

        for i in range(0, total_models):
            m1, _ = models[i]
            for j in range(0, total_models):
                m2, _ = models[j]
                distance = 0
                if i == j:
                    distance = 0
                else:
                    for layer in m1:
                        l1 = m1[layer]
                        # l1 = l1.view(len(l1), 1)

                        l2 = m2[layer]
                        # l2 = l2.view(len(l2), 1)
                        distance += numpy.linalg.norm(l1 - l2)
                distance_list[i] += distance

            if min_distance_sum > distance_list[i]:
                min_distance_sum = distance_list[i]
                min_index = i

        # Assign the model with min distance with others as the aggregated model
        m, _ = models[min_index]
        for layer in m:
            accum[layer] = accum[layer] + m[layer]

        return accum


from flask import jsonify, request, Flask
from flask_cors import CORS

from backend.fetchers.fetch_categories import fetch_categories, fetch_subcategories
from backend.fetchers.fetch_cities import fetch_cities
from backend.fetchers.fetch_hotels import fetch_hotels
from backend.fetchers.fetch_pois import fetch_pois
from backend.fetchers.fetch_routes import fetch_routes

from urllib.parse import parse_qs

app = Flask(__name__)
CORS(app)

@app.route('/api/cities', methods=['GET'])
def get_locations():
    # Retrieve the list of available locations from your data source
    cities = fetch_cities()
    print("here")

    # Return the locations as a response
    return jsonify(cities)

@app.route('/')
def hello_world():
    return 'Hello, World!'

@app.route('/api/hotels', methods=['GET'])
def get_hotels():
    city = request.args.get('city')
    if city:
        hotels = fetch_hotels(city)
    else:
        hotels = []

    # Return the locations as a response
    return jsonify(hotels)

@app.route('/api/pois', methods=['GET'])
def get_pois():
    categories = request.args.get('categories')
    categories = categories.split(",")
    city = request.args.get('city')
    pois = fetch_pois(city, categories)

    return jsonify(pois)

@app.route('/api/categories', methods=['GET'])
def get_categories():
    city = request.args.get('city')
    categories = fetch_categories(city)

    return jsonify(categories)

@app.route('/api/subcategories', methods=['GET'])
def get_subcategories():
    city = request.args.get('city')
    subcategories = fetch_subcategories(city)

    return jsonify(subcategories)

@app.route('/api/routes', methods=['GET'])
def get_routes():
    params = parse_qs(request.args.get('params'))
    city = params.get('city')[0]
    hotel = params.get('hotel')[0]
    categories = params.get('categories')[0].split(",")
    compulsory_pois = params.get('compulsory_pois', [])
    if compulsory_pois:
        compulsory_pois = compulsory_pois[0].split(",")
    num_days = int(params.get('days')[0])
    max_distance = int(params.get('max_distance')[0])
    max_locations = int(params.get('max_locations')[0])

    routes, distance = fetch_routes(city, hotel, categories, compulsory_pois, num_days, max_distance, max_locations)
    result = {
        'routes': routes,
        'distance': distance
    }

    return result

#app.run(debug=True)


# Using dates with Pandas
# Objects for points in time and time periods
# Time can be a series or DF columns, but a DF Index converts the object into Time Series
# Basic building block:
    # pd.Timestamp
import pandas as pd
import numpy as np
from datetime import datetime
time_stamp = pd.Timestamp(datetime(2017, 1, 1))

# can also use a date string instead of an object
pd.Timestamp('2017-01-01') == time_stamp
time_stamp.year
time_stamp.day_name()

period = pd.Period('2017-01')
# Period defaults to a month
period.asfreq('D')
period + 2  #(returns 2 months ahead)
pd.Timestamp('2017-01-01', 'M') + 1

# time series requires a sequence of dates
index = pd.date_range(start='2018-01-01', periods=12, freq='M')
# can also specify an enddate instead of number of periods
# default is daily frequency
pd.DateTimeIndex    #sequence of Timestamp objects with frequency info
# can index into it   index[0]

pd.DataFrame({'data': index}).info()

# data = np.random.random((size=12,2))
# pd.DataFrame(data=data,  index=index).info()
# creates time series with 12 monthly timestamps

# Exercises:

# Create the range of dates here
seven_days = pd.date_range(start='2017-1-1', periods=7)

# Iterate over the dates and print the number and name of the weekday
for day in seven_days:
    print(day.dayofweek, day.day_name())

# Transforming time series
google = pd.read_csv('google.csv')
google.info()

# to convert strings to datetime64
pd.to_datetime()

# date into index   .set_index()
google.set_index('date', inplace=True)

google.price.plot(title='Google Stock Price')
plt.tight_layout(); plt.show()

# selecting/indexing using strings to parse to dates
google['2015'].info()
google['2014-3': '2016-2'].info()
google.loc['2016-6-1', 'price']     #need to use full date and column label with .loc

# to set frequency
google.asfreq('D').info()   #set calendar day freq
google.asfreq('B').info()   #set as business day freq

# Exercises:
data = pd.read_csv('nyc.csv')

# Inspect data
print(data.info())

# Convert the date column to datetime64
data.date = pd.to_datetime(data.date)

# Set date column as index
data.set_index('date', inplace=True)

# Inspect data
print(data.info())

# Plot data
data.plot(subplots=True)
plt.show()

data = pd.read_csv('nyc.csv')

# Inspect data
print(data.info())

# Convert the date column to datetime64
data.date = pd.to_datetime(data.date)

# Set date column as index
data.set_index('date', inplace=True)

# Inspect data
print(data.info())

# Plot data
data.plot(subplots=True)
plt.show()

# Inspect data
print(co.info())

# Set the frequency to calendar daily
co = co.asfreq('D')

# Plot the data
co.plot(subplots=True)
plt.show()

# Set frequency to monthly
co = co.asfreq('M')

# Plot the data
co.plot(subplots=True)
plt.show()

google = pd.read_csv('google.csv', parse_dates=['date'], index_col='date')

# moving data between past and future
# defaults to periods=1
google['shifted'] = google.price.shift()
google['lagged'] = google.price.shift(periods=-1)

# to calculate one period percent change (called financial return in finance)
# returns relative change in price in percent
google['change'] = google.price.div(google.shifted)

google['return'] = google.change.sub(1).mul(100)

# .diff calculates the change between values at different points in time
# can be used to calculate one period returns
google['diff'] = google.price.diff()

# percent change for two adjacent periods
google['pct_change']  = google.price.pct_change().mul(100)

# default periods = 1
google['return_3d']  = google.price.pct_change(periods=3).mul(100)

# Import data here
google = pd.read_csv('google.csv', parse_dates=['Date'], index_col='Date')

# Set data frequency to business daily
google = google.asfreq('B')

# Create 'lagged' and 'shifted'
google['lagged'] = google.Close.shift(periods=-90)
google['shifted'] = google.Close.shift(periods=90)

# Plot the google price series
google.plot('lagged', 'shifted')
plt.show()

# Created shifted_30 here
yahoo['shifted_30'] = yahoo.price.shift(periods=30)

# Subtract shifted_30 from price
yahoo['change_30'] = yahoo.price.sub(yahoo.shifted_30)

# Get the 30-day price difference
yahoo['diff_30'] = yahoo.price.diff(periods=30)

# Inspect the last five rows of price
print(yahoo.tail(5))

# Show the value_counts of the difference between change_30 and diff_30
print(yahoo.change_30.sub(yahoo.diff_30).value_counts())

# Create daily_return
google['daily_return'] = google.Close.pct_change().mul(100)

# Create monthly_return
google['monthly_return'] = google.Close.pct_change(periods=30).mul(100)

# Create annual_return
google['annual_return'] = google.Close.pct_change(periods=360).mul(100)

# Plot the result
google.plot(subplots=True)
plt.show()

# Compare growth rates
# stock prices: hard to compare at different levels
# one solution: normalize price series to start at 100
    # divide all time series by its first value and multiply by 100
first_price = google.price.iloc[0]
normalized = google.price.div(first_price).mul(100)
normalized.plot(title="Google Normalized Series")

# normalizing multiple series
prices = pd.read_csv('stock_prices.csv', parse_dates=['date'], index_col='date')
normalized = prices.div(prices.iloc[0])

index = pd.read_csv('benchmark.csv', parse_dates=['date'], index_col='date')
prices = pd.concat([prices, index], axis=1).dropna()

normalized = prices.div(prices.iloc[0]).mul(100)
normalized.plot()

# plotting performance difference
diff = normalized[tickers].sub(normalized['SP500'], axis=0)

# Exercises:
# Import data here
prices = pd.read_csv('asset_classes.csv', parse_dates=['DATE'], index_col='DATE')

# Inspect prices here
print(prices.info())

# Select first prices
first_prices = prices.iloc[0]

# Create normalized
normalized = prices.div(first_prices).mul(100)

# Plot normalized
normalized.plot()
plt.show()

# Import stock prices and index here
stocks = pd.read_csv('nyse.csv', parse_dates=['date'], index_col='date')
dow_jones = pd.read_csv('dow_jones.csv', parse_dates=['date'], index_col='date')

# Concatenate data and inspect result here
data = pd.concat([stocks, dow_jones], axis=1)
print(data.info())

# Normalize and plot your data here
normalized = data.div(data.iloc[0]).mul(100).plot()
plt.show()

# Create tickers
tickers = ['MSFT', 'AAPL']

# Import stock data here
stocks = pd.read_csv('msft_aapl.csv', parse_dates=['date'], index_col='date')

# Import index here
sp500 = pd.read_csv('sp500.csv', parse_dates=['date'], index_col='date')

# Concatenate stocks and index here
data = pd.concat([stocks, sp500], axis=1).dropna()

# Normalize data
normalized = data.div(data.iloc[0]).mul(100)

# Subtract the normalized index from the normalized stock prices, and plot the result
diff = normalized[tickers].sub(normalized['SP500'], axis=0).plot()
plt.show()

# Changing time series frequency: resampling
# frequency conversion affects the data
# upsampling creates new rows that you need to tell pandas how to fill or interpolate
# downsampling - you need to tell pandas how to aggregate existing data

dates = pd.date_range(start='2016', periods=4, freq='Q')
data = range(1,5)
quarterly = pd.Series(data=data, index=dates)

# default for quarterly is Dec for end of 4th quarter
monthly = quarterly.asfreq('M')

# ways to fill in missing values
monthly = monthly.to_frame('baseline')
monthly['ffill'] = quarterly.asfreq('M', method='ffill')   #forward fill
monthly['bfill'] = quarterly.asfreq('M', method='bfill')    #backward fill
monthly['value'] = quarterly.asfreq('M', fill_value=0)      #replace with a value

# you can reindex too
dates = pd.date_range(start='2016', periods=12, freq='M')
quarterly.reindex(dates)

# exercises
# Set start and end dates
start = '2016-1-1'
end = '2016-2-29'

# Create monthly_dates here
monthly_dates = pd.date_range(start=start, end=end, freq='M')

# Create and print monthly here
monthly = pd.Series(data=[1,2], index=monthly_dates)
print(monthly)

# Create weekly_dates here
weekly_dates = pd.date_range(start=start, end=end, freq='W')

# Print monthly, reindexed using weekly_dates
print(monthly.reindex(weekly_dates))
print(monthly.reindex(weekly_dates, method='bfill'))
print(monthly.reindex(weekly_dates, method='ffill'))

# Import data here
data = pd.read_csv('unemployment.csv', parse_dates=['date'], index_col='date')

# Show first five rows of weekly series
print(data.asfreq('W').head())

# Show first five rows of weekly series with bfill option
print(data.asfreq('W', method='bfill').head())

# Create weekly series with ffill option and show first five rows
weekly_ffill = data.asfreq('W', method='ffill')
print(weekly_ffill.head())

# Plot weekly_fill starting 2015 here
weekly_ffill.loc['2015':].plot()
plt.show()

# Upsampling and interpoliation with .resample()
# works similar to groupby()
# groups data within resampling period and applies one or more methods to each group
# can be used to fill or interpolate values during upsampling
# or for applying aggregation for downsampling

unrate = pd.read_csv('unrate.csv', parse_dates['Date'], index_col='Date')

# resampling creates new date for frequency offset
# resample can assign a frequency if none exists
# Calendar Month End    'M'
# Calendar Month Start  'MS'
# Business Month End    'BM'    may deviate based on weekends or holidays
# Business Month Start  'BMS'

unrate.asfreq('MS').info()
unrate.resample('MS')   #creates a Resampler object
unrate.asfreq('MS').equals(unrate.resample('MS').asfreq())  #returns true

# resample only returns data when calling another method
gdp_1 = gdp.resample('MS').ffill().add_suffix('ffill')
gdp_2 = gdp.resample('MS').interpolate().add_suffix('_inter')
df1 = pd.DataFrame([1,2,3], columns=['df1'])
df2 = pd.DataFrame([4,5,6], columns=['df2'])
pd.concat([df1, df2])
pd.concat([df1, df2], axis=1)  #concats the DF's horizontally, alinging the row index

# Exercises:
# Inspect data here
print(monthly.info())

# Create weekly dates
weekly_dates = pd.date_range(start=monthly.index.min(), end=monthly.index.max(), freq='W')

# Reindex monthly to weekly data
weekly = monthly.reindex(weekly_dates)

# Create ffill and interpolated columns
weekly['ffill'] = weekly.UNRATE.ffill()
weekly['interpolated'] = weekly.UNRATE.interpolate()

# Plot weekly
weekly.plot()
plt.show()

# Import & inspect data here
data = pd.read_csv('debt_unemployment.csv', parse_dates=['date'], index_col='date')
print(data.info())

# Interpolate and inspect here
interpolated = data.interpolate()
print(interpolated.info())

# Plot interpolated data here
interpolated.plot(secondary_y='Unemployment')
plt.show()

# downsampling (day to month, hour to day)
# how to summarize existing data
ozone = ozone.resample('D').asfreq()
ozone.reample('M').mean().head()    #creates monthly freq with default month-end offset with monthly average

# can use more than one agg method
ozone.resample('M').agg(['mean', 'std']).head()

ozone = ozone.loc['2016':]
ax = ozone.plot()
monthly = ozone.resample('M').mean()
monthly.add_suffix('_monthly').plot(ax=ax)

data = pd.read_csv('ozone_pm25.csv', parse_dates=['date'], index_col='date')
data = data.resample('D').asfreq()

data = data.resample('BM').mean()

df.resample('M').first().head(4)
df.resample('MS').first().head(4)

# Exercises
# Import and inspect data here
ozone = pd.read_csv('ozone.csv', parse_dates=['date'], index_col='date')
ozone.info()

# Calculate and plot the weekly average ozone trend
ozone.resample('W').mean().plot()
plt.show()

# Calculate and plot the monthly average ozone trend
ozone.resample('M').mean().plot()
plt.show()

# Calculate and plot the annual average ozone trend
ozone.resample('A').mean().plot()
plt.show()

# Import and inspect data here
stocks = pd.read_csv('stocks.csv', parse_dates=['date'], index_col='date')
print(stocks.info())

# Calculate and plot the monthly averages
monthly_average = stocks.resample('M').mean()
monthly_average.plot(subplots=True)
plt.show()

# Import and inspect gdp_growth here
gdp_growth = pd.read_csv('gdp_growth.csv', parse_dates=['date'], index_col='date')
gdp_growth.info()

# Import and inspect djia here
djia = pd.read_csv('djia.csv', parse_dates=['date'], index_col='date')
djia.info()

# Calculate djia quarterly returns here
djia_quarterly = djia.resample('QS').first()
djia_quarterly_return = djia_quarterly.pct_change().mul(100)

# Concatenate, rename and plot djia_quarterly_return and gdp_growth here
data = pd.concat([gdp_growth, djia_quarterly_return], axis=1)
data.columns = ['gdp', 'djia']

data.plot()
plt.show()

# Import data here
sp500 = pd.read_csv('sp500.csv', parse_dates=['date'], index_col='date')
sp500.info()

# Calculate daily returns here
daily_returns = sp500.squeeze().pct_change()

# Resample and calculate statistics
stats = daily_returns.resample('M').agg(['mean', 'median', 'std'])

# Plot stats here
stats.plot()
plt.show()

# Rolling windows functions
# Calculate metrics for subperiods inside the window
# creates a new time series of metrics (a summary of several data poitns in the original time series)
# Two types:
    # rolling - same size, slides over the times series
    # expanding - grow with the tiem series and contain all prior values
# Calculating a rolling average:
data.rolling(window=30).mean()

# window = # of business days
# mean will only be calculated if the winosw has no missing values
# can change that by making the min_periods parameter smaller than the window
data.rolling(window='30D').mean()
# this means calendar days

r90 = data.rolling(window='90D').mean()
google.join(r90.add_suffix('_mean_90')).plot()

data['mean90'] = r90
r360 = data['price'].rolling(window='360D').mean()
data['mean360'] = r360; data.plot()

# can do multiple rolling metrics
r = data.price.rolling('90D').agg(['mean', 'std'])
r.plot(subplots=True)

# or like this:
rolling = data.google.rolling('360D')
q10 = rolling.quantile(0.1).to_frame('q10')
median = rolling.median().to_frame('median')
q90 = rolling.quantile(0.9).to_frame('q90')
pd.concat([q10, median, q90], axis=1).plot()

# Exercises

# Import and inspect ozone data here
data = pd.read_csv('ozone.csv', parse_dates=['date'], index_col='date')
print(data.info())

# Calculate 90d and 360d rolling mean for the last price
data['90D'] = data['Ozone'].rolling(window='90D').mean()
data['360D'] = data['Ozone'].rolling(window='360D').mean()

# Plot data
data['2010':].plot(title='New York City')
plt.show()

# Import and inspect ozone data here
data = pd.read_csv('ozone.csv', parse_dates=['date'], index_col='date').dropna()

# Calculate the rolling mean and std here
rolling_stats = data.Ozone.rolling(360).agg(['mean', 'std'])

# Join rolling_stats with ozone data
stats = data.join(rolling_stats)

# Plot stats
stats.plot(subplots=True);
plt.show()

# Resample, interpolate and inspect ozone data here
data = data.resample('D').interpolate()
print(data.info())

# Create the rolling window
rolling = data.Ozone.rolling(360)

# Insert the rolling quantiles to the monthly returns
data['q10'] = rolling.quantile(0.1).to_frame('q10')
data['q50'] = rolling.quantile(0.5).to_frame('q50')
data['q90'] = rolling.quantile(0.9).to_frame('q90')

# Plot the data
data.plot(subplots=True)
plt.show()

# Expanding window functions
# calculates emtrics for periods up to current date
# new time series that reflects all historical values up to that point
# good for running rate of return, running min/max
# can either use .expanding() (works just like .rolling())
# or use .cumsum(), .cumprod(), .cummin()/max()

df = pd.DataFrame({'data': range(5)})
df['expanding sum'] = df.data.expanding().sum()
df['cumulative sum'] = df.data.cumsum()

data = pd.read_csv('sp500.csv', parse_dates=['date'], index_col="date")
# to calculate running rate of return
pr = data.SP500.pct_change()
pr_plus_one = pr.add(1)
cumulative_return = pr_plus_one.cumprod().sub(1)
cumulative_return.mul(100).plot()       #this makes a percentage and plots

data['running_min'] = data.SP500.expanding().min()
data['running_max'] = data.SP500.expanding().max()
data.plot()

def multi_period_return(period_returns):
    return np.prod(period_returns + 1) -1

pr = data.SP500.pct_change()
r = pr.rolling('360D').apply(multi_period_return)
data['Rolling 1 yr Return'] = r.mul(100)
data.plot(subplots=True)

# Exercises
# Calculate differences
differences = data.diff().dropna()

# Select start price
start_price = data.first('D')

# Calculate cumulative sum
cumulative_sum = start_price.append(differences).cumsum()

# Validate cumulative sum equals data
print(data.equals(cumulative_sum))

# Define your investment
investment = 1000

# Calculate the daily returns here
returns = data.pct_change()

# Calculate the cumulative returns here
returns_plus_one = returns.add(1)
cumulative_return = returns_plus_one.cumprod()

# Calculate and plot the investment return here
cumulative_return.mul(investment).plot()
plt.show()

# Import numpy
import numpy as np

# Define a multi_period_return function
def multi_period_return(period_returns):
    return np.prod(period_returns + 1) -1

# Calculate daily returns
daily_returns = data.pct_change()

# Calculate rolling_annual_returns
rolling_annual_returns = daily_returns.rolling('360D').apply(multi_period_return)

# Plot rolling_annual_returns
rolling_annual_returns.mul(100).plot()
plt.show()

# Using random walk to simulate daily stock price predictions
from numpy.random import normal, seed
from scipy.stats import norm
seed(42)
random_returns = normal(loc=0, scale=0.01, size=1000)
sns.distplot(random_returns, fit=norm, kde=False)

return_series = pd.Series(random_returns)
random_prices = return_series.add(1).cumprod().sub(1)
random_prices.mul(100).plot()

data = pd.read_csv('sp500.csv', parse_dates=['date'], index_col='date')
data['returns'] = data.SP500.pct_change()
data.plot(subplots=True)

sns.displot(data.returns.dropna().mul(100), fit=norm)

from numpy.random import choice
sample = data.returns.dropna()
n_obs = data.returns.count()
random_walk = choice(sample, size=n_obs)
random_walk = pd.Series(random_walk, index=sample.index)
random_walk.head()

start = data.SP500.first('D')
sp500_random = start.append(random_walk.add(1))
sp500_random.head()

data['SP500_random'] = sp500_random.cumprod()
data[['SP500', 'SP500_random']].plot()

# Exercises
# Set seed here
seed(42)

# Create random_walk
random_walk = normal(loc=.001, scale=.01, size=2500)

# Convert random_walk to pd.series
random_walk = pd.Series(random_walk)

# Create random_prices
random_prices = random_walk.add(1).cumprod()

# Plot random_prices here
random_prices.mul(1000).plot()
plt.show()

# Set seed here
seed(42)

# Calculate daily_returns here
daily_returns = fb.pct_change().dropna()

# Get n_obs
n_obs = daily_returns.count()

# Create random_walk
random_walk = choice(daily_returns, size=n_obs )

# Convert random_walk to pd.series
random_walk = pd.Series(random_walk)

# Plot random_walk distribution
sns.distplot(random_walk)
plt.show()

# Select fb start price here
start = fb.price.first('D')

# Add 1 to random walk and append to start
random_walk = random_walk + 1
random_price = start.append(random_walk)

# Calculate cumulative product here
random_price = random_price.cumprod()

# Insert into fb and plot
fb['random'] = random_price
fb.plot()
plt.show()

# Correlation - measures linear relationships
# correlation coefficient - how similar is the pairwise movement of two variables around their averages
# between -1 and 1
# the closer to 1 or -1, the more linear the relationship

data = pd.read_csv('assets.csv', parse_dates=['date'], index_col='date')
data = data.dropna().info()
daily_returns = data.pct_change()
sns.jointplot(x='sp500', y='nasdaq', data=daily_returns)

correlations = returns.corr()
sns.heatmap(correlations, annot=True)

# Exercises
# Inspect data here
print(data.info())

# Calculate year-end prices here
annual_prices = data.resample('A').last()

# Calculate annual returns here
annual_returns = annual_prices.pct_change()

# Calculate and print the correlation matrix here
correlations = annual_returns.corr()
print(correlations)

# Visualize the correlations as heatmap here
sns.heatmap(correlations, annot=True)
plt.show()

# Manipulating time series data
# Value weighted index
# share price * number of shares => market value
# Larger companies have a larger weight and will have a larger impact on index

nyse = pd.read_csv('listings.xlsx', sheet_name='nyse', na_values='n/a')
nyse.set_index('Stock Symbol', inplace=True)
nyse.dropna(subset=['Sector'], inplace=True)
nyse['Market Capitalization'] /= 1e6
components = nyse.groupby(['Sector'])['Market Capitalization'].nlargest(1)
components.sort_values(ascending=False)
tickers = components.index.get_level_values('Stock Symbol')
tickers.tolist()

columns = ['Company Name', 'Market Capitalization', 'Last Sale']
component_info = nyse.loc[tickers, columns]
pd.options.display.float_format = '{:,.2f}'.format

data = pd.read_csv('stocks.csv', parse_dates=["Date"], index_col='Date').loc[:, tickers.tolist()]

# Exercises
# Inspect listings
print(listings.info())

# Move 'stock symbol' into the index
listings.set_index('Stock Symbol', inplace=True)

# Drop rows with missing 'sector' data
listings.dropna(subset=['Sector'], inplace=True)

# Select companies with IPO Year before 2019
listings = listings[listings['IPO Year'] < 2019]

# Inspect the new listings data
print(listings.info())

# Show the number of companies per sector
print(listings.groupby('Sector').size().sort_values(ascending=False))

# Select largest company for each sector
components = listings.groupby(['Sector'])['Market Capitalization'].nlargest(1)

# Print components, sorted by market cap
print(components.sort_values(ascending=False))

# Select stock symbols and print the result
tickers = components.index.get_level_values('Stock Symbol')
print(tickers)

# Print company name, market cap, and last price for each component
info_cols = ['Company Name', 'Market Capitalization', 'Last Sale']
print(listings.loc[tickers, info_cols].sort_values('Market Capitalization', ascending=False))

# Print tickers
print(tickers)

# Import prices and inspect result
stock_prices = pd.read_csv('stock_prices.csv', parse_dates=['Date'], index_col='Date')
print(stock_prices.info())

# Calculate the returns
price_return = stock_prices.iloc[-1].div(stock_prices.iloc[0]).sub(1).mul(100)

# Plot horizontal bar chart of sorted price_return
price_return.sort_values().plot(kind='barh', title='Stock Price Returns')
plt.show()

# number of shares
shares = components['Market Capitalization'].div(components['Last Sale'])

data = pd.read_csv('stocks.csv', parse_dates=['Date'], index_col='Date').loc[:, tickers.tolist()]
market_cap_series = data.mul(no_shares)
market_cap_series.first('D').append(market_cap_series.last('D'))

agg_mcap = market_cap_series.sum(axis=1)  #total market cap
agg_mcap(title='Aggregate Market Cap')

index = agg_mcap.div(agg_mcap.iloc[0]).mul(100) #creates value based index
index.plot(title='Market-Cap Weighted Index')

# Inspect listings and print tickers
print(listings.info())
print(tickers)

# Select components and relevant columns from listings
components = listings.loc[tickers, ['Market Capitalization', 'Last Sale']]

# Print the first rows of components
print(components.head())

# Calculate the number of shares here
no_shares = components['Market Capitalization'].div(components['Last Sale'])

# Print the sorted no_shares
print(no_shares.sort_values(ascending=False))

# Select the number of shares
no_shares = components['Number of Shares']
print(no_shares.sort_values())

# Create the series of market cap per ticker
market_cap = stock_prices.mul(no_shares)

# Select first and last market cap here
first_value = market_cap.iloc[0]
last_value = market_cap.iloc[-1]

# Concatenate and plot first and last market cap here
pd.concat([first_value, last_value], axis=1).plot(kind='barh')
plt.show()

# Aggregate and print the market cap per trading day
raw_index = market_cap_series.sum(axis=1)
print(raw_index)

# Normalize the aggregate market cap here
index = raw_index.div(raw_index.iloc[0]).mul(100)
print(index)

# Plot the index here
index.plot(title='Market-Cap Weighted Index')
plt.show()

# Evaluated index performance
# look at index return
# performance vs. benchmakr

agg_market_cap = market_cap_series.sum(axis=1)
index = agg_market_cap.div(agg_market_cap.iloc[0]).mul(100)
index.plot(title='Market-Cap Weighted Index')

agg_market_cap.ilocc[-1] - agg_market_cap.iloc[0]

# Value contribution by stock
change = market_cap_series.first('D').append(market_cap_series.last('D'))
change.diff().iloc[-1].sort_values()

market_cap = components['Market Capitalization']
weights = market_cap.div(market_cap.sum())
weights.sort_values().mul(100)
# Value weighted component returns
index_return = (index.iloc[-1] / index.iloc[0] -1) * 100
weighted_returns = weights.mul(index_return)
weighted_returns.sort_values().plot(kind='barh')

# performance vs benchmark
data = index.to_frame('Index')
data['SP500'] = pd.read_csv('sp500.csv', parse_dates=['Date'], index_col='Date')
data.SP500 = data.SP500.div(data.SP500.iloc[0], axis=0).mul(100)

def multi_period_return(r):
    return (np.prod(r +1) -1) * 100
data.pct_change().rolling('30D').apply(multi_period_return).plot()

# Calculate and print the index return here
index_return = (index.iloc[-1] / index.iloc[0] -1) * 100
print(index_return)

# Select the market capitalization
market_cap = components['Market Capitalization']

# Calculate the total market cap
total_market_cap = market_cap.sum()

# Calculate the component weights, and print the result
weights = market_cap.div(total_market_cap)
print(weights.sort_values())

# Calculate and plot the contribution by component
weights.mul(index_return).sort_values().plot(kind='barh')
plt.show()

# Calculate and print the index return here
index_return = (index.iloc[-1] / index.iloc[0] -1) * 100
print(index_return)

# Select the market capitalization
market_cap = components['Market Capitalization']

# Calculate the total market cap
total_market_cap = market_cap.sum()

# Calculate the component weights, and print the result
weights = market_cap.div(total_market_cap)
print(weights.sort_values())

# Calculate and plot the contribution by component
weights.mul(index_return).sort_values().plot(kind='barh')
plt.show()

# Inspect data
print(data.info())
print(data.head())

# Create multi_period_return function here
def multi_period_return(r):
    return (np.prod(r + 1) -1) * 100

# Calculate rolling_return_360
rolling_return_360 = data.pct_change().rolling('360D').apply(multi_period_return)

# Plot rolling_return_360 here
rolling_return_360.plot(title='Rolling 360D Return')
plt.show()

# Index Correlation
data = DataReader(tickers, 'google', start='2016', end='2017')['Close']
data.info()

daily_returns = data.pct_change()
correlations = daily_returns.corr()
sns.heatmap(correlations, annot=True)
plt.xticks(rotation=45)
plt.title('Daily Return Correlations')

# Saving to Excel
correlations.to_excel(excel_writer = 'correlations.xls',
    sheet_name='corrrelations',
    startrow=1,
    startcol=1)

# Saving to multiple worksheets
data.index = data.index.date
with pd.Excel_writer('stock_data.xlsx') as writer:
    corr.to_excel(excel_writer=writer, sheet_name='correlations')
    data.to_excel(excel_writer=writer, sheet_name='prices')
    data.pct_change.to_excel(excel_writer=writer, sheet_name='returns')

# Exercises
# Inspect stock_prices here
print(stock_prices.info())

# Calculate the daily returns
returns = stock_prices.pct_change()

# Calculate and print the pairwise correlations
correlations = returns.corr()
print(correlations)

# Plot a heatmap of daily return correlations
sns.heatmap(correlations, annot=True)
plt.title('Daily Returns Correlations')
plt.show()

# Inspect index and stock_prices
print(index.info())
print(stock_prices.info())

# Join index to stock_prices, and inspect the result
data = stock_prices.join(index)
print(data.info())

# Create index & stock price returns
returns = data.pct_change()

# Export data and data as returns to excel
with pd.ExcelWriter('data.xls') as writer:
    data.to_excel(excel_writer=writer, sheet_name='data')
    returns.to_excel(excel_writer=writer, sheet_name='returns')