# -*- coding: utf-8 -*-
"""
Created on Thu May 19 09:37:43 2022

@author: cvskf
"""

import os
import importlib
import inspect

for file in os.listdir():
    print(file)
    if file.startswith('gbElements'):

        version_=file[-7:-3]
        print(version_)
        version=version_.replace('_','.')
        print(version)

        with open(os.path.join(os.pardir,
                               os.pardir,
                               'docs',
                               'auto_auto_methods_pages',
                               f'{version}_index.rst'),
                  'w') as f:

            f.write(f'Schema Version {version}\n')
            f.write('===================\n\n')
            f.write('.. toctree::\n\n')

            my_module = importlib.import_module(f'xgbxml.auto.gbElements_{version_}')

            print(my_module)

            for kls_name,kls in inspect.getmembers(my_module, inspect.isclass):

                f.write(f'   {version}/{kls_name}\n')

                continue  # to avoid recreating the large set of files below each time.

                with open(os.path.join(os.pardir,
                                       os.pardir,
                                       'docs',
                                       'auto_auto_methods_pages',
                                       f'{version}',
                                       f'{kls_name}.rst'),
                          'w') as f1:

                    f1.write(f'{kls_name}\n')
                    x='='*len(kls_name)
                    f1.write(f'{x}\n\n')
                    f1.write(f'.. autoclass:: xgbxml.auto.gbElements_{version_}.{kls_name}\n')
                    f1.write('   :members:\n')
                    f1.write('   :undoc-members:\n\n')

    #break


#!/usr/bin/env python3

import argparse
import os
import sys

from environs import Env

from src.service.consumer_service import consumer_service
from src.service.logging_service import logging_service
from src.service.producer_service import producer_service

def main():
    """ Derived from https://github.com/aiven/aiven-examples/blob/master/kafka/python/main.py """

    parser = argparse.ArgumentParser()
    parser.add_argument('--kafka-service-uri', help="Kafka Service URI in the form host:port",
                        required=True)
    parser.add_argument('--db-service-uri', help="PostgreSQL Database service URI in the form host:port. "
                                                 "Can be left blank if only running the producer.",
                        required=True)
    parser.add_argument('--ca-path', help="Path to project CA certificate",
                        required=True)
    parser.add_argument('--key-path', help="Path to the Kafka Access Key (obtained from Aiven Console)",
                        required=True)
    parser.add_argument('--cert-path', help="Path to the Kafka Certificate Key (obtained from Aiven Console)",
                        required=True)

    parser.add_argument('--consumer', action='store_true', default=False, help="Run Kafka consumer for the "
                                                                               "processing monitor")
    parser.add_argument('--producer', action='store_true', default=False, help="Run Kafka producer for the "
                                                                               "processing monitor")
    parser.add_argument('--logger', action='store_true', default=False,
                        help="Run logger that prints out recent updates to the DB.")

    env = Env()
    env.read_env()
    validate_env(env)

    args = parser.parse_args()
    validate_args(args)

    if args.producer:
        producer_service(
            args.kafka_service_uri,
            args.ca_path,
            args.cert_path,
            args.key_path,
            env.str("WEBSITE_MONITOR_AVAILABILITY_TOPIC"),
            env.int("PING_INTERVAL_SECONDS"))

    elif args.consumer:
        consumer_service(args.kafka_service_uri,
                         args.ca_path,
                         args.cert_path,
                         args.key_path,
                         env.str("WEBSITE_MONITOR_AVAILABILITY_TOPIC"),
                         env.str("WEBSITE_MONITOR_TABLE_NAME"),
                         args.db_service_uri)
    elif args.logger:
        logging_service(args.db_service_uri, env.str("WEBSITE_MONITOR_TABLE_NAME"))

def validate_env(env):
    for env_param in ("WEBSITE_MONITOR_AVAILABILITY_TOPIC", "PING_INTERVAL_SECONDS", "WEBSITE_MONITOR_TABLE_NAME"):
        try:
            env.str(env_param)
        except:
            fail(f"Failed to retrieve env_param :: {env_param} :: from env file.\n"
                 f"You can set these to your liking in the env file.")

def validate_args(args):
    for path_option in ("ca_path", "key_path", "cert_path"):
        path = getattr(args, path_option)
        if not os.path.isfile(path):
            fail(f"Failed to open --{path_option.replace('_', '-')} at path: {path}.\n"
                 f"You can retrieve these details from Overview tab in the Aiven Console")
    if args.producer and args.consumer:
        fail("--producer and --consumer are mutually exclusive")
    elif not args.producer and not args.consumer and not args.logger:
        fail("--producer, --logger or --consumer are required")

def fail(message):
    print(message, file=sys.stderr)
    exit(1)

if __name__ == '__main__':
    main()


from django.test import LiveServerTestCase
from selenium import webdriver

class HomepageTestCase(LiveServerTestCase):
    def setUp(self):
        self.selenium = webdriver.Chrome()
        self.selenium.maximize_window()
        super(HomepageTestCase, self).setUp()

    def tearDown(self):
        self.selenium.quit()
        super(HomepageTestCase, self).tearDown()

    def test_can_visit_homepage(self):
        self.selenium.get(
            '%s%s' % (self.live_server_url,  "/")
        )

        self.assertIn("Growth Studio - Enjoy Create & Share", self.selenium.title)


from __future__ import absolute_import, division, print_function

from marshmallow import fields

from polyaxon.schemas.base import BaseConfig, BaseSchema

class CelerySchema(BaseSchema):
    taskTrackStarted = fields.Bool(allow_none=True)
    brokerPoolLimit = fields.Int(allow_none=True)
    confirmPublish = fields.Bool(allow_none=True)
    workerPrefetchMultiplier = fields.Int(allow_none=True)
    workerMaxTasksPerChild = fields.Int(allow_none=True)
    workerMaxMemoryPerChild = fields.Int(allow_none=True)
    taskAlwaysEager = fields.Bool(allow_none=True)

    @staticmethod
    def schema_config():
        return CeleryConfig

class CeleryConfig(BaseConfig):
    SCHEMA = CelerySchema
    REDUCED_ATTRIBUTES = [
        "taskTrackStarted",
        "brokerPoolLimit",
        "confirmPublish",
        "workerPrefetchMultiplier",
        "workerMaxTasksPerChild",
        "workerMaxMemoryPerChild",
        "taskAlwaysEager",
    ]

    def __init__(
        self,  # noqa
        taskTrackStarted=None,
        brokerPoolLimit=None,
        confirmPublish=None,
        workerPrefetchMultiplier=None,
        workerMaxTasksPerChild=None,
        workerMaxMemoryPerChild=None,
        taskAlwaysEager=None,
    ):
        self.taskTrackStarted = taskTrackStarted
        self.brokerPoolLimit = brokerPoolLimit
        self.confirmPublish = confirmPublish
        self.workerPrefetchMultiplier = workerPrefetchMultiplier
        self.workerMaxTasksPerChild = workerMaxTasksPerChild
        self.workerMaxMemoryPerChild = workerMaxMemoryPerChild
        self.taskAlwaysEager = taskAlwaysEager


# pylint: disable=cell-var-from-loop
import math
from functools import partial

import numpy as np
import torch
import torch.utils.data
from ray import tune
from ray.tune import CLIReporter
from ray.tune.schedulers import ASHAScheduler

# data to csv
import pandas as pd

from halutmatmul.modules import (
    create_bit_matrix,
    create_selection_matrix,
    halut_matmul_forward,
)
import halutmatmul.halutmatmul as hm

dtype = torch.float32

device = torch.device(
    "cpu"
)  # torch.device("cuda:1" if torch.cuda.is_available() else "cpu")

data_path = "/usr/scratch2/vilan1/janniss/halut/resnet9-tanh-all"

Cs = [64, 32, 16]
data = []

layers = [
    "conv2.0",
    "res1.0.0",
    "res1.1.0",
    "conv3.0",
    "conv4.0",
    "res2.0.0",
    "res2.1.0",
]
layer_factor = {
    "conv2.0": 1,
    "res1.0.0": 2,
    "res1.1.0": 2,
    "conv3.0": 2,
    "conv4.0": 4,
    "res2.0.0": 4,
    "res2.1.0": 4,
}

def getBack(var_grad_fn, all_shapes: list) -> None:
    print(var_grad_fn)
    for n in var_grad_fn.next_functions:
        if n[0]:
            try:
                tensor = getattr(n[0], "variable")
                print(n[0])
                print("Tensor with grad found:", tensor.shape)
                print(" - gradient:", tensor.grad.shape)
                if len(tensor.grad.shape) > 1:
                    # count zeros
                    print(
                        "LUT grad - zeros:",
                        torch.sum(tensor.grad == 0),
                        torch.numel(tensor.grad),
                    )
                elif tensor.grad.shape[0] > 1:
                    print("threshold grad", tensor.grad, torch.sum(tensor.grad == 0))
                all_shapes.append(tensor.grad.shape)
                print()
            except AttributeError:
                getBack(n[0], all_shapes)

class HalutMatmul(torch.nn.Module):
    def __init__(self, C, K, S, B, T, L, dims, t_grad=True, l_grad=True):
        super().__init__()
        self.C = C
        self.K = K
        self.S = torch.nn.Parameter(S, requires_grad=False)
        self.B = torch.nn.Parameter(B, requires_grad=False)
        self.T = torch.nn.Parameter(T, requires_grad=t_grad)
        self.L = torch.nn.Parameter(L, requires_grad=l_grad)
        self.D = torch.nn.Parameter(dims, requires_grad=False)
        self.temp = torch.nn.Parameter(torch.ones(1).to(device), requires_grad=False)

    def halut_updates(self):
        self.temp.data = torch.clamp(self.temp.data, 0.1, 1.0)
        # print("temp", self.temp.data)

    def forward(self, I):
        return halut_matmul_forward(
            I,
            self.T,
            self.L,
            self.S,
            self.B,
            self.C,
            self.K,
            self.D,
            temperature=self.temp,
        )

def train_one_epoch(model, criterion, optimizer, data_loader, device, epoch):
    model.train()

    def halut_updates(module, prefix=""):
        if isinstance(module, (HalutMatmul)):
            module.halut_updates()
            return

        for child_name, child_module in module.named_children():
            child_prefix = f"{prefix}.{child_name}" if prefix != "" else child_name
            halut_updates(child_module, prefix=child_prefix)

    optimizer.zero_grad()
    total_loss = 0

    for _, (input) in enumerate(data_loader):
        input = input[0].to(device)
        output = model(input)
        target = input.mm(W)
        loss = criterion(output, target)
        total_loss += loss.item()
        loss.backward()
        # if idx % 100 == 0:
        # print("gradient threshold", model.T.grad)
        # print("gradient lut", model.L.grad)
        # getBack(loss.grad_fn, [])
        optimizer.step()
        optimizer.zero_grad()
        halut_updates(model)

    print(
        f"({epoch}) Average loss train: {total_loss / len(data_loader)}, "
        f"lr: {optimizer.param_groups[0]['lr']}"
    )

def evaluate(model, criterion, data_loader, device):
    model.eval()
    total_loss = 0
    with torch.inference_mode():
        for _, (input) in enumerate(data_loader):
            input = input[0].to(device)
            output = model(input)
            target = input.mm(W)
            loss = criterion(output, target)
            total_loss += loss.item()
    print("Average loss evaluate: {}".format(total_loss / len(data_loader)))
    return total_loss / len(data_loader)

def train(
    model,
    criterion,
    optimizer,
    data_loader_train,
    data_loader_val,
    device,
    epochs,
    lr_scheduler,
    is_plateu=False,
):
    evaluate(model, criterion, data_loader_val, device)
    for i in range(epochs):
        train_one_epoch(model, criterion, optimizer, data_loader_train, device, i)
        if is_plateu:
            lr_scheduler.step(evaluate(model, criterion, data_loader_val, device))
        else:
            lr_scheduler.step()
        evaluate(model, criterion, data_loader_val, device)

for c_ in Cs:
    for layer in layers:
        row = {}
        test_layer_A = f"{layer}_A.npy"
        test_layer_B = f"{layer}_B.npy"

        I = (
            torch.from_numpy(np.load(data_path + "/" + test_layer_A))
            .to(dtype)
            .to(device)
        )
        W = (
            torch.from_numpy(np.load(data_path + "/" + test_layer_B))
            .to(dtype)
            .to(device)
        )

        index_random = torch.randperm(I.shape[0])
        I = I[index_random]

        N = I.shape[0]
        # train_input = I[: N - (N // 10)]
        # val_input = I[N - (N // 10) :]
        train_input = I
        val_input = I

        # C = 64
        C = c_ * layer_factor[layer]
        K = 16
        M = W.shape[1]
        N = I.shape[0]
        D = I.shape[1]

        S = create_selection_matrix(C, K, dtype=dtype).to(device)
        B = create_bit_matrix(C, K, dtype=dtype).to(device)
        T = torch.randn((C * 15), dtype=dtype).to(device)
        T = torch.nn.init.uniform_(T, a=0, b=1)
        L = torch.randn((M, C, K), dtype=dtype).to(device)

        L = torch.nn.init.kaiming_uniform_(L, a=math.sqrt(5))

        depth = int(math.sqrt(K))

        result = torch.matmul(I, W)

        row["C"] = C
        row["K"] = K
        row["M"] = M
        row["N"] = N
        row["D"] = D
        row["layer"] = test_layer_A  # type: ignore

        halut_learned_init = hm.learn_halut_offline(
            train_input.detach().cpu().numpy(), W.detach().cpu().numpy(), C, K
        )
        halut_lut_init = (
            torch.from_numpy(halut_learned_init[hm.HalutOfflineStorage.LUT])
            .to(dtype)
            .to(device)
        )
        halut_T_init = (
            torch.from_numpy(halut_learned_init[hm.HalutOfflineStorage.THRESHOLDS])
            .to(dtype)
            .to(device)
        )
        halut_dims_init = (
            torch.from_numpy(halut_learned_init[hm.HalutOfflineStorage.DIMS])
            .to(torch.int64)
            .to(device)
        )
        L = halut_lut_init
        # L = torch.nn.init.kaiming_uniform_(L)
        T = halut_T_init
        # T = torch.nn.init.uniform_(T)
        dims = halut_dims_init
        # in_channels = I.shape[1] // 9
        # dims = torch.zeros(in_channels * 4, dtype=torch.int64)
        # for i in range(in_channels):
        #     # random select idx out of list with no duplicates
        #     # ensure no duplicates
        #     channel_dims = torch.tensor(
        #         np.random.choice([0, 1, 2, 3, 4, 5, 6, 7, 8], size=4, replace=False),
        #         dtype=torch.int64,
        #     )
        #     dims[i * 4 : (i + 1) * 4] = channel_dims + i * 9

        # train
        batch_size = 256
        epochs = 250
        model = HalutMatmul(C, K, S, B, T, L, dims)
        optimizer = torch.optim.Adam(model.parameters(), lr=0.00004)
        # criterion = torch.nn.HuberLoss(reduction="mean")
        # criterion = torch.nn.L1Loss(reduction="mean")
        criterion = torch.nn.MSELoss(reduction="mean")

        train_dataset = torch.utils.data.TensorDataset(train_input)
        data_loader_train = torch.utils.data.DataLoader(
            train_dataset, batch_size=batch_size, shuffle=True
        )
        val_dataset = torch.utils.data.TensorDataset(val_input)
        data_loader_val = torch.utils.data.DataLoader(
            val_dataset, batch_size=batch_size, shuffle=False
        )

        lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
            optimizer=optimizer, T_max=epochs
        )

        # train(model, criterion,
        # optimizer, data_loader_train, data_loader_val, device, epochs, lr_scheduler)

        configs = {
            "lr": tune.grid_search([0.0005]),  # tune.uniform(0.0001, 0.001)
            "batch_size": tune.grid_search([2048]),
            "epochs": tune.grid_search([100]),
            "optimizer": tune.grid_search(["adam"]),  # sgd
            "criterion": tune.grid_search(["mse"]),  # mae, huber
            "lr_scheduler": tune.grid_search(["cosine"]),  # plateau, step
            "learn_tensors": tune.grid_search(
                [
                    [True, False, True],
                ]
            ),
        }

        configs = {
            "lr": 0.001,
            "batch_size": 128,
            "epochs": 20,
            "optimizer": "adam",
            "criterion": "mse",
            "lr_scheduler": "cosine",
            "learn_tensors": [True, True],
        }

        row["epochs"] = configs["epochs"]
        row["batch_size"] = configs["batch_size"]
        row["lr"] = configs["lr"]
        row["optimizer"] = configs["optimizer"]
        row["criterion"] = configs["criterion"]
        row["lr_scheduler"] = configs["lr_scheduler"]

        def train_with_config(config):
            I = (
                torch.from_numpy(np.load(data_path + "/" + test_layer_A))
                .to(dtype)
                .to(device)
            )
            train_input = I
            val_input = I
            model = HalutMatmul(C, K, S, B, T, L, dims, *config["learn_tensors"])
            batch_size = config["batch_size"]
            train_dataset = torch.utils.data.TensorDataset(train_input)
            data_loader_train = torch.utils.data.DataLoader(
                train_dataset, batch_size=batch_size, shuffle=True
            )
            val_dataset = torch.utils.data.TensorDataset(val_input)
            data_loader_val = torch.utils.data.DataLoader(
                val_dataset, batch_size=batch_size, shuffle=False
            )

            params = {
                "other": [],
                "thresholds": [],
                "temperature": [],
                "lut": [],
            }

            def _add_params(module, prefix=""):
                for name, p in module.named_parameters(recurse=False):
                    if not p.requires_grad:  # removes parameter from optimizer!!
                        # filter(lambda p: p.requires_grad, model.parameters())
                        continue
                    if isinstance(module, (HalutMatmul)):
                        if name == "T":
                            params["thresholds"].append(p)
                            continue
                        if name == "temp":
                            params["temperature"].append(p)
                            continue
                        if name == "L":
                            params["lut"].append(p)
                            continue
                    # if prefix in ("conv1", "linear"):
                    #     continue
                    print("add to other", prefix, name)
                    params["other"].append(p)

                for child_name, child_module in module.named_children():
                    child_prefix = (
                        f"{prefix}.{child_name}" if prefix != "" else child_name
                    )
                    _add_params(child_module, prefix=child_prefix)

            _add_params(model)

            custom_lrs = {
                "other": config["lr"],
                "temperature": config["lr"] * 0,
                "thresholds": config["lr"] * 1,
                "lut": config["lr"],
            }
            param_groups = []
            # pylint: disable=consider-using-dict-items
            for key in params:
                if len(params[key]) > 0:
                    # pylint: disable=consider-iterating-dictionary
                    if key in custom_lrs.keys():
                        param_groups.append(
                            {"params": params[key], "lr": custom_lrs[key]}
                        )
                    else:
                        param_groups.append({"params": params[key]})

            epochs = config["epochs"]
            if config["optimizer"] == "sgd":
                optimizer = torch.optim.SGD(param_groups, lr=config["lr"])
            elif config["optimizer"] == "adam":
                optimizer = torch.optim.Adam(param_groups, lr=config["lr"])

            if config["lr_scheduler"] == "plateau":
                lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
                    optimizer=optimizer, mode="min", factor=0.1, patience=10
                )
            elif config["lr_scheduler"] == "step":
                lr_scheduler = torch.optim.lr_scheduler.StepLR(
                    optimizer=optimizer, step_size=epochs // 4, gamma=0.1
                )
            elif config["lr_scheduler"] == "cosine":
                lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
                    optimizer=optimizer, T_max=epochs
                )

            if config["criterion"] == "mse":
                criterion = torch.nn.MSELoss(reduction="mean")
            elif config["criterion"] == "l1":
                criterion = torch.nn.L1Loss(reduction="mean")
            elif config["criterion"] == "huber":
                criterion = torch.nn.HuberLoss(reduction="mean")

            train(
                model,
                criterion,
                optimizer,
                data_loader_train,
                data_loader_val,
                device,
                epochs,
                lr_scheduler,
                is_plateu=config["lr_scheduler"] == "plateau",
            )

            mse = evaluate(
                model, torch.nn.MSELoss(reduction="mean"), data_loader_val, device
            )
            mae = evaluate(
                model, torch.nn.L1Loss(reduction="mean"), data_loader_val, device
            )
            huber = evaluate(
                model, torch.nn.HuberLoss(reduction="mean"), data_loader_val, device
            )
            return mse, mae, huber

        mse, mae, huber = train_with_config(configs)
        print("Best trial final validation loss: {}".format(mse))  # type: ignore
        print("Best trial final validation MAE: {}".format(mae))  # type: ignore
        print("Best trial final validation Huber" ": {}".format(huber))  # type: ignore

        # print("Final results", model.T, model.L)
        print("Final max", torch.max(model.T), torch.max(model.L))

        output = model(val_input)
        target = val_input.mm(W)

        print("Final loss", criterion(output, target))
        print("Final max", torch.max(output), torch.max(target))

        # print("Compare", output[0], target[0])
        print("Final MSE", torch.nn.MSELoss(reduction="mean")(output, target))
        print("Final MAE", torch.nn.L1Loss(reduction="mean")(output, target))
        print("Final Huber", torch.nn.HuberLoss(reduction="mean")(output, target))

        print("config", configs)

        row["mse_backprop"] = mse
        row["mae_backprop"] = mae
        row["huber_backprop"] = huber
        row["max_backprop"] = torch.max(output).item()  # type: ignore

        halut_learned = hm.learn_halut_offline(
            I.detach().cpu().numpy(), W.detach().cpu().numpy(), C, K
        )
        halut_lut = (
            torch.from_numpy(halut_learned[hm.HalutOfflineStorage.LUT])
            .to(dtype)
            .to(device)
        )
        halut_T = (
            torch.from_numpy(halut_learned[hm.HalutOfflineStorage.THRESHOLDS])
            .to(dtype)
            .to(device)
        )
        halut_dims = (
            torch.from_numpy(halut_learned[hm.HalutOfflineStorage.DIMS])
            .to(torch.int64)
            .to(device)
        )

        halut_learned_model = HalutMatmul(C, K, S, B, halut_T, halut_lut, halut_dims)
        halut_learned_output = halut_learned_model(val_input)
        print("Learned loss", criterion(halut_learned_output, target))
        print("Learned max", torch.max(halut_learned_output), torch.max(target))
        print(
            "Learned MSE",
            torch.nn.MSELoss(reduction="mean")(halut_learned_output, target),
        )
        print(
            "Learned MAE",
            torch.nn.L1Loss(reduction="mean")(halut_learned_output, target),
        )
        print(
            "Learned Huber",
            torch.nn.HuberLoss(reduction="mean")(halut_learned_output, target),
        )

        print("Compare", halut_learned_output[0], target[0])

        row["mse_learned"] = torch.nn.MSELoss(reduction="mean")(
            halut_learned_output, target
        ).item()
        row["mae_learned"] = torch.nn.L1Loss(reduction="mean")(
            halut_learned_output, target
        ).item()
        row["huber_learned"] = torch.nn.HuberLoss(reduction="mean")(
            halut_learned_output, target
        ).item()
        row["max_learned"] = torch.max(halut_learned_output).item()  # type: ignore

        for t in [4, 6, 7, 8, 10, 16, 32]:
            # convert to empty type as empty_strided is not supported
            scale = torch.max(I) - torch.min(I)
            scale = scale / (2**t - 1)
            I_quant = torch.fake_quantize_per_tensor_affine(  # type: ignore
                I, scale, 0, quant_min=-(2 ** (t - 1)), quant_max=2 ** (t - 1) - 1
            )
            W_quant = torch.fake_quantize_per_tensor_affine(  # type: ignore
                W, scale, 0, quant_min=-(2 ** (t - 1)), quant_max=2 ** (t - 1) - 1
            )

            result = I_quant.mm(W_quant)
            result_scale_back = result
            print(
                f"MSE INT{t}",
                torch.nn.MSELoss(reduction="mean")(result_scale_back, target),
            )
            print(
                f"MAE INT{t}",
                torch.nn.L1Loss(reduction="mean")(result_scale_back, target),
            )
            print(
                f"Huber INT{t}",
                torch.nn.HuberLoss(reduction="mean")(result_scale_back, target),
            )
            row[f"int{t}_mse"] = torch.nn.MSELoss(reduction="mean")(
                result_scale_back, target
            ).item()
            row[f"int{t}_mae"] = torch.nn.L1Loss(reduction="mean")(
                result_scale_back, target
            ).item()
            row[f"int{t}_huber"] = torch.nn.HuberLoss(reduction="mean")(
                result_scale_back, target
            ).item()
        data.append(row)

# data to csv
df = pd.DataFrame(data)
df.to_csv("accuracy_comparison_all.csv")


import requests
import sqlite3
import pandas as pd

engine = sqlite3.connect("pokedex.db")
cursor = engine.cursor()

names = []
types = []

for i in range(1,100):
    data = requests.get(f"https://pokeapi.co/api/v2/pokemon/{i}/").json()
    names.append(data["name"])
    types.append(data["types"][0]["type"]["name"])

df = pd.DataFrame({"name": names, "type": types})

df.to_sql("pokeinfo", engine, if_exists="replace", index=False)


import pandas as pd
import numpy as np
import random
import regex as re
import sqlite3
import os
import io
import csv

from flask import Flask, jsonify
from flasgger import Swagger

app = Flask(__name__)

from flask import request
from flasgger import Swagger, LazyString, LazyJSONEncoder
from flasgger import swag_from

#string input tweet
def readdatacsvframe(input_tweet= None , csv_tweet = None):
    conn = sqlite3.connect('test_database2')
    c = conn.cursor()
    c.execute('CREATE TABLE IF NOT EXISTS products (Tweet text, price number)')
    conn.commit()
    c.execute('DELETE FROM products ')
    conn.commit()
    if input_tweet :
        print ("input_tweetinput_tweetinput_tweetinput_tweet")
        print (input_tweet)
        c.execute('INSERT INTO products(Tweet) values ("%s") ' % input_tweet)
        conn.commit()

    if csv_tweet  :
        print ("asdfasdfsadffasdfsadfsdfasdfsadfasdfsadf")
        print(csv_tweet)
        dfcsv_tweet = pd.read_csv(csv_tweet,encoding='latin-1')
        dfcsv_tweet.to_sql('products' , conn, if_exists='replace', index = False)
        conn.commit()

    pd.set_option('display.max_rows', None)

    dfabbusive = pd.read_csv('abusive.csv')

    dfss = pd.read_sql_query("SELECT * FROM products", conn)

    tweettolist3= list(set(dfss["Tweet"]))

    dfabbusive.to_sql('productsabbusive', conn, if_exists='replace', index = False)

    dfabbusive = pd.read_sql_query("SELECT * FROM productsabbusive", conn)

    abbusivetolist3= list(set(dfabbusive["ABUSIVE"]))

    tweettolistsetelahregex = []

    for i in tweettolist3:

        pattern = r",*(\s*\b(?:{}))\b".format("|".join(abbusivetolist3))
        hasildiregex= re.sub(pattern, "__SENSOR__", i)

        tweettolistsetelahregex.append(hasildiregex)

    df2s = pd.DataFrame (tweettolistsetelahregex, columns = ['column_setelahdifilter'])
    df2s

    df2s.to_sql('hasilsetelahdicleaning', conn, if_exists='replace', index = False)

    c.execute('DELETE FROM products ')
    conn.commit()

    outputhasil = ""

    if input_tweet:
        print ("ini twetttt input biasa")
        outputhasil = df2s.values.tolist()[0][0]
    if csv_tweet :
        print ("ini csv tweet")
        outputhasil = df2s.values.tolist()

    return outputhasil

app.json_encoder = LazyJSONEncoder
swagger_template = dict(
    info = {
        'title': LazyString(lambda: 'API Documentation for Data Processing and Modeling'),
        'version': LazyString(lambda: '1.0.0'),
        'description': LazyString(lambda: 'Dokumentasi API untuk Data Processing dan Modeling')
    },
    host = LazyString(lambda: request.host)
)
swagger_config = {
    "headers": [],
    "specs": [
        {
            "endpoint": 'docs',
            "route": '/docs.json'
        }
    ],
    "static_url_path": "/flasgger_static",
    "swagger_ui": True,
    "specs_route": "/docs/"
}
swagger = Swagger(app, template=swagger_template,config=swagger_config)

@swag_from("docs/hello_world.yml", methods=['GET'])
@app.route('/', methods=['GET'])
def hello_world():

    resulthasil = readdatacsvframe("ampas")

    json_response = {
        'status_code': 200,
        'description': "Menyapa Hello World",
        'data': resulthasil
    }

    response_data = jsonify(json_response)
    return response_data

@swag_from("docs/text.yml", methods=['GET'])
@app.route('/text', methods=['GET'])
def text():
    json_response = {
        'status_code': 200,
        'description': "Original Teks",
        'data': "Halo, apa kabar semua?"
    }

    response_data = jsonify(json_response)
    return response_data

@swag_from("docs/text_clean.yml", methods=['GET'])
@app.route('/text-clean', methods=['GET'])
def text_clean():
    json_response = {
        'status_code': 200,
        'description': "Original Teks",
        'data': re.sub(r'[^a-zA-Z0-9]', ' ', "Halo, apa kabar semua?")
    }

    response_data = jsonify(json_response)
    return response_data

@swag_from("docs/text_processing.yml", methods=['POST'])
@app.route('/text-processingssss', methods=['POST'])
def text_processing():

    text = request.form.get('text')
    resulthasil = readdatacsvframe(text)

    json_response = {
        'status_code': 200,
        'description': "Original Teks",
        #'data': re.sub(r'[^a-zA-Z0-9]', '###', text)
        'data': resulthasil

    }

    response_data = jsonify(json_response)
    return response_data

@swag_from("docs/text_fileupload.yml", methods=['POST'])
@app.route('/text-uploadfile', methods=['POST'])
def text_uploadfile():

    #fileoke = csv.reader(request.files['file'])
    # fileoke = csv.reader(request.files['file'])
    filesaveku = "data_uploaded.csv"
    request.files['file'].save(filesaveku)

    #fileku = open(filesaveku)

    # dfssss = pd.DataFrame(csv.reader(request.files['file']))
    # csvfile =dfssss.to_csv('data_uploadedss.csv', sep='\t', encoding='utf-8')
    #dfssss  = pd.DataFrame({'A' : ['ampas', 1], 'B' : [1, 6]})
    #csvfile =dfssss.to_csv('data_uploadedss.csv', sep='\t', encoding='utf-8')

    #dfabbusivesss = pd.read_csv('abusive.csv')

        #
        #import ipdb; ipdb.set_trace()

    resultnya = readdatacsvframe(csv_tweet=filesaveku)

    json_response = {
        'status_code': 200,
        'description': "Original Teks",
        #'data': re.sub(r'[^a-zA-Z0-9]', '###', text)
        'data': resultnya

    }

    response_data = jsonify(json_response)
    return response_data

if __name__ == '__main__':
    app.run()


import pygame
from constantes import *
from mapa import Mapa,  Bordas, blocos_destrutiveis, items_posição
from blocos import Blocos_indestrutiveis, Blocos_destrutiveis
from personagem import Personagem
from item import Item
from sprites import *
from timer import Timer
from inimigos import Inimigo
from bomba import Fogo

def main():
    pygame.mixer.music.set_volume(0.5)
    tema = pygame.mixer.music.load("sons\Tema.mp3")
    pygame.mixer.music.play(-1)

    #Criando objetos e o grupo de seus sprites
    pygame.font.init()
    mapa = Mapa(CELULAS_ALTURA*TAMANHO_CELULA, CELULAS_LARGURA*TAMANHO_CELULA, TAMANHO_MENU, TAMANHO_BORDAS)
    bordas = [Bordas(i[0], i[1]) for i in mapa.bordas]
    blocos_ind = [Blocos_indestrutiveis(i[0], i[1]) for i in blocos_indestrutiveis]
    blocos_dest = [Blocos_destrutiveis(i[0], i[1]) for i in blocos_destrutiveis]

    #Criando os itens
    itens = ["velocidade", "tempo", "vida", "portal", "item_bomba"]
    for i, tipo in enumerate(itens):
        Item(mapa.tela, items_posição[i], tipo)

    #Criando os inimigos
    numero_inimigos = 4
    for _ in range(numero_inimigos):
        Inimigo("polemonio", blocos_dest)

    #Criando o personagem
    player = Personagem(25, 125, blocos_dest)
    [sprites.add(i) for i in bordas + blocos_ind + blocos_dest]

    #Definindo a fonte dos textos
    fonte = pygame.font.SysFont('arial', 40, True, True)

    # Gerando o timer
    tempo = 60
    timer = Timer()

    #Iniciando pygame e loop para o jogo ficar rodando até fecharem
    clock = pygame.time.Clock()
    Rodar = True
    pygame.init()

    while Rodar:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                Rodar == False
                exit()

        #Gerando o timer na tela
        timer.__str__() ; tempo += player.buff_tempo
        timer_impressao = f'{tempo-int(timer.sec)}s'
        texto_timer = fonte.render(timer_impressao, False, (255, 255, 255))

        #Gerando a vida na tela
        mensagem = f'{player.vida}'
        mapa.tela.fill((0,0,0))
        texto_formatado = fonte.render(mensagem, False, (255, 255, 255))
        mapa.tela.blit(mapa.background, (0, TAMANHO_BORDAS+TAMANHO_MENU))

        #Desenhando os sprites na tela
        vilao.draw(mapa.tela)
        vilao.update()
        mapa.tela.blit(texto_timer, (TAMANHO_CELULA*CELULAS_LARGURA - 100 -TAMANHO_CELULA//2,0))
        mapa.tela.blit(texto_formatado, (50,0))
        player.sprite.draw(mapa.tela)
        sprites.draw(mapa.tela)
        sprites.update()
        player.sprite.update()

        #Desenhando os sprites do relogio e do coração
        mapa.tela.blit(mapa.relogio, (TAMANHO_CELULA*CELULAS_LARGURA-50, 0))
        mapa.tela.blit(mapa.coração, (0, 0))
        pygame.display.update()
        clock.tick(60)

        #Checa se o tempo acabou, reseta o timer e dá dano no personagem
        if tempo-int(timer.sec) < 0:
            player.dano()
            tempo += 60

        #checa se o personagem ainda tem vidas
        if player.vida <= 0:
            Rodar = False
            win_lose("perdeu")

        #Acaba o jogo quando o personagem encontra o portal
        elif player.portal:
            Rodar = False
            win_lose("ganhou")

        #Depois de perder ou ganhar, se você apertar qualquer botão do teclado, a tela é fechada automaticamente

#Tela de início
def start():
    largura = CELULAS_LARGURA*TAMANHO_CELULA
    altura = CELULAS_ALTURA*TAMANHO_CELULA
    tela_start = pygame.display.set_mode((largura, altura))
    start_back = pygame.image.load(sprite["inicio"])
    start_back = pygame.transform.scale(start_back, (largura, altura))
    pygame.display.set_caption("CINberman - Start_screen")
    pygame.init()
    no_menu = True
    while no_menu:
        tela_start.blit(start_back, (0,0))
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                no_menu == False
                exit()
        botao = pygame.key.get_pressed()
        if botao[pygame.K_RETURN]:
            no_menu = False
            main()
        if botao[pygame.K_ESCAPE]:
            no_menu = False
            exit()
        pygame.display.update()

#Tela de fim de jogo, tanto de derrota quanto para vitória
def win_lose(W_L):
    largura = CELULAS_LARGURA*TAMANHO_CELULA
    altura = CELULAS_ALTURA*TAMANHO_CELULA
    tela_wl = pygame.display.set_mode((largura, altura))
    wl_back = pygame.image.load(sprite[W_L])
    wl_back = pygame.transform.scale(wl_back, (largura, altura))
    if W_L == "ganhou":
        pygame.display.set_caption("CINberman - Você ganhou!")
    elif W_L == "perdeu":
        pygame.display.set_caption("CINberman - Você perdeu!")
    pygame.init()
    no_wl = True
    while no_wl:
        tela_wl.blit(wl_back, (0,0))
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                no_wl == False
                exit()
            if event.type == pygame.KEYDOWN:
                no_wl == False
                exit()
        pygame.display.update()

start()


import asyncio
import contextlib
import functools
import discord
from datetime import datetime, timedelta, timezone
from typing import Tuple, Optional, List, no_type_check
from redbot.core import commands, checks
from redbot.core.config import Config
from redbot.core.i18n import Translator, cog_i18n
from redbot.core.utils.menus import menu, DEFAULT_CONTROLS

from .message import SchedulerMessage
from .logs import get_logger
from .tasks import Task
from .converters import Schedule, non_numeric, TempMute
from .checks import can_run_command

_ = Translator("And I think it's gonna be a long long time...", __file__)

@cog_i18n(_)
class Scheduler(commands.Cog):
    """
    A somewhat sane scheduler cog
    """

    __version__ = "1.0.27"
    __author__ = "mikeshardmind(Sinbad), DiscordLiz"
    __flavor_text__ = "Unhidden remindme."

    def __init__(self, bot):
        self.bot = bot
        self.config = Config.get_conf(
            self, identifier=78631113035100160, force_registration=True
        )
        self.config.register_channel(tasks={})  # Serialized Tasks go in here.
        self.log = get_logger("sinbadcogs.scheduler")
        self.bg_loop_task = bot.loop.create_task(self.bg_loop())
        self.scheduled = {}  # Might change this to a list later.
        self.tasks = []
        self._iter_lock = asyncio.Lock()

    def cog_unload(self):
        self.bg_loop_task.cancel()
        for task in self.scheduled.values():
            task.cancel()
        self.log.handlers = []

    # This never should be needed,
    # but it doesn't hurt to add and could cover a weird edge case.
    __del__ = cog_unload

    async def _load_tasks(self):
        chan_dict = await self.config.all_channels()
        for channel_id, channel_data in chan_dict.items():
            channel = self.bot.get_channel(channel_id)
            if (
                not channel
                or not channel.permissions_for(channel.guild.me).read_messages
            ):
                continue
            tasks_dict = channel_data.get("tasks", {})
            for t in Task.bulk_from_config(bot=self.bot, **tasks_dict):
                self.tasks.append(t)

    async def _remove_tasks(self, *tasks: Task):
        async with self._iter_lock:
            for task in tasks:
                self.tasks.remove(task)
                await self.config.channel(task.channel).clear_raw("tasks", task.uid)

    async def bg_loop(self):
        await self.bot.wait_until_ready()
        await asyncio.sleep(2)
        _guilds = [
            g for g in self.bot.guilds if g.large and not (g.chunked or g.unavailable)
        ]
        await self.bot.request_offline_members(*_guilds)

        async with self._iter_lock:
            await self._load_tasks()
        while self is self.bot.get_cog("Scheduler"):
            sleep_for = await self.schedule_upcoming()
            await asyncio.sleep(sleep_for)

    async def delayed_wrap_and_invoke(self, task: Task, delay: int):
        await asyncio.sleep(delay)
        task.update_objects(self.bot)
        chan = task.channel
        if not chan.permissions_for(chan.guild.me).read_messages:
            return
        message = await task.get_message(self.bot)
        context = await self.bot.get_context(message)
        context.assume_yes = True
        await self.bot.invoke(context)
        for cog_name in ("CustomCommands", "Alias"):
            cog = self.bot.get_cog(cog_name)
            if cog:
                await cog.on_message(message)
        # TODO: allow registering additional cogs to process on_message for.

    async def schedule_upcoming(self) -> int:
        """
        Schedules some upcoming things as tasks.

        """

        # TODO: improve handlng of next time return

        async with self._iter_lock:
            to_pop = []
            for k, v in self.scheduled.items():
                if v.done():
                    to_pop.append(k)
                    try:
                        v.result()
                    except Exception:
                        self.log.exception("Dead task ", exc_info=True)

            for k in to_pop:
                self.scheduled.pop(k, None)

        to_remove: list = []

        for task in self.tasks:
            delay = task.next_call_delay
            if delay < 30 and task.uid not in self.scheduled:
                self.scheduled[task.uid] = asyncio.create_task(
                    self.delayed_wrap_and_invoke(task, delay)
                )
                if not task.recur:
                    to_remove.append(task)

        await self._remove_tasks(*to_remove)

        return 15

    @property
    def task_class(self):
        """ API class Access """
        return Task

    async def submit_task(self, task: Task):
        """ externally safe API """
        async with self._iter_lock:
            self.tasks.append(task)

    async def fetch_task_by_attrs_exact(self, **kwargs) -> List[Task]:
        def pred(item):
            try:
                return kwargs and all(getattr(item, k) == v for k, v in kwargs.items())
            except AttributeError:
                return False

        async with self._iter_lock:
            return [t for t in self.tasks if pred(t)]

    async def fetch_task_by_attrs_lax(
        self, lax: Optional[dict] = None, strict: Optional[dict] = None
    ) -> List[Task]:
        def pred(item):
            try:
                if strict:
                    if not all(getattr(item, k) == v for k, v in strict.items()):
                        return False
            except AttributeError:
                return False
            if lax:
                return any(getattr(item, k, None) == v for k, v in lax.items())
            return True

        async with self._iter_lock:
            return [t for t in self.tasks if pred(t)]

    async def fetch_tasks_by_guild(self, guild: discord.Guild) -> List[Task]:

        async with self._iter_lock:
            return [t for t in self.tasks if t.channel in guild.text_channels]

    # Commands go here

    @checks.mod_or_permissions(manage_guild=True)
    @commands.guild_only()
    @commands.command(usage="<eventname> <command> <args>")
    @no_type_check
    async def schedule(self, ctx, event_name: non_numeric, *, schedule: Schedule):
        """
        Schedule something

        Usage:
            [p]schedule eventname command args

        args:

            you must provide one of:

                --start-in interval
                --start-at time

            you may also provide:

                --every interval

            for recurring tasks

        intervals look like:

            5 minutes
            1 minute 30 seconds
            1 hour
            2 days
            30 days
            (etc)

        times look like:
            February 14 at 6pm EDT

        times default to UTC if no timezone provided.

        Example use:

            [p]schedule autosync bansync True --start-at 12AM --every 1 day

        Example use with other parsed commands:

        [p]schedule autosyndicate syndicatebans --sources 133049272517001216 --auto-destinations -- --start-at 12AM --every 1 hour

        This can also execute aliases.
        """
        schedule: Tuple[str, datetime, Optional[timedelta]]

        command, start, recur = schedule

        t = Task(
            uid=ctx.message.id,
            nicename=event_name,
            author=ctx.author,
            content=command,
            channel=ctx.channel,
            initial=start,
            recur=recur,
        )

        if await self.fetch_task_by_attrs_exact(
            author=ctx.author, channel=ctx.channel, nicename=event_name
        ):
            return await ctx.send("You already have an event by that name here.")

        async with self._iter_lock:
            async with self.config.channel(ctx.channel).tasks() as tsks:
                tsks.update(t.to_config())
            self.tasks.append(t)

        ret = (
            f"Task Scheduled. You can cancel this task with "
            f"`{ctx.clean_prefix}unschedule {ctx.message.id}` "
            f"or with `{ctx.clean_prefix}unschedule {event_name}`"
        )

        if recur and t.next_call_delay < 60:
            ret += (
                "\nWith the intial start being set so soon, "
                "you might have missed an initial use being scheduled by the loop. "
                "you may find the very first expected run of this was missed or otherwise seems late. "
                "Future runs will be on time."  # fractions of a second in terms of accuracy.
            )

        await ctx.send(ret)

    @commands.guild_only()
    @commands.command()
    async def unschedule(self, ctx, info):
        """
        unschedule something.
        """

        tasks = await self.fetch_task_by_attrs_lax(
            lax={"uid": info, "nicename": info},
            strict={"author": ctx.author, "channel": ctx.channel},
        )

        if not tasks:
            await ctx.send(
                f"Hmm, I couldn't find that task. (try `{ctx.clean_prefix}showscheduled`)"
            )

        elif len(tasks) > 1:
            self.log.warning(
                f"Mutiple tasks where should be unique. Task data: {tasks}"
            )
            return await ctx.send(
                "There seems to have been breakage here. Cleaning up and logging incident."
            )

        else:
            await self._remove_tasks(*tasks)
            await ctx.tick()

    @commands.guild_only()
    @commands.command()
    async def showscheduled(self, ctx: commands.Context, all_channels: bool = False):
        """ shows your scheduled tasks in this, or all channels """

        if all_channels:
            tasks = await self.fetch_tasks_by_guild(ctx.guild)
            tasks = [t for t in tasks if t.author == ctx.author]
        else:
            tasks = await self.fetch_task_by_attrs_exact(
                author=ctx.author, channel=ctx.channel
            )

        if not tasks:
            return await ctx.send("No scheduled tasks")

        await self.task_menu(ctx, tasks)

    async def task_menu(self, ctx, tasks, message: Optional[discord.Message] = None):

        color = await ctx.embed_color()

        async def task_killer(
            cog: "Scheduler",
            page_mapping: dict,
            ctx: commands.Context,
            pages: list,
            controls: dict,
            message: discord.Message,
            page: int,
            timeout: float,
            emoji: str,
        ):
            to_cancel = page_mapping.pop(page)
            await cog._remove_tasks(to_cancel)
            if page_mapping:
                tasks = list(page_mapping.values())
                await cog.task_menu(ctx, tasks, message)
            else:
                with contextlib.suppress(discord.NotFound):
                    await message.delete()

        count = len(tasks)
        embeds = [
            t.to_embed(index=i, page_count=count, color=color)
            for i, t in enumerate(tasks, 1)
        ]

        controls = DEFAULT_CONTROLS.copy()
        page_mapping = {i: t for i, t in enumerate(tasks)}
        actual_task_killer = functools.partial(task_killer, self, page_mapping)
        controls.update({"\N{NO ENTRY SIGN}": actual_task_killer})
        await menu(ctx, embeds, controls, message=message)

    @commands.command(name="remindme", usage="<what to be reminded of> <args>")
    async def reminder(self, ctx, *, reminder: Schedule):
        """
        Schedule a reminder DM from the bot

        Usage:
            [p]remindme to do something [args]

        args:

            you must provide one of:

                --start-in interval
                --start-at time

            you may also provide:

                --every interval

            for recurring reminders

        intervals look like:

            5 minutes
            1 minute 30 seconds
            1 hour
            2 days
            30 days
            (etc)

        times look like:
            February 14 at 6pm EDT

        times default to UTC if no timezone provided.

        example usage:
            `[p]remindme get some fresh air --start-in 4 hours`
        """

        command, start, recur = reminder

        t = Task(
            uid=ctx.message.id,
            nicename=f"reminder-{ctx.message.id}",
            author=ctx.author,
            content=f"schedhelpers selfwhisper {command}",
            channel=ctx.channel,
            initial=start,
            recur=recur,
        )

        async with self._iter_lock:
            async with self.config.channel(ctx.channel).tasks() as tsks:
                tsks.update(t.to_config())
            self.tasks.append(t)

        await ctx.tick()

    @commands.check(lambda ctx: ctx.message.__class__.__name__ == "SchedulerMessage")
    @commands.group(hidden=True, name="schedhelpers")
    async def helpers(self, ctx):
        """ helper commands for scheduler use """
        pass

    @helpers.command(name="say")
    async def say(self, ctx, *, content):
        await ctx.send(content)

    @helpers.command(name="selfwhisper")
    async def swhisp(self, ctx, *, content):
        with contextlib.suppress(discord.HTTPException):
            await ctx.author.send(content)

    @commands.admin_or_permissions(manage_guild=True)
    @commands.guild_only()
    @commands.group()
    async def scheduleradmin(self, ctx):
        """ Administrative commands for scheduler """
        pass

    @scheduleradmin.command()
    async def viewall(self, ctx):
        """ view all scheduled events in a guild """

        tasks = await self.fetch_tasks_by_guild(ctx.guild)

        if not tasks:
            return await ctx.send("No scheduled tasks")

        await self.task_menu(ctx, tasks)

    @scheduleradmin.command()
    async def kill(self, ctx, *, task_id):
        """ kill another user's task (id only) """

        tasks = await self.fetch_task_by_attrs_exact(uid=task_id)

        if not tasks:
            await ctx.send(
                f"Hmm, I couldn't find that task. (try `{ctx.clean_prefix}showscheduled`)"
            )

        elif len(tasks) > 1:
            self.log.warning(
                f"Mutiple tasks where should be unique. Task data: {tasks}"
            )
            return await ctx.send(
                "There seems to have been breakage here. Cleaning up and logging incident."
            )

        else:
            await self._remove_tasks(*tasks)
            await ctx.tick()

    @scheduleradmin.command()
    async def killchannel(self, ctx, channel: discord.TextChannel):
        """ kill all in a channel """

        tasks = await self.fetch_task_by_attrs_exact(channel=channel)

        if not tasks:
            return await ctx.send("No scheduled tasks in that channel.")

        await self._remove_tasks(*tasks)
        await ctx.tick()

    @commands.guild_only()
    @commands.command(aliases=["tm"])
    @can_run_command("mute server")
    async def tempmute(self, ctx, user: discord.Member, *, mute: TempMute):
        """
        binding for mute + scheduled unmute
        This exists only until it is added to core red

        args can be
            --until time
        or
            --for interval

        intervals look like:

            5 minutes
            1 minute 30 seconds
            1 hour
            2 days
            30 days
            (etc)

        times look like:
            February 14 at 6pm EDT

        times default to UTC if no timezone provided.
        """

        reason, unmute_time = mute

        now = datetime.now(timezone.utc)

        mute_task = Task(
            uid=f"mute-{ctx.message.id}",
            nicename=f"mute-{ctx.message.id}",
            author=ctx.author,
            content=f"mute server {user.id} {reason}",
            channel=ctx.channel,
            initial=now,
            recur=None,
        )

        mute_task_2 = Task(
            uid=f"muterole-{ctx.message.id}",
            nicename=f"muterole-{ctx.message.id}",
            author=ctx.author,
            content=f"addrole nadeko-mute {user.id}",
            channel=ctx.channel,
            initial=now,
            recur=None,
        )

        unmute_task = Task(
            uid=f"unmute-{ctx.message.id}",
            nicename=f"unmute-{ctx.message.id}",
            author=ctx.author,
            content=f"unmute server {user.id} Scheduler: Scheduled Unmute - {reason}",
            channel=ctx.channel,
            initial=unmute_time,
            recur=None,
        )

        unmute_task_2 = Task(
            uid=f"unmuterole-{ctx.message.id}",
            nicename=f"unmuterole-{ctx.message.id}",
            author=ctx.author,
            content=f"removerole nadeko-mute {user.id}",
            channel=ctx.channel,
            initial=unmute_time,
            recur=None,
        )

        async with self._iter_lock:
            self.scheduled[mute_task.uid] = asyncio.create_task(
                self.delayed_wrap_and_invoke(mute_task, 0)
            )

            self.scheduled[mute_task_2.uid] = asyncio.create_task(
                self.delayed_wrap_and_invoke(mute_task_2, 0)
            )

            async with self.config.channel(ctx.channel).tasks() as tsks:
                tsks.update(unmute_task.to_config())
            self.tasks.append(unmute_task)

            async with self.config.channel(ctx.channel).tasks() as tsks:
                tsks.update(unmute_task_2.to_config())
            self.tasks.append(unmute_task_2)


import cv2 as cv
import numpy as np

image = cv.imread('CVBVP-Daten/OpenCV-03/Img03a.jpg')
if not image.data:
    print("Error loading image")
    exit

## Aufgabe 1

#cv.imshow("Image:", image)
#cv.imshow("Image mit getauschten Farbkanaelen:", cv.cvtColor(image, cv.COLOR_BGR2RGB))

height = image.shape[0]
width = image.shape[1]

#https://pyimagesearch.com/2021/01/23/splitting-and-merging-channels-with-opencv/
(B, G, R) = cv.split(image)

grayscaleimage = np.empty((height, width))
fullgrayscaleimage = image.copy()

for y in range(height):
    for x in range (width):
        pixel = image[y][x]
        grayscalevalue = int((pixel[0]+pixel[1]+pixel[2])/3)
        fullgrayscaleimage[y][x][0] = grayscalevalue
        fullgrayscaleimage[y][x][1] = grayscalevalue
        fullgrayscaleimage[y][x][2] = grayscalevalue
        grayscaleimage[y][x] = grayscalevalue

cv.imshow("Image:", fullgrayscaleimage)
cv.imshow("Image2:", grayscaleimage)
cv.imshow("Image3:", cv.cvtColor(image, cv.COLOR_BGR2GRAY))
if cv.waitKey(0) == ord('q'):
        exit()
cv.destroyAllWindows()
cv.imwrite('CVBVP-Daten/OpenCV-03/out/blue.jpg', B)
cv.imwrite('CVBVP-Daten/OpenCV-03/out/green.tif', G)
cv.imwrite('CVBVP-Daten/OpenCV-03/out/red.png', R)
cv.imwrite('CVBVP-Daten/OpenCV-03/out/gray.png', fullgrayscaleimage)
cv.imwrite('CVBVP-Daten/OpenCV-03/out/convertedcolorspace.png', cv.cvtColor(image, cv.COLOR_BGR2RGB))

## Aufgabe 2


import os
import warnings
import yaml

import mlflow
import shutil
import uuid
import mlflow.utils.environment
import mlflow.utils.model_utils

from mlflow.exceptions import MlflowException, BAD_REQUEST
from mlflow import pyfunc
from mlflow.models import Model
from mlflow.models.model import MLMODEL_FILE_NAME
from mlflow.models.signature import ModelSignature
from mlflow.models.utils import ModelInputExample
from mlflow.utils.docstring_utils import format_docstring, LOG_MODEL_PARAM_DOCS
from mlflow.utils.environment import (
    _validate_env_arguments,
)
from mlflow.utils.requirements_utils import _get_pinned_requirement

from mlflow.utils.model_utils import (
    _validate_and_prepare_target_save_path,
)
from mlflow.utils.file_utils import (
    write_to,
)

from h2o_mlflow_flavors.utils import unzip_specific_file
from h2o_mlflow_flavors.utils import match_file_from_name_pattern
from h2o_mlflow_flavors.utils import unzip_specific_folder
from h2o_mlflow_flavors.utils import zip_folder

import h2o_mlflow_flavors

FLAVOR_NAME = "h2o_driverless_ai"
MOJO_FILE = "mojo-pipeline/pipeline.mojo"
MOJO_ZIP_FILE_NAME = "mojo"
PY_SCORING_WHL_FILE_PATTERN = "scoring-pipeline/scoring_h2oai_experiment.*\.whl"
PY_SCORING_SUMMARY_FILE_PATTERN = "scoring-pipeline/h2oai_experiment_summary.*\.zip"
PY_SCORING_FILE_NAME = "scorer"
PY_SCORING_CUSTOM_RECIPES_FOLDER = "scoring-pipeline/tmp"

def save_model(
        h2o_dai_artifact_location,
        h2o_dai_model_directory,
        path,
        model_type,
        conda_env=None,
        mlflow_model=None,
        settings=None,
        signature: ModelSignature = None,
        input_example: ModelInputExample = None,
        pip_requirements=None,
        extra_pip_requirements=None,
):
    _validate_env_arguments(conda_env, pip_requirements, extra_pip_requirements)

    path = os.path.abspath(path)
    _validate_and_prepare_target_save_path(path)
    model_data_subpath = "model.h2o.dai"
    model_data_path = os.path.join(path, model_data_subpath)

    shutil.copytree(h2o_dai_model_directory, model_data_path + "/")
    # os.makedirs(model_data_path)

    if mlflow_model is None:
        mlflow_model = Model()
    if signature is not None:
        mlflow_model.signature = signature
    if input_example is not None:
        _save_example(mlflow_model, input_example, path)

    if settings is None:
        settings = {}
    settings["dai_source_file"] = h2o_dai_artifact_location
    settings["dai_model"] = _get_file_name(h2o_dai_model_directory)
    with open(os.path.join(model_data_path, "h2o_dai.yaml"), "w") as settings_file:
        yaml.safe_dump(settings, stream=settings_file)

    mlflow_model.add_flavor(
        FLAVOR_NAME,  type=model_type
    )

    mlflow_model.save(os.path.join(path, MLMODEL_FILE_NAME))

def _get_file_name(path):
    return os.path.basename(path)

def _get_dir_path(path):
    return os.path.dirname(path)

def log_model(h2o_dai_artifact_location,
            artifact_path,
            model_type="dai/mojo_pipeline",
            h2o_dai_model_download_location="/tmp/"+str(uuid.uuid1()),
            conda_env=None,
            registered_model_name=None,
            signature: ModelSignature = None,
            input_example: ModelInputExample = None,
            pip_requirements=None,
            extra_pip_requirements=None,
            **kwargs,
    ):

        valid_dai_model_types = ['dai/mojo_pipeline', 'dai/scoring_pipeline']
        if model_type not in valid_dai_model_types:
            raise MlflowException.invalid_parameter_value("Invalid value for model_type. Valid values are pipeline/mojo or scoring-pipeline")

        h2o_dai_model_directory = determine_model_file(model_type, h2o_dai_artifact_location, h2o_dai_model_download_location)

        return Model.log(
            artifact_path=artifact_path,
            flavor=h2o_mlflow_flavors.driverless,
            registered_model_name=registered_model_name,
            h2o_dai_artifact_location=h2o_dai_artifact_location,
            conda_env=conda_env,
            signature=signature,
            input_example=input_example,
            pip_requirements=pip_requirements,
            extra_pip_requirements=extra_pip_requirements,
            model_type = model_type,
            h2o_dai_model_directory=h2o_dai_model_directory,
            **kwargs,
        )

def determine_model_file(model_type, h2o_dai_model, h2o_dai_model_download_location):
    if model_type == 'dai/mojo_pipeline':
        return _minimise_mojo_scoring_model(h2o_dai_model, h2o_dai_model_download_location)
    elif model_type == 'dai/scoring_pipeline':
        return _minimise_python_scoring_pipeline_model(h2o_dai_model, h2o_dai_model_download_location)

def _minimise_mojo_scoring_model(h2o_dai_model, h2o_dai_model_download_location):
    if match_file_from_name_pattern(h2o_dai_model, MOJO_FILE) is None:
        raise MlflowException.invalid_parameter_value("Not a valid DAI MOJO Pipeline - pipeline.mojo not present in the provided model.")
    location = h2o_dai_model_download_location + "/"
    minimal_model_file_location = location + "model"
    unzip_specific_file(h2o_dai_model, MOJO_FILE, directory=minimal_model_file_location)
    return minimal_model_file_location

def _minimise_python_scoring_pipeline_model(h2o_dai_model, h2o_dai_model_download_location):
    location = h2o_dai_model_download_location + "/"
    minimal_model_file_location = location + "model"
    wheel_file = match_file_from_name_pattern(h2o_dai_model, PY_SCORING_WHL_FILE_PATTERN)
    summary_file = match_file_from_name_pattern(h2o_dai_model, PY_SCORING_SUMMARY_FILE_PATTERN)

    if wheel_file is None or summary_file is None:
        raise MlflowException.invalid_parameter_value("Not a valid DAI Scoring Pipeline - Experiment wheel or Summary file not present in the provided model")

    unzip_specific_file(h2o_dai_model, wheel_file, summary_file, directory=minimal_model_file_location)
    unzip_specific_folder(h2o_dai_model, PY_SCORING_CUSTOM_RECIPES_FOLDER, directory=minimal_model_file_location)
    return minimal_model_file_location

def load_model(model_uri, dst_path=None):
    raise MlflowException(
        message="Executing DAI models within MLflow is currently not supported",
        error_code=BAD_REQUEST)


import numpy as np
from random import uniform
import math
import pickle
from torch import nn
from torch.masked import masked_tensor
import torch
from bot import NeuralNetwork, inodes, onodes
from dataset import PositionDataset
from torch.utils.data import DataLoader

class Trainer():
    def __init__(self):
        return

    def loss_fn():
        return

    def train_one_epoch():
        return
"""
trainer = Trainer()

# Code to load NN parameters
model = NeuralNetwork().to("cpu")
model.load_state_dict(torch.load("./botModels/gen1.bot"))
model.eval()

# Code to load training data
data_file = open("./trainingData/pos1690.pickle", "rb")
posData = pickle.load(data_file)
data_file.close()

# Code to get the test run MCTS output, this should be done during the test runs for the future
indexes = posData[2].nonzero(as_tuple = False)
myOtherCoolTensor = []
nextToInsert = 0
for i in range(onodes - 1):
    if i in indexes:
        myOtherCoolTensor.append(posData[1][nextToInsert])
        nextToInsert += 1
    else:
        myOtherCoolTensor.append(0.)

actOutput = torch.tensor(myOtherCoolTensor)
print(actOutput)

# Code to get the normalized NN MCTS prediction
prediction = model(posData[0])
myCoolTensor = torch.where(posData[2], prediction[0], float('-inf'))
predictedOutput = nn.Softmax(dim=0)(myCoolTensor)

# Loss
print(nn.CrossEntropyLoss()(predictedOutput, actOutput))
"""

#mybatch = []
#for i in range(5):
#    data_file = open("./trainingData/pos" + str(i) + ".pickle", "rb")
#    posData = pickle.load(data_file)
#    data_file.close()
#    mybatch.append(posData[0])

#def AZLossFcn(predMTCS, actMTCS, predRes, actRes):
#    return nn.CrossEntropyLoss()(predMTCS, actMCTS)

#a,b,c,d = next(iter(loader))

#gamePos, train_MCTS, mask, train_result = list(next(iter(loader)))[0]
#print(gamePos)

def AZLossFcn(predMCTS, actMCTS, predRes, actRes):
    loss1 = nn.CrossEntropyLoss(reduction='mean')(predMCTS, actMCTS)
    loss2 = nn.MSELoss()(predRes, actRes.unsqueeze(dim=1))
    return (loss1 + loss2).sum()

def training(learnRate):
    training_data = PositionDataset("./AI/trainingData")
    loader = DataLoader(training_data, batch_size=32, shuffle=True)
    model = NeuralNetwork().to("cpu")
    model.load_state_dict(torch.load("./AI/botModels/currbot.bot"))
    model.eval()
    n_epochs = 500
    for epoch in range(n_epochs):
        gamePos, train_MCTS, mask, train_result = next(iter(loader))
        gamePos.requires_grad = True
        NNpred, NNvaluation = model(gamePos)
        predictedOutput = torch.where(mask, NNpred, float('-inf'))
        #predictedOutput = nn.Softmax(dim=1)(predictedOutput)
        loss = AZLossFcn(predictedOutput, train_MCTS, NNvaluation, train_result.float())

        optimizer = torch.optim.SGD(model.parameters(), lr=learnRate, momentum=0.9)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    torch.save(model.state_dict(), "./AI/botModels/nextbot.bot")

#training(0.1)


"""# Visualization
We use t-SNE plot to observe the distribution of extracted features.
"""

import argparse

import matplotlib.pyplot as plt
import numpy as np
import os
from model import Model
import torch
from sklearn import manifold
from model import source_transform, target_transform
from torch.utils.data import ImageFolder, DataLoader

"""## Step1: Load checkpoint and evaluate to get extracted features"""

# Hints:
# Set features_extractor to eval mode
# Start evaluation and collect features and labels

# Extract feature extractor from model
model = Model()

state_dict = torch.load('model.pth')['model']
model.load_state_dict(state_dict)
model = model.feature_extractor
model.eval()

root_dir = "/tmp2/edwardlee/dataset/real_or_drawing/"

src = ImageFolder(
    os.path.join(root_dir, 'train_data'), transform=source_transform
)[:]

tgt = ImageFolder(
    os.path.join(root_dir, 'test_data'), transform=target_transform
)[:5000]

img = torch.cat([src, tgt], 0)

"""## Step2: Apply t-SNE and normalize"""

# process extracted features with t-SNE
X = model(img)
X_tsne = manifold.TSNE(n_components=2, init='random', random_state=5, verbose=1).fit_transform(X)

# Normalization the processed features
x_min, x_max = X_tsne.min(0), X_tsne.max(0)
X_norm = (X_tsne - x_min) / (x_max - x_min)

x, y = X_norm[:, 0], X_norm[:, 1]

"""## Step3: Visualization with matplotlib"""

# Data Visualization
# Use matplotlib to plot the distribution
# The shape of X_norm is (N,2)

plt.figure(figsize=(10, 10))
plt.scatter(x, y, c='red', s=1, label='Source')
plt.savefig('tsne.png')


###########################
# Problem One
###########################
# There is no unit testing
# for this problem
###########################
import matplotlib.pyplot as plt
import random as rn
import math

# translates a random int into a step along the random walk
# parameters: int i for the step index, numpy array x for tracking the left/right location at index i,
# numpy array y for tracking the forward/backward location at index i
# return: none
def step(x,y,i):

    # We have already setup the below line to return a list of four values
    # [1,2,3,4] out of which you will select the actions and act accordingly.
    # i.e., if you select 1 from direction below then move right and so on for other values.
    direction = rn.randint(1,4)
    # TODO: implement this function
    direction = rn.randit(1,4)
    if direction == 1:
        x[i] = x[i-1] + 1
        y[i] = y[i-1]
    elif direction == 2:
        x[i] = x[i-1] - 1
        y[i] = y[i-1]
    elif direction == 3:
        x[i] = x[i-1]
        y[i] = y[i-1] + 1
    else:
        x[i] = x[i-1]
        y[i] = y[i-1] - 1

# do not change code below for graphit() function.
# this function actually draws the plot that you see in the
# PDF. Your plot will be different from the one given in the
# PDF due to the randomness in the walk.
def graphit(x,y,n):

    plt.title("Random {0} Walk\nLast Location {1},{2}".format(n,int(x[n-1]),int(y[n-1])) )
    plt.plot(x,y)
    plt.plot([x[1],x[1]],[y[1]-10,y[1]+10], "b-")
    plt.plot([x[1]-10,x[1]+10],[y[1],y[1]], "b-")
    plt.plot([x[n-1]-10,x[n-1]+10],[y[n-1],y[n-1]], "r-")
    plt.plot([x[n-1],x[n-1]],[y[n-1]-10,y[n-1]+10], "r-")
    plt.show()

###########################
# Problem Two
###########################

# input: 2 points in 2-dimensions example (1,2) or (3,4)
# output: the distance between them
def distance(p1, p2):
    distance = math.sqrt((p1[0]-p2[0]) ** 2 + (p1[1]-p2[1]) ** 2)
    if distance == 0:
        return 0
    else:
        return distance

# input:  list of pairs, where each pair is a point in 2-dimensions
# output: [point1, point2, distane] where distance is the minimum out of all possible pair of points in xlst
# point1 is first point
# point2 is second point
# distance is the distance between the points
def brute(xlst):
    first = 0
    second = 0
    minimum = distance(xlst[0],xlst[1])
    for i in range (len(xlst)):
        for j in range(len(xlst)):
            if i != j:
                d = distance(xlst[i], xlst[j])
                if d < minimum:
                    minimum = d
                    first = i
                    second = j
    return [xlst[first],xlst[second],minimum]

###########################
# Problem Three
###########################

# input: N and time (set to 40)
# output: calculated productivity value

def productivity(N, T = 40):
    P = N * T * (0.55 - 0.00005 * N * (N-1))
    return P

"""
Do Not Change the functions below.
All you have to do is to complete the productivity function above
The below functions are used to create the plot as shown in the PDF.
"""
def fp(f):
    h = .00001
    return lambda x: (f(x + h) - f(x-h))/(2*h)

def newton(f, fp, x, tau):
    def n(x):
        while f(x) > tau:
            x = x - f(x)/fp(x)
        return x
    return n(x)

###############
# PROBLEM Four
###############
# You CANNOT use ANY modules in this problem
###############

# input: a list of elements
# ouptut: a list of lists, where each sublist is a permutation of the original input list
def permutation(lst):
    list = []
    if len(lst) == 0:
        return []
    if len(lst) == 1:
        return [lst]
    for i in range(len(lst)):
        x = lst[i]
        remaining = lst[:i] + lst[i+1:]
        for j in permutation(remaining):
            list.append([x] + j)
    return list

###############
# PROBLEM Five
###############

class Vector:
    def __init__(self, *x):
        self.__v = x

    def get_tuple(self):
        return self.__v

    def __add__(self,other):
        vec = []
        for i in range(len(self.__v)):
            for j in range(len(other.__v)):
                if i == j:
                    the_sum = self.__v[i] + other.__v[j]
                    vec.append(the_sum)

    def __sub__(self,other):
        vec = []
        for i in range(len(self.__v)):
            for j in range(len(other.__v)):
                if i == j:
                    subt = self.__v[i] - other.__v[j]
                    vec.append(subt)

    def __rmul__(self,other):
        return self.__mul__(other)

    def __mul__(self,other):
        my_sum = 0
        vec = []
        for i in range(len(self.__v)):
            for j in range(len(other.__v)):
                if i == j:
                    m = self.__v[i] * other.__v[i]
                    my_sum += m
        return [my_sum]

    def __abs__(self):
        total = 0
        for i in range(len(self.__v)):
            total += (self.__v[i])**2
        a = math.sqrt(total)
        return a

    def __neg__(self):
        vec = []
        for i in range(len(self.__v[i])):
            negative = 0 - self.__v[i]
            vec.append(negative)
        return vec

    def __eq__(self, other):
        pass

    def __repr__(self):
        pass

if __name__ == "__main__":

    # You are free to uncomment the below statements to test the code
    # but comment them back before submitting your final work.

# #PROBLEM 1
    # #number of steps
    # n = 100000   #You should change the number of steps to see
    #             #to see how it affects the plot
    # x = np.zeros(n)
    # y = np.zeros(n)

    # #fill array with step values
    # for i in range(1,n):
    #     step(x,y,i)

    # graphit(x,y,n)

##PROBLEM 2

    # x = [(rn.randint(1,50), rn.randint(1,50)) for _ in range(10)]
    # print(x)
    # print(brute(x))

##PROBLEM 3
    # f = fp(productivity)
    # x = math.ceil(newton(f,fp(f),62,.01))
    # y = math.ceil(productivity(x))

    # print("The maximum productivity is P({0}) ~ {1} person x hrs".format(x,y))

    # t = np.arange(0.0, 100.0)
    # fig,ax = plt.subplots()

    # ax.plot(t, productivity(t),'g')
    # ax.plot(x,productivity(x), 'bo--')
    # ax.set(xlabel ="Number of people", ylabel="person x hrs", title = "Maximal Productivity P({0}) ~ {1}".format(x,y))

    # ax.grid()
    # plt.show()

## PROBLEM 4
    # print(permutation([1,2,3]))
    # print(permutation([1,2,3,4]))

## PROBLEM 5
    # x,y,w = Vector(1,2),Vector(3,-1),Vector(*(10,10))
    # z,a = Vector(0,3,2),Vector(-1,-1,-1)
    # print(x,y,z,a)
    # print(x+y,z+a)
    # print(x*y,z*a)
    # print(5*x,5*z)
    # print(abs(x),abs(z))
    # print(-x,-z)
    # print(x - y + y == x, 2 * z == z + z)

    pass


# System imports
import pandas as pd
import numpy as np
from datetime import datetime, timedelta

# Third-party imports
from pathlib import Path
import yfinance as yf

# Local imports
from portfolio.transactions import Trades
import datehandler

class Portfolio:
    def __init__(self):
        '''View portfolio data and returns
        '''
        # Dates and file paths
        self.today = datetime.today()
        self.save_dir = Path(__file__).parents[1] / 'data'
        self.portfolio_csv_name = f'portfolio_holdings.csv'
        self.portfolio_csv_path = self.save_dir / self.portfolio_csv_name

        # Initialise trade transaction data
        self.t = Trades()
        self.trades = self.t.all

        # Create portfolio from trades
        self.holdings = self.build()

    def build(self):
        '''Builds portfolio holdings from transaction data

        Returns:
            Dataframe -- (Date, Ticker): [Volume, Close]
        '''

        # Construct range of dates over portfolio period
        p_inception = self.trades.index[0][0]
        p_dates = datehandler.date_list(p_inception, self.today)

        # Create base dataframe
        df_base_dates = pd.DataFrame(index=p_dates)
        df_base_dates.index.name = 'Date'

        df_pfolio = pd.DataFrame()  # Initialise the main portfolio df as empty

        # Build list of tickers seen throughout investment period
        tickers = list(sorted(set(self.trades.reset_index().Ticker.to_list())))

        # Use yfinance to get close prices
        lookup_tickers = [f'{ticker}.AX' for ticker in tickers]  # Only supports ASX stocks
        lookup_tickers = ' '.join(lookup_tickers)

        prices = yf.download(lookup_tickers, start=p_dates[0], end=p_dates[-1])
        close = prices['Adj Close'] # Using Adj Close instead of Close to account for stocksplits, dividends
        close.columns = tickers

        # Step through transactions dataframe, ticker by ticker
        for ticker in tickers:
            vol_df = self.t.by_ticker(ticker)['Volume']  # Get volumes

            # Merge with base_dates_df and cumulative sum the volumes
            ticker_df = pd.merge_ordered(vol_df, df_base_dates, on='Date').set_index('Date')
            ticker_df = ticker_df.cumsum().ffill()
            ticker_df = ticker_df[ticker_df['Volume'].isna() == False]

            # Update volumes based on stocksplits
            ticker_df = self.stocksplits(ticker_df,ticker)

            # Merge with close prices
            ticker_df = pd.merge_ordered(ticker_df, close[ticker], on='Date').set_index('Date')
            ticker_df.rename({ticker: 'Close'}, axis=1, inplace=True)

            # Merge ticker_df with portfolio_df
            ticker_df['Ticker'] = ticker # create ticker column for later indexing
            df_pfolio = pd.concat([df_pfolio, ticker_df])

        df_pfolio = df_pfolio.reset_index().set_index(['Date','Ticker']).sort_index()  # Finalise portfolio
        df_pfolio = df_pfolio.assign(Value = lambda x: x['Volume'] * x['Close'])  # Calculate portfolio value
        print('\nPortfolio successfully built')

        return df_pfolio

    def stocksplits(self, df, ticker):
        '''Adjusts volumes according to historical stocksplits

        Arguments:
            df {DataFrame} -- Dataframe with date (index) and volume (series) data
            ticker {String} -- Ticker to explore. Only ASX securities supported

        Returns:
            DataFrame -- Returns dataframe, but with volume scaled by the cumulative stocksplits
        '''
        # Terminal output
        print(f'\rChecking {ticker}', end='', flush=True)

        try:  # Dividends and stock splits
            stock = yf.Ticker(f'{ticker}.AX')
            df_actions = stock.actions
        except TypeError:
            print(f'\rSomething went wrong!',flush=True)
            return df

        # Add stocksplits column
        if len(df_actions[df_actions['Stock Splits'] > 0]) > 0:
            df['Stock Splits'] = df_actions['Stock Splits'].replace(0,np.nan)

            # Set all dates with 0 holdings to have np.nan holdings
            df['Volume'] = df['Volume'].replace(0,np.nan)

            stocksplit_cumulative = df.loc[df['Volume'].first_valid_index():]['Stock Splits'].cumprod()
            stocksplit_cumulative = stocksplit_cumulative.fillna(method='ffill').fillna(1)

            # Update holding volume
            df['Volume'] *= stocksplit_cumulative
            df['Volume'] = np.ceil(df['Volume'])

            # Update vwap
            # df['Vwap'] /= stocksplit_cumulative

            df = df.drop(columns='Stock Splits')  # Finalise dataframe to return by removing excess columns

        return df

    def plot(self, view='default'):
        # Portfolio value
        value_df = self.holdings.groupby('Date').sum()['Value']
        value_df = value_df[value_df > 0]

        # The portfolio cost for equity
        cost_df = self.trades.assign(Value = lambda x: x['Volume'] * x['EffectivePrice'])
        cost_df = cost_df.groupby('Date').sum()['Value'].cumsum()

        plot_df = pd.merge_ordered(value_df, cost_df, on='Date', fill_method='ffill').set_index('Date')
        plot_df.columns = ['Value', 'Cost']
        plot_df = plot_df.assign(pl_line = lambda x: x['Value'] - x['Cost'])
        plot_df = plot_df.assign(pl_pct = lambda x: x['pl_line']/x['Cost'])

        if view == 'default':
            cols = ['Value','Cost']
        elif view == 'pl':
            cols = ['pl_line']
        elif view == 'pl_pct':
            cols = ['pl_pct']

        plot_df[cols].plot()

        return plot_df


#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
@File    :   create_sketch.py
@Time    :   2023/03/28 10:10:16
@Author  :   Weihao Xia (xiawh3@outlook.com)
@Version :   1.0
@Desc    :   This script is used to generate sketches.
             please ensure torch==0.4.1 and torchvision==0.2.1
'''

import os
import cv2
import numpy as np
import argparse

import hashlib
import platform
import subprocess
# import urllib.request

from torchvision import transforms
from torchvision.utils import save_image
from torch.utils.serialization import load_lua

def sobel(img):
    opImgx = cv2.Sobel(img, cv2.CV_8U, 0, 1, ksize=3)
    opImgy = cv2.Sobel(img, cv2.CV_8U, 1, 0, ksize=3)
    return cv2.bitwise_or(opImgx, opImgy)

def sketch(frame):
    frame = cv2.GaussianBlur(frame, (3, 3), 0)
    invImg = 255 - frame
    edgImg0 = sobel(frame)
    edgImg1 = sobel(invImg)
    edgImg = cv2.addWeighted(edgImg0, 0.75, edgImg1, 0.75, 0)
    opImg = 255 - edgImg
    return opImg

def get_sketch_image(image_path):
    original = cv2.imread(image_path)
    original = cv2.cvtColor(original, cv2.COLOR_BGR2GRAY)
    sketch_image = sketch(original)
    return sketch_image[:, :, np.newaxis]

def download_model(modelname, filename, fileurl, filemd5):
    '''
    download the sketch simplification model (sketch_gan.t7) from official implementation:
    https://github.com/bobbens/sketch_simplification
    '''
    # check if the model already exists
    if os.path.isfile(filename):
        print (filename)
        print(f"Model '{modelname}' already exists. Skipping download.")
    else:
        #  call wget command to download the file
        print(f"Downloading the sketch simplification {modelname} model...")
        # urllib.request.urlretrieve(fileurl, filename)
        subprocess.call(["wget", "-q", "--show-progress", "--continue", "-O", filename, "--", fileurl])

        # verify the MD5 checksum
        print("checking integrity (md5sum)...")
        with open(filename, "rb") as f:
            md5 = hashlib.md5(f.read()).hexdigest()
        if md5 != filemd5:
            print("Integrity check failed. File is corrupt!")
            print(f"Try running this script again and if it fails remove '{filename}' before trying again.")
            os.remove(filename)
            raise ValueError("MD5 checksum does not match")

        print(f"Model '{modelname}' downloaded successfully")

if __name__ == "__main__":

    # define a dictionary to store the model information
    models = {
        "GAN": {
            "filename": "model_gan.t7",
            "fileurl": "https://esslab.jp/~ess/data/sketch_gan.t7",
            "filemd5": "3a5b4088f2490ca4b8140a374e80c878"
        },
        "MSE": {
            "filename": "model_mse.t7",
            "fileurl": "https://esslab.jp/~ess/data/sketch_mse.t7",
            "filemd5": "12317df9a0a2a7220629f5f361b45b82"
        },
        "PENCIL(1)": {
            "filename": "model_pencil1.t7",
            "fileurl": "https://esslab.jp/~ess/data/pencil_artist1.t7",
            "filemd5": "33d553ff3a50d6522e79a73002b0025c"
        },
        "PENCIL(2)": {
            "filename": "model_pencil2.t7",
            "fileurl": "https://esslab.jp/~ess/data/pencil_artist2.t7",
            "filemd5": "537b3ad9d46b2a82b65883be747a7ba9"
        }
    }

    # parse command-line arguments
    parser = argparse.ArgumentParser(description="sketch data generation")
    parser.add_argument("modelname", type=str, choices=models.keys(), help="name of the model to download")
    parser.add_argument("use_cuda", type=bool, default=True, help="use cuda or not")
    parser.add_argument("data_path", type=str, default="celeba_image", help="path to read images")
    parser.add_argument("save_path", type=str, default="celeba_sketch", help="path to save sketches")
    args = parser.parse_args()

    # download the specified model
    print("downloading pretrained models...")
    model = models[args.modelname]
    download_model(args.modelname, model["filename"], model["fileurl"], model["filemd5"])
    print("download finished!")

    # load the model
    cache = load_lua(args.modelname) # sketch_gan.t7
    model = cache.model
    immean = cache.mean
    imstd = cache.std
    model.evaluate()

    if not os.path.exists(args.save_path):
        os.makedirs(args.save_path)

    images = [os.path.join(args.data_path, f) for f in os.listdir(args.data_path)]

    for idx, image_path in enumerate(images):
        if idx % 50 == 0:
            print("{} out of {}".format(idx, len(images)))
        data = get_sketch_image(image_path)
        data = ((transforms.ToTensor()(data) - immean) / imstd).unsqueeze(0)
        if args.use_cuda:
            pred = model.cuda().forward(data.cuda()).float()
        else:
            pred = model.forward(data)
        save_image(pred[0], os.path.join(args.save_path, "{}_edges.jpg".format(image_path.split("/")[-1].split('.')[0])))


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import uuid

from tensorflow.core.protobuf import saver_pb2
from tensorflow.python.eager import def_function
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_spec
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_io_ops
from tensorflow.python.ops import io_ops
from tensorflow.python.ops import string_ops
from tensorflow.python.training.saving import saveable_object
from tensorflow.python.training.saving import saveable_object_util
from tensorflow.python.training import training_util
from tensorflow.python.util import nest

class _SingleDeviceSaver(object):
  """Saves and restores checkpoints from the current device."""

  def __init__(self, saveable_objects):
    """Specify a list of `SaveableObject`s to save and restore.

    Args:
      saveable_objects: A list of `SaveableObject`s.
    """
    saveable_objects = list(saveable_objects)
    for saveable in saveable_objects:
      if not isinstance(saveable, saveable_object.SaveableObject):
        raise ValueError(
            "Expected a list of SaveableObjects, got %s." % (saveable,))
    self._saveable_objects = saveable_objects

  def save(self, file_prefix):
    """Save the saveable objects to a checkpoint with `file_prefix`.

    Args:
      file_prefix: A string or scalar string Tensor containing the prefix to
        save under.
    Returns:
      An `Operation`, or None when executing eagerly.
    """
    tensor_names = []
    tensors = []
    tensor_slices = []
    ev_key_types = []
    has_ev = False
    for saveable in self._saveable_objects:
      if isinstance(saveable, saveable_object_util.EmbeddingVariableSaveable):
        tensors.append(training_util.get_or_create_global_step())
        tensor_names.append("global_step")
        tensor_slices.append("")
        has_ev = True
        break

    for saveable in self._saveable_objects:
      if isinstance(saveable, saveable_object_util.EmbeddingVariableSaveable):
        tensor_names.append(saveable.name)
        tensors.append(saveable.handle_op)
        tensor_slices.append("")
        ev_key_types.append(saveable.key_type)
        continue
      for spec in saveable.specs:
        tensor_names.append(spec.name)
        tensors.append(spec.tensor)
        tensor_slices.append(spec.slice_spec)
    with ops.device("cpu:0"):
      return io_ops.save_v2(file_prefix, tensor_names, tensor_slices, tensors,
              ev_key_types, has_ev)

  def restore(self, file_prefix):
    """Restore the saveable objects from a checkpoint with `file_prefix`.

    Args:
      file_prefix: A string or scalar string Tensor containing the prefix for
        files to read from.

    Returns:
      A dictionary mapping from SaveableObject names to restore operations.
    """
    restore_specs = []
    tensor_structure = []
    for saveable in self._saveable_objects:
      saveable_tensor_structure = []
      tensor_structure.append(saveable_tensor_structure)
      for spec in saveable.specs:
        saveable_tensor_structure.append(spec.name)
        restore_specs.append((spec.name, spec.slice_spec, spec.dtype))
    tensor_names, tensor_slices, tensor_dtypes = zip(*restore_specs)
    with ops.device("cpu:0"):
      restored_tensors = io_ops.restore_v2(
          file_prefix, tensor_names, tensor_slices, tensor_dtypes)
    structured_restored_tensors = nest.pack_sequence_as(
        tensor_structure, restored_tensors)
    restore_ops = {}
    for saveable, restored_tensors in zip(self._saveable_objects,
                                          structured_restored_tensors):
      restore_ops[saveable.name] = saveable.restore(
          restored_tensors, restored_shapes=None)
    return restore_ops

def sharded_filename(filename_tensor, shard, num_shards):
  """Append sharding information to a filename.

  Args:
    filename_tensor: A string tensor.
    shard: Integer.  The shard for the filename.
    num_shards: An int Tensor for the number of shards.

  Returns:
    A string tensor.
  """
  return gen_io_ops.sharded_filename(filename_tensor, shard, num_shards)

class MultiDeviceSaver(object):
  """Saves checkpoints directly from multiple devices.

  Note that this is a low-level utility which stores Tensors in the keys
  specified by `SaveableObject`s. Higher-level utilities for object-based
  checkpointing are built on top of it.
  """

  def __init__(self, saveable_objects):
    """Specify a list of `SaveableObject`s to save and restore.

    Args:
      saveable_objects: A list of `SaveableObject`s.
    """
    saveable_objects = list(saveable_objects)
    saveables_by_device = {}
    for saveable in saveable_objects:
      if not isinstance(saveable, saveable_object.SaveableObject):
        raise ValueError(
            "Expected a dictionary of SaveableObjects, got {}."
            .format(saveable))
      saveables_by_device.setdefault(saveable.device, []).append(saveable)
    self._single_device_savers = {
        device: _SingleDeviceSaver(saveables)
        for device, saveables in saveables_by_device.items()}

  def to_proto(self):
    """Serializes to a SaverDef referencing the current graph."""
    filename_tensor = array_ops.placeholder(
        shape=[], dtype=dtypes.string, name="saver_filename")
    save_tensor = self._traced_save(filename_tensor)
    restore_op = self._traced_restore(filename_tensor).op
    return saver_pb2.SaverDef(
        filename_tensor_name=filename_tensor.name,
        save_tensor_name=save_tensor.name,
        restore_op_name=restore_op.name,
        version=saver_pb2.SaverDef.V2)

  @def_function.function(
      input_signature=(tensor_spec.TensorSpec(shape=(), dtype=dtypes.string),),
      autograph=False)
  def _traced_save(self, file_prefix):
    save_op = self.save(file_prefix)
    with ops.device("cpu:0"):
      with ops.control_dependencies([save_op]):
        return array_ops.identity(file_prefix)

  @def_function.function(
      input_signature=(tensor_spec.TensorSpec(shape=(), dtype=dtypes.string),),
      autograph=False)
  def _traced_restore(self, file_prefix):
    restore_ops = self.restore(file_prefix)
    with ops.device("cpu:0"):
      with ops.control_dependencies(restore_ops.values()):
        return array_ops.identity(file_prefix)

  def save(self, file_prefix):
    """Save the saveable objects to a checkpoint with `file_prefix`.

    Args:
      file_prefix: A string or scalar string Tensor containing the prefix to
        save under.
    Returns:
      An `Operation`, or None when executing eagerly.
    """
    # IMPLEMENTATION DETAILS: most clients should skip.
    #
    # Suffix for any well-formed "checkpoint_prefix", when sharded.
    # Transformations:
    # * Users pass in "save_path" in save() and restore().  Say "myckpt".
    # * checkpoint_prefix gets fed <save_path><sharded_suffix>.
    #
    # Example:
    #   During runtime, a temporary directory is first created, which contains
    #   files
    #
    #     <train dir>/myckpt_temp/
    #        part-?????-of-?????{.index, .data-00000-of-00001}
    #
    #   Before .save() finishes, they will be (hopefully, atomically) renamed to
    #
    #     <train dir>/
    #        myckpt{.index, .data-?????-of-?????}
    #
    # Users only need to interact with the user-specified prefix, which is
    # "<train dir>/myckpt" in this case.  Save() and Restore() work with the
    # prefix directly, instead of any physical pathname.  (On failure and
    # subsequent restore, an outdated and orphaned temporary directory can be
    # safely removed.)
    sharded_suffix = "_temp_%s/part" % uuid.uuid4().hex

    with ops.device("cpu:0"):
      tmp_checkpoint_prefix = string_ops.string_join(
          [file_prefix, sharded_suffix])

    num_shards = len(self._single_device_savers)
    sharded_saves = []
    sharded_prefixes = []
    num_shards_tensor = constant_op.constant(num_shards, name="num_shards")
    last_device = None
    for shard, (device, saver) in enumerate(
        sorted(self._single_device_savers.items())):
      last_device = device
      with ops.device(saveable_object_util.set_cpu0(device)):
        shard_prefix = sharded_filename(tmp_checkpoint_prefix, shard,
                                        num_shards_tensor)
      sharded_prefixes.append(shard_prefix)
      with ops.device(device):
        # _SingleDeviceSaver will use the CPU device when necessary, but initial
        # read operations should be placed on the SaveableObject's device.
        sharded_saves.append(saver.save(shard_prefix))

    with ops.control_dependencies(sharded_saves):
      # Co-locates the merge step with the last device.
      with ops.device(saveable_object_util.set_cpu0(last_device)):
        # V2 format write path consists of a metadata merge step.  Once merged,
        # attempts to delete the temporary directory, "<user-fed prefix>_temp".
        return gen_io_ops.merge_v2_checkpoints(
            sharded_prefixes, file_prefix, delete_old_dirs=True)

  def restore(self, file_prefix):
    """Restore the saveable objects from a checkpoint with `file_prefix`.

    Args:
      file_prefix: A string or scalar string Tensor containing the prefix for
        files to read from.

    Returns:
      A dictionary mapping from SaveableObject names to restore operations.
    """
    restore_ops = {}
    # Sort by device name to avoid propagating non-deterministic dictionary
    # ordering in some Python versions.
    for device, saver in sorted(self._single_device_savers.items()):
      with ops.device(device):
        restore_ops.update(saver.restore(file_prefix))
    return restore_ops