import os
import numpy as np
import copy
import torch
import pandas as pd
from torch.utils.data import Subset, Dataset
from torchvision import transforms
from torchvision.datasets import ImageFolder, DatasetFolder
from omegaconf import DictConfig, OmegaConf
from typing import Union, List, Optional, Tuple
import matplotlib.pyplot as plt
from PIL import Image
import logging
from tqdm import tqdm

from .meta.office_caltech import DefaultImageLoader

logger = logging.getLogger(__name__)

from fade.file import FileManager
from . import read_fed_dataset

def simple_stat(arr):
    res = {
        "mean": np.mean(arr),
        "std": np.std(arr),
        "max": np.max(arr),
        "min": np.min(arr),
        "median": np.median(arr),
    }
    for metric, value in res.items():
        print(f"  {metric}: {value:.4g}", end=", ")
    print("")
    return res

def basic_stat_fed_dict(data_dict, verbose=False, group_by="none"):
    users = data_dict['users']
    num_users = len(users)
    num_samples = data_dict['num_samples']
    # if group_by == "group":
    #     groups = data_dict[]

    if verbose:
        example_user = users[0]
        shape = None

        classes = []
        n_classes = []
        mtl_flag = True
        for k in tqdm(users, desc="stat users"):
            if 'idx' in data_dict['user_data'][k]:
                cs = np.unique(data_dict['dataset'].targets[i] for i in data_dict['user_data'][k]['idx']).tolist()
                sample, label = data_dict['dataset'][data_dict['user_data'][k]['idx'][0]]
            elif 'dataset' in data_dict['user_data'][k]:
                # labels = [y for x, y in data_dict['user_data'][k]['dataset']]
                if isinstance(data_dict['user_data'][k]['dataset'], Subset):
                    ds = data_dict['user_data'][k]['dataset'].dataset
                    labels = [ds.targets[i] for i in data_dict['user_data'][k]['dataset'].indices]
                else:
                    ds = data_dict['user_data'][k]['dataset']
                    labels = ds.targets
                labels = np.array(labels)
                cs = np.unique(labels).tolist()
                sample, label = data_dict['user_data'][k]['dataset'][0]

                if len(labels.shape) > 1 and mtl_flag:
                    print(f" Multi-task w/ {labels.shape[1]} tasks")
                    mtl_flag = False
            elif 'x' in data_dict['user_data'][k] and 'y' in data_dict['user_data'][k]:
                cs = np.unique(data_dict['user_data'][k]['y']).tolist()
                sample = data_dict['user_data'][k]['x'][0]
            else:
                raise ValueError(f"Expected keys are: x, y, dataset, idx. Not found "
                                 f"wanted data keys in {data_dict['user_data'][k].keys()}")
            classes.append(cs)
            n_classes.append(len(cs))
            if shape is None:
                shape = sample.shape
            else:
                if shape != sample.shape:
                    logger.warning(f"User {k} has sample of shape not equal to other users' sample "
                                   f"shape: {shape} != {sample.shape} (others)")

            # if len(data_dict['hierarchies']) > 0:
            #     groups = data_dict['hierarchies']
            #     # TODO class balance for each group
            #
            #     # group balance check
            #     uni_groups, n_user_by_group = np.unique(groups, return_counts=True)
            #     print(f" Num user by group: "
            #           f"{dict((g, nu) for g, nu in zip(uni_groups, n_user_by_group))}")
        # classes = np.array(classes)

        print(f" Num of users: {num_users}")
        print(f" Sample: shape {shape}, type {type(sample)}")
        print(f" Num of samples (by user):", end=" ")
        for stat_method in ["sum", "mean", "median", "min", "max", "std"]:
            print(f"{stat_method} {eval('np.'+stat_method+'(num_samples)'):7.1f}", end=", ")
        print()
        if False:  # FIXME: ad-hoc 'float' in str(classes[0].dtype):
            print("mean of objective (per user)", [np.mean(c) for c in classes])
        else:
            classes_ = []
            for cs in classes:
                classes_.extend(cs)
            print(f" Total num of classes: {len(np.unique(classes_))}")
            print(f" Num of classes per user: "
                  f"mean {np.mean(n_classes):.3f}, min {np.min(n_classes)}, max {np.max(n_classes)}")
    return num_samples, num_users

def basic_stat(data_dict, verbose=False):
    """
    Args:
        data_dict: Dict<user_id: Dict<'x': samples, 'y': labels> >.
    """
    num_users = len(data_dict)
    sample = data_dict[list(data_dict.keys())[0]]['x']
    num_samples = np.array([len(data_dict[k]['y']) for k in data_dict])

    if verbose:
        shape = sample.shape[1:]
        classes = [np.unique(data_dict[k]['y']) for k in data_dict]
        n_classes = np.array([len(np.unique(data_dict[k]['y'])) for k in data_dict])

        print(f" Num of users: {num_users}")
        print(f" Sample: shape {shape}, type {type(sample)}")
        print(f" Num of samples (by user):", end=" ")
        for stat_method in ["mean", "median", "min", "max", "std"]:
            print(f"{stat_method} {eval('np.'+stat_method+'(num_samples)'):7.1f}", end=", ")
        if 'float' in str(classes[0].dtype):
            print("mean of objective (per user)", [np.mean(c) for c in classes])
        else:
            print(f" Total num of classes: {len(np.unique(np.concatenate(classes, axis=0)))}")
            print(f" Num of classes per user: "
                  f"mean {np.mean(n_classes):.1f}, min {np.min(n_classes)}, max {np.max(n_classes)}")
    print()
    return num_samples, num_users

def user_class_shards(data_dict, visualize=False, ax=None, show_xticks=True, by_percent=False):
    """Stat the class shards in users.

    Args:
        data_dict: Dict<user_id: Dict<'x': samples, 'y': labels> >.
        visualize: Plot if True.
        by_percent: Transform the count to frequency (percent).

    Returns:
        ucs: An dict<User_name: Dict<class: #samples>> where each element is the number of
            samples of each class in each user.
        ucs_arr: An array of shape (n_user, n_class).
    """
    ucs = dict()
    all_classes = []
    for _id in data_dict:
        y = data_dict[_id]['y']
        classes, cnts = np.unique(y, return_counts=True)
        ucs[_id] = dict((c, cnt) for c, cnt in zip(classes, cnts))
        all_classes = np.union1d(classes, all_classes)
    users = list(data_dict.keys())
    ucs_arr = []
    for _id in users:
        ucs_arr.append([ucs[_id][c] if c in ucs[_id] else 0 for c in all_classes])
    ucs_arr = np.cumsum(np.array(ucs_arr), axis=1)  # shape: (n_user, n_class)
    if by_percent:
        ucs_arr = ucs_arr / ucs_arr[:, -1:] * 100.
    if visualize:
        for ic in np.arange(len(all_classes))[::-1]:
            ax.bar(list(range(len(ucs_arr))), ucs_arr[:, ic], label=f"class-{all_classes[ic]}")
    if show_xticks:
        ax.set(xticks=list(range(len(ucs_arr))))
    return ucs, ucs_arr

def save_data(data_name, data_dict):
    """Save data into files.

    Args:
        data_name ():
        data_dict (dict): Should include keys: 'train' and 'test' at least.
    """
    for s in data_dict:
        fname = FileManager.data(os.path.join(data_name, s, f"{s}.pt"), is_dir=False, overwrite=True)
        with open(fname, 'wb') as outfile:
            print(f"Dumping {s} data => {fname}")
            torch.save(data_dict[s], outfile)

def load_fed_dataset(dataset: str, subset="train",
                     list_keys=("users", "hierarchies", "num_samples"),
                     dict_keys=("user_data",)):
    """Load dataset in fed format without unpacking.
    If multiple files exist, we will load alll and combine into one dict.

    Returns:
        fed_data_dict: A dict like {"users": [], "hierarchies": [], "user_data": {}}
    """
    # os.path.join('data', dataset, 'data', 'train')
    if "-" in dataset:
        dataset, data_fld = dataset.split("-")
    else:
        # data_fld = 'data'
        data_fld = ''

    data_dir = FileManager.data(os.path.join(dataset, data_fld, subset), is_dir=True,
                                      create_new=False)
    # test_data_dir = FileManager.data(os.path.join(dataset, data_fld, 'test'), is_dir=True,
    #                                  create_new=False)
    # clients = []
    # groups = []
    fed_data_dict = {}
    # fed_data_dict = {"users": [], "hierarchies": [], "user_data": {}, "num_samples": []}

    all_files = [f for f in os.listdir(data_dir) if f.endswith('.json') or f.endswith('.pt')]
    for f in all_files:
        file_path = os.path.join(data_dir, f)
        print(f"Load data <== {file_path}")
        if file_path.endswith(".pt"):
            with open(file_path, 'rb') as inf:
                cdata = torch.load(inf)
        elif file_path.endswith(".json"):
            raise ValueError(f"Found json file. Unless you intend to do this, you should use `.pt` file for dataset. "
                             f"`.json` is slow in loading. Try to remove .json file to fix this.")
            # with open(file_path, 'r') as inf:
            #     cdata = json.load(inf)
        else:
            raise TypeError(f"file: {file_path}")
        for k in list_keys:
            if k in cdata:
                fed_data_dict.setdefault(k, []).extend(cdata[k])
        for k in dict_keys:
            if k in cdata:
                fed_data_dict.setdefault(k, {}).update(cdata[k])
        if len(cdata.keys()) > 4:
            raise ValueError(f"Expected three keys. But we get {len(cdata.keys())} "
                             f"keys from the dataset ({dataset}, {subset}): {list(cdata.keys())}")
    return fed_data_dict

def get_dataset_config(name):
    return OmegaConf.load(os.path.join(os.path.dirname(__file__), name + ".yaml")).dataset

def read_data(dataset, load_config=False):
    """parses data in given train and test data directories

    assumes:
    - the data in the input directories are .json files with
        keys 'users' and 'user_data'
    - the set of train set users is the same as the set of test set users

    Args:
        dataset (str|DictConfig):
        load_config (bool): Enforce to use hydra config to access dataset if `dataset`
            is str.

    Return:
        clients: list of client ids
        groups: list of group ids; empty list if none found
        train_data: dictionary of train data
        test_data: dictionary of test data
    """
    logger.warning(f"read_data will be deprecated. Use read_fed_dataset instead.")
    if isinstance(dataset, str) and load_config:
        dataset = get_dataset_config(dataset)
    if isinstance(dataset, DictConfig):
        return read_fed_dataset(dataset)

    train_dict = load_fed_dataset(dataset, "train")
    test_dict = load_fed_dataset(dataset, "test")
    groups = train_dict["hierarchies"] if "hierarchies" in train_dict else []
    if "user_data" in train_dict:
        data_type = "fed"
    # elif "task_data" in train_dict:
    #     data_type = "mtl"
    else:
        raise ValueError(f"train_dict keys: {train_dict.keys()}")
    if data_type == "fed":
        train_data = train_dict["user_data"]

        test_data = test_dict["user_data"]
    # elif data_type == "mtl":
    #     train_data = train_dict["task_data"]
    #
    #     test_data = test_dict["task_data"]
    else:
        raise ValueError(f"data_type: {data_type}")

    users = list(sorted(train_data.keys()))
    # TODO verify the consistency of client keys.

    return users, groups, train_data, test_data

def extract_user_data(user_index, data):  # , dataset):
    """`data` is from the output of `read_fed_data`"""
    if isinstance(data, dict):
        train_data = data["train"]["user_data"]
        test_data = data["test"]["user_data"]
        groups = data["train"]["hierarchies"] if "hierarchies" in data["train"] else []
        users = data["train"]['users']
    else:
        users, groups, train_data, test_data = data
    user_id = users[user_index]
    if len(groups) > 0:
        if isinstance(groups[user_index], list):
            # TODO fix this.
            print(f"WARNING: found vec groups.")
            assert not isinstance(groups[user_index][0], str)
            # FIXME ad-hoc only use the first group if the groups include multiple values.
            group = (groups[user_index][0], groups[user_index][0])
        else:
            group_names = np.unique(groups).tolist()
            group_name = groups[user_index]
            # print(f"### np.unique(groups): {group_names}")
            # note: the unique return SORTED array of groups.
            group = (group_names.index(group_name), group_name)
    else:
        group = (0, 'all')
    data_dict = {'train': train_data, 'test': test_data}
    new_data_dict = {}
    for subset in data_dict:
        subset_data = data_dict[subset]
        if 'idx' in subset_data[user_id]:
            assert isinstance(data, dict), f"data is not dict but {type(data)}"
            assert 'dataset' in data[subset], f"Not found 'dataset' in {subset} data. {data[subset].keys()}"
            dataset = data[subset]['dataset']  # type: Dataset
            idxs = subset_data[user_id]['idx']
            subset_data = Subset(dataset, idxs)
        elif 'dataset' in subset_data[user_id]:
            subset_data = subset_data[user_id]['dataset']
        else:
            subset_data = subset_data[user_id]
            X_set, y_set = subset_data['x'], subset_data['y']
            if not isinstance(X_set, torch.Tensor):
                X_set = torch.Tensor(X_set).type(torch.float32)
                y_set = torch.Tensor(y_set).type(torch.int64)
            subset_data = [(x, y) for x, y in zip(X_set, y_set)]
        new_data_dict[subset] = subset_data
    return user_id, group, new_data_dict['train'], new_data_dict['test']

def grid_imshow(
        tensor: Union[torch.Tensor, List[torch.Tensor]],
        nrow: int = 8,
        padding: int = 2,
        normalize: bool = False,
        img_range: Optional[Tuple[int, int]] = None,
        scale_each: bool = False,
        pad_value: int = 0,
        title=None,
) -> torch.Tensor:
    """Make a grid of images.

    Args:
        tensor (Tensor or list): 4D mini-batch Tensor of shape (B x C x H x W)
            or a list of images all of the same size.
        nrow (int, optional): Number of images displayed in each row of the grid.
            The final grid size is ``(B / nrow, nrow)``. Default: ``8``.
        padding (int, optional): amount of padding. Default: ``2``.
        normalize (bool, optional): If True, shift the image to the range (0, 1),
            by the min and max values specified by :attr:`range`. Default: ``False``.
        img_range (tuple, optional): tuple (min, max) where min and max are numbers,
            then these numbers are used to normalize the image. By default, min and max
            are computed from the tensor.
        scale_each (bool, optional): If ``True``, scale each image in the batch of
            images separately rather than the (min, max) over all images. Default: ``False``.
        pad_value (float, optional): Value for the padded pixels. Default: ``0``.

    Example:
        See this notebook `here <https://gist.github.com/anonymous/bf16430f7750c023141c562f3e9f2a91>`_

    """
    if not (torch.is_tensor(tensor) or
            (isinstance(tensor, list) and all(torch.is_tensor(t) for t in tensor))):
        raise TypeError('tensor or list of tensors expected, got {}'.format(type(tensor)))

    # if list of tensors, convert to a 4D mini-batch Tensor
    if isinstance(tensor, list):
        tensor = torch.stack(tensor, dim=0)

    if tensor.dim() == 2:  # single image H x W
        tensor = tensor.unsqueeze(0)
    if tensor.dim() == 3:  # single image
        if tensor.size(0) == 1:  # if single-channel, convert to 3-channel
            tensor = torch.cat((tensor, tensor, tensor), 0)
        tensor = tensor.unsqueeze(0)

    if tensor.dim() == 4 and tensor.size(1) == 1:  # single-channel images
        tensor = torch.cat((tensor, tensor, tensor), 1)

    if normalize is True:
        tensor = tensor.clone()  # avoid modifying tensor in-place
        if img_range is not None:
            assert isinstance(img_range, tuple), \
                "img_range has to be a tuple (min, max) if specified. min and max are numbers"

        def norm_ip(img, min, max):
            img.clamp_(min=min, max=max)
            img.add_(-min).div_(max - min + 1e-5)

        def norm_range(t, img_range):
            if img_range is not None:
                norm_ip(t, img_range[0], img_range[1])
            else:
                norm_ip(t, float(t.min()), float(t.max()))

        if scale_each is True:
            for t in tensor:  # loop over mini-batch dimension
                norm_range(t, img_range)
        else:
            norm_range(tensor, img_range)

    if tensor.size(0) == 1:
        return tensor.squeeze(0)

    # make the mini-batch of images into a grid
    nmaps = tensor.size(0)
    xmaps = min(nrow, nmaps)
    ymaps = int(np.ceil(float(nmaps) / xmaps))
    height, width = int(tensor.size(2) + padding), int(tensor.size(3) + padding)
    num_channels = tensor.size(1)
    fig, axs = plt.subplots(ymaps, xmaps, figsize=(.03 * width * xmaps, .03 * height * ymaps))
    axs = np.atleast_2d(axs)
    # grid = tensor.new_full((num_channels, height * ymaps + padding, width * xmaps + padding), pad_value)
    k = 0
    for y in range(ymaps):
        for x in range(xmaps):
            ax = axs[y, x]
            if k < nmaps:
                npimg = tensor[k].numpy()
                ax.imshow(np.transpose(npimg, (1, 2, 0)), interpolation='nearest')
                if title is not None:
                    ax.set(title=title(k))
            ax.axis('off')
            # Tensor.copy_() is a valid method but seems to be missing from the stubs
            # https://pytorch.org/docs/stable/tensors.html#torch.Tensor.copy_
            # grid.narrow(1, y * height + padding, height - padding).narrow(  # type: ignore[attr-defined]
            #     2, x * width + padding, width - padding
            # ).copy_(tensor[k])

            k = k + 1
    return fig, axs

class GrayscaleToRgb:
    """Convert a grayscale image to rgb"""
    def __call__(self, image):
        image = np.array(image)
        image = np.dstack([image, image, image])
        return Image.fromarray(image)

class ToBinary:
    """Convert a grayscale image to rgb"""
    def __init__(self, scale=255):
        self.scale = scale

    def __call__(self, image):
        ret = (image > 0).int() * self.scale
        return ret

class SqueezeTarget:
    """Try to squeeze dimensions of targets.
    """

    def __call__(self, y):
        """
        Args:
            y (): Label of a sample.

        Returns:
            Tensor: Converted image.
        """
        return y[0] if len(y) == 1 else y

    def __repr__(self):
        return 'squeeze dimension of targets'

def make_static_dataset_copy(dataset):
    dataset = copy.deepcopy(dataset)
    if isinstance(dataset, Subset):
        ds = dataset.dataset
    else:
        ds = dataset
    if hasattr(ds, 'transform'):
        ds.transform = remove_random_transforms(ds.transform)
    else:
        logger.debug(f"Not found transform. Skip")
    # assert not hasattr(ds, 'target_transform') or ds.target_transform is None
    return dataset

def remove_random_transforms(trans):
    """Remove the random transform in `trans`."""
    if trans is None:
        return trans
    assert isinstance(trans, transforms.Compose)
    new_trans = []
    logger.debug(f"old trans: {trans}")
    for t in trans.transforms:
        if isinstance(t, (transforms.Resize, transforms.ToTensor, transforms.Normalize,
                          DefaultImageLoader)):
            new_trans.append(t)
        elif isinstance(t, transforms.RandomCrop):
            t_ = transforms.CenterCrop(t.size)
            new_trans.append(t_)
            logger.debug(f" Replace {t} by {t_}")
        elif isinstance(t, transforms.RandomResizedCrop):
            t_ = transforms.Resize(t.size)
            new_trans.append(t_)
            logger.debug(f" Replace {t} by {t_}")
        elif isinstance(t, (transforms.RandomHorizontalFlip, transforms.RandomVerticalFlip, transforms.RandomRotation)):
            logger.debug(f" Skip transform: {t}")
        else:
            raise TypeError(f"Unable to translate transform: {t}")
    new_trans = transforms.Compose(new_trans)
    logger.debug(f"New trans: {new_trans}")
    return new_trans

def update_dataset_targets(dataset, new_targets, indices=None):
    """Update dataset targets in place."""
    if indices is None:
        indices = list(range(len(dataset)))
    if isinstance(dataset, Subset):
        indices = dataset.indices
        update_dataset_targets(dataset.dataset, new_targets, indices=indices)
        # for ii in indices:
        #     dataset.dataset.targets[indices[ii]] = new_targets[ii]
    elif isinstance(dataset, DatasetFolder):
        assert len(indices) == len(new_targets), f"len(indices) ({len(indices)}) != len(new_targets) ({len(new_targets)})"
        for i in range(len(indices)):
            s = dataset.samples[indices[i]]
            dataset.samples[indices[i]] = (s[0], new_targets[i])
        # for i in range(len(dataset.samples)):
        #     dataset.samples[indices[i]] = (s[0], 0)
        dataset.targets = [s[1] for s in dataset.samples]
    elif isinstance(dataset, list) and isinstance(dataset[0], tuple) and len(dataset[0])==2:
        for i in range(len(indices)):
            s = dataset[indices[i]]
            dataset[indices[i]] = (s[0], new_targets[i])
    else:
        raise TypeError(f"Type: {type(dataset)}")
        # TODO use this for other known types. Do not use this for all Dataset.
        # assert hasattr(dataset, 'targets')
        # for ii in indices:
        #     dataset.targets[indices[ii]] = new_targets[ii]

def merge_fed_dataset_users(dataset):
    """Merge all users into a single table."""
    assert isinstance(dataset, dict)

    dfs = dict((k, []) for k in ['x', 'y', 'group', 'user'])  # a list of all users' dataframes
    for user, group in zip(dataset['users'], dataset['hierarchies']):
        X = dataset['user_data'][user]['x']
        y = dataset['user_data'][user]['y']
        df = {'x': X.tolist(), 'y': y.tolist(), 'group': [group]*len(y), 'user': [user]*len(y)}
        for k in df:
            dfs[k].extend(df[k])
    return dfs

def compute_mean_distance(data_dict):
    users = data_dict["users"]
    n_user = len(users)
    D = np.zeros((n_user, n_user))
    for i in range(n_user):
        X1 = data_dict['user_data'][users[i]]['x']
        for j in range(i+1, n_user):
            X2 = data_dict['user_data'][users[j]]['x']
            d = np.linalg.norm(np.mean(X1, axis=0) - np.mean(X2, axis=0))
            D[i, j] = d
            D[j, i] = d
    return D

def plot_sample_size_dist(fed_dict, subset):
    users = fed_dict[subset]["users"]
    n_user = len(fed_dict[subset]["users"])
    from .utils import extract_user_data
    n_smp_by_class = {}
    for i_user in range(n_user):
        user_id, group, train_set, test_set = extract_user_data(i_user, fed_dict)
        subset_data = train_set if subset == "train" else test_set
        for _, y in subset_data:
            n_smp_by_class.setdefault(y, [0] * n_user)
            n_smp_by_class[y][i_user] += 1
    all_classes = sorted(list(n_smp_by_class.keys()))
    n_smp_mat = np.array([n_smp_by_class[cl] for cl in all_classes]).T
    # print(n_smp_mat)
    m_classes, m_users = np.meshgrid(all_classes, np.arange(0, n_user))
    m_classes = m_classes.reshape((-1,))
    m_users = m_users.reshape((-1,))
    n_smp_mat = n_smp_mat.reshape((-1,))

    import seaborn as sns
    import matplotlib.pyplot as plt
    max_size = 240
    n_smp_mat = n_smp_mat * 1. * max_size / np.max(n_smp_mat)
    fig, ax = plt.subplots(1, 1)
    plt.scatter(m_classes, m_users, s=n_smp_mat, alpha=0.5)
    # sns.scatterplot(x=m_classes, y=m_users, size=n_smp_mat, ax=ax)
    ax.set(xlabel="class", ylabel="user", yticks=list(range(n_user)), yticklabels=users)
    return ax

def get_dataloader_options(dataset):
    if isinstance(dataset, Subset):
        dataset = dataset.dataset
    return dataset.dataloader_options() \
        if hasattr(dataset, "dataloader_options") else {}


from django.shortcuts import get_object_or_404
from blog.models import Post, Category, Like, Comment, View
from users.models import NewUser
from .serializers import PostSerializer, CommentSerializer, CreatePostSerializer, LikeSerializer, CommentSerializer, ViewSerializer, PostRankingSerializer
# from .serializers import  UpdateViewSerializer, UpdateLikeSerializer, UpdateCommentSerializer,
from rest_framework import  filters, generics, permissions
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework import status
from rest_framework.parsers import MultiPartParser, FormParser
from django_filters.rest_framework import DjangoFilterBackend
# Display Posts
from rest_framework.pagination import PageNumberPagination

class CustomPageNumberPagination(PageNumberPagination):
    page_size = 6
    page_size_query_param = 'page_size'
    max_page_size = 100

    def get_paginated_response(self, data):
        res = super().get_paginated_response(data)
        res.data['page_size'] = 6
        return res

class PostList(generics.ListAPIView):
    permission_classes = [permissions.AllowAny]
    serializer_class = PostSerializer
    queryset = Post.objects.all()
    filter_backends = [DjangoFilterBackend]
    filterset_fields = ['category__slug', 'author__user_name']
    # Thêm trình xử lý phân trang vào view
    pagination_class = CustomPageNumberPagination

class PostRankingList (generics.ListAPIView):
    permission_classes = [permissions.AllowAny]
    serializer_class = PostRankingSerializer
    queryset = Post.objects.all()
    filter_backends = [filters.OrderingFilter]

class PostDetail(generics.RetrieveAPIView):
    permission_classes = [permissions.IsAuthenticated]
    serializer_class = PostSerializer

    def get_object(self):
        item = self.kwargs.get('slug')
        post = get_object_or_404(Post, slug=item)
        view = View.objects.create(post=post, author=self.request.user)
        post.num_view += 1
        post.author.num_view += 1
        post.save()
        post.author.save()
        return post

class LikeAPIView(APIView):
    permission_classes = [permissions.IsAuthenticated]

    def post(self, request, slug):
        try:
            post = Post.objects.get(slug=slug)
            # Kiểm tra xem người dùng đã Like bài viết chưa
            user = request.user
            if Like.objects.filter(post=post, author=user).exists():
                return Response({"detail": "Bạn đã Like bài viết này rồi."}, status=status.HTTP_400_BAD_REQUEST)

            like = Like.objects.create(post=post, author=user)
            serializer = LikeSerializer(like)
            post.author.num_like += 1
            post.num_like += 1
            post.save()
            post.author.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        except Post.DoesNotExist:
            return Response({"detail": "Bài viết không tồn tại."}, status=status.HTTP_404_NOT_FOUND)

class CommentAPIView(APIView):
    permission_classes = [permissions.IsAuthenticated]

    def post(self, request, slug):
        try:
            post = Post.objects.get(slug=slug)
            # Lấy dữ liệu từ request.data để tạo Comment
            author = request.user
            content = request.data.get('content', '')

            comment = Comment.objects.create(
                post=post, author=author, content=content)
            serializer = CommentSerializer(comment)
            post.num_comment += 1
            post.author.num_comment += 1
            post.save()
            post.author.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        except Post.DoesNotExist:
            return Response({"detail": "Bài viết không tồn tại."}, status=status.HTTP_404_NOT_FOUND)

class ViewAPIView(APIView):
    permission_classes = [permissions.IsAuthenticated]

    def post(self, request, slug):
        try:
            post = Post.objects.get(slug=slug)
            # Kiểm tra xem người dùng đã View bài viết chưa
            user = request.user
            view = View.objects.create(post=post, author=user)
            serializer = ViewSerializer(view)
            post.num_view += 1
            post.author.num_view += 1
            post.save()
            post.author.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        except Post.DoesNotExist:
            return Response({"detail": "Bài viết không tồn tại."}, status=status.HTTP_404_NOT_FOUND)

class PostListDetailfilter(generics.ListAPIView):

    serializer_class = PostSerializer
    queryset = Post.objects.all()
    filter_backends = [filters.SearchFilter]
    # '^' Starts-with search.
    # '=' Exact matches.
    search_fields = ['^slug']

# Post Admin

# class CreatePost(generics.CreateAPIView):
#     permission_classes = [permissions.IsAuthenticated]
#     queryset = Post.objects.all()
#     serializer_class = PostSerializer

class CreatePost(APIView):
    permission_classes = [permissions.IsAuthenticated]
    parser_classes = [MultiPartParser, FormParser]

    def post(self, request, format=None):
        category_slug = request.data.get('category')
        category = Category.objects.get(slug=category_slug)
        data_copy = request.data.copy()
        data_copy['category'] = category.id

        serializer = CreatePostSerializer(data=data_copy)
        if serializer.is_valid():
            serializer.save()
            request.user.num_post += 1
            request.user.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        else:
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class UserPostDetail(generics.RetrieveAPIView):
    permission_classes = [permissions.IsAuthenticated]
    serializer_class = PostSerializer
    queryset = Post.objects.all()

class EditPost(generics.UpdateAPIView):
    permission_classes = [permissions.IsAuthenticated]
    serializer_class = PostSerializer
    queryset = Post.objects.all()

    def update(self, request, *args, **kwargs):
        category_slug = request.data.get('category', None)
        category = Category.objects.get(slug=category_slug)

        request.data['category'] = category.id

        return super().update(request, *args, **kwargs)

class DeletePost(generics.RetrieveDestroyAPIView):
    permission_classes = [permissions.IsAuthenticated]
    serializer_class = PostSerializer
    queryset = Post.objects.all()

    def perform_destroy(self, instance):
        # Xác định tác giả của bài viết trước khi xóa nó
        author = instance.author
        author.num_post -= 1
        author.num_comment -= instance.num_comment
        author.num_view -= instance.num_view
        author.num_like -= instance.num_like
        author.save()
        # Xóa bài viết
        instance.delete()

""" Concrete View Classes
# CreateAPIView
Used for create-only endpoints.
# ListAPIView
Used for read-only endpoints to represent a collection of model instances.
# RetrieveAPIView
Used for read-only endpoints to represent a single model instance.
# DestroyAPIView
Used for delete-only endpoints for a single model instance.
# UpdateAPIView
Used for update-only endpoints for a single model instance.
# ListCreateAPIView
Used for read-write endpoints to represent a collection of model instances.
RetrieveUpdateAPIView
Used for read or update endpoints to represent a single model instance.
# RetrieveDestroyAPIView
Used for read or delete endpoints to represent a single model instance.
# RetrieveUpdateDestroyAPIView
Used for read-write-delete endpoints to represent a single model instance.
"""


### IMPORTANDO BIBLIOTECAS
from typing import Concatenate
import pandas as pd
from pathlib import Path
import matplotlib.pyplot as plt
import pandas_datareader.data as web
import win32com.client as win32
from datetime import date, datetime, timedelta

### CAMINHOS DE ARQUIVO
CAMINHO_CARTEIRA = Path('C:/Users/paulo/Desktop/paulocesarm/python study/#HASHTAG/ENCONTROS AO VIVO/03 - ANÁLISES FINANCEIRS/CarteiraMentoria.xlsx')
CAMINHO_TD = 'https://www.tesourotransparente.gov.br/ckan/dataset/f0468ecc-ae97-4287-89c2-6d8139fb4343/resource/e5f90e3a-8f8d-4895-9c56-4bb2f7877920/download/VendasTesouroDireto.csv'
tesouro_direto = 'https://www.tesourotransparente.gov.br/ckan/dataset/f0468ecc-ae97-4287-89c2-6d8139fb4343/resource/e5f90e3a-8f8d-4895-9c56-4bb2f7877920/download/VendasTesouroDireto.csv'

### CAMINHO PARA SALVAR GRÁFICOS
VALOR_INVESTIDO_ATIVO = r'C:/Users/paulo/Desktop/paulocesarm/python study/#HASHTAG/ENCONTROS AO VIVO/03 - ANÁLISES FINANCEIRS/valor_investido_ativo.png'
VALOR_INVESTIDO_TIPO = r'C:/Users/paulo/Desktop/paulocesarm/python study/#HASHTAG/ENCONTROS AO VIVO/03 - ANÁLISES FINANCEIRS/valor_investido_tipo.png'
VALOR_IBOV = r'C:/Users/paulo/Desktop/paulocesarm/python study/#HASHTAG/ENCONTROS AO VIVO/03 - ANÁLISES FINANCEIRS/valores_ibov.png'
COMPARATIVO_CARTEIRA_IBOV = r'C:/Users/paulo/Desktop/paulocesarm/python study/#HASHTAG/ENCONTROS AO VIVO/03 - ANÁLISES FINANCEIRS/comparativo_carteira_ibov.png'

### DEFININDO DATA ATUAL
data_atual = date.today()
data_atual = data_atual.strftime('%Y-%m-%d')

def enviando_email():

    outlook = win32.Dispatch('outlook.application') #definindo aplicação pelo outlook
    mail = outlook.CreateItem(0) #criando o email
    mail.To = 'paulocesaradvice@gmail.com' #defindo o destinatário
    mail.Subject = 'Relatório Gerencial da Carteira' #definindo título
    mail.Body = f''' Olá Paulo, tudo bem?
    Conforme solicitado, levantamos os principais gráficos referente a sua carteira e enviamos em anexo.
    Caso tenha alguma dúvida, não hesite em entrar em contato conosco.''' #corpo do email#corpo do email
    mail.Attachments.Add(VALOR_INVESTIDO_ATIVO) #adicionando anexo
    mail.Attachments.Add(VALOR_INVESTIDO_TIPO) #adicionando anexo
    mail.Attachments.Add(VALOR_IBOV) #adicionando anexo
    mail.Attachments.Add(COMPARATIVO_CARTEIRA_IBOV) #adicionando anexo
    mail.Send() #enviando :)

    return print("Email Enviado")

def main():

    ### PEGANDO COTAÇÕES
    ## indices: ^iINDICE
    ## ações brasileiras: TICKER.SA
    ## source = yahoo
    ## datas: 'aaaa-mm-dd'

    ## IBOVESPA
    ibov_df = web.DataReader('^BVSP', data_source='yahoo', start = '2020-01-02', end = f'{data_atual}') #web.DataReader('TICKER', data_source='fonte', start = 'data inicio', end = 'data final')
    print(ibov_df.info()) #printando info do dataframe para análise

    ## MINHA CARTEIRA
    carteira = pd.read_excel(CAMINHO_CARTEIRA)
    carteira_tipo = carteira.groupby('Tipo')['Valor Investido'].sum()
    carteira_df = pd.DataFrame()
    for ativo in carteira['Ativos']:
        if 'Tesouro' not in ativo: #retirando o tesouro direto do FOR (Não da para retirar as informações dele pelo link do yahoo)
            carteira_df[ativo] = web.DataReader('{}.SA'.format(ativo), data_source='yahoo', start = '2020-01-01', end = f'{data_atual}')['Adj Close'] #pegando os dados dos ativos da carteira
        else:
            pass

    carteira_df = carteira_df.ffill() #preenche o vazio com o dado da linha anterior, a coluna do SMALL11 estava faltando 2 dados. Fizemos isso para preencher.
    print(carteira_df.info()) #printando info do dataframe para análise

    ## TESOURO
    tesouro_df = pd.read_csv(CAMINHO_TD, sep = ';', decimal = ',') #lendo o arquivo CSV com os dados do tesouro (através de um link direto de csv)
    print(tesouro_df.info()) #printando informações do dataframe para análise
    tesouro_df['Data Venda'] = pd.to_datetime(tesouro_df['Data Venda'], format = '%d/%m/%Y') #passando o formato de data de dd/mm/aaaa para o datetime padrão do pandas ('aaaa-mm-dd'), o format não é o formato que ele vai ficar e sim o que ele é.
    tesouro_df = tesouro_df[tesouro_df['Tipo Titulo'] == 'Tesouro Selic'] #pegando somente as linhas que o tipo título é o tesouro selic
    tesouro_df = tesouro_df[['Data Venda', 'PU']] #filtrando base para pegar apenas as colunas de Data Venda e PU
    tesouro_df = tesouro_df.rename(columns = {'Data Venda':'Date' , 'PU': 'Tesouro Selic'}) #renomeando as colunas de Data Venda e PU.

    ## UNINDO DADOS DA MINHA CATEIRA COM O TESOURO
    carteira_df = carteira_df.merge(tesouro_df[['Date','Tesouro Selic']], on = 'Date', how = 'left') #colando os dados do lado e juntando a coluna em comum. (sempre tem que ter uma coluna em comum para usar o merge!)
    print(carteira_df.info()) #printando informações do dataframe para análise

    ### REALIZANDO CALCULO DE INDICADORES
    ## TRATANDO DADOS
    carteira_copia = carteira_df.copy() #criando uma cópia do dataframe da carteira, para não mudar o original na hora dos cálculos

    for i,ativo in enumerate(carteira['Ativos']):
        carteira_copia[ativo] = carteira_copia[ativo]*carteira.loc[i, 'Qtde'] #substituindo os valores das colunas dos ativos pelo Total Investido

    carteira_copia = carteira_copia.set_index('Date') #setando a coluna de data para ser o indice para que no SUM ela não seja contabilizada.

    carteira_copia['Total'] = carteira_copia.sum(axis = 1) #criando coluna de total somando as linhas das colunas restantes (o axis = 1 muda o padrão de somar coluna para somar linha)
    ## CALCULANDO INDICADORES
    carteira_copia_norm = carteira_copia/carteira_copia.iloc[0] #colocando na mesma base para que a comparação no gráfico seja real (partam do mesmo ínicio)
    ibov_df_norm = ibov_df/ibov_df.iloc[0] #colocando na mesma base para que a comparação no gráfico seja real
    rentabilidde_carteira = carteira_copia['Total'].iloc[-1]/carteira_copia['Total'].iloc[0] - 1 #calculando a rentabilidade da carteira
    rentabilidade_ibov = ibov_df['Adj Close'].iloc[-1]/ibov_df['Adj Close'].iloc[0] - 1 #calculando a rentabilidade do ibovespa
    print('Rentabilidade da Carteira {:.1%}'.format(rentabilidde_carteira)) #printando a rentabilidade da carteira
    print('Rentabilidade da IBOV {:.1%}'.format(rentabilidade_ibov)) #printando a rentabilidade do ibovespa

    ### PLOTANDO GRÁFICOS
    ## PLOTANDO GRÁFICO DE DISTRIBUIÇÃO DO VALOR INVESTIDO EM PERCENTUAL POR ATIVO DA CARTEIRA
    grafico1 = carteira['Valor Investido'].plot.pie(labels = carteira['Ativos'], y = 'Valor Investido', legend = False, title ="Distruibuição dos ativos da Carteira",figsize = (15,5), autopct = "%.1f%%") # plotando gráfico de pizza
    grafico1.set_ylabel('') #tirando título do eixo y
    grafico1.set_xlabel('') #tirando título do eixo x
    fig1 = grafico1.figure #atribuindo gráfico a essa variavel
    fig1.savefig(VALOR_INVESTIDO_ATIVO) #salvando gráfico no caminho especificado
    ## PLOTANDO GRÁFICO DE DISTRIBUIÇÃO DO VALOR INVESTIDO EM PERCENTUAL POR TIPO DE ATIVO DA CARTEIRA
    grafico2 = carteira.groupby('Tipo')['Valor Investido'].sum().plot.pie(y = 'Valor Investido', legend = False, title ="Distruibuição dos tipos da Carteira",figsize = (15,5), autopct = "%.1f%%")
    grafico2.set_ylabel('') #tirando o título do eixoo y do gráfico 2
    grafico2.set_xlabel('') #tirando título do eixo x grafico 2
    fig2 = grafico2.figure #atribuindo gráfico a essa variavel
    fig2.savefig(VALOR_INVESTIDO_TIPO)
    ## PLOTANDO GRÁFICO COM OS VALORES DO IBOV AO LONGO DO TEMPO
    grafico3 = ibov_df['Adj Close'].plot(figsize = (15,5))
    grafico3.set_xlabel('')
    fig3 = grafico3.figure #atribuindo gráfico a essa variavel
    fig3.savefig(VALOR_IBOV) #salvando gráfico no caminho especificado
    ## PLOTANDO GRÁFICO COMPARATIVO DA CATEIRA COM O IBOVESPA
    grafico4 =carteira_copia_norm['Total'].plot(figsize = (15,5), label = 'Carteira')
    grafico4 = ibov_df_norm['Adj Close'].plot(figsize = (15,5), label = 'IBOV')
    plt.legend()
    fig4 = grafico4.figure #atribuindo gráfico a essa variavel
    fig4.savefig(COMPARATIVO_CARTEIRA_IBOV) #salvando gráfico no caminho especificado

    print("Iniciando rotina de envio de email")
    enviar_email = enviando_email()

    return None

funcao = main()


import numpy as np
import pandas as pd
import os
import faiss
from .fast_similarity_matching import FSM
from .utils import m_estimate, dim_estimate, apply_dim_reduct, apply_dim_reduct_inference, preprocess, evaluate_clusters
from sklearn.metrics.pairwise import euclidean_distances, cosine_similarity
from sklearn.metrics import silhouette_score, adjusted_rand_score, adjusted_mutual_info_score, normalized_mutual_info_score
from sklearn.cluster import KMeans
from sklearn.svm import SVC
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
import warnings
warnings.filterwarnings('ignore')
import OCAT.example as example
from sklearn import svm

##############################################
# In: data_list   [(a,n)...(z,n)]    -- list of datasets
#     m                              -- num of anchors
# Out:
#     anchor list [(m,n)...(m,n)]    -- list of anchors
# Description: The function generates the corresponding anchors for the datasets
##############################################
def find_anchors(data_list, m):
    anchor_list = []
    for i, X in enumerate(data_list):
        X = X.astype(np.float32)
        d = X.shape[1]
        kmeans = faiss.Kmeans(d, m[i], niter=20, verbose=False)
        kmeans.train(X)
        anchors = kmeans.centroids
        anchor_list.append(anchors)
    return anchor_list

##############################################
# In: X      (c,n)
# Out:
#     S     (c,n)
# Description: The function sorts data in descending order and
#      picks the first kk th that sums up to 1 and scales them
#      to be of range 0 to 1
##############################################
def SimplexPr(X):
    C, N = X.shape
    #to sort in descending order
    T = np.sort(X,axis=0) [::-1]
    S = np.array(X).astype(np.float)
    for i in range(N):
        kk = -1
        t = T[:,i]
        #find kk where first kk element >= 1 otherwise kk = last index
        for j in range(C):
            tep = t[j] - (np.sum(t[0:j+1]) - 1)/(j+1)
            if tep <= 0:
                kk = j-1
                break
        if kk == -1:
            kk = C-1
        #scale X to be at 1
        theta = (np.sum(t[0:kk+1]) - 1)/(kk+1)
        #theta = np.expand_dims(theta, axis=1)
        S[:,i] = (X[:,i] - theta).clip(min=0).flatten()
        #X = np.subtract(X, theta.clip(min=0))
    return S

####################################################################
# In: X      (d,1)  -- data from one cell
#     U      (d,s)  -- Anchor data, s is the # of the closest anchors
#     cn     ()     -- # of iterations, 5-20
# Out:
#     z     (s,1)   --
# Description: This function updates z cn times to find the best z
#      such that x <- U * z
#
###################################################################
def LAE (x,U,cn):
    d, s = U.shape
    #print("d, s: ", d, s)
    z0 = np.ones((s,1))/s
    z1 = z0
    delta = np.zeros((1,cn+2))
    delta[0][0] = 0
    delta[0][1] = 1
    beta = np.zeros((1,cn+1))
    beta[0][0] = 1
    for t in range(cn):
        alpha = (delta[0][t]-1)/delta[0][t+1]
        v = z1 + alpha*(z1-z0)
        dif = x - np.matmul(U,v)
        gv = np.matmul(dif.transpose(),dif/2)
        dgv = np.matmul(U.transpose(),np.matmul(U,v)-x)
        for j in range(d+1):
            b = 2**j*beta[0][t]
            z = SimplexPr(v-dgv/b)
            dif = x - np.matmul(U,z)
            gz = np.matmul(dif.transpose(),dif/2)
            dif = z - v
            gvz = gv + np.matmul(dgv.transpose(),dif) + b * np.matmul(dif.transpose(),dif/2)
            if gz <= gvz:
                beta[0][t+1] = b
                z0 = z1
                z1 = z
                break
        if beta[0][t+1] == 0:
            beta[0][t+1] = b
            z0 = z1
            z1 = z
        delta[0][t+2] = ( 1+np.sqrt(1+4*delta[0][t+1]**2) )/2
        if np.sum(abs(z1-z0)) <= 1e-4:
            break
    z = z1
    return z, np.sum(abs(z1-z0))

####################################################################
# In: TrainData  (d,n)  -- input data matrix, d dimension, n # of samples
#     Anchor     (d,m)  -- Anchor data, m # of anchors
#     s          ()     -- # of closest anchors
#     flag       ()     -- 0 gives a Gaussian kernel-defined Z and 1 gives a LAE-optimized Z
#     cn         ()     -- # of iterations for LAE, usually set to 5-20
# Out:
#     Z          (n,m)  -- anchor-to-data regression weight matrix
#     rL         (m,m)  -- reduced graph Laplacian matrix
#
###################################################################
def AnchorGraph (TrainData, Anchor, s, flag, cn = 5):
    d,m = Anchor.shape
    _, n = TrainData.shape
    Similarity = cosine_similarity(TrainData.transpose(), Anchor.transpose())
    val = np.sort(Similarity, axis=1)[:, -s:]
    pos = np.argsort(Similarity, axis=1)[:, -s:]
    #flatten matrix and reshape back to nxm
    ind = ((pos)*n + np.repeat(np.array([range(n)]).transpose(),s,1)).astype(int)
    # Gaussian kernel-defined Z
    if flag == 0:
        sigma = np.mean(np.sqrt(val[:,s-1]))
        val = np.exp(-val/(sigma*sigma))
        val = np.repeat(np.array([1/np.sum(val,1)]).transpose(),s,1)*val
    #LAE-optimized Z
    if flag == 1:
        val = val/np.expand_dims(np.sum(val,1), axis=1)
    else:
        Anchor = np.matmul(Anchor,np.diag(1/np.sqrt(np.sum(Anchor*Anchor,axis=0))))
        for i in range(n):
            x = TrainData[:,i]
            x = (x/np.linalg.norm(x,2)).reshape((len(x),1))
            U = Anchor[:,pos[i,:]]
            a = example.LAE(x,U,cn)
            val[i,:] = a[0].flatten()
    #expand s # closest anchors to m
    Z = np.zeros((n* m))
    Z[ind] = [val]
    Z = Z.reshape((m,n)).transpose().astype(np.float32)
    #Z = np.matmul(Z, np.diag(np.sqrt(np.sum(Z,0)**-1)))
    return Z

####################################################################
# In:   Z          (n,m)  -- normalized anchor-to-data regression weight matrix
#       zW         (n,m)  -- unnormalized anchor-to-data regression weight matrix
# Out:  ZW         (n,m)  -- approximation for ZW, where W=(n,n) similarity matrix
###################################################################
def Z_to_ZW(Z):
    W_anchor = np.matmul(Z.T, Z)
    W_diag = np.diag(np.sqrt(np.sum(W_anchor,0)**-1))
    W_anchor = np.linalg.multi_dot([W_diag, W_anchor, W_diag])
    ZW = np.matmul(Z, W_anchor)
    return ZW, W_anchor

####################################################################
# In:   Z          (n,m)  -- input regression weight matrix
# Out:
#       Z          (n,m)  -- matrix norm normalized regression weight matrix
###################################################################
def norm(Z):
    Z_norm = np.linalg.norm(Z, axis=1)
    Z_norm = np.expand_dims(Z_norm, axis=1)
    Z = np.divide(Z, Z_norm)
    Z = np.nan_to_num(Z)
    return Z

####################################################################
# In:   data_list       [(a,dim)...(z,dim)]    -- list of datasets (dim PCs)
#       m_list                                 -- num of anchors
#       s_list                                 -- num of anchors to be selected
#       p                                      -- percentage of NNs to consider
#       cn                                     -- rounds of optimization
#       if_inference                           -- flag for cell inference
#       true_known                             -- flag for true labels known
# Out:  ZW              (a+...+z, m)           -- OCAT feature matrix
###################################################################
def sparse_encoding_integration(data_list, m_list, s_list=None, p=0.3, cn=5, if_inference=True, true_known=False):
    # find anchors
    anchor_list = find_anchors(data_list, m_list)
    if true_known:
        anchor_list = [np.concatenate(anchor_list,axis=0)]
    # construct sparse anchor graph
    Z_list = []
    for i, dataset in enumerate(data_list):
        dataset_Z_list = []
        for j, anchor in enumerate(anchor_list):
            Z = AnchorGraph(dataset.transpose(), anchor.transpose(), s_list[j], 2, cn)
            dataset_Z_list.append(Z)
        dataset_Z = np.concatenate(dataset_Z_list, axis=1)
        Z_list.append(dataset_Z)
    Z = np.nan_to_num(np.concatenate(Z_list, axis=0))
    ZW, W_anchor = Z_to_ZW(Z)
    ZW = norm(np.nan_to_num(ZW))
    if if_inference:
        return ZW, anchor_list, s_list, W_anchor
    else:
        return ZW

####################################################################
# In:   data_list       [(a,dim)...(z,dim)]    -- list of datasets (dim PCs)
#       m_list                                 -- num of anchors
#       s_list                                 -- num of anchors to be selected
#       dim                                    -- num of dimensions after dim reduct
#       p                                      -- percentage of NNs to consider
#       log_norm                               -- if apply log norm
#       l2_norm                                -- if apply l2 norm
#       if_inference                           -- if prepare for cell inference
#       random_seed                            -- random seed
#       labels_true                            -- the true labels for each dataset in the data_list
# Out:  ZW              (a+...+z, m)           -- OCAT feature matrix
#       db_list                                -- anchor_list, s_list, W_anchor, Wm
#                                                 from reference dataset for cell inference
###################################################################
def run_OCAT(data_list, m_list=None, s_list=None, dim=None, p=0.3, log_norm=True, l2_norm=True, tfidf=0, mode='FSM', if_inference=False, random_seed=42, labels_true=[]):
    if m_list == None:
        m_list = m_estimate(data_list)
    if s_list ==None:
        s_list = [round(p*m) for m in m_list]
    if dim == None:
        dim = dim_estimate(data_list)

    data_list = preprocess(data_list, log_norm=log_norm, l2_norm=l2_norm, tfidf=tfidf)
    if if_inference:
        data_list, Wm = apply_dim_reduct(data_list, dim=dim, mode=mode, random_seed=random_seed)
        true_known = False
        if list(labels_true):
            true_known = True
            data_combined = np.concatenate(data_list, axis=0)
            labels_true_combined = np.concatenate(labels_true, axis=0)
            data_list = [data_combined[labels_true_combined==i,:] for i in np.unique(labels_true_combined)]
            labels_true_reconstruct = [labels_true_combined[labels_true_combined==i] for i in np.unique(labels_true_combined)]
            labels_true_reconstruct = np.concatenate(labels_true_reconstruct,axis=0)
            m_sum = np.sum(m_list)
            # extract the cell numbers in each true cluster
            m_list = [data_list[i].shape for i in range(len(np.unique(labels_true_combined)))]
            # assign m based on the cell proportion
            total_cell_num = data_combined.shape[0]
            m_list = [int(m_sum*i[0]/total_cell_num) if int(m_sum*i[0]/total_cell_num)>1 else 1 for i in m_list]

            s_list = [round(p*m) for m in m_list]
            s_list = [np.sum(s_list)]
            print('New m_list based on true cell type cluster: ',m_list)

        ZW, anchor_list, s_list, W_anchor = sparse_encoding_integration(data_list, m_list=m_list, s_list=s_list, p=p, cn=5, if_inference=True, true_known=true_known)
        if labels_true:
            db_list = [anchor_list, s_list, W_anchor, Wm, m_list, labels_true_reconstruct]
        else:
            db_list = [anchor_list, s_list, W_anchor, Wm]
        return ZW, db_list
    else:
        data_list, _ = apply_dim_reduct(data_list, dim=dim, mode=mode, random_seed=random_seed)
        ZW = sparse_encoding_integration(data_list, m_list=m_list, s_list=s_list, p=p, cn=5, if_inference=False)
        return ZW

def gene_alignment(ref_genes,inf_genes):
    ref_gene_dict = {j: i for i,j in enumerate(ref_genes)}
    inf_gene_dict = {j: i for i,j in enumerate(inf_genes)}

    genes_intersection = set(ref_genes) & set(inf_genes)
    genes_intersection = sorted(genes_intersection, key=lambda x: ref_gene_dict[x])

    ref_gene_idx = [ref_gene_dict.get(var) for var in genes_intersection]
    inf_gene_idx = [inf_gene_dict.get(var) for var in genes_intersection]

    return ref_gene_idx, inf_gene_idx

####################################################################
# In:   data_list       [(a,dim)...(z,dim)]    -- list of inference datasets (dim PCs)
#       labels_db                              -- cell type annotations from reference dataset
#       db_list                                -- reference db info returned from run_OCAT
#       true_known                             -- if the true labels for reference db is known
#       ZW_db                                  -- OCAT features of the reference dataset
#       log_norm                               -- if apply log norm
#       l2_norm                                -- if apply l2 norm
#       cn                                     -- rounds of optimization
# Out:  ZW              (a+...+z, m)           -- OCAT features of the inference dataset
#       labels                                 -- inferred cell type labels from inference dataset
###################################################################
def run_cell_inference(data_list, db_list, ref_genes=[],inf_genes=[], labels_db = [], true_known=False, ZW_db=list(), log_norm=True, l2_norm=True, cn=5):
    if data_list[0].shape[1]==db_list[3].shape[0]:
        print("Assumed the genes are aligned and in the same order since there is no genes inputs and dimension matched")
    else:
        assert ref_genes and inf_genes, "Must input reference gene label and Query gene labels if the gene dimensions don't match"

    if true_known:
        [anchor_list, s_list, W_anchor, Wm, m_list, labels_db] = db_list
    else:
        assert  labels_db!=[], "Must input labels_db when true_known = False"
        assert ZW_db!=list(), "Must input ZW_db if the reference datasets has no true labels"
        [anchor_list, s_list, W_anchor, Wm] = db_list

    if list(ref_genes) and list(inf_genes):
        ref_gene_idx, inf_gene_idx = gene_alignment(ref_genes, inf_genes)
        Wm=Wm[ref_gene_idx,:]
        data_list = [data_list[0][:,inf_gene_idx]]

    data_list = preprocess(data_list, log_norm=log_norm, l2_norm=l2_norm)
    data_list = apply_dim_reduct_inference(data_list, Wm)
    Z_list = []
    for i, dataset in enumerate(data_list):
        dataset_Z_list = []
        for j, anchor in enumerate(anchor_list):
            Z = AnchorGraph(dataset.transpose(), anchor.transpose(), s_list[j], 2, cn)
            dataset_Z_list.append(Z)
        dataset_Z = np.concatenate(dataset_Z_list, axis=1)
        Z_list.append(dataset_Z)
    Z = np.nan_to_num(np.concatenate(Z_list, axis=0))

    if true_known:
        all_index = np.cumsum([0]+ m_list)
        anchor_num_index = list(zip(all_index[:len(all_index)-1], all_index[1:]))
        ## add up all the weights for the each true cluster and choose the cluster with max as assigned label
        labels = np.argmax([[np.sum(data[x:y]) for x,y in anchor_num_index] for data in Z],axis=1)
        labels = np.unique(labels_db)[labels]
        return Z, labels
    else:
        ZW = np.matmul(Z, W_anchor)
        ZW = norm(np.nan_to_num(ZW))

        clf = SVC(random_state=42)
        clf.fit(ZW_db, labels_db)
        labels = clf.predict(ZW)
    return ZW, labels

def post_processing_pca(Z, topk=20):
    # center data by standard scaling
    scaler=StandardScaler()
    scaler.fit(Z)
    Z = scaler.transform(Z)
    # take topk PCs
    pca = PCA(n_components=topk, svd_solver='arpack')
    pca_result = pca.fit_transform(Z)
    return pca_result

def tune_hyperparameters(data_list, if_tune_m=True, m_range=None, if_tune_dim=True, dim_range=None, if_tune_p=False, p_range=None, log_norm=True, l2_norm=True, true_labels=None, verbose=True):
    # Specify data normalization
    data_list = preprocess(data_list, log_norm=log_norm, l2_norm=l2_norm)
    num_datasets = len(data_list)
    # Impute m if None
    if m_range==None:
        m_est = max(m_estimate(data_list))
        if if_tune_m:
            m_range = [m_est+i*5 for i in range(-3, 3)]
        else:
            m_range = [m_est]
            print('WARNING no value of m is given, default m={} for the dataset(s) from estimation.'.format(m_est))
    # Impute dim if None
    if dim_range==None:
        dim_est = dim_estimate(data_list)
        if if_tune_dim:
            dim_range = [dim_est+i*10 for i in range(-2, 2)]
        else:
            dim_range = [dim_est]
            print('WARNING no value of dim is given, default dim={} for the dataset(s) from estimation.'.format(dim_est))
    # Impute p if None
    if p_range==None:
        if if_tune_p:
            p_range = [0.1, 0.3, 0.5]
        else:
            p_range = [0.3]
            print('WARNING no value of p is given, default p=0.3 for the dataset(s) from estimation.')
    # If ground truth given, find n_clusters
    if true_labels is not None:
        n_clusters = len(np.unique(true_labels))
    out = []
    if verbose:
        print('Testing hyperparameters in the range below:')
        print('Range for m: {}'.format(m_range))
        print('Range for dim: {}'.format(dim_range))
        print('Range for p: {}'.format(p_range))
    for m in m_range:
        for n_dim in dim_range:
            for p in p_range:
                if m*p < 3:
                    print('Skip m={} and p={} as the number of ghost cells is smaller than 3.'.format(m, p))
                    continue
                ZW = run_OCAT(data_list=data_list, m_list=[m]*num_datasets, dim=n_dim, p=p, log_norm=False, l2_norm=False)
                if true_labels is None:
                    labels_pred, n_clusters = evaluate_clusters(ZW, return_num_cluster=True)
                    sil_score = silhouette_score(ZW, labels_pred)
                    out.append([m, n_dim, p, n_clusters, sil_score])
                else:
                    labels_pred = evaluate_clusters(ZW, num_cluster=n_clusters)
                    NMI_cell = normalized_mutual_info_score(true_labels, labels_pred)

                    AMI_cell = adjusted_mutual_info_score(true_labels, labels_pred)

                    ARI_cell = adjusted_rand_score(true_labels, labels_pred)
                    out.append([m, n_dim, p, NMI_cell, AMI_cell, ARI_cell])
    out = np.array(out)
    if true_labels is not None:
        df = pd.DataFrame(data=out, columns=['m', 'n_dim', 'p', 'NMI_score', 'AMI_score', 'ARI_score'])
    else:
        df = pd.DataFrame(data=out, columns=['m', 'n_dim', 'p', 'n_clusters', 'silhoutte_score'])
    if verbose:
        print(df)
    return df


import pytest
import ray
import time
import shutil
import numpy as np

from gym import spaces

from malib import settings
from malib.envs.agent_interface import AgentInterface
from malib.algorithm import random
from malib.utils.typing import BehaviorMode, ParameterDescription, Status

from tests.parameter_server import FakeParameterServer

@pytest.mark.parametrize(
    "observation_space,action_space",
    [
        (spaces.Box(low=-1.0, high=1.0, shape=(1,)), spaces.Discrete(3)),
        (
            spaces.Box(low=-1.0, high=1.0, shape=(4,)),
            spaces.Box(low=-1.0, high=1.0, shape=(2,)),
        ),
    ],
    scope="class",
)
class TestEnvAgentInterface:
    @pytest.fixture(autouse=True)
    def init(self, observation_space, action_space):
        if not ray.is_initialized():
            ray.init()

        try:
            parameter_server = ray.get_actor(settings.PARAMETER_SERVER_ACTOR)
        except ValueError:
            parameter_server = FakeParameterServer.remote()

        self.interface = AgentInterface(
            agent_id="test",
            observation_space=observation_space,
            action_space=action_space,
            parameter_server=parameter_server,
        )

        self.env_id = "test"
        desc = dict(
            registered_name=random.NAME,
            observation_space=observation_space,
            action_space=action_space,
            model_config={},
            custom_config={},
        )
        self.interface.add_policy(
            self.env_id,
            "policy_0",
            desc,
            ParameterDescription(
                time_stamp=time.time(),
                identify="policy_0",
                env_id=self.env_id,
                id="policy_0",
                version=0,
                description=desc,
            ),
        )

    def test_set_behavior_mode(self):
        self.interface.set_behavior_mode(BehaviorMode.EXPLOITATION)
        assert self.interface.behavior_mode == BehaviorMode.EXPLOITATION
        self.interface.set_behavior_mode(BehaviorMode.EXPLORATION)
        assert self.interface.behavior_mode == BehaviorMode.EXPLORATION

    def test_set_behavior_dist(self, observation_space, action_space):
        PPOOL_SIZE = 10
        for i in range(PPOOL_SIZE):
            pid = f"policy_{i}"
            desc = dict(
                registered_name=random.NAME,
                observation_space=observation_space,
                action_space=action_space,
                model_config={},
                custom_config={},
            )
            parameter_desc = ParameterDescription(
                time_stamp=time.time(),
                identify=pid,
                env_id=self.env_id,
                id=pid,
                version=0,
                description=desc,
            )
            self.interface.add_policy(self.env_id, pid, desc, parameter_desc)

        dist = dict(zip([f"policy_{i}" for i in range(PPOOL_SIZE)], [1 / PPOOL_SIZE]))
        self.interface.set_behavior_dist(dist)

    def test_serialize(self):
        # since pickle does not support __post_init__
        # while install dill has no need, we just test __setstate__ and __getstate__
        state = self.interface.__getstate__()
        self.interface.__setstate__(state)

    def test_reset(self, observation_space, action_space):
        PPOOL_SIZE = 10
        for i in range(PPOOL_SIZE):
            pid = f"policy_{i}"
            desc = dict(
                registered_name=random.NAME,
                observation_space=observation_space,
                action_space=action_space,
                model_config={},
                custom_config={},
            )
            parameter_desc = ParameterDescription(
                time_stamp=time.time(),
                identify=pid,
                env_id=self.env_id,
                id=pid,
                version=0,
                description=desc,
            )
            self.interface.add_policy(self.env_id, pid, desc, parameter_desc)

        # reset without any policy figured out?
        self.interface.reset()
        assert self.interface.behavior_policy is not None, (
            self.interface.behavior_policy,
            list(self.interface.policies.keys()),
        )

        # reset with policy figure out?
        self.interface.reset(policy_id="policy_2")
        assert (
            self.interface.behavior_policy == "policy_2"
        ), self.interface.behavior_policy
        assert np.isclose(sum(self.interface.sample_dist.values()), 1.0)
        for e in self.interface.sample_dist.values():
            assert e == 1 / PPOOL_SIZE

        # reset with sample dist figureout?
        sample_dist = [1.0 / PPOOL_SIZE] * PPOOL_SIZE
        sample_dist = dict(zip([f"policy_{i}" for i in range(PPOOL_SIZE)], sample_dist))
        self.interface.reset(sample_dist=sample_dist)
        assert self.interface.behavior_policy is not None

        sample_dist = [0.0] * PPOOL_SIZE
        sample_dist[-1] = 1.0
        sample_dist = dict(zip([f"policy_{i}" for i in range(PPOOL_SIZE)], sample_dist))
        self.interface.reset(sample_dist=sample_dist)
        assert self.interface.behavior_policy == "policy_{}".format(PPOOL_SIZE - 1), (
            self.interface.behavior_policy,
            list(self.interface.policies.keys()),
        )

    def test_compute_action(self, observation_space: spaces.Space):
        self.interface.reset()
        x = self.interface.compute_action(
            observation=np.asarray(
                [observation_space.sample() for _ in range(4)]
            ).reshape((-1,) + observation_space.shape),
            rnn_state=[self.interface.get_initial_state() for _ in range(4)],
        )
        assert len(x) == 3
        action = []
        action_dist = []
        rnn_state = []
        for e in x[0]:
            action.append(e)
        for e in x[1]:
            action_dist.append(e)
        for e in x[2]:
            rnn_state.append(e)
        assert len(action) == len(action_dist) == 4

    def test_weight_update(self):
        self.interface.reset()
        pid = self.interface.behavior_policy
        status = self.interface.update_weights([pid], waiting=True)
        assert isinstance(status, dict) and status[pid] == Status.SUCCESS, status
        status = self.interface.update_weights([pid], waiting=False)
        assert isinstance(status, dict) and status[pid] == Status.SUCCESS, status

    def test_observation_transformation(self, observation_space):
        self.interface.reset()
        res1 = self.interface.transform_observation(observation_space.sample())
        res2 = self.interface.transform_observation(
            [observation_space.sample() for _ in range(4)]
        )
        target_shape = observation_space.shape
        if target_shape == (1,):
            target_shape = ()
        assert res1["obs"].shape == target_shape
        assert res2["obs"].shape == (4,) + target_shape

    def test_save_and_close(self):
        self.interface.reset()
        pid = self.interface.behavior_policy
        tmp_dir = "/tmp/model_{}".format(time.time())
        self.interface.save(tmp_dir)
        self.interface.close()
        shutil.rmtree(tmp_dir)

    @classmethod
    def teardown_class(cls):
        ray.shutdown()


from tkinter import ttk
from tkinter import *

from tkinter import filedialog
from PIL import Image, ImageTk
import sqlite3
from adress.count_screen import Counting

class Win(Counting):
    """Class for Main Window."""

    def __init__(self):
        self.connection = sqlite3.connect("database/adress_cat.db")
        self.win = Tk()
        self.win.title=("Adress-Management")
        self.win.geometry('800x600')
        self.co0 = "#ffffff"
        self.co1 = "#000000"
        self.co2 = "#20214f"
        self.win.configure(background=self.co0)
        self.win.resizable(width=False, height=False)

    def Delete_Win(self):
        self.win.withdraw()
        self.win.destroy()
        from delete_contact_gui import MainWinDelete
        win = MainWinDelete()
        win.Window()
        win.win.mainloop()

    def Add_Win(self):
        self.win.withdraw()
        self.win.destroy()
        from add_contact_gui import MainWin
        win = MainWin()
        win.Window_Main()
        win.win.mainloop()

    def Query_Win(self):
        self.win.withdraw()
        self.win.destroy()
        from gui_query import MainWinQuery
        win = MainWinQuery()
        win.Window()
        win.win.mainloop()

    def Update_Win(self):
        self.win.withdraw()
        self.win.destroy()
        from gui_update_record import MainWinUpdate
        wind = MainWinUpdate()
        wind.MainWinUpdate()
        wind.wind.mainloop()

    def Window(self):
        # top
        top = Frame(self.win, width=800, height=50, bg=self.co2)
        top.grid(row=0, column=0, padx=0, pady=1)

        # bottom
        bottom = Frame(self.win, width=800, height=150, bg=self.co2)
        bottom.place(x=0, y=480)

        # header
        header = Label(top, text="Adress-Management ✆", height=1, font=("Bahnschrift 22 bold"), bg= self.co2, fg=self.co0)
        header.place(x=280, y=2)

        # Instruction
        center = Label(text="Bitte wählen Sie Ihre gewünschte Aktion", height=1, font=("Calibri 18 bold"), bg= self.co0, fg=self.co1)
        center.place(x=220, y=80)

        # Add Contact Button
        self.bAdd = Button(self.win, text="Kontakt Hinzufügen", font=("Bahnschrift 14 bold"), bg=self.co2, fg=self.co0, command=self.Add_Win)
        self.bAdd.place(x=290, y=140, width=260, height=40)

        # Update Button
        self.bUpdate = Button(self.win, text="Kontakt Aktualisieren", font=("Bahnschrift 14 bold"), bg=self.co2, fg=self.co0, command=self.Update_Win)
        self.bUpdate.place(x=290, y=200, width=260, height=40)

        # Query Button
        self.bQuery = Button(self.win, text="Kontakt Abfragen", font=("Bahnschrift 14 bold"), bg=self.co2, fg=self.co0, command=self.Query_Win)
        self.bQuery.place(x=290, y=260, width=260, height=40)

        # Delete Button
        self.bDelete = Button(self.win, text="Kontakt Löschen", font=("Bahnschrift 14 bold"), bg=self.co2, fg=self.co0, command=self.Delete_Win)
        self.bDelete.place(x=290, y=320, width=260, height=40)

        self.get_school()
        frame = Frame(self.win, bg="blue", width=80, height=80)
        frame.place(x=230, y=390)
        school_text = Label(self.win, text="Schulkollegen", fg=self.co1, bg=self.co0)
        school_text.place(x=230, y= 368)
        label = Label(frame, text="🏫" +"\n" + str(self.school) , font=("Arial", 22), bg="white")
        label.place(x=4, y=4, width=72, height=72)

        self.get_family()
        frame = Frame(self.win, bg="green", width=80, height=80)
        frame.place(x=330, y=390)
        family_text = Label(self.win, text="Familie", fg=self.co1, bg=self.co0)
        family_text.place(x=348, y= 368)
        label = Label(frame, text="👨👩👧👦" +"\n" + str(self.family) , font=("Arial", 22), bg="white")
        label.place(x=4, y=4, width=72, height=72)

        self.get_work()
        frame = Frame(self.win, bg="orange", width=80, height=80)
        work_text = Label(self.win, text="Arbeitskollegen", fg=self.co1, bg=self.co0)
        work_text.place(x=424, y= 368)
        frame.place(x=430, y=390)
        label = Label(frame, text="💼" +"\n"+ str(self.work) , font=("Arial", 22), bg="white")
        label.place(x=4, y=4, width=72, height=72)

        self.get_friends()
        frame = Frame(self.win, bg="red", width=80, height=80)
        friend_text = Label(self.win, text="Freunde", fg=self.co1, bg=self.co0)
        frame.place(x=530, y=390)
        friend_text.place(x=544, y= 368)
        label = Label(frame, text="🧍" + "\n" + str(self.friend) , font=("Arial", 22), bg="white")
        label.place(x=4, y=4, width=72, height=72)

def main():
    win = Win()
    win.Window()
    win.win.mainloop()

if __name__ == "__main__":
    main()

{'MainWinDelete': 'delete_contact_gui.MainWinDelete', 'MainWin': 'add_contact_gui.MainWin', 'MainWinQuery': 'gui_query.MainWinQuery', 'MainWinUpdate': 'gui_update_record.MainWinUpdate'}


import os
import argparse
import collections
import numpy as np
import pandas as pd
import scipy.stats as stats
from sklearn import linear_model
from sklearn.decomposition import PCA

def parse_args():

    parser = argparse.ArgumentParser(description = 'Preprocess gene expression')

    parser.add_argument('-g',
                        type = str,
                        required = True,
                        dest = 'gxfile',
                        metavar = 'FILE',
                        help = 'input gene expression file')

    parser.add_argument('-o',
                        type = str,
                        required = True,
                        dest = 'outfile',
                        metavar = 'FILE',
                        help = 'output gene expression file')

    parser.add_argument('-c',
                        type = str,
                        required = True,
                        dest = 'covfile',
                        metavar = 'FILE',
                        help = 'input covariates file')

    parser.add_argument('-m',
                        nargs = '*',
                        default = ['norm'],
                        dest = 'methods',
                        help = 'which method to apply: norm, qn, cclm, cclasso, knn or any combination')

    opts = parser.parse_args()
    return opts

def normalize_center(Y):
    # requires  G x N matrix, where G is the number of genes, and Y is the number of samples
    # returns   G x N matrix
    newY = (Y - np.mean(Y, axis = 1).reshape(-1, 1)) / np.std(Y, axis = 1).reshape(-1, 1)
    return newY

def read_expression(filename):
    # returns G x N gene expression
    #         list of N donor ids
    #         list of G gene names
    gx = list()
    genes = list()
    with open(filename, 'r') as infile:
        header = next(infile)
        for line in infile:
            linesplit = line.strip().split()
            thisgx = np.array([float(x) for x in linesplit[1:]])
            gx.append(thisgx)
            genes.append(linesplit[0])
    gx = np.array(gx)
    donors = header.strip().split()[1:]
    return gx, donors, genes

def write_expression(GX, donors, genes, filename):
    with open(filename, 'w') as fout:
        header = 'gene_ids\t' + '\t'.join(donors) + '\n'
        fout.write(header)
        for i in range(GX.shape[0]):
            line = '{:s}\t'.format(genes[i]) + '\t'.join(['{:g}'.format(x) for x in GX[i, :]]) + '\n'
            fout.write(line)

def normalize_quantiles(M):
    """
    Note: replicates behavior of R function normalize.quantiles from library("preprocessCore")

    Reference:
     [1] Bolstad et al., Bioinformatics 19(2), pp. 185-193, 2003

    Adapted from https://github.com/andrewdyates/quantile_normalize
    """

    Q = M.argsort(axis=0)
    m,n = M.shape

    # compute quantile vector
    quantiles = np.zeros(m)
    for i in range(n):
        quantiles += M[Q[:,i],i]
    quantiles = quantiles / n

    for i in range(n):
        # Get equivalence classes; unique values == 0
        dupes = np.zeros(m, dtype=np.int)
        for j in range(m-1):
            if M[Q[j,i],i]==M[Q[j+1,i],i]:
                dupes[j+1] = dupes[j]+1

        # Replace column with quantile ranks
        M[Q[:,i],i] = quantiles

        # Average together equivalence classes
        j = m-1
        while j >= 0:
            if dupes[j] == 0:
                j -= 1
            else:
                idxs = Q[j-dupes[j]:j+1,i]
                M[idxs,i] = np.median(M[idxs,i])
                j -= 1 + dupes[j]
        assert j == -1

    return M

#def inverse_quantile_normalize(M):
#    """
#    After quantile normalization of samples, standardize expression of each gene
#    """
#    R = stats.mstats.rankdata(M,axis=1)  # ties are averaged
#    Q = stats.norm.ppf(R/(M.shape[1]+1))
#    return Q

def inverse_normal_transform(M):
    """
    Transform rows to a standard normal distribution
    """
    R = stats.mstats.rankdata(M, axis=1)  # ties are averaged
    if isinstance(M, pd.DataFrame):
        Q = pd.DataFrame(stats.norm.ppf(R/(M.shape[1]+1)), index=M.index, columns=M.columns)
    else:
        Q = stats.norm.ppf(R/(M.shape[1]+1))
    return Q

def edgeR_calcNormFactors(counts_df, ref=None, logratio_trim=0.3, sum_trim=0.05, acutoff=-1e10, verbose=False):
    """
    Calculate TMM (Trimmed Mean of M values) normalization.
    Reproduces edgeR::calcNormFactors.default
    Scaling factors for the library sizes that minimize the log-fold changes between the samples for most genes.
    Effective library size: TMM scaling factor * library size
    References:
     [1] Robinson & Oshlack, 2010
     [2] R functions:
          edgeR::calcNormFactors.default
          edgeR:::.calcFactorWeighted
          edgeR:::.calcFactorQuantile
    """

    # discard genes with all-zero counts
    Y = counts_df.values.copy()
    allzero = np.sum(Y>0,axis=1)==0
    if np.any(allzero):
        Y = Y[~allzero,:]

    # select reference sample
    if ref is None:  # reference sample index
        f75 = np.percentile(Y/np.sum(Y,axis=0), 75, axis=0)
        ref = np.argmin(np.abs(f75-np.mean(f75)))
        if verbose:
            print('Reference sample index: '+str(ref))

    N = np.sum(Y, axis=0)  # total reads in each library

    # with np.errstate(divide='ignore'):
    with warnings.catch_warnings():
        warnings.simplefilter('ignore')
        logR = np.log2((Y/N).T / (Y[:,ref]/N[ref])).T  # log fold change; Mg in [1]
        absE = 0.5*(np.log2(Y/N).T + np.log2(Y[:,ref]/N[ref])).T  # average log relative expression; Ag in [1]
        v = (N-Y)/N/Y
        v = (v.T + v[:,ref]).T  # w in [1]

    ns = Y.shape[1]
    tmm = np.zeros(ns)
    for i in range(ns):
        fin = np.isfinite(logR[:,i]) & np.isfinite(absE[:,i]) & (absE[:,i] > acutoff)
        n = np.sum(fin)

        loL = np.floor(n*logratio_trim)+1
        hiL = n + 1 - loL
        loS = np.floor(n*sum_trim)+1
        hiS = n + 1 - loS
        rankR = stats.rankdata(logR[fin,i])
        rankE = stats.rankdata(absE[fin,i])
        keep = (rankR >= loL) & (rankR <= hiL) & (rankE >= loS) & (rankE <= hiS)
        # in [1], w erroneously defined as 1/v ?
        tmm[i] = 2**(np.nansum(logR[fin,i][keep]/v[fin,i][keep]) / np.nansum(1/v[fin,i][keep]))

    tmm = tmm / np.exp(np.mean(np.log(tmm)))
    return tmm

def edgeR_cpm(counts_df, tmm=None, normalized_lib_sizes=True):
    """
    Return edgeR normalized/rescaled CPM (counts per million)
    Reproduces edgeR::cpm.DGEList
    """
    lib_size = counts_df.sum(axis=0)
    if normalized_lib_sizes:
        if tmm is None:
            tmm = edgeR_calcNormFactors(counts_df)
        lib_size = lib_size * tmm
    return counts_df / lib_size * 1e6

def gtex_v6_preproc(GX):
    M = normalize_quantiles(GX)
    #R = inverse_quantile_normalize(M)
    R = inverse_normal_transform(M)
    return R

def gtex_v6_preproc_sim(GX):
    R = inverse_normal_transform(GX)
    return R

def gtex_v6p_preproc(GX):
    tmm_counts_df = edgeR_cpm(counts_df, normalized_lib_sizes=True)
    R = inverse_normal_transform(tmm_counts_df)
    return norm_df

def covcorrlm(X, W):
    # inputs: X of shape G x N (G = number of genes, N = number of samples)
    #         W of shape C x N (C = number of covariates)
    Wnorm = (W - np.mean(W, axis = 1).reshape(-1, 1)) / np.std(W, axis = 1).reshape(-1, 1)
    linreg = linear_model.LinearRegression()
    linreg.fit(Wnorm.T, X.T)
    Xcorr = X - linreg.predict(Wnorm.T).T
    return Xcorr

def covcorrlasso(X, W, alpha = 0.05):
    # inputs: X of shape G x N (G = number of genes, N = number of samples)
    #         W of shape C x N (C = number of covariates)
    Wnorm = (W - np.mean(W, axis = 1).reshape(-1, 1)) / np.std(W, axis = 1).reshape(-1, 1)
    Xcorr = np.zeros(X.shape)
    #coefs = np.zeros((X.shape[0], W.shape[0]))
    for i in range(X.shape[0]):
        lassoreg = linear_model.Lasso(alpha = alpha)
        lassoreg.fit(Wnorm.T, X[i, :])
        Xcorr[i] = X[i, :] - lassoreg.predict(Wnorm.T).T
    return Xcorr

def covcorrknn(X, kneighbor):
    # input: X of shape G x N (G = number of genes, N = number of samples)
    XT = X.T
    pca = PCA(n_components=min(X.shape[0], X.shape[1]))
    pca.fit(XT) # requires N x G
    Xpca = pca.transform(XT)

    def gene_distance(a, b):
        return np.linalg.norm(a - b)

    nsample = XT.shape[0]
    distance_matrix = np.zeros((nsample, nsample))
    for i in range(nsample):
        for j in range(i+1, nsample):
            dist = gene_distance(Xpca[i,:], Xpca[j,:])
            distance_matrix[i, j] = dist
            distance_matrix[j, i] = dist

    gx_knn = np.zeros_like(XT)
    neighbor_list = list()

    for i in range(nsample):
        neighbors = np.argsort(distance_matrix[i, :])[:kneighbor + 1][1:]
        gx_knn[i, :] = XT[i, :] - np.mean(XT[neighbors, :], axis = 0)

    return gx_knn.T

def pp_options(method, _X, cov = None):
    # a pythonic dictionary runs through all functions and creates the full dictionary
    # hence if-else is used.
    gx = _X.copy()
    #if method == 'raw':
    #    print('   > Keep raw file')
    #    res = gx
    if method == 'norm':
        print('   > Center and scale')
        res = normalize_center(gx)
    elif method == 'qnonly':
        print('   > QN only (no inverse normal transformation)')
        res = normalize_quantiles(gx)
    elif method == 'qn':
        print('   > QN (misnomer. actually only inverse normal transformation)')
        #res = gtex_v6_preproc(gx)
        res = gtex_v6_preproc_sim(gx)
    elif method == 'cclm':
        print('   > Covariate correction with linear model')
        res = covcorrlm(gx, cov)
    elif method == 'cclasso':
        print('   > Adaptive covariate correction using LASSO')
        res = covcorrlasso(gx, cov)
    elif method.startswith('knn'):
        kneighbor = int(method[3:])
        print('   > KNN correction')
        res = covcorrknn(gx, kneighbor)
    return res

if __name__ == '__main__':

    opts = parse_args()

    # Read gene expression
    print ("Reading gene expression.")
    gx, gxdonors, genes = read_expression(opts.gxfile)

    # Read covariates
    if any(['cc' in x for x in opts.methods]):
        print ("Reading covariates.")
        cov, covdonors, covnames = read_expression(opts.covfile)
        covidx = np.array([covdonors.index(x) for x in gxdonors])
        covsort = cov[:, covidx]
    else:
        cov = None

    # Perform the preprocessing
    gxpp = collections.defaultdict(lambda: None)
    gxpp['raw'] = gx

    for method in opts.methods:
        print(f'Applying {method}')
        msteps = method.split('_')
        prevstep = 'raw'
        curstep = ''

        for mstep in msteps:
            curstep = f'{mstep}' if curstep == '' else f'{curstep}_{mstep}'
            print(f'  Step: {curstep}')
            if gxpp[curstep] is None:
                gxpp[curstep] = pp_options(mstep, gxpp[prevstep], cov = cov)
            prevstep = curstep

    # Write output
    print ("Writing all outputs.")
    oprefix = os.path.splitext(opts.outfile)[0]
    osuffix = os.path.splitext(opts.outfile)[1]
    for method in opts.methods:
        ofilename = f'{oprefix}_{method}{osuffix}'
        write_expression(gxpp[method], gxdonors, genes, ofilename)


import xlwt
import xlrd
import os
import time
import math
import itertools

# rederence:
# https://blog.csdn.net/YeChao3/article/details/83660615

# https://blog.csdn.net/liao392781/article/details/85338491
# Python3学习（三十八）：如何遍历一个目录下的同类型文件（csv、log等）

# https://blog.csdn.net/u013300049/article/details/79313979?depth_1-utm_source=distribute.pc_relevant_right.none-task&utm_source=distribute.pc_relevant_right.none-task
# itertools

# 读取课表信息，存于tab的列表当中
tab = []# 储存每个人的课表信息
info = []# 储存对应的 姓名_学号 信息

files = os.listdir('./course')
xls = list(filter(lambda x:x[-4:]=='.xls', files))
xlsx = list(filter(lambda x:x[-5:]=='.xlsx', files))

print("读取到以下表格文件：\n{}{}\n数量是：{}".format(xls,xlsx,len(xls)+len(xlsx)))

# 打开文件
for i in range(len(xls)):
    data = xlrd.open_workbook(filename="./course/"+xls[i])
    tab.append(data.sheets()[0])
    info.append(data.sheets()[0].cell(0,0).value[7:-7]+xls[i][7:-4])
    # print(info)
for i in range(len(xlsx)):
    data = xlrd.open_workbook("./course/"+xlsx[i])
    tab.append(data.sheets()[0])
    info.append(data.sheets()[0].cell(0,0).value[7:-7]+xlsx[i][7:-5])
    # print(info)
    # xlrd.open_workbook(filename=,encoding_override="utf-8")
'''amount = eval(input("请输入文件的数量"))
for i in range(1,amount+1):
    #data = xlrd.open_workbook("path of file")
    data = xlrd.open_workbook("arrange (" + str(i) + ").xls")
    tab.append(data.sheets()[0])
'''
# print输出人员信息，检查是否有误
print("\n经处理一共有{}张课表信息，人员如下：".format(len(tab)))
for i in range(len(tab)):
    print(info[i])

# 将所有成员信息写入convert当中的convert.xls内
# wb_c表格ws_c表，设置姓名班级对应的特殊昵称
wb_c = xlwt.Workbook(encoding='ascii')
ws_c = wb_c.add_sheet('Convert')
print('请在convert文件夹里的convert.xls里编辑姓名学号后一格对应的昵称,或者直接新建一个convert0.xls的文件，格式与前一个相同\n')
for i in range(len(tab)):
    ws_c.write(i,0,label=info[i])
wb_c.save('./convert/convert.xls')

if_nickname = input("是否需要对应昵称？\n1  是\n2  否\n")

if if_nickname == '1':
    input("请打开目录convert文件中的表格，在B列姓名右方写出对应昵称，另存为文件为convert0.xls后按回车键继续")
    # 需要更改昵称的话，在这里写代码
    # 打开convert0.xls表格
    try:
        data_conv = xlrd.open_workbook('./convert/convert0.xls')
    except:
        try:
            data_conv = xlrd.open_workbook('./convert/convert0.xlsx')
        except:
            print("格式错误")
    t_c = data_conv.sheets()[0]
    for i in range(len(info)):
        for j in range(t_c.nrows):
            # print(t_c.cell(j,0).value)
            if t_c.cell(j,0).value == info[i]:
                info[i] = t_c.cell(j,1).value
    print(info)

# 读取课程表周数[['6'], ['1-5', '7-14'], ['13-15']，['4', '6', '12', '14', '16']]
# 返回值是一个列表，包含单个周数信息，比如 [1, 2, 3, 4, 5, 6, 7, 8, 13, 14, 15]
def read(a):
    free = []
    if a == ' ' or a == '':
        None
    else:
        a = a.split('\n\n')
        for i_a in range(len(a)):
            a[i_a] = a[i_a].split('\n')
        a[0].remove('')
        for i_a in range(len(a)):
            try:
                if a[i_a] != ['']:
                    a[i_a][2] = a[i_a][2].replace('[周]','')
                    a[i_a][2] = a[i_a][2].replace('[单周]','')
                    a[i_a][2] = a[i_a][2].replace('[双周]','')
                    free.append(a[i_a][2].split(','))
            except:
                print("出现错误，请将此状况报告给开发者，谢谢")
    free2 = []
    if free == []:
        None
    else:
        for i_2 in range(len(free)):
            for j_2 in range(len(free[i_2])):
                if '-' in free[i_2][j_2]:
                    list_free2 = free[i_2][j_2].split('-')
                    for i_33 in range(eval(list_free2[0]),eval(list_free2[-1])+1):
                        free2.append(i_33)
                else:
                    free2.append(eval(free[i_2][j_2]))
    #print(free2)
    return free2

# 有课的课表
# 先建立一个框架，空白的
lis_busy = []
lis_free = []
week_l = 0
week_r = 19
for i_l in range(week_r):
    lis_busy.append([])
    lis_free.append([])
    for i_ll in range(7):
        lis_busy[i_l].append([])
        lis_free[i_l].append([])
        for i_lll in range(6):
            lis_busy[i_l][i_ll].append([])
            lis_free[i_l][i_ll].append([])

# worksheet可以通过tab[]的索引
# 注意，为了方便起见，这里不在从0开始计数
for i in range(len(tab)):
    #
    for row in range(3, 9):
        for col in range(1, 8):
            fr = read(tab[i].cell(row, col).value)
            for i_week in range(len(lis_busy)):
                if i_week in fr:
                    lis_busy[i_week][col-1][row-3].append(info[i])
                else:
                    lis_free[i_week][col-1][row-3].append(info[i])
            # print('第四周星期二{}'.format(lis[4][2]))
# print(lis_busy)
# print(lis_free)
# 整合各列表存于wbd当中，同时储存在lis里
wb_busy = xlwt.Workbook(encoding='ascii')
'''[week1[Mon[[0102][0304][0506][0708][0910][1112]]Tue[]Wed[]Thu[]Fri[]Sat[]Sun[]
          ]
    week2[]
    ......]'''

# wb_busy.save('./busytime/busytime.xls')
# print(lis)
# 到此为止，已经建立了一个list，有没时间的人都写了进去，开心，任务完成了一半
'''
lis_busy 有课时间表
lis_free 无课时间表
格式：有没空的人 = lis_busy[第一周][星期八][78节课]
info     人员名单
'''

'''
x = itertools.combinations_with_replacement(info, 100)
print(list(x)[30:100])
'''

# 安排傍晚值班的程序
'''
def arrange_evening(bgwk,bgdy,edwk,eddy,people):
    print("step1 计算人均工作最大次数：", end='')
    days = (edwk-bgwk)*7+eddy-bgdy+1
    average_timesum = math.ceil(days * people / len(info))
    print(average_timesum)
    weekday_cal = []
    wk = bgwk
    dy = bgdy
    for i in range(days):
        temp_weekday_cal = (wk,dy)#week,day
        if dy == 6:
            dy = 0
            wk += 1
        else:
            dy += 1
        weekday_cal.append(temp_weekday_cal)
    print("step2 新建列表")
    #ws_wholeday.write(0, 0, label='')
    list_wholeday = []# 表格
    list_pre = []
    for i in range(days):
        list_wholeday.append([])
        list_pre.append({})
    list_used = {}# 统计已经安排过的人的次数以及时间
    for i in info:
        list_used[i] = 0
    # 预统计
    for i in range(len(list_wholeday)):
        for j in range(block):
            temp_dict_peo = {}
            for k in info:
                temp_dict_peo[k]=suit_wholeday_value(k,j,weekday_cal[i][0],weekday_cal[i][1])
            list_pre[i][j] = temp_dict_peo
    print("step3 开始第一次安排")
    # 开始遍历每一天
    for i in range(len(list_wholeday)):
        # 遍历每一天当中的时间段
        for j in range(block):
            temp_dict = {}
            temp_peo = []
            if_arrange = 0
            # 遍历预统计当中的人
            for p in range(people):

                for k in sorted(list_pre[i][j]):
                    # 判断100的人以及未安排到的人
                    if list_pre[i][j][k] >= 60 and list_used[k] < average_timesum and not k in temp_peo and not if_in_list(k,list_wholeday[i]):
                        temp_peo.append(k)
                        list_used[k] += 1
                        if_arrange = 1
                        break
                if if_arrange == 0:
                    temp_peo.append(' ')
                if_arrange = 0
            list_wholeday[i].append(temp_peo)
    #print(list_wholeday)
    print("step4 开始第二次安排")

    print("step5 开始写入信息")
    wb_wholeday = xlwt.Workbook(encoding='ascii')
    ws_wholeday = wb_wholeday.add_sheet('值班表')
    ws_times = wb_wholeday.add_sheet("值班次数统计")
    for i in range(len(tab)):
        ws_times.write(i, 0, label=info[i])
        ws_times.write(i, 1, label=list_used[info[i]])
    ws_wholeday.write(0, 0, label='周数星期')
    ws_wholeday.write(0, 1, label='时间段')
    ws_wholeday.write(0, 2, label='人员')
    for i in range(days):
        ws_wholeday.write(i * block + 1, 0, label='第'+str(weekday_cal[i][0]+1)+'周 星期'+str(weekday_cal[i][1]+1))
        ws_wholeday.write(i * block + 1, 1, label='8:00-10:00')
        ws_wholeday.write(i * block + 2, 1, label='10:00-12:00')
        ws_wholeday.write(i * block + 3, 1, label='12:00-下午课前')
        ws_wholeday.write(i * block + 4, 1, label='下午56节课时间段')
        ws_wholeday.write(i * block + 5, 1, label='下午78节课时间段')
        if block == 6:
            ws_wholeday.write(i * block + 6, 1, label='下午78节课后-傍晚')
        for j in range(len(list_wholeday[i])):
            for k in range(len(list_wholeday[i][j])):
                ws_wholeday.write(i * block + 1 + j, k+2, label=list_wholeday[i][j][k])

    filename = input("step6 请输入保存的文件名:")
    wb_wholeday.save('./'+filename+'.xls')
    print("Succcessful")
'''

# 计算全天值班的合适概率
def suit_wholeday_value(mem, num, week, day):
    output = 100
    if num == 0:#8-10
        if mem in lis_busy[week][day][0]:
            output = 0
        elif mem in lis_busy[week][day][1]:
            output -= 10
    elif num == 1:#10-12
        if mem in lis_busy[week][day][1]:
            output = 0
        elif mem in lis_busy[week][day][0]:
            output -= 10
    elif num == 2:#12-before class 56
        if mem in lis_busy[week][day][1]:
            output -= 10
        if mem in lis_busy[week][day][2]:
            output -= 10
    elif num == 3:#class5-6
        if mem in lis_busy[week][day][2]:
            output = 0
        elif mem in lis_busy[week][day][3]:
            output -= 10
    elif num == 4:#class7-8
        if mem in lis_busy[week][day][3]:
            output = 0
        elif mem in lis_busy[week][day][2]:
            output -= 10
    elif num == 5:#after class78
        if mem in lis_busy[week][day][3]:
            output -= 10
        if mem in lis_busy[week][day][4]:
            output -= 10
    return output

# 判断子列表是否含有某元素
def if_in_list(k, lis):
    state = 0
    for i in lis:
        if k in i:
            state = 1
    if state == 1:
        return True
    else:
        return False

# 安排整天的程序
def arrange_wholeday(bgwk,bgdy,edwk,eddy,people,ifdawn):
    print("开始安排")
    if ifdawn == 1:
        block = 6
    else:
        block = 5

    print("step1 计算人均工作最大次数：", end='')
    days = (edwk-bgwk)*7+eddy-bgdy+1
    average_timesum = math.ceil(block * days * people / len(info))
    print(average_timesum)
    weekday_cal = []
    wk = bgwk
    dy = bgdy
    for i in range(days):
        temp_weekday_cal = (wk,dy)#week,day
        if dy == 6:
            dy = 0
            wk += 1
        else:
            dy += 1
        weekday_cal.append(temp_weekday_cal)
    print("step2 新建列表")
    #ws_wholeday.write(0, 0, label='')
    list_wholeday = []# 表格
    list_pre = []
    for i in range(days):
        list_wholeday.append([])
        list_pre.append({})
    list_used = {}# 统计已经安排过的人的次数以及时间
    for i in info:
        list_used[i] = 0
    # 预统计
    for i in range(len(list_wholeday)):
        for j in range(block):
            temp_dict_peo = {}
            for k in info:
                temp_dict_peo[k]=suit_wholeday_value(k,j,weekday_cal[i][0],weekday_cal[i][1])
            list_pre[i][j] = temp_dict_peo
    print("step3 开始第一次安排")
    # 开始遍历每一天
    for i in range(len(list_wholeday)):
        # 遍历每一天当中的时间段
        for j in range(block):
            temp_dict = {}
            temp_peo = []
            if_arrange = 0
            # 遍历预统计当中的人
            for p in range(people):

                for k in sorted(list_pre[i][j]):
                    # 判断100的人以及未安排到的人
                    if list_pre[i][j][k] >= 60 and list_used[k] < average_timesum \
                            and not k in temp_peo and not if_in_list(k,list_wholeday[i]):
                        temp_peo.append(k)
                        list_used[k] += 1
                        if_arrange = 1
                        break
                if if_arrange == 0:
                    temp_peo.append(' ')
                if_arrange = 0
            list_wholeday[i].append(temp_peo)
    #print(list_wholeday)
    print("step4 开始第二次安排")

    print("step5 开始写入信息")
    wb_wholeday = xlwt.Workbook(encoding='ascii')
    ws_wholeday = wb_wholeday.add_sheet('值班表')
    ws_times = wb_wholeday.add_sheet("值班次数统计")
    for i in range(len(tab)):
        ws_times.write(i, 0, label=info[i])
        ws_times.write(i, 1, label=list_used[info[i]])
    ws_wholeday.write(0, 0, label='周数星期')
    ws_wholeday.write(0, 1, label='时间段')
    ws_wholeday.write(0, 2, label='人员')
    for i in range(days):
        ws_wholeday.write(i * block + 1, 0, label='第'+str(weekday_cal[i][0]+1)+'周 星期'+str(weekday_cal[i][1]+1))
        ws_wholeday.write(i * block + 1, 1, label='8:00-10:00')
        ws_wholeday.write(i * block + 2, 1, label='10:00-12:00')
        ws_wholeday.write(i * block + 3, 1, label='12:00-下午课前')
        ws_wholeday.write(i * block + 4, 1, label='下午56节课时间段')
        ws_wholeday.write(i * block + 5, 1, label='下午78节课时间段')
        if block == 6:
            ws_wholeday.write(i * block + 6, 1, label='下午78节课后-傍晚')
        for j in range(len(list_wholeday[i])):
            for k in range(len(list_wholeday[i][j])):
                ws_wholeday.write(i * block + 1 + j, k+2, label=list_wholeday[i][j][k])

    filename = input("step6 请输入保存的文件名:")
    wb_wholeday.save('./'+filename+'.xls')
    print("Succcessful")

# 转换4个输入时间为输出的四个字符
def time_convert1():
    try:
        temp = input("请以空格分隔来输入，输入完成后回车\n开始周数 开始星期数 结束周数 结束星期数 单次人数\n请输入:")
        temp = temp.split(" ")
        a = eval(temp[0])-1
        b = eval(temp[1])-1
        c = eval(temp[2])-1
        d = eval(temp[3])-1
        e = eval(temp[4])
        return a, b, c, d, e
    except:
        print("输入错误")

# 转换4个输入时间为输出的四个字符
def time_convert2():
    try:
        temp = input("请以空格分隔来输入，输入完成后回车\n开始周数 开始星期数 结束周数 结束星期数 单次人数 是否需要下午课以后（是1 否0）\n请输入:")
        temp = temp.split(" ")
        a = eval(temp[0])-1
        b = eval(temp[1])-1
        c = eval(temp[2])-1
        d = eval(temp[3])-1
        e = eval(temp[4])
        f = eval(temp[5])
        return a, b, c, d, e, f
    except:
        print("输入错误")

def main():
    choice = input("请输入数字后回车进行选择\n1 安排傍晚值班\n2 安排全天值班\n3 退出\n请输入:")
    if choice == '1':
        print("安排傍晚值班")
        #(a,b,c,d,e)=time_convert1()
        #arrange_evening(a,b,c,d,e)
        print("此功能还在开发中，敬请期待")
    elif choice == '2':
        print("安排全天值班")
        (a,b,c,d,e,f)=time_convert2()
        arrange_wholeday(a,b,c,d,e,f)
    elif choice == '3':
        quit()

main()