from pathlib import Path

import torch
# from torch.autograd.grad_mode import F
from torch.utils.data import Dataset
import datasets.transforms_video as T

import os
from PIL import Image
import json
import numpy as np
import random

from pycocotools import mask as coco_mask

class MeViSDataset(Dataset):
    """
    A dataset class for the MeViS dataset which was first introduced in the paper:
    "MeViS: A Large-scale Benchmark for Video Segmentation with Motion Expressions"
    """

    def __init__(self, img_folder: Path, ann_file: Path, transforms, return_masks: bool,
                 num_frames: int, max_skip: int):
        self.img_folder = img_folder
        self.ann_file = ann_file
        self._transforms = transforms
        self.return_masks = return_masks  # not used
        self.num_frames = num_frames
        self.max_skip = max_skip
        # create video meta data
        self.prepare_metas()

        mask_json = os.path.join(str(self.img_folder) + '/mask_dict.json')
        print(f'Loading masks form {mask_json} ...')
        with open(mask_json) as fp:
            self.mask_dict = json.load(fp)

        print('\n video num: ', len(self.videos), ' clip num: ', len(self.metas))
        print('\n')

    def prepare_metas(self):
        # read expression data
        with open(str(self.ann_file), 'r') as f:
            subset_expressions_by_video = json.load(f)['videos']
        self.videos = list(subset_expressions_by_video.keys())

        self.metas = []
        for vid in self.videos:
            # vid_meta = subset_metas_by_video[vid]
            vid_data = subset_expressions_by_video[vid]
            vid_frames = sorted(vid_data['frames'])
            vid_len = len(vid_frames)

            for exp_id, exp_dict in vid_data['expressions'].items():
                for frame_id in range(0, vid_len, self.num_frames):
                    meta = {}
                    meta['video'] = vid
                    meta['exp'] = exp_dict['exp']
                    meta['obj_id'] = [int(x) for x in exp_dict['obj_id']]
                    meta['anno_id'] = [str(x) for x in exp_dict['anno_id']]
                    meta['frames'] = vid_frames
                    meta['frame_id'] = frame_id
                    meta['category'] = 0
                    self.metas.append(meta)

    @staticmethod
    def bounding_box(img):
        rows = np.any(img, axis=1)
        cols = np.any(img, axis=0)
        rmin, rmax = np.where(rows)[0][[0, -1]]
        cmin, cmax = np.where(cols)[0][[0, -1]]
        return rmin, rmax, cmin, cmax  # y1, y2, x1, x2

    def __len__(self):
        return len(self.metas)

    def __getitem__(self, idx):
        instance_check = False
        while not instance_check:
            meta = self.metas[idx]  # dict

            video, exp, anno_id, category, frames, frame_id = \
                meta['video'], meta['exp'], meta['anno_id'], meta['category'], meta['frames'], meta['frame_id']
            # clean up the caption
            exp = " ".join(exp.lower().split())
            category_id = 0
            vid_len = len(frames)

            num_frames = self.num_frames
            # random sparse sample
            sample_indx = [frame_id]
            if self.num_frames != 1:
                # local sample
                sample_id_before = random.randint(1, 3)
                sample_id_after = random.randint(1, 3)
                local_indx = [max(0, frame_id - sample_id_before), min(vid_len - 1, frame_id + sample_id_after)]
                sample_indx.extend(local_indx)

                # global sampling
                if num_frames > 3:
                    all_inds = list(range(vid_len))
                    global_inds = all_inds[:min(sample_indx)] + all_inds[max(sample_indx):]
                    global_n = num_frames - len(sample_indx)
                    if len(global_inds) > global_n:
                        select_id = random.sample(range(len(global_inds)), global_n)
                        for s_id in select_id:
                            sample_indx.append(global_inds[s_id])
                    elif vid_len >= global_n:  # sample long range global frames
                        select_id = random.sample(range(vid_len), global_n)
                        for s_id in select_id:
                            sample_indx.append(all_inds[s_id])
                    else:
                        select_id = random.sample(range(vid_len), global_n - vid_len) + list(range(vid_len))
                        for s_id in select_id:
                            sample_indx.append(all_inds[s_id])
            sample_indx.sort()

            # read frames and masks
            imgs, labels, boxes, masks, valid = [], [], [], [], []
            for j in range(self.num_frames):
                frame_indx = sample_indx[j]
                frame_name = frames[frame_indx]
                img_path = os.path.join(str(self.img_folder), 'JPEGImages', video, frame_name + '.jpg')
                # mask_path = os.path.join(str(self.img_folder), 'Annotations', video, frame_name + '.png')
                img = Image.open(img_path).convert('RGB')
                # h, w = img.shape
                mask = np.zeros(img.size[::-1], dtype=np.float32)
                for x in anno_id:
                    frm_anno = self.mask_dict[x][frame_indx]
                    if frm_anno is not None:
                        mask += coco_mask.decode(frm_anno)

                # create the target
                label = torch.tensor(category_id)

                if (mask > 0).any():
                    y1, y2, x1, x2 = self.bounding_box(mask)
                    box = torch.tensor([x1, y1, x2, y2]).to(torch.float)
                    valid.append(1)
                else:  # some frame didn't contain the instance
                    box = torch.tensor([0, 0, 0, 0]).to(torch.float)
                    valid.append(0)
                mask = torch.from_numpy(mask)

                # append
                imgs.append(img)
                labels.append(label)
                masks.append(mask)
                boxes.append(box)

            # transform
            w, h = img.size
            labels = torch.stack(labels, dim=0)
            boxes = torch.stack(boxes, dim=0)
            boxes[:, 0::2].clamp_(min=0, max=w)
            boxes[:, 1::2].clamp_(min=0, max=h)
            masks = torch.stack(masks, dim=0)
            target = {
                'frames_idx': torch.tensor(sample_indx),  # [T,]
                'labels': labels,  # [T,]
                'boxes': boxes,  # [T, 4], xyxy
                'masks': masks,  # [T, H, W]
                'valid': torch.tensor(valid),  # [T,]
                'caption': exp,
                'orig_size': torch.as_tensor([int(h), int(w)]),
                'size': torch.as_tensor([int(h), int(w)])
            }

            # "boxes" normalize to [0, 1] and transform from xyxy to cxcywh in self._transform
            imgs, target = self._transforms(imgs, target)
            imgs = torch.stack(imgs, dim=0)  # [T, 3, H, W]

            # FIXME: handle "valid", since some box may be removed due to random crop
            if torch.any(target['valid'] == 1):  # at leatst one instance
                instance_check = True
            else:
                idx = random.randint(0, self.__len__() - 1)

        return imgs, target

def make_coco_transforms(image_set, max_size=640):
    normalize = T.Compose([
        T.ToTensor(),
        T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

    scales = [288, 320, 352, 392, 416, 448, 480, 512]

    if image_set == 'train':
        return T.Compose([
            T.RandomHorizontalFlip(),
            T.PhotometricDistort(),
            T.RandomSelect(
                T.Compose([
                    T.RandomResize(scales, max_size=max_size),
                    T.Check(),
                ]),
                T.Compose([
                    T.RandomResize([400, 500, 600]),
                    T.RandomSizeCrop(384, 600),
                    T.RandomResize(scales, max_size=max_size),
                    T.Check(),
                ])
            ),
            normalize,
        ])

    # we do not use the 'val' set since the annotations are inaccessible
    if image_set == 'val':
        return T.Compose([
            T.RandomResize([360], max_size=640),
            normalize,
        ])

    raise ValueError(f'unknown {image_set}')

def build(image_set, args):
    root = Path(args.rovos_path)
    assert root.exists(), f'provided YTVOS path {root} does not exist'
    PATHS = {
        "train": (root / "train", root / "train" / "meta_expressions.json"),
        "val": (root / "valid", root / "meta_expressions" / "val" / "meta_expressions.json"),  # not used actually
    }
    img_folder, ann_file = PATHS['train']
    dataset = ROVOSDataset(img_folder, ann_file, transforms=make_coco_transforms(image_set, max_size=args.max_size), return_masks=args.masks,
                           num_frames=args.num_frames, max_skip=args.max_skip)
    return dataset

if __name__ == '__main__':
    root = Path('data/mevis_release')
    image_set = 'train'
    PATHS = {
        "train": (root / "train", root / "train" / "meta_expressions.json"),
    }
    img_folder, ann_file = PATHS['train']

    dataset = ROVOSDataset(img_folder, ann_file, transforms=T.ToTensor(), return_masks=True, num_frames=5, max_skip=3)

    img, meta = dataset[0]


# -*- coding: utf-8 -*-

import xml.etree.ElementTree as ET

import cv2
import numpy as np
from const import DEFAULT_WIDTH, IMAGE_SIZE
from rendering import render_entity

class Bunch:
    """Dummy class"""
    def __init__(self, **kwds):
        self.__dict__.update(kwds)

def get_real_bbox(entity_bbox, entity_type, entity_size, entity_angle):
    assert entity_type != "none"
    center = (int(entity_bbox[1] * IMAGE_SIZE), int(entity_bbox[0] * IMAGE_SIZE))
    M = cv2.getRotationMatrix2D(center, entity_angle, 1)
    unit = min(entity_bbox[2], entity_bbox[3]) * IMAGE_SIZE / 2
    delta = DEFAULT_WIDTH * 1.5 / IMAGE_SIZE
    if entity_type == "circle":
        radius = unit * entity_size
        real_bbox = [center[1] * 1.0 / IMAGE_SIZE, center[0] * 1.0 / IMAGE_SIZE, 2 * radius / IMAGE_SIZE + delta, 2 * radius / IMAGE_SIZE + delta]
    else:
        if entity_type == "triangle":
            dl = int(unit * entity_size)
            homo_pts = np.array([[center[0], center[1] - dl, 1],
                                 [center[0] + int(dl / 2.0 * np.sqrt(3)), center[1] + int(dl / 2.0), 1],
                                 [center[0] - int(dl / 2.0 * np.sqrt(3)), center[1] + int(dl / 2.0), 1]],
                                np.int32)
        if entity_type == "square":
            dl = int(unit / 2 * np.sqrt(2) * entity_size)
            homo_pts = np.array([[center[0] - dl, center[1] - dl, 1],
                                 [center[0] - dl, center[1] + dl, 1],
                                 [center[0] + dl, center[1] + dl, 1],
                                 [center[0] + dl, center[1] - dl, 1]],
                                np.int32)
        if entity_type == "pentagon":
            dl = int(unit * entity_size)
            homo_pts = np.array([[center[0], center[1] - dl, 1],
                                 [center[0] - int(dl * np.cos(np.pi / 10)), center[1] - int(dl * np.sin(np.pi / 10)), 1],
                                 [center[0] - int(dl * np.sin(np.pi / 5)), center[1] + int(dl * np.cos(np.pi / 5)), 1],
                                 [center[0] + int(dl * np.sin(np.pi / 5)), center[1] + int(dl * np.cos(np.pi / 5)), 1],
                                 [center[0] + int(dl * np.cos(np.pi / 10)), center[1] - int(dl * np.sin(np.pi / 10)), 1]],
                                np.int32)
        if entity_type == "hexagon":
            dl = int(unit * entity_size)
            homo_pts = np.array([[center[0], center[1] - dl, 1],
                                 [center[0] - int(dl / 2.0 * np.sqrt(3)), center[1] - int(dl / 2.0), 1],
                                 [center[0] - int(dl / 2.0 * np.sqrt(3)), center[1] + int(dl / 2.0), 1],
                                 [center[0], center[1] + dl, 1],
                                 [center[0] + int(dl / 2.0 * np.sqrt(3)), center[1] + int(dl / 2.0), 1],
                                 [center[0] + int(dl / 2.0 * np.sqrt(3)), center[1] - int(dl / 2.0), 1]],
                                np.int32)
        after_pts = np.dot(M, homo_pts.T)
        min_x = min(after_pts[1, :]) / IMAGE_SIZE
        max_x = max(after_pts[1, :]) / IMAGE_SIZE
        min_y = min(after_pts[0, :]) / IMAGE_SIZE
        max_y = max(after_pts[0, :]) / IMAGE_SIZE
        real_bbox = [(min_x + max_x) / 2, (min_y + max_y) / 2, max_x - min_x + delta, max_y - min_y + delta]
    return list(np.round(real_bbox, 4))

def get_mask(entity_bbox, entity_type, entity_size, entity_angle):
    dummy_entity = Bunch()
    dummy_entity.bbox = entity_bbox
    dummy_entity.type = Bunch(get_value=lambda : entity_type)
    dummy_entity.size = Bunch(get_value=lambda : entity_size)
    dummy_entity.color = Bunch(get_value=lambda : 0)
    dummy_entity.angle = Bunch(get_value=lambda : entity_angle)
    mask = render_entity(dummy_entity) / 255
    return mask

# ref: https://www.kaggle.com/stainsby/fast-tested-rle
# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode
def rle_encode(img):
    '''
    img: numpy array, 1 - mask, 0 - background
    Returns run length as string formated
    '''
    pixels = img.flatten()
    pixels = np.concatenate([[0], pixels, [0]])
    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
    runs[1::2] -= runs[::2]
    return "[" + ",".join(str(x) for x in runs) + "]"

def rle_decode(mask_rle, shape):
    '''
    mask_rle: run-length as string formated (start length)
    shape: (height,width) of array to return
    Returns numpy array, 1 - mask, 0 - background
    '''
    s = mask_rle[1:-1].split(",")
    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]
    starts -= 1
    ends = starts + lengths
    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)
    for lo, hi in zip(starts, ends):
        img[lo:hi] = 1
    return img.reshape(shape)


"""
Helper for drawing convolutional neural network weights.

This may be removed in the future.
"""
import numpy as np
import ubelt as ub
try:
    import torch  # NOQA
except ImportError:
    print('warning: torch not available')
    torch = None

def make_conv_images(conv, color=None, norm_per_feat=True):
    """
    Convert convolutional weights to a list of visualize-able images

    Args:
        conv (torch.nn.Conv2d): a torch convolutional layer
        color (bool): if True output images are colorized
        norm_per_feat (bool): if True normalizes over each feature separately,
            otherwise normalizes all features together.

    Returns:
        ndarray:

    TODO:
        - [ ] better normalization options

    Example:
        >>> # xdoctest: +REQUIRES(module:torch)
        >>> conv = torch.nn.Conv2d(3, 9, (5, 7))
        >>> weights_tohack = conv.weight[0:7].data.numpy()
        >>> weights_flat = make_conv_images(conv, norm_per_feat=False)
        >>> # xdoctest: +REQUIRES(--show)
        >>> import kwimage
        >>> import kwplot
        >>> stacked = kwimage.stack_images_grid(weights_flat, chunksize=5, overlap=-1)
        >>> kwplot.imshow(stacked)
        >>> kwplot.show_if_requested()

    Ignore:
        >>> # xdoctest: +REQUIRES(module:torch)
        >>> import torchvision
        >>> conv = torchvision.models.resnet50(pretrained=True).conv1
        >>> #conv = torchvision.models.vgg11(pretrained=True).features[0]
        >>> weights_flat = make_conv_images(conv, norm_per_feat=False)
        >>> # xdoctest: +REQUIRES(--show)
        >>> import kwplot
        >>> import kwimage
        >>> stacked = kwplot.stack_images_grid(weights_flat, chunksize=8, overlap=-1)
        >>> kwplot.autompl()
        >>> kwplot.imshow(stacked)
        >>> kwplotwil.show_if_requested()
    """
    # get relavent data out of pytorch module
    weights = conv.weight.data.cpu().numpy()
    in_channels = conv.in_channels
    # out_channels = conv.out_channels
    spatial_dims = list(conv.kernel_size)
    n_space_dims = len(spatial_dims)

    if color is None:
        # color if possible
        color = (in_channels == 3)

    # Normalize layer weights between 0 and 1
    if norm_per_feat:
        # if normaxis=0 norm over output channels
        minval = weights.min(axis=(0, 1), keepdims=True)
        maxval = weights.max(axis=(0, 1), keepdims=True)
    else:
        minval = weights.min()
        maxval = weights.max()

    weights_norm = (weights - minval) / (maxval - minval)

    # If there are 3 input channels, we can visualize features in a colorspace
    if color:
        # Move colorable channels to the end (handle 1, 2 and 3d convolution)
        ifeat_axes = [0]
        color_axes = [1]
        space_axes = list(range(2, 2 + n_space_dims))
        axes = ifeat_axes + space_axes + color_axes
        weights_norm = weights_norm.transpose(*axes)
        color_dims = [in_channels]
    else:
        color_dims = []

    # flatten all non-spacetime/color dimensions
    weights_flat = weights_norm.reshape(-1, *(spatial_dims + color_dims))
    return weights_flat

def plot_convolutional_features(conv, limit=144, colorspace='rgb', fnum=None,
                                nCols=None, voxels=False, alpha=.2,
                                labels=False, normaxis=None, _hack_2drows=False):
    """Plots the convolutional layers to a matplotlib pyplot.

    The convolutional filters (kernels) are stored into a grid and saved to disk
    as a Maplotlib figure.  The convolutional filters, if it has one channel,
    will be stored as an intensity imgage.  If a colorspace is specified and
    there are three input channels, the convolutional filters will be
    represented as an RGB image.

    In the event that 2 or 4+ filters are
    displayed, the different channels will be flattened and showed as distinct
    outputs in the grid.

    TODO:
        - [ ] refactor to use make_conv_images

    Args:
        conv (torch.nn.modules.conv._ConvNd): torch convolutional layer with weights to draw

        limit (int): the limit on the number of filters drawn in the
            figure, achieved by simply dropping any filters past the limit
            starting at the first filter.  Detaults to 144.

        colorspace (str): the colorspace seen by the convolutional filter
            (if applicable), so we can convert to rgb for display.

        voxels (bool): if True, and we have a 3d conv, show the voxels

        alpha (float): only applicable if voxels=True

        stride (list): only applicable if voxels=True

    Returns:
        matplotlib.figure.Figure: fig - a Matplotlib figure

    References:
        https://matplotlib.org/devdocs/gallery/mplot3d/voxels.html

    Example:
        >>> # xdoctest: +REQUIRES(module:torch)
        >>> conv = torch.nn.Conv2d(3, 9, (5, 7))
        >>> plot_convolutional_features(conv, colorspace=None, fnum=None, limit=2)

    Example:
        >>> # xdoctest: +REQUIRES(--comprehensive)
        >>> # xdoctest: +REQUIRES(module:torch)
        >>> import torchvision
        >>> # 2d uncolored gray-images
        >>> conv = torch.nn.Conv3d(1, 2, (3, 4, 5))
        >>> plot_convolutional_features(conv, colorspace=None, fnum=1, limit=2)

        >>> # 2d colored rgb-images
        >>> conv = torch.nn.Conv3d(3, 2, (6, 4, 5))
        >>> plot_convolutional_features(conv, colorspace='rgb', fnum=1, limit=2)

        >>> # 2d uncolored rgb-images
        >>> conv = torch.nn.Conv3d(3, 2, (6, 4, 5))
        >>> plot_convolutional_features(conv, colorspace=None, fnum=1, limit=2)

        >>> # 3d gray voxels
        >>> conv = torch.nn.Conv3d(1, 2, (6, 4, 5))
        >>> plot_convolutional_features(conv, colorspace=None, fnum=1, voxels=True,
        >>>                             limit=2)

        >>> # 3d color voxels
        >>> conv = torch.nn.Conv3d(3, 2, (6, 4, 5))
        >>> plot_convolutional_features(conv, colorspace='rgb', fnum=1,
        >>>                             voxels=True, alpha=1, limit=3)

        >>> # hack the nice resnet weights into 3d-space
        >>> # xdoctest: +REQUIRES(--network)
        >>> import torchvision
        >>> model = torchvision.models.resnet50(pretrained=True)
        >>> conv = torch.nn.Conv3d(3, 1, (7, 7, 7))
        >>> weights_tohack = model.conv1.weight[0:7].data.numpy()
        >>> # normalize each weight for nice colors, then place in the conv3d
        >>> for w in weights_tohack:
        ...     w[:] = (w - w.min()) / (w.max() - w.min())
        >>> weights_hacked = weights_tohack.transpose(1, 0, 2, 3)[None, :]
        >>> conv.weight.data[:] = torch.FloatTensor(weights_hacked)

        >>> plot_convolutional_features(conv, colorspace='rgb', fnum=1, voxels=True, alpha=.6)

        >>> plot_convolutional_features(conv, colorspace='rgb', fnum=2, voxels=False, alpha=.9)

    Example:
        >>> # xdoctest: +REQUIRES(--network)
        >>> # xdoctest: +REQUIRES(module:torch)
        >>> import torchvision
        >>> model = torchvision.models.resnet50(pretrained=True)
        >>> conv = model.conv1
        >>> plot_convolutional_features(conv, colorspace='rgb', fnum=None)

    """
    import kwplot
    kwplot.autompl()
    import matplotlib.pyplot as plt

    # get relavent data out of pytorch module
    weights = conv.weight.data.cpu().numpy()
    in_channels = conv.in_channels
    # out_channels = conv.out_channels
    kernel_size = conv.kernel_size
    conv_dim = len(kernel_size)

    # TODO: use make_conv_images in the 2d case here

    if voxels:
        # use up to 3 spatial dimensions
        spatial_axes = list(kernel_size[-3:])
    else:
        # use only 2 spatial dimensions
        spatial_axes = list(kernel_size[-2:])
    color_axes = []

    output_axis = 0

    # If there are 3 input channels, we can visualize features in a colorspace
    if colorspace is not None and in_channels == 3:
        # Move colorable channels to the end (handle 1, 2 and 3d convolution)
        axes = [0] + list(range(2, 2 + conv_dim)) + [1]
        weights = weights.transpose(*axes)
        color_axes = [in_channels]
        output_axis = 0
    else:
        pass

    # Normalize layer weights between 0 and 1
    if normaxis is None:
        minval = weights.min()
        maxval = weights.max()
    else:
        # if normaxis=0 norm over output channels
        minval = weights.min(axis=output_axis, keepdims=True)
        maxval = weights.max(axis=output_axis, keepdims=True)

    weights_norm = (weights - minval) / (maxval - minval)

    if _hack_2drows:
        # To agree with jason's visualization for a paper figure
        if not voxels:
            weights_norm = weights_norm.transpose(1, 0, 2, 3)

    # flatten everything but the spatial and requested color dims
    weights_flat = weights_norm.reshape(-1, *(spatial_axes + color_axes))

    num_plots = min(weights_flat.shape[0], limit)
    dim = int(np.ceil(np.sqrt(num_plots)))

    if voxels:
        from mpl_toolkits.mplot3d import Axes3D  # NOQA
        filled = np.ones(spatial_axes, dtype=np.bool)
        # np.ones(spatial_axes)
        # d, h, w = np.indices(spatial_axes)

    fnum = kwplot.ensure_fnum(fnum)
    fig = kwplot.figure(fnum=fnum)
    fig.clf()
    if nCols is None:
        nCols = dim
    pnum_ = kwplot.PlotNums(nCols=nCols, nSubplots=num_plots)

    def plot_kernel3d(i):
        img = weights_flat[i]

        # fig = kwplot.figure(fnum=fnum, pnum=pnum_[i])
        ax = fig.add_subplot(*pnum_[i], projection='3d')
        # ax = fig.gca(projection='3d')

        alpha_ = (filled * alpha)[..., None]
        colors = img

        if not color_axes:
            import kwimage
            # transform grays into colors
            grays = kwimage.atleast_nd(img, 4)
            colors = np.concatenate([grays, grays, grays], axis=3)

        if colorspace and color_axes:
            import kwimage
            # convert into RGB
            for d in range(len(colors)):
                colors[d] = kwimage.convert_colorspace(colors[d],
                                                       src_space=colorspace,
                                                       dst_space='rgb')
        facecolors = np.concatenate([colors, alpha_], axis=3)

        # shuffle dims so height is upwards and depth move away from us.
        dim_labels = ['d', 'h', 'w']
        axes = [2, 0, 1]

        dim_labels = list(ub.take(dim_labels, axes))
        facecolors = facecolors.transpose(*(axes + [3]))
        filled_ = filled.transpose(*axes)
        spatial_axes_ = list(ub.take(spatial_axes, axes))

        # ax.voxels(filled_, facecolors=facecolors, edgecolors=facecolors)
        if False:
            ax.voxels(filled_, facecolors=facecolors, edgecolors='k')
        else:
            # hack to show "occluded" voxels
            # stride = [1, 3, 1]
            stride = [2, 2, 2]
            slices = tuple(slice(None, None, s) for s in stride)
            spatial_axes2 = list(np.array(spatial_axes_) * stride)
            filled2 = np.zeros(spatial_axes2, dtype=np.bool)
            facecolors2 = np.empty(spatial_axes2 + [4], dtype=np.float32)
            filled2[slices] = filled_
            facecolors2[slices] = facecolors
            edgecolors2 = [0, 0, 0, alpha]
            # 'k'
            # edgecolors2 = facecolors2

            # Shrink the gaps, which let you see occluded voxels
            x, y, z = np.indices(np.array(filled2.shape) + 1).astype(float) // 2
            x[0::2, :, :] += 0.05
            y[:, 0::2, :] += 0.05
            z[:, :, 0::2] += 0.05
            x[1::2, :, :] += 0.95
            y[:, 1::2, :] += 0.95
            z[:, :, 1::2] += 0.95

            ax.voxels(x, y, z, filled2, facecolors=facecolors2, edgecolors=edgecolors2)

        for xyz, dlbl in zip(['x', 'y', 'z'], dim_labels):
            getattr(ax, 'set_' + xyz + 'label')(dlbl)

        for xyz in ['x', 'y', 'z']:
            getattr(ax, 'set_' + xyz + 'ticks')([])

        ax.set_aspect('equal')
        if not labels or i < num_plots - 1:
            # show axis only on the last plot
            ax.grid(False)
            plt.axis('off')

    for i in ub.ProgIter(range(num_plots), desc='plot conv layer',
                         enabled=False):
        if voxels:
            plot_kernel3d(i)
        else:
            img = weights_flat[i]
            kwplot.imshow(img, fnum=fnum, pnum=pnum_[i],
                          interpolation='nearest', colorspace=colorspace)
    return fig


from django.shortcuts import render, redirect

from newExam.expense_app.forms import ExpenseForm, DeleteExpenseForm
from newExam.expense_app.models import Expense
from newExam.profile_app.models import Profile

def create_expense(request):
    if request.method == 'POST':
        form = ExpenseForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('home')
    else:
        form = ExpenseForm()
    context = {'form': form, 'profile': Profile.objects.first()}
    return render(request, 'expense-create.html', context)

def edit_expense(request, pk):
    expense = Expense.objects.get(pk=pk)
    if request.method == 'POST':
        form = ExpenseForm(request.POST, instance=expense)
        if form.is_valid():
            form.save()
            return redirect('home')
    else:
        form = ExpenseForm(instance=expense)
    context = {'form': form, 'profile': Profile.objects.first()}
    return render(request, 'expense-create.html', context)

def delete_expense(request, pk):
    expense = Expense.objects.get(pk=pk)
    if request.method == 'POST':
        expense.delete()
        return redirect('home')
    else:
        form = DeleteExpenseForm(instance=expense)
    context = {'form': form, 'profile': Profile.objects.first()}
    return render(request, 'expense-delete.html', context)


# -*- coding: utf-8 -*-
r"""
Interfaces to R

This is the reference to the Sagemath R interface, usable from any
Sage program.

The %r interface creating an R cell in the sage
notebook is decribed in the Notebook manual.

The %R and %%R interface creating an R line or an R cell in the
Jupyter notebook are briefly decribed at the end of this page. This
documentation will be expanded and placed in the Jupyter notebook
manual when this manual exists.

The following examples try to follow "An Introduction to R" which can
be found at http://cran.r-project.org/doc/manuals/R-intro.html .

EXAMPLES:

Simple manipulations; numbers and vectors

The simplest data structure in R is the numeric vector which
consists of an ordered collection of numbers.  To create a
vector named $x$ using the R interface in Sage, you pass the
R interpreter object a list or tuple of numbers::

    sage: x = r([10.4,5.6,3.1,6.4,21.7]); x
    [1] 10.4  5.6  3.1  6.4 21.7

You can invert elements of a vector x in R by using the
invert operator or by doing 1/x::

    sage: ~x
    [1] 0.09615385 0.17857143 0.32258065 0.15625000 0.04608295
    sage: 1/x
    [1] 0.09615385 0.17857143 0.32258065 0.15625000 0.04608295

The following assignment creates a vector $y$ with 11 entries which
consists of two copies of $x$ with a 0 in between::

    sage: y = r([x,0,x]); y
    [1] 10.4  5.6  3.1  6.4 21.7  0.0 10.4  5.6  3.1  6.4 21.7

Vector Arithmetic

The following command generates a new vector $v$ of length 11 constructed
by adding together (element by element) $2x$ repeated 2.2 times, $y$
repeated just once, and 1 repeated 11 times::

    sage: v = 2*x+y+1; v
    [1] 32.2 17.8 10.3 20.2 66.1 21.8 22.6 12.8 16.9 50.8 43.5

One can compute the sum of the elements of an R vector in the following
two ways::

    sage: sum(x)
    [1] 47.2
    sage: x.sum()
    [1] 47.2

One can calculate the sample variance of a list of numbers::

    sage: ((x-x.mean())^2/(x.length()-1)).sum()
    [1] 53.853
    sage: x.var()
    [1] 53.853

    sage: x.sort()
    [1] 3.1  5.6  6.4 10.4 21.7
    sage: x.min()
    [1] 3.1
    sage: x.max()
    [1] 21.7
    sage: x
    [1] 10.4  5.6  3.1  6.4 21.7

    sage: r(-17).sqrt()
    [1] NaN
    sage: r('-17+0i').sqrt()
    [1] 0+4.123106i

Generating an arithmetic sequence::

    sage: r('1:10')
    [1] 1  2  3  4  5  6  7  8  9 10

Because ``from`` is a keyword in Python, it can't be used
as a keyword argument.  Instead, ``from_`` can be passed, and
R will recognize it as the correct thing::

    sage: r.seq(length=10, from_=-1, by=.2)
    [1] -1.0 -0.8 -0.6 -0.4 -0.2  0.0  0.2  0.4  0.6  0.8

    sage: x = r([10.4,5.6,3.1,6.4,21.7])
    sage: x.rep(2)
    [1] 10.4  5.6  3.1  6.4 21.7 10.4  5.6  3.1  6.4 21.7
    sage: x.rep(times=2)
    [1] 10.4  5.6  3.1  6.4 21.7 10.4  5.6  3.1  6.4 21.7
    sage: x.rep(each=2)
    [1] 10.4 10.4  5.6  5.6  3.1  3.1  6.4  6.4 21.7 21.7

Missing Values::

    sage: na = r('NA')
    sage: z = r([1,2,3,na])
    sage: z
    [1]  1  2  3 NA
    sage: ind = r.is_na(z)
    sage: ind
    [1] FALSE FALSE FALSE  TRUE
    sage: zero = r(0)
    sage: zero / zero
    [1] NaN
    sage: inf = r('Inf')
    sage: inf-inf
    [1] NaN
    sage: r.is_na(inf)
    [1] FALSE
    sage: r.is_na(inf-inf)
    [1] TRUE
    sage: r.is_na(zero/zero)
    [1] TRUE
    sage: r.is_na(na)
    [1] TRUE
    sage: r.is_nan(inf-inf)
    [1] TRUE
    sage: r.is_nan(zero/zero)
    [1] TRUE
    sage: r.is_nan(na)
    [1] FALSE

Character Vectors::

    sage: labs = r.paste('c("X","Y")', '1:10', sep='""'); labs
    [1] "X1"  "Y2"  "X3"  "Y4"  "X5"  "Y6"  "X7"  "Y8"  "X9"  "Y10"

Index vectors; selecting and modifying subsets of a data set::

    sage: na = r('NA')
    sage: x = r([10.4,5.6,3.1,6.4,21.7,na]); x
    [1] 10.4  5.6  3.1  6.4 21.7   NA
    sage: x['!is.na(self)']
    [1] 10.4  5.6  3.1  6.4 21.7

    sage: x = r([10.4,5.6,3.1,6.4,21.7,na]); x
    [1] 10.4  5.6  3.1  6.4 21.7   NA
    sage: (x+1)['(!is.na(self)) & self>0']
    [1] 11.4  6.6  4.1  7.4 22.7
    sage: x = r([10.4,-2,3.1,-0.5,21.7,na]); x
    [1] 10.4 -2.0  3.1 -0.5 21.7   NA
    sage: (x+1)['(!is.na(self)) & self>0']
    [1] 11.4  4.1  0.5 22.7

Distributions::

    sage: r.options(width="60")
    $width
    [1] 100

    sage: rr = r.dnorm(r.seq(-3,3,0.1))
    sage: rr
     [1] 0.004431848 0.005952532 0.007915452 0.010420935
     [5] 0.013582969 0.017528300 0.022394530 0.028327038
     [9] 0.035474593 0.043983596 0.053990967 0.065615815
    [13] 0.078950158 0.094049077 0.110920835 0.129517596
    [17] 0.149727466 0.171368592 0.194186055 0.217852177
    [21] 0.241970725 0.266085250 0.289691553 0.312253933
    [25] 0.333224603 0.352065327 0.368270140 0.381387815
    [29] 0.391042694 0.396952547 0.398942280 0.396952547
    [33] 0.391042694 0.381387815 0.368270140 0.352065327
    [37] 0.333224603 0.312253933 0.289691553 0.266085250
    [41] 0.241970725 0.217852177 0.194186055 0.171368592
    [45] 0.149727466 0.129517596 0.110920835 0.094049077
    [49] 0.078950158 0.065615815 0.053990967 0.043983596
    [53] 0.035474593 0.028327038 0.022394530 0.017528300
    [57] 0.013582969 0.010420935 0.007915452 0.005952532
    [61] 0.004431848

Convert R Data Structures to Python/Sage::

    sage: rr = r.dnorm(r.seq(-3,3,0.1))
    sage: sum(rr._sage_())
    9.9772125168981...

Or you get a dictionary to be able to access all the information::

    sage: rs = r.summary(r.c(1,4,3,4,3,2,5,1))
    sage: rs
       Min. 1st Qu.  Median    Mean 3rd Qu.    Max.
      1.000   1.750   3.000   2.875   4.000   5.000
      sage: d = rs._sage_()
      sage: d['DATA']
      [1, 1.75, 3, 2.875, 4, 5]
      sage: d['_Names']
      ['Min.', '1st Qu.', 'Median', 'Mean', '3rd Qu.', 'Max.']
      sage: d['_r_class']
      ['summaryDefault', 'table']

It is also possible to access the plotting capabilities of R
through Sage.  For more information see the documentation of
r.plot() or r.png().

THE JUPYTER NOTEBOOK INTERFACE (work in progress).

The %r interface described in the Sage notebook manual is not useful
in the Jupyter notebook : it creates a inferior R interpreter which
cannot be escaped.

The RPy2 library allows the creation of an R cell in the Jupyter
notebook analogous to the %r escape in command line or %r cell in a
Sage notebook.

The interface is loaded by a cell containing the sole code :

"%load_ext rpy2.ipython"

After execution of this code, the %R and %%R magics are available :

- %R allows the execution of a single line of R code. Data exchange is
   possible via the -i and -o options. Do "%R?" in a standalone cell
   to get the documentation.

- %%R allows the execution in R of the whole text of a cell, with
    similar options (do "%%R?" in a standalone cell for
    documentation).

A few important points must be noted :

- The R interpreter launched by this interface IS (currently)
  DIFFERENT from the R interpreter used br other r... functions.

- Data exchanged via the -i and -o options have a format DIFFERENT
  from the format used by the r... functions (RPy2 mostly uses arrays,
  and bugs the user to use the pandas Python package).

- R graphics are (beautifully) displayed in output cells, but are not
  directly importable. You have to save them as .png, .pdf or .svg
  files and import them in Sage for further use.

In its current incarnation, this interface is mostly useful to
statisticians needing Sage for a few symbolic computations but mostly
using R for applied work.

AUTHORS:

- Mike Hansen (2007-11-01)
- William Stein (2008-04-19)
- Harald Schilly (2008-03-20)
- Mike Hansen (2008-04-19)
- Emmanuel Charpentier (2015-12-12, RPy2 interface)
"""

##########################################################################
#
#       Copyright (C) 2007 William Stein <wstein@gmail.com>
#                     2007 Mike Hansen   <mhansen@gmail.com>
#                     2008 Harald Schilly <harald.schilly@gmail.com>
#
#  Distributed under the terms of the GNU General Public License (GPL)
#
#                  http://www.gnu.org/licenses/
#
##########################################################################
from __future__ import print_function, absolute_import
from six.moves import range
import six

from .expect import Expect, ExpectElement, ExpectFunction, FunctionElement
from sage.env import DOT_SAGE
import re
import sage.rings.integer
from sage.structure.element import parent
from sage.misc.cachefunc import cached_method
from sage.interfaces.tab_completion import ExtraTabCompletion
from sage.docs.instancedoc import instancedoc

COMMANDS_CACHE = '%s/r_commandlist.sobj'%DOT_SAGE
PROMPT = '__SAGE__R__PROMPT__> '
prompt_re = re.compile("^>", re.M)

#there is a mirror network, but lets take #1 for now
RRepositoryURL = "http://cran.r-project.org/"
RFilteredPackages = ['.GlobalEnv']

# crosscheck with https://svn.r-project.org/R/trunk/src/main/names.c
# but package:base should cover this. i think.
RBaseCommands = ['c', "NULL", "NA", "True", "False", "Inf", "NaN"]

class R(ExtraTabCompletion, Expect):
    def __init__(self,
                 maxread=None, script_subdirectory=None,
                 server_tmpdir = None,
                 logfile=None,
                 server=None,
                 init_list_length=1024,
                 seed=None):
        """
        An interface to the R interpreter.

        R is a comprehensive collection of methods for statistics,
        modelling, bioinformatics, data analysis and much more.
        For more details, see http://www.r-project.org/about.html

        Resources:

        * http://r-project.org/ provides more information about R.
        * http://rseek.org/ R's own search engine.

        EXAMPLES::

             sage: r.summary(r.c(1,2,3,111,2,3,2,3,2,5,4))
             Min. 1st Qu.  Median    Mean 3rd Qu.    Max.
             1.00    2.00    3.00   12.55    3.50  111.00

        TESTS::

            sage: r == loads(dumps(r))
            True
        """
        Expect.__init__(self,

                  # The capitalized version of this is used for printing.
                  name = 'r',

                  # This is regexp of the input prompt.  If you can change
                  # it to be very obfuscated that would be better.   Even
                  # better is to use sequence numbers.
                  # options(prompt=\"<prompt> \")
                  prompt = '> ', #default, later comes the change

                  # This is the command that starts up your program
                  # See #25806 for the --no-readline switch which fixes hangs for some
                  command = "R --no-readline --vanilla --quiet",

                  server=server,
                  server_tmpdir=server_tmpdir,

                  script_subdirectory = script_subdirectory,

                  # If this is true, then whenever the user presses Control-C to
                  # interrupt a calculation, the whole interface is restarted.
                  restart_on_ctrlc = False,

                  # If true, print out a message when starting
                  # up the command when you first send a command
                  # to this interface.
                  verbose_start = False,

                  logfile=logfile,

                  # If an input is longer than this number of characters, then
                  # try to switch to outputting to a file.
                  eval_using_file_cutoff=1024)

        self.__seq = 0
        self.__var_store_len = 0
        self.__init_list_length = init_list_length
        self._prompt_wait = [self._prompt]
        self._seed = seed

    def set_seed(self, seed=None):
        """
        Set the seed for R interpreter.

        The seed should be an integer.

        EXAMPLES::

            sage: r = R()
            sage: r.set_seed(1)
            1
            sage: r.sample("1:10", 5)
            [1] 3 4 5 7 2
        """
        if seed is None:
            seed = self.rand_seed()
        self.eval('set.seed(%d)' % seed)
        self._seed = seed
        return seed

    def _start(self):
        """
        Start up the R interpreter and sets the initial prompt and options.

        This is called the first time the R interface is actually used.

        EXAMPLES::

            sage: r = R()
            sage: r._start()
        """
        Expect._start(self)

        # width is line width, what's a good value? maximum is 10000!
        # pager needed to replace help view from less to printout
        # option device= is for plotting, is set to x11, NULL would be better?
        self._change_prompt(PROMPT)
        self.eval('options(prompt=\"%s\",continue=\"%s\", width=100,pager="cat",device="png")'%(PROMPT, PROMPT))
        self.expect().expect(PROMPT)
        self.eval('options(repos="%s")'%RRepositoryURL)
        self.eval('options(CRAN="%s")'%RRepositoryURL)

        # don't abort on errors, just raise them!
        # necessary for non-interactive execution
        self.eval('options(error = expression(NULL))')

        # set random seed
        self.set_seed(self._seed)

    def png(self, *args, **kwds):
        """
        Creates an R PNG device.

        This should primarily be used to save an R graphic to a custom file.  Note
        that when using this in the notebook, one must plot in the same cell that
        one creates the device.  See r.plot() documentation for more information
        about plotting via R in Sage.

        These examples won't work on the many platforms where R still gets
        built without graphics support.

        EXAMPLES::

            sage: filename = tmp_filename() + '.png'
            sage: r.png(filename='"%s"'%filename)             # optional -- rgraphics
            NULL
            sage: x = r([1,2,3])
            sage: y = r([4,5,6])
            sage: r.plot(x,y) # This saves to filename, but is not viewable from command line; optional -- rgraphics
            null device
                      1
            sage: import os; os.unlink(filename) # We remove the file for doctesting; optional -- rgraphics

        We want to make sure that we actually can view R graphics, which happens
        differently on different platforms::

            sage: s = r.eval('capabilities("png")') # Should be on Linux and Solaris
            sage: t = r.eval('capabilities("aqua")') # Should be on all supported Mac versions
            sage: "TRUE" in s+t                      # optional -- rgraphics
            True
        """
        #Check to see if R has PNG support
        s = self.eval('capabilities("png")')
        t = r.eval('capabilities("aqua")')
        if "TRUE" not in s+t:
            raise RuntimeError("R was not compiled with PNG support")

        from sage.server.support import EMBEDDED_MODE
        if EMBEDDED_MODE:
            self.setwd('"%s"'%os.path.abspath('.'))
        return RFunction(self, 'png')(*args, **kwds)

    def convert_r_list(self, l):
        r"""
        Converts an R list to a Python list.

        EXAMPLES::

            sage: s = 'c(".GlobalEnv", "package:stats", "package:graphics", "package:grDevices", \n"package:utils", "package:datasets", "package:methods", "Autoloads", \n"package:base")'
            sage: r.convert_r_list(s)
            ['.GlobalEnv',
             'package:stats',
             'package:graphics',
             'package:grDevices',
             'package:utils',
             'package:datasets',
             'package:methods',
             'Autoloads',
             'package:base']
        """
        pl = []
        l = "".join(l.split("\n"))
        l = l[2:len(l)-1]
        for l in l.split(","):
            pl.append(l.strip().strip('"'))
        return pl

    def install_packages(self, package_name):
        """
        Install an R package into Sage's R installation.

        EXAMPLES::

            sage: r.install_packages('aaMI')       # not tested
            ...
            R is free software and comes with ABSOLUTELY NO WARRANTY.
            You are welcome to redistribute it under certain conditions.
            Type 'license()' or 'licence()' for distribution details.
            ...
            Please restart Sage in order to use 'aaMI'.
        """
        cmd = """options(repos="%s"); install.packages("%s")"""%(RRepositoryURL, package_name)
        os.system("time echo '%s' | R --vanilla"%cmd)
        print("Please restart Sage in order to use '%s'." % package_name)

    def _repr_(self):
        """
        Return string representation of this R interface.

        EXAMPLES::

            sage: r                 # indirect doctest
            R Interpreter
        """
        return 'R Interpreter'

    def __reduce__(self):
        """
        Used in serializing an R interface.

        EXAMPLES::

            sage: rlr, t = r.__reduce__()
            sage: rlr(*t)
            R Interpreter
        """
        return reduce_load_R, tuple([])

    def __getattr__(self, attrname):
        """
        Called when you get an attribute of the R interface.  This
        manufactures an R function, which is a Python function that
        can then be called with various inputs.

        EXAMPLES::

            sage: c = r.c; c
            c
            sage: type(c)
            <class 'sage.interfaces.r.RFunction'>
        """
        if attrname[:1] == "_":
            raise AttributeError
        return RFunction(self, attrname)

    def _quit_string(self):
        r"""
        Return the string that when typed into R causes the R
        interpreter to exit.

        EXAMPLES::

            sage: r._quit_string()
            'quit(save="no")'
        """
        return 'quit(save="no")'

    def _read_in_file_command(self, filename):
        r"""
        Return the R command (as a string) to read in a file named
        filename into the R interpreter.

        EXAMPLES::

            sage: r._read_in_file_command('file.txt')
            'file=file("file.txt",open="r")\nsource(file)'
        """
        return 'file=file("%s",open="r")\nsource(file)'%filename

    def read(self, filename):
        r"""
        Read filename into the R interpreter by calling R's source function on a
        read-only file connection.

        EXAMPLES::

            sage: filename = tmp_filename()
            sage: f = open(filename, 'w')
            sage: _ = f.write('a <- 2+2\n')
            sage: f.close()
            sage: r.read(filename)
            sage: r.get('a')
            '[1] 4'
        """
        self.eval( self._read_in_file_command(filename) )

    def _install_hints(self):
        """
        EXAMPLES::

            sage: print(r._install_hints())
            R is currently installed with Sage.
        """
        return "R is currently installed with Sage.\n"

    def _source(self, s):
        """
        Returns the source code of an R function as a string.

        INPUT:

        - s -- the name of the function as a string

        EXAMPLES::

            sage: print(r._source("c"))
            function (...)  .Primitive("c")
        """
        if s[-2:] == "()":
            s = s[-2:]
        return self.eval('%s'%s)

    def source(self, s):
        """
        Display the R source (if possible) about the function named s.

        INPUT:

        - s -- a string representing the function whose source code you want to see

        OUTPUT: string -- source code

        EXAMPLES::

            sage: print(r.source("c"))
            function (...)  .Primitive("c")
        """
        return self._source(s)

    def version(self):
        """
        Return the version of R currently running.

        OUTPUT: tuple of ints; string

        EXAMPLES::

            sage: r.version() # not tested
            ((3, 0, 1), 'R version 3.0.1 (2013-05-16)')
            sage: rint, rstr = r.version()
            sage: rint[0] >= 3
            True
            sage: rstr.startswith('R version')
            True
        """
        major_re = re.compile(r'^major\s*(\d.*?)$', re.M)
        minor_re = re.compile(r'^minor\s*(\d.*?)$', re.M)
        version_string_re = re.compile(r'^version.string\s*(R.*?)$', re.M)

        s = self.eval('version')

        major = int( major_re.findall(s)[0].strip() )
        minor = tuple(int(i) for i in minor_re.findall(s)[0].strip().split(".") )
        version_string = version_string_re.findall(s)[0].strip()

        return ( (major,) + minor, version_string )

    def library(self, library_name):
        """
        Load the library library_name into the R interpreter.

        This function raises an ImportError if the given library
        is not known.

        INPUT:

        - library_name -- string

        EXAMPLES::

            sage: r.library('grid')
            sage: 'grid' in r.eval('(.packages())')
            True
            sage: r.library('foobar')
            Traceback (most recent call last):
            ...
            ImportError: ...
        """
        ret = self.eval('require("%s")' % library_name)
        try:
            ret = ret.decode('utf-8')
        except UnicodeDecodeError:
            ret = ret.decode('latin-1')
        # try hard to parse the message string in a locale-independent way
        if ' library(' in ret:       # locale-independent key-word
            raise ImportError("%s"%ret)
        else:
            try:
                # We need to rebuild keywords!
                del self.__tab_completion
            except AttributeError:
                pass
            self._tab_completion(verbose=False, use_disk_cache=False)

    require = library #overwrites require

    def available_packages(self):
        """
        Returns a list of all available R package names.

        This list is not necessarily sorted.

        OUTPUT: list of strings

        .. note::

            This requires an internet connection. The CRAN server is
            that is checked is defined at the top of sage/interfaces/r.py.

        EXAMPLES::

            sage: ap = r.available_packages()   # optional - internet
            sage: len(ap) > 20                  # optional - internet
            True
        """
        p = self.new('available.packages("%s/src/contrib")'%RRepositoryURL)
        s = str(p).splitlines()[1:]
        v = [x.split()[0].strip("'") for x in s]
        return v
        #The following was more structural, but breaks on my machine.  (stein)
        #p = p._sage_()
        #s = p['_Dim'][0]
        #l = [[p['DATA'][i],p['DATA'][s+1+i]] for i in range(0,s)]
        #return l

    def _object_class(self):
        """
        Return the underlying class of elements of the R interface object.

        OUTPUT: a Python class

        EXAMPLES::

            sage: r._object_class()
            <class 'sage.interfaces.r.RElement'>
        """
        return RElement

    def _true_symbol(self):
        """
        Return the symbol that represents True in R.

        OUTPUT: string

        EXAMPLES::

            sage: r._true_symbol()
            '[1] TRUE'

        This is used behinds the scenes to implement comparison::

            sage: r('1') == r('1')
            [1] TRUE
            sage: r('1') == r('2')
            [1] FALSE
        """
        # return the string rep of truth, i.e., what the system outputs
        # when you type 1==1.
        return "[1] TRUE"

    def _false_symbol(self):
        """
        Return the symbol that represents True in R.

        OUTPUT: string

        EXAMPLES::

            sage: r._false_symbol()
            '[1] FALSE'
        """
        # return the string rep of false, i.e., what the system outputs
        # when you type 1==2.
        return "[1] FALSE"

    def _equality_symbol(self):
        """
        EXAMPLES::

            sage: r._equality_symbol()
            '=='
        """
        # return the symbol for checking equality, e.g., == or eq.
        return "=="

    def help(self, command):
        """
        Returns help string for a given command.

        INPUT:
        - command -- a string

        OUTPUT: HelpExpression -- a subclass of string whose __repr__ method is __str__, so it prints nicely

        EXAMPLES::

            sage: r.help('c')
            c                     package:base                     R Documentation
            ...

            .. note::

                This is similar to typing r.command?.
        """
        s = self.eval('help("%s")'%command).strip()     # ?cmd is only an unsafe shortcut
        import sage.plot.plot
        if sage.plot.plot.EMBEDDED_MODE:
            s = s.replace('_\x08','')
        return HelpExpression(s)

    def _assign_symbol(self):
        """
        Return the symbol used in R for assignment, which is ' <- '.

        OUTPUT: string

        EXAMPLES::

            sage: r._assign_symbol()
            ' <- '
        """
        return " <- "

    def _left_list_delim(self):
        """
        Return the left delimiter for lists in R, which is 'c('

        OUTPUT: string

        EXAMPLES::

            sage: r._left_list_delim()
            'c('
        """
        return "c("

    def _right_list_delim(self):
        """
        Return the right delimiter for lists in R, which is 'c('

        OUTPUT: string

        EXAMPLES::

            sage: r._right_list_delim()
            ')'
        """
        return ")"

    def console(self):
        """
        Runs the R console as a separate new R process.

        EXAMPLES::

            sage: r.console()                    # not tested
                R version 2.6.1 (2007-11-26)
                Copyright (C) 2007 The R Foundation for Statistical Computing
                ISBN 3-900051-07-0
                ...
        """
        r_console()

    def function_call(self, function, args=None, kwds=None):
        """
        Return the result of calling an R function, with given args and keyword args.

        OUTPUT: RElement -- an object in R

        EXAMPLES::

            sage: r.function_call('length', args=[ [1,2,3] ])
            [1] 3
        """
        args, kwds = self._convert_args_kwds(args, kwds)
        self._check_valid_function_name(function)
        return self.new("%s(%s)"%(function, ",".join([s.name() for s in args] +
                                                     [self._sage_to_r_name(key)+'='+kwds[key].name() for key in kwds ] )))

    def call(self, function_name, *args, **kwds):
        r"""
        This is an alias for :meth:`function_call`.

        EXAMPLES::

            sage: r.call('length', [1,2,3])
            [1] 3
        """
        return self.function_call(function_name, args=args, kwds=kwds)

    def _an_element_impl(self):
        """
        Returns an element belonging to the R interpreter.  This is used
        behind the scenes when doing things like comparisons, etc.

        OUTPUT: RElement -- an R element.

        EXAMPLES::

            sage: r._an_element_impl()
            [1] 0
            sage: type(_)
            <class 'sage.interfaces.r.RElement'>
        """
        return self(0)

    def set(self, var, value):
        """
        Set the variable var in R to what the string value evaluates to in R.

        INPUT:

        - var -- a string
        - value -- a string

        EXAMPLES::

            sage: r.set('a', '2 + 3')
            sage: r.get('a')
            '[1] 5'
        """
        cmd = '%s <- %s'%(var,value)
        out = self.eval(cmd)
        if out.find("error") != -1:
            raise TypeError("Error executing code in R\nCODE:\n\t%s\nR ERROR:\n\t%s"%(cmd, out))

    def get(self, var):
        """
        Returns the string representation of the variable var.

        INPUT:

        - var -- a string

        OUTPUT: string

        EXAMPLES::

            sage: r.set('a', 2)
            sage: r.get('a')
            '[1] 2'
        """
        s = self.eval('%s'%var)
        #return self._remove_indices_re.sub("", s).strip()
        return s

    def na(self):
        """
        Returns the NA in R.

        OUTPUT: RElement -- an element of R

        EXAMPLES::

            sage: r.na()
            [1] NA
        """
        return self('NA')

    def completions(self, s):
        """
        Return all commands names that complete the command starting with the
        string s.   This is like typing s[Ctrl-T] in the R interpreter.

        INPUT:

        - s -- string

        OUTPUT: list -- a list of strings

        EXAMPLES::

            sage: dummy = r._tab_completion(use_disk_cache=False)    #clean doctest
            sage: r.completions('tes')
            ['testInheritedMethods', 'testPlatformEquivalence', 'testVirtual']
        """
        return [name for name in self._tab_completion() if name[:len(s)] == s]

    def _commands(self):
        """
        Return list of all commands defined in R.

        OUTPUT: list -- a sorted list of strings

        EXAMPLES::

            sage: l = r._commands()
            sage: 'AIC' in l
            True
            sage: len(l) > 200
            True
        """
        v = RBaseCommands

        ll = self.eval('dput(search())') # loaded libs
        ll = self.convert_r_list(ll)

        for lib in ll:
            if lib in RFilteredPackages:
                continue

            if lib.find("package:") != 0:
                continue #only packages

            raw = self.eval('dput(objects("%s"))'%lib)
            raw = self.convert_r_list(raw)
            raw = [x.replace(".","_") for x in raw]

            #TODO are there others? many of them are shortcuts or
            #should be done on another level, like selections in lists
            #instead of calling obj.[[( fun-args) or other crazy stuff like that

            #TODO further filtering, check if strings are now
            #really functions, in R: exists(s, mode = "function"))
            # (apply to vector with sapply(vec,func))

            #filter only python compatible identifiers
            valid = re.compile('[^a-zA-Z0-9_]+')
            raw = [x for x in raw if valid.search(x) is None]
            v += raw

        v.sort()
        return v

    def _tab_completion(self, verbose=True, use_disk_cache=True):
        """
        Return list of all R functions.

        INPUT:

        - verbose -- bool (default: True); if True, display debugging information
        - use_disk_cache -- bool (default: True); if True, use the disk cache of
          tab completions to save time.

        OUTPUT: list -- list of string

        EXAMPLES::

            sage: t = r._tab_completion(verbose=False)
            sage: len(t) > 200
            True
        """
        try:
            return self.__tab_completion
        except AttributeError:
            import sage.misc.persist
            if use_disk_cache:
                try:
                    self.__tab_completion = sage.misc.persist.load(COMMANDS_CACHE)
                    return self.__tab_completion
                except IOError:
                    pass
            if verbose and use_disk_cache:
                print("\nBuilding R command completion list (this takes")
                print("a few seconds only the first time you do it).")
                print("To force rebuild later, delete %s." % COMMANDS_CACHE)
            v = self._commands()
            self.__tab_completion = v
            if len(v) > 200 and use_disk_cache:
                sage.misc.persist.save(v, COMMANDS_CACHE)
            return v

    def plot(self, *args, **kwds):
        """
        The R plot function.  Type r.help('plot') for much more extensive
        documentation about this function.  See also below for a brief
        introduction to more plotting with R.

        If one simply wants to view an R graphic, using this function is
        is sufficient (because it calls dev.off() to turn off the device).

        However, if one wants to save the graphic to a specific file, it
        should be used as in the example below to write the output.

        EXAMPLES:

        This example saves a plot to the standard R output, usually
        a filename like ``Rplot001.png`` - from the command line, in
        the current directory, and in the cell directory in the notebook::

            sage: d=r.setwd('"%s"'%SAGE_TMP)    # for doctesting only; ignore if you are trying this
            sage: r.plot("1:10")                # optional -- rgraphics
            null device
                      1

        To save to a specific file name, one should use :meth:`png` to set
        the output device to that file.  If this is done in the notebook, it
        must be done in the same cell as the plot itself::

            sage: filename = tmp_filename() + '.png'
            sage: r.png(filename='"%s"'%filename) # Note the double quotes in single quotes!; optional -- rgraphics
            NULL
            sage: x = r([1,2,3])
            sage: y = r([4,5,6])
            sage: r.plot(x,y)         # optional -- rgraphics
            null device
                      1
            sage: import os; os.unlink(filename) # For doctesting, we remove the file; optional -- rgraphics

        Please note that for more extensive use of R's plotting
        capabilities (such as the lattices package), it is advisable
        to either use an interactive plotting device or to use the
        notebook.  The following examples are not tested, because they
        differ depending on operating system::

            sage: r.X11() # not tested - opens interactive device on systems with X11 support
            sage: r.quartz() # not tested - opens interactive device on OSX
            sage: r.hist("rnorm(100)") # not tested - makes a plot
            sage: r.library("lattice") # not tested - loads R lattice plotting package
            sage: r.histogram(x = "~ wt | cyl", data="mtcars") # not tested - makes a lattice plot
            sage: r.dev_off() # not tested, turns off the interactive viewer

        In the notebook, one can use r.png() to open the device, but
        would need to use the following since R lattice graphics do
        not automatically print away from the command line::

            sage: filename = tmp_filename() + '.png' # Not needed in notebook, used for doctesting
            sage: r.png(filename='"%s"'%filename) # filename not needed in notebook, used for doctesting; optional -- rgraphics
            NULL
            sage: r.library("lattice")
            sage: r("print(histogram(~wt | cyl, data=mtcars))") # plot should appear; optional -- rgraphics
            sage: import os; os.unlink(filename) # We remove the file for doctesting, not needed in notebook; optional -- rgraphics
        """
        # We have to define this to override the plot function defined in the
        # superclass.
        from sage.server.support import EMBEDDED_MODE
        if EMBEDDED_MODE:
            self.setwd('"%s"'%os.path.abspath('.'))
        RFunction(self, 'plot')(*args, **kwds)
        return RFunction(self, 'dev.off')()

    def _strip_prompt(self, code):
        """
        Remove the standard R prompt from the beginning of lines in code.

        INPUT:

        - code -- a string

        OUTPUT: a string

        EXAMPLES::

            sage: s = '> a <- 2\n> b <- 3'
            sage: r._strip_prompt(s)
            ' a <- 2\n b <- 3'
        """
        return prompt_re.sub("", code)

    def eval(self, code, globals=None, locals=None, synchronize=True, *args, **kwds):
        """
        Evaluates a command inside the R interpreter and returns the output
        as a string.

        EXAMPLES::

            sage: r.eval('1+1')
            '[1] 2'
        """
        # TODO split code at ";" outside of quotes and send them as individual
        #      lines without ";".
        return Expect.eval(self, code, synchronize=synchronize, *args, **kwds)

    def _r_to_sage_name(self, s):
        """
        Returns a Sage/Python identifier from an R one.  This involves
        replacing periods with underscores, <- with __, and prepending
        _ in front of Python keywords.

        INPUT:

        - s -- a string

        OUTPUT: a string

        EXAMPLES::

            sage: f = r._r_to_sage_name
            sage: f('t.test')
            't_test'
            sage: f('attr<-')
            'attr__'
            sage: f('parent.env<-')
            'parent_env__'
            sage: f('class')
            'class_'
        """
        from keyword import iskeyword
        s = s.replace('.', '_')
        s = s.replace('<-', '__')
        if iskeyword(s):
            s += '_'
        return s

    def _sage_to_r_name(self, s):
        r"""
        The reverse of :meth:`_r_to_sage_name`.  See the docs for that method.

        EXAMPLES::

            sage: f = r._sage_to_r_name
            sage: f('t_test')
            't.test'
            sage: f('attr__')
            'attr<-'
            sage: f('parent_env__')
            'parent.env<-'
            sage: r._r_to_sage_name(f('parent_env__'))
            'parent_env__'
            sage: f('class_')
            'class'
        """
        if len(s) > 1 and s[-2:] == "__":
            s = s[:-2] + '<-'
        if len(s) > 0 and s[-1] == '_':
            s = s[:-1]
        s = s.replace('_', '.')
        return s

    def __getitem__(self, s):
        """
        Returns the RFunction with name s.

        INPUT:

        - s -- a string
        OUTPUT: RFunction -- the R function that in R has name s

        EXAMPLES::

            sage: r['as.data.frame']
            as.data.frame
            sage: r['print']
            print
        """
        return RFunction(self, s, r_name=True)

    def chdir(self, dir):
        """
        Changes the working directory to ``dir``

        INPUT:

        - ``dir`` -- the directory to change to.

        EXAMPLES::

            sage: import tempfile
            sage: tmpdir = tempfile.mkdtemp()
            sage: r.chdir(tmpdir)

        Check that ``tmpdir`` and ``r.getwd()`` refer to the same
        directory.  We need to use ``realpath()`` in case ``$TMPDIR``
        (by default ``/tmp``) is a symbolic link (see :trac:`10264`).

        ::

            sage: os.path.realpath(tmpdir) == sageobj(r.getwd())  # known bug (trac #9970)
            True
        """
        self.execute('setwd(%r)' % dir)

# patterns for _sage_()
rel_re_param = re.compile(r'\s([\w\.]+)\s=')
rel_re_range = re.compile(r'([\d]+):([\d]+)')
rel_re_integer = re.compile(r'([^\d])([\d]+)L')
rel_re_terms = re.compile(r'terms\s*=\s*(.*?),')
rel_re_call = re.compile(r'call\s*=\s*(.*?)\),')

@instancedoc
class RElement(ExtraTabCompletion, ExpectElement):

    def _tab_completion(self):
        """
        Return a list of all methods of this object.

        .. note::

            Currently returns all R commands.

        EXAMPLES::

            sage: a = r([1,2,3])
            sage: t = a._tab_completion()
            sage: len(t) > 200
            True
        """
        # TODO: rewrite it, just take methods(class=class(self))
        return self.parent()._tab_completion()

    def tilde(self, x):
        """
        The tilde regression operator in R.

        EXAMPLES::

            sage: x = r([1,2,3,4,5])
            sage: y = r([3,5,7,9,11])
            sage: a = r.lm( y.tilde(x) ) # lm( y ~ x )
            sage: d = a._sage_()
            sage: d['DATA']['coefficients']['DATA'][1]
            2
        """
        par = self.parent()
        rx = par(x)
        return par.new("%s ~ %s" % (self.name(), rx.name()))

    stat_model = tilde

    def __len__(self):
        """
        Return the length of this object.

        OUTPUT: integer

        EXAMPLES::

            sage: x = r([10.4,5.6,3.1,6.4,21.7])
            sage: len(x)
            5
        """
        return int(self.parent().eval('dput(length(%s))'%self.name())[:-1] )

    def __getattr__(self, attrname):
        """
        Return attribute of this object, which is an R function with this object
        as the first input.

        INPUT:

        - attrname -- string

        OUTPUT: RFunctionElement

        EXAMPLES::

            sage: x = r([1,2,3])
            sage: length = x.length
            sage: type(length)
            <class 'sage.interfaces.r.RFunctionElement'>
            sage: length()
            [1] 3
        """
        self._check_valid()
        if attrname[:1] == "_":
            raise AttributeError
        return RFunctionElement(self, attrname)

    def __getitem__(self, n):
        """
        Return element(s) of self.

        INPUT:

        - n -- an integer, a tuple, a string that makes sense to R, or an RElement

        OUTPUT: RElement

        EXAMPLES::

            sage: x = r([10.4,5.6,3.1,6.4,21.7])
            sage: x[0]
            numeric(0)
            sage: x[1]
            [1] 10.4
            sage: x[-1]
            [1] 5.6  3.1  6.4 21.7
            sage: x[-2]
            [1] 10.4  3.1  6.4 21.7
            sage: x[-3]
            [1] 10.4  5.6  6.4 21.7
            sage: x['c(2,3)']
            [1]  5.6 3.1
            sage: key = r.c(2,3)
            sage: x[key]
            [1]  5.6 3.1
            sage: m = r.array('1:3',r.c(2,4,2))
            sage: m
            , , 1
                 [,1] [,2] [,3] [,4]
            [1,]    1    3    2    1
            [2,]    2    1    3    2
            , , 2
                 [,1] [,2] [,3] [,4]
            [1,]    3    2    1    3
            [2,]    1    3    2    1
            sage: m[1,2,2]
            [1] 2
            sage: m[1,r.c(1,2),1]
            [1] 1 3
        """
        P = self._check_valid()
        if isinstance(n, six.string_types):
            n = n.replace('self', self._name)
            return P.new('%s[%s]'%(self._name, n))
        elif parent(n) is P:  # the key is RElement itself
            return P.new('%s[%s]'%(self._name, n.name()))
        elif not isinstance(n,tuple):
            return P.new('%s[%s]'%(self._name, n))
        else:
            L = []
            for i in range(len(n)):
                if parent(n[i]) is P:
                    L.append(n[i].name())
                else:
                    L.append(str(n[i]))
            return P.new('%s[%s]'%(self._name, ','.join(L)))

    def __bool__(self):
        """
        Implements bool(self).

        .. note::

            bool(self) will only return True if self == 0 contains a FALSE in its representation.

        EXAMPLES::

            sage: x = r([10.4,5.6,3.1,6.4,21.7])
            sage: bool(x)
            True
            sage: y = r([0,0,0,0])
            sage: bool(y)
            False
            sage: bool(r(0))
            False
            sage: bool(r(1))
            True
        """
        return "FALSE" in repr(self==0)

    __nonzero__ = __bool__

    def _comparison(self, other, symbol):
        """
        Used to implement comparison of two objects.

        INPUT:

        - other -- RElement
        - symbol -- string

        OUTPUT: RElement -- output is an R element; not a bool!

        TESTS::

            sage: x = r([10.4,5.6,3.1,6.4,21.7])
            sage: x._comparison(10.4, "==")
            [1] TRUE FALSE FALSE FALSE FALSE
        """
        P = self.parent()
        other = P(other)
        return P('%s %s %s'%(self.name(), symbol, other.name()))

    def __eq__(self, other):
        """
        Equality testing term by term.

        INPUT:

        - other -- RElement

        OUTPUT: RElement -- an R element; not a bool!

        EXAMPLES:

        Notice that comparison is term by term and returns an R element. ::

            sage: x = r([10.4,5.6,3.1,6.4,21.7])
            sage: x == 10.4
            [1] TRUE FALSE FALSE FALSE FALSE
        """
        return self._comparison(other, "==")

    def __lt__(self, other):
        """
        Less than testing term by term.

        INPUT:

        - other -- RElement

        OUTPUT: RElement -- an R element; not a bool!

        EXAMPLES:

        Notice that comparison is term by term and returns an R element. ::

            sage: x = r([10.4,5.6,3.1,6.4,21.7])
            sage: x < 7
            [1] FALSE  TRUE  TRUE  TRUE FALSE
        """
        return self._comparison(other, "<")

    def __gt__(self, other):
        """
        Greater than testing term by term.

        INPUT:

        - other -- RElement

        OUTPUT: RElement -- an R element; not a bool!

        EXAMPLES:

        Notice that comparison is term by term and returns an R element. ::

            sage: x = r([10.4,5.6,3.1,6.4,21.7])
            sage: x > 8
            [1] TRUE FALSE FALSE FALSE  TRUE
        """
        return self._comparison(other, ">")

    def __le__(self, other):
        """
        Less than or equal testing term by term.

        INPUT:

        - other -- RElement

        OUTPUT: RElement -- an R element; not a bool!

        EXAMPLES::

            sage: x = r([10.4,5.6,3.1,6.4,21.7])
            sage: x <= 10.4
            [1] TRUE  TRUE  TRUE  TRUE FALSE
        """
        return self._comparison(other, "<=")

    def __ge__(self, other):
        """
        Greater than or equal testing term by term.

        INPUT:

        - other -- RElement

        OUTPUT: RElement -- an R element; not a bool!

        EXAMPLES::

            sage: x = r([10.4,5.6,3.1,6.4,21.7])
            sage: x >= 10.4
            [1] TRUE FALSE FALSE FALSE  TRUE
        """
        return self._comparison(other, ">=")

    def __ne__(self, other):
        """
        Not equal testing term by term.

        INPUT:

        - other -- RElement

        OUTPUT: RElement -- an R element; not a bool!

        EXAMPLES::

            sage: x = r([10.4,5.6,3.1,6.4,21.7])
            sage: x != 10.4
            [1] FALSE  TRUE  TRUE  TRUE  TRUE

        """
        return self._comparison(other, "!=")

    def dot_product(self, other):
        """
        Implements the notation self . other.

        INPUT:

        - self, other -- R elements

        OUTPUT: R element

        EXAMPLES::

            sage: c = r.c(1,2,3,4)
            sage: c.dot_product(c.t())
                 [,1] [,2] [,3] [,4]
            [1,]    1    2    3    4
            [2,]    2    4    6    8
            [3,]    3    6    9   12
            [4,]    4    8   12   16

            sage: v = r([3,-1,8])
            sage: v.dot_product(v)
                 [,1]
            [1,]   74
        """
        P = self._check_valid()
        Q = P(other)
        # the R operator is %*% for matrix multiplication
        return P('%s %%*%% %s'%(self.name(), Q.name()))

    def _subs_dots(self, x):
        r"""
        Replace dots by underscores; used internally to implement
        conversation from R expression to Sage objects.

        INPUT:

        - x -- regular expression match: ``<type '_sre.SRE_Match'>``

        OUTPUT: string

        EXAMPLES::

            sage: import re
            sage: a = r([1,2,3])
            sage: rel_re_param = re.compile(r'\s([\w\.]+)\s=')
            sage: rel_re_param.sub(a._subs_dots, ' test.test =')
             ' test_test ='
        """
        return x.group().replace('.','_')

    def _subs_range(self, x):
        r"""
        Change endpoints of ranges.  This is used internally in the
        code for converting R expressions to Sage objects.

        INPUT:

        - x -- regular expression match: ``<type '_sre.SRE_Match'>``

        OUTPUT: string

        EXAMPLES::

            sage: import re
            sage: a = r([1,2,3])
            sage: rel_re_range = re.compile(r'([\d]+):([\d]+)')
            sage: rel_re_range.sub(a._subs_range, ' 1:10')
            ' range(1,11)'
        """
        g = x.groups()
        g1 = int(g[1]) + 1
        return 'range(%s,%s)' % (g[0],  g1)

    def _subs_integer(self, x):
        r"""
        Replaces strings like 'dL' with 'Integer(d)' where d is some
        integer.  This is used internally in the code for converting R
        expressions to Sage objects.

        EXAMPLES::

            sage: import re
            sage: a = r([1,2,3])
            sage: rel_re_integer = re.compile(r'([^\d])([\d]+)L')
            sage: rel_re_integer.sub(a._subs_integer, ' 1L 2L')
            ' Integer(1) Integer(2)'
            sage: rel_re_integer.sub(a._subs_integer, '1L 2L')
            '1L Integer(2)'

        """
        return '%sInteger(%s)'%x.groups()

    def _convert_nested_r_list(self, exp):
        """
        Converts a string representing a (possibly) nested list in R
        to a (possibly) nested Python list.  This is used internally
        in the code for converting R expressions to Sage objects.

        INPUT:

        - exp -- a string

        OUTPUT: a string

        EXAMPLES::

            sage: a = r([1,2,3])
            sage: s = 'c(1, 2, 3)'
            sage: a._convert_nested_r_list(s)
            '[1, 2, 3]'
        """
        from re import compile as re_compile
        from re import split   as re_split
        splt = re_compile(r'(c\(|\(|\))') # c( or ( or )
        lvl = 0
        ret = []
        for token in re_split(splt, exp):
            if token == 'c(':
                ret.append('[')
                lvl += 1
            elif token == '(':
                ret.append(token)
                if lvl > 0 : lvl += 1
            elif token == ')':
                if lvl == 1:
                    ret.append(']')
                    lvl -= 1
                else:
                    ret.append(token)
                    if lvl > 0:
                        lvl -= 1
            else:
                ret.append(token)

        return ''.join(ret)

    def _r_list(self, *args, **kwds):
        """
        This is used internally in the code for converting R
        expressions to Sage objects.

        EXAMPLES::

            sage: a = r([1,2,3])
            sage: sorted(a._r_list(1,2,3,k=5).items())
            [('#0', 1), ('#1', 2), ('#2', 3), ('k', 5)]
        """
        ret = dict(kwds)
        i = 0
        for k in args:
            ret['#%s' % i] = k
            i += 1
        return ret

    def _r_structure(self, __DATA__, **kwds):
        """
        This is used internally in the code for converting R
        expressions to Sage objects.

        EXAMPLES::

            sage: a = r([1,2,3])
            sage: d = a._r_structure('data', a=1, b=2)
            sage: sorted(d.items())
            [('DATA', 'data'), ('a', 1), ('b', 2)]
            sage: a._r_structure([1,2,3,4], _Dim=(2,2))
            [1 3]
            [2 4]

        """
        if '_Dim' in kwds: #we have a matrix
            # TODO what about more than 2 dimensions?
            #      additional checks!!
            try:
                from sage.matrix.constructor import matrix
                d = kwds.get('_Dim')
                # TODO: higher dimensions? happens often in statistics
                if len(d) != 2:
                    raise TypeError
                #since R does it the other way round, we assign
                #transposed and then transpose the matrix :)
                m = matrix(d[1], d[0], [i for i in __DATA__])
                return m.transpose()
            except TypeError:
                pass
        d = dict(DATA=__DATA__)
        d.update(kwds)
        return d

    def _sage_(self):
        r"""
        Returns Sage representation of the R object.

        R objects are basic C structures, of different kind, that can
        be stacked together.  This is similar to Python lists with
        variable objects, including lists of lists.  If R lists have
        names, they are translated to a Python dictionary, with anonymous
        list entries called ``#{number}``.

        OUTPUT: object -- Python object

        EXAMPLES::

            sage: rs = r.summary(r.c(1,4,3,4,3,2,5,1))
            sage: d = rs._sage_()
            sage: sorted(d.items())
            [('DATA', [1, 1.75, 3, 2.875, 4, 5]),
             ('_Names', ['Min.', '1st Qu.', 'Median', 'Mean', '3rd Qu.', 'Max.']),
             ('_r_class', ['summaryDefault', 'table'])]
        """
        self._check_valid()
        P = self.parent()

        # This is the core of the trick: using dput
        # dput prints out the internal structure of R's data elements
        # options via .deparseOpts(control=...)
        # TODO: dput also works with a file, if things get huge!
        # [[However, using a file for output often isn't necessary
        # since pipe buffering works pretty well for output.
        # That said, benchmark this.  -- William Stein]]
        exp = P.eval('dput(%s)'%self.name())

        # Preprocess expression
        # example what this could be:
        # structure(list(statistic = structure(0.233549683248457, .Names = "t"),
        # parameter = structure(5.58461538461538, .Names = "df"), p.value = 0.823657802106985,
        # conf.int = structure(c(-2.41722062247400, 2.91722062247400
        # ), conf.level = 0.95), estimate = structure(c(2.75, 2.5), .Names = c("mean of x",
        # "mean of y")), null.value = structure(0, .Names = "difference in means"),
        # alternative = "two.sided", method = "Welch Two Sample t-test",
        # data.name = "c(1, 2, 3, 5) and c(1, 2, 3, 4)"), .Names = c("statistic",
        # "parameter", "p.value", "conf.int", "estimate", "null.value",
        # "alternative", "method", "data.name"), class = "htest")

        # R's structure (from help):
        # structure(.Data, ...)
        #    .Data: an object which will have various attributes attached to it.
        #    ...: attributes, specified in 'tag=value' form, which will be
        #         attached to data.
        #For historical reasons (these names are used when deparsing),
        # attributes '".Dim"', '".Dimnames"', '".Names"', '".Tsp"' and
        # '".Label"' are renamed to '"dim"', '"dimnames"', '"names"',
        # '"tsp"' and '"levels"'.

        # we want this in a single line
        exp.replace('\n','')
        exp = "".join(exp.split("\n"))

        # python compatible parameters
        exp = rel_re_param.sub(self._subs_dots, exp)

        # Rename class since it is a Python keyword
        exp = re.sub(' class = ', ' _r_class = ',exp)

        # Change 'structure' to '_r_structure'
        # TODO: check that we are outside of quotes ""
        exp = re.sub(r' structure\(', ' _r_structure(', exp)
        exp = re.sub(r'^structure\(', '_r_structure(', exp)  # special case

        # Change 'list' to '_r_list'
        exp = re.sub(r' list\(', ' _r_list(', exp)
        exp = re.sub(r'\(list\(', '(_r_list(', exp)

        # Change 'a:b' to 'range(a,b+1)'
        exp = rel_re_range.sub(self._subs_range, exp)

        # Change 'dL' to 'Integer(d)'
        exp = rel_re_integer.sub(self._subs_integer, exp)

        # Wrap the right hand side of terms = ... in quotes since it
        # has a ~ in it.
        exp = rel_re_terms.sub(r'terms = "\1",', exp)

        # Change call = ..., to call = "...",
        exp = rel_re_call.sub(r'call = "\1",', exp)

        # seems to work for
        # rr = r.summary(r.princomp(r.matrix(r.c(1,2,3,4,3,4,1,2,2),4)))
        # rr._sage_()
        # but the call expression get's evaluated. why?!? TODO

        # translation:
        # c is an ordered list
        # list is a dictionary (where _Names give the entries names.
        #    map entries in names to (value, name) in each entry?
        # structure is .. see above .. structure(DATA,**kw)
        # TODO: thinking of just replacing c( with ( to get a long tuple?

        exp = self._convert_nested_r_list(exp)

        # Set up the globals
        globs = {'NA':None, 'NULL':None, 'FALSE':False, 'TRUE':True,
                 '_r_list':self._r_list, '_r_structure':self._r_structure,
                 'Integer':sage.rings.integer.Integer,
                 'character':str}

        return eval(exp, globs, globs)

    def _latex_(self):
        r"""
        Return LaTeX representation of this R object.

        This calls the ``latex`` command in R.

        OUTPUT: a latex expression (basically a string)

        EXAMPLES::

            sage: latex(r(2))  # optional - Hmisc (R package)
            2
        """
        from sage.misc.latex import LatexExpr
        self._check_valid()
        P = self.parent()
        # latex is in Hmisc, this is currently not part of Sage's R!!!
        try:
            P.library('Hmisc')
        except ImportError:
            raise RuntimeError("The R package 'Hmisc' is required for R to LaTeX conversion, but it is not available.")
        return LatexExpr(P.eval('latex(%s, file="");' % self.name()))

@instancedoc
class RFunctionElement(FunctionElement):
    def __reduce__(self):
        """
        EXAMPLES::

            sage: a = r([1,2,3])
            sage: a.mean
            mean
            sage: dumps(a.mean)
            Traceback (most recent call last):
            ...
            NotImplementedError: pickling of R element methods is not yet supported
        """
        raise NotImplementedError("pickling of R element methods is not yet supported")

    def _instancedoc_(self):
        """
        Returns the help for self as a string.

        EXAMPLES::

            sage: a = r([1,2,3])
            sage: length = a.length
            sage: print(length.__doc__)
            length                 package:base                 R Documentation
            ...
            <BLANKLINE>
        """
        M = self._obj.parent()
        return M.help(self._name)

    def _sage_src_(self):
        """
        Returns the source code of self.

        EXAMPLES::

            sage: a = r([1,2,3])
            sage: length = a.length
            sage: print(length._sage_src_())
            function (x)  .Primitive("length")
        """
        M = self._obj.parent()
        return M.source(self._name)

    def __call__(self, *args, **kwds):
        """
        EXAMPLES::

            sage: a = r([1,2,3])
            sage: length = a.length
            sage: length()
            [1] 3
        """
        return self._obj.parent().function_call(self._name, args=[self._obj] + list(args), kwds=kwds)

@instancedoc
class RFunction(ExpectFunction):
    def __init__(self, parent, name, r_name=None):
        """
        A Function in the R interface.

        INPUT:

        - parent -- the R interface
        - name -- the name of the function for Python
        - r_name -- the name of the function in R itself (which can have dots in it)

        EXAMPLES::

            sage: length = r.length
            sage: type(length)
            <class 'sage.interfaces.r.RFunction'>
            sage: loads(dumps(length))
            length
        """
        self._parent = parent
        if r_name:
            self._name = name
        else:
            self._name = parent._sage_to_r_name(name)

    def __eq__(self, other):
        """
        EXAMPLES::

            sage: r.mean == loads(dumps(r.mean))
            True
            sage: r.mean == r.lr
            False
        """
        return (isinstance(other, RFunction) and
            self._name == other._name)

    def __ne__(self, other):
        """
        EXAMPLES::

            sage: r.mean != loads(dumps(r.mean))
            False
            sage: r.mean != r.lr
            True
        """
        return not (self == other)

    def _instancedoc_(self):
        """
        Returns the help for self.

        EXAMPLES::

            sage: length = r.length
            sage: print(length.__doc__)
            length                 package:base                 R Documentation
            ...
            <BLANKLINE>
        """
        M = self._parent
        return M.help(self._name)

    def _sage_src_(self):
        """
        Returns the source of self.

        EXAMPLES::

            sage: length = r.length
            sage: print(length._sage_src_())
            function (x)  .Primitive("length")

        """
        M = self._parent
        return M.source(self._name)

    def __call__(self, *args, **kwds):
        """
        EXAMPLES::

            sage: length = r.length
            sage: length([1,2,3])
            [1] 3
        """
        return self._parent.function_call(self._name, args=list(args), kwds=kwds)

def is_RElement(x):
    """
    Return True if x is an element in an R interface.

    INPUT:

    - x -- object

    OUTPUT: bool

    EXAMPLES::

        sage: from sage.interfaces.r import is_RElement
        sage: is_RElement(2)
        False
        sage: is_RElement(r(2))
        True
    """
    return isinstance(x, RElement)

# An instance of R
r = R()

def reduce_load_R():
    """
    Used for reconstructing a copy of the R interpreter from a pickle.

    EXAMPLES::

        sage: from sage.interfaces.r import reduce_load_R
        sage: reduce_load_R()
        R Interpreter
    """
    return r

import os
def r_console():
    """
    Spawn a new R command-line session.

    EXAMPLES::

        sage: r.console()                    # not tested
            R version 2.6.1 (2007-11-26)
            Copyright (C) 2007 The R Foundation for Statistical Computing
            ISBN 3-900051-07-0
            ...
    """
    from sage.repl.rich_output.display_manager import get_display_manager
    if not get_display_manager().is_in_terminal():
        raise RuntimeError('Can use the console only in the terminal. Try %%r magics instead.')
    # This will only spawn local processes
    os.system('R --vanilla')

def r_version():
    """
    Return the R version.

    EXAMPLES::

        sage: r_version() # not tested
        ((3, 0, 1), 'R version 3.0.1 (2013-05-16)')
        sage: rint, rstr = r_version()
        sage: rint[0] >= 3
        True
        sage: rstr.startswith('R version')
        True
    """
    return r.version()

class HelpExpression(str):
    """
    Used to improve printing of output of r.help.
    """
    def __repr__(self):
        """
        Return string representation of self.

        OUTPUT: string

        EXAMPLES::

            sage: a = sage.interfaces.r.HelpExpression("This\nis\nR!")
            sage: type(a)
            <class 'sage.interfaces.r.HelpExpression'>
            sage: a
            This
            is
            R!
        """
        return str(self)

{'EMBEDDED_MODE': 'sage.server.support.EMBEDDED_MODE', 'sage.plot.plot': 'sage.plot.plot', 'sage.misc.persist': 'sage.misc.persist', 'iskeyword': 'keyword.iskeyword'}


import json
from pathlib import Path
from enum import Enum

class Subject(Enum):
    Math = 0
    SocialScience = 1
    Science = 2
    English = 3

def InsertRecord():
    try:
        Record = json.load(open('C:/users/garry/Desktop/StudentRecord/stdRec.txt'))
    except:
        Record = {}
    Name = {}
    Marks = {}
    Percentage = {}
    M = []
    per = 0
    rno = input("\nEnter Student's Roll Number : ")
    Name['Name'] = input("\nEnter Student's Name : ")
    for i in range(4):
        M.append(int(input('\nEnter Marks in ' + str(Subject(i).name) + ' : ')))
    for m in M:
        per += m
    per /= 4
    Marks['Marks'] = M
    Percentage['Percentage'] = per
    Record[rno] = Name, Marks, Percentage
    json.dump(Record, open('C:/users/garry/Desktop/StudentRecord/stdRec.txt', 'w'))
    print('\nRecord Inserted')

def PrintToppersList():
    try:
        rec = json.load(open('C:/users/garry/Desktop/StudentRecord/stdRec.txt'))
        mathTopper = next(iter(rec))
        socialScienceTopper = next(iter(rec))
        scienceTopper = next(iter(rec))
        englishTopper = next(iter(rec))
        for i in rec:
            for j in range(4):
                if j == 0:
                    if rec[i][1]['Marks'][j] > rec[mathTopper][1]['Marks'][j]:
                        mathTopper = i
                elif j == 1:
                    if rec[i][1]['Marks'][j] > rec[socialScienceTopper][1]['Marks'][j]:
                        socialScienceTopper = i
                elif j == 2:
                    if rec[i][1]['Marks'][j] > rec[scienceTopper][1]['Marks'][j]:
                        scienceTopper = i
                elif j == 3:
                    if rec[i][1]['Marks'][j] > rec[englishTopper][1]['Marks'][j]:
                        englishTopper = i
        print('\nMath Topper : ' + rec[mathTopper][0]['Name'])
        print('\nSocial Science Topper : ' + rec[socialScienceTopper][0]['Name'])
        print('\nScience Topper : ' + rec[scienceTopper][0]['Name'])
        print('\nEnglish Topper : ' + rec[englishTopper][0]['Name'])
    except:
        print('\n\nEmpty File. Enter a record first\n\n')

def PrintStudentRecord():
    try:
        rec = json.load(open('C:/users/garry/Desktop/StudentRecord/stdRec.txt'))
        print('\nRollNo    Name        Math    SocialScience    Science    English    Percentage')
        for i in rec:
            r = i
            n = rec[i][0]['Name']
            m = rec[i][1]['Marks']
            p = rec[i][2]['Percentage']
            print(r, '       ', n, '    ', '            '.join([str(ls) for ls in m]), '    ', p)
        print()
    except:
        print('\n\nEmpty File. Enter a record first\n\n')

while True:
    f = Path('C:/users/garry/Desktop/StudentRecord/stdRec.txt')
    if f.is_file():
        uc = int(input('\nPress 1 to veiw existing Student Record\nPress 2 to Add a record\nPress 3 to get the list of Toppers in each Subject\nPress 4 to exit the Program\n\nEnter Your Choice : '))
        if uc == 1:
            PrintStudentRecord()
        elif uc == 2:
            InsertRecord()
        elif uc == 3:
            PrintToppersList()
        elif uc == 4:
            exit()
    else:
        temp = open('C:/users/garry/Desktop/StudentRecord/stdRec.txt', 'w')
        temp.close()


#-*- coding: UTF-8 -*-
from common import MyExcel
import time
from common.SignatureGenerator import *
from common.read_config import read_config
import requests
import configparser
# 1、从excel 表中读取需要算签名数据
filePath = "E:\\PycharmProjects\\python-api\\TestDatas\\TestData.xlsx"
me = MyExcel.MyExcel(filePath)
caseDatas = me.get_all_testDatas()
print(len(caseDatas))
appkey = read_config().read_config("common.conf", "http", "app_key")
for i in range(0,len(caseDatas)):
    params_map =eval(caseDatas[i]["request_data"])
    url = caseDatas[i]["url"]
    signt = str(int(time.time() * 1000))
    params_map.update(signt=signt,appkey=appkey)
    sign = SignatureGenerator.generate(url, params_map)
    params_map.update(sign=sign)
    caseDatas[i]["request_data"] = str(params_map)
    caseDatas[i]["url"] = read_config().read_config("common.conf", "http", "server_ip")+url
print(caseDatas[1]["expected_data"])
# param_map=eval(caseDatas[0]["request_data"])
# url = caseDatas[0]["url"]
# print(param_map)
print(url)
# 2.进行更新param数据中得signt
# signt =str(int(time.time()*1000))
# param_map.update(signt=signt)
# print(param_map)
#
# # 3、进行签名加密
# sign = SignatureGenerator.generate(url, param_map)
# print(sign)
#
# # 4、更新param_map 字典中得参数
#
# param_map.update(sign=sign)
# print(param_map)
# # 5、gei请求
#
# # 读取配置文件中得ip
# config=configparser.ConfigParser()
# config.read("common.conf")
# urls="http://"+config.get("http","server_ip")+"/"+url
# print(urls)
#
#
# request = requests.get(url=urls,params=param_map)
# result =request.content
# print(request.url)
# print(result)


import os
os.sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from passlib.hash import sha256_crypt
from flask_restful import reqparse
from functools import wraps
from flask import request, jsonify
from models import models
from flask_jwt_extended import (
    verify_jwt_in_request, get_jwt_claims,
    get_jwt_identity
)

def pw_encrypt(pw):
    return sha256_crypt.hash(pw)

def verify_pw(pw_str, pw_hash):
    return sha256_crypt.verify(pw_str, pw_hash)

def verify_signup_input(f):
    @wraps(f)
    def decorated(*args, **kwargs):
        parser = reqparse.RequestParser(bundle_errors=True)
        parser.add_argument('name', type=str, required=True, help="Name cannot be blank!")
        parser.add_argument('email', type=str, required=True, help="Email cannot be blank!")
        parser.add_argument('password', type=str, required=True, help="Password cannot be blank!")
        parser.parse_args()

        return f(*args, **kwargs)
    return decorated

def get_role_id(role='regular'):
    role = models.Role.query.filter_by(name=role).first()
    return role.id

def get_request_type_id(type='pending'):
    request_type = models.Request_Type.query.filter_by(name=type).first()
    return request_type.id

def validate_request(*expected_args):
    def validate_input(f):
        @wraps(f)
        def decorated(*args, **kwargs):
            if not request.json:
                return {
                    'status': 'fail',
                    "data": {"message": "Request must be a valid JSON"}
                }, 400

            payload = request.get_json()
            if payload:
                for value in expected_args:
                    if value == 'status' and value not in payload: continue
                    if value not in payload or not payload[value]:
                        return {
                            "status": "fail",
                            "data": {"message": value + " is required"}
                        }, 400
                    if type(payload[value]) != str or not payload[value].strip(' '):
                        return {
                            "status": "fail",
                            "data": {"message": value + " must contain valid strings"}
                        }, 400
            return f(*args, **kwargs)
        return decorated
    return validate_input

# Here is a custom decorator that verifies the JWT is present in
# the request, as well as insuring that this user has a role of
# `admin` in the access token
def admin_required(fn):
    @wraps(fn)
    def authenticated(*args, **kwargs):
        verify_jwt_in_request()
        claims = get_jwt_claims()
        if claims['role'] != 'admin':
            return jsonify(dict(
                status="fail",
                messsage="Access Denied!"
            )), 401
        return fn(*args, **kwargs)
    return authenticated

def validate_user_update_request(fn):
    @wraps(fn)
    def decorated(*args, **kwargs):
        current_user = get_jwt_identity()
        request_id = request.url.rstrip('/').split('/')[-1]
        user_request = models.Request.query.get(request_id)

        if not user_request:
            return {
                "status": "fail",
                "message": "Request not found!"
            }, 404

        if current_user['id'] != str(user_request.owner_id):
            return {
                "status": "fail",
                "message": "Request denied!"
            }, 403

        if user_request.status in ['approved', 'disapproved', 'resolved']:
            return {
                "status": "fail",
                "message": "Request cannot be edited!"
            }, 403

        return fn(*args, **kwargs)
    return decorated