text:
"""Common utilities"""

import argparse
import asyncio
from collections import defaultdict
import configparser
from contextlib import contextmanager
import datetime
from dateutil import parser as dateparser
from dateutil.relativedelta import relativedelta
import functools
import itertools
import io
import logging
import math
import types

import numpy as np
import pandas as pd
from pandas.io.common import get_filepath_or_buffer

try:
    from pandas.io.common import _get_handle as get_handle
except ImportError:
    from pandas.io.common import get_handle
from pyexcel_xlsx import get_data as get_xlsx, save_data as save_xlsx
from pyexcel_xls import get_data as get_xls, save_data as save_xls
from tlbx import (
    st,
    raiseif,
    raiseifnot,
    json,
    dbg as _dbg,
    dbgsql as _dbgsql,
    info as _info,
    warn as _warn,
    error as _error,
    get_caller,
    is_str,
    Arg,
    Parent,
)
import yaml

default_logger = logging.getLogger("glide")
default_logger.setLevel(logging.INFO)

XLS = "xls"
XLSX = "xlsx"

_CLASS_LIST_DOCSTRING_TEMPLATE = """

**{heading}:**

{classes_str}
"""

def find_class_in_dict(cls, d, include=None, exclude=None):
    """Get a list of keys that are an instance of a class in a dict"""
    names = []
    for key, value in d.copy().items():
        if not isinstance(value, type):
            continue
        if issubclass(value, cls):
            if include and include not in key:
                continue
            if exclude and exclude in key:
                continue
            names.append(key)
    return names

def get_class_list_docstring(heading, classes):
    """Helper to generate a part of a module docstring from a list of classes"""
    classes_str = "- " + "\n- ".join(classes)
    return _CLASS_LIST_DOCSTRING_TEMPLATE.format(
        heading=heading, classes_str=classes_str
    )

def closer(x):
    """Helper to call close on x"""
    x.close()

def not_none(*args):
    return all([x is not None for x in args])

def split_count_helper(data, split_count):
    """Helper to override the split count if data len is shorter"""
    if hasattr(data, "__len__"):
        return min(len(data), split_count)
    return split_count

# -------- Date utils

def date_from_str(s):
    today = datetime.date.today()
    if s == "today":
        return today
    elif s == "yesterday":
        return today - datetime.timedelta(days=1)
    elif s == "tomorrow":
        return today + datetime.timedelta(days=1)
    elif s == "start_of_month":
        return today.replace(day=1)
    elif s == "start_of_next_month":
        return (today + relativedelta(months=+1)).replace(day=1)
    elif s == "start_of_last_month":
        return (today.replace(day=1) - datetime.timedelta(days=1)).replace(day=1)

    return dateparser.parse(s)

def to_datetime(d):
    if type(d) == datetime.datetime:
        return d
    if type(d) == datetime.date:
        return datetime.datetime.combine(d, datetime.time()).replace(microsecond=0)
    if type(d) == str:
        return to_datetime(date_from_str(d))
    raise AssertionError("Unsupported type: %s" % type(d))

def to_date(d):
    if type(d) == datetime.date:
        return d
    if type(d) == datetime.datetime:
        return d.date()
    if type(d) == str:
        return to_date(date_from_str(d))
    raise AssertionError("Unsupported type: %s" % type(d))

def datetime_cmp(d1, d2):
    d1 = to_datetime(d1)
    d2 = to_datetime(d2)
    return (d1 > d2) - (d1 < d2)

def get_date_windows(start_date, end_date, reverse=False):
    start_date = to_datetime(start_date)
    end_date = to_datetime(end_date)

    start_date = start_date.date()
    if datetime_cmp(end_date, end_date.date()) != 0:
        end_date = end_date.date() + datetime.timedelta(days=1)
    else:
        end_date = end_date.date()

    delta = end_date - start_date
    num_windows = delta.days
    orig_end_date = end_date
    end_date = start_date
    date_windows = []

    for i in range(num_windows):
        end_date = end_date + datetime.timedelta(days=1)
        date_windows.append((start_date, end_date))
        start_date = end_date
        if end_date >= orig_end_date:
            break

    if reverse:
        date_windows.reverse()

    return date_windows

def get_datetime_windows(
    start_date,
    end_date,
    window_size_hours=None,
    num_windows=None,
    reverse=False,
    add_second=True,
):
    """Produce a list of start/end date tuples

    Parameters
    ----------
    start_date : date, datetime, or str
        The absolute start date of the range
    end_date : date, datetime, or str
        The absolute end date of the range
    window_size_hours : float, optional
        The size of the windows in hours. May be a float to represent partial hours.
    num_windows : int, optional
        The number of windows to split the date range into. One of num_windows or
        window_size_hours must be specified.
    reverse : bool, optional
        If true return the windows in reverse order
    add_second : bool, optional
        If true, offset the start of each window to be one second past the end
        date of the previous window.

    Returns
    -------
    dt_windows : list
        A list of tuples of start / end datetime pairs

    """
    raiseifnot(
        window_size_hours or num_windows,
        "One of window_size_hours or num_windows must be specified",
    )
    raiseif(
        window_size_hours and num_windows,
        "Only one of window_size_hours or num_windows can be specified",
    )

    start_date = to_datetime(start_date)
    end_date = to_datetime(end_date)
    total_secs = (end_date - start_date).total_seconds()

    if num_windows:
        if num_windows == 1:
            return [(start_date, end_date)]
        window_size_hours = (total_secs / num_windows) / 3600

    total_hours = total_secs / 3600
    num_windows = num_windows or int(math.ceil(total_hours / window_size_hours))
    window_delta = datetime.timedelta(hours=window_size_hours)
    orig_end_date = end_date
    end_date = start_date
    dt_windows = []

    for i in range(num_windows):
        end_date = min(end_date + window_delta, orig_end_date)
        dt_windows.append((start_date, end_date))
        seconds = 1 if add_second else 0
        start_date = end_date + datetime.timedelta(seconds=seconds)
        if end_date >= orig_end_date:
            break

    if reverse:
        dt_windows.reverse()

    return dt_windows

def _process_date_action_dest(dest, values, end_date):
    today = datetime.date.today()
    now = datetime.datetime.now()
    if dest == "yesterday":
        start_date = today - datetime.timedelta(days=1)
        end_date = today
    elif dest == "today":
        start_date = today
    elif dest == "days_back":
        start_date = today - datetime.timedelta(days=values)
    elif dest == "hours_back":
        start_date = now - datetime.timedelta(hours=values)
    elif dest == "date_range":
        if len(values) == 1:
            start_date = date_from_str(values[0])
        elif len(values) == 2:
            start_date = date_from_str(values[0])
            end_date = date_from_str(values[1])
        else:
            raise AssertionError("date_range only accepts one or two arguments")
    else:
        raise AssertionError("Unrecognized date action %s" % dest)
    return start_date, end_date

class DateTimeWindowAction(argparse.Action):
    """An argparse Action for handling datetime window CLI args"""

    def __call__(self, parser, namespace, values, option_string=None):
        end_date = datetime.datetime.now()

        if self.dest == "date_window_size":
            if getattr(namespace, "date_windows", None):
                start_date = namespace.date_windows[0][0]
                end_date = namespace.date_windows[0][1]
                setattr(
                    namespace,
                    "date_windows",
                    get_datetime_windows(
                        start_date,
                        end_date,
                        window_size_hours=values,
                        reverse=getattr(namespace, "reverse_windows", False),
                    ),
                )
            else:
                # This will get used later once date_windows is populated
                setattr(namespace, self.dest, values)
                return
        else:
            start_date, end_date = _process_date_action_dest(
                self.dest, values, end_date
            )

        if not hasattr(namespace, "date_windows"):
            if getattr(namespace, "date_window_size", None):
                setattr(
                    namespace,
                    "date_windows",
                    get_datetime_windows(
                        start_date,
                        end_date,
                        window_size_hours=namespace.date_window_size,
                        reverse=getattr(namespace, "reverse_windows", False),
                    ),
                )
            else:
                setattr(namespace, "date_windows", [(start_date, end_date)])

class DateWindowAction(argparse.Action):
    """An argparse Action for handling date window CLI args"""

    def __call__(self, parser, namespace, values, option_string=None):
        today = datetime.date.today()
        end_date = today + datetime.timedelta(days=1)
        start_date, end_date = _process_date_action_dest(self.dest, values, end_date)

        raiseifnot(
            isinstance(start_date, datetime.date),
            "start date must be a datetime.date object",
        )
        raiseifnot(
            isinstance(end_date, datetime.date),
            "end date must be a datetime.date object",
        )

        date_windows = get_date_windows(
            start_date, end_date, reverse=getattr(namespace, "reverse_windows", False)
        )
        setattr(namespace, "date_windows", date_windows)

@Parent()
@Arg("--today", nargs=0, action=DateTimeWindowAction, help="Use today as date range")
@Arg(
    "--yesterday",
    nargs=0,
    action=DateTimeWindowAction,
    help="Use yesterday as date range",
)
@Arg(
    "--days-back",
    type=int,
    action=DateTimeWindowAction,
    help="Use date range starting N days back",
)
@Arg(
    "--hours-back",
    type=int,
    action=DateTimeWindowAction,
    help="Use date range starting N hours back",
)
@Arg(
    "--date-range",
    nargs="+",
    metavar=("start_date", "end_date"),
    type=str,
    action=DateTimeWindowAction,
    help="Specify custom date range start/end",
)
@Arg(
    "--date-window-size",
    type=float,
    action=DateTimeWindowAction,
    help="Date window size in hours",
)
def datetime_window_cli():
    """An argparse parent CLI that adds datetime window support"""
    pass

@Parent()
@Arg("--today", nargs=0, action=DateWindowAction, help="Use today as date range")
@Arg(
    "--yesterday", nargs=0, action=DateWindowAction, help="Use yesterday as date range"
)
@Arg(
    "--days-back",
    type=int,
    action=DateWindowAction,
    help="Use date range starting N days back",
)
@Arg(
    "--date-range",
    nargs="+",
    metavar=("start_date", "end_date"),
    type=str,
    action=DateWindowAction,
    help="Specify custom date range start/end",
)
def date_window_cli():
    """An argparse parent CLI that adds date window support"""
    pass

# -------- Config utils

def _config_helper(data, key):
    if key:
        if callable(key):
            return key(data)
        return data[key]
    return data

def load_json_config(filename, key=None):
    """Load a config from a json file, optionally extracting a key"""
    with open(filename, "r") as f:
        config = json.load(f)
    return _config_helper(config, key)

def load_yaml_config(filename, key=None):
    """Load a config from a yaml file, optionally extracting a key"""
    with open(filename, "r") as f:
        config = yaml.load(f, Loader=yaml.FullLoader)
    return _config_helper(config, key)

def load_ini_config(filename, key=None):
    """Load a config from an ini file, optionally extracting a key"""
    config = configparser.ConfigParser()
    config.read(filename)
    return _config_helper(config, key)

# -------- Type utils

def is_function(f):
    """Test if f is a function"""
    return isinstance(f, (types.FunctionType, functools.partial))

def is_pandas(o):
    """Test if an object is a Pandas object"""
    return isinstance(o, (pd.DataFrame, pd.Series))

def is_file_obj(o):
    """Test if an object is a file object"""
    return isinstance(o, (io.TextIOBase, io.BufferedIOBase, io.RawIOBase, io.IOBase))

# -------- Iterable utils

def nchunks(a, n):
    """Divide iterable a into n chunks"""
    k, m = divmod(len(a), n)
    return (a[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n))

def divide_data(data, n):
    """Divide data into n chunks, with special handling for pandas objects"""
    if is_pandas(data):
        return np.array_split(data, n)
    else:
        return nchunks(data, n)

def flatten(l):
    """Flatten a list of iterables"""
    raiseifnot(
        l and len(l) > 0, "flatten requires a list of lists or list of pandas objects"
    )
    if is_pandas(l[0]):
        return pd.concat(l)
    else:
        return list(itertools.chain.from_iterable(l))

def size(o, default=None):
    """Helper to return the len() of an object if it is available"""
    if hasattr(o, "__len__"):
        return len(o)
    return default

# https://stackoverflow.com/questions/6822725/rolling-or-sliding-window-iterator
def window(seq, size=2):
    """Returns a sliding window over data from the iterable"""
    it = iter(seq)
    result = tuple(itertools.islice(it, size))
    if len(result) == size:
        yield result
    for elem in it:
        result = result[1:] + (elem,)
        yield result

def iterize(o):
    """Automatically wrap certain objects that you would not normally process
    item by item.

    TODO: this function should probaly be improved/generalized.
    """
    if (
        is_pandas(o)
        or is_str(o)
        or is_file_obj(o)
        or isinstance(o, dict)
        or callable(o)
    ):
        return [o]
    return o

def listify(o):
    """Ensure an object is a list by wrapping if necessary"""
    if isinstance(o, list):
        return o
    return [o]

def join(tables, on=None, how="left", rsuffixes=None):
    """Join a list of iterables or DataFrames"""

    if rsuffixes:
        raiseifnot(
            len(rsuffixes) == (len(tables) - 1), "Must have one rsuffix per right table"
        )

    convert_df = True
    if isinstance(tables[0], pd.DataFrame):
        raiseifnot(
            all([isinstance(t, pd.DataFrame) for t in tables]),
            "Expected all items to be DataFrames",
        )
        convert_df = False

    df = None
    for i, table in enumerate(tables):
        if convert_df:
            table = pd.DataFrame.from_records(table, index=on, coerce_float=True)
            # https://pandas.pydata.org/pandas-docs/stable/user_guide/missing_data.html
            # Force new dtypes to avoid int->float coersion when null values are present
            table = table.convert_dtypes()

        if i == 0:
            df = table
            continue

        rsuffix = ""
        if rsuffixes:
            rsuffix = rsuffixes[i - 1]

        df = df.join(table, on=on, how=how, rsuffix=rsuffix)

    if not convert_df:
        return df

    if how != "outer":
        df = df.reset_index()
    df.replace({np.nan: None}, inplace=True)
    rows = df.to_dict(orient="records")
    return rows

# -------- File utils

def open_filepath_or_buffer(f, open_flags="r", compression=None, is_text=True):
    """Use pandas IO functions to return a handle from a filepath
    or buffer.

    Parameters
    ----------
    f : str or buffer
        filepath or buffer to open
    open_flags : str, optional
        mode to open file
    compression : str, optional
        compression arg passed to pandas functions
    is_text : bool
        Whether file/buffer is in text format, Passed through to pandas helpers.

    Returns
    -------
    f : file-like
        A file-like object
    handles : list of file-like
        A list of file-like objects opened. Seems mostly relevant for zipped archives.
    close : bool
        A flag indicating whether the caller should close the file object when done

    """
    f, _, compression, should_close = get_filepath_or_buffer(f, compression=compression)

    close = False or should_close
    if isinstance(f, str):
        close = True

    f, handles = get_handle(f, open_flags, compression=compression, is_text=is_text)

    # TODO: review behavior when handles has multiple files
    return f, handles, close

@contextmanager
def get_file_handle(*args, **kwargs):
    """Context manager pass through to open_filepath_or_buffer. This will
    automatically close the file if and only if it was opened here. If file
    handles are passed in it is assumed the caller will manage them."""
    f, _, close = open_filepath_or_buffer(*args, **kwargs)
    try:
        yield f
    finally:
        if close:
            f.close()

# -------- Excel utils

def excel_file_type(f):
    """Best guess at Excel file type from name"""
    if isinstance(f, str):
        if f.endswith(XLS):
            return XLS
        if f.endswith(XLSX):
            return XLSX
        raise AssertionError("Unsupported Excel file: %s" % f)
    else:
        if hasattr(f, "name") and f.name.endswith(XLS):
            return XLS
        # Just assumes it's an .xlsx file
        return XLSX

def read_excel(f, **kwargs):
    """Read data from an Excel file using pyexcel

    Parameters
    ----------
    f : str or buffer
        Excel file to read from
    **kwargs
        Keyword arguments passed to pyexcel

    """
    excel_type = excel_file_type(f)
    if excel_type == XLS:
        data = get_xls(f, **kwargs)
    else:
        data = get_xlsx(f, **kwargs)
    return data

def save_excel(f, data, **kwargs):
    """Write data to an Excel file using pyexcel

    Note
    ----
    If f is a file that ends in .xls, pyexcel_xls will be used, otherwise it
    defaults to pyexcel_xlsx.

    Parameters
    ----------
    f : str or buffer
        Excel file to write to
    data : dict
        Data to write to the file. This is expected to be a dict of
        {sheet_name: sheet_data} format.
    **kwargs
        Keyword arguments passed to pyexcel's save_data

    """
    excel_type = excel_file_type(f)
    if excel_type == XLS:
        save_xls(f, data, **kwargs)
    else:
        save_xlsx(f, data, **kwargs)

# -------- Asyncio utils

def get_or_create_event_loop():
    """Get an existing asyncio Event Loop or create one if necessary"""
    created = False
    try:
        loop = asyncio.get_running_loop()
    except RuntimeError:
        loop = asyncio.get_event_loop()
        if loop.is_closed():
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
            created = True
    return loop, created

def cancel_asyncio_tasks(tasks, loop, cancel_timeout=None):
    """Cancel a set of asyncio tasks

    Parameters
    ----------
    tasks : iterable
        An iterable of asyncio tasks to cancel
    loop
        asyncio Event Loop
    cancel_timeout : int or float, optional
        A timeout to use when waiting for tasks to finish cancelling

    """
    [task.cancel() for task in tasks]
    _, not_done = loop.run_until_complete(asyncio.wait(tasks, timeout=cancel_timeout))
    if not_done:
        warn("%d tasks did not cancel immediately" % len(not_done))

# -------- Logging utils

def dbg(msg, **kwargs):
    """Call tlbx dbg with glide logger"""
    kwargs["logger"] = kwargs.get("logger", default_logger)
    kwargs["label"] = kwargs.get("label", get_caller())
    _dbg(msg, **kwargs)

def dbgsql(msg, **kwargs):
    """Call tlbx dbgsql with glide logger"""
    kwargs["logger"] = kwargs.get("logger", default_logger)
    kwargs["label"] = kwargs.get("label", get_caller())
    _dbgsql(msg, **kwargs)

def info(msg, **kwargs):
    """Call tlbx info with glide logger"""
    kwargs["logger"] = kwargs.get("logger", default_logger)
    kwargs["label"] = kwargs.get("label", get_caller())
    _info(msg, **kwargs)

def warn(msg, **kwargs):
    """Call tlbx warn with glide logger"""
    kwargs["logger"] = kwargs.get("logger", default_logger)
    kwargs["label"] = kwargs.get("label", get_caller())
    _warn(msg, **kwargs)

def error(msg, **kwargs):
    """Call tlbx error with glide logger"""
    kwargs["logger"] = kwargs.get("logger", default_logger)
    kwargs["label"] = kwargs.get("label", get_caller())
    _error(msg, **kwargs)

max_stars_repo_name: LemontechSA/glide

max_issues_repo_name: LemontechSA/glide

max_forks_repo_name: LemontechSA/glide


text:
import numpy as np
import pandas as pd
import os
import glob
import re
import time
import datetime
import itertools

import matplotlib.pyplot as plt

filename = 'Sine_Long.csv'
print("Plotting Input file: ",filename)

# load dataframes and reindex
Xdf_loc = pd.read_csv(filename, sep=" ", header = None,)

# print(Xdf_loc.iloc[:3])

Xdf_loc['Milliseconds'] = Xdf_loc[0]

plt.figure()

Xdf_loc[1].plot()

plt.show()

ext: py

reason: import numpy


text:
import pandas as pd
import requests
import numpy as np
import xlrd
import csv
import redis
import json
import operator
import re

def get_spreadsheet(key):
	fileobj_csv = requests.get('https://docs.google.com/spreadsheet/ccc?key='+key+'&output=csv').text
	return fileobj_csv
def find_dataset(identifier,datasetid):
	dataset = None
	dataObj = json_loads_redis(identifier)
	for item in dataObj['datasets']:
		if item['_id'] == datasetid:
			dataset = item['filters']
			break
	return dataset
def store_in_redis(Identifier,strobj):
	conn = redis.Redis()
	conn.set(Identifier,strobj)

def json_loads_redis(identifier):
	conn = redis.Redis()
	strobj = conn.get(identifier)
	return json.loads(strobj)

def ret_sheetnames(fileobj):
	wb = xlrd.open_workbook(file_contents = fileobj.read())
	return {'sheet_names':wb.sheet_names()}

class Filters:
	@classmethod
	def limit(self,df,limit):
		limit = len(df) if limit >len(df) else limit
		return df.head(limit)
	@classmethod
	def limit_obj(self,df,lim,identifier,skip):
		skip = 0 if skip == None else skip
		lim = len(df)-skip if len(df)<(skip+lim) else lim
		return ObjFile.obj(df,identifier,lim,skip)
	@classmethod
	def range(self,df,fro,to):
		return df[fro:to]
	@classmethod
	def sort_col(self,df,column,sort):
		try:
			isAsc = True if sort == 1 else False
			return df.sort_index(by = column, ascending = isAsc)
		except:
			return df
	@classmethod
	def obj_where(self,df,column,values):
		try:
			return df.ix[[item in values for item in df[column]]]
		except:
			return df
	@classmethod
	def groupby(self,df,colgroup,colagr,agr):
		try:
			if agr == 'count':
				return pd.DataFrame(df.groupby(colgroup)[colagr].count()).reset_index()
			elif agr == 'sum':
				return pd.DataFrame(df.groupby(colgroup)[colagr].sum()).reset_index()
			elif agr == 'mean':
				return pd.DataFrame(df.groupby(colgroup)[colagr].mean()).reset_index()
			elif agr == 'max':
				return pd.DataFrame(df.groupby(colgroup)[colagr].max()).reset_index()
			elif agr == 'min':
				return pd.DataFrame(df.groupby(colgroup)[colagr].min()).reset_index()
			else:
				return df
		except:
			return df

class DataView():
	@classmethod
	def return_df(self,identifier):
		data = json_loads_redis(identifier)['data']
		return pd.DataFrame(data)
	@classmethod
	def filter_view(self,identifier,limit=None,skip=None,filters=None):
		df = self.return_df(identifier)
		if(filters!=None):
			filters = sorted(filters,key=operator.itemgetter('priority'))
			for item in filters:
				if (item['select'] != ''):
					df = Filters.obj_where(df,item['column'],item['select'])
				if (item['aggregate']['colagr']!=''):
					df = Filters.groupby(df,item['column'],item['aggregate']['colagr'],item['aggregate']['agr'])
					print df
				if (item['sort'] != ''):
					df = Filters.sort_col(df,item['column'],int(item['sort']))
				if (item['limit']!=''):
					df = Filters.limit(df,int(item['limit']))

		limit = len(df) if limit == None else limit
		return Filters.limit_obj(df,limit,identifier,skip)
	@classmethod
	def chart_view(self,identifier,datasetid):
		dataset = find_dataset(identifier,datasetid)
		df = self.return_df(identifier)
		return self.filter_view(identifier,filters = dataset)

class ObjFile():
	@classmethod
	def putString(self,obj):
		obj = json.dumps(obj)
		return str(obj)
	@classmethod
	def obj(self,df,identifier,limit,skip):
		dataObj = {'data':{},
		 'columns':None,
		 'unique':{},
		 'types':None,
		 'identifier':None,
		 'numrow':0
		}
		dataObj['identifier'] =identifier
		dataObj['columns'] = df.columns.tolist()
		dataObj['types'] = []
		dataObj['numrow'] = len(df)
		for column in df.columns:
			dataObj['data'][column] = df[column].values.tolist()[skip:skip+limit]
			try:
				dataObj['types'].append(str(type(df[column].values[0])))
			except:
				dataObj['types']=None
			dataObj['unique'][column] = df[column].dropna().unique().tolist()
		return  dataObj

class Response():
	message = {
			'status':None,
			'error_message':None
			}

	@classmethod
	def success(self):
		self.message['status'] = 200
		self.message['error_message'] = None
		return self.message
	@classmethod
	def invalid_head(self):
		self.message['status'] = 500
		self.message['error_message'] = 'Header should be unique and cannot be null'
		return self.message
	@classmethod
	def	non_unique_dt(self,cols):
		self.message['status'] = 500
		error_cols = ",".join(cols)
		self.message['error_message'] = 'non unique datatypes in columns '+ error_cols
		return self.message
	@classmethod
	def file_not_sup(self):
		self.message['status'] = 500
		self.message['error_message'] = 'file not supported or corrupted'
		return self.message
	@classmethod
	def unsup_spreadsheet(self):
		self.message['status'] = 500
		self.message['error_message'] = 'spreadsheet not opening'
		return self.message

class Validator():
	@classmethod
	def validate_header(self,fileobj,filetype,sheetno):
		is_valid_head = True
		values = []
		if(filetype == 'xls') or (filetype == 'xlsx'):
			wb = xlrd.open_workbook(file_contents = fileobj.read())
			sheet = wb.sheet_by_index(sheetno)
			for i in range(0,sheet.ncols):
				cell = sheet.cell(0,i)
				if(cell.ctype == xlrd.XL_CELL_EMPTY):
					is_valid_head = False
					break
				else:
					if cell.value not in values:
						value = re.sub(r'[.]','_',cell.value)
						values.append(value)
					else:
						is_valid_head = False
						break
		elif filetype == 'csv':
			reader = csv.reader(fileobj)
			header = reader.next()
			for head in header:
				if head == '':
					is_valid_head = False
					break
				else:
					if head not in values:
						value = re.sub(r'[.]','_',head)
						values.append(value)
					else:
						is_valid_head = False
						break

		return is_valid_head

	@classmethod
	def check_unique(self,col_series):
		is_unique = True
		if len(col_series) > 1:
			col_type = type(col_series.values[0])
			col_series = col_series[1:]
			for item in col_series.values:
				if type(item)!= col_type:
					is_unique = False
					break;
		return is_unique

	@classmethod
	def valid_xl_csv(self,filetype,fileobj,Identifier,sheetno):
		res_message = {}
		non_unique_columns = []
		try:
			if(self.validate_header(fileobj,filetype,sheetno)):
				fileobj.seek(0)
				if (filetype == 'xls')|(filetype == 'xlsx'):
					df = pd.read_excel(fileobj,sheetname=sheetno)
				elif filetype == 'csv':
					df = pd.read_csv(fileobj)
				for item in df.columns:
					series = df[item].dropna()
					is_unique = self.check_unique(series)
					if is_unique == False:
						non_unique_columns.append(item)
				if(non_unique_columns):
					res_message = Response.non_unique_dt(non_unique_columns)
				else:
					res_message = Response.success()
					dataobject = ObjFile.obj(df,Identifier,len(df),0)
					store_in_redis(Identifier,ObjFile.putString(dataobject))

			else:
				res_message = Response.invalid_head()
		except:
			res_message = Response.file_not_sup()
		return res_message

	@classmethod
	def valid_spreadsheet(self,key):
		try:
			fileobj_csv = get_spreadsheet(key)
			return self.valid_xl_csv('csv',fileobj_csv,key)
		except:
			return Response.unsup_spreadsheet()

	@classmethod
	def validate(self,Identifier,types,sheetno,fileobj=None,formats=None):
		if types == 'link':
			return self.valid_spreadsheet(Identifier)
		if types == 'file':
			return self.valid_xl_csv(formats,fileobj,Identifier,sheetno)
'''f=open('testfile/2.xls','rb')
a=Validator.validate('123','file',0,f,'xls')
print a'''
'''a={
	'identifier':'123',
	'limit':10,
	'skip':0,
	'filters':[
                {
                    "limit" : "10",
                    "sort" : "1",
                    "select" : [
                        "ANDHRA PRADESH",
                        "ARUNACHAL PRADESH",
                        "LAKSHADWEEP",
                        "PUDUCHERRY"
                    ],
                    "column" : "STATE",
                    "priority" : 1,
                    "aggregate":{'colagr':'','agr':''}
                }
            ]
}
print DataView.filter_view(a['identifier'],a['limit'],a['skip'])'''
'''print DataView.chart_view('123','550665e2f94139fc049a8ccd')'''

max_stars_repo_name: CSC-ORG/Dynamic-dashboard

max_issues_repo_name: CSC-ORG/Dynamic-dashboard

max_forks_repo_name: CSC-ORG/Dynamic-dashboard


text:
##################### generated by xml-casa (v2) from subvs.xml #####################
##################### 359882fd4602fc4cb310cc193180fe92 ##############################
from __future__ import absolute_import
from casashell.private.stack_manip import find_local as __sf__
from casashell.private.stack_manip import find_frame as _find_frame
from casatools.typecheck import validator as _pc
from casatools.coercetype import coerce as _coerce
from suncasatasks import subvs as _subvs_t
from collections import OrderedDict
import numpy
import sys
import os

import shutil

def static_var(varname, value):
    def decorate(func):
        setattr(func, varname, value)
        return func
    return decorate

class _subvs:
    """
    subvs ---- Vector-subtraction in UV using selected time ranges and spectral channels as background

    Split is the general purpose program to make a new data set that is a
    subset or averaged form of an existing data set.  General selection
    parameters are included, and one or all of the various data columns
    (DATA, LAG_DATA and/or FLOAT_DATA, and possibly MODEL_DATA and/or
    CORRECTED_DATA) can be selected.

    Split is often used after the initial calibration of the data to make a
    smaller measurement set with only the data that will be used in
    further flagging, imaging and/or self-calibration.  split can
    average over frequency (channels) and time (integrations).

    --------- parameter descriptions ---------------------------------------------

    vis         Name of input measurement set
    outputvis   Name of output measurement set
    timerange   Select the time range of the input visbility to be subtracted from
    spw         Select the spectral channels of the input visibility to be subtracted from
    mode        Operation: linear, highpass
    subtime1    Select the first time range as the background for uv subtraction
    subtime2    Select the second time range as the background for uv subtraction
    smoothaxis  Select the axis along which smooth is performed
    smoothtype  Select the smooth type
    smoothwidth Select the width of the smoothing window
    splitsel    Split the selected timerange and spectral channels as outputvis
    reverse     Reverse the sign of the background-subtracted data (for absorptive structure)
    overwrite   Overwrite the already existing output measurement set

    --------- examples -----------------------------------------------------------

    Subvs is a task to do UV vector-subtraction, by selecting time ranges
    in the data as background. Subvs can be used to subtract the background
    continuum emission to separate the time-dependent emission, e.g. solar
    coherent radio bursts.

    Keyword arguments:
    vis -- Name of input visibility file (MS)
    default: none; example: vis='ngc5921.ms'
    outputvis -- Name of output uv-subtracted visibility file (MS)
    default: none; example: outputvis='ngc5921_src.ms'
    timerange -- Time range of performing the UV subtraction:
    default='' means all times.  examples:
    timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
    timerange = 'hh:mm:ss~hh:mm:ss'
    spw -- Select spectral window/channel.
    default = '' all the spectral channels. Example: spw='0:1~20'
    mode -- operation mode
    default 'linear'
    mode = 'linear': use a linear fit for the background to be subtracted
    mode = 'lowpass': act as a lowpass filter---smooth the data using different smooth
    types and smooth window size. Can be performed along either time
    or frequency axis
    mode = 'highpass': act as a highpass filter---smooth the data first, and
    subtract the smoothed data from the original. Can be performed along either time
    or frequency axis
    mode = 'linear' expandable parameters:
    subtime1 -- Time range 1 of the background to be subtracted from the data
    default='' means all times.  format:
    timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
    timerange = 'hh:mm:ss~hh:mm:ss'
    subtime2 -- Time range 2 of the backgroud to be subtracted from the data
    default='' means all times.  examples:
    timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
    timerange = 'hh:mm:ss~hh:mm:ss'
    mode = 'lowpass' or 'highpass' expandable parameters:
    smoothaxis -- axis of smooth
    Default: 'time'
    smoothaxis = 'time': smooth is along the time axis
    smoothaxis = 'freq': smooth is along the frequency axis
    smoothtype -- type of the smooth depending on the convolving kernel
    Default: 'flat'
    smoothtype = 'flat': convolving kernel is a flat rectangle,
    equivalent to a boxcar moving smooth
    smoothtype = 'hanning': Hanning smooth kernel. See numpy.hanning
    smoothtype = 'hamming': Hamming smooth kernel. See numpy.hamming
    smoothtype = 'bartlett': Bartlett smooth kernel. See numpy.bartlett
    smoothtype = 'blackman': Blackman smooth kernel. See numpy.blackman
    smoothwidth -- width of the smooth kernel
    Default: 5
    Examples: smoothwidth=5, meaning the width is 5 pixels
    splitsel -- True or False. default = False. If splitsel = False, then the entire input
    measurement set is copied as the output measurement set (outputvis), with
    background subtracted at selected timerange and spectral channels.
    If splitsel = True,then only the selected timerange and spectral channels
    are copied into the output measurement set (outputvis).
    reverse -- True or False. default = False. If reverse = False, then the times indicated
    by subtime1 and/or subtime2 are treated as background and subtracted; If reverse
    = True, then reverse the sign of the background-subtracted data. The option can
    be used for mapping absorptive structure.
    overwrite -- True or False. default = False. If overwrite = True and
    outputvis already exists, the selected subtime and spw in the
    output measurment set will be replaced with background subtracted
    visibilities

    """

    _info_group_ = """misc"""
    _info_desc_ = """Vector-subtraction in UV using selected time ranges and spectral channels as background"""

    __schema = {'vis': {'type': 'cReqPath', 'coerce': _coerce.expand_path}, 'outputvis': {'type': 'cPath', 'coerce': _coerce.expand_path}, 'timerange': {'type': 'cStr', 'coerce': _coerce.to_str}, 'spw': {'type': 'cStr', 'coerce': _coerce.to_str}, 'mode': {'type': 'cStr', 'coerce': _coerce.to_str, 'allowed': [ 'linear', 'lowpass', 'highpass' ]}, 'subtime1': {'type': 'cStr', 'coerce': _coerce.to_str}, 'subtime2': {'type': 'cStr', 'coerce': _coerce.to_str}, 'smoothaxis': {'type': 'cStr', 'coerce': _coerce.to_str}, 'smoothtype': {'type': 'cStr', 'coerce': _coerce.to_str}, 'smoothwidth': {'type': 'cInt'}, 'splitsel': {'type': 'cBool'}, 'reverse': {'type': 'cBool'}, 'overwrite': {'type': 'cBool'}}

    def __init__(self):
        self.__stdout = None
        self.__stderr = None
        self.__root_frame_ = None

    def __globals_(self):
        if self.__root_frame_ is None:
            self.__root_frame_ = _find_frame( )
            assert self.__root_frame_ is not None, "could not find CASAshell global frame"
        return self.__root_frame_

    def __to_string_(self,value):
        if type(value) is str:
            return "'%s'" % value
        else:
            return str(value)

    def __validate_(self,doc,schema):
        return _pc.validate(doc,schema)

    def __do_inp_output(self,param_prefix,description_str,formatting_chars):
        out = self.__stdout or sys.stdout
        description = description_str.split( )
        prefix_width = 23 + 14 + 4
        output = [ ]
        addon = ''
        first_addon = True
        while len(description) > 0:
            ## starting a new line.....................................................................
            if len(output) == 0:
                ## for first line add parameter information............................................
                if len(param_prefix)-formatting_chars > prefix_width - 1:
                    output.append(param_prefix)
                    continue
                addon = param_prefix + ' #'
                first_addon = True
                addon_formatting = formatting_chars
            else:
                ## for subsequent lines space over prefix width........................................
                addon = (' ' * prefix_width) + '#'
                first_addon = False
                addon_formatting = 0
            ## if first word of description puts us over the screen width, bail........................
            if len(addon + description[0]) - addon_formatting + 1 > self.term_width:
                ## if we're doing the first line make sure it's output.................................
                if first_addon: output.append(addon)
                break
            while len(description) > 0:
                ## if the next description word puts us over break for the next line...................
                if len(addon + description[0]) - addon_formatting + 1 > self.term_width: break
                addon = addon + ' ' + description[0]
                description.pop(0)
            output.append(addon)
        out.write('\n'.join(output) + '\n')

    #--------- return nonsubparam values ----------------------------------------------

    def __vis_dflt( self, glb ):
        return ''

    def __vis( self, glb ):
        if 'vis' in glb: return glb['vis']
        return ''

    def __spw_dflt( self, glb ):
        return ''

    def __spw( self, glb ):
        if 'spw' in glb: return glb['spw']
        return ''

    def __mode_dflt( self, glb ):
        return 'linear'

    def __mode( self, glb ):
        if 'mode' in glb: return glb['mode']
        return 'linear'

    def __outputvis_dflt( self, glb ):
        return ''

    def __outputvis( self, glb ):
        if 'outputvis' in glb: return glb['outputvis']
        return ''

    def __timerange_dflt( self, glb ):
        return ''

    def __timerange( self, glb ):
        if 'timerange' in glb: return glb['timerange']
        return ''

    def __splitsel_dflt( self, glb ):
        return True

    def __splitsel( self, glb ):
        if 'splitsel' in glb: return glb['splitsel']
        return True

    def __reverse_dflt( self, glb ):
        return False

    def __reverse( self, glb ):
        if 'reverse' in glb: return glb['reverse']
        return False

    def __overwrite_dflt( self, glb ):
        return False

    def __overwrite( self, glb ):
        if 'overwrite' in glb: return glb['overwrite']
        return False

    #--------- return inp/go default --------------------------------------------------
    def __smoothaxis_dflt( self, glb ):
        if self.__mode( glb ) == "lowpass": return "time"
        if self.__mode( glb ) == "highpass": return "time"
        return None
    def __smoothwidth_dflt( self, glb ):
        if self.__mode( glb ) == "lowpass": return int(5)
        if self.__mode( glb ) == "highpass": return int(5)
        return None
    def __subtime2_dflt( self, glb ):
        if self.__mode( glb ) == "linear": return ""
        return None
    def __subtime1_dflt( self, glb ):
        if self.__mode( glb ) == "linear": return ""
        return None
    def __smoothtype_dflt( self, glb ):
        if self.__mode( glb ) == "lowpass": return "hanning"
        if self.__mode( glb ) == "highpass": return "hanning"
        return None

    #--------- return subparam values -------------------------------------------------
    def __subtime1( self, glb ):
        if 'subtime1' in glb: return glb['subtime1']
        dflt = self.__subtime1_dflt( glb )
        if dflt is not None: return dflt
        return ''
    def __subtime2( self, glb ):
        if 'subtime2' in glb: return glb['subtime2']
        dflt = self.__subtime2_dflt( glb )
        if dflt is not None: return dflt
        return ''
    def __smoothaxis( self, glb ):
        if 'smoothaxis' in glb: return glb['smoothaxis']
        dflt = self.__smoothaxis_dflt( glb )
        if dflt is not None: return dflt
        return 'time'
    def __smoothtype( self, glb ):
        if 'smoothtype' in glb: return glb['smoothtype']
        dflt = self.__smoothtype_dflt( glb )
        if dflt is not None: return dflt
        return 'flat'
    def __smoothwidth( self, glb ):
        if 'smoothwidth' in glb: return glb['smoothwidth']
        dflt = self.__smoothwidth_dflt( glb )
        if dflt is not None: return dflt
        return int(5)

    #--------- subparam inp output ----------------------------------------------------
    def __vis_inp(self):
        description = ''
        value = self.__vis( self.__globals_( ) )
        (pre,post) = ('','') if self.__validate_({'vis': value},{'vis': self.__schema['vis']}) else ('\x1B[91m','\x1B[0m')
        self.__do_inp_output('%-14.14s = %s%-23s%s' % ('vis',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
    def __outputvis_inp(self):
        description = ''
        value = self.__outputvis( self.__globals_( ) )
        (pre,post) = ('','') if self.__validate_({'outputvis': value},{'outputvis': self.__schema['outputvis']}) else ('\x1B[91m','\x1B[0m')
        self.__do_inp_output('%-14.14s = %s%-23s%s' % ('outputvis',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
    def __timerange_inp(self):
        description = ''
        value = self.__timerange( self.__globals_( ) )
        (pre,post) = ('','') if self.__validate_({'timerange': value},{'timerange': self.__schema['timerange']}) else ('\x1B[91m','\x1B[0m')
        self.__do_inp_output('%-14.14s = %s%-23s%s' % ('timerange',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
    def __spw_inp(self):
        description = ''
        value = self.__spw( self.__globals_( ) )
        (pre,post) = ('','') if self.__validate_({'spw': value},{'spw': self.__schema['spw']}) else ('\x1B[91m','\x1B[0m')
        self.__do_inp_output('%-14.14s = %s%-23s%s' % ('spw',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
    def __mode_inp(self):
        description = ''
        value = self.__mode( self.__globals_( ) )
        (pre,post) = ('','') if self.__validate_({'mode': value},{'mode': self.__schema['mode']}) else ('\x1B[91m','\x1B[0m')
        self.__do_inp_output('\x1B[1m\x1B[47m%-14.14s =\x1B[0m %s%-23s%s' % ('mode',pre,self.__to_string_(value),post),description,13+len(pre)+len(post))
    def __subtime1_inp(self):
        if self.__subtime1_dflt( self.__globals_( ) ) is not None:
             description = ''
             value = self.__subtime1( self.__globals_( ) )
             (pre,post) = ('','') if self.__validate_({'subtime1': value},{'subtime1': self.__schema['subtime1']}) else ('\x1B[91m','\x1B[0m')
             self.__do_inp_output('   \x1B[92m%-11.11s =\x1B[0m %s%-23s%s' % ('subtime1',pre,self.__to_string_(value),post),description,9+len(pre)+len(post))
    def __subtime2_inp(self):
        if self.__subtime2_dflt( self.__globals_( ) ) is not None:
             description = ''
             value = self.__subtime2( self.__globals_( ) )
             (pre,post) = ('','') if self.__validate_({'subtime2': value},{'subtime2': self.__schema['subtime2']}) else ('\x1B[91m','\x1B[0m')
             self.__do_inp_output('   \x1B[92m%-11.11s =\x1B[0m %s%-23s%s' % ('subtime2',pre,self.__to_string_(value),post),description,9+len(pre)+len(post))
    def __smoothaxis_inp(self):
        if self.__smoothaxis_dflt( self.__globals_( ) ) is not None:
             description = ''
             value = self.__smoothaxis( self.__globals_( ) )
             (pre,post) = ('','') if self.__validate_({'smoothaxis': value},{'smoothaxis': self.__schema['smoothaxis']}) else ('\x1B[91m','\x1B[0m')
             self.__do_inp_output('   \x1B[92m%-11.11s =\x1B[0m %s%-23s%s' % ('smoothaxis',pre,self.__to_string_(value),post),description,9+len(pre)+len(post))
    def __smoothtype_inp(self):
        if self.__smoothtype_dflt( self.__globals_( ) ) is not None:
             description = ''
             value = self.__smoothtype( self.__globals_( ) )
             (pre,post) = ('','') if self.__validate_({'smoothtype': value},{'smoothtype': self.__schema['smoothtype']}) else ('\x1B[91m','\x1B[0m')
             self.__do_inp_output('   \x1B[92m%-11.11s =\x1B[0m %s%-23s%s' % ('smoothtype',pre,self.__to_string_(value),post),description,9+len(pre)+len(post))
    def __smoothwidth_inp(self):
        if self.__smoothwidth_dflt( self.__globals_( ) ) is not None:
             description = ''
             value = self.__smoothwidth( self.__globals_( ) )
             (pre,post) = ('','') if self.__validate_({'smoothwidth': value},{'smoothwidth': self.__schema['smoothwidth']}) else ('\x1B[91m','\x1B[0m')
             self.__do_inp_output('   \x1B[92m%-11.11s =\x1B[0m %s%-23s%s' % ('smoothwidth',pre,self.__to_string_(value),post),description,9+len(pre)+len(post))
    def __splitsel_inp(self):
        description = ''
        value = self.__splitsel( self.__globals_( ) )
        (pre,post) = ('','') if self.__validate_({'splitsel': value},{'splitsel': self.__schema['splitsel']}) else ('\x1B[91m','\x1B[0m')
        self.__do_inp_output('%-14.14s = %s%-23s%s' % ('splitsel',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
    def __reverse_inp(self):
        description = ''
        value = self.__reverse( self.__globals_( ) )
        (pre,post) = ('','') if self.__validate_({'reverse': value},{'reverse': self.__schema['reverse']}) else ('\x1B[91m','\x1B[0m')
        self.__do_inp_output('%-14.14s = %s%-23s%s' % ('reverse',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
    def __overwrite_inp(self):
        description = ''
        value = self.__overwrite( self.__globals_( ) )
        (pre,post) = ('','') if self.__validate_({'overwrite': value},{'overwrite': self.__schema['overwrite']}) else ('\x1B[91m','\x1B[0m')
        self.__do_inp_output('%-14.14s = %s%-23s%s' % ('overwrite',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))

    #--------- global default implementation-------------------------------------------
    @static_var('state', __sf__('casa_inp_go_state'))
    def set_global_defaults(self):
        self.set_global_defaults.state['last'] = self
        glb = self.__globals_( )
        if 'smoothaxis' in glb: del glb['smoothaxis']
        if 'outputvis' in glb: del glb['outputvis']
        if 'vis' in glb: del glb['vis']
        if 'reverse' in glb: del glb['reverse']
        if 'smoothwidth' in glb: del glb['smoothwidth']
        if 'subtime2' in glb: del glb['subtime2']
        if 'splitsel' in glb: del glb['splitsel']
        if 'mode' in glb: del glb['mode']
        if 'subtime1' in glb: del glb['subtime1']
        if 'overwrite' in glb: del glb['overwrite']
        if 'smoothtype' in glb: del glb['smoothtype']
        if 'spw' in glb: del glb['spw']
        if 'timerange' in glb: del glb['timerange']

    #--------- inp function -----------------------------------------------------------
    def inp(self):
        print("# subvs -- %s" % self._info_desc_)
        self.term_width, self.term_height = shutil.get_terminal_size(fallback=(80, 24))
        self.__vis_inp( )
        self.__outputvis_inp( )
        self.__timerange_inp( )
        self.__spw_inp( )
        self.__mode_inp( )
        self.__subtime1_inp( )
        self.__subtime2_inp( )
        self.__smoothaxis_inp( )
        self.__smoothtype_inp( )
        self.__smoothwidth_inp( )
        self.__splitsel_inp( )
        self.__reverse_inp( )
        self.__overwrite_inp( )

    #--------- tget function ----------------------------------------------------------
    @static_var('state', __sf__('casa_inp_go_state'))
    def tget(self,file=None):
        from casashell.private.stack_manip import find_frame
        from runpy import run_path
        filename = None
        if file is None:
            if os.path.isfile("subvs.last"):
                filename = "subvs.last"
        elif isinstance(file, str):
            if os.path.isfile(file):
                filename = file
        if filename is not None:
            glob = find_frame( )
            newglob = run_path( filename, init_globals={ } )
            for i in newglob:
                glob[i] = newglob[i]
            self.tget.state['last'] = self
        else:
            print("could not find last file, setting defaults instead...")
            self.set_global_defaults( )

    def __call__( self, vis=None, outputvis=None, timerange=None, spw=None, mode=None, subtime1=None, subtime2=None, smoothaxis=None, smoothtype=None, smoothwidth=None, splitsel=None, reverse=None, overwrite=None ):
        def noobj(s):
           if s.startswith('<') and s.endswith('>'):
               return "None"
           else:
               return s
        _prefile = os.path.realpath('subvs.pre')
        _postfile = os.path.realpath('subvs.last')
        _return_result_ = None
        _arguments = [vis,outputvis,timerange,spw,mode,subtime1,subtime2,smoothaxis,smoothtype,smoothwidth,splitsel,reverse,overwrite]
        _invocation_parameters = OrderedDict( )
        if any(map(lambda x: x is not None,_arguments)):
            # invoke python style
            # set the non sub-parameters that are not None
            local_global = { }
            if vis is not None: local_global['vis'] = vis
            if outputvis is not None: local_global['outputvis'] = outputvis
            if timerange is not None: local_global['timerange'] = timerange
            if spw is not None: local_global['spw'] = spw
            if mode is not None: local_global['mode'] = mode
            if splitsel is not None: local_global['splitsel'] = splitsel
            if reverse is not None: local_global['reverse'] = reverse
            if overwrite is not None: local_global['overwrite'] = overwrite

            # the invocation parameters for the non-subparameters can now be set - this picks up those defaults
            _invocation_parameters['vis'] = self.__vis( local_global )
            _invocation_parameters['outputvis'] = self.__outputvis( local_global )
            _invocation_parameters['timerange'] = self.__timerange( local_global )
            _invocation_parameters['spw'] = self.__spw( local_global )
            _invocation_parameters['mode'] = self.__mode( local_global )
            _invocation_parameters['splitsel'] = self.__splitsel( local_global )
            _invocation_parameters['reverse'] = self.__reverse( local_global )
            _invocation_parameters['overwrite'] = self.__overwrite( local_global )

            # the sub-parameters can then be set. Use the supplied value if not None, else the function, which gets the appropriate default
            _invocation_parameters['subtime1'] = self.__subtime1( _invocation_parameters ) if subtime1 is None else subtime1
            _invocation_parameters['subtime2'] = self.__subtime2( _invocation_parameters ) if subtime2 is None else subtime2
            _invocation_parameters['smoothaxis'] = self.__smoothaxis( _invocation_parameters ) if smoothaxis is None else smoothaxis
            _invocation_parameters['smoothtype'] = self.__smoothtype( _invocation_parameters ) if smoothtype is None else smoothtype
            _invocation_parameters['smoothwidth'] = self.__smoothwidth( _invocation_parameters ) if smoothwidth is None else smoothwidth

        else:
            # invoke with inp/go semantics
            _invocation_parameters['vis'] = self.__vis( self.__globals_( ) )
            _invocation_parameters['outputvis'] = self.__outputvis( self.__globals_( ) )
            _invocation_parameters['timerange'] = self.__timerange( self.__globals_( ) )
            _invocation_parameters['spw'] = self.__spw( self.__globals_( ) )
            _invocation_parameters['mode'] = self.__mode( self.__globals_( ) )
            _invocation_parameters['subtime1'] = self.__subtime1( self.__globals_( ) )
            _invocation_parameters['subtime2'] = self.__subtime2( self.__globals_( ) )
            _invocation_parameters['smoothaxis'] = self.__smoothaxis( self.__globals_( ) )
            _invocation_parameters['smoothtype'] = self.__smoothtype( self.__globals_( ) )
            _invocation_parameters['smoothwidth'] = self.__smoothwidth( self.__globals_( ) )
            _invocation_parameters['splitsel'] = self.__splitsel( self.__globals_( ) )
            _invocation_parameters['reverse'] = self.__reverse( self.__globals_( ) )
            _invocation_parameters['overwrite'] = self.__overwrite( self.__globals_( ) )
        try:
            with open(_prefile,'w') as _f:
                for _i in _invocation_parameters:
                    _f.write("%-11s = %s\n" % (_i,noobj(repr(_invocation_parameters[_i]))))
                _f.write("#subvs( ")
                count = 0
                for _i in _invocation_parameters:
                    _f.write("%s=%s" % (_i,noobj(repr(_invocation_parameters[_i]))))
                    count += 1
                    if count < len(_invocation_parameters): _f.write(",")
                _f.write(" )\n")
        except: pass
        try:
            _return_result_ = _subvs_t( _invocation_parameters['vis'],_invocation_parameters['outputvis'],_invocation_parameters['timerange'],_invocation_parameters['spw'],_invocation_parameters['mode'],_invocation_parameters['subtime1'],_invocation_parameters['subtime2'],_invocation_parameters['smoothaxis'],_invocation_parameters['smoothtype'],_invocation_parameters['smoothwidth'],_invocation_parameters['splitsel'],_invocation_parameters['reverse'],_invocation_parameters['overwrite'] )
        except Exception as e:
            from traceback import format_exc
            from casatasks import casalog
            casalog.origin('subvs')
            casalog.post("Exception Reported: Error in subvs: %s" % str(e),'SEVERE')
            casalog.post(format_exc( ))
            _return_result_ = False
        try:
            os.rename(_prefile,_postfile)
        except: pass
        return _return_result_

subvs = _subvs( )


text:
import wpcv
import math
import numpy as np

import math
import numpy as np
from .polygon_utils import organize_quad_points,rotate_points,bounding_rect
##########################Box Operations#############################
def resize_box(box,size):
    cx,cy,w,h=ltrb_to_ccwh(box)
    nw,nh=size
    l=cx-nw//2
    r=cx+nw//2
    t=cy-nh//2
    b=cy+nh//2
    return [l,t,r,b]
def rescale_box(box,scale):
    '''rescale with  image'''
    if isinstance(scale,(tuple,list)):
        scaleX,scaleY=scale
    else:
        scaleX=scaleY=scale
    l,t,r,b=box
    l*=scaleX
    r*=scaleX
    t*=scaleY
    b*=scaleY
    return [l,t,r,b]

def rotate_boxes(boxes,angle,cx,cy,h,w):
    '''原创'''
    boxes=[list(box) for box in boxes]
    quads = [ltrb_to_quad(box) for box in boxes]
    quads=rotate_points(quads,angle,cx,cy,h,w)
    boxes=[bounding_rect(quad) for quad in quads]
    boxes=np.array(boxes)
    return boxes
def rotate_box(box, angle, cx, cy, h, w):
    return rotate_boxes([box],angle,cx,cy,h,w)
def shift_box(box,offset,limits=None):
    l,t,r,b=box
    ofx,ofy=offset
    l+=ofx
    r+=ofx
    t+=ofy
    b+=ofy
    return limit_box([l,t,r,b],limits=limits)
def translate_box(box,offset,limits=None):
    return shift_box(box,offset,limits)
def flip_box_horizontal(box,imsize):
    imw,imh=imsize
    l,t,r,b=box
    l=imw-r
    r=imw-l
    return [l,t,r,b]
def flip_box_vertical(box,imsize):
    imw,imh=imsize
    l,t,r,b=box
    t=imh-b
    b=imh-t
    return [l,t,r,b]
def pad_box(box,pad_size=5,pad_ratio=None,limits=None):
    from math import inf
    limits=limits or (-inf,-inf,inf,inf)
    l,t,r,b=box
    bh=b-t
    bw=r-l
    if pad_ratio is not None:
        if isinstance(pad_ratio,(tuple,list)):
            if len(pad_ratio)==2:
                pad_l=pad_r=int(min(bh,bw)*pad_ratio[0])
                pad_t=pad_b=int(min(bh,bw)*pad_ratio[1])
            else:
                assert len(pad_ratio)==4
                pad_l= int(min(bh, bw) * pad_ratio[0])
                pad_t= int(min(bh, bw) * pad_ratio[1])
                pad_r= int(min(bh, bw) * pad_ratio[2])
                pad_b= int(min(bh, bw) * pad_ratio[3])
        else:
            pad_l=pad_t=pad_r=pad_b=int(min(bh,bw)*pad_ratio)
    else:
        if isinstance(pad_size,(tuple,list)):
            if len(pad_size)==2:
                pad_l=pad_r=pad_size[0]
                pad_t=pad_b=pad_size[1]
            else:
                assert len(pad_size)==4
                pad_l,pad_t,pad_r,pad_b=pad_size
        else:
            pad_l=pad_t=pad_r=pad_b=pad_size
    l-=pad_l
    t-=pad_t
    r+=pad_r
    b+=pad_b
    return limit_box([l,t,r,b],limits=limits)

def limit_box(box,limits=None):
    if limits is None:return box
    if len(limits)==2:
        ml,mt=0,0
        mr,mb=limits
    else:
        assert len(limits)==4
        ml,mt,mr,mb=limits
    l,t,r,b=box
    l=max(ml,l)
    t=max(mt,t)
    r=min(mr,r)
    b=min(mb,b)
    if l>=r:
        return None
    if t>=b:return None
    return [l,t,r,b]
############################Sort Operations#############################
def min_enclosing_box(boxes):
    boxes=np.array(boxes)
    ml=np.min(boxes[:,0])
    mt=np.min(boxes[:,1])
    mr=np.max(boxes[:,2])
    mb=np.max(boxes[:,3])
    return [ml,mt,mr,mb]
def box_in_area(box,size):
    l,t,r,b=box
    mr,mb=size
    if l<0 or t<0 or r>mr or b>mb:
        return False
    else:
        return True
############################ConvertingTools#############################
def ltrb_to_quad(box):
    l,t,r,b=box
    quad=[[l,t],[r,t],[r,b],[l,b]]
    return quad

def ltrb_to_ccwh(box):
    l, t, r, b = box
    w = r - l
    h = b - t
    cx = (l + r) // 2
    cy = (t + b) // 2
    return [cx,cy,w,h]
def ccwh_to_ltrb(box):
    cx,cy,w,h=box
    l=cx-w//2
    t=cy-h//2
    r=cx+w//2
    b=cy+h//2
    return [l,t,r,b]
def oowh_to_ltrb(box):
    '''o:origin'''
    x, y, w, h = box
    return [int(i) for i in (x, y, x + w, y + h)]
def ltrb_to_oowh(box):
    l, t, r, b = box
    return [l, t, r - l, b - t]
def xywh_to_ltrb(box):
    '''same with oowh'''
    x,y,w,h=box
    return [int(i) for i in (x,y,x+w,y+h)]
def ltrb_to_xywh(box):
    l,t,r,b=box
    return [l,t,r-l,b-t]

def quad_to_four_corners(quad):
    (x0, y0), (x1, y1), (x2, y2), (x3, y3) = quad
    return [x0,y0,x1,y1,x2,y2,x3,y3]
def four_corners_to_quad(box):
    x0,y0,x1,y1,x2,y2,x3,y3 = box
    return [(x0, y0), (x1, y1), (x2, y2), (x3, y3)]
###########################Calculations##############################
def calc_iou(box1,box2):
    l1,t1,r1,b1=box1
    l2,t2,r2,b2=box2
    w1,h1=r1-l1,b1-t1
    w2,h2=r2-l2,b2-t2
    width=min(r1,r2)-max(l1,l2)
    height=min(b1,b2)-max(t1,t2)
    width=max(width,0)
    height=max(height,0)
    area=width*height
    return area/(w1*h1+w2*h2-area)
def calc_iou2(box1,box2):
    width=min(box1[2],box2[2])-max(box1[0],box2[0])
    height=min(box1[3],box2[3])-max(box1[1],box2[1])
    width=max(width,0)
    height=max(height,0)
    area=width*height
    return area/((box1[2]-box1[0])*(box1[3]-box1[1])+(box2[2]-box2[0])*(box2[3]-box2[1])-area)
def calc_iou_batch_with_batch(boxes1,boxes2):
    box1=boxes1
    box2=boxes2
    import numpy as np
    box1=np.array(box1)
    box2=np.array(box2)
    width = np.min(np.vstack([box1[...,2], box2[...,2]]),axis=0) - np.max(np.vstack([box1[...,0], box2[...,0]]),axis=0)
    height = np.min(np.vstack([box1[...,3], box2[...,3]]),axis=0) - np.max(np.vstack([box1[...,1], box2[...,1]]),axis=0)
    width = np.clip(width,0,np.inf)
    height = np.clip(height,0,np.inf)
    area = width * height
    return area / ((box1[...,2] - box1[...,0]) * (box1[...,3] - box1[...,1]) + (box2[...,2] - box2[...,0]) * (box2[...,3] - box2[...,1]) - area)
def calc_iou_batch(boxes,box):
    '''calculate iou between batch bboxes with one bbox , 原创'''
    box1=boxes
    box2=box
    import numpy as np
    box1 = np.array(box1)
    box2 = np.array(box2)
    shape=box1.shape
    width = np.min(np.vstack([box1[..., 2], np.full(shape[:-1],box2[2])]), axis=0) - np.max(np.vstack([box1[..., 0], np.full(shape[:-1],box2[0])]),axis=0)
    height = np.min(np.vstack([box1[..., 3], np.full(shape[:-1],box2[3])]), axis=0) - np.max(np.vstack([box1[..., 1], np.full(shape[:-1],box2[1])]),axis=0)
    width = np.clip(width, 0, np.inf)
    height = np.clip(height, 0, np.inf)
    area = width * height
    return area / ((box1[...,2] - box1[...,0]) * (box1[...,3] - box1[...,1]) + (box2[2] - box2[0]) * (box2[3] - box2[1]) - area)

def calc_box_area(box):
    l,t,r,b=box
    return (r-l)*(b-t)

max_stars_repo_name: Peiiii/detro

max_issues_repo_name: Peiiii/detro

max_forks_repo_name: Peiiii/detro


text:
import torch
from torch import nn
from torchvision import models, transforms
import torchvision
import os
from time import time
import onnx
import onnxruntime
from onnx import numpy_helper
import numpy as np

def get_test_data(dataloader, size):
    X_test, Y_test = next(iter(dataloader))
    batch_size = len(X_test)
    n = size//batch_size
    for i, batch in enumerate(dataloader):
        if i < n:
            X_tmp, Y_tmp = batch
            X_test = torch.cat((X_test, X_tmp), 0)
            Y_test = torch.cat((Y_test, Y_tmp), 0)
    return X_test, Y_test

input_size = 224
batch_size = 128

mean=[0.485, 0.456, 0.406]
std=[0.229, 0.224, 0.225]

transformresnetTrain=transforms.Compose([ # Cette fois on utilise pas de grayscale car nous avons un gros modele pré-entrainé
        transforms.RandomResizedCrop(input_size), # selection aléatoire d'une zone de la taille voulue (augmentation des données en apprentissage)
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize(mean, std)
    ])
transformresnetTest=transforms.Compose([
        transforms.Resize(input_size), # selection de la zone centrale de la taille voulue
        transforms.CenterCrop(input_size),
        transforms.ToTensor(),
        transforms.Normalize(mean, std)
    ])

resnet_trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transformresnetTrain)
resnet_trainloader = torch.utils.data.DataLoader(resnet_trainset, batch_size=batch_size, pin_memory=True, shuffle=True)

resnet_testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transformresnetTest)
resnet_testloader = torch.utils.data.DataLoader(resnet_testset, batch_size=batch_size, pin_memory=True, shuffle=True)

X_test, Y_test = get_test_data(resnet_testloader, 1000)

PATH = "models/"
model = models.resnet18(pretrained=True)
model.fc = nn.Linear(512, 10)
model.load_state_dict(torch.load(os.path.join(PATH,"resnet.pth"),map_location='cpu'))
model.eval()

# for t in (20,40,60,80,100,120):
#     t0 = time()
#     model(X_test[:t])
#     print("FPS:", t, " --> seconds:", (time() - t0))

test_size = 100
dummy_input = torch.randn(test_size, 3, input_size, input_size)
torch.onnx.export(model,
                  dummy_input,
                  str(PATH+"resnet.onnx"),
                  export_params=True,
                  do_constant_folding=True,
                  input_names = ['modelInput'],
                  output_names = ['modelOutput'])

X_test = X_test[:test_size]

t0 = time()
pred = model(X_test)
print("Time for ",test_size," images without ONNX inference", (time() - t0))
print(pred.shape)

sess = onnxruntime.InferenceSession(str(PATH+"resnet.onnx"))
input_name = sess.get_inputs()[0].name
output_name = sess.get_outputs()[0].name
t0 = time()
pred = sess.run([output_name], {input_name: np.array(X_test).astype(np.float32)})[0]
print("Time for ",test_size," images with ONNX inference", (time() - t0))
print(pred.shape)

max_stars_repo_name: hanzopgp/JetsonAutonomousDriving

max_issues_repo_name: hanzopgp/JetsonAutonomousDriving

max_forks_repo_name: hanzopgp/JetsonAutonomousDriving


text:
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle
from datetime import datetime
from experiment_handler.time_synchronisation import convert_timestamps
from experiment_handler.label_data_reader import read_experiment_phases, read_location_labels
from feature_calculations.colocation.common import get_location_of_persons_at_samples

def read_roi_scene_recognitions_for_person(exp_root, person_id):
    filepath = os.path.join(exp_root, "processed_data", "scene_recognition_results", person_id + "_et_scene_classes.pkl")
    data = pd.read_pickle(filepath)
    return data

def generate_sample_times(start, end, step):
    return np.arange(start, end, step)

def calc_feature_distances(exp_root, times, reference_time="video", filename="tmp.pkl"):
    p1_features = read_roi_scene_recognitions_for_person(exp_root, "P1")
    p2_features = read_roi_scene_recognitions_for_person(exp_root, "P2")
    p3_features = read_roi_scene_recognitions_for_person(exp_root, "P3")
    p4_features = read_roi_scene_recognitions_for_person(exp_root, "P4")

    differences = []

    for t in times:
        t_et = convert_timestamps(exp_root, t, reference_time, "P1_eyetracker")
        ts_d = p1_features['timestamp'] - t_et
        idx = ts_d.abs().argmin()
        p1_f = p1_features.loc[idx]['predictions']

        t_et = convert_timestamps(exp_root, t, reference_time, "P2_eyetracker")
        ts_d = p2_features['timestamp'] - t_et
        idx = ts_d.abs().argmin()
        p2_f = p2_features.loc[idx]['predictions']

        t_et = convert_timestamps(exp_root, t, reference_time, "P3_eyetracker")
        ts_d = p3_features['timestamp'] - t_et
        idx = ts_d.abs().argmin()
        p3_f = p3_features.loc[idx]['predictions']

        t_et = convert_timestamps(exp_root, t, reference_time, "P4_eyetracker")
        ts_d = p4_features['timestamp'] - t_et
        idx = ts_d.abs().argmin()
        p4_f = p4_features.loc[idx]['predictions']

        diff = {}
        diff["timestamp"] = t
        diff["P1-P2"] = np.linalg.norm(p1_f - p2_f)
        diff["P1-P3"] = np.linalg.norm(p1_f - p3_f)
        diff["P1-P4"] = np.linalg.norm(p1_f - p4_f)
        diff["P2-P3"] = np.linalg.norm(p2_f - p3_f)
        diff["P2-P4"] = np.linalg.norm(p2_f - p4_f)
        diff["P3-P4"] = np.linalg.norm(p3_f - p4_f)

        differences.append(diff)

    diff_df = pd.DataFrame(differences)
    diff_df.to_pickle(filename)
    return diff_df

def get_colocation_labels(locations_at_samples):
    colocation = {}
    times = locations_at_samples[:, 0]

    colocation["P1"] = {
        "P2": np.ones((len(locations_at_samples), 2)),
        "P3": np.ones((len(locations_at_samples), 2)),
        "P4": np.ones((len(locations_at_samples), 2))
    }
    colocation["P2"] = {
        "P1": np.ones((len(locations_at_samples), 2)),
        "P3": np.ones((len(locations_at_samples), 2)),
        "P4": np.ones((len(locations_at_samples), 2))
    }
    colocation["P3"] = {
        "P1": np.ones((len(locations_at_samples), 2)),
        "P2": np.ones((len(locations_at_samples), 2)),
        "P4": np.ones((len(locations_at_samples), 2))
    }
    colocation["P4"] = {
        "P1": np.ones((len(locations_at_samples), 2)),
        "P3": np.ones((len(locations_at_samples), 2)),
        "P2": np.ones((len(locations_at_samples), 2))
    }

    colocation["P1"]["P2"][:, 0] = times
    colocation["P1"]["P2"][(locations_at_samples[:, 1] - locations_at_samples[:, 2]) != 0, 1] = 0
    colocation["P1"]["P3"][:, 0] = times
    colocation["P1"]["P3"][(locations_at_samples[:, 1] - locations_at_samples[:, 3]) != 0, 1] = 0
    colocation["P1"]["P4"][:, 0] = times
    colocation["P1"]["P4"][(locations_at_samples[:, 1] - locations_at_samples[:, 4]) != 0, 1] = 0

    colocation["P2"]["P1"][:, 0] = times
    colocation["P2"]["P1"][(locations_at_samples[:, 1] - locations_at_samples[:, 2]) != 0, 1] = 0
    colocation["P2"]["P3"][:, 0] = times
    colocation["P2"]["P3"][(locations_at_samples[:, 2] - locations_at_samples[:, 3]) != 0, 1] = 0
    colocation["P2"]["P4"][:, 0] = times
    colocation["P2"]["P4"][(locations_at_samples[:, 2] - locations_at_samples[:, 4]) != 0, 1] = 0

    colocation["P3"]["P1"][:, 0] = times
    colocation["P3"]["P1"][(locations_at_samples[:, 1] - locations_at_samples[:, 3]) != 0, 1] = 0
    colocation["P3"]["P2"][:, 0] = times
    colocation["P3"]["P2"][(locations_at_samples[:, 2] - locations_at_samples[:, 3]) != 0, 1] = 0
    colocation["P3"]["P4"][:, 0] = times
    colocation["P3"]["P4"][(locations_at_samples[:, 3] - locations_at_samples[:, 4]) != 0, 1] = 0

    colocation["P4"]["P1"][:, 0] = times
    colocation["P4"]["P1"][(locations_at_samples[:, 1] - locations_at_samples[:, 4]) != 0, 1] = 0
    colocation["P4"]["P3"][:, 0] = times
    colocation["P4"]["P3"][(locations_at_samples[:, 3] - locations_at_samples[:, 4]) != 0, 1] = 0
    colocation["P4"]["P2"][:, 0] = times
    colocation["P4"]["P2"][(locations_at_samples[:, 2] - locations_at_samples[:, 4]) != 0, 1] = 0

    return colocation

def test_colocation(coloc_features, threshold):
    colocation = {}

    colocation["P1"] = {
        "P2": np.ones((len(coloc_features), 2)),
        "P3": np.ones((len(coloc_features), 2)),
        "P4": np.ones((len(coloc_features), 2))
    }
    colocation["P2"] = {
        "P1": np.ones((len(coloc_features), 2)),
        "P3": np.ones((len(coloc_features), 2)),
        "P4": np.ones((len(coloc_features), 2))
    }
    colocation["P3"] = {
        "P1": np.ones((len(coloc_features), 2)),
        "P2": np.ones((len(coloc_features), 2)),
        "P4": np.ones((len(coloc_features), 2))
    }
    colocation["P4"] = {
        "P1": np.ones((len(coloc_features), 2)),
        "P3": np.ones((len(coloc_features), 2)),
        "P2": np.ones((len(coloc_features), 2))
    }

    value = 1
    #value = np.random.randint(0, 2)
    for idx, row in coloc_features.iterrows():
        colocation["P1"]["P2"][idx, 0] = row['timestamp']
        colocation["P1"]["P2"][idx, 1] = value

        colocation["P1"]["P3"][idx, 0] = row['timestamp']
        colocation["P1"]["P3"][idx, 1] = value

        colocation["P1"]["P4"][idx, 0] = row['timestamp']
        colocation["P1"]["P4"][idx, 1] = value

        colocation["P2"]["P1"][idx, 0] = row['timestamp']
        colocation["P2"]["P1"][idx, 1] = value

        colocation["P2"]["P4"][idx, 0] = row['timestamp']
        colocation["P2"]["P4"][idx, 1] = value

        colocation["P2"]["P3"][idx, 0] = row['timestamp']
        colocation["P2"]["P3"][idx, 1] = value

        colocation["P3"]["P1"][idx, 0] = row['timestamp']
        colocation["P3"]["P1"][idx, 1] = value

        colocation["P3"]["P4"][idx, 0] = row['timestamp']
        colocation["P3"]["P4"][idx, 1] = value

        colocation["P3"]["P2"][idx, 0] = row['timestamp']
        colocation["P3"]["P2"][idx, 1] = value

        colocation["P4"]["P1"][idx, 0] = row['timestamp']
        colocation["P4"]["P1"][idx, 1] = value

        colocation["P4"]["P3"][idx, 0] = row['timestamp']
        colocation["P4"]["P3"][idx, 1] = value

        colocation["P4"]["P2"][idx, 0] = row['timestamp']
        colocation["P4"]["P2"][idx, 1] = value

    return colocation

def detect_colocation(coloc_features, threshold):
    colocation = {}

    colocation["P1"] = {
        "P2": np.ones((len(coloc_features), 2)),
        "P3": np.ones((len(coloc_features), 2)),
        "P4": np.ones((len(coloc_features), 2))
    }
    colocation["P2"] = {
        "P1": np.ones((len(coloc_features), 2)),
        "P3": np.ones((len(coloc_features), 2)),
        "P4": np.ones((len(coloc_features), 2))
    }
    colocation["P3"] = {
        "P1": np.ones((len(coloc_features), 2)),
        "P2": np.ones((len(coloc_features), 2)),
        "P4": np.ones((len(coloc_features), 2))
    }
    colocation["P4"] = {
        "P1": np.ones((len(coloc_features), 2)),
        "P3": np.ones((len(coloc_features), 2)),
        "P2": np.ones((len(coloc_features), 2))
    }

    for idx, row in coloc_features.iterrows():
        colocation["P1"]["P2"][idx, 0] = row['timestamp']
        colocation["P1"]["P2"][idx, 1] = 0 if row["P1-P2"] > threshold else 1

        colocation["P1"]["P3"][idx, 0] = row['timestamp']
        colocation["P1"]["P3"][idx, 1] = 0 if row["P1-P3"] > threshold else 1

        colocation["P1"]["P4"][idx, 0] = row['timestamp']
        colocation["P1"]["P4"][idx, 1] = 0 if row["P1-P4"] > threshold else 1

        colocation["P2"]["P1"][idx, 0] = row['timestamp']
        colocation["P2"]["P1"][idx, 1] = 0 if row["P1-P2"] > threshold else 1

        colocation["P2"]["P4"][idx, 0] = row['timestamp']
        colocation["P2"]["P4"][idx, 1] = 0 if row["P2-P4"] > threshold else 1

        colocation["P2"]["P3"][idx, 0] = row['timestamp']
        colocation["P2"]["P3"][idx, 1] = 0 if row["P2-P3"] > threshold else 1

        colocation["P3"]["P1"][idx, 0] = row['timestamp']
        colocation["P3"]["P1"][idx, 1] = 0 if row["P1-P3"] > threshold else 1

        colocation["P3"]["P4"][idx, 0] = row['timestamp']
        colocation["P3"]["P4"][idx, 1] = 0 if row["P3-P4"] > threshold else 1

        colocation["P3"]["P2"][idx, 0] = row['timestamp']
        colocation["P3"]["P2"][idx, 1] = 0 if row["P2-P3"] > threshold else 1

        colocation["P4"]["P1"][idx, 0] = row['timestamp']
        colocation["P4"]["P1"][idx, 1] = 0 if row["P1-P4"] > threshold else 1

        colocation["P4"]["P3"][idx, 0] = row['timestamp']
        colocation["P4"]["P3"][idx, 1] = 0 if row["P3-P4"] > threshold else 1

        colocation["P4"]["P2"][idx, 0] = row['timestamp']
        colocation["P4"]["P2"][idx, 1] = 0 if row["P2-P4"] > threshold else 1

    return colocation

def _get_score_for_single_pair(labels, detections):
    """

    Parameters
    ----------
    labels: numpy array (Nx2)
        Co-location ground truth (first column = sample time, second column = 0 if not colocated 1 if colocated)
    detections: numpy array (Nx2)
        Detected Co-locations (first column = sample time, second column = 0 if not colocated 1 if colocated)

    Returns
    -------
        scores: list
            List of four items. 1: number of true positives 2: number false positives
            3: number of false negatives 4: number of true negatives

    """
    tmp = labels[:, 1] - detections[:, 1]

    fn = np.count_nonzero(tmp == 1)
    fp = np.count_nonzero(tmp == -1)
    tp = np.count_nonzero(labels[tmp == 0, 1])
    tn = len(tmp) - fn - fp - tp
    return [tp, fp, fn, tn]

def calculate_score(colocation_labels, colocation_detections):
    """
    Calculating some standard frame-by-frame scores for colocation detections

    Parameters
    ----------
    colocation_labels
    colocation_detections

    Returns
    -------
        p: float
            Precision
        r: float
            Recall
        a: float
            Accuracy
        f1: float
            F1-score
        fpr: float
            False Positive Rate

    """
    p1p2_scores = _get_score_for_single_pair(colocation_labels["P1"]["P2"], colocation_detections["P1"]["P2"])
    p1p3_scores = _get_score_for_single_pair(colocation_labels["P1"]["P3"], colocation_detections["P1"]["P3"])
    p1p4_scores = _get_score_for_single_pair(colocation_labels["P1"]["P4"], colocation_detections["P1"]["P4"])
    p2p3_scores = _get_score_for_single_pair(colocation_labels["P2"]["P3"], colocation_detections["P2"]["P3"])
    p2p4_scores = _get_score_for_single_pair(colocation_labels["P2"]["P4"], colocation_detections["P2"]["P4"])
    p3p4_scores = _get_score_for_single_pair(colocation_labels["P3"]["P4"], colocation_detections["P3"]["P4"])

    scores = np.array([
        p1p2_scores, p1p3_scores, p1p4_scores, p2p3_scores, p2p4_scores, p3p4_scores
    ])

    sums = np.sum(scores, axis=0)
    if sums[0] > 0:
        p = sums[0] / (sums[0] + sums[1])
    else:
        p = 0

    r = sums[0] / (sums[0] + sums[2])
    a = (sums[0] + sums[3]) / (sums[0] + sums[3] + sums[1] + sums[2])
    f1 = 2*sums[0] / (2*sums[0] + sums[1] + sums[2])
    fpr = sums[1] / (sums[1] + sums[3])

    return p, r, a, f1, fpr

def create_colocation_plots(colocation_labels, colocation_detections):
    # get step size:
    stepsize = colocation_detection["P1"]["P2"][1][0] - colocation_detection["P1"]["P2"][0][0]

    fig = plt.figure(figsize=(8,4))
    ax = plt.gca() # fig.add_axes([0, 0, 1, 1])
    plt.title("Co-location of persons", loc="left")

    yticks = [
        "P1 - P2",
        "P1 - P3",
        "P1 - P4",
        "P2 - P3",
        "P2 - P4",
        "P3 - P4",
    ]

    height = 0.05

    gt_color = "#1f77b4"
    for values in colocation_labels["P1"]["P2"]:
        if values[1] > 0:
            y_pos = (yticks.index("P1 - P2") + 1) * (1/7)
            plt.axvspan(values[0] - stepsize / 2, values[0] + stepsize / 2, y_pos, y_pos + height)
    for values in colocation_labels["P1"]["P3"]:
        if values[1] > 0:
            y_pos = (yticks.index("P1 - P3") + 1) * (1/7)
            plt.axvspan(values[0] - stepsize / 2, values[0] + stepsize / 2, y_pos, y_pos + height)

    for values in colocation_labels["P1"]["P4"]:
        if values[1] > 0:
            y_pos = (yticks.index("P1 - P4") + 1) * (1/7)
            plt.axvspan(values[0] - stepsize / 2, values[0] + stepsize / 2, y_pos, y_pos + height)

    for values in colocation_labels["P2"]["P3"]:
        if values[1] > 0:
            y_pos = (yticks.index("P2 - P3") + 1) * (1/7)
            plt.axvspan(values[0] - stepsize / 2, values[0] + stepsize / 2, y_pos, y_pos + height)

    for values in colocation_labels["P2"]["P4"]:
        if values[1] > 0:
            y_pos = (yticks.index("P2 - P4") + 1) * (1/7)
            plt.axvspan(values[0] - stepsize / 2, values[0] + stepsize / 2, y_pos, y_pos + height)

    for values in colocation_labels["P3"]["P4"]:
        if values[1] > 0:
            y_pos = (yticks.index("P3 - P4") + 1) * (1/7)
            plt.axvspan(values[0] - stepsize / 2, values[0] + stepsize / 2, y_pos, y_pos + height)

    det_color = "orange"
    for values in colocation_detections["P1"]["P2"]:
        if values[1] > 0:
            y_pos = (yticks.index("P1 - P2") + 1) * (1/7)
            plt.axvspan(values[0] - stepsize / 2, values[0] + stepsize / 2, y_pos - height, y_pos, color=det_color)
    for values in colocation_detections["P1"]["P3"]:
        if values[1] > 0:
            y_pos = (yticks.index("P1 - P3") + 1) * (1/7)
            plt.axvspan(values[0] - stepsize / 2, values[0] + stepsize / 2, y_pos - height, y_pos, color=det_color)

    for values in colocation_detections["P1"]["P4"]:
        if values[1] > 0:
            y_pos = (yticks.index("P1 - P4") + 1) * (1/7)
            plt.axvspan(values[0] - stepsize / 2, values[0] + stepsize / 2, y_pos - height, y_pos, color=det_color)

    for values in colocation_detections["P2"]["P3"]:
        if values[1] > 0:
            y_pos = (yticks.index("P2 - P3") + 1) * (1/7)
            plt.axvspan(values[0] - stepsize / 2, values[0] + stepsize / 2, y_pos - height, y_pos, color=det_color)

    for values in colocation_detections["P2"]["P4"]:
        if values[1] > 0:
            y_pos = (yticks.index("P2 - P4") + 1) * (1/7)
            plt.axvspan(values[0] - stepsize / 2, values[0] + stepsize / 2, y_pos - height, y_pos, color=det_color)

    for values in colocation_detections["P3"]["P4"]:
        if values[1] > 0:
            y_pos = (yticks.index("P3 - P4") + 1) * (1/7)
            plt.axvspan(values[0] - stepsize / 2, values[0] + stepsize / 2, y_pos - height, y_pos, color=det_color)

    plt.grid()
    plt.ylim([0, len(yticks)+1])
    plt.yticks(range(1, len(yticks)+1), yticks)
    plt.tight_layout()
    plt.subplots_adjust(top=0.83)

    box_height = 0.07
    box_y = 1.02

    left_box_x = 0.5
    right_box_x = 0.75
    det_legend = Rectangle((right_box_x, box_y), 0.2, box_height, color=det_color, transform=ax.transAxes, clip_on=False)
    gt_legend = Rectangle((left_box_x, box_y), 0.2, box_height, color=gt_color, transform=ax.transAxes, clip_on=False)

    ax.text(right_box_x + 0.1, box_y + 0.5*box_height, 'Detections',
            horizontalalignment='center',
            verticalalignment='center',
            fontsize=10, color='#444433',
            transform=ax.transAxes)

    ax.text(left_box_x + 0.1, box_y + 0.5*box_height, 'Ground Truth',
            horizontalalignment='center',
            verticalalignment='center',
            fontsize=10, color='white',
            transform=ax.transAxes)

    ax.add_patch(det_legend)
    ax.add_patch(gt_legend)
    plt.show()

def find_best_threshold_workflow(coloc_features, coloc_labels, threshold_min, threshold_max, th_stepsize):
    """
    Testing a range of threshold and printing evaluations scores (Precision, recall and f1 score)
    """

    roc = []
    for threshold in range(threshold_min, threshold_max+th_stepsize, th_stepsize):
        threshold = float(threshold)/1000.0
        detected_colocations = detect_colocation(coloc_features, threshold)
        scores = calculate_score(coloc_labels, detected_colocations)

        print("\n----------- Score for threshold = " + str(threshold) + " ------------")
        print("Precision: " + str(scores[0]) + ", Recall: " + str(scores[1]) + ", Accuracy: " + str(scores[2]) + ", F1-score: " + str(scores[3]))
        print("----------- ------------------------- ------------")
        roc.append([scores[4], scores[1]])

    roc = np.array(roc)

    plt.plot(roc[:, 0], roc[:, 1])
    plt.ylim([0, 1])
    plt.xlim([0, 1])
    plt.xlabel("False Positive Rate")
    plt.ylabel("True Positive Rate")
    plt.grid()
    plt.show()

if __name__ == '__main__':
    exp_root = "/Volumes/DataDrive/igroups_recordings/igroups_experiment_8"
    force_recalc = False
    sample_step = 0.5

    phases = read_experiment_phases(exp_root)
    times = generate_sample_times(phases['assembly'][0], phases['disassembly'][1], sample_step)
    #times = generate_sample_times(sample_start, sample_end, sample_step)

    start = datetime.now()

    if force_recalc or not os.path.exists("tmp.pkl"):
        print("Warning: (re)generating features could take several minutes to compute.")
        coloc_features = calc_feature_distances(exp_root, times, filename="tmp.pkl")
    else:
        coloc_features = pd.read_pickle("tmp.pkl")

    # Read labels:
    location_labels = read_location_labels(exp_root)
    locations = get_location_of_persons_at_samples(location_labels, times, exp_root)
    colocation_labels = get_colocation_labels(locations)

    if False:
        colocation_detection = detect_colocation(coloc_features, 1000)
        plt.figure(figsize=(8, 4))
        plt.title("P1 - P2")
        plt.plot(colocation_detection["P1"]["P2"][:, 0], colocation_detection["P1"]["P2"][:, 1])
        plt.grid()
        plt.draw()
        plt.figure(figsize=(8, 4))
        plt.title("P1 - P3")
        plt.plot(colocation_detection["P1"]["P3"][:, 0], colocation_detection["P1"]["P3"][:, 1])
        plt.grid()
        plt.draw()
        plt.figure(figsize=(8, 4))
        plt.title("P1 - P4")
        plt.plot(colocation_detection["P1"]["P4"][:, 0], colocation_detection["P1"]["P4"][:, 1])
        plt.grid()
        plt.show()

    #find_best_threshold_workflow(coloc_features, colocation_labels, 1, 30, 1)

    #print(coloc_features)
    colocation_detection = detect_colocation(coloc_features, 0.0115)
    #print(colocation_detection)

    print("Runtime:", datetime.now() - start)
    score = calculate_score(colocation_labels, colocation_detection)
    print(score)
    #exit()

    create_colocation_plots(colocation_labels, colocation_detection)


text:
"""
Utilities functions

Created by Kunhong Yu
Date: 2021/07/05
"""
import os

import torch as t
import matplotlib.pyplot as plt
from torch.nn.functional import one_hot
from torch.nn import functional as F
from PIL import Image
import numpy as np

#################
#     Config    #
#################
class Config(object):
    """
    Args :
        --dataset: default is 'mnist' also support 'fashion_mnist'
        --beta: hyper-parameter to trade off negative labels, default is 0.5
        --epochs: training epochs, default is 20
        --batch_size: default is 100
    """
    dataset = 'mnist'

    beta = 0.5
    epochs = 20
    batch_size = 100

    device = 'cuda' if t.cuda.is_available() else 'cpu'

    def parse(self, **kwargs):
        for k, v in kwargs.items():
            if not hasattr(self, k):
                print(k + ' does not exist, will be added!')

            setattr(self, k, v)

##################
#      Loss      #
##################
def margin_loss(x, y, beta):
    """Define margin loss according to the paper
    Args :
        --x: input tensor, shape [m, 10, 16]
        --y: input label tensor, shape [m, ]
        --beta: hyper-parameter to trade off negative labels
    return :
        --loss
    """
    T = one_hot(y, 10) # [m, 10]
    loss = T * (F.relu(0.9 - x) ** 2) + beta * (1 - T) * (F.relu(x - 0.1) ** 2) # margin loss
    loss = t.mean(t.sum(loss, dim = -1))

    return loss

##################
#    Visualize   #
##################
def show_batch(batch, index = 0, dataset = 'mnist'):
    """Show batch of images
    Args :
        --battch: batch image in numpy form
        --index: index of batch, default is 0
        --dataset: default is 'mnist'
    """
    f, ax = plt.subplots(len(batch), 1, figsize = (5, 20))
    f.suptitle(f'Batch {index + 1} \n Input || Recon', fontsize = 30)
    for i in range(len(batch)):
        x = batch[i].permute(1, 2, 0) if dataset == 'cifar10' else batch[i]
        ax[i].imshow(x)
        ax[i].axis('off')

    if not os.path.exists('./results/re_imgs/'):
        os.makedirs('./results/re_imgs/')
    plt.savefig(f'./results/re_imgs/re_img_{index + 1}_{dataset}.png')
    plt.close()

def show_manipulation(x, model, index = 0, dataset = 'mnist'):
    """This function is used to show manipulation of our capsules in the DigitCaps layer
    Args :
        --x: input DigitCaps, which have shape [1, 10, 16]
        --model: model instance
        --index: batch index
        --dataset: 'mnist' as default
    """
    model.eval()
    step_size = 0.05
    #re_model = model.reconstruction
    f, ax = plt.subplots(16, 11, figsize = (11, 16))
    f.suptitle(f'Recon for all variations in batch {index}')
    with t.no_grad():
        for i in range(16):
            start = -0.3
            for j in range(11):
                start += step_size
                x[0 : 1, :, i] = x[0 : 1, :, i] + start
                x_ = model.reconstruction_module(x)
                x_ = x_.view(3, 28, 28).cpu().numpy() if dataset == 'cifar10' else x_.view(28, 28).cpu().numpy()
                if dataset == 'cifar10':
                    x_ = x_.transpose(1, 2, 0)
                ax[i][j].imshow(x_)
                ax[i][j].grid(False)
                ax[i][j].axis('off')
                x[0: 1, :, i] = x[0: 1, :, i] - start

    plt.savefig(f'./results/re_imgs/re_img_man_{index + 1}_{dataset}.png')
    plt.close()

def generate_animnated(dataset = 'mnist', man = False):
    """
    Generate one animated image
    Args :
        --dataset: default is 'mnist'
        --man: manipulate True or False
    """
    path = os.path.join('./results/re_imgs')
    files = os.listdir(path)
    files = list(filter(lambda x : x.count(dataset) and (x.count('man') if man else not x.count('man')), files))

    plt.ion()

    for file in files[:30]:
        plt.cla()
        img = Image.open(os.path.join(path, file))
        img = np.asarray(img)
        plt.imshow(img, cmap = 'gray')
        plt.axis('off')
        plt.pause(0.6)

    plt.ioff()
    plt.close()

if __name__ == '__main__':
    generate_animnated(dataset = 'fashion_mnist', man = False)

max_stars_repo_name: yuranusduke/DynamicRoutingCapsule

max_issues_repo_name: yuranusduke/DynamicRoutingCapsule

max_forks_repo_name: yuranusduke/DynamicRoutingCapsule