from typing import List

class ParsingError(Exception):
    pass

# 'IF' isn't reserved... treat it as a normal internal function.
RESERVED_WORDS = (['ALL', 'AND', 'ANY', 'ARRAY', 'AS', 'ASC',
                   'ASSERT_ROWS_MODIFIED', 'AT', 'BETWEEN', 'BY', 'CASE',
                   'CAST', 'COLLATE', 'CONTAINS', 'CREATE', 'CROSS',
                   'CUBE', 'CURRENT', 'DEFAULT', 'DEFINE', 'DESC',
                   'DISTINCT', 'ELSE', 'END', 'ENUM', 'ESCAPE', 'EXCEPT',
                   'EXCLUDE', 'EXISTS', 'EXTRACT', 'FALSE', 'FETCH',
                   'FOLLOWING', 'FOR', 'FROM', 'FULL', 'GROUP', 'GROUPING',
                   'GROUPS', 'HASH', 'HAVING', 'IGNORE', 'IN', 'INNER',
                   'INTERSECT', 'INTERVAL', 'INTO', 'IS', 'JOIN',
                   'LATERAL', 'LEFT', 'LIKE', 'LIMIT', 'LOOKUP', 'MERGE',
                   'NATURAL', 'NEW', 'NO', 'NOT', 'NULL', 'NULLS', 'OF',
                   'ON', 'OR', 'ORDER', 'OUTER', 'OVER', 'PARTITION',
                   'PRECEDING', 'PROTO', 'RANGE', 'RECURSIVE', 'RESPECT',
                   'RIGHT', 'ROLLUP', 'ROWS', 'SELECT', 'SET', 'SOME',
                   'STRUCT', 'TABLESAMPLE', 'THEN', 'TO', 'TREAT', 'TRUE',
                   'UNBOUNDED', 'UNION', 'UNNEST', 'USING', 'WHEN', 'MINUS',
                   'WHERE', 'WINDOW', 'WITH', 'WITHIN', 'QUALIFY', 'RECORDS'])

class SQLLexer:

    def __init__(self, sql_str):
        self._str = sql_str
        self._pos = 0
        self._comments = []

        self.consume_all_space()

    CONTEXT = 25

    def error(self, msg):

        # Find beginning and end of snippet.
        pos = self._pos
        linec = self._str[0:pos].count('\n') + 1
        line_start = 0
        if linec > 1:
            line_start = self._str[0:pos + 1].rindex('\n') + 1

        snippet_beg = int(max(line_start, pos - SQLLexer.CONTEXT))
        try:
            snippet_end = int(
                min(self._str[line_start:].index('\n') + line_start + 1,
                    pos + SQLLexer.CONTEXT))
        except ValueError:
            snippet_end = int(min(len(self._str), pos + SQLLexer.CONTEXT))

        # Raise exception
        raise ParsingError('\n'.join([
            f'{msg} at [{linec}:{pos - line_start + 1}]',
            f'Snippet: {self._str[snippet_beg:snippet_end]}',
            f'         {"-" * (pos - snippet_beg)}^'
        ]))

    def get_comments(self) -> List[str]:
        rcomments = self._comments
        self._comments = []
        return rcomments

    def peek(self, elem):
        """Peek for an element.

        Args:
          elem: Element. If a string, an identifier is searched for. Otherwise
              an operator.

        Returns:
          Returns True/False if the element is available to consume.
        """
        if isinstance(elem, list):
            rpos = self._pos
            for el in elem:
                if not self.peek(el):
                    self._pos = rpos
                    return False
                self._pos += len(el)
                self.consume_all_space()
            self._pos = rpos
            return True

        # If elem does not start with an alpha character,
        # just check if it matches the next available text
        if not elem[0].isalpha():
            return self._str[self._pos:self._pos + len(elem)] == elem

        # If it *doesn't* match the upper case of next available next,
        # then it doesn't match.
        if self._str[self._pos:self._pos + len(elem)].upper() != elem:
            return False

        # If this is the end of the string, it's a match
        if self._pos + len(elem) >= len(self._str):
            return True

        # Check the next character -- if it's not an alpha character,
        # and not a digit or underscore then we have a match.
        ch = self._str[self._pos + len(elem)]
        if not ch.isalpha() and not ch.isdigit() and ch != '_':
            return True

        return False

    def consume_any(self, elems):
        """Consume any of a list of elements.

        Args:
            elems: List of elements
                See peek() for what is an element
                Tested by consume(elem)

        Returns:
            Returns elem from elems that was consumed or None
        """
        for e in elems:
            if self.consume(e):
                return e
        return None

    def consume(self, elem):
        """Consume for an element or list of elements.

        Args:
          elem(s): Element or list of elements.
              See peek() for what is an element.
              If a list, then all elements must be consumed to be satisfied.

        Returns:
          Returns True/False if the element(s) is/are available to consume.
        """
        if isinstance(elem, list):
            rpos = self._pos
            for e in elem:
                if not self.consume(e):
                    self._pos = rpos
                    return None
            return elem

        if self.peek(elem):
            self._pos += len(elem)
            self.consume_all_space()
            return elem

        return None

    def expect(self, elem):
        return (self.consume(elem) or
                self.error('Expected "' + elem + '"'))

    def expect_end(self):
        if self._pos < len(self._str):
            self.error('Expected end')

    def consume_all_space(self):
        """Consume all space and comments.

        Returns:
          This moves the position forward to the next non-space non-comment.
        """
        while self._pos < len(self._str):

            if self._str[self._pos].isspace():
                self._pos += 1
                continue

            # It's a SQL comment
            if self._str[self._pos:self._pos + 2] == '--':
                start = self._pos + 2
                try:
                    move = self._str[self._pos:].index('\n')
                except ValueError:
                    move = len(self._str) - self._pos
                self._comments.append(self._str[start:start + move - 2].strip())
                self._pos += move
                continue

            # It's a hash comment
            if self._str[self._pos] == '#':
                try:
                    move = self._str[self._pos:].index('\n')
                except ValueError:
                    move = len(self._str) - self._pos
                self._pos += move
                continue

            # It's a /* */ comment
            if self._str[self._pos:self._pos + 2] == '/*':
                start = self._pos + 2
                while self._pos < len(self._str):
                    if self._str[self._pos:self._pos + 2] == '*/':
                        self._comments.append(
                            self._str[start:self._pos].strip())
                        self._pos += 2
                        break
                    self._pos += 1
                continue

            # Something else - stop now!
            break

    def consume_identifier(self):
        """Consume identifier.

        Returns:
          This returns an identifier (that's not reserved) or None if
          one isn't available.
        """
        if self._pos >= len(self._str):
            return None

        is_escaped = None
        ch = self._str[self._pos]
        if ch in ('`', '"'):
            is_escaped = ch

        if not (is_escaped or
                ch.isalpha() or
                ch == ':' or
                ch == '_'):
            return None

        pos = self._pos

        if is_escaped:
            pos += 1

        v = []

        if ch == ':':
            v.append(ch)
            pos += 1

        while pos < len(self._str):
            ch = self._str[pos]
            if is_escaped:
                if ch == is_escaped:
                    break
            else:
                if not (ch.isalpha() or ch == '_' or ch.isdigit()):
                    break
            v.append(ch)
            pos += 1

        v = ''.join(v)
        if v.upper() in RESERVED_WORDS:
            return None

        if is_escaped:
            pos += 1

        self._pos = pos

        self.consume_all_space()

        return v

    def consume_number(self):
        """Consume number.

        Returns:
          This returns a number (float or integer) or None
          if one isn't available.
        """
        if self._pos == len(self._str):
            return None

        if not self._str[self._pos].isdigit():
            return None

        # Do the parsing itself.
        pos = self._pos
        is_frac = False
        divisor = 1
        num = 0
        while pos < len(self._str):
            ch = self._str[pos]
            if ch.isdigit():
                num *= 10
                num += int(ch)
                if is_frac:
                    divisor *= 10
            elif not is_frac and ch == '.':
                is_frac = True
            else:
                break
            pos += 1

        # TODO(scannell) need 'e'

        # TODO(scannell) This is awkward. Fix.
        self._pos = pos
        self.consume_all_space()

        if is_frac:
            return float(num) / divisor

        return num

    def consume_string(self):
        """Consume string.

        Returns:
          This returns a string or None if one isn't available.
        """
        spos = self._pos
        flag = None

        if spos == len(self._str):
            return None

        if self._str[spos] == 'b' or self._str[spos] == 'r':
            flag = self._str[spos]
            spos += 1

        if spos == len(self._str) or self._str[spos] not in ('"', '\''):
            return None

        if self._str[spos:spos + 3] == '"""':
            quotechar = self._str[spos:spos + 3]
        else:
            quotechar = self._str[spos]

        spos += len(quotechar)
        epos = spos
        while (epos < len(self._str) and
               self._str[epos:epos + len(quotechar)] != quotechar):
            epos += 1

        if epos == len(self._str):
            self.error('Expected string')

        self._pos = epos + len(quotechar)

        self.consume_all_space()

        return (self._str[spos:epos], flag)


"""
this library contains a number of useful functions that are general purpose
data processing functions.
"""

###############################################################################
### import libraries
#from __init__ import (_np,_copy,_pd,_plt,_plot,_math)

### import libraries

# common libraries
import numpy as _np
import matplotlib.pyplot as _plt
import copy as _copy
import math as _math

# hbtepLib libraries
import _plotTools as _plot

###############################################################################
### misc functions

def convertDataToStairstepData(x,y):
    """
    When dealting with discrete data, often it makes sense to show the data
    look like stair-steps insteady of a the more typical smooth plot.  This
    function makes this happen #TODO(John) there is a better way to explain
    this...

    Example
    -------
    x=np.arange(0,1,0.01)
    y=np.sin(2*np.pi*12.345*x)
    (xOut,yOut)=hbt.process.convertDataToStaircaseDate(x,y)
    plt.plot(xOut,yOut)

    """

    xOut=_np.zeros(len(x)*2)
    yOut=_np.zeros(len(x)*2)
    dx=x[1]-x[0];
    for i in range(0,len(x)):
#        print yOut[i*2]
#        print y[i]
        xOut[2*i]=x[i]
        if i == len(x)-1:
            xOut[2*i+1]=xOut[2*i]+dx
        else:
            xOut[2*i+1]=x[i+1]
        yOut[2*i]=y[i]
        yOut[2*i+1]=y[i]
    return (xOut,yOut)

def findNearest(array,value):
    """
    search through `array` and returns the `index` of the cell closest to the
    `value`.   `array` should be sorted in ascending order

    Parameters
    ----------
    array : numpy.array
        data array to search through
    value : float (or int)
        value to look for in array

    Return
    ------
    index : int
        index of value in array that is closest to value

    References
    ----------
    http://stackoverflow.com/questions/2566412/find-nearest-value-in-numpy-array
    """
    index = (_np.abs(array-value)).argmin()
    # value = array[index]
    return index
    # return index, value # uncomment to return both the index AND the value

def rmse(data, targets=0):
    """
    Root mean square error function.
    If targets = 0, this is also the root mean square function.

    Parameters
    ----------
    data : numpy.array
        Data being considered
    target : numpy.array of floats
        values being compared against

    Return
    ------
    : numpy.array
        root mean square error of data

    References
    ----------
    # http://stackoverflow.com/questions/17197492/root-mean-square-error-in-python
    # http://statweb.stanford.edu/~susan/courses/s60/split/node60.html
    """
    return _np.sqrt(((data - targets) ** 2).mean())

def rms(data):
    """
    Root mean square function.

    Parameters
    ----------
    data : numpy.array
        Data to be processed

    Return
    ------
    : numpy.array
        root mean square of data

    References
    ----------
    # http://stackoverflow.com/questions/17197492/root-mean-square-error-in-python
    # http://statweb.stanford.edu/~susan/courses/s60/split/node60.html
    """
    return _np.sqrt(((data - 0) ** 2).mean())

def rejectOutliers(data, sigma=2):
    """
    remove outliers from set of data

    Parameters
    ----------
    data : numpy.ndarray
        data array being considered
    sigma : int
        the number of std. devs. about which to reject data.  E.g. sigma=2
        rejects outliers outside of +-2*sigma

    Return
    ------
     : numpy.ndarray
        Same as databut missing entires considered as outliers
    indicesToKeep : numpy.ndarray (of bool)
        Boolean indices associated with entries of data that are kept

    References
    ----------
    http://stackoverflow.com/questions/11686720/is-there-a-numpy-builtin-to-reject-outliers-from-a-list
    """
    indicesToKeep=abs(data - _np.mean(data)) < sigma* _np.std(data)
    return data[indicesToKeep],indicesToKeep

def rmPhaseJumps(time,data,cut=_np.pi):
    """
    Removed phase jumps of greater than "cut", defaults to pi
    Assumes phase is wrapped.
    Inserts NaN at jump points in data and time vector
    """
    indx=_np.argwhere(_np.abs(_np.diff(data))>=cut)+1
    if indx[-1]==len(data):indx[-1]-=1# in case last point jumps
    data=list(data)
    time=list(time)
    # Iterate over jump points, insert NaNs
    for i in indx:
        data.insert(i,_np.NaN)
        time.insert(i,_np.NaN)
        indx += 1
    return _np.array(time),_np.array(data)

def wrapPhase(data):
    """
    Wraps phase data so that it repeats every 2pi.
    This is important for phase data when you want it to all fit nicely on a
    plot.

    Parameters
    ----------
    data : numpy.ndarray
        data being wrapped

    Return
    ------
    outData : numpy.ndarray
        wrapped data array

    """
    inData=data*1.0
    outData=_np.zeros(inData.size);
    inData-=_np.pi;
    for i in range(0,_np.size(inData)-1):
        a=_np.floor(inData[i]/(_np.pi*2))
        outData[i]=inData[i]-(a+1)*2*_np.pi+_np.pi
    return outData

def unwrapPhase(inData):
    """
    Takes in phase array (in radians).  I think it needs to be centered about 0.
    Unwraps phase data so that it is continuous.
    This is important for phase data when you want to take it's derivative to
    get frequency.

    Parameters
    ----------
    data : numpy.ndarray
        data being unwrapped

    Return
    ------
    outData : numpy.ndarray
        unwrapped data array

    """
    outData=_np.zeros(_np.size(inData));
    offset=0;
    outData[0]=inData[0];
    for i in range(0,_np.size(inData)-1):
        if inData[i] > _np.pi/4 and inData[i+1] < -_np.pi/4:
            offset=offset+2*_np.pi;
        elif inData[i] < -_np.pi/4 and inData[i+1] > _np.pi/4:
            offset=offset-2*_np.pi;
        outData[i+1]=inData[i+1]+offset;
    return outData

def hasNan(inArray):
    """
    searches array, inArray, for any occurances of NaN.  returns True if
    yes, returns False if no.

    Parameters
    ----------
    inArray : numpy.ndarray
        data array being considered for NaN entries

    Return
    ------
        : bool
        True if NaNs are in array, False otherwise
    """
    count = 0;
    for i in range(0,len(inArray)):
        if _math.isnan(inArray[i]):
            count+=1;

    print("There was/were %d instances of NaN" % count)

    if count == 0:
        return False
    if count != 0:
        return True

def sort2Arrays(array1, array2):
    """
    sorts array1 and array2 in ascending order of array1

    outdated:  replaced by sortArrays() below
    """
    array1, array2 = zip(*sorted(zip(array1, array2)));
    array1=_np.array(array1);
    array2=_np.array(array2);
    return array1, array2

def sortArrays(arrays,sortIndex):
    """
    sorts a list of n arrays.  sortIndex is the index of the array to sort all arrays.

    example use:  [V, I, phi]=sortArrays([V,I,phi],1) to sort all arrays by ascending I

    reference:  https://stackoverflow.com/questions/6618515/sorting-list-based-on-values-from-another-list
    """
    sortedIndices=arrays[sortIndex].argsort()
    for i in range(len(arrays)):
        arrays[i]=arrays[i][sortedIndices]
    return arrays

def downSampleData(downX,upX,data):
    """
    Down samples data by finding the nearest x values on data that match the
    downselecting x (downX)

    Sometimes, you want to compare two different sets of data that do not have
    the same time basis.  This function down-samples the data set with more
    data points so that its x-data matches the other

    Parameters
    ----------
    downX : numpy.ndarray
        the x-data that will provide the down smpaled reference to the
        upsampled x-data
    upX : numpy.ndarray
        the upsampled x-data that will be downsampled
    data : list (of numpy.array)
        the upsampled y-data that will be downsampled

    Returns
    -------
    out : list (of np.ndarray)
        list of trimmed y-data

    Notes
    -----
    upX is not actually trimmed in this instance.  it is assumed that you user
    will use downX as their new time basis
    """
    if type(data) is not list:
        data=[data]

    m=len(downX)
    indices=_np.zeros(m,dtype=_np.int16)
    out = []

    for i in range(0,m):
        indices[i]=int(findNearest(upX,downX[i]))

    for i in range(0,len(data)):
        out.append(data[i][indices])
    return out

def upSampleData(upX,downX,data):
    """
    Up samples data by linear interpolating

    Similar to downSampleData() but up-samples instead

    Parameters
    ----------
    upX : np.ndarray
        the x-data that will be used in up-sampling the under-sampled data
    downX : np.ndarray
        the undersampled x-data
    data : list (of np.ndarray)
        the y-data to be up-samples.  downX is its time-base before
        up-sampling.  upX will be its time-base after up-sampling

    Returns
    -------
    out : list (of np.ndarray)
        list of up-sampled y-data
    """
    out = _np.interp(upX,downX,data)
    return out

def linearizeDataMatrix(data):
    """
    data is assumed to be a list of arrays

    this function converts the data to a single, appended array
    """
    m=len(data);
    temp=_np.array([])
    for i in range(0,m):
        temp=_np.append(temp,data[i])
    return temp

###############################################################################
### filters and smoothing algorithms

def downSample(data,time,new_dT,shftStart=False):
    if new_dT%_np.mean(_np.diff(time))>=1e-7:
        raise SyntaxError("Sampling rate must be integer multiple of original: %e vs %e"%(_np.mean(_np.diff(time)),new_dT))
    dS=(new_dT/_np.mean(_np.diff(time))).astype(int)

    # Verify data shape
    if data.shape[0] != len(time):data=data.T

    # Make smoothing kernel
    boxMat=_np.zeros((len(time)/dS,len(time)))
    for i in range(len(boxMat)):boxMat[i,i*dS:(i+1)*dS]=1./dS

    return _np.matmul(boxMat,data),_np.matmul(boxMat,time)-shftStart*new_dT*(3./2)#(dS/2)#*_np.mean(_np.diff(time))

def nPoleFilter(data,xData=None,numPoles=1,alpha=0.0625,filterType='lowPass',plot=False):
    """
    n-pole filter.

    Parameters
    ----------
    data : numpy.ndarray
        data to be smoothed
    xData : numpy.ndarray or NoneType
        (optional) array of x-data
    numPoles : int
        number of poles for the filter
    alpha : float
        weight of the filter, float between 0 and 1.  Close to zero for a low
        pass and close to 1 for a high pass
    filterType : str
        'lowPass' - Low pass filter
        'highPhass' - High pass filter
    plot : bool
        plots results if true

    Returns
    -------
    filteredData : 2D numpy.ndarray
        filtered data

    References
    ----------
    http://techteach.no/simview/lowpass_filter/doc/filter_algorithm.pdf
    https://en.wikipedia.org/wiki/Low-pass_filter#Discrete-time_realization
    https://en.wikipedia.org/wiki/High-pass_filter#Discrete-time_realization

    Notes
    -----
    this method is pulled from Qian Peng's 2016 GPU code.
    his highpass filter does NOT follow this code

    Example #1
    ----------
    t=np.arange(0,.01,6e-8);
    y2=np.sin(2*np.pi*300*t)+np.sin(2*np.pi*30000*t)
    alpha=0.00625;
    hbt.process.nPoleFilter(y2,t,numPoles=2,filterType='lowPass',plot=True,alpha=alpha)
    hbt.process.nPoleFilter(y2,t,numPoles=2,filterType='highPass',plot=True,alpha=1.0-0.000625)

    Example #2
    ----------
    t=np.arange(0,.01,6e-6);
    from scipy.signal import square
    y=square(t,0.001)
    hbt.process.nPoleFilter(y,t,numPoles=2,filterType='lowPass',plot=True)
    hbt.process.nPoleFilter(y,t,numPoles=2,filterType='highPass',plot=True)

    """

    # initialize data arrays
    procData=_np.zeros((numPoles+1,len(data)));
    procData[0,:]=data;

    # filter.  for loop controls the number of poles
    for i in range(1,numPoles+1):

        if filterType=='lowPass':
            for j in range(0,len(data)-1):
                procData[i,j+1]=procData[i,j]+alpha*(procData[i-1,j]-procData[i,j])
        elif filterType=='highPass':
            for j in range(0,len(data)-1):
                procData[i,j+1]=alpha*(procData[i,j]+procData[i-1,j+1]-procData[i-1,j])

    # plot results
    if plot==True:
        p1=_plot.plot(title=str(numPoles)+" pole "+filterType+" filter, alpha="+str(alpha),
                   xLabel="x-axis",yLabel='y-axis')

        if type(xData)==type(None):
            xData=_np.arange(0,len(data));

        p1.addTrace(xData=xData,yData=data,yLegendLabel='raw')

        for i in range(1,numPoles+1):
            p1.addTrace(xData=xData,yData=procData[i,:],yLegendLabel=str(i))

        p1.plot()

    # return results
    return procData

def savgolFilter(data,numPoints,polynomialOrder,plot=False):
    """
    Ssavitzky-Golay moving average smoothing filter.  Applies a nth order
    polynomial to a moving window of data.

    Parameters
    ----------
    data : numpy.ndarray
        data to be smoothed
    numPoints : int
        number of points for smoothing
    method : str
        method to use
        'box' - box car type of smoothing
        'gaussian' - guassian or normal smoothing
    plot : bool
        plots results if true

    Returns
    -------
    smoothedData : numpy.ndarray
        smoothed data

    References
    ----------
    https://docs.scipy.org/doc/scipy-0.16.1/reference/generated/scipy.signal.savgol_filter.html
    https://stackoverflow.com/questions/20618804/how-to-smooth-a-curve-in-the-right-way

    Notes
    -----
    this is a wrapper for scipy.signal.savgol_filter

    I think I prefer the gaussian convolution algorithm over this one (john)
    """

    def plotResults():
        """
        plots results (before and after smoothing)
        """
        p1=_plot.plot()
        x=_np.arange(0,len(data))
        p1.xData=[x,x]
        p1.yData=[data,smoothedData]
        p1.yLegendLabel=['raw data','smoothed data']
        p1.marker=['.','']
        p1.linestyle=['','-']
        p1.title='%d point, Savitzky-Golay smoothing of order %s'% (numPoints,polynomialOrder)
        p1.plot()

    # import savgol package
    from scipy.signal import savgol_filter

    # perform filter
    smoothedData=savgol_filter(data,numPoints,polynomialOrder)

    # plot results if requested
    if plot==True or plot=='all':
        plotResults()

    return smoothedData

def convolutionSmoothing(data,numPoints,method='gaussian',plot=False):
    """
    Convolution moving average smoothing filter

    Parameters
    ----------
    data : numpy.ndarray
        data to be smoothed
    numPoints : int
        number of points for smoothing.  should be an odd number
    method : str
        method to use
        'box' - box car type of smoothing.  keywords: boxcar
        'gaussian' - guassian or normal smoothing
    plot : bool
        plots results if true

    Returns
    -------
    smoothedData : numpy.ndarray
        smoothed data

    References
    ----------
    https://stackoverflow.com/questions/20618804/how-to-smooth-a-curve-in-the-right-way

    Example 1
    ---------
    # randon noise on sine way
    x = np.arange(0,2*np.pi,.1)
    y1 = np.sin(x) + np.random.random(len(x)) * 0.8
    convolutionSmoothing(y,numPoints,method='box',plot='all')

    Example 2
    ---------
    # step function
    x = np.arange(0,2*np.pi,.1)
    y2=np.zeros(len(x))
    y2[np.where(x>np.pi)[0]]=1
    convolutionSmoothing(y,numPoints,method='box',plot='all')

    Notes
    -----
    I want to justiy some of my weird code below.  When numPoints is an even
    number, the filter centers itself 1.5 points ahead of itself in the window
    which results in a small time/phase shift in the filtered data.  When it's
    an odd number, it centers itself 1.0 points ahead in the window and
    smaller time/phase shift.  By requiring that numPoints is odd and
    temporarily removing the first point in the data (I put it back in the
    end), I center the filter in the window and have no time/phase shift.

    The convolution filters "mess up" the last set of points (on the order of
    numPoints).  If you want to get around this, you should give it more data
    than you actually want and trim the end off.
    """
    def plotResults():
        """
        plots before and after of smoothing
        """
        p1=_plot.plot()
        x=_np.arange(0,len(data))
        p1.xData=[x,x]
        p1.yData=[data,smoothedData]
        p1.yLegendLabel=['raw data','smoothed data']
        p1.marker=['.','']
        p1.linestyle=['','-']
        p1.title='%d point, %s smoothing'% (numPoints,method)
        p1.plot()

    def plotSmoothingFunction():
        """
        plots smoothing function
        """
        p1=_plot.plot()
        x=_np.arange(0,len(smoothingFunction))
        p1.xData=[x]
        p1.yData=[smoothingFunction]
        p1.yLegendLabel=['smoothing function']
        p1.marker=['o']
        p1.linestyle=['-']
        p1.title='%d point, %s smoothing function'% (numPoints,method)
        p1.plot()

    # make sure numPoints is an odd number
    if (numPoints % 2 == 0): # is an even number
        numPoints+=1;
        print("Warning: numPoints was not an odd number.  +1 was added.  "
              "numPoints is now %d" % numPoints)

    # temporarily remove first point (see Notes)
    tempPoint=data[0]
    data=data[1:]

    # box smoothing
    if method=='box':
        smoothingFunction=_np.ones(numPoints)/numPoints

    # gaussian smoothing
    elif method=='gaussian' or method == 'normal':
        temp=_np.arange(numPoints)
        sigma=numPoints/(2*_np.pi);
        smoothingFunction=_np.exp(-((temp-(numPoints-1)/2.)/sigma)**2/2)
        smoothingFunction/=_np.sum(smoothingFunction)

    # perform smoothing
    smoothedData = _np.convolve(data, smoothingFunction, mode='same')

    # plot if requested
    if plot==True:
        plotResults()
    if plot=='all':
        plotSmoothingFunction()
        plotResults()

    # add point back to smoothedData (see Notes
    smoothedData=_np.append(tempPoint,smoothedData)

    return smoothedData

def gaussianFilter(t,y,timeFWHM,filterType='high',plot=False,plotGaussian=False):
    """
    Low and pass filters using scipy's gaussian convolution filter

    Parameters
    ----------
    t : numpy.array
        time
    y : numpy.array
        time dependent data
    timeFWHM : float
        full width at half maximum of the gaussian with units in time.  this
        effectively sets the corner frequency of the filter
    filterType : str
        'high' - high-pass filter
        'low' - low-pass filter
    plot : bool
        plots the results
    plotGaussian : bool
        plots the gaussian distribution used for the filter

    Returns
    -------
    yFiltered : numpy.array
        filtered time dependent data

    References
    ----------
    https://en.wikipedia.org/wiki/Full_width_at_half_maximum
    https://docs.scipy.org/doc/scipy-0.19.0/reference/generated/scipy.signal.gaussian.html
    https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.gaussian_filter1d.html
    """

    dt=t[1]-t[0]

#    from scipy import signal

    from scipy.ndimage import gaussian_filter1d

    def fwhmToGaussFilterStd(fwhm,dt):

        std=1.0/_np.sqrt(8*_np.log(2))*fwhm/dt
        return std

    std=fwhmToGaussFilterStd(timeFWHM,dt)
#    yFiltered=signal.gaussian(len(t), std=std)

#    if filterType=='low':
    yFiltered=gaussian_filter1d(y*1.0,std,mode='nearest')
#    elif filterType=='high':
#        yFiltered=y-gaussian_filter1d(y*1.0,std)

    if plot==True:

        _plt.figure()
        _plt.plot(t,y,label='Raw')
        _plt.plot(t,yFiltered,label='Low-pass')
        _plt.plot(t,y-yFiltered,label='High-pass')
        _plt.legend()

    if plotGaussian==True:

        from scipy import signal
        _plt.figure()
        _plt.plot(t,signal.gaussian(len(t), std=std),label='gaussian')
        _plt.legend()

    if filterType=='low':
        return yFiltered
    else:
        return y-yFiltered

def gaussianLowPassFilter(y,t,timeWidth=1./20000,plot=False,plotGaussian=False):
    """
    Low pass filter using scipy's gaussian filters

    Parameters
    ----------
    y : numpy.array
        time dependent data
    t : numpy.array
        time
    timeWidth : float
        full width at half maximum of the gaussian with units in time.  this
        effectively sets the corner frequency of the filter
        (f_{corner} \approx 1/timeWidth)
    plot : bool
        plots the results
    plotGaussian : bool
        plots the gaussian distribution used for the filter

    Returns
    -------
    yFiltered : numpy.array
        filtered time dependent data

    References
    ----------
    https://en.wikipedia.org/wiki/Full_width_at_half_maximum
    https://docs.scipy.org/doc/8scipy-0.19.0/reference/generated/scipy.signal.gaussian.html
    https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.gaussian_filter1d.html

    Example
    -------
    import numpy as np
    t=np.arange(0,10e-3,2e-6)
    y = np.random.randn(len(t)).cumsum()
    y+=np.sin(2*np.pi*1000+np.pi*2*np.random.rand())
    y+=np.sin(2*np.pi*3300+np.pi*2*np.random.rand())
    y+=np.sin(2*np.pi*10000+np.pi*2*np.random.rand())
    y+=np.sin(2*np.pi*33000+np.pi*2*np.random.rand())
    gaussianLowPassFilter(y,t,timeWidth=1e-4,plot=True,plotGaussian=True)
    """

    from scipy.ndimage import gaussian_filter1d

    dt=t[1]-t[0]
    #1/(dt*timeWidth*2*_np.pi)#
    sigma= (1./(2*_np.pi))*timeWidth/dt#2.355*timeWidth/dt#  #TODO(John)  This equation is wrong.  Should be dividing by 2.355, not multiplying.  Fix here and with all dependencies
    yFiltered=gaussian_filter1d(y,sigma)

    if plot==True:

        _plt.figure()
        _plt.plot(t,y,label='Raw')
        _plt.plot(t,yFiltered,label='Filtered')
        _plt.legend()
        _plt.grid()

    if plotGaussian==True:

        from scipy import signal
        _plt.figure()
        _plt.plot(t,signal.gaussian(len(t), std=sigma),label='gaussian')
        _plt.legend()

    return yFiltered

def gaussianHighPassFilter(y,t,timeWidth=1./20000,plot=False,plotGaussian=False):
    """
    High pass filter using scipy's gaussian filters

    Parameters
    ----------
    y : numpy.array
        time dependent data
    t : numpy.array
        time
    timeWidth : float
        full width at half maximum of the gaussian with units in time.  this
        effectively sets the corner frequency of the filter
        (f_{corner} \approx 1/timeWidth)
    plot : bool
        plots the results
    plotGaussian : bool
        plots the gaussian distribution used for the filter

    Returns
    -------
    yFiltered : numpy.array
        filtered time dependent data

    References
    ----------
    https://en.wikipedia.org/wiki/Full_width_at_half_maximum
    https://docs.scipy.org/doc/scipy-0.19.0/reference/generated/scipy.signal.gaussian.html
    https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.gaussian_filter1d.html

    Example
    -------
    import numpy as np
    t=np.arange(0,10e-3,2e-6)
    y = np.random.randn(len(t)).cumsum()
    y+=np.sin(2*np.pi*1000+np.pi*2*np.random.rand())
    y+=np.sin(2*np.pi*3300+np.pi*2*np.random.rand())
    y+=np.sin(2*np.pi*10000+np.pi*2*np.random.rand())
    y+=np.sin(2*np.pi*33000+np.pi*2*np.random.rand())
    gaussianHighPassFilter(y,t,timeWidth=1./20000,plot=True,plotGaussian=True)
    """
    fit=gaussianLowPassFilter(y,t,timeWidth,plot=False,plotGaussian=plotGaussian)
    yFiltered= y-fit

    if plot==True:

        _plt.figure()
        _plt.plot(t,y,label='Raw')
        _plt.plot(t,fit,label='Fit')
        _plt.plot(t,yFiltered,label='Filtered')
        _plt.legend()
        _plt.grid()

    return yFiltered, fit

def butterworthFilter(y, x,filterOrder=2, samplingRate=1/(2*1e-6),
                      cutoffFreq=20*1e3, filterType='low',plot=False):
    """
    Apply a digital butterworth filter on your data

    Parameters
    ----------
    y : numpy.ndarray
        unfiltered dependent data
    x : numpy.ndarray
        independent data
    filterOrder : int
        Butterworth filter order
    samplingRate : float
        Data sampling rate.
    cutoffFreq : float
        cutoff frequency for the filter
    filterType : str
        filter type.  'low' is lowpass filter
    plot : bool or str
        - True - plots filter results.
        - 'all'- plots filter results and filter response (psuedo-BODE plot)

    Returns
    -------
    filteredData : numpy.ndarray
        Filtered dependent data

    References
    ----------
    https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.freqz.html
    https://stackoverflow.com/questions/25191620/creating-lowpass-filter-in-scipy-understanding-methods-and-units
    """

    from scipy.signal import butter, lfilter, freqz

    def butter_lowpass(cutoff, fs, order=5):
        nyq = 0.5 * fs
        normal_cutoff = cutoff / nyq
        b, a = butter(order, normal_cutoff, btype=filterType, analog=False)
        return b, a

    def butter_lowpass_filter(data, cutoff, fs, order=5):
        b, a = butter_lowpass(cutoff, fs, order=order)
        y = lfilter(b, a, data)
        return y

    def plotOfFreqResponse():

        # Get the filter coefficients so we can check its frequency response.
        b, a = butter_lowpass(cutoffFreq, samplingRate, filterOrder)

        # calc frequency response
        w, h = freqz(b, a, worN=8000)
        gain=_np.abs(h)
        phase=_np.unwrap(_np.angle(h))*180/_np.pi

        # generate gain plot
        p1=_plot.plot()
        p1.xLim=[0, 0.5*samplingRate]
        p1.xLabel='Frequency [Hz]'
        p1.subtitle="Gain Response"
        p1.yLabel='Gain'
        p1.yLim=[0,1.]
        p1.title='Butterworth filter. Order=%d. Cutoff Freq=%.1f. Sampling Rate = %.1f Hz. Type = %s.' % (filterOrder, cutoffFreq, samplingRate, filterType)

        p1.xData.append(0.5*samplingRate*w/_np.pi)
        p1.yData.append(gain)
        p1.yLegendLabel.append('Frequency Response')

        p1.xData.append(_np.array([cutoffFreq,cutoffFreq]))
        p1.yData.append(_np.array([0,1])) #0.5*_np.sqrt(2)
        p1.yLegendLabel.append('Cuttoff Frequency')

        # generate phase plot
        p2=_plot.plot()
        p2.xLim=[0, 0.5*samplingRate]
        p2.xLabel='Frequency [Hz]'
        p2.subtitle="Phase Response"
        p2.yLabel='Phase [Degrees]'
        p2.yLim=[_np.min(phase),0]

        p2.xData.append(0.5*samplingRate*w/_np.pi)
        p2.yData.append(phase)
        p2.yLegendLabel.append('Frequency Response')

        p2.xData.append(_np.array([cutoffFreq,cutoffFreq]))
        p2.yData.append(_np.array([_np.min(phase),0])) #0.5*_np.sqrt(2)
        p2.yLegendLabel.append('Cuttoff Frequency')

        # combine into subplot
        sp1=_plot.subPlot([p1,p2],plot=False)

        return sp1

    def plotOfResults():
        p1=_plot.plot()

        p1.xData.append(x)
        p1.yData.append(y)
        p1.yLegendLabel.append('Unfiltered Data')

        p1.xData.append(x)
        p1.yData.append(filteredData)
        p1.yLegendLabel.append('Filtered Data')

        return p1

    filteredData=butter_lowpass_filter(y, cutoffFreq,
                                       samplingRate, filterOrder)

    if plot==True:
        plotOfResults().plot()

    if plot == 'all':
        plotOfResults().plot()
        plotOfFreqResponse().plot()

    return filteredData

###############################################################################
### fitting functions and related

class polyFitData:
    """
    Polynomial fit function.

    Parameters
    ----------
    yData : 'numpy.array'
        dependent variable
    xData : 'numpy.array'
        independent variable
    order : int
        order of polynomial fit.  1 = linear, 2 = quadratic, etc.
    plot : bool
        Causes a plot of the fit to be generated

    Attributes
    ----------
    coefs : 'numpy.array'
        array of fit coefficients, starts at highest order.
    ffit : 'numpy.lib.polynomial.poly1d'
        function that returns yFit data given ANY numpy.array of x values
    fitData : 'numpy.array'
        yFit data corresponding to xData
    plotOfFit :
        custom plot class of data.

    Notes
    -----
    This is merely a wrapper function for the numpy.polyfit function.  However,
    it also plots the result automatically.

    output:
    fitData is the y fit data.
    """

    def __init__(self, yData, xData,order=2, plot=True):
        title = str(order)+' order Polynomial fit'

        ### do fit
        self.coefs=_np.polyfit(xData, yData, order)
        self.ffit = _np.poly1d(self.coefs)
        self.fitData=self.ffit(xData)

        ### generate plot
        self.plotOfFit=_plot.plot();
        self.plotOfFit.xLabel='x'
        self.plotOfFit.yLabel='y'
        self.plotOfFit.title=title;

        ### raw data
        self.plotOfFit.xData.append(xData)
        self.plotOfFit.yData.append(yData)
        self.plotOfFit.marker.append('.')
        self.plotOfFit.linestyle.append('')
        self.plotOfFit.alpha.append(.15)
        self.plotOfFit.yLegendLabel.append('raw data')

        ### fit
        x=_np.linspace(_np.min(xData),_np.max(xData),1000);
        self.plotOfFit.xData.append(x)
        self.plotOfFit.yData.append(self.ffit(x))
        self.plotOfFit.marker.append('')
        self.plotOfFit.linestyle.append('-')
        self.plotOfFit.alpha.append(1.)
        self.plotOfFit.yLegendLabel.append('poly fit order %d'%order)

        if plot==True:
            self.plotOfFit.plot()

class genericCurveFit:
    """
    generic curve fitting function that uses scipy.optimize.curve_fit solution

    I "think" I like the genericLeastSquaresFit code better than this function.  See below.

    func = fit function
    indepVars = independent variables.  for multivariable, use indepVars = (x,y) etc.
    depVars = dependent variable.  this is the single dependent variable that we are trying to model
    guess = guess parameters.  use: guess = 8., 2., 7. etc.

    note:  this function CAN be upgraded to include bounds

    references:
        https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.curve_fit.html

    Example use
    -----------
    def dumbModel(variables,a,b,c):
        x,y=variables
        return _np.log(a) + b*_np.log(x) + c*_np.log(y)

    # some artificially noisy data to fit
    x = _np.linspace(0.1,1.1,101)
    y = _np.linspace(1.,2., 101)
    a, b, c = 10., 4., 6.
    z = dumbModel((x,y), a, b, c) * 1 + _np.random.random(101) / 100

    # initial guesses for a,b,c:
    p0 = 8., 2., 7.

    # solve
    indepVars=(x,y)
    a=genericCurveFit(dumbModel, indepVars, z, p0)

    """

    def __init__(self,func, indepVars,depVar,guess, bounds=None,plot=True , maxNumIterations = None, fileName=''):
        self.indepVars=indepVars;
        self.depVars=depVar
        self.fileName=fileName

        from scipy.optimize import curve_fit

        if bounds == None:
            self.fitParams,self.covMatrix = curve_fit(func, indepVars, depVar, guess) #, max_nfev = 1e2 , max_nfev=maxNumIterations
        else:
            self.fitParams,self.covMatrix = curve_fit(func, indepVars, depVar, guess, bounds=bounds) #, max_nfev = 1e2 , max_nfev=maxNumIterations
#        print self.fitParams
#        print self.covMatrix
        self.fitSoln=func(indepVars, *self.fitParams)
#        print fitSoln

        self.R2=rSquared(indepVars, self.fitSoln)
        print(r"$R^2$ = %.3E" % self.R2)

        if plot==True:
            self.plot();

    def plot(self):
        _plt.figure()
        _plt.plot(self.fitSoln, self.depVars, '.',
                  label='scipy.optimize.curve_fit solution',alpha=0.10)
#        _plt.plot(self.solnRobust, self.depData, 'x', label='robust least squares soln')
        _plt.plot(_np.array([_np.min(self.depVars),_np.max(self.depVars)]),
                  _np.array([_np.min(self.depVars),_np.max(self.depVars)]),
                  color='r',linestyle='--',linewidth=5)
        _plt.xlabel('Fit Solution')
        _plt.ylabel('Dependent Data')
        _plt.legend()
        _plt.grid()
#        alphaB, alphaV, alphaR, phi0, C, C2 = self.fitParams
        #        _plt.ylim([-5,30])
        _plt.axes().set_aspect('equal') #, 'datalim'
        if self.fileName != '':
            _plt.savefig(self.fileName+'.png')

def _expFunction(x, a,b,c):
    """
    Basic exponential function.  Used primarily with fitting functions.
    Output = a*_np.exp(x/b)+c

    Parameters
    ----------
    x : numpy.ndarray
        Independent variable
    a : float
        Fitting parameter.  Output = a*_np.exp(x/b)+c
    b : float
        Fitting parameter.  Output = a*_np.exp(x/b)+c
    c : float
        Fitting parameter.  Output = a*_np.exp(x/b)+c

    Returns
    -------
    : numpy.ndarray
        Output = a*_np.exp(x/b)+c

    """
    return a*_np.exp(x/b)+c

def _cosFunction(x, a,b,c,d):
    """
    Cosine function.  Used primarily with fitting functions.
    Output = a*_np.cos(x*d*2*_np.pi+b)+c

    Parameters
    ----------
    x : numpy.ndarray
        Independent variable
    a : float
        Fitting parameter.  Output = a*_np.cos(x*d*2*_np.pi+b)+c
    b : float
        Fitting parameter.  Output = a*_np.cos(x*d*2*_np.pi+b)+c
    c : float
        Fitting parameter.  Output = a*_np.cos(x*d*2*_np.pi+b)+c
    d : float
        Fitting parameter.  Output = a*_np.cos(x*d*2*_np.pi+b)+c

    Returns
    -------
    : numpy.ndarray
        Output = a*_np.cos(x*d*2*_np.pi+b)+c
    """
    return a*_np.cos(x*d*2*_np.pi+b)+c

def singlePowerTerm(x, a,b,c):
    """
    Basic power term.  Used primarily with fitting functions.
    Output = a*(x)**b+c

    Parameters
    ----------
    x : numpy.ndarray
        Independent variable
    a : float
        Fitting parameter.  Output = a*(x)**b+c
    b : float
        Fitting parameter.  Output = a*(x)**b+c
    c : float
        Fitting parameter.  Output = a*(x)**b+c

    Returns
    -------
    : numpy.ndarray
        Output = a*(x)**b+c
    """
    return a*(x)**b+c

class cosFit:
    """
    Cos fit function.  a*_np.cos(x*d*2*_np.pi+b)+c

    Parameters
    ----------
    y : numpy.ndarray
        dependent data
    x : numpy.ndarray
        independent array
    guess : list
        list of four floats [a, b, c, d]=[amplitude, phase offset, amplitude offest, linear frequency].  these are the guess values.
    plot : bool
        causes the results to be plotted

    Attributes
    ----------
    fit : genericLeastSquaresFit

    Notes
    -----
    if you are receiving the error: "ValueError: Residuals are not finite in
    the initial point.", most likely, you need to play with your initial
    conditions to get them closer to the right answer before the fit will work
    correctly.

    Example use
    -----------
    # import library first.  I set it as hbt.pd

    >>> y=np.array([11.622967, 12.006081, 11.760928, 12.246830, 12.052126, 12.346154, 12.039262, 12.362163, 12.009269, 11.260743, 10.950483, 10.522091,  9.346292,  7.014578,  6.981853,  7.197708,  7.035624,  6.785289, 7.134426,  8.338514,  8.723832, 10.276473, 10.602792, 11.031908, 11.364901, 11.687638, 11.947783, 12.228909, 11.918379, 12.343574, 12.046851, 12.316508, 12.147746, 12.136446, 11.744371,  8.317413, 8.790837, 10.139807,  7.019035,  7.541484,  7.199672,  9.090377,  7.532161,  8.156842,  9.329572, 9.991522, 10.036448, 10.797905])
    >>> x=np.linspace(0,2*np.pi,48)
    >>> c=hbt.process.cosFit(y,x,guess=[2,0,10,.3])
    # note that this example took me quite a bit of guessing with the guess
    # values before everything fit correctly.

    """
    def __init__(self,y,x,guess,plot=True):
        self.fit=genericLeastSquaresFit(x=x,paramsGuess=guess,y=y,
                                        function=_cosFunction,plot=plot)

class expFit:
    """
    Exponential fit function

    Parameters
    ----------
    y : numpy.ndarray
        dependent data
    x : numpy.ndarray
        independent array
    guess : list
        list of three floats.  these are the guess values.
    plot : bool
        causes the results to be plotted

    Attributes
    ----------
    fit : genericLeastSquaresFit

    Notes
    -----
    if you are receiving the error: "ValueError: Residuals are not finite in
    the initial point.", most likely, you need to play with your initial
    conditions to get them closer to the right answer before the fit will work
    correctly.

    Example use
    -----------
    # import library first.  I set it as hbt.pd
    >>> x = np.array([399.75, 989.25, 1578.75, 2168.25, 2757.75, 3347.25, 3936.75, 4526.25, 5115.75, 5705.25])
    >>> y = np.array([109,62,39,13,10,4,2,0,1,2])
    >>> hbt.pd.expFit(y,x,guess=[10,-100,1])

    """
    def __init__(self,y,x,guess=[1,1,1], plot=True):
        self.fit=genericLeastSquaresFit(x=x,paramsGuess=guess,y=y,
                                        function=_expFunction,plot=plot)

class genericLeastSquaresFit:
    """
    Least squares fitting function(class)
    This is a wrapper for scipy.optimize.least_squares

    Parameters
    ----------
    y : numpy.ndarray
        dependent data
    x : numpy.ndarray (multidimensional)
        independent array(s).  multiple ind. variables are supported.
    paramsGuess : list (of floats)
        guess values for the fit parameters
    function : function
        function to attempt to fit the data to.  see exdamples _cosFunction and
        _expFunction to see how this function should be constructed
    yTrue : (optional) numpy.ndarray
        if you know what the actual fit should be (for example when testing
        this code against a known), include it here, and it will plot
        alongside the other data.  also useful for debugging.
    plot : bool
        causes the results to be plotted

    Attributes
    ----------
    fitParams : numpy.ndarray
        array of fit parameters, in the same order as the guess
    plotOfFit :
    plotOfFitDep :
        custom plot function of fit data plotted against dependent data.  this
        is important if there is more than 1 dependent data array.
    rSquared : float
        r^2 result of the fit
    res :
        fit output from scipy.optimize.least_squares
    yFit : numpy.ndarray
        y-fit data that corresponds with the independent data

    Notes
    -----
    i've found that the guess values often NEED to be somewhat close to the
    actual values for the solution to converge

    i've implemented several specific functions that implement this function.
    see expFit and cosFit

    Example use
    ------------
    # define expoential function
    def _expFunction(x, a,b,c):
        return a*_np.exp(x/b)+c
    # generate noisy exponential signal
    x1=_np.linspace(-1,1,100);
    a=_np.zeros(len(x1));
    b=_np.zeros(len(x1));
    c=_np.zeros(len(x1));
    for i in range(0,len(x1)):
        a[i]=(random.random()-0.5)/4. + 1.
        b[i]=(random.random()-0.5)/4. + 1.
        c[i]=(random.random()-0.5)/4. + 1.
    y1=1+_np.pi*_np.exp(x1/_np.sqrt(2)) # actual solution
    y2=1*a+_np.pi*b*_np.exp(x1/_np.sqrt(2)/c) # noisy solution
    # perform fit
    d=genericLeastSquaresFit(x1,[1,1,1],y2, _expFunction, y1,plot=True)
    """

    def __init__(self, x, paramsGuess, y, function,yTrue=[],plot=True ):

        def fit_fun(paramsGuess,x,y):
            return function(x, *paramsGuess) - y

        from scipy.optimize import least_squares

        self.x=x
        self.y=y
        self.yTrue=yTrue

        # perform least squares fit and record results
        self.res=least_squares(fit_fun, paramsGuess, args=[x,y])  #args=(y)
        self.yFit=function(x, *self.res.x)
        self.fitParams=self.res.x;

        # calculate r^2
        self.rSquared=rSquared(y,self.yFit)

        # print results to screen
        print(r'R2 =  %.5E' % self.rSquared)
        print('fit parameters')
        print('\n'.join('{}: {}'.format(*k) for k in enumerate(self.res.x)))

        # plot data
        if plot==True:
            if type(x) is _np.ndarray or len(x)==1:
                self.plotOfFit().plot()
            self.plotOfFitDep().plot()

    def plotOfFit(self):
        """
        plots raw and fit data vs. its indep. variable.

        Notes
        -----
        this function only works if there is a single indep. variable
        """
        # make sure that there is only a single indep. variable
        if type(self.x) is _np.ndarray or len(self.x)==1:
            p1=_plot.plot();
            p1.yLabel='y'
            p1.xLabel='x'
            p1.title=r'Fit results.  R$^2$ = %.5f' % self.rSquared

            if isinstance(self.x,list):
                x=self.x[0]
            else:
                x=self.x

            # raw data
            p1.xData.append(x)
            p1.yData.append(self.y)
            p1.yLegendLabel.append('raw data')
            p1.marker.append('.')
            p1.linestyle.append('')
            p1.color.append('b')
            p1.alpha.append(0.3)

            # fit data
            p1.xData.append(x)
            p1.yData.append(self.yFit)
            p1.yLegendLabel.append('fit')
            p1.marker.append('')
            p1.linestyle.append('-')
            p1.color.append('r')
            p1.alpha.append(1.)

            # the true data (if applicable)
            if self.yTrue!=[]:
                p1.xData.append(x)
                p1.yData.append(self.yTrue)
                p1.yLegendLabel.append('True Soln')
                p1.marker.append('')
                p1.linestyle.append('-')
                p1.plotOfFit.color.append('k')
                p1.plotOfFit.alpha.append(1.)

            return p1

    def plotOfFitDep(self):
        """
        plot of fit vs dependent data.  important if there are multiple
        independent variables.
        """
        p1=_plot.plot();
        p1.yLabel='fit data'
        p1.xLabel='raw data'
        p1.aspect="equal"

        p1.xData.append(self.y)
        p1.yData.append(self.yFit)
        p1.yLegendLabel.append('actual fit')
        p1.marker.append('.')
        p1.linestyle.append('')
        p1.color.append('b')
        p1.alpha.append(0.3)

        p1.xData.append(_np.array([_np.min(self.y),_np.max(self.y)]))
        p1.yData.append(_np.array([_np.min(self.y),_np.max(self.y)]))
        p1.yLegendLabel.append('ideal fit line')
        p1.marker.append('')
        p1.linestyle.append('-')
        p1.color.append('r')
        p1.alpha.append(1.)
        p1.legendLoc=  'upper left'
        p1.title=r'Fit quality.  R$^2$ = %.5f' % self.rSquared

        return p1

def rSquared(y,f):
    """
    calculates R^2 of data fit

    Parameters
    ----------
    y : numpy.ndarray
        data being fit to, the dependent variable (NOT THE INDEPENDENT VARIABLE).  y is a functino of x, i.e. y=y(x)
    f : float
        fit data

    Returns
    -------
    : float
        R^2 = 1 - \frac{\sum (f-y)^2 }{\sum (y-<y>)^2 }

    Reference
    ---------
    https://en.wikipedia.org/wiki/Coefficient_of_determination
    """
    yAve=_np.average(y);
    SSres = _np.sum( (y-f)**2 )
    SStot = _np.sum( (y-yAve)**2 )
    return 1-SSres/SStot

###############################################################################
### data management related

def listArrayToNumpyArray(inData):
    """
    Converts data from format a list of numpy.ndarrays to a 2D numpy.ndarray
    e.g. inData=[_np.array, _np.array, _np.array] to outData=_np.array([3,:])

    Parameters
    ----------
    inData : list (of numpy.ndarray)
        e.g. inData=[array([ 10.,  10.,  10.,  10.]),
                     array([ 15.,  15.,  15.,  15.]),
                     array([ 2.,  2.,  2.,  2.])]

    Returns
    -------
    outData : numpy.ndarray (2D)
        e.g. outData=   array([[ 10.,  15.,   2.],
                               [ 10.,  15.,   2.],
                               [ 10.,  15.,   2.],
                               [ 10.,  15.,   2.]])

    Notes
    -----
    -Note that this code requires that all arrays in the list have the same
    length
    -This code is so obvious that having a wrapper for it is kinda dumb...

    """
    outData = _np.array(inData)
    return outData

###############################################################################
### string manipulation related

def extractIntsFromStr(string):
    """
    Extracts all integers from a string.

    Parameters
    ----------
    string : str
        str with numbers embedded

    Returns
    -------
    numbers : list (of int)
        list of numbers that were within string

    Example
    -------
    print(extractNumsFromStr("123HelloMy65Is23"))

    Notes
    -----
    Does not work with decimal points.  Integers only.
    """
    import re

    # get list of numbers
    numbers=re.findall(r'\d+',string)

    # convert to integers
    for i in range(0,len(numbers)):
        numbers[i]=int(numbers[i])

    return numbers


import dropbox
import os
from dropbox.files import WriteMode

class Transferdata:
    def __init__(self,access_token):
        self.access_token = access_token

    def uploadfile(self,source,destination):
        dbx = dropbox.Dropbox(self.access_token)
        for root,folder,files in os.walk(source):
            for i in files:
                localpath = os.path.join(root,i)
                relativepath = os.path.relpath(localpath,source)
                dropboxpath = os.path.join(destination,relativepath)

                f = open(localpath , "rb")
                dbx.files_upload(f.read() , dropboxpath,mode = WriteMode("overwrite"))

def main():
    access_token = "DchsX-ktqsoAAAAAAAAAARUvSDUQoqXpzi89BUTTOWOEzbqQ_RUBa1px_9HBD6L5"
    filetransfer = Transferdata(access_token)

    source = str(input("enter your Source : - "))
    destination = input("enter the destination : - ")

    filetransfer.uploadfile(source,destination)
    print("file has been moved!")

main()


# By Marco Anaya

# python3 plot.py (module)

import sys
import os
import math
import string

import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator
from matplotlib.backends.backend_pdf import PdfPages

# supports coloring 12 categories, repeating once until 24
colors = dict(zip(string.ascii_uppercase,
            ['#E58606', '#5D69B1', '#52BCA3', '#99C945', '#CC61B0', '#24796C',
            '#DAA51B', '#2F8AC4', '#764E9F', '#ED645A', '#CC3A8E', '#A5AA99',
            '#E58606', '#5D69B1', '#52BCA3', '#99C945', '#CC61B0', '#24796C',
            '#DAA51B', '#2F8AC4', '#764E9F', '#ED645A', '#CC3A8E', '#A5AA99',
            '#E58606', '#5D69B1', '#52BCA3']))

min_size = 5

def plot(project, module, class_distributions=None, class_performance_per_req=None):
    class_perf, class_distr = class_performance_per_req, class_distributions

    csv_path = "./" + project + "/" + module + '/csv/'
    graph_path = "./" + project + "/" + module + '/graphs/'
    if class_perf is None:
        class_perf = pd.read_csv(csv_path + module + '_class_performance_per_req.csv')
    if class_distr is None:
        class_distr = pd.read_csv(csv_path + module + '_class_distributions.csv')

    class_perf = class_perf[class_perf['Class Size'] > min_size]
    class_distr = class_distr[class_distr['Class Size'] > min_size]

    # Runs plot_grade() for each grade level, generating three graphs
    grades = list(filter(lambda entry: os.path.isdir(csv_path + entry), os.listdir(csv_path)))
    print('Graphing ' + module + ':')
    for grade in grades:

        plot_grade(graph_path, module, grade, class_perf, class_distr)
        print('  performance and distribution graphs for grade ' + str(grade) + '.')

    # Generates teacher analysis for the whole module
    plot_module(graph_path, module, grades, class_perf)
    print('  performance by classroom and grade.')
    plot_module_by_req(graph_path, module, grades, class_perf)
    print('  performance for a single requirement by classroom and grade.')
    print('Done.')

def plot_grade(path, module, grade, class_perf, class_distr):
    df = class_perf
    df2 = class_distr

    pdf = PdfPages(path + module + '-' + grade + '.pdf')

    df = df.loc[df['Grade'] == grade]

    by_req_data = df.iloc[:, 4:len(df.columns) - 1]

    tIDs = list(df['Teacher ID'])
    cIDs = list(df['Studio ID'])
    class_sizes = list(df.iloc[:, 3])

    # construct per-requirement graph
    plt.subplots(figsize=(10, 6))
    plt.title((module + ' grade ' + grade + ' Classroom Performance Per Requirement').title())

    req_count = len(by_req_data.columns)

    class_count = len(by_req_data.index)
    pos = list(range(req_count))
    width = .7 / (class_count)

    labels = [tID + '-' + str(cID) + ' (n=' + str(class_size) + ')' for (tID, cID, class_size) in zip(tIDs, cIDs, class_sizes)]

    # plotting a set of bars for each grade level
    for i, (index, row) in enumerate(by_req_data.iterrows()):
        plt.bar(pos if i == 0 else [p + width * i for p in pos],
            row,
            width * .9,
            alpha=0.7,
            color=colors[tIDs[i]],
            label=labels[i],
            zorder=3)
    plt.xlabel('Requirements')
    plt.xticks([p + (width * .9 * class_count / 2) for p in pos], labels=[p + 1 for p in pos])
    plt.ylabel('Percent Complete')
    plt.ylim([0, 100])
    plt.grid(axis='y', zorder=0, which='both')
    plt.legend(bbox_to_anchor=(1.04, 1), loc='upper right', ncol=1)

    plt.savefig(path + 'pngs/' + module + '-' + grade + '-per-req.png', bbox_inches='tight')
    pdf.savefig(bbox_inches='tight')
    plt.close()

    # construct totals graph
    if class_count > 1:
        total_data = df['Total']
        plt.subplots(figsize=(10, 6))
        plt.title((module + ' grade ' + grade + ' Classroom Performance Totals').title())
        plt.bar(
            list(range(class_count)),
            total_data,
            alpha=0.7,
            color=[colors[tID] for tID in tIDs],
            zorder=3
        )
        plt.xlabel('Classroom')
        plt.xticks(list(range(class_count)), labels=labels, rotation=20, horizontalalignment='right')
        plt.ylabel('Percent Complete')
        plt.ylim([0, 100])
        plt.grid(axis='y', zorder=0, which='both')

        plt.savefig(path + 'pngs/' + module + '-' + grade + '-totals.png', bbox_inches='tight')
        pdf.savefig(bbox_inches='tight')
        plt.close()

    df2 = df2.loc[df2['Grade'] == grade]
    distr_data = df2.iloc[:, 4:]

    # construct distribution graph
    fig, axs = plt.subplots(math.ceil(class_count / 2), 2, figsize=(10, 5 * math.ceil(class_count / 2)))

    for i, (ax, (index, row)) in enumerate(zip(axs.reshape(-1), distr_data.iterrows())):
        pos = list(distr_data.columns)
        ax.bar(pos, row, .95,
            alpha=0.7, zorder=3, color=colors[tIDs[i]])

        if class_count > 1:
            ax.set_title(('Classroom ' + str(cIDs[i])).title())
        else:
            ax.set_title(('Classroom ' + str(cIDs[i]) + ' Distribution').title())

        ax.set_xlabel('Score')
        ax.set_xticks(pos)
        ax.set_xticklabels = row.index

        ax.set_ylabel('Students')
        ax.yaxis.set_major_locator(MaxNLocator(integer=True))

        ax.grid(axis='y', zorder=0, which='major', alpha=0.5)

    if class_count % 2 != 0:
        axs.reshape(-1)[-1].remove()
    if class_count > 1:
        fig.suptitle('Grade Distributions')

    plt.savefig(path + 'pngs/' + module + '-' + grade + '-distributions.png', bbox_inches='tight')
    pdf.savefig(bbox_inches='tight')

    pdf.close()
    plt.close()

# plots teacher analysis using TOTALS column from CLASS PERFORMANCE PER REQUIREMENT
def plot_module(path, module, grades, class_perf):
    df = class_perf
    totals_data = [df.loc[df['Grade'] == grade] for grade in grades]

    # construct graph measuring performance by classroom and grade
    fig, axs = plt.subplots(1, len(grades), sharey=True, sharex=True, figsize=(3 * len(grades), 6))
    fig.suptitle((module + ' Requirement Completion by Classroom and Grade').title())
    fig.text(0.5, 0.05, 'Grades', ha='center', va='center')
    fig.text(0.08, 0.5, 'Classroom Completion (%)', ha='center', va='center', rotation='vertical')

    for i, [d, ax] in enumerate(zip(totals_data, axs)):
        bars = len(d.index)
        labels = [list(d['Teacher ID'])[i] + '-' + str(list(d['Studio ID'])[i]) + ' (n=' + str(list(d.iloc[:, 3])[i]) + ')' for i in range(bars)]
        bar = ax.bar(
            list(range(bars)),
            d['Total'],
            .9,
            color=[colors[tID] for tID in list(d['Teacher ID'])],
            alpha=0.7,
            zorder=3
        )
        ax.set_ylim([0, 100])
        ax.set_xlabel(grades[i])
        ax.tick_params(labelbottom=False)
        ax.legend(bar, labels, bbox_to_anchor=(1, -0.1), loc='upper right', ncol=1)

    plt.savefig(path + 'pngs/' + module + '-by-classroom.png', bbox_inches='tight')
    plt.savefig(path + module + '-teacher-analysis.pdf', bbox_inches='tight')
    plt.close()

# plots teacher analysis by requirement
def plot_module_by_req(path, module, grades, class_perf):
    df = class_perf
    totals_data = [df.loc[df['Grade'] == grade] for grade in grades]

    for req_index in range(4,len(df.columns)-1):
        # construct graph measuring performance by classroom and grade
        fig, axs = plt.subplots(1, len(grades), sharey=True, sharex=True, figsize=(3 * len(grades), 6))
        fig.suptitle((module + ' Single Requirement Completion by Classroom and Grade').title())
        fig.text(0.5, 0.93, 'Requirement: ' + df.columns[req_index] + ' [' + str(req_index-3) + ']', ha='center', va='center')
        fig.text(0.5, 0.05, 'Grades', ha='center', va='center')
        fig.text(0.08, 0.5, 'Classroom Completion for Requirement (%)', ha='center', va='center', rotation='vertical')

        for i, [d, ax] in enumerate(zip(totals_data, axs)):
            bars = len(d.index)
            labels = [list(d['Teacher ID'])[i] + '-' + str(list(d['Studio ID'])[i]) + ' (n=' + str(list(d.iloc[:, 3])[i]) + ')' for i in range(bars)]
            bar = ax.bar(
                list(range(bars)),
                d.iloc[:,req_index],
                .9,
                color=[colors[tID] for tID in list(d['Teacher ID'])],
                alpha=0.7,
                zorder=3
            )
            ax.set_ylim([0, 100])
            ax.set_xlabel(grades[i])
            ax.tick_params(labelbottom=False)
            ax.legend(bar, labels, bbox_to_anchor=(1, -0.1), loc='upper right', ncol=1)

        plt.savefig(path + 'pngs/' + module + "-req" + str(req_index-3) + '-analysis.png', bbox_inches='tight')
        plt.savefig(path + module + "-req" + str(req_index-3) + '-analysis.pdf', bbox_inches='tight')
        plt.close()

if __name__ == '__main__':
    plot(sys.argv[1], sys.argv[2])


import json
import numpy as np
import os
from skipgram import build_model, train, traning_op
from genmetapaths import MetaPathGenerator
from dataset import Dataset
import sys
sys.path.append("./src/embs/metapath2vec/")

def mp2vec(walk_txt,
           node_type_txt,
           epochs=2,
           lr=0.01,
           d=100,
           window=1,
           negative_samples=5,
           care_type=1):
    index_file = "./model/mp2vec/index2nodeid.json"
    emb_file = "./model/mp2vec/node_embeddings.npz"
    if not os.path.exists(index_file) or not os.path.exists(emb_file):
        dataset = Dataset(random_walk_txt=walk_txt,
                          node_type_mapping_txt=node_type_txt, window_size=window)
        center_node_placeholder, context_node_placeholder, negative_samples_placeholder, loss = build_model(
            BATCH_SIZE=1, VOCAB_SIZE=len(dataset.nodeid2index), EMBED_SIZE=d, NUM_SAMPLED=negative_samples)
        optimizer = traning_op(loss, LEARNING_RATE=lr)
        print("training starts.")
        train(center_node_placeholder, context_node_placeholder, negative_samples_placeholder,
              loss, dataset, optimizer, NUM_EPOCHS=epochs, BATCH_SIZE=1, NUM_SAMPLED=negative_samples,
              care_type=care_type, LOG_DIRECTORY="./model/mp2vec", LOG_INTERVAL=-1,
              MAX_KEEP_MODEL=10)
    else:
        print("Embedding file exists. Skip training.")
    index2nodeid = json.load(open(index_file))
    index2nodeid = {int(k): v for k, v in index2nodeid.items()}
    nodeid2index = {v: int(k) for k, v in index2nodeid.items()}
    node_embeddings = np.load(emb_file)['arr_0']
    # node embeddings of "yi"
    embeidng_dict = {}
    for node in nodeid2index.keys():
        embeidng_dict[node] = node_embeddings[nodeid2index[node]]
    return embeidng_dict


import json
import os
import datetime

from analysis.analysis_constants import (
    REPLACE_LIST, USER_DATA_PATH, EDIT_TYPES,
    MSG_TYPES, HIGH_LEVEL_PROCESS, EDITING_PROCESS,
    PARTICIPANT_DATABASE, EDIT_TYPES,
)

class Participant:
    def __init__(self, email, id, order, study, videos, baseline):
        self.email = email
        self.id = id
        self.order = order
        self.study_id = study
        self.videos = videos
        self.baseline = baseline
        self.json_files = []
        self.found_file = False
        for filename in os.listdir(USER_DATA_PATH):
            if filename.endswith(".json"):
                if filename.startswith(email):
                    self.json_files.append(os.path.join(USER_DATA_PATH, filename))

        self.data = {}
        if len(self.json_files) == 0:
            self.found_file = False
        else:
            self.found_file = True
            for json_file in self.json_files:
                with open(json_file, "r") as f:
                    cur_data = json.load(f)
                    log_length = len(cur_data.get("logs", []))
                    cur_log_length = len(self.data.get("logs", []))
                    if log_length > cur_log_length:
                        # print("!!!", json_file, log_length)
                        self.data = cur_data

            self.adjust_data()

    def exists(self):
        return self.found_file

    def video_id(self):
        return self.videos[0]

    def video_name(self):
        if self.video_id() == "video-1":
            return "cooking"
        elif self.video_id() == "video-2":
            return "talking-head"
        return "unknown"

    ### user logs
    def logs(self):
        return self.data.get("logs", [])

    def get_specific_logs(self, msg):
        logs = self.logs()
        processing_logs = []
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            if log["msg"] == msg:
                processing_logs.append(log)
        return processing_logs

    def get_unique_msgs(self):
        logs = self.logs()
        unique_msgs = []
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            if log["msg"] not in unique_msgs:
                unique_msgs.append(log["msg"])
        return unique_msgs

    def get_unqiue_msg_types(self):
        unique_msgs = self.get_unique_msgs()
        unique_msg_types = {}
        for msg in unique_msgs:
            msg_type = msg[0:4]
            if msg_type not in unique_msg_types:
                unique_msg_types[msg_type] = []
            unique_msg_types[msg_type].append(msg)
        return unique_msg_types

    def adjust_data(self):
        logs = self.logs()
        new_logs = []
        for log in logs:
            if self.email == "kh.mukashev@gmail.com":
                ## skip logs after 12:00:00
                date = datetime.datetime.fromtimestamp(log["time"] / 1000)
                if date.hour >= 12:
                    continue
            if self.email == "ysamargarita2002@gmail.com":
                date = datetime.datetime.fromtimestamp(log["time"] / 1000)
                if date.hour >= 18 and date.minute >= 30:
                    continue
            if self.email == "kamila240373@gmail.com":
                date = datetime.datetime.fromtimestamp(log["time"] / 1000)
                if date.month < 10 or date.day < 1 or date.day > 3 or date.hour < 15 or date.hour > 17:
                    continue
            if log["msg"] in REPLACE_LIST:
                if REPLACE_LIST[log["msg"]] == None:
                    continue
                else:
                    log["msg"] = REPLACE_LIST[log["msg"]]
            new_logs.append(log)
        self.data["logs"] = new_logs

    def count_msg(self, msg):
        logs = self.logs()
        count = 0
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            if log["msg"] == msg:
                count += 1
        return count

    def count_msg_type(self, msg_type):
        logs = self.logs()
        count = 0
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            if log["msg"].startswith(msg_type):
                count += 1
        return count

    def get_processing_logs(self):
        processing_logs = self.get_specific_logs("processingComplete")
        formatted_logs = []
        for log in processing_logs:
            date = datetime.datetime.fromtimestamp(log["time"] / 1000)
            formatted_date = date.strftime("%H:%M:%S")
            formatted_logs.append({
                "intent_id": log["data"]["intentId"],
                "text": log["data"]["text"],
                "sketch": log["data"]["sketch"],
                "mode": log["data"]["mode"],
                "explanation": json.dumps(log["data"]["explanation"], separators=(',', ': ')),
                "formatted_time": formatted_date,
                "time": log["time"],
            })
        formatted_logs.sort(key=lambda x: x["time"])
        return formatted_logs

    def count_processing_logs(self):
        PROCESSING_COMPLETE_MSG = "processingComplete"
        logs = self.logs()
        count = 0
        count_iterations = 0
        count_processings = {}

        combined_data = []
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            if log["msg"] == PROCESSING_COMPLETE_MSG:
                data = log["data"]
                if data["intentId"] not in count_processings.keys():
                    count_processings[data["intentId"]] = 0
                count_processings[data["intentId"]] += 1
                combined_data.append(data)

        iterations_list = [num - 1 for num in count_processings.values()]
        count_iterations = sum(iterations_list)
        count = sum(count_processings.values())

        processed_data = []
        total_count_edits = 0
        count_from_scratch = 0
        count_add_more = 0
        count_has_text = 0
        count_has_sketch = 0
        count_has_text_and_sketch = 0
        for data in combined_data:
            point = {
                "text": data["text"],
                "hasSketch": len(data["sketch"]) > 0,
                "count_edits": len(data["edits"]),
                "start": data["start"],
                "finish": data["finish"],
                "mode": data["mode"],
            }
            processed_data.append(point)
            total_count_edits += len(data["edits"])
            if data["mode"] == "from-scratch":
                count_from_scratch += 1
            elif data["mode"] == "add-more":
                count_add_more += 1

            if len(data["text"]) > 0:
                count_has_text += 1
            if len(data["sketch"]) > 0:
                count_has_sketch += 1
            if len(data["text"]) > 0 and len(data["sketch"]) > 0:
                count_has_text_and_sketch += 1
        return {
            "total": count,
            "total_count_suggested": total_count_edits,
            "total_count_iterations": count_iterations,
            "iterations_list": iterations_list,
            "count_from_scratch": count_from_scratch,
            "count_add_more": count_add_more,
            "count_has_text": count_has_text,
            "count_has_sketch": count_has_sketch,
            "count_has_text_and_sketch": count_has_text_and_sketch,
        }

    def count_decision(self):
        REJECT_MSG = "timelineDecisionReject"
        ACCEPT_MSG = "timelineDecisionAccept"
        logs = self.logs()
        reject_count = 0
        accept_count = 0
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            if log["msg"] == REJECT_MSG:
                reject_count += 1
            elif log["msg"] == ACCEPT_MSG:
                accept_count += 1
        return {
            "reject": reject_count,
            "accept": accept_count,
        }

    ### user tasks
    def tasks(self):
        return self.data.get("tasks", [])

    def get_task(self, task_id):
        tasks = self.tasks()
        for task in tasks:
            if task["projectMetadata"]["projectId"] == task_id:
                return task
        return None

    def get_all_intents(self):
        return self.data.get("intents", [])

    def get_intent(self, intent_id):
        all_intents = self.get_all_intents()
        for intent in all_intents:
            if intent["id"] == intent_id:
                return intent

    def get_task_summary(self, task_id):
        accepted_suggestion_ids = []
        logs = self.logs()
        for log in logs:
            if log["msg"] == "timelineDecisionAccept" and log["videoId"] == self.video_id():
                data = log["data"]
                if "addedEdits" in data:
                    accepted_suggestion_ids.extend(data["addedEdits"])

        task = self.get_task(task_id)
        intents = task["intents"]

        count_intents = len(intents)
        count_edits = 0
        count_history_points = 0
        count_edits_from_suggestion = 0
        edit_types_count = {}

        intents_data = []
        for intent_id in intents:
            intent = self.get_intent(intent_id)
            intents_data.append({
                "intent_id": intent_id,
                "edit_type": intent["editOperationKey"],
                "count_edits": len(intent["activeEdits"]),
                "history_points": len(intent["history"]),
                "count_suggestions": len(intent["suggestedEdits"]),
            })
            count_edits += len(intent["activeEdits"])
            count_history_points += len(intent["history"]) - 1

            edits_from_suggested = []
            for edit_id in intent["activeEdits"]:
                if edit_id in accepted_suggestion_ids:
                    edits_from_suggested.append(edit_id)

            count_edits_from_suggestion += len(edits_from_suggested)

            edit_type = intent["editOperationKey"]
            if edit_type not in edit_types_count:
                edit_types_count[edit_type] = {
                    "count_intents": 0,
                    "count_edits": 0,
                    "count_history_points": 0,
                    "count_edits_from_suggestion": 0,
                    "edit_type": edit_type,
                }
            edit_types_count[edit_type]["count_intents"] += 1
            edit_types_count[edit_type]["count_edits"] += len(intent["activeEdits"])
            edit_types_count[edit_type]["count_history_points"] += len(intent["history"]) - 1
            edit_types_count[edit_type]["count_edits_from_suggestion"] += len(edits_from_suggested)

        return {
            "count_intents": count_intents,
            "count_edits": count_edits,
            "count_history_points": count_history_points,
            "count_edits_from_suggestion": count_edits_from_suggestion,
            "edit_types_count": edit_types_count,
            #"intents": intents_data,
        }

    def get_edit_process(self, high_level = False):
        logs = self.logs()
        process = []
        for log in logs:
            if log["videoId"] != self.video_id():
                continue
            process.append({
                "msg": log["msg"].strip(),
                "data": log["data"],
                "time": log["time"],
            })
        process.sort(key=lambda x: x["time"])

        # combine the same logs near each other
        for log in process:
            found_edit_types = []
            for edit_type in EDITING_PROCESS.keys():
                if log["msg"] in EDITING_PROCESS[edit_type]:
                    found_edit_types.append(edit_type)
            if len(found_edit_types) == 0:
                log["msg"] = "unknown"
            else:
                found_edit_type = ""
                if len(found_edit_types) > 1:
                    if "suggested" in log["data"] and log["data"]["suggested"] == True:
                        found_edit_type = "examine"
                    else:
                        found_edit_type = "manual"
                    #print("!!!", log["msg"], found_edit_types)
                else:
                    found_edit_type = found_edit_types[0]
                if high_level:
                    log["msg"] = HIGH_LEVEL_PROCESS[found_edit_type]
                else:
                    log["msg"] = found_edit_type

        # filter out anything or unknown
        new_process = []
        for log in process:
            if log["msg"] == "anything" or log["msg"] == "unknown":
               continue
            new_process.append(log)
        process = new_process

        combined_process = []
        for log in process:
            if len(combined_process) == 0:
                combined_process.append(log)
                continue
            last_log = combined_process[-1]
            if last_log["msg"] == log["msg"]:
                last_log["time"] = log["time"]
            else:
                combined_process.append(log)

        # add duration
        for log in combined_process:
            log["duration"] = 0
        for i in range(1, len(combined_process)):
            log = combined_process[i]
            prev_log = combined_process[i - 1]
            log["duration"] = log["time"] - prev_log["time"]

        formatted_process = []
        for log in combined_process:
            date = datetime.datetime.fromtimestamp(log["time"] / 1000)
            duration = datetime.datetime.fromtimestamp(log["duration"] / 1000)
            formatted_date = date.strftime("%H:%M:%S")
            formatted_duration = duration.strftime("%M:%S")

            formatted_process.append({
                "msg": log["msg"],
                "date": formatted_date,
                "duration": formatted_duration,
                "time": log["time"],
            })
        return formatted_process


import logging
import os.path
from typing import List

from kodi.AppConfiguration import AppConfiguration
from kodi.exception.WrongTypePlaylistException import WrongTypePlaylistException
from kodi.fs.FileItem import FileItem
from kodi.fs.FileItemUtils import FileItemsResponse
from kodi.playlist.Playlist import Playlist
from kodi.playlist.PlaylistItem import PlaylistItem
from kodi.playlist.playlist_constant import TITLE_TYPE, TITLE_ITEM

logger = logging.getLogger(__name__)

class PlaylistUtils:
    @classmethod
    def load_from_file(cls, path: str) -> Playlist:
        a_path = os.path.abspath(path)
        if not os.path.exists(a_path):
            logger.error("Does not find playlist file: " + a_path)
            return Playlist(a_path, list())
        playlist_items = list()
        with open(a_path, "r") as playlist_file:
            line = playlist_file.readline()
            if not line:
                logger.warning("Found empty file of playlist: " + a_path)
                return Playlist(a_path, list())
            if line.rstrip() != TITLE_TYPE:
                raise WrongTypePlaylistException(path)
            cnt = 0
            line = playlist_file.readline()
            while line:
                if line.startswith(TITLE_ITEM):
                    line = playlist_file.readline()
                    playlist_items.append(PlaylistItem(line))
                    cnt += 1
                line = playlist_file.readline()
            if cnt == 0:
                logger.info("Playlist is empty")
            else:
                logger.info("Playlist contains " + str(cnt) + " items")

        return Playlist(a_path, playlist_items)

    @staticmethod
    def save_to_file(playlist: Playlist) -> str:
        with open(playlist.path, "w") as dest_playlist:
            dest_playlist.write(playlist.to_playlist())
        return playlist.path

    @staticmethod
    def create_actual_playlist(app_config: AppConfiguration, delta: FileItemsResponse,
                               store_items: List[FileItem],
                               current_playlist: Playlist) -> Playlist:
        new_playlist = Playlist(current_playlist.path, list())
        current_playlist_items = current_playlist.items.copy()
        sync_playlist_items = list()
        for store_item in store_items:
            existed = False
            for current_playlist_item in current_playlist_items:
                if store_item.name == current_playlist_item.name:
                    existed = True
                    sync_playlist_items.append(current_playlist_item)
                    continue
            if not existed:
                sync_playlist_items.append(PlaylistItem(store_item.path))

        for new_file_item in delta.new_items:
            new_playlist.items.append(PlaylistItem(os.path.join(app_config.directories.store, new_file_item.name)))

        cleaned_playlist_items = list()
        for playlist_item in sync_playlist_items:
            deleted = False
            for deleted_file_item in delta.deleted_items:
                if playlist_item.name == deleted_file_item.name:
                    deleted = True
            if not deleted:
                cleaned_playlist_items.append(playlist_item)

        for updated_file_item in delta.updated_items:
            existed = False
            for new_playlist_item in cleaned_playlist_items:
                if new_playlist_item.name == updated_file_item.name:
                    existed = True
            if not existed:
                cleaned_playlist_items.append(
                    PlaylistItem(os.path.join(app_config.directories.store, updated_file_item.name)))

        new_playlist.items += cleaned_playlist_items
        return new_playlist


from flask import Flask, redirect, url_for, render_template, request
import Task3
from Task3 import change_map, write_to_file
app = Flask(__name__)

@app.route("/")
def home():
    return render_template("index.html")

@app.route("/register", methods=["POST"])
def register():
    name = request.form.get("domain")
    token = request.form.get("tocken")
    return f"{name} {token}"

@app.route("/register/map", methods=["POST"])
def take_elements():
    elements = register().split()
    name = elements[0]
    tocker = elements[1]
    file_with_friends = write_to_file(name, tocker)
    your_map = change_map("friends.json")
    return your_map.get_root().render()

if __name__ == "__main__":
    app.run(debug=True)


# python 3

from collections import deque

def find_min_max(arr, k):
    n = len(arr)
    ans = 0

    queue_min = deque()
    queue_max = deque()

    for i in range(n):
        if queue_max and queue_max[0] <= i - k:
            queue_max.popleft()
        if queue_min and queue_min[0] <= i - k:
            queue_min.popleft()

        while queue_max and arr[queue_max[-1]] <= arr[i]:
            queue_max.pop()
        queue_max.append(i)

        while queue_min and arr[queue_min[-1]] >= arr[i]:
            queue_min.pop()
        queue_min.append(i)

        if i >= k - 1:
            ans += arr[queue_min[0]] + arr[queue_max[0]]

    return ans

def main():
    arr = list(map(int, input().split()))
    k = int(input())
    print(find_min_max(arr, k))

if __name__ == '__main__':
    main()


from fabric import Connection

host_list = ('root@demo.seafile.top', 'root@download.seafile.top')

for host in host_list:

    c = Connection(host)

    result = c.run('uname -s', hide=True)
    print("Ran {0.command!r} on {0.connection.host}, got stdout:\n{0.stdout}".format(result))

    result = c.put('lian-test', remote='/opt/')
    print("Uploaded {0.local} to {0.remote}".format(result))


from flask import render_template, flash, redirect, url_for

from app.firestore_service import get_todos, put_todo, delete_todo, update_todo
from app.forms import TodoForm

from flask_login import login_required, current_user

from . import tasks

# Private Area Example
@tasks.route('/list', methods=['GET', 'POST'])
@login_required
def list():
  user_id = current_user.id

  task_form = TodoForm()

  if task_form.validate_on_submit():
    put_todo(user_id, task_form.description.data)
    flash('New Task created!', 'success')
    return redirect(url_for('tasks.list'))

  context = {
    'todos': get_todos(user_id),
    'task_form': task_form
  }
  return render_template('tasks.html', **context)

@tasks.route('/delete/<string:todo_id>', methods=['GET'])
@login_required
def delete(todo_id):
  user_id = current_user.id
  delete_todo(user_id=user_id, todo_id=todo_id)

  flash('Task removed', 'success')

  return redirect(url_for('tasks.list'))

@tasks.route('/status/<string:todo_id>/<int:done>', methods=['GET'])
@login_required
def status(todo_id, done):
  user_id = current_user.id

  update_todo(user_id=user_id, todo_id=todo_id, done=done)

  flash('Task updated', 'success')

  return redirect(url_for('tasks.list'))


# -*- coding = 'utf-8' -*-
"""

--------------------------------------------------------

File Name : evaluate

Description : 总的预测函数

Author : leiliang

Date : 2020/8/4 10:59 上午

--------------------------------------------------------

"""
import logging
from base_algorithm import BaseAlgorithm
from utils import transform_table_data_to_html, format_dataframe

log = logging.getLogger(__name__)

class predictModel(BaseAlgorithm):
    def __init__(self):
        BaseAlgorithm.__init__(self)
        self.get_config()
        if self.config["oneSample"]:
            self.table_data = None
        else:
            self.table_data = self.exec_sql(self.config['tableName'], self.config['X'], None)
        self.model = self.load_model_by_database(self.config['algorithm'], self.config['model'])

    def get_config(self):
        '''
            前端传过来的参数
        {
            "algorithm": "",
            "model": "",
            "oneSample": False,
            "table": "",
            "X": "",
        }
        :return:
        '''
        self.config = {}
        try:
            self.config['algorithm'] = self.web_data['algorithm']
            self.config['model'] = self.web_data['model']
            self.config['oneSample'] = self.web_data['oneSample']
            self.config['tableName'] = self.web_data.get('tableName', None)
            self.config['X'] = self.web_data.get('X')
        except Exception as e:
            log.info(e)
            raise e

    def model_predict(self):
        try:
            res = {}
            if self.config['oneSample']:
                if not self.config['X']:
                    raise ValueError("feature must not be empty when one-sample")
                X = [[float(x) for x in self.config['X']]]
                predict = self.model.predict(X)[0] if isinstance(self.model.predict(X)[0], str) else "{:.0f}".format(
                    self.model.predict(X)[0])
                res.update({
                    "data": [[",".join([str(s) for s in self.config['X']]), predict]],
                    "title": "单样本预测结果",
                    "col": ["样本特征", "模型预测结果"],
                })
            else:
                # 从数据库拿数据
                if not self.config['tableName'] or self.config['tableName'] == "":
                    raise ValueError("cannot find table data when predict many samples")
                data = self.table_data
                log.info("输入数据大小:{}".format(len(data)))
                data = data.astype(float)
                data["predict"] = self.model.predict(data.values)
                if data["predict"].dtypes != "object":
                    data = format_dataframe(data, {"predict": ".0f"})
                res.update(transform_table_data_to_html({
                    "data": data.values.tolist(),
                    "title": "多样本预测结果",
                    "col": data.columns.tolist(),
                    "row": data.index.tolist()
                }))
            response_data = {"res": res,
                             "code": "200",
                             "msg": "ok!"}
            return response_data
        except Exception as e:
            # raise e
            return {"data": "", "code": "500", "msg": "{}".format(e.args)}


import streamlit as st
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression

data=pd.read_csv("data//Salary_Data.csv")
lr=LinearRegression()
x = np.array(data['YearsExperience']).reshape(-1,1)
lr.fit(x,np.array(data['Salary']))
st.title("Salary Predictor")

nav= st.sidebar.radio("Navigation",['Home' , 'Prediction' , 'Contribute'])
if nav=='Home':
    st.image("data//salary.jpj")
    if st.checkbox("Show Table"):
        st.table(data)
    graph= st.selectbox("select graph",["Non-interactive","Interactive"])
    val= st.slider('Filter data ',0,20,value=5)
    data=data.loc[data['YearsExperience']>val]
    if graph =="Non-interactive" :
        fig,ax =plt.subplots()
        plt.figure(figsize=(10,5))
        ax.scatter(data['YearsExperience'],data['Salary'])
        plt.xlabel('years of experience')
        plt.ylabel('Salary')
        plt.title('Scatter plt')
        st.pyplot(fig)

    if graph =="Interactive" :
        fig, ax = plt.subplots()
        plt.figure(figsize=(10, 5))
        ax.scatter(data['YearsExperience'], data['Salary'])
        plt.xlabel('years of experience')
        plt.ylabel('Salary')
        plt.title('Scatter plt')
        st.pyplot_chart(fig)

if nav == "Prediction":
    st.image("data//salary.jpj")
    st.header("Know your Salary")
    val = st.number_input("Enter you exp",0.00,20.00,step=0.25)
    val = np.array(val).reshape(1,-1)
    pred =lr.predict(val)[0]

    if st.button("Predict"):
        st.success(f"Your predicted salary is {round(pred)}")
if nav == "Contribute":
    st.image("data//salary.jpj")
    st.header("Contribute to our dataset")
    ex = st.number_input("Enter your Experience",0.0,20.0)
    sal = st.number_input("Enter your Salary",0.00,1000000.00,step = 1000.0)
    if st.button("submit"):
        to_add = {"YearsExperience":[ex],"Salary":[sal]}
        to_add = pd.DataFrame(to_add)
        to_add.to_csv("data//Salary_Data.csv",mode='a',header = False,index= False)
        st.success("Submitted")


# Dependencies
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression, LinearRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score

# Loading data
url = "https://raw.githubusercontent.com/callxpert/datasets/master/iris.data.txt"
names = ['sepal-length', 'sepal-width', 'petal-length', 'petal-width', 'class']
dataset = pd.read_csv(url, names=names)

# Data details
print('Data shape: ', dataset.shape)
print('------------------------------------------------------------------------------')
print('Data first 10:\n ', dataset.head(10))
print('------------------------------------------------------------------------------')
print('Data stats:\n ', dataset.describe())
print('------------------------------------------------------------------------------')

# Class distribution HOW MANY OBJECT HAVE DIFFERENT FLOWER
print('How many objects have different flower:\n', dataset.groupby('class').size())
print('------------------------------------------------------------------------------')

# Box and whisker plots
dataset.plot(kind='box', subplots=True, layout=(2,2), sharex=False, sharey=False)
#plt.show()

# Histogram
dataset.hist()
plt.show()

# Scatter plot matrix IMPORTANT FEATURES WITH RESPECT TO OTHER
pd.plotting.scatter_matrix(dataset, alpha=0.5, figsize=(8, 8), diagonal='kde')
plt.show()

# Splitting the data
array = dataset.values
X = array[:,0:4]
Y = array[:,4]
features_train, features_test, labels_train, labels_test = train_test_split(X, Y, test_size=0.2, random_state=7)

# Parameters for GridSearch for different algorithms
param_SVC = {'kernel':('linear','rbf','poly'), 'C':[1,10,100,1000], 'gamma':[0.0001,0.001,0.01,0.1,1], 'degree':[0,1,2,3]}
param_KN = {'n_neighbors':[1,2,3,4,5], 'algorithm':('auto', 'ball_tree', 'kd_tree'), 'leaf_size':[10,20,30]}
param_RF = {'n_estimators':[2,3,4,5,6,7,8,10,12,14], 'min_samples_split':[2,3,4,5,6], 'min_samples_leaf':[1,2,3,4]}
# param_LogisticR = {'penalty':('l1','12')}

# Making a models and fit the data
clf_SVC = GridSearchCV(SVC(),param_SVC)
clf_SVC.fit(features_train,labels_train)

clf_KN = GridSearchCV(KNeighborsClassifier(), param_KN)
clf_KN.fit(features_train,labels_train)

clf_RF = GridSearchCV(RandomForestClassifier(), param_RF)
clf_RF.fit(features_train,labels_train)

clf_LogisticR = LogisticRegression()
clf_LogisticR.fit(features_train,labels_train)

# Prediction with help of different models
pred_SVC = clf_SVC.predict(features_test)
pred_KN = clf_KN.predict(features_test)
pred_RF = clf_RF.predict(features_test)
pred_LogisticR = clf_LogisticR.predict(features_test)

# Printing the score
print('SVC score: ', accuracy_score(labels_test, pred_SVC))
print('KN score: ', accuracy_score(labels_test, pred_KN))
print('RF score: ', accuracy_score(labels_test, pred_RF))
print('Logistic score: ', accuracy_score(labels_test, pred_LogisticR))


import pandas as pd
from sklearn.ensemble import RandomForestClassifier

# Load the training data
df = pd.read_csv('golf_ball_data.csv')

# Split the data into features and labels
X = df[['gender', 'handicap', 'driver_ball_speed', 'price_range']]
y = df['recommended_ball']

# Train a random forest classifier on the data
model = RandomForestClassifier()
model.fit(X, y)

# Ask the user for their information
gender = input('What is your gender (male/female)? ')
handicap = int(input('What is your golf handicap? '))
driver_ball_speed = int(input('What is your driver ball speed (in mph)? '))
price_range = int(input('What is your price range (1-5)? '))
current_ball = input('What golf ball do you currently use? ')
like_current_ball = input('Do you like your current golf ball (yes/no)? ')

# If the user likes their current golf ball, add it to the training data
if like_current_ball == 'yes':
    new_data = {'gender': [gender], 'handicap': [handicap], 'driver_ball_speed': [driver_ball_speed], 'price_range': [price_range], 'recommended_ball': [current_ball]}
    new_df = pd.DataFrame(data=new_data)
    df = df.append(new_df, ignore_index=True)
    X = df[['gender', 'handicap', 'driver_ball_speed', 'price_range']]
    y = df['recommended_ball']
    model.fit(X, y)

# Make a prediction based on the user's information
prediction = model.predict([[gender, handicap, driver_ball_speed, price_range]])

# Recommend a golf ball to the user
print(f'We recommend using the {prediction[0]} golf ball.')