text:
import os
import cPickle
import numpy as np
import tensorflow as tf
from sklearn.metrics import confusion_matrix

from utils import birds_utils
from utils import utils_vgg16
from params import Params

######### Modifiable Settings ##########
gpu         = Params.gpu_ewc      # Used GPU
nb_cl       = Params.nb_cl        # Classes per group
nb_gr       = Params.nb_groups    # Number of groups
nb_sa       = Params.num_samples  # for calculating Fisher Information
lr_init     = Params.lr_init      # Starting learning rate
wght_decay  = Params.wght_decay   # Weight Decay
train_path  = Params.train_path   # Path to read images
data_size   = Params.data_size    # Image size
ratio       = Params.ratio        # Trade-off between old tasks and new task
eval_single = Params.eval_single  # True: Evaluate different task separately
########################################

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
os.environ["CUDA_VISIBLE_DEVICES"] = gpu

# Sanity Initializations
model_vgg = []
acc_task = np.zeros((nb_gr, nb_gr))

# Loading CUB-200 Birds dataset
print 'Loading full dataset...'
full_birds = birds_utils.load_birds(train_path, data_size)

# Experiment name: results will be loaded from this folder.
Ex_name = 'EWC_vgg16' + str(nb_gr) + '_lr' + str(lr_init) + '_sample_' + str(nb_sa) + '_ratio_' + str(ratio) + '/'
save_path = Params.save_path + Ex_name
print 'Files will be loaded from: ' + save_path
print '---'

# Evaluate performance
for it in range(nb_gr):

    # Initialization
    confusion_mat = []
    for _ in range(2):
        confusion_mat.append([])

    # Define placeholders
    x = tf.placeholder("float", [None, data_size, data_size, 3])
    y = tf.placeholder("float", [None, nb_cl * nb_gr])

    # Get the data for this task
    print('Retrieving data for task {}...'.format(it + 1))
    _, _, tst_img, _, _, tst_lbl = birds_utils.disjoint_birds(full_birds, range(0, (it+1) * nb_cl))

    # Building the graph
    if it == 0:
        with tf.variable_scope('Vgg16'):
            model_vgg = utils_vgg16.Vgg16(x)
    else:
        with tf.variable_scope('Vgg16'):
            model_vgg.build(x, rotate=False, pre_init=True)
    correct_pred = tf.equal(tf.argmax(model_vgg.y, 1), tf.argmax(y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

    # Loading weights
    save_weights = cPickle.load(open(save_path + 'model-iteration' + str(nb_cl) + '-%i.pickle' % it, 'rb'))

    # Inference and metrics
    with tf.Session(config=config) as sess:
        sess.run(tf.global_variables_initializer())
        variables_graph = tf.get_collection(tf.GraphKeys.WEIGHTS)
        void0 = sess.run([(variables_graph[i]).assign(save_weights[i]) for i in range(len(variables_graph))])

        fg = np.arange(len(tst_img))
        # Evaluation routine
        for i in range(int(np.floor(len(tst_img)))):
            batch_x = tst_img[fg[i:(i+1)]]
            batch_y = tst_lbl[fg[i:(i+1)]]
            acc, sc = sess.run([accuracy, model_vgg.y], feed_dict={x: batch_x, y: np.eye(nb_gr*nb_cl)[batch_y]})
            tmp = np.argsort(sc[:, batch_y[0]/nb_cl*nb_cl:(batch_y[0]/nb_cl+1)*nb_cl], axis=1)[:, -1:]   # Top 1 accuracy
            if eval_single:
                # Single-head evaluation
                confusion_mat[0].append(batch_y)
                confusion_mat[1].append(np.argsort(sc[:, :(it+1)*nb_cl], axis=1)[:, -1:])
            else:
                # Multi-head evaluation
                confusion_mat[0].append(batch_y)
                confusion_mat[1].append(tmp+batch_y/nb_cl*nb_cl)

        confusion_mat = np.asarray(confusion_mat)
        confusion_mat = np.reshape(confusion_mat, (confusion_mat.shape[0], confusion_mat.shape[1]))
        for k in range(it+1):
            cmat = confusion_matrix(confusion_mat[0], confusion_mat[1])
            tmp = cmat.diagonal()/(cmat.sum(axis=1)*1.0)
            acc_task[it, k] = np.sum(tmp[k*nb_cl:(k+1)*nb_cl]/nb_cl)

    tf.reset_default_graph()

print('Accuracy for each task is:' + str(acc_task))

max_stars_repo_name: xialeiliu/RotateNetworks

max_issues_repo_name: xialeiliu/RotateNetwoks

max_forks_repo_name: xialeiliu/RotateNetwoks


text:
"""
pyqtgraphPlotHelpers.py

Routines to help use pyqtgraph and make cleaner plots
as well as get plots read for publication.

Intially copied from PlotHelpers.py for matplotlib.
Modified to allow us to use a list of axes, and operate on all of those,
or to use just one axis if that's all that is passed.
Therefore, the first argument to these calls can either be a pyqtgraph axis object,
or a list of axes objects.  2/10/2012 pbm.

Created by Paul Manis on 2010-03-09.
Copyright 2010-2019  Paul Manis
Distributed under MIT/X11 license. See license.txt for more infofmation.

"""
from __future__ import print_function
from __future__ import absolute_import

import string

stdFont = 'Arial'

import scipy.stats
import numpy as np
import pyqtgraph as pg
from PyQt5 import QtGui
from . import talbotetalticks as ticks # logical tick formatting...

"""
Basic functions:
"""

def nice_plot(plotlist, spines=['left', 'bottom'], position=10, direction='inward', axesoff = False):
    """ Adjust a plot so that it looks nicer than the default matplotlib plot
        Also allow quick access to things we like to do for publication plots, including:
        using a calbar instead of an axes: calbar = [x0, y0, xs, ys]
        inserting a reference line (grey, 3pt dashed, 0.5pt, at refline = y position)

        Parameters
        ----------
        plotlist : list
            a plot handle or list of plot handles to which the "niceplot" will be applied

        spines : list
            a list of which axes should have spines. Not relevant for pyqtgraph

        position : int
            not relevant for pyqtgraph

        direction : string
            need to implement for pyqtgraph

        axesoff : boolean
            flag that forces plots to turn axes off

        Returns
        -------
        Nothing
    """

    if not isinstance(plotlist, list):
        plotlist = [plotlist]
    for pl in plotlist:
        if axesoff is True:
            pl.hideAxis('bottom')
            pl.hideAxis('left')

def noaxes(plotlist, whichaxes = 'xy'):
    """ take away all the axis ticks and the lines

    Parameters
    ----------
        plotlist : list
            list of plot handles
        whichaxes : string
            string describing which axes to remove: 'x', 'y', or 'xy' for both

    """
    if not isinstance(plotlist, list):
        plotlist = [plotlist]
    for pl in plotlist:
        if 'x' in whichaxes:
            pl.hideAxis('bottom')
        if 'y' in whichaxes:
            pl.hideAxis('left')

def setY(ax1, ax2):
    """
    Set the Y axis of all the plots in ax2 to be like ax1

    Parameters
    ----------
        ax1 : pyqtgraph plot instance
        ax2 : list
            list of target plots that will have the axes properties of ax1
    """
    if type(ax1) is list:
        print('PlotHelpers: cannot use list as source to set Y axis')
        return
    if type(ax2) is not list:
        ax2 = [ax2]
    y = ax1.getAxis('left')
    refy = y.range # return the current range
    for ax in ax2:
        ax.setRange(refy)

def setX(ax1, ax2):
    """
    Set the X axis of all the plots in ax2 to be like ax1

    Parameters
    ----------
        ax1 : pyqtgraph plot instance
        ax2 : list
            list of target plots that will have the axes properties of ax1
    """
    if type(ax1) is list:
        print('PlotHelpers: cannot use list as source to set X axis')
        return
    if type(ax2) is not list:
        ax2 = [ax2]
    x = ax1.getAxis('bottom')
    refx = x.range
    for ax in ax2:
        ax.setrange(refx)

def labelPanels(axl, axlist=None, font='Arial', fontsize=18, weight = 'normal'):
    """
    Label the panels like a specific panel

    Parameters
    ----------
    axl : dict or list

    axlist : list, optional
        list of labels to use for the axes, defaults to None

    font : str, optional
        Font to use for the labels, defaults to Arial

    fontsize : int, optional
        Font size in points for the labels, defaults to 18

    weight : str, optional
        Font weight to use, defaults to 'normal'

    """

    if type(axl) is dict:
        axt = [axl[x] for x in axl]
        axlist = axl.keys()
        axl = axt
    if type(axl) is not list:
        axl = [axl]
    if axlist is None:
        axlist = string.uppercase(1,len(axl)) # assume we wish to go in sequence

    for i, ax in enumerate(axl):
        labelText = pg.TextItem(axlist[i])
        y = ax.getAxis('left').range
        x = ax.getAxis('bottom').range
        ax.addItem(labelText)
        labelText.setPos(x[0], y[1])

def listAxes(axd):
    """
    make a list of the axes from the dictionary of axes

    Parameters
    ----------
    axd : dict
        a dict of axes, whose values are returned in a list

    Returns
    -------
    list : a list of the axes

    """
    if type(axd) is not dict:
        if type(axd) is list:
            return axd
        else:
            print('listAxes expects dictionary or list; type not known (fix the code)')
            raise
    axl = [axd[x] for x in axd]
    return axl

def cleanAxes(axl):
    """
    """
    if type(axl) is not list:
        axl = [axl]
    # does nothing at the moment, as axes are already "clean"
    # for ax in axl:
    #
    #    update_font(ax)

def formatTicks(axl, axis='xy', fmt='%d', font='Arial'):
    """
    Convert tick labels to intergers
    to do just one axis, set axis = 'x' or 'y'
    control the format with the formatting string
    """
    if type(axl) is not list:
        axl = [axl]

def autoFormatTicks(axl, axis='xy', font='Arial'):
    if type(axl) is not list:
        axl = [axl]
    for ax in axl:
        if 'x' in axis:
            b = ax.getAxis('bottom')
            x0 = b.range
 #           setFormatter(ax,  x0, x1, axis = 'x')
        if 'y' in axis:
            l = ax.getAxis('left')
            y0= l.range
 #           setFormatter(ax, y0, y1, axis = 'y')

def setFormatter(ax, x0, x1, axis='x'):
    datarange = np.abs(x0-x1)
    mdata = np.ceil(np.log10(datarange))
    # if mdata > 0 and mdata <= 4:
    #     majorFormatter = FormatStrFormatter('%d')
    # elif mdata > 4:
    #     majorFormatter = FormatStrFormatter('%e')
    # elif mdata <= 0 and mdata > -1:
    #     majorFormatter = FormatStrFormatter('%5.1f')
    # elif mdata < -1 and mdata > -3:
    #     majorFormatatter = FormatStrFormatter('%6.3f')
    # else:
    #     majorFormatter = FormatStrFormatter('%e')
    # if axis == 'x':
    #     ax.xaxis.set_major_formatter(majorFormatter)
    # else:
    #     ax.yaxis.set_major_formatter(majorFormatter)

def update_font(axl, size=6, font=stdFont):
    pass
    # if type(axl) is not list:
    #     axl = [axl]
    # fontProperties = {'family':'sans-serif','sans-serif':[font],
    #         'weight' : 'normal', 'size' : size}
    # for ax in axl:
    #     for tick in ax.xaxis.get_major_ticks():
    #           tick.label1.set_family('sans-serif')
    #           tick.label1.set_fontname(stdFont)
    #           tick.label1.set_size(size)
    #
    #     for tick in ax.yaxis.get_major_ticks():
    #           tick.label1.set_family('sans-serif')
    #           tick.label1.set_fontname(stdFont)
    #           tick.label1.set_size(size)
    #     ax.set_xticklabels(ax.get_xticks(), fontProperties)
    #     ax.set_yticklabels(ax.get_yticks(), fontProperties)
    #     ax.xaxis.set_smart_bounds(True)
    #     ax.yaxis.set_smart_bounds(True)
    #     ax.tick_params(axis = 'both', labelsize = 9)

def lockPlot(axl, lims, ticks=None):
    """
        This routine forces the plot of invisible data to force the axes to take certain
        limits and to force the tick marks to appear.
        call with the axis and lims = [x0, x1, y0, y1]
    """
    if type(axl) is not list:
        axl = [axl]
    plist = []
    for ax in axl:
        y = ax.getAxis('left')
        x = ax.getAxis('bottom')
        x.setRange(lims[0], lims[1])
        y.setRange(lims[2], lims[3])

def calbar(plotlist, calbar=None, axesoff=True, orient='left', unitNames=None):
    """ draw a calibration bar and label it up. The calibration bar is defined as:
        [x0, y0, xlen, ylen]

        Parameters
        ----------
        plotlist : list
            a plot item or a list of plot items for which a calbar will be applied
        calbar : list, optional
            a list with 4 elements, describing the calibration bar
                [xposition, yposition, xlength, ylength] in units of the data inthe plot
                defaults to None
        axesoff : boolean, optional
                Set true to turn off the standard axes, defaults to True
        orient : text, optional
                'left': put vertical part of the bar on the left
                'right': put the vertical part of the bar on the right
                defaults to 'left'
        unitnames: str, optional
                a dictionary with the names of the units to append to the calibration bar
                lengths. Example: {'x': 'ms', 'y': 'nA'}
                defaults to None
        Returns
        -------
        Nothing
    """

    if type(plotlist) is not list:
        plotlist = [plotlist]
    for pl in plotlist:
        if axesoff is True:
            noaxes(pl)
        Vfmt = '%.0f'
        if calbar[2] < 1.0:
            Vfmt = '%.1f'
        Hfmt = '%.0f'
        if calbar[3] < 1.0:
            Hfmt = '%.1f'
        if unitNames is not None:
            Vfmt = Vfmt + ' ' + unitNames['x']
            Hfmt = Hfmt + ' ' + unitNames['y']
        Vtxt = pg.TextItem(Vfmt % calbar[2], anchor=(0.5, 0.5), color=pg.mkColor('k'))
        Htxt = pg.TextItem(Hfmt % calbar[3], anchor=(0.5, 0.5), color=pg.mkColor('k'))
       # print pl
        if calbar is not None:
            if orient == 'left': # vertical part is on the left
                pl.plot([calbar[0], calbar[0], calbar[0]+calbar[2]],
                    [calbar[1]+calbar[3], calbar[1], calbar[1]],
                    pen=pg.mkPen('k'), linestyle = '-', linewidth = 1.5)
                ht = Htxt.setPos(calbar[0]+0.05*calbar[2], calbar[1]+0.5*calbar[3])
            elif orient == 'right': # vertical part goes on the right
                pl.plot([calbar[0] + calbar[2], calbar[0]+calbar[2], calbar[0]],
                    [calbar[1]+calbar[3], calbar[1], calbar[1]],
                    pen=pg.mkPen('k'), linestyle = '-', linewidth = 1.5)
                ht = Htxt.setPos(calbar[0]+calbar[2]-0.05*calbar[2], calbar[1]+0.5*calbar[3])
            else:
                print("PlotHelpers.py: I did not understand orientation: %s" % (orient))
                print("plotting as if set to left... ")
                pl.plot([calbar[0], calbar[0], calbar[0]+calbar[2]],
                    [calbar[1]+calbar[3], calbar[1], calbar[1]],
                    pen=pg.mkPen('k'), linestyle = '-', linewidth = 1.5)
                ht = Htxt.setPos(calbar[0]+0.05*calbar[2], calbar[1]+0.5*calbar[3])
                Htxt.setText(Hfmt % calbar[3])
            xc = float(calbar[0]+calbar[2]*0.5)  # always centered, below the line
            yc = float(calbar[1]-0.1*calbar[3])
            vt = Vtxt.setPos(xc, yc)
            Vtxt.setText(Vfmt % calbar[2])
            pl.addItem(Htxt)
            pl.addItem(Vtxt)

def refline(axl, refline=None, color=[64, 64, 64], linestyle='--' ,linewidth=0.5, orient='horizontal'):
    """
    Draw a reference line at a particular level of the data on the y axis

    Parameters
    ----------
    axl : list
        axis handle or list of axis handles
    refline : float, optional
            the position of the reference line, defaults to None
    color : list, optional
            the RGB color list for the line, in format [r,g,b], defaults to [64, 64, 64] (faint grey line)
    linestyle : str, optional
            defines the linestyle to be used: -- for dash, . for doted, - for solid, -. for dash-dot,
            -.. for -.., etc.
            defaults to '--' (dashed)
    linewidth : float, optional
            width of the line, defaults to 0.5
    """
    if type(axl) is not list:
        axl = [axl]
    if linestyle == '--':
        style = pg.QtCore.Qt.DashLine
    elif linestyle == '.':
        style = pg.QtCore.Qt.DotLine
    elif linestyle == '-':
        style = pg.QtCore.Qt.SolidLine
    elif linestyle == '-.':
        style = pg.QtCore.Qt.DsahDotLine
    elif linestyle == '-..':
        style = pg.QtCore.Qt.DashDotDotLine
    else:
        style = pg.QtCore.Qt.SolidLine # default is solid
    if orient == 'horizontal':
        for ax in axl:
            if refline is not None:
                x = ax.getAxis('bottom')
                xlims = x.range
                ax.plot(xlims, [refline, refline], pen=pg.mkPen(color, width=linewidth, style=style))
    if orient == 'vertical':
        for ax in axl:
            if refline is not None:
                y = ax.getAxis('left')
                ylims = y.range
                ax.plot([refline, refline], [ylims[0]+0.5, ylims[1]-0.5], pen=pg.mkPen(color, width=linewidth, style=style))

def tickStrings(values, scale=1, spacing=None, tickPlacesAdd=1):
    """Return the strings that should be placed next to ticks. This method is called
    when redrawing the axis and is a good method to override in subclasses.

    Parameters
    ----------
    values : array or list
         An array or list of tick values
    scale : float, optional
        a scaling factor (see below), defaults to 1
    spacing : float, optional
        spaceing between ticks (this is required since, in some instances, there may be only
    one tick and thus no other way to determine the tick spacing). Defaults to None
    tickPlacesToAdd : int, optional
        the number of decimal places to add to the ticks, default is 1

    Returns
    -------
    list : a list containing the tick strings

    The scale argument is used when the axis label is displaying units which may have an SI scaling prefix.
    When determining the text to display, use value*scale to correctly account for this prefix.
    For example, if the axis label's units are set to 'V', then a tick value of 0.001 might
    be accompanied by a scale value of 1000. This indicates that the label is displaying 'mV', and
    thus the tick should display 0.001 * 1000 = 1.
    Copied rom pyqtgraph; we needed it here.
    """
    if spacing is None:
        spacing = np.mean(np.diff(values))
    places = max(0, np.ceil(-np.log10(spacing*scale))) + tickPlacesAdd
    strings = []
    for v in values:
        vs = v * scale
        if abs(vs) < .001 or abs(vs) >= 10000:
            vstr = "%g" % vs
        else:
            vstr = ("%%0.%df" % places) % vs
        strings.append(vstr)
    return strings

def crossAxes(axl, xyzero=[0., 0.], limits=[None, None, None, None], **kwds):
    """
    Make the plot(s) have crossed axes at the data points set by xyzero, and optionally
    set axes limits

    Parameters
    ----------
    axl : pyqtgraph plot/axes instance or list
        the plot to modify
    xyzero : list
        A 2-element list for the placement of x=0 and y=0, defaults to [0., 0.]
    limits : list
        A 4-element list with the min and max limits of the axes, defaults to all None
    **kwds : keyword arguments to pass to make_crossedAxes

    """
    if type(axl) is not list:
        axl = [axl]
    for ax in axl:
        make_crossedAxes(ax, xyzero, limits, **kwds)

def make_crossedAxes(ax, xyzero=[0., 0.], limits=[None, None, None, None], ndec=3,
            density=(1.0, 1.0), tickl = 0.0125, insideMargin=0.05, pointSize=12, tickPlacesAdd=(0,0)):
    """
    Parameters
    ----------
    axl : pyqtgraph plot/axes instance or list
        the plot to modify
    xyzero : list
        A 2-element list for the placement of x=0 and y=0, defaults to [0., 0.]
    limits : list
        A 4-element list with the min and max limits of the axes, defaults to all None
    ndec : int
        Number of decimals (would be passed to talbotTicks if that was being called)
    density : tuple
        tick density (for talbotTicks), defaults to (1.0, 1.0)
    tickl : float
        Tick length, defaults to 0.0125
    insideMargin : float
        Inside margin space for plot, defaults to 0.05 (5%)
    pointSize : int
         point size for tick text, defaults to 12
    tickPlacesAdd : tuple
        number of decimal places to add in tickstrings for the ticks, pair for x and y axes, defaults to (0,0)

    Returns
    -------
    Nothing

    """
    # get axis limits
    aleft = ax.getAxis('left')
    abottom = ax.getAxis('bottom')
    aleft.setPos(pg.Point(3., 0.))
    yRange = aleft.range
    xRange =  abottom.range
    hl = pg.InfiniteLine(pos=xyzero[0], angle=90, pen=pg.mkPen('k'))
    ax.addItem(hl)
    vl = pg.InfiniteLine(pos=xyzero[1], angle=0, pen=pg.mkPen('k'))
    ax.addItem(vl)
    ax.hideAxis('bottom')
    ax.hideAxis('left')
    # now create substitue tick marks and labels, using Talbot et al algorithm
    xr = np.diff(xRange)[0]
    yr = np.diff(yRange)[0]
    xmin, xmax = (np.min(xRange) - xr * insideMargin, np.max(xRange) + xr * insideMargin)
    ymin, ymax = (np.min(yRange) - yr * insideMargin, np.max(yRange) + yr * insideMargin)
    xtick = ticks.Extended(density=density[0], figure=None, range=(xmin, xmax), axis='x')
    ytick = ticks.Extended(density=density[1], figure=None, range=(ymin, ymax), axis='y')
    xt = xtick()
    yt = ytick()
    ytk = yr*tickl
    xtk = xr*tickl
    y0 = xyzero[1]
    x0 = xyzero[0]
    tsx = tickStrings(xt, tickPlacesAdd=tickPlacesAdd[0])
    tsy = tickStrings(yt, tickPlacesAdd=tickPlacesAdd[1])
    for i, x in enumerate(xt):
        t = pg.PlotDataItem(x=x*np.ones(2), y=[y0-ytk, y0+ytk], pen=pg.mkPen('k'))
        ax.addItem(t)  # tick mark
        # put text in only if it does not overlap the opposite line
        if x == y0:
            continue
        txt = pg.TextItem(tsx[i], anchor=(0.5, 0), color=pg.mkColor('k')) #, size='10pt')
        txt.setFont(pg.QtGui.QFont('Arial', pointSize=pointSize))
        txt.setPos(pg.Point(x, y0-ytk))
        ax.addItem(txt) #, pos=pg.Point(x, y0-ytk))
    for i, y in enumerate(yt):
        t = pg.PlotDataItem(x=np.array([x0-xtk, x0+xtk]), y=np.ones(2)*y, pen=pg.mkPen('k'))
        ax.addItem(t)
        if y == x0:
            continue
        txt = pg.TextItem(tsy[i], anchor=(1, 0.5), color=pg.mkColor('k')) # , size='10pt')
        txt.setFont(pg.QtGui.QFont('Arial', pointSize=pointSize))
        txt.setPos(pg.Point(x0-xtk, y))
        ax.addItem(txt) #, pos=pg.Point(x, y0-ytk))

class polarPlot():
    """
    Create a polar plot, as a PlotItem for pyqtgraph.
    """

    def __init__(self, plot=None):
        """
        Instantiate a plot as a polar plot

        Parameters
        ---------
        plot : pyqtgraph plotItem
            the plot that will be converted to a polar plot, defaults to None
            if None, then a new PlotItem will be created, accessible
            as polarPlot.plotItem

        """

        if plot is None:
            self.plotItem = pg.PlotItem()  # create a plot item for the plot
        else:
            self.plotItem = plot
        self.plotItem.setAspectLocked()
        self.plotItem.hideAxis('bottom')
        self.plotItem.hideAxis('left')
        self.gridSet = False
        self.data = None
        self.rMax = None

    def setAxes(self, steps=4, rMax=None, makeGrid=True):
        """
        Make the polar plot axes

        Parameters
        ----------
        steps : int, optional
            The number of radial steps for the grid, defaults to 4
        rMax : float, optional
            The maximum radius of the plot, defaults to None (the rMax is 1)
        makeGrid : boolean, optional
            Whether the grid will actually be plotted or not, defaults to True

        """
        if makeGrid is False or self.gridSet:
            return
        if rMax is None:
            if self.data is None:
                rMax = 1.0
            else:
                rMax = np.max(self.data['y'])
        self.rMax = rMax
        # Add radial grid lines (theta markers)
        gridPen = pg.mkPen(width=0.55, color='k', style=pg.QtCore.Qt.DotLine)
        ringPen = pg.mkPen(width=0.75, color='k', style=pg.QtCore.Qt.SolidLine)
        for th in np.linspace(0., np.pi*2, 8, endpoint=False):
            rx = np.cos(th)*rMax
            ry = np.sin(th)*rMax
            self.plotItem.plot(x=[0, rx], y=[0., ry], pen=gridPen)
            ang = th*360./(np.pi*2)
            # anchor is odd: 0,0 is upper left corner, 1,1 is lower right corner
            if ang < 90.:
                x=0.
                y=0.5
            elif ang == 90.:
                x=0.5
                y=1
            elif ang < 180:
                x=1.0
                y=0.5
            elif ang == 180.:
                x=1
                y=0.5
            elif ang < 270:
                x=1
                y=0
            elif ang== 270.:
                x=0.5
                y=0
            elif ang < 360:
                x=0
                y=0
            ti = pg.TextItem("%d" % (int(ang)), color=pg.mkColor('k'), anchor=(x,y))
            self.plotItem.addItem(ti)
            ti.setPos(rx, ry)
        # add polar grid lines (r)
        for gr in np.linspace(rMax/steps, rMax, steps):
            circle = pg.QtGui.QGraphicsEllipseItem(-gr, -gr, gr*2, gr*2)
            if gr < rMax:
                circle.setPen(gridPen)
            else:
                circle.setPen(ringPen)
            self.plotItem.addItem(circle)
            ti = pg.TextItem("%d" % (int(gr)), color=pg.mkColor('k'), anchor=(1, 1))
            ti.setPos(gr, 0.)
            self.plotItem.addItem(ti)
        self.gridSet = True

    def plot(self, r, theta, vectors=False, arrowhead=True, normalize=False, sort=False, **kwds):
        """
        plot puts the data into a polar plot.
        the plot will be converted to a polar graph

        Parameters
        ----------
        r : list or numpy array
            a list or array of radii
        theta : list or numpy array
            a list or array of angles (in radians) corresponding to the values in r
        vectors : boolean, optional
            vectors True means that plot is composed of vectors to each point radiating from the origin, defaults to False
        arrowhead : boolean, optional
            arrowhead True plots arrowheads at the end of the vectors, defaults to True
        normalize : boolean, optional
            normalize forces the plot to be scaled to the max values in r, defaults to False
        sort : boolean, optional
            causes data r, theta to be sorted by theta, defaults to False
        **kwds are passed to the data plot call.

        """

        # sort r, theta by r

        rs = np.array(r)
        thetas = np.array(theta)

        if sort:
            indx = np.argsort(thetas)
            theta = thetas
            if not isinstance(indx, np.int64):
                for i, j in enumerate(indx):
                    rs[i] = r[j]
                    thetas[i] = theta[j]

        # Transform to cartesian and plot
        if normalize:
            rs = rs/np.max(rs)
        x = rs * np.cos(thetas)
        y = rs * np.sin(thetas)
        try:
            len(x)
        except:
            x = [x]
            y = [y]
        if vectors:  # plot r,theta as lines from origin
            for i, xi in enumerate(x):
    #            print x[i], y[i]
                if arrowhead:
                    arrowAngle = -(thetas[i]*360/(2*np.pi)+180) # convert to degrees, and correct orientation
                    arrow = pg.ArrowItem(angle=arrowAngle, tailLen=0, tailWidth=1.5, **kwds)
                    arrow.setPos(x[i], y[i])
                    self.plotItem.addItem(arrow)
                self.plotItem.plot([0., x[i]], [0., y[i]], **kwds)

        else:
            self.plotItem.plot(x, y, **kwds)
        self.rMax = np.max(y)
        self.data = {'x': x, 'y': y}

    def hist(self, r, theta, binwidth=np.pi/6., normalize=False, density=False, mode='straight', **kwds):
        """
        plot puts the data into a polar plot as a histogram of the number of observations
        within a wedge
        the plot will be converted to a polar graph

        Parameters
        ----------
        r : list or numpy array
            a list or array of radii
        theta : list or numpy array
            a list or array of angles (in radians) corresponding to the values in r
        binwidth : bin width, in radians optional
            vectors True means that plot is composed of vectors to each point radiating from the origin, defaults to 30 degrees (np.pi/6)
        normalize : boolean, optional
            normalize forces the plot to be scaled to the max values in r, defaults to False
        density : boolean, optional
            plot a count histogram, or a density histogram weighted by r values, defaults to False
        mode : str, optional
            'straight' selects straight line between bars. 'arc' makes the end of the bar an arc (truer representation), defaults to 'straight'
        **kwds are passed to the data plot call.

        Returns
        -------
        tuple : (list of rHist, list of bins)
            The histogram that was plotted (use for statistical comparisions)
        """

        rs = np.array(r)
        thetas = np.array(theta)
        twopi = np.pi*2.0
        for i, t in enumerate(thetas):  # restrict to positive half plane [0....2*pi]
            while t < 0.0:
                t += twopi
            while t > twopi:
                t -= twopi
            thetas[i] = t
        bins = np.arange(0, np.pi*2+1e-12, binwidth)
        # compute histogram
        (rhist, rbins) = np.histogram(thetas, bins=bins, weights=rs, density=density)
        # Transform to cartesian and plot
        if normalize:
            rhist = rhist/np.max(rhist)
        xo = rhist * np.cos(bins[:-1])  # get cartesian form
        xp = rhist * np.cos(bins[:-1] + binwidth)
        yo = rhist * np.sin(bins[:-1])
        yp = rhist * np.sin(bins[:-1] + binwidth)
        arcinc = np.pi/100.  # arc increments
        for i in range(len(xp)):
            if mode == 'arc':
                self.plotItem.plot([xo[i], 0., xp[i]], [yo[i], 0., yp[i]], **kwds) # "v" segement
                arcseg = np.arange(bins[i], bins[i+1], arcinc)
                x = np.array(rhist[i] * np.cos(arcseg))
                y = np.array(rhist[i] * np.sin(arcseg))
                self.plotItem.plot(x, y, **kwds)

            else:
                self.plotItem.plot([0., xo[i], xp[i], 0.], [0., yo[i], yp[i], 0.], **kwds)
        self.data = {'x': xo, 'y': yo}
        self.rMax = np.max(yo)
        return (rhist, rbins)

    def circmean(self, alpha, axis=None):
        """
        Compute the circular mean of a set of angles along the axis

        Parameters
        ----------
        alpha : numpy array
            the angles to compute the circular mean of
        axis : int
            The axis of alpha for the computatoin, defaults to None

        Returns
        -------
        float : the mean angle
        """
        mean_angle = np.arctan2(np.mean(np.sin(alpha),axis),np.mean(np.cos(alpha),axis))
        return mean_angle

def talbotTicks(axl, **kwds):
    """
    Adjust the tick marks using the talbot et al algorithm, on an existing plot.
    """
    if type(axl) is not list:
        axl = [axl]
    for ax in axl:
        do_talbotTicks(ax, **kwds)

def do_talbotTicks(ax, ndec=3,
                   density=(1.0, 1.0), insideMargin=0.05, pointSize=None, tickPlacesAdd=(0,0)):
    """
    Change the axis ticks to use the talbot algorithm for ONE axis
    Paramerters control the ticks

    Parameters
    ----------
    ax : pyqtgraph axis instance
        the axis to change the ticks on
    ndec : int
        Number of decimals (would be passed to talbotTicks if that was being called)
    density : tuple
        tick density (for talbotTicks), defaults to (1.0, 1.0)
    insideMargin : float
        Inside margin space for plot, defaults to 0.05 (5%)
    pointSize : int
         point size for tick text, defaults to 12
    tickPlacesAdd : tuple
        number of decimal places to add in tickstrings for the ticks, pair for x and y axes, defaults to (0,0)

    """
    # get axis limits
    aleft = ax.getAxis('left')
    abottom = ax.getAxis('bottom')
    yRange = aleft.range
    xRange =  abottom.range
    # now create substitue tick marks and labels, using Talbot et al algorithm
    xr = np.diff(xRange)[0]
    yr = np.diff(yRange)[0]
    xmin, xmax = (np.min(xRange) - xr * insideMargin, np.max(xRange) + xr * insideMargin)
    ymin, ymax = (np.min(yRange) - yr * insideMargin, np.max(yRange) + yr * insideMargin)
    xtick = ticks.Extended(density=density[0], figure=None, range=(xmin, xmax), axis='x')
    ytick = ticks.Extended(density=density[1], figure=None, range=(ymin, ymax), axis='y')
    xt = xtick()
    yt = ytick()
    xts = tickStrings(xt, scale=1, spacing=None, tickPlacesAdd = tickPlacesAdd[0])
    yts = tickStrings(yt, scale=1, spacing=None, tickPlacesAdd = tickPlacesAdd[1])
    xtickl = [[(x, xts[i]) for i, x in enumerate(xt)] , []]  # no minor ticks here
    ytickl = [[(y, yts[i]) for i, y in enumerate(yt)] , []]  # no minor ticks here

    #ticks format: [ (majorTickValue1, majorTickString1), (majorTickValue2, majorTickString2), ... ],
    aleft.setTicks(ytickl)
    abottom.setTicks(xtickl)
    # now set the point size (this may affect spacing from axis, and that would have to be adjusted - see the pyqtgraph google groups)
    if pointSize is not None:
        b = pg.QtGui.QFont()
        b.setPixelSize(pointSize)
        aleft.tickFont = b
        abottom.tickFont = b

def violinPlotScatter(ax, data, symbolColor='k', symbolSize=4, symbol='o'):
    """
    Plot data as violin plot with scatter and error bar

    Parameters
    ----------
    ax : pyqtgraph plot instance
        is the axs to plot into
    data : dict
        dictionary containing {pos1: data1, pos2: data2}, where pos is the x position for the data in data. Each data
        set iis plotted as a separate column
    symcolor : string, optional
        color of the symbols, defaults to 'k' (black)
    symbolSize : int, optional
        Size of the symbols in the scatter plot, points, defaults to 4
    symbol : string, optoinal
        The symbol to use, defaults to 'o' (circle)
    """

    y = []
    x = []
    xb=np.arange(0,len(data.keys()), 1)
    ybm = [0]*len(data.keys()) # np.zeros(len(sdat.keys()))
    ybs = [0]*len(data.keys()) # np.zeros(len(sdat.keys()))
    for i, k in enumerate(data.keys()):
        yvals = np.array(data[k])
        xvals = pg.pseudoScatter(yvals, spacing=0.4, bidir=True) * 0.2
        ax.plot(x=xvals+i, y=yvals, pen=None, symbol=symbol, symbolSize=symbolSize,
            symbolBrush=pg.mkBrush(symbolColor))
        y.append(yvals)
        x.append([i]*len(yvals))
        ybm[i] = np.nanmean(yvals)
        ybs[i] = np.nanstd(yvals)
        mbar = pg.PlotDataItem(x=np.array([xb[i]-0.2, xb[i]+0.2]), y=np.array([ybm[i], ybm[i]]), pen={'color':'k', 'width':0.75})
        ax.addItem(mbar)
        bar = pg.ErrorBarItem(x=xb, y=np.array(ybm), height=np.array(ybs), beam=0.2, pen={'color':'k', 'width':0.75})
    violin_plot(ax, y, xb, bp=False)
    ax.addItem(bar)
    ticks = [[(v, k) for v, k in enumerate(data.keys())], []]
    ax.getAxis('bottom').setTicks(ticks)

def violin_plot(ax, data, pos, dist=.0, bp=False):
    '''
    create violin plots on an axis
    '''

    if data is None or len(data) == 0:
        return  # skip trying to do the plot

    dist = max(pos)-min(pos)
    w = min(0.15*max(dist,1.0),0.5)
    for i, d in enumerate(data):
        if d == [] or len(d) == 0:
            continue
        k = scipy.stats.gaussian_kde(d) #calculates the kernel density
        m = k.dataset.min() #lower bound of violin
        M = k.dataset.max() #upper bound of violin
        y = np.arange(m, M, (M-m)/100.) # support for violin
        v = k.evaluate(y) #violin profile (density curve)
        v = v / v.max() * w #scaling the violin to the available space
        c1 = pg.PlotDataItem(y=y, x=pos[i]+v, pen=pg.mkPen('k', width=0.5))
        c2 = pg.PlotDataItem(y=y, x=pos[i]-v, pen=pg.mkPen('k', width=0.5))
        #mean = k.dataset.mean()
        #vm = k.evaluate(mean)
        #vm = vm * w
        #ax.plot(x=np.array([pos[i]-vm[0], pos[i]+vm[0]]), y=np.array([mean, mean]), pen=pg.mkPen('k', width=1.0))
        ax.addItem(c1)
        ax.addItem(c2)
        #ax.addItem(hbar)
        f = pg.FillBetweenItem(curve1=c1, curve2=c2, brush=pg.mkBrush((255, 255, 0, 96)))
        ax.addItem(f)

    if bp:
       pass
       # bpf = ax.boxplot(data, notch=0, positions=pos, vert=1)
       # pylab.setp(bpf['boxes'], color='black')
       # pylab.setp(bpf['whiskers'], color='black', linestyle='-')

def labelAxes(plot, xtext, ytext, **kwargs):
    """
    helper to label up the plot

    Parameters
    -----------
    plot : plot item
    xtext : string
        text for x axis
    ytext : string
        text for y axis
    **kwargs : keywords
        additional arguments to pass to pyqtgraph setLabel
    """
    plot.setLabel('bottom', xtext, **kwargs)
    plot.setLabel('left', ytext, **kwargs)

def labelPanels(plot, label=None, **kwargs):
    r"""
    Helper to label up the plot

    Parameters
    ----------
    plot : plot item

    label : plot panel label (for example, "A", "A1")

    \**kwargs : arguments for setPlotLabel
    """

    if label is not None:
        setPlotLabel(plot, plotlabel="%s" % label, **kwargs)
    else:
        setPlotLabel(plot, plotlabel="")

def labelTitles(plot, title=None, **kwargs):
    """
    Set the title of a plotitem. Basic HTML formatting is allowed, along
    with "size", "bold", "italic", etc..
    If the title is not defined, then a blank label is used
    A title is a text label that appears centered above the plot, in
    QGridLayout (position 0,2) of the plotitem.
    params
    -------
    :param plotitem: The plot item to label
    :param title: The text string to use for the label
    :kwargs: keywords to pass to the pg.LabelItem
    :return: None

    """

    if title is not None:
        plot.setTitle(title="<b><large>%s</large></b>" % title, visible=True, **kwargs)
    else:  # clear the plot title
        plot.setTitle(title=" ")

def setPlotLabel(plotitem, plotlabel='', **kwargs):
    """
    Set the plotlabel of a plotitem. Basic HTML formatting is allowed, along
    with "size", "bold", "italic", etc..
    If plotlabel is not defined, then a blank label is used
    A plotlabel is a text label that appears the upper left corner of the
    QGridLayout (position 0,0) of the plotitem.
    params
    -------
    :param plotitem: The plot item to label
    :param plotlabel: The text string to use for the label
    :kwargs: keywords to pass to the pg.LabelItem
    :return: None

    """

    plotitem.LabelItem = pg.LabelItem(plotlabel, **kwargs)
    plotitem.LabelItem.setMaximumHeight(30)
    # plotitem.layout.setRowFixedHeight(0, 30)
    try:
        plotitem.layout.addItem(plotitem.LabelItem, 0, 0)
        plotitem.LabelItem.setVisible(True)
    except:
        pass # not a valid thing to do with the plotitem...

class LayoutMaker():
    def __init__(self, win=None, cols=1, rows=1, letters=True, titles=False, labelEdges=True,
             margins=4, spacing=4, ticks='default', items='plots'):
        self.sequential_letters = string.ascii_uppercase
        self.cols = cols
        self.rows = rows
        self.letters = letters
        self.titles = titles
        self.edges = labelEdges
        self.margins = margins
        self.spacing = spacing
        self.rcmap = [None]*cols*rows
        self.plots = None
        self.items = items
        self.win = win
        self.ticks = ticks
        self._makeLayout(letters=letters, titles=titles, margins=margins, spacing=spacing)
        #self.addLayout(win)

    # def addLayout(self, win=None):
    #     if win is not None:
    #         win.setLayout(self.gridLayout)

    def getCols(self):
        return self.cols

    def getRows(self):
        return self.rows

    def mapFromIndex(self, index):
        """
        for a given index, return the row, col tuple associated with the index
        """
        return self.rcmap[index]

    def getPlot(self, index):
        """
        return the plot item in the list corresponding to the index n
        """
        if isinstance(index, tuple):
            r, c = index
        elif isinstance(index, int):
            r, c = self.rcmap[index]
        else:
            raise ValueError ('pyqtgraphPlotHelpers, LayoutMaker plot: index must be int or tuple(r,c)')
        return self.plots[r][c]

    def plot(self, index, x=None, y=None, pen=None, **kwargs):
        if x is None or y is None or pen is None:
            return None
        p = self.getPlot(index).plot(x=x,y=y, pen=pen, **kwargs)
        if self.ticks == 'talbot':
            talbotTicks(self.getPlot(index))
        return p

    def _makeLayout(self, letters=True, titles=True, margins=4, spacing=4):
        """
        Create a multipanel plot.
        The pyptgraph elements (widget, gridlayout, plots) are stored as class variables.
        The layout is always a rectangular grid with shape (cols, rows)
        if letters is true, then the plot is labeled "A, B, C..." Indices move horizontally first, then vertically
        margins sets the margins around the outside of the plot
        spacing sets the spacing between the elements of the grid
        If a window was specified (self.win is not None) then the grid layout will derive from that window's central
        item; otherwise we just make a gridLayout that can be put into another container somewhere.
        """
        import string
        if self.win is None:
            self.app = pg.mkQApp()
            self.win = QtGui.QWidget()
        self.layout = QtGui.QGridLayout()
        self.win.setLayout(self.layout)
        #     self.gridLayout = self.win.ci.layout  # the window's 'central item' is the main gridlayout.
        # else:
        #     print('b')
        #     self.gridLayout = pg.QtGui.QGridLayout()  # just create the grid layout to add to another item
        self.layout.setContentsMargins(margins, margins, margins, margins)
        self.layout.setSpacing(spacing)
        self.plots = [[0 for x in range(self.cols)] for x in range(self.rows)]
        self.gl = [[0 for x in range(self.cols)] for x in range(self.rows)]
        i = 0
        for r in range(self.rows):
            for c in range(self.cols):
                self.rcmap[i] = (r, c)
                if self.items == 'plots':
                    thisplot = pg.PlotWidget()
                    self.layout.addWidget(thisplot, r, c)
                    self.plots[r][c] =  thisplot # pg.PlotWidget()
                    if letters:
                        labelPanels(self.plots[r][c], label=self.sequential_letters[i], size='14pt', bold=True)
                    if titles:
                        labelTitles(self.plots[r][c], title=self.sequential_letters[i], size='14pt', bold=False)

                elif self.items == 'images':
                    imgview = pg.ImageView()
                    # imgview.ui.roiBtn.hide()
 #                    imgview.ui.menuBtn.hide()
                    # imgview.ui.histogram.hide()
                    textlabel = pg.TextItem(f"t = 0", anchor=(0, 1.1))
                    self.layout.addWidget(imgview, r, c, 1, 1)
                    self.plots[r][c] = imgview
                    v = self.layout.itemAtPosition(r, c) # imgview # self.gridLayout.addItem(pg.ViewBox(), row=r, col=c)
                    self.gl[r][c] = v
                    # if letters:
                    #     labelPanels(self.plots[r][c], label=self.sequential_letters[i], size='14pt', bold=True)
                    if titles:
                        self.plots[r][c].getImageItem().getViewBox().setWindowTitle(self.sequential_letters[i], size='14pt')
                        #labelTitles(self.plots[r][c], title=self.sequential_letters[i], size='14pt', bold=False)

                i += 1
                if i > 25:
                    i = 0
        if self.items == 'plots':
            self.labelEdges('T(s)', 'Y', edgeOnly=self.edges)

    def labelEdges(self, xlabel='T(s)', ylabel='Y', edgeOnly=True, **kwargs):
        """
        label the axes on the outer edges of the gridlayout, leaving the interior axes clean
        """
        (lastrow, lastcol) = self.rcmap[-1]
        i = 0
        for (r,c) in self.rcmap:
            if c == 0:
                ylab = ylabel
            elif edgeOnly:
                ylab = ''
            else:
                ylab = ylabel
            if r == self.rows-1:  # only the last row
                xlab = xlabel
            elif edgeOnly:  # but not other rows
                xlab = ''
            else:
                xlab = xlabel  # otherwise, label it
            labelAxes(self.plots[r][c], xlab, ylab, **kwargs)
            i += 1

    def axesEdges(self, edgeOnly=True):
        """
        text labesls only on the axes on the outer edges of the gridlayout,
        leaving the interior axes clean
        """
        (lastrow, lastcol) = self.rcmap[-1]
        i = 0
        for (r,c) in self.rcmap:
            xshow = True
            yshow = True
            if edgeOnly and c > 0:
                yshow = False
            if edgeOnly and r < self.rows:  # only the last row
                yshow = False
            ax = self.getPlot((r,c))
            leftaxis = ax.getAxis('left')
            bottomaxis = ax.getAxis('bottom')
            #print dir(self.plots[r][c])
            leftaxis.showValues = yshow
            bottomaxis.showValues = xshow
            i += 1

    def columnAutoScale(self, col, axis='left'):
        """
        autoscale the columns according to the max value in the column.
        Finds outside range of column data, then sets the scale of all plots
        in the column to that range
        """
        atmax = None
        atmin = None
        for (r, c) in self.rcmap:
            if c != col:
                continue
            ax = self.getPlot((r,c))
            thisaxis = ax.getAxis(axis)
            amin, amax = thisaxis.range
            if atmax is None:
                atmax = amax
            else:
                if amax > atmax:
                    atmax = amax
            if atmin is None:
                atmin = amin
            else:
                if amin > atmin:
                    atmin = amin

        self.columnSetScale(col, axis=axis, range=(atmin, atmax))
        return(atmin, atmax)

    def columnSetScale(self, col, axis='left', range=(0., 1.)):
        """
        Set the column scale
        """
        for (r, c) in self.rcmap:
            if c != col:
                continue
            ax = self.getPlot((r,c))
            if axis == 'left':
                ax.setYRange(range[0], range[1])
            elif axis == 'bottom':
                ax.setXRange(range[0], range[1])

            if self.ticks == 'talbot':
                talbotTicks(ax)

    def title(self, index, title='', **kwargs):
        """
        add a title to a specific plot (specified by index) in the layout
        """
        labelTitles(self.getPlot(index), title=title, **kwargs)

def figure(title = None, background='w'):
    if background == 'w':
        pg.setConfigOption('background', 'w')  # set background to white
        pg.setConfigOption('foreground', 'k')
    pg.mkQApp()
    win = pg.GraphicsWindow(title=title)
    return win

def show():
    pg.QtGui.QApplication.instance().exec_()

def test_layout(win):
    """
    Test the various plot types and modifications provided by the helpers above,
    in the context of a layout with various kinds of plots.
    """
    layout = LayoutMaker(cols=4,rows=2, win=win, labelEdges=True, ticks='talbot')
    x=np.arange(0, 10., 0.1)
    y = np.sin(x*3.)  # make an interesting signal
    r = np.random.random(10)  # and a random signal
    theta = np.linspace(0, 2.*np.pi, 10, endpoint=False)  # r, theta for polar plots
    for n in range(4*2):
        if n not in [1,2,3,4]:
            layout.plot(n, x, y)
        p = layout.getPlot(n)
        if n == 0:  # crossed axes plot
            crossAxes(p, xyzero=[5., 0.], density=(0.75, 1.5), tickPlacesAdd=(1, 0), pointSize=12)
            layout.title(n, 'Crossed Axes')
        if n in [1,2,3]:  # two differnt forms of polar plots
            if n == 1:
                po = polarPlot(p)
                po.setAxes(rMax=np.max(r))
                po.plot(r, theta, pen=pg.mkPen('r'))
                layout.title(n, 'Polar Path')

            if n == 2:
                po = polarPlot(p)
                po.plot(r, theta, vectors=True, pen=pg.mkPen('k', width=2.0))
                po.setAxes(rMax=np.max(r))
                po.plot([np.mean(r)], [po.circmean(theta)],
                 vectors=True, pen=pg.mkPen('r', width=2.0))
                layout.title(n, 'Polar Arrows')
            if n == 3:
                po = polarPlot(p)
                po.hist(r, theta, binwidth=np.pi/6., normalize=False, density=False, pen='r')
                po.hist(r, theta, binwidth=np.pi/6., normalize=False, density=False, mode='arc', pen='b')
                po.setAxes(rMax=None)
                layout.title(n, 'Polar Histogram')

        if n == 4:  # violin plot with scatter plot data
            data = {2: [3,5,7,9,2,4,6,8,7,2,3,1,2.5], 3: [5, 6, 7, 9, 2, 8, 10, 9.5, 11]}
            violinPlotScatter(p, data, symbolColor='r')
            p.setYRange(0, 12)
            layout.title(n, 'Violin Plots with PseudoScatter')

        if n == 5:  # clean plot for physiology with baseline reference and a calibration bar
            calbar(p, calbar=[7.0, -1.5, 2.0, 0.5], axesoff=True, orient='left', unitNames={'x': 'ms', 'y': 'nA'})
            refline(p, refline=0., color = [64, 64, 64], linestyle = '--' ,linewidth = 0.5)
            layout.title(n, 'Calbar and Refline')

    #talbotTicks(layout.getPlot(1))
    layout.columnAutoScale(col=3, axis='left')
    show()

def test_crossAxes(win):
    layout = LayoutMaker(cols=1,rows=1, win=win, labelEdges=True)
    x=np.arange(-1, 1., 0.01)
    y = np.sin(x*10.)
    layout.plot(0, x, y)
    p = layout.getPlot(0)
    crossAxes(p, xyzero=[0., 0.], limits=[None, None, None, None], density=1.5, tickPlacesAdd=1, pointSize=12)
    show()

def test_polarPlot(win):
    layout = LayoutMaker(cols=1,rows=1, win=win, labelEdges=True)
    po = polarPlot(layout.getPlot((0,0)))  # convert rectangular plot to polar
    po.setAxes(steps=4, rMax=100, makeGrid=True)  # build the axes
    nvecs = 50
    #th = np.linspace(-np.pi*2, np.pi*2-np.pi*2/nvecs, nvecs)
    th = np.linspace(-np.pi*4, 0, nvecs)
    r = np.linspace(10, 100, nvecs)
    po.plot(r, th, vectors=True, arrowhead=True, symbols='o', pen=pg.mkPen('k', width=1.5))  # plot with arrowheads
    nvecs=8
    th = np.linspace(-np.pi*2, np.pi*2-np.pi*2/nvecs, nvecs)
    r = np.linspace(10, 100, nvecs)
    #po.plot(r, th, vectors=True, arrowhead=False, symbols='o', pen=pg.mkPen('r', width=1.5))  # plot with just lines

    show()

if __name__ == '__main__':
    win = figure(title='testing')
    test_layout(win)
    #test_crossAxes(win)
    #test_polarPlot(win)

ext: py

max_stars_repo_name: pbmanis/pylibrary

max_issues_repo_name: pbmanis/pylibrary

max_forks_repo_name: pbmanis/pylibrary

reason: import numpy,import scipy


text:
"""
This is the implementation of the Item MAD rating metric.
It proceeds from a user-wise computation, and average the values over the users.
"""

__version__ = '0.1'
__author__ = 'Vito Walter Anelli, Claudio Pomo'
__email__ = 'vitowalter.anelli@poliba.it, claudio.pomo@poliba.it'

import numpy as np
import pandas as pd
from elliot.evaluation.metrics.base_metric import BaseMetric

class ItemMADrating(BaseMetric):
    r"""
    Item MAD Rating-based

    This class represents the implementation of the Item MAD rating recommendation metric.

    For further details, please refer to the `paper <https://dl.acm.org/doi/abs/10.1145/3269206.3271795>`_

    .. math::
        \mathrm {MAD}={avg}_{i, j}({MAD}(R^{(i)}, R^{(j)}))

    To compute the metric, add it to the config file adopting the following pattern:

    .. code:: yaml

        complex_metrics:
        - metric: ItemMADrating
          clustering_name: ItemPopularity
          clustering_file: ../data/movielens_1m/i_pop.tsv
    """

    def __init__(self, recommendations, config, params, eval_objects, additional_data):
        """
        Constructor
        :param recommendations: list of recommendations in the form {user: [(item1,value1),...]}
        :param config: SimpleNameSpace that represents the configuration of the experiment
        :param params: Parameters of the model
        :param eval_objects: list of objects that may be useful for the computation of the different metrics
        """
        super().__init__(recommendations, config, params, eval_objects, additional_data)
        self._cutoff = self._evaluation_objects.cutoff
        self._relevance = self._evaluation_objects.relevance.binary_relevance

        self._item_clustering_path = self._additional_data.get("clustering_file", False)
        self._item_clustering_name = self._additional_data.get("clustering_name", "")
        if self._item_clustering_path:
            self._item_clustering = pd.read_csv(self._additional_data["clustering_file"], sep="\t", header=None)
            self._n_clusters = self._item_clustering[1].nunique()
            self._item_clustering = dict(zip(self._item_clustering[0], self._item_clustering[1]))
        else:
            self._n_clusters = 1
            self._item_clustering = {}

        self._sum = np.zeros(self._n_clusters)
        self._n_items = np.zeros(self._n_clusters)

        self._item_count = {}
        self._item_gain = {}

    def name(self):
        """
        Metric Name Getter
        :return: returns the public name of the metric
        """
        return f"ItemMADrating_{self._item_clustering_name}"

    def __item_mad(self, user_recommendations, cutoff, user_relevant_items):
        """
        Per User Item MAD rating
        :param user_recommendations: list of user recommendation in the form [(item1,value1),...]
        :param cutoff: numerical threshold to limit the recommendation list
        :param user_relevant_items: list of user relevant items in the form [item1,...]
        :return: the value of the Precision metric for the specific user
        """
        for i, r in user_recommendations[:cutoff]:
            self._item_count[i] = self._item_count.get(i, 0) + 1
            self._item_gain[i] = self._item_gain.get(i, 0) + (r if i in user_relevant_items else 0)

    def eval(self):
        """
        Evaluation function
        :return: the overall averaged value of Item MAD rating
        """

        for u, u_r in self._recommendations.items():
            if len(self._relevance.get_user_rel(u)):
                self.__item_mad(u_r, self._cutoff, self._relevance.get_user_rel(u))

        for item, gain in self._item_gain.items():
            v = gain/self._item_count[item]
            cluster = self._item_clustering.get(item, None)

            if cluster is not None:
                self._sum[cluster] += v
                self._n_items[cluster] += 1

        avg = [self._sum[i]/self._n_items[i] for i in range(self._n_clusters)]
        differences = []
        for i in range(self._n_clusters):
            for j in range(i+1, self._n_clusters):
                differences.append(abs(avg[i] - avg[j]))
        return np.average(differences)

    def get(self):
        return [self]

reason: import numpy


text:
#-*-coding: cp1254-*-
###Hareket Algılama2##

import numpy as np
import cv2

cam=cv2.VideoCapture('videolar/test.mp4')

fgbg = cv2.createBackgroundSubtractorMOG2()

while(1):
	ret,frame=cam.read()
	if frame is not None:
		fgmask=fgbg.apply(frame)
		cv2.imshow('frame',fgmask)

		if cv2.waitKey(30) & 0xFF == 27:
			break
	else:
		break
cam.release()
cv2.destroyAllWindows()

max_stars_repo_path: Goruntu Isleme/ornek8.py

max_stars_repo_name: deryaksz/RACLAB

max_issues_repo_path: Goruntu Isleme/ornek8.py

max_issues_repo_name: deryaksz/RACLAB

max_forks_repo_path: Goruntu Isleme/ornek8.py

max_forks_repo_name: deryaksz/RACLAB


text:
import os
import numpy as np
from pdf2embeddings.scraper import DocumentScraper
from pdf2embeddings.arrange_text import CorpusGenerator
from pdf2embeddings.embedder import Embedder
from pdf2embeddings.process_user_queries import query_embeddings

def test_end_to_end_runner():
    scraper = DocumentScraper(
        os.getenv("FIXTURES_DIR"), os.path.join(os.getenv("FIXTURES_DIR"), 'words_to_replace.json')
    )
    df_by_page = scraper.document_corpus_to_pandas_df()
    generator = CorpusGenerator(df_by_page)
    df_by_sentence = generator.df_by_page_to_df_by_sentence()
    list_of_sentences = df_by_sentence['sentence'].values.tolist()
    assert list_of_sentences == [
        'Mr Michael went to the store to buy some eggs.',
        'Joel rolled down the street on his skateboard.',
        'test / this is a first sentence',
        "Take a look, then, at Tuesday's elections in New York City, New Jersey and Virginia:"
    ]

    embedder = Embedder(list_of_sentences)
    models_to_be_run = ['Word2Vec_tfidf_weighted', 'Word2Vec', 'BERT', 'ELMo']
    for model in models_to_be_run:
        if model == 'Word2Vec_tfidf_weighted':
            sentence_embeddings, w2v_tfidf, tfidf_vectorizer = embedder.compute_word2vec_embeddings(tfidf_weights=True)
            df_by_sentence = embedder.add_embeddings_to_corpus_df(
                df_by_sentence, sentence_embeddings, 'Word2Vec_with_TfIdf_weights'
            )
        elif model == 'Word2Vec':
            sentence_embeddings, w2v, _ = embedder.compute_word2vec_embeddings(tfidf_weights=False)
            df_by_sentence = embedder.add_embeddings_to_corpus_df(df_by_sentence, sentence_embeddings, 'Word2Vec')
        elif model == 'BERT':
            bert_model = 'bert-base-nli-stsb-mean-tokens'  # This line is specific to BERT
            sentence_embeddings, bert = embedder.compute_bert_embeddings(bert_model)
            df_by_sentence = embedder.add_embeddings_to_corpus_df(df_by_sentence, sentence_embeddings, 'BERT')
        elif model == 'ELMo':
            sentence_embeddings, elmo = embedder.compute_elmo_embeddings()
            df_by_sentence = embedder.add_embeddings_to_corpus_df(df_by_sentence, sentence_embeddings, 'ELMo_layer_3')
        else:
            raise KeyError(f'The model {model} is not recognized as input.')

    w2v_emb, df_by_sentence = query_embeddings(
        list_of_sentences[0], df_by_sentence, 'Word2Vec', 'Word2Vec', w2v, metric_colname='w2v_distance_test1'
    )
    w2v_tfidf_emb, df_by_sentence = query_embeddings(
        list_of_sentences[0], df_by_sentence, 'Word2Vec_with_TfIdf_weights', 'Word2Vec_TfIdf_weighted', w2v_tfidf,
        metric_colname='w2v_tfidf_weighted_distance_test1', tfidf_vectorizer=tfidf_vectorizer
    )
    elmo_emb, df_by_sentence = query_embeddings(
        list_of_sentences[0], df_by_sentence, 'ELMo_layer_3', 'ELMo', elmo, metric_colname='elmo_distance_test1'
    )
    bert_emb, df_by_sentence = query_embeddings(
        list_of_sentences[0], df_by_sentence, 'BERT', 'BERT', bert, metric_colname='bert_distance_test1'
    )

    df_by_sentence.sort_values('w2v_distance_test1', ascending=True, inplace=True)
    df_by_sentence.reset_index(inplace=True, drop=True)
    assert df_by_sentence['sentence'][0] == "Mr Michael went to the store to buy some eggs."
    np.testing.assert_array_equal(w2v_emb, df_by_sentence['Word2Vec'][0])

    df_by_sentence.sort_values('w2v_tfidf_weighted_distance_test1', ascending=True, inplace=True)
    df_by_sentence.reset_index(inplace=True, drop=True)
    assert df_by_sentence['sentence'][0] == "Mr Michael went to the store to buy some eggs."
    np.testing.assert_array_equal(w2v_tfidf_emb, df_by_sentence['Word2Vec_with_TfIdf_weights'][0])

    df_by_sentence.sort_values('elmo_distance_test1', ascending=True, inplace=True)
    df_by_sentence.reset_index(inplace=True, drop=True)
    assert df_by_sentence['sentence'][0] == "Mr Michael went to the store to buy some eggs."
    # np.testing.assert_array_almost_equal(elmo_emb, df_by_sentence['ELMo_layer_3'][0])
    # This test does not work, see https://github.com/allenai/allennlp/issues/3995#

    df_by_sentence.sort_values('bert_distance_test1', ascending=True, inplace=True)
    df_by_sentence.reset_index(inplace=True, drop=True)
    assert df_by_sentence['sentence'][0] == "Mr Michael went to the store to buy some eggs."
    np.testing.assert_array_almost_equal(bert_emb, df_by_sentence['BERT'][0])

    w2v_emb, df_by_sentence = query_embeddings(
        "New York", df_by_sentence, 'Word2Vec', 'Word2Vec', w2v, metric_colname='w2v_distance_test2'
    )
    w2v_tfidf_emb, df_by_sentence = query_embeddings(
        "New York", df_by_sentence, 'Word2Vec_with_TfIdf_weights', 'Word2Vec_TfIdf_weighted', w2v_tfidf,
        metric_colname='w2v_tfidf_weighted_distance_test2', tfidf_vectorizer=tfidf_vectorizer
    )
    elmo_emb, df_by_sentence = query_embeddings(
        "New York", df_by_sentence, 'ELMo_layer_3', 'ELMo', elmo, metric_colname='elmo_distance_test2'
    )
    bert_emb, df_by_sentence = query_embeddings(
        "New York", df_by_sentence, 'BERT', 'BERT', bert, metric_colname='bert_distance_test2'
    )

    df_by_sentence.sort_values('w2v_distance_test2', ascending=True, inplace=True)
    df_by_sentence.reset_index(inplace=True, drop=True)
    assert df_by_sentence['sentence'][0] == \
        "Take a look, then, at Tuesday's elections in New York City, New Jersey and Virginia:"

    df_by_sentence.sort_values('w2v_tfidf_weighted_distance_test2', ascending=True, inplace=True)
    df_by_sentence.reset_index(inplace=True, drop=True)
    assert df_by_sentence['sentence'][0] == \
        "Take a look, then, at Tuesday's elections in New York City, New Jersey and Virginia:"

    df_by_sentence.sort_values('elmo_distance_test2', ascending=True, inplace=True)
    df_by_sentence.reset_index(inplace=True, drop=True)
    assert df_by_sentence['sentence'][0] == \
        "Take a look, then, at Tuesday's elections in New York City, New Jersey and Virginia:"

    df_by_sentence.sort_values('bert_distance_test2', ascending=True, inplace=True)
    df_by_sentence.reset_index(inplace=True, drop=True)
    assert df_by_sentence['sentence'][0] == \
        "Take a look, then, at Tuesday's elections in New York City, New Jersey and Virginia:"


text:
import os

import click

import numpy as np
import pandas as pd
import torch

from rampwf.utils.importing import import_module_from_source

from .data_processing import get_metadata_dictionary
from .data_processing import rollout
from .model_env import make_model_env_class

def model_based_rl(agent_name, submission,
                   n_epochs, min_epoch_steps, min_random_steps,
                   episodic_update, initial_trace,
                   seed):
    """Main script of model based RL loop."""

    np.random.seed(seed)
    if seed is not None:
        torch.manual_seed(seed)
    print(f'Random seed: {seed}')

    if min_random_steps is None:
        min_random_steps = min_epoch_steps

    print((f'Using {submission} as model and {agent_name} as '
           f'agent for {n_epochs} epochs with at least {min_epoch_steps} '
           f'steps per epoch and {min_random_steps} random steps.'))

    problem_module_path = 'problem.py'
    problem_module = import_module_from_source(problem_module_path, 'problem')

    env_module_path = 'env.py'
    env_module = import_module_from_source(env_module_path, 'env')
    system_env_object = env_module.Env
    system_env = system_env_object()
    system_env.seed()

    # reward function
    reward_module_path = 'reward_function.py'
    reward_module = import_module_from_source(
        reward_module_path, 'reward_function')
    reward_func = reward_module.reward_func

    # agent
    agent_module_path = os.path.join('agents', agent_name + '.py')
    agent_module = import_module_from_source(agent_module_path, agent_name)
    agent_object = agent_module.Agent

    # metadata
    metadata_path = os.path.join('data', 'metadata.json')
    metadata = get_metadata_dictionary(metadata_path)

    # create a directory to store the results
    output_dir = os.path.join(
        'submissions', submission, 'mbrl_outputs', agent_name, f'seed_{seed}')

    # model
    if submission == 'real_system':
        model_env = system_env
    else:
        submission_path = os.path.join('submissions', submission)
        ModelEnv = make_model_env_class(system_env_object)
        model_env = ModelEnv(
            submission_path, problem_module, reward_func,
            metadata, output_dir, seed)

    # retrieving feature names
    observation_names = metadata["observation"]
    action_names = metadata["action"]
    restart_name = metadata["restart_name"]
    reward_name = metadata["reward"]

    trace_header = (
        observation_names + action_names + reward_name +
        [restart_name] + ['epoch_id'])

    epoch_output_dir = os.path.join(output_dir, f'epoch_0')
    if initial_trace:
        # epoch 0 is the initial trace
        print('Epoch 0: Initial trace.')

        # no need to train if the model environment is the real environment
        if hasattr(model_env, 'train_model'):
            model_env.train_model()

        agent = agent_object(model_env, epoch_output_dir=epoch_output_dir,
                             seed=None)
        epoch_start = 1
    else:
        # random agent
        agent = agent_object(model_env, epoch_output_dir=epoch_output_dir,
                             random_action=True, seed=None)
        epoch_start = 0

    for epoch in range(epoch_start, n_epochs):
        # use the agent on the real system, collect the trace to update the
        # model and update the agent using the updated model

        if epoch == 0:
            # random policy
            min_rollout_steps = min_random_steps
        else:
            min_rollout_steps = min_epoch_steps
            agent.random_action = False

        trace = rollout(
            system_env, len(action_names),
            epoch=epoch, min_epoch_steps=min_rollout_steps,
            agent=agent, episodic_update=episodic_update)

        # save new trace to disk
        epoch_output_dir = os.path.join(output_dir, f'epoch_{epoch}')
        if not os.path.exists(epoch_output_dir):
            os.makedirs(epoch_output_dir)
        trace_path = os.path.join(epoch_output_dir, 'trace.csv')
        trace_df = pd.DataFrame(data=trace, columns=trace_header)
        trace_df.to_csv(trace_path, index=False)

        # update model if it remains epochs to compute.
        if epoch <= n_epochs - 2 and hasattr(model_env, 'train_model'):
            model_env.train_model(epoch=epoch)

@click.command()
@click.option('--agent-name', default='random_shooting', show_default=True,
              type=click.STRING, help="Agent.")
@click.option("--submission", default="real_system", show_default=True,
              type=click.STRING,
              help="Model submission. Choose 'real_system' if you want to "
              "use the real environment.")
@click.option("--n-epochs", default=100, show_default=True, type=click.INT,
              help="The number of epochs. If the submission is not the real "
              "system, the model is updated at each epoch. If initial-trace "
              "is set to True the first epoch is assumed to be the initial "
              "trace.")
@click.option("--min-epoch-steps", default=200, show_default=True,
              type=click.INT,
              help="The minimum number of steps for each epoch given that "
              "each epoch ends by a complete episode.")
@click.option("--min-random-steps", default=None, show_default=True,
              type=click.INT,
              help="The minimum number of steps done at the first epoch"
              " with the random policy if initial-trace is set to False. "
              "If None then it is equal to min-epoch-steps.")
@click.option("--episodic-update", default=False, show_default=True,
              type=click.BOOL,
              help="Whether to update the model after each episode such that "
              "one epoch is exaclty one episode.")
@click.option("--initial-trace", default=False, show_default=True,
              type=click.BOOL, help="Whether an initial trace is available. "
              "If True, the initial trace should be stored under trace.csv in "
              "submissions/<submission>/mbrl_outputs/<agent_name>/seed_<seed>/"
              "epoch_0/.")
@click.option("--seed", default=0, show_default=True,
              help="Seed of the random number generator. Only the numpy and "
              "pytorch global random generators are seeded.")
def model_based_rl_command(agent_name, submission,
                           n_epochs, min_epoch_steps, min_random_steps,
                           episodic_update, initial_trace,
                           seed):
    return model_based_rl(
        agent_name, submission, n_epochs, min_epoch_steps, min_random_steps,
        episodic_update, initial_trace, seed
    )

if __name__ == "__main__":
    model_based_rl_command()


text:
import cv2
import numpy as np
from scipy import ndimage

"""
opencv 处理图像
"""

"""
1. 不同空间的色彩转换
计算机视觉中常用的色彩空间: 灰度, BGR, HSV(Hue, Saturation, Value)
灰度: 去除色彩信息将其转换为灰阶, 弧度色彩空间对中间处理特别有效, 比如人脸识别

BGR: 蓝-绿-红色彩空间, 每一个像素点由一个三元数组来表示.

HSV: H是色调, S是饱和度, V是黑暗程度
"""

"""
2. 傅里叶变换
高通滤波器(HPF): 检测图像的某个区域, 然后根据像素与邻近像素的亮度差值来提升该像素的
亮度.

kernel:
[
  [  0,  -0.25,  0  ]
  [-0.25,  1,  -0.25]
  [  0,  -0.25,  0  ]
]

kernel是值一组权重的集合, 它会应用在源图像的每一个区域, 并由此生成目标图像的一个像素.
例如, 大小为7的kernel意味着每个49(7*7)个源图像的像素会产生目标图像的一个像素.

在计算玩中央像素与周围临近像素的亮度差值之和以后, 如果亮度变化很大, 中央像素的亮度就会
增加(反之不会). 即, 如果一个像素比他周围的像素更突出, 就会提升它的亮度
"""

def pass_filter():
    kernel_3 = np.array([[-1, -1, -1],
                         [-1, 8, -1],
                         [-1, -1, -1]])

    kernel_5 = np.array([[-1, -1, -1, -1, -1],
                         [-1, 1, 2, 1, -1],
                         [-1, 2, 4, 2, -1],
                         [-1, 1, 2, 1, -1],
                         [-1, -1, -1, -1, -1]])

    img = cv2.imread("images/female.jpg", 0)

    k3 = ndimage.convolve(img, kernel_3)  # 卷积, numpy只接受一位数组, 因此这里使用ndimage
    k5 = ndimage.convolve(img, kernel_5)

    blurred = cv2.GaussianBlur(img, (11, 11), 0)  # 高斯模糊, 低通滤波器
    g_hpf = img - blurred

    cv2.imshow("3*3", k3)  # k3
    cv2.imshow("5*5", k5)  # k5
    cv2.imshow("g_hpf", g_hpf)  # Gauss
    cv2.waitKey()
    cv2.destroyAllWindows()

"""
低通滤波(LPF): 在像素与周围像素亮度差值小于一个特定值时, 平滑该像素的亮度.
主要用于去噪和模糊化. 比如, 高斯模糊是最常用的模糊滤波器之一, 它是一个削减
高频信号强度的低通滤波器.
"""

"""
3. 边缘检测:
opencv提供的边缘检测滤波函数, Laplacian(), Sobel(), Scharr(). 这些滤波
函数都会将非边缘区域转为黑色, 讲边缘区域转为白色或其他饱和的颜色, 但是这些函数很
容易将噪声错误地识别为边缘. 缓解这个问题的方法是在找到边缘之前对图像进行模糊处理.

opencv提供的模糊滤波函数, blur()[简单的算术平均], mediaBlur() 以及 GaussianBlur()

边缘检测函数和模糊滤波函数的参数很多, 但是总有一个ksize参数, 奇数, 表示滤波核的
宽和高(以像素为单位)

mediaBlur() 去除数字化的视频噪声非常有效, 特别是去除彩色图像的噪声

Laplacian() 会产生明显的边缘线条, 灰度图像也是如此
"""

def stroke_edges(src, dst, blurKsize=7, edgeKsize=5):
    if blurKsize >= 3:
        blurredSrc = cv2.medianBlur(src, blurKsize)
        graySrc = cv2.cvtColor(blurredSrc, cv2.COLOR_BGR2GRAY)
    else:
        graySrc = cv2.cvtColor(src, cv2.COLOR_BGR2GRAY)

    cv2.Laplacian(graySrc, cv2.CV_8U, graySrc, ksize=edgeKsize)
    normalizedInverseAlpha = (1.0 / 255) * (255 - graySrc)  # 归一化处理
    channels = cv2.split(src)

    for channel in channels:
        channel[:] = channel * normalizedInverseAlpha
    cv2.merge(channels, dst)

"""
4. 使用定制内核做卷积
核: 一组权重, 它决定如何通过临近像素点来计算新的像素点. 核也称为卷积矩阵, 它对一个区域的
像素做调和(mix up) 或 卷积运算.

opencv提供filter2D()函数, 它产生一个卷积矩阵.

filter2D(src, ddepth, kernel, dst)
    第二个参数指定目标图像每个通道的位深度(比如, cv2.CV_8U表示每个通道为8位), 如果为负
    值, 则表示目标图像和源图像有同样的位深度
    第三个参数指明核大小,是一个奇数

核矩阵权重加起来为1, 不会改变图像的亮度
核矩阵权重加起来为0, 边缘检测核, 把边缘转为白色, 把非边缘转为黑色

锐化, 边缘检测, 模糊等滤波器都使用了高度对称的核
"""

"""
边缘检测:
Canny()  边缘检测. 高斯滤波器去噪 -> 计算梯度 -> 在边缘上使用NMS(非最大抑制) -> 在检测到的
边缘上使用双阈值去除假阳性 -> 分析所有的边缘及其之间的连接

轮廓检测:
    cv2.threshold(src, thresh, maxval, type) 图像二值化处理
    cv2.findContours(image, mode, method)
        image是输入图像, 输入的图像会被修改
        mode层次类型, cv2.RETR_TREE会得到图像中轮廓的整体层次结果, 以此建立轮廓之间的关系.
        cv2.RETR_EXTERNAL, 只是得到最外面的轮廓
        method轮廓逼近方法

        返回值: 修改后的图像, 图像的轮廓, 层次
    cv2.drawContours(image, contours, contourIdx, color, thickness=None) 轮廓画像
        contours, 轮廓数组
        contourIdx, 要绘制的轮廓数组的, 负数表示绘制全部轮廓
        color, 轮廓颜色
        thickness, 密度
"""

def contour_detected():
    image = cv2.pyrDown(cv2.imread("images/contour.jpg", cv2.IMREAD_UNCHANGED))

    # 二值化处理
    ret, thresh = cv2.threshold(cv2.cvtColor(image.copy(), cv2.COLOR_BGR2GRAY),
                                127, 255, cv2.THRESH_BINARY)

    # 轮廓查找
    _, contours, hierarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL,
                                              cv2.CHAIN_APPROX_SIMPLE)

    # 轮廓处理
    for c in contours:
        # 计算简单的边界框, 将轮廓信息转换成(x,y)坐标, 并加上矩形的高度和宽度
        x, y, w, h = cv2.boundingRect(c)
        # 画出矩形
        cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)

        # 计算目标的最小矩形区域
        rect = cv2.minAreaRect(c)  # 查找最小矩形
        box = cv2.boxPoints(rect)  # 计算最小矩形的坐标(4个)
        box = np.int0(box)  # 将坐标整数化
        # 画出矩形
        cv2.drawContours(image, [box], 0, (0, 0, 255), 3)

        # 计算目标的最小闭圆区域
        (x, y), radius = cv2.minEnclosingCircle(c)  # 计算最小闭圆的中心坐标和半径
        center = (int(x), int(y))
        radius = int(radius)
        # 画出圆形
        image = cv2.circle(image, center, radius, (0, 255, 0), 2)

    # 轮廓绘画
    cv2.drawContours(image, contours, -1, (255, 0, 0), 1)
    cv2.imshow("c", image)
    cv2.waitKey()
    cv2.destroyAllWindows()

"""
凸轮廓与Douglas-Peucker:
    cv2.approxPolyDP(curve, epsilon, closed) 计算近似的多边形框
        curve, 轮廓
        epsilon, 表示源轮廓周长与近似多边形周长的最大差值(此值越小, 近似多边形与源轮廓越接近)
        closed, 表示这个多边形是否闭合

    epsilon, 使用cv.arcLength(curve, closed)估计

    为了计算凸形状, 需要使用 cv2.convexHull(curve) 处理轮廓

直线和圆检测:
Hough变换是直线和形状检测背后的理论基础
直线检测, 通过HoughLines和HoughLinesP函数完成, HoughLines使用标准的Hough变换, HoughLinesP
使用概率Hough变换.

cv2.HoughLinesP(image, rho, theta, threshold, lines=None, minLineLength=None, maxLineGap=None)
    image, 单通道二值图像(一个经过去噪声并只有边缘的图像比较好).
    rho, theta, 线段的几何表示, 一般分别取1和np.pi/180
    threshold, 阈值, 低于该阈值的直线会被忽略
    minLineLength, 最小直线长度(更短的直线会被删除)
    maxLineGap, 最大线段间隙(一条线段的间隙长度大于这个值会被分成两条分开的线段)

圆检测, 通过HoughCircles函数, 与使用HoughLines函数类似
cv2.HoughCircles(image, method, dp, minDist, circles=None, param1=None, param2=None,
                 minRadius=None, maxRadius=None)
    image, 单通道二值图像
    method, 定义检测图像中圆的方法. 目前唯一实现的方法是cv2.HOUGH_GRADIENT
    dp, 累加器分辨率与图像分辨率的反比. dp获取越大,累加器数组越小. 一般为1
    minDist, 检测到的圆的中心与(x,y)坐标之间的最小距离. 如果minDist太小,
    则可能导致检测到多个相邻的圆. 如果minDist太大,则可能导致很多圆检测不到.

    param1, 用于处理边缘检测的梯度值方法. 一般为100
    param2, cv2.HOUGH_GRADIENT方法的累加器阈值. 阈值越小,检测到的圈子越多. 一般为30
    minRadius, 半径的最小大小(以像素为单位)
    maxRadius, 半径的最大大小(以像素为单位)
"""

def line_detected():
    img = cv2.imread("images/female.jpg")
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    edges = cv2.Canny(gray, 50, 120)  # Canny边缘检测滤波器处理过的单通道二值图像
    minLineLength = 20
    maxLineGap = 5

    lines = cv2.HoughLinesP(edges, 1, np.pi / 180, 100,
                            minLineLength=minLineLength,
                            maxLineGap=maxLineGap)

    for x1, y1, x2, y2 in lines[0]:
        cv2.line(img, (x1, y1), (x2, y2), (0, 255, 0), 2)

    cv2.imshow("edges", edges)
    cv2.imshow("lines", img)
    cv2.waitKey()
    cv2.destroyAllWindows()

def circle_detected():
    img = cv2.imread("images/code.png")
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    edges = cv2.medianBlur(gray, 11)
    circle_image = cv2.cvtColor(edges, cv2.COLOR_GRAY2BGR)
    cv2.imshow("blur", circle_image)

    circles = cv2.HoughCircles(edges, cv2.HOUGH_GRADIENT, 1, 20,
                               param1=100, param2=30, minRadius=20, maxRadius=60)

    circles = np.uint16(np.around(circles))

    for c in circles[0, :]:
        cv2.circle(img, (c[0], c[1]), c[2], (0, 255, 0), 2)
        cv2.circle(img, (c[0], c[1]), 2, (0, 0, 255), 3)

    cv2.imshow("circle", img)
    cv2.waitKey()
    cv2.destroyAllWindows()

if __name__ == '__main__':
    circle_detected()

reason: import numpy,from scipy


text:
import numpy as np
import torch
import time, pdb

def converter(data):
    if isinstance(data, torch.Tensor):
        data = data.cpu().data.numpy().flatten()
    return data.flatten()

def fast_hist(label_pred, label_true, num_classes):
    # pdb.set_trace()
    hist = np.bincount(num_classes * label_true.astype(int) + label_pred, minlength=num_classes ** 2)
    hist = hist.reshape(num_classes, num_classes)
    return hist

class Metric_mIoU():
    def __init__(self, class_num):
        self.class_num = class_num
        self.hist = np.zeros((self.class_num, self.class_num))

    def update(self, predict, target):
        predict, target = converter(predict), converter(target)

        self.hist += fast_hist(predict, target, self.class_num)

    def reset(self):
        self.hist = np.zeros((self.class_num, self.class_num))

    def get_miou(self):
        miou = np.diag(self.hist) / (
                np.sum(self.hist, axis=1) + np.sum(self.hist, axis=0) -
                np.diag(self.hist))
        miou = np.nanmean(miou)
        return miou

    def get_acc(self):
        acc = np.diag(self.hist) / self.hist.sum(axis=1)
        acc = np.nanmean(acc)
        return acc

    def get(self):
        return self.get_miou()

class MultiLabelAcc():
    def __init__(self):
        self.cnt = 0
        self.correct = 0

    def reset(self):
        self.cnt = 0
        self.correct = 0

    def update(self, predict, target):
        predict, target = converter(predict), converter(target)
        self.cnt += len(predict)
        self.correct += np.sum(predict == target)

    def get_acc(self):
        return self.correct * 1.0 / self.cnt

    def get(self):
        return self.get_acc()

class AccTopk():
    def __init__(self, background_classes, k):
        self.background_classes = background_classes
        self.k = k
        self.cnt = 0
        self.top_k_correct = 0

    def reset(self):
        self.cnt = 0
        self.top_k_correct = 0

    def update(self, predict, target):
        predict, target = converter(predict), converter(target)
        self.cnt += len(predict)
        background_idx = (predict == self.background_classes) + (target == self.background_classes)
        self.top_k_correct += np.sum(predict[background_idx] == target[background_idx])
        not_background_idx = np.logical_not(background_idx)
        self.top_k_correct += np.sum(np.absolute(predict[not_background_idx] - target[not_background_idx]) < self.k)

    def get(self):
        return self.top_k_correct * 1.0 / self.cnt

def update_metrics(metric_dict, pair_data):
    for i in range(len(metric_dict['name'])):
        metric_op = metric_dict['op'][i]
        data_src = metric_dict['data_src'][i]
        metric_op.update(pair_data[data_src[0]], pair_data[data_src[1]])

def reset_metrics(metric_dict):
    for op in metric_dict['op']:
        op.reset()

if __name__ == '__main__':
    # p = np.random.randint(5, size=(800, 800))
    # t = np.zeros((800, 800))
    # me = Metric_mIoU(5)
    # me.update(p,p)
    # me.update(p,t)
    # me.update(p,p)
    # me.update(p,t)
    # print(me.get_miou())
    # print(me.get_acc())

    a = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0])
    b = np.array([1, 1, 2, 2, 2, 3, 3, 4, 4, 0])
    me = AccTopk(0, 5)
    me.update(b, a)
    print(me.get())

max_stars_repo_name: MhinaX/Ultra-Fast-Lane-Detection

max_issues_repo_name: MhinaX/Ultra-Fast-Lane-Detection

max_forks_repo_name: MhinaX/Ultra-Fast-Lane-Detection


text:
#!/usr/bin/env python2
# -*- coding: utf-8 -*-

#       Copyright (C) 2005-2007 Carabos Coop. V. All rights reserved
#       Copyright (C) 2008-2013 Vicent Mas. All rights reserved
#
#       This program is free software: you can redistribute it and/or modify
#       it under the terms of the GNU General Public License as published by
#       the Free Software Foundation, either version 3 of the License, or
#       (at your option) any later version.
#
#       This program is distributed in the hope that it will be useful,
#       but WITHOUT ANY WARRANTY; without even the implied warranty of
#       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#       GNU General Public License for more details.
#
#       You should have received a copy of the GNU General Public License
#       along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#       Author:  Vicent Mas - vmas@vitables.org

"""
When a user edits the attributes of a node using the Properties dialog and
presses `OK` the attributes correctness is checked by this module. Datatypes
and overflow conditions are checked. If no errors are detected the new set of
attributes is stored in the `PyTables` node. Otherwise the user is reported
about the error and the dialog remains open so the user can fix her mistake.
"""

__docformat__ = 'restructuredtext'

import tables
import numpy

from PyQt4 import QtGui

import vitables.utils

translate = QtGui.QApplication.translate

def checkSyntax(value):
    """Check the syntax of a `Python` expression.

    :Parameters value: the Python expression to be evaluated
    """

    if value[0] in ("'", '"'):
        # Quotes are not permitted in the first position
        return False
    try:
        eval(value)
    except (ValueError, SyntaxError):
        return False
    else:
        return True

def formatStrValue(dtype, str_value):
    """
    Format the string representation of a value accordingly to its datatype.

    This function catches datatype errors for boolean values.

    :Parameters:

    - dtype: the value data type
    - str_value: the string representation of a value
    """

    if dtype == u'bool':
        # Every listed value is valid but none of them would
        # pass the out of range test later so we use a fake
        # valid value.
        # Beware that numpy.array(True).astype('bool')[()] fails
        # with a TypeError so we use '1' as a fake value
        if str_value in [u'1', u'TRUE', u'True', u'true']:
            str_value = u'1'
        elif str_value in [u'0', u'FALSE', u'False', u'false']:
            str_value = u'0'
        else:
            raise ValueError
    elif dtype.startswith(u'complex'):
        # Valid complex literal strings do not have parenthesis
        if str_value.startswith(u'(') and str_value.endswith(u')'):
            str_value = str_value[1:-1]

    return str_value

def checkValue(dtype, str_value):
    """
    Check for overflow errors and dtype errors on integer and float values.

    By default, when overflow occurs in the creation of a ``numpy`` array,
    it is silently converted to the desired datatype:

    >>> numpy.array(-4).astype(numpy.uint8)
    array(252, dtype=uint8)
    >>> numpy.array(1420).astype(numpy.int8)
    array(-116, dtype=int8)

    This behavior can be acceptable for a library, but not for an end
    user application as ``ViTables`` so we have to catch such cases.

    :Parameters:

    - dtype: the value datatype
    - str_value: the string representation of a value
    """

    dtypes_map = {
        u'int8': numpy.int8, u'int16': numpy.int16,
        u'int32': numpy.int32, u'int64': numpy.int64,
        u'uint8': numpy.uint8, u'uint16': numpy.uint16,
        u'uint32': numpy.uint32, u'uint64': numpy.uint64,
        u'float32': numpy.float32, u'float64': numpy.float64,
        }

    # For Python objects and strings no overflow can occur
    if dtype not in dtypes_map:
        return str_value

    # astype() doesn't support unicode arguments
    dtype_enc = dtype.encode('utf_8')
    new_array = numpy.array(str_value).astype(dtype_enc)
    # Catches unexpected results from conversions
    # Examples: numpy.array('-23').astype('unint8') -> mismatch dtype
    # or numpy.array('99999').astype('int8') -> overflow
    if unicode(new_array[()]) != str_value:
        raise RuntimeWarning
    # If no errors are found return the original value
    return str_value

class AttrEditor(object):
    """
    Setup the attributes entered in the Properties dialog.

    When the user edits the Attributes Set Instance (see `PyTables` manual
    for details) of a given node via the Properties dialog and presses `OK`
    the validity of the new set of attributes is checked. If it is fine
    then the old `ASI` is replaced by the new one and the dialog is closed.
    If an error is found in the new set of attributes then the dialog
    remains opened until the user fixes the mistake.

    :Parameters:

    - `asi`: the Attributes Set Instance being updated
    - `title`: the TITLE attribute entered by the user in the Properties dialog
    - `user_table`: the table of user attributes edited by the user in the
        Properties dialog
    """

    def __init__(self, asi, title, user_table):
        """Initialise the attributes checker."""

        self.asi = asi

        # A dictionary with the attributes that have to be checked
        self.edited_attrs = {}
        model = user_table.model()
        rows = model.rowCount()
        # Parse the table and get string representations of its cell contents
        for row in range(0, rows):
            # As ViTables doesn't support editing ND-array attributes they
            # are marked in order to be found later
            name_item = model.item(row, 0)
            if not name_item.isEditable():
                multidim = True
            else:
                multidim = False
            name = model.item(row, 0).text()
            value = model.item(row, 1).text()
            dtype_index = model.indexFromItem(model.item(row, 2))
            dtype = user_table.indexWidget(dtype_index).currentText()
            self.edited_attrs[row] = (name, value, dtype, multidim)

        # Add the TITLE attribute to the dictionary
        if title is not None:
            self.edited_attrs[rows] = (u'TITLE', title, u'string', False)

    def checkAttributes(self):
        """
        Check the user attributes table.

        If empty or repeated names, values mismatching the
        attribute type or out of range values are found then an error is
        reported. If the table is OK the node `ASI` is updated and the
        Properties dialog is closed.
        """

        # Error message for mismatching value/type pairs
        dtype_error = translate('AttrEditor', """\nError: "{0}" value """
            """mismatches its data type.""",
            'User attributes table editing error')

        # Error message for overflow values
        range_error = translate('AttrEditor', """\nError: "{0}" value """
            """is out of range.""",
            'User attributes table editing error')

        # Some times it's difficult to find out if the error
        # is due to a mismatch or to an overflow
        conversion_error = translate('AttrEditor', """\nError: "{0}" value """
            """is out of range or mismatches its data type.""",
            'User attributes table editing error')

        # Error message for syntax errors in Python attributes
        syntax_error = translate('AttrEditor', """\nError: "{0}" """
            """cannot be converted to a Python object.""",
            'User attributes table editing error')

        rows_range = self.edited_attrs.keys()

        # Check for empty Name cells
        for row in rows_range:
            name = self.edited_attrs[row][0]
            # Empty Value cells are acceptable for string attributes
            # but empty Name cells are invalid
            if name == u'':
                return (False,
                        translate('AttrEditor',
                            "\nError: empty field Name in the row {0:d}",
                            'User attrs editing error').format(int(row + 1)))

        # Check for repeated names
        names_list = []
        for row in rows_range:
            name = self.edited_attrs[row][0]
            if not name in names_list:
                names_list.append(name)
            else:
                return (False,
                        translate('AttrEditor',
                            '\nError: attribute name "{0}" is repeated.',
                            'User attrs table editing error').format(name))

        # Check for dtype, range and syntax correctness of scalar attributes
        for row in rows_range:
            name, value, dtype, multidim = self.edited_attrs[row]
            if multidim == True:
                continue
            if dtype == 'python':
                # Check the syntax of the Python expression
                if not checkSyntax(value):
                    return (False, syntax_error.format(name))
            else:
                # Format properly the string representation of value
                # and check for dtype and overflow errors
                try:
                    value = formatStrValue(dtype, value)
                    value = checkValue(dtype, value)
                except OverflowError:
                    return (False, range_error.format(name))
                except ValueError:
                    return (False, dtype_error.format(name))
                except RuntimeWarning:
                    return (False, conversion_error.format(name))

            # If the attribute passes every test then its entry in the
            # dictionary of edited attributes is updated
            self.edited_attrs[row] = name, value, dtype, multidim

        return (True, None)

    def setAttributes(self):
        """
        Update edited attributes.

        This method is called when the user has succesfuly edited some
        attribute in the Properties dialog. If it happens then this method
        updates the `ASI` of the inspected `PyTables` node.
        """

        # Get rid of deleted attributes
        if u'TITLE' in self.edited_attrs:
            all_attrs = frozenset(self.asi._v_attrnamesuser + [u"TITLE"])
        else:
            all_attrs = frozenset(self.asi._v_attrnamesuser)
        edited_attrs_names = frozenset([self.edited_attrs[row][0]
                                        for row in self.edited_attrs.keys()])
        for attr in (all_attrs - edited_attrs_names):
            try:
                self.asi._v_node._f_delAttr(attr)
            except (tables.NodeError, AttributeError):
                vitables.utils.formatExceptionInfo()

        for row in self.edited_attrs.keys():
            # Scalar attributes are stored as
            # numpy scalar arrays of the proper type
            name, value, dtype, multidim = self.edited_attrs[row]
            if multidim == True:
                continue

            if dtype == u'python':
                value = eval(u'{0}'.format(value))
            else:
                dtype_enc = dtype.encode('utf_8')
                value = numpy.array(value).astype(dtype_enc)[()]

            # Updates the ASI
            try:
                setattr(self.asi, name, value)
            except AttributeError:
                vitables.utils.formatExceptionInfo()

max_stars_repo_name: lefevre-fraser/openmeta-mms

max_issues_repo_name: lefevre-fraser/openmeta-mms

max_forks_repo_name: lefevre-fraser/openmeta-mms


text:
import numpy
import pandas as pd
import math as m

#Moving Average
def MA(df, n):
    MA = pd.Series(df['Close'].rolling(n).mean(), name = 'MA_' + str(n))
    df = df.join(MA)
    return df

def MACD(df, n_fast, n_slow):
    """Calculate MACD, MACD Signal and MACD difference

    :param df: pandas.DataFrame
    :param n_fast:
    :param n_slow:
    :return: pandas.DataFrame
    """
    EMAfast = pd.Series(df['Close'].ewm(span=n_fast, min_periods=n_slow).mean())
    EMAslow = pd.Series(df['Close'].ewm(span=n_slow, min_periods=n_slow).mean())
    MACD = pd.Series(EMAfast - EMAslow, name='MACD_' + str(n_fast) + '_' + str(n_slow))
    MACDsign = pd.Series(MACD.ewm(span=9, min_periods=9).mean(), name='MACDsign_' + str(n_fast) + '_' + str(n_slow))
    MACDdiff = pd.Series(MACD - MACDsign, name='MACDdiff_' + str(n_fast) + '_' + str(n_slow))
    df = df.join(MACD)
    df = df.join(MACDsign)
    df = df.join(MACDdiff)
    return df

#Average Directional Movement Index
def ADX(df, n, n_ADX):
    """Calculate the Average Directional Movement Index for given data.

    :param df: pandas.DataFrame
    :param n:
    :param n_ADX:
    :return: pandas.DataFrame
    """
    i = 0
    UpI = []
    DoI = []
    while i + 1 <= df.index[-1]:
        UpMove = df.loc[i + 1, 'High'] - df.loc[i, 'High']
        DoMove = df.loc[i, 'Low'] - df.loc[i + 1, 'Low']
        if UpMove > DoMove and UpMove > 0:
            UpD = UpMove
        else:
            UpD = 0
        UpI.append(UpD)
        if DoMove > UpMove and DoMove > 0:
            DoD = DoMove
        else:
            DoD = 0
        DoI.append(DoD)
        i = i + 1
    i = 0
    TR_l = [0]
    while i < df.index[-1]:
        TR = max(df.loc[i + 1, 'High'], df.loc[i, 'Close']) - min(df.loc[i + 1, 'Low'], df.loc[i, 'Close'])
        TR_l.append(TR)
        i = i + 1
    TR_s = pd.Series(TR_l)
    ATR = pd.Series(TR_s.ewm(span=n, min_periods=n).mean())
    UpI = pd.Series(UpI)
    DoI = pd.Series(DoI)
    PosDI = pd.Series(UpI.ewm(span=n, min_periods=n).mean() / ATR)
    NegDI = pd.Series(DoI.ewm(span=n, min_periods=n).mean() / ATR)
    ADX = pd.Series((abs(PosDI - NegDI) / (PosDI + NegDI)).ewm(span=n_ADX, min_periods=n_ADX).mean(),
                    name='ADX_' + str(n) + '_' + str(n_ADX))
    df = df.join(ADX)
    return df

#RSI
def RSI(df, n):
    """Calculate Relative Strength Index(RSI) for given data.

    :param df: pandas.DataFrame
    :param n:
    :return: pandas.DataFrame
    """
    i = 0
    UpI = [0]
    DoI = [0]
    while i + 1 <= df.index[-1]:
        UpMove = df.loc[i + 1, 'High'] - df.loc[i, 'High']
        DoMove = df.loc[i, 'Low'] - df.loc[i + 1, 'Low']
        if UpMove > DoMove and UpMove > 0:
            UpD = UpMove
        else:
            UpD = 0
        UpI.append(UpD)
        if DoMove > UpMove and DoMove > 0:
            DoD = DoMove
        else:
            DoD = 0
        DoI.append(DoD)
        i = i + 1
    UpI = pd.Series(UpI)
    DoI = pd.Series(DoI)
    PosDI = pd.Series(UpI.ewm(span=n, min_periods=n).mean())
    NegDI = pd.Series(DoI.ewm(span=n, min_periods=n).mean())
    RSI = pd.Series(PosDI / (PosDI + NegDI), name='RSI_' + str(n))
    df = df.join(RSI)
    return df
#Mass Index
def mass_index(df):
    """Calculate the Mass Index for given data.

    :param df: pandas.DataFrame
    :return: pandas.DataFrame
    """
    Range = df['High'] - df['Low']
    EX1 = Range.ewm(span=9, min_periods=9).mean()
    EX2 = EX1.ewm(span=9, min_periods=9).mean()
    Mass = EX1 / EX2
    MassI = pd.Series(Mass.rolling(25).sum(), name='Mass Index')
    df = df.join(MassI)
    return df

def get_technical_indicators(dataset):
    # Create 7 and 21 days Moving Average
    dataset['ma7'] = dataset['Close'].rolling(window=7).mean()
    dataset['ma21'] = dataset['Close'].rolling(window=21).mean()

    # Create MACD
    dataset['26ema'] = pd.DataFrame.ewm(dataset['Close'], span=26).mean()
    dataset['12ema'] = pd.DataFrame.ewm(dataset['Close'], span=12).mean()
    dataset['MACD'] = (dataset['12ema']-dataset['26ema'])

    # Create Bollinger Bands
    dataset['20sd'] = dataset['Close'].rolling(20).std()
    dataset['upper_band'] = dataset['ma21'] + (dataset['20sd']*2)
    dataset['lower_band'] = dataset['ma21'] - (dataset['20sd']*2)

    # Create Exponential moving average
    dataset['ema'] = dataset['Close'].ewm(com=0.5).mean()

    # Create Momentum
    dataset['momentum'] = dataset['Close']-1

    return dataset


text:
# This code is supporting material for the book
# Building Machine Learning Systems with Python
# by Willi Richert and Luis Pedro Coelho
# published by PACKT Publishing
#
# It is made available under the MIT License

import numpy as np
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.datasets import load_svmlight_file
from sklearn.linear_model import LinearRegression
from sklearn.cross_validation import KFold

# Whether to use Elastic nets (otherwise, ordinary linear regression is used)

# Load data:
data, target = load_svmlight_file('data/E2006.train')

lr = LinearRegression()

# Compute error on training data to demonstrate that we can obtain near perfect
# scores:

lr.fit(data, target)
pred = lr.predict(data)

print('RMSE on training, {:.2}'.format(np.sqrt(mean_squared_error(target, pred))))
print('R2 on training, {:.2}'.format(r2_score(target, pred)))
print('')

pred = np.zeros_like(target)
kf = KFold(len(target), n_folds=5)
for train, test in kf:
    lr.fit(data[train], target[train])
    pred[test] = lr.predict(data[test])

print('RMSE on testing (5 fold), {:.2}'.format(np.sqrt(mean_squared_error(target, pred))))
print('R2 on testing (5 fold), {:.2}'.format(r2_score(target, pred)))

max_stars_repo_path: Chapter 7/lr10k.py

max_issues_repo_path: Chapter 7/lr10k.py

max_forks_repo_path: Chapter 7/lr10k.py