text:
from scores import *
import inquirer
from functools import lru_cache as cache
import pandas as pd
from IPython import get_ipython
import itertools
from statistics import mean
from numba import jit, vectorize
import timeit
import numpy as np
from itertools import permutations
from numpy import vectorize
import random
import pyfiglet

# TODO, add pre-flop 2 card hand, show stats for pre-flop
# TODO, make more game like
# TODO, declear winner

def start_game():
    result = pyfiglet.figlet_format("PyPoker")
    print(result)
    print('Welcome to PyPoker a TexasHoldem style poker similuation')
    print('You can choose to play a random hand, or choose your own cards')

def flop_message():
    result = "** Flop **"
    print(result)

def turn_message():
    result = "** Turn **"
    print(result)

def river_message():
    result = "** River **"
    print(result)

def multi_choice():
    while True:
        questions = [inquirer.Checkbox(
            'Choose',
            message="Choose how you would like to play",
            choices=['Random Hand','Choose Your Own'],
        )]
        answers = inquirer.prompt(questions)['Choose']
        break
    return answers

# refactor, this is called multiple times
def calc_flop(c4, c3, flop):
    flopscore = expected_value(flop,combi)
    current = df.loc[df['value'] >= flopscore[0]].index[0]/2598960*100
    future  = df.loc[df['value'] >= flopscore[1]].index[1]/2598960*100
    print('My current value is  %s and the average expected value is %s' % (current,future))
    players = float(input('enter number of players: '))
    pot = float(input('enter pot value: '))
    price = float(input('enter value of your bet: '))
    if current > future:
        should_call(players,current,pot,price)
    else:
        should_call(players,future,pot,price)

def calc_winner(current, pre_flop):
    pre_flop = []
    for i in range(8):
        rand_card = random.randint(0,53)
        pre_flop.append(deck[rand_card])

    flopscore = expected_value(pre_flop,combi)
    bot_score = df.loc[df['value'] >= flopscore[0]].index[0]/2598960*100

    if current < bot_score:
        print("you lose, bot had a better hand")
    else:
        print("you win, you beat the bot")

    print("Your score:", current)
    print("Bot score:", bot_score)

# Numba has two compilation modes: nopython and object
# Produces much faster code, to prevent Numba from falling back, and instead raise an error pass nopython=true
@jit(nopython=True)
def common(a,b):
    common=[]
    l1 = [i for i in a]
    l2 = [i for i in b]
    common = len([i for i in l1 if i in l2])
    return common

@jit(nopython=True)
def numba_4():
    results = []
    len1 = c4.shape[0]
    len2 = combi.shape[0]
    for i in range(len1):
        for j in range(len2):
            if common(c4[i],combi[j]) == 4:
                results.append(combi[j])
    return results

@jit(nopython=True)
def numba_3():
    results = []
    len1 = c3.shape[0]
    len2 = combi.shape[0]
    for i in range(len1):
        for j in range(len2):
            if common(c3[i],combi[j]) == 4:
                results.append(combi[j])
    return results

@cache(maxsize=None)
def opti_3():
    values= numba_3()
    return [score_hand(i) for i in values]

@cache(maxsize=None)
def opti_4():
    values= numba_4()
    return [score_hand(i) for i in values]

def expected_value(hand,combi):
    if len(hand) == 5:
        maxi = score_hand(hand)
        mean= np.mean(opti_3() + opti_4())
    elif len(hand) == 6:
        maxi = max([score_hand(i) for i in combinations(hand,5)])
        mean = np.mean(opti_4())
    elif len(hand) >= 7:
        maxi = max([score_hand(i) for i in combinations(hand,5)])
        mean= maxi
    values = [maxi, mean]
    return values

def should_call(players,percentile,pot,price):
    pwin = (percentile/100)**players
    ev = pwin*pot
    if ev <= 0:
        print('you should fold')
    if ev > 0:
        print('You should bet as long as it is less than %s $' % ev)
    print('The expected value betting %s is %s $' % (price,ev-price))
    return pwin*100

# this cannot be in a method or it breaks the whole thing due to scope
flop = []
pre_flop = []
start_game()

"""
 Pre-Flop
"""

# Multiple choice prompt
answers_pre = multi_choice()
if answers_pre != 'Choose Your Own':
    for i in range(2):
        rand_card = random.randint(0,53)
        try:
            flop.append(deck[rand_card])
            pre_flop.append(deck[rand_card])
        except IndexError:
            flop = []
            flop.append(deck[rand_card])
            pre_flop.append(deck[rand_card])

# if the user chooses to pick their own cards
if len(flop) == 0:
    # Prompt the user what they're about to enter
    print('Enter you card in the format suit value, this is for your flop so it will be 5 cards')
    print('For Example S9 would be the 9 of spades')
    for i in range(0,2):
        flop.append(str(input('enter card: ')))

def show_hand():
# show the hand
    if len(flop) <= 5:
        print("_________________________________")
        print("         Your Hand")
        print(flop)
        print("_________________________________")
    elif len(flop) <= 6:
        print("_________________________________")
        print("         Your Hand")
        print("Turn Card", turn)
        print(flop)
        print("_________________________________")
    else:
        print("_________________________________")
        print("         Your Hand")
        print("River Card", river)
        print(flop)
        print("_________________________________")

print("** Hole Cards **")
show_hand()

"""
 Flop
"""

flop_message()

# Multiple choice prompt
answers = multi_choice()
if answers != 'Choose Your Own':
    for i in range(3):
        card_rand = random.randint(0,53)
        try:
            flop.append(deck[card_rand])
        except IndexError:
            flop = []
            flop.append(deck[card_rand])

# if the user chooses to pick their own cards
if len(flop) == 0:
    # Prompt the user what they're about to enter
    print('Enter you card in the format suit value, this is for your flop so it will be 5 cards')
    print('For Example S9 would be the 9 of spades')
    for i in range(0,5):
        flop.append(str(input('enter card: ')))

show_hand()

c4 = combinations(flop,4)
c3 = combinations(flop,3)
calc_flop(c4, c3, flop)

"""
 Turn
"""

turn_message()

turn = []
# Multiple choice prompt
answers_turn = multi_choice()
if answers_turn != 'Choose Your Own':
    for i in range(1):
        rand_card = random.randint(0,53)
        try:
            turn.append(deck[rand_card])
        except IndexError:
            turn = []
            turn.append(deck[rand_card])

if len(turn) < 1:
    turn.append(str(input('enter card: ')))
flop.append(turn[0])

# show the hand
show_hand()
calc_flop(c4, c3, flop)
"""
 River
"""

river_message()

river = []
answers_turn = multi_choice()
if answers_turn != 'Choose Your Own':
    for i in range(1):
        j = random.randint(0,53)
        river.append(deck[j])

if len(turn) < 1:
    river.append(str(input('enter card: ')))
flop.append(river[0])
# show the hand
show_hand()

flop.append(river[0])
combiriver = expected_value(flop,combi)
current = df.loc[df['value'] >= combiriver[0]].index[0]/2598960*100
print('My final value is %s' % current)
players = float(input('enter number of players: '))
pot = float(input('enter pot value: '))
price = float(input('enter value of your bet: '))
should_call(players,current, pot,price)
calc_winner(current, pre_flop)

max_stars_repo_name: BrendanGlancy/akron

max_issues_repo_name: BrendanGlancy/akron

max_forks_repo_name: BrendanGlancy/akron

reason: import numpy,from numpy,from numba


text:
from typing import Union

import cv2
import numpy as np

def is_line(w: int, h: int, max_height, multiplier=0.01) -> bool:
    """
    checks whether the dimensions match that of a line
    """
    return w > h * multiplier and h < max_height

def is_approx_rect(contour: np.ndarray, confidence: int = 0.95) -> bool:
    """
    returns true if contour is approximately a rectangle for the given confidence
    """
    _, _, w, h = cv2.boundingRect(contour)
    area = w * h
    contour_area = cv2.contourArea(contour)
    return area >= confidence * contour_area

def get_approx_polygon(contour: np.ndarray, points: int = 4) -> Union[np.ndarray, None]:
    """
    gets the approximate polygon from a contour
    """
    num_points = contour.shape[0]

    # contour has less # points than target
    if num_points < points:
        return None

    processed_contour = contour
    arc_length = cv2.arcLength(processed_contour, closed=True)
    epsilon = 0.2 * arc_length
    while num_points > points:
        processed_contour = cv2.approxPolyDP(processed_contour, epsilon=epsilon, closed=True)
        num_points = processed_contour.shape[0]
        epsilon *= 1.1

    return processed_contour

ext: py

max_stars_repo_name: thoaif/mv-bank-receipt-ocr

max_issues_repo_name: thoaif/mv-bank-receipt-ocr

max_forks_repo_name: thoaif/mv-bank-receipt-ocr

reason: import numpy


text:
######################################################################
######################################################################
#  Copyright Tsung-Hsien Wen, Cambridge Dialogue Systems Group, 2016 #
######################################################################
######################################################################
import os
import re
import sys
import json
import math
import operator
import random
import itertools
import numpy as np

from FeatParser import *
from DataLexicaliser import *
from semo.RNNLG.utils.nlp import *

class DataReader(object):
    def __init__(self, seed, domain, obj,
            vocabfile, trainfile, validfile, testfile,
            percentage=1.0, verbose=0, lexCutoff=4):

        self.percentage = percentage # percentage of data used
        # container for data
        self.data   = {'train':[],'valid':[],'test':[]}
        self.mode   = 'train'     # mode  for accessing data
        self.index  = 0          # index for accessing data
        self.obj    = obj

        # load vocab from file
        self._loadVocab(vocabfile)
        self._loadTokenMap()

        ## set input feature cardinality
        self._setCardinality()

        ## init formatter/lexicaliser
        self.formatter      = SoftDActFormatter()
        self.lexicaliser    = ExactMatchDataLexicaliser()
        #self.hardformatter  = HardDActFormatter()

        ## for lexicalising SLOT_TYPE
        self.lexicaliser.typetoken = domain

        # initialise dataset
        self._setupData(trainfile,validfile,testfile)
        #self._testDelexicalisation()

        # obtain pos tags
        #self.obtainTags()

        if verbose : self._printStats()

    def readall(self,mode='train',processed=False):
        alldata = []
        for feat, dact, sent, base in self.data[mode]:
            a,sv,s,v = self.genFeatVec(feat,self.cardinality,self.dfs)
            alldata.append([a,sv,s,v,sent,dact,base])
        return alldata

    def read(self,mode='train',batch=1):
        ## default implementation for read() function
        ## support batch reading & random shuffling
        if self.mode!=mode:
            self.mode = mode
            index = 0

        # end of data , reset index & return None
        if self.index>=len(self.data[mode]):
            data = None
            self.index = 0
            # shuffle data except for testing
            if mode!='test' : random.shuffle(self.data[mode])
            return data

        # reading a batch
        start = self.index
        end   = self.index+batch \
                if self.index+batch<len(self.data[mode])\
                else len(self.data[mode])
        data = self.data[mode][start:end]
        self.index += batch

        # post processing data: a,sv,s,v,sent,dact,base
        proc_data = [[],[],[],[],[],[],[]]
        max_leng  = [0,0,0,0,0]
        for feat,dact,sent,base in data:
            a,sv,s,v = self.genFeatVec(feat,self.cardinality,self.dfs)
            # testing case,
            if mode=='test':
                sent = [self.lexicalise(sent[i],dact) \
                        for i in range(len(sent))]
                base = [self.lexicalise(base[i],dact) \
                        for i in range(len(base))]
                # put in container
                xvec = [a,sv,s,v,sent,dact,base]
                for j in range(len(proc_data)):
                    proc_data[j].append(xvec[j])
            else:
                # delexicalised text & formatted feature
                if self.obj=='ml': # ML training, 1 sent per example
                    sent = self.delexicalise(sent,dact)
                    sent = self.genInputSent(sent, self.vocab)
                    # put in container
                    xvec = [a,sv,s,v,sent,dact,base]
                    for j in range(len(proc_data)):
                        proc_data[j].append(xvec[j])
                    # padding length
                    for j in range(0,5):
                        if len(xvec[j])>max_leng[j]:
                            max_leng[j] = len(xvec[j])
                else: # TODO:DT training, 1 da/multiple sents per example
                    print a,sv
                    print sent
                    raw_input()
        # padding to have the same sent length
        lengs = [[],[],[],[],[]]
        if mode!='test': # if testing set, not need to process sentence
            for i in range(0,5):
                for j in range(len(proc_data[i])):
                    lengs[i].append(len(proc_data[i][j]))
                    proc_data[i][j].extend([-1]*(max_leng[i]-len(proc_data[i][j])))
        # padding to have the same batch size
        for x in range(end-start,batch):
            for i in range(0,5):
                proc_data[i].append(proc_data[i][-1])
                lengs[i].append(len(proc_data[i][-1]))
        # input/output
        a,sv,s,v =  np.array(proc_data[0],dtype=np.int32),\
                    np.array(proc_data[1],dtype=np.int32),\
                    np.array(proc_data[2],dtype=np.int32),\
                    np.array(proc_data[3],dtype=np.int32)
        if mode!='test':
            words= np.swapaxes(np.array(proc_data[4],dtype=np.int32),0,1)
        else:
            words = proc_data[4]
        dact=proc_data[5]
        base=proc_data[6]
        # current batch size
        b_size = end-start
        lengs  = np.array(lengs,dtype=np.int32)

        return a,sv,s,v, words, dact, base, b_size, lengs

    def delexicalise(self,sent,dact):
        feat = self.formatter.parse(dact,keepValues=True)
        return self.lexicaliser.delexicalise(sent,feat['s2v'])

    def lexicalise(self,sent,dact):
        feat = self.formatter.parse(dact,keepValues=True)
        return self.lexicaliser.lexicalise(sent,feat['s2v'])

    def format(self,dact):
        return self.formatter.format(dact)

    def _setCardinality(self):

        self.cardinality = []
        fin = file('semo/RNNLG/resource/feat_template.txt')
        self.dfs = [0,0,0,0,0]
        for line in fin.readlines():
            self.cardinality.append(line.replace('\n',''))
            if line.startswith('a.'):
                self.dfs[1]+=1
            elif line.startswith('sv.'):
                self.dfs[2]+=1
            elif line.startswith('s.'):
                self.dfs[3]+=1
            elif line.startswith('v.'):
                self.dfs[4]+=1
        for i in range(0,len(self.dfs)-1):
            self.dfs[i+1] = self.dfs[i] + self.dfs[i+1]

    def _printStats(self):
        print '==============='
        print 'Data statistics'
        print '==============='
        print 'Train: %d' % len(self.data['train'] )
        print 'Valid: %d' % len(self.data['valid'] )
        print 'Test : %d' % len(self.data['test']  )
        print 'Feat : %d' % len(self.cardinality)
        print '==============='

    def _testDelexicalisation(self):
        for data in self.data['train']+self.data['valid']+self.data['test']:
            dact,sent,base = data
            self.lexicalise(self.delexicalise(sent,dact),dact)

    def _setupData(self,trainfile,validfile,testfile):

        # load data from file
        train_group = True if self.obj=='dt' else False
        self.data['train'] = self._loadData(trainfile,train_group)
        self.data['valid'] = self._loadData(validfile,train_group)
        self.data['test' ] = self._loadData(testfile, False, True)
        # cut train/valid data by proportion
        self.data['train'] = self.data['train']\
                [:int(self.percentage*len(self.data['train']))]
        self.data['valid'] = self.data['valid']\
                [:int(self.percentage*len(self.data['valid']))]

    def _loadData(self,filename,group=True,multiref=False):

        fin = file(filename)
        # remove comment lines
        for i in range(5):
            fin.readline()
        data = json.load(fin)
        fin.close()

        container = []
        for dact,sent,base in data:
            # word tokens
            sent = self.delexicalise(
                    normalize(re.sub(' [\.\?\!]$','',sent)),dact)
            base = self.delexicalise(
                    normalize(re.sub(' [\.\?\!]$','',base)),dact)
            feat = self.formatter.format(dact)
            container.append( [feat,dact,sent,base] )

        # grouping several sentences w/ the same dialogue act
        # for testing set, or DT on train/valid
        if group or multiref:
            # grouping data points according to unique DAs
            a2ref = {}
            for feat,dact,sent,base in container:
                if a2ref.has_key(tuple(feat)):
                    a2ref[ tuple(feat)][0].append(dact)
                    a2ref[ tuple(feat)][1].append(sent)
                    a2ref[ tuple(feat)][2].append(base)
                else:
                    a2ref[ tuple(feat)] = [[dact],[sent],[base]]
            # return grouped examples
            if group:
                reordered_container = []
                for feat,bundle in a2ref.items():
                    reordered_container.append([feat,
                        bundle[0],bundle[1],bundle[2]])
                return reordered_container
            # return examples w/ multiple references
            if multiref:
                reordered_container = []
                for feat,dact,sent,base in container:
                    reordered_container.append([feat,dact,
                        a2ref[tuple(feat)][1],
                        a2ref[tuple(feat)][2]])
                return reordered_container
        # if no grouping nor multiref, return directly
        else:   return container

    def _loadVocab(self,vocabfile):
        fin = file(vocabfile)
        self.vocab = []#['<unk>','</s>']
        for wrd in fin.readlines():
            wrd = wrd.replace('\n','')
            self.vocab.append(wrd)

    def _loadTokenMap(self,mapfile='semo/RNNLG/resource/detect.pair'):
        fin  = file(mapfile)
        self.tokenMap = json.load(fin)['general'].items()
        fin.close()
        # make it 1-to-1 relation
        self.feat2token = {}
        for k,v in self.tokenMap:
            for x in ['1','2','3']:
                key = 'sv.'+k+'._'+x
                self.feat2token[key] = v

    def tokenMap2Indexes(self):
        maxleng = 0
        idxmap = [[] for x in range(len(self.vocab))]
        for k,v in self.feat2token.iteritems():
            try:
                idxmap[self.vocab.index(v)].append(self.cardinality.index(k)-self.dfs[1])
                if len(idxmap[self.vocab.index(v)])>maxleng:
                    maxleng = len(idxmap[self.vocab.index(v)])
            except: pass
        for i in range(len(idxmap)):
            idxmap[i].extend([-1]*abs(len(idxmap[i])-maxleng))
        idxmap = np.array(idxmap,dtype='int32')
        return idxmap

    def readVecFile(self,filename,vocab):
        fin = file(filename)
        # discard comment lines
        for i in range(5):
            fin.readline()
        word2vec = {}
        for line in fin.readlines():
            tokens = line.replace('\n','').split()
            word = tokens[0]
            vec = [float(x) for x in tokens[1:]]
            if word in vocab:
                word2vec[vocab.index(word)] = np.array(vec)
        return word2vec

    def genFeatVec(self,feat,cardinality,dfs):
        a,sv,s,v = [],[],[],[]
        a.append(cardinality.index('a.'+feat[0][-1]))
        for item in feat[1:]:
            si,vi = item
            if 'sv.'+si+'.'+vi in cardinality:
                sv.append(  cardinality.index('sv.'+si+'.'+vi)-dfs[1])
            if 's.'+si in cardinality:
                s.append(   cardinality.index('s.'+si)-dfs[2] )
            if 'v.'+vi in cardinality:
                v.append(   cardinality.index('v.'+vi)-dfs[3] )
        if len(feat[1:])==0:
            sv.append(  cardinality.index('sv.NONE.NONE')-dfs[1])
            s.append(   cardinality.index('s.NONE')-dfs[2])
            v.append(   cardinality.index('v.NONE')-dfs[3])
        return a,sv,s,v
        """
        featvec = [0.0 for x in range(len(cardinality))]
        featvec[ cardinality.index('a.'+feat[0][-1]) ] = 1.0
        for item in feat[1:]:
            si,vi = item
            featvec[ cardinality.index('s.'+si) ] = 1.0
            featvec[ cardinality.index('v.'+vi) ] = 1.0
            featvec[ cardinality.index('sv.'+si+'.'+vi) ] = 1.0
        if len(feat[1:])==0:
            featvec[ cardinality.index('s.NONE')] = 1.0
            featvec[ cardinality.index('v.NONE')] = 1.0
            featvec[ cardinality.index('sv.NONE.NONE')] = 1.0
        featvec = np.array(featvec,dtype=np.float32)
        return featvec
        """

    def genInputSent(self,sent,vocab):
        words = ['</s>'] + sent.split() + ['</s>']
        wordids = [vocab.index(w) if w in vocab else 0 for w in words]
        return wordids

max_stars_repo_name: WowCZ/strac

max_issues_repo_name: WowCZ/strac

max_forks_repo_name: WowCZ/strac


text:
import sys
import time
import socket
import collections
import numpy as np
import pyqtgraph as pg
from pyqtgraph.Qt import QtCore, QtGui

import requests
import PIL.Image
from io import BytesIO

CAMERA_IP = "192.168.4.4" # can be improved by using mDNS
IMAGE_FRESH_INTERVAL = 20 # ms

class App(QtGui.QMainWindow):
    def __init__(self, parent=None):
        super(App, self).__init__(parent)

        #### Create Gui Elements ###########
        self.mainbox = pg.LayoutWidget()
        self.setCentralWidget(self.mainbox)

        # set up image widget
        self.img_w = pg.GraphicsLayoutWidget()
        self.mainbox.addWidget(self.img_w, row=0, col=0)
        # image view box
        self.view = self.img_w.addViewBox()
        self.view.setAspectLocked(True)
        self.view.setRange(QtCore.QRectF(0,0, 800, 600))
        #  image plot
        self.img = pg.ImageItem(border='w')
        self.view.addItem(self.img)

        # a text label widget for info dispaly
        self.label = QtGui.QLabel()
        self.mainbox.addWidget(self.label, row=1, col=0)

        #### Set Data  #####################

        self.fps = 0.
        self.lastupdate = time.time()

        # bring up camera and stream video
        self.setup_camera()

    def calculate_fps(self):
        now = time.time()
        dt = (now-self.lastupdate)
        if dt <= 0:
            dt = 0.000000000001
        fps2 = 1.0 / dt
        self.lastupdate = now
        self.fps = self.fps * 0.9 + fps2 * 0.1

    def update_label(self):
        tx = 'Mean Frame Rate:  {fps:.3f} FPS'.format(fps=self.fps )
        self.label.setText(tx)

    def setup_camera(self):
        while True:
            # set a large frame size
            cmd = {'var': 'framesize', 'val':"8"}
            resp = requests.get('http://'+ CAMERA_IP +'/control', params=cmd)
            if resp.status_code == 200:
                break

        self.stream_iter = requests.get('http://'+ CAMERA_IP +':81/stream', stream=True).iter_content(chunk_size=1024*1024*8)
        self.video_buf = bytes()
        # schedule the next update call
        QtCore.QTimer.singleShot(1, self.update_img)

    def update_img(self):
        try:
            self.video_buf += next(self.stream_iter)
        except:
            # schedule the next update call
            QtCore.QTimer.singleShot(IMAGE_FRESH_INTERVAL, self.update_img)
            return
        a = self.video_buf.find(b'\xff\xd8')
        b = self.video_buf.find(b'\xff\xd9')

        if a != -1 and b != -1:
            chunk_content = self.video_buf[a:b+2]
            self.video_buf = self.video_buf[b+2:]

            img_capture = PIL.Image.open(BytesIO(chunk_content))
            img_np = np.array(img_capture.rotate(-90))

            self.img.setImage(img_np)

        self.calculate_fps()
        self.update_label()

        # schedule the next update call
        QtCore.QTimer.singleShot(IMAGE_FRESH_INTERVAL, self.update_img)

if __name__ == '__main__':

    app = QtGui.QApplication(sys.argv)
    thisapp = App()
    thisapp.show()
    sys.exit(app.exec_())


text:
# so I think the aim here is that we construct stuff which kind of works, but I really don't kno
# we add hulls to the things, I think seeign a straightforward algorithmic implementation woudl be good
# and further it would be really cool if we had something that works nicely, so let's try this out and see if it can shed light on the algorithm at all?
# so it just iteratively consructs the envelope function g(x) as the list of the things cmoing from it, which is nice, as tangents to the plane and the rest, so that is really cool
# and doesn't necessarily make sense per se, but that doesn't matter overmuch, so let's work on this and copy this to see if we get a greater understanding of how the algorithm fits together

import numpy as np
import random
import matplotlib.pyplot as plt

class ARS():
	#PDF must be univariate(!!) and log-concave!
	#doesn't exlpoit lower hull atm

	def __init__(self, f, fprima, xi=[-4,1,4], lb=np.Inf, ub=nb.Inf, use_lower=False, ns=50, **fargs):
		#so params:
		#f: a function that compues log(f(u...) for given u where f(u) is proportioante to the density we want to sample from - i.e. this is our function evaluation
		#fprima is the derivative d/du log(f(U,...))
		#zi is the ordered vector of starting points in which log(fu is defined) to initisliae the hulls
	#d is domain limits
	#use_lower means that lower squeezing will be used which is more efficient as it eliminates/reduces useless chcks
	#lb is the lower bound of the domain
	#ub is the upper bound of the domain
	#ns is the maximum number of points for the hull
	#fargs is argments for f and frprima

	self.lb = lb
	self.ub = ub
	self.f = f
	self.fprima = fprima
	self.fargs = fargs

	self.ns = ns
	self.x = np.array(xi)#intiailise x, the vector of absicassae at which functoin h has been evaluated
	self.h = self.f(self.x, **self.fargs) # this is the result of our functoin
	self.hprime = self.fprima(self.x, **self.fargs)

	#avoid under/overflow errors as the envelope and pdf are only proportional so we can choose any constant of roportoinality
	self.offset = np.amax(self.h)
	self.h = self.h - self.offset

	#derivative at first point in xi must be > 0
	#and at last point in xi < 0 # concave restriction
	if not(self.hprime[0] > 0):
		print self.hprime
		raise IOError('initial anchor points must span mode of pdf')
	if not(self.hprime[-1]<0):
		print self.hprime
		raise IOError('initial anchor points must span mode of pdf')
	self.insert()

	#okay draws n samples and update upper and lower hulls
	def draw(self,N):
		samples = np.zeros(N)
		n = 0
		while n < N:
			[xt,i] = self.sampleUpper()
			#should perform squeezing test but haven't yet in their implenetation ugh
			ht = self.f(xt, **self.fargs)
			hprimet = self.fprima(xt, **self.fargs)
			ht = ht - self.offset
			ut = self.h[i[ + (xt - self.x[i])*self.hprime[i]

			#accept sample? doesn't use lower atm
			u = random.random()
			if u(np.exp(ht-ut):
				samples[n] =xt
				n+=1

			#update hull with new function evluations
			if self.u.__len__() < self.ns:
				self.insert([xt],[ht],[hprimet])
		return samples

    def insert(self,xnew=[],hnew=[],hprimenew=[]):
        '''
        Update hulls with new point(s) if none given, just recalculate hull from existing x,h,hprime
        '''
        if xnew.__len__() > 0:
            x = np.hstack([self.x,xnew])
            idx = np.argsort(x)
            self.x = x[idx]
            self.h = np.hstack([self.h, hnew])[idx]
            self.hprime = np.hstack([self.hprime, hprimenew])[idx]

        self.z = np.zeros(self.x.__len__()+1)

        # This is the formula explicitly stated in Gilks.
        # Requires 7(N-1) computations
        # Following line requires 6(N-1)
        # self.z[1:-1] = (np.diff(self.h) + self.x[:-1]*self.hprime[:-1] - self.x[1:]*self.hprime[1:]) / -np.diff(self.hprime);

        self.z[1:-1] = (np.diff(self.h) - np.diff(self.x*self.hprime))/-np.diff(self.hprime)

        self.z[0] = self.lb; self.z[-1] = self.ub
        N = self.h.__len__()
        self.u = self.hprime[[0]+range(N)]*(self.z-self.x[[0]+range(N)]) + self.h[[0]+range(N)]

        self.s = np.hstack([0,np.cumsum(np.diff(np.exp(self.u))/self.hprime)])
        self.cu = self.s[-1]

    def sampleUpper(self):
        '''
        Return a single value randomly sampled from the upper hull and index of segment
        '''
        u = random.random()

        # Find the largest z such that sc(z) < u
        i = np.nonzero(self.s/self.cu < u)[0][-1]

        # Figure out x from inverse cdf in relevant sector
        xt = self.x[i] + (-self.h[i] + np.log(self.hprime[i]*(self.cu*u - self.s[i]) +
        np.exp(self.u[i]))) / self.hprime[i]

        return [xt,i]

    def plotHull(self):
        '''
        Plot the piecewise linear hull using matplotlib
        '''
        xpoints = self.z
        #ypoints = np.hstack([0,np.diff(self.z)*self.hprime])
        ypoints = np.exp(self.u)
        plt.plot(xpoints,ypoints)
        plt.show()
        '''
        for i in range(1,self.z.__len__()):
            x1 = self.z[i]
            y1 = 0
            x2 = self.z[i+1]
            y2 = self.z[i+1]-self.z[i] * hprime[i]
        '''

max_stars_repo_name: Bmillidgework/Misc-Maths

max_issues_repo_name: Bmillidgework/Misc-Maths

max_forks_repo_name: Bmillidgework/Misc-Maths


text:
########################################################################
# SwarmOps - Heuristic optimization for Python.
# Copyright (C) 2003-2016 Magnus Erik Hvass Pedersen.
# See the file README.md for instructions.
# See the file LICENSE.txt for license details.
# SwarmOps on the internet: http://www.Hvass-Labs.org/
########################################################################

########################################################################
# Parent-classes for doing optimization. These provide various logistics,
# printing of status messages, running multiple optimizations, etc.
########################################################################

import numpy as np
from swarmops.FitnessTrace import FitnessTrace
from swarmops import tools

##################################################

class SingleRun:
    """
    Parent-class for performing a single optimization run.
    The class provides various logistics that are common to all
    optimizers. If you make a new optimizer, then you should derive
    from this class.
    """

    def __init__(self, max_evaluations, run_number=0, trace_len=0, display_interval=0):
        """
        Create object instance and initialize variables for keeping track
        of the optimization progress.
        Then call self._optimize() to perform the actual optimization.

        :param max_evaluations:
            Maximum number of fitness evaluations for the problem.

        :param run_number:
            The optimization run number, if performing multiple optimization runs.

        :param trace_len:
            Approximate length of the fitness-trace.
            Default is zero which means that no fitness-trace will be created.

        :param display_interval:
            Approximate interval between printing status messages.
            Default is zero which means that no status messages will be printed.

        :return:
            Object instance.
        """

        # Copy arguments to instance variables.
        self.max_evaluations = max_evaluations
        self.run_number = run_number
        self.display_interval = display_interval

        # Initialize the counter for the next status-display.
        self.display_next = 0

        # Create an object used for tracing the fitness at regular intervals.
        self.fitness_trace = FitnessTrace(trace_len, max_evaluations)

        # Initialize best-known position and fitness.
        self.best = None
        self.best_fitness = np.inf

        # Print status at the beginning of the optimization run.
        self._start_run()

        # Perform the actual optimization iterations.
        self._optimize()

        # Print status at the end of the optimization run.
        self._end_run()

    def _optimize(self):
        """
        Perform the actual optimization.
        This function should be implemented by the child-class.
        The function does not return anything, instead it will
        call self._update_best() to update the best-known solution
        during optimization.
        """

        # Raise an exception if the child-class has not implemented this function.
        raise NotImplementedError

    def _start_run(self):
        """
        Print status at the beginning of the optimization run.
        """

        if self.display_interval > 0:
            msg = "Starting optimization run {0} using {1} ..."
            print(msg.format(self.run_number, self.name))

    def _iteration(self, i):
        """
        Print status and trace fitness during optimization.
        This function should be called regularly during optimization.
        """

        if self.display_interval > 0 and i >= self.display_next:
            # Print the status.
            msg = "Run: {0}, Iteration: {1}, Best Fitness: {2:.4e}"
            print(msg.format(self.run_number, i, self.best_fitness))

            # Increment the counter for the next status-display.
            self.display_next = i + self.display_interval

        # Trace the fitness.
        self.fitness_trace.trace(i, self.best_fitness)

    def _end_run(self):
        """
        Print status at the end of the optimization run.
        """

        if self.display_interval > 0:
            # Print the status.
            msg = "Finished optimization run {0}, Best fitness: {1:.4e}"
            print(msg.format(self.run_number, self.best_fitness))

    def _update_best(self, fitness, x):
        """
        Update the best-known solution and fitness if an improvement.

        WARNING: This function does NOT copy the array x so you must
        ensure that the optimizer does not modify the array you pass
        in as x here.

        :param fitness: New fitness.
        :param x: New solution.
        :return: Boolean whether the fitness was an improvement.
        """

        # If the fitness is an improvement over the best-known fitness.
        improvement = fitness < self.best_fitness
        if improvement:
            # Update the best-known fitness and position.
            self.best_fitness = fitness
            self.best = x

        return improvement

##################################################

class MultiRun:
    """
    Perform multiple optimization runs with an optimizer and
    calculate statistics on the results.

    This has been separated into its own class for the sake of
    modularity so it is easier to read and maintain. There is only a
    tiny overhead of instantiating the SingleRun-class for each run.
    """

    def __init__(self, optimizer, num_runs, problem, parallel=True, *args, **kwargs):
        """
        Create object instance and perform multiple optimization runs.
        To retrieve the results, access the object variables afterwards.

        The parameters are the same as for the Optimize-class,
        except for the following:

        :param optimizer:
            Class for the optimizer e.g. PSO or DE.

        :param num_runs:
            Number of optimization runs to perform.

        :param problem:
            The problem to be optimized. Instance of Problem-class.

        :param parallel:
            Perform the optimization runs in parallel (True) or serial (False).

        :param args:
            Arguments to pass along to the Optimize-class.

        :param kwargs:
            Arguments to pass along to the Optimize-class.

        :return:
            Object instance. Get the optimization results from the object's variables.
            -   best_solution is the best-found solution.
            -   best_fitness is the associated fitness of the best-found solution.
            -   fitness_trace is a 2-d numpy array with the fitness trace of each run.
        """

        # Copy arguments to instance variables.
        self.problem = problem
        self.optimizer = optimizer

        # Store the args and kwargs to be passed on to the optimizer.
        self.args = args
        self.kwargs = kwargs

        if not parallel:
            # Run the optimizer multiple times on one processor.
            self.runs = [self._optimize(run_number=i) for i in range(num_runs)]
        else:
            import multiprocessing as mp

            # Create a pool of workers sized according to the CPU cores available.
            pool = mp.Pool()

            # Run the optimizer multiple times in parallel.
            # We must use a helper-function for this.
            self.runs = pool.map(self._optimize_parallel, range(num_runs))

            # Close the pool of workers and wait for them all to finish.
            pool.close()
            pool.join()

        # Put the best solutions from all the optimization runs into an array.
        self.solution = np.array([run.best for run in self.runs])

        # Put the best fitness from all the optimization runs into an array.
        self.fitness = np.array([run.best_fitness for run in self.runs])

        # Put the fitness traces from all the optimization runs into an array.
        # This only works when the fitness-traces are all the same length.
        # If you make changes to the optimizers so they can stop early, then
        # you may have to change this.
        self.fitness_trace = np.array([run.fitness_trace.fitness for run in self.runs])

        # Index for the best fitness (minimization).
        i = np.argmin(self.fitness)

        # Best optimization run. This is an instance of the SingleRun-class.
        self.best_run = self.runs[i]

        # Best fitness of all the optimization runs.
        self.best_fitness = self.fitness[i]

        # Best solution of all the optimization runs.
        self.best = self.solution[i]

    def _optimize(self, run_number):
        """
        Helper-function used for execution of an optimization run. Non-parallel.

        :param run_number: Counter for the optimization run.
        :return: Instance of the SingleRun-class.
        """

        return self.optimizer(problem=self.problem, run_number=run_number,
                              *self.args, **self.kwargs)

    def _optimize_parallel(self, run_number):
        """
        Helper-function used for parallel execution of an optimization run.

        :param run_number: Counter for the optimization run.
        :return: Instance of the SingleRun-class.
        """

        # Create a new Pseudo-Random Number Generator for the thread.
        tools.new_prng()

        # Do the optimization.
        return self._optimize(run_number=run_number)

    def refine(self):
        """
        Refine the best result from heuristic optimization using SciPy's L-BFGS-B method.
        This may significantly improve the results on some optimization problems,
        but it is sometimes very slow to execute.

        NOTE: This function imports SciPy, which should make it possible
        to use the rest of this source-code library even if SciPy is not installed.
        SciPy should first be loaded when calling this function.

        :return:
            A tuple with:
            -   The best fitness found.
            -   The best solution found.
        """

        # SciPy requires bounds in another format.
        bounds = list(zip(self.problem.lower_bound, self.problem.upper_bound))

        # Start SciPy optimization at best found solution.
        import scipy.optimize
        res = scipy.optimize.minimize(fun=self.problem.fitness,
                                      x0=self.best,
                                      method="L-BFGS-B",
                                      bounds=bounds)

        # Get best fitness and parameters.
        refined_fitness = res.fun
        refined_solution = res.x

        return refined_fitness, refined_solution

    def print_statistics(self):
        """
        Print statistics for the fitness.
        """

        # Print header.
        print("{0} - Optimized by {1}".format(self.problem.name_full, self.optimizer.name))
        print("Fitness Statistics:")

        # Print mean and standard deviation.
        print("- Mean:\t\t{0:.4e}".format(self.fitness.mean()))
        print("- Std.:\t\t{0:.4e}".format(self.fitness.std()))

        # Print quartiles.
        print("- Min:\t\t{0:.4e}".format(self.fitness.min()))
        print("- 1st Qrt.:\t{0:.4e}".format(np.percentile(self.fitness, 0.25)))
        print("- Median:\t{0:.4e}".format(np.percentile(self.fitness, 0.5)))
        print("- 3rd Qrt.:\t{0:.4e}".format(np.percentile(self.fitness, 0.75)))
        print("- Max:\t\t{0:.4e}".format(self.fitness.max()))

    def plot_fitness_trace(self, y_log_scale=True, filename=None):
        """
        Plot the fitness traces.

        NOTE: This function imports matplotlib, which should make it possible
        to use the rest of this source-code library even if it is not installed.
        matplotlib should first be loaded when calling this function.

        :param y_log_scale: Use log-scale for y-axis.
        :param filename: Output filename e.g. "foo.svg". If None then plot to screen.
        :return: Nothing.
        """

        import matplotlib.pyplot as plt

        # Setup plotting.
        plt.grid()

        # Axis labels.
        plt.xlabel("Iteration")
        plt.ylabel("Fitness (Lower is better)")

        # Title.
        title = "{0} - Optimized by {1}".format(self.problem.name_full, self.optimizer.name)
        plt.title(title)

        # Use log-scale for Y-axis.
        if y_log_scale:
            plt.yscale("log", nonposy="clip")

        # Plot the fitness-trace for each optimization run.
        for run in self.runs:
            # Array with iteration counter for the optimization run.
            iteration = run.fitness_trace.iteration

            # Array with fitness-trace for the optimization run.
            fitness_trace = run.fitness_trace.fitness

            # Plot the fitness-trace.
            plt.plot(iteration, fitness_trace, 'r-', color='black', alpha=0.25)

        # Plot to screen or file.
        if filename is None:
            # Plot to screen.
            plt.show()
        else:
            # Plot to file.
            plt.savefig(filename, bbox_inches='tight')
            plt.close()

##################################################

max_stars_repo_name: Hvass-Labs/SwarmOps

max_issues_repo_name: Hvass-Labs/SwarmOps

max_forks_repo_name: Hvass-Labs/SwarmOps

reason: import numpy,import scipy


text:
import json
import torch
from sqlnet.utils import *
import numpy as np
import datetime
#import mxnet as mx
#from bert_embedding import BertEmbedding

LOCAL_TEST=False

if LOCAL_TEST:
    N_word=100
    B_word=6
    USE_SMALL=True
else:
    N_word=300
    B_word=42
    USE_SMALL=False

sql_data, table_data, val_sql_data, val_table_data,\
        test_sql_data, test_table_data, TRAIN_DB, DEV_DB, TEST_DB = \
        load_dataset(0, use_small=USE_SMALL)
word_emb = load_word_emb('glove/glove.%dB.%dd.txt'%(B_word,N_word),
        use_small=USE_SMALL)
#ctx = mx.gpu(0)
#bert = BertEmbedding(ctx=ctx)
print("Length of word vocabulary: {}".format(len(word_emb)))
print(np.array(word_emb).shape)
word_to_idx = {'<UNK>':0, '<BEG>':1, '<END>':2}
word_num = 3
embs = [np.zeros(N_word,dtype=np.float32) for _ in range(word_num)]

def check_and_add(tok):
    #Check if the tok is in the vocab. If not, add it.
    global word_num
    if tok not in word_to_idx and tok in word_emb:
        word_to_idx[tok] = word_num
        word_num += 1
        embs.append(word_emb[tok])

for sql in sql_data:
    for tok in sql['question_tok']:
        check_and_add(tok)
for tab in table_data.values():
    for col in tab['header_tok']:
        for tok in col:
            check_and_add(tok)
for sql in val_sql_data:
    for tok in sql['question_tok']:
        check_and_add(tok)
for tab in val_table_data.values():
    for col in tab['header_tok']:
        for tok in col:
            check_and_add(tok)
for sql in test_sql_data:
    for tok in sql['question_tok']:
        check_and_add(tok)
for tab in test_table_data.values():
    for col in tab['header_tok']:
        for tok in col:
            check_and_add(tok)

print("Length of used word vocab: {}".format(len(word_to_idx)))

print(np.array(embs).shape)
emb_array = np.stack(embs, axis=0)
with open('glove/word2idx.json', 'w') as outf:
    json.dump(word_to_idx, outf)
np.save(open('glove/usedwordemb.npy', 'wb'), emb_array)


text:
import ctypes, warnings
from ctypes import byref
import numpy as np

# treat timout errors seperately
ACQIRIS_ERROR_MEZZIO_ACQ_TIMEOUT = -1074116395
ACQIRIS_ERROR_ACQ_TIMEOUT = -1074116352
ACQIRIS_ERROR_PROC_TIMEOUT = -1074116350
ACQIRIS_ERROR_LOAD_TIMEOUT = -1074116349
ACQIRIS_ERROR_READ_TIMEOUT = -1074116348
ACQIRIS_ERROR_WAIT_TIMEOUT = -1074116346
ACQIRIS_ERROR_FLASH_ACCESS_TIMEOUT = -1074116048
ACQIRIS_ERROR_NO_DATA = -1074116601

TIMEOUTS = (ACQIRIS_ERROR_MEZZIO_ACQ_TIMEOUT,
            ACQIRIS_ERROR_ACQ_TIMEOUT,
            ACQIRIS_ERROR_PROC_TIMEOUT,
            ACQIRIS_ERROR_LOAD_TIMEOUT,
            ACQIRIS_ERROR_READ_TIMEOUT,
            ACQIRIS_ERROR_WAIT_TIMEOUT,
            ACQIRIS_ERROR_FLASH_ACCESS_TIMEOUT,
            ACQIRIS_ERROR_NO_DATA)

# ignore error codes below the warning limit
WARNING_LIMIT = 0

# error type returned by this class
class Error(Exception):
    pass

class TimeoutError(Error):
    pass

# open dll
AgDLL = ctypes.WinDLL('AgMD1Fundamental.dll')

# define data types used by this dll
ViString = ctypes.c_char_p
ViBoolean = ctypes.c_bool
ViSession = ctypes.c_uint32
ViStatus = ctypes.c_int32
ViInt32 = ctypes.c_int32
ViUInt32 = ctypes.c_uint32
ViReal64 = ctypes.c_double
ViRsrc = ViString
ViChar = ctypes.c_char_p

# structs
class AqReadParameters(ctypes.Structure):
    _fields_ = [('dataType', ViInt32),
                ('readMode', ViInt32),
                ('firstSegment', ViInt32),
                ('nbrSegments', ViInt32),
                ('firstSampleInSeg', ViInt32),
                ('nbrSamplesInSeg', ViInt32),
                ('segmentOffset', ViInt32),
                ('dataArraySize', ViInt32),
                ('segDescArraySize', ViInt32),
                ('flags', ViInt32),
                ('reserved', ViInt32),
                ('reserved2', ViReal64),
                ('reserved3', ViReal64)]

class AqSegmentDescriptor(ctypes.Structure):
    _fields_ = [('horPos', ViReal64),
                ('timeStampLo', ViUInt32),
                ('timeStampHi', ViUInt32)]

class AqSegmentDescriptorAvg(ctypes.Structure):
    _fields_ = [('horPos', ViReal64),
                ('timeStampLo', ViUInt32),
                ('timeStampHi', ViUInt32),
                ('actualTriggersInSeg', ViUInt32),
                ('avgOvfl', ViInt32),
                ('avgStatus', ViInt32),
                ('avgMax', ViInt32),
                ('flags', ViUInt32),
                ('reserved', ViInt32)]

class AqSegmentDescriptorSeqRaw(ctypes.Structure):
    _fields_ = [('horPos', ViReal64),
                ('timeStampLo', ViUInt32),
                ('timeStampHi', ViUInt32),
                ('indexFirstPoint', ViUInt32),
                ('actualSegmentSize', ViUInt32),
                ('reserved', ViInt32)]

class AqDataDescriptor(ctypes.Structure):
    _fields_ = [('returnedSamplesPerSeg', ViInt32),
                ('indexFirstPoint', ViInt32),
                ('sampTime', ViReal64),
                ('vGain', ViReal64),
                ('vOffset', ViReal64),
                ('returnedSegments', ViInt32),
                ('nbrAvgWforms', ViInt32),
                ('actualTriggersInAcqLo', ViUInt32),
                ('actualTriggersInAcqHi', ViUInt32),
                ('actualDataSize', ViUInt32),
                ('reserved2', ViInt32),
                ('reserved3', ViReal64)]

class AcqirisDigitizer():
    """Represent the Acqiris digitizer, redefines the dll functions in python"""

    def __init__(self):
        """The init case defines a session ID, used to identify the instrument"""
        # create a session id
        self.session = ViSession()
        # store data buffer, to avoid re-allocation
        self.lBuffer = [None for a in range(2)]

    def callFunc(self, sFunc, *args, **kargs):
        """General function caller with restype=ViStatus, also checks for errors"""
        # get function from DLL
        func = getattr(AgDLL, sFunc)
        func.restype = ViStatus
        # call function, raise error if needed
        status = func(*args)
        if 'bIgnoreError' in kargs:
            bIgnoreError = kargs['bIgnoreError']
        else:
            bIgnoreError = False
        if status and not bIgnoreError and status<WARNING_LIMIT:
            sError = self.getError(status)
            # special treatment of TimeoutError
            if status in TIMEOUTS:
                raise TimeoutError(sError)
            else:
                raise Error(sError)

    def getError(self, status):
        """Convert the error in status to a string"""
        nBuffer = 512
        msgBuffer = ctypes.create_string_buffer(nBuffer)
        # ViStatus status = Acqrs_errorMessage(ViSession instrumentID,
        # ViStatus errorCode, ViChar errorMessage[],ViInt32 errorMessageSize);
        AgDLL['Acqrs_errorMessage'](self.session, status, msgBuffer,
                                    ViInt32(nBuffer))
        return msgBuffer.value

    def init(self, sResource='', bIdQuery=False, bReset=False):
        # ViStatus status = Acqrs_init(ViRsrc resourceName, ViBoolean IDQuery,
        # ViBoolean resetDevice, ViSession* instrumentID)
        if not isinstance(sResource, bytes):
            sResource = sResource.encode()
        self.callFunc('Acqrs_init', ViRsrc(sResource), ViBoolean(bIdQuery),
                      ViBoolean(bReset), byref(self.session))

    def InitWithOptions(self, sResource='', bIdQuery=False, bReset=False,
                         sOption='CAL=FALSE'):
        # ViStatus status = Acqrs_InitWithOptions(ViRsrc resourceName, ViBoolean IDQuery,
        # ViBoolean resetDevice, ViString optionsString, ViSession* instrumentID);
        if not isinstance(sResource, bytes):
            sResource = sResource.encode()
        self.callFunc('Acqrs_InitWithOptions', ViRsrc(sResource), ViBoolean(bIdQuery),
                      ViBoolean(bReset), ViString(sOption), byref(self.session))

    def close(self):
        # ViStatus status = Acqrs_close(ViSession instrumentID);
        self.callFunc('Acqrs_close', self.session, bIgnoreError=True)

    def closeAll(self):
        # ViStatus status = Acqrs_closeAll(void);
        self.callFunc('Acqrs_closeAll', bIgnoreError=True)

    def configHorizontal(self, sampInterval, delayTime):
        # ViStatus status = AcqrsD1_configHorizontal(ViSession instrumentID,
        # ViReal64 sampInterval, ViReal64 delayTime);
        self.callFunc('AcqrsD1_configHorizontal', self.session, ViReal64(sampInterval),
                      ViReal64(delayTime))

    def getHorizontal(self):
        # ViStatus status = AcqrsD1_getHorizontal(ViSession instrumentID,
        # ViReal64* sampInterval, ViReal64* delayTime);
        sampInterval = ViReal64()
        delayTime = ViReal64()
        self.callFunc('AcqrsD1_getHorizontal', self.session, byref(sampInterval),
                      byref(delayTime))
        # return both results
        return (sampInterval.value, delayTime.value)

    def configExtClock(self, clockType):
        """
        clockType      = 0        Internal Clock (default at start-up)
                       = 1        External Clock (continuous operation)
                       = 2        External Reference (10 MHz)
                       = 4        External Clock (start/stop operation)"""
        # ViStatus ACQ_CC AcqrsD1_configExtClock(ViSession instrumentID, ViInt32 clockType,
        #    ViReal64 inputThreshold, ViInt32 delayNbrSamples,
        #    ViReal64 inputFrequency, ViReal64 sampFrequency);
        self.callFunc('AcqrsD1_configExtClock', self.session,
                      ViInt32(clockType), ViReal64(0.0), ViInt32(0),
                      ViReal64(10E6), ViReal64(10E6))

    def getExtClock(self):
        # ViStatus ACQ_CC AcqrsD1_getExtClock(ViSession instrumentID, ViInt32* clockType,
        # ViReal64* inputThreshold, ViInt32* delayNbrSamples,
        # ViReal64* inputFrequency, ViReal64* sampFrequency);
        clockType = ViInt32()
        inputThreshold = ViReal64()
        delayNbrSamples = ViInt32()
        inputFrequency = ViReal64()
        sampFrequency = ViReal64()
        self.callFunc('AcqrsD1_getExtClock', self.session, byref(clockType),
                      byref(inputThreshold), byref(delayNbrSamples),
                      byref(inputFrequency), byref(sampFrequency))
        return (clockType.value, inputThreshold.value, delayNbrSamples.value, inputFrequency.value, sampFrequency.value)

    def configMemory(self, nbrSamples, nbrSegments):
        # ViStatus status = AcqrsD1_configMemory(ViSession instrumentID,
        # ViInt32 nbrSamples, ViInt32 nbrSegments)
        self.callFunc('AcqrsD1_configMemory', self.session,
                      ViInt32(nbrSamples), ViInt32(nbrSegments))

    def configMemoryEx(self, nbrSamples, nbrSegments, nbrBanks=2, flags=1):
#ACQ_DLL ViStatus ACQ_CC AcqrsD1_configMemoryEx(ViSession instrumentID, ViUInt32 nbrSamplesHi,
#    ViUInt32 nbrSamplesLo, ViInt32 nbrSegments, ViInt32 nbrBanks, ViInt32 flags);
#
#  Configures the memory control parameters of the digitizer.
#  'nbrSamplesHi'      = reserved for future use, must be set to 0.
#    'nbrSamplesLo'      = nominal number of samples to record (per segment!).
#    'nbrSegments'       = number of segments to acquire per bank
#                          1 corresponds to the normal single-trace acquisition mode.
#    'nbrBanks'          = number of banks in which the memory will be split,
#                          for buffered reading (SAR).
#                          1 corresponds to the normal acquisition mode.
#    'flags'             = 0 no flags.
#                        = 1 force use of internal memory (for digitizers with extended
#                          memory options only).
        self.callFunc('AcqrsD1_configMemoryEx', self.session,
                      ViUInt32(0), ViUInt32(nbrSamples),
                      ViInt32(nbrSegments), ViInt32(nbrBanks), ViInt32(flags))

    def freeBank(self):
#ACQ_DLL ViStatus ACQ_CC AcqrsD1_freeBank(ViSession instrumentID, ViInt32 reserved);
        self.callFunc('AcqrsD1_freeBank', self.session, ViInt32(0))

    def getMemory(self):
        # ViStatus status = AcqrsD1_getMemory(ViSession instrumentID,
        # ViInt32* nbrSamples, ViInt32* nbrSegments);
        nbrSamples = ViInt32()
        nbrSegments = ViInt32()
        self.callFunc('AcqrsD1_getMemory', self.session,
                      byref(nbrSamples), byref(nbrSegments))
        return (nbrSamples.value, nbrSegments.value)

    def configTrigSource(self, channel, trigCoupling, trigSlope, trigLevel1,
                         trigLevel2=0.0):
        """Channel is negative for externa trig.
        Trig coupling:    0, DC
                          1, AC
                          2, HF Reject (if available)
                          3, DC, 50 Ohm (ext. trigger only, if available)
                          4, AC, 50 Ohm (ext. trigger only, if available)
        Trig slope:       0, Positive
                          1, Negative"""
        # ViStatus status = AcqrsD1_configTrigSource(ViSession instrumentID,
        # ViInt32 channel, ViInt32 trigCoupling, ViInt32 trigSlope,
        # ViReal64 trigLevel1, ViReal64 trigLevel2);
        trigLevel2 = 0.0
        self.callFunc('AcqrsD1_configTrigSource', self.session,
                      ViInt32(channel), ViInt32(trigCoupling), ViInt32(trigSlope),
                      ViReal64(trigLevel1), ViReal64(trigLevel2))

    def getTrigSource(self, channel):
        """Channel is negative for externa trig.
        Trig coupling:    0, DC
                          1, AC
                          2, HF Reject (if available)
                          3, DC, 50 Ohm (ext. trigger only, if available)
                          4, AC, 50 Ohm (ext. trigger only, if available)
        Trig slope:       0, Positive
                          1, Negative"""
        # ViStatus status = AcqrsD1_getTrigSource(ViSession instrumentID, ViInt32 channel,
        # ViInt32* trigCoupling,
        # ViInt32* trigSlope, ViReal64* trigLevel1, ViReal64* trigLevel2);
        trigCoupling = ViInt32()
        trigSlope = ViInt32()
        trigLevel1 = ViReal64()
        trigLevel2 = ViReal64()
        self.callFunc('AcqrsD1_getTrigSource', self.session, ViInt32(channel),
                      byref(trigCoupling), byref(trigSlope),
                      byref(trigLevel1), byref(trigLevel2))
        return (trigCoupling.value, trigSlope.value, trigLevel1.value, trigLevel2.value)

    def configTrigClass(self, sourcePattern):
        """
        Source pattern: Ch1 = 0x00000001
                        Ch2 = 0x00000002
                        Ext1 = 0x80000000
        """
        # ViStatus status = AcqrsD1_configTrigClass(ViSession instrumentID,
        # ViInt32 trigClass, ViInt32 sourcePattern, ViInt32 validatePattern,
        # ViInt32 holdType, ViReal64 holdoffTime, ViReal64 reserved)
        self.callFunc('AcqrsD1_configTrigClass', self.session,
                      ViInt32(0), ViInt32(sourcePattern), ViInt32(0),
                      ViInt32(0), ViReal64(0.0), ViReal64(0.0))

    def getTrigClass(self):
        """
        Source pattern: Ch1 = 0x00000001
                        Ch2 = 0x00000002
                        Ext1 = 0x80000000
        """
        # ViStatus status = AcqrsD1_getTrigClass(ViSession instrumentID,
        # ViInt32* trigClass, ViInt32* sourcePattern, ViInt32* validatePattern,
        # ViInt32* holdType, ViReal64* holdoffTime, ViReal64* reserved);
        trigClass = ViInt32()
        sourcePattern = ViInt32()
        validatePattern = ViInt32()
        holdType = ViInt32()
        holdoffTime = ViReal64()
        reserved = ViReal64()
        self.callFunc('AcqrsD1_getTrigClass', self.session,
                      byref(trigClass), byref(sourcePattern),
                      byref(validatePattern), byref(holdType),
                      byref(holdoffTime), byref(reserved))
        return (sourcePattern.value, trigClass.value)

    def configVertical(self, channel, fullScale, offset, coupling, bandwidth):
        """
        Coupling:   0: Ground (Averagers ONLY)
                    1: DC, 1 MOhm
                    2: AC, 1 MOhm
                    3: DC, 50 Ohm
                    4: AC, 50 Ohm
        Bandwidth:  0: Full
                    1: 25 MHz
                    2: 700 MHz
                    3: 200 MHz
                    4: 20 MHz
                    5: 35 MHz
        """
        # ViStatus status = AcqrsD1_configVertical(ViSession instrumentID,
        # ViInt32 channel,ViReal64 fullScale, ViReal64 offset, ViInt32 coupling,
        # ViInt32 bandwidth)
        self.callFunc('AcqrsD1_configVertical', self.session,
                      ViInt32(channel), ViReal64(fullScale), ViReal64(offset),
                      ViInt32(coupling), ViInt32(bandwidth))

    def getVertical(self, channel):
        """
        Coupling:   0: Ground (Averagers ONLY)
                    1: DC, 1 MOhm
                    2: AC, 1 MOhm
                    3: DC, 50 Ohm
                    4: AC, 50 Ohm
        Bandwidth:  0: Full
                    1: 25 MHz
                    2: 700 MHz
                    3: 200 MHz
                    4: 20 MHz
                    5: 35 MHz
        """
        # ViStatus status = AcqrsD1_getVertical(ViSession instrumentID,
        # ViInt32 channel, ViReal64* fullScale,
        # ViReal64* offset, ViInt32* coupling, ViInt32* bandwidth);
        fullScale = ViReal64()
        offset = ViReal64()
        coupling = ViInt32()
        bandwidth = ViInt32()
        self.callFunc('AcqrsD1_getVertical', self.session, ViInt32(channel),
                      byref(fullScale), byref(offset),
                      byref(coupling), byref(bandwidth))
        return (fullScale.value, offset.value, coupling.value, bandwidth.value)

    def configMode(self, mode, modifier=0, flags=0):
        """
        0 = normal data acquisition
        1 = AC/SC stream data to DPU
        2 = averaging mode (only in real-time averagers)
        3 = buffered data acquisition (only in AP101/AP201
        analyzers)
        5 = PeakTDC mode
        6 = frequency counter mode
        7 = SSR mode (AP235/240)/ Zero-Suppress (U1084)
        12 = DDC mode (M9202A)
        14 = Custom firmware
        """
        # ViStatus status = AcqrsD1_configMode(ViSession instrumentID,
        # ViInt32 mode, ViInt32 modifier, ViInt32 flags);
        self.callFunc('AcqrsD1_configMode', self.session,
                      ViInt32(mode), ViInt32(modifier), ViInt32(flags))

    def getMode(self):
        """
        0 = normal data acquisition
        1 = AC/SC stream data to DPU
        2 = averaging mode (only in real-time averagers)
        3 = buffered data acquisition (only in AP101/AP201
        analyzers)
        5 = PeakTDC mode
        6 = frequency counter mode
        7 = SSR mode (AP235/240)/ Zero-Suppress (U1084)
        12 = DDC mode (M9202A)
        14 = Custom firmware
        """
        # ViStatus status = AcqrsD1_getMode(ViSession instrumentID,
        # ViInt32* mode, ViInt32* modifier, ViInt32* flags)
        mode = ViInt32()
        modifier = ViInt32()
        flags = ViInt32()
        self.callFunc('AcqrsD1_getMode', self.session,
                      byref(mode), byref(modifier), byref(flags))
        return (mode.value, modifier.value, flags.value)

    def configAveraging(self, channel, **params):
        """Setup parameters for averaging mode"""
        for key, value in params.items():
            self.configAvgConfig(channel, key, value)

    def configAvgConfig(self, channelNbr, parameterString, value):
        if not isinstance(parameterString, bytes):
            parameterString = parameterString.encode()
        lFloat = (b"NoiseBase", b"StartDeltaPosPeakV", b"Threshold", b"ValidDeltaPosPeakV")
        # proceed depending on Int or Float
        if parameterString in lFloat:
            # ViStatus status = AcqrsD1_configAvgConfigReal64(ViSession instrumentID,
            # ViInt32 channelNbr, ViString parameterString, ViReal64 value);
            self.callFunc('AcqrsD1_configAvgConfigReal64', self.session,
                          ViInt32(channelNbr), ViString(parameterString), ViReal64(value))
        else:
            # ViStatus status = AcqrsD1_configAvgConfigInt32(ViSession instrumentID,
            # ViInt32 channelNbr, ViString parameterString, ViInt32 value);
            self.callFunc('AcqrsD1_configAvgConfigInt32', self.session,
                          ViInt32(channelNbr), ViString(parameterString), ViInt32(value))

    def getAvgConfig(self, channelNbr, parameterString):
        if not isinstance(parameterString, bytes):
            parameterString = parameterString.encode()
        lFloat = (b"NoiseBase", b"StartDeltaPosPeakV", b"Threshold", b"ValidDeltaPosPeakV")
        # proceed depending on Int or Float
        if parameterString in lFloat:
            # ViStatus status = AcqrsD1_getAvgConfigReal64(ViSession instrumentID,
            # ViInt32 channelNbr, ViString parameterString, ViReal64 *value);
            value = ViReal64()
            self.callFunc('AcqrsD1_getAvgConfigReal64', self.session, ViInt32(channelNbr),
                          ViString(parameterString), byref(value))
        else:
            # AcqrsD1_getAvgConfigInt32(ViSession instrumentID, ViInt32 channelNbr,
            # ViString parameterString, ViInt32 *value);
            value = ViInt32()
            self.callFunc('AcqrsD1_getAvgConfigInt32', self.session, ViInt32(channelNbr),
                          ViString(parameterString), byref(value))
        return value.value

    def acqDone(self):
#ACQ_DLL ViStatus ACQ_CC AcqrsD1_acqDone(ViSession instrumentID, ViBoolean* done);
#//! Checks if the acquisition has terminated.
#/*! Returns 'done' = VI_TRUE if the acquisition is terminated
#
#    Returns one of the following ViStatus values:
#    ACQIRIS_ERROR_IO_READ    if a link error has been detected (e.g. PCI link lost).
#    VI_SUCCESS otherwise. */
        value = ViBoolean()
        self.callFunc('AcqrsD1_acqDone', self.session, byref(value))
        return value.value

    def waitForEndOfAcquisition(self, timeout=60000, funcStop=None):
        # ViStatus status = AcqrsD1_waitForEndOfAcquisition (ViSession instrumentID,
        # ViInt32 timeout);
        # max timout is 1 second
        iMaxTime = 1000
        # do multiple calls to check stop function once in a while
        while timeout > 0:
            try:
                self.callFunc('AcqrsD1_waitForEndOfAcquisition', self.session,
                              ViInt32(int(iMaxTime)))
                # completed, break out of loop
                break
            except TimeoutError:
                timeout -= iMaxTime
                if timeout > 0:
                    # check if stopped
                    if funcStop is not None and funcStop():
                        # stop acquisition, break loop
                        self.stopAcquisition()
                        return False
                    # try again
                    continue
                else:
                    self.stopAcquisition()
                    # re-raise timeout error
                    raise
        return True

    def acquire(self):
        # ViStatus status = AcqrsD1_acquire(ViSession instrumentID);
        self.callFunc('AcqrsD1_acquire', self.session)

    def stopAcquisition(self):
        # ViStatus status = AcqrsD1_stopAcquisition(ViSession instrumentID);
        self.callFunc('AcqrsD1_stopAcquisition', self.session)

    def stopProcessing(self):
        # ViStatus status = AcqrsD1_stopProcessing(ViSession instrumentID);
        self.callFunc('AcqrsD1_stopProcessing', self.session)

    # define datatype
    DATATYPE_BYTE = 0
    DATATYPE_SHORT = 1
    DATATYPE_INT = 2
    DATATYPE_DOUBLE = 3

    def readData(self, channel, readPar):
        """
        readPar.dataType:
            DATATYPE_BYTE = 0
            DATATYPE_SHORT = 1
            DATATYPE_INT = 2
            DATATYPE_DOUBLE = 3
        """
        #
        nbrSegments = readPar.nbrSegments
        nbrSamples = readPar.nbrSamplesInSeg * nbrSegments

        # allocate data array based on the requested type
        if readPar.dataType == self.DATATYPE_BYTE:
            dataType = ctypes.c_byte
        elif readPar.dataType == self.DATATYPE_SHORT:
            dataType = ctypes.c_short
        elif readPar.dataType == self.DATATYPE_INT:
            dataType = ctypes.c_int
        elif readPar.dataType == self.DATATYPE_DOUBLE:
            dataType = ctypes.c_double
        # get buffer
        bufSize = nbrSamples + 32
        dataArray = self.lBuffer[channel-1]
        # if no old buffer, reallocate
        if dataArray is None or bufSize != len(dataArray):
            dataArray = (dataType * bufSize)()
            self.lBuffer[channel-1] = dataArray
#        dataArray = (dataType * (nbrSamples + 32))()
        # allocate data descriptor struct
        descriptor = AqDataDescriptor()
        # allocate segment descriptor array
        if readPar.readMode in [0, 1, 3]:
            segDescType = AqSegmentDescriptor
        elif readPar.readMode in [2, 5, 6]:
            segDescType = AqSegmentDescriptorAvg
        elif readPar.readMode in [11]:
            segDescType = AqSegmentDescriptorSeqRaw
        else:
            raise Exception('Unknown readMode: %d' % (readPar.readMode))
        segDesc = (segDescType * nbrSegments)()

        # set the sizes of data and segment descriptor arrays
        readPar.dataArraySize = ctypes.sizeof(dataArray)
        readPar.segDescArraySize = ctypes.sizeof(segDesc)
        #
        # ViStatus status = AcqrsD1_readData(ViSession instrumentID,
        # ViInt32 channel, AqReadParameters* readPar, ViAddr dataArray,
        # AqDataDescriptor* descriptor, ViAddr segDesc);
        self.callFunc('AcqrsD1_readData', self.session, ViInt32(channel),
                      byref(readPar), dataArray, byref(descriptor), segDesc)
        return (dataArray, descriptor, segDesc)

    def readChannelsToNumpy(self, nSample, lChannel=[1], nAverage=1, nSegment=1,
                            timeout=10000, bAverageMode=False, bArm=True, bMeasure=True):
        """Convenience method for getting multiple channels to a list of numpy
        arrays"""
        if bArm:
            if bAverageMode:
                # average
                self.readPar = AqReadParameters(dataType=self.DATATYPE_DOUBLE,
                                                  readMode=2,
                                                  nbrSegments=nSegment,
                                                  nbrSamplesInSeg=nSample,
                                                  firstSegment=0,
                                                  firstSampleInSegment=0)
            else:
                # single trace
                self.readPar = AqReadParameters(dataType=self.DATATYPE_DOUBLE,
                                                  readMode=0,
                                                  nbrSegments=nSegment,
                                                  nbrSamplesInSeg=nSample,
                                                  firstSegment=0,
                                                  firstSampleInSegment=0)
            # start acquisition
            self.acquire()
        if bMeasure:
            if not self.waitForEndOfAcquisition(timeout):
                return ([np.array([])]*len(lChannel), 1.0)
            # get waveforms for all channels
            lData = []
            for channel in lChannel:
                (dataArray, descriptor, segDesc) = self.readData(channel, self.readPar)
                vData = self.getResultAsNumpyArray(dataArray, descriptor)
                lData.append(vData)
            # get time data
            dt = descriptor.sampTime
            # return new numpy 2d array
            return (lData, dt)

    def getResultAsNumpyArray(self, dataArray, descriptor):
        """Return the result from the Acqiris card to a numpy array"""
        indexFirstPoint = descriptor.indexFirstPoint
        nbrSamplesPerSeg = descriptor.returnedSamplesPerSeg
        nbrSegments = descriptor.returnedSegments
        # don't show warnings about wrong data size
        with warnings.catch_warnings():
            warnings.simplefilter('ignore')
            vData = np.ctypeslib.as_array(dataArray)
        vData = vData[indexFirstPoint:indexFirstPoint + nbrSamplesPerSeg*nbrSegments]
        return vData

    def getRoundRobinData(self, nSample, nSegment, nAverage,
                          bConfig=True, bArm=True, bMeasure=True,
                          funcStop=None, funcProgress=None):
        """Get round-robin type data by continuous segmented acquisition"""
#        import logging
#        lg = logging.getLogger('LabberDriver')
        # total number of segments*nSample is 1024*8192
        nMax = 1024*8192
#        nMax = 1024*8192
        # calculate number of calls, and number of segments per call
        nOneRound = nSample * nSegment
        # max number of averages per call
        nMaxAvCall = int(np.floor(nMax / float(nOneRound) ))
        # number of calls to make
        nCall = int(np.ceil(nAverage / float(nMaxAvCall) ))
        # actual number of averaeges per call
        nAvCall = int(np.ceil(nAverage / float(nCall) ))
        # total number of segments per call
        nSegCall = nSegment * nAvCall
        # always get both channels
        lChannel = [1, 2]
        # configure
        if bConfig:
            # set mode (averagin + SAR)
            self.configMode(2, 0, 10)
            # config memory (2 banks, use internal memory)
            self.configMemory(nSample, nSegCall)
            self.configMemoryEx(nSample, nSegCall, 2, 1)
            # configure averaging for both channels
            for ch in lChannel:
                self.configAveraging(ch, NbrSamples=nSample, NbrSegments=nSegCall,
                                     NbrWaveforms=1, StartDelay=0, StopDelay=0)
            # define read parameters
            self.readPar = AqReadParameters(dataType=self.DATATYPE_DOUBLE,
                                       readMode=2,
                                       nbrSegments=nSegCall,
                                       nbrSamplesInSeg=nSample,
                                       firstSegment=0,
                                       firstSampleInSegment=0)
        # start acquisition
        if bArm:
            self.acquire()
        if not bMeasure:
            return
        # allocate memory
        lData = [np.zeros((nSegment, nSample)) for n in range(2)]
        n1 = 0
        while n1 < nCall:
            # wait to acquire data for one buffer
            if not self.waitForEndOfAcquisition(timeout=600E3, funcStop=funcStop):
                return (lData, 1.0)
            for n2, channel in enumerate(lChannel):
                # get data as numpy array
                (dataArray, descriptor, segDesc) = self.readData(channel, self.readPar)
                vData = self.getResultAsNumpyArray(dataArray, descriptor)
                # get time stamps
                vL = np.array([o.timeStampLo for o in segDesc], dtype=np.uint64)
                vH = np.array([o.timeStampHi for o in segDesc], dtype=np.uint64)
                vTimeStamp = vL + vH * np.uint64(2**32)
                # if first call, get typical trig time step for reference
                if n1==0 and n2==0:
                    t0 = vTimeStamp[0]
                    dt = vTimeStamp[1] - vTimeStamp[0]
                # convert time stamps to index
                vIndx = np.array((vTimeStamp - t0 + dt/np.uint64(2)) /dt, dtype=int)
                iSeg = vIndx[0] % nSegCall
                # if not starting at first segment, rotate. ignore if too few index values
                if iSeg != 0 and vIndx[0] > (nSegCall*n1):
                    vData = np.roll(vData, iSeg*nSample)
                # add data to corresponding segment
                if len(vData)>0:
                    lData[n2] += np.sum(vData.reshape(nAvCall, nSegment, nSample), 0)
            # free data bank and continue with next values
            self.freeBank()
            n1 += 1
            # check if stopped
            if funcStop is not None:
                if funcStop():
                    break
            # report progress
            if funcProgress is not None:
                funcProgress(float(n1)/float(nCall))
        self.stopAcquisition()
        # finallly, divide summed arrays to get average
        for n in range(2):
            lData[n] /= float(nAverage)
        # get time data
        dt = descriptor.sampTime
        # remove reference
        self.readPar = None
        # turn off segmenting, to avoid limits on averaging in hardware
        nSegCall = 1
        self.configMemory(nSample, nSegCall)
        self.configMemoryEx(nSample, nSegCall, 1, 0)
        # configure averaging for both channels
        for ch in lChannel:
            self.configAveraging(ch, NbrSamples=nSample, NbrSegments=nSegCall,
                                 NbrWaveforms=1, StartDelay=0, StopDelay=0)
        self.configMode(2, 0, 10)
        # return data
        return (lData, dt)

def reportProgress(value):
    print ('Progress: %.0f %%' % (100*value))

if __name__ == '__main__':
    #
    # test driver
    Digitizer = AcqirisDigitizer()
    Digitizer.init('PCI::INSTR0')
    # trigger
    Digitizer.configTrigSource(-1, 0, 0, 300.)
    Digitizer.configTrigClass(0x80000000)
    print('Trig', Digitizer.getTrigClass())
    Digitizer.configHorizontal(1E-9, 0.0)
#    Digitizer.configTrigSource(2, 0, 1, 0.05)
#    Digitizer.configTrigClass(0x00000002L)
    # test, single trace
#    Digitizer.configMode(0, 0, 0)
#    vRes = Digitizer.readChannelsToNumpy(nSample=1000, lChannel=[1], bAverageMode=False)[0]
#    print (vRes)
    print ('start acquire')

    ((vCh1, vCh2), dt) = Digitizer.getRoundRobinData(2048, 26, 1000,
                                             funcProgress=reportProgress)

#    # test, average
#    Digitizer.configMode(2, 0, 0)
#    Digitizer.configAveraging(1, NbrSamples=1024, NbrSegments=1,
#                         NbrWaveforms=100, StartDelay=0, StopDelay=0)
#    vRes2 = Digitizer.readChannelsToNumpy(nSample=1000, lChannel=[1], bAverageMode=True)[0]

    # close digitizer
    Digitizer.close()
    Digitizer.closeAll()

    # plot data
    import matplotlib.pyplot as plt
    fig, ax = plt.subplots(1, 1)
#    ax.plot(vRes[0], 'r-', label='raw')
#    ax.plot(vAll[:1E4], 'k-', label='averaged')
    ax.plot(vCh1.flatten(), 'k-', label='averaged')
    plt.show()

max_stars_repo_name: roniwinik/Drivers

max_issues_repo_name: roniwinik/Drivers

max_forks_repo_name: roniwinik/Drivers


text:
from os import path
from random import shuffle

import json
import os
import numpy as np

def present(workdir, file):
    if path.exists(path.join(workdir, file)):
        print(" ", "[+] %s exists" % file)
    else:
        raise Exception("[-] %s does not exist" % file)

def write_sample(fp, sample, sample_dict):
    fp.write('\n')
    fp.write(sample)
    fp.write(',')
    fp.write(','.join(sample_dict[sample]))

def features_to_dict(features: dict) -> dict:
    result = dict()
    for feature in features:
        result[feature] = features[feature].tolist()
    return result

def features_from_dict(input: dict) -> dict:
    result = dict()
    for feature in input:
        result[feature] = np.array(input[feature])
    return result

class Scaler:
    def __init__(self):
        self.mean = {}
        self.stddev = {}
        self.count = 0

    def fit(self, features: dict) -> dict:
        result = {}
        for feature_id in features:
            result[feature_id] = self.fit_subspace(features[feature_id], feature_id)
        return result

    def fit_subspace(self, features, subspace):
        return (np.array(features) - self.mean[subspace]) / self.stddev[subspace]

    def to_dict(self):
        result = dict()
        result["mean"] = features_to_dict(self.mean)
        result["stddev"] = features_to_dict(self.stddev)
        result["count"] = self.count
        return result

    @staticmethod
    def from_dict(input: dict):
        result = Scaler()
        result.count = input["count"]
        result.mean = features_from_dict(input["mean"])
        result.stddev = features_from_dict(input["stddev"])
        return result

class ScalerBuilder:
    def __init__(self):
        self.features = {}

    def add_features(self, features: dict) -> None:
        for feature_id in features:
            sample = np.array(features[feature_id])
            if feature_id not in self.features:
                self.features[feature_id] = []
            self.features[feature_id].append(sample)

    def bake(self) -> Scaler:
        scaler = Scaler()
        for feature_id in self.features:
            samples = self.features[feature_id]
            means = np.zeros(len(samples[0]))
            stddevs = np.zeros(len(samples[0]))
            scaler.count = len(samples)

            for sample in samples:
                means += sample
            means /= scaler.count
            scaler.mean[feature_id] = means

            for sample in samples:
                stddevs += (sample - means)**2
            stddevs /= scaler.count
            stddevs = np.sqrt(stddevs)

            stddevs[stddevs == 0.0] = 1.0
            scaler.stddev[feature_id] = stddevs

        return scaler

def write_feature_sample(fp, sample, not_first: set, scaler: Scaler, balance=False):
    negatives = 0
    for i, features in enumerate(sample["features"]):
        correct = sample["candidates"][i]["profile"]["screenName"].casefold() == sample["entry"]["twitterId"].casefold()
        if not correct:
            negatives += 1
            if negatives > 5: # and balance:
                continue
        if fp in not_first:
            fp.write('\n')
        else:
            not_first.add(fp)

        if correct:
            fp.write("1")
        else:
            fp.write("0")

        fp.write('\t')
        fp.write(json.dumps(features_to_dict(scaler.fit(features))))

def load_dataset_and_produce_folds(workdir: str, num_folds: int):
    samples = []
    sample_dict = {}
    header = None
    with open(path.join(workdir, "gold.csv"), 'r') as reader:
        for line in reader:
            line = line.rstrip()
            if header is None:
                header = line
                continue
            line = line.split(",")
            entity = line[0]
            samples.append(entity)
            sample_dict[entity] = line[1:]
    print("Loaded %d samples" % len(samples))
    shuffle(samples)

    folds = [set() for _ in range(num_folds)]
    for i, sample in enumerate(samples):
        fold = i % num_folds
        folds[fold].add(sample)
    print("Generated folds with amounts: [%s]" % ", ".join([str(len(fold)) for fold in folds]))

    splits_dir = path.join(workdir, "splits")
    os.mkdir(splits_dir)

    test_writers = []
    train_writers = []
    for i, fold in enumerate(folds):
        split_dir = path.join(splits_dir, str(i))
        os.mkdir(split_dir)
        writer = open(path.join(split_dir, "test.csv"), 'w')
        writer.write(header)
        test_writers.append(writer)
        writer = open(path.join(split_dir, "train.csv"), 'w')
        writer.write(header)
        train_writers.append(writer)
    for i, fold in enumerate(folds):
        for sample in fold:
            for j in range(len(folds)):
                if i == j:
                    write_sample(test_writers[j], sample, sample_dict)
                else:
                    write_sample(train_writers[j], sample, sample_dict)
    [writer.close() for writer in test_writers+train_writers]

    print("Making the first pass on a dataset to collect scaler statistics")
    scaler_builders = []
    for i in range(len(folds)):
        scaler_builders.append(ScalerBuilder())
    counter = 0
    with open(path.join(workdir, "dataset.json"), 'r') as reader:
        for line in reader:
            sample = json.loads(line)
            for i in range(len(folds)):
                if not sample["entry"]["resourceId"] in folds[i]:
                    for features in sample["features"]:
                        scaler_builders[i].add_features(features)
            counter += 1
            if counter % 5000 == 0:
                print("  Done %d samples" % counter)

    print("Generating scalers")
    scalers = []
    for i in range(len(folds)):
        scaler = scaler_builders[i].bake()
        split_dir = path.join(splits_dir, str(i))
        json.dump(scaler.to_dict(), open(path.join(split_dir, "scaler.json"), 'w'))
        scalers.append(scaler)
        print("  Done scaler %d" % i)
    del scaler_builders

    print("Dumping train and test sets")
    test_writers = []
    train_writers = []
    for i, fold in enumerate(folds):
        split_dir = path.join(splits_dir, str(i))
        test_writers.append(open(path.join(split_dir, "test.json"), 'w'))
        train_writers.append(open(path.join(split_dir, "train.json"), 'w'))
    not_first = set()
    counter = 0
    with open(path.join(workdir, "dataset.json"), 'r') as reader:
        for line in reader:
            sample = json.loads(line)
            for i in range(len(folds)):
                if sample["entry"]["resourceId"] in folds[i]:
                    write_feature_sample(test_writers[i], sample, not_first, scalers[i])
                else:
                    write_feature_sample(train_writers[i], sample, not_first, scalers[i], True)
            counter += 1
            if counter % 5000 == 0:
                print("  Done %d samples" % counter)
    [writer.close() for writer in test_writers+train_writers]

max_stars_repo_name: Remper/alignments

max_issues_repo_name: Remper/alignments

max_forks_repo_name: Remper/alignments


text:
import pytest
import numpy as np
from pg import DB
from odinapi.views.level1b_scandata_exporter_v2 import (
    ScandataExporter,
    smr_lofreqcorr,
    unsplit_normalmode,
    apply_calibration_step2,
)
from odinapi.views.smr_frequency import (
    Smrl1bFreqspec,
    Smrl1bFreqsort,
    get_bad_ssb_modules,
)
from odinapi.views.freq_calibration import (
    Freqcorr572,
    fit_scan_median_spectrum,
    identify_species_by_lines
)

class DatabaseConnector(DB):
    def __init__(self, host, port):
        DB.__init__(self, dbname='odin', user='odinop', host=host, port=port)

@pytest.fixture
def scan_sample_data(odin_postgresql):
    backend = 'AC2'
    freqmode = 22
    scanid = 7016113563
    db_connection = DatabaseConnector(*odin_postgresql)
    scan_data_exporter = ScandataExporter(
        backend, db_connection)
    scan_data_exporter.get_db_data(
        freqmode, scanid)
    scan_data_exporter.decode_specdata()
    assert len(scan_data_exporter.spectra['stw']) == 94
    scan_data_exporter = apply_calibration_step2(
        db_connection, scan_data_exporter)
    scan_data_exporter = unsplit_normalmode(
        scan_data_exporter)
    assert len(scan_data_exporter.spectra['stw']) == 47
    smr_lofreqcorr(scan_data_exporter)
    smr_freq_spec = Smrl1bFreqspec()
    smr_freq_sort = Smrl1bFreqsort()
    frequency_grid = smr_freq_spec.get_frequency(
        scan_data_exporter.spectra, 0)
    spectra = np.array(
        scan_data_exporter.spectra['spectrum'])
    bad_modules = get_bad_ssb_modules(
        scan_data_exporter.spectra['backend'][0],
        spectra,
        frequency_grid,
        False)
    frequency_grid = frequency_grid.flatten()
    frequency_grid, _, _, channels_id = (
        smr_freq_sort.get_sorted_ac_spectrum(
            frequency_grid,
            spectra[0],
            bad_modules))
    return {
        'frequency_grids': np.tile(
            frequency_grid, (spectra.shape[0] - 2, 1)
        ),
        'spectra': spectra[2::, channels_id],
        'altitudes':
            scan_data_exporter.spectra['altitude'][2::]}

@pytest.fixture
def freq_corr_572(scan_sample_data):
    return Freqcorr572(
        scan_sample_data['frequency_grids'],
        scan_sample_data['spectra'],
        scan_sample_data['altitudes'])

class TestFreqcorr572:

    def test_altitude_check(self, odinapi_service, freq_corr_572):
        assert freq_corr_572.altitude_check() == 1

    def test_get_initial_fit_values(self, odinapi_service, freq_corr_572):
        [initial_guess_is_ok, f_initial, tb_initial] = \
            freq_corr_572.get_initial_fit_values()
        assert np.all([
            initial_guess_is_ok,
            f_initial,
            tb_initial] == [
                1,
                pytest.approx(576.5938, abs=1e-3),
                pytest.approx(82.1877, abs=1e-3)])

    def test_fit_data(self, odinapi_service, freq_corr_572):
        freq_corr_572.get_initial_fit_values()
        [fit_is_ok, freq1, freq2, _, _] = fit_scan_median_spectrum(
            576.5938,
            82.1877,
            freq_corr_572.frequency_grid,
            freq_corr_572.median_tb
        )
        assert np.all([
            fit_is_ok,
            freq1,
            freq2] == [
                True,
                pytest.approx(576.5938, abs=1e-3),
                pytest.approx(576.840, abs=1e-3)])

    def test_get_tb_profile(self, odinapi_service, freq_corr_572):
        tb_profiles = freq_corr_572.get_tb_profile(
            576.593, 576.840)
        assert tb_profiles.shape == (45, 2)
        assert np.all([
            tb_profiles[0, 0],
            tb_profiles[44, 0],
            tb_profiles[0, 1],
            tb_profiles[44, 1]] == [
                pytest.approx(-2.2, abs=1e-1),
                pytest.approx(198.3, abs=1e-1),
                pytest.approx(-7.0, abs=1e-1),
                pytest.approx(195.6, abs=1e-1)])

    def test_identify_species_by_profile(self, odinapi_service, freq_corr_572):
        tb_profile = freq_corr_572.get_tb_profile(
            576.593, 576.840)
        co_found = freq_corr_572.identify_species_by_profile(
            tb_profile)
        assert co_found

    def test_identify_species_by_lines(self, odinapi_service, freq_corr_572):
        freq_corr_572.get_initial_fit_values()
        [fit_is_ok, _, _, amplitude_1, amplitude_2] = (
            fit_scan_median_spectrum(
                82.1877,
                576.5938,
                freq_corr_572.frequency_grid,
                freq_corr_572.median_tb
            )
        )
        co_found = identify_species_by_lines(
            amplitude_1, amplitude_2)
        assert co_found

    def test_get_frequency_offset_of_scan(
        self, odinapi_service, freq_corr_572,
    ):
        (scan_correction_is_ok, frequency_offset) = (
            freq_corr_572.get_frequency_offset_of_scan()
        )
        assert np.all([
            scan_correction_is_ok,
            frequency_offset] == [
                True,
                pytest.approx(0.3254, abs=1e-4)])

    def test_run_frequency_correction(self, odinapi_service, freq_corr_572):
        (
            scan_correction_is_ok,
            single_spectrum_correction_is_ok,
            frequency_offset_of_scan,
            frequencies_offset_per_spectrum
        ) = freq_corr_572.run_frequency_correction()
        assert np.all([
            scan_correction_is_ok,
            frequency_offset_of_scan,
            ] == [
                True,
                pytest.approx(0.3254, abs=1e-4)])
        assert np.all(single_spectrum_correction_is_ok)
        assert np.max(
            np.abs(frequencies_offset_per_spectrum)) < 5e-3

max_stars_repo_name: Odin-SMR/odin-api

max_issues_repo_name: Odin-SMR/odin-api

max_forks_repo_name: Odin-SMR/odin-api


text:
import json
from datetime import datetime

import jsonschema
import mock
import numpy
import pytest
import utils

from aa import js

EARLY_DATE = datetime(2001, 1, 1, 1, 1)
LATE_DATE = datetime(2010, 2, 3, 4, 5)
EMPTY_ARRAY = numpy.array((0,))

@pytest.fixture
def json_fetcher():
    return js.JsonFetcher("localhost", 5000)

@pytest.mark.parametrize("filename", ["event", "string_event", "waveform"])
def test_json_matches_schema(filename):
    schema_string = utils.load_from_file("aa_schema.json")
    schema_obj = json.loads(schema_string)
    json_string = utils.load_from_file(filename + ".json")
    json_obj = json.loads(json_string)
    jsonschema.validate(json_obj, schema_obj)

def test_JsonFetcher_constructs_url_correctly(json_fetcher):
    assert json_fetcher._url == "http://localhost:5000/retrieval/data/getData.json"

def test_JsonFetcher_decodes_empty_json_correctly(dummy_pv, empty_data, json_fetcher):
    mock_response = utils.mock_response(json_str="[]")
    json_fetcher._fetch_data = mock_response
    aa_data = json_fetcher.get_values(dummy_pv, EARLY_DATE, LATE_DATE)
    assert aa_data == empty_data

def test_JsonFetcher_decodes_single_event_correctly(dummy_pv, json_fetcher):
    event_json = utils.load_from_file("event.json")
    mock_response = utils.mock_response(json_str=event_json)
    json_fetcher._fetch_data = mock.MagicMock(return_value=mock_response)
    aa_data = json_fetcher.get_values(dummy_pv, EARLY_DATE, LATE_DATE)
    assert aa_data.pv == dummy_pv
    assert aa_data.values[0] == 1.23
    assert aa_data.timestamps[0] == 1502963093.000000123
    assert aa_data.severities[0] == 1

def test_JsonFetcher_decodes_string_event_correctly(dummy_pv, json_fetcher):
    event_json = utils.load_from_file("string_event.json")
    mock_response = utils.mock_response(json_str=event_json)
    json_fetcher._fetch_data = mock.MagicMock(return_value=mock_response)
    aa_data = json_fetcher.get_values(dummy_pv, EARLY_DATE, LATE_DATE)
    assert aa_data.pv == dummy_pv
    assert aa_data.values[0] == "2015-01-08 19:47:01 UTC"
    assert aa_data.timestamps[0] == 1507712433.235971000
    assert aa_data.severities[0] == 0

def test_JsonFetcher_decodes_waveform_events_correctly(
    dummy_pv, json_fetcher, data_2d_2_events
):
    waveform_json = utils.load_from_file("waveform.json")
    mock_response = utils.mock_response(json_str=waveform_json)
    json_fetcher._fetch_data = mock.MagicMock(return_value=mock_response)
    aa_data = json_fetcher.get_values(dummy_pv, EARLY_DATE, LATE_DATE)
    assert aa_data == data_2d_2_events

def test_JsonFetcher_decodes_enum_events_correctly(dummy_pv, json_fetcher):
    enum_json = utils.load_from_file("enum_event.json")
    mock_response = utils.mock_response(json_str=enum_json)
    json_fetcher._fetch_data = mock.MagicMock(return_value=mock_response)
    aa_data = json_fetcher.get_values(dummy_pv, EARLY_DATE, LATE_DATE)
    assert aa_data.enum_options[5] == "Special"
    assert aa_data.enum_strings[0] == "User"
    assert aa_data[0].enum_string[0] == "User"
    assert aa_data.values[0] == 4

max_stars_repo_name: philipp-leitl/aapy

max_issues_repo_name: philipp-leitl/aapy

max_forks_repo_name: philipp-leitl/aapy


text:
# Tests that the environment works correctly

import matplotlib
matplotlib.use('TkAgg')
import sklearn
import pandas as pd
import scipy
import numpy as np
import ete3
import graphviz
import skopt

from sklearn import tree
from sklearn.tree import DecisionTreeClassifier
from graphviz import Source
from xgboost import XGBClassifier
from skopt import BayesSearchCV

# Auxiliary functions
def plottree(decisiontree, features_names=None, class_names=None):
    """Returns a graphviz visualization of a scikit-learn decision tree

    Inputs
        - decisiontree: tree to visualize
        - feature_names: iterable with the names of the features present in the data.
        - class_names: iterable with the names of the classes present in the data, in increasing order.

    If the call to this function is the last line of a notebook cell, the tree is rendered automatically.
    """
    dot_data = tree.export_graphviz(
        decisiontree,
        out_file=None,
        filled=True,
        rounded=True,
        rotate=True,
        feature_names=features_names,
        class_names=class_names
    )
    return Source(dot_data)

def getdata():
    X = np.array([[0, 0], [1, 1]])
    y = [0, 1]
    return X, y

def buildtree(X, y):
    clf = DecisionTreeClassifier()
    clf.fit(X, y)
    return clf

# Tests

def test_tree():
    """Trains a Tree"""
    X, y = getdata()
    clf = buildtree(X, y)
    clf.predict([[2., 2.]])

def plot_tree():
    """Plots a Tree"""
    X, y = getdata()
    clf = buildtree(X, y)
    plottree(clf)

def test_xgb():
    """Trains an XGB model"""
    X, y = getdata()
    clf = XGBClassifier()
    clf.fit(X, y)
    clf.predict([[2., 2.]])

max_stars_repo_name: cperales/teaching-environments

max_issues_repo_name: cperales/teaching-environments

max_forks_repo_name: cperales/teaching-environments


text:
# dubins_parameters
#   - Dubins parameters that define path between two configurations
#
# mavsim_matlab
#     - Beard & McLain, PUP, 2012
#     - Update history:
#         3/26/2019 - RWB

import numpy as np
import sys
sys.path.append('..')

class dubins_parameters:
    def __init__(self):
        self.p_s = np.inf*np.ones((3,1))  # the start position in re^3
        self.chi_s = np.inf  # the start course angle
        self.p_e = np.inf*np.ones((3,1))  # the end position in re^3
        self.chi_e = np.inf  # the end course angle
        self.radius = np.inf  # turn radius
        self.length = np.inf  # length of the Dubins path
        self.center_s = np.inf*np.ones((3,1))  # center of the start circle
        self.dir_s = np.inf  # direction of the start circle
        self.center_e = np.inf*np.ones((3,1))  # center of the end circle
        self.dir_e = np.inf  # direction of the end circle
        self.r1 = np.inf*np.ones((3,1))  # vector in re^3 defining half plane H1
        self.r2 = np.inf*np.ones((3,1))  # vector in re^3 defining position of half plane H2
        self.r3 = np.inf*np.ones((3,1))  # vector in re^3 defining position of half plane H3
        self.n1 = np.inf*np.ones((3,1))  # unit vector in re^3 along straight line path
        self.n3 = np.inf*np.ones((3,1))  # unit vector defining direction of half plane H3

    def update(self, ps, chis, pe, chie, R):
        ell = np.linalg.norm(ps - pe)
        if ell < 2 * R:
            print('Error in Dubins Parameters: The distance between nodes must be larger than 2R.')
        else:

            self.p_s =
            self.chi_s =
            self.p_e =
            self.chi_e =
            self.radius =
            self.length =
            self.center_s =
            self.dir_s =
            self.center_e =
            self.dir_e =
            self.r1 =
            self.n1 =
            self.r2 =
            self.r3 =
            self.n3 =

def rotz(theta):
    return np.array([[np.cos(theta), -np.sin(theta), 0],
                    [np.sin(theta), np.cos(theta), 0],
                    [0, 0, 1]])

def mod(x):
    # make x between 0 and 2*pi
    while x < 0:
        x += 2*np.pi
    while x > 2*np.pi:
        x -= 2*np.pi
    return x