text:
# Make logratio matrix.

# USAGE:

# python3 make_logratio_matrix.py vocablength metadata_file name_of_outfile pathtodata1 pathtodata2 etc

# this is written to produce
# a matrix of Laplace-smoothed log ratios; i.e.,
# log( (word_uses_for_women + 1 / all_words_for_women) /
#      (word_uses_for_men + 1 / all_words_for_men)
#    )

# It is also written to ensure that we have the same number of characters
# created by women and created by men in each year, and the same number
# of characters-per-year across the timeline.

# In order to do this it requires a metadata_file listing
# the HTvolume ID, inferred publication date, and author gender
# for each volume.

# Example of usage:
# make_logratio_matrix2.py 6000 null new_bio_logratio.csv ../natalie/out_files/all_pre23bio_new.tsv ../natalie/out_files/all_post23bio_Sep11.tsv

VOCABLENGTH = 6000
N = 3000

import csv, sys, os, random
import numpy as np
import pandas as pd
import math
# from random import shuffle
# from random import random as randomprob
from sklearn.linear_model import LinearRegression
from collections import Counter

csv.field_size_limit(sys.maxsize)

arguments = sys.argv

if len(arguments) < 5:
    print('This module requires command-line args to run. In order:')
    print(' vocablength: integer size of vocabulary retained')
    print(' metadata_file: csv listing volID, pubdate, and authgender')
    print(' outfile: filename to write')
    print(' datafiles: all subsequent args will be interpreted as sources')
    sys.exit(0)

VOCABLENGTH = int(arguments[1])

metadata_path = arguments[2]
if not os.path.isfile(metadata_path):
    print('I cannot find ' + metadata_path)
    print('This will be okay if the data itself contains authgender.')
else:
    metadata = pd.read_csv(metadata_path)

out_filename = arguments[3]
if os.path.isfile(out_filename + '.csv'):
    print(out_filename + ' already exists, and I refuse to overwrite it.')
    sys.exit(0)

files_to_use = []
for f in arguments[4:]:
    if os.path.isfile(f):
        files_to_use.append(f)
    else:
        print("Cannot find " + f)

# We need a lexicon of personal names, to keep those out of our
# vocabulary. Obviously, they'll be gendered!

personalnames = set()
with open("../lexicons/PersonalNames.txt", encoding="utf-8") as f:
    names = f.readlines()

for line in names:
    name = line.rstrip()
    personalnames.add(name)
    personalnames.add('said-' + name)

# Okay, here's the core logic of this module. We need to do five things:
# 1) Create a grid of all characters by authgender and publication date
# 2) Select (with replacement!) an equal number of characters from each
#    cell of that grid.
# 3) Make a vocabulary based on the most common words among those characters.
# 4) Sum up vocab words from actually-selected characters.
# 5) Calculate logratios between usage by masculine and feminine chargenders.

# This sequence is fastest/easiest if we do everything in memory, and I *think*
# that's possible.

def initialize_grid():
    grid = dict()
    for i in range (1780, 2008):
        grid[i] = dict()
        for g in ['f', 'm']:
            grid[i][g] = []

    return grid

def fill_grid(filepath, grid, wordsdict, genderdict):
    '''
    We build three underlying data structures.
    1) A dictionary of character ids by year and authgender,
    2) a dictionary where character-id keys point to wordlists.
    3) a dictionary where character-id keys point to chargenders.
    '''

    with open(filepath, encoding = 'utf-8') as f:
        reader = csv.DictReader(f, delimiter = '\t')
        for row in reader:
            if 'chargender' in row:
                chargender = row['chargender']
            else:
                chargender = row['gender']

            if chargender.startswith('u'):
                continue
                # for purposes of this calculation we are
                # excluding characters of unknown gender

            if 'authgender' in row:
                authgender = row['authgender']
            else:
                docid = row['docid']
                if docid in metadata.index:
                    authgender = metadata.loc[docid, 'authgender']
                else:
                    continue

            if authgender != 'm' and authgender != 'f':
                continue

            pubdate = int(row['pubdate'])
            if pubdate < 1780 or pubdate > 2007:
                continue
            charid = row['charid']

            grid[pubdate][authgender].append(charid)
            genderdict[charid] = chargender
            wordsdict[charid] = row['words'].split()

    # no return values are needed, because the parameters
    # are mutable and have been mutated!

def select_chars(grid, n):
    selected = initialize_grid()

    for y in range (1780, 2008):
        for g in ['f', 'm']:
            options = grid[y][g]
            if len(options) < 1:
                continue

            for n1 in range(n):
                selected[y][g].append(random.choice(options))

    return selected

def make_vocab(selected, wordsdict, vocab):
    for y in range (1780, 2008):
        for g in ['f', 'm']:
            for c in selected[y][g]:
                words = wordsdict[c]
                for w in words:
                    if not w.startswith('said-') and w not in personalnames:
                        vocab[w] += 1

# Let's make the grid.

grid = initialize_grid()
wordsdict = dict()
genderdict = dict()

for f in files_to_use:
    fill_grid(f, grid, wordsdict, genderdict)

print('Characters read from file.')

selected = select_chars(grid, N)

print('Characters selected.')

# Let's create the vocabulary.

vocab = Counter()

make_vocab(selected, wordsdict, vocab)

vocabcount = len(vocab)
print("The data includes " + str(vocabcount) + " words")

wordsinorder = [x[0] for x in vocab.most_common(VOCABLENGTH)]

vocabulary = dict()
vocabset = set()

for idx, word in enumerate(wordsinorder):
    vocabulary[word] = idx
    vocabset.add(word)
print("Vocabulary sorted, top " + str(VOCABLENGTH) + " kept.")

vecbyyear = dict()
vecbyyear['m'] = dict()
vecbyyear['f'] = dict()
datevector = list(range(1780, 2008))

for g in ['f', 'm']:
    for i in range(1780, 2008):
        vecbyyear[g][i] = np.zeros((VOCABLENGTH))

# Now we actually count words

for y in range (1780, 2008):
    for g in ['f', 'm']:
        for c in selected[y][g]:
            words = wordsdict[c]
            for w in words:
                if w in vocabset:
                    idx = vocabulary[w]
                    np.add.at(vecbyyear[g][y], idx, 1)

def dunnings(vectora, vectorb):
    assert len(vectora) == len(vectorb)
    veclen = len(vectora)
    totala = np.sum(vectora)
    totalb = np.sum(vectorb)
    totalboth = totala + totalb

    dunningvector = np.zeros(veclen)

    for i in range(veclen):
        if vectora[i] == 0 or vectorb[i] == 0:
            continue
            # Cause you know you're going to get div0 errors.

        try:
            probI = (vectora[i] + vectorb[i]) / totalboth
            probnotI = 1 - probI
            expectedIA = totala * probI
            expectedIB = totalb * probI
            expectedNotIA = totala * probnotI
            expectedNotIB = totalb * probnotI
            expected_table = np.array([[expectedIA, expectedNotIA],
                [expectedIB, expectedNotIB]])
            actual_table = np.array([[vectora[i], (totala - vectora[i])],
                [vectorb[i], (totalb - vectorb[i])]])
            G = np.sum(actual_table * np.log(actual_table / expected_table))

            # We're going to use a signed version of Dunnings, so features where
            # B is higher than expected will be negative.

            if expectedIB > vectorb[i]:
                G = -G

            dunningvector[i] = G

        except:
            pass
            # There are a million ways to get a div-by-zero or log-zero error
            # in that calculation. I could check them all, or just do this.
            # The vector was initialized with zeroes, which are the default
            # value I want for failed calculations anyhow.

    return dunningvector

def pure_rank_matrix(femalevectorsbyyear, malevectorsbyyear, datevector):
    rankmatrix = []
    magnitudematrix = []

    for i in datevector:
        d = dunnings(femalevectorsbyyear[i], malevectorsbyyear[i])

        # transform this into a nonparametric ranking
        decorated = [x for x in zip(d, [x for x in range(len(d))])]
        decorated.sort()

        negativeidx = -sum(d < 0)
        positiveidx = 1

        numzeros = sum(d == 0)

        ranking = np.zeros(len(d))
        for dvalue, index in decorated:
            # to understand what follows, it's crucial to remember that
            # we're iterating through decorated in dvalue order

            if dvalue < 0:
                ranking[index] = negativeidx
                negativeidx += 1
            elif dvalue > 0:
                ranking[index] = positiveidx
                positiveidx += 1
            else:
                # dvalue is zero
                pass

        checkzeros = sum(ranking == 0)
        if numzeros != checkzeros:
            print('error in number of zeros')

        rawmagnitude = femalevectorsbyyear[i] + malevectorsbyyear[i]
        normalizedmagnitude = rawmagnitude / np.sum(rawmagnitude)
        assert len(ranking) == len(normalizedmagnitude)
        rank_adjusted_by_magnitude = ranking * normalizedmagnitude

        rankmatrix.append(ranking)
        magnitudematrix.append(rank_adjusted_by_magnitude)

    return np.array(magnitudematrix), np.array(rankmatrix)

def diff_proportion(vecbyyear, datevector):
    diffmatrix = []

    for yr in datevector:
        if np.sum(vecbyyear['m'][yr]) == 0:
            mvec = np.full(len(vecbyyear['m'][yr]), 0)
        else:
            mvec = (vecbyyear['m'][yr] * 5000) / np.sum(vecbyyear['m'][yr])

        if np.sum(vecbyyear['f'][yr]) == 0:
            fvec = np.full(len(vecbyyear['f'][yr]), 0)
        else:
            fvec = (vecbyyear['f'][yr] * 5000) / np.sum(vecbyyear['f'][yr])

        dvec = fvec - mvec
        diffmatrix.append(dvec)

    return np.array(diffmatrix)

def log_ratios(vecbyyear, datevector):

    diffmatrix = []

    for yr in datevector:

        # We add one to all the word counts;
        # let's call that Laplacian smoothing.
        # It has the effect of getting rid of
        # nan values, in a way that I think is relatively
        # principled: it will tend to move
        # infrequent words toward the origin.

        vecbyyear['m'][yr] = vecbyyear['m'][yr] + 1
        mvec = vecbyyear['m'][yr] / np.sum(vecbyyear['m'][yr])

        vecbyyear['f'][yr] = vecbyyear['f'][yr] + 1
        fvec = vecbyyear['f'][yr] / np.sum(vecbyyear['f'][yr])

        ratios = fvec / mvec

        log_ratios = np.log(ratios)

        diffmatrix.append(log_ratios)

    return np.array(diffmatrix)

diffmatrix = log_ratios(vecbyyear, datevector)

def writematrix(amatrix, outpath):
    global wordsinorder, datevector

    with open(outpath, mode = 'w', encoding = 'utf-8') as f:
        writer = csv.writer(f)
        writer.writerow(['thedate'] + wordsinorder)
        for i, date in enumerate(datevector):
            writer.writerow(np.insert(amatrix[i, : ], 0, date))

writematrix(diffmatrix, '../data/' + out_filename + '.csv')

print('Linear regression to infer slopes.')
datevector = np.array(datevector)
outrows = []
for i in range(VOCABLENGTH):
    thiscolumn = diffmatrix[ : , [i]]
    # note: the brackets around i extract it as a *column* rather than row

    notmissing = thiscolumn != 0
    # still a column

    y = thiscolumn[notmissing].transpose()

    # that's a cheap hack to create an array w/ more than one column,
    # which the linear regression seems to want

    x = datevector[notmissing.transpose()[0]]
    # We have to transpose the column "notmissing" to index a row.

    x = x[ : , np.newaxis]
    # Then we have to make x a row of an array with two
    # dimensions (even though it only has one row).

    vectorlen = len(x)

    word = wordsinorder[i]

    model = LinearRegression()

    model.fit(x, y)
    slope = model.coef_[0]
    intercept = model.intercept_
    standard_deviation = np.std(y)

    nineteenth = np.mean(thiscolumn[0:120])
    twentieth =np.mean(thiscolumn[120:])
    change = twentieth - nineteenth

    approachmid = abs(np.nanmean(thiscolumn[0:60])) - abs(np.nanmean(thiscolumn[150:210]))
    approachstd = approachmid / standard_deviation

    # note that it's important we use thiscolumn rather than y here, because y has been reduced
    # in length

    out = dict()
    out['word'] = word
    out['slope'] = slope
    out['mean'] = np.mean(thiscolumn)
    out['intercept'] = intercept
    out['change'] = change
    out['approachmid'] = approachmid
    out['approachstd'] = approachstd
    outrows.append(out)

with open('../data/' + out_filename + '.slopes.csv', mode = 'w', encoding = 'utf-8') as f:
    writer = csv.DictWriter(f, fieldnames = ['word', 'slope', 'mean', 'intercept', 'change', 'approachmid', 'approachstd'])
    writer.writeheader()
    for row in outrows:
        writer.writerow(row)

with open('../data/' + out_filename + '.gendersums.tsv', mode = 'w', encoding = 'utf-8') as f:
    f.write('year\twomenbywomen\tmenbywomen\twomenbymen\tmenbymen\n')
    for y in range (1780, 2008):
        chargenders = dict()
        for authorgender in ['m', 'f']:
            chargenders[authorgender] = dict()
            chargenders[authorgender]['m'] = 0
            chargenders[authorgender]['f'] = 0
            for c in selected[y][authorgender]:
                chargen = genderdict[c]
                chargenders[authorgender][chargen] += 1

        outline = '\t'.join([str(x) for x in [y, chargenders['f']['f'], chargenders['f']['m'], chargenders['m']['f'], chargenders['m']['m']]])
        f.write(outline + '\n')

max_stars_repo_name: tedunderwood/biographies

max_issues_repo_name: afcarl/biographies

max_forks_repo_name: afcarl/biographies


text:
# import necessary libraries

import numpy as np
import pandas as pd # data processing
import re # for regex
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize # returns the syllables from a single word
from nltk.stem import SnowballStemmer # it is a stemming algorithm
from sklearn.feature_extraction.text import CountVectorizer #  transform a given text into a vector on the basis of the frequency (count) of each word that occurs in the entire text
from sklearn.model_selection import train_test_split # split the dataset into train and test data set
from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB
from sklearn.metrics import accuracy_score
import pickle

# load the dataset
data = pd.read_csv('Artificial-Intelligence/Sentiment Analysis/Movie review sentiment analysis/data/IMDB-Dataset.csv')
print(data.shape)
data.head()
data.info()

""" [OUTPUT]
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 50000 entries, 0 to 49999
 #   Column     Non-Null Count  Dtype
---  ------     --------------  -----
 0   review     50000 non-null  object
 1   sentiment  50000 non-null  object
dtypes: object(2)
memory usage: 781.4+ KB
"""

# no null values in the dataset
data.sentiment.value_counts()

"""[OUTPUT]
positive    25000
negative    25000
Name: sentiment, dtype: int64
"""
# encode the sentiments to by mapping positive to 0 and negative to 1
data.sentiment.replace('positive',1, inplace = True)
data.sentiment.replace('negative',0, inplace = True)
data.head(10)

# data.review[0]

# step 1: Remove the HTML Tags

def clean_tags(text):
  cleaned = re.compile(r'<.*?>')
  return re.sub(cleaned,'',text)

data.review = data.review.apply(clean_tags)
# data.review[0]

# step 2: Remove all the special charecters
def clean_special(text):
  rem = ''
  for i in text:
    if i.isalnum():
      rem = rem + i
    else:
      rem = rem + ' '
  return rem

data.review = data.review.apply(clean_special)
# data.review[0]

# step 3: Converting to lowercases
def lower(text):
  return text.lower()

data_review = data.review.apply(lower)
# data_review[0]

# step 4: Remove stopwords

def clean_stopwords(text):
  stop_words = set(stopwords.words('english'))
  words = word_tokenize(text)
  return [w for w in words if w not in stop_words]

data.review = data.review.apply(clean_stopwords)
# data.review[0]

# step 5: Stem the words
def clean_stem(text):
  stem_snow = SnowballStemmer('english')
  return " ".join([stem_snow.stem(w) for w in text])

data.review = data.review.apply(clean_stem)
# data.review[0]
# data.head()

# creating The model
# creating a bag of words
X = np.array(data.iloc[:0].values)
y = np.array(data.sentiment.values)
cv = CountVectorizer(max_features=1000)
X = cv.fit_transform(data.review).toarray()
print("X shape =",X.shape)
print("Y shape =",y.shape)
# print(X)

# split the dataset in training and test dataset
trainx,testx,trainy,testy = train_test_split(X,y,test_size =0.2,random_state =9)
print("Train shapes : X = {}, y = {}".format(trainx.shape,trainy.shape))
print("Test shapes : X = {}, y = {}".format(testx.shape,testy.shape))

# Model defination and training
gb,mb,bb = GaussianNB(), MultinomialNB(alpha =1 , fit_prior=True ),BernoulliNB(alpha =1.0, fit_prior=True)
gb.fit(trainx, trainy)
mb.fit(trainx, trainy)
bb.fit(trainx, trainy)

pred_gb = gb.predict(testx)
pred_mb = mb.predict(testx)
pred_bb = bb.predict(testx)

print("Gaussian = ",accuracy_score(testy,pred_gb))
print("Multinomial = ",accuracy_score(testy,pred_mb))
print("Bernoulli = ",accuracy_score(testy,pred_bb))

"""[output]
Gaussian =  0.7817
Multinomial =  0.8299
Bernoulli =  0.8371
"""
# save the model
pickle.dump(bb, open('Movie_review_model.pkl','wb'))

word_dict = cv.vocabulary_
pickle.dump(word_dict,open('bow.pkl','wb'))

ext: py

max_stars_repo_path: Sentiment Analysis/Movie review sentiment analysis/main.py

max_stars_repo_name: Suhas-Prabhu/Artificial-Intelligence

max_issues_repo_name: Suhas-Prabhu/Artificial-Intelligence

max_forks_repo_name: Suhas-Prabhu/Artificial-Intelligence

reason: import numpy


text:
#-----------------------------------------------------------------------
# Skeleton 1-2/2D Darwin OpenMP PIC code
# written by Viktor K. Decyk, Adam Tableman, and Qiyang Hu, UCLA
import math
import numpy
from fmdpush1 import *
from dtimer import *
from fomplib import *

int_type = numpy.int32
double_type = numpy.float64
float_type = numpy.float32
complex_type = numpy.complex64

# indx = exponent which determines grid points in x direction:
# nx = 2**indx.
indx =   9
# npx = number of electrons distributed in x direction.
npx =  18432
# tend = time at end of simulation, in units of plasma frequency.
# dt = time interval between successive calculations.
# qme = charge on electron, in units of e.
tend = 10.0; dt = 0.1; qme = -1.0
# vtx/vty = thermal velocity of electrons in x/y direction
# vx0/vy0 = drift velocity of electrons in x/y direction.
vtx = 1.0; vty = 1.0; vx0 = 0.0; vy0 = 0.0
# vtx/vz0 = thermal/drift velocity of electrons in z direction
vtz = 1.0; vz0 = 0.0
# ax = smoothed particle size in x direction
# ci = reciprocal of velocity of light.
ax = .912871; ci = 0.1
# idimp = number of particle coordinates = 4
# ipbc = particle boundary condition: 1 = periodic
idimp = 4; ipbc = 1
# omx/omy/omz = magnetic field electron cyclotron frequency in x/y/z
omx = 0.4; omy = 0.0; omz = 0.0
# ndc = number of corrections in darwin iteration
ndc = 1
# wke/we = particle kinetic/electrostatic field energy
# wf/wm/wt = magnetic field/transverse electric field/total energy
wke = numpy.zeros((1),float_type)
we = numpy.zeros((1),float_type)
wf = numpy.zeros((1),float_type)
wm = numpy.zeros((1),float_type)
wt = numpy.zeros((1),float_type)
zero = 0.0
# mx = number of grids in x in sorting tiles
mx = 32
# xtras = fraction of extra particles needed for particle management
xtras = 0.2
# declare scalars for standard code
wpmax = numpy.empty((1),float_type)
wpmin = numpy.empty((1),float_type)

# declare scalars for OpenMP code
nppmx = numpy.empty((1),int_type)
irc = numpy.zeros((1),int_type)

# declare and initialize timing data
itime = numpy.empty((4),numpy.int32)
tdpost = 0.0; tguard = 0.0; tfft = 0.0; tfield = 0.0
tdjpost = 0.0; tdcjpost = 0.0; tpush = 0.0; tsort = 0.0
dtime = numpy.empty((1),double_type)

# nvp = number of shared memory nodes (0=default)
nvp = 0
#nvp = int(input("enter number of nodes: "))
# initialize for shared memory parallel processing
init_omp(nvp)

# initialize scalars for standard code
# np = total number of particles in simulation
# nx = number of grid points in x direction
np = npx; nx = int(math.pow(2,indx)); nxh = int(nx/2)
nxe = nx + 2; nxeh = int(nxe/2)
# mx1 = number of tiles in x direction
mx1 = int((nx - 1)/mx + 1)
# nloop = number of time steps in simulation
# ntime = current time step
nloop = int(tend/dt + .0001); ntime = 0
qbme = qme
affp = float(nx)/float(np)

# allocate data for standard code
# part = particle array
part = numpy.empty((idimp,np),float_type,'F')
# qe = electron charge density with guard cells
qe = numpy.empty((nxe),float_type,'F')
# fxe = smoothed longitudinal electric field with guard cells
fxe = numpy.empty((nxe),float_type,'F')
# cue = electron current density with guard cells
cue = numpy.empty((2,nxe),float_type,'F')
# dcu = acceleration density with guard cells
dcu = numpy.empty((2,nxe),float_type,'F')
# cus = transverse electric field with guard cells
cus = numpy.empty((2,nxe),float_type,'F')
# amu = momentum flux with guard cells
amu = numpy.empty((2,nxe),float_type,'F')
# exyze = smoothed total electric field with guard cells
exyze = numpy.empty((3,nxe),float_type,'F')
# byze = smoothed magnetic field with guard cells
byze = numpy.empty((2,nxe),float_type,'F')
# ffc, ffe = form factor arrays for poisson solvers
ffc = numpy.empty((nxh),complex_type,'F')
ffe = numpy.empty((nxh),complex_type,'F')
# mixup = bit reverse table for FFT
mixup = numpy.empty((nxh),int_type,'F')
# sct = sine/cosine table for FFT
sct = numpy.empty((nxh),complex_type,'F')
# kpic = number of particles in each tile
kpic = numpy.empty((mx1),int_type,'F')
# gxe, gyze = scratch arrays for fft
gxe = numpy.empty((nxe),float_type,'F')
gyze = numpy.empty((2,nxe),float_type,'F')

# prepare fft tables
wfft1rinit(mixup,sct,indx,nxh)
# calculate form factor: ffc
isign = 0
pois1(qe,fxe,isign,ffc,ax,affp,we,nx)
# initialize electrons
distr1h(part,vtx,vty,vtz,vx0,vy0,vz0,npx,idimp,np,nx,ipbc)

# find number of particles in each of mx, tiles: updates kpic, nppmx
dblkp1l(part,kpic,nppmx,idimp,np,mx,mx1,irc)
if (irc[0] != 0):
   print "dblkp1l error, irc=", irc[0]
   exit(0)
# allocate vector particle data
nppmx0 = int((1.0 + xtras)*nppmx)
ntmax = int(xtras*nppmx)
npbmx = int(xtras*nppmx)
# ppart = tiled particle array
ppart = numpy.empty((idimp,nppmx0,mx1),float_type,'F')
# ppbuff = buffer array for reordering tiled particle array
ppbuff = numpy.empty((idimp,npbmx,mx1),float_type,'F')
# ncl = number of particles departing tile in each direction
ncl = numpy.empty((2,mx1),int_type,'F')
# ihole = location/destination of each particle departing tile
ihole = numpy.empty((2,ntmax+1,mx1),int_type,'F')
# copy ordered particle data for OpenMP: updates ppart and kpic
ppmovin1l(part,ppart,kpic,nppmx0,idimp,np,mx,mx1,irc)
if (irc[0] != 0):
   print "ppmovin1l overflow error, irc=", irc[0]
   exit(0)
# sanity check
ppcheck1l(ppart,kpic,idimp,nppmx0,nx,mx,mx1,irc)
if (irc[0] != 0):
   print "ppcheck1l error, irc=", irc[0]
   exit(0)

# find maximum and minimum initial electron density
qe.fill(0.0)
gppost1l(ppart,qe,kpic,qme,nppmx0,idimp,mx,nxe,mx1)
aguard1l(qe,nx,nxe)
fwpminmx1(qe,qbme,wpmax,wpmin,nx,nxe)
wpm = 0.5*(wpmax[0] + wpmin[0])*affp
# accelerate convergence: update wpm
if (wpm <= 10.0):
   wpm = 0.75*wpm
print "wpm=",wpm
q2m0 = wpm/affp
# calculate form factor: ffe
isign = 0
epois13(dcu,cus,isign,ffe,ax,affp,wpm,ci,wf,nx,nxeh,nxh)

# initialize transverse electric field
cus.fill(0.0)

# * * * start main iteration loop * * *

for ntime in xrange(0,nloop):
#  print "ntime = ", ntime

# deposit current with OpenMP: updates cue
   dtimer(dtime,itime,-1)
   cue.fill(0.0)
   gjppost1l(ppart,cue,kpic,qme,zero,nppmx0,idimp,nx,mx,nxe,mx1,ipbc)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tdjpost = tdjpost + time

# deposit charge with OpenMP: updates qe
   dtimer(dtime,itime,-1)
   qe.fill(0.0)
   gppost1l(ppart,qe,kpic,qme,nppmx0,idimp,mx,nxe,mx1)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tdpost = tdpost + time

# add guard cells with standard procedure: updates qe, cue
   dtimer(dtime,itime,-1)
   aguard1l(qe,nx,nxe)
   acguard1l(cue,nx,nxe)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tguard = tguard + time

# transform charge to fourier space with standard procedure:
# updates qe, gxe
   dtimer(dtime,itime,-1)
   isign = -1
   fft1rxx(qe,gxe,isign,mixup,sct,indx,nxe,nxh)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfft = tfft + time

# calculate longitudinal force/charge in fourier space with standard
# procedure: updates fxe, we
   dtimer(dtime,itime,-1)
   isign = -1
   pois1(qe,fxe,isign,ffc,ax,affp,we,nx)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfield = tfield + time

# transform longitudinal electric force to real space with standard
# procedure: updates fxe, gxe
   dtimer(dtime,itime,-1)
   isign = 1
   fft1rxx(fxe,gxe,isign,mixup,sct,indx,nxe,nxh)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfft = tfft + time

# transform current to fourier space with standard procedure:
# updates cue, gyze
   dtimer(dtime,itime,-1)
   isign = -1
   fft1r2x(cue,gyze,isign,mixup,sct,indx,nxe,nxh)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfft = tfft + time

# calculate magnetic field in fourier space with standard procedure:
# updates byze, wm
   dtimer(dtime,itime,-1)
   bbpois13(cue,byze,ffc,ci,wm,nx,nxeh,nxh)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfield = tfield + time

# transform magnetic force to real space with standard procedure:
# updates byze, gyze
   dtimer(dtime,itime,-1)
   isign = 1
   fft1r2x(byze,gyze,isign,mixup,sct,indx,nxe,nxh)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfft = tfft + time

# add constant to magnetic field with standard procedure: updates byze
   dtimer(dtime,itime,-1)
   baddext1(byze,omy,omz,nx,nxe)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfield = tfield + time

# copy guard cells with standard procedure: updates fxe, byze
   dtimer(dtime,itime,-1)
   dguard1l(fxe,nx,nxe)
   cguard1l(byze,nx,nxe)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tguard = tguard + time

# add longitudinal and old transverse electric fields with standard
# procedure: updates exyze
   dtimer(dtime,itime,-1)
   addvrfield13(exyze,cus,fxe,nxe)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfield = tfield + time

# deposit electron acceleration density and momentum flux with OpenMP:
# updates dcu, amu
   dtimer(dtime,itime,-1)
   dcu.fill(0.0); amu.fill(0.0)
   gdjppost1l(ppart,exyze,byze,dcu,amu,kpic,omx,qme,qbme,dt,idimp,
              nppmx0,nx,mx,nxe,mx1)
# add old scaled electric field with standard procedure: updates dcu
   ascfguard1l(dcu,cus,q2m0,nx,nxe)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tdcjpost = tdcjpost + time

# add guard cells with standard procedure: updates dcu, amu
   dtimer(dtime,itime,-1)
   acguard1l(dcu,nx,nxe)
   acguard1l(amu,nx,nxe)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tguard = tguard + time

# transform acceleration density and momentum flux to fourier space
# with standard procedure: updates dcu, amu, gyze
   dtimer(dtime,itime,-1)
   isign = -1
   fft1r2x(dcu,gyze,isign,mixup,sct,indx,nxe,nxh)
   fft1r2x(amu,gyze,isign,mixup,sct,indx,nxe,nxh)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfft = tfft + time

# take transverse part of time derivative of current with standard
# procedure: updates dcu
   dtimer(dtime,itime,-1)
   adcuperp13(dcu,amu,nx,nxeh)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfield = tfield + time

# calculate transverse electric field with standard procedure:
# updates cus, wf
   dtimer(dtime,itime,-1)
   isign = -1
   epois13(dcu,cus,isign,ffe,ax,affp,wpm,ci,wf,nx,nxeh,nxh)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfield = tfield + time

# transform transverse electric field to real space with standard
# procedure: updates cus, gyze
   dtimer(dtime,itime,-1)
   isign = 1
   fft1r2x(cus,gyze,isign,mixup,sct,indx,nxe,nxh)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfft = tfft + time

# copy guard cells with standard procedure: updates cus
   dtimer(dtime,itime,-1)
   cguard1l(cus,nx,nxe)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tguard = tguard + time

# add longitudinal and transverse electric fields with standard
# procedure: exyze = cus + fxe, updates exyze
# cus needs to be retained for next time step
   dtimer(dtime,itime,-1)
   addvrfield13(exyze,cus,fxe,nxe)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tfield = tfield + time

# inner iteration loop
   for k in xrange(0,ndc):

# deposit electron current and acceleration density and momentum flux
# with OpenMP: updates cue, dcu, amu
      dtimer(dtime,itime,-1)
      cue.fill(0.0); dcu.fill(0.0); amu.fill(0.0)
      gdcjppost1l(ppart,exyze,byze,cue,dcu,amu,kpic,omx,qme,qbme,dt,
                  idimp,nppmx0,nx,mx,nxe,mx1)
# add scaled electric field with standard procedure: updates dcu
      ascfguard1l(dcu,cus,q2m0,nx,nxe)
      dtimer(dtime,itime,1)
      time = float(dtime)
      tdcjpost = tdcjpost + time

# add guard cells for current, acceleration density, and momentum flux
# with standard procedure: updates cue, dcu, amu
      dtimer(dtime,itime,-1)
      acguard1l(cue,nx,nxe)
      acguard1l(dcu,nx,nxe)
      acguard1l(amu,nx,nxe)
      dtimer(dtime,itime,1)
      time = float(dtime)
      tguard = tguard + time

# transform current to fourier space with standard procedure:
# update cue, gyze
      dtimer(dtime,itime,-1)
      isign = -1
      fft1r2x(cue,gyze,isign,mixup,sct,indx,nxe,nxh)
      dtimer(dtime,itime,1)
      time = float(dtime)
      tfft = tfft + time

# calculate magnetic field in fourier space with standard procedure:
# updates byze, wm
      dtimer(dtime,itime,-1)
      bbpois13(cue,byze,ffc,ci,wm,nx,nxeh,nxh)
      dtimer(dtime,itime,1)
      time = float(dtime)
      tfield = tfield + time

# transform magnetic force to real space with standard procedure:
# updates byze, gyze
      dtimer(dtime,itime,-1)
      isign = 1
      fft1r2x(byze,gyze,isign,mixup,sct,indx,nxe,nxh)
      dtimer(dtime,itime,1)
      time = float(dtime)
      tfft = tfft + time

# add constant to magnetic field with standard procedure: updates bzye
      dtimer(dtime,itime,-1)
      baddext1(byze,omy,omz,nx,nxe)
      dtimer(dtime,itime,1)
      time = float(dtime)
      tfield = tfield + time

# transform acceleration density and momentum flux to fourier space
# with standard procedure: updates dcu, amu, gyze
      dtimer(dtime,itime,-1)
      isign = -1
      fft1r2x(dcu,gyze,isign,mixup,sct,indx,nxe,nxh)
      fft1r2x(amu,gyze,isign,mixup,sct,indx,nxe,nxh)
      dtimer(dtime,itime,1)
      time = float(dtime)
      tfft = tfft + time

# take transverse part of time derivative of current with standard
# procedure: updates dcu
      dtimer(dtime,itime,-1)
      adcuperp13(dcu,amu,nx,nxeh)
      dtimer(dtime,itime,1)
      time = float(dtime)
      tfield = tfield + time

# calculate transverse electric field with standard procedure:
# updates cus, wf
      dtimer(dtime,itime,-1)
      isign = -1
      epois13(dcu,cus,isign,ffe,ax,affp,wpm,ci,wf,nx,nxeh,nxh)
      dtimer(dtime,itime,1)
      time = float(dtime)
      tfield = tfield + time

# transform transverse electric field to real space with standard
      dtimer(dtime,itime,-1)
      isign = 1
      fft1r2x(cus,gyze,isign,mixup,sct,indx,nxe,nxh)
      dtimer(dtime,itime,1)
      time = float(dtime)
      tfft = tfft + time

# copy guard cells with standard procedure: updates byze, cus
      dtimer(dtime,itime,-1)
      cguard1l(byze,nx,nxe)
      cguard1l(cus,nx,nxe)
      dtimer(dtime,itime,1)
      time = float(dtime)
      tguard = tguard + time

# add longitudinal and transverse electric fields with standard
# procedure: exyze = cus + fxyze, updates exyze
# cus needs to be retained for next time step
      dtimer(dtime,itime,-1)
      addvrfield13(exyze,cus,fxe,nxe)
      dtimer(dtime,itime,1)
      time = float(dtime)
      tfield = tfield + time
   pass

# push particles with OpenMP:
   wke[0] = 0.0
   dtimer(dtime,itime,-1)
# updates ppart, wke
#  gbppush13l(ppart,exyze,byze,kpic,omx,qbme,dt,dt,wke,idimp,nppmx0,nx,
#             mx,nxe,mx1,ipbc)
# updates ppart, ncl, ihole, wke, irc
   gbppushf13l(ppart,exyze,byze,kpic,ncl,ihole,omx,qbme,dt,dt,wke,idimp,
               nppmx0,nx,mx,nxe,mx1,ntmax,irc)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tpush = tpush + time
   if (irc[0] != 0):
      print "gbppushf13l error, irc=", irc[0]
      exit(0)

# reorder particles by tile with OpenMP:
   dtimer(dtime,itime,-1)
# updates ppart, ppbuff, kpic, ncl, ihole, and irc
#  pporder1l(ppart,ppbuff,kpic,ncl,ihole,idimp,nppmx0,nx,mx,mx1,npbmx,
#            ntmax,irc)
# updates ppart, ppbuff, kpic, ncl, and irc
   pporderf1l(ppart,ppbuff,kpic,ncl,ihole,idimp,nppmx0,mx1,npbmx,ntmax,
              irc)
   dtimer(dtime,itime,1)
   time = float(dtime)
   tsort = tsort + time
   if (irc[0] != 0):
      print "pporderf1l error, ntmax, irc=", ntmax, irc[0]
      exit(0)

   if (ntime==0):
      wt = we + wm
      print "Initial Total Field, Kinetic and Total Energies:"
      print "%14.7e %14.7e %14.7e" % (wt, wke, wke + wt)
      print "Initial Electrostatic, Transverse Electric and Magnetic " \
      "Field Energies:"
      print "%14.7e %14.7e %14.7e" % (we, wf, wm)
ntime = ntime + 1

# * * * end main iteration loop * * *

print "ntime, ndc = ", ntime, ndc
wt = we + wm
print "Final Total Field, Kinetic and Total Energies:"
print "%14.7e %14.7e %14.7e" % (wt, wke, wke + wt)
print "Final Electrostatic, Transverse Electric and Magnetic Field " \
"Energies:"
print "%14.7e %14.7e %14.7e" % (we, wf, wm)

print ""
print "deposit time = ", tdpost
print "current deposit time = ", tdjpost
print "current derivative deposit time = ", tdcjpost
tdpost = tdpost + tdjpost + tdcjpost
print "total deposit time = ", tdpost
print "guard time = ", tguard
print "solver time = ", tfield
print "fft time = ", tfft
print "push time = ", tpush
print "sort time = ", tsort
tfield = tfield + tguard + tfft
print "total solver time = ", tfield
time = tdpost + tpush + tsort
print "total particle time = ", time
wt = time + tfield
print "total time = ", wt
print ""

wt = 1.0e+09/(float(nloop)*float(np))
print "Push Time (nsec) = ", tpush*wt
print "Deposit Time (nsec) = ", tdpost*wt
print "Sort Time (nsec) = ", tsort*wt
print "Total Particle Time (nsec) = ", time*wt

max_stars_repo_name: gcasabona/cuda

max_issues_repo_name: gcasabona/cuda

max_forks_repo_name: gcasabona/cuda


text:
import logging
import os.path

from astropy.logger import AstropyLogger

from sunpy import config, log

level_to_numeric = {'CRITICAL': 50, 'ERROR': 40,
                    'WARNING': 30, 'INFO': 20, 'DEBUG': 10, 'NOTSET': 0}

def test_logger_name():
    assert log.name == 'sunpy'

def test_is_the_logger_there():
    assert isinstance(log, logging.Logger)
    assert isinstance(log, AstropyLogger)

def test_is_level_configed():
    """
    Test to make sure that the logger follows the config:

    log_level
    """
    config_level_numeric = level_to_numeric.get(config.get('logger', 'log_level'))
    assert log.getEffectiveLevel() == config_level_numeric

def test_is_log_to_file_configed():
    """
    Test to make sure that the logger follows the config:

    log_to_file, log_file_level, log_file_path
    """

    if config.get('logger', 'log_to_file') == 'True':
        #  there must be two handlers, one streaming and one to file.
        assert len(log.handlers) == 2
        #  one of the handlers must be FileHandler
        assert isinstance(log.handlers[0], logging.FileHandler) or isinstance(
            log.handlers[1], logging.FileHandler)
        fh = None
        if isinstance(log.handlers[0], logging.FileHandler):
            fh = log.handlers[0]

        if isinstance(log.handlers[1], logging.FileHandler):
            fh = log.handlers[1]

        if fh is not None:
            log_file_level = config.get('logger', 'log_file_level')
            assert level_to_numeric.get(log_file_level) == fh.level

            log_file_path = config.get('logger', 'log_file_path')
            assert os.path.basename(fh.baseFilename) == os.path.basename(log_file_path)

def test_origin():
    with log.log_to_list() as log_list:
        log.info('test1')

    assert log_list[0].origin == 'sunpy.util.tests.test_logger'
    assert log_list[0].message.startswith('test1')

def send_to_log(message, kind='INFO'):
    """
    A simple function to demonstrate the logger generating an origin.
    """
    if kind.lower() == 'info':
        log.info(message)
    elif kind.lower() == 'debug':
        log.debug(message)

# no obvious way to do the following
# TODO: test for the following configs  use_color, log_warnings, log_exceptions, log_file_format

reason: from astropy


text:
import cv2
import numpy as np

cv2.setUseOptimized(False)
#cv2.setUseOptimized(True)

print(cv2.useOptimized())
t1 = cv2.getTickCount()
img = cv2.imread('C:/Users/Hardik/Documents/OpeCV/Tut1/Messi.jpg')
cv2.imshow('image',img)
px = img[100,100,2]
print px

#img[100,100] = [255,255,255]
#print img[100,100]

img.itemset((100,100,1),100)
print img.item(100,100,0)

print img.shape
print img.size
print img.dtype
t2 = cv2.getTickCount()
time = (t2-t1)/cv2.getTickFrequency()
print time
cv2.waitKey(0)

max_stars_repo_path: Core Operations/Performance measurement and optimisation/`CheckOptimisation.py


text:
#!/usr/bin/env python3

#tensorflow/examples/courses/udacity_intro_to_tensorflow_for_deep_learning/l04c01_image_classification_with_cnns.ipynb

# Image Classification with Convolutional Neural Networks
import tensorflow as tf

import os
import math
import numpy as np
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt

import logging
logger = tf.get_logger()
logger.setLevel(logging.ERROR)

from clothes import *

### Build the model
#
#Building the neural network requires configuring the layers of the model, then
#compiling the model.
#
#### Setup the layers
#The basic building block of a neural network is the *layer*. A layer extracts a
#representation from the data fed into it. Hopefully, a series of connected
#layers results in a representation that is meaningful for the problem at hand.
#
#Much of deep learning consists of chaining together simple layers. Most layers,
#like `tf.keras.layers.Dense`, have internal parameters which are adjusted
#("learned") during training.

model = tf.keras.Sequential([
    tf.keras.layers.Conv2D(32, (3,3), padding='same', activation=tf.nn.relu,
                           input_shape=(28, 28, 1)),
    tf.keras.layers.MaxPooling2D((2, 2), strides=2),
    tf.keras.layers.Conv2D(64, (3,3), padding='same', activation=tf.nn.relu),
    tf.keras.layers.MaxPooling2D((2, 2), strides=2),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(128, activation=tf.nn.relu),
    tf.keras.layers.Dense(10, activation=tf.nn.softmax)
])

#This network layers are:
#
#* **"convolutions"** `tf.keras.layers.Conv2D and MaxPooling2D`— Network start
#with two pairs of Conv/MaxPool. The first layer is a Conv2D filters (3,3) being
#applied to the input image, retaining the original image size by using padding,
#and creating 32 output (convoluted) images (so this layer creates 32 convoluted
#                                            images of the same size as input).
#After that, the 32 outputs are reduced in size using a MaxPooling2D (2,2) with
#a stride of 2. The next Conv2D also has a (3,3) kernel, takes the 32 images as
#input and creates 64 outputs which are again reduced in size by a MaxPooling2D
#layer. So far in the course, we have described what a Convolution does, but we
#haven't yet covered how you chain multiples of these together. We will get back
#to this in lesson 4 when we use color images. At this point, it's enough if you
#understand the kind of operation a convolutional filter performs
#
#* **output** `tf.keras.layers.Dense` — A 128-neuron, followed by 10-node
#*softmax* layer. Each node represents a class of clothing. As in the previous
#layer, the final layer takes input from the 128 nodes in the layer before it,
#and outputs a value in the range `[0, 1]`, representing the probability that
#the image belongs to that class. The sum of all 10 node values is 1.
#
#> Note: Using `softmax` activation and `SparseCategoricalCrossentropy()` has
#issues and which are patched by the `tf.keras` model. A safer approach, in
#general, is to use a linear output (no activation function) with
#`SparseCategoricalCrossentropy(from_logits=True)`.

#### Compile the model
#
#Before the model is ready for training, it needs a few more settings. These are
#added during the model's *compile* step:
#
#* *Loss function* — An algorithm for measuring how far the model's outputs are
#from the desired output. The goal of training is this measures loss.
#* *Optimizer* —An algorithm for adjusting the inner parameters of the model in
#order to minimize loss.
#* *Metrics* —Used to monitor the training and testing
#steps. The following example uses *accuracy*, the fraction of the images that
#are correctly classified.

model.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(),
              metrics=['accuracy'])

### Train the model
#
#First, we define the iteration behavior for the train dataset:
#1. Repeat forever by specifying `dataset.repeat()` (the `epochs` parameter described
#                                          below limits how long we perform
#                                          training).
#2. The
#`dataset.shuffle(60000)` randomizes the order so our model cannot learn
#anything from the order of the examples.
#3. And `dataset.batch(32)` tells
#`model.fit` to use batches of 32 images and labels when updating the model
#variables.
#
#Training is performed by calling the `model.fit` method: 1. Feed the training
#data to the model using `train_dataset`.  2. The model learns to associate
#images and labels.  3. The `epochs=5` parameter limits training to 5 full
#iterations of the training dataset, so a total of 5 * 60000 = 300000 examples.
#
#(Don't worry about `steps_per_epoch`, the requirement to have this flag will soon be removed.)

BATCH_SIZE = 32
train_dataset = train_dataset.cache().repeat().shuffle(num_train_examples).batch(BATCH_SIZE)
test_dataset = test_dataset.cache().batch(BATCH_SIZE)

saved = '../data/cnn.h5'
if os.path.exists(saved):
    model = tf.keras.models.load_model(saved)
else:
    model.fit(train_dataset, epochs=10,
             verbose=2, steps_per_epoch=math.ceil(num_train_examples/BATCH_SIZE))
    model.save(saved)

#As the model trains, the loss and accuracy metrics are displayed. This model
#reaches an accuracy of about 0.97 (or 97%) on the training data.

## Evaluate accuracy
#Next, compare how the model performs on the test dataset. Use all examples we
#have in the test dataset to assess accuracy.

test_loss, test_accuracy = model.evaluate(test_dataset, steps=math.ceil(num_test_examples/32))
print('Accuracy on test dataset:', test_accuracy)

#As it turns out, the accuracy on the test dataset is smaller than the accuracy
#on the training dataset. This is completely normal, since the model was trained
#on the `train_dataset`. When the model sees images it has never seen during
#training, (that is, from the `test_dataset`), we can expect performance to go
#down.

show_results(model, test_dataset)

# And, as before, the model predicts a label of 6 (shirt).

# # Exercises

# Experiment with different models and see how the accuracy results differ. In
# particular change the following parameters: *   Set training epochs set to 1 *
# Number of neurons in the Dense layer following the Flatten one. For example, go
# really low (e.g. 10) in ranges up to 512 and see how accuracy changes *   Add
# additional Dense layers between the Flatten and the final Dense(10), experiment
# with different units in these layers *   Don't normalize the pixel values, and
# see the effect that has

# Remember to enable GPU to make everything run faster
# (Runtime -> Change runtime type -> Hardware accelerator -> GPU).
# Also, if you run into trouble, simply reset the entire environment and start
# from the beginning: *   Edit -> Clear all outputs *   Runtime -> Reset all
# runtimes


text:
import numpy as np
import omnidirectional_camera as ocam
import cv2
import matplotlib.pyplot as plt

# This is the "focal length" of the camera, you can test with different values
focal_length = 200

# These are the coefficients that describe the fisheye camera (based on the omnidirectional model)
model_coefficients = np.array([730.949123, 315.876984, -177.960849, -352.468231, -678.144608, -615.917273, -262.086205, -42.961956])

# Load the test fish-eye camera image
image = cv2.imread('.//test_data/fish_eye_camera/test_fisheye.jpg')

# Calculate the look-up-table for converting the image into perspective camera image
x, y = ocam.perspective_lut(image.shape, (505.480427, 381.777786), focal_length, model_coefficients)

# Convert the image into perspective camera image
image_perspective = cv2.remap(image, x, y, cv2.INTER_LINEAR)

# Show the re-mapped image
plt.imshow(image_perspective)
plt.show()


text:
import ci_reduce.common as common
import astropy.io.fits as fits
import glob
import numpy as np
import os
import matplotlib.pyplot as plt
from scipy.stats import scoreatpercentile

# currently I have access to 4 independent bias exposures for each of
# CIE, CIN, CIS, and CIW, and none for CIC
# these available data come from "forDK.tar.gz" samples, where
# CIC was in "simulation mode"

engdir = '/project/projectdirs/desi/spectro/data/20190330'

def get_bias_frame_names():
    flist = glob.glob(engdir + '/*/ci*.fits.fz')
    flist = np.array(flist)

    exptime = np.zeros(len(flist))
    flavor = []
    for i, f in enumerate(flist):
        h = fits.getheader(f, extname='CI')
        exptime[i] = h['EXPTIME']
        flavor.append(h['FLAVOR'])

    flavor = np.array(flavor)
    flist = flist[(exptime == 0) & (flavor == 'zero')]

    flist.sort()
    return flist

def read_bias_frames(ci_extname):
    flist = get_bias_frame_names()
    imgs = [fits.getdata(f, extname=ci_extname) for f in flist]

    return imgs

def master_bias_1camera(ci_extname):
    # use common to check that CI extention name is valid

    bias_frames = read_bias_frames(ci_extname)
    bias_frames = np.asarray(bias_frames)
    bias_med = np.median(bias_frames, axis=0)
    return bias_med

def master_bias_header_cards(hdu, ci_extname):
    h = hdu.header
    h['FLAVOR'] = 'ZERO'
    h['EXTNAME'] = ci_extname
    h['EXPTIME'] = 0.0
    # zero idea whatsoever what temperature the forDK.tar.gz bias frames were
    # acquired at; just putting in 0 here since the dark current will be
    # zeroed out by EXPTIME = 0 anyway

    # CAMTEMP header card name comes from DESI-4000v3, "CI Camera" tab
    # night of March 30 biases did not have CAMTEMP available
    #h['CAMTEMP'] = 0.0
    return h

def write_master_bias(outname=None):

    par = common.ci_misc_params()

    if outname is None:
        outname = par['master_bias_filename']
        outname = os.path.join(os.environ[par['etc_env_var']], outname)

    assert(not os.path.exists(outname))

    ci_extnames = common.valid_image_extname_list()

    hdus = []
    for ci_extname in ci_extnames:
        print('Working on master bias for: ' + ci_extname)
        bias_image = master_bias_1camera(ci_extname)
        bias_image = bias_image.astype('float32')
        if len(hdus) == 0:
            hdu = fits.PrimaryHDU(bias_image)
        else:
            hdu = fits.ImageHDU(bias_image)
        hdu.header = master_bias_header_cards(hdu, ci_extname)

        hdus.append(hdu)

    hdul = fits.HDUList(hdus)

    hdul.writeto(outname)

def plot_master_biases():
    # compare biases based on forDK samples to those post-installation

    ci_extnames = common.valid_image_extname_list()

    for ci_extname in ci_extnames:
        if ci_extname == 'CIC':
            continue
        bias_new = fits.getdata('/global/cscratch1/sd/ameisner/CI_master_bias.fits', extname=ci_extname)

        bias_old = fits.getdata('/project/projectdirs/desi/users/ameisner/CI/ci_reduce_etc/CI_master_bias.fits.gz', extname=ci_extname)

        print(ci_extname)
        print(np.median(bias_new), 'new')
        print(np.median(bias_old), 'old')

        bias_old -= np.median(bias_old)
        bias_new -= np.median(bias_new)

        vmax = max(scoreatpercentile(bias_old, 97.5), scoreatpercentile(bias_new, 97.5))
        vmin = max(scoreatpercentile(bias_old, 2.5), scoreatpercentile(bias_old, 2.5))

        print(vmin, vmax)
        print('~'*30)
        plt.imshow(bias_old, vmin=vmin, vmax=vmax, cmap='gray')
        outname = '/global/cscratch1/sd/ameisner/old_bias_' + ci_extname + '.png'
        plt.gca().get_xaxis().set_visible(False)
        plt.gca().get_yaxis().set_visible(False)

        dpi = 300
        plt.savefig(outname, dpi=dpi, bbox_inches='tight')

        plt.cla()

        plt.imshow(bias_new, vmin=vmin, vmax=vmax, cmap='gray', origin='lower')
        outname = outname.replace('old', 'new')

        plt.gca().get_xaxis().set_visible(False)
        plt.gca().get_yaxis().set_visible(False)

        plt.savefig(outname, dpi=dpi, bbox_inches='tight')

reason: import numpy,from scipy,import astropy


text:
import numpy as np
import pandas as pd
from random import random
from sec_etl import *
from reports.cash_flow.etl import wrapper_get_cashflow_df_and_col
from reports.cash_flow.investing_activity import wrapper_get_ia_values_and_label
from reports.cash_flow.operating_activity import wrapper_get_oa_values_and_label
from reports.document_entity_information.total_outstanding_shares import get_total_outstanding_shares
import time
from utils import make_cashflow_colnames, update_dict_with_cashflow_vals, get_save_path, get_fname_from_sourcedata

# For windows
OSX = False

# User selected file to process
fname = get_fname_from_sourcedata(OSX)
tickers = load_tickers(fname)

# Create column headers
investing_colnames = make_cashflow_colnames('investing')
operating_colnames = make_cashflow_colnames('operating')

# init dict to store results
info_dict = {k: [] for k in ('ticker',
                             'company',
                             'industry',
                             'filing_type',
                             'filing_date',
                             'dler_info',
                             'total_outstanding_shares',
                             'investing_activity_label',
                             *investing_colnames,
                             'operating_activity_label',
                             *operating_colnames,)}

for i, ticker in enumerate(tickers, 1):
    # Meta + Excel File ETL
    filing, soup = get_filing_soup(ticker)
    company = get_company(soup)
    industry = get_industry(soup)
    xl_url, fdate = get_xl_url(filing, soup)
    xl_file, dler_info = get_xl_file(xl_url)

    # Print ticker info in command line
    nl = '\n'
    print(f'#: {i}{nl}'
          f'TICKER: {ticker}{nl}'
          f'COMPANY: {company}{nl}'
          f'INDUSTRY: {industry}{nl}'
          f'FILING: {filing}{nl}'
          f'FDATE: {fdate}{nl}'
          f'EXCEL: {xl_url}')

    # random sleep ~0.5 second per ticker. ~120 tickers per min
    # to avoid getting timed out
    time.sleep(random()/2)

    # change to results dict
    info_dict['ticker'].append(ticker)
    info_dict['company'].append(company)
    info_dict['industry'].append(industry)
    info_dict['filing_type'].append(filing)
    info_dict['filing_date'].append(fdate)
    info_dict['dler_info'].append(dler_info)

    if xl_file and filing == '10-K':
        # get & log TOTAL OUTSTANDING SHARES (possible to log nan)
        total_outstanding_shares = get_total_outstanding_shares(xl_file)
        info_dict['total_outstanding_shares'].append(total_outstanding_shares)

        # ETL cashflow data
        cashflow_df, cashflow_col = wrapper_get_cashflow_df_and_col(xl_file)
        if (cashflow_df is None) or (cashflow_col is None):
            # log INVESTING ACTIVITIES
            info_dict['investing_activity_label'].append(np.nan)
            info_dict = update_dict_with_cashflow_vals(info_dict,
                                                       investing_colnames,
                                                       len(investing_colnames) * [np.nan])

            # log OPERATING ACTIVITIES
            info_dict['operating_activity_label'].append(np.nan)
            info_dict = update_dict_with_cashflow_vals(info_dict,
                                                       operating_colnames,
                                                       len(operating_colnames) * [np.nan])

        else:
            # get & log INVESTING ACTIVITY
            ia_values, ia_label = wrapper_get_ia_values_and_label(cashflow_df, cashflow_col)
            if (ia_values is None) or (ia_label is None):
                info_dict['investing_activity_label'].append(np.nan)
                info_dict = update_dict_with_cashflow_vals(info_dict,
                                                           investing_colnames,
                                                           len(investing_colnames) * [np.nan])
            else:
                # IA ACTUALY LOGGING
                print('Investing Activity: ', ia_label)
                info_dict['investing_activity_label'].append(ia_label)
                info_dict = update_dict_with_cashflow_vals(info_dict,
                                                           investing_colnames,
                                                           ia_values)

            # get & log OPERATING ACTIVITY
            oa_values, oa_label = wrapper_get_oa_values_and_label(cashflow_df, cashflow_col)
            if (oa_values is None) or (oa_label is None):
                info_dict['operating_activity_label'].append(np.nan)
                info_dict = update_dict_with_cashflow_vals(info_dict,
                                                           operating_colnames,
                                                           len(operating_colnames) * [np.nan])
            else:
                # OA ACTUALY LOGGING
                print('Operating Activity: ', oa_label)
                info_dict['operating_activity_label'].append(oa_label)
                info_dict = update_dict_with_cashflow_vals(info_dict,
                                                           operating_colnames,
                                                           oa_values)

    else:
        # log total outstanding shares
        info_dict['total_outstanding_shares'].append(np.nan)

        # log INVESTING ACTIVITIES
        info_dict['investing_activity_label'].append(np.nan)
        info_dict = update_dict_with_cashflow_vals(info_dict,
                                                   investing_colnames,
                                                   len(investing_colnames) * [np.nan])

        # log OPERATING ACTIVITIES
        info_dict['operating_activity_label'].append(np.nan)
        info_dict = update_dict_with_cashflow_vals(info_dict,
                                                   operating_colnames,
                                                   len(operating_colnames) * [np.nan])
    print()

# convert to datafame & save
info_df = pd.DataFrame(data=info_dict)
spath = get_save_path(fname, OSX)
info_df.to_csv(spath, index=False)

print('#'*50)
print(f'FINISHED')
print('#'*50)

max_stars_repo_name: marwin-ko/mfc-sec-etl

max_issues_repo_name: marwin-ko/mfc-sec-etl

max_forks_repo_name: marwin-ko/mfc-sec-etl


text:
import numpy as np
from qiskit.circuit import QuantumCircuit, ParameterVector
from surfer.qfi import ReverseQFI

x = ParameterVector('x', 2)
circuit = QuantumCircuit(1)
circuit.rx(x[0], 0)
circuit.rx(x[1], 0)

circuit = circuit.bind_parameters({x[0]: 0})

values = np.ones(circuit.num_parameters)
print(circuit.num_parameters)

rev = ReverseQFI()
revd = rev.compute(circuit, values)

print("Reverse mode")
print(revd)

max_stars_repo_name: Cryoris/surfer

max_issues_repo_name: Cryoris/surfer

max_forks_repo_name: Cryoris/surfer


text:
__author__ = "Nestor Bermudez"
__license__ = "MIT"
__version__ = "1.0.0"
__email__ = "nab6@illinois.edu"
__status__ = "Development"

import numpy as np
import tensorflow as tf

flags = tf.flags
FLAGS = flags.FLAGS

flags.DEFINE_string("embeddings_path", None,
                    "Path to the embeddings file")
flags.DEFINE_string("output_dir", None,
                    "Path to the output dir")
flags.DEFINE_integer("n_graphs", 6,
                    "Number of graphs present per organism")
flags.DEFINE_list("organisms", ["human", "yeast"],
                  "List of organisms present in the data")
flags.DEFINE_list("org_nodes", [18362, 6400],
                  "Number of nodes per organism network")

def main(_):
    all_embs = np.load(FLAGS.embeddings_path)
    n_dims = all_embs.shape[1]
    print("Shape of embeddings: ", all_embs.shape)
    assert (all_embs.shape[0] == FLAGS.n_graphs * np.sum(FLAGS.org_nodes))

    offset = 0
    for org, size in zip(FLAGS.organisms, FLAGS.org_nodes):
        org_embs = all_embs[offset:offset + FLAGS.n_graphs * size]
        org_embs = org_embs.reshape(FLAGS.n_graphs, size, -1)
        org_embs = np.mean(org_embs, axis=0)
        assert (org_embs.shape == (size, n_dims))

        org_embs.dump('{}/embs_{}.npy'.format(FLAGS.output_dir, org))
        offset += FLAGS.n_graphs * size

if __name__ == '__main__':
    tf.app.run()

max_stars_repo_name: nbermudezs/HoGEm

max_issues_repo_name: nbermudezs/HoGEm

max_forks_repo_name: nbermudezs/HoGEm


text:
import datetime

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
import numpy as np

import matplotlib.pyplot as plt
import matplotlib
# matplotlib.use('Agg')

# logistic regression modeling
lr_model = LogisticRegression()

from numpy import genfromtxt

import pandas as pd

tbl_df = pd.read_csv('../samples/data/data_mining_DB_clients_tbl.csv')

bis_df = pd.read_csv('../samples/data/data_mining_DB_clients_tbl_bis.csv')

"""
Clients :

Id
CDSEXE
MTREV
NBENF      : nombre d’enfants
CDSITFAM   : situation familiale, catégorielle
DTADH      : date adhésion à l’organisme
CDTMT      : statut (siège ou bien tsmt) = catégorielle

CDDEM      : code démission
DTDEM      : date démission
ANNEE_DEM  : année démission
CDMOTDEM   : motif démission - Rien si non-démissionnaire ou si motif inconnu.

CDCATCL    : catégorie (sociétaire / adhérent)
AGEAD      : âge du client à l’adhésion
rangagead  : âge du client à l’adhésion par tranches
agedem     : âge du client à la démission
rangagedem : âge du client à la démission par tranches
rangdem    : date démission sous format mois année
adh        : durée en années de l’adhésion
rangadh    : durée en années de l’adhésion par tranches
"""

"""
Clients BIS:

Id         : auto increment
CDSEXE     : catégorielle
DTNAIS     : date naissance adhérent
MTREV      : numérique
NBENF      : nombre d’enfants
CDSITFAM   : situation familiale, catégorielle
DTADH      : date adhésion à l’organisme
CDTMT      : statut (siège ou bien tsmt) = catégorielle
CDMOTDEM   : motif démission. Rien si non-démissionnaire
CDCATCL    : catégorie (sociétaire / adhérent)
Bpadh      : variable inconnue
DTDEM      : date démission (31/12/1900 si non démissionnaire)
"""

# tbl_array = genfromtxt('samples/data/data_mining_DB_clients_tbl.csv')

print(tbl_df)

result = pd.concat([tbl_df, bis_df], ignore_index=True, sort=False)
# print(result)

result['ANNEE_DEM'] = np.where(result['DTDEM'] != '1900-12-31', result['DTDEM'].apply(lambda x: int(x.split('-')[0])), result['ANNEE_DEM'])

# print(result)

result['AGEAD'] = np.where(result['AGEAD'].isnull() & result['DTNAIS'].notnull(), (result['DTADH'].apply(lambda x: int(x.split('-')[0]))) - (result['DTNAIS'].apply(lambda x: 0 if type(x) == float else int(x.split('-')[0]))), result['AGEAD'])

result['AGEAD'] = np.where(result['AGEAD'], result['AGEAD'].apply(lambda x: int(result['AGEAD'].median()) if x > 500 else x), result['AGEAD'])

# year(DTADH) - year(DTNAISS)

# print(result)

result = result.drop(['Id'], axis=1)
result.insert(0, 'Id', result.index + 1)

# print(result)

result['adh'] = np.where(
    result['adh'].isnull(),  # & result['DTDEM'] != '1900-12-31'
    (result['DTDEM'].apply(lambda x: int(x.split('-')[0]))) - (result['DTADH'].apply(lambda x: 0 if type(x) == float else int(x.split('-')[0]))),
    result['adh']
)

# 2007 - 1900 = 107
result['adh'] = np.where(result['adh'], result['adh'].apply(lambda x: 107 + x if x < 0 else x), result['adh'])

# year(DTDEM) - year(DTADH) si démissionnaire

# print(result)

result['agedem'] = np.where(result['agedem'].isnull() & result['DTDEM'].notnull() & result['DTNAIS'].notnull(),
                            (result['DTDEM'].apply(lambda x: int(x.split('-')[0]))) - (result['DTNAIS'].apply(lambda x: 0 if type(x) == float else int(x.split('-')[0]))),
                            result['agedem'])

result['agedem'] = np.where(result['agedem'], result['agedem'].apply(lambda x: 0 if x < 0 else x), result['agedem'])
result['agedem'] = np.where(result['agedem'], result['agedem'].apply(lambda x: 0 if x == 1900 else x), result['agedem'])

# year(DTDEM) - year(DTNAISS))

# print(result)

# Clean structs

result = result.drop(['rangdem'], axis=1)
result = result.drop(['rangadh'], axis=1)
result = result.drop(['rangagead'], axis=1)
result = result.drop(['rangagedem'], axis=1)

# print(result)

CREATE_DB = True

result.to_csv('result.csv')

# === Prev Table, used for previsions ===
prev = result

# == Create predict boolean
date_1901 = datetime.datetime(year=1901, month=1, day=1)
prev['dem'] = ((~ prev['CDDEM'].isnull()) | (prev['DTDEM'] != '1900-12-31') | (~ prev['ANNEE_DEM'].isnull())).astype(int)

# Set as first col
prev.insert(0, 'dem', prev.pop("dem"))

# == Drop dates, and other useless
prev = prev.drop(['Id'], axis=1)
prev = prev.drop(['DTADH'], axis=1)

prev = prev.drop(['CDDEM'], axis=1)
prev = prev.drop(['DTDEM'], axis=1)
prev = prev.drop(['ANNEE_DEM'], axis=1)
prev = prev.drop(['CDMOTDEM'], axis=1)
prev = prev.drop(['agedem'], axis=1)
prev = prev.drop(['DTNAIS'], axis=1)
# Incomplete
prev = prev.drop(['Bpadh'], axis=1)

# Unsure about this one ?
prev = prev.drop(['adh'], axis=1)

# == Fix Types
prev['AGEAD'] = prev['AGEAD'].astype(int)

# == Enum to dummy
dummies = ['CDTMT', 'CDSITFAM', 'CDTMT', 'CDCATCL']
prev = pd.get_dummies(prev, prefix=dummies, columns=dummies)

prev.to_csv('previsions.csv', index=False)

if CREATE_DB:

    from sqlalchemy import create_engine
    engine = create_engine('sqlite:///clients_tbl.db', echo=False)

    tbl_df.to_sql('clients', con=engine, if_exists='replace')
    bis_df.to_sql('bis', con=engine, if_exists='replace')
    result.to_sql('result', con=engine, if_exists='replace')
    prev.to_sql('prev', con=engine, if_exists='replace')
    engine.execute("SELECT * FROM clients").first()
    engine.execute("SELECT * FROM bis").first()
    engine.execute("SELECT * FROM result").first()
    engine.execute("SELECT * FROM prev").first()

max_stars_repo_name: TheRainbowPhoenix/BITume

max_issues_repo_name: TheRainbowPhoenix/BITume

max_forks_repo_name: TheRainbowPhoenix/BITume

reason: import numpy,from numpy


text:
"""
Created on Apr 18, 2017

@author: Christopher Bruns
"""

import os

import numpy
from OpenGL import GL
from OpenGL.GL.shaders import compileShader, compileProgram
from OpenGL.arrays import vbo

from openvr.glframework.glmatrix import identity, pack, rotate_y, scale
from openvr.glframework import shader_string

class TriangleActor(object):
    def __init__(self):
        self.vao = None
        # hard-code shader parameter location index
        self.mvp_location = 0
        self.program = None
        # Create triangle geometry: corner 2D location and colors
        self.vertices = vbo.VBO(numpy.array([
            [-0.6, -0.4, 1.0, 0.0, 0.0],  # x, y, r, g, b
            [0.6, -0.4, 0.0, 1.0, 0.0],
            [0.0, 0.6, 0.0, 0.0, 1.0],
        ], dtype='float32'))

    def init_gl(self):
        # Create vertex array object, apparently required for modern OpenGL
        self.vao = GL.glGenVertexArrays(1)
        GL.glBindVertexArray(self.vao)
        self.vertices.bind()
        # hard-code shader parameter location indices
        vpos_location = 0
        vcol_location = 1
        GL.glEnableVertexAttribArray(vpos_location)
        float_size = self.vertices.dtype.itemsize  # 4 bytes per float32
        GL.glVertexAttribPointer(vpos_location, 2, GL.GL_FLOAT, False,
                                 float_size * 5, self.vertices + float_size * 0)
        GL.glEnableVertexAttribArray(vcol_location)
        GL.glVertexAttribPointer(vcol_location, 3, GL.GL_FLOAT, False,
                                 float_size * 5, self.vertices + float_size * 2)
        # Create GLSL shader program
        vertex_shader = compileShader(
            """#version 450 core
            #line 50

            layout(location = %d) uniform mat4 MVP = mat4(1);

            layout(location = %d) in vec2 vPos;
            layout(location = %d) in vec3 vCol;

            out vec3 color;

            void main()
            {
                gl_Position = MVP * vec4(vPos, 0.0, 1.0);
                color = vCol;
            }
            """ % (self.mvp_location, vpos_location, vcol_location),
            GL.GL_VERTEX_SHADER)
        fragment_shader = compileShader(
            """#version 450 core
            #line 68

            in vec3 color;
            out vec4 fragColor;

            void main()
            {
                fragColor = vec4(color, 1);
            }
            """,
            GL.GL_FRAGMENT_SHADER)
        self.program = compileProgram(vertex_shader, fragment_shader)

    def display_gl(self, model_view, projection):
        GL.glBindVertexArray(self.vao)
        GL.glUseProgram(self.program)
        mvp = numpy.matrix(model_view) * projection
        GL.glUniformMatrix4fv(self.mvp_location, 1, False, pack(mvp))
        GL.glDrawArrays(GL.GL_TRIANGLES, 0, 3)

    def dispose_gl(self):
        if self.vao:
            GL.glDeleteVertexArrays(1, [self.vao, ])
        self.vertices.delete()
        GL.glDeleteProgram(self.program)

class ObjActor(object):
    def __init__(self, obj_stream):
        self.model_matrix = identity()
        self.vao = None
        self.shader = None
        self.vertexes = list()
        vertex_normals = list()
        self.normal_for_vertex = dict()
        self.faces = list()
        fh = obj_stream
        for line in fh:
            if line.startswith('#'):
                # e.g. "# Blender v2.65 (sub 0) OBJ File"
                continue  # ignore comments
            elif line.startswith('o '):
                # e.g. "o teapot.005"
                continue  # ignore object names
            elif line.startswith('v '):
                # e.g. "v -0.498530 0.712498 -0.039883"
                vec3 = [float(x) for x in line.split()[1:4]]
                self.vertexes.append(vec3)
            elif line.startswith('vn '):
                # e.g. "vn -0.901883 0.415418 0.118168"
                vec3 = [float(x) for x in line.split()[1:4]]
                vertex_normals.append(vec3)
            elif line.startswith('s '):
                continue  # ignore whatever "s" is
                # print(line)
            elif line.startswith('f '):
                face = list()
                for c in line.split()[1:]:
                    v, n = [int(x) for x in c.split('/')[0:3:2]]
                    face.append(v - 1)  # vertex index
                    self.normal_for_vertex[v - 1] = vertex_normals[n - 1]
                    self.faces.append(face)
                    # print(line)
                    # print(face)
            else:
                print(line)
                break
        self.vbo = list()
        ibo = list()
        for i in range(len(self.vertexes)):
            v = self.vertexes[i]
            n = self.normal_for_vertex[i]
            self.vbo.append(v)
            self.vbo.append(n)
        for f in self.faces:
            for v in f:
                ibo.append(v)
                # todo: only works for single triangle faces at the moment...
        self.element_count = len(ibo)
        self.vbo = numpy.array(self.vbo, 'float32')
        ibo = numpy.array(ibo, 'int16')
        self.vbo = vbo.VBO(self.vbo)
        self.ibo = vbo.VBO(ibo, target=GL.GL_ELEMENT_ARRAY_BUFFER)

    def init_gl(self):
        self.vao = GL.glGenVertexArrays(1)
        GL.glBindVertexArray(self.vao)
        self.ibo.bind()
        self.vbo.bind()
        GL.glEnableVertexAttribArray(0)  # vertex location
        float_size = self.vbo.dtype.itemsize  # 4 bytes per float32
        GL.glVertexAttribPointer(0, 3, GL.GL_FLOAT, False,
                                 6 * float_size, self.vbo + 0 * float_size)
        GL.glEnableVertexAttribArray(1)  # vertex normal
        GL.glVertexAttribPointer(1, 3, GL.GL_FLOAT, False,
                                 6 * float_size, self.vbo + 3 * float_size)
        vertex_shader = compileShader(
            shader_string("""
            layout(location = 0) in vec3 in_Position;
            layout(location = 1) in vec3 in_Normal;

            layout(location = 0) uniform mat4 projection = mat4(1);
            layout(location = 1) uniform mat4 model_view = mat4(1);

            out vec3 normal;

            void main()
            {
                gl_Position = projection * model_view * vec4(in_Position, 1.0);
                mat4 normal_matrix = transpose(inverse(model_view));
                normal = normalize((normal_matrix * vec4(in_Normal, 0)).xyz);
            }
            """),
            GL.GL_VERTEX_SHADER)
        fragment_shader = compileShader(
            shader_string("""            in vec3 normal;
            out vec4 fragColor;

            vec4 color_by_normal(in vec3 n) {
                return vec4(0.5 * (normalize(n) + vec3(1)), 1);
            }

            void main()
            {
                fragColor = color_by_normal(normal);
            }
            """),
            GL.GL_FRAGMENT_SHADER)
        self.shader = compileProgram(vertex_shader, fragment_shader)
        GL.glEnable(GL.GL_DEPTH_TEST)

    def display_gl(self, model_view, projection):
        GL.glBindVertexArray(self.vao)
        GL.glUseProgram(self.shader)
        m = self.model_matrix * model_view
        GL.glUniformMatrix4fv(0, 1, False, pack(projection))
        GL.glUniformMatrix4fv(1, 1, False, pack(m))
        GL.glDrawElements(GL.GL_TRIANGLES, self.element_count, GL.GL_UNSIGNED_SHORT, None)

    def dispose_gl(self):
        if self.vao:
            GL.glDeleteVertexArrays(1, [self.vao, ])
            self.ibo.delete()
            self.vbo.delete()
            GL.glDeleteProgram(self.shader)
            self.vao = None

class TeapotActor(ObjActor):
    def __init__(self):
        src_folder = os.path.dirname(os.path.abspath(__file__))
        obj_path = os.path.join(src_folder, 'wt_teapot.obj')
        with open(obj_path) as fh:
            super(TeapotActor, self).__init__(obj_stream=fh)

if __name__ == "__main__":
    from openvr.glframework.glfw_app import GlfwVrApp
    import glfw
    teapot = TeapotActor()
    s = 0.2  # size of teapot in meters
    with GlfwVrApp(actors=[teapot, ]) as app:
        while not glfw.window_should_close(app.window):
            # scale teapot to original Melitta model aspect ratio
            teapot.model_matrix = scale(s, s*4/3, s) * rotate_y(glfw.get_time())
            app.render_scene()


text:
import tvm
import numpy as np
from tvm.contrib import verilog

def test_buffer_fifo():
    # Test the tvm_buffer.v module as a fifo

    # Find file will search root/verilog and root/tests/verilog
    sess = verilog.session([
        verilog.find_file("test_buffer_fifo.v"),
        verilog.find_file("tvm_buffer.v")
    ])

    # Get the handles by their names
    rst = sess.main.rst
    enq = sess.main.enq
    write_data = sess.main.write_data
    read_data = sess.main.read_data
    read_data_valid = sess.main.read_data_valid

    # Simulation input data
    test_data = np.arange(16).astype('int8')

    # Initial state
    rst.put_int(1)
    enq.put_int(0)
    write_data.put_int(0)

    # De-assert reset
    sess.yield_until_next_cycle()
    rst.put_int(0)

    # Main simulation loop
    read_idx = 0
    write_idx = 0
    while read_idx < len(test_data):
        # write logic
        if (write_idx < len(test_data)):
            enq.put_int(1)
            write_data.put_int(write_idx)
            write_idx += 1
        else:
            enq.put_int(0)
        # read logic
        if (read_data_valid.get_int()):
            assert(read_data.get_int()==test_data[read_idx])
            read_idx += 1
        # step
        sess.yield_until_next_cycle()

if __name__ == "__main__":
    test_buffer_fifo()


text:
#Add libraries we need
import librosa
import numpy as np
from flask import Flask

#Add the Darcy AI Pipeline object
from darcyai.pipeline import Pipeline

#Add our custom InputStream and Perceptor classes
from audio_input_stream import AudioInputStream
from audio_analysis_perceptor import AudioAnalysisPerceptor

#Define a class to hold our demo application
class AudioAnalyzer():
    def __init__(self):
        #Use our custom audio input stream
        mic = AudioInputStream(sample_len_sec=1)
        self.__rate = mic.get_sample_rate()

        #Create a Darcy AI pipeline with the audio input stream
        self.__pipeline = Pipeline(input_stream=mic)

        #Create an instance of our custom audio analysis percpetor and add it to the pipeline
        audio_perceptor = AudioAnalysisPerceptor()
        self.__pipeline.add_perceptor("audio_analyzer",
                                      audio_perceptor,
                                      input_callback=self.__audio_perceptor_input_callback)

    #Define our "run" method, which is just calling "run" on the pipeline
    def run(self):
        self.__pipeline.run()

    #Define a callback to work with the incoming audio data before it passes into the audio analysis perceptor
    def __audio_perceptor_input_callback(self, input_data, pom, config):
        data = np.frombuffer(input_data.data, dtype=np.float32)
        return data[:193]
        return self.__extract_features(data)

    #Perform audio feature extractions using Numpy that we will pass to the audio analysis AI perceptor
    def __extract_features(self, data):
        features = np.empty((0, 193))

        stft = np.abs(librosa.stft(data))
        mfccs = np.mean(librosa.feature.mfcc(y=data, sr=self.__rate, n_mfcc=40).T, axis=0)
        chroma = np.mean(librosa.feature.chroma_stft(S=stft, sr=self.__rate).T, axis=0)
        mel = np.mean(librosa.feature.melspectrogram(data, sr=self.__rate).T, axis=0)
        contrast = np.mean(librosa.feature.spectral_contrast(S=stft, sr=self.__rate).T, axis=0)
        tonnetz = np.mean(librosa.feature.tonnetz(y=librosa.effects.harmonic(data), sr=self.__rate).T, axis=0)
        ext_features = np.hstack([mfccs, chroma, mel, contrast, tonnetz])
        features = np.vstack([features, ext_features]).astype(np.float32)

        return features

#In the main thread, start the application by instantiating our demo class and calling "run"
if __name__ == "__main__":
    audio_analyzer = AudioAnalyzer()
    audio_analyzer.run()

max_stars_repo_name: darcyai/darcyai-sdk

max_issues_repo_name: darcyai/darcyai-sdk

max_forks_repo_name: darcyai/darcyai-sdk


text:
from bs4 import BeautifulSoup
import requests
from time import sleep
from pprint import pprint
import numpy as np
import pandas as pd
from tqdm.notebook import tqdm
import random

pd.set_option('display.max_rows', 10)
pd.set_option('display.max_columns',None)

# 読み込んだデータの進捗を保存するためのリストを用意しておく
pre_results = []

#Engineer_Agent_Scraping
def EAS(url, pre_results=[]):

    e_list = pre_results

    if (len(e_list) > 0):
        progress = list(pd.DataFrame(e_list)['ページ'])
    else:
        progress = []

    #403 forbiddenのエラーを回避するために、requestsメソッドのヘッダーにユーザーエージェントの設定をする
    #利用規約でスクレイピング禁止を明言していないくせに...!
    headers = {
        "User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15"
    }

    target_url = url.format(1)
    r = requests.get(target_url, headers = headers)
    sleep(random.randint(1,3))
    soup = BeautifulSoup(r.content, 'html.parser')
    end = soup.find('span', class_='project-list-header__num').text
    print("Scraping Start!")

    for page in tqdm(range(1, (int(end)//20+1)+1)):
        try:

            target_url = url.format(page)
            r = requests.get(target_url, headers = headers)
            sleep(random.randint(1,3))
            soup = BeautifulSoup(r.content, 'html.parser')
            contents = soup.find_all('a', class_='project-article-card__title')

            for page_num, content in enumerate(tqdm(contents)):
                if (int(str(page)+'{0:02d}'.format(page_num+1)) in progress):
                    continue

                #カラムは後から追加できるので、今はページのみ宣言
                e = {
                    "ページ":int(str(page)+'{0:02d}'.format(page_num+1))
                }

                print("page{}:".format(e["ページ"]),target_url)

                rc = requests.get(content.get('href'), headers = headers)
                sleep(random.randint(1,3))
                soupc = BeautifulSoup(rc.content, 'html.parser')

                e["タイトル"] = soupc.find('div', class_='project-article-card__title').text

                cont_names = soupc.find_all('div', class_='project-article-card__item-name')
                cont_values = soupc.find_all('div', class_='project-article-card__item-value')

                #スクレイピングした情報を格納(格納されなかったカラムはNaNになる)
                for cont_num, cont_name in enumerate(cont_names):
                    e[cont_name.text] = cont_values[cont_num].text

                e_list.append(e)

        except IndexError:
            continue
        except:
            pre_results = e_list
            return e_list

    pre_results = e_list
    return e_list

# 引数にpre_resultsを入れておけば、中断したところから素早く再開できる！
url = "https://tech-stock.com/projects/{}/?careerIds=&careerIds=&careerIds=&onlyRecruiting=true"
t_list = EAS(url, pre_results)
test = pd.DataFrame(t_list)
test

# pickleファイルに保存する
test.to_pickle('Engineer_Offers.pickle')


text:
import sys
import time
import numpy as np
import pandas as pd
import math
import warnings
from transformUtils import transformFUnion

# Make numpy values easier to read.
np.set_printoptions(precision=3, suppress=True)
warnings.filterwarnings('ignore') #cleaner, but not recommended

def readNprep():
    # Read,isolate target and combine training and test data
    train = pd.read_csv("~/datasets/homeCreditTrain.csv", delimiter=",", header=None)
    test = pd.read_csv("~/datasets/homeCreditTest.csv", delimiter=",", header=None)
    train = train.iloc[1:,:] #remove header
    train.drop(1, axis=1, inplace=True); #remove target
    train.columns = [*range(0,121)] #rename header from 0 to 120
    test = test.iloc[1:,:]
    home = pd.concat([train, test])
    # Replace NaNs with before/after entries
    home.fillna(method='pad', inplace=True)
    home.fillna(method='bfill', inplace=True)
    print(home.head())
    print(home.info())
    return home

X = readNprep()

X_c = X.copy(deep=True)
t1 = time.time()
#X_prep = transformFUnion_backup(X_c)
X_prep = transformFUnion(X_c, "homecredit_spec1.json", "homecredit_sk.dat")
#print(X_prep.toarray()[:5, :]) #print first 5 rows
print("Elapsed time for transformations using FeatureUnion = %s sec" % (time.time() - t1))
#np.savetxt("X_prep_sk.csv", X_prep.toarray(), fmt='%1.2f', delimiter=',') #dense

max_stars_repo_name: damslab/reproducibility

max_issues_repo_name: damslab/reproducibility

max_forks_repo_name: damslab/reproducibility


text:
import tensorflow as tf
import numpy as np
from skimage import color, transform

class ModuelTF(object):
    def __init__(self, input_size=[32, 32], n_output=10, path="./datadir", data_index=1000, threshold=0.7):
        self.classes = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
        self.input_size = input_size
        self.n_output = n_output
        self.path = path
        self.data_index = data_index
        self.threshold = threshold
        self.init = tf.global_variables_initializer()
        filepath = self.path + "/myfirst.ckpt_" + str(self.data_index)
        self.output, self.x, self.p_keep_conv, self.p_keep_hidden = self.pre_work()
        self.result = tf.argmax(self.output, 1)
        saver = tf.train.Saver()
        self.sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True))
        init = tf.global_variables_initializer()
        self.sess.run(init)
        tf.reset_default_graph()
        saver.restore(self.sess, filepath)

    def model1(self, _input, _w, _b, p_keep_conv, p_keep_hidden):
        # _input: must be (-1, 16, 16, 1) shape
        _conv1 = tf.nn.conv2d(_input, _w['wc1'], strides=[1, 1, 1, 1], padding='SAME')
        # _conv1 (-1, 16, 16, 8)
        _conv1_r = tf.nn.relu(tf.nn.bias_add(_conv1, _b['bc1']))
        _pool1 = tf.nn.max_pool(_conv1_r, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
        # _pool1 (-1, 8, 8, 8)
        _pool1_drp = tf.nn.dropout(_pool1, p_keep_conv)
        _conv2 = tf.nn.conv2d(_pool1_drp, _w['wc2'], strides=[1, 1, 1, 1], padding='SAME')
        # _conv2 (-1, 8, 8, 16)
        _conv2 = tf.nn.relu(tf.nn.bias_add(_conv2, _b['bc2']))
        _pool2 = tf.nn.max_pool(_conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
        # _pool2 (-1, 4, 4, 16)
        _pool2_drp = tf.nn.dropout(_pool2, p_keep_conv)
        _dense1 = tf.reshape(_pool2_drp, [-1, _w['wd1'].get_shape().as_list()[0]])
        _fc1 = tf.nn.relu(tf.add(tf.matmul(_dense1, _w['wd1']), _b['bd1']))
        _fc1_drp = tf.nn.dropout(_fc1, p_keep_hidden)
        _out = tf.add(tf.matmul(_fc1_drp, _w['wd2']), _b['bd2'])
        return _out

    def forward(self, pic):
        pic = np.array(pic)
        pic = pic.astype(np.float64)
        resized = transform.resize(pic, (self.input_size[0], self.input_size[1]))
        resized = resized.reshape(1, self.input_size[0], self.input_size[1], 3)

        label_index, pred = self.sess.run([self.result, self.output],
                                    feed_dict={self.x: resized, self.p_keep_conv: 1.0, self.p_keep_hidden: 1.0})
        return label_index, pred

    def pre_work(self):
        with tf.device('/gpu:1'):
            weights = {
                'wc1': tf.Variable(tf.random_normal([3, 3, 3, 8], stddev=0.1)),
                'wc2': tf.Variable(tf.random_normal([3, 3, 8, 16], stddev=0.1)),
                'wd1': tf.Variable(tf.random_normal([8 * 8 * 16, 64], stddev=0.1)),
                'wd2': tf.Variable(tf.random_normal([64, self.n_output], stddev=0.1)),
            }

            biases = {
                'bc1': tf.Variable(tf.random_normal([8], stddev=0.1)),
                'bc2': tf.Variable(tf.random_normal([16], stddev=0.1)),
                'bd1': tf.Variable(tf.random_normal([64], stddev=0.1)),
                'bd2': tf.Variable(tf.random_normal([self.n_output], stddev=0.1)),
            }
        x = tf.placeholder(tf.float32, [None, self.input_size[0], self.input_size[1], 3])
        p_keep_conv = tf.placeholder(tf.float32)
        p_keep_hidden = tf.placeholder(tf.float32)

        output = self.model1(x, weights, biases, p_keep_conv, p_keep_hidden)
        return output, x, p_keep_conv, p_keep_hidden

    def get_result(self, pic):
        index, pred = self.forward(pic)
        label = self.classes[index[0]]
        z = zip(label, pred)
        label_new = ""
        for b, c in z:
            prob = max(np.exp(c) / sum(np.exp(c)))
            print(prob)
            if prob < float(self.threshold):
                label_new = "NULL"
            else:
                label_new = b
        return label_new

    def close_session(self):
        self.sess.close()


text:
# Copyright (c) 2018, University of Michigan
# Author: Alex Gorodetsky
# Email: goroda@umich.edu
# License: MIT

from __future__ import print_function
import numpy as np
import scipy.special

class TotalOrder:
    """ Total order index set iterator

    >>> for c in TotalOrder(2, 3):
            print(c)
    >>> [0, 0]
    [0, 1]
    [0, 2]
    [0, 3]
    [1, 0]
    [1, 1]
    [1, 2]
    [2, 0]
    [2, 1]
    [3, 0]

    """
    def __init__(self, dimension, order):
        assert dimension > 0, "Dimension must be greater than zero"
        assert order > -1, "Order must be a positive integer"

        self.d_minus_1 = dimension - 1
        self.on_dim = dimension - 1
        self.current_sum = 0
        self.current_inds = [0]*dimension
        self.start = True
        self.max_sum = order
        self.count = int(scipy.special.binom(dimension + order, dimension))
        self.max_indices = [order] * dimension

    def get_count(self):
        return self.count

    def get_max_indices(self):
        return self.max_indices

    def _print_current_state(self):
        """ Diagnostics Functions """
        print("\n ")
        print("on_dim = ", self.on_dim)
        print("d_minus_1 = ", self.d_minus_1)
        print("current_sum = ", self.current_sum)
        print("max_sum = ", self.max_sum)
        print("current_inds = ", self.current_inds)
        print("start = ", self.start)

    def __iter__(self):
        return self

    def _basefunc(self):

        if 0 <= self.on_dim < self.d_minus_1:
            if self.current_sum < self.max_sum:
                self.current_inds[self.on_dim] = self.current_inds[self.on_dim] + 1
                self.current_sum = self.current_sum + 1
                self.on_dim = self.d_minus_1
                return self.current_inds
            else: #self.current_sum == self.max_sum
                return self._reset()
        elif self.on_dim == self.d_minus_1:
            if self.current_sum < self.max_sum:
                self.current_inds[self.on_dim] = self.current_inds[self.on_dim] + 1
                self.current_sum = self.current_sum + 1
                return self.current_inds
            else:
                return self._reset()
        else: #self.on_dim == -1
            self.on_dim = self.d_minus_1
            self.current_sum = 0
            self.current_inds = [0]*(self.d_minus_1+1)
            self.start = True
            return None

    def _reset(self):
        self.current_sum = self.current_sum - self.current_inds[self.on_dim]
        self.current_inds[self.on_dim] = 0
        self.on_dim = self.on_dim - 1
        return self._basefunc()

    def __next__(self):
        # self._print_current_state()
        if self.start is True:
            self.start = False
            return self.current_inds
        else:
            retval = self._basefunc()
            if retval:
                return retval
            else:
                raise StopIteration

    next = __next__ # for python2

def totalorder1():
    for c in TotalOrder(4, 0):
        print(c)

def totalorder2():
    for c in TotalOrder(1, 2):
        print(c)

def totalorder3():
    for c in TotalOrder(3, 5):
    # for c in TotalOrder(20, 3):
        print(c)

def totalorder4():
    d = list(TotalOrder(3, 5))
    for c in d:
        print(c)

if __name__ == "__main__":

    print("Totalorder 1")
    totalorder1()

    print("\n\n")

    print("Totalorder 2")
    totalorder2()

    print("\n\n")

    print("Totalorder 3")
    totalorder3()

max_stars_repo_name: goroda/pyindex

max_issues_repo_name: goroda/pyindex

max_forks_repo_name: goroda/pyindex

reason: import numpy,import scipy


text:
import pytest
import emgfit as emg
import numpy as np

class Test_spectrum:
    # Create simulated spectrum data
    from emgfit.sample import simulate_events
    true_sigma = 7.77901056381226e-05
    true_theta = 0.6591808159640057
    true_eta_m1 = 0.7393102752716145
    true_eta_m2 = 0.2606897247283855
    true_tau_m1 = 4.4723478031626915e-05
    true_tau_m2 = 0.00011112601042960299
    true_eta_p1 = 0.7315780388972555
    true_eta_p2 = 0.2684219611027445
    true_tau_p1 = 7.130854298242941e-05
    true_tau_p2 = 0.0002741372066519157
    true_bkg_c = 1.036125336704966
    shape_pars = {'sigma' : true_sigma,
                  'theta' : true_theta,
                  'eta_m1': true_eta_m1,
                  'eta_m2': true_eta_m2,
                  'tau_m1': true_tau_m1,
                  'tau_m2': true_tau_m2,
                  'eta_p1': true_eta_p1,
                  'eta_p2': true_eta_p2,
                  'tau_p1': true_tau_p1,
                  'tau_p2': true_tau_p2,
                  'bkg_c' : true_bkg_c}

    # Get literature mass values from AME2020
    m_e = 0.000548579909065 # CODATA value from physics.nist.gov
    m_Ni58 = 57.935341650
    m_err_Ni58 = 0.374e-06
    m_Co58 = 57.935751292
    m_err_Co58 = 1.237e-06
    m_Mn58 = 57.940066643
    m_err_Mn58 = 2.900e-06
    m_Sn116 = 115.901742825
    m_err_Sn116 = 0.103
    ME_Sn116_keV = -91525.979

    true_mus = [m_Ni58 - m_e, m_Co58 - m_e, m_Mn58 - m_e, m_Sn116/2 - m_e] #[57.93479320009094, 57.935203, 57.93959511435116,
               #    115.90064566418187/2]
    true_amps = [0.38916170, 0.05940254, 0.94656384, 0.20934518]
    true_N_events = 67636
    x_min = true_mus[0] - 0.004
    x_max = true_mus[-1] + 0.005
    bin_width = 2.37221e-05
    N_bins = int((x_max - x_min)/bin_width)

    # Set random seed for reproducibility, other seeds can result in
    # assertion errors below
    np.random.seed(12)
    data = simulate_events(shape_pars, true_mus, true_amps, true_bkg_c,
                           true_N_events, x_min, x_max, out='hist',
                           N_bins=N_bins)

    def test_grabbing_of_AME_values(self):
        # Define reference literature values

        m_Ni58_AME16 = 57.935341780 - self.m_e
        m_err_Ni58_AME16 = 0.400e-06
        m_Co58_AME16 = 57.935751429 - self.m_e
        m_err_Co58_AME16 = 1.245e-06
        atol = 1e-09 # tolerance [u] up to which absolute agreement is demanded

        # Instantiate spectrum object
        spec = emg.spectrum(df=self.data, show_plot=False)
        spec.add_peak(57.9, species="Ni58:-1e")
        spec.add_peak(57.95, species="Co58:-1e", lit_src="AME2016")

        # Test defaulting to most recent AME database
        p0 = spec.peaks[0]
        msg0 = "default m_AME value of 'Ni58:-1e' deviates from AME2020 value"
        assert np.isclose(p0.m_AME, self.m_Ni58, atol=atol), msg0
        msg1 = "default m_AME_error of 'Ni58:-1e' deviates from AME2020 value"
        assert np.isclose(p0.m_AME_error, self.m_err_Ni58, atol=atol), msg1

        # Test switching to older AME database via add_peak()
        p1 = spec.peaks[1]
        msg2 = "AME2016 value invoked with add_peak() deviates from reference"
        assert np.isclose(p1.m_AME, m_Co58_AME16, atol), msg2
        msg3 = "AME2016 error invoked with add_peak() deviates from reference"
        assert np.isclose(p1.m_AME_error, m_err_Co58_AME16, atol), msg3
        msg4 = "Flagging for AME2016 values invoked with add_peak() failed"
        assert 'lit_src: AME2016' in p1.comment, msg4

        # Test switching to older AME database via assign_species()
        spec.assign_species("Ni58:-1e", peak_index=0, lit_src = 'AME2016')
        msg5 = "AME2016 value invoked with assign_species() deviates from ref."
        assert np.isclose(p0.m_AME, m_Ni58_AME16, atol), msg5
        msg6 = "AME2016 error invoked with assign_species() deviates from ref."
        assert np.isclose(p0.m_AME_error, m_err_Ni58_AME16, atol), msg6
        msg7 = "Flagging for AME16 values invoked with assign_species() failed"
        assert 'lit_src: AME2016' in p0.comment, msg7

    def test_fitting_accuracy(self):
        """Check accuracy of fitting using simulated spectrum and test
        calculation of literature values for doubly charged and isomeric species

        """
        # Instantiate spectrum object, calibrate peak shape and fit all peaks
        spec = emg.spectrum(df=self.data,show_plot=False)
        spec.detect_peaks(thres=0.0053, plot_smoothed_spec=False,
                          plot_2nd_deriv=False, plot_detection_result=False)
        msg0 = "Incorrect number of peaks detected."
        assert len(spec.peaks) == len(self.true_mus), msg0
        spec.assign_species(["Ni58:-1e","Co58:-1e","Mn58?:-1e","Sn116:-2e"])
        spec.assign_species("Mn58m?:-1e", peak_index=2, Ex=71.77, Ex_error=0.05)
        spec.determine_peak_shape(species_shape_calib="Mn58m?:-1e",
                                  show_plots=False)
        spec.fit_peaks(species_mass_calib="Ni58:-1e",show_plots=False)

        # Perform accuracy checks
        for p in spec.peaks:
            if p.species == "Ni58:-1e":
                continue # skip calibrant
            msg1 = "ME deviates from literature by more than 1 sigma."
            assert p.m_dev_keV <= p.mass_error_keV, msg1

            # Check calculation of (atomic) ME for doubly charged species
            if p.species == "Sn116:-2e":
                ME_dev_keV = p.atomic_ME_keV - self.ME_Sn116_keV
                msg2 = str("Respective deviation of ionic mass and atomic mass "
                           "excess from literature differ by > 1 sigma for "
                           "Sn116:-2e.")
                assert abs(ME_dev_keV - p.m_dev_keV) < p.mass_error_keV, msg2

max_stars_repo_name: RobbenRoll/emgfit

max_issues_repo_name: RobbenRoll/emgfit

max_forks_repo_name: RobbenRoll/emgfit