text:
"""
Finetune Bertweet to predict market sentiment of financial tweets
"""
import numpy as np
from datasets import load_dataset, load_metric
from transformers import TrainingArguments, Trainer
from fintweet_sentiment_classification_model import BertweetForSequenceClassification
from utils import tokenize_function
"""
FILE specification
- columns=['body', 'label']
- dtype=[str, int]
- 0 <= label < num_classes
    - negative: 0
    - neutral : 1
    - positive: 2
"""

TRAIN_FILE='./sentiment-classification-train.csv'
EVAL_FILE='./sentiment-classification-eval.csv'
TEST_FILE='./sentiment-classification-test.csv'

raw_datasets = load_dataset('csv', data_files={
  "train": TRAIN_FILE,
  "eval": EVAL_FILE,
  "test": TEST_FILE
})

metric = load_metric("accuracy")
def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return metric.compute(predictions=predictions, references=labels)

tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)

train_dataset = tokenized_datasets["train"]
eval_dataset = tokenized_datasets["eval"]
test_dataset = tokenized_datasets["test"]

model = BertweetForSequenceClassification.from_pretrained("vinai/bertweet-base", num_labels=3)
training_args = TrainingArguments("fintweet-sentiment-classifier", evaluation_strategy="epoch")
trainer = Trainer(
    model=model, args=training_args, train_dataset=train_dataset, eval_dataset=eval_dataset, compute_metrics=compute_metrics,
)

trainer.train()
trainer.evaluate()
_, _, metrics = trainer.predict(test_dataset=test_dataset)
print(metrics)

# TODO test accuracy: 91% -> accuracy 개선하기
model.save_pretrained("./fintweet-sentiment-classifier")

max_stars_repo_name: retroinspect/tweetstock

max_issues_repo_name: retroinspect/tweetstock

max_forks_repo_name: retroinspect/tweetstock


text:
import csv

import gspread
import pandas as pd
import numpy as np
import requests

from bs4 import BeautifulSoup
from datetime import date, datetime, timedelta
from pytz import timezone

from log_gsheets import login_sheets

def generate_past_dates():
  url_list= []
  calendar_list = []

  tz_date = datetime.now(timezone('America/Bahia'))
  #tz_date  = datetime.strptime("2022-03-21", '%Y-%m-%d').date() #Usar para capturar outras datas

  for i in range(30):
    tz_date = tz_date - timedelta(days=1)
    past_date = tz_date.strftime('%d/%m/%Y')
    past_url_date = tz_date.strftime('%Y-%m-%d')
    past_url = f"https://www.gov.br/planalto/pt-br/acompanhe-o-planalto/agenda-do-presidente-da-republica/{past_url_date}"

    url_list.append(past_url)
    calendar_list.append(past_date)

  return url_list, calendar_list

def generate_past_datas():
  url_list, calendar_list = generate_past_dates()

  dict_agenda = []
  list_soup = []

  for url in url_list:
    reply = requests.get(url)
    html_past = reply.text
    bsoup_past = BeautifulSoup(html_past, "html.parser")
    list_soup.append(bsoup_past)

    for soup_past in list_soup:
      #pegar hora do início
      tag_start = soup_past.find_all("time", {"class": "compromisso-inicio"})
      start_hour = [i.text for i in tag_start]
      if start_hour == []:
          start_hour = float('NaN')

      #pegar hora do fim
      tag_end= soup_past.find_all("time", {"class": "compromisso-fim"})
      end_hour = [h.text for h in tag_end]
      if end_hour == []:
          end_hour = float('NaN')

      #pegar compromisso
      tag_appointment = soup_past.find_all("h2", {"class": "compromisso-titulo"})
      appointment = [c.text for c in tag_appointment]
      if appointment == []:
          appointment = ["Sem compromisso oficial"]

      #pegar local
      tag_place = soup_past.find_all("div", {"class": "compromisso-local"})
      place = [l.text for l in tag_place]
      if place == []:
          place = float('NaN')

    dict_agenda.append({"hora_inicio":start_hour, "hora_fim": end_hour, "compromisso": appointment, "local":place})

  return dict_agenda

def generate_past_df():
  url_list, calendar_list = generate_past_dates()
  dict_agenda = generate_past_datas()

  # criando dataframe:
  df_dict = pd.DataFrame(dict_agenda)
  df_dict["data"]=pd.Series(calendar_list)

  # usando explode para explodir as listas:
  stacked = df_dict.stack().explode().reset_index()
  stacked["uid"] = stacked.groupby(["level_0", "level_1"]).cumcount()
  past_df = stacked.pivot(["level_0", "uid"], "level_1", 0).reset_index(drop=True).rename_axis(None, axis=1)
  past_df["data"] = past_df["data"].fillna(method="ffill")
  past_df = past_df[["data", "hora_inicio", "hora_fim", "compromisso", "local"]]

  #substituindo NaN por " ":
  past_df.replace([np.inf, -np.inf], np.nan, inplace=True)
  past_df.fillna(' ', inplace=True)

  return past_df

#para quando rodar o código com datas específicas
#def send_past_sheets():
    #past_df = generate_past_df()
    #spreadsheet = login_sheets()

    #worksheet = spreadsheet.worksheet("dados_agenda")
    #worksheet.append_rows(past_df.values.tolist())

#send_past_sheets()

#if __name__ == "__main__":
    #send_past_sheets()

ext: py

max_stars_repo_name: oquejair/oquejair

max_issues_repo_name: oquejair/oquejair

max_forks_repo_name: oquejair/oquejair

reason: import numpy


text:
'''
SOVBAlg.py

Implementation of stochastic online VB (soVB) for bnpy models
'''
import os
import numpy as np
import scipy.sparse

from bnpy.learnalg.LearnAlg import LearnAlg
from bnpy.learnalg.LearnAlg import makeDictOfAllWorkspaceVars
from bnpy.learnalg import ElapsedTimeLogger
from bnpy.util.SparseRespUtil import sparsifyResp

class SOVBAlg(LearnAlg):

    def __init__(self, **kwargs):
        ''' Creates stochastic online learning algorithm,
            with fields rhodelay, rhoexp that define learning rate schedule.
        '''
        super(type(self), self).__init__(**kwargs)
        self.rhodelay = self.algParams['rhodelay']
        self.rhoexp = self.algParams['rhoexp']
        self.LPmemory = dict()

    def fit(self, hmodel, DataIterator, SS=None):
        ''' Run stochastic variational to fit hmodel parameters to Data.

        Returns
        --------
        Info : dict of run information.

        Post Condition
        --------
        hmodel updated in place with improved global parameters.
        '''
        self.set_start_time_now()
        LP = None
        rho = 1.0  # Learning rate
        nBatch = float(DataIterator.nBatch)

        # Set-up progress-tracking variables
        iterid = -1
        lapFrac = np.maximum(0, self.algParams['startLap'] - 1.0 / nBatch)
        if lapFrac > 0:
            # When restarting an existing run,
            #  need to start with last update for final batch from previous lap
            DataIterator.lapID = int(np.ceil(lapFrac)) - 1
            DataIterator.curLapPos = nBatch - 2
            iterid = int(nBatch * lapFrac) - 1

        # Save initial state
        self.saveParams(lapFrac, hmodel)

        # Custom func hook
        self.eval_custom_func(
            isInitial=1, **makeDictOfAllWorkspaceVars(**vars()))
        ElapsedTimeLogger.writeToLogOnLapCompleted(lapFrac)

        if self.algParams['doMemoELBO']:
            SStotal = None
            SSPerBatch = dict()
        else:
            loss_running_sum = 0
            loss_per_batch = np.zeros(nBatch)
        while DataIterator.has_next_batch():

            # Grab new data
            Dchunk = DataIterator.get_next_batch()
            batchID = DataIterator.batchID
            Dchunk.batchID = batchID

            # Update progress-tracking variables
            iterid += 1
            lapFrac += 1.0 / nBatch
            self.lapFrac = lapFrac
            nLapsCompleted = lapFrac - self.algParams['startLap']
            self.set_random_seed_at_lap(lapFrac)

            # E step
            self.algParamsLP['batchID'] = batchID
            self.algParamsLP['lapFrac'] = lapFrac  # logging
            if batchID in self.LPmemory:
                batchLP = self.load_batch_local_params_from_memory(batchID)
            else:
                batchLP = None
            LP = hmodel.calc_local_params(Dchunk, batchLP,
                doLogElapsedTime=True,
                **self.algParamsLP)
            rho = (1 + iterid + self.rhodelay) ** (-1.0 * self.rhoexp)
            if self.algParams['doMemoELBO']:
                # SS step. Scale at size of current batch.
                SS = hmodel.get_global_suff_stats(Dchunk, LP,
                                                  doLogElapsedTime=True,
                                                  doPrecompEntropy=True)
                if self.algParams['doMemoizeLocalParams']:
                    self.save_batch_local_params_to_memory(
                        batchID, LP)
                # Incremental updates for whole-dataset stats
                # Must happen before applification.
                if batchID in SSPerBatch:
                    SStotal -= SSPerBatch[batchID]
                if SStotal is None:
                    SStotal = SS.copy()
                else:
                    SStotal += SS
                SSPerBatch[batchID] = SS.copy()

                # Scale up to size of whole dataset.
                if hasattr(Dchunk, 'nDoc'):
                    ampF = Dchunk.nDocTotal / float(Dchunk.nDoc)
                    SS.applyAmpFactor(ampF)
                else:
                    ampF = Dchunk.nObsTotal / float(Dchunk.nObs)
                    SS.applyAmpFactor(ampF)
                # M step with learning rate
                hmodel.update_global_params(SS, rho, doLogElapsedTime=True)
                # ELBO step
                assert not SStotal.hasAmpFactor()
                loss = -1 * hmodel.calc_evidence(
                    SS=SStotal,
                    doLogElapsedTime=True,
                    afterGlobalStep=not self.algParams['useSlackTermsInELBO'])
            else:
                # SS step. Scale at size of current batch.
                SS = hmodel.get_global_suff_stats(Dchunk, LP,
                    doLogElapsedTime=True)

                # Scale up to size of whole dataset.
                if hasattr(Dchunk, 'nDoc'):
                    ampF = Dchunk.nDocTotal / float(Dchunk.nDoc)
                    SS.applyAmpFactor(ampF)
                else:
                    ampF = Dchunk.nObsTotal / float(Dchunk.nObs)
                    SS.applyAmpFactor(ampF)

                # M step with learning rate
                hmodel.update_global_params(SS, rho, doLogElapsedTime=True)

                # ELBO step
                assert SS.hasAmpFactor()
                cur_batch_loss = -1 * hmodel.calc_evidence(
                    Dchunk, SS, LP, doLogElapsedTime=True)
                if loss_per_batch[batchID] != 0:
                    loss_running_sum -= loss_per_batch[batchID]
                loss_running_sum += cur_batch_loss
                loss_per_batch[batchID] = cur_batch_loss
                loss = loss_running_sum / nBatch

            # Display progress
            self.updateNumDataProcessed(Dchunk.get_size())
            if self.isLogCheckpoint(lapFrac, iterid):
                self.printStateToLog(hmodel, loss, lapFrac, iterid, rho=rho)

            # Save diagnostics and params
            if self.isSaveDiagnosticsCheckpoint(lapFrac, iterid):
                self.saveDiagnostics(lapFrac, SS, loss)
            if self.isSaveParamsCheckpoint(lapFrac, iterid):
                self.saveParams(lapFrac, hmodel, tryToSparsifyOutput=1)
                # don't save SS here, since its for one batch only
            self.eval_custom_func(**makeDictOfAllWorkspaceVars(**vars()))

            if self.isLastBatch(lapFrac):
                ElapsedTimeLogger.writeToLogOnLapCompleted(lapFrac)
            # .... end loop over data

        # Finished! Save, print and exit
        self.printStateToLog(hmodel, loss, lapFrac, iterid, isFinal=1)
        self.saveParams(lapFrac, hmodel, SS)
        self.eval_custom_func(
            isFinal=1, **makeDictOfAllWorkspaceVars(**vars()))

        return self.buildRunInfo(Data=DataIterator, loss=loss, SS=SS)

    def load_batch_local_params_from_memory(self, batchID, doCopy=0):
        ''' Load local parameter dict stored in memory for provided batchID

        TODO: Fastforward so recent truncation changes are accounted for.

        Returns
        -------
        batchLP : dict of local parameters specific to batchID
        '''
        batchLP = self.LPmemory[batchID]
        if isinstance(batchLP, str):
            ElapsedTimeLogger.startEvent('io', 'loadlocal')
            batchLPpath = os.path.abspath(batchLP)
            assert os.path.exists(batchLPpath)
            F = np.load(batchLPpath, allow_pickle=True)
            indptr = np.arange(
                0, (F['D']+1)*F['nnzPerDoc'],
                F['nnzPerDoc'])
            batchLP = dict()
            batchLP['DocTopicCount'] = scipy.sparse.csr_matrix(
                (F['data'], F['indices'], indptr),
                shape=(F['D'], F['K'])).toarray()
            ElapsedTimeLogger.stopEvent('io', 'loadlocal')
        if doCopy:
            # Duplicating to avoid changing the raw data stored in LPmemory
            # Usually for debugging only
            batchLP = copy.deepcopy(batchLP)
        return batchLP

    def save_batch_local_params_to_memory(self, batchID, batchLP):
        ''' Store certain fields of the provided local parameters dict
              into "memory" for later retrieval.
            Fields to save determined by the memoLPkeys attribute of this alg.
        '''
        batchLP = dict(**batchLP) # make a copy
        allkeys = list(batchLP.keys())
        for key in allkeys:
            if key != 'DocTopicCount':
                del batchLP[key]
        if len(list(batchLP.keys())) > 0:
            if self.algParams['doMemoizeLocalParams'] == 1:
                self.LPmemory[batchID] = batchLP
            elif self.algParams['doMemoizeLocalParams'] == 2:
                ElapsedTimeLogger.startEvent('io', 'savelocal')
                spDTC = sparsifyResp(
                    batchLP['DocTopicCount'],
                    self.algParams['nnzPerDocForStorage'])
                wc_D = batchLP['DocTopicCount'].sum(axis=1)
                wc_U = np.repeat(wc_D, self.algParams['nnzPerDocForStorage'])
                spDTC.data *= wc_U
                savepath = self.savedir.replace(os.environ['BNPYOUTDIR'], '')
                if os.path.exists('/ltmp/'):
                    savepath = '/ltmp/%s/' % (savepath)
                else:
                    savepath = '/tmp/%s/' % (savepath)
                from distutils.dir_util import mkpath
                mkpath(savepath)
                savepath = os.path.join(savepath, 'batch%d.npz' % (batchID))
                # Now actually save it!
                np.savez(savepath,
                    data=spDTC.data,
                    indices=spDTC.indices,
                    D=spDTC.shape[0],
                    K=spDTC.shape[1],
                    nnzPerDoc=spDTC.indptr[1])
                self.LPmemory[batchID] = savepath
                del batchLP
                del spDTC
                ElapsedTimeLogger.stopEvent('io', 'savelocal')

max_stars_repo_name: raphael-group/bnpy

max_issues_repo_name: raphael-group/bnpy

max_forks_repo_name: raphael-group/bnpy

reason: import numpy,import scipy


text:
import matplotlib.pyplot as plt
import matplotlib as mpl
import scipy as sp
import matplotlib.cm as cm
import numpy as np
import pandas as pd
import seaborn as sns

with open("output.txt") as jurassicpark:
	jurassicpark_txt = jurassicpark.read()
	#jurassicpark.close()
	jurassicpark_tokens = jurassicpark_txt.split() #with no arguments, splits on whitespace
	print(len(jurassicpark_tokens))

lastword = jurassicpark_tokens[-1]
print(lastword)

strangestring = ",".join(jurassicpark_tokens)
strangestring[:100]

jurassicparklc_tokens = [word.lower() for word in jurassicpark_tokens]
count = jurassicparklc_tokens.count("had")
print(count)

#find unique set of words, count how often used with count method on lists
uniquelctokens = set(jurassicparklc_tokens)

tokendict = {}
for ut in uniquelctokens:
	tokendict[ut] = jurassicparklc_tokens.count(ut)

el = sorted(tokendict.items(), key= lambda args: args[1], reverse = True)[:100]
print(el)

topfreq = el[:20]
print(topfreq)
with sns.axes_style('whitegrid'):
	pos = np.arange(len(topfreq))
	plt.bar(pos, [e[1] for e in topfreq]);
	plt.xticks(pos+0.4, [e[0] for e in topfreq]);
	plt.xlabel("Words")
	plt.ylabel("Count")
	plt.title("Word Frequency in output text")

plt.show()

max_stars_repo_name: matthapenney/word-rnn-tensorflow

max_issues_repo_name: matthapenney/word-rnn-tensorflow

max_forks_repo_name: matthapenney/word-rnn-tensorflow


text:
# see license.txt
# python vecplt42.py 250 gfs-0250/speed-0000.jpg map3.pbm ../gfs-0250/uv2048.png ../gfs-0250/uv1024.png

from PIL import Image
from PIL import ImageDraw
from PIL import ImageChops
from PIL import ImageEnhance
import numpy as np
import sys

press=int(sys.argv[1])
uvname="gfs-%04d/uv-0000.npz"%press
mapname=sys.argv[3]
pngname=sys.argv[4]
pngname2=sys.argv[5]
imname=sys.argv[2]
map0=Image.open(imname)
ovl=Image.open(mapname)
ovl2=ovl.convert("RGB")
en=ImageEnhance.Brightness(ovl2)
ovl3=en.enhance(0.4)
map=Image.composite(ovl3,map0,ovl)

maps=map.copy()

degrad=np.pi/180.0

NX=45
NY=20
#x,y=np.meshgrid(np.linspace(0,360,45,endpoint=False),np.linspace(-84,84,22))
x,y=np.meshgrid(np.linspace(0,360,NX,endpoint=False),np.linspace(-76,76,NY))
#x,y=np.meshgrid(np.linspace(0,1,45,endpoint=False),np.linspace(-84,84,22))
x=np.reshape(x,(NX*NY,1))[:,0]
y=np.reshape(y,(NX*NY,1))[:,0]
x0=np.copy(x)
y0=np.copy(y)

dr=ImageDraw.Draw(map)

def addmap(dr,x,y,col,szflg):
    nx=map.size[0]
    ny=map.size[1]
    n=x.shape[0]
    for j in range(0,n):
        lt=y[j]
        ln=x[j]+180
        ln=ln-360*(ln>=360)
        ln=np.round(ln/360.0*nx)
        lt=np.round((90-lt)/180.0*ny)
#        if szflg:
#            ln=int(ln)
#            lt=int(lt)
#            xy=(ln-1,lt-1,ln+1,lt+1)
##            dr.chord(xy,0,350,fill=col,outline="black")
#            dr.chord(xy,0,350,fill=col)
#        else:
#            dr.point((ln,lt),fill=col)
        if szflg:
            ln=int(ln)
            lt=int(lt)
            xy=(ln-1,lt-1,ln+1,lt+1)
            dr.line([ln-1,lt,ln+1,lt],fill=col)
            dr.line([ln,lt-1,ln,lt+1],fill=col)
        else:
            dr.point((ln,lt),fill=col)

R_e=6378
msecdegday=360.0*86400/2/np.pi/R_e/1000;

uvf=np.load(uvname)
u=uvf['u']
v=uvf['v']
lon=np.arange(0,360,0.25)
lat=np.arange(90.0,-90.1,-0.25)
#uvf['lat']
#lon=np.aragne(0
#lon=uvf['lon']
nx=len(lon);
dx=lon[1]-lon[0];
ny=len(lat);
dy=lat[0]-lat[1];

def interp(u,v,xa,ya):
    ua=np.zeros((xa.shape[0],2))
    for j in range(0,xa.shape[0]):
        x=xa[j]
        y=ya[j]
        ix=int(np.floor(x/dx))
        sx=x/dx-ix
        if ix>=nx: ix=0
        ix1=ix+1
        if ix1>=nx: ix1=0
        iy=int(np.floor((90-y)/dy))
        sy=(90-y)/dy-iy
        if iy>=ny: iy=0
        iy1=iy+1
        if iy1>=ny: iy1=0
        try:
            ui=np.array([[u[iy,ix],u[iy,ix1]],[u[iy1,ix],u[iy1,ix1]]])
            vi=np.array([[v[iy,ix],v[iy,ix1]],[v[iy1,ix],v[iy1,ix1]]])
        except:
            print x,y,ix,iy,ix1,iy1
        w=np.array([[(1-sy)*(1-sx),(1-sy)*sx],[sy*(1-sx),sy*sx]])
        uf=np.sum(ui*w)/np.cos(degrad*y)*msecdegday
        vf=np.sum(vi*w)*msecdegday
        ua[j,0]=uf
        ua[j,1]=vf
#  print x,y,s,uf,vf
    return ua

dt=1.0/1024.0
t=0

#while t<1:
while t<0.5:
#    print t
    vel=interp(u,v,x,y)
    dx0 = vel[:,0]*dt
    dy0 = vel[:,1]*dt
    t += dt
    x += dx0
    y += dy0
    x += 360*(x<0)-360*(x>=360)
    vel=interp(u,v,x,y)
    dx0=vel[:,0]*dt-dx0
    dy0=vel[:,1]*dt-dy0
    x += 0.5*dx0
    y += 0.5*dy0
    x += 360*(x<0)-360*(x>=360)
    addmap(dr,x,y,"white",False)

addmap(dr,x0,y0,"red",True)
#addmap(dr,x0,y0,"red",False)

map.save(pngname)
# execfile("vecplt2.py")
map0=map.resize((1024,512),Image.ANTIALIAS)
map0.save(pngname2)

max_stars_repo_name: grflierl/met

max_issues_repo_name: grflierl/met

max_forks_repo_name: grflierl/met


text:
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import math
import time

start = time.thread_time()

data0 = np.array(pd.read_csv("./code/graph1.csv", header=2))
data = data0  # 从csv文件获取数据
d = -1  # 精度
plt.axis("equal")
plt.plot(data[:, 0], data[:, 1], '-o', markersize=1)
length = 0
times = 0
dots = 0

def unit(v):  # 单位化
    return(v/np.linalg.norm(v))

def angle(v):  # 取辐角
    return(math.atan2(v[1], v[0]))

def inangle(v1, v2):  # 向量夹角
    return(math.acos(round(np.dot(v1, np.transpose(v2)) / (np.linalg.norm(v1)*np.linalg.norm(v2)), 9)))

def cross(v1, v2):  # 平面内向量叉乘
    return(v1[0]*v2[1]-v2[0]*v1[1])

def ifcross(p1, p2, q1, q2):
    v11 = q1-p1
    v12 = q2-p1
    v21 = q1-p2
    v22 = q2-p2
    if cross(v11, v12)*cross(v21, v22) < 0 and cross(v11, v21)*cross(v12, v22) < 0:
        return(1)
    else:
        return(0)

def delcross(temp):
    i = 0
    while i < temp.shape[0]-3:
        global times
        j = i
        while j < i+(temp.shape[0]-i)//4:
            if ifcross(temp[i, :], temp[i+1, :], temp[j, :], temp[j+1, :]):
                temp = np.row_stack((temp[0:i, :], temp[j+1:, :]))
                continue
            else:
                j += 1
        i += 1
    return(temp)

def draw(data):  # 画等高线
    global dots
    data = np.insert(data, data.shape[0], values=data[1, :], axis=0)
    data = np.insert(data, 0, values=data[data.shape[0]-3, :], axis=0)
    temp = np.array([0, 0])
    i = 0
    while i < data.shape[0]-2:
        v1 = data[i+1, :]-data[i, :]
        v2 = data[i+2, :]-data[i+1, :]
        if 0 < inangle(v1, v2) < math.pi*0.9:  # 一般情况在菱形中使用向量得到内缩点
            u = d/(math.sin(inangle(v1, v2)))
            if cross(v1, v2) > 0:
                new = data[i+1, :]+(unit(v2)-unit(v1))*u
            else:
                new = data[i+1, :]-(unit(v2)-unit(v1))*u
        else:
            if inangle(v1, v2) == 0:  # 两向量平行的特殊情况
                if angle(v1) > 0:
                    new = data[i+1, :] + unit([v1[1], -v1[0]])*abs(d)
                else:
                    new = data[i+1, :] - unit([-v1[1], v1[0]])*abs(d)
            else:  # 排除转角过大的点
                i += 1
                continue
        i += 1
        temp = np.row_stack((temp, new))
        dots += 1
    temp = np.delete(temp, 0, axis=0)
    temp = ifwide(temp, data)  # 与上一级间距控制
    temp = iflong(temp)  # 同级点间距控制
    return(temp)

def iflong(data):  # 同级点间距控制
    i = 0
    while i < data.shape[0]-1:  # 遍历所有数据
        if np.linalg.norm(data[i+1, :]-data[i, :]) > 2*abs(d):  # 两点间距过大的添加中点
            new = np.array([(data[i+1, 0]+data[i, 0])/2,
                            (data[i+1, 1]+data[i, 1])/2])
            data = np.insert(data, i+1, new,  axis=0)
            continue
        else:
            i = i+1
    return(data)

def ifwide(data, last):  # 与上一级间距控制
    i = 0
    while i < data.shape[0]:  # 遍历该级所有数据
        j = max([0, i-20])
        while j < min(last.shape[0], i+20):  # 遍历上级部分数据
            if np.linalg.norm(data[i, :]-last[j, :]) < abs(d)*0.999:  # 小于一个精度的直接删除
                data[i] = [0, 0]
                break
            else:
                j += 1
        i += 1
    i = 0
    while i < data.shape[0]:
        if data[i, 0] == data[i, 1] == 0:
            data = np.delete(data, i, axis=0)
            continue
        i += 1
    return(data)

def ifdivide(data):  # 判断区域划分
    for i in range(data.shape[0]-2):
        for j in range(i, data.shape[0]-2):
            x1 = data[i, 0]
            y1 = data[i, 1]
            x2 = data[j, 0]
            y2 = data[j, 1]
            if 0 < math.sqrt((x2-x1)**2+(y2-y1)**2) < abs(d) and j-i > 3:  # 间距过近且向量方向差超过90度
                v1 = data[i+2, :]-data[i, :]
                v2 = data[j+2, :]-data[j, :]
                if abs(angle(v1)-angle(v2)) > math.pi/2:
                    return(np.array([i, j]))
    return(np.array([0, 0]))

def drawline(data):  # 判断是否需要分割
    global length
    global times
    while True:
        temp = data[-1]
        if data[0].shape[0] < 12:
            writecsv(temp)
            break
        if temp.shape[0] < 12:
            del data[len(data)-1]
            print('-')
            continue
        index = ifdivide(temp)  # 分割点序号
        if index[0] == 0 and index[1] == 0:
            if times > 0:
                temp = delcross(temp)
                writecsv(temp)
                plt.plot(temp[:, 0], temp[:, 1], '-', color='r', markersize=3)
            data[-1] = draw(temp)
            times += 1
            print(times)
            for j in range(data[-1].shape[0]-1):
                length = length + \
                    math.sqrt((data[-1][j+1, 0]-data[-1][j, 0])**2 +
                              (data[-1][j+1, 1]-data[-1][j, 1])**2)
        else:
            data.append(temp[math.floor(index[0])+1: math.floor(index[1]), :])
            data[-1] = np.row_stack((data[-1], data[-1][0:1, :]))
            temp1 = temp[0:math.floor(index[0])+1, :]
            temp2 = temp[math.floor(index[1]):temp.shape[0], :]
            data[-2] = np.row_stack((temp1, temp2))
    pass

def writecsv(data):
    global times
    dataframe = pd.DataFrame(data={'x': data[:, 0], 'y': data[:, 1]})
    dataframe.to_csv(f"./code/contour{times}.csv",
                     index=False, mode='w', sep=',')
    pass

data = list([data])
drawline(data)

end = time.thread_time()
print('Length of curve: %s mm' % length)
print('Number of turns: %s' % times)
print('Number of dots:  %s' % dots)
print('Running time:    %s Seconds' % (end-start))

plt.show()


text:
import jax
from pyscfad import gto, scf, cc

mol = gto.Mole()
mol.atom = 'H 0. 0. 0.; F 0. 0. 1.1'
mol.basis = 'ccpvdz'
mol.verbose = 5
mol.incore_anyway = True
mol.build()

def ccsd(mol, dm0=None, t1=None, t2=None):
    mf = scf.RHF(mol)
    mf.kernel(dm0)
    mycc = cc.RCCSD(mf)
    mycc.kernel(t1=t1, t2=t2)
    return mycc.e_tot

jac = jax.jacrev(ccsd)(mol)
print(jac.coords)

max_stars_repo_name: yangdatou/pyscfad

max_issues_repo_name: yangdatou/pyscfad

max_forks_repo_name: yangdatou/pyscfad

reason: import jax


text:
# Copyright 2018 The TensorFlow Constrained Optimization Authors. All Rights
# Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
# ==============================================================================
"""Defines `LagrangianOptimizer`.

This optimizer minimizes a `ConstrainedMinimizationProblem` by introducing
Lagrange multipliers, and using `tf.train.Optimizer`s to jointly optimize over
the model parameters and Lagrange multipliers.

For the purposes of constrained optimization, at least in theory, external
regret minimization of the Lagrangian formulation suffices if the
`ConstrainedMinimizationProblem` we're optimizing doesn't have any
"proxy_constraints", while swap regret minimization of the proxy-Lagrangian
formulation (using the `ProxyLagrangianOptimizer`) is recommended if
"proxy_constraints" are present.

For more specifics, please refer to:

> Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex
> Constrained Optimization". ALT'19.
> [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500)

The formulation used by the LagrangianOptimizer--which is simply the usual
Lagrangian formulation--can be found in Definition 1, and is discussed in
Section 3. This optimizer is most similar to Algorithm 3 in Appendix C.3, with
the two differences being that it uses proxy constraints (if they're provided)
in the update of the model parameters, and uses `tf.train.Optimizer`s, instead
of SGD, for the "inner" updates.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf

from tensorflow_constrained_optimization.python.train import constrained_optimizer

def _project_multipliers_wrt_euclidean_norm(multipliers, radius):
  """Projects its argument onto the feasible region.

  The feasible region is the set of all vectors with nonnegative elements that
  sum to at most "radius".

  Args:
    multipliers: rank-1 `Tensor`, the Lagrange multipliers to project.
    radius: float, the radius of the feasible region.

  Returns:
    The rank-1 `Tensor` that results from projecting "multipliers" onto the
    feasible region w.r.t. the Euclidean norm.

  Raises:
    TypeError: if the "multipliers" `Tensor` is not floating-point.
    ValueError: if the "multipliers" `Tensor` does not have a fully-known shape,
      or is not one-dimensional.
  """
  if not multipliers.dtype.is_floating:
    raise TypeError("multipliers must have a floating-point dtype")
  multipliers_shape = multipliers.get_shape()
  if multipliers_shape.ndims is None:
    raise ValueError("multipliers must have known shape")
  if multipliers_shape.ndims != 1:
    raise ValueError(
        "multipliers must be one dimensional (instead is %d-dimensional)" %
        multipliers_shape.ndims)
  dimension = multipliers_shape[0].value
  if dimension is None:
    raise ValueError("multipliers must have fully-known shape")

  def while_loop_condition(iteration, multipliers, inactive, old_inactive):
    """Returns false if the while loop should terminate."""
    del multipliers  # Needed by the body, but not the condition.
    not_done = (iteration < dimension)
    not_converged = tf.reduce_any(tf.not_equal(inactive, old_inactive))
    return tf.logical_and(not_done, not_converged)

  def while_loop_body(iteration, multipliers, inactive, old_inactive):
    """Performs one iteration of the projection."""
    del old_inactive  # Needed by the condition, but not the body.
    iteration += 1
    scale = tf.minimum(0.0, (radius - tf.reduce_sum(multipliers)) / tf.maximum(
        1.0, tf.reduce_sum(inactive)))
    multipliers = multipliers + (scale * inactive)
    new_inactive = tf.cast(multipliers > 0, multipliers.dtype)
    multipliers = multipliers * new_inactive
    return (iteration, multipliers, new_inactive, inactive)

  iteration = tf.constant(0)
  inactive = tf.ones_like(multipliers, dtype=multipliers.dtype)

  # We actually want a do-while loop, so we explicitly call while_loop_body()
  # once before tf.while_loop().
  iteration, multipliers, inactive, old_inactive = while_loop_body(
      iteration, multipliers, inactive, inactive)
  iteration, multipliers, inactive, old_inactive = tf.while_loop(
      while_loop_condition,
      while_loop_body,
      loop_vars=(iteration, multipliers, inactive, old_inactive),
      name="euclidean_projection")

  return multipliers

class LagrangianOptimizer(constrained_optimizer.ConstrainedOptimizer):
  """A `ConstrainedOptimizer` based on the Lagrangian formulation.

  This `ConstrainedOptimizer` uses the given `tf.train.Optimizer`s to jointly
  minimize over the model parameters, and maximize over Lagrange multipliers,
  with the latter maximization using additive updates and an algorithm that
  minimizes external regret.

  For more specifics, please refer to:

  > Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex
  > Constrained Optimization". ALT'19.
  > [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500)

  The formulation used by this optimizer--which is simply the usual Lagrangian
  formulation--can be found in Definition 1, and is discussed in Section 3. It
  is most similar to Algorithm 3 in Appendix C.3, with the two differences being
  that it uses proxy constraints (if they're provided) in the update of the
  model parameters, and uses `tf.train.Optimizer`s, instead of SGD, for the
  "inner" updates.
  """

  def __init__(self,
               optimizer,
               constraint_optimizer=None,
               maximum_multiplier_radius=None):
    """Constructs a new `LagrangianOptimizer`.

    The difference between "optimizer" and "constraint_optimizer" (if the latter
    is provided) is that the former is used for learning the model parameters,
    while the latter us used for the Lagrange multipliers. If no
    "constraint_optimizer" is provided, then "optimizer" is used for both.

    Args:
      optimizer: tf.train.Optimizer, used to optimize the objective and
        proxy_constraints portion of ConstrainedMinimizationProblem. If
        constraint_optimizer is not provided, this will also be used to optimize
        the Lagrange multipliers.
      constraint_optimizer: optional tf.train.Optimizer, used to optimize the
        Lagrange multipliers.
      maximum_multiplier_radius: float, an optional upper bound to impose on the
        sum of the Lagrange multipliers.

    Raises:
      ValueError: if the maximum_multiplier_radius parameter is nonpositive.
    """
    super(LagrangianOptimizer, self).__init__(optimizer=optimizer)
    self._constraint_optimizer = constraint_optimizer

    if maximum_multiplier_radius and (maximum_multiplier_radius <= 0.0):
      raise ValueError("maximum_multiplier_radius must be strictly positive")

    self._maximum_multiplier_radius = maximum_multiplier_radius

  @property
  def constraint_optimizer(self):
    """Returns the `tf.train.Optimizer` used for the Lagrange multipliers."""
    return self._constraint_optimizer

  def _create_multipliers(self, num_constraints):
    """Creates the internal state used by _minimize_constrained."""
    # This method is only here so that the tests have a place they can hook into
    # to examine the internal state. Otherwise, this would be done inline in
    # _minimize_constrained().
    #
    # For a LagrangianOptimizer, the internal state is simply a tensor of
    # Lagrange multipliers with shape (m,), where m is the number of
    # constraints.
    #
    # We initialize to a numpy array, instead of a Tensor, so that if this
    # method is called inside the scope of a tf.control_dependencies() block,
    # the dependencies will not be applied to initial_multipliers.
    #
    # FUTURE WORK: make the dtype a parameter.
    initial_multipliers = np.zeros((num_constraints,), dtype=np.float32)
    return tf.Variable(
        initial_multipliers,
        trainable=False,
        name="lagrangian_state",
        dtype=tf.float32)

  def _projection_op(self, multipliers, name=None):
    with tf.colocate_with(multipliers):
      if self._maximum_multiplier_radius:
        projected_multipliers = _project_multipliers_wrt_euclidean_norm(
            multipliers, self._maximum_multiplier_radius)
      else:
        projected_multipliers = tf.maximum(multipliers, 0.0)
      return tf.assign(multipliers, projected_multipliers, name=name)

  def _minimize_constrained(self,
                            minimization_problem,
                            global_step=None,
                            var_list=None,
                            gate_gradients=tf.train.Optimizer.GATE_OP,
                            aggregation_method=None,
                            colocate_gradients_with_ops=False,
                            name=None,
                            grad_loss=None):
    """Returns an `Operation` for minimizing the constrained problem.

    The "optimizer" constructor parameter will be used to update the model
    parameters, while the Lagrange multipliers will be updated using
    "constraint_optimizer" (if provided) or "optimizer" (if not).

    Args:
      minimization_problem: ConstrainedMinimizationProblem, the problem to
        optimize.
      global_step: as in `tf.train.Optimizer`'s minimize() method.
      var_list: as in `tf.train.Optimizer`'s minimize() method.
      gate_gradients: as in `tf.train.Optimizer`'s minimize() method.
      aggregation_method: as in `tf.train.Optimizer`'s minimize() method.
      colocate_gradients_with_ops: as in `tf.train.Optimizer`'s minimize()
        method.
      name: as in `tf.train.Optimizer`'s minimize() method.
      grad_loss: as in `tf.train.Optimizer`'s minimize() method.

    Raises:
      TypeError: if the "minimization_problem" `Tensor`s have different dtypes.

    Returns:
      `Operation`, the train_op.
    """
    objective = minimization_problem.objective

    constraints = minimization_problem.constraints
    proxy_constraints = minimization_problem.proxy_constraints
    if proxy_constraints is None:
      proxy_constraints = constraints

    # Make sure that the objective, constraints and proxy constraints all have
    # the same dtype.
    if (objective.dtype.base_dtype != constraints.dtype.base_dtype or
        objective.dtype.base_dtype != proxy_constraints.dtype.base_dtype):
      raise TypeError("objective, constraints and proxy_constraints must have "
                      "the same dtypes")

    # Flatten both constraints tensors to 1d.
    num_constraints = minimization_problem.num_constraints
    constraints = tf.reshape(constraints, shape=(num_constraints,))
    proxy_constraints = tf.reshape(proxy_constraints, shape=(num_constraints,))

    # Create the Lagrange multipliers.
    multipliers = self._create_multipliers(num_constraints)

    loss = (
        objective + tf.tensordot(
            tf.cast(multipliers, proxy_constraints.dtype), proxy_constraints,
            1))
    multipliers_gradient = tf.cast(constraints, multipliers.dtype)

    update_ops = []
    if self._constraint_optimizer is None:
      # If we don't have a separate constraint_optimizer, then we use
      # self._optimizer for both the update of the model parameters, and that of
      # the Lagrange multipliers.
      grads_and_vars = self.optimizer.compute_gradients(
          loss,
          var_list=var_list,
          gate_gradients=gate_gradients,
          aggregation_method=aggregation_method,
          colocate_gradients_with_ops=colocate_gradients_with_ops,
          grad_loss=grad_loss)
      # We negate multipliers_gradient since we want to maximize over the
      # Lagrange multipliers.
      grads_and_vars.append((-multipliers_gradient, multipliers))
      update_ops.append(
          self.optimizer.apply_gradients(grads_and_vars, name="update"))
    else:
      # If we have a separate constraint_optimizer, then we use self._optimizer
      # for the update of the model parameters, and self._constraint_optimizer
      # for that of the Lagrange multipliers.
      grads_and_vars = self.optimizer.compute_gradients(
          loss,
          var_list=var_list,
          gate_gradients=gate_gradients,
          aggregation_method=aggregation_method,
          colocate_gradients_with_ops=colocate_gradients_with_ops,
          grad_loss=grad_loss)
      # We negate multipliers_gradient since we want to maximize over the
      # Lagrange multipliers.
      multiplier_grads_and_vars = [(-multipliers_gradient, multipliers)]

      gradients = [
          gradient for gradient, _ in grads_and_vars + multiplier_grads_and_vars
          if gradient is not None
      ]
      with tf.control_dependencies(gradients):
        update_ops.append(
            self.optimizer.apply_gradients(grads_and_vars, name="update"))
        update_ops.append(
            self._constraint_optimizer.apply_gradients(
                multiplier_grads_and_vars, name="optimizer_state_update"))

    with tf.control_dependencies(update_ops):
      if global_step is None:
        # If we don't have a global step, just project, and we're done.
        return self._projection_op(multipliers, name=name)
      else:
        # If we have a global step, then we need to increment it in addition to
        # projecting.
        projection_op = self._projection_op(
            multipliers, name="optimizer_state_project")
        with tf.colocate_with(global_step):
          global_step_op = tf.assign_add(
              global_step, 1, name="global_step_increment")
        return tf.group(projection_op, global_step_op, name=name)


text:
import glob
import numpy as np
import os
import time

start = time.time()
valid = open("/home/longhn/handwriting_0307/valid.txt", "w", encoding="utf8")
all_files = []
for ext in ["*.png", "*.jpeg", "*.jpg"]:
  images = glob.glob(os.path.join("/home/longhn/handwriting_0307/train", ext))
  all_files += images
num_img = len(all_files)
print("total images: %d " %num_img)
num_val = int(num_img*0.1)
rand_idx = np.random.randint(0, len(all_files), num_val)
with open("/home/longhn/handwriting_0307/train.txt", "w", encoding="utf8") as writefile:
    i = 0
    for file in glob.glob("/home/longhn/handwriting_0307/train/*.txt"):
        with open(file, "r", encoding="utf8") as readfile:
            label = readfile.read()
        file = '/'.join(file.split('/')[-3:])
        print(file)
        path = file.replace("txt", "jpg")
        if not i in rand_idx:
            writefile.write(f"{path}\t{label}\n")
            i += 1
        else:
            valid.write(f"{path}\t{label}\n")
            i += 1

# Creat test file
test = open("/home/longhn/handwriting_0307/test.txt", "w", encoding="utf8")
i = 0
for file in glob.glob("/home/longhn/handwriting_0307/test/*.txt"):
    with open(file, "r", encoding="utf8") as readfile:
        label = readfile.read()
    file = '/'.join(file.split('/')[-3:])
    print(file)
    path = file.replace("txt", "jpg")
    test.write(f"{path}\t{label}\n")

# test = open("/home/longhn/Desktop/result_label_SMD/Anotation_0506/p1/test.txt", "w", encoding="utf8")
# i = 0
# for file in glob.glob("/home/longhn/Desktop/result_label_SMD/Anotation_0506/p1/*.txt"):
#     with open(file, "r", encoding="utf8") as readfile:
#         label = readfile.read()
#     file = '/'.join(file.split('/')[-3:])
#     print(file)
#     path = file.replace("txt", "jpg")
#     test.write(f"{path}\t{label}\n")
stop = time.time()
train = stop-start
print('train: ', train)


text:
import argparse
import os
import ctypes

import matplotlib.pyplot as plt
import numpy as np

from img2rei import ReiFileHeader, ReiFrameHeader

def bytes_to_array(data: bytes, width: int, height: int):
    assert len(data) == width * height * 2

    def bytes_to_pixel(pxl_data: bytes):
        assert len(pxl_data) == 2

        val = int.from_bytes(pxl_data, byteorder='little', signed=False)

        return np.array([(val >> 11)/31, ((val >> 5) & 0x3f)/63, (val & 0x1f)/31])

    def bytes_to_row(row_data: bytes):
        return np.array([*map(bytes_to_pixel, [row_data[i:i+2] for i in range(0, len(row_data), 2)])])

    return np.array([*map(bytes_to_row, [data[i:i+width*2] for i in range(0, len(data), width*2)])])

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('img')

    args = parser.parse_args()
    if not os.path.isfile(args.img):
        print('error: cannot find input img {:s}'.format(args.img))
        quit(-1)

    try:
        file_hdr_size = ctypes.sizeof(ReiFileHeader)
        frame_hdr_size = ctypes.sizeof(ReiFrameHeader)
        with open(args.img, 'rb') as f:
            file_hdr = ReiFileHeader.from_buffer_copy(f.read(file_hdr_size))

            if file_hdr.magic != 0x5718:
                raise Exception(f'{args.img} is not an rei file.')

            print('file header')
            print(f'  hdr_len:    {file_hdr.hdr_len}')
            print(f'  ver:        {file_hdr.ver}')
            print(f'  fps:        {file_hdr.fps}')
            print(f'  num_frames: {file_hdr.num_frames}')

            f.seek(file_hdr.hdr_len, 0)

            for i in range(file_hdr.num_frames):
                frame_hdr = ReiFrameHeader.from_buffer_copy(f.read(frame_hdr_size))

                print(f'frame {i}')
                print(f'  width: {frame_hdr.width}')
                print(f'  height: {frame_hdr.height}')
                print(f'  pixel_format: {frame_hdr.pixel_format}')
                print(f'  frame_size: {frame_hdr.frame_size}')
                print(f'  crypto_mode: {frame_hdr.crypto_mode}')
                print(f'  iv_len: {frame_hdr.iv_len}')
                f.seek(frame_hdr.iv_len, 1)

                frame_data = f.read(frame_hdr.frame_size - frame_hdr_size - frame_hdr.iv_len)
                img = bytes_to_array(frame_data, frame_hdr.width, frame_hdr.height)
                assert img.shape[0] == frame_hdr.height
                assert img.shape[1] == frame_hdr.width
                assert img.shape[2] == 3
                plt.imshow(img)
                plt.show()
    except Exception as e:
        print(f'error: {e}')
        quit()

max_stars_repo_name: ub-rms/rushmore

max_issues_repo_name: ub-rms/rushmore

max_forks_repo_name: ub-rms/rushmore


text:
# Creates a figure for a beta distribution
# Author : Aleyna Kara (@karalleyna)

from scipy.stats import beta
import matplotlib.pyplot as plt
import numpy as np

a, b = 2, 2
theta_idx = 1
colors = ["tab:blue", "tab:orange"]

x = np.linspace(beta.ppf(0, a, b),beta.ppf(1, a, b), 100)

plt.figure(figsize=(7,7))
plt.xlim(0, 1)
plt.xticks([1])
plt.yticks([1])
plt.plot(x, beta.pdf(x, a, b), '-', c=colors[theta_idx-1], linewidth=3)
plt.xlabel(f'$\Theta_{theta_idx}$', fontsize='15')
plt.ylabel(f'f($\Theta_{theta_idx}$)', fontsize='15')

plt.savefig(f"action_{theta_idx}_{a}{b}.png", bbox_inches='tight')
plt.show()

max_stars_repo_name: sandutsar/pyprobml

max_issues_repo_name: brickbitbot/pyprobml

max_forks_repo_name: brickbitbot/pyprobml

reason: import numpy,from scipy


text:
# I’ve attached a figure with all the time series that went into our estimates. GP = Greenland peripheral glaciers, M&D = Missing & Disappeared glaciers. I’ve also attached a data file with all the trends that I computed over the AR6 periods that also include the uncertainties. It’s a Python/numpy file, and this command should load the file:

import numpy as np
import re
import matplotlib.pyplot as plt
stats = np.load('stats_basin_global.npy',allow_pickle=True).all()

# Then, the trends for observed GMSL are stored as dictionaries within this file. To show all the components, then run:

# stats["global"].keys()

# The observed GMSL trends then show up if you type

# stats["global"]["obs"]

# Which should give the time series and at the end the trends:

# 'trend_1900_2018': array([ 1.5590526 , -0.32608284,  0.32570602]),
# 'trend_1957_2018': array([ 1.77520187, -0.29996452,  0.30013279]),
# 'trend_1993_2018': array([ 3.35012592, -0.46057008,  0.46348239]),
# 'trend_1900_1949': array([ 1.34955675, -0.50915415,  0.50806184]),
# 'trend_1950_1999': array([ 0.98695231, -0.3456766 ,  0.36832388]),
# 'trend_1900_1956': array([ 1.55189052, -0.47334641,  0.46303397]),
# 'trend_1957_1993': array([ 0.87219705, -0.37207373,  0.38602746]),
# 'trend_1901_2018': array([ 1.56653093, -0.32566036,  0.29980854]),
# 'trend_1901_1990': array([ 1.44329056, -0.3724046 ,  0.37391159]),
# 'trend_1971_2018': array([ 2.24392976, -0.30802869,  0.30662267]),
# 'trend_2006_2018': array([ 4.27840446, -1.04768399,  1.09513025])}

# The first number is the mean of the estimate,
# and both numbers give the lower and upper bound relative to the mean.

# The individual components are also there:

std290 = 3.2897

for name,element in stats["global"].items():
    S = element["tseries"]
    t = np.arange(len(S))+1900
    for key,row in element.items():
        if not (key.startswith('trend')):
            continue
        period = key.replace('trend_','').replace('_','\t')
        sigma = (row[2]-row[1])/std290

        #also calculate our own trend....
        yrs=[float(x) for x in period.split('\t')]
        ix = (t>=yrs[0]) & (t<=yrs[1])
        if np.any(np.isnan(S[ix,1])):
            p1=[np.nan,np.nan]
            psigma=np.nan
        else:
            (p1,C) = np.polyfit(t[ix]-2000,S[ix,1],1,cov=True)
            psigma = np.polyfit(t[ix]-2000,(S[ix,0]-S[ix,2])/std290,1)
            psigma = np.sqrt(psigma[0]**2+C[0,0])
        print(f'{name}\t{period}\t{row[0]:.3f}\t{sigma:.3f}\t{p1[0]:.3f}\t{psigma:.3f}')
# Out[18]: dict_keys(['obs', 'steric', 'grd_glac', 'grd_GrIS', 'grd_AIS', 'grd_tws', 'grd_total', 'grd_tws_natural', 'grd_tws_gwd', 'grd_tws_dam', 'grd_ice', 'altimetry', 'budget', 'diff', 'obs_steric'])

# stats[“global”][“grd_glac”] is the glacier contribution, grd_GrIS = Greenland, “steric” (not obs_steric) is the steric component etc.

# Hope this helps!

# Cheers,

# Thomas
s1 = stats["global"]["obs_steric"]["tseries"]
t = np.arange(len(s1))+1900
s2 = stats["global"]["steric"]["tseries"]
plt.plot(t,s1,label='obs_steric')
plt.plot(t,s2,label='steric')
plt.legend()


text:
import random
import math
import networkx as nx
import matplotlib.pyplot as plt
import math
import threading, queue
import graph
import state_tree as st_tree
import samplePoints as spoint

while True:

    print()
    print("\tALGORITMOS DE BUSQUEDA")
    print()
    print("+ Generador de grafo 01 (1)")
    print("+ Generador de grafo 02 (2)")
    print("+ Salir (3)")
    print()

    opcion_generar_grafo = int(input("Opcion: "))

    if(opcion_generar_grafo == 1):

        """Iniciando programa"""

        G = nx.Graph()
        num_nodes = int(input("Number of nodes: "))
        num_conexions = int(input("Number conexions: "))

        """ Generando nodos aleatorios que no repiten posicion """

        G.add_nodes_from(range(0,num_nodes))

        for i in range (0,num_nodes):
            G.add_node(i,pos=(0,0),value = i, n_edges = 0, path = False)

        pos_good = spoint.get_good_points(num_nodes,4)

        pos = dict()
        i=0
        while len(pos) < num_nodes:
            xpos = pos_good[i][0] #random.randint(0,300)
            ypos = pos_good[i][1] #random.randint(0,300)

            if (xpos,ypos) not in pos.values():
                pos[i] = (xpos,ypos)
                i+=1

        for i in range(0,len(G.nodes)):
            for key in pos:
                if G.nodes[i]['value'] == key:
                    G.nodes[i]['pos'] = pos[key]

        position_node =nx.get_node_attributes(G,'pos')

        """ Conectando nodos a nodos mas cercanos """

        list_node_start = [node for node in pos]
        list_points = [point for point in pos.values()]

        for i in range(0,len(list_node_start)):
            list_node_edges = graph.close_nodes_points(list_points[i],list_points)

            lista_node_end = graph.busca_nodos(list_node_edges,pos)

            """ revisar nodos con muchas conexiones y cero conexiones"""
            lista_node_end = graph.validar_conexion(list_node_start[i],lista_node_end,num_conexions,G)

            #evaluar para no hacer doble enlace

            for j in range(0,len(lista_node_end)):
                G.add_edge(list_node_start[i],lista_node_end[j])

        edges_reverse = []

        for edge in G.edges:
            edges_reverse.append((edge[1],edge[0]))

        for edge in G.edges:
            for j in range(0,len(edges_reverse)):
                if edge == edges_reverse[j]:
                    G.remove_edge(edges_reverse[j])

        nx.draw_networkx_nodes(G,position_node)
        nx.draw_networkx_edges(G,position_node)

        labels = {}
        for i in range(0,num_nodes):
            labels[i] = i

        nx.draw_networkx_labels(G,position_node,labels,font_size=12)

        plt.show()

        """ Busquedas"""
        print()
        print("\tBusquedas")
        print()
        nodo_inicio = int(input("nodo inicial: "))
        nodo_objetivo = int(input("nodo objetivo: "))

        opcion_arbol = 0
        while(opcion_arbol != 1 and opcion_arbol!=2):
            print()
            print("+ Busqueda amplitud sin arbol  [1]")
            print(" (se recorre el grafo como arbol a traves de una cola - Rapido, no contruye todo el arbol) ")
            print()
            print("+ Busqueda amplitud con arbol  [2]")
            print(" (se construye todo el arbol de estados y luego recorrer con una cola - Lento , construye arbol ,con varias conexiones)")
            print()
            opcion_arbol = int(input("Opcion: "))

        if(opcion_arbol == 1):
            st_tree.busqueda_amplitud_sin_arbol(G,num_nodes,position_node,nodo_inicio,nodo_objetivo)

        elif(opcion_arbol == 2):
            st_tree.busqueda_amplitud_arbol(G,num_nodes,position_node,nodo_inicio,nodo_objetivo)

        st_tree.hill_climbing(G,num_nodes,nodo_inicio,nodo_objetivo)

    elif(opcion_generar_grafo == 2):

        Grafo = nx.Graph()
        print("Ingrese el numero de nodos: ")
        n_nodos=int(input())
        print("\n Ingrese el numero de conexiones: ")
        n_conexiones=int(input())

        lengthss={}
        path_find = False
        distancia_nodo_objetivo={}
        for i in range (0,n_nodos):
            Grafo.add_node(i,pos=(random.randint(0,100),random.randint(0,100)),value = i, n_edges = 0, path = False)

        for i in range (0, n_nodos):
            for j in range(0, n_nodos):
                if i != j:
                    lengthss[j]=(round(math.sqrt(((Grafo.nodes[i]['pos'][1]-Grafo.nodes[j]['pos'][1])**2)+((Grafo.nodes[j]['pos'][0]-Grafo.nodes[i]['pos'][0])**2)),2))
                length_sort={k: v for k, v in sorted(lengthss.items(), key=lambda item: item[1])}
            for h in range(0, n_conexiones):
                for g in list(length_sort):
                    if Grafo.nodes[i]['n_edges'] < n_conexiones and Grafo.nodes[next(iter(length_sort))]['n_edges'] < n_conexiones :
                        Grafo.add_edge(i,next(iter(length_sort)))
                        Grafo.nodes[i]['n_edges'] +=1
                        Grafo.nodes[next(iter(length_sort))]['n_edges'] +=1
                    length_sort.pop(list(length_sort.keys())[0])
            lengthss = {}

        pos=nx.get_node_attributes(Grafo,'pos')
        nx.draw(Grafo,pos,with_labels=True)
        plt.show()

        """ Busquedas"""
        print()
        print("\tBusquedas")
        print()
        nodo_inicio = int(input("nodo inicial: "))
        nodo_objetivo = int(input("nodo objetivo: "))

        opcion_arbol = 0
        while(opcion_arbol != 1 and opcion_arbol!=2):
            print()
            print("+ Busqueda amplitud sin arbol  [1]")
            print(" (se recorre el grafo como arbol a traves de una cola - Rapido, no contruye todo el arbol) ")
            print()
            print("+ Busqueda amplitud con arbol  [2]")
            print(" (se construye todo el arbol de estados y luego recorrer con una cola - Lento , construye arbol ,con varias conexiones)")
            print()
            opcion_arbol = int(input("Opcion: "))

        if(opcion_arbol == 1):
            st_tree.busqueda_amplitud_sin_arbol(Grafo,n_nodos,pos,nodo_inicio,nodo_objetivo)
        elif(opcion_arbol == 2):
            st_tree.busqueda_amplitud_arbol(Grafo,n_nodos,pos,nodo_inicio,nodo_objetivo)

        st_tree.hill_climbing(Grafo,n_nodos,nodo_inicio,nodo_objetivo)

    elif(opcion_generar_grafo == 3):
        break

max_stars_repo_name: dennisMSF/IA

max_issues_repo_name: dennisMSF/IA

max_forks_repo_name: dennisMSF/IA

reason: import networkx


text:
"""@file equal_error_rate.py
contains the EER class"""

import os
import numpy as np
import scorer
import copy
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt

class EER(scorer.Scorer):
	""""""
	def __init__(self, conf, evalconf, dataconf, store_dir, numbatches, task, scorer_name, checkpoint_file):
		"""EER constructor

		Args:
			conf: the scorer configuration as a dictionary
			evalconf: the evaluator configuration as a ConfigParser
			dataconf: the database configuration
			store_dir: the directory where the reconstructions are
			numbatches: the number of batches to process
		"""

		super(EER, self).__init__(conf, evalconf, dataconf, store_dir, numbatches, task, scorer_name, checkpoint_file)

		thr_range_parse = map(float, conf['thr_range'].split(' '))
		self.thr_range = np.arange(thr_range_parse[0], thr_range_parse[1], thr_range_parse[2])

		self.nr_true_S = int(conf['num_true_speakers'])

		self.target = np.concatenate([np.ones(self.nr_true_S, dtype=np.bool), np.zeros(self.nrS - self.nr_true_S, dtype=np.bool)])

		if 'make_figure' in conf and conf['make_figure'] == 'True':
			self.make_figure = True
		else:
			self.make_figure = False

	def _get_score(self, handled_output, enr_vecs, labels):
		"""score the handled output.

		Args:
			handled_output

		Returns:
			the score"""

		score_to_binary = [[outp > thr for thr in self.thr_range] for outp in handled_output]

		score_dict = list()
		for spk_ind, spk_target in zip(range(self.nrS), labels):
			spk_binaries = score_to_binary[spk_ind]
			score_dict.append(dict())
			score_dict[spk_ind]['binary'] = spk_binaries
			score_dict[spk_ind]['target'] = spk_target

			correct = np.equal(spk_binaries, spk_target)
			score_dict[spk_ind]['false'] = np.logical_not(correct)

		return score_dict

	def get_enr_vecs(self, utt_ind):
		enr_vecs = None
		labels = self.target
		return enr_vecs, labels

	def storable_result(self):
		storable_result = copy.deepcopy(self.results)
		for utt in storable_result:
			for spk_ind in range(len(storable_result[utt]['score'])):
				storable_result[utt]['score'][spk_ind]['binary'] = map(int, storable_result[utt]['score'][spk_ind]['binary'])
				storable_result[utt]['score'][spk_ind]['false'] = map(int, storable_result[utt]['score'][spk_ind]['false'])
				storable_result[utt]['score'][spk_ind]['target'] = int(storable_result[utt]['score'][spk_ind]['target'])

	def summarize(self):
		"""summarize the results of all utterances

		"""
		#
		utts = self.results.keys()

		num_ground_trues = 0
		num_ground_falses = 0
		num_false_negative = np.zeros(np.shape(self.thr_range))
		num_false_positive = np.zeros(np.shape(self.thr_range))
		for utt in utts:
			utt_result = self.results[utt]
			for spk_ind in range(self.nrS):
				if utt_result['score'][spk_ind]['target']:
					num_ground_trues += 1
					num_false_negative += utt_result['score'][spk_ind]['false']
				else:
					num_ground_falses += 1
					num_false_positive += utt_result['score'][spk_ind]['false']

		false_negative_rate = num_false_negative/num_ground_trues
		false_positive_rate = num_false_positive/num_ground_falses

		eer, thr_ind = get_eer(false_negative_rate, false_positive_rate)
		result_summary = {'eer': eer, 'thr': self.thr_range[thr_ind]}
		#

		print ''
		print 'Result for %s (using %s): ' % ('eer', self.__class__.__name__)
		print 'EER=%.2f%% (threshold=%.3f)' % (result_summary['eer']*100.0, result_summary['thr'])

		if self.make_figure:
			fig = plt.figure()
			ax = fig.add_subplot(1, 1, 1)
			ax.scatter(false_positive_rate*100.0, false_negative_rate*100.0, edgecolors='blue', facecolors='none')
			ax.plot([eer * 100.0], [eer * 100.0], marker='o', markersize=6, color="red")
			ax.annotate('EER=%.1f%% (thr=%.2f)' % (eer*100.0, self.thr_range[thr_ind]), (eer * 100.0, eer * 100.0))
			ax.set_xlim(0.0, 100.0)
			ax.set_ylim(0.0, 100.0)
			ax.set_xlabel('False positive rate (%)')
			ax.set_ylabel('False negative rate (%)')
			fig.savefig(os.path.join(self.store_dir, 'eer_graph.png'))
		return result_summary

def get_eer(false_negative_rate, false_positive_rate):
	if false_positive_rate[0] < false_negative_rate[0]:
		raise BaseException('')

	found_eer = False
	for ind in range(len(false_negative_rate)):
		false_neg_ra = false_negative_rate[ind]
		false_pos_ra = false_positive_rate[ind]
		if false_pos_ra < false_neg_ra:
			# crossed the equal error line. We define the EER now as the intersection between the equal error line and
			# the line going through the threshold point right before and right after the crossing of the equal error
			# line.
			x1 = false_negative_rate[ind-1]
			y1 = false_positive_rate[ind-1]
			x2 = false_negative_rate[ind]
			y2 = false_positive_rate[ind]

			eer = ((x2 - x1)*y1 - (y2 - y1)*x1) / ((x2 - x1) - (y2 - y1))
			found_eer = True
			break

	if not found_eer:
		raise BaseException('Did not manage to find an EER.')

	return eer, ind


text:
import pdb
import numpy as np

class PermutationTestResult:
    def __init__(self, t0, t):
        self.t0 = t0
        self.t = t

def permuteDiffPairedMeans(data1, data2, nResamples):
    def computeMeanDiffPairedMeans(data):
        data1 = data[:int(len(data)/2)]
        data2 = data[int(len(data)/2):]
        meanDiff = np.mean(data1-data2)
        return meanDiff

    data0 = np.concatenate((data1, data2))
    t0 = computeMeanDiffPairedMeans(data=data0)
    t = np.empty(nResamples)
    for i in range(nResamples):
        t[i] = computeMeanDiffPairedMeans(data=data0[np.random.randint(low=0, high=len(data0), size=len(data0))])
    # p = float(len(np.nonzero(np.absolute(t)>abs(t0))[0]))/len(t)
    answer = PermutationTestResult(t0=t0, t=t)
    return answer

max_stars_repo_name: joacorapela/svGPFA

max_issues_repo_name: joacorapela/svGPFA

max_forks_repo_name: joacorapela/svGPFA


text:
# https://matplotlib.org/tutorials/introductory/usage.html#sphx-glr-tutorials-introductory-usage-py

import matplotlib
import matplotlib.pyplot as plt
import numpy as np

def my_plotter(ax, data1, data2, param_dict={}):
    ax.plot(data1, data2, **param_dict)
    ax.set(xlabel='Time (s)', ylabel='Amplitutde', title='Sine')
    ax.grid()

def test_backend(backend):
    matplotlib.use(backend)
    t = np.arange(0.0, 2.0, 0.01)
    s = 1 + np.sin(2 * np.pi * t)
    fig, ax = plt.subplots()
    my_plotter(ax, t, s)
    fig.savefig("sine-backend.{}".format(backend))

test_backend('pdf')
test_backend('svg')
test_backend('ps')

max_stars_repo_name: drichardson/examples

max_issues_repo_name: drichardson/examples

max_forks_repo_name: drichardson/examples


text:
import collections
import glob
import os
import random
from tqdm import tqdm

random.seed(12345)

# import numpy as np

# np.set_printoptions(threshold=np.nan)

def read_voxceleb_structure(directory, test_only=False, sample=False):
    voxceleb = []
    speakers = set()
    for subset in os.listdir(directory):
        if not os.path.isdir(os.path.join(directory, subset)):
            continue
        if subset == 'dev' and (test_only or sample):
            continue
        if subset != 'dev' and subset != 'test' and subset != 'sample_dev':
            continue
        # root_dir/subset/speaker_id/uri/wav
        key = os.path.join(directory, subset) + '/*/*/*.wav'
        print('==> Reading {} set'.format(subset))
        for file_path in tqdm(glob.glob(key)):
            subset, speaker, uri, file = file_path.split('/')[-4:]
            if '.wav' in file:
                voxceleb.append({'filename': file, 'speaker_id': speaker, 'uri': uri, 'subset': subset, 'file_path': file_path})
                speakers.add(speaker)
    print('Found {} files with {} different speakers.'.format(len(voxceleb), len(speakers)))

    if sample:
        voxceleb_dev = [datum for datum in voxceleb if datum['subset']=='sample_dev']
    else:
        voxceleb_dev = [datum for datum in voxceleb if datum['subset']=='dev']
    voxceleb_test = [datum for datum in voxceleb if datum['subset']=='test']

    # return voxceleb
    return voxceleb_dev, voxceleb_test

def generate_test_dir(voxceleb):
    voxceleb_test = []
    speakers_test = collections.defaultdict(list)
    for item in voxceleb:
        if item['subset'] == 'test':
            voxceleb_test.append(item['file_path'])
            speakers_test[item['speaker_id']].append(item['file_path'])
    voxceleb_test = [item['file_path'] for item in voxceleb if item['subset'] == 'test']

    # # negative
    pairs = list(zip(voxceleb_test, sorted(voxceleb_test, key=lambda x: random.random())))
    # positive
    for values in speakers_test.values():
        pairs += list(zip(values, sorted(values, key=lambda x: random.random())))

    csv_path = os.path.join(directory, 'test_pairs.csv')
    with open(csv_path, 'w') as f:
        for item in tqdm(pairs):
            item = list(item)
            for i in range(len(item)):
                item[i] = item[i].split('/test/')[-1].strip()

            # print('>>>>>>>>>>>', item, item[0].split('/')[0], item[1].split('/')[0])

            issame = '1' if item[0].split('/')[0] == item[1].split('/')[0] else '0'
            line = ' '.join([issame, item[0], item[1]])
            f.write(line + '\n')
        # f.write('\n'.join([','.join(item) for item in pairs]))
    print('==> Generated and saved test pairs to {}'.format(csv_path))

if __name__ == '__main__':
    directory = '/data5/xin/voxceleb/raw_data/'
    voxceleb = read_voxceleb_structure(directory, test_only=True)
    generate_test_dir(voxceleb)


text:
from typing import Dict

import numpy as np

import torch
from torch import nn

from enums.metric_type import MetricType

from entities.metric import Metric

from models.model_base import ModelBase
from models.transformer_encoder_decoder.transformer_encoder import TransformerEncoder
from models.transformer_encoder_decoder.transformer_decoder import TransformerDecoder

from services.arguments.transformer_arguments_service import TransformerArgumentsService
from services.data_service import DataService
from services.metrics_service import MetricsService
from services.vocabulary_service import VocabularyService
from services.log_service import LogService
from services.tokenize.base_tokenize_service import BaseTokenizeService

class TransformerModel(ModelBase):
    def __init__(
            self,
            arguments_service: TransformerArgumentsService,
            data_service: DataService,
            vocabulary_service: VocabularyService,
            metrics_service: MetricsService,
            log_service: LogService,
            tokenize_service: BaseTokenizeService):
        super().__init__(data_service, arguments_service)

        self._vocabulary_service = vocabulary_service
        self._metrics_service = metrics_service
        self._log_service = log_service
        self._tokenize_service = tokenize_service

        self._device = arguments_service.device
        self._metric_types = arguments_service.metric_types

        self.encoder = TransformerEncoder(
            input_dim=tokenize_service.vocabulary_size,
            hid_dim=arguments_service.hidden_dimension,
            n_layers=arguments_service.number_of_layers,
            n_heads=arguments_service.number_of_heads,
            pf_dim=arguments_service.hidden_dimension,
            dropout=arguments_service.dropout,
            device=arguments_service.device,
            include_pretrained=arguments_service.include_pretrained_model,
            pretrained_hidden_size=arguments_service.pretrained_model_size,
            max_length=5000)

        self.decoder = TransformerDecoder(
            output_dim=vocabulary_service.vocabulary_size(),
            hid_dim=arguments_service.hidden_dimension,
            n_layers=arguments_service.number_of_layers,
            n_heads=arguments_service.number_of_heads,
            pf_dim=arguments_service.hidden_dimension,
            dropout=arguments_service.dropout,
            device=arguments_service.device,
            include_pretrained=arguments_service.include_pretrained_model,
            pretrained_hidden_size=arguments_service.pretrained_model_size,
            max_length=5000)

        self.src_pad_idx = 0
        self.trg_pad_idx = vocabulary_service.pad_token
        self.device = arguments_service.device

        self.initialize_weights()

    def initialize_weights(self):
        if hasattr(self, 'weight') and self.weight.dim() > 1:
            nn.init.xavier_uniform_(self.weight.data)

    def make_src_mask(self, src):

        # src = [batch size, src len]

        src_mask = (src != self.src_pad_idx).unsqueeze(1).unsqueeze(2)

        # src_mask = [batch size, 1, 1, src len]

        return src_mask

    def make_trg_mask(self, trg):

        # trg = [batch size, trg len]

        trg_pad_mask = (trg != self.trg_pad_idx).unsqueeze(1).unsqueeze(3)

        # trg_pad_mask = [batch size, 1, trg len, 1]

        trg_len = trg.shape[1]

        trg_sub_mask = torch.tril(torch.ones(
            (trg_len, trg_len), device=self.device)).bool()

        # trg_sub_mask = [trg len, trg len]

        trg_mask = trg_pad_mask & trg_sub_mask

        # trg_mask = [batch size, 1, trg len, trg len]

        return trg_mask

    def forward(self, input_batch, debug=False, **kwargs):
        source, targets, lengths, pretrained_representations, _ = input_batch

        # source = source[:, :-1]
        cut_targets = targets[:, :-1]

        # src = [batch size, src len]
        # trg = [batch size, trg len]

        src_mask = self.make_src_mask(source)
        trg_mask = self.make_trg_mask(cut_targets)

        # src_mask = [batch size, 1, 1, src len]
        # trg_mask = [batch size, 1, trg len, trg len]

        enc_src = self.encoder(source, src_mask, pretrained_representations)

        # enc_src = [batch size, src len, hid dim]

        output, _ = self.decoder(cut_targets, enc_src, trg_mask, src_mask)

        # output = [batch size, trg len, output dim]
        # attention = [batch size, n heads, trg len, src len]

        return output, targets

    def calculate_accuracies(self, batch, outputs, output_characters=False) -> Dict[MetricType, float]:
        output, targets = outputs
        output_dim = output.shape[-1]
        predictions = output.max(dim=2)[1].cpu().detach().numpy()

        targets = targets[:,1:].cpu().detach().numpy()
        predicted_characters = []
        target_characters = []

        for i in range(targets.shape[0]):
            indices = np.array(
                (targets[i] != self._vocabulary_service.pad_token), dtype=bool)
            predicted_characters.append(predictions[i][indices])
            target_characters.append(targets[i][indices])

        metrics = {}

        if MetricType.JaccardSimilarity in self._metric_types:
            predicted_tokens = [self._vocabulary_service.ids_to_string(
                x) for x in predicted_characters]
            target_tokens = [self._vocabulary_service.ids_to_string(
                x) for x in target_characters]
            jaccard_score = np.mean([self._metrics_service.calculate_jaccard_similarity(
                target_tokens[i], predicted_tokens[i]) for i in range(len(predicted_tokens))])

            metrics[MetricType.JaccardSimilarity] = jaccard_score

        character_results = None
        if MetricType.LevenshteinDistance in self._metric_types:
            predicted_strings = [self._vocabulary_service.ids_to_string(
                x) for x in predicted_characters]
            target_strings = [self._vocabulary_service.ids_to_string(
                x) for x in target_characters]

            levenshtein_distance = np.mean([self._metrics_service.calculate_normalized_levenshtein_distance(
                predicted_strings[i], target_strings[i]) for i in range(len(predicted_strings))])

            metrics[MetricType.LevenshteinDistance] = levenshtein_distance

            if output_characters:
                character_results = []
                _, _, lengths, _, ocr_texts_tensor = batch
                ocr_texts = ocr_texts_tensor.cpu().detach().tolist()
                for i in range(len(ocr_texts)):
                    input_string = self._vocabulary_service.ids_to_string(
                        ocr_texts[i])

                    character_results.append(
                        [input_string, predicted_strings[i], target_strings[i]])

        return metrics, character_results

    def compare_metric(self, best_metric: Metric, new_metric: Metric) -> bool:
        if best_metric.is_new:
            return True

        best_loss = round(best_metric.get_current_loss(), 2)
        new_loss = round(new_metric.get_current_loss(), 2)

        if best_loss == new_loss:
            best_levenshtein = best_metric.get_accuracy_metric(MetricType.LevenshteinDistance)
            new_levenshtein = new_metric.get_accuracy_metric(MetricType.LevenshteinDistance)
            return new_levenshtein <= best_levenshtein
        else:
            return best_loss > new_loss

max_stars_repo_name: ktodorov/eval-historical-texts

max_issues_repo_name: ktodorov/eval-historical-texts

max_forks_repo_name: ktodorov/eval-historical-texts


text:
import numpy as np

# Import PyCOMPSs
# from exaqute.ExaquteTaskPyCOMPSs import *   # to execute with runcompss
# from exaqute.ExaquteTaskHyperLoom import *  # to execute with the IT4 scheduler
from exaqute.ExaquteTaskLocal import *      # to execute with python3

def geometricModel(parameters,point):
    """
    Compute y = C*exp(-w.x) where C = parameters[0] and w = parameters[1:]
    """
    return parameters[0]*np.exp(sum([-parameters[i+1]*point[i] for i in range(len(point))]))

@ExaquteTask(returns=1)
def geometricModel_Task(parameters,point):
    """
    Compute y = C*exp(-w.x) where C = parameters[0] and w = parameters[1:]
    """
    return geometricModel(parameters,point)

max_stars_repo_name: HubertBalcerzak/Kratos

max_issues_repo_name: HubertBalcerzak/Kratos

max_forks_repo_name: HubertBalcerzak/Kratos


text:
from behave import given, then, when
from PIL import Image
import numpy
import os
import dtlpy as dl

@given(u'I convert to Numpy.NdArray an item with the name "{item_path}" and add it to context.array')
def step_impl(context, item_path):
    item_path = os.path.join(os.environ['DATALOOP_TEST_ASSETS'], item_path)
    image = Image.open(item_path)
    context.array = numpy.array(image)

@given(u'I save item_metadata context.item_metadata')
def step_impl(context):
    context.item_metadata = {'City': 'NY', 'Country': 'USA'}

@when(u'I Upload an Numpy.NdArray (context.array) item with the name "{item_name}"')
def step_impl(context, item_name):
    try:
        context.item = context.dataset.items.upload(context.array,
                                                    remote_name=item_name,
                                                    item_metadata=context.item_metadata)
    except dl.exceptions.BadRequest:
        # For Scenario: Upload illegal Numpy.NdArray item
        # In such case item is not upload and "There are no items" scansion has to be executed
        pass

@then(u'Item is correctly uploaded to platform')
def step_impl(context):
    item_get = context.dataset.items.get(item_id=context.item.id)

    for key in context.item_metadata:
        assert key in context.item.metadata
    assert item_get.id == context.item.id

@when(u'I Download as Numpy.NdArray the uploaded item')
def step_impl(context):
    context.download_array = context.item.download(save_locally=False, to_array=True)

@then(u'Download Numpy.NdArray item and context.array size equal')
# I not check numpy.array_equal since the  numpy.ndarray to io.BytesIO can the RGB values
def step_impl(context):
    assert context.download_array.size, context.array.size

@when(u'Download as Numpy.NdArray the .mp4')
def step_impl(context):
    context.download_array = context.item_mp4.download(save_locally=False, to_array=True)

@given(u'I remove log files')
def step_impl(context):
    for f in os.listdir("."):
        if any(x in f for x in ["log", ".json"]):
            os.remove(f)

@then(u'Log file is exist')
def step_impl(context):
    for f in os.listdir("."):
        if any(x in f for x in ["log", ".json"]):
            return
    assert False

@then(u'Log file does not exist')
def step_impl(context):
    for f in os.listdir("."):
        if any(x in f for x in ["log", ".json"]):
            assert False

max_stars_repo_name: dataloop-ai/dtlpy

max_issues_repo_name: dataloop-ai/dtlpy

max_forks_repo_name: dataloop-ai/dtlpy


text:
from collections import defaultdict
import faiss
import numpy as np
import struct
import torch
from torch import nn
from typing import Dict, List

from LibVQ.utils import dist_gather_tensor

class IVFCPU(nn.Module):
    """
    Efficiently training IVF on CPU. In this way you can use a large-scale IVF index.
    """
    def __init__(self,
                 center_vecs: np.ndarray,
                 id2center: Dict[int, int]):
        """
        :param center_vecs: Vectors for all centers
        :param id2center: Mapping Doc id to center id
        """
        super(IVFCPU, self).__init__()
        self.center_vecs = center_vecs
        self.id2center = id2center
        self.center_grad = np.zeros_like(center_vecs)
        self.ivf_centers_num = len(center_vecs)

    @classmethod
    def from_faiss_index(cls, index_file):
        print(f'loading IVF from Faiss index: {index_file}')

        index = faiss.read_index(index_file)
        if isinstance(index, faiss.IndexPreTransform):
            ivf_index = faiss.downcast_index(index.index)
        else:
            ivf_index = index

        coarse_quantizer = faiss.downcast_index(ivf_index.quantizer)
        coarse_embeds = faiss.vector_to_array(coarse_quantizer.xb)
        center_vecs = coarse_embeds.reshape((-1, ivf_index.d))

        invlists = ivf_index.invlists
        id2center = {}
        for i in range(len(center_vecs)):
            ls = invlists.list_size(i)
            list_ids = faiss.rev_swig_ptr(invlists.get_ids(i), ls)

            for docid in list_ids:
                id2center[docid] = i

        ivf = cls(center_vecs, id2center)
        return ivf

    @classmethod
    def from_spann_index(cls, index_file):
        print(f'loading IVF from spann index: {index_file}')

        with open(f'{index_file}/SPTAGHeadVectors.bin', 'rb') as fq:
            count = struct.unpack('i', fq.read(4))[0]
            dimension = struct.unpack('i', fq.read(4))[0]
            center_vecs = np.frombuffer(fq.read(count * dimension * 4), dtype=np.float32).reshape((count, dimension))

        id2center = defaultdict(int)
        with open(f'{index_file}/output.txt', 'r') as f:
            cid = 0
            for line in f:
                line = line.strip().split(',')
                for id in line:
                    id2center[int(id)] = cid
                cid += 1

        assert cid == len(center_vecs)
        ivf = cls(center_vecs, id2center)
        return ivf

    def get_batch_centers(self,
                          batch_centers_index: List[int],
                          device: torch.device):
        c_embs = self.center_vecs[batch_centers_index]
        self.batch_centers_index = batch_centers_index
        self.batch_center_vecs = torch.FloatTensor(c_embs).to(device)
        self.batch_center_vecs.requires_grad = True

    def merge_and_dispatch(self,
                           doc_ids: List[int],
                           neg_ids: List[int],
                           world_size: int):
        dc_ids = [self.id2center[i] for i in doc_ids]
        nc_ids = [self.id2center[i] for i in neg_ids]

        if world_size > 1:
            all_dc_ids = dist_gather_tensor(torch.LongTensor(dc_ids).cuda(), world_size=world_size, detach=True)
            all_dc_ids = list(all_dc_ids.detach().cpu().numpy())
            all_nc_ids = dist_gather_tensor(torch.LongTensor(nc_ids).cuda(), world_size=world_size, detach=True)
            all_nc_ids = list(all_nc_ids.detach().cpu().numpy())
        else:
            all_dc_ids, all_nc_ids = dc_ids, nc_ids

        batch_cids = sorted(list(set(all_dc_ids + all_nc_ids)))
        cid2bid = {}
        for i, c in enumerate(batch_cids):
            cid2bid[c] = i

        batch_dc_ids = torch.LongTensor([cid2bid[x] for x in dc_ids])
        batch_nc_ids = torch.LongTensor([cid2bid[x] for x in nc_ids])
        return batch_cids, batch_dc_ids, batch_nc_ids

    def select_centers(self,
                       doc_ids: List[int],
                       neg_ids: List[int],
                       device: torch.device,
                       world_size: int):
        batch_cids, batch_dc_ids, batch_nc_ids = self.merge_and_dispatch(doc_ids, neg_ids, world_size=world_size)

        self.get_batch_centers(batch_cids, device)
        batch_dc_ids, batch_nc_ids = batch_dc_ids.to(device), batch_nc_ids.to(device)
        dc_emb = self.batch_center_vecs.index_select(dim=0, index=batch_dc_ids)
        nc_emb = self.batch_center_vecs.index_select(dim=0, index=batch_nc_ids)
        return dc_emb, nc_emb

    def grad_accumulate(self,
                        world_size: int):
        if world_size > 1:
            grad = dist_gather_tensor(self.batch_center_vecs.grad.unsqueeze(0), world_size=world_size, detach=True)
            grad = torch.mean(grad, dim=0).detach().cpu().numpy()
        else:
            grad = self.batch_center_vecs.grad.detach().cpu().numpy()

        self.center_grad[self.batch_centers_index] += grad

    def update_centers(self,
                       lr: float):
        self.center_vecs = self.center_vecs - lr * self.center_grad

    def zero_grad(self):
        self.center_grad = np.zeros_like(self.center_grad)

    def save(self,
             save_file: str):
        np.save(save_file, self.center_vecs)

max_stars_repo_name: staoxiao/LibVQ

max_issues_repo_name: staoxiao/LibVQ

max_forks_repo_name: staoxiao/LibVQ


text:
#!/usr/bin/python

import csv
import dask.array as da
import h5py
import math
import numpy as np
import argparse
import time

def coarsen(f, type, tile_size=256):
    """
    Create data pyramid.
    """
    grid = f["resolutions"]["1"][type]
    top_n = grid.shape[0]
    max_zoom = math.ceil(math.log(top_n / tile_size) / math.log(2))

    chunk_size = tile_size * 16
    curr_size = grid.shape
    dask_dset = da.from_array(grid, chunks=(chunk_size, chunk_size))

    r = f["resolutions"]
    curr_resolution = 1

    while curr_resolution < 2 ** max_zoom:
        curr_size = tuple(np.array(curr_size) / 2)
        print("coarsening")
        curr_resolution *= 2

        print("curr_size:", curr_size)
        group_name = "{}{}".format(
            curr_resolution, "" if type == "values" else "-" + type
        )
        g = r.create_group(group_name)
        values = g.require_dataset(
            type, curr_size, dtype="f4", compression="lzf", fillvalue=np.nan
        )

        dask_dset = dask_dset.rechunk((chunk_size, chunk_size))
        dask_dset = da.coarsen(np.nansum, dask_dset, {0: 2, 1: 2})
        da.store(dask_dset, values)

def parse(input_handle, output_hdf5, height, width, delimiter, first_n, is_labelled):
    reader = csv.reader(input_handle, delimiter=delimiter)
    if is_labelled:
        first_row = next(reader)
        col_labels = first_row[1 : (first_n + 1) if first_n else None]
        output_hdf5.create_dataset(
            "col_labels",
            data=np.array(col_labels, dtype=h5py.special_dtype(vlen=str)),
            compression="lzf",
        )

    tile_size = 256
    limit = max(height, width)
    max_zoom = math.ceil(math.log(limit / tile_size) / math.log(2))
    max_dimension = tile_size * 2 ** max_zoom

    g = output_hdf5.create_group("resolutions")
    g1 = g.create_group("1")
    val_ds = g1.create_dataset(
        "values",
        (max_dimension, max_dimension),
        dtype="f4",
        compression="lzf",
        fillvalue=np.nan,
    )
    nan_ds = g1.create_dataset(
        "nan_values",
        (max_dimension, max_dimension),
        dtype="f4",
        compression="lzf",
        fillvalue=0,
    )

    start_time = time.time()
    counter = 0
    row_labels = []
    for row in reader:
        if is_labelled:
            offset = 1
            row_labels.append(row[0])
        else:
            offset = 0

        val_np = np.array([float(p) for p in row[offset:]])
        val_ds[counter, : len(val_np)] = val_np

        nan_np = np.array([0 for p in row[offset:]])
        nan_ds[counter, : len(nan_np)] = nan_np

        counter += 1
        if counter == first_n:
            break

        time_elapsed = time.time() - start_time
        time_per_entry = time_elapsed / counter

        time_remaining = time_per_entry * (height - counter)
        print(
            "counter:",
            counter,
            "sum(x):",
            sum(val_np),
            "time remaining: {:d} seconval_ds".format(int(time_remaining)),
        )

    if is_labelled:
        output_hdf5.create_dataset(
            "row_labels",
            data=np.array(row_labels, dtype=h5py.special_dtype(vlen=str)),
            compression="lzf",
        )
    coarsen(output_hdf5, "values")
    coarsen(output_hdf5, "nan_values")
    output_hdf5.close()

def get_height(input_path, is_labelled=True):
    """
    We need to scan the file once just to see how many lines it contains.
    If it is tall and narrow, the first tile will need to be larger than just
    looking at the width of the first row would suggest.
    """
    with open(input_path) as f:
        for i, l in enumerate(f):
            pass
    if is_labelled:
        return i
    else:
        return i + 1

def get_width(input_path, is_labelled, delimiter="\t"):
    """
    Assume the number of elements in the first row is the total width.
    """
    with open(input_path, "r", newline="") as input_handle:
        reader = csv.reader(input_handle, delimiter=delimiter)
        len_row = len(next(reader))
        if is_labelled:
            return len_row - 1
        return len_row

def main():
    parser = argparse.ArgumentParser(
        description="""
        Given a tab-delimited file, produces an HDF5 file with mrmatrix
        ("multi-resolution matrix") structure: Under the "resolutions"
        group are datasets, named with successive powers of 2,
        which represent successively higher aggregations of the input.
    """
    )
    parser.add_argument("input_file", help="TSV file path")
    parser.add_argument("output_file", help="HDF5 file")
    parser.add_argument(
        "-d",
        "--delimiter",
        type=str,
        default="\t",
        metavar="D",
        help="Delimiter; defaults to tab",
    )
    parser.add_argument(
        "-n",
        "--first-n",
        type=int,
        default=None,
        metavar="N",
        help="Only read first N columns from first N rows",
    )
    parser.add_argument(
        "-l",
        "--labelled",
        action="store_true",
        help="TSV Matrix has column and row labels",
    )
    args = parser.parse_args()

    height = get_height(args.input_file, is_labelled=args.labelled)
    width = get_width(
        args.input_file, is_labelled=args.labelled, delimiter=args.delimiter
    )
    print("height:", height)
    print("width:", width)

    f_in = open(args.input_file, "r", newline="")

    parse(
        f_in,
        h5py.File(args.output_file, "w"),
        height=height,
        width=width,
        delimiter=args.delimiter,
        first_n=args.first_n,
        is_labelled=args.labelled,
    )

    f = h5py.File(args.output_file, "r")
    print("sum1:", np.nansum(f["resolutions"]["1"]["values"][0]))

if __name__ == "__main__":
    main()

max_stars_repo_name: sergpolly/clodius

max_issues_repo_name: sergpolly/clodius

max_forks_repo_name: sergpolly/clodius


text:
import math
import numpy as np
import tensorflow as tf

if __name__ == "__main__":
    data = np.loadtxt("..\data_set\data.txt", dtype=float)
    ## 训练样本
    train_data = data[0: 200]
    data_len = 200
    filename = "..\DNN_tfData_tfrecords\data_path\dataTrain.tfrecords"
    writer = tf.python_io.TFRecordWriter(filename)
    for i in range(0, data_len):
        example = tf.train.Example(features=tf.train.Features(feature={
            "label": tf.train.Feature(float_list=tf.train.FloatList(value=[data[i, 0]])),
            "feature_array": tf.train.Feature(float_list=tf.train.FloatList(value=data[i, 1:17]))
        }))
        writer.write(example.SerializeToString())
    writer.close()
    ## 测试样本
    test_data = data[200: 300]
    data_len = 100
    # 将样本存于第一个文件中
    filename = "..\DNN_tfData_tfrecords\data_path\dataTest.tfrecords"
    writer = tf.python_io.TFRecordWriter(filename)
    for i in range(0, data_len):
        example = tf.train.Example(features=tf.train.Features(feature={
            "label": tf.train.Feature(float_list=tf.train.FloatList(value=[data[i, 0]])),
            "feature_array": tf.train.Feature(float_list=tf.train.FloatList(value=data[i, 1:17]))
        }))
        writer.write(example.SerializeToString())
    writer.close()

max_stars_repo_name: lmxing/keras

max_issues_repo_name: lmxing/keras

max_forks_repo_name: lmxing/keras


text:
# --------------
import pandas as pd
from sklearn import preprocessing

#path : File path

# Code starts here
dataset=pd.read_csv(path)

# read the dataset

# look at the first five columns

dataset.head()
# Check if there's any column which is not useful and remove it like the column id

dataset.drop('Id',axis=1,inplace=True)
# check the statistical description
dataset.describe()

# --------------
# We will visualize all the attributes using Violin Plot - a combination of box and density plots
import seaborn as sns
from matplotlib import pyplot as plt

#names of all the attributes

cols=list(dataset.columns)
#number of attributes (exclude target)
size=len(cols)-1

#x-axis has target attribute to distinguish between classes

x=dataset.iloc[:,-1]
y=dataset.iloc[:,:-1]
#y-axis shows values of an attribute

#Plot violin for all attributes
#fig, ax = pyplot.subplots(figsize =(9, 7))

for feat in cols :
    ax=sns.violinplot( x = dataset[feat])
    plt.show()

# --------------
import numpy
#threshold=0.75
upper_threshold = 0.5
lower_threshold=-0.5

# no. of features considered after ignoring categorical variables
num_features = 10
# create a subset of dataframe with only 'num_features'
subset_train=dataset.iloc[:,0:10]
data_corr=subset_train.corr()
sns.heatmap(data_corr)
correlation=data_corr.unstack().sort_values(kind='quicksort')
corr_var_list=correlation[((correlation>upper_threshold) & (correlation!=1)|(correlation< lower_threshold)
&(correlation!=1))]
#Print correlations and column names

# --------------
#Import libraries
from sklearn import cross_validation
from sklearn.preprocessing import StandardScaler

# Identify the unnecessary columns and remove it
dataset.drop(columns=['Soil_Type7', 'Soil_Type15'], inplace=True)

X=dataset

Y=X[['Cover_Type']].copy()
X.drop("Cover_Type",axis=1,inplace=True)
# Scales are not the same for all variables. Hence, rescaling and standardization may be necessary for some algorithm to be applied on it.
#cross_validation.KFold(10, n_folds=2, shuffle=True, random_state=1):

X_train,X_test,Y_train,Y_test= cross_validation.train_test_split(X, Y, test_size=0.2,random_state=0)
#Standardized
#Apply transform only for non-categorical data

scaler=StandardScaler()
X_train_con=X_train.select_dtypes(include=['int32','int64','float'])
X_test_con=X_test.select_dtypes(include=['int32','int64','float'])
list_con=list(X_train_con.columns)
for feat in list_con:
    X_train_con[feat]=scaler.fit_transform(X_train_con[[feat]])
X_train_temp=X_train_con
for feat in list_con:
    X_test_con[feat]=scaler.fit_transform(X_test[[feat]])
X_test_temp=X_test_con
X_train_cat=X_train.select_dtypes(exclude=['int32','int64','float'])
X_test_cat=X_test.select_dtypes(exclude=['int32','int64','float'])
scaled_features_train_df=pd.concat([X_train_temp,X_train_cat],1)
scaled_features_test_df=pd.concat([X_test_temp,X_test_cat],1)
print(scaled_features_train_df)

#Concatenate non-categorical data and categorical

# --------------
from sklearn.feature_selection import SelectPercentile
from sklearn.feature_selection import f_classif

# Write your solution here:

skb=SelectPercentile(score_func=f_classif,percentile=20)

predictors=skb.fit_transform(X_train1,Y_train)
scores=skb.scores_.tolist()
#print(X_train1)
Features=X.columns
dataframe=pd.DataFrame({'Features':Features,'scores':scores})
dataframe=dataframe.sort_values(by=['scores'],ascending=False)
top_k_predictors=dataframe[dataframe.scores>=dataframe.scores.quantile(.80)].Features.tolist()
print(dataframe.scores.quantile(0.80))
#print(np.percentile(dataframe.scores,80))
print(top_k_predictors)
#x_new = selector.transform(X) # not needed to get the score
#scores = selector.scores_

# --------------
from sklearn.multiclass import OneVsRestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix, precision_score
clf=OneVsRestClassifier(LogisticRegression())
clf1=OneVsRestClassifier(LogisticRegression())
#model_fit_all_features=LogisticRegression()

#model_fit_all_features=clf1.fit(X_train,Y_train)
#prediction_all_feature=model_fit_all_features.predict(X_test)
#score_all_features=accuracy_score(Y_test,prediction_all_feature)
#model_fit_top_features=clf.fit(scaled_features_train_df,top_k_predictors)
#XX=scaled_features_train_df[top_k_predictors]
#model_fit_top_features.fit(XX,Y_train)
#XX_test=scaled_features_test_df[top_k_predictors]
#y_pred=model_fit_top_features.predict(X_test)
#predictions_top_features=model_fit_top_features.predict(scaled_features_test_df)
#score_top_features=accuracy_score(Y_test,predictions_top_features)

clf = LogisticRegression(random_state=0, multi_class='ovr')
clf1 = LogisticRegression(random_state=0, multi_class='ovr')

model_fit_all_features = clf1.fit(X_train, Y_train)
predictions_all_features = clf1.predict(X_test)

score_all_features = accuracy_score(predictions_all_features, Y_test)

print('score_all_features', score_all_features)
# print(scaled_features_train_df.head())
# print(scaled_features_train_df[top_k_predictors].shape)

model_fit_top_features = clf.fit(scaled_features_train_df[top_k_predictors], Y_train)
predictions_top_features = clf.predict(scaled_features_test_df[top_k_predictors])
score_top_features = accuracy_score(predictions_top_features, Y_test)

print('score_top_features',score_top_features)

max_stars_repo_name: anupriya-babbar/ga-learner-dsmp-repo

max_issues_repo_name: anupriya-babbar/ga-learner-dsmp-repo

max_forks_repo_name: anupriya-babbar/ga-learner-dsmp-repo