text:
import os
import threading

import numpy as np
import tensorflow as tf

from a3c_network import *

# Sample code from here
def main(args):
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    sess = tf.InteractiveSession(config=config)

    import lf2gym
    from lf2gym import WebDriver, Character
    env = lf2gym.make(
                startServer=True,
                driverType=WebDriver.Chrome,
                characters=[Character.Firen, Character[args.player]],
                versusPlayer=True)
    env.start_recording()

    num_actions = env.action_space.n
    global_network = Worker(-1, num_actions, 1e-4, sess, AIchar=env.characters[0])

    saver = tf.train.Saver()
    latest_ckpt = tf.train.latest_checkpoint(args.loadpath)
    saver.restore(sess, latest_ckpt)
    print("restore from", latest_ckpt)

    global_network.test(env)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--loadpath', type=str, default='/path/to/model/', help="Path to load model.")
    parser.add_argument('--player', type=str, default='Bandit', help="Player that you want to play.")
    parser.add_argument('--comment', type=str, default='', help="Comment.")
    args = parser.parse_args()
    main(args)

# Terminal:
# For train: python a3c.py --train --savepath 'model_a3c/'
# For test: python a3c.py --test --loadpath 'model_a3c/'

ext: py

reason: import numpy


text:
import time
import numpy as np
import Adafruit_PCA9685

#pwm = Adafruit_PCA9685.PCA9685()
#pwm.set_pwm_freq(60)

class rgb_led:
    def __init__(self, red_channel = 5, green_channel = 6, blue_channel = 7,frequency=60):
        self.pwm = Adafruit_PCA9685.PCA9685()
        #self.pwm.set_pwm_freq(frequency)
        self.red_channel = red_channel
        self.green_channel = green_channel
        self.blue_channel = blue_channel
        #self.color=color
        self.led=(self.red_channel,self.green_channel,self.blue_channel)
        RGB_LED_set_color(self.pwm , self.led , (0,25,0))

    def set_color(self,color):
        RGB_LED_set_color(self.pwm , self.led , color)

    def run(self):
        pass

    def shutdown(self):
        RGB_LED_set_color(self.pwm , self.led , (0,25,0))

class turn_signal():
    def __init__(self, left_led=None, right_led=None):
        self.left_led = left_led
        self.right_led = right_led

    def run(self,angle):

        if angle > 0:

            self.right_led.set_color(((25 * abs(angle)),(25 * abs(angle)),0))
            self.left_led.set_color((0,0,0))

        if angle < 0:
            self.left_led.set_color(((25 * abs(angle)),(25 * abs(angle)),0))
            self.right_led.set_color((0,0,0))

        if angle == 0:
            self.left_led.set_color((0,0,0))
            self.right_led.set_color((0,0,0))

class status_indicator():
    def __init__(self, status_led=None):
        self.status_led = status_led

    def run(self,user_mode,recording):
        #print(str(user_mode))

        if recording:
            self.status_led.set_color((255,0,0))
        else:
            if user_mode == 'face':
                self.status_led.set_color((0,50,50))
            elif user_mode == 'user':
                self.status_led.set_color((0,50,0))
            elif user_mode == 'local':
                self.status_led.set_color((0,0,255))
            elif user_mode == 'local_angle':
                self.status_led.set_color((255,0,255))

def main():
    #import Adafruit_PCA9685
    pwm = Adafruit_PCA9685.PCA9685()
    pwm.set_pwm_freq(60)

    # set up the LED pins.  (R,G,B)
    LEDS = (
        (5,6,7),
        (9,10,11),
        (13,14,15)
    )

    #initiales LEDS to purple
    for led in LEDS:
        RGB_LED_set_color(pwm,led,(20,0,50))

    # set LEDs to RGB dim purple.. RED and GREEN scaled down
    #RGB_LED_set_color(pwm,LEDS[0],(255,0,255),scale=0.01)
    #RGB_LED_set_color(pwm,LEDS[1],(255,0,255),scale = 0.01)
    #RGB_LED_set_color(pwm,LEDS[2],(255,0,255),scale=0.01)

    #RGB_LED_set_color(pwm,LEDS[0],(20,0,50))
    #RGB_LED_set_color(pwm,LEDS[1],(20,0,50))
    #RGB_LED_set_color(pwm,LEDS[2],(20,0,50))

    ##time.sleep(3)
    #t = 1
    #l = 255
    #pwm.set_pwm(11,0,4095)
    '''
    while True:
        RGB_LED_set_color(pwm,LED_0,(l,0,0))
        time.sleep(t)

        RGB_LED_set_color(pwm,LED_0,(0,l,0))
        time.sleep(t)

        RGB_LED_set_color(pwm,LED_0,(0,0,l))
        time.sleep(t)

        RGB_LED_set_color(pwm,LED_0,(0,0,0))
        time.sleep(t)

        RGB_LED_set_color(pwm,LED_1,(l,0,0))
        time.sleep(t)

        RGB_LED_set_color(pwm,LED_1,(0,l,0))

        time.sleep(t)
        RGB_LED_set_color(pwm,LED_1,(0,0,l))
        time.sleep(t)

        RGB_LED_set_color(pwm,LED_1,(0,0,0))
        time.sleep(t)

        RGB_LED_set_color(pwm,LED_2,(l,0,0))
        time.sleep(t)

        RGB_LED_set_color(pwm,LED_2,(0,l,0))
        time.sleep(t)

        RGB_LED_set_color(pwm,LED_2,(0,0,l))
        time.sleep(t)

        RGB_LED_set_color(pwm,LED_2,(0,0,0))
        time.sleep(t)

        '''

def RGB_LED_set_color(driver , pins=(13,14,15) , RGB=(255,255,255) , CommonAnode=True, scale=1):

    # I only have Common Anode RGB leds, I THINK that Common Cathod LED's work differntly.
    # To DO.  Add Common Cathode support
    if CommonAnode:
        #invert rgb input values so 255,0,0 input is RED and not the oposite..
        outRGB= (
            int(abs(((RGB[0] * scale) - 255 )*(16))),
            int(abs(((RGB[1] * scale) - 255 )*(16))),
            int(abs(((RGB[2] * scale) - 255 )*(16)))
            )

        #print('input: ' + str(RGB) + 'inveted' + str(outRGB))
    else:
        outRGB = RGB
        #for val in RGB:
        #for i in range(0 , RGB.count())
        #    print(str(RGB[i]))
            #invertedI =np.invert(np.array(val),dtype=np.int8)
            #print(invertedI)
            #RGB[i] = invertedI

    # set the values for each R G B channel
    driver.set_pwm(pins[0],0,outRGB[0])
    driver.set_pwm(pins[1],0,outRGB[1])
    driver.set_pwm(pins[2],0,outRGB[2])

if __name__ == "__main__":
    main()

max_stars_repo_name: wheatgrinder/donkey

max_issues_repo_name: wheatgrinder/donkey

max_forks_repo_name: wheatgrinder/donkey


text:
import os
import pickle
import numpy as np
import h5py
from sklearn.externals import joblib
from gala import imio, agglo, features, classify

fman = features.default.snemi3d()

def train(index):
    outfn = 'training-data-%i.h5' % index
    if os.path.exists(outfn):
        data, labels = classify.load_training_data_from_disk(outfn,
                                                             names=['data',
                                                                    'labels'])
    else:
        ws_tr = imio.read_image_stack('watershed-%i.lzf.h5' % index)
        pr_tr = imio.read_image_stack('probabilities-inv-norm-%i.lzf.h5' % index)
        gt_tr = imio.read_image_stack('ground-truth-%i.lzf.h5' % index)
        g = agglo.Rag(ws_tr, pr_tr,
                      feature_manager=fman)
        data, labels = g.learn_agglomerate(gt_tr, fman, min_num_epochs=4)[0][:2]
        classify.save_training_data_to_disk([data, labels],
                                            fn=outfn,
                                            names=['data', 'labels'])
    print('total training data:', data.shape)
    print('size in MB:', data.size * data.itemsize / 1e6)
    rf = classify.DefaultRandomForest(n_jobs=6)
    rf.fit(data, labels[:, 0])
    policy = agglo.classifier_probability(fman, rf)
    return policy

def test(index, policy):
    ws = imio.read_image_stack('watershed-%i.lzf.h5' % index)
    pr = imio.read_image_stack('probabilities-inv-norm-%i.lzf.h5' % index)
    g = agglo.Rag(ws, pr, merge_priority_function=policy,
                  feature_manager=fman)
    g.agglomerate(np.inf)
    return g.tree

def train_test_pair(training_index, testing_index):
    print('training %i' % training_index)
    policy = train(training_index)
    print('testing %i' % testing_index)
    tree = test(testing_index, policy)
    with open('results-%i-tr%i.pickle' % (testing_index, training_index),
              'wb') as fout:
        pickle.dump(tree, fout, protocol=-1)
    return tree

def write_saalfeld(fn, raw, labels, res=np.array([12., 1, 1])):
    imio.write_h5_stack(raw, fn, group='raw')
    imio.write_h5_stack(labels, fn, group='labels')
    f = h5py.File(fn, 'a')
    f['/raw'].attrs['resolution'] = res
    f['/labels'].attrs['resolution'] = res
    f.close()

if __name__ == '__main__':
    index_pairs = [(3 - ts, ts) for ts in range(4)]
    trees = joblib.Parallel(n_jobs=4)(joblib.delayed(train_test_pair)(*p)
                                      for p in index_pairs)
    trees = dict(zip(index_pairs, trees))
    with open('results.pickle', 'wb') as fout:
        pickle.dump(trees, fout, protocol=-1)
    images = imio.read_image_stack('/groups/saalfeld/saalfeldlab/concha/sample_A/crop/raw/*.tiff')
    wss = [imio.read_image_stack('watershed-%i.lzf.h5' % i) for i in range(4)]
    maps = [t.get_map(0.5) for t in trees]
    segs = [m[ws] for m, ws in zip(maps, wss)]

    seg = np.zeros(images.shape, dtype=np.uint64)
    seg[:, :625, :625] = segs[0]
    seg[:, :625, 625:] = segs[1] + np.max(segs[0])
    seg[:, 625:, :625] = segs[2] + np.max(segs[0]) + np.max(segs[1])
    seg[:, 625:, 625:] = segs[3] + np.max(segs[0]) + np.max(segs[1]) + np.max(segs[2])

    write_saalfeld('/groups/saalfeld/saalfeldlab/concha/sample_A/juan/corners-segments2.h5',
                   images, seg)

max_stars_repo_name: jni/gala-scripts

max_issues_repo_name: jni/gala-scripts

max_forks_repo_name: jni/gala-scripts


text:
import numpy as np

class Env:
    def __init__(self, dt=0.05, m=0, f=0, max_iters=1000):
        """Initialise a shear flow environment."""
        # The initial starting position
        self._X = np.zeros([2, 1])
        # The goal position
        self._G = np.array([[0], [1]])
        # The time step
        self._dt = dt
        # The control vector
        self._w = np.zeros([2, 1])
        # The current gradient
        self._m = m
        # The current velocity at starting point
        self._f = f
        # Initialise the current at starting position
        self._v = np.array([[self._f], [0]])
        # Initialise the iteration counter
        self._i = 0
        # Store the termination counter
        self._I = max_iters

    def get(self):
        """Get the distance vector to destination, previous speed of the agent, current velocity."""
        return np.vstack((self._G-self._X, self._w, self._v))

    def reset(self):
        """Reset the environment."""
        # The initial starting position
        self._X = np.zeros([2, 1])
        # Initialise the current at starting position
        self._v = np.array([[self._f], [0]])
        # The control vector
        self._w = np.zeros([2, 1])
        # Initialise the iteration counter
        self._i = 0

    def _shearflow(self):
        """Get the velocity components for the current position."""
        # TODO Come up with a more sophisticated current model.
        self._v[0] = self._m*self._X[1] + self._f
        return self._v

    def update(self, w):
        """Given the control vector, update the environment and return the reward."""
        # Calculate the movement
        X_dot = w - self._shearflow()
        # Update the position.
        self._X = self._X + X_dot*self._dt
        # Calculate the distance
        distance = np.linalg.norm(self._G-self._X)
        # Check if the goal has been reached.
        success = distance <= 1e-3
        self._i = self._i + 1
        success = success or self._i >= self._I
        # Calculate the energy change due to change in control vector.
        energy = np.linalg.norm(w-self._w)**2
        # Update the control velocity with w
        self._w = w
        return success, distance, energy, self._dt

max_stars_repo_name: FletcherFT/NeuroVolution

max_issues_repo_name: FletcherFT/NeuroVolution

max_forks_repo_name: FletcherFT/NeuroVolution


text:
import os
import pandas as pd
import numpy as np
from itertools import chain

from codifyComplexes.CodifyComplexException import CodifyComplexException
from .DataLoaderClass import  DataLoader

class AbstractProtocol(DataLoader):
  '''
    This class is a template for codification protocols
  '''

  DESIRED_ORDER=["L", "R", "P"]
  def __init__(self, dataRootPath, cMapPath, prevStepPaths, singleChainfeatsToInclude, pairfeatsToInclude=None, verbose=False):
    '''
      @param dataRootPath: str. A path to computedFeatures directory that contains needed features. Example:
                computedFeatures/
                  common/
                    contactMaps/
                  seqStep/
                    conservation/
                    ...
                  structStep/
                    PSAIA/
                    VORONOI/
                    ...

      @param cMapPath: str. A path to a dir that contains the contact map of the protein complex

      @param prevStepPaths: str or str[]. A path to previous results files directory. If it is None, contactMaps files will be used
                                 to define which residue pairs are in contact. Can also be a str[] if multiple feedback_path's
                                 wanted

      @param singleChainfeatsToInclude: List that contains the paths where single chain features needed for complex
                                      codification are located. Must have the following format:
                                      ["featName":(relativePath_from_dataRootPath, listOfColumnNumbers, dictForNamedColums)]
                                          dictForNamedColums= {"myFeatName":colNumber} or Empty
      @param pairfeatsToInclude: List that contains the paths where pair features needed for complex
                                      codification are located. Must have the following format:
                                      ["featName":(relativePath_from_dataRootPath, listOfColumnNumbers, dictForNamedColums)]
                                          dictForNamedColums= {"myFeatName":colNumber} or {}
      @param verbose: bool.
    '''
    DataLoader.__init__(self, dataRootPath, singleChainfeatsToInclude, pairfeatsToInclude, verbose)
    self.dataRootPath= dataRootPath
    self.cMapPath= cMapPath
    self.verbose= verbose
    if prevStepPaths is None:
      self.prevFnamesList=None
    else:
      self.prevFnamesList= prevStepPaths if isinstance(prevStepPaths,list) else [prevStepPaths]
      self.prevFnamesList= [os.path.join(onePath, fname) for onePath in self.prevFnamesList for fname in os.listdir(onePath)
                                          if fname.endswith(".res.tab")]
    if not self.useCorrMut:
      self.pairfeatsToInclude = [ elem for elem in self.pairfeatsToInclude if elem[0]!="corrMut"]
      if len(self.pairfeatsToInclude)==0:
        self.pairfeatsToInclude = None

  def applyProtocol( self,  prefixComplex, prefixL, prefixR):
    '''
      This method is the basic skeleton for applyProtocol of subclasses
      Given a prefix that identifies the complex and prefixes that identifies
      the ligand and the receptor, this method integrates the information that
      is contained in self.dataRootPath and is described in self.singleChainfeatsToInclude

      @param prefixComplex: str. A prefix that identifies a complex
      @param prefixL: str. A prefix that identifies the ligand of the complex
      @param prefixR: str. A prefix that identifies the receptor of the complex
      @return df: pandas.DataFrame. A pandas.Dataframe in which each row represents
                      a pair of amino acids in direct form (L to R).
                      Column names are:
                      'chainIdL', 'structResIdL', 'resNameL', 'chainIdR', 'structResIdR', 'resNameR', 'categ'
                       [propertiesP .... propertiesL     .... propertiesR]
    '''

    if self.prevFnamesList is None:
      cmapNameList= list(self.getFullNamesIterInPath( prefixComplex, self.cMapPath))
      if len(cmapNameList)>1:
        raise ValueError("There are more than 1 Contact map for %s in %s path"%(prefixComplex,self.cMapPath))
      allPairsCodified= self.loadDataFile(cmapNameList)
    else:
      allPairsCodified= self.loadPreviousResults(prefixComplex)
    if not self.pairfeatsToInclude is None:
      allPairsCodified= self.addPairFeatures(prefixComplex, allPairsCodified)
    lFeats= self.loadSingleChainFeatures( prefixL, "l")
    rFeats= self.loadSingleChainFeatures( prefixR, "r")
#    print(lFeats["chainIdL"].unique(), rFeats["chainIdR"].unique(), allPairsCodified["chainIdL"].unique(), allPairsCodified["chainIdR"].unique())
    allPairsCodified= self.combinePairwiseAndSingleChainFeats(allPairsCodified,lFeats, rFeats)
    assert allPairsCodified.shape[0]>1, "Error, %s dataset is empty"%prefixComplex
    #Reorder columns
    allPairsCodified= self.reorderColumns(allPairsCodified)
    return allPairsCodified

  def reorderColumns(self, allPairsCodified):
    colNames= list(allPairsCodified.columns)
    categIndex= colNames.index("categ")
    lFeatNames= [elem for elem in colNames[(categIndex+1):] if elem[-1]=="L"]
    rFeatNames= [elem for elem in colNames[(categIndex+1):] if elem[-1]=="R"]
    pairwiseFeatNames= [elem for elem in colNames[(categIndex+1):] if elem.endswith("_P")]
    colOrder= list(colNames[:(categIndex+1)])
    for featType in AbstractProtocol.DESIRED_ORDER:
      if featType=="L":
        colOrder+=lFeatNames
      elif featType=="R":
        colOrder+=rFeatNames
      elif featType=="P":
        colOrder+=pairwiseFeatNames
      else:
        raise ValueError("just L, R or P allowed in idAbstractProtocol.DESIRED_ORDER")

    allPairsCodified= allPairsCodified[ colOrder ]
    allPairsCodified.columns= colOrder

    return allPairsCodified

  def loadPreviousResults(self, prefixComplex):
    '''
      Loads previous results. Returns a pandas.DataFrame that contains in each row
      the scores of previous steps for a given pair of amino acids.
      @param prefixComplex: str. A prefixOneChainType that identifies the receptor or ligand

      @return df: pandas.DataFrame. A pandas.Dataframe in which each row represents
                      one amino acid
                      Column names are:
                      'chainIdL', 'structResIdL', 'resNameL', 'chainIdR', 'structResIdR', 'resNameR',
                      'categ', [prev_step_scores]
    '''
    previousResultsList= [fname for fname in self.prevFnamesList if fname.endswith(prefixComplex+".res.tab")]
    assert len(previousResultsList)>0, "No previous results"
    for fname in previousResultsList:
      if self.verbose: print("loading previous resuls for %s"%(prefixComplex))
      prevResults= self.loadDataFile( iter([fname]) )
      prevResults.loc[:,"prediction_norm"]= (prevResults["prediction"] - np.mean(prevResults["prediction"])) / np.std(prevResults["prediction"])
      break # break to add just one type of previous predictions
    if len(previousResultsList)>1:
      for fname in previousResultsList[1:]: #Load the remaining previous predictions
        if self.verbose: print("loading previous resuls for %s"%(prefixComplex))

        prevNrow= prevResults.shape[0]
        newData= self.loadDataFile( iter([fname] ))
        newData.loc[:,"prediction_norm"]= (newData["prediction"] - np.mean(newData["prediction"])) / np.std(newData["prediction"])

        prevResults= pd.merge(prevResults, newData, how='inner', on=["chainIdL",  "structResIdL","resNameL",
                                                                     "chainIdR",  "structResIdR","resNameR","categ"])
        curNrow= prevResults.shape[0]
        if prevNrow<1:
          raise CodifyComplexException(("Error merging previous results in %s. There are 0 rows "+
                                        "in previous features")%(prefixComplex ))
        elif (abs(float(prevNrow- curNrow)) / prevNrow) > 0.2:
          print("%s Nrows previously/now %d/%d  %s"%(prefixComplex, prevNrow, curNrow,fname))
          raise CodifyComplexException(("Error merging previous results  in %s. There are a different number of residues "+
                                        "compared to previous file")%(prefixComplex))
    return prevResults

  def loadSingleChainFeatures(self, prefixOneChainType, chainType):
    '''
      Loads all features files computed for ligand or receptor chains. Returns a pandas.DataFrame
      that contains in each row all features from all files for each amino acid. Just amino acids
      that appears in each file will be included. Others will be ruled out (intersection)
      @param prefixOneChainType: str. A prefixOneChainType that identifies the receptor or ligand
      @param chainType: str. "l" for ligand and "r" for receptor
      @return df: pandas.DataFrame. A pandas.Dataframe in which each row represents
                      one amino acid
                      Column names are:
                      'chainId%s', 'structResId%s', 'resName%s', [properties]
                      %s is L if chainType=="l" and R if  chainType=="r"
    '''

    assert chainType=="l" or chainType=="r"
    chainType= chainType.upper()
    oneChainTypeFeats= None
#    self.verbose=True
    for featNum in range(len(self.singleChainfeatsToInclude)): #Load just one single chain feature
      featName, params= self.getParamsForLoadingFile( prefixOneChainType, featNum)
      if self.verbose: print("loading %s for %s"%(featName,prefixOneChainType))
      oneChainTypeFeats= self.loadDataFile(*params)
      break # break to add just one type of feature. Next will be added in following loop
    if len(self.singleChainfeatsToInclude)>1:
      for featNum in range(len(self.singleChainfeatsToInclude))[1:]: #Load the remaining single chain feature
        featName, params= self.getParamsForLoadingFile( prefixOneChainType, featNum)
        if self.verbose: print("loading %s for %s"%(featName, prefixOneChainType))
        prevNrow= oneChainTypeFeats.shape[0]
        newData= self.loadDataFile(*params)
#        print(newData.head())
#        print(newData.shape, newData["chainId"].unique())
#        print(prefixOneChainType, chainType, featName)
#        raw_input("enter to continue")
#        print(oneChainTypeFeats.head())
#        print(oneChainTypeFeats.shape, oneChainTypeFeats["chainId"].unique())
#        raw_input("enter to continue")
#        print(list(oneChainTypeFeats.columns), list(newData.columns),oneChainTypeFeats.shape)
        oneChainTypeFeats= pd.merge(oneChainTypeFeats, newData, how='inner', on=["chainId",  "structResId","resName"])
#        print(oneChainTypeFeats.columns, oneChainTypeFeats.shape)
#        raw_input()
        curNrow= oneChainTypeFeats.shape[0]
        if prevNrow<1:
          raise CodifyComplexException(("Error merging previous single chain feature %s in %s. There are 0 rows "+
                                        "in previous feature to %s")%(featName, prefixOneChainType, featName))
        elif (abs(float(prevNrow- curNrow)) / prevNrow) > 0.2:
          print("Nrows previously/now", prevNrow, curNrow)
          print("%s Nrows previously/now %d/%d  %s"%(prefixOneChainType, prevNrow, curNrow,featName))
          raise CodifyComplexException(("Error merging single chain feature %s in %s. There are a different number of residues "+
                                        "in %s compared to previous features")%(featName, prefixOneChainType, featName))

    oneChainTypeFeats.rename(columns={elem:elem+chainType for elem in list(oneChainTypeFeats.columns.values)}, inplace=True)
    return oneChainTypeFeats

  def addPairFeatures(self, prefix, allPairs):
    '''
      Loads all files computed for pairwise features and adds them to pairs residue df contained in allPairs.
      Returns a pandas.DataFrame that contains in each row all pairwise features from all files for each pair of amino acids.
      Just amino acid pairs that appears in each file will be included. Others will be ruled out (intersection)
      @param prefix: str. A prefix that identifies the complex
      @param allPairs: pandas.DataFrame. A pandas.Dataframe in which each row represents one amino acid pair
                        Column names are:
                        "chainIdL",  "structResIdL","resNameL","chainIdR",  "structResIdR","resNameR", "categ", [previousScores]
      @return df: pandas.DataFrame. A pandas.Dataframe in which each row represents one amino acid pair
                      Column names are:
                      "chainIdL",  "structResIdL","resNameL","chainIdR",  "structResIdR","resNameR", "categ", [properties]
    '''

    pairTypeFeats= None
    self.verbose=True
    for featNum in range(len(self.pairfeatsToInclude)): #Load just one single chain feature
      featName, params= self.getParamsForLoadingFile( prefix, featNum, lookForPairFeats=True)
      if self.verbose: print("loading %s for %s"%(featName,prefix))
      pairTypeFeats= self.loadDataFile(*params)
      break # break to add just one type of feature. Next will be added in following loop
    if len(self.pairfeatsToInclude)>1:
      for featNum in range(len(self.pairfeatsToInclude))[1:]: #Load the remaining single chain feature
        featName, params= self.getParamsForLoadingFile( prefix, featNum, lookForPairFeats=True)
        if self.verbose: print("loading %s for %s"%(featName, prefix))
        prevNrow= pairTypeFeats.shape[0]
        newData= self.loadDataFile(*params)
        pairTypeFeats= pd.merge(pairTypeFeats, newData, how='inner', on=["chainIdL",  "structResIdL","resNameL",
                                                                         "chainIdR",  "structResIdR","resNameR"])
        curNrow= pairTypeFeats.shape[0]
        if prevNrow<1:
          raise CodifyComplexException(("Error merging previous pair feature %s in %s. There are 0 rows "+
                                        "in previous feature to %s")%(featName, prefix, featName))
        elif (abs(float(prevNrow- curNrow)) / prevNrow) > 0.2:
          print("Nrows previously/now", prevNrow, curNrow)
          print("%s Nrows previously/now %d/%d  %s"%(prefix, prevNrow, curNrow,featName))
          raise CodifyComplexException(("Error merging pair feature %s in %s. There are a different number of residues "+
                                        "in %s compared to previous features")%(featName, prefix, featName))

    #check if _P has being assigned to allPairs pairwise features
    nPs= sum( (1 for elem in pairTypeFeats.columns.values if elem.endswith("_P")))
    if nPs>0: #if so add _P to pairwise features
      pairTypeFeats.rename(columns={elem:elem+"_P" for elem in set(pairTypeFeats.columns.values
                                                           ).difference(AbstractProtocol.ARE_STR_TYPE_COLUMNS)}, inplace=True)
#    print(allPairs.shape)
    prevNrow= allPairs.shape[0]
    allPairs= pd.merge(allPairs, pairTypeFeats, how='inner', on=["chainIdL",  "structResIdL","resNameL",
                                                                       "chainIdR",  "structResIdR","resNameR"])

#    print(allPairs.head())
#    print(allPairs.shape)
#    raw_input("enter")
    curNrow= allPairs.shape[0]
    if abs(float(prevNrow- curNrow)) / prevNrow > 0.05:
      print("Nrows previously/now", prevNrow, curNrow)
      print("%s Nrows previously/now %d/%d  %s"%(prefix, prevNrow, curNrow,featName))
      raise CodifyComplexException(("Error merging pair feature %s in %s. There are a different number of residues "+
                                    "in %s compared to previous features")%(featName, prefix, featName))
    return allPairs

  def combinePairwiseAndSingleChainFeats(self, pairFeats, singleFeatsL, singleFeatsR):
    '''
      Merges pairFeats pandas.DataFrame with singleFeatsL and singleFeatsR dataFrames.
      singleFeatsL has n rows (as many as ligand residues) and singleFeatsR has m rows
      (as many as ligand residues), and pairsFeats has ~ n*m rows
      (some amino acids pairs might not be considered due to several reasons).

      @param pairFeats: pandas.DataFrame. A pandas.Dataframe in which each row represents the properties of
                        a residue of the receptor
                        Column names are:
                        'chainIdL', 'structResIdL', 'resNameL', 'chainIdR', 'structResIdR', 'resNameR', 'categ'
                         [propertiesP]

      @param singleFeatsL: pandas.DataFrame. A pandas.Dataframe in which each row represents the properties of
                           a residue of the ligand
                           Column names are:
                           'chainIdL', 'structResIdL', 'resNameL',[propertiesL]

      @param singleFeatsR: pandas.DataFrame. A pandas.Dataframe in which each row represents the properties of
                           a residue of the receptor
                           Column names are:
                           'chainIdR', 'structResIdR', 'resNameR',[propertiesR]

      @return directPairs: pandas.DataFrame. A pandas.Dataframe in which each row represents
                      a pair of amino acids in direct form (L to R).
                      Column names are:
                      'chainIdL', 'structResIdL', 'resNameL', 'chainIdR', 'structResIdR', 'resNameR', 'categ'
                      [properties_P .... propertiesL     .... propertiesR]

    '''
    if singleFeatsL.shape[1]!= singleFeatsR.shape[1]:
      print( "singleFeatsL and singleFeatsR have different number of variables")
      featsLNo_chain= set( (elem[:-1] for elem in singleFeatsL.columns))
      featsRNo_chain= set( (elem[:-1] for elem in singleFeatsR.columns))
      print("L %d R %d"%(len(singleFeatsL.columns),len(singleFeatsR.columns)) ,"L dif R",
            sorted(featsLNo_chain.difference(featsRNo_chain)), "R diff L",
            sorted(featsRNo_chain.difference(featsLNo_chain)) )
      raise ValueError( "singleFeatsL and singleFeatsR have different number of variables")
    otherPairColumns= set(pairFeats.columns.values).difference(set(AbstractProtocol.ARE_STR_TYPE_COLUMNS+["categ"]))
    if len(otherPairColumns)>0: #add _P suffix to all pairwise features
      pairFeats.rename(columns={elem:elem+"_P" for elem in otherPairColumns}, inplace=True)
    directPairs= pd.merge(pairFeats,   singleFeatsL, how='inner', on=None)
    directPairs= pd.merge(directPairs, singleFeatsR, how='inner', on=None)
    return directPairs

  def prepareDataForPairwiseAggregat(self, df):
    '''
      abstract method
    '''
    return None

  def addPairwiseAggregation(self, df):
    '''
      Adds environment pairwise features to a df that contains pairwise features (named featName_P)
      @param df: pandas.DataFrame. A pandas.Dataframe in which each row represents
                      a pair of amino acids in direct form (ligand, receptor)
                      Column names are:
                      'chainIdL', 'structResIdL', 'resNameL', 'chainIdR', 'structResIdR', 'resNameR', 'categ'
                      [properties_P .... propertiesL .... propertiesR] #no defined order for properties

      @return newDf: pandas.DataFrame. A pandas.Dataframe in which each row represents
                      a pair of amino acids in direct form (L to R). New columns will have been added, all then
                      named %sAggr%d%s(numAggr/factorAggr, numberOfNewFeature, ligand/receptor)
                      Column names are:
                      'chainIdL', 'structResIdL', 'resNameL', 'chainIdR', 'structResIdR', 'resNameR', 'categ'
                      [ properties_P propertiesR propertiesL ] #no defined order for properties
    '''

    pairwiseDf, ids2RowL, ids2RowR, neigsids2rowL, neigsids2rowR = self.prepareDataForPairwiseAggregat(df)
    nElems= df.shape[0]

    l_2_r_neigs= [[] for i in range(df.shape[0])]
    r_2_l_neigs= [[] for i in range(df.shape[0])]
    l_neigs_2_r_neigs= [[] for i in range(df.shape[0])]
    for idL in ids2RowL:
      rowsInvolvingL= ids2RowL[idL]
      rowsNeigsL= neigsids2rowL[idL]
      for idR in ids2RowR:
        rowsInvolvingR= ids2RowR[idR]
        rowsNeigsR= neigsids2rowR[idR]
        row_Index= tuple(rowsInvolvingL.intersection(rowsInvolvingR))[0]
        l_2_r_neigs[row_Index]= tuple(rowsInvolvingL.intersection(rowsNeigsR))
        r_2_l_neigs[row_Index]= tuple(rowsInvolvingR.intersection(rowsNeigsL))
        l_neigs_2_r_neigs[row_Index]= tuple(rowsNeigsR.intersection(rowsNeigsL))

    numericAggreL2r= computeNumericAggr(pairwiseDf, l_2_r_neigs)
    numericAggreR2l= computeNumericAggr(pairwiseDf, r_2_l_neigs)
    numericAggreN2N= computeNumericAggr(pairwiseDf, l_neigs_2_r_neigs)
    numericAggreL2r.columns= [ "l2r-pair"+elem for elem in numericAggreL2r.columns]
    numericAggreR2l.columns= [ "r2l-pair"+elem for elem in numericAggreR2l.columns]
    numericAggreN2N.columns= [ "n2n-pair"+elem for elem in numericAggreN2N.columns]
    df= pd.concat([df,  numericAggreL2r, numericAggreR2l, numericAggreN2N ], axis=1)
    return df

def computeNumericAggr(df, selectedRows):
  '''
    Compute aggregation functions (min, max, mean and sum) for the rows of data. Each row is aggregated
    over the rows included in selectedRows
    @param data: pandas.DataFrame(nrow, nfeatures). The numeric data to aggregate. Each row i is averaged, sum... with
                                          the rows such that selectedRows[i,:]==True
    @param selectedRows: [[]]: int (nrow, variableLength)): Each row i is averaged, with the rows included in
                                                    selectedRows[i]
    @return np.array (nrow, 4*nfeatures)
  '''
#  s= time.time()
  data= df.values
  nVars= data.shape[1]
  aggregatedResults= -(2**10)* np.ones( (data.shape[0], nVars* 4))
  splitPoint2= 2*nVars
  splitPoint3= 3*nVars
  splitPoint4= 4*nVars
  for i in range(data.shape[0]):
    dataToAggregate= data[selectedRows[i], :]

    if dataToAggregate.shape[0]>0:
      aggregatedResults[i, 0:nVars]=                np.mean( dataToAggregate, axis=0)
      aggregatedResults[i, nVars:splitPoint2]=      np.max(  dataToAggregate, axis=0)
      aggregatedResults[i, splitPoint2:splitPoint3]= np.min( dataToAggregate, axis=0)
      aggregatedResults[i, splitPoint3:splitPoint4]= np.sum( dataToAggregate, axis=0)

  aggregatedResults= pd.DataFrame(aggregatedResults)
  aggregatedResults.columns= list(chain.from_iterable([[ "numAggr-"+oper+"-"+name for name in df.columns] for
                                                      oper in ["mean", "max", "min", "sum"] ]))
  return aggregatedResults

def computeFactorAggr(df, selectedRows):
  '''
    Compute aggregation function (sum) for the rows of data. Each row is aggregated
    over the rows included in selectedRows
    @param data: pandas.DataFrame(nrow, nfeatures). The numeric data to aggregate. Each row i is averaged,  with
                                          the rows such that selectedRows[i,:]==True
    @param selectedRows: [[]]: int (nrow, variableLength)): Each row i is added with the rows included in
                                                    selectedRows[i]

    @return np.array (nrow, nfeatures)
  '''
  data= df.values
#  aggregatedResults= -(2**16)* np.ones( (data.shape[0], data.shape[1]))
  aggregatedResults= -(2**10)* np.ones( (data.shape[0], data.shape[1]))
  for i in range(data.shape[0]):
    dataToAggregate= data[selectedRows[i],:]
    if dataToAggregate.shape[0]>0:
      aggregatedResults[i, :]= np.sum( dataToAggregate, axis=0)
  aggregatedResults= pd.DataFrame(aggregatedResults)
  aggregatedResults.columns= [ "factorAggr-"+name for name in df.columns]
  return aggregatedResults

max_stars_repo_name: bioinsilico/BIPSPI

max_issues_repo_name: bioinsilico/BIPSPI

max_forks_repo_name: bioinsilico/BIPSPI


text:
from helper import *
import pandas as pd
import os
import glob
import re
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import numpy as np
from sklearn.decomposition import PCA
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.metrics import mean_squared_error

import warnings
warnings.filterwarnings("ignore")

def test_feat(cols, df):
    combined = df
    X = combined[cols]
    y = combined.latency
    # randomly split into train and test sets, test set is 80% of data
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8, random_state=1)

    clf = DecisionTreeRegressor()
    clf = clf.fit(X_train,y_train)
    y_pred = clf.predict(X_test)
    acc1 = mean_squared_error(y_test, y_pred)
    #print("Decision Tre Accuracy:", acc1, '\n')

    clf2 = RandomForestRegressor(n_estimators=10)
    clf2 = clf2.fit(X_train,y_train)
    y_pred2 = clf2.predict(X_test)
    acc2= mean_squared_error(y_test, y_pred2)
    #print("Random Forest Accuracy:", acc2, '\n')

    clf3 = ExtraTreesRegressor(n_estimators=10)
    clf3 = clf3.fit(X_train,y_train)
    y_pred3 = clf3.predict(X_test)
    acc3= mean_squared_error(y_test, y_pred3)
    #print("Extra Trees Accuracy:", acc3, '\n')

    pca = PCA()
    X_transformed = pca.fit_transform(X_train)
    cl = DecisionTreeRegressor()
    cl.fit(X_transformed, y_train)
    newdata_transformed = pca.transform(X_test)
    y_pred4 = cl.predict(newdata_transformed)
    acc4 = mean_squared_error(y_test, y_pred4)
    #print("PCA Accuracy:", acc4, '\n')

    return [acc1, acc2, acc3, acc4 ]

def getAllCombinations(object_list):
    lst = ['total_bytes', 'max_bytes', 'proto', '1->2Bytes','2->1Bytes', '1->2Pkts', '2->1Pkts', 'total_pkts', 'loss']
    uniq_objs = set(lst)
    combinations = []
    for obj in uniq_objs:
        for i in range(0,len(combinations)):
            combinations.append(combinations[i].union([obj]))
        combinations.append(set([obj]))
    print("all combinations generated")
    return combinations

def test_mse(all_comb, df):
    all_comb2 = pd.Series(all_comb).apply(lambda x: list(x))
    dt = []
    rf = []
    et = []
    pca = []
    counter = 0
    for i in all_comb2:
        accs = test_feat(i, df)
        #print(accs)
        dt.append(accs[0])
        rf.append(accs[1])
        et.append(accs[2])
        pca.append(accs[3])
        counter +=1
        #print("combination:" + str(counter))

    dictin = {'feat': all_comb, 'dt': dt, 'rf': rf, 'et': et, 'pca': pca}

    feat_df = pd.DataFrame(dictin)
    return feat_df

def best_performance(df):
    print("finding best performance")
    sorted = df.sort_values(by=['dt', 'rf', 'et', 'pca'], ascending = False)
    path = os.path.join(os.getcwd() , "outputs")
    #combined.to_csv("combined_latency.csv")
    sorted.to_csv(os.path.join(path, "sorted_df.csv"), index = False)
    return sorted[:1]


text:
#!/usr/bin/env python

from __future__ import absolute_import, division, print_function, unicode_literals

import argparse
import numpy as np
import pkg_resources

from astropy.table import Table

from measure_extinction.merge_obsspec import merge_stis_obsspec

def read_stis_archive_format(filename):
    """
    Read the STIS archive format and make it a *normal* table
    """
    t1 = Table.read(filename)
    ot = Table()
    for ckey in t1.colnames:
        if len(t1[ckey].shape) == 2:
            ot[ckey] = t1[ckey].data[0, :]

    return ot

if __name__ == "__main__":

    # commandline parser
    parser = argparse.ArgumentParser()
    parser.add_argument("starname", help="name of star (filebase)")
    parser.add_argument(
        "--waveregion",
        choices=["UV", "Opt"],
        default="Opt",
        help="wavelength region to merge",
    )
    parser.add_argument(
        "--inpath",
        help="path where original data files are stored",
        default=pkg_resources.resource_filename("measure_extinction", "data/Orig"),
    )
    parser.add_argument(
        "--outpath",
        help="path where merged spectra will be stored",
        default=pkg_resources.resource_filename("measure_extinction", "data/Out"),
    )
    parser.add_argument(
        "--ralph", action="store_true", help="Ralph Bohlin reduced data"
    )
    parser.add_argument("--outname", help="Output filebase")
    parser.add_argument("--png", help="save figure as a png file", action="store_true")
    parser.add_argument("--eps", help="save figure as an eps file", action="store_true")
    parser.add_argument("--pdf", help="save figure as a pdf file", action="store_true")
    args = parser.parse_args()

    if args.ralph:
        sfilename = "%s/%s/%s.mrg" % (args.inpath, args.waveregion, args.starname)
        stable = Table.read(
            sfilename,
            format="ascii",
            data_start=23,
            names=[
                "WAVELENGTH",
                "COUNT-RATE",
                "FLUX",
                "STAT-ERROR",
                "SYS-ERROR",
                "NPTS",
                "TIME",
                "QUAL",
            ],
        )
        stable = [stable]
    else:
        sfilename = "%s/%s/M31Ext2/%s" % (
            args.inpath,
            args.waveregion,
            args.starname.lower(),
        )
        t1 = read_stis_archive_format(sfilename + "10_x1d.fits")
        t2 = read_stis_archive_format(sfilename + "20_x1d.fits")
        t1.rename_column("ERROR", "STAT-ERROR")
        t2.rename_column("ERROR", "STAT-ERROR")
        t1["NPTS"] = np.full((len(t1["FLUX"])), 1.0)
        t2["NPTS"] = np.full((len(t2["FLUX"])), 1.0)
        t1["NPTS"][t1["FLUX"] == 0.0] = 0.0
        t2["NPTS"][t2["FLUX"] == 0.0] = 0.0
        stable = [t1, t2]

    rb_stis_opt = merge_stis_obsspec(stable, waveregion=args.waveregion)
    if args.outname:
        outname = args.outname
    else:
        outname = args.starname.lower()
    stis_opt_file = "%s_stis_%s.fits" % (outname, args.waveregion)
    rb_stis_opt.write("%s/%s" % (args.outpath, stis_opt_file), overwrite=True)

reason: import numpy,from astropy


text:
from PIL import Image
import sys
import os
from os import listdir
import numpy
import util
import json
import io

black = 0
white = 255
metrics_array_size = 20
matrix_size = 9

def output_json(matrix, output_file_name):
    output = {}
    numbers, count = matrix.shape
    for number in range(numbers):
        if number > 0:
            output[number] = matrix[number].tolist()
    print(json.dumps(output, indent=4))
    output_file = open(output_file_name, 'w')
    output_file.write(json.dumps(output, indent=4))
    output_file.close()

def main():
    if len(sys.argv) < 3:
        raise Exception("Input parameter 1: folder of folders of training data images. Input parameter 2: folder for JSON output representing image metrics.")
    # input fold of images organized into folders named by the number in in the image
    input_folder = sys.argv[1]
    # output folder for json metrics
    output_folder = sys.argv[2]
    # matrix to keep totals for where bits are found in the example images on both x and y axes
    x_metrics = numpy.zeros((matrix_size+1, metrics_array_size), dtype=float)
    y_metrics = numpy.zeros((matrix_size+1, metrics_array_size), dtype=float)

    for image_folder_name in listdir(input_folder):
        image_folder = input_folder + image_folder_name
        if os.path.isdir(image_folder):
            print("Processing: %s..." % image_folder_name)
            # current_number is the number in the images in this folder
            current_number = int(image_folder_name)
            for image_file in listdir(image_folder):
                image_file_name, image_file_extension = os.path.splitext(image_file)
                if image_file_extension == '' or image_file_name[0] == '.':
                    print("Skipping, not an image file.")
                else:
                    image = Image.open("%s/%s" % (image_folder, image_file))
                    # arrays to hold the x and y axis totals for this image
                    x_histogram, y_histogram = util.count_pixels(image)
                    # because images come in different sizes we need to normalize the length of the array
                    # of counts to be a constant
                    normalized_x = util.morph_array_to_size(x_histogram, metrics_array_size)
                    normalized_y = util.morph_array_to_size(y_histogram, metrics_array_size)
                    print("%s:\tx: %s, %s\ty: %s, %s" % (image_file, sum(x_histogram), sum(normalized_x),
                                                         sum(y_histogram), sum(normalized_y)))
                    print("x:\t%s" % normalized_x)
                    print("y:\t%s" % normalized_y)

                    # now add totals from this image to totals
                    for n in range(metrics_array_size):
                        x_metrics[current_number][n] += normalized_x[n]
                        y_metrics[current_number][n] += normalized_y[n]
                    print("x %s total:\t%s" % (current_number, x_metrics[current_number]))
                    print("y %s total:\t%s" % (current_number, y_metrics[current_number]))
                    print("Finished image %s" % image_file)

            print("Finished: %s" % image_folder)
            # Now that we have counts for all images of this number, we need to compute the % distribution
            print("x:")
            print(x_metrics[current_number])
            print("y:")
            print(y_metrics[current_number])
            total = sum(x_metrics[current_number])
            for n in range(x_metrics[current_number].size):
                x_metrics[current_number][n] = x_metrics[current_number][n] / total
            total = sum(y_metrics[current_number])
            for n in range(y_metrics[current_number].size):
                y_metrics[current_number][n] = y_metrics[current_number][n] / total
            print("x:")
            print(x_metrics[current_number])
            print("y:")
            print(y_metrics[current_number])

    output_json(x_metrics, output_folder + "x.json")
    output_json(y_metrics, output_folder + "y.json")

if __name__ == '__main__':
    main()


text:
import numpy as np
from vmf_utils import aic as vmf_aic, tic as vmf_tic, fit_mean_direction, fit_concentration, log_vMF_gradient
from gaussian_utils import aic, aic_spherical, tic, tic_spherical

def von_mises_correction_aic(Dnew, Dc):
    """
    :param Dnew[nxd matrix]: set 1
    :param Dc[mxd matrix]: set 2

    :return [float]: semantic similarity measure (approximation of the bayes factors)
    """

    # If both sentences are empty, the similarity is undefined
    if (Dnew==0).all() or (Dc==0).all():
        return np.nan

    D = np.concatenate((Dnew, Dc), axis=0)

    aic_x = -vmf_aic(Dnew)
    aic_y = -vmf_aic(Dc)
    aic_xy = -vmf_aic(D)
    similarity = aic_xy - (aic_x + aic_y)

    return similarity

def von_mises_correction_tic(Dnew, Dc):
    """
    :param Dnew[nxd matrix]: set 1
    :param Dc[mxd matrix]: set 2

    :return [float]: semantic similarity measure (approximation of the bayes factors)
    """

    # If both sentences are empty, the similarity is undefined
    if (Dnew==0).all() or (Dc==0).all():
        return np.nan

    D = np.concatenate((Dnew, Dc), axis=0)

    tic_x = -vmf_tic(Dnew)
    tic_y = -vmf_tic(Dc)
    tic_xy = -vmf_tic(D)
    similarity = tic_xy - (tic_x + tic_y)

    return similarity

def gaussian_correction_aic(Dnew, Dc):
    """
    :param Dnew[nxd matrix]: set 1
    :param Dc[mxd matrix]: set 2

    :return [float]: semantic similarity measure (approximation of the bayes factors)
    """

    # If both sentences are empty, the similarity is undefined
    if Dnew.shape[0] < 2 or Dc.shape[0] < 2:
        return np.nan

    D = np.concatenate((Dnew, Dc), axis=0)

    aic_x = -aic(Dnew)
    aic_y = -aic(Dc)
    aic_xy = -aic(D)
    similarity = aic_xy - (aic_x + aic_y)

    return similarity

def gaussian_correction_aic_fast(Dnew, Dc):
    """
    :param Dnew[nxd matrix]: set 1
    :param Dc[mxd matrix]: set 2

    :return [float]: semantic similarity measure (approximation of the bayes factors)
    """

    # If both sentences are empty, the similarity is undefined
    K, D = Dnew.shape
    L, D = Dc.shape
    if Dnew.shape[0] < 2 or Dc.shape[0] < 2:
        return np.nan

    mu_1 = np.mean(Dnew, axis=0)
    mu_2 = np.mean(Dc, axis=0)
    mu_1_sq = np.mean(Dnew ** 2, axis=0)
    mu_2_sq = np.mean(Dc ** 2, axis=0)
    p1 = K * 1.0 / (K + L)
    mu_3 = p1 * mu_1 + (1 - p1) * mu_2
    mu_3_sq = p1 * mu_1_sq + (1 - p1) * mu_2_sq

    reg = 1e-5
    v_1 = mu_1_sq - mu_1 ** 2 + reg
    v_2 = mu_2_sq - mu_2 ** 2 + reg
    v_3 = mu_3_sq - mu_3 ** 2 + reg

    ll_fast_x = K * np.sum(np.log(v_1))
    ll_fast_y = L * np.sum(np.log(v_2))
    ll_fast_xy = (K + L) * np.sum(np.log(v_3))
    similarity_fast = - ll_fast_xy + ll_fast_x + ll_fast_y

    return similarity_fast + 4 * D

def spherical_gaussian_correction_aic(Dnew, Dc):
    """
    :param Dnew[nxd matrix]: set 1
    :param Dc[mxd matrix]: set 2

    :return [float]: semantic similarity measure (approximation of the bayes factors)
    """

    # If both sentences are empty, the similarity is undefined
    if Dnew.shape[0] < 2 or Dc.shape[0] < 2:
        return np.nan

    D = np.concatenate((Dnew, Dc), axis=0)

    aic_x = -aic_spherical(Dnew)
    aic_y = -aic_spherical(Dc)
    aic_xy = -aic_spherical(D)

    return aic_xy - (aic_x + aic_y)

def gaussian_correction_tic(Dnew, Dc):
    """
    :param Dnew[nxd matrix]: set 1
    :param Dc[mxd matrix]: set 2

    :return [float]: semantic similarity measure (approximation of the bayes factors)
    """

    # If both sentences are empty, the similarity is undefined
    if Dnew.shape[0] < 2 or Dc.shape[0] < 2:
        return np.nan

    D = np.concatenate((Dnew, Dc), axis=0)

    tic_x = -tic(Dnew)
    tic_y = -tic(Dc)
    tic_xy = -tic(D)

    similarity = tic_xy - (tic_x + tic_y)

    return similarity

def spherical_gaussian_correction_tic(Dnew, Dc):
    """
    :param Dnew[nxd matrix]: set 1
    :param Dc[mxd matrix]: set 2

    :return [float]: semantic similarity measure (approximation of the bayes factors)
    """

    # If both sentences are empty, the similarity is undefined
    if Dnew.shape[0] < 2 or Dc.shape[0] < 2:
        return np.nan

    D = np.concatenate((Dnew, Dc), axis=0)

    tic_x = -tic_spherical(Dnew)
    tic_y = -tic_spherical(Dc)
    tic_xy = -tic_spherical(D)

    similarity = tic_xy - (tic_x + tic_y)

    return similarity

aic_total = 0
aic_spherical_total = 0
count = 0

def comparison(Dnew, Dc):
    global aic_total, aic_spherical_total, count
    aic_x = -aic(Dnew)
    aic_y = -aic(Dc)
    aic_sum = aic_x + aic_y

    aic_spherical_x = -aic_spherical(Dnew)
    aic_spherical_y = -aic_spherical(Dc)
    aic_spherical_sum = aic_spherical_x + aic_spherical_y

    aic_total += aic_sum
    aic_spherical_total += aic_spherical_sum
    count += 1

    print(aic_total * 1.0 / count, aic_spherical_total * 1.0 / count, aic_total * 1.0 / aic_spherical_total)
    return np.random.randn()

NAME_TO_SIM = {
    'von_mises_correction_tic': von_mises_correction_tic,
    'von_mises_correction_aic': von_mises_correction_aic,
    'gaussian_correction_aic': gaussian_correction_aic,
    'gaussian_correction_aic_fast': gaussian_correction_aic_fast,
    'gaussian_correction_tic': gaussian_correction_tic,
    'spherical_gaussian_correction_tic': spherical_gaussian_correction_tic,
    'spherical_gaussian_correction_aic': spherical_gaussian_correction_aic,
    'comparison': comparison
}

def get_similarity_by_name(sim_name):
    return NAME_TO_SIM[sim_name]

max_stars_repo_name: KeksimusPrime/SentEval

max_issues_repo_name: KeksimusPrime/SentEval

max_forks_repo_name: KeksimusPrime/SentEval


text:
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright 2015-2018 by ExopyPulses Authors, see AUTHORS for more details.
#
# Distributed under the terms of the BSD license.
#
# The full license is in the file LICENCE, distributed with this software.
# -----------------------------------------------------------------------------
"""Definition of a pulse, the elementary building block of sequences.

"""
import numpy as np
from atom.api import (Unicode, Enum, Typed, Property, set_default)
from exopy.utils.atom_util import (update_members_from_preferences)

from .shapes.base_shape import AbstractShape
from .shapes.modulation import Modulation
from .item import Item

class Pulse(Item):
    """ Represent a pulse to perfom during a sequence.

    """
    #: The kind of pulse can be either logical or ananlogical.
    kind = Enum('Logical', 'Analogical').tag(pref=True)

    #: Channel of the executioner which should perfom the pulse.
    channel = Unicode().tag(pref=True)

    #: Waveform
    waveform = Property()

    #: Modulation to apply to the pulse. Only enabled in analogical mode.
    modulation = Typed(Modulation, ()).tag(pref=True)

    #: Shape of the pulse. Only enabled in analogical mode.
    shape = Typed(AbstractShape).tag(pref=True)

    linkable_vars = set_default(['start', 'stop', 'duration'])

    def eval_entries(self, root_vars, sequence_locals, missings, errors):
        """ Attempt to eval the string parameters of the pulse.

        Parameters
        ----------
        root_vars : dict
            Dictionary of global variables for the all items. This will
            tipically contains the i_start/stop/duration and the root vars.

        sequence_locals : dict
            Dictionary of variables whose scope is limited to this item
            parent.

        missings : set
            Set of unfound local variables.

        errors : dict
            Dict of the errors which happened when performing the evaluation.

        Returns
        -------
        flag : bool
            Boolean indicating whether or not the evaluation succeeded.

        """
        success = super(Pulse, self).eval_entries(root_vars, sequence_locals,
                                                  missings, errors)

        if success and self.kind == 'Analogical':
            # Shapes are not allowed to modify global vars hence the empty
            # dict
            success &= self.modulation.eval_entries({}, sequence_locals,
                                                    missings, errors)
            self.shape.index = self.index
            success &= self.shape.eval_entries({}, sequence_locals,
                                               missings, errors)

        return success

    @classmethod
    def build_from_config(cls, config, dependencies):
        """ Create a new instance using the provided infos for initialisation.

        Parameters
        ----------
        config : dict(str)
            Dictionary holding the new values to give to the members in string
            format, or dictionnary like for instance with prefs.

        dependencies : dict
            Dictionary holding the necessary classes needed when rebuilding.

        Returns
        -------
        pulse : pulse
            Newly created and initiliazed sequence.

        Notes
        -----
        This method is fairly powerful and can handle a lot of cases so
        don't override it without checking that it works.

        """
        pulse = cls()

        #: Initialize the shape object with the right class, so that after
        #: update_members_from_preferences can do all the rest (initialize
        #: the shape's members)
        if 'shape' in config:
            shape_config = config['shape']
            if not shape_config == 'None':
                s_id = shape_config.pop('shape_id')
                s_cls = dependencies['exopy.pulses.shape'][s_id]
                shape = s_cls()
                pulse.shape = shape

        update_members_from_preferences(pulse, config)

        return pulse

    def traverse(self, depth=-1):
        """Yield a task and all of its components.

        The base implementation simply yields the task itself.

        Parameters
        ----------
        depth : int
            How deep should we explore the tree of tasks. When this number
            reaches zero deeper children should not be explored but simply
            yielded.

        """
        for i in super(Pulse, self).traverse(depth=depth):
            yield i

        if self.kind == 'Analogical':

            yield self.modulation

            if self.shape:
                yield self.shape

    def clean_cached_values(self):
        """Also clean modualtion and shape if necessary.

        """
        super(Pulse, self).clean_cached_values()
        if self.kind == 'Analogical':
            self.modulation.clean_cached_values()
            self.shape.clean_cached_values()

    # =========================================================================
    # --- Private API ---------------------------------------------------------
    # =========================================================================

    def _get_waveform(self):
        """ Getter for the waveform property.

        """
        context = self.root.context
        n_points = context.len_sample(self.duration)
        if self.kind == 'Analogical':
            time = np.linspace(self.start, self.stop, n_points, False)
            mod = self.modulation.compute(time, context.time_unit)
            shape = self.shape.compute(time, context.time_unit)
            return mod * shape
        else:
            return np.ones(n_points, dtype=np.int8)


text:
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc
from sklearn.metrics import roc_auc_score
import csv
import sys
import pandas as pd
import numpy as np
import seaborn as sns

def ro_curve(y_pred, y_label, method_name):
    '''
        y_pred is a list of length n.  (0,1)
        y_label is a list of same length. 0/1
        https://scikit-learn.org/stable/auto_examples/model_selection/plot_roc.html#sphx-glr-auto-examples-model-selection-plot-roc-py
    '''
    y_label = np.array(y_label)
    y_pred = np.array(y_pred)
    fpr = dict()
    tpr = dict()
    roc_auc = dict()
    fpr[0], tpr[0], _ = roc_curve(y_label, y_pred)
    roc_auc[0] = auc(fpr[0], tpr[0])
    lw = 3
    sns.lineplot(fpr[0],tpr[0],label= method_name + ' (area = %0.2f)' % roc_auc[0],lw=lw)
    # plt.plot(fpr[0], tpr[0],
        #  lw=lw, label= method_name + ' (area = %0.2f)' % roc_auc[0])
    # plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
    return

def col_pic():
    y_label = []
    y_pred = []
    plt.figure()

    # folder=[
    #     "fold1_0.775.csv",
    #     "fold2_0.675.csv",
    #     "fold3_0.575.csv",
    #     "fold4_0.675.csv",
    #     "fold5_0.675.csv"
    # ]
    folder=[
        "eval_All_0.73.csv",
    ]
    for i in range(1):
        f1 = pd.read_csv("result/"+folder[i])
        y_label=f1.loc[:,"label"].to_numpy()
        y_pred=f1.loc[:,"prob"].to_numpy()
        ro_curve(y_pred,y_label,"Fold" + str(i+1))
    sns.lineplot([0, 1], [0, 1], color='navy', linestyle='--')  # 对角

    fontsize = 14
    plt.title("AUC",fontsize=fontsize)
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.01])
    # plt.xticks(font="Times New Roman",size=18,wei ght="bold")
    # plt.yticks(font="Times New Roman",size=18,weight="bold")
    plt.xlabel('False Positive Rate', fontsize = fontsize)
    plt.ylabel('True Positive Rate', fontsize = fontsize)
    #plt.title('Receiver Operating Characteristic Curve', fontsize = fontsize)
    plt.legend(loc="lower right")
    # plt.savefig("figure/ROC_FOLD" + ".png",dpi=700)
    plt.savefig("figure/ROC_All" + ".png",dpi=700)

def main():
    col_pic()

if __name__=="__main__":
    main()


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Feb  8 09:33:41 2018

@author: sreimond
"""

from ..statistics import least_squares
from ..algebra import matrices
import numpy as np
import os, sys
import inspect
import warnings

def read_groops_normals( file_path ):
    """
    Imports normal equation and right hand side from GROOPS output path and
    returns LeastSquares class instance.
    """
    # LS class
    LS = least_squares.RegularizedOrdinaryLeastSquares()
    # full file names
    f_norm = file_path + '-normals.txt'
    f_rghs = file_path + '-normals.rightHandSide.txt'
    f_info = file_path + '-normals.info.xml'
    f_x = file_path + '-x.txt'
    f_sigmax = file_path + '-sigmax.txt'
    # check for existence
    c1 = not os.path.exists( f_norm )
    c2 = not os.path.exists( f_rghs )
    c3 = not os.path.exists( f_info )
    if c1 or c2 or c3:
        warnings.warn('Groops files not found!')
        return LS
    # read info file
    f = open( f_info, 'r' )
    for line in f:
        if '<lPl>' in line:
            lPl = float(line[line.index("<lPl>") + 5:line.rindex("</lPl>")])
        if '<parameterCount>' in line:
            nx = int(line[line.index("<parameterCount>") +
                     16:line.rindex("</parameterCount>")])
        if '<observationCount>' in line:
            nl = int(line[line.index("<observationCount>") +
                     18:line.rindex("</observationCount>")])
    f.close()
    LS.bb = lPl
    LS.row_count = nl
    LS.col_count = nx
    LS.redundancy = nl-nx
    # read normals file
    N = np.zeros( (nx,nx), dtype=np.float_ )
    f = open( f_norm, 'r' )
    next(f), next(f)
    ii, ij = -1, -1
    for line in f:
        ii += 1
        ij += 1
        N[ii,ij:nx] = [float(ni) for ni in line.split()]
    f.close()
    N += np.triu( N, 1).T
    LS.normal_equation = matrices.MatrixEquation()
    LS.normal_equation.define_coefficient_matrix(matrices.return_matrix_from_numpy_array(N))
#    # read normals file (slow)
#    N_triu = matrices.UpperTriangularMatrix(LS.col_count)
#    f = open( f_norm, 'r' )
#    next(f), next(f)
#    ix = -1
#    for line in f:
#        values = [float(ni) for ni in line.split()]
#        for value in values:
#            ix += 1
#            N_triu.set_element(ix,value)
#    f.close()
#    LS.normal_equation = matrices.MatrixEquation()
#    LS.normal_equation.define_coefficient_matrix(N_triu.return_full_matrix())
    # read right hand side file
    n = np.genfromtxt( f_rghs, unpack=True, skip_header=2 )
    LS.normal_equation.define_right_hand_side(matrices.return_matrix_from_numpy_array(n))
    # x and sigmax
    x, sigmax = np.ones(LS.col_count)*np.nan, np.ones(LS.col_count)*np.nan
    if os.path.exists( f_x ):
        x = np.genfromtxt( f_x, unpack=True, skip_header=2 )
    if os.path.exists( f_sigmax ):
        sigmax = np.genfromtxt( f_sigmax, unpack=True, skip_header=2 )
    LS.normal_equation.x = matrices.return_matrix_from_numpy_array(x)
    LS.x = matrices.return_matrix_from_numpy_array(x)
    LS.sigma_x = matrices.return_matrix_from_numpy_array(sigmax)
    xNx = LS.x.return_transpose().multiply(LS.normal_equation.A).multiply(LS.x).elements
    nx2 = 2.0 * LS.normal_equation.b.return_transpose().multiply(LS.x).elements
    xRx = 0
    LS.rss = xNx - nx2 + LS.bb
    LS.xss = xRx
    LS.variance_unit_weight = LS.rss/LS.redundancy
    LS.parameter_covariance_matrix = LS.normal_equation.A.return_inverse()
    LS.parameter_covariance_matrix.elements *= LS.variance_unit_weight
    LS.sigma_x = LS.parameter_covariance_matrix.return_diagonal_vector()
    LS.sigma_x.elements **= 0.5
    LS.bias = None
    return LS

max_stars_repo_name: sreimond/picasso

max_issues_repo_name: sreimond/picasso

max_forks_repo_name: sreimond/picasso


text:
from typing import List

from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import RepeatedStratifiedKFold

import pandas as pd
import numpy as np

from .machine_learning_utils import get_default_classifier

from tqdm.notebook import tqdm

from ..mylogger import get_handler
import logging

handler = get_handler()

log = logging.getLogger(__name__)
log.handlers[:] = []
log.addHandler(handler)
log.setLevel(logging.DEBUG)

# PARAM_GRID_SIMPLE = {
#     'bootstrap': [True, False],
#     'max_depth': [2, 5, 10],
#     'max_features': ['auto', 'sqrt'],
#     'n_estimators': [50, 100, 200, 400],
#     'min_samples_split': [2, 5, 10]
# }

class FineTuner:

    PARAM_GRID_LEVEL_0 = {
        'max_depth': [2, 5, 10],
        'n_estimators': [10, 25, 50, 75, 100, 200, 400],
        'min_samples_split': [2, 5],
        'max_features': ['auto', 'sqrt', None],
        'class_weight': ['balanced', None]
    }

    PARAM_GRID_LEVEL_1 = {
        'bootstrap': [True, False],
        'max_depth': list(np.arange(2, 15)) + [None],
        'min_samples_leaf': [1, 2, 4],
        'n_estimators': [5] + list(np.arange(10, 110, 10)) + list(np.arange(120, 620, 20)),
        'min_samples_split': [2, 5],
        'max_features': ['auto', 'sqrt', None],
        'class_weight': ['balanced', None]
    }

    def __init__(
            self,
            n_iter: int,
            n_repeats_cv: int,
            n_jobs: int,
            verbose: int,
            Xs_determined,
            n_experiment: int,
            random_seeds: List[int],
            data_materials,
            param_grid_level=0
    ):
        self.n_iter = n_iter
        self.n_repeats_cv = n_repeats_cv
        self.n_jobs = n_jobs
        self.verbose = verbose
        self.n_experiment = n_experiment
        self.random_seeds = random_seeds
        self.data_materials = data_materials
        self.Xs_determined = Xs_determined

        self.randomized_search_objects = None
        self.classifiers_attributes_data = None
        self.best_estimators = None

        self.param_grid = self.get_param_grid(param_grid_level)

    def get_param_grid(self, level):
        if level == 0:
            return self.PARAM_GRID_LEVEL_0

        elif level == 1:
            return self.PARAM_GRID_LEVEL_1

        else:
            raise ValueError(f"Level {level} is invalid argument.")

    # def run_randomized_search(self) -> None:
    #     log.debug("Running randomized search for each experiment ..")
    #     randomized_search_objects = []
    #     for exp in tqdm(range(self.n_experiment)):
    #         random_seed = self.random_seeds[exp]
    #         X_fine_tuning = self.data_materials[self.Xs_determined][exp]
    #         y_fine_tuning = self.data_materials["ys_train"][exp]
    #         randomized_search = self.get_randomized_search(random_seed)
    #         randomized_search.fit(X_fine_tuning, y_fine_tuning)
    #         randomized_search_objects.append(randomized_search)
    #     self.randomized_search_objects = randomized_search_objects
    #     self.save_randomized_search_info()
    #     self.save_best_estimators()

    def run_search(self, search_type="randomized") -> None:
        log.debug(f"Running {search_type} search for each experiment ..")
        log.debug(f"PARAM_GRID: {self.param_grid}")
        search_objects = []
        # searcher = self.get_randomized_search if search_type == "randomized" else self.get_grid_search
        for exp in tqdm(range(self.n_experiment)):
            random_seed = self.random_seeds[exp]
            X_fine_tuning = self.data_materials[self.Xs_determined][exp]
            y_fine_tuning = self.data_materials["ys_train"][exp]
            search_obj = self.get_searcher(search_type, random_seed)
            search_obj.fit(X_fine_tuning, y_fine_tuning)
            search_objects.append(search_obj)
        self.randomized_search_objects = search_objects
        self.save_randomized_search_info()
        self.save_best_estimators()

    def get_searcher(self, search_type, random_seed):
        if search_type == "randomized":
            return self.get_randomized_search(random_seed)
        elif search_type == "grid":
            return self.get_grid_search(random_seed)
        else:
            raise ValueError(f"{search_type} is invalid argument.")

    def get_randomized_search(self, random_seed) -> RandomizedSearchCV:

        if self.n_iter is None:
            raise ValueError("Using RANDOMIZED SEARCH. Parameter `n_iter` cannot be None.")

        clf = get_default_classifier(random_state=random_seed)

        randomized_search = RandomizedSearchCV(
            clf, self.param_grid,
            n_iter=self.n_iter,
            random_state=random_seed,
            cv=RepeatedStratifiedKFold(
                n_splits=10, n_repeats=self.n_repeats_cv, random_state=random_seed
            ),
            scoring='balanced_accuracy',
            return_train_score=True,
            n_jobs=self.n_jobs,
            verbose=self.verbose
        )

        return randomized_search

    def get_grid_search(self, random_seed) -> GridSearchCV:

        if self.n_iter is not None:
            log.warning("Using GRID SEARCH. Parameter `n_iter` will be ignored.")

        clf = get_default_classifier(random_state=random_seed)

        grid_search = GridSearchCV(
            clf,
            self.param_grid,
            cv=RepeatedStratifiedKFold(
                n_splits=10, n_repeats=self.n_repeats_cv, random_state=random_seed
            ),
            scoring='balanced_accuracy',
            return_train_score=True,
            n_jobs=self.n_jobs,
            verbose=self.verbose
        )

        return grid_search

    def save_randomized_search_info(self):
        experiment_repeat_to_randomized_search_info = {}

        for exp in range(self.n_experiment):
            experiment_repeat_to_randomized_search_info[F'EXP_{exp + 1}'] = [
                self.randomized_search_objects[exp].best_params_,
                self.randomized_search_objects[exp].best_estimator_,
                self.randomized_search_objects[exp].best_score_]

        classifiers_attributes_data = pd.DataFrame(
            experiment_repeat_to_randomized_search_info,
            index=['best_params_', 'best_estimator_', 'best_score_']
        ).T

        self.classifiers_attributes_data = classifiers_attributes_data

    def save_best_estimators(self):
        best_estimators = []
        for search_obj in self.randomized_search_objects:
            best_estimators.append(search_obj.best_estimator_)
        self.best_estimators = best_estimators

max_stars_repo_name: ibrahimberb/Predicting-Mutation-Effects

max_issues_repo_name: ibrahimberb/Predicting-Mutation-Effects

max_forks_repo_name: ibrahimberb/Predicting-Mutation-Effects


text:
# import os
#
# import lydorn_utils.geo_utils as utils
# import numpy as np
# import rasterio
# from PIL import Image, ImageDraw
# from rasterio.windows import Window
#
# # from mmdet.apis import inference_detector, init_detector, show_result_pyplot
#
# PATH = "../../data/toulouse/unlabled"
# DIRECTORY_CROPPED_IMAGE = PATH + "/train/"
# DIRECTORY_MASK_IMAGE = PATH + "/mask/"
#
#
# #
# # def get_prediction(file):
# #     # Choose to use a config and initialize the detector
# #     config = PATH + '/checkpoints/reppoints_moment_r101_fpn_gn-neck+head_2x_toulouse.py'
# #     # Setup a checkpoint file to load
# #     checkpoint = 'checkpoints/epoch_24.pth'
# #     # initialize the detector
# #     model = init_detector(config, checkpoint, device='cuda:0')
# #     result = inference_detector(model, file)
# #     # Let's plot the result
# #     show_result_pyplot(model, file, result, score_thr=0.3)
#
#
# def get_segmentation(image_name):
#     with rasterio.open(DIRECTORY_CROPPED_IMAGE + image_name) as src:
#         mask = Image.new('1', (src.width, src.height), "#000000")
#         mask_draw = ImageDraw.Draw(mask)
#         poligons = utils.get_polygons_from_osm(DIRECTORY_CROPPED_IMAGE + image_name, tag="building", ij_coords=False)
#         for p in poligons:
#             result = list(map(tuple, np.array(p).astype(int)))
#             mask_draw.polygon(result, fill='#ee7621', outline='#ee7621')
#         mask.save(DIRECTORY_MASK_IMAGE + image_name)
#
#
# if __name__ == '__main__':
#     for subdir, dirs, files in os.walk(DIRECTORY_CROPPED_IMAGE):
#         for file in files:
#             if file.lower().endswith('.tif'):
#                 if not os.path.exists(DIRECTORY_MASK_IMAGE + file):
#                     print("get mask for " + file)
#                     get_segmentation(file)
#                     # get_prediction(file)
#                 # else:
#                 # print("mask " + file + " already exists")

max_stars_repo_name: galyalina/mmdetection

max_issues_repo_name: galyalina/mmdetection

max_forks_repo_name: galyalina/mmdetection


text:
#from __future__ import absolute_import
#from __future__ import division
#from __future__ import print_function

#from kaggle_environments.envs.football.helpers import *

import tensorflow as tf
import numpy as np
from tf_agents.trajectories import time_step as ts

import abc
import tensorflow as tf
import numpy as np
from tf_agents.environments import py_environment
from tf_agents.environments import tf_environment
from tf_agents.environments import tf_py_environment
from tf_agents.environments import utils
from tf_agents.specs import array_spec
from tf_agents.environments import wrappers
from tf_agents.environments import suite_gym
from tf_agents.trajectories import time_step as ts
from tf_agents.utils import common

import gfootball
import gfootball.env as football_env

data_dic = {
    'ball_owned_team': array_spec.BoundedArraySpec(shape=(1,), dtype=np.int32, minimum=0, maximum=2,
                                                   name='ball_owned_team'),
    'steps_left': array_spec.BoundedArraySpec(shape=(1,), dtype=np.float32, minimum=0.0, name='steps_left'),
    'ball_owned_player': array_spec.BoundedArraySpec(shape=(1,), dtype=np.int32, minimum=0, maximum=11,
                                                     name='ball_owned_player'),
    'game_mode': array_spec.BoundedArraySpec(shape=(1,), dtype=np.int32, minimum=0, maximum=6, name='game_mode'),
    'designated': array_spec.BoundedArraySpec(shape=(1,), dtype=np.int32, minimum=-1, maximum=11, name='designated'),
    'active': array_spec.BoundedArraySpec(shape=(1,), dtype=np.int32, minimum=-1, maximum=11, name='active'),
    'left_team_active': array_spec.BoundedArraySpec(shape=(11,), dtype=np.int32, minimum=0, maximum=1,
                                                    name='left_team_active'),
    'left_team': array_spec.BoundedArraySpec(shape=(11, 2), dtype=np.float32, minimum=-1.5, maximum=1.5,
                                             name='left_team'),
    'right_team_active': array_spec.BoundedArraySpec(shape=(11,), dtype=np.int32, minimum=0, maximum=1,
                                                     name='right_team_active'),
    'ball_direction': array_spec.BoundedArraySpec(shape=(3,), dtype=np.float32, name='ball_direction'),
    'ball': array_spec.BoundedArraySpec(shape=(3,), dtype=np.float32, name='ball'),
    'left_team_tired_factor': array_spec.BoundedArraySpec(shape=(11,), dtype=np.float32, minimum=0.0,
                                                          name='left_team_tired_factor'),
    'left_team_direction': array_spec.BoundedArraySpec(shape=(11, 2), dtype=np.float32, minimum=-1.5, maximum=1.5,
                                                       name='left_team_direction'),
    'score': array_spec.BoundedArraySpec(shape=(2,), dtype=np.float32, minimum=0.0, name='score'),
    'left_team_roles': array_spec.BoundedArraySpec(shape=(1,), dtype=np.int32, minimum=0, maximum=10,
                                                   name='left_team_roles'),
    'right_team_tired_factor': array_spec.BoundedArraySpec(shape=(11,), dtype=np.float32, minimum=0.0,
                                                           name='right_team_tired_factor'),
    'right_team': array_spec.BoundedArraySpec(shape=(11, 2), dtype=np.float32, minimum=-1.5, maximum=1.5,
                                              name='right_team'),
    'right_team_direction': array_spec.BoundedArraySpec(shape=(11, 2), dtype=np.float32, minimum=-1.5, maximum=1.5,
                                                        name='right_team_direction'),
    'right_team_yellow_card': array_spec.BoundedArraySpec(shape=(11,), dtype=np.int32, minimum=0, maximum=1,
                                                          name='right_team_yellow_card'),
    'right_team_roles': array_spec.BoundedArraySpec(shape=(1,), dtype=np.int32, minimum=0, maximum=10,
                                                    name='right_team_roles'),
    'ball_rotation': array_spec.BoundedArraySpec(shape=(3,), dtype=np.float32, name='ball_rotation'),
    'left_team_yellow_card': array_spec.BoundedArraySpec(shape=(11,), dtype=np.int32, minimum=0, maximum=1,
                                                         name='left_team_yellow_card'),
    'sticky_actions': array_spec.BoundedArraySpec(shape=(10,), dtype=np.int32, minimum=0, maximum=1,
                                                  name='sticky_actions'),
}

class MyFootballEnv(py_environment.PyEnvironment):

    @staticmethod
    def convert_observation_to_tf(obs, data_cfg, propertie_list=[]):

        if (len(propertie_list) == 0):
            propertie_list = ['right_team_direction', 'right_team_tired_factor',
                              'left_team_roles', 'left_team_direction', 'ball_direction',
                              'ball_owned_player', 'right_team_yellow_card', 'ball',
                              'right_team', 'steps_left', 'ball_rotation',
                              'ball_owned_team', 'game_mode', 'left_team_yellow_card',
                              'left_team', 'right_team_roles', 'right_team_active',
                              'left_team_active', 'left_team_tired_factor', 'score',
                              'designated', 'active', 'sticky_actions']
        rp = {}

        for name in propertie_list:

            if (('ball_owned_player' == name) and (obs[0][name] == -1)):
                obs[0][name] = 11
            if (('ball_owned_team' == name) and (obs[0][name] == -1)):
                obs[0][name] = 2

            if ('right_team_roles' == name):
                if ((obs[0]['ball_owned_team'] == 1) and (
                        (obs[0]['ball_owned_player'] != -1) or (obs[0]['ball_owned_player'] != 11))):
                    obs[0][name] = obs[0][name][obs[0]['ball_owned_player']]
                else:
                    obs[0][name] = 10

            if ('left_team_roles' == name):
                if ((obs[0]['ball_owned_team'] == 0) and (
                        (obs[0]['ball_owned_player'] != -1) or (obs[0]['ball_owned_player'] != 11))):
                    obs[0][name] = obs[0][name][obs[0]['ball_owned_player']]
                else:
                    obs[0][name] = 10

            if (data_cfg[name].shape == (1,)):
                rp[name] = np.array([np.squeeze(obs[0][name])]).astype(data_cfg[name].dtype)
            else:
                a = np.zeros(data_cfg[name].shape, dtype=data_cfg[name].dtype)
                a[:len(obs[0][name])] = obs[0][name]
                obs[0][name] = a.copy()
                rp[name] = np.array(obs[0][name]).astype(data_cfg[name].dtype)

        return rp

    @staticmethod
    def construct_obs_spec(data_cfg, cfg):
        rp = {}

        for name in cfg:
            rp[name] = data_cfg[name]

        return rp

    def __init__(self, propertie_list=[], scenario='11_vs_11_kaggle'):
        super().__init__()
        self._data_dic = data_dic

        if (len(propertie_list) == 0):
            propertie_list = ['right_team_direction', 'right_team_tired_factor',
                              'left_team_roles', 'left_team_direction', 'ball_direction',
                              'ball_owned_player', 'right_team_yellow_card', 'ball',
                              'right_team', 'steps_left', 'ball_rotation',
                              'ball_owned_team', 'game_mode', 'left_team_yellow_card',
                              'left_team', 'right_team_roles', 'right_team_active',
                              'left_team_active', 'left_team_tired_factor', 'score',
                              'designated', 'active', 'sticky_actions']

        self.propertie_list = propertie_list

        self.env = football_env.create_environment(
            env_name=scenario,
            stacked=False,
            representation='raw',
            rewards='scoring, checkpoints',
            write_goal_dumps=False,
            write_full_episode_dumps=False,
            render=False,
            write_video=False,
            dump_frequency=1,
            logdir='./',
            extra_players=None,
            number_of_left_players_agent_controls=1,
            number_of_right_players_agent_controls=0
        )

        self._state = self.convert_observation_to_tf(self.env.reset(), self._data_dic, self.propertie_list)

        self._action_spec = array_spec.BoundedArraySpec(
            shape=(), dtype=np.int32, minimum=0, maximum=18, name='action')

        # representation of the enviroment: price + open position state
        self._observation_spec = self.construct_obs_spec(self._data_dic, self.propertie_list)

        # used for idndication of the end of episode
        self._episode_ended = False

        pass

    def action_spec(self):
        return self._action_spec

    def observation_spec(self):
        return self._observation_spec

    def _reset(self):
        self._episode_ended = False
        self._state = self.convert_observation_to_tf(self.env.reset(), self._data_dic, self.propertie_list)
        return ts.restart(self._state)

    def _step(self, action):

        if self._episode_ended:
            # The last action ended the episode. Ignore the current action and start
            # a new episode.
            return self.reset()

        self._state, reward, self._episode_ended, info = self.env.step(action)
        self._state = self.convert_observation_to_tf(self._state, self._data_dic, self.propertie_list)

        if self._episode_ended:
            return ts.termination(self._state, reward)
        else:
            return ts.transition(self._state, reward=reward, discount=1.0)

def convert_obs_to_tf_env_obs(obs):
    cfg = ['ball_owned_team',
           'steps_left',
           'ball_owned_player',
           'game_mode',
           # 'designated',
           'active',
           # 'left_team_active',
           'left_team',
           # 'right_team_active',
           'ball_direction',
           'ball',
           # 'left_team_tired_factor',
           'left_team_direction',
           'score',
           # 'left_team_roles',
           # 'right_team_tired_factor',
           'right_team',
           'right_team_direction',
           # 'right_team_yellow_card',
           # 'right_team_roles',
           'ball_rotation',
           # 'left_team_yellow_card',
           'sticky_actions']

    o = MyFootballEnv.convert_observation_to_tf(obs['players_raw'], data_dic, cfg)
    for key, value in o.items():
        o[key] = tf.expand_dims(tf.convert_to_tensor(value), axis=0)

    return o

policy_dir = './policy/'
saved_policy = tf.compat.v2.saved_model.load(policy_dir)

def agent(obs):
    global saved_policy

    st = tf.constant(np.array(np.array([0], dtype=np.int32)))
    rw = tf.constant(np.array(np.array([0], dtype=np.float32)))
    ds = tf.constant(np.array(np.array([0], dtype=np.float32)))
    ts_obs = {}
    ts_obs = convert_obs_to_tf_env_obs(obs)
    t = ts.TimeStep(st, rw, ds, ts_obs)

    a = saved_policy.action(t).action.numpy()[0]

    return [a]


text:
import numpy as np
from utils import *

def avg_color(blocks): #it averages every color in each block structure the same
    # temp = upgrade(blocks[0][0])
    for row in range(len(blocks)):
        for column in range(len(blocks[row])):
            blocks[row][column] = avg_helper(blocks[row][column])

#helper function that averages a single block
def avg_helper(block):
    h, w, d = block.shape #sets a variable for the 3 dimensional array
    S = np.sum(block, axis=(0, 1), dtype = np.dtype(np.uint16)) #add all the elements in each row of the array or each column
    # Integer-divide, convert back to uint8, and tile to shape of original block
    return pixel_to_block((S // (h*w)).astype(np.dtype(np.uint8)), w, h)

max_stars_repo_name: aidan-fitz/pixelator

max_issues_repo_name: aidan-fitz/pixelator

max_forks_repo_name: aidan-fitz/pixelator


text:
#  Copyright (c) polakowo
#  Licensed under the MIT license.

import logging
import random
import numpy as np
import torch
import torch.nn.functional as F
from transformers import GPT2LMHeadModel, GPT2Tokenizer
import ipdb

# Enable logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
handler = logging.FileHandler(filename='discord.log', encoding='utf-8', mode='w')
handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
logger.addHandler(handler)

def set_seed(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)

def top_k_top_p_filtering(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')):
    """ Filter a distribution of logits using top-k and/or nucleus (top-p) filtering
        Args:
            logits: logits distribution shape (batch size x vocabulary size)
            top_k > 0: keep only top k tokens with highest probability (top-k filtering).
            top_p > 0.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering).
                Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751)
        From: https://gist.github.com/thomwolf/1a5a29f6962089e871b94cbd09daf317
    """
    # ipdb.set_trace()
    logger.debug("logits dimension: " + str(logits.dim())) # batch size 1 for now - could be updated for more but the code would be less clear
    logger.debug("logits: " + str(logits)) # batch size 1 for now - could be updated for more but the code would be less clear
    top_k = min(top_k, logits.size(-1))  # Safety check
    if top_k > 0:
        # Remove all tokens with a probability less than the last token of the top-k
        indices_to_remove = logits < torch.topk(logits, top_k, largest = False)[0][..., -1, None]
        logger.debug("indices_to_remove: " + str(indices_to_remove))
        logits[indices_to_remove] = filter_value
        logger.debug("logits after topk filter: " + str(logits))
    if top_p > 0.0:
        sorted_logits, sorted_indices = torch.sort(logits, descending=True)
        cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
        # Remove tokens with cumulative probability above the threshold
        sorted_indices_to_remove = cumulative_probs > top_p
        # Shift the indices to the right to keep also the first token above the threshold
        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
        sorted_indices_to_remove[..., 0] = 0
        # scatter sorted tensors to original indexing
        indices_to_remove = sorted_indices_to_remove.scatter(dim=1, index=sorted_indices, src=sorted_indices_to_remove)
        # indices_to_remove = sorted_indices[sorted_indices_to_remove]
        logits[indices_to_remove] = filter_value
    return logits

def sample_sequence(model, tokenizer, context_ids, config):
    # Parse parameters
    no_cuda = config.getboolean('model', 'no_cuda')
    num_samples = config.getint('decoder', 'num_samples')
    max_length = config.getint('decoder', 'max_length')
    temperature = config.getfloat('decoder', 'temperature')
    top_k = config.getint('decoder', 'top_k')
    top_p = config.getfloat('decoder', 'top_p')

    device = torch.device("cuda" if torch.cuda.is_available() and not no_cuda else "cpu")
    context_tensor = torch.tensor(context_ids, dtype=torch.long, device=device)
    logger.debug("tensor built: " + str(context_tensor))
    logger.debug("tensor shape (before): " + str(context_tensor.shape))
    context_tensor = context_tensor.unsqueeze(0).repeat(num_samples, 1)
    logger.debug("tensor unsqueezed: " + str(context_tensor))
    generated = context_tensor
    logger.debug("Generated shape (before): " + str(generated.shape))
    with torch.no_grad():
        while True:
            inputs = {'input_ids': generated}
            outputs = model(**inputs)  # Note: we could also use 'past' with GPT-2/Transfo-XL/XLNet/CTRL (cached hidden-states)
            logits = outputs[0][:, -1, :]
            next_token_logits = logits / (temperature if temperature > 0 else 1.)
            logger.debug("next_token_logits: " + str(next_token_logits))
            filtered_logits = top_k_top_p_filtering(next_token_logits, top_k=top_k, top_p=top_p)
            logger.debug("filtered logits: " + str(filtered_logits))
            if temperature == 0.0: # greedy sampling:
                next_token = torch.argmax(filtered_logits, dim=-1).unsqueeze(-1)
                logger.debug("next_token: " + str(next_token))
            else:
                next_token = torch.multinomial(F.softmax(filtered_logits, dim=-1), num_samples=1)
                logger.debug("next_token: " + str(next_token))
            generated = torch.cat((generated, next_token), dim=1)
            logger.debug("next_token added: " + str(next_token))
            logger.debug("Generated shape (after): " + str(generated.shape))
            if (generated[:, len(context_ids):] == tokenizer.eos_token_id).any(dim=1).all():
                # EOS token id found in each sample
                break
            if generated.shape[1] - len(context_ids) >= max_length:
                # Maximum length reached
                break
    return generated

def select_using_mmi(mmi_model, mmi_tokenizer, candidates, config):
    # Parse parameters
    no_cuda = config.getboolean('model', 'no_cuda')

    device = torch.device("cuda" if torch.cuda.is_available() and not no_cuda else "cpu")
    scores = []
    for i, candidate in enumerate(candidates):
        context = []
        for response in reversed(candidate):
            context.extend(response)
            context.append(mmi_tokenizer.eos_token_id)
        context_ids = mmi_tokenizer.encode(context)
        context_tensor = torch.tensor(context_ids, dtype=torch.long, device=device)
        loss, _, _ = mmi_model(input_ids=context_tensor, labels=context_tensor)
        scores.append(-loss.float())

    scores = torch.stack(scores, dim=0)
    # The smaller the loss, the higher must be the probability of selecting it
    winner = torch.multinomial(F.softmax(scores, dim=0), num_samples=1).item()
    return winner

def generate_response(model: GPT2LMHeadModel, tokenizer: GPT2Tokenizer, context: str, config, mmi_model=None, mmi_tokenizer=None):
    # Parse parameters
    use_mmi = config.getboolean('model', 'use_mmi')
    num_samples = config.getint('decoder', 'num_samples')
    max_length = config.getint('decoder', 'max_length')
    seed = config.get('decoder', 'seed')
    seed = int(seed) if seed is not None else None

    # Make answers reproducible only if wanted
    if seed is not None:
        set_seed(seed)

    # Generate response
    context_ids = tokenizer.encode(context, max_length=max_length, pad_to_max_length=False)
    logger.debug("found context_ids: " + str(context_ids))
    samples = sample_sequence(model, tokenizer, context_ids, config)
    samples = samples[:, len(context_ids):].tolist()
    logger.debug("found samples: " + str(samples))
    texts = []
    for sample in samples:
        text = tokenizer.decode(sample, clean_up_tokenization_spaces=True)
        text = text[: text.find(tokenizer.eos_token)]
        texts.append(text)

    if use_mmi:
        assert(num_samples > 1, "MMI requires num_samples > 1")
        candidates = [context + text for text in texts]
        best_i = select_using_mmi(mmi_model, mmi_tokenizer, candidates, config)
        return [texts[best_i]]
    return texts