text:
#!/usr/bin/env python
#-*- coding:utf-8 -*-

##############################################
# File Name: importance_selection.py
# Author: Xuhao Du
# Email: duxuhao88@gmail.com
##############################################

from scipy.stats import pearsonr
from collections import OrderedDict
import random
import time
import numpy as np
import sys

def _reachlimit(func):
    def wrapper(c):
        temp = func(c)
        if (len(c._TemplUsedFeatures) >= c._FeaturesQuanLimitation) | ((time.time() - c._StartTime) >= c._TimeLimitation):
            if (len(c._TemplUsedFeatures) >= c._FeaturesQuanLimitation):
                print("Maximum features limit reach!")
            if ((time.time() - c._StartTime) >= c._TimeLimitation):
                print("Time's up!")
            print('{0}\nbest score:{1}\nbest {2} features combination: {3}'.format('*-*' * 50,
                                                                                c._score,
                                                                                c._FeaturesQuanLimitation,
                                                                                c._TemplUsedFeatures))
            sys.exit()
        return temp
    return wrapper

class _importsance_selection(object):

    def __init__(self, clf, df, RecordFolder, start, label,
                 direction, LossFunction, FeaturesQuanLimitation, TimeLimitation,
                 fit_params=None, validatefunction=0,important_update=1,
                 selectkey='', selectbatch=1, selectfrac=1):
        self._clf = clf
        self._fit_params = fit_params
        self._LossFunction = LossFunction
        self._df = df
        self._RecordFolder = RecordFolder
        self._TemplUsedFeatures, self._Label = start, label
        self._Startcol = ['None']
        self._validatefunction = validatefunction
        self._TimeLimitation = TimeLimitation * 60
        self._FeaturesQuanLimitation = FeaturesQuanLimitation
        self._StartTime = time.time()
        self._fit_params = fit_params
        self._frac = selectfrac
        self._batch = selectbatch
        self._key = selectkey
        self._direction = direction
        self._update_imp = important_update

    def _evaluate(self, a, b):
        if self._direction == 'ascend':
            return a > b
        else:
            return a < b

    def select(self):
        self._StartTime = time.time()
        if self._direction == 'ascend':
            self._score, self._greedyscore = -np.inf, np.inf
        else:
            self._score, self._greedyscore = np.inf, -np.inf
        print('test performance of initial features combination')
        self.bestscore, self._bestfeature = self._score, self._TemplUsedFeatures[:]
        self._validation(self._TemplUsedFeatures[:], str(0), 'baseline')
        selectcol = self._TemplUsedFeatures[:]
        removelist = [i for i in selectcol if self._key in i]
        if self._frac != 1:
            n = int(len(removelist) * self._frac)
        else:
            n = int(self._batch)
        if n < 1:
            n = 1
        print('Remove Batch: {}'.format(n))
        iter_num = 0
        while len(selectcol) > 1:
            temp = selectcol[:]
            if self._update_imp | (iter_num == 0):
                importances = sorted([[i,j] for i,j in zip(self._clf.feature_importances_,list(OrderedDict.fromkeys(temp)))])
                index_step = 0
            deletenum = 0
            removed = []
            while (deletenum < n) & (index_step < len(temp)):
                if (importances[index_step][1] in removelist) & (importances[index_step][1] in temp):
                    temp.remove(importances[index_step][1])
                    removed.append(importances[index_step][1])
                    deletenum += 1
                index_step += 1
            self._validation(temp[:], str(iter_num), str(removed))
            iter_num += 1
            selectcol = temp[:]

        print('{0}\nbest score:{1}\nbest features combination: {2}'.format('*-*' * 50,
                                                                           self.bestscore,
                                                                           self._bestfeature))
        with open(self._RecordFolder, 'a') as f:
            f.write('{0}\nbest score:{1}\nbest features combination: {2}'.format('*-*' * 50,
                                                                           self.bestscore,
                                                                           self._bestfeature))
        return self._bestfeature

    def _validation(self,
                    selectcol,
                    num,
                    rmfeature): #get the score with the new features list and update the best features combination
        """ set up your cross validation here"""
        self.chekcLimit()
        selectcol = list(OrderedDict.fromkeys(selectcol))
        tempdf = self._df
        X, y = tempdf, tempdf[self._Label]
        totaltest, self._clf = self._validatefunction(X, y, selectcol, self._clf, self._LossFunction) #, self._fit_params)
        print('remove features: {}'.format(rmfeature))
        print('Mean loss: {}'.format(totaltest))
        if self._ScoreUpdate():
            with open(self._RecordFolder, 'a') as f: #record all the imporved combination
                f.write('{0}  {1}:\n{2}\t{3}\n'.format(num, rmfeature,
                                                            np.round(np.mean(totaltest),6),
                                                            selectcol[:]))
                f.write('*{}\n'.format(np.round(np.mean(totaltest),6)))
                for s in selectcol[:]:
                    f.write('{} '.format(s))
                f.write('\n')
            self._TemplUsedFeatures, self._score = selectcol[:], np.mean(totaltest)

    @_reachlimit
    def chekcLimit(self):
        return True

    @_reachlimit
    def _ScoreUpdate(self):
        if self._direction == 'ascend':
            start = -np.inf
        else:
            start = np.inf
        if self._score == start:
            return True
        elif self._evaluate(self._score, self.bestscore):
            self.bestscore, self._bestfeature = self._score, self._TemplUsedFeatures[:]
        return True

class Select(object):
    """This is a class for importances features selection

    The functions needed to be called before running include:

        ImportDF(pd.dataframe, str) - import you complete dataset including the label column

        ImportLossFunction(func, str) - import your self define loss function,
                                        eq. logloss, accuracy, etc

        InitialFeatures(list) - Initial your starting features combination,
                                if the initial features combination include
                                all features, the backward sequence searching
                                will run automatically

        InitialNonTrainableFeatures(list) - Initial the non-trainable features

        run(func) - start selecting features
    """

    def __init__(self):
        self._NonTrainableFeatures = []
        self._temp = []
        self._logfile = 'record.log'
        self._FeaturesLimit = np.inf
        self._TimeLimit = np.inf
        self._sampleratio = 1
        self._samplestate = 0
        self._samplemode = 1
        self._frac = 1
        self._batch = 1
        self._key = ''
        self._update_imp = 1

    def SetLogFile(self, fn):
        """Setup the log file

        Args:
            fn: str, filename
        """
        self._logfile = fn

    def ImportDF(self, df, label):
        """Import pandas dataframe to the class

        Args:
            df: pandas dataframe include all features and label.
            label: str, label name
        """
        self._df = df
        self._label = label

    def ImportLossFunction(self, modelscore, direction):
        """Import the loss function

        Args:
            modelscore: the function to calculate the loss result
                        with two input series
            direction: str, ‘ascent’ or descent, the way you want
                       the score to go
        """
        self._modelscore = modelscore
        self._direction = direction

    def InitialFeatures(self,features):
        """Initial your starting features combination

        Args:
            features: list, the starting features combination
        """
        self._temp = features

    def SelectRemoveMode(self, frac = 1, batch = 1, key = ''):
        self._frac = frac
        self._batch = batch
        self._key = key

    def SetFeaturesLimit(self, FeaturesLimit):
        """Set the features quantity limitation, when selected features reach
           the quantity limitation, the algorithm will exit

        Args:
            FeaturesLimit: int, the features quantity limitation
        """
        self._FeaturesLimit = FeaturesLimit

    def SetTimeLimit(self, TimeLimit):
        """Set the running time limitation, when the running time
           reach the time limit, the algorithm will exit

        Args:
            TimeLimit: double, the maximum time in minutes
        """
        self._TimeLimit = TimeLimit

    def SetSample(self, ratio, samplestate=0, samplemode=1, update_importance = 1):
        """Set the sample of all data

        Args:
            ratio: double, sample ratio
            samplestate: int, seed
            samplemode: positive int, if 0, every time they
                        sample the same subset, default = 1
        """
        self._sampleratio = ratio
        self._samplestate = samplestate
        self._samplemode = samplemode
        self._update_imp = update_importance

    def SetClassifier(self, clf, fit_params=None):
        """Set the classifier and its fit_params

        Args:
            clf: estimator object, defined algorithm to train and evaluate features
            fit_params, dict, optional, parameters to pass to the fit method
        """
        self.clf = clf
        self.fit_params = fit_params = fit_params if fit_params is not None else {}

    def run(self,validate):
        """start running the selecting algorithm

        Args:
            validate: validation method, eq. kfold, last day, etc
        """
        with open(self._logfile, 'a') as f:
            f.write('\n{}\n%{}%\n'.format('Start!','-'*60))
        print("Features Quantity Limit: {}".format(self._FeaturesLimit))
        print("Time Limit: {} min(s)".format(self._TimeLimit))
        a = _importsance_selection(df = self._df, clf = self.clf,
                                    RecordFolder = self._logfile,
                                    LossFunction = self._modelscore,
                                    label = self._label,
                                    start = self._temp,
                                    FeaturesQuanLimitation = self._FeaturesLimit,
                                    TimeLimitation = self._TimeLimit,
                                    selectkey = self._key,
                                    selectbatch = self._batch,
                                    selectfrac = self._frac,
                                    direction = self._direction,
                                    validatefunction = validate,
                                    important_update = self._update_imp,
                                    )
        best_features_comb = a.select()
        #best_features_comb = a._bestfeature
        with open(self._logfile, 'a') as f:
            f.write('\n{}\n{}\n%{}%\n'.format('Done',self._temp,'-'*60))
        return best_features_comb

max_stars_repo_name: ranchlin/Feature-Selection

max_issues_repo_name: luobaozhu/Feature-Selection

max_forks_repo_name: luobaozhu/Feature-Selection


text:
import pytest

def test_composition():
    from pyxmolpp2 import XYZ, Rotation, Translation, Degrees, UniformScale

    R = Rotation(XYZ(1, 1, 1), Degrees(39))
    T = Translation(XYZ(7, 1, 2))
    S = UniformScale(1.5)
    G = R * T * S * R * T

    def check(G):
        r = XYZ(5, 6, 4)
        assert ((G * G.inverted()).transform(r) - r).len() == pytest.approx(0)

    check(R)
    check(T)
    check(S)
    check(G)

    for a in [R, T, S, G]:
        for b in [R, T, S, G]:
            check(a * b)

def test_transformation_3d():
    import numpy as np
    from pyxmolpp2 import XYZ, Rotation, Translation, Degrees

    R = Rotation(XYZ(1, 0, 0), Degrees(45))
    T = Translation(XYZ(1, 2, 5))

    G = T * R

    m = G.matrix3d()
    v = G.vector3d()

    assert pytest.approx(m[0, 0]) == 1
    assert pytest.approx(m[0, 1]) == 0
    assert pytest.approx(m[0, 2]) == 0
    assert pytest.approx(np.sqrt(2) / 2) == m[2, 1]

    assert v.x == 1
    assert v.y == 2
    assert v.z == 5

def test_rotation_decomposition():
    import numpy as np
    from pyxmolpp2 import XYZ, Rotation, Degrees

    for ax, theta in [
        (XYZ(0, 0, 1), Degrees(30)),
        (XYZ(0, 0, 1), Degrees(70)),
        (XYZ(0, 1, 0), Degrees(70)),
        (XYZ(1, 0, 0), Degrees(70)),
        (XYZ(1, 1, 0), Degrees(70)),
        (XYZ(1, 1, 1), Degrees(70)),
        (XYZ(1, 0, 1), Degrees(0)),
        (XYZ(1, 1, 1), Degrees(0)),
    ]:
        R = Rotation(ax, theta)

        ax1, theta1 = R.axis(), R.theta()

        R2 = Rotation(ax1, theta1)

        assert np.allclose(R.matrix3d(), R2.matrix3d()), (
            "\n".join(map(lambda x: "%25.18e" % x, (R.matrix3d() - R2.matrix3d()).flatten())))

def test_angle_extraction():
    import numpy as np
    from pyxmolpp2 import Rotation

    r = Rotation(np.identity(3))
    assert np.isclose(r.theta().degrees, 0.0)

    r = Rotation(
        np.array(
            [1.000000000000000444e+00,
             1.526556658859591229e-16,
             6.938893903907233000e-18,
             1.526556658859591229e-16,
             1.000000000000000666e+00,
             -5.551115123125786400e-17,
             0.000000000000000000e+00,
             -5.551115123125786400e-17,
             1.000000000000000444e+00, ]
        ).reshape((3, 3))
    )
    assert np.isclose(r.theta().degrees, 0.0)

def test_bad_rotation_matrix_input():
    import numpy as np
    from pyxmolpp2 import Rotation

    with pytest.raises(TypeError):
        r = Rotation(np.array([1, 2, 3]))

ext: py

reason: import numpy


text:
import os
import numpy as np
import keras
from keras import models, layers
from PIL import Image
from numpy import asarray
from cv2 import cv2
from keras.models import load_model
from os.path import join, dirname, realpath
from flask import Flask,render_template,request
import skimage
from skimage.transform import resize
from matplotlib import pyplot as plt

app = Flask(__name__)
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
app.config["CACHE_TYPE"] = "null"
app.static_folder = 'static'
f = ""
original_filename =""
cwd = str(os.getcwd())

UPLOADS_PATH = join(dirname(realpath(__file__)), 'static\\')

encoder = load_model("encoder.h5")
decoder = load_model("decoder.h5")
compressor = load_model("compressor.h5")

def preprocess(path):
    img = Image.open(path)
    image = np.asarray(img)
    image = image.astype('float32') / 255.
    img = np.expand_dims(image,axis=-1)
    img = np.expand_dims(image,axis=-0)
    return img

@app.route('/')
def upload_image():
    if os.path.exists(cwd+"\\static\\decompressed.jpg"):
        os.remove(cwd+"\\static\\decompressed.jpg")
    if os.path.exists(cwd+"\\static\\intermediate.jpg"):
        os.remove(cwd+"\\static\\intermediate.jpg")
    if os.path.exists(cwd+"\\static\\compressed.npy"):
        os.remove(cwd+"\\static\\compressed.npy")
    return render_template('index.html')

@app.route('/imageuploader',methods=['GET','POST'])
def image_upload():
    if request.method=='POST':
        f = request.files['image']
        f.save(os.path.join(UPLOADS_PATH,f.filename))
        original_filename = f.filename

        path = cwd+"\\static\\"+f.filename
        image = Image.open(path)
        img = preprocess(path)

        output = encoder.predict(img)
        path = cwd+"\\static\\compressed"
        np.save(path, output)

        img = np.resize(image, (28, 28,1))
        code = compressor.predict(img[None])[0]
        path = cwd+"\\static\\intermediate.jpg"
        plt.imsave(path,code.reshape([code.shape[-1]//2,-1]))
        return render_template('compress.html',filename = f.filename)

def postprocess(img):
    image = np.reshape(img, (28, 28, 1))
    image = image * 255
    return image

@app.route('/npyuploader',methods=['GET','POST'])
def npy_upload():
    if request.method=='POST':
        print(original_filename)
        f = request.files['npy']
        f.save(os.path.join(UPLOADS_PATH,f.filename))

        path = cwd+"\\static\\"+f.filename
        compressed_img = np.load(path)

        decompressed_img = decoder.predict(compressed_img)
        decompressed_img = postprocess(decompressed_img)

        path = cwd+"\\static\\decompressed.jpg"
        cv2.imwrite(path, decompressed_img)

        return render_template('decompress.html',filename = original_filename)

if __name__ == '__main__':
    app.run()

reason: import numpy,from numpy


text:
'''
    Export/Dataframes/counter
    _________________________

    Linkage counters for hierarchical dataframes.

    :copyright: (c) 2015 The Regents of the University of California.
    :license: GNU GPL, see licenses/GNU GPLv3.txt for more details.
'''

# load modules/submodules
from collections import defaultdict

import numpy as np

from xldlib.utils import logger

# HELEPRS
# -------

def lenif(iterable):
    '''Returns the length of the iterable removing falsey items'''

    return len([i for i in iterable if i])

# TEMPLATES
# ---------

def counts_template():
    return {
        'count': defaultdict(int),
        'interaction': defaultdict(set)
    }

# COUNTERS
# --------

@logger.init('spreadsheet', level='DEBUG')
class IndexMemo(dict):
    '''Subclass to provide facile index setting/memoization'''

    def __init__(self, dataframe, linkageindex=1):
        super(IndexMemo, self).__init__()

        self.dataframe = dataframe
        self.linkageindex = linkageindex

    #      MAGIC

    def __getitem__(self, key, dict_getitem=dict.__getitem__):
        '''Checks the linkage string previous set or creates a new index'''

        if key in self:
            return dict_getitem(self, key)
        else:
            index = self.dataframe.get_last_index()
            column = self.dataframe.get_column(self.linkageindex)
            self[key] = index
            self.dataframe.loc[index, column] = key
            return index

@logger.init('spreadsheet', level='DEBUG')
class LinkageCounts(defaultdict):
    '''Subclass to provide facile linkage counting'''

    def __init__(self, counter, factory=list):
        super(LinkageCounts, self).__init__(factory)

        self.counter = counter

    #     SETTERS

    def setlinkage(self, linkage):
        self[linkage.string].append(self.counter(linkage))

    #     GETTERS

    def get_counts(self, key):
        return len(self[key])

    def get_sum(self, key):
        return sum(self[key])

@logger.init('spreadsheet', level='DEBUG')
class FileCounts(defaultdict):
    '''Subclass to provide facile linkage counting'''

    def __init__(self, counter, headers=None, factory=counts_template):
        super(FileCounts, self).__init__(factory)

        self.counter = counter
        if headers is not None:
            self.headers = {j: i for i, j in enumerate(headers)}

    #     SETTERS

    def setlinkage(self, linkage, key=None, ratio_obj=None):
        '''
        Sets the linkage counts using a counter mechanism if counts
        are not provided, else, it uses a ratio_obj added to add the
        normalized values and the remaining items.
        '''

        if key is None:
            key = linkage.file

        if ratio_obj is None:
            self.__setlinkagesingle(linkage, key)
        else:
            self.__setlinkageratio(linkage, key, ratio_obj)

    def __setlinkagesingle(self, linkage, key):
        '''Sets the linkage entry for a single counts item (no counters)'''

        self[key]['count'][linkage.string] += self.counter(linkage)
        self[key]['interaction'][linkage.getsubunit()] = None

    def __setlinkageratio(self, linkage, key, ratio_obj):
        '''Sets an array of values for linkage counts from a ratio object'''

        zeros = np.zeros(ratio_obj.counts.size)
        array = self[key]['count'].setdefault(linkage.string, zeros)

        # push the normalized value to the end
        counts = ratio_obj.counts.tolist()
        counts.append(counts.pop(ratio_obj.index))
        array += np.array(counts)

        # add interaction values
        subunit = linkage.getsubunit()
        for header, index in self.headers.items():
            if counts[index]:
                self[key]['interaction'][header].add(subunit)

        if counts[-1]:
            self[key]['interaction']['normalize'].add(subunit)

    #     GETTERS

    def get_counts(self, key, header=None):
        '''Returns the count of linkages'''

        if header is None:
            return len(self[key]['count'])
        else:
            keycounts = self.counts_from_index(key, self.headers[header])
            normalization = self.counts_from_index(key, -1)
            return '{0}/{1}'.format(lenif(keycounts), lenif(normalization))

    def get_sum(self, key, header=None):
        '''Returns the total sum of counts within the file'''

        if header is None and hasattr(self, "headers"):
            return sum(i.sum() for i in self[key]['count'].values())
        elif header is None:
            return sum(self[key]['count'].values())
        else:
            keycounts = self.counts_from_index(key, self.headers[header])
            normalization = self.counts_from_index(key, -1)
            return '{0}/{1}'.format(sum(keycounts), sum(normalization))

    def get_interactions(self, key, header=None):
        '''Returns the total number of interactions'''

        if header is None:
            return len(self[key]['interaction'])
        else:
            length = len(self[key]['interaction'][header])
            normalize = len(self[key]['interaction']['normalize'])
            return '{0}/{1}'.format(length, normalize)

    def counts_from_index(self, key, index):
        return (int(i[index]) for i in self[key]['count'].values())

    #      TOTALS

    def get_totalcounts(self):
        total = set()
        for values in self.values():
            total.update(list(values['count']))
        return len(total)

    def get_totalsum(self):
        return sum(self.get_sum(i) for i in self)

    def get_totalinteractions(self):
        total = set()
        for values in self.values():
            total.update(values['interaction'])
        return len(total)

max_stars_repo_name: Alexhuszagh/XLDiscoverer

max_issues_repo_name: Alexhuszagh/XLDiscoverer

max_forks_repo_name: Alexhuszagh/XLDiscoverer


text:
import torch
import random
import copy
import numpy as np
import pdb
import cv2
import os

def patch_shuffle(image_data, label_data, region_size=16):
    #shuffle_flag = random.uniform(0,1.)
    shuffle_flag = np.random.rand()
    if shuffle_flag > 0.5:
        return image_data, label_data

    c, h, w = image_data.shape
    x_interval =  h // region_size - 1
    y_interval = w // region_size - 1

    x_index1 = random.randint(0, x_interval) * 16
    y_index1 = random.randint(0, y_interval) * 16

    x_index2 = random.randint(0, x_interval) * 16
    y_index2 = random.randint(0, y_interval) * 16

    while x_index1==x_index2 and y_index1==y_index2:
        x_index2 = random.randint(0, x_interval)
        y_index2 = random.randint(0, y_interval)

    image = copy.deepcopy(image_data)
    label = copy.deepcopy(label_data)

    im_patch1 = image[:,x_index1:x_index1+region_size, y_index1:y_index1+region_size]
    im_patch2 = image[:,x_index2:x_index2+region_size, y_index2:y_index2+region_size]

    gt_patch1 = label[:,x_index1:x_index1+region_size, y_index1:y_index1+region_size]
    gt_patch2 = label[:,x_index2:x_index2+region_size, y_index2:y_index2+region_size]

    image_data[:,x_index1:x_index1+region_size, y_index1:y_index1+region_size] = im_patch2
    image_data[:,x_index2:x_index2+region_size, y_index2:y_index2+region_size] = im_patch1

    label_data[:,x_index1:x_index1+region_size, y_index1:y_index1+region_size] = gt_patch2
    label_data[:,x_index2:x_index2+region_size, y_index2:y_index2+region_size] = gt_patch1

    image_data, label_data = random_offset(image_data, label_data, x_index1, x_index2, y_index1, y_index2, x_interval, y_interval)
    #pdb.set_trace()
    #cv2.imwrite('im.png', (image.permute(1,2,0).numpy()[:,:,::-1] + 0.5)*255)
    #cv2.imwrite('im_sf.png', (image_data.permute(1,2,0).numpy()[:,:,::-1] + 0.5)*255)
    #cv2.imwrite('label_sf.png', (label_data.permute(1,2,0).numpy()[:,:,0])*50)
    #pdb.set_trace()

    return image_data, label_data

def random_offset(image_data, label_data, x_index1, x_index2, y_index1, y_index2, x_interval, y_interval):
    h_or_v_flag = np.random.rand() #determinte which direction to conduct offset
    H_offset = h_or_v_flag > 0.5
    region_size = 16
    offset_dis = random.randint(0,16)
    if offset_dis == 0:
        return image_data, label_data

    if H_offset:
        #random offset along horizon direction
        x_idx = random.randint(0, x_interval) * 16
        start_idx = random.randint(0, y_interval)
        end_idx = random.randint(start_idx + 1, y_interval+1) * 16
        start_idx = start_idx * 16

        im_patch = image_data[:, x_idx:x_idx+region_size, start_idx:end_idx]
        gt_patch = label_data[:,x_idx:x_idx+region_size, start_idx:end_idx]
        #patch_len = end_idx - start_idx

        replace_or_zero = np.random.rand()
        if replace_or_zero > 0.5: #replace
            if replace_or_zero > 0.75:#forward
                new_im_patch = torch.cat([im_patch[:, :, -offset_dis:], im_patch[:, :, :-offset_dis]], dim=2)
                new_gt_patch = torch.cat([gt_patch[:, :, -offset_dis:], gt_patch[:, :, :-offset_dis]], dim=2)
            else:#backward
                new_im_patch = torch.cat([im_patch[:,:, offset_dis:], im_patch[:, :, :offset_dis]], dim=2)
                new_gt_patch = torch.cat([gt_patch[:,:, offset_dis:], gt_patch[:, :, :offset_dis]], dim=2)

            image_data[:, x_idx:x_idx+region_size, start_idx:end_idx] = new_im_patch
            label_data[:,x_idx:x_idx+region_size, start_idx:end_idx] = new_gt_patch
        else: #random fill
            random_im_patch = torch.rand(3, 16, offset_dis) * 2 -1 # to -1---1
            random_gt_patch = torch.ones(1,16,offset_dis) * 50
            if replace_or_zero < 0.25:#forward
                new_im_patch = torch.cat([random_im_patch, im_patch[:, :, :-offset_dis]], dim=2)
                new_gt_patch = torch.cat([random_gt_patch, gt_patch[:, :, :-offset_dis]], dim=2)
            else:#backward
                new_im_patch = torch.cat([im_patch[:,:, offset_dis:], random_im_patch], dim=2)
                new_gt_patch = torch.cat([gt_patch[:,:, offset_dis:], random_gt_patch], dim=2)

            image_data[:, x_idx:x_idx+region_size, start_idx:end_idx] = new_im_patch
            label_data[:,x_idx:x_idx+region_size, start_idx:end_idx] = new_gt_patch
    else:
        #random offset along horizon direction
        y_idx = random.randint(0, y_interval) * 16
        start_idx = random.randint(0, x_interval)
        end_idx = random.randint(start_idx + 1, x_interval+1) * 16
        start_idx = start_idx * 16

        im_patch = image_data[:, start_idx:end_idx, y_idx:y_idx+region_size]
        gt_patch = label_data[:, start_idx:end_idx, y_idx:y_idx+region_size]
        patch_len = end_idx - start_idx

        replace_or_zero = np.random.rand()
        if replace_or_zero > 0.5: #replace
            if replace_or_zero > 0.75:#forward
                new_im_patch = torch.cat([im_patch[:,-offset_dis:, :], im_patch[:, :-offset_dis,:]], dim=1)
                new_gt_patch = torch.cat([gt_patch[:,-offset_dis:, :], gt_patch[:, :-offset_dis,:]], dim=1)
            else:#backward
                new_im_patch = torch.cat([im_patch[:,offset_dis:,:], im_patch[:, :offset_dis, :]], dim=1)
                new_gt_patch = torch.cat([gt_patch[:,offset_dis:,:], gt_patch[:, :offset_dis, :]], dim=1)

            image_data[:, start_idx:end_idx, y_idx:y_idx+region_size] = new_im_patch
            label_data[:, start_idx:end_idx, y_idx:y_idx+region_size] = new_gt_patch
        else: #random fill
            random_im_patch = torch.rand(3, offset_dis, 16) * 2 -1
            random_gt_patch = torch.ones(1,offset_dis, 16) * 50
            if replace_or_zero < 0.25:#forward
                new_im_patch = torch.cat([random_im_patch, im_patch[:, :-offset_dis, :]], dim=1)
                new_gt_patch = torch.cat([random_gt_patch, gt_patch[:, :-offset_dis,:]], dim=1)
            else:#backward
                new_im_patch = torch.cat([im_patch[:,offset_dis:, :], random_im_patch], dim=1)
                new_gt_patch = torch.cat([gt_patch[:,offset_dis:, :], random_gt_patch], dim=1)

            image_data[:, start_idx:end_idx, y_idx:y_idx+region_size] = new_im_patch
            label_data[:, start_idx:end_idx, y_idx:y_idx+region_size] = new_gt_patch

    return image_data, label_data

max_stars_repo_name: YanFangCS/AINET

max_issues_repo_name: YanFangCS/AINET

max_forks_repo_name: YanFangCS/AINET


text:
import numpy
import FileUtil
import random
import time
import Load_dataset
numpy.random.seed(123)
random.seed(123)

import time
import sys
import subprocess
import os
from Represent_luo import RepresentationLayer
import draw_design
import Eval

from keras.models import Sequential,Model
from keras.layers import Dense, Dropout, Activation
from keras.layers import Embedding,Input
from keras.layers.recurrent import LSTM, SimpleRNN,GRU
from keras.layers.merge import Concatenate,Add
from keras.layers.wrappers import Bidirectional,TimeDistributed
from keras.layers import Conv1D, MaxPooling1D,GlobalMaxPooling1D,GlobalAveragePooling1D,LocallyConnected1D
from keras.layers.core import Masking, Reshape,Lambda, Flatten
from keras.optimizers import RMSprop, SGD, Adam, Adadelta, Adagrad,Nadam,Adamax
#from keras.layers.normalization import BatchNormalization
from keras import initializers
from AttentionLayer import Attention
from keras import backend as K

if __name__ == '__main__':

    s = {
         'pos_dim':5,
         'pos_fea':True,
         'ner_dim':5,
         'ner_fea':True,
         'sent_max_num':15, # number of backprop through time steps
         'word_max_num':30,
         'ppi_max_len':400,
         'ppi_pre':True,
         'ppilayer_train':True,
         'model_save':True,
         'hidden_dims':50,
         'mini_batch':32,
         'epochs':20}

    print s
    model_file='/home/BIO/luoling/biocreativeVI/model/NN/hiernn-token_pos_ner-ppipre-50d-tune.model_post1'

    train_result_file='/home/BIO/luoling/biocreativeVI/result/NN/hiernn-token_pos_ner-ppipre-50d-tune.result_train_post'
    dev_result_file='/home/BIO/luoling/biocreativeVI/result/NN/hiernn-token_pos_ner-ppipre-50d-tune.result_dev_post'
    test_result_file='/home/BIO/luoling/biocreativeVI/result/NN/hiernn-token_pos_ner-ppipre-50d-tune.result_test_post'

#    folder = '/home/BIO/luoling/biocreativeVI/model/word_embedding_model/'
#    folder = '/home/BIO/luoling/PPIAC_CNN/'
    folder = '/home/BIO/luoling/biocreativeVI/model/word_embedding_model/fastText/'
#    rep1 = RepresentationLayer(folder + 'word2vec_model/bc_ex3_token_50.word2vec_model',vec_size=50, frequency=800000)
#    rep1 = RepresentationLayer(folder + 'PMtask_Triage_Pubmed_ori.sg50_w2v',vec_size=50, frequency=900000)
    rep1 = RepresentationLayer(folder + 'BioCreativevi.vec_new',POS_vocab_file='./fea_vocab/POS.vocab',NER_vocab_file='./fea_vocab/NER.vocab',vec_size=50, frequency=800000)

    # instanciate the model
    sent_input=Input(shape=(s['word_max_num'],), dtype='int32', name='sent_input')

    word_emb=Embedding(rep1.vec_table.shape[0], rep1.vec_table.shape[1], \
                        weights=[rep1.vec_table], input_length=s['word_max_num'])
    sent_vec=word_emb(sent_input)

    POS_fea=Input(shape=(s['word_max_num'],), dtype='int32', name='POS_fea')
    NER_fea=Input(shape=(s['word_max_num'],), dtype='int32', name='NER_fea')

    #POS_fea=Input(shape=(s['max_len'],), dtype='int32', name='POS_fea')
    pos_emb=Embedding(len(rep1.pos_fea_index),s['pos_dim'] , input_length=s['word_max_num'])
    pos_vec=pos_emb(POS_fea)
    pos_vec=Dropout(0.2)(pos_vec)
    pos_lstm=LSTM(5,implementation=2,dropout=0.2, recurrent_dropout=0.2,return_sequences=True)(pos_vec)
    pos_pool=GlobalMaxPooling1D()(pos_lstm)
    posEncoder=Model([POS_fea],pos_pool)

    #NER_fea=Input(shape=(s['max_len'],), dtype='int32', name='NER_fea')
    ner_emb=Embedding(len(rep1.ner_fea_index),s['ner_dim'] , input_length=s['word_max_num'])
    ner_vec=ner_emb(NER_fea)
    ner_vec=Dropout(0.2)(ner_vec)
    ner_lstm=LSTM(5,implementation=2,dropout=0.2, recurrent_dropout=0.2,return_sequences=True)(ner_vec)
    ner_pool=GlobalMaxPooling1D()(ner_lstm)
    nerEncoder=Model([NER_fea],ner_pool)
    #sent_vec = Concatenate()([sent_vec, pos_vec, ner_vec])

    sent_vec_drop=Dropout(0.2)(sent_vec)
#    model.add(BatchNormalization())
    sent_lstm=LSTM(50,implementation=2,dropout=0.2, recurrent_dropout=0.2,return_sequences=True)(sent_vec_drop)
#    sent_tanh=TimeDistributed(Dense(50, activation = "tanh"))(sent_lstm)
#    sent_pool=Attention()(sent_lstm)
    sent_pool=GlobalMaxPooling1D()(sent_lstm)
    #sentEncoder=Model([sent_input,POS_fea,NER_fea],sent_pool)
    sentEncoder=Model([sent_input],sent_pool)

    doc_input=Input(shape=(s['sent_max_num'],s['word_max_num']), dtype='int32', name='doc_input')
    doc_input_pos=Input(shape=(s['sent_max_num'],s['word_max_num']), dtype='int32', name='doc_input_pos')
    doc_input_ner=Input(shape=(s['sent_max_num'],s['word_max_num']), dtype='int32', name='doc_input_ner')

    doc_encoder = TimeDistributed(sentEncoder)(doc_input)
    doc_encoder_pos = TimeDistributed(posEncoder)(doc_input_pos)
    doc_encoder_ner = TimeDistributed(nerEncoder)(doc_input_ner)
    if s['pos_fea']==True and s['ner_fea']==False:
        doc_encoder=Concatenate()([doc_encoder,doc_encoder_pos])
    elif s['pos_fea']==False and s['ner_fea']==True:
        doc_encoder=Concatenate()([doc_encoder,doc_encoder_ner])
    elif s['pos_fea']==True and s['ner_fea']==True:
        doc_encoder=Concatenate()([doc_encoder,doc_encoder_pos,doc_encoder_ner])
    else:
        doc_encoder=Concatenate()(doc_encoder)
    doc_lstm_left=LSTM(100,implementation=2,dropout=0.2, recurrent_dropout=0.2,return_sequences=False)(doc_encoder)
    doc_lstm_right=LSTM(100,implementation=2,dropout=0.2, recurrent_dropout=0.2,return_sequences=False,go_backwards=True)(doc_encoder)
    bilstm=Concatenate()([doc_lstm_left,doc_lstm_right])

    word=Input(shape=(s['ppi_max_len'],), dtype='int32', name='word')
    if s['ppi_pre']==True:
        #word=Input(shape=(s['ppi_max_len'],), dtype='int32', name='word')
        ppi_emb=Embedding(rep1.vec_table.shape[0], rep1.vec_table.shape[1], weights=[rep1.vec_table], input_length=s['ppi_max_len'],name='emb',trainable=s['ppilayer_train'])
        ppi_vec=ppi_emb(word)
        #ppi_vec=Dropout(0.2)(ppi_vec)
	ppi_lstm=Bidirectional(LSTM(50,implementation=2,dropout=0.2,recurrent_dropout=0.2,return_sequences=True,trainable=s['ppilayer_train']),name='ppi_bilstm')(ppi_vec)
	ppi_att=Attention(name='ppi_att',trainable=s['ppilayer_train'])(ppi_lstm)
	ppi_fc=Dense(100,activation='relu',name='ppi_dense1',trainable=s['ppilayer_train'])(ppi_att)
	ppi_fc=Dense(50,activation='relu',name='ppi_dense2',trainable=s['ppilayer_train'])(ppi_fc)
        bilstm=Concatenate()([bilstm,ppi_fc])
    predict=Dense(2,activation='softmax',name='lastpre')(bilstm)

    model= Model(inputs=[doc_input,doc_input_pos,doc_input_ner,word],outputs=predict)

    model.load_weights(model_file, by_name=True)

    model.summary()

    devfile='/home/BIO/luoling/biocreativeVI/data/ppiac/PMtask_Triage_DevSet_ir_scon.hiernn_PPIAC_fea'
    fin_dev=open(devfile,'r')
    dev=[]
    sent=[]
    for line in fin_dev:
        line=line.strip()
        if line!='':
            sent.append(line)
        else:
            dev.append(sent)
            sent=[]
    #dev_x1,dev_y1=rep1.represent_instances_3D(dev, s['sent_max_num'],s['word_max_num'])
    dev_x1,dev_pos1,dev_ner1,dev_y1=rep1.represent_instances_3D_fea(dev, s['sent_max_num'],s['word_max_num'])
    fin_dev.close()

    trainfile='/home/BIO/luoling/biocreativeVI/data/ppiac/PMtask_Triage_TrainingSet_ir_scon.hiernn_PPIAC_fea'
    fin_train=open(trainfile,'r')
    train=[]
    sent=[]
    for line in fin_train:
        line=line.strip()
        if line!='':
            sent.append(line)
        else:
            train.append(sent)
            sent=[]
    #train_x1,train_y1=rep1.represent_instances_3D(train, s['sent_max_num'],s['word_max_num'])
    train_x1,train_pos1,train_ner1,train_y1=rep1.represent_instances_3D_fea(train, s['sent_max_num'],s['word_max_num'])
    fin_train.close()

    testfile='/home/BIO/luoling/biocreativeVI/data/ppiac/PMtask_TestSet_gold.hiernn_PPIAC_fea'
    fin_test=open(testfile,'r')
    test=[]
    sent=[]
    for line in fin_test:
        line=line.strip()
        if line!='':
            sent.append(line)
        else:
            test.append(sent)
            sent=[]
    #test_x1,test_y1=rep1.represent_instances_3D(test, s['sent_max_num'],s['word_max_num'])
    test_x1,test_pos1,test_ner1,test_y1=rep1.represent_instances_3D_fea(test, s['sent_max_num'],s['word_max_num'])
    fin_test.close()
#    print train_x1

    train_token_text,train_pos_text,train_ner_text=Load_dataset.read_dataset('/home/BIO/luoling/biocreativeVI/data/ppiac/fea-add/PMtask_Triage_TrainingSet_ir_scon.conll.PPIAC')
    dev_token_text,dev_pos_text,dev_ner_text=Load_dataset.read_dataset('/home/BIO/luoling/biocreativeVI/data/ppiac/fea-add/PMtask_Triage_DevSet_ir_scon.conll.PPIAC')
    test_token_text,test_pos_text,test_ner_text=Load_dataset.read_dataset('/home/BIO/luoling/biocreativeVI/data/ppiac/test/PMtask_TestSet_gold.fea_conll.PPIAC')
#    print test_token_text[0],test_pos_text[0],test_ner_text[0]
#    train1 = [line.strip() for line in open(folder + 'corpus/testfile1.txt')

    test_token,test_pos,test_ner, test_y = rep1.represent_instances_fea(test_token_text,test_pos_text,test_ner_text, max_len=s['ppi_max_len'])
    dev_token, dev_pos, dev_ner, dev_y = rep1.represent_instances_fea(dev_token_text, dev_pos_text, dev_ner_text, max_len=s['ppi_max_len'])
    train_token,train_pos, train_ner,train_y = rep1.represent_instances_fea(train_token_text,train_pos_text,train_ner_text, max_len=s['ppi_max_len'])

    inds = range(train_y1.shape[0])
    numpy.random.shuffle(inds)
    batch_num = len(inds) / s['mini_batch']

    # train with early stopping on validation set

    best_f1 = -numpy.inf
    iter = 0
    max_f=[0,0,0,0,-1]
    test_f=[0,0,0,0,-1]
    max_res=0
    each_print=False
    max_minibatch=0
    temp=0.6
    pit_x=[]

    train_res = model.predict([train_x1,train_pos1,train_ner1,train_token],batch_size=200)
    dev_res = model.predict([dev_x1,dev_pos1,dev_ner1,dev_token],batch_size=200)
    test_res = model.predict([test_x1,test_pos1,test_ner1,test_token],batch_size=200)
    F=Eval.eval_mulclass(train_y1, train_res, True,True)
    F=Eval.eval_mulclass(dev_y1, dev_res, True,True)
    F=Eval.eval_mulclass(test_y1, test_res, True,True)
    if s['model_save']==True:
        FileUtil.writeFloatMatrix(train_res, train_result_file)
        FileUtil.writeFloatMatrix(dev_res, dev_result_file)
        FileUtil.writeFloatMatrix(test_res, test_result_file)

max_stars_repo_name: lingluodlut/PPIAC

max_issues_repo_name: lingluodlut/PPIAC

max_forks_repo_name: lingluodlut/PPIAC


text:
import sys
import random
import numpy as np
import matplotlib.pyplot as plt
import streamlit as st
import os
import torch_modulation_recognition as tmr

NUM_PER_CLASS = 6000

params = {
    "INPUT_TYPE": "random",
    "IDX": 0,
    "BUTTON_PRESSED": False
}

@st.cache(allow_output_mutation=True)
def load_data():
    data = tmr.data.RadioML2016()
    return data

def get_params(data):

    # st.sidebar.title('Privacy Encoder')
    st.sidebar.title("Options")

    # Get signal type
    params["MOD"] = st.sidebar.selectbox("Modulation", list(tmr.data.MODULATIONS.keys()))
    params["SNR"] = st.sidebar.selectbox("Signal-to-Noise Ratio", tmr.data.SNRS)

    # Get input type
    options = ["idx", "random"]
    params["INPUT_TYPE"] = st.sidebar.selectbox("Input Options", options)

    if params["INPUT_TYPE"] == "idx":
        params["IDX"] = st.sidebar.slider(
            'idx',
            min_value=0,
            max_value=NUM_PER_CLASS,
            value=params["IDX"]
        )
    elif params["INPUT_TYPE"] == "random":
        params["IDX"] = random.randrange(NUM_PER_CLASS)

    params["BUTTON_PRESSED"] = st.sidebar.button("Click here to plot")

    return params

def main(data):

    params = get_params(data)

    params["SIGNALS"] = data.get_signals(mod=params["MOD"], snr=params["SNR"])

    if params["BUTTON_PRESSED"]:
        fig, ax = plt.subplots(nrows=2, ncols=1, sharey=True, sharex=True)
        plt.xlabel("time (s)")
        fig.suptitle("({}) Modulation: {}, Signal-to-Noise Ratio: {} (In-Phase (I) and Quadrature (Q) Channels)".format(params["IDX"], params["MOD"], params["SNR"]))
        ax[0].plot(params["SIGNALS"][(params["MOD"], params["SNR"])][params["IDX"], 0, :])
        ax[1].plot(params["SIGNALS"][(params["MOD"], params["SNR"])][params["IDX"], 1, :])
        st.pyplot()
        xx=params["SIGNALS"][(params["MOD"], params["SNR"])][params["IDX"], 0, :]
        yy=params["SIGNALS"][(params["MOD"], params["SNR"])][params["IDX"], 1, :]

        pwd=os.path.abspath(os.curdir)
        logfile = open(pwd+"\\log.txt", mode = 'a',encoding='ASCII')
        print("Tpye: {}\n x:\n {}\n y:\n {} ".format(params["MOD"], xx,yy),end=' ',file=logfile)
        logfile.close()
        return xx , yy
if __name__ == "__main__":
    data = load_data()
    main(data)

max_stars_repo_name: azyslzp/LZP-Signal-recognition

max_issues_repo_name: azyslzp/LZP-Signal-recognition

max_forks_repo_name: azyslzp/LZP-Signal-recognition


text:
import numpy as np
from audio import istft_, create_spectrogram
from sklearn.cluster import KMeans
import config

# Compute the reconstruction of the signal from the filtered spectrogram
def reconstruct_signal(filtered_spec, orig_spec, fs=config.fs, fftsize=config.fftsize):

	if orig_spec != None :
		angle = np.angle(orig_spec)
		phi = np.exp(1.0j*np.unwrap(angle)) # the angle must be 2pi modulo
	else :
		phi = np.random.randn(*orig_spec.shape)

	return istft_(filtered_spec*phi)

# Produce the embeddings of a mixture spectogram through a Deep Learning modle
def produce_embeddings(model, X):
	# If there is only 1 mixture (no batch)
	if len(X.shape) == 2:
		X = np.reshape(X, (1, X.shape[0], X.shape[1]))

	return model.embeddings(X)

def produce_masks(embeddings, nb_speakers):

	Kmean = KMeans(n_clusters = nb_speakers, n_jobs = -1)

	Kmean.fit(np.reshape(embeddings, (-1, embeddings.shape[2])))

	masks = [np.eye(nb_speakers)[label] for label in Kmean.labels_]

	masks = np.reshape(masks, embeddings.shape)

	return masks

def apply_masks(X, masks):
	return [masks[:, :, i]*X for i in range(masks.shape[2])]

def separate(signal, sample_rate, nb_speakers, model):

	spectrogram = create_spectrogram(signal, sample_rate)

	X = np.sqrt(abs(spectrogram))

	min_ = X.min()
	max_ = X.max()
	X = (X - min_)/(max_ - min_) # Normalized absolute spectrogram

	#Pass the normalized spec into the model
	embeddings = produce_embeddings(model, X)

	masks = produce_masks(embeddings, nb_speakers)

	filtered_specs = apply_masks(X, masks)

	signals = [reconstruct_signal(spec, spectrogram) for spec in filtered_specs]

	return signals

max_issues_repo_name: jhuiac/Adaptive-MultiSpeaker-Separation

max_forks_repo_name: jhuiac/Adaptive-MultiSpeaker-Separation


text:
#!/usr/bin/env python
# encoding: utf-8
'''

zscores_for_copy_number_when_mutated.py

Created by Joan Smith
on 2017-7-7.

Given a set of cnv files, mutation files and clinical files, all from TCGA
calculate cnv zscores partitioned by whether or not the gene in question is mutated

Copyright (c) 2017. All rights reserved.

'''

import pandas as pd
import numpy as np
import argparse
import sys
import os
import pdb
import collections
import glob
import rpy2
from multiprocessing import Pool

sys.path.append('../common/')
import utilities as util
import analysis

sys.path.append('../mutation-analysis')
import zscores_for_mutants as mutation_zscores

def get_options():
  parser = argparse.ArgumentParser(description='Get mutation, cnv, and clinical directories. optional output dir')
  parser.add_argument('-i', action='store', dest='cnv_directory')
  parser.add_argument('-m', action='store', dest='mutation_directory')
  parser.add_argument('-c', action='store', dest='clinical_directory')
  parser.add_argument('-o', action='store', dest='output_directory', default='.')
  namespace = parser.parse_args()

  return (namespace.cnv_directory, namespace.mutation_directory, namespace.clinical_directory,
    namespace.output_directory)

def mutations_for_gene(df):
  mutated_patients = df['identifier'].unique()
  return pd.DataFrame({'mutated': np.ones(len(mutated_patients))}, index=mutated_patients)

def prep_mutation_data(mutation, clinical_data):
  mutation, clinical_data_w_seq_patients, num_patients = mutation_zscores.prep_data(mutation, clinical_data)

  # include only nonsilent mutations
  non_silent = mutation.where(mutation[u'Variant_Classification'] != 'Silent')
  mutation = non_silent.dropna(subset=[u'Variant_Classification'])

  mutation = mutation.reset_index()
  mutation['Hugo_Symbol'] = '\'' + mutation['Hugo_Symbol'].astype(str)

  gene_mutation_df = mutation.groupby(['Hugo_Symbol']).apply(mutations_for_gene)
  gene_mutation_df.index.set_names(['Hugo_Symbol', 'patient'], inplace=True)
  gene_mutation_df = gene_mutation_df.reset_index()
  gene_patient_mutations = gene_mutation_df.pivot(index='Hugo_Symbol', columns='patient', values='mutated')

  return gene_patient_mutations.transpose()

def calculate_cox(data, gene):
  data_cox_dict = collections.defaultdict(lambda: np.nan)
  if data[gene].count() > 10:
    try:
      data_cox_dict = analysis.do_cox(data.time,
                                 data.censor,
                                 data[gene])
    except rpy2.rinterface.RRuntimeError as e:
      print 'WARN: skipped', gene, 'due to R error'
  return data_cox_dict

def make_zscores(copy_number, mutation, clinical, outdir):
  clinical_data = util.get_clinical_data(clinical)
  cnv = pd.read_csv(copy_number, index_col=0)
  mutation = prep_mutation_data(mutation, clinical_data)

  cnv_by_patient = cnv.transpose()
  clinical_and_cnv = cnv_by_patient.join(clinical_data, how='inner')

  clinical_and_mutation_patients = list(set(mutation.index).intersection(set(clinical_and_cnv.index)))
  clinical_and_cnv_with_mutations = clinical_and_cnv.loc[clinical_and_mutation_patients]

  cancer_type = util.get_cancer_type(copy_number)
  outfile = os.path.join(outdir, cancer_type + '.cnv_with_mutation_zscores.csv')
  formatstring = '{0}, {1}, {2}, {3}, {4}, {5}, {6}\n'

  with open(outfile, 'w') as out:
    out.write('gene,mutated zscore,mutated pvalue,mutated patients,non-mutated zscore, non-mutated pvalue, non-mutated patients\n')
    for gene in clinical_and_cnv_with_mutations:
      if gene not in ('time', 'censor'): # skip metadata
        clinical_gene = clinical_and_cnv_with_mutations[[gene, 'time', 'censor']]

        if gene in mutation:
          mutations_for_gene = mutation[gene].rename('mutation')
          with_mutation = clinical_gene.join(mutations_for_gene.dropna(), how='inner')
          without_mutation = clinical_gene.join(mutations_for_gene[mutations_for_gene != 1], how='inner')
        else:
          with_mutation = pd.DataFrame({gene: []})
          without_mutation = clinical_gene

        without_mutation_cox_dict = calculate_cox(without_mutation, gene)
        with_mutation_cox_dict = calculate_cox(with_mutation, gene)
        out.write(formatstring.format(gene,
          with_mutation_cox_dict['z'], with_mutation_cox_dict['p'], with_mutation_cox_dict['n'],
          without_mutation_cox_dict['z'], without_mutation_cox_dict['p'], without_mutation_cox_dict['n']))

def multiprocess_zscores(args):
  copy_number = args[0]
  mutation = args[1]
  clinical = args[2]
  outdir = args[3]
  make_zscores(copy_number, mutation, clinical, outdir)

def main(argv=None):
  cnv_dir, mutation_dir, clinical_dir, outdir = get_options()
  cnv_files = os.listdir(cnv_dir)
  cnv_files = util.remove_extraneous_files(cnv_files)
  cnv_files = [os.path.join(cnv_dir, i) for i in cnv_files]

  zscore_inputs = []
  for cnv in cnv_files:
    cancer_type = util.get_cancer_type(cnv)
    mutation = glob.glob(os.path.join(mutation_dir, cancer_type + '*'))[0]
    clinical = glob.glob(os.path.join(clinical_dir, '*' + cancer_type + '*'))[0]
    zscore_inputs.append([cnv, mutation, clinical, outdir])

  p = Pool(4)
  p.map(multiprocess_zscores, zscore_inputs)

if __name__ == "__main__":
  main()


text:
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD.
# See https://nomad-lab.eu for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
import logging
import json
import numpy as np

from nomad.units import ureg
from nomad.parsing.parser import FairdiParser
from nomad.datamodel.metainfo.simulation.run import Run, Program
from nomad.datamodel.metainfo.workflow import (
    Workflow, Elastic, EquationOfState, EOSFit, Thermodynamics, Stability, Decomposition,
    Phonon)
from nomad.datamodel.metainfo.simulation.system import System, Atoms
from nomad.datamodel.metainfo.simulation.method import (
    Method, DFT, Electronic, XCFunctional, Functional, BasisSet, BasisSetCellDependent)
from nomad.datamodel.metainfo.simulation.calculation import (
    Calculation, Dos, DosValues, BandStructure, BandEnergies)
from mpparser.metainfo.mp import Composition, Symmetry

class MPParser(FairdiParser):
    def __init__(self):
        super().__init__(
            name='parsers/mp', code_name='MaterialsProject',
            code_homepage='https://materialsproject.org',
            mainfile_mime_re=r'(application/json)|(text/.*)',
            mainfile_name_re=r'.*mp.+materials\.json',
            mainfile_contents_re=(r'"pymatgen_version":'))

    def init_parser(self):
        try:
            self.data = json.load(open(self.filepath))
        except Exception:
            self.logger.error('Failed to load json file.')

    def parse_elastic(self, source):
        sec_workflow = self.archive.m_create(Workflow)
        sec_workflow.type = 'elastic'
        sec_elastic = sec_workflow.m_create(Elastic)
        sec_elastic.energy_stress_calculator = 'VASP'
        sec_elastic.calculation_method = 'stress'
        source = source.get('elasticity', source)
        sec_elastic.elastic_constants_order = source.get('order', 2)

        deformations = source.get('deformations')
        if deformations is not None:
            sec_elastic.n_deformations = len(deformations)

        elastic_tensor = source.get('elastic_tensor')
        if elastic_tensor is not None:
            sec_elastic.elastic_constants_matrix_second_order = elastic_tensor * ureg.GPa

        compliance_tensor = source.get('compliance_tensor')
        if compliance_tensor is not None:
            sec_elastic.compliance_matrix_second_order = compliance_tensor * (1 / ureg.GPa)

        if source.get('g_reuss') is not None:
            sec_elastic.shear_modulus_reuss = source['g_reuss'] * ureg.GPa
        if source.get('g_voigt') is not None:
            sec_elastic.shear_modulus_voigt = source['g_voigt'] * ureg.GPa
        if source.get('g_vrh') is not None:
            sec_elastic.shear_modulus_hill = source['g_vrh'] * ureg.GPa
        if source.get('homogeneous_poisson') is not None:
            sec_elastic.poisson_ratio_hill = source['homogeneous_poisson']
        if source.get('k_reuss') is not None:
            sec_elastic.bulk_modulus_reuss = source['k_reuss'] * ureg.GPa
        if source.get('k_voigt') is not None:
            sec_elastic.bulk_modulus_voigt = source['k_voigt'] * ureg.GPa
        if source.get('k_vrh') is not None:
            sec_elastic.bulk_modulus_hill = source['k_vrh'] * ureg.GPa

    def parse_eos(self, source):
        sec_workflow = self.archive.m_create(Workflow)
        sec_workflow.type = 'equation_of_state'
        sec_eos = sec_workflow.m_create(EquationOfState)
        if source.get('volumes') is not None:
            sec_eos.volumes = source['volumes'] * ureg.angstrom ** 3
        if source.get('energies') is not None:
            sec_eos.energies = source['energies'] * ureg.eV
        for fit_function, result in source.get('eos', {}).items():
            sec_eos_fit = sec_eos.m_create(EOSFit)
            sec_eos_fit.function_name = fit_function
            if result.get('B') is not None:
                sec_eos_fit.bulk_modulus = result['B'] * ureg.eV / ureg.angstrom ** 3
            if result.get('C') is not None:
                sec_eos_fit.bulk_modulus_derivative = result['C']
            if result.get('E0') is not None:
                sec_eos_fit.equilibrium_energy = result['E0'] * ureg.eV
            if result.get('V0') is not None:
                sec_eos_fit.equilibrium_volume = result['V0'] * ureg.angstrom ** 3
            if result.get('eos_energies') is not None:
                sec_eos_fit.fitted_energies = result['eos_energies'] * ureg.eV

    def parse_thermo(self, data):
        sec_workflow = self.archive.m_create(Workflow)
        sec_workflow.type = 'thermodynamics'
        sec_thermo = sec_workflow.m_create(Thermodynamics)
        sec_stability = sec_thermo.m_create(Stability)
        sec_stability.formation_energy = data.get(
            'formation_energy_per_atom', 0) * data.get('nsites', 1) * ureg.eV
        sec_stability.delta_formation_energy = data.get('energy_above_hull', 0) * ureg.eV
        sec_stability.is_stable = data.get('is_stable')
        if data.get('decomposes_to') is not None:
            for system in data.get('decomposes_to'):
                sec_decomposition = sec_stability.m_create(Decomposition)
                sec_decomposition.formula = system.get('formula')
                sec_decomposition.fraction = system.get('amount')

    def parse_phonon(self, data):
        sec_workflow = self.archive.m_create(Workflow)
        sec_workflow.type = 'phonon'
        sec_phonon = sec_workflow.m_create(Phonon)
        # TODO is vasp always mp calculator?
        sec_phonon.force_calculator = 'vasp'

        calculations = self.archive.run[-1].calculation
        calc = calculations[-1] if calculations else self.archive.run[-1].m_create(Calculation)

        if data.get('ph_dos') is not None:
            sec_dos = calc.m_create(Dos, Calculation.dos_phonon)
            sec_dos.energies = data['ph_dos']['frequencies'] * ureg.THz * ureg.h
            dos = data['ph_dos']['densities'] * (1 / (ureg.THz * ureg.h))
            sec_dos.total.append(DosValues(value=dos))

        if data.get('ph_bs') is not None:
            sec_phonon.with_non_analytic_correction = data['ph_bs'].get('has_nac')
            sec_bs = calc.m_create(BandStructure, Calculation.band_structure_phonon)
            bands = np.transpose(data['ph_bs']['bands'])
            qpoints = data['ph_bs']['qpoints']
            labels = data['ph_bs']['labels_dict']
            hisym_qpts = list(labels.values())
            labels = list(labels.keys())
            endpoints = []
            for i, qpoint in enumerate(qpoints):
                if qpoint in hisym_qpts:
                    endpoints.append(i)
                if len(endpoints) < 2:
                    continue
                sec_segment = sec_bs.m_create(BandEnergies)
                energies = bands[endpoints[0]: endpoints[1] + 1]
                sec_segment.energies = np.reshape(energies, (1, *np.shape(energies))) * ureg.THz * ureg.h
                sec_segment.kpoints = qpoints[endpoints[0]: endpoints[1] + 1]
                sec_segment.endpoints_labels = [labels[hisym_qpts.index(qpoints[i])] for i in endpoints]
                endpoints = []

        calc.system_ref = self.archive.run[-1].system[0]

        # TODO add eigendisplacements

    def parse_tasks(self, data):
        if len(data['calcs_reversed']) == 0:
            return

        xc_func_mapping = {
            'PAW_PBE': ['GGA_X_PBE', 'GGA_C_PBE']
        }

        sec_method = self.archive.run[-1].m_create(Method)
        sec_xc_functional = XCFunctional()
        for potcar_type in data['calcs_reversed'][0].get('input', {}).get('potcar_type', []):
            for xc_functional in xc_func_mapping.get(potcar_type, []):
                if '_X_' in xc_functional or xc_functional.endswith('_X'):
                    sec_xc_functional.exchange.append(Functional(name=xc_functional))
                elif '_C_' in xc_functional or xc_functional.endswith('_C'):
                    sec_xc_functional.correlation.append(Functional(name=xc_functional))
                elif 'HYB' in xc_functional:
                    sec_xc_functional.hybrid.append(Functional(name=xc_functional))
                else:
                    sec_xc_functional.contributions.append(Functional(name=xc_functional))

        sec_method.dft = DFT(xc_functional=sec_xc_functional)
        sec_method.electronic = Electronic(method="DFT")

        encut = data['calcs_reversed'][0].get('input', {}).get('incar', {}).get('ENCUT')
        prec = data['calcs_reversed'][0].get('input', {}).get('incar', {}).get('PREC')
        if encut is not None and prec is not None:
            sec_basis = sec_method.m_create(BasisSet)
            sec_basis.type = 'plane waves'
            sec_basis_set_cell_dependent = sec_basis.m_create(BasisSetCellDependent)
            sec_basis_set_cell_dependent.kind = 'plane waves'
            prec = 1.3 if 'acc' in prec else 1.0
            sec_basis_set_cell_dependent.planewave_cutoff = encut * prec * ureg.eV

        self.archive.run[-1].calculation[0].method_ref = sec_method

    def parse(self, filepath, archive, logger):
        self.filepath = os.path.abspath(filepath)
        self.archive = archive
        self.logger = logger if logger is not None else logging.getLogger(__name__)
        self.maindir = os.path.dirname(self.filepath)

        self.init_parser()

        sec_run = archive.m_create(Run)
        sec_run.program = Program(name='MaterialsProject', version="1.0.0")

        #  TODO system should be referenced
        structure = self.data.get('structure')
        if structure is not None:
            labels = [site['label'] for site in structure.get('sites')]
            positions = [site['xyz'] for site in structure.get('sites')]
            cell = structure.get('lattice', {}).get('matrix')
            sec_system = sec_run.m_create(System)
            sec_atoms = sec_system.m_create(Atoms)
            if cell is not None:
                sec_atoms.lattice_vectors = cell * ureg.angstrom
                sec_atoms.periodic = [True, True, True]
            if positions:
                sec_atoms.positions = positions * ureg.angstrom
            if labels:
                sec_atoms.labels = labels

        for key, val in self.data.get('composition', {}).items():
            sec_system.x_mp_composition.append(
                Composition(x_mp_label=key, x_mp_value=val))

        for key, val in self.data.get('composition_reduced', {}).items():
            sec_system.x_mp_composition_reduced.append(
                Composition(x_mp_label=key, x_mp_value=val))

        symmetry = self.data.get('symmetry')
        if symmetry is not None:
            sec_symmetry = sec_system.m_create(Symmetry)
            for key, val in symmetry.items():
                try:
                    setattr(sec_symmetry, 'x_mp_%s' % key, val)
                except Exception:
                    pass

        # misc
        sec_system.x_mp_elements = self.data.get('elements', [])
        for key, val in self.data.items():
            try:
                setattr(sec_system, 'x_mp_%s' % key, val)
            except Exception:
                pass

        # temporary fix to go through workflow normalization
        sec_calc = sec_run.m_create(Calculation)
        sec_calc.system_ref = sec_system

        # TODO should we use the MP api for workflow results?
        workflow_files = [f for f in os.listdir(
            self.maindir) if f.endswith('.json') and f != os.path.basename(self.filepath)]
        for filename in workflow_files:
            try:
                data = json.load(open(os.path.join(self.maindir, filename)))
            except Exception:
                continue
            # make sure data matches that of system
            # TODO maybe better to simply compare filename prefix so no need to load data
            if data.get('material_id', data.get('task_id')) != self.data.get('material_id'):
                continue

            if 'elasticity' in data:
                self.parse_elastic(data)
            if 'eos' in data:
                self.parse_eos(data)
            if 'ph_bs' in data or 'ph_dos' in data:
                self.parse_phonon(data)
            if 'property_name' in data and data.get('property_name') == 'thermo':
                self.parse_thermo(data)
            if 'calcs_reversed' in data:
                self.parse_tasks(data)

max_stars_repo_name: nomad-coe/nomad-parser-mp

max_issues_repo_name: nomad-coe/nomad-parser-mp

max_forks_repo_name: nomad-coe/nomad-parser-mp


text:
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot as plt
import json

sns.set_style('whitegrid')
import os

def concatenate(dataframe):
    pd.concat([dataframe['epochs'], dataframe['causal_train']], axis=1)

path = os.path.join(os.getcwd(), 'Results')
models = os.listdir(path)
experiment = 'small_experiment'
model_hp = {}
for model in models:
    if model.startswith(experiment):
        try:
            with open(os.path.join(os.getcwd(), 'Jsons', model + '.json'), 'r') as f:
                hp = json.load(f)
                model_hp[model] = hp
        except FileNotFoundError:
            pass

dfs = []
for (dirpath, dirnames, filenames) in os.walk(path):
    for file in filenames:
        if file.endswith('.csv') and file.startswith(experiment):
            df = pd.read_csv(os.path.join(dirpath, file), skiprows=[1])
            if len(df) > 6:
                df = df.rename(
                    columns={k: k + '_train' if i % 2 == 1 else k.replace('.1', '') + '_validation' for (i, k) in
                             enumerate(df.columns)})
                df = df.rename(columns={'Unnamed: 0_validation': 'epochs'})
                name = file.replace('_accuracy.csv', '')
                minidx = df.total_validation.argmin()
                model_hp[name]['max_validation_accuracy'] = 1 - df.total_validation.iat[minidx]
                model_hp[name]['first_total_validation_error'] = 1 - df.total_validation.iat[0]
                model_hp[name]['epoch'] = df.epochs.iloc[minidx]
                model_hp[name]['train_error'] = df.total_train.iat[minidx]
                model_hp[name]['symmetry_success'] = df.symmetry_validation.iat[minidx]
                model_hp[name]['success_average'] = (df.symmetry_validation.iat[minidx] + 1 - df.total_validation.iat[minidx]) / 2
                # concatenate(df)
                # dfs.append(df)
            else:
                del model_hp[file.replace('_accuracy.csv', '')]

hyper_df = pd.DataFrame(model_hp).T
# hyper_df.reset_index(level=0, inplace=True)
hyper_df.sort_values(by=['symmetry_success'], inplace=True)
sns.scatterplot(x='max_validation_accuracy', y='symmetry_success', data=hyper_df, hue='learning_rate')
plt.savefig(experiment + '_accuracy_vs_symmetry_lr')
plt.close()
sns.scatterplot(x='max_validation_accuracy', y='symmetry_success', data=hyper_df, hue='dropout_rate')
plt.savefig(experiment + 'accuracy_vs_symmetry_dropout')
plt.close()
sns.scatterplot(x='max_validation_accuracy', y='symmetry_success', data=hyper_df, hue='n_hiddens')
plt.savefig(experiment + 'accuracy_vs_symmetry_hidden_neurons')
plt.close()

with open('best_symmetry.json', 'w') as f:
    json.dump(hyper_df.sort_values(by=['symmetry_success']).tail().save_model_name.tolist(), f)
with open('best_validation_accuracy.json', 'w') as f:
    json.dump(hyper_df.sort_values(by=['max_validation_accuracy']).tail().save_model_name.tolist(), f)
with open('best_average_accuracy.json', 'w') as f:
    json.dump(hyper_df.sort_values(by=['success_average']).tail().save_model_name.tolist(), f)
a = 0

max_stars_repo_name: jonatanbarkan/CausalInference

max_issues_repo_name: jonatanbarkan/CausalInference

max_forks_repo_name: jonatanbarkan/CausalInference


text:
import numpy as np
import random
import collections

def observeGame(model, env, epsilon=1, render=False):
     # Initialise environment and render
    state = env.reset()
    score = 0
    memory = collections.deque()
    done = False
    while not done:
        if render:
            env.render()
        # Pick action allowing random action
        # to allow the model to learn new moves
        random_number = random.uniform(0,1)
        if random_number < epsilon:
            action = random.choice(range(env.action_space.n))
        else:
            action = np.argmax(model.predict( np.expand_dims(state, axis=0) ))
        # Carry action and save <s,a,r,s'>
        new_state, reward, done, info = env.step(action)
        memory.append([state,action,reward,new_state,done])
        score += reward
        state = new_state

    return memory, score

def learnFromMemory(model, env, memory, gamma=0.9, batch_size=5000):
    # Take a random sample of the memory
    # to train the model
    if len(memory) < batch_size:
        recollection = memory
    else:
        recollection = random.sample(memory,batch_size)
    batch = []
    targets = []
    for event in recollection:
        state     = event[0]
        action    = event[1]
        reward    = event[2]
        new_state = event[3]
        done      = event[4]
        # Define the targets for the action
        # using the model prediction for all actions
        # but the one that was choosen which is updated
        # with Bellman's equation (if game is note done).
        target = model.predict( np.expand_dims(state, axis=0) )[0]
        if done:
            target[action] = reward
        else:
            target[action] = reward + gamma * np.max(model.predict( np.expand_dims(new_state, axis=0) )[0])
        batch.append(state)
        targets.append(target)

    trainModel(X=batch,y=targets,model=model)

    return model

def trainModel(X, y, model=False):
    model.fit({'input': X}, {'targets': y}, n_epoch=5, snapshot_step=500, show_metric=True, run_id='openai_learning')

max_stars_repo_name: tdelubac/DeepGamer

max_issues_repo_name: tdelubac/DeepGamer

max_forks_repo_name: tdelubac/DeepGamer


text:
"""
 Copyright (C) 2018-2020 Intel Corporation

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import unittest

import numpy as np

from extensions.front.tf.identityN_to_identity import IdentityN_to_Identity
from mo.utils.ir_engine.compare_graphs import compare_graphs
from mo.utils.unittest.graph import result, regular_op_with_shaped_data, \
    regular_op_with_empty_data, build_graph, connect, empty_data

nodes = {
    **regular_op_with_shaped_data('placeholder_0', [1, 227, 227, 3], {'type': 'Parameter'}),
    **regular_op_with_shaped_data('placeholder_1', [1, 227, 227, 3], {'type': 'Parameter'}),

    **regular_op_with_empty_data('identityN', {'op': 'IdentityN', 'type': None, 'data_types': [np.int32, np.float],
                                               'name': 'my_identity'}),
    **empty_data('identityN_1_d'),
    **regular_op_with_empty_data('identity0', {'op': 'Identity', 'type': None, 'data_type': np.int32,
                                               'name': 'my_identity/0_port'}),
    **regular_op_with_empty_data('identity1', {'op': 'Identity', 'type': None, 'data_type': np.float,
                                               'name': 'my_identity/1_port'}),

    **result('output0'),
    **result('output1'),
}

class TestIdentityN(unittest.TestCase):
    def test_identityN(self):
        graph = build_graph(nodes, [
            *connect('placeholder_0', '0:identityN'),
            *connect('placeholder_1', '1:identityN'),
            *connect('identityN:0', 'output0'),
            ('identityN', 'identityN_1_d', {'out': 1}),
            ('identityN_1_d', 'output1', {'out': 1}),
        ], nodes_with_edges_only=True)

        IdentityN_to_Identity().find_and_replace_pattern(graph)

        graph_ref = build_graph(nodes, [
            *connect('placeholder_0', 'identity0'),
            *connect('placeholder_1', 'identity1'),
            *connect('identity0', 'output0'),
            *connect('identity1', 'output1'),
        ], nodes_with_edges_only=True)

        (flag, resp) = compare_graphs(graph, graph_ref, 'output0', check_op_attrs=True)
        self.assertTrue(flag, resp)

    def test_identityN_unused_ports(self):
            graph = build_graph(nodes, [
                *connect('placeholder_0', '0:identityN'),
                *connect('placeholder_1', '1:identityN'),
                *connect('identityN:0', 'output0'),
            ], nodes_with_edges_only=True)

            IdentityN_to_Identity().find_and_replace_pattern(graph)

            graph_ref = build_graph(nodes, [
                *connect('placeholder_0', 'identity0'),
                *connect('identity0', 'output0'),
            ], nodes_with_edges_only=True)

            (flag, resp) = compare_graphs(graph, graph_ref, 'output0', check_op_attrs=True)
            self.assertTrue(flag, resp)

max_stars_repo_name: anton-potapov/openvino

max_issues_repo_name: anton-potapov/openvino

max_forks_repo_name: mmakridi/openvino


text:
#!/usr/bin/env python3

#### PYTHON IMPORTS ################################################################################
import csv
import os
from collections import OrderedDict

#### THIRD-PARTY IMPORTS ###########################################################################
import nltk
import numpy as np

#### PACKAGE IMPORTS ###############################################################################

#### GLOBALS #######################################################################################

#### FUNCTIONS #####################################################################################
def _readDocuments(data_path):
    """
    Read documents from the given path.

    GIVEN:
      data_path (str)   path to read data from

    RETURN:
      documents (list)  list of lists, where each sublist contains a CVE ID, cluster number, and
                        the unique words in the description
    """
    documents = list()
    with open(data_path, newline="") as f:
        csv_reader = csv.reader(f, delimiter=",", quotechar="\"", quoting=csv.QUOTE_MINIMAL)

        # Skip header row
        next(csv_reader)

        for row in csv_reader:
            new_row = list()
            new_row.append(row[0])              # CVE ID
            new_row.append(row[1])              # Cluster Number
            words_set = list(set(row[2].split(" ")))
            new_row.append(words_set)           # Unique Words
            documents.append(new_row)

    return documents

def _readDocuments2(data_path):
    documents = list()
    with open(data_path, newline="") as f:
        csv_reader = csv.reader(f, delimiter=",", quotechar="\"", quoting=csv.QUOTE_MINIMAL)

        # Skip header row
        next(csv_reader)

        for row in csv_reader:
            new_row = list()
            new_row.append(row[0])
            new_row.append(row[1])
            new_row.append(row[2])
            documents.append(new_row)

    return documents

def _getClusters(documents):
    """
    Aggregate all of the words from the documents in each cluster together.

    GIVEN:
      documents (list)  list of lists, where each sublist contains a CVE ID, cluster number, and
                        the unique words in the description

    RETURN:
      clusters (dict)   OrderedDict where each key is a cluster number and each value is a list of
                        words from the document descriptions
    """
    clusters = OrderedDict()
    for doc in documents:
        if doc[1] in clusters.keys():
            # Append words from each document to the cluster
            clusters[doc[1]].extend(doc[2])
        else:
            clusters.update({doc[1]: doc[2]})

    new_clusters = OrderedDict()
    for k, v in clusters.items():
        new_v = list()
        for word in v:
            new_v.append(word)

        new_clusters.update({k: new_v})

    return new_clusters

def _getClusters2(documents):
    """
    Read the clusters (and documents) from disk. Compute bigrams and frequencies for all documents
    within a cluster.

    GIVEN:
      documents (list)  list of lists, where each sublist contains a CVE ID, cluster number, and
                        the unique words in the description

    RETURN:
      clusters (dict)   OrderedDict where each key is a cluster number and each value is a nested
                        list of bigrams and their frequencies
    """
    clusters = OrderedDict()
    for doc in documents:
        bigrams = list(nltk.bigrams(nltk.word_tokenize(doc[2])))
        if doc[1] in clusters.keys():
            clusters[doc[1]].extend(bigrams)
        else:
            clusters.update({doc[1]: bigrams})

    new_clusters = OrderedDict()
    for k, v in clusters.items():
        new_v = nltk.FreqDist(v)
        new_clusters.update({k: new_v})

    return new_clusters

def _getNumDocsPerCluster(documents):
    """
    Count how many documents are assigned to each cluster.

    GIVEN:
      documents (list)  list of lists, where each sublist contains a CVE ID, cluster number, and
                        the unique words from the description

    RETURN:
      clusters (dict)   dictionary where each key is a cluster number and each value is the number
                        of documents in that cluster
    """
    clusters = dict()
    for doc in documents:
        if doc[1] in clusters.keys():
            clusters[doc[1]] += 1
        else:
            clusters.update({doc[1]: 1})

    return clusters

def _getWordCounts(words):
    """
    Helper function for _getClusterWords(...). Compute frequencies for words.

    GIVEN:
      words (list)  words in a cluster

    RETURN:
      d (dict)      OrderedDict where each key is a word and each value is its frequency
    """
    tokens, counts = np.unique(words, return_counts=True)
    d = dict(zip(tokens, counts))
    d = OrderedDict({k: v for k, v in sorted(d.items(), key=lambda item: item[1])})

    return d

def _getTopicWords(word_counts, num_docs):
    """
    Helper function for _getClusterWords(...). Select the words with high enough frequencies and
    put them in a list.

    GIVEN:
      word_counts (dict)    OrderedDict where each key is a word and each value is its frequency
      num_docs (int)        the number of documents in the cluster

    RETURN:
      topic_words (list)    list of words with their frequencies
    """
    topic_words = list()
    for word, count in word_counts.items():
        if count >= num_docs / 2:
            topic_words.append("{} ({})".format(word, count))

    return topic_words

def _getClusterWords(clusters, docs_per_cluster):
    """
    Compute word frequencies for the words in a cluster.

    GIVEN:
      clusters (dict)           OrderedDict where each key is a cluster number and each value is a
                                list of words from the document descriptions
      docs_per_cluster (dict)   dictionary where each key is a cluster number and each value is the
                                number of documents in that cluster

    RETURN:
      cluster_words (dict)      OrderedDict where each key is a cluster number and each value is
                                a list of words and their frequencies
    """
    cluster_words = OrderedDict()
    for k, v in clusters.items():
        num_docs = docs_per_cluster[k]
        word_counts = _getWordCounts(v)
        topic_words = _getTopicWords(word_counts, num_docs)
        cluster_words.update({k: topic_words})

    return cluster_words

def _printClusterWords(cluster_words):
    """
    Helper function to pretty print the clusters and their word frequencies.

    GIVEN:
      cluster_words (dict)  OrderedDict where each key is a cluster number and each value is a
                            list of words and their frequencies

    """
    num_clusters = len(cluster_words.keys())
    for i in range(num_clusters):
        words = cluster_words[str(i)]
        print("{}: {}".format(i, words))

def _printClusterBigrams(clusters):
    """
    Helper function to pretty print the clusters and their most frequent bigrams.

    GIVEN:
      clusters (dict)   OrderedDict where each key is a cluster number and each value is a nested
                        list of bigrams and their frequencies
    """
    num_clusters = len(clusters.keys())
    for i in range(num_clusters):
        cluster_bigrams = clusters[str(i)]
        temp = cluster_bigrams.most_common(15)
        bigrams = ""
        for b in temp:
            bigram = "{} {} ({})".format(
                b[0][0],
                b[0][1],
                b[1]
            )
            bigrams = bigrams + bigram + ", "

        print("{}: {}".format(i, bigrams))

#### MAIN ##########################################################################################
def label(args, RESULTS_PATH):
    print("Most Frequent Words:")
    documents = _readDocuments(args.clusters)
    docs_per_cluster = _getNumDocsPerCluster(documents)
    clusters = _getClusters(documents)
    cluster_words = _getClusterWords(clusters, docs_per_cluster)
    _printClusterWords(cluster_words)

    print("\nMost Frequent Bigrams:")
    documents = _readDocuments2(args.clusters)
    clusters = _getClusters2(documents)
    _printClusterBigrams(clusters)


text:
'''
 Based on the following tutorial:
   http://docs.opencv.org/3.0-beta/doc/py_tutorials/py_video/py_lucas_kanade/py_lucas_kanade.html
'''

import numpy as np
import cv2

# Start the webcam
#cap = cv2.VideoCapture(0)
cap = cv2.VideoCapture('output.mp4')

frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
source_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
source_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

print("{0} frames {1}x{2}".format(frames,source_width,source_height))

nframes = 10 # how many frames we want to analyze
target_width = source_width // 4
target_height = source_height // 4
skip_frames = frames / nframes - 1

def convert(frame):
    return cv2.resize(frame,(target_width,target_height))

# Take the first frame and convert it to gray
ret, frame = cap.read()
frm=convert(frame)
gray = cv2.cvtColor(frm, cv2.COLOR_BGR2GRAY)
# Create the HSV color image
hsvImg = np.zeros_like(frm)
hsvImg[..., 1] = 255

# Play until the user decides to stop
for i in range(frames-1):
    # Save the previous frame data
    previousGray = gray
    ret, frame = cap.read()
    frm = convert(frame)
    gray = cv2.cvtColor(frm, cv2.COLOR_BGR2GRAY)

    # Calculate the dense optical flow
    flow = cv2.calcOpticalFlowFarneback(previousGray, gray, None, 0.5, 3, 15, 3, 5, 1.2, 0)
    vec = np.average(flow,axis=(0,1))
    mask = flow[:,:,0]==0 and flow[:,:,1]==0
    flow = flow-vec
    flow[mask]=0

    # Obtain the flow magnitude and direction angle
    mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])

    # Update the color image
    hsvImg[..., 0] = 0.5 * ang * 180 / np.pi
    hsvImg[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
    rgbImg = cv2.cvtColor(hsvImg, cv2.COLOR_HSV2BGR)

    #cv2.imwrite('c:/temp/opt/orig{}.png'.format(i),frm)
    #cv2.imwrite('c:/temp/opt/opt{}.png'.format(i), rgbImg)

    # Display the resulting frame
    #cv2.imshow('dense optical flow', np.hstack((frame, rgbImg)))
    cv2.imshow('dense optical flow',rgbImg)

    k = cv2.waitKey(30) & 0xff
    if k == 27:
        break
    #j=skip_frames
    #while ret and (j>0):
    #    ret, _ = cap.read()
    #    j-=1

# When everything is done, release the capture and close all windows
cap.release()
cv2.destroyAllWindows()


text:
__author__ = "Tomasz Rybotycki"

import abc

from numpy import ndarray

class NetworkSimulationStrategy(abc.ABC):
    @classmethod
    def __subclasshook__(cls, subclass):
        return hasattr(subclass, "simulate") and callable(subclass.simulate)

    @abc.abstractmethod
    def simulate(self, input_state: ndarray) -> ndarray:
        raise NotImplementedError

max_stars_repo_name: Tomev/BoSS

max_issues_repo_name: Tomev/BoSS

max_forks_repo_name: Tomev/BoSS

reason: from numpy


text:
"""Synthetic observation data testing.
"""

import copy
import unittest
import numpy

import tigernet
from .network_objects import network_empirical_simplified_wcm

import platform

os = platform.platform()[:7].lower()
if os == "windows":
    WINDOWS = True
    DECIMAL = -1
else:
    WINDOWS = False
    DECIMAL = 1
# WINDOWS = False
# DECIMAL = 1

####################################################################################
############################# ORIG-XXXX--Segments ##################################
####################################################################################

class TestEmpiricalObservationsOrigToXXXXSegments(unittest.TestCase):
    def setUp(self):
        self.network = copy.deepcopy(network_empirical_simplified_wcm)

        # empirical observations
        obs = tigernet.testing_data("CensusBlocks_Leon_FL_2010")

        # associate observations with the network
        args = self.network, obs.copy()
        kwargs = {"df_name": "obs1", "df_key": "GEOID"}
        self.net_obs = tigernet.Observations(*args, **kwargs)

    @unittest.skipIf(WINDOWS, "Skipping Windows due to precision issues.")
    def test_net_no_snap(self):
        known_mtx = numpy.array(
            [
                [0.0, 3285.69020577, 3883.24256199, 3581.0882225],
                [3285.69020577, 0.0, 645.01865974, 3264.87455121],
                [3883.24256199, 645.01865974, 0.0, 3909.89321095],
                [3581.0882225, 3264.87455121, 3909.89321095, 0.0],
            ]
        )
        args = copy.deepcopy(self.net_obs), copy.deepcopy(self.network)
        kwargs = {
            "destination_observations": None,
            "snap_dist": False,
            "distance_type": "network",
        }
        observed_mtx = tigernet.obs2obs_cost_matrix(*args, **kwargs)
        numpy.testing.assert_array_almost_equal(
            observed_mtx[:4, :4], known_mtx, decimal=DECIMAL
        )

        known_mtx_sum = 22099816.17479256
        observed_mtx_sum = observed_mtx.sum()
        self.assertAlmostEqual(observed_mtx_sum, known_mtx_sum, delta=1)

    @unittest.skipIf(WINDOWS, "Skipping Windows due to precision issues.")
    def test_net_snap(self):
        known_mtx = numpy.array(
            [
                [0.0, 3389.37545944, 3975.65875829, 3687.64355986],
                [3389.37545944, 0.0, 727.84658824, 3361.84162077],
                [3975.65875829, 727.84658824, 0.0, 3995.59122314],
                [3687.64355986, 3361.84162077, 3995.59122314, 0.0],
            ]
        )
        args = copy.deepcopy(self.net_obs), copy.deepcopy(self.network)
        kwargs = {
            "destination_observations": None,
            "snap_dist": True,
            "distance_type": "network",
        }
        observed_mtx = tigernet.obs2obs_cost_matrix(*args, **kwargs)
        numpy.testing.assert_array_almost_equal(
            observed_mtx[:4, :4], known_mtx, decimal=DECIMAL
        )

        known_mtx_sum = 23230504.28050229
        observed_mtx_sum = observed_mtx.sum()
        self.assertAlmostEqual(observed_mtx_sum, known_mtx_sum, delta=1)

####################################################################################
############################# ORIG-XXXX--Nodes #####################################
####################################################################################

class TestEmpiricalObservationsOrigToXXXXNodes(unittest.TestCase):
    def setUp(self):
        self.network = copy.deepcopy(network_empirical_simplified_wcm)

        # empirical observations
        obs = tigernet.testing_data("CensusBlocks_Leon_FL_2010")

        # associate observations with the network
        args = self.network, obs.copy()
        kwargs = {"df_name": "obs1", "df_key": "GEOID", "snap_to": "nodes"}
        self.net_obs = tigernet.Observations(*args, **kwargs)

    @unittest.skipIf(WINDOWS, "Skipping Windows due to precision issues.")
    def test_net_no_snap(self):
        known_mtx = numpy.array(
            [
                [0.0, 3317.88793311, 4003.17771763, 3387.79485659],
                [3317.88793311, 0.0, 940.71263515, 3273.42506072],
                [4003.17771763, 940.71263515, 0.0, 3836.53500068],
                [3387.79485659, 3273.42506072, 3836.53500068, 0.0],
            ]
        )
        args = self.net_obs, self.network
        kwargs = {
            "destination_observations": None,
            "snap_dist": False,
            "distance_type": "network",
        }
        observed_mtx = tigernet.obs2obs_cost_matrix(*args, **kwargs)
        numpy.testing.assert_array_almost_equal(
            observed_mtx[:4, :4], known_mtx, decimal=DECIMAL
        )

        known_mtx_sum = 22077455.792563077
        observed_mtx_sum = observed_mtx.sum()
        self.assertAlmostEqual(observed_mtx_sum, known_mtx_sum, delta=1)

    @unittest.skipIf(WINDOWS, "Skipping Windows due to precision issues.")
    def test_net_snap(self):
        known_mtx = numpy.array(
            [
                [0.0, 3501.0453002, 4171.59260934, 3611.91094868],
                [3501.0453002, 0.0, 1027.01398167, 3415.42760763],
                [4171.59260934, 1027.01398167, 0.0, 3963.79507221],
                [3611.91094868, 3415.42760763, 3963.79507221, 0.0],
            ]
        )
        args = self.net_obs, self.network
        kwargs = {
            "destination_observations": None,
            "snap_dist": True,
            "distance_type": "network",
        }
        observed_mtx = tigernet.obs2obs_cost_matrix(*args, **kwargs)
        numpy.testing.assert_array_almost_equal(
            observed_mtx[:4, :4], known_mtx, decimal=DECIMAL
        )

        known_mtx_sum = 23904433.77183481
        observed_mtx_sum = observed_mtx.sum()
        self.assertAlmostEqual(observed_mtx_sum, known_mtx_sum, delta=1)

####################################################################################
############################# ORIG-DEST--Segments ##################################
####################################################################################

class TestEmpiricalObservationsOrigToDestSegments(unittest.TestCase):
    def setUp(self):
        self.network = copy.deepcopy(network_empirical_simplified_wcm)

        # empirical origins
        obs1 = tigernet.testing_data("CensusBlocks_Leon_FL_2010")
        # empirical destinations
        obs2 = tigernet.testing_data("WeightedParcels_Leon_FL_2010")

        # associate origins with the network
        args = self.network, obs1.copy()
        kwargs = {"df_name": "obs1", "df_key": "GEOID"}
        self.net_obs1 = tigernet.Observations(*args, **kwargs)

        # associate destinations with the network
        args = self.network, obs2.copy()
        kwargs = {"df_name": "obs2", "df_key": "PARCEL_ID"}
        self.net_obs2 = tigernet.Observations(*args, **kwargs)

    @unittest.skipIf(WINDOWS, "Skipping Windows due to precision issues.")
    def test_net_no_snap(self):
        known_mtx = numpy.array(
            [
                [2308.71804518, 1823.15385777, 1629.12550059, 1597.86997059],
                [2412.05369383, 2167.34025136, 2869.67781642, 3774.02731159],
                [3057.07235357, 2604.44689325, 3306.78445831, 4371.5796678],
                [2295.60401708, 5094.9154798, 4900.88712262, 3106.05917224],
            ]
        )
        args = copy.deepcopy(self.net_obs1), copy.deepcopy(self.network)
        kwargs = {
            "destination_observations": self.net_obs2,
            "snap_dist": False,
            "distance_type": "network",
        }
        observed_mtx = tigernet.obs2obs_cost_matrix(*args, **kwargs)
        numpy.testing.assert_array_almost_equal(
            observed_mtx[:4, :4], known_mtx, decimal=DECIMAL
        )

        known_mtx_sum = 469638820.3745194
        observed_mtx_sum = observed_mtx.sum()
        self.assertAlmostEqual(observed_mtx_sum, known_mtx_sum, delta=1)

    @unittest.skipIf(WINDOWS, "Skipping Windows due to precision issues.")
    def test_net_snap(self):
        known_mtx = numpy.array(
            [
                [2396.63085253, 1918.5571568, 1727.08007354, 1702.91268741],
                [2490.37823338, 2253.15528259, 2958.04412157, 3869.4817606],
                [3124.12783575, 2678.99286711, 3383.88170609, 4455.76505945],
                [2376.79864032, 5183.60059472, 4992.12351146, 3204.38370495],
            ]
        )
        args = copy.deepcopy(self.net_obs1), copy.deepcopy(self.network)
        kwargs = {
            "destination_observations": self.net_obs2,
            "snap_dist": True,
            "distance_type": "network",
        }
        observed_mtx = tigernet.obs2obs_cost_matrix(*args, **kwargs)
        numpy.testing.assert_array_almost_equal(
            observed_mtx[:4, :4], known_mtx, decimal=DECIMAL
        )

        known_mtx_sum = 489306860.5320058
        observed_mtx_sum = observed_mtx.sum()
        self.assertAlmostEqual(observed_mtx_sum, known_mtx_sum, delta=1)

####################################################################################
############################# ORIG-DEST--Nodes #####################################
####################################################################################

class TestEmpiricalObservationsOrigToDestNodes(unittest.TestCase):
    def setUp(self):
        self.network = copy.deepcopy(network_empirical_simplified_wcm)

        # empirical origins
        obs1 = tigernet.testing_data("CensusBlocks_Leon_FL_2010")
        # empirical destinations
        obs2 = tigernet.testing_data("WeightedParcels_Leon_FL_2010")

        # associate origins with the network
        args = self.network, obs1.copy()
        kwargs = {"df_name": "obs1", "df_key": "GEOID", "snap_to": "nodes"}
        self.net_obs1 = tigernet.Observations(*args, **kwargs)

        # associate destinations with the network
        args = self.network, obs2.copy()
        kwargs = {"df_name": "obs2", "df_key": "PARCEL_ID", "snap_to": "nodes"}
        self.net_obs2 = tigernet.Observations(*args, **kwargs)

    @unittest.skipIf(WINDOWS, "Skipping Windows due to precision issues.")
    def test_net_no_snap(self):
        known_mtx = numpy.array(
            [
                [2064.59726946, 1899.48030691, 1749.06065623, 1567.86895727],
                [2562.49978182, 2285.24409928, 2943.97295784, 3776.2240256],
                [3125.60972178, 2648.05559975, 3306.78445831, 4461.51381012],
                [2062.69976958, 4977.94856303, 4827.52891235, 3122.6351043],
            ]
        )
        args = self.net_obs1, self.network
        kwargs = {
            "destination_observations": self.net_obs2,
            "snap_dist": False,
            "distance_type": "network",
        }
        observed_mtx = tigernet.obs2obs_cost_matrix(*args, **kwargs)
        numpy.testing.assert_array_almost_equal(
            observed_mtx[:4, :4], known_mtx, decimal=DECIMAL
        )

        known_shape = (92, 1969)
        observed_shape = observed_mtx.shape
        self.assertEqual(observed_shape, observed_shape)

        known_mtx_sum = 467794809.4908291
        observed_mtx_sum = observed_mtx.sum()
        self.assertAlmostEqual(observed_mtx_sum, known_mtx_sum, delta=1)

    @unittest.skipIf(WINDOWS, "Skipping Windows due to precision issues.")
    def test_net_snap(self):
        known_mtx = numpy.array(
            [
                [2336.13968088, 2090.68486173, 1923.01392459, 1802.63808338],
                [2751.92864805, 2394.33510891, 3035.812681, 3928.87960652],
                [3300.29611263, 2742.40413399, 3383.88170609, 4599.42691566],
                [2293.08736082, 5127.99829766, 4960.32736052, 3316.24941022],
            ]
        )
        args = self.net_obs1, self.network
        kwargs = {
            "destination_observations": self.net_obs2,
            "snap_dist": True,
            "distance_type": "network",
        }
        observed_mtx = tigernet.obs2obs_cost_matrix(*args, **kwargs)
        numpy.testing.assert_array_almost_equal(
            observed_mtx[:4, :4], known_mtx, decimal=DECIMAL
        )

        known_shape = (92, 1969)
        observed_shape = observed_mtx.shape
        self.assertEqual(observed_shape, observed_shape)

        known_mtx_sum = 503083944.4664013
        observed_mtx_sum = observed_mtx.sum()
        self.assertAlmostEqual(observed_mtx_sum, known_mtx_sum, delta=1)

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: martinfleis/tigernet

max_issues_repo_name: martinfleis/tigernet

max_forks_repo_name: jGaboardi/tigernet