text:
import streamlit as st
import pandas as pd
import numpy as np
import pickle
from sklearn.ensemble import RandomForestClassifier

st.write("""

# Penguin Prediction App

This app predicts the **Palmer Penguin** species!

Data obtained from the [palmerpenguins library](https://github.com/allisonhorst/palmerpenguins) in R by allison Horst.
""")

st.sidebar.header('User input Features')

st.sidebar.markdown("""
[Example CSV input file](https://github.com/zubin131/streamlit-data-apps/blob/main/penguin_classifier/penguins_example.csv)
""")

#collect user input as dataframe
uploaded_file = st.sidebar.file_uploader("Upload your input CSV file", type=["csv"])
if uploaded_file is not None:
    input_df = pd.read_csv(uploaded_file)
else:
    def user_input_features():
        island = st.sidebar.selectbox('Island',('Biscoe', 'Dream', 'Torgersen'))
        sex = st.sidebar.selectbox('Sex',('male', 'female'))
        bill_length_mm = st.sidebar.slider('Bill Length (mm)', 32.1,59.6,43.9)
        bill_depth_mm = st.sidebar.slider('Bill Depth (mm)', 13.1, 21.5,17.2)
        flipper_length_mm = st.sidebar.slider('Flipper Length (mm)', 172.0, 231.0, 201.0)
        body_mass_g = st.sidebar.slider('Body Mass (g)', 2700.0,6300.0, 4207.0)
        data = {'island': island,
                'bill_length_mm': bill_length_mm,
                'bill_depth_mm': bill_depth_mm,
                'flipper_length_mm': flipper_length_mm,
                'body_mass_g': body_mass_g,
                'sex':sex}
        features = pd.DataFrame(data, index=[0])
        return features
    input_df = user_input_features()

# combines user input features with entire penguin dataset
#this will be useful for the encoding phase
penguins_raw = pd.read_csv('penguins_cleaned.csv')
penguins = penguins_raw.drop(columns=['species'])
df = pd.concat([input_df, penguins], axis=0)

#encoding of ordinal features
encode = ['sex', 'island']
for col in encode:
    dummmy = pd.get_dummies(df[col], prefix=col)
    df = pd.concat([df,dummmy], axis=1)
    del df[col]

df = df[:1] #selects only the first row / user input data

#displays the user input features
st.subheader('User Input Features')

if uploaded_file is not None:
    st.write(df)
else:
    st.write('Awaiting CSV file to be uploaded. Currently using example input parameters (shown below).')
    st.write(df)

#reads in saved model
load_clf = pickle.load(open('penguin_clf.pkl', 'rb'))

#apply model to make predictions
prediction = load_clf.predict(df)
prediction_proba = load_clf.predict_proba(df)

st.subheader('Prediction')
penguin_species = np.array(['Adelie', 'Chinstrap', 'Gentoo'])
st.write(penguin_species[prediction])

st.subheader('Prediction Probability')
st.write(prediction_proba)


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

###
# Name: Jarod Penniman, Jared Love
# Student ID: 2258875, 1818306
# Email: penni112@mail.chapman.edu, love115@mail.chapman.edu
# Course: PHYS220/MATH220/CPSC220 Fall 2017
# Assignment: CW 06
###

"""Code creates an 'n x n' matrix such that, when mulitplied by a vector,
returns the approximate derivative of the vector given by the forward, symmetric, and
backward differences appropriately."""

import numpy as np
import matplotlib.pyplot as plt

def derivative(a,b,n):
	"""derivative(a,b,n)
	Function to create the derivative matrix.  Takes 3 inputs:
		- a = float, starting point of the domain
		- b = float, end point of domain
		- n = number of points created in the domain
	Output: an 'n x n' 2-dimensional numpy array that, when matrix multiplied by a 1-D array,
	returns the approximate derivative at each point."""
	t = np.linspace(a,b,n)
	dx = t[1]-t[0]
	empty = np.zeros(n, dtype = 'float64')
	a,b = np.meshgrid(empty, empty)
	A = a + b
	#Forward Difference
	A[0,0] = -1/dx
	A[0,1] = 1/dx
	#Symmetric Difference
	index = np.arange(1,n-1)
	A[index, index-1] = -1/(2*dx)
	A[index, index+1] = 1/(2*dx)
	#Backward Difference
	A[n-1,n-2] = -1/dx
	A[n-1,n-1] = 1/dx
	return A

def second_derivative(a,b,n):
    """second_derivative(a,b,n)
    Function to create the second derivative matrix.  Takes 3 inputs:
        - a = float, starting point of the domain
        - b = float, end point of domain
        - n = number of points created in the domain
    Output: an 'n x n' 2-dimensional numpy array that, when matrix multiplied by a 1-D array,
    returns the approximate second derivative at each point."""
    t = np.linspace(a,b,n)
    dx = t[1]-t[0]
    empty = np.zeros(n, dtype = 'float64')
    a,b = np.meshgrid(empty, empty)
    A = a + b
    #Forward Difference
    A[0,0] = 1
    A[0,1] = -2
    A[0,2] = 1
    #Symmetric/Forward Difference
    A[1,0] = 2
    A[1,1] = -3
    A[1,3] = 1
    #Symmetric Difference
    index = np.arange(2,n-2)
    A[index, index-2] = 1/2
    A[index, index] = -1
    A[index, index+2] = 1/2
    #Symmetric/Backward Difference
    A[n-2,n-4] = 1
    A[n-2,n-2] = -3
    A[n-2,n-1] = 2
    #Backward Difference
    A[n-1,n-3] = 1
    A[n-1,n-2] = -2
    A[n-1,n-1] = 1
    return A*(1/(2*dx**2))

def f(a,b,n):
    """Function to return an array of the square of the input (a float)"""
    t = np.linspace(a,b,n)
    return t**2

def s(a,b,n):
    """Function to return an array of the sine of the input (float)"""
    t = np.linspace(a,b,n)
    sin = np.vectorize(np.sin)
    sin = sin(t)
    return sin

def g(a,b,n):
    """Function to return an array of the gaussian function"""
    def gauss(x):
        """Local function to return the Gaussian function dependent on the input float"""
        c = 1/(np.sqrt(2*np.pi))
        gauss = c*np.exp(-x**2/2)
        return gauss
    t = np.linspace(a,b,n)
    gs = np.vectorize(gauss)
    gs = gs(t)
    return gs

def plot_function(a,b,n,f,string):
    """plot_function(a,b,n,f,string)
    Plots a function on the domain specified by a and b, with n points, for
    a function f, labeled with a string describing the function"""
    t = np.linspace(a,b,n)
    deriv = np.dot(derivative(a,b,n),f)
    sec_deriv = np.dot(second_derivative(a,b,n),f)
    plt.plot(t,f,'b', label=string)
    plt.plot(t,deriv,'r', label='Derivative')
    plt.plot(t,sec_deriv,'g', label='Second Derivative')
    plt.title(string)
    plt.legend()
    plt.show()

ext: py

reason: import numpy


text:
import numpy as np

graphcolor = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120),
              (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150),
              (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148),
              (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199),
              (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)]

colors = []
for i in range(int(len(graphcolor) / 2)):
    colors.append((np.asarray(graphcolor[i*2]) + np.asarray(graphcolor[i*2+1])) / 2)

#colors = [(97,146,187),(230,141,60),(88,172,82)]

grey= (0.5, 0.5, 0.5)
graphcolor = [(c1/255.0,c2/255.0,c3/255.0) for c1,c2,c3 in graphcolor]
colors = [(c1/255.0,c2/255.0,c3/255.0) for c1,c2,c3 in colors]
markers=['o', '^', 's', 'D', '*', 'x', '.', 'H', '>', '<', 'v', 'd', '_']
linestyles = ['-', '--', '-.', ':']

def lighter(c, p=0.45, n=230.0/255.0):
    return tuple(a * p + (1-p) * n for a in c)

def darker(c):
    return lighter(c, 0.45, 25.0/255.0)

def shade(c,i,t):
    if (t % 2 == 1) and (int(i) == int(t/2)):
        return c
    elif i < t /2:
        return lighter(c, 0.75, 230.0/255.0)
    else:
        return lighter(c, 0.75, 25.0/255.0)

dark_blue = c_rss = colors[0]
purple = c_rsspp = colors[4]
light_orange = graphcolor[2]
dark_orange = graphcolor[3]
light_red = graphcolor[6]
dark_red = graphcolor[7]
green = c_metron = colors[2]
c_sprayer = colors[6]

m_rss= 'o'
m_rsspp = 'd'

max_stars_repo_name: rsspp/experiments

max_issues_repo_name: rsspp/experiments

max_forks_repo_name: rsspp/experiments


text:
# vim: fdm=marker
'''
author:     Fabio Zanini
date:       12/09/14
content:    Plot the phylogenetic tree of the consensus sequences.
'''
# Modules
import sys
import os
import argparse
import numpy as np
from Bio import SeqIO, AlignIO, Phylo
import matplotlib.pyplot as plt

from hivwholeseq.sequencing.samples import SampleSeq
from hivwholeseq.patients.patients import load_patients, Patient

# Script
if __name__ == '__main__':

    # Parse input args
    parser = argparse.ArgumentParser(description='Get phylogenetic tree of consensi',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--patients', nargs='+',
                        help='Patients to analyze')
    parser.add_argument('--regions', nargs='+', required=True,
                        help='Regions to analyze (e.g. V3 F6)')
    parser.add_argument('--verbose', type=int, default=0,
                        help='Verbosity level [0-4]')

    args = parser.parse_args()
    pnames = args.patients
    regions = args.regions
    VERBOSE = args.verbose
    use_plot = args.plot

    patients = load_patients()
    if pnames:
        patients = patients.loc[pnames]

    for pname, patient in patients.iterrows():
        patient = Patient(patient)
        patient.discard_nonsequenced_samples()

        for region in regions:
            if VERBOSE >= 1:
                print pname, region

            ali = patient.get_consensi_alignment(region)
            tree = patient.get_consensi_tree(region, format='json')

            if use_plot:
                fig, ax = plt.subplots(figsize=(15, 12))
                Phylo.draw(tree, do_show=False, axes=ax)
                ax.set_title(pname+', '+region)

                x_max = max(tree.depths().itervalues())
                ax.set_xlim(0.998, 0.998 + (x_max - 0.998) * 1.5)
                ax.grid(True)

    if use_plot:
        plt.ion()
        plt.show()

max_stars_repo_name: neherlab/hivwholeseq

max_issues_repo_name: iosonofabio/hivwholeseq

max_forks_repo_name: iosonofabio/hivwholeseq


text:
#-*- coding:utf-8 -*-
import os
import sys
import numpy as np
from imp import reload
from PIL import Image, ImageOps

from keras.layers import Input
from keras.models import Model
# import keras.backend as K
from keras.utils import multi_gpu_model
import dl_resnet_crnn as densenet
#import densenet
os.environ["CUDA_VISIBLE_DEVICES"] = "4,5"
GPU_NUM = 2
#reload(densenet)

encode_dct =  {}
char_set = open('chn.txt', 'r', encoding='utf-8').readlines()
#char_set = open('japchn.txt', 'r', encoding='utf-8').readlines()
for i in range (0, len(char_set)):
	c = char_set[i].strip('\n')
	encode_dct[c] = i

char_set.append('卍')
#char_set = ''.join([ch.strip('\n') for ch in char_set] + ['卍'])

#characters = ''.join([chr(i) for i in range(32, 127)] + ['卍'])
nclass = len(char_set)

mult_model, basemodel = densenet.get_model(False, 32, nclass)
#input = Input(shape=(32, None, 1), name='the_input')
#y_pred= densenet.dense_cnn(input, nclass)
#basemodel = Model(inputs=input, outputs=y_pred)

#model = Model(inputs=[input, labels, input_length, label_length], outputs=loss_out)

mp = 'weights_densenet-02-1.29.h5'
modelPath = os.path.join(os.getcwd(), './models/' + mp)
#modelPath = sys.argv[1]
'''
load models
'''

def load_model_weights(modelPath):
	if not os.path.exists(modelPath):
		print ("ERROR Load Model : ", modelPath)
		return None

	mult_model, basemodel = densenet.get_model(False, 32, nclass)
	multi_model = multi_gpu_model(basemodel, gpus=GPU_NUM)
	multi_model.load_weights(modelPath)
	weights = basemodel.get_weights()
	return weights

'''
modelPaths = [
'weights_chn_eng_eroded_v3_resnet-08-1.13.h5',
'weights_chn_eng_eroded_v3_resnet-09-1.21.h5',
'weights_chn_eng_eroded_v3_resnet-10-1.16.h5',
'weights_chn_eng_eroded_v3_resnet-11-1.14.h5',
'weights_chn_eng_eroded_v3_resnet-12-1.20.h5',
'weights_chn_eng_eroded_v3_resnet-13-1.11.h5'
]
'''

modelPaths = [
'weights_chn_eng_eroded_v7_resnet-03-1.14.h5',
'weights_chn_eng_eroded_v7_resnet-04-1.15.h5'
]

weights_list = []
for modelPath in modelPaths:
	weights_list.append(load_model_weights("./models/" + modelPath) )

print (weights_list[0])

new_weights = list()

for weights_list_tuple in zip(*weights_list):
	new_weights.append(
		[np.array(weights_).mean(axis=0)\
			for weights_ in zip(*weights_list_tuple)])

multi_model, basemodel = densenet.get_model(False, 32, nclass)
basemodel.set_weights(new_weights)
basemodel.save("avg_model.h5")

	#basemodel.save(sys.argv[2])
	#basemodel.save("./new_model.h5")
	#basemodel = multi_model
	#model.load_weights(modelPath)


text:
import os
import sys
from glob import glob
import argparse

import numpy as np

from rsfmri import utils
from rsfmri import register as reg
from rsfmri import transform

import logging

def split(rawdir, destdir, sid, logger, globstr='B*func4d.nii*'):
    """ split files into 3D vols,
    return is None if it fails"""
    files, nfiles = utils.get_files(rawdir, globstr)
    logger.info('SUBID: {0}, nfiles: {1}'.format(sid, nfiles))
    if not nfiles == 1:
        logger.error('Raw functional not found: {0}'.format(globstr))
        return None
    rawfunc = files[0]
    funcs =  utils.fsl_split4d(rawfunc, destdir, sid)
    return funcs

def make_realignst(funcs, TR, logger):
    """use spm to reaign and slicetime correct files"""
    stvars = utils.get_slicetime_vars(funcs, TR=TR)
    meanfunc, realigned, params = utils.spm_realign(funcs)
    if meanfunc is None:
        logger.error('{0}: spm realignment failed'%(funcs))
        return None, None
    st_realigned = utils.spm_slicetime(realigned, stdict = stvars)
    utils.zip_files(realigned[1:]) # first is original func
    utils.zip_files([meanfunc])
    if st_realigned is None:
        logger.error('{0}: slice timing failed'.format(realigned))
        return None, None

    movement_array = np.loadtxt(params)
    return st_realigned, movement_array

def plot_write_movement(destdir, sid, movement, logger):
    transform.plot_movement(movement, destdir)
    outfile = os.path.join(destdir, '{0}_movement.xls'.format(sid))
    transform.movementarr_to_pandas(movement, outfile)

def process_subject(subdir, tr, logger, despike=False):
    """ process one subject despike (optional), realign and
    do slicetime correction via SPM tools"""
    globstr = 'B*func4d.nii*'
    workdirnme = utils.defaults['realign_spm']
    if despike:
        workdirnme = utils.defaults['despike'] + workdirnme
        globstr = 'ds' + globstr
    _, sid = os.path.split(subdir)
    rawdir = os.path.join(subdir, 'raw')

    workdir, exists = utils.make_dir(subdir, workdirnme)

    if exists:
        logger.error('{0}: skipping {1} exists'.format(subdir, workdir))
        return None

    funcs = split(rawdir, workdir, sid, logger, globstr)
    if funcs is None:
        logger.error('Raw dir missing data: {0}'.format(rawdir))
        return None

    staligned, move_arr = make_realignst(funcs, tr, logger)
    if staligned is None:
        return None

    ## Make aligned_4d
    aligned4d = os.path.join(workdir, 'align4d_{0}.nii.gz'.format(sid))
    aligned4d = utils.fsl_make4d(staligned, aligned4d)
    logger.info(aligned4d)
    ## spm only has 6 params, add empty 7th for plotting
    move_arr = np.hstack((move_arr,
                          np.zeros((move_arr.shape[0],1))))
    plot_write_movement(workdir, sid, move_arr, logger)
    ## zip files
    utils.zip_files(funcs)
    utils.zip_files(staligned)
    logger.info('{0} : finished'.format(sid))

def process_all(datadir, globstr, tr, logger, despike=False):
    gstr = os.path.join(datadir, globstr)
    subjects = sorted(glob(gstr))
    if len(subjects) < 1:
        raise IOError('{0}: returns no subjects'.format(gstr))
    for subjectdir in subjects:
        logger.info(subjectdir)
        res = process_subject(subjectdir, tr, logger, despike=despike)
        #res  = None
        if res is None:
            logger.error('{0}: skipped'.format(subjectdir))
            continue
        logger.info('{0}: finished {1}'.format(subjectdir, __file__))

if __name__ == '__main__':

    epilog = """
    python realignst_spm.py /home/jagust/graph/data/mri1.5/tr220 2.2 -d
    """

    parser = argparse.ArgumentParser(
        epilog = epilog,
        description='Run subject through realign, slicetime (optional despike)')
    parser.add_argument(
        'datadir',
        type=str,
        help='directory holding data')
    parser.add_argument(
        'TR',
        type=float,
        help='(float) Repetition Time (TR) of timeseries data')
    parser.add_argument(
        '-d', '--despike',
        dest='despike',
        action='store_true'
        )
    parser.add_argument('-g','--globstr',
        type=str,
        dest='globstr',
        default='B*',
        help='globstring to select subject directories ("B*")')

    try:
        args = parser.parse_args()
        print args
    except:
        parser.print_help()
        sys.exit()

    datadir, tr = args.datadir, args.TR
    globstr, despike = args.globstr, args.despike
    logger = logging.getLogger('realignst_spm')
    logger.setLevel(logging.DEBUG)
    ts = reg.timestr()
    fname = os.path.split(__file__)[-1].replace('.py', '')
    logfile = os.path.join(datadir,
                           'logs',
                           '{0}_logger_{1}.log'.format(fname, ts))
    fh = logging.FileHandler(logfile)
    fh.setLevel(logging.DEBUG)
    logger.addHandler(fh)
    logger.info(os.getenv('USER'))
    logger.info(ts)
    logger.info(args)
    #datadir, tr, despike = args.datadir, args.TR, args.despike
    process_all(datadir, 'B*', tr, logger, despike)
    #process_all(datadir, globstr, tr, logger, despike=False)
    #process_all(datadir, 'B13*', repetition_time, logger, despike)


text:
import torch
import numpy as np
import os, argparse, cv2
from lib.Network_Res2Net_GRA_NCD_FeatureViz import Network
from utils.dataloader import test_dataset

def heatmap(feat_viz, ori_img, save_path=None):
    feat_viz = torch.mean(feat_viz, dim=1, keepdim=True).data.cpu().numpy().squeeze()
    feat_viz = (feat_viz - feat_viz.min()) / (feat_viz.max() - feat_viz.min() + 1e-8)

    ori_img = ori_img.data.cpu().numpy().squeeze()
    ori_img = ori_img.transpose((1, 2, 0))
    ori_img = ori_img * np.array((0.229, 0.224, 0.225)) + np.array((0.485, 0.456, 0.406))
    ori_img = ori_img[:, :, ::-1]
    # img = (img - img.min()) / (img.max() - img.min() + 1e-8)
    ori_img = np.uint8(255 * ori_img)
    feat_viz = np.uint8(255 * feat_viz)
    feat_viz = cv2.applyColorMap(feat_viz, cv2.COLORMAP_JET)
    feat_viz = cv2.resize(feat_viz, (320, 320))
    ori_img = cv2.resize(ori_img, (320, 320))
    # print(feat_viz.shape, ori_img.shape)
    feat_viz = cv2.addWeighted(ori_img, 0.5, feat_viz, 0.5, 0)

    cv2.imwrite(save_path, feat_viz)
    # cv2.imshow('img', feat_viz)
    # cv2.waitKey(0)
    # cv2.destroyAllWindows()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--testsize', type=int, default=352, help='testing size')
    parser.add_argument('--pth_path', type=str, default='./snapshot/20201214-Network_Res2Net_GRA_NCD/Net_epoch_best.pth')
    opt = parser.parse_args()

    for _data_name in ['CAMO', 'COD10K', 'CHAMELEON']:
        data_path = '/media/nercms/NERCMS/GepengJi/2020ACMMM/Dataset/COD_New_data/TestDataset/{}/'.format(_data_name)
        save_path = './res/{}/Feature_Viz/{}/'.format(opt.pth_path.split('/')[-2], _data_name)
        model = Network()
        model.load_state_dict(torch.load(opt.pth_path))
        model.cuda()
        model.eval()

        os.makedirs(save_path, exist_ok=True)
        image_root = '{}/Imgs/'.format(data_path)
        gt_root = '{}/GT/'.format(data_path)
        test_loader = test_dataset(image_root, gt_root, opt.testsize)

        for i in range(test_loader.size):
            image, gt, name = test_loader.load_data()
            gt = np.asarray(gt, np.float32)
            gt /= (gt.max() + 1e-8)
            image = image.cuda()

            res5, res4, res3, res2, feat_viz = model(image)
            for i in range(0, 3):
                for j in range(0, 4):
                    for k in range(0, 2):
                        cur_feat_viz = feat_viz[i][j][k]
                        label = 'feat' if k == 0 else 'guid'
                        img_name = name.split('.')[0] + '_level{}_GRA{}_'.format(i+3, j+1) + label + '.png'
                        heatmap(feat_viz=cur_feat_viz, ori_img=image, save_path=save_path+img_name)
                        print('> Dataset: {}, Image: {}'.format(_data_name, save_path+img_name))
            # res = res2
            # res = F.upsample(res, size=gt.shape, mode='bilinear', align_corners=False)
            # res = res.sigmoid().data.cpu().numpy().squeeze()
            # res = (res - res.min()) / (res.max() - res.min() + 1e-8)
            # misc.imsave(save_path+name, res)


text:
import pandas as pd
import numpy as np
from sklearn.utils import shuffle

df = pd.read_csv("../allchest.csv")
df_list = list(map(lambda x: df[df['ID'] == x], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17]))
df_list = list(map(lambda x: x[x['label'] <= 4], df_list))
del df

train = []
test = []

for df in df_list:
	df.reset_index(inplace=True, drop=True)
	df = shuffle(df, random_state=42)
	df.reset_index(inplace=True, drop=True)
	train.append(df.iloc[:int(0.6 * df.shape[0]), :])
	test.append(df.iloc[int(0.6 * df.shape[0]):, :])
del df_list

train = pd.concat(train, axis=0)
test = pd.concat(test, axis=0)
train.reset_index(inplace=True, drop=True)
test.reset_index(inplace=True, drop=True)
train = train[['label', 'ID', 'chestACCx', 'chestACCy', 'chestACCz', 'chestECG', 'chestEMG', 'chestEDA', 'chestTemp', 'chestResp']]
test = test[['label', 'ID', 'chestACCx', 'chestACCy', 'chestACCz', 'chestECG', 'chestEMG', 'chestEDA', 'chestTemp', 'chestResp']]

train = np.array(train, dtype=np.float32)
test = np.array(test, dtype=np.float32)

np.save("train.npy", train)
np.save("test.npy", test)

max_stars_repo_name: naveenggmu/StressAffectDetection

max_issues_repo_name: naveenggmu/StressAffectDetection

max_forks_repo_name: naveenggmu/StressAffectDetection


text:
import numpy as np
import keras.backend as K
from keras import losses
from keras.models import Model
from keras.optimizers import Adam
from keras.layers import Input
from keras.layers import MaxPool2D
from keras.layers import Activation
from keras.layers import BatchNormalization
from keras.layers import UpSampling2D
from keras.layers import LeakyReLU
from keras.layers import Conv2D
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.layers import concatenate

def eacc(y_true, y_pred):
    return K.mean(K.equal(K.round(y_true), K.round(y_pred)))

def l1(y_true, y_pred):
    return K.mean(K.abs(y_pred - y_true))

def create_conv(filters, kernel_size, inputs, name=None, bn=True, dropout=0., padding='same', activation='relu'):
    conv = Conv2D(filters, kernel_size, padding=padding,
                  kernel_initializer='he_normal', name=name)(inputs)

    if bn:
        conv = BatchNormalization()(conv)

    if activation == 'relu':
        conv = Activation(activation)(conv)
    elif activation == 'leakyrelu':
        conv = LeakyReLU()(conv)

    if dropout != 0:
        conv = Dropout(dropout)(conv)

    return conv

def create_model_gen(input_shape, output_channels):
    inputs = Input(input_shape)
    conv1 = create_conv(64, (3, 3), inputs, 'conv1_1', activation='leakyrelu')
    conv1 = create_conv(64, (3, 3), conv1, 'conv1_2', activation='leakyrelu')
    pool1 = MaxPool2D((2, 2))(conv1)

    conv2 = create_conv(128, (3, 3), pool1, 'conv2_1', activation='leakyrelu')
    conv2 = create_conv(128, (3, 3), conv2, 'conv2_2', activation='leakyrelu')
    pool2 = MaxPool2D((2, 2))(conv2)

    conv3 = create_conv(256, (3, 3), pool2, 'conv3_1', activation='leakyrelu')
    conv3 = create_conv(256, (3, 3), conv3, 'conv3_2', activation='leakyrelu')
    pool3 = MaxPool2D((2, 2))(conv3)

    conv4 = create_conv(512, (3, 3), pool3, 'conv4_1', activation='leakyrelu')
    conv4 = create_conv(512, (3, 3), conv4, 'conv4_2', activation='leakyrelu')
    pool4 = MaxPool2D((2, 2))(conv4)

    conv5 = create_conv(1024, (3, 3), pool4, 'conv5_1', activation='leakyrelu')
    conv5 = create_conv(1024, (3, 3), conv5, 'conv5_2', activation='leakyrelu')

    up6 = create_conv(512, (2, 2), UpSampling2D((2, 2))(conv5), 'up6')
    merge6 = concatenate([conv4, up6], axis=3)
    conv6 = create_conv(512, (3, 3), merge6, 'conv6_1', activation='relu')
    conv6 = create_conv(512, (3, 3), conv6, 'conv6_2', activation='relu')

    up7 = create_conv(256, (2, 2), UpSampling2D((2, 2))(conv6), 'up7')
    merge7 = concatenate([conv3, up7], axis=3)
    conv7 = create_conv(256, (3, 3), merge7, 'conv7_1', activation='relu')
    conv7 = create_conv(256, (3, 3), conv7, 'conv7_2', activation='relu')

    up8 = create_conv(128, (2, 2), UpSampling2D((2, 2))(conv7), 'up8')
    merge8 = concatenate([conv2, up8], axis=3)
    conv8 = create_conv(128, (3, 3), merge8, 'conv8_1', activation='relu')
    conv8 = create_conv(128, (3, 3), conv8, 'conv8_2', activation='relu')

    up9 = create_conv(64, (2, 2), UpSampling2D((2, 2))(conv8))
    merge9 = concatenate([conv1, up9], axis=3)
    conv9 = create_conv(64, (3, 3), merge9, 'conv9_1', activation='relu')
    conv9 = create_conv(64, (3, 3), conv9, 'conv9_2', activation='relu')
    conv9 = Conv2D(output_channels, (1, 1), padding='same', name='conv9_3')(conv9)

    model = Model(inputs=inputs, outputs=conv9, name='generator')

    return model

def create_model_dis(input_shape):
    inputs = Input(input_shape)
    conv1 = create_conv(64, (3, 3), inputs, 'conv1', activation='leakyrelu', dropout=.8)
    pool1 = MaxPool2D((2, 2))(conv1)

    conv2 = create_conv(128, (3, 3), pool1, 'conv2', activation='leakyrelu', dropout=.8)
    pool2 = MaxPool2D((2, 2))(conv2)

    conv3 = create_conv(256, (3, 3), pool2, 'conv3', activation='leakyrelu', dropout=.8)
    pool3 = MaxPool2D((2, 2))(conv3)

    conv4 = create_conv(512, (3, 3), pool3, 'conv4', activation='leakyrelu', dropout=.8)
    pool4 = MaxPool2D((2, 2))(conv4)

    conv5 = create_conv(512, (3, 3), pool4, 'conv5', activation='leakyrelu', dropout=.8)

    flat = Flatten()(conv5)
    dense6 = Dense(1, activation='sigmoid')(flat)

    model = Model(inputs=inputs, outputs=dense6, name='discriminator')

    return model

def create_model_gan(input_shape, generator, discriminator):
    input = Input(input_shape)

    gen_out = generator(input)
    dis_out = discriminator(concatenate([gen_out, input], axis=3))

    model = Model(inputs=[input], outputs=[dis_out, gen_out], name='dcgan')

    return model

def create_models(input_shape_gen, input_shape_dis, output_channels, lr, momentum, loss_weights):
    optimizer = Adam(lr=lr, beta_1=momentum)

    model_gen = create_model_gen(input_shape=input_shape_gen, output_channels=output_channels)
    model_gen.compile(loss=losses.mean_absolute_error, optimizer=optimizer)

    model_dis = create_model_dis(input_shape=input_shape_dis)
    model_dis.trainable = False

    model_gan = create_model_gan(input_shape=input_shape_gen, generator=model_gen, discriminator=model_dis)
    model_gan.compile(
        loss=[losses.binary_crossentropy, l1],
        metrics=[eacc, 'accuracy'],
        loss_weights=loss_weights,
        optimizer=optimizer
    )

    model_dis.trainable = True
    model_dis.compile(loss=losses.binary_crossentropy, optimizer=optimizer)

    return model_gen, model_dis, model_gan


text:
import numpy as np

from Dist import Dist
from pysgpp import (DataVector, DataMatrix, KernelDensityEstimator,
                    KernelType_GAUSSIAN,
                    createOperationRosenblattTransformationKDE,
                    createOperationInverseRosenblattTransformationKDE,
                    KernelType_EPANECHNIKOV)
from pysgpp.extensions.datadriven.uq.operations.general import isNumerical, isList, isMatrix
import pysgpp.extensions.datadriven.uq.jsonLib as ju
from EstimatedDist import EstimatedDist
from pysgpp.pysgpp_swig import BandwidthOptimizationType_NONE, \
    BandwidthOptimizationType_MAXIMUMLIKELIHOOD, \
    BandwidthOptimizationType_SILVERMANSRULE

class KDEDist(EstimatedDist):
    """
    KDE using SG++ implementation
    """

    def __init__(self,
                 trainData,
                 kde=None,
                 kernelType=KernelType_GAUSSIAN,
                 bandwidhts=None,
                 bandwidthOptimizationType=BandwidthOptimizationType_SILVERMANSRULE,
                 bounds=None):
        super(KDEDist, self).__init__(trainData.shape[1], trainData, bounds)

        trainData_matrix = DataMatrix(self.trainData)
        if kde is not None:
            self.dist = kde
        elif bandwidhts is not None:
            self.dist = KernelDensityEstimator(trainData_matrix,
                                               kernelType,
                                               BandwidthOptimizationType_NONE)
            bandwidhts_vec = DataVector(bandwidhts)
            self.dist.setBandwidths(bandwidhts_vec)
        else:
            # just learn the kernel density
            self.dist = KernelDensityEstimator(trainData_matrix,
                                               kernelType,
                                               bandwidthOptimizationType)

    def pdf(self, x):
        x = self._convertEvalPoint(x)
        x_matrix = DataMatrix(x)
        res_vec = DataVector(x.shape[0])
        self.dist.pdf(x_matrix, res_vec)
        res = res_vec.array()

        if len(res) == 1:
            return res[0]
        else:
            return res

    def cdf(self, x, shuffle=False):
        x = self._convertEvalPoint(x)
        x_matrix = DataMatrix(x)
        res_matrix = DataMatrix(x_matrix.getNrows(), x_matrix.getNcols())
        res_matrix.setAll(0.0)

        # do the transformation
        opRosen = createOperationRosenblattTransformationKDE(self.dist)
        if shuffle:
            opRosen.doShuffledTransformation(x_matrix, res_matrix)
        else:
            opRosen.doTransformation(x_matrix, res_matrix)

        # transform the outcome
        res = res_matrix.array()
        if res.shape[0] == 1 and res.shape[1] == 1:
            return res[0, 0]
        else:
            return res

    def ppf(self, x, shuffle=False):
        x = self._convertEvalPoint(x)
        x_matrix = DataMatrix(x)
        res_matrix = DataMatrix(x_matrix.getNrows(), x_matrix.getNcols())
        res_matrix.setAll(0.0)

        # do the transformation
        opRosen = createOperationInverseRosenblattTransformationKDE(self.dist)
        if shuffle:
            opRosen.doShuffledTransformation(x_matrix, res_matrix)
        else:
            opRosen.doTransformation(x_matrix, res_matrix)

        # transform the outcome
        res = res_matrix.array()
        if res.shape[0] == 1 and res.shape[1] == 1:
            return res[0, 0]
        else:
            return res

    def rvs(self, n=1, shuffle=False):
        unif = np.random.rand(self.dim * n).reshape(n, self.dim)
        return self.ppf(unif, shuffle=shuffle)

    def mean(self, n=1e4):
        return self.dist.mean()

    def var(self):
        return self.dist.variance()

    def cov(self):
        covMatrix = DataMatrix(np.zeros((self.dim, self.dim)))
        self.dist.cov(covMatrix)
        return covMatrix.array()

    def getBandwidths(self):
        bandwidths = DataVector(self.getDim())
        self.dist.getBandwidths(bandwidths)
        return bandwidths.array()

    def marginalizeToDimX(self, idim):
        margLearner = self.dist.margToDimX(idim)
        return KDEDist(trainData=np.vstack((self.trainData[:, idim])),
                       kde=margLearner)

    def marginalize(self, idim):
        margLearner = self.dist.marginalize(idim)
        return KDEDist(trainData=np.delete(self.trainData, idim, axis=1),
                       kde=margLearner)

    def __str__(self):
        ans = "KDE - %s"

        if self.dist.getKernel().getType() == KernelType_GAUSSIAN:
            ans = ans % "Gaussian"
        elif self.dist.getKernel().getType() == KernelType_EPANECHNIKOV:
            ans = ans % "Epanechnikov"
        else:
            ans = ans % "unknown kernel"
        return ans

    def toJson(self):
        """
        Returns a string that represents the object

        Arguments:

        Return A string that represents the object
        """
        serializationString = '"module" : "' + \
                              self.__module__ + '",\n'

        for attrName, attrValue in [("_KDEDist__kernelType", str(self.dist.getKernel().getType())),
                                    ("_KDEDist__bandwidths", self.getBandwidths()),
                                    ("_KDEDist__trainData", self.trainData),
                                    ("_KDEDist__bounds", self.bounds)]:
            serializationString += ju.parseAttribute(attrValue, attrName)

        s = serializationString.rstrip(",\n")

        return "{" + s + "}"

    @classmethod
    def fromJson(cls, jsonObject):
        """
        Restores the Beta object from the json object with its
        attributes.

        Arguments:
        jsonObject -- json object

        Return the restored UQSetting object
        """
        # restore surplusses
        key = '_KDEDist__trainData'
        if key in jsonObject:
            trainData = np.array(jsonObject[key])
        else:
            raise AttributeError("KDEDist: fromJson - trainData is missing")

        key = '_KDEDist__kernelType'
        if key in jsonObject:
            kernelType = jsonObject[key]
            if kernelType == "0":
                kernelType = KernelType_GAUSSIAN
            elif kernelType == "1":
                kernelType = KernelType_EPANECHNIKOV
            else:
                raise AttributeError("KDEDist: fromJson - kernel type unknown")
        else:
            raise AttributeError("KDEDist: fromJson - kernelType is missing")

        key = '_KDEDist__bounds'
        bounds = None
        if key in jsonObject:
            bounds = np.array(jsonObject[key])

        key = '_KDEDist__bandwidths'
        bandwidths = None
        if key in jsonObject:
            bandwidths = np.array(jsonObject[key])

        return KDEDist(trainData, bounds=bounds, bandwidhts=bandwidths,
                       kernelType=kernelType)

max_stars_repo_name: valentjn/thesis

max_issues_repo_name: valentjn/thesis

max_forks_repo_name: valentjn/thesis


text:
# This program uses the 10 day rate of change(ROC) tehcnical indicator to either go
# long a market or short a market based on if the ROC is greater than 25 (Long market)
# or less than -25 (short market)

import numpy

def myTradingSystem(DATE, OPEN, HIGH, LOW, CLOSE, VOL, exposure, equity, settings):
    nMarkets = CLOSE.shape[1]
    pos = numpy.zeros(nMarkets)
    for markets in range(nMarkets):
        # ROC = ((todays close - 10 days ago close) / (ten days ago close)) * 100
        settings['rateOfChange'][markets] = ((CLOSE[-1, markets] - CLOSE[-10, markets]) /
                                             CLOSE[-10, markets]) * 100
        if settings['rateOfChange'][markets] > 25 and settings['relativePos'][markets] == 0:
            settings['relativePos'][markets] = -.05
            settings['countPos'] += 1
        elif settings['rateOfChange'][markets] < -25 and settings['relativePos'][markets] == 0:
            settings['relativePos'][markets] = .05
            settings['countPos'] += 1
    for markets in range(nMarkets):
        if settings['relativePos'][markets] == -.05 and settings['rateOfChange'][markets] <= 0:
            settings['relativePos'][markets] = 0
            settings['countPos'] -= 1
        elif settings['relativePos'][markets] == .05 and settings['rateOfChange'][markets] >= 0:
            settings['relativePos'][markets] = 0
            settings['countPos'] -= 1
    # allocating positions, weigh portfolio
    independentVar = settings['countPos'] / (1 + (.05 * (settings['countPos'] - 1)))
    longShortFactor = independentVar * .05
    for markets in range(nMarkets):
        if markets == 0:
            pos[0] = independentVar
        elif settings['relativePos'][markets] == .05:
            pos[markets] = longShortFactor
        elif settings['relativePos'][markets] == -.05:
            pos[markets] = -longShortFactor
    settings['countPos'] = 0

    return pos, settings

def mySettings():
    settings = {}

    settings['markets'] = ['CASH', 'F_AD', 'F_BO', 'F_BP', 'F_C', 'F_CC', 'F_CD',
                           'F_CL', 'F_CT', 'F_DX', 'F_EC', 'F_ED', 'F_ES', 'F_FC', 'F_FV', 'F_GC',
                           'F_HG', 'F_HO', 'F_JY', 'F_KC', 'F_LB', 'F_LC', 'F_LN', 'F_MD', 'F_MP',
                           'F_NG', 'F_NQ', 'F_NR', 'F_O', 'F_OJ', 'F_PA', 'F_PL', 'F_RB', 'F_RU',
                           'F_S', 'F_SB', 'F_SF', 'F_SI', 'F_SM', 'F_TU', 'F_TY', 'F_US', 'F_W', 'F_XX',
                           'F_YM']

    settings['beginInSample'] = '20000104'
    # settings['endInSample'] = '20170710'
    settings['lookback'] = 10
    settings['budget'] = 10 ** 6
    settings['slippage'] = 0.05
    settings['relativePos'] = numpy.zeros(45)
    settings['rateOfChange'] = numpy.zeros(45)
    settings['countPos'] = 0

    return settings

# Evaluate trading system defined in current file.
if __name__ == '__main__':
    import quantiacsToolbox

    results = quantiacsToolbox.runts(__file__)
    print(results['stats'])

max_stars_repo_name: Nawter/Quantiacs

max_issues_repo_name: Nawter/Quantiacs

max_forks_repo_name: Nawter/Quantiacs


text:
import os
import sys
import numpy as np
import pandas as pd
from pathlib import Path
# App
import methylprep
#patching
import unittest
try:
    # python 3.4+ should use builtin unittest.mock not mock package
    from unittest.mock import patch
except ImportError:
    from mock import patch

class TestPipeline():

    @staticmethod
    def test_pipeline_two_samples():
        """ pass in --sample_name with 2 samples -- from fake command line args """
        test_data_dir = 'docs/example_data/GSE69852'
        test_outputs = [
            Path(test_data_dir, 'control_probes.pkl'),
            #Path(test_data_dir, 'beta_values.pkl'),
            #Path(test_data_dir, 'm_values.pkl'),
            #Path(test_data_dir, 'meth_values.pkl'),
            #Path(test_data_dir, 'unmeth_values.pkl'),
            Path(test_data_dir, 'noob_meth_values.pkl'),
            Path(test_data_dir, 'noob_unmeth_values.pkl'),
            Path(test_data_dir, 'sample_sheet_meta_data.pkl'),
            #Path(test_data_dir, '9247377085', '9247377085_R04C02_processed.csv'),
            #Path(test_data_dir, '9247377093', '9247377093_R02C01_processed.csv'),
            ]
        for outfile in test_outputs:
            if outfile.exists():
                outfile.unlink()
        # CLI not working in unit tests
        #testargs = ["__program__", '-d', test_data_dir, '--no_export', '--sample_name', 'AdultLiver1', 'FetalLiver1', '--minfi', '--betas']
        #with patch.object(sys, 'argv', testargs):
        test_data_containers = methylprep.run_pipeline(test_data_dir, export=False, sample_name=['AdultLiver1', 'FetalLiver1'])
        # spot checking the output.
        test1 = test_data_containers[0]._SampleDataContainer__data_frame
        test1_v162_ref = [
            ['cg00000029',     1144.0,       1369.0,        0.003,           1.0,    0.455233, -0.259035],
            ['cg00000108',     3841.0,       3926.0,        0.000,           1.0,    0.494528, -0.031578],
            ['cg00000109',     3828.0,        190.0,        0.000,           1.0,    0.952713,  4.332520],
            ['cg00000165',      555.0,       2580.0,        0.006,           1.0,    0.177033, -2.216811],
            ['cg00000236',     3545.0,        428.0,        0.000,           1.0,    0.892273,  3.050103],
            ['rs9363764',      3038.0,       2009.0,        0.001,           1.0,    0.601942,  0.596644],
            ['rs939290',        360.0,       5615.0,        0.001,           1.0,    0.060251, -3.963217],
            ['rs951295',       5317.0,       4706.0,        0.000,           1.0,    0.530480,  0.176111],
            ['rs966367',        120.0,       4104.0,        0.002,           1.0,    0.028409, -5.095924],
            ['rs9839873',      4545.0,        112.0,        0.000,           1.0,    0.975950,  5.342710]
        ]
        test1_v163_ref = [
            ['cg00000029',     1137.0,       1380.0,        0.003,           1.0,    0.451, -0.279],
            ['cg00000108',     3834.0,       3935.0,        0.000,           1.0,    0.493, -0.037],
            ['cg00000109',     3821.0,        190.0,        0.000,           1.0,    0.953,  4.330],
            ['cg00000165',      551.0,       2593.0,        0.006,           1.0,    0.175, -2.234],
            ['cg00000236',     3538.0,        431.0,        0.000,           1.0,    0.891,  3.037],
            ['rs9363764',      3030.0,       2021.0,        0.001,           1.0,    0.599,  0.584],
            ['rs939290',        359.0,       5619.0,        0.001,           1.0,    0.060, -3.968],
            ['rs951295',       5312.0,       4700.0,        0.000,           1.0,    0.530,  0.177],
            ['rs966367',        120.0,       4112.0,        0.002,           1.0,    0.028, -5.099],
            ['rs9839873',      4540.0,        112.0,        0.000,           1.0,    0.976,  5.341]
        ]
        """ FULL 6-digit values for 1.6.3
        0.451728 -0.279436
        0.493500 -0.037513
        0.952630  4.329879
        0.175254 -2.234498
        0.891408  3.037174
        0.599881  0.584248
        0.060054 -3.968258
        0.530563  0.176594
        0.028355 -5.098734
        0.975924  5.341122
        """
        test1_ref = pd.DataFrame(data=test1_v163_ref, columns=['IlmnID', 'noob_meth',  'noob_unmeth', 'poobah_pval', 'quality_mask', 'beta_value',   'm_value']).set_index('IlmnID').astype('float32')
        test1_sub = test1.loc[ test1.index.isin(test1_ref.index) ].astype('float32')

        # print(test1_sub.head())

        if not np.isclose(test1_sub, test1_ref, atol=0.01).all():
            print(test1_sub)
            print(test1_ref)
            raise AssertionError()

        for outfile in test_outputs:
            if not outfile.exists():
                raise FileNotFoundError(f"Expected {outfile.name} to be generated by run_pipeline() but it was missing.")
            else:
                print('+', outfile)
                outfile.unlink()

max_stars_repo_name: LifeEGX/methylprep

max_issues_repo_name: LifeEGX/methylprep

max_forks_repo_name: LifeEGX/methylprep