text:
""" test autodir.lib.dlayer
"""
import os
import tempfile
import numpy
import automol
import autodir

PREFIX = tempfile.mkdtemp()
print(PREFIX)

def test__species():
    """ tets fsys.species
    """
    prefix = os.path.join(PREFIX, 'species')
    os.mkdir(prefix)

    args_lst = [
        ('InChI=1S/O', 3),
        ('InChI=1S/O', 1),
        ('InChI=1S/C2H2F2/c3-1-2-4/h1-2H/b2-1+', 1),
        ('InChI=1S/C2H2F2/c3-1-2-4/h1-2H/b2-1-', 1),
        ('InChI=1S/C5H5O/c1-2-3-4-5-6/h1-5H/b4-3-', 2),
    ]

    # fs_ = autodir.filesystem()

    # for args in args_lst:
    #     assert not fs_.species.dir.exists(prefix, args)
    #     assert not fs_.species.file.info.exists(prefix, args)
    #     fs_.species.dir.create(prefix, args)
    #     assert fs_.species.dir.exists(prefix, args)
    #     assert fs_.species.file.info.exists(prefix, args)

    # assert fs_.species.dir.created_names(prefix) == (
    #     'SPC/C2H2F2/WFLOTYSKFUPZQB/1/OWOJBTED',
    #     'SPC/C2H2F2/WFLOTYSKFUPZQB/1/UPHRSURJ',
    #     'SPC/C5H5O/ZMKDIXNSHZUDML/2/ARJAWSKD',
    #     'SPC/O/QVGXLLKOCUKJST/1/UHFFFAOY',
    #     'SPC/O/QVGXLLKOCUKJST/3/UHFFFAOY'
    # )

def test__theory():
    """ tets fsys.theory
    """
    prefix = os.path.join(PREFIX, 'theory')
    os.mkdir(prefix)

    spc_args = ('InChI=1S/CH3/h1H3', 2)
    args_lst = [
        spc_args + ('hf', 'sto-3g', True),
        spc_args + ('hf', 'sto-3g', False),
        spc_args + ('b3lyp', 'sto-3g', False),
        spc_args + ('b3lyp', '6-31g*', False),
    ]

    # fs_ = autodir.filesystem()

    # for args in args_lst:
    #     assert not fs_.theory.dir.exists(prefix, args)
    #     assert not fs_.theory.file.info.exists(prefix, args)
    #     fs_.theory.dir.create(prefix, args)
    #     assert fs_.theory.dir.exists(prefix, args)
    #     assert fs_.theory.file.info.exists(prefix, args)

    # print(fs_.theory.dir.created_names(prefix))
    # # assert fs_.theory.dir.created_names(prefix, spc_args) == (
    # #     '_0gh69R', '_0gh69U', 'ezvh69R', 'ezvh69U', 'ezvlpJR', 'ezvlpJU')

def test__conformer():
    """ tets fsys.conformer
    """
    prefix = os.path.join(PREFIX, 'conformer')
    os.mkdir(prefix)

    nconfs = 10
    cids = (autodir.lib.name.new_conformer_id() for _ in range(nconfs))
    args_lst = [(cid,) for cid in cids]

    fs_ = fsys.conformer()
    assert not fs_.conformer_trunk.dir.exists(prefix)
    fs_.conformer_trunk.dir.create(prefix)
    assert fs_.conformer_trunk.dir.exists(prefix)

    # create the trunk information file
    ref_trunk_inf_obj = autodir.lib.info.conformer_trunk(
        nsamp=7, tors_info={'d3': (0, 6.283185307179586),
                            'd4': (0, 6.283185307179586)})
    fs_.conformer_trunk.file.info.write(ref_trunk_inf_obj, prefix)
    trunk_inf_obj = fs_.conformer_trunk.file.info.read(prefix)
    assert trunk_inf_obj == ref_trunk_inf_obj

    # create the trunk vmatrix file
    ref_trunk_vma = (('C', (None, None, None), (None, None, None)),
                     ('O', (0, None, None), ('r1', None, None)),
                     ('O', (0, 1, None), ('r2', 'a1', None)),
                     ('H', (0, 1, 2), ('r3', 'a2', 'd1')),
                     ('H', (0, 1, 2), ('r4', 'a3', 'd2')),
                     ('H', (1, 0, 2), ('r5', 'a4', 'd3')),
                     ('H', (2, 0, 1), ('r6', 'a5', 'd4')))
    fs_.conformer_trunk.file.vmatrix.write(ref_trunk_vma, prefix)
    trunk_vma = fs_.conformer_trunk.file.vmatrix.read(prefix)
    assert trunk_vma == ref_trunk_vma

    for args in args_lst:
        fs_.conformer.dir.create(prefix, args)

        ref_geom_inf_obj = autodir.lib.info.run(
            job='optimization', prog='psi4', method='mp2', basis='sto-3g')
        ref_grad_inf_obj = autodir.lib.info.run(
            job='gradient', prog='psi4', method='mp2', basis='sto-3g')
        ref_hess_inf_obj = autodir.lib.info.run(
            job='hessian', prog='psi4', method='mp2', basis='sto-3g')
        ref_geom_inp_str = '<geometry input file>'
        ref_grad_inp_str = '<gradient input file>'
        ref_hess_inp_str = '<hessian input file>'
        ref_ene = -187.38518070487598
        ref_geo = (('C', (0.066541036329, -0.86543409422, -0.56994517889)),
                   ('O', (0.066541036329, -0.86543409422, 2.13152981129)),
                   ('O', (0.066541036329, 1.6165813318, -1.63686376233)),
                   ('H', (-1.52331011945, -1.99731957213, -1.31521725797)),
                   ('H', (1.84099386813, -1.76479255185, -1.16213243427)),
                   ('H', (-1.61114836922, -0.17751142359, 2.6046492029)),
                   ('H', (-1.61092727126, 2.32295906780, -1.19178601663)))
        ref_grad = ((0.00004103632, 0.00003409422, 0.00004517889),
                    (0.00004103632, 0.00003409422, 0.00002981129),
                    (0.00004103632, 0.00008133180, 0.00006376233),
                    (0.00001011945, 0.00001957213, 0.00001725797),
                    (0.00009386813, 0.00009255185, 0.00003243427),
                    (0.00004836922, 0.00001142359, 0.00004920290),
                    (0.00002727126, 0.00005906780, 0.00008601663))
        # (I'm not bothering with the hessian for now)

        # writes
        fs_.conformer.file.geometry_information.write(
            ref_geom_inf_obj, prefix, args)
        fs_.conformer.file.gradient_information.write(
            ref_grad_inf_obj, prefix, args)
        fs_.conformer.file.hessian_information.write(
            ref_hess_inf_obj, prefix, args)
        fs_.conformer.file.geometry_input.write(ref_geom_inp_str, prefix, args)
        fs_.conformer.file.gradient_input.write(ref_grad_inp_str, prefix, args)
        fs_.conformer.file.hessian_input.write(ref_hess_inp_str, prefix, args)
        fs_.conformer.file.energy.write(ref_ene, prefix, args)
        fs_.conformer.file.geometry.write(ref_geo, prefix, args)
        fs_.conformer.file.gradient.write(ref_grad, prefix, args)

        # reads
        geom_inf_obj = fs_.conformer.file.geometry_information.read(prefix,
                                                                    args)
        grad_inf_obj = fs_.conformer.file.gradient_information.read(prefix,
                                                                    args)
        hess_inf_obj = fs_.conformer.file.hessian_information.read(prefix,
                                                                   args)
        geom_inp_str = fs_.conformer.file.geometry_input.read(prefix, args)
        grad_inp_str = fs_.conformer.file.gradient_input.read(prefix, args)
        hess_inp_str = fs_.conformer.file.hessian_input.read(prefix, args)
        ene = fs_.conformer.file.energy.read(prefix, args)
        geo = fs_.conformer.file.geometry.read(prefix, args)
        grad = fs_.conformer.file.gradient.read(prefix, args)

        # check read values
        assert geom_inf_obj == ref_geom_inf_obj
        assert grad_inf_obj == ref_grad_inf_obj
        assert hess_inf_obj == ref_hess_inf_obj
        assert geom_inp_str == ref_geom_inp_str
        assert grad_inp_str == ref_grad_inp_str
        assert hess_inp_str == ref_hess_inp_str
        assert numpy.isclose(ene, ref_ene)
        assert automol.geom.almost_equal(geo, ref_geo)
        assert numpy.allclose(grad, ref_grad)

    assert len(fs_.conformer.dir.created_names(prefix)) == nconfs

if __name__ == '__main__':
    test__species()
    test__theory()
    # test__conformer()

max_stars_repo_name: avcopan/filesystem

max_issues_repo_name: avcopan/filesystem

max_forks_repo_name: avcopan/filesystem


text:
import numpy as np
import argparse
import matplotlib.pyplot as plt

from glob import glob
from ntpath import basename
from imageio import imread
from skimage.measure import compare_ssim
from skimage.measure import compare_psnr
from skimage.color import rgb2gray

def parse_args():
    parser = argparse.ArgumentParser(description='script to compute all statistics')
    parser.add_argument('--data-path', help='Path to ground truth data', type=str)
    parser.add_argument('--output-path', help='Path to output data', type=str)
    parser.add_argument('--debug', default=0, help='Debug', type=int)
    args = parser.parse_args()
    return args

def compare_mae(img_true, img_test):
    img_true = img_true.astype(np.float32)
    img_test = img_test.astype(np.float32)
    return np.sum(np.abs(img_true - img_test)) / np.sum(img_true + img_test)

args = parse_args()
for arg in vars(args):
    print('[%s] =' % arg, getattr(args, arg))

path_true = args.data_path
path_pred = args.output_path

psnr = []
ssim = []
mae = []
names = []
index = 1

files = list(glob(path_true + '/*.jpg')) + list(glob(path_true + '/*.png'))
for fn in sorted(files):
    name = basename(str(fn))
    names.append(name)

    img_gt = (imread(str(fn)) / 255.0).astype(np.float32)
    img_pred = (imread(path_pred + '/' + basename(str(fn))) / 255.0).astype(np.float32)

    img_gt = rgb2gray(img_gt)
    img_pred = rgb2gray(img_pred)

    if args.debug != 0:
        plt.subplot('121')
        plt.imshow(img_gt)
        plt.title('Groud truth')
        plt.subplot('122')
        plt.imshow(img_pred)
        plt.title('Output')
        plt.show()

    psnr.append(compare_psnr(img_gt, img_pred, data_range=1))
    ssim.append(compare_ssim(img_gt, img_pred, data_range=1, win_size=51))
    mae.append(compare_mae(img_gt, img_pred))
    if np.mod(index, 100) == 0:
        print(
            str(index) + ' images processed',
            "PSNR: %.4f" % round(np.mean(psnr), 4),
            "SSIM: %.4f" % round(np.mean(ssim), 4),
            "MAE: %.4f" % round(np.mean(mae), 4),
        )
    index += 1

np.savez(args.output_path + '/metrics.npz', psnr=psnr, ssim=ssim, mae=mae, names=names)
print(
    "PSNR: %.4f" % round(np.mean(psnr), 4),
    "PSNR Variance: %.4f" % round(np.var(psnr), 4),
    "SSIM: %.4f" % round(np.mean(ssim), 4),
    "SSIM Variance: %.4f" % round(np.var(ssim), 4),
    "MAE: %.4f" % round(np.mean(mae), 4),
    "MAE Variance: %.4f" % round(np.var(mae), 4)
)

ext: py

max_stars_repo_name: Xiefan-Guo/Paper-PyTorch

max_issues_repo_name: Xiefan-Guo/Paper-PyTorch

max_forks_repo_name: Xiefan-Guo/Paper-PyTorch

reason: import numpy


text:
import numpy as np

def rml_estimator(y, sigma2i, iterations=50):
    """Implementation of the robust maximum likelihood estimator.

        ::

            @book{demidenko2013mixed,
              title={Mixed models: theory and applications with R},
              author={Demidenko, Eugene},
              year={2013},
              publisher={John Wiley \& Sons}
            }

    """
    w = 1 / sigma2i
    sw = np.sum(w, axis=0)
    beta0 = np.sum(y * w, axis=0) / sw
    sigma2ML = np.sum((y - np.mean(y, axis=0)) ** 2 / (len(beta0) - 1), axis=0)
    eps = np.zeros(beta0.shape)
    betaML = None
    for _ in xrange(iterations):
        w = 1 / (sigma2i + sigma2ML)
        sw = np.sum(w, axis=0)
        sw2 = np.sum(w ** 2, axis=0)
        betaML = np.sum(y * w, axis=0) / sw
        sigma2ML_new = (
            sigma2ML
            * np.sum(((y - betaML) ** 2) * (w ** 2), axis=0)
            / (sw - (sw2 / sw))
        )
        eps = np.abs(sigma2ML - sigma2ML_new)
        sigma2ML = sigma2ML_new
    var_betaML = 1 / np.sum(1 / (sigma2i + sigma2ML), axis=0)
    return betaML, var_betaML, eps


text:
import base64, cv2, json, flask, time
import numpy as np
# from PIL import Image
from flask import Flask, request
from urllib.parse import unquote
from google.cloud import firestore
from elasticsearch import Elasticsearch
import tflite_runtime.interpreter as tflite

es = Elasticsearch(["http://35.190.172.118:9200/"])
emotionNames = ["Afraid", "Angry", "Disgusted", "Happy", "Neutral", "Sad", "Surprised"]

# Initialisation
app = Flask(__name__)

@app.route("/uploadImage", methods=["POST"])
def infer():
    print("Loading image...")
    try:
        imageData = str(request.form.getlist('imageData')[0])
        imageData = unquote(imageData) # Url decode body
        imageData = imageData.split(",")[1] # Remove b64 header
        imageData = base64.b64decode(imageData) # decode base64 to bytes
        nparr = np.frombuffer(imageData, np.uint8) # Load image into numpy array
        img = cv2.imdecode(nparr, 0) # Convert to grayscale cv2 image
    except Exception as e:
        print(f"Image data error - {e}")
        response = flask.jsonify({"body": f"Image data error - {e}"})
        response.headers.add("Access-Control-Allow-Origin", "*")
        return response, 400

    imgArr = np.zeros((48,48)) # Empty array, will store the image data

    # Load Haar-Cascade
    try:
        print("Loading Haar-Cascade...")
        faceCascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

        # Detect the faces in the image
        print("Detecting faces in image...")
        faces = faceCascade.detectMultiScale(img, 1.1, 4)
    except Exception as e:
        print(f"Could not load Haar-Cascade - {e}")
        response = flask.jsonify({"body": f"Error identifying faces in image"})
        response.headers.add("Access-Control-Allow-Origin", "*")
        return response, 500

    if(len(faces)) == 0:
        print("No faces found.")
        response = flask.jsonify({"body": "No faces found"})
        response.headers.add("Access-Control-Allow-Origin", "*")
        return response, 200

    for (x, y, w, h) in faces:
        # Capture image of face, resize to 48x48
        faceImage = img[y:y+w,x:x+h]
        faceImage = cv2.resize(faceImage,(48,48))
        # Convert to numpy array and expand dimensions to be used as inputs for models
        imgArr = np.array(faceImage, dtype=np.float32)
        imgArr = np.expand_dims(imgArr, axis=0)

    try:
        # Load model
        print("Loading model...")
        interpreter = tflite.Interpreter(model_path="model.tflite")
        interpreter.allocate_tensors()

        # Load input data to model
        inputDetails = interpreter.get_input_details()
        outputDetails = interpreter.get_output_details()
        inputData = np.expand_dims(imgArr, axis=3)
        interpreter.set_tensor(inputDetails[0]['index'], inputData)
        print("Successfully loaded model, running...")

        # Run model
        interpreter.invoke()
        predictions = interpreter.get_tensor(outputDetails[0]['index'])
        emotion = emotionNames[np.argmax(predictions[0])]
        print(emotion)
    except Exception as e:
        print(f"Emotion recognition error - {e}")
        response = flask.jsonify({"body": f"Error identifying emotion"})
        response.headers.add("Access-Control-Allow-Origin", "*")
        return response, 500

    response = flask.jsonify({
        "statusCode": 200,
        "body": "FER successful",
        "emotion": emotion
    })
    response.headers.add("Access-Control-Allow-Origin", "*")
    return response, 200

@app.route("/uploadTrainingDetails", methods=["POST"])
def storeTrainingData():
    try:
        modelName = str(request.form.getlist('modelName')[0])
        imageNames = json.loads(request.form.getlist('imageNames')[0])
        typeToIdentify = str(request.form.getlist('typeToIdentify')[0])
        responseIndex = str(request.form.getlist('responseIndex')[0])
        emotion = str(request.form.getlist('emotion')[0])
    except Exception as e:
        print(f"Failed to get parameters - {e}")
        response = flask.jsonify({"body": "Invalid parameters"})
        response.headers.add("Access-Control-Allow-Origin", "*")
        return response, 400

    if not(emotion in emotionNames):
        print(f"Invalid emotion {emotion}")
        response = flask.jsonify({"body": f"Invalid emotion {emotion}"})
        response.headers.add("Access-Control-Allow-Origin", "*")
        return response, 400

    if len(imageNames) != 2:
        print(f"Incorrect number of image names")
        response = flask.jsonify({"body": f"Invalid number of image names ({len(imageNames)} found, should be 2)"})
        response.headers.add("Access-Control-Allow-Origin", "*")
        return response, 400

    # Check that the response is a valid positive integer
    try:
        responseIndex = int(responseIndex)
        if responseIndex < 0 or responseIndex > 1:
            print(f"Invalid response index")
            response = flask.jsonify({"body": f"Invalid response index {responseIndex}"})
            response.headers.add("Access-Control-Allow-Origin", "*")
            return response, 400
    except:
        print(f"Invalid response index datatype")
        response = flask.jsonify({"body": f"Invalid response index datatype"})
        response.headers.add("Access-Control-Allow-Origin", "*")
        return response, 400

    responseIndex = str(responseIndex)

    try:
        # Instantiate Firebase connection
        db = firestore.Client()

        # Create string to format image names for DB write
        imageNameStr = ""
        for image in imageNames:
            imageNameStr += image+","
        imageNameStr = imageNameStr[:-1]

        # Write training data to DB
        timestamp = int(time.time())
        document = db.collection("inferenceData").document(f"{timestamp}")
        doc = {
            "modelName": modelName,
            "imageNames": imageNameStr,
            "typeToIdentify": typeToIdentify,
            "responseIndex": responseIndex,
            "emotion": emotion
        }
        # Write to Firestore
        document.set(doc)
    except Exception as e:
        print(f"Could not write to Firestore - {e}")
        response = flask.jsonify({"body": f"Error storing training details"})
        response.headers.add("Access-Control-Allow-Origin", "*")
        return response, 500

    try:
        # Write to Elasticsearch
        res = es.index(index="test-index", body=doc)
    except Exception as e:
        print(f"Could not write to Elasticsearch - {e}")
        response = flask.jsonify({"body": f"Error storing training details"})
        response.headers.add("Access-Control-Allow-Origin", "*")
        return response, 500

    response = flask.jsonify({"body": "Successfully written to db"})
    response.headers.add("Access-Control-Allow-Origin", "*")
    return response, 200

if __name__ == "__main__":
    app.run()


text:
import cv2
import os
import time
import numpy as np
from filters import Filters

class Main:
    def __init__(self):
        # Main parameters
        # self.path = os.getcwd()
        # self.dataset_path = os.path.join(self.path, 'dataset')
        # self.output_path = os.path.join(self.path, 'out')

        self.dataset_path = '/media/marcos/Dados/Projects/Datasets/Exams'
        self.output_path = '/media/marcos/Dados/Projects/Results/Qualificacao/AlgoPPL/frames'
        self.labe_output_path = '/media/marcos/Dados/Projects/Results/Qualificacao/AlgoPPL/label'

        self.exams = ['benchmark_final.avi']
        # self.exams = os.listdir(self.dataset_path)

        self.detail_presentation = True
        self.save_output = True
        self.execute_invisible = True
        self.save_csv = True

        self.sleep_pause = 3

        self.filters = Filters(self.detail_presentation)

    def add_label(self, title, information):
        with open('{}/{}_label.csv'.format(self.labe_output_path, title), 'a', newline='') as file:
            file.write('{}\n'.format(information))
            file.close()

    def start_process(self):
        exam_jump_process = 0
        exam_process = 0
        for exam in self.exams:
            if exam_process >= exam_jump_process:
                name_file = exam.replace('.mp4', '').replace('.avi', '')
                csv_title = 'frame,center_x,center_y,radius,eye_size'
                self.add_label(name_file, csv_title)

                movie = cv2.VideoCapture("{}/{}".format(self.dataset_path, exam))
                self.pupil_process(movie, name_file)
            exam_process += 1

    def pupil_process(self, exam, name_file):
        number_frame = 0
        while True:
            _, frame = exam.read()

            if frame is None:
                break

            path_out = '/media/marcos/Dados/Projects/ProjectPupilometer/pupilometer/out'
            name_out = 'original_{}'.format(number_frame)
            file_out = '{}/{}.png'.format(path_out, name_out)
            cv2.imwrite(file_out, frame)

            center, radius, images = self.filters.pupil_analysis(frame, number_frame)
            final = np.copy(frame)

            final = cv2.circle(final, (center[0], center[1]), radius, (174, 174, 174), 2)

            if not self.execute_invisible:
                cv2.namedWindow('Training', cv2.WINDOW_NORMAL)
                cv2.imshow('Training', images['final'])

            if self.save_output:
                file_out = '{}/{}.png'.format(self.output_path, number_frame)
                cv2.imwrite(file_out, final)

            if self.save_csv:
                information = '{},{},{},{},{}'.format(number_frame, center[0], center[1], radius, 0)
                self.add_label(name_file, information)

            if cv2.waitKey(1) & 0xFF == ord('p'):  # Pause
                time.sleep(self.sleep_pause)

            number_frame += 1

        exam.release()
        cv2.destroyAllWindows

if __name__ == '__main__':
    main = Main()
    main.start_process()

max_stars_repo_name: marcosvsilva/AlgoPPL

max_issues_repo_name: marcosvsilva/AlgoPPL

max_forks_repo_name: marcosvsilva/AlgoPPL


text:
#!/usr/bin/env python2
#-W ignore::DeprecationWarning
# -*- coding: utf-8 -*-
"""
Created on Wed May 31 08:29:12 2017

@author: fboers
"""
import sys,os
#sys.path.append('../../')

import numpy as np
import warnings
with warnings.catch_warnings():
     warnings.filterwarnings("ignore",category=DeprecationWarning)
     import mne

#--- jumeg clases
#from jumeg.filter.jumeg_filter import jumeg_filter
from jumeg.jumeg_base import jumeg_base as jb
from jumeg.epocher.jumeg_epocher import jumeg_epocher

verbose    = True
DO_EVENTS  = True
DO_EPOCHER = True
DO_FILTER  = False

#--- file to process
fpath = '/data/exp/FREEVIEWING/epocher/'
ffif  = '211776_FREEVIEW01_180115_1414_1_c,rfDC,meeg_bcc,tr-raw.fif'
ffif  = '211776_FREEVIEW01_180115_1414_1_c,rfDC,meeg_bcc,tr,nr,ar-raw.fif'

fname = fpath + ffif
raw   = None

#--- set template path
template_name = 'FreeView'
template_path = "/epocher/"

imo_list=["ImoIODBc"]
fv_list=["FVImoBc","FVsaccardeBc","FVfixationBc"]
me_list=["MEImoBc","MEsaccardeBc","MEfixationBc"]
se_list=["SEImoBc","SEsaccardeBc","SEfixationBc"]

condition_list= imo_list  + fv_list + me_list + se_list +["SACBc","FIXBc"]
#condition_list= ["SACBc","FIXBc"]
#condition_list= fv_list

#--- Epocher
if DO_EVENTS:
   print"-"*50
   print "---> EPOCHER Events"
   print "  -> FIF File: "+ fname
   print "  -> Template: "+ template_name+"\n"

   evt_param = { "condition_list":condition_list,
                 "template_path": template_path,
                 "template_name": template_name,
                 "verbose"      : verbose
               }

   raw,fname = jumeg_epocher.apply_events(fname,raw=raw,**evt_param)

#--- EPOCHER
if DO_EPOCHER:
   ep_param={
          "condition_list": condition_list,
          "template_path" : template_path,
          "template_name" : template_name,
          "verbose"       : verbose,
          "event_extention": ".eve",
          "save_condition":{"events":True,"epochs":True,"evoked":True},
          #"weights"       :{"mode":"equal","method":"median","skip_first":null}
         # "exclude_events":{"eog_events":{"tmin":-0.4,"tmax":0.6} } },
           }
#---
   if DO_FILTER:
      from jumeg.filter.jumeg_filter import jumeg_filter
      jf = jumeg_filter()
      jb.picks.exclude_trigger
      jf.vebose = verbose
      jf.apply_filter(raw._data,picks=jb.picks.exclude_trigger(raw) ) # inplace

#---
   print "---> EPOCHER Epochs"
   print "  -> File            : "+ fname
   print "  -> Epocher Template: "+ template_name+"\n"
   raw,fname = jumeg_epocher.apply_epochs(fname=fname,raw=raw,**ep_param)

#---
#--- check
# import pandas as pd
# fhdf=epocher_hdf_fname
# HDF=pd.HDFStore(fhdf)
# HDF['/epocher/ET_sacc_onset']
# HDF.close()

#d=np.ones([10,4])
#a=np.arange(4)+11
#d.T+a
#df=pd.DataFrame(dd,index=range(10), columns=list("abcd"))

   # jee = JuMEG_Epocher_Epochs()
   # jee.epocher_update(fname=fname,raw=raw,**events)
   # jee.marker.type_result='hit'
   # jee.epocher_run(fname=fname,raw=raw,**events)

#fpath = '/home/fboers/MEGBoers/data/exp/FREEVIEWING/epocher/'
#fep  = fpath + '211776_FREEVIEW01_180115_1414_1_c,rfDC,meeg_bcc,tr,nr,ar,ImoIOD_evt-epo.fif'
#fepbc= fpath + '211776_FREEVIEW01_180115_1414_1_c,rfDC,meeg_bcc,tr,nr,ar,ImoIOD_evt_bc-epo.fif'
#ep=mne.read_epochs(fep)
#epbc=mne.read_epochs(fepbc)
#
#ep[0]._data.shape

"""
import pandas as pd
fpath = '/home/fboers/MEGBoers/data/exp/FREEVIEWING/epocher/'
fhdf  = '211776_FREEVIEW01_180115_1414_1_FreeView-epocher.hdf5'
HDFobj=pd.HDFStore(fpath+fhdf)
HDFobj.keys()

key='/epocher/FVsaccardeBc'

df=HDFobj[key]
print df.columns

tsl0=1000
tsl1=50000
resp_type_input ='iod_onset'
resp_type_offset='iod_offset'

early_idx_in = df[ ( tsl0 <= df[resp_type_input] ) & ( df[ resp_type_input ] < tsl1 ) ].index

early_idx_off = df[ ( tsl0 <= df[resp_type_offset] ) & ( df[ resp_type_offset ] < tsl1 ) ].index
early_idx     = np.unique( np.concatenate((early_idx_in,early_idx_off), axis=0) )

In [46]: c=np.arange(10)

In [47]: df['a']+=c

In [48]: df
Out[48]:
      a     b     c     d
0  12.0  13.0  14.0  15.0
1  13.0  13.0  14.0  15.0
2  14.0  13.0  14.0  15.0
3  15.0  13.0  14.0  15.0
4  16.0  13.0  14.0  15.0
5  17.0  13.0  14.0  15.0
6  18.0  13.0  14.0  15.0
7  19.0  13.0  14.0  15.0
8  20.0  13.0  14.0  15.0
9  21.0  13.0  14.0  15.0

In [49]: r=df[ ( 12 <= df['a'] ) & ( df[ 'a' ] <= 18 )].index

In [50]: r
Out[50]: Int64Index([0, 1, 2, 3, 4, 5, 6], dtype='int64')

In [51]:

In [51]: r.any()
Out[51]: True

In [52]: df['b'][r]
Out[52]:
0    13.0
1    13.0
2    13.0
3    13.0
4    13.0
5    13.0
6    13.0
Name: b, dtype: float64

In [53]: df['a'][r].isin([14,15,18])
Out[53]:
0    False
1    False
2     True
3     True
4    False
5    False
6     True
Name: a, dtype: bool

In [54]: found=df['a'][r].isin([14,15,18])

In [55]: np.where(found==False)
Out[55]: (array([0, 1, 4, 5]),)

In [56]: np.where(found==False)[0]
Out[56]: array([0, 1, 4, 5])

"""

max_stars_repo_name: fboers/jumeg

max_issues_repo_name: fboers/jumeg

max_forks_repo_name: fboers/jumeg


text:
from rdtools.degradation import degradation_year_on_year
import numpy as np
import pandas as pd

def analyze_yoy(pfit):
    out = {}

    for k in ['photocurrent_ref', 'saturation_current_ref',
              'resistance_series_ref',
              'conductance_shunt_extra', 'diode_factor', 'i_sc_ref',
              'v_oc_ref', 'i_mp_ref', 'v_mp_ref', 'p_mp_ref', 'v_mp_ref_est',
              'i_mp_ref_est', 'p_mp_ref_est',
              'nNsVth_ref']:
        Rd_pct, Rd_CI, calc_info = degradation_year_on_year(pd.Series(pfit[k]),
                                                            recenter=False)
        renorm = np.median(pfit[k])
        if renorm == 0:
            renorm = np.nan

        out[k] = {
            'change_per_year': Rd_pct * 1e-2,
            'percent_per_year': Rd_pct / renorm,
            'change_per_year_CI': np.array(Rd_CI) * 1e-2,
            'percent_per_year_CI': np.array(Rd_CI) / renorm,
            'calc_info': calc_info,
            'median': np.median(pfit[k])}

    return out

max_stars_repo_name: cwhanse/pvpro

max_issues_repo_name: cwhanse/pvpro

max_forks_repo_name: cwhanse/pvpro


text:
# -*- coding: utf-8 -*-
"""cifar10.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1hvQQB0awUJf5zku5A5Z_GByf80C56hqD
"""

from keras.datasets import cifar10
from keras.layers import Dense , Conv2D ,  Flatten , MaxPool2D , BatchNormalization , Dropout
from keras.models import Sequential
from keras.utils import to_categorical
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

#PREPROCESSING

(X_train , y_train) ,(X_test , y_test) = cifar10.load_data()
def custom_preprocess(data):
  data = data.astype('float')
  data /= 255.0
  return data
X_train  = custom_preprocess(X_train)
X_test = custom_preprocess(X_test)

plt.imshow(X_train[0])

X = np.vstack((X_train , X_test))
Y = np.vstack((y_train , y_test))
# Y = to_categorical(Y)
X[0].shape

from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import f1_score

## Optimizer colleciton:
optzer = []
optzer_name = []
#-------------------------------------------------------------------------------
from keras.optimizers import Adam
##ref : https://arxiv.org/pdf/1412.6980v8.pdf
optzer.append(Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False))
optzer_name.append('Adam')
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
from keras.optimizers import SGD
optzer.append(SGD(learning_rate=0.01, momentum=0.0, nesterov=False))
optzer_name.append('SGD')
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
from keras.optimizers import RMSprop
## ref: http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf
optzer.append(RMSprop(learning_rate=0.001, rho=0.9))
optzer_name.append('RMSprop')
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
from keras.optimizers import Adagrad
## ref: http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf
optzer.append(Adagrad(learning_rate=0.01))
optzer_name.append('Adagrad')
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
from keras.optimizers import Adadelta
## ref: https://arxiv.org/pdf/1212.5701.pdf
optzer.append(Adadelta(learning_rate=1.0, rho=0.95))
optzer_name.append('Adadelta')
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
from keras.optimizers import Adamax
## ref: https://arxiv.org/pdf/1412.6980v8.pdf
optzer.append(Adamax(learning_rate=0.002, beta_1=0.9, beta_2=0.999))
optzer_name.append('Adamax')
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
from keras.optimizers import Nadam
## ref: http://cs229.stanford.edu/proj2015/054_report.pdf
optzer.append(Nadam(learning_rate=0.002, beta_1=0.9, beta_2=0.999))
optzer_name.append('Nadam')
#-------------------------------------------------------------------------------

seed = 7
np.random.seed(seed)

kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)
cvscores = []
History = []
F1_score = []
for train, test in kfold.split(X, Y):

  model = Sequential()
  model.add( Conv2D(input_shape = (32, 32,3) , filters = 32 , kernel_size=(3,3) , padding = 'same' , activation = 'relu' , kernel_initializer= 'he_uniform'))
  model.add(BatchNormalization())
  model.add( Conv2D(filters = 32 , kernel_size = (3,3) , padding = 'same' , activation = 'relu' , kernel_initializer = 'he_uniform'))
  model.add( MaxPool2D(2,2))
  model.add( Dropout(0.2))

  model.add( Conv2D(filters = 64 , kernel_size = (3,3) , padding = 'same' , activation = 'relu' , kernel_initializer = 'he_uniform'))
  model.add(BatchNormalization())
  model.add( Conv2D(filters = 64 , kernel_size = (3,3) , padding = 'same' , activation = 'relu' , kernel_initializer = 'he_uniform'))
  model.add( MaxPool2D(2,2))
  model.add( Dropout(0.4))

  model.add( Conv2D(filters = 128 , kernel_size = (3,3) , padding = 'same' , activation = 'relu' , kernel_initializer = 'he_uniform'))
  model.add(BatchNormalization())
  model.add( Conv2D(filters = 128 , kernel_size = (3,3) , padding = 'same' , activation = 'relu' , kernel_initializer = 'he_uniform'))
  model.add( MaxPool2D(2,2))
  model.add( Dropout(0.5))

  model.add(Flatten())
  model.add(Dense(units = 128 , activation = 'relu' , kernel_initializer = 'he_uniform'))
  model.add(Dense(units = 10 , activation = 'softmax'))
  model.compile(optimizer = 'adam' , loss = 'categorical_crossentropy' , metrics = ['accuracy'])
  History.append(model.fit(x = X[train], y = to_categorical(Y[train]) , epochs=50 , batch_size = 128 , validation_data=(X[test] , to_categorical(Y[test]))))
  scores = model.evaluate(X[test] , to_categorical(Y[test]))
  cvscores.append(scores[1] * 100)
  F1_score.append(f1_score( Y[test] ,  model.predict_classes(X[test]) , average = 'micro') )

print("%.2f%% (+/- %.2f%%)" % (np.mean(cvscores), np.std(cvscores)))

# create data generator
datagen = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True)
# prepare iterator
it_train = datagen.flow(X_train, y_train, batch_size=64)

# fit model
steps = int(X_train.shape[0] / 64)
history = model.fit_generator(it_train, steps_per_epoch=steps, epochs=100, validation_data=(testX, testY), verbose=0)

print("Total Accuracy is %.2f%% (+/- %.2f%%)" % (np.mean(cvscores), np.std(cvscores)))
print("F1 Score is %.2f%% (+/- %.2f%%)" % (np.mean(F1_score), np.std(F1_score)))

#USING MULTI LEVEL PERCEPTRON

model = Sequential()
model.add(Dense(256, activation='relu', input_dim=(32*32*3)))
model.add(Dense(256, activation='relu'))
model.add(Dense(10, activation='softmax'))

#### Loop over different optimizers
for algo in optzer:
    model.compile(optimizer=algo,
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])
    history = model.fit(X_train.reshape(50000 , 32*32*3),to_categorical(y_train), epochs=15, batch_size=32, verbose=2, validation_split=0.2)
    score = model.evaluate(X_test.reshape(10000 , 32*32*3), to_categorical(y_test), batch_size=128)
    print(model.metrics_names)
    print(score)

    #MODEL SUMMARY AND PLOTS

    def summarize_diagnostics(history, optimizer):
        plt.subplot(211)
        plt.title('Cross Entropy Loss - Optimizer: ' + optimizer)
        plt.xlabel(' ')
        plt.ylabel(' ')
        plt.plot(history.history['loss'], color='blue', label='train')
        plt.plot(history.history['val_loss'], color='orange', label='test')
        plt.subplot(212)
        plt.title('Classification Accuracy - Optimizer: ' + optimizer)
        plt.xlabel(' ')
        plt.ylabel(' ')
        plt.plot(history.history['acc'], color='blue', label='train')
        plt.plot(history.history['val_acc'], color='orange', label='test')

    i = 1
    opttcount = 0
    for h in History:
      summarize_diagnostics(h, optzer_name[opttcount ])
      plt.show()
      print('For Fold: ',i)
      print('-------------------------------------------------------------')
      i+=1
      opttcount += 1

max_stars_repo_name: siavashtab/CNN-optimizers

max_issues_repo_name: siavashtab/CNN-optimizers

max_forks_repo_name: siavashtab/CNN-optimizers


text:
import os,sys, numpy as np
from mcni.neutron_storage.idf_usenumpy import totalintensity, count
from matplotlib import pyplot as plt
from scipy.optimize import differential_evolution
from mcvine import run_script
# from sampleassembly_program import makeSAXML

from mcni.neutron_storage.idf_usenumpy import totalintensity, count
from mantid2mcvine.nxs import template as nxs_template

parent_dir = os.path.abspath(os.pardir)
beam_path=os.path.join(parent_dir, 'beam')
mantid_path=os.path.join(parent_dir, 'mantid')
result_path=os.path.join(parent_dir, 'results')

libpath = os.path.join(parent_dir, 'c3dp_source')

if not libpath in sys.path:
    sys.path.insert(0, libpath)

from collimator_geometry_vulcan import create as create_collimator_geometry, Parameter_error
import convert2nxs as det2nxs
import rotateAndtranslate_detector_for_reduction_mantid as rot
import reduce_nexasdata_using_mantid as red
import masking_nexus_givenKernel as mask

class Peak:
    def __init__(self, label='Si 111', d_spacing=3.345, d_min=3., d_max=3.5):
        self.label = label
        self.d_spacing = d_spacing
        self.dmin = d_min
        self.dmax = d_max

class Conf:
    def __init__(self, moderatorTosample_z, angleMons, sampleTodet_z ):
        self.moderatorTosample_z= moderatorTosample_z
        self.angleMons= angleMons
        self.sampleTodet_z = sampleTodet_z

Si_111_peak = Peak(label='Si 111', d_spacing=3.345, d_min=3., d_max=3.5)
Al_111_peak = Peak(label='Al 111', d_spacing=2.3, d_min=2.2, d_max=2.5)
Cu_111_peak = Peak(label='Cu 111', d_spacing=2.08, d_min=2., d_max=2.1)
Si_220_peak = Peak(label='Si 220', d_spacing=1.9, d_min=1.86, d_max=2)
Cu_200_peak = Peak(label='Cu 200', d_spacing=1.8, d_min=1.78, d_max=1.85)

class PresureCell(object):

    def parameters(self, Snap_angle = False, coll_sim = False  ,
                   source_file = 'clampcellSi_scattered-neutrons_1e9_det-50_105_new',
        sampleassembly_fileName = 'collimator_plastic',ncount=1e3, sourceTosample_x = 0.0,
        sourceTosample_y = 0.0, sourceTosample_z = 0.0, moderatorTosample_z=-42.254,
        angleMons = [150, -90] ,
        sampleTodetector_z=[2, 2], detector_width=[0.35], detector_height=[0.76],
        number_pixels_in_height=[256],
        number_pixels_in_width=[8], number_of_box_in_height=[1],
        number_of_box_in_width=[9], masking = False,
        masked_template = 'coll_plastic_SCAT_masked.nxs', number_of_detectors= 1,
        peaks={'sample':[Si_111_peak,Si_220_peak ], 'cell' : [Al_111_peak,Cu_111_peak,Cu_200_peak]} ):

        self.Snap_angle = Snap_angle
        self.coll_sim = coll_sim
        self.source_file = source_file
        self.sampleassembly_fileName = sampleassembly_fileName
        self.ncount=ncount
        self.sourceTosample_x = sourceTosample_x
        self.sourceTosample_y = sourceTosample_y
        self.sourceTosample_z = sourceTosample_z
        self.angleMons = angleMons
        self.detector_width = detector_width
        self.detector_height = detector_height
        self.sampleTodetector_z = sampleTodetector_z
        self.number_pixels_in_height = number_pixels_in_height
        self.number_of_box_in_height= number_of_box_in_height
        self.number_pixels_in_width = number_pixels_in_width
        self.number_of_box_in_width = number_of_box_in_width
        self.masking = masking
        self.masked_template = masked_template
        self.peaks= peaks
        self.moderatorTosample_z= moderatorTosample_z
        self.number_of_detectors = number_of_detectors
        self.number_of_total_DetectorPixels = 0
        # for i in range(self.number_of_detectors):
        #     self.number_of_total_DetectorPixels += self.number_pixels_in_height[i]\
        #                                       *self.number_of_box_in_height[i]\
        #                                       *self.number_pixels_in_width[i]\
        #                                       *self.number_of_box_in_width[i]

        self.number_of_total_DetectorPixels =  self.number_pixels_in_height \
                                               * self.number_of_box_in_height \
                                               * self.number_pixels_in_width \
                                               * self.number_of_box_in_width

    def source_neutrons(self):
        scattered =os.path.join( beam_path, self.source_file)
        return(scattered)

    def diffraction_pattern_calculation(self,params=[501.9, 444.5]):

        channel_len, coll_front_end_from_center = params

        if  self.coll_sim:

            create_collimator_geometry(
                coll_front_end_from_center=coll_front_end_from_center,
                chanel_length=channel_len,
                Snap_angle= self.Snap_angle,
                detector_angles=[150]) #150

            name = "length_%s-dist_%s" % (channel_len, coll_front_end_from_center)

        else:
            name = self.sampleassembly_fileName

        simdir = os.path.join(
            parent_dir,
            "out/%s" %
            (name))

        # instr = os.path.join(libpath, 'myinstrument_multipleDet.py')

        instr = os.path.join(libpath, 'myinstrument.py')

        # scattered_neutrons ='clampcellSi_scattered-neutrons_1e9_det-50_105_new'

        # scatterer = {(sampleassembly_fileName, 'shapeColl', 'coll_geometry', 'H', 'xyz'),
        #              }
        #
        # sa.makeSAXML(sampleassembly_fileName, scatterer)

        # sample= 'collimator_plastic'
        import shutil
        if os.path.exists(simdir):
            shutil.rmtree(simdir)
        # run_script.run_mpi(instr, simdir,
        #                    beam=self.source_neutrons(), ncount=self.ncount,
        #                    nodes=20, angleMons=self.angleMons,
        #                    sample=self.sampleassembly_fileName,
        #                    sourceTosample_x=self.sourceTosample_x, sourceTosample_y=self.sourceTosample_y,
        #                    sourceTosample_z=self.sourceTosample_z, sampleTodetector_z=self.sampleTodetector_z,
        #                    detector_width=self.detector_width, detector_height=self.detector_height,
        #                    number_pixels_in_height=self.number_pixels_in_height,
        #                    number_of_box_in_height=self.number_of_box_in_height,
        #                    number_pixels_in_width=self.number_pixels_in_width,
        #                    number_of_box_in_width= self.number_of_box_in_width,
        #                    number_detectors=self.number_of_detectors,
        #                    overwrite_datafiles=True)

        run_script.run_mpi(instr, simdir,
                           beam=self.source_neutrons(), ncount=self.ncount,
                           nodes=20,
                           angleMon1=150,
                           sample=self.sampleassembly_fileName,
                           sourceTosample_x=self.sourceTosample_x,
                           sourceTosample_y=self.sourceTosample_y,
                           sourceTosample_z=self.sourceTosample_z,
                           sampleTodetector_z=self.sampleTodetector_z,
                           detector_width=self.detector_width,
                           detector_height=self.detector_height,
                           number_pixels_in_height=self.number_pixels_in_height,
                           number_of_box_in_height=self.number_of_box_in_height,
                           number_pixels_in_width=self.number_pixels_in_width,
                           number_of_box_in_width=self.number_of_box_in_width,
                           multiple_detectors=False,
                           overwrite_datafiles=True)

        SNAP_definition_file = os.path.join(mantid_path,
                                            'SNAPOneDet_virtual_definition.xml')  # Vulcan_virtual_Definition.xml

        template = os.path.join(mantid_path, 'template_snapOneDet.nxs')  # vulcan_template.nxs

        nxs_template.create(
            SNAP_definition_file,
            ntotpixels=self.number_of_total_DetectorPixels, outpath=template, pulse_time_end=1e5
        )

        nexus_file_path = os.path.join( mantid_path, '{}.nxs'.format(name))

        det2nxs.create_nexus(simdir, nexus_file_path, template, N=self.number_of_total_DetectorPixels)

        nexus_file_correctDet_path = os.path.join(mantid_path, '{}_correctDet.nxs'.format(name))

        conf = Conf(self.moderatorTosample_z, self.angleMons, self.sampleTodetector_z)
        rot.detector_position_for_reduction(nexus_file_path, conf,
                                            SNAP_definition_file, nexus_file_correctDet_path)

        if self.masking :
            masked_file_path = os.path.join(mantid_path, '{}_masked.nxs'.format(name))

            masked_template_path = os.path.join(mantid_path, '{}'.format(self.masked_template) )

            mask.masking(nexus_file_correctDet_path, masked_template_path, masked_file_path)

        else:
            masked_file_path = nexus_file_correctDet_path

        binning = [0.5, 0.01, 4.]
        d_sim, I_sim, error = red.mantid_reduction(masked_file_path, binning)

        plt.figure()
        plt.errorbar (d_sim, I_sim, error)
        plt.xlabel('d_spacing ()')
        plt.show()

        np.save(os.path.join(result_path, 'I_d_{sample}.npy'.format(sample=name)), [d_sim, I_sim, error])

        return (d_sim, I_sim, error)

    # def peak_detection(self, para):

    def collimator_inefficiency(self, params):

        dcs, I_d, error = self.diffraction_pattern_calculation (params)

        scattered = os.path.join(beam_path, 'clampcellSi_neutron')

        ncount= 1e9

        sample_peaks = self.peaks['sample']

        cell_peaks = self.peaks['cell']

        scattered_beam_intensity = totalintensity(scattered) * ncount / count(scattered)

        sample_peak_int=0.0
        cell_peak_int =0.0

        for sample_peak in sample_peaks:
            sample_peak_int += I_d[ (dcs<sample_peak.dmax) & (dcs>sample_peak.dmin)].sum() #I_d[ (dcs<3.5) & (dcs>3)].sum()

        for cell_peak in cell_peaks:
            cell_peak_int +=  I_d[ (dcs<cell_peak.dmax) & (dcs>cell_peak.dmin)].sum()  #I_d[np.logical_and(dcs<2.2, dcs>2)].sum()

        # collimator_ineff=(cell_peak_int/sample_peak_int)+(1-sample_peak_int/scattered_beam_intensity)

        collimator_ineff = (cell_peak_int / sample_peak_int)  # new objective function suggested by Garrett

        print ('coll_len,:', params[0] , 'focal_distance,:', params[1]  ,'collimator_performance: ', (sample_peak_int/cell_peak_int))

        return (collimator_ineff)

    def collimator_performance (self, params):
        return (1/self.collimator_inefficiency(params))

    def objective_func(self,params):
        try:
            return (self.collimator_inefficiency(params))

        except Parameter_error as e:
            return (1e10)

    def optimize(self, params_bounds = [(2.0, 8.0), (17.0, 50.0)]):
        # objective_func.counter = 0

        result = differential_evolution(self.objective_func, params_bounds,popsize=4,maxiter=10)
        with open(os.path.join(result_path, 'optimized_Collimator_Dimension.dat'), "w") as res:
            res.write(result.x)
        return(result.x)

reason: from scipy


text:
"""
CTQA Phantom Centering

Provides the functionality for detecting the center of a phantom within an image
"""

import cv2
import pydicom
import numpy as np
import matplotlib.pyplot as plt
import uuid
from os import listdir
from os.path import isfile, join

def threshold_image(original_img, threshold):
  """Applies a threshold to a grayscale image"""
  # original_img is numpy array
  # threshold is Hounsfield units
  img = np.array(original_img)
  img[img > threshold] = 255
  img[img <= threshold] = 0

  return img

def get_circles(pixel_array, threshold=-100):
  """Applies Hough Circles to a grayscale image"""
  # pixel_array should be a 2D Array
  # threshold defaults to 100 Hounsfield units
  # 100 = the equivalent lower bound of soft tissue

  # Thresholding image
  orig_img = np.array(pixel_array)
  img = threshold_image(orig_img, threshold)
  img = np.uint8(img)

  # Hough circles detection
  circles = cv2.HoughCircles(img,cv2.HOUGH_GRADIENT,2.4,100)
  # returning a single circle's coords
  circle_coords = np.uint16(np.around(circles))[0,:]

  return circle_coords

def save_image_circles(pixel_array, circles, path):
  """Saves grayscale image with circle drawn"""
  orig_img = np.uint8(pixel_array)
  cimg = cv2.cvtColor(orig_img,cv2.COLOR_GRAY2BGR)

  for i in circles:
      # draw the outer circle
      cv2.circle(cimg,(i[0],i[1]),i[2],(0,255,0),2)
      # draw the center of the circle
      cv2.circle(cimg,(i[0],i[1]),2,(0,0,255),3)

  save_file = join(path, str(uuid.uuid4()) + '.jpg')
  cv2.imwrite(save_file, cimg)

def set_window(pixel_array, window_level, window_width):
  """Sets the window level and width for the image"""
  scale_img = np.array(pixel_array)
  # Getting low and high values
  low = window_level - window_width/2
  high = window_level + window_width/2

  def bin_function(i, window_level, window_width, high, low):
    """Bins image with high and low levels"""
    # Binning pixel
    if i < low:
      i = 0
    elif i > high:
      i = 255
    else:
      i = (((i - (window_level - 0.5)) / (window_width - 1)) + 0.5) * 255;
    return i
  bin_function = np.vectorize(bin_function)
  scale_img = np.uint8(bin_function(pixel_array, window_level, window_width, high, low))

  return scale_img

def get_scaled_image(ds):
  """Retrieves an appropriately scaled image from a pydicom object"""
  # ds should be a pydicom loaded DICOM CT object
  if hasattr(ds, 'RescaleIntercept') and hasattr(ds, 'RescaleSlope'):
      intercept = ds.RescaleIntercept
      slope = ds.RescaleSlope
      img = np.array(ds.pixel_array)
      img = (img*slope) + intercept
  else:
      img = np.array(ds.PixelData)

  return img

if __name__ == "__main__":
  root = 'audit_files'

  for f in listdir(root):
    save_loc = 'phantom_circle_select'
    save_file = join(save_loc, f+'.jpg')
    curr_file = join(root, f)
    if not isfile(curr_file) or f[0] == '.':
      continue

    print(curr_file)
    ds = pydicom.dcmread(curr_file)
    img = get_scaled_image(ds)

    threshold = -100

    # For showing what the threshold image looks like
    # orig_img = np.uint8(threshold_image(img, threshold))

    # For showing a regular CT image
    orig_img = set_window(img, 0, 50)

    cimg = cv2.cvtColor(orig_img,cv2.COLOR_GRAY2BGR)

    # Hough circles detection
    circles = get_circles(img, threshold=threshold)

    for i in circles:
      # draw the outer circle
      cv2.circle(cimg,(i[0],i[1]),i[2],(0,255,0),5)
      # draw the center of the circle
      print(i[0],i[1])
      cv2.circle(cimg,(i[0],i[1]),2,(0,0,255),5)

    cv2.imwrite(save_file, cimg)

max_stars_repo_name: Brikwerk/ctqa

max_issues_repo_name: Brikwerk/ctqa

max_forks_repo_name: Brikwerk/ctqa


text:
import numpy as np
import sys
import os
import pickle
import glob
import time
import types

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

import torch as tc
import torch.tensor as T
from torch import nn, optim
from .utils import *

class SGD:
    def __init__(self, params, model):
        ## set default parameters
        if not hasattr(params, "weight_decay_sgd"):
            params.weight_decay_sgd = 0.0

        self.params = params
        self.model = model
        if self.params.use_gpu:
            self.device = tc.device("cuda:0")
        else:
            self.device = tc.device("cpu")

        self.params.exp_root = os.path.join(self.params.snapshot_root, self.params.exp_name)
        if not os.path.exists(self.params.exp_root):
            os.makedirs(self.params.exp_root)
        self.param_fn = "model_params"
        self.opt_params = None
        if not hasattr(params, "vis"):
            self.params.vis = False

        self.model.to(self.device)

    def set_opt_params(self, opt_params):
        self.opt_params = opt_params

    def train_post_epoch_processing(self):
        pass

    def set_stop_criterion(self):
        pass

    def stop_criterion(self, i):
        return False

    def train(self, ld_tr, ld_val):
        self.ld_tr = ld_tr
        self.ld_val = ld_val

        ## init predictor
        self.model = self.model.to(self.device)

        ## load
        if hasattr(self.params, "load_model") and self.params.load_model:
            if self.model.load(
                os.path.join(self.params.exp_root, self.param_fn)):
                return

        ## optimizer parameters
        if self.opt_params is None:
            print("self.opt_params is empty")
            return
        else:
            opt_params = self.opt_params

        ## optimizer
        if self.params.optimizer == "Adam":
            opt = optim.Adam(opt_params, lr=self.params.lr)
        elif self.params.optimizer == "AMSGrad":
            opt = optim.Adam(opt_params, lr=self.params.lr, amsgrad=True)
        elif self.params.optimizer == "SGD":
            opt = optim.SGD(opt_params, lr=self.params.lr, momentum=self.params.momentum, weight_decay=self.params.weight_decay_sgd)
        else:
            raise NotImplementedError

        scheduler = optim.lr_scheduler.StepLR(
            opt, self.params.lr_decay_epoch, self.params.lr_decay_rate)

        ## train
        val_error_best = np.inf
        self.val_error_best = np.inf
        self.set_stop_criterion()
        for i in range(self.params.n_epochs):
            self.current_epoch = i+1
            t_start = time.time()
            self.model.train()
            loss = self.train_epoch(ld_tr, opt)
            # update lr
            scheduler.step()

            # print iteration status for every epoch
            if (i+1) % 1 == 0:
                print('[%d/%d, lr=%f, %f sec.] loss: %f'%
                      (i+1, self.params.n_epochs,
                       opt.param_groups[0]['lr'], time.time()-t_start,
                       loss))
                if self.params.vis:
                    self.vis(ld_tr)

            # eval on given datasets
            if (i+1) % self.params.n_epoch_eval == 0:
                self.test([ld_tr, ld_val], ["train", "val"])

            # save best model for every epoch
            if self.params.keep_best:
                val_error = self.validate(ld_val, i)
                if val_error <= val_error_best:
                    val_error_best = val_error
                    print("[model saved] val_eror_best = %4.2f%%"%(val_error_best*100.0))
                    if self.params.save_model:
                        self.model.save(os.path.join(
                            self.params.snapshot_root,
                            self.params.exp_name,
                            self.param_fn))
                    self.val_error_best = val_error_best
                print()
            else:
                if self.params.save_model:
                    self.model.save(os.path.join(self.params.snapshot_root, self.params.exp_name, self.param_fn))

            # post-epoch process
            self.train_post_epoch_processing()
            # stop
            if self.stop_criterion(i):
                break

        self.model.eval()
        if self.params.save_model:
            if self.params.keep_best == False:
                # save the last model
                self.model.save(os.path.join(self.params.snapshot_root, self.params.exp_name, self.param_fn))
            else:
                # load the best model
                self.model.load(os.path.join(self.params.snapshot_root, self.params.exp_name, self.param_fn))

    def train_epoch(self, ld_tr, opt):
        if hasattr(self.params, "label_weight"):
            loss_fn = nn.CrossEntropyLoss(weight=self.params.label_weight)
        else:
            loss_fn = nn.CrossEntropyLoss()

        for xs_src, ys_src in ld_tr:
            xs_src = xs_src.to(self.device)
            ys_src = ys_src.to(self.device)

            # init for backprop
            opt.zero_grad()
            # compute loss
            fhs = self.model(xs_src)
            loss = loss_fn(fhs, ys_src)
            # backprop
            loss.backward()
            # update parameters
            opt.step()
        return loss

    def validate(self, ld, i_epoch):
        self.model.eval()
        error, _, _ = compute_cls_error([ld], self.model, self.device)
        return error

    def test(self, lds, ld_names, model=None):
        if model is None:
            model = self.model

        model.eval()

        ## classification error
        if ld_names is not None:
            assert(len(lds) == len(ld_names))
        errors = []

        for i, ld in enumerate(lds):
            error, n_error, n_total = compute_cls_error([ld], model, self.device)

            if ld_names is not None:
                print("# %s classification error: %d / %d  = %.2f%%"%(
                    ld_names[i], n_error, n_total, error * 100.0))
            else:
                print("# classification error: %d / %d  = %.2f%%"%(n_error, n_total, error * 100.0))
            errors.append(error.unsqueeze(0))
        errors = tc.cat(errors)

        if len(lds) > 1:
            ## combined classification error
            error, n_error, n_total = compute_cls_error(lds, model, self.device)
            print("# Combined classification error: %d / %d  = %.2f%%"%(n_error, n_total, error * 100.0))

            ## average classification error
            print("# Average classification error = %.2f%%"%(errors.mean() * 100.0))

        return errors

    def vis(self, ld):
        # parameters
        x_rng = [-1.5, 1.5]
        y_rng = [-1.5, 1.5]
        delta_plot = 0.005
        ## draw all data
        xs = []
        ys = []
        for x, y in ld:
            xs.append(x)
            ys.append(y)
        xs = tc.cat(xs, 0)
        ys = tc.cat(ys)
        # init
        plt.figure(1)
        plt.clf()
        # plot positive data
        plt.plot(xs[ys==1, 0].cpu().detach().numpy(), xs[ys==1, 1].cpu().detach().numpy(), 'go')
        plt.plot(xs[ys==0, 0].cpu().detach().numpy(), xs[ys==0, 1].cpu().detach().numpy(), 'ro')
        # beautify plot
#         plt.xlim(x_rng)
#         plt.ylim(y_rng)
        plt.gca().set_aspect('equal')
        plt.grid()
        plt.gca().get_xaxis().set_visible(False)
        plt.gca().get_yaxis().set_visible(False)

        ## plot a decision boundary
        xi = np.arange(x_rng[0], x_rng[1]+delta_plot, delta_plot)
        yi = np.arange(y_rng[0], y_rng[1]+delta_plot, delta_plot)
        X, Y = np.meshgrid(xi, yi)
        xs_grid = np.hstack((np.expand_dims(X.flatten(), 1), np.expand_dims(Y.flatten(), 1)))
        phs = self.model.prob_pred(tc.tensor(xs_grid, dtype=tc.float).to(self.device))
        phs_grid = phs[:, 1].cpu().detach().numpy()
        # plot a decision boundary
        levels = np.arange(0.0, 1.01, 0.1)
        zs_grid = np.reshape(phs_grid, X.shape)
        CS = plt.contour(xi, yi, zs_grid, levels=levels, linewidths=0.5, colors='k')
        plt.contourf(xi, yi, zs_grid, levels=levels, cmap="RdYlGn", alpha=0.5)
        plt.clabel(CS, inline=False, fontsize=10, colors="k")
        # save figure
        fig_fn = os.path.join(
            self.params.snapshot_root,
            self.params.exp_name,
            "data_plot.png")
        plt.savefig(fig_fn, bbox_inches='tight')


text:
import numpy as np
from collections import OrderedDict
from ConfigSpace.util import deactivate_inactive_hyperparameters
from openbox.optimizer.generic_smbo import SMBO
from openbox.utils.config_space import ConfigurationSpace
from openbox.utils.constants import SUCCESS
from openbox.core.base import Observation

class AdaptiveTuner(object):
    def __init__(self, objective_function, config_space, importance_list,
                 strategy='default', max_run=100, step_size=10, random_state=1):
        self.objective_function = objective_function
        self.importance_list = importance_list
        self.strategy = strategy
        self.config_space = config_space
        self.config_space.seed(random_state)
        self.step_size = step_size
        self.max_run = max_run
        self.random_state = random_state
        self._hp_cnt = 0
        self._delta = 2
        self._trial_num = 0
        self.history_dict = OrderedDict()
        self.hp_size = len(self.importance_list)

        # Obtain the default values.
        hps = self.config_space.get_hyperparameters()
        self.defaults = dict()
        for _hp in hps:
            self.defaults[_hp.name] = _hp.default_value

        # Safeness check.
        if len(importance_list) != len(self.config_space.get_hyperparameters()):
            raise ValueError('The length of importance list should be equal to the hyperspace\'s')
        hyper_names = [hp.name for hp in self.config_space.get_hyperparameters()]
        for _hp in importance_list:
            if _hp not in hyper_names:
                raise ValueError('HP name %s is not in the hyper-parameter space.' % _hp)

    def run(self):
        while self._trial_num < self.max_run:
            self.iterate()
            self._trial_num += self.step_size

    def get_configspace(self):
        hp_num = self._hp_cnt + self._delta
        if hp_num > self.hp_size:
            hp_num = self.hp_size

        hps = self.config_space.get_hyperparameters()
        cs = ConfigurationSpace()
        for _id in range(hp_num):
            _hp_id = self.importance_list[_id]
            for _hp in hps:
                if _hp.name == _hp_id:
                    cs.add_hyperparameter(_hp)

        history_list = list()
        if len(self.history_dict.keys()) > 0 and self._hp_cnt < self.hp_size:
            for _config in self.history_dict.keys():
                # Impute the default value for new hyperparameter.
                _config_dict = _config.get_dictionary().copy()
                for _idx in range(self._hp_cnt, hp_num):
                    new_hp = self.importance_list[_idx]
                    # print('hp_num=', self._hp_cnt, 'new hp is', new_hp)
                    _config_dict[new_hp] = self.defaults[new_hp]
                history_list.append((_config_dict, self.history_dict[_config]))
        if len(history_list) == 0:
            history_list = [(_config, self.history_dict[_config]) for _config in self.history_dict.keys()]
        return cs, history_list

    def evaluate_wrapper(self, config):
        # Impute the missing hyper-parameters with default values.
        config_dict = config.get_dictionary()
        included_keys = config_dict.keys()
        all_keys = self.defaults.keys()
        for _missing_key in list(set(all_keys) - set(included_keys)):
            config_dict[_missing_key] = self.defaults[_missing_key]
        _config = deactivate_inactive_hyperparameters(configuration_space=self.config_space,
                                                      configuration=config_dict)
        return {'objs': (self.objective_function(_config),)}

    def iterate(self):
        config_space, hist_list = self.get_configspace()
        # print(self._hp_cnt, config_space)
        # print(self._hp_cnt, hist_list)

        # Set the number of initial number.
        if len(hist_list) > 0:
            init_num = 0
        else:
            init_num = 3

        # Set the number of iterations.
        # eta = 3
        # if self._hp_cnt > 0:
        #     iter_num = eta ** (self._hp_cnt + 1) - eta ** self._hp_cnt
        #     if eta ** (self._hp_cnt + 1) > self.max_run:
        #         iter_num = self.max_run - eta ** self._hp_cnt
        # else:
        #     iter_num = eta
        iter_num = self.step_size

        smbo = SMBO(self.evaluate_wrapper, config_space,
                    advisor_type=self.strategy,
                    max_runs=iter_num,
                    init_num=init_num,
                    task_id='smbo%d' % self._hp_cnt,
                    random_state=self.random_state)

        # Set the history trials.
        for _config_dict, _perf in hist_list:
            config = deactivate_inactive_hyperparameters(configuration_space=config_space,
                                                         configuration=_config_dict)
            _observation = Observation(config, SUCCESS, None, (_perf,), None)
            smbo.config_advisor.history_container.update_observation(_observation)
        smbo.run()

        # Save the runhistory.
        self.history_dict = OrderedDict()
        for _config, perf in zip(smbo.config_advisor.history_container.configurations,
                                 smbo.config_advisor.history_container.perfs):
            self.history_dict[_config] = perf

        self._hp_cnt += self._delta
        if self._hp_cnt > self.hp_size:
            self._hp_cnt = self.hp_size

    def get_history(self):
        return list(self.history_dict.items())

    def get_incumbent(self):
        items = list(self.history_dict.items())
        inc_idx = np.argmin([_item[1] for _item in items])
        return items[inc_idx]


text:
#coding: utf-8
import numpy as np
from scipy import optimize

"""
Conjugate Gradientによるコスト関数を最小化する
パラメータの求め方
"""

def J(theta, *args):
    """最小化を目指すコスト関数を返す"""
    theta1, theta2 = theta
    return (theta1 - 5) ** 2 + (theta2 - 5) ** 2

def gradient(theta, *args):
    """コスト関数の偏微分を返す
    各パラメータで偏微分した関数リストを返す"""
    theta1, theta2 = theta
    # Jをtheta1で偏微分した関数
    gt1 = 2 * (theta1 - 5)
    # Jをtheta2で偏微分した関数
    gt2 = 2 * (theta2 - 5)
    return np.array((gt1, gt2))

if __name__ == "__main__":
    # パラメータの初期値
    initial_theta = np.array([0, 0])

    # Conjugate Gradientによるコスト関数最小化
    theta = optimize.fmin_cg(J, initial_theta, fprime=gradient)
    print "conjugate gradient:", theta

    # BFGSによるコスト関数最小化
    theta = optimize.fmin_bfgs(J, initial_theta, fprime=gradient)
    print "BFGS:", theta

max_stars_repo_name: aidiary/PRML

max_issues_repo_name: sojvai/PRML

max_forks_repo_name: sojvai/PRML

reason: import numpy,from scipy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function
from flask import Flask, render_template, jsonify, json, request, session, flash, redirect

import os
import sys
import time
import numpy as np
import json

sys.path.append('pythonFlask/control')
from TargetControllerClass import TargetController
from SimilarityControllerClass import SimilarityController
from DrugControllerClass import DrugController

app = Flask(__name__) # , template_folder='view'
app.secret_key = 'any random string'

@app.route('/')
def index():
	return render_template("index.html")

@app.route('/targetsForm')
def targetsForm():
	iframe = "/static/angularJS/iframe/demo.html"
	return render_template("compound-target-form.html", iframe = iframe)

@app.route('/similarityForm')
def similarityForm():
	hora = time.time()
	iframe = "/static/angularJS/iframe/demo.html"
	return render_template("compound-similarity-form.html", iframe = iframe, time = hora)

@app.route('/drugForm')
def drugForm():
	iframe = "/static/angularJS/iframe/demo.html"
	return render_template("compound-info-form.html", iframe = iframe)

@app.route('/targetResults', methods=['GET', 'POST'])
def targetResults():
	if request.method == 'POST':
		targets = []
		labels = []
		values = []

		targetController = TargetController(request.json['action'], request.json['compound'])
		targets = targetController.doAction()
		if (len(targets) != 0):
			arrayClasses = []
			for target in targets:
				arrayClasses.append(json.dumps(target['target_class']))
			for c in arrayClasses:
				if (c not in labels):
					labels.append(c)
			for c in labels:
				values.append(arrayClasses.count(c))

		return jsonify(status=True, message='Connection successfully', results=targets, labels=labels, values=values)
	else:
		return jsonify(status=False, message='Error request method')

@app.route('/similarityResults', methods=['GET', 'POST'])
def similarityResults():
	if request.method == 'POST':
		similarMols = []
		similarityController = SimilarityController(request.json['action'], request.json['compound'])
		similarMols = similarityController.doAction()
		return jsonify(status=True, message='Connection successfully', results=similarMols)
	else:
		return jsonify(status=False, message='Error request method')

@app.route('/drugResults', methods=['GET', 'POST'])
def drugResults():
	if request.method == 'POST':
		drugInfo = []
		drugController = DrugController(request.json['action'], request.json['compound'])
		drugInfo = drugController.doAction()
		path = 'static/images/queryMolecule/queryMolecule.png'
		img = ""
		if(os.path.isfile(path)):
			img = path
		return jsonify(status=True, message='Connection successfully', results=drugInfo, image = img)
	else:
		return jsonify(status=False, message='Error request method')

if __name__ == "__main__":
	app.run(host= '0.0.0.0', debug = True, use_reloader = True, port = 8000)

max_stars_repo_name: albags/TargetPredictionApp

max_issues_repo_name: albags/TargetPredictionApp

max_forks_repo_name: albags/TargetPredictionApp


text:
import inviwopy
# import inviwopy.glm as glm
import numpy as np
import h5py
from envisionpy.utils.exceptions import *
from .baseNetworks.VolumeSubnetwork import VolumeSubnetwork

# TODO: Link the tf properties instead and copying will be done automatically.
#       Inviwo bug is preventing this as of writing.

# TODO: Closing one of the connected volume visualisations while this one is running
#       Will result in unsupported behaviour. Either block those from closing before
#       this, notify this about them closing, or restart this when volume is closed.

class MultiVolume(VolumeSubnetwork):
    '''
    Manages a subnetwork for rendering of multiple volumes from other visualisations.
    Takes multiple volumes and transfer functions and renders them in a single image.
    Used to for example render both Electron Density and ELF in the same canvas.

    Based on a modified VolumeSubnetwork.
    '''
    def __init__(self, inviwoApp, hdf5_path, hdf5_outport, xpos=0, ypos=0):
        # Set up a generic volume rendering network.
        VolumeSubnetwork.__init__(self, inviwoApp, hdf5_path, hdf5_outport, xpos, ypos, True)
        self.modify_network(xpos, ypos)

        self.used_inports = [False] * 4
        self.other_transferfunctions = [None] * 4

    @staticmethod
    def valid_hdf5(hdf5_file):
        return (
            hdf5_file.get('CHG') != None and
            len(hdf5_file.get('CHG').keys()) != 0 and
            hdf5_file.get('basis') != None and
            hdf5_file.get('scaling_factor') != None)

    def get_ui_data(self):
        return []

    def copy_transferfunctions(self):
        # Copy all the transferfunctions from connected visualisations.
        # TODO: Link the properties instead and this will be done automatically.
        #       Inviwo bug is preventing this as of writing.
        for i, connected in enumerate(self.used_inports):
            if not connected: continue
            tf_points = self.other_transferfunctions[i].getValues()
            self.transfer_functions[i].clear()
            for p in tf_points:
                self.transfer_functions[i].add(p)

            if tf_points[0].pos != 0:
                self.transfer_functions[i].add(
                    tf_points[0].pos * 0.99,
                    inviwopy.glm.vec4(1.0, 1.0, 1.0, 0.0))

    def connect_volume(self, volume_outport, transfer_func_prop, camera_prop):
        if all(self.used_inports):
            raise EnvisionError('No more than 4 volumes may be connected.')

        self.network.addLink(self.camera_prop, camera_prop)
        self.network.addLink(camera_prop, self.camera_prop)

        idx = self.used_inports.index(False)
        self.used_inports[idx] = True
        self.other_transferfunctions[idx] = transfer_func_prop
        self.network.addConnection(volume_outport, self.volume_inports[idx])

        # Inviwo bug preventing linking of TransferFunctionProperty.
        # self.network.addLink(transfer_func_prop.tf, self.transfer_functions[idx])
        # self.network.addLink(self.transfer_functions[idx], transfer_func_prop.tf)
        self.copy_transferfunctions() # Remove when bug is fixed.

    def modify_network(self, xpos, ypos):
        self.remove_processor('HDF5 path')
        self.remove_processor('Hdf5Selection')
        merger = self.add_processor('org.inviwo.VolumeMerger', 'VolumeMerger', xpos, ypos + 3)

        boundingBox = self.get_processor('Volume Bounding Box')
        cubeProxy = self.get_processor('Cube Proxy Geometry')
        raycaster = self.get_processor('Raycaster')
        volumeSlice = self.get_processor('VolumeSlice')
        isoRaycaster = self.get_processor('IsoRaycaster')
        self.network.addConnection(merger.getOutport('outputVolume'), boundingBox.getInport('volume'))
        self.network.addConnection(merger.getOutport('outputVolume'), cubeProxy.getInport('volume'))
        self.network.addConnection(merger.getOutport('outputVolume'), raycaster.getInport('volume'))
        self.network.addConnection(merger.getOutport('outputVolume'), isoRaycaster.getInport('volume'))
        self.network.addConnection(merger.getOutport('outputVolume'), volumeSlice.getInport('volume'))

        self.volume_inports = [
            merger.getInport('inputVolume'),
            merger.getInport('volume2'),
            merger.getInport('volume3'),
            merger.getInport('volume4')
        ]
        self.transfer_functions = [
            getattr(raycaster, 'transfer-functions').transferFunction1,
            getattr(raycaster, 'transfer-functions').transferFunction2,
            getattr(raycaster, 'transfer-functions').transferFunction3,
            getattr(raycaster, 'transfer-functions').transferFunction4
        ]
        pass

max_stars_repo_name: adamskor/ENVISIoN

max_issues_repo_name: adamskor/ENVISIoN

max_forks_repo_name: adamskor/ENVISIoN


text:
# -*- coding: utf-8 -*-
#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
:py:class:`BasicDataSetReader`
"""
import csv
import os
import traceback
import logging
from collections import namedtuple
import numpy as np
from paddle import fluid

from text2sql.framework.register import RegisterSet
from text2sql.framework.reader.data_set_reader.base_dataset_reader import BaseDataSetReader

@RegisterSet.data_set_reader.register
class BasicDataSetReader(BaseDataSetReader):
    """BasicDataSetReader:一个基础的data_set_reader，实现了文件读取，id序列化，token embedding化等基本操作
    """

    def create_reader(self):
        """ 初始化paddle_py_reader，必须要初始化，否则会抛出异常
        :return:
        """
        if not self.fields:
            raise ValueError("fields can't be None")

        shapes = []
        types = []
        levels = []
        for item in self.fields:
            if not item.field_reader:
                raise ValueError("{0}'s field_reader is None".format(item.name))
            i_shape, i_type, i_level = item.field_reader.init_reader()
            shapes.extend(i_shape)
            types.extend(i_type)
            levels.extend(i_level)

        self.paddle_py_reader = fluid.layers.py_reader(
            capacity=50,
            shapes=shapes,
            name=self.name,
            dtypes=types,
            lod_levels=levels,
            use_double_buffer=True)

        logging.debug("%s create py_reader shape = %s , types = %s , level = %s : ", self.name, shapes, types, levels)

    def instance_fields_dict(self):
        """调用pyreader，得到fields_id, 视情况构造embedding，然后结构化成dict类型返回给组网部分
        :return: 实例化的dict，保存了各个field的id和embedding(可以没有，是情况而定), 给trainer用
        """
        fields_id = fluid.layers.read_file(self.paddle_py_reader)
        logging.debug("fluid.layers.read_file...")
        fields_instance = self.convert_fields_to_dict(fields_id)
        return fields_instance

    def convert_fields_to_dict(self, field_list, need_emb=True):
        """实例化fields_dict，保存了各个field的id和embedding(可以没有，是情况而定),
        当need_emb=False的时候，可以直接给predictor调用
        :param field_list:
        :param need_emb:
        :return: dict
        """
        start_index = 0
        fields_instance = {}
        for index, filed in enumerate(self.fields):
            logging.debug('reading fields from index %d', start_index)
            item_dict = filed.field_reader.structure_fields_dict(field_list, start_index, need_emb=need_emb)
            fields_instance[filed.name] = item_dict
            start_index += filed.field_reader.get_field_length()

        return fields_instance

    def data_generator(self):
        """
        :return:
        """
        assert os.path.isdir(self.config.data_path), \
                "%s must be a directory that stores data files" % self.config.data_path
        data_files = os.listdir(self.config.data_path)

        assert len(data_files) > 0, "%s is an empty directory" % self.config.data_path

        def wrapper():
            """
            :return:
            """
            for epoch_index in range(self.config.epoch):
                self.current_example = 0
                self.current_epoch = epoch_index

                for input_file in data_files:
                    if os.path.basename(input_file).startswith('.'):
                        logging.warn('dot(.) started file is ignore: %s', input_file)
                        continue
                    examples = self.read_files(os.path.join(self.config.data_path, input_file))
                    if self.config.shuffle:
                        np.random.shuffle(examples)

                    for batch_data in self.prepare_batch_data(examples, self.config.batch_size):
                        yield batch_data

        return wrapper

    def read_files(self, file_path, quotechar=None):
        """读取明文文件
        :param file_path
        :return: 以namedtuple数组形式输出明文样本对应的实例
        """
        with open(file_path, "r") as f:
            try:
                examples = []
                reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
                len_fields = len(self.fields)
                field_names = []
                for filed in self.fields:
                    field_names.append(filed.name)

                self.Example = namedtuple('Example', field_names)

                for line in reader:
                    if len(line) == len_fields:
                        example = self.Example(*line)
                        examples.append(example)
                    else:
                        logging.warn('fileds in file %s not match: got %d, expect %d'\
                                % (file_path, len(line), len_fields))
                return examples

            except Exception:
                logging.error("error in read tsv")
                logging.error("traceback.format_exc():\n%s" % traceback.format_exc())

    def prepare_batch_data(self, examples, batch_size):
        """将明文样本按照py_reader需要的格式序列化成一个个batch输出
        :param examples:
        :param batch_size:
        :return:
        """
        batch_records = []
        for index, example in enumerate(examples):
            self.current_example = index
            if len(batch_records) < batch_size:
                batch_records.append(example)
            else:
                yield self.serialize_batch_records(batch_records)
                batch_records = [example]

        if batch_records:
            yield self.serialize_batch_records(batch_records)

    def serialize_batch_records(self, batch_records):
        """
        :param batch_records:
        :return:
        """
        return_list = []
        example = batch_records[0]
        for index, key in enumerate(example._fields):
            text_batch = []
            for record in batch_records:
                text_batch.append(record[index])

            id_list = self.fields[index].field_reader.convert_texts_to_ids(text_batch)
            return_list.extend(id_list)

        return return_list

    def get_train_progress(self):
        """Gets progress for training phase."""
        return self.current_example, self.current_epoch

    def get_num_examples(self):
        """get number of example"""
        data_files = os.listdir(self.config.data_path)
        assert len(data_files) > 0, "%s is an empty directory" % self.config.data_path
        for input_file in data_files:
            examples = self.read_files(os.path.join(self.config.data_path, input_file))
            self.num_examples += len(examples)
        return self.num_examples

max_stars_repo_name: pkulzb/Research

max_issues_repo_name: pkulzb/Research

max_forks_repo_name: pkulzb/Research


text:
import torch
import torch.nn as nn
import numpy as np

class W2V_model(nn.Module):
    """This class represents the Word2Vec model"""

    def __init__(self, vocab_size, hidden_dim=300):
        super(W2V_model, self).__init__()
        self.hidden_dim = hidden_dim
        self.vocab_size = vocab_size

        self.fc1 = nn.Linear(vocab_size, hidden_dim) #projection layer
        self.fc2 = nn.Linear(hidden_dim, hidden_dim) #hidden layer
        self.fc3 = nn.Linear(hidden_dim, vocab_size) #output layer

        nn.init.kaiming_normal_(self.fc1.weight)
        nn.init.kaiming_normal_(self.fc2.weight)
        nn.init.kaiming_normal_(self.fc3.weight)

    def forward(self, x):
        x = self.fc1(x)
        x = self.fc2(x)
        x = self.fc3(x)
        return x

class W2V_SGNS_model(nn.Module):
    """ This class represents the Word2Vec Negative sampling model"""

    def __init__(self, vocab_size, hidden_dim=300):
        super(W2V_SGNS_model, self).__init__()

        self.hidden_dim = hidden_dim
        self.vocab_size = vocab_size

        self.embedding = nn.Embedding(self.vocab_size, self. hidden_dim)
        self.context = nn.Embedding(self.vocab_size, self.hidden_dim)
        self.sigmoid = nn.Sigmoid()

    def forward(self, input_idxs, context_idxs):
        """
        Does a forward pass given the input and output word indices.
        Performs a dot prooduct between them and applies sigmoid to get
        output.
        Arguments:
        input_idxs : Input word indices (type list)
        output_idxs : Context word indices (type list)
        """

        input_embeddings = self.embedding(input_idxs)
        context_embeddings = self.context(context_idxs)
        output = self.sigmoid(torch.sum(input_embeddings * context_embeddings, dim=1)).view(-1, 1)
        return output


text:
from heapq import heappush, heappop  # Recommended.
import numpy as np

from math import sqrt

from flightsim.world import World
from proj1_3.code.occupancy_map import OccupancyMap # Recommended.

def graph_search(world, resolution, margin, start, goal, astar):
    """
    Parameters:
        world,      World object representing the environment obstacles
        resolution, xyz resolution in meters for an occupancy map, shape=(3,)
        margin,     minimum allowed distance in meters from path to obstacles.
        start,      xyz position in meters, shape=(3,)
        goal,       xyz position in meters, shape=(3,)
        astar,      if True use A*, else use Dijkstra
    Output:
        path,       xyz position coordinates along the path in meters with
                    shape=(N,3). These are typically the centers of visited
                    voxels of an occupancy map. The first point must be the
                    start and the last point must be the goal. If no path
                    exists, return None.
    """

    # While not required, we have provided an occupancy map you may use or modify.
    occ_map = OccupancyMap(world, resolution, margin)
    # Retrieve the index in the occupancy grid matrix corresponding to a position in space.
    start_index = tuple(occ_map.metric_to_index(start))
    start_cost = 0
    start_x,start_y,start_z = start_index
    goal_index = tuple(occ_map.metric_to_index(goal))
    goal_x,goal_y,goal_z = goal_index
    goal_cost = np.inf
    heuristic = sqrt((start_x-goal_x)**2+(start_y-goal_y)**2+(start_z-goal_z)**2)
    start_f = heuristic
    goal_f = np.inf
    # Initialization
    Q = []    # priority queue of open nodes
    G = []    # all the graph
    path = np.array([list(goal)]) # initialize path

    # world representing in voxel
    x = occ_map.map.shape[0]
    y = occ_map.map.shape[1]
    z = occ_map.map.shape[2]
    index = 0
    pos = np.zeros((x,y,z))    # position in a list

    class grid(object):
        """
        This class used to store features of a voxel
        """
        def __init__(self,idx,cost_to_come=np.inf,parent=np.NaN,heu=heuristic,cost=np.inf):
            self.idx = idx
            self.cost_to_come = cost_to_come
            self.parent = parent
            self.heu = heu
            self.cost = cost

    # store the features of all node in an object list
    for i in range(x):
        for j in range(y):
            for k in range(z):

                pos[i][j][k]=index    # store index

                if start_index == (i,j,k):
                    vox = grid(idx=(i,j,k),cost_to_come=0)
                    start_pos = index
                elif goal_index == (i,j,k):
                    vox = grid(idx=(i,j,k))
                    goal_pos = index
                else:
                    vox = grid(idx=(i,j,k))

                index+=1
                G.append(vox)    # store all the voxel object in a list *** Do Not Change the idx***

    if astar:
        heappush(Q,(start_f,G[start_pos].idx))
        heappush(Q,(goal_f,G[goal_pos].idx))

        # A* algorithm
        while (goal_cost,goal_index) in Q and Q[0][0]<np.inf:
            u = Q[0]    # node that has smllest cost
            ux,uy,uz = u[1] # node's coordination
            heappop(Q)    # alwasy pop out node that has smallest cost

            if (ux,uy,uz) == goal_index:
                break

            # push neigbor(u) into Q
            for dx in range(-1,2):
                for dy in range(-1,2):
                    for dz in range(-1,2):

                        if dx==0 and dy == 0 and dz==0:
                            continue

                        # update new node
                        new_x = dx+ux
                        new_y = dy+uy
                        new_z = dz+uz
                        is_valid_neigbor = occ_map.is_valid_index((new_x,new_y,new_z))
                        if is_valid_neigbor:
                            is_occupy = occ_map.is_occupied_index((new_x,new_y,new_z))
                            if not is_occupy:
                                idx_in_G = int(pos[new_x][new_y][new_z])
                                v = G[idx_in_G]
                                c_u_v = sqrt(dx**2+dy**2+dz**2)
                                idx_of_u = int(pos[ux][uy][uz])
                                d = G[idx_of_u].cost_to_come+c_u_v
                                if d<v.cost_to_come:
                                    heuristic = sqrt((new_x-goal_x)**2+(new_y-goal_y)**2+(new_z-goal_z)**2)
                                    f = heuristic+d    # calculate the cost of a node
                                    G[idx_in_G] = grid((new_x,new_y,new_z),d,(ux,uy,uz),heuristic,f)    # update new node's feature
                                    heappush(Q,(G[idx_in_G].cost,G[idx_in_G].idx))    # update the open list

    else:
        heappush(Q,(start_cost,G[start_pos].idx))
        heappush(Q,(goal_cost,G[goal_pos].idx))

        # Dijstra Algorithms starts here#

        while (goal_cost,goal_index) in Q and Q[0][0]<np.inf:
            u = Q[0]    # node that has smllest cost
            ux,uy,uz = u[1] # node's coordination
            heappop(Q)    # alwasy pop out node that has smallest cost

            # push neigbor(u) into h
            for dx in range(-1,2):
                for dy in range(-1,2):
                    for dz in range(-1,2):

                        if dx==0 and dy == 0 and dz==0:
                            continue

                        # update new node
                        new_x = dx+ux
                        new_y = dy+uy
                        new_z = dz+uz
                        is_valid_neigbor = occ_map.is_valid_index((new_x,new_y,new_z))
                        if is_valid_neigbor:
                            is_occupy = occ_map.is_occupied_index((new_x,new_y,new_z))
                            if not is_occupy:
                                idx_in_G = int(pos[new_x][new_y][new_z])
                                v = G[idx_in_G]
                                c_u_v = sqrt(dx**2+dy**2+dz**2)
                                idx_of_u = int(pos[ux][uy][uz])
                                d = G[idx_of_u].cost_to_come+c_u_v
                                if d<v.cost_to_come:
                                    G[idx_in_G] = grid((new_x,new_y,new_z),d,(ux,uy,uz))    # update new node's feature
                                    heappush(Q,(G[idx_in_G].cost_to_come,G[idx_in_G].idx))    # update open list

    # trace parent
    if np.any(np.isnan(G[goal_pos].parent)):
        return None
    else:
        parent_x,parent_y,parent_z = G[goal_pos].parent
        parent = occ_map.index_to_metric_center((parent_x,parent_y,parent_z))
        path = np.r_[path,np.array([list(parent)])]
        idx_in_G = int(pos[parent_x][parent_y][parent_z])
        while 1:
            parent_x,parent_y,parent_z = G[idx_in_G].parent
            parent = occ_map.index_to_metric_center((parent_x,parent_y,parent_z))
            path = np.r_[path,np.array([list(parent)])]
            idx_in_G = int(pos[parent_x][parent_y][parent_z])
            if idx_in_G == start_pos:
                break
        path = np.r_[path,np.array([list(start)])]
        path = np.flipud(path)

        return path


text:
# --------------
#Importing header files

import pandas as pd
from sklearn.model_selection import train_test_split

# Code starts here
data = pd.read_csv(path)
X = data.drop(['customer.id','paid.back.loan'], 1)
y = data['paid.back.loan']

X_train, X_test, y_train, y_test = train_test_split(X,y,test_size = 0.3, random_state = 0)

# Code ends here

# --------------
#Importing header files
import matplotlib.pyplot as plt

# Code starts here
fully_paid = y_train.value_counts()
fully_paid.plot(kind = 'bar')

# Code ends here

# --------------
#Importing header files
import numpy as np
from sklearn.preprocessing import LabelEncoder

# Code starts here
X_train['int.rate'] = X_train['int.rate'].map(lambda x: str(x)[:-1])
X_train['int.rate']  =X_train['int.rate'].astype(np.float64)
X_train['int.rate']  = X_train['int.rate'] /100

X_test['int.rate'] = X_test['int.rate'].map(lambda x: str(x)[:-1])
X_test['int.rate']  =X_test['int.rate'].astype(np.float64)
X_test['int.rate'] = X_test['int.rate'] /100

num_df = X_train.select_dtypes(exclude ='object')
cat_df = X_train.select_dtypes(include ='object')
num_df.dtypes

# Code ends here

# --------------
#Importing header files
import seaborn as sns

# Code starts here
cols = list(num_df)
fig, axes = plt.subplots(nrows = 9 , ncols = 1, figsize=(20,10))

# Code ends here

# --------------
# Code starts here
cols = list(cat_df.columns)
fig, axes = plt.subplots(2,2, figsize=(20,10))
for i in range(0,2):
    for j in range(0,2):
        sns.countplot(x=X_train[cols[i*2+j]], hue=y_train,ax=axes[i,j])

# Code ends here

# --------------
#Importing header files
from sklearn.tree import DecisionTreeClassifier

# Code starts here
# for i in cat_df:
#     X_train =  X_train.fillna('NA')
#     le = LabelEncoder()
#     cat_df = le.fit_transform(cat_df)

    # X_test = X_test.fillna('NA')
    # le.fit_transform(X_test.select_dtypes(include ='object'))
for i in cols:
    X_train[i].fillna('NA')
    le = LabelEncoder()
    X_train[i] = le.fit_transform(X_train[i])

for i in cols:
    X_test[i].fillna('NA')
    le1 = LabelEncoder()
    X_test[i] = le1.fit_transform(X_test[i])

y_train = y_train.replace({'No':0, 'Yes':1})
y_test = y_test.replace({'No':0, 'Yes':1})

model = DecisionTreeClassifier(random_state= 0)
model.fit(X_train, y_train)
acc= model.score(X_test, y_test)

# Code ends here

# --------------
#Importing header files
from sklearn.model_selection import GridSearchCV
from sklearn import tree
#Parameter grid
parameter_grid = {'max_depth': np.arange(3,10), 'min_samples_leaf': range(10,50,10)}

# Code starts here
model_2 = tree.DecisionTreeClassifier(criterion='entropy', random_state= 0)
p_tree = GridSearchCV(estimator=model_2, param_grid=parameter_grid, cv=5)
p_tree.fit(X_train, y_train)
acc_2 = p_tree.score(X_test,y_test)

# Code ends here

# --------------
#Importing header files

from io import StringIO
from sklearn.tree import export_graphviz
from sklearn import tree
from sklearn import metrics
from IPython.display import Image
import pydotplus

# Code starts here
dot_data = export_graphviz(decision_tree=p_tree.best_estimator_, out_file=None, feature_names=X.columns, filled = True, class_names=['loan_paid_back_yes','loan_paid_back_no'])
graph_big = pydotplus.graph_from_dot_data(dot_data)

# show graph - do not delete/modify the code below this line
img_path = user_data_dir+'/file.png'
graph_big.write_png(img_path)

plt.figure(figsize=(20,15))
plt.imshow(plt.imread(img_path))
plt.axis('off')
plt.show()

# Code ends here


text:
import torch
import torchvision
import torchvision.transforms as transforms
import numpy as np
from PIL import Image

class IMBALANCECIFARIND(torchvision.datasets.CIFAR10):
    cls_num = 10

    def __init__(self, root, imb_type='step', imb_factor=0.01, rand_number=0, train=True,
                 transform=None, target_transform=None,
                 download=False, weight_path = None):
        super(IMBALANCECIFARIND, self).__init__(root, train, transform, target_transform, download)
        np.random.seed(rand_number)
        img_num_list = self.get_img_num_per_cls(self.cls_num, imb_type, imb_factor)
        self.weight_path = weight_path
        self.gen_imbalanced_data(imb_type)

    def get_img_num_per_cls(self, cls_num, imb_type, imb_factor):
        img_max = len(self.data) / cls_num
        img_num_per_cls = []
        if imb_type == 'exp':
            for cls_idx in range(cls_num):
                num = img_max * (imb_factor**(cls_idx / (cls_num - 1.0)))
                img_num_per_cls.append(int(num))
        elif imb_type == 'step':
            for cls_idx in range(cls_num // 2):
                img_num_per_cls.append(int(img_max))
            for cls_idx in range(cls_num // 2):
                img_num_per_cls.append(int(img_max * imb_factor))
        else:
            img_num_per_cls.extend([int(img_max)] * cls_num)
        return img_num_per_cls

    def gen_imbalanced_data(self, imb_type):
        if imb_type == 'exp':
            self.data = np.load('data/cifar_exp_data.npy').astype('uint8')
            self.targets = np.load('data/cifar_exp_label.npy').astype('int64')
        if imb_type == 'step':
            self.data = np.load('data/cifar_step_data.npy').astype('uint8')
            self.targets = np.load('data/cifar_step_label.npy').astype('int64')

        self.weights = np.load(self.weight_path).astype('float32')

    def get_cls_num_list(self):
        cls_num_list = []
        for i in range(self.cls_num):
            cls_num_list.append(self.num_per_cls_dict[i])
        return cls_num_list

    def __getitem__(self, index: int):

        img, target = self.data[index], self.targets[index]
        weight = self.weights[index]

        img = Image.fromarray(img)

        if self.transform is not None:
            img = self.transform(img)

        if self.target_transform is not None:
            target = self.target_transform(target)

        return img, target, weight

class IMBALANCECIFAR(torchvision.datasets.CIFAR10):
    cls_num = 10

    def __init__(self, root, imb_type='step', imb_factor=0.01, rand_number=0, train=True,
                 transform=None, target_transform=None,
                 download=False):
        super(IMBALANCECIFAR, self).__init__(root, train, transform, target_transform, download)
        np.random.seed(rand_number)
        img_num_list = self.get_img_num_per_cls(self.cls_num, imb_type, imb_factor)
        self.gen_imbalanced_data(imb_type)

    def get_img_num_per_cls(self, cls_num, imb_type, imb_factor):
        img_max = len(self.data) / cls_num
        img_num_per_cls = []
        if imb_type == 'exp':
            for cls_idx in range(cls_num):
                num = img_max * (imb_factor**(cls_idx / (cls_num - 1.0)))
                img_num_per_cls.append(int(num))
        elif imb_type == 'step':
            for cls_idx in range(cls_num // 2):
                img_num_per_cls.append(int(img_max))
            for cls_idx in range(cls_num // 2):
                img_num_per_cls.append(int(img_max * imb_factor))
        else:
            img_num_per_cls.extend([int(img_max)] * cls_num)
        return img_num_per_cls

    def gen_imbalanced_data(self, imb_type):
        if imb_type == 'exp':
            self.data = np.load('data/cifar_exp_data.npy').astype('uint8')
            self.targets = np.load('data/cifar_exp_label.npy').astype('int64')
        if imb_type == 'step':
            self.data = np.load('data/cifar_step_data.npy').astype('uint8')
            self.targets = np.load('data/cifar_step_label.npy').astype('int64')

    def get_cls_num_list(self):
        cls_num_list = []
        for i in range(self.cls_num):
            cls_num_list.append(self.num_per_cls_dict[i])
        return cls_num_list


text:
import pandas as pd
import numpy as np

def readline(line):
    current_line_list=[]
    for tuple_str in line.split('|'):
        name_value = tuple_str.split()
        if len(name_value)==1:
            current_line_list.append((name_value[0],None))
        elif len(name_value)==2:
            current_line_list.append((name_value[0],name_value[1]))
        else:
            index = tuple_str.find(name_value[1])
            current_line_list.append((name_value[0],tuple_str[index:]))
    return current_line_list

def generate_df(line_lists):
    datas=[]
    for i in range(len(line_lists)):
        row=[np.NAN]*len(columns)
        for j in range(len(line_lists[i])):
            key = line_lists[i][j][0]
            column_num =  columns_map[key]
            row[column_num] = line_lists[i][j][1]
        datas.append(row)
    return datas

def save(filename,datas):
    user_feature_df = pd.DataFrame(datas,columns=columns)
    user_feature_df.to_csv(filename,sep=',',index=False)

columns_map={}
columns=['uid','age','gender','LBS','consumptionAbility', 'education','topic1', 'topic2', 'topic3', 'kw1', 'kw2','kw3','carrier', 'ct',  'marriageStatus', 'interest1', 'interest2','interest3','interest4','interest5','house', 'os','appIdAction','appIdInstall']
for i in range(len(columns)):
    columns_map[columns[i]]=i

# 如果想全部保存到一个文件,也很简单,那么只要把MAX_NUM设置为>userFeature.data的行数即可
MAX_NUM=1000000
line_lists=[]
import os
if not os.path.exists("../tmp/"):
    os.mkdir("../tmp/")

with open('../data/userFeature.data') as f:
    count=0
    for line in f:
        current_line_list=readline(line)
        line_lists.append(current_line_list)
        count +=1
        if count%MAX_NUM==0:
            datas = generate_df(line_lists)
            file_index = (count//MAX_NUM)
            filename = "../tmp/MyUserFeature"+str(file_index)+".csv"
            save(filename,datas)
            line_lists=[]
    if count%MAX_NUM!=0:
        datas = generate_df(line_lists)
        file_index = (count//MAX_NUM)+1
        filename = "../tmp/MyUserFeature"+str(file_index)+".csv"
        save(filename,datas)
    f.close()

del line_lists, datas
user_feature=pd.concat([pd.read_csv('MyUserFeature' + str(i) + '.csv') for i in range(1,13)]).reset_index(drop=True)
user_feature.to_csv('../data/MyUserFeature.csv', index=False)