import scrapy

class GilenofilhoSpider(scrapy.Spider):

    name = 'gilenofilho'
    allowed_domains = ['www.gilenofilho.com.br']
    start_urls = [
        'http://www.gilenofilho.com.br/',
    ]

    def parse(self, response):
        links = response.xpath("//div[@class='col-md-8']/article/a/@href")
        for link in links.extract():
            yield scrapy.Request(
                'http://www.gilenofilho.com.br{}'.format(link),
                self.parse_post
            )
        next_page = response.xpath(
            "//a[@aria-label='Next']/@href").extract_first()
        if next_page:
            yield scrapy.Request(
                url='http://www.gilenofilho.com.br{}'.format(next_page),
                callback=self.parse
            )

    def parse_post(self, response):
        texts = response.css(
            "article > p:nth-child(n+3):nth-last-child(n+5)::text, article > ul > li::text"
        ).extract()
        yield {
            'text': '\n'.join(texts),
            'category': response.xpath(
                "//main/div[1]/div[1]/article/p[1]/a[2]/text()"
            ).extract_first()
        }


#Sriracha Sauce-Sara Aseer and Damian Wasilewicz
#SoftDev pd06
#K06 -- Yummy Mongo Py
#2019-02-06

import pymongo

#setup
SERVER_ADDR = "68.183.28.211"
connection = pymongo.MongoClient(SERVER_ADDR)
db = connection.test
collection = db.restaurants

#Specified BOROUGH
b = "Queens"
def bFind(borough):
    found = collection.find({"borough":borough})
    for item in found:
        print("Displaying restaurants in borough: " + borough + "\n")
        print("Name: "+ item['name'] + "\n")

bFind(b)

#Specified ZIPCODE

def zFind(zipcode):
    found = collection.find({"address.zipcode":zipcode})
    for item in found:
        print("Displaying restaurants in zipcode: " + zipcode + "\n")
        print("ZipCode: "+ item['name']+ "\n")

zFind("11219")
zFind("11374")

#Specified Zip and Grade
def zsFind(zipcode,g):
    found = collection.find({"address.zipcode":zipcode,"grades.grade":g})
    for item in found:
        print("Displaying restaurants in zipcode: " + zipcode + " and grade: " + g + "\n")
        print("Zip + Grade: "+ item['name']+ "\n")

zsFind("11374","A")
zsFind("11374","C")

#All restaurants in Zip w/ below score
def sFind(zipcode,s):
    found = collection.find({"address.zipcode":zipcode,"grades.score":{"$lt": s}})
    for item in found:
        print("displaying with zip: " + zipcode + " and score below :" + s + "\n")
        print("Zip+Score: " + item['name']+ "\n")

sFind("11374",3)

def superFind(b,c,s):
    found = collection.find({"borough":b, "cuisine": c, "grades.score":{"$gt": s}})
    for item in found:
        print("Borough+Cuisine+Score(above): " + item['name'])

superFind("Manhattan","Hamburgers",8)


# 数字人体挑战赛
# coding = utf-8

import kfbReader
import cv2 as cv
import numpy as np
import json
import os

'''
# 函数名称：Init()
# 函数功能：初始化函数
# 传入参数：scale：读取倍率
# 传出参数：filenamelist：返回一个包含文件和文件夹名字的列表
'''
def Init():
	# 设置文件路径
	global filepath
	filepath = "E:\\2_Competitions\\2_14_数字人体\\3_技术资料\\0_Data\\pos_0\\"
	# 设置读取倍率
	global scale
	scale =  20

'''
# 函数名称：ReadReturnFileNames()
# 函数功能：读取指定文件夹内，所有的文件名字，
# 		   如果是文件会读取整个文件名，包括后缀名，
# 		   如果是文件夹，也会读取文件夹的名字
# 传入参数：filename：传入要读取的文件夹路径，字符类型
# 传出参数：filenamelist：返回一个包含文件和文件夹名字的列表
'''
def ReadReturnFileNames(filepath):
	# filepath = filepath
	filenamelist = []															# 新建一个空的列表，用来存储读取到的文件名字
	for files in os.listdir(filepath):  										# 不仅仅是文件，当前目录下的文件夹也会被认为遍历到
			filenamelist.append(files)
	return filenamelist

Init()																			# 执行初始化函数

filepath = "E:\\2_Competitions\\2_14_数字人体\\3_技术资料\\0_Data\\pos_0\\"
filenamelist = ReadReturnFileNames(filepath)									# 输入路径，返回文件名列表

for filename in filenamelist:
	filename = filename[:-4]													# 消除文件名字的后缀 .kfb
	file1 = "E:\\2_Competitions\\2_14_数字人体\\3_技术资料\\0_Data\\pos_0\\"+filename+".kfb"
	label1 = "E:\\2_Competitions\\2_14_数字人体\\3_技术资料\\0_Data\\labels\\"+filename+".json"

	# 实例化 reader 类，读取提供的图像
	read = kfbReader.reader()
	read.ReadInfo(file1, scale, True)

	# 定义存储数据的空列表
	roi_x = []																	# 用来存储 roi 数据
	roi_y = []
	roi_w = []
	roi_h = []
	pos_x = []							# 用来存储 pos 数据
	pos_y = []
	pos_w = []
	pos_h = []
	roi   = []							# 用来存储绘制的 roi 图形

	# 打开 label 文件
	with  open(label1,"r") as f:
		labelContent = json.load(f)		# 获取 label1 标签内容

	# 遍历 label 文件中的标记框的参数
	for dic in labelContent:
		# 如果属于 roi 类，则存储到 roi 开头的列表
		if dic["class"] ==  "roi":
			roi_x.append(dic['x'])
			roi_y.append(dic['y'])
			roi_w.append(dic['w'])
			roi_h.append(dic['h'])
		# 如果属于 pos 类，则存储到 pos 开头的列表
		if dic["class"] ==  "pos":
			pos_x.append(dic['x'])
			pos_y.append(dic['y'])
			pos_w.append(dic['w'])
			pos_h.append(dic['h'])
	# 打印提取到的 roi 和 pos 数据
	# print(roi_x,'roi_x',roi_y,'roi_y',roi_w,'roi_w',roi_h,'roi_h')
	# print(pos_x,'pos_x',pos_y,'pos_y',pos_w,'pos_w',pos_h,'pos_h')

	# 利用数据画出 roi 图和 pos 标记图
	for i in range(len(roi_x)):
		# 利用官方 sdk 读取 kfb 图片，存储到 roi 数组中
		roi.append( read.ReadRoi(roi_x[i], roi_y[i], roi_w[i], roi_h[i], scale) )
		for j in range(len(pos_x)):
			# 循环读取 pos 数据， 绘制矩形框标记在 roi 区域
			x = pos_x[j] - roi_x[i]
			y = pos_y[j] - roi_y[i]
			w = pos_w[j] + x
			h = pos_h[j] + y
			if x>0 and x<10000 :
				# 打印绘制矩形框的数据
				# print(x,y,w,h,'rectangle')
				#cv.rectangle(roi[i], (x,y), (w,h), (0,255,0), 2)
		# 显示已经标记绿色矩形框的图
		# cv.imshow('roi'+str(i), roi[i])
		# 保存处理过的图像
				cv.imwrite(str(filename)+'_'+str(i)+'.jpg', roi[i])
				# 写入文件后再次打开，并保存标记区域
				img = cv.imread(str(filename)+'_'+str(i)+'.jpg')
				imgsave = img[y:h,x:w]
				cv.imwrite(str(filename)+'_roi'+str(i)+'_pos'+str(j)+'.jpg',imgsave)
	# 将列表清空，方便再次存储新的图像数据
	roi_x = []							# 用来存储新的 roi 数据
	roi_y = []
	roi_w = []
	roi_h = []
	pos_x = []							# 用来存储新的 pos 数据
	pos_y = []
	pos_w = []
	pos_h = []
	roi   = []							# 用来存储新的绘制的 roi 图形


"""
Homework for OCN506, Glover, 5/4/2020
"""

#imports
import sys, os
sys.path.append('shared')
import numpy as np
import pickle
import numpy.matlib
import my_module as mm
import matplotlib.pyplot as plt
import argparse

# use argparse to provide user input (practice!)
parser = argparse.ArgumentParser()
parser.add_argument('-cols', '--cols', default='1000', type=int)
parser.add_argument('-d','--dir_name',default='ocn506/',type=str)
args = parser.parse_args()

# number of columns must be even for the reshape line.
# Only proceed if even, else print error message
if (args.cols % 2)==0:
    # make an array 5 x cols (defined by user input):
    y = np.random.rand(5,args.cols)

    # sin(y*pi):
    y = np.sin(y*np.pi)

    # nan all values less than 0.1
    y[y<0.1]=np.nan

    # calculate the mean of each col, ignoring nans
    y_mean = np.nanmean(y,axis=0)
    y_std = np.nanstd(y,axis=0)

    # reshape y
    new_cols=args.cols//2
    y = np.reshape(y,(10,new_cols))

    # Make an x variable, same size as y:
    x = np.linspace(0,1,len(y[0]))
    x = np.matlib.repmat(x,len(y),1)

    # Multiply, element-wise
    z = np.multiply(x,y)

    # plot the first row of x vs y:
    plt.close('all')
    fig=plt.figure()
    plt.plot(x[0,:],y[0,:])
    plt.plot(x[0,:],z[0,:])
    plt.show()

    # make sure the output directory exists
    out_dir = '../../output/' + args.dir_name
    mm.make_dir(out_dir)

    # define the output filename and save as a pickle file
    out_y = out_dir + 'out_y05042020.p'
    pickle.dump(y, open(out_y, 'wb')) # 'wb' is for write binary

    # reload y with a different variable name
    y_reload = pickle.load(open(out_y, 'rb')) # 'rb is for read binary

    # check that y and y_reload are the same (ignore nans)
    if np.allclose(y,y_reload,equal_nan=True)==True:
        print('success')
    else:
        print('keep trying')

else:
    print('number of columns must be even')


#! /usr/bin/env python3

from collections import Counter

count = 0

hexamer_5 = Counter()
hexamer_3 = Counter()

import sys

filename = sys.argv[1]

for line in open(filename):
    line = line.rstrip()

    if count == 0:
        name = line
        count += 1

    elif count == 1:
        seq = line
        count += 1

    elif count == 2:
        count += 1

    elif count == 3:
        qual = line
        count = 0

def fiveprime_hexamer(filename):

    hexamer_5 = seq[0:6]
    hexamers_5[hexamer_5] += 1

    for hexamer, count in hexamers_5.most_common(1):
        print("The most common 5'end hexamer sequence is", hexamer)

def threeprime_hexamer(filename):

    hexamer_3 = seq[-6:]
    hexamers_3[hexamer_3] += 1

    for hexamer, count in hexamers_3.most_common(1):
         print("The most common 3'end hexamer sequence is", hexamer)

fiveprime_hexamer(filename)
threeprime_hexamer(filename)


from __future__ import print_function, division

def autowrapt_decimal(_module):
    import sys
    import cdecimal

    sys.modules['decimal'] = cdecimal
    print('module decimal patched to {}'.format(sys.modules['decimal'].__name__))
    cdecimal.tmp_context = cdecimal.Context

    def patched(*args, **kwargs):
        if 'rounding' in kwargs and kwargs['rounding'] is None:
            kwargs['rounding'] = 6
        return cdecimal.tmp_context(*args, **kwargs)
    cdecimal.Context = patched

    print('cdecimal.Context monkey-patch for boto(if rounding=None, then 6 passed to Context instead) applied\n')

def autowrapt_psycopg2(module):
    import ujson
    module.json = ujson
    print('psycopg2._json.json monkey-patched to ujson instead of json\n')

def autowrapt_dynamodb(module):
    from decimal import Decimal
    from boto.dynamodb.types import Binary

    def deepcopy(data):
        """Expected dicts nesting level 1. Lists, sets, and nested dicts
         contains only simple types"""
        res = {}
        for k, v in data.items():
            if type(v) in [Decimal, Binary, str, int, unicode, long, bool]:
                res[k] = v
            elif type(v) in [set, dict, list]:
                res[k] = type(v)(v)
            elif v is None:
                res[k] = v
            else:
                raise ValueError('Unknown type: {}'.format(type(v)))
        return res

    module.deepcopy = deepcopy
    print('boto.dynamodb2.items.deepcopy monkey-patched\n')


import math
from statistics import median
import numpy as np

# made all operations as they were more eficient than numpy
def myadd(x, y):
    # print("add")
    # print("x : ",len(x))
    # print("y : ", len(y))
    z = []
    for i in range(0,len(x)):
        z.append(x[i] + y[i])
    return z

def mysub(x, y):
    # print("sub")
    # print("x : ", len(x))
    # print("y : ", len(y))
    z = []
    for i in range(0,len(x)):
        z.append(x[i] - y[i])
    return z

def mymul(x, y):
    # print("mul")
    # print("x : ", len(x))
    # print("y : ", len(y))
    z = []
    for i in range(0,len(x)):
        z.append(x[i] * y[i])
    return z

def mydiv(x, y):
    # print("div")
    # print("x : ", len(x))
    # print("y : ", len(y))
    z = []
    for i in range(0,len(x)):
        if y[i] == 0. or x[i] == -0.:
            z.append(x[i])
        else:
            z.append(x[i] / y[i])
    return z

def myln(x):
    z = []
    for i in x:
        if i == 0. or i == -0.:
            z.append(i)
        else:
            z.append(math.log(abs(i)))
    return z

def mysqrt(x):
    z = []
    for i in x:
        if i <= 0. or i == -0.:
            z.append(i)
        else:
            z.append(math.sqrt(i))
    return z

# <= 20 for threshold, mroe than that is simply 0 and would overflow if unchecked
def sigmoid(x):
    z = []
    for i in x:
        if i <= -20:
            z.append(i)
        else:
            z.append(1 / (1 + math.exp(-i)))
    return z

def tanh(x):
    z = []
    for i in x:
        if i <= -20:
            z.append(i)
        else:
            z.append(2 / (1 + math.exp(-i)) - 1)
    return z

def logsum(x):
    z = []
    for i in x:
        if i <= 0:
            z.append(math.exp(i) - 1)
        else:
            z.append(i + math.log(i))
    return z

def softExp(x):
    z = []
    alpha = -0.2
    for i in x:
        if i < 0:
            z.append(-1 * (math.log(1-alpha*(i+alpha)/alpha)))
        elif i == 0:
            z.append(i)
        else:
            z.append(math.exp(alpha*i)/alpha + alpha)
    return z

def medianAccs(l):
    training = []
    test = []
    dim = []
    node = []
    training.extend([i[0] for i in l])
    test.extend([i[1] for i in l])
    dim.extend([i[2] for i in l])
    node.extend([i[3] for i in l])
    mediumTraining = [median(t) for t in zip(*training)]    # zips each corresponding element of a list of tuples
    mediumTest = [median(t) for t in zip(*test)]            # zips each corresponding element of a list of tuples
    mediumDim = [median(t) for t in zip(*dim)]            # zips each corresponding element of a list of tuples
    mediumNode = [median(t) for t in zip(*node)]            # zips each corresponding element of a list of tuples
    return mediumTraining, mediumTest, mediumDim, mediumNode


#!/usr/bin/env python
# encoding: utf-8
"""
dustMap.py

Created by José Sánchez-Gallego on 13 May 2014.
Licensed under a 3-clause BSD license.

Revision history:
    13 May 2014 J. Sánchez-Gallego
      Initial version

"""

import glob
import os

import numpy as np
from astropy.wcs import WCS
from scipy.interpolate import griddata

from .ccm_unred import ccmUnred

try:
    import fitsio
    fitsLib = 'fitsio'
except Exception:
    try:
        from astropy.io import fits
        fitsLib = 'fits'
    except Exception:
        raise ImportError('either fitsio or astropy.io.fits are needed.')

try:
    from astropysics import coords
    astropysicsLib = True
except Exception:
    try:
        from astropy.coordinates import SkyCoord
        astropysicsLib = False
    except Exception:
        raise ImportError('either astropysics or astropy are needed.')

try:
    __defaultMaps__ = glob.glob(
        os.path.join(os.environ['DUST_DIR'], 'maps', 'SFD_dust_4096_*.fits'))
except Exception:
    __defaultMaps__ = None

class DustMap(object):
    """A class to analyse dust maps and derive values.

    This class allows to load dust maps and determine extinction values at
    specific coordinates. Any dust map with units E(B-V) can be used as long
    as valid WCS information is stored in the header of the FITS file. Both FK5
    or Galactic coordinates can be used.

    Once the class is instantialised, it can be called by providing a list of
    coordinates. The returned dictionary includes the E(B-V) value(s) for the
    input coordinate(s), as well as the iIncrease and gIncrease values at the
    position. iIncrease is defined as the reddening correction for an unitary
    flux in the SDSS i-band:

    iIncrease = 1 / iOut ** 2

    where iOut is the reddened unitary flux in the i-band at the position.

    The returned values are masked arrays with elements masked if the requested
    coordinate was outside the footprint of all of the dust maps.

    Parameters
    ----------
    maps : string, list of strings, optional
        The path or list of paths for the dust maps to be used. If not
        specified, dust maps are searched at the path $DUST_DIR/maps/.

    Example
    -------
    The class can be initialised as ::
      >> from sdss.manga import DustMap
      >> dMap = DustMap()

    And then called as ::
      >> dustValues = dMap(12.5, 30)

    This will return the dust parameters at the position RA=12.5 deg,
    Dec=30 deg. It's possible to call the DustMap instance with a list of
    coordinates. To determine the dust parameters at (12.5, 30) and
    (210, -15) ::
      >> dustValues = dMap([12.5, 210], [30, -15])

    By default, equatorial coordinates are assumed. Galactic coordinates can be
    used by adding the keyword `coordSys='galactic'`. The normal behaviour is
    not to interpolate the values of the dust map. This can be overridden by
    defining `interpolate=True`. The output dictionary is of the form ::
      >> print(dustValues)
      {'EBV': masked_array(data = [0.05857369377550144 0.08294817811791555],
              mask = [False False],
        fill_value = 1e+20),
 'gIncrease': masked_array(data = [1.5123812061910469 1.7964882075835737],
              mask = [False False],
        fill_value = 1e+20),
 'galCoords': [[122.56131852693547, -32.870627753783275],
  [326.54366100057314, 44.705445415210264]],
 'iIncrease': masked_array(data = [1.257851242757 1.383847964727236],
              mask = [False False],
        fill_value = 1e+20)}

    """

    def __init__(self, maps=__defaultMaps__, **kwargs):

        if maps is None:
            raise ValueError('no maps defined and $DUST_DIR is not defined.')

        self.maps = [maps] if not hasattr(maps, '__getitem__') else maps

        if False in map(os.path.exists, self.maps):
            raise NameError('dust map(s) does not exist.')

        if len(self.maps) == 0:
            raise ValueError('no dust maps available.')

        self.loadMaps()

    def __call__(self, xx, yy, coordSys='fk5', interpolate=False):

        if coordSys not in ['fk5', 'galactic']:
            raise ValueError('coordSys must be fk5 or galactic.')

        xx = np.atleast_1d(xx)
        yy = np.atleast_1d(yy)
        if len(xx) != len(yy):
            raise ValueError('x and y coordinates must be of the same length.')

        if coordSys == 'galactic':
            coordsGal = np.array([xx, yy]).T
        else:
            if astropysicsLib:
                ccICRS = [coords.ICRSCoordinates(xx[ii], yy[ii]) for ii in range(len(xx))]
                ccGal = [cc.convert(coords.GalacticCoordinates) for cc in ccICRS]
                coordsGal = np.array([[cc.l.degrees for cc in ccGal],
                                      [cc.b.degrees for cc in ccGal]]).T
            else:
                ccSkyCoords = SkyCoord(xx, yy, frame='icrs', unit='deg')
                ccGal = ccSkyCoords.galactic
                coordsGal = np.array([ccGal.l.deg, ccGal.b.deg]).T

        return self._getValues(coordsGal, interpolate=interpolate)

    def _getValues(self, cc, interpolate=True):

        # Central wavelengths for u, g, r, i, z
        lambdaIn = np.array([3551., 4686., 6165., 7481., 8931.])
        # Unitary flux
        fluxIn = np.array([1., 1., 1., 1., 1.])

        ebv = np.ma.zeros(cc.shape[0])
        ebv.mask = True
        iIncrease = ebv.copy()
        gIncrease = ebv.copy()

        for ii in range(len(self._hdu)):

            ww = self._wcs[ii]
            data = self._data[ii]

            pixels = ww.wcs_world2pix(cc, 0)

            for jj in range(len(pixels)):

                if not np.ma.is_masked(ebv.mask[jj]) and ebv[jj] != 0.0:
                    continue

                xx, yy = pixels[jj]
                x0 = int(xx)
                y0 = int(yy)

                # If the pixel coordinates are within the footprint of the
                # image, stores the value of E(B-V). This value can be
                # overwritten later if interpolate=True, but this ensures that
                # there is a stored value if interpolate=True but the boundary
                # conditions are invalid (the pixel is too close to the border
                # of the image).
                if (x0 >= 0.0 and y0 >= 0.0 and x0 <= data.shape[1] - 1 and
                        y0 <= data.shape[0] - 1):
                    ebv[jj] = data[y0, x0]

                if interpolate is True:
                    # Checks that the boundary conditions are ok for the
                    # interpolation.
                    if (xx < 1.0 or yy < 1.0 or xx > data.shape[1] - 1 or yy > data.shape[0] - 1):
                        continue

                    mgrid = np.mgrid[y0 - 1:y0 + 2, x0 - 1:x0 + 2]
                    grid = np.array([mgrid[0].flatten(), mgrid[1].flatten()]).T
                    slice = data[y0 - 1:y0 + 2, x0 - 1:x0 + 2].flatten()
                    ebv[jj] = griddata(grid, slice, (yy, xx))

            # del data

        # Calculates iIncrease and gIncrease from E(B-V)
        for ii in range(len(ebv)):

            if np.ma.is_masked(ebv[ii]) is True:
                continue

            # Calculates the reddened flux (we use -ebv[ii]).
            fluxOut = ccmUnred(lambdaIn, fluxIn, -ebv[ii])
            iIncrease[ii] = 1. / fluxOut[3]**2
            gIncrease[ii] = 1. / fluxOut[1]**2

        return {
            'galCoords': cc.tolist(),
            'EBV': ebv,
            'iIncrease': iIncrease,
            'gIncrease': gIncrease
        }

    def loadMaps(self):

        self._hdu = []
        self._header = []
        self._wcs = []
        self._data = []

        for mm in self.maps:

            if fitsLib == 'fitsio':
                hduList = fitsio.FITS(mm)
                self._hdu.append(hduList[-1])

                header = hduList[-1].read_header()
                self._header.append(header)
                self._wcs.append(WCS(header))
                self._data.append(hduList[-1].read())

            else:
                hduList = fits.open(mm)
                self._hdu.append(hduList[-1])

                header = hduList[-1].header
                self._header.append(header)
                self._wcs.append(WCS(header))
                self._data.append(hduList[-1].data)


from datetime import datetime
import os

import database_func

class Date:
    """
    Adds date based on what file it is.
    Params::
    filename - The Full path to file oncluding file name
    """

    def __init__(self, folder, filename):
        self.folder = folder
        self.filename = filename
        self.path = "F:\Work_Python_Scripts\Windows_info\\"
        self.date = str(datetime.now().strftime("%Y-%m-%d %H-%M"))

    def format_date(self):
        temp_list = []
        new = self.path + self.folder + "\\" + self.date + ".txt"
        old = self.path +  self.folder + "\\" + self.filename + ".txt"
        self.rename(old, new)

    def rename(self, old, new):
        os.rename(old, new)

def get_computername():
    """
    Returns a Computer name from text file which is then iterated over in
    powershell()
    """
    with open (r'f:\Work_Python_Scripts\Windows_info\powershell\NoBOM.txt', 'r') as f:

        x = f.read().splitlines()

        return x

def add_computer():
    """
    Reads in Text file created by powershell() assigns it to a variable then
    passes the Info to Computer Class which creates an object.
    Lastly the function passes the returned object to db.update_data
    """

    file1 = 'f:\Work_Python_Scripts\Windows_info\powershell\SysConfig.txt'
    file = r'f:\Work_Python_Scripts\Windows_info\powershell\ou.txt'

    try:
        with open(file1, 'r') as f1:

            text = f1.read().splitlines()

            if text:
                name = text[0]
                username = text[1]
                windows = text[2]
                cpu = text[3]
                currentamount = text[4]
                totalslots = text[5]
                lastlogon = text[6]
                ipaddress = text[7]
                splitted = text[8].split(",")
                location = splitted[2][3:]

                computer = database_func.Computer(name=name, username=username,
                                                  windows=windows, cpu=cpu,
                                                  currentamount=currentamount,
                                                  totalslots=totalslots,
                                                  lastlogon=lastlogon,
                                                  ipaddress=ipaddress,
                                                  location=location)
                database_func.update_data(computer)

            else:
                print("")

    except FileNotFoundError:
        pass

def log_main(added, updated, not_connect, deleted):

    log = r"F:\Work_Python_Scripts\Windows_info\logs\main.txt"

    with open (log, 'w') as f:
        f.write("Computers that were added\n")
        f.write("-" * 95)
        f.write("\n")
        f.writelines(f"{line}\n" for line in added)
        f.write("Computers that were updated\n")
        f.write("-" * 95)
        f.write("\n")
        f.writelines(f"{line}\n" for line in updated)
        f.write("Computers that could not be connect to\n")
        f.write("-" * 95)
        f.write("\n")
        f.writelines(f"{line}\n" for line in not_connect)
        f.write("Deleted Computers\n")
        f.write("-" * 95)
        f.write("\n")
        f.writelines(f"{line}\n" for line in deleted)
    p = Date("logs", "main")
    p.format_date()

def change_directory(cwd):
    powershell=r"F:\Work_Python_Scripts\Windows_Info\powershell\\"
    python=r"F:\Work_Python_Scripts\Windows_info\python\\"
    logs=r"F:\Work_Python_Scripts\Windows_info\logs\\"

    if  cwd == python:
        os.chdir(python)

    elif cwd == powershell:
        os.chdir(powershell)

    else:
        os.chdir(logs)


# -*- coding: utf-8 -*-
#
# Copyright (C) 2018-2020 CERN.
# Copyright (C) 2018-2020 RERO.
#
# Invenio-Circulation is free software; you can redistribute it and/or modify
# it under the terms of the MIT License; see LICENSE file for more details.

"""Invenio Circulation custom transitions."""

from flask import current_app
from invenio_db import db

from invenio_circulation.proxies import current_circulation

from ..api import can_be_requested, get_available_item_by_doc_pid, \
    get_document_pid_by_item_pid, get_pending_loans_by_doc_pid, \
    is_item_at_desk_available_for_checkout
from ..errors import ItemDoNotMatchError, ItemNotAvailableError, \
    LoanMaxExtensionError, RecordCannotBeRequestedError, \
    TransitionConditionsFailedError, TransitionConstraintsViolationError
from ..transitions.base import Transition
from ..transitions.conditions import is_same_location

def _ensure_valid_loan_duration(loan, initial_loan):
    """Validate start and end dates for a loan."""
    loan.setdefault("start_date", loan["transaction_date"])

    if not loan.get("end_date"):
        get_loan_duration = current_app.config["CIRCULATION_POLICIES"][
            "checkout"
        ]["duration_default"]
        duration = get_loan_duration(loan, initial_loan)
        loan["end_date"] = loan["start_date"] + duration

    is_duration_valid = current_app.config["CIRCULATION_POLICIES"]["checkout"][
        "duration_validate"
    ]
    if not is_duration_valid(loan):
        msg = "The loan duration from '{0}' to '{1}' is not valid.".format(
            loan["start_date"].isoformat(), loan["end_date"].isoformat()
        )
        raise TransitionConstraintsViolationError(description=msg)

def _ensure_item_attached_to_loan(loan):
    """Validate that an item is attached to a loan."""
    if not loan.get("item_pid"):
        msg = "No item assigned to loan '{0}'.".format(loan.id)
        raise TransitionConditionsFailedError(description=msg)

def ensure_same_item(f):
    """Validate that the item PID exists and cannot be changed."""
    def inner(self, loan, initial_loan, **kwargs):
        item_pid = kwargs.get("item_pid")

        if item_pid:
            if not current_app.config["CIRCULATION_ITEM_EXISTS"](item_pid):
                msg = "Item '{0}:{1}' not found in the system".format(
                    item_pid["type"], item_pid["value"]
                )
                raise ItemNotAvailableError(description=msg)

            wrong_pid_value = loan.get("item_pid") and \
                item_pid["value"] != loan["item_pid"]["value"]
            wrong_pid_type = loan.get("item_pid") and \
                item_pid["type"] != loan["item_pid"]["type"]
            if wrong_pid_value or wrong_pid_type:
                msg = (
                    "Cannot change item '{0}:{1}' while performing an "
                    "action on this loan".format(
                        item_pid["type"], item_pid["value"]
                    )
                )
                raise ItemDoNotMatchError(description=msg)

        return f(self, loan, initial_loan, **kwargs)

    return inner

def _update_document_pending_request_for_item(item_pid, **kwargs):
    """Update pending loans on a Document with no Item attached yet.

    :param item_pid: a dict containing `value` and `type` fields to
        uniquely identify the item.
    """
    document_pid = get_document_pid_by_item_pid(item_pid)
    for pending_loan in get_pending_loans_by_doc_pid(document_pid):
        pending_loan["item_pid"] = item_pid
        pending_loan.commit()
        db.session.commit()
        current_circulation.loan_indexer().index(pending_loan)

def _ensure_same_location(item_pid, location_pid, destination, error_msg):
    """Validate that item location is same as given location."""
    if not is_same_location(item_pid, location_pid):
        error_msg += " Transition to '{}' has failed.".format(destination)
        raise TransitionConditionsFailedError(description=error_msg)

def _ensure_not_same_location(item_pid, location_pid, destination, error_msg):
    """Validate that item location is not the same as given location."""
    if is_same_location(item_pid, location_pid):
        error_msg += " Transition to '{}' has failed.".format(destination)
        raise TransitionConditionsFailedError(description=error_msg)

def _validate_item_pickup_transaction_locations(loan, destination, **kwargs):
    """Validate the loan item, pickup and transaction locations."""
    item_location_pid = \
        current_app.config["CIRCULATION_ITEM_LOCATION_RETRIEVER"](
            loan["item_pid"])
    kwargs["item_location_pid"] = item_location_pid
    validate_item_pickup_transaction_locations = current_app.config[
        "CIRCULATION_LOAN_LOCATIONS_VALIDATION"]
    if not validate_item_pickup_transaction_locations(
            loan, destination, **kwargs):
        raise TransitionConditionsFailedError()

def _get_item_location(item_pid):
    """Retrieve Item location based on PID."""
    return current_app.config["CIRCULATION_ITEM_LOCATION_RETRIEVER"](item_pid)

def _ensure_default_pickup_location(loan, context):
    """Set default pickup location if no one."""
    if not context.get("pickup_location_pid") \
            or "pickup_location_pid" not in loan:
        loan['pickup_location_pid'] = _get_item_location(loan['item_pid'])

class ToItemOnLoan(Transition):
    """Action to checkout."""

    def before(self, loan, initial_loan, **kwargs):
        """Validate checkout action."""
        super().before(loan, initial_loan, **kwargs)

        self.ensure_item_is_available_for_checkout(loan)

        _ensure_default_pickup_location(loan, kwargs)

        _ensure_valid_loan_duration(loan, initial_loan)

class ItemAtDeskToItemOnLoan(Transition):
    """Check-out action to perform a loan when item ready at desk."""

    def before(self, loan, initial_loan, **kwargs):
        """Validate checkout action."""
        super().before(loan, initial_loan, **kwargs)

        self.ensure_at_desk_item_is_available_for_checkout(loan)

        _ensure_default_pickup_location(loan, kwargs)

        _ensure_valid_loan_duration(loan, initial_loan)

    def ensure_at_desk_item_is_available_for_checkout(self, loan):
        """Validate that an item at desk is available for checkout."""
        self._check_item_before_availability(loan)

        # patron_pid is mandatory for next steps
        if 'patron_pid' not in loan:
            msg = "Patron not set for loan with pid '{}'".format(loan['pid'])
            raise TransitionConstraintsViolationError(description=msg)

        is_available = is_item_at_desk_available_for_checkout(
            loan['item_pid'],
            loan['patron_pid']
        )
        if not is_available:
            raise ItemNotAvailableError(
                item_pid=loan['item_pid'], transition=self.dest)

def check_request_on_document(f):
    """Decorator to check if the request is on document."""
    def inner(self, loan, initial_loan, **kwargs):
        document_pid = kwargs.get("document_pid")
        if document_pid and not kwargs.get("item_pid"):
            if not can_be_requested(loan):
                msg = "Cannot create a request for the document '{}'".format(
                    loan.get("document_pid")
                )
                raise RecordCannotBeRequestedError(description=msg)

            if self.assign_item:
                available_item_pid = get_available_item_by_doc_pid(
                    document_pid
                )
                if available_item_pid:
                    kwargs["item_pid"] = available_item_pid

        if kwargs.get("item_pid") and not kwargs.get("pickup_location_pid"):
            # if no pickup location was specified in the request,
            # assign a default one
            kwargs["pickup_location_pid"] = _get_item_location(
                kwargs["item_pid"]
            )

        return f(self, loan, initial_loan, **kwargs)
    return inner

class CreatedToPending(Transition):
    """Action to request to loan an item."""

    def __init__(
        self, src, dest, trigger="next", permission_factory=None, **kwargs
    ):
        """Constructor."""
        super().__init__(
            src,
            dest,
            trigger=trigger,
            permission_factory=permission_factory,
            **kwargs
        )
        self.assign_item = kwargs.get("assign_item", True)

    @check_request_on_document
    def before(self, loan, initial_loan, **kwargs):
        """Check if the loan request can be created."""
        super().before(loan, initial_loan, **kwargs)

        if not can_be_requested(loan):
            msg = "Cannot create a request for the loan '{}'".format(loan)
            raise RecordCannotBeRequestedError(description=msg)

class PendingToItemAtDesk(Transition):
    """Validate pending request to prepare the item at desk of its location."""

    def before(self, loan, initial_loan, **kwargs):
        """Validate if the item is for this location or should transit."""
        super().before(loan, initial_loan, **kwargs)

        # check if a request on document has no item attached
        _ensure_item_attached_to_loan(loan)
        # validate the item, pickup and transaction locations of the loan
        _validate_item_pickup_transaction_locations(loan, self.dest, **kwargs)

class PendingToItemInTransitPickup(Transition):
    """Validate pending request to send the item to the pickup location."""

    def before(self, loan, initial_loan, **kwargs):
        """Validate if the item is for this location or should transit."""
        super().before(loan, initial_loan, **kwargs)

        # check if a request on document has no item attached
        _ensure_item_attached_to_loan(loan)
        # validate the item, pickup and transaction locations of the loan
        _validate_item_pickup_transaction_locations(loan, self.dest, **kwargs)

class ItemOnLoanToItemOnLoan(Transition):
    """Extend action to perform a item loan extension."""

    def update_extension_count(self, loan):
        """Check number of extensions and update it."""
        extension_count = loan.get("extension_count", 0)
        extension_count += 1

        get_extension_max_count_func = current_app.config[
            "CIRCULATION_POLICIES"
        ]["extension"]["max_count"]
        extension_max_count = get_extension_max_count_func(loan)
        if extension_count > extension_max_count:
            raise LoanMaxExtensionError(
                loan_pid=loan["pid"], extension_count=extension_max_count
            )
        loan["extension_count"] = extension_count

    def update_loan_end_date(self, loan, initial_loan):
        """Update the end date of the extended loan."""
        get_extension_duration_func = current_app.config[
            "CIRCULATION_POLICIES"
        ]["extension"]["duration_default"]
        duration = get_extension_duration_func(loan, initial_loan)

        should_extend_from_end_date = current_app.config[
            "CIRCULATION_POLICIES"
        ]["extension"]["from_end_date"]
        if not should_extend_from_end_date:
            # extend from the transaction_date instead
            loan["end_date"] = loan["transaction_date"]

        loan["end_date"] += duration

    @ensure_same_item
    def before(self, loan, initial_loan, **kwargs):
        """Validate extension action."""
        super().before(loan, initial_loan, **kwargs)
        self.update_extension_count(loan)
        self.update_loan_end_date(loan, initial_loan)

class ItemOnLoanToItemInTransitHouse(Transition):
    """Check-in action when returning an item not to its belonging location."""

    @ensure_same_item
    def before(self, loan, initial_loan, **kwargs):
        """Validate check-in action."""
        super().before(loan, initial_loan, **kwargs)

        _ensure_not_same_location(
            loan["item_pid"],
            loan["transaction_location_pid"],
            self.dest,
            error_msg="Item should be returned (already in house).",
        )

class ItemOnLoanToItemReturned(Transition):
    """Check-in action when returning an item to its belonging location."""

    def __init__(
        self, src, dest, trigger="next", permission_factory=None, **kwargs
    ):
        """Constructor."""
        super().__init__(
            src,
            dest,
            trigger=trigger,
            permission_factory=permission_factory,
            **kwargs
        )
        self.assign_item = kwargs.get("assign_item", True)

    @ensure_same_item
    def before(self, loan, initial_loan, **kwargs):
        """Validate check-in action."""
        super().before(loan, initial_loan, **kwargs)
        _ensure_same_location(
            loan["item_pid"],
            loan["transaction_location_pid"],
            self.dest,
            error_msg="Item should be in transit to house.",
        )

        # set end loan date as transaction date when completing loan
        loan["end_date"] = loan["transaction_date"]

    def after(self, loan, initial_loan):
        """Check for pending requests on this item after check-in."""
        super().after(loan, initial_loan)
        if self.assign_item:
            _update_document_pending_request_for_item(loan["item_pid"])

class ItemInTransitHouseToItemReturned(Transition):
    """Check-in action when returning an item to its belonging location."""

    def __init__(
        self, src, dest, trigger="next", permission_factory=None, **kwargs
    ):
        """Constructor."""
        super().__init__(
            src,
            dest,
            trigger=trigger,
            permission_factory=permission_factory,
            **kwargs
        )
        self.assign_item = kwargs.get("assign_item", True)

    @ensure_same_item
    def before(self, loan, initial_loan, **kwargs):
        """Validate check-in action."""
        super().before(loan, initial_loan, **kwargs)

        _ensure_same_location(
            loan["item_pid"],
            loan["transaction_location_pid"],
            self.dest,
            error_msg="Item should be in transit to house.",
        )

    def after(self, loan, initial_loan):
        """Check for pending requests on this item after check-in."""
        super().after(loan, initial_loan)
        if self.assign_item:
            _update_document_pending_request_for_item(loan["item_pid"])

class ToCancelled(Transition):
    """When cancelling a loan, ensure that the item is not changed."""

    @ensure_same_item
    def before(self, loan, initial_loan, **kwargs):
        """Validate cancel action."""
        super().before(loan, initial_loan, **kwargs)


# -*- coding: utf-8 -*-

from collective.transmogrifier.interfaces import ISection
from collective.transmogrifier.interfaces import ISectionBlueprint
from collective.transmogrifier.utils import defaultMatcher
from zope.interface import classProvides
from zope.interface import implements

class WorkflowConverter(object):
    classProvides(ISectionBlueprint)
    implements(ISection)

    def __init__(self, transmogrifier, name, options, previous):
        self.transmogrifier = transmogrifier
        self.name = name
        self.options = options
        self.previous = previous
        self.context = transmogrifier.context
        self.pathkey = defaultMatcher(options, 'path-key', name, 'path')

    def __iter__(self):
        for item in self.previous:
            pathkey = self.pathkey(*item.keys())[0]
            if not pathkey:
                yield item
                continue

            path = item[pathkey]
            obj = self.context.unrestrictedTraverse(path, None)
            if obj is None:
                yield item
                continue

            yield item


'''
Lrasmy@Zhilab  Jan 2021

# This script processes originally extracted data on a distributed platform
# This code version create data with multiple labels for both survival and Binary classification can create files for a predefined split of patients or can randomly split
# also it can build upon existing typoes dictionary or creates its own
# and builds pickled lists including a full list that includes all information for case and controls
## it outputs pickled list of the following shape
#[[pt1_id,label,[
#                  [[delta_time 0],[list of Medical codes in Visit0]],
#                  [[delta_time between V0 and V1],[list of Medical codes in Visit2]],
#                   ......]],
# [pt2_id,label,[[[delta_time 0],[list of Medical codes in Visit0 ]],[[delta_time between V0 and V1],[list of Medical codes in Visit2]],......]]]
#
# This version is for multiple outcomes prediction, therefore the label looks like ['mort_label','LOS','vent_label','time_to_intub','Readmission_label','plos_label']; please note the LOS is equivalent to time to event for in-hospital mortality event.
#
# Usage: feed this script with Case file and Control files each is just a three columns like pt_id | medical_code | visit_date and execute like:
#
# python data_preprocessing_v5.py <Data File> <labels File> <types dictionary if available,otherwise use 'NA' to build new one> <output Files Prefix> <path and prefix to pts file if available,otherwise use 'NA' for random slit into 7:1:2, or 'nosplit' for avoid splitting>
# you can optionally activate <case_samplesize> <control_samplesize> based on your cohort definition
# This file will later split the data to training , validation and Test sets of ratio
# Output files include
# <output file>.pts: List of unique Patient ids. Created for validation and comparison purposes
# <output file>.types: Python dictionary that maps string diagnosis codes to integer diagnosis codes.
# Main output files for the baseline RNN models are <output file>.combined
'''

import sys
from optparse import OptionParser
try:
    import cPickle as pickle
except:
    import pickle
import numpy as np
import random
import pandas as pd
from datetime import datetime as dt
from datetime import timedelta
import glob
#import timeit ( for time tracking if required)

if __name__ == '__main__':

    dataFile= sys.argv[1]
    labelFile= sys.argv[2]
    typeFile= sys.argv[3]
    outFile = sys.argv[4]
    pts_file_pre = sys.argv[5]
    #cls_type= sys.argv[6]
    #samplesize_pts = int(sys.argv[7])
    parser = OptionParser()
    (options, args) = parser.parse_args()

    #_start = timeit.timeit()
    debug=False
    #np.random.seed(1)
    time_list = []
    dates_list =[]
    label_list = []
    pt_list = []

    ## loading Case
    print('loading data')
    ##### please uncomment glob lines for distributed data
    #all_files1 = glob.glob(dataFile + "/*.csv")
    #li1 = []
    #for filename in all_files1:
    #    df = pd.read_csv(filename)
    #    li1.append(df)
    #data_dat = pd.concat(li1).drop_duplicates()
    data_dat=pd.read_table(dataFile)
    data_dat.columns = ["Pt_id", "ICD", "Time"]

    #data_dat=data_dat[~(data_dat["ICD"].str.startswith('SNOMED') |data_dat["ICD"].str.startswith('L'))] ### use if you need to exclude or include only certain type of codes

    print('loaded data for: ',data_dat["Pt_id"].nunique())

    print('loading labels')

    #all_files = glob.glob(labelFile + "/*.csv")
    #li = []
    #for filename in all_files:
    #    df = pd.read_csv(filename)
    #    li.append(df)
    #data_lbl_v1 = pd.concat(li).drop_duplicates()
    data_lbl_v1=pd.read_table(labelFile)
    data_lbl_v1.columns = ["Pt_id", "mort_label","LOS","vent_label","time_to_intub","Readmission_label","plos_label"]
    data_lbl=pd.merge(data_dat["Pt_id"].drop_duplicates(),data_lbl_v1, how='inner').drop_duplicates()
    print('loaded labels for: ',data_lbl_v1["Pt_id"].nunique() , ' after primary cleaning ',data_lbl["Pt_id"].nunique())
    print('Mortality Case counts: ',data_lbl[data_lbl["mort_label"]==1]["Pt_id"].nunique())
    print('Intubation Case counts: ',data_lbl[data_lbl["vent_label"]==1]["Pt_id"].nunique())
    print('Intubation Case with tti >=1 : ',data_lbl[(data_lbl["vent_label"]==1)& (data_lbl["time_to_intub"]>=1)]["Pt_id"].nunique())
    print('LOS>7 : ',data_lbl[data_lbl["LOS"]>7]["Pt_id"].nunique())
    print('pLOS>7 : ',data_lbl[data_lbl["plos_label"]==1]["Pt_id"].nunique())
    print('Readmission case counts : ',data_lbl[data_lbl["Readmission_label"]==1]["Pt_id"].nunique())

    ### An example of sampling code: Control Sampling
    #print('pt sampling')
    #data_sk=data_dat["Pt_id"]
    #data_sk=data_sk.drop_duplicates()
    #data_sk_samp=data_sk.sample(n=samplesize_pts) ## that is an input arg 7
    #data_dat=data_dat[data_dat["Pt_id"].isin(data_sk_samp.values.tolist())]
    #data_lbl=data_lbl[data_lbl["Pt_id"].isin(data_sk_samp.values.tolist())]

    ## loading the types
    #### If using pretrained models, you should start from the shared types file

    if typeFile=='NA':
       types={"zero_pad":0}
       print('new types dictionary')
    else:
      with open(typeFile, 'rb') as t2:
             types=pickle.load(t2)
      print('types dictionary loaded')

    #end_time = timeit.timeit()
    #print ("consumed time for data loading",(_start -end_time)/1000.0 )

    full_list=[]
    index_date = {}
    time_list = []
    dates_list =[]
    label_list = []
    pt_list = []
    dur_list=[]
    newVisit_list = []
    count=0

    for Pt, group in data_dat.groupby('Pt_id'):
            data_i_c = []
            data_dt_c = []
            for Time, subgroup in group.sort_values(['Time'], ascending=False).groupby('Time', sort=False): ### ascending=True normal order ascending=False reveresed order
                        data_i_c.append(np.array(subgroup['ICD']).tolist())
                        data_dt_c.append(dt.strptime(Time, '%Y-%m-%d'))
            if len(data_i_c) > 0:
                 # creating the duration in days between visits list, first visit marked with 0   (last in reversed order)
                    v_dur_c=[]
            if len(data_dt_c)<=1:
                     v_dur_c=[0]
            else:
                     for jx in range (len(data_dt_c)):
                        if jx==0:
                             v_dur_c.append(jx)
                        else:
                            #xx = ((dt.strptime(data_dt_c[jx-1], '%d-%b-%y'))-(dt.strptime(data_dt_c[jx], '%d-%b-%y'))).days ## use if original data have time information or different date format
                            #xx = (data_dt_c[jx]- data_dt_c[jx-1]).days ### normal order
                            xx = (data_dt_c[jx-1] - data_dt_c[jx]).days ## reversed order
                            v_dur_c.append(xx)

            ### Diagnosis recoding
            newPatient_c = []
            for visit in data_i_c:
                      newVisit_c = []
                      for code in visit:
                                    if code in types: newVisit_c.append(types[code])
                                    else:
                                          types[code] = max(types.values())+1
                                          newVisit_c.append(types[code])
                      newPatient_c.append(newVisit_c)

            if len(data_i_c) > 0: ## only save non-empty entries
                  label_list.append(data_lbl.loc[data_lbl.Pt_id == Pt, ['mort_label','LOS','vent_label','time_to_intub','Readmission_label','plos_label']].values.squeeze().tolist())  #### LR ammended for multilabel
                  pt_list.append(Pt)
                  newVisit_list.append(newPatient_c)
                  dur_list.append(v_dur_c)

            count=count+1
            if count % 1000 == 0: print ('processed %d pts' % count)

    ### Creating the full pickled lists ### uncomment if you need to dump the all data before splitting

    pickle.dump(types, open(outFile+'.types', 'wb'), -1)

    ### Create the combined list for the Pytorch RNN
    fset=[]
    print ('Reparsing')
    for pt_idx in range(len(pt_list)):
                pt_sk= pt_list[pt_idx]
                pt_lbl= label_list[pt_idx]
                pt_vis= newVisit_list[pt_idx]
                pt_td= dur_list[pt_idx]
                d_gr=[]
                n_seq=[]
                d_a_v=[]
                for v in range(len(pt_vis)):
                        nv=[]
                        nv.append([pt_td[v]])
                        nv.append(pt_vis[v])
                        n_seq.append(nv)
                n_pt= [pt_sk,pt_lbl,n_seq]
                fset.append(n_pt)

    ### Random split to train ,test and validation sets, if needed
    print ("Splitting")

    if pts_file_pre=='nosplit':
      nptfile = outFile +'.pts.all'
      pickle.dump(pt_list, open(nptfile, 'wb'),protocol=2)
      filename_all=outFile+'.combined.all'
      pickle.dump(fset, open(filename_all, 'wb'), -1)

    else:
      if pts_file_pre=='NA':
          print('random split')
          dataSize = len(pt_list)
          #np.random.seed(0)
          ind = np.random.permutation(dataSize)
          nTest = int(0.2 * dataSize)
          nValid = int(0.1 * dataSize)
          test_indices = ind[:nTest]
          valid_indices = ind[nTest:nTest+nValid]
          train_indices = ind[nTest+nValid:]
      else:
          print ('loading previous splits')
          pt_train=pickle.load(open(pts_file_pre+'.train', 'rb'))
          pt_valid=pickle.load(open(pts_file_pre+'.valid', 'rb'))
          pt_test=pickle.load(open(pts_file_pre+'.test', 'rb'))
          test_indices = np.intersect1d(pt_list, pt_test,assume_unique=True, return_indices=True)[1]
          valid_indices= np.intersect1d(pt_list, pt_valid,assume_unique=True, return_indices=True)[1]
          train_indices= np.intersect1d(pt_list, pt_train,assume_unique=True, return_indices=True)[1]

      for subset in ['train','valid','test']:
          if subset =='train':
              indices = train_indices
          elif subset =='valid':
              indices = valid_indices
          elif subset =='test':
              indices = test_indices
          else:
              print ('error')
              break

          ### split the full combined set to the same as individual files
          #### only using Pts file for later, so dumping them

          subset_p = [pt_list[i] for i in indices]
          nptfile = outFile +'.pts.'+subset
          pickle.dump(subset_p, open(nptfile, 'wb'),protocol=2)
          subset_nfull = [fset[i] for i in indices]
          csfilename=outFile+'.combined.'+subset
          pickle.dump(subset_nfull, open(csfilename, 'wb'), -1)


# -*- coding: utf-8 -*-
#%% NumPyの読み込み
import numpy as np
#   SciPyのstatsモジュールの読み込み
import scipy.stats as st
#   Pandasの読み込み
import pandas as pd
#   PyMCの読み込み
import pymc as pm
#   MatplotlibのPyplotモジュールの読み込み
import matplotlib.pyplot as plt
#   日本語フォントの設定
from matplotlib.font_manager import FontProperties
import sys
if sys.platform.startswith('win'):
    FontPath = 'C:\\Windows\\Fonts\\meiryo.ttc'
elif sys.platform.startswith('darwin' ):
    FontPath = '/System/Library/Fonts/ヒラギノ角ゴシック W4.ttc'
elif sys.platform.startswith('linux'):
    FontPath = '/usr/share/fonts/truetype/takao-gothic/TakaoPGothic.ttf'
else:
    sys.exit('このPythonコードが対応していないOSを使用しています．')
jpfont = FontProperties(fname=FontPath)
#   PandasからMatplotlibへのコンバーター
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()
#%% 使用電力量データの読み込み
"""
    電灯電力需要実績月報・用途別使用電力量・販売電力合計・10社計
    電気事業連合会ウェブサイト・電力統計情報より入手
    http://www.fepc.or.jp/library/data/tokei/index.html
"""
data = pd.read_csv('electricity.csv', index_col=0)
y0 = np.log(data.values.reshape((data.shape[0]//3, 3)).sum(axis=1))
y = 100 * (y0 - y0[0])
n = y.size
series_date = pd.date_range(start='1/1/1989', periods=n, freq='Q')
#%% 確率的トレンド+季節変動
trend_coef = np.array([2.0, -1.0])
seasonal_coef = np.array([-1.0, -1.0, -1.0])
timeseries_decomp = pm.Model()
with timeseries_decomp:
    sigma = pm.HalfCauchy('sigma', beta=1.0)
    tau = pm.HalfCauchy('tau', beta=1.0)
    omega = pm.HalfCauchy('omega', beta=1.0)
    trend = pm.AR('trend', trend_coef, sigma=tau, shape=n)
    seasonal = pm.AR('seasonal', seasonal_coef, sigma=omega, shape=n)
    observation = pm.Normal('y', mu=trend+seasonal, sigma=sigma, observed=y)
#%% 事後分布からのサンプリング
param_names = ['sigma', 'tau', 'omega']
n_draws = 5000
n_chains = 4
n_tune = 2000
with timeseries_decomp:
    trace = pm.sample(draws=n_draws, chains=n_chains, tune=n_tune,
                      target_accept=0.95, random_seed=123)
    print(pm.summary(trace, var_names=param_names))
#%% 事後分布のグラフの作成
series_name = ['原系列', '平滑値', 'トレンド', '季節変動', 'ノイズ']
k = len(param_names)
fig1, ax1 = plt.subplots(k, 2, num=1, figsize=(8, 1.5*k), facecolor='w')
for index in range(k):
    mc_trace = trace.posterior[param_names[index]].values.flatten()
    x_min = mc_trace.min() - 0.2 * np.abs(mc_trace.min())
    x_max =  mc_trace.max() + 0.2 * np.abs(mc_trace.max())
    x = np.linspace(x_min, x_max, 250)
    posterior = st.gaussian_kde(mc_trace).evaluate(x)
    ax1[index, 0].plot(mc_trace, 'k-', linewidth=0.1)
    ax1[index, 0].set_xlim(1, n_draws*n_chains)
    ax1[index, 0].set_ylabel('$\\{label:s}$'.format(label=param_names[index]),
                             fontproperties=jpfont)
    ax1[index, 1].plot(x, posterior, 'k-')
    ax1[index, 1].set_xlim(x_min, x_max)
    ax1[index, 1].set_ylim(0, 1.1*posterior.max())
    ax1[index, 1].set_ylabel('確率密度', fontproperties=jpfont)
ax1[k-1, 0].set_xlabel('乱数系列', fontproperties=jpfont)
ax1[k-1, 1].set_xlabel('周辺事後分布', fontproperties=jpfont)
plt.tight_layout()
plt.savefig('pybayes_fig_decomp_posterior.png', dpi=300)
plt.show()
#%% 時系列の分解
trend = trace.posterior['trend'].values.reshape(n_draws*n_chains, n).mean(axis=0)
seasonal = trace.posterior['seasonal'].values.reshape(n_draws*n_chains, n).mean(axis=0)
noise = y - trend - seasonal
series = np.vstack((y, trend + seasonal, trend, seasonal, noise)).T
results = pd.DataFrame(series, index=series_date, columns=series_name)
fig2, ax2 = plt.subplots(4, 1, sharex='col',
                         num=2, figsize=(8, 6), facecolor='w')
for index in range(4):
    ts_name = series_name[index+1]
    ax2[index].plot(results[ts_name], 'k-', label=ts_name)
    ax2[index].set_ylabel(ts_name, fontproperties=jpfont)
ax2[0].plot(results[series_name[0]], 'k:', label=series_name[0])
ax2[0].set_xlim(series_date[0], series_date[-1])
ax2[0].legend(loc='lower right', frameon=False, prop=jpfont)
plt.tight_layout()
plt.savefig('pybayes_fig_decomp_timeseries.png', dpi=300)
plt.show()


'''
Created on Sep 28, 2014

@author: jesse
'''

from pygame.threads import Thread
import logging
import time

logging.basicConfig(level = logging.WARNING)

class Parameters(object):
    '''
    Takes care reading and writing parameters.
    '''

    toc = None
    parameters = {}
    _cf = None

    def __init__(self, cf):
        '''
        Creates a new set of sensors
        '''
        self._cf = cf

    def setup(self):
        # Print the param TOC
        self.toc = self._cf.param.toc.toc
        for group in sorted(self.toc.keys()):
            for param in sorted(self.toc[group].keys()):
                self._cf.param.add_update_callback(group = group, name = param,
                                           cb = self.updateValue)

    def start(self):
        t = Thread(target = self.updateLoop)
        t.daemon = True
        t.start()

    def updateValue(self, name, value):
        parts = name.split(".")
        if parts[0] not in self.parameters:
            self.parameters[parts[0]] = {}
        group = self.parameters[parts[0]]
        group[parts[1]] = value

    def requestUpdate(self):
        for group in sorted(self.toc.keys()):
            for param in sorted(self.toc[group].keys()):
                self._cf.param.request_param_update("{0}.{1}".format(group, param))

    def setValue(self, paramname, value):
        self._cf.param.set_value(paramname, "{:.2f}".format(value))
        self._cf.param.request_param_update(paramname)

    def updateLoop(self):
        while True:
            self.requestUpdate()
            time.sleep(5)


# -*- coding: utf-8 -*-
"""
Created on Mon Feb 25 15:04:43 2019

@author:

To represent mini-batch method
"""

import sys,os
sys.path.append(os.pardir)
from dataset.mnist import load_mnist
import numpy as np
from common.functions import softmax, cross_entropy_error

(x_train,y_train),(x_test,y_test) = \
     load_mnist(normalize=True, one_hot_label=True)

batch_size = 10
train_size = x_train.shape[0]
batch_mask = np.random.choice(train_size,batch_size)
x_batch = x_train[batch_mask]
y_batch = y_train[batch_mask]

class simpleNet:
    def __init__(self):
        self.w = np.random.randn(2,3)

    def predict(self,x):
        return np.dot(x,self.w)

    def loss(self,x,t):
        z=self.predict(self.w,x)
        y=softmax(z)
        loss=cross_entropy_error(y,t)

        return loss

net=simpleNet()
print(net.w)


import itertools
import json
import math

def get_formatter(test_type):
    if test_type == "iozone" or test_type == "fio":
        return format_io_stat
    elif test_type == "pgbench":
        return format_pgbench_stat
    else:
        raise Exception("Cannot get formatter for type %s" % test_type)

def format_io_stat(res):
    if len(res) != 0:
        bw_mean = 0.0
        for measurement in res:
            bw_mean += measurement["bw_mean"]

        bw_mean /= len(res)

        it = ((bw_mean - measurement["bw_mean"]) ** 2 for measurement in res)
        bw_dev = sum(it) ** 0.5

        meta = res[0]['__meta__']

        sync = meta['sync']
        direct = meta['direct_io']

        if sync and direct:
            ss = "d+"
        elif sync:
            ss = "s"
        elif direct:
            ss = "d"
        else:
            ss = "a"

        key = "{0} {1} {2} {3}k".format(meta['action'], ss,
                                        meta['concurence'],
                                        meta['blocksize'])

        data = json.dumps({key: (int(bw_mean), int(bw_dev))})

        return data

def format_pgbench_stat(res):
    """
    Receives results in format:
    "<num_clients> <num_transactions>: <tps>
     <num_clients> <num_transactions>: <tps>
     ....
    "
    """
    if res:
        data = {}
        grouped_res = itertools.groupby(res, lambda x: x[0])
        for key, group in grouped_res:
            results = list(group)
            sum_res = sum([r[1] for r in results])
            mean = sum_res/len(results)
            sum_sq = sum([(r[1] - mean) ** 2 for r in results])
            if len(results) > 1:
                dev = math.sqrt(sum_sq / (len(results) - 1))
            else:
                dev = 0
            data[key] = (mean, dev)
        return data


from flask import Flask, request
import os
import librosa
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler

app = Flask(__name__)

@app.route('/', methods = ['GET', 'POST'])
def hello():
    filelist = os.listdir('static')
    train_df = pd.DataFrame(filelist)
    train_df = train_df.rename(columns={0:'file'})
    train_df = train_df.sample(frac=1).reset_index(drop=True)
    speaker = []
    for i in range(0, len(train_df)):
        speaker.append(train_df['file'][i][0:-5])
    train_df['speaker'] = speaker
    train_features = train_df.apply(extract_features, axis=1)
    features_test = []
    for i in range(0, len(train_features)):
        features_test.append(np.concatenate((train_features[i][0], train_features[i][1],
                    train_features[i][2], train_features[i][3],
                    train_features[i][4]), axis=0))
    X_test = np.array(features_test)
    print(X_test)
    ss = StandardScaler()
    X_test = ss.fit_transform(X_test)
    return print(X_test)

def extract_features(files):
    file_name = os.path.join(os.path.abspath('static')+'/'+str(files.file))
    X, sample_rate = librosa.load(file_name, res_type='kaiser_fast')
    mfccs = np.mean(librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=40).T,axis=0)
    stft = np.abs(librosa.stft(X))
    chroma = np.mean(librosa.feature.chroma_stft(S=stft, sr=sample_rate).T,axis=0)
    mel = np.mean(librosa.feature.melspectrogram(X, sr=sample_rate).T,axis=0)
    contrast = np.mean(librosa.feature.spectral_contrast(S=stft, sr=sample_rate).T,axis=0)
    tonnetz = np.mean(librosa.feature.tonnetz(y=librosa.effects.harmonic(X),
    sr=sample_rate).T,axis=0)
    return mfccs, chroma, mel, contrast, tonnetz