import gym
from gym import wrappers
import numpy as np

np.random.seed(0)

def softmax(x):
    """Compute softmax values for each sets of scores in x."""
    return np.exp(x) / np.sum(np.exp(x), axis=0)

def weighed_avg(arr, weights):
	"""return weighted avg of reward history"""
	res = np.zeros(np.shape(arr[0]))

	soft_weights = softmax(weights)

	for i in range(len(arr)):
		res += arr[i] * soft_weights[i]

	return res

def run_episode(env, parameters):
	observation = env.reset()
	totalreward = 0
	time_steps = 0
	done = False

	while not done:
		env.render()
		#initalize random weights
		action = 0 if np.matmul(parameters, observation)<0 else 1
		observation, reward, done, info = env.step(action)
		totalreward += reward
		time_steps += 1

	return (totalreward, time_steps)

#training
def train(submit):
	env = gym.make('CartPole-v0')
	env = wrappers.Monitor(env, '/tmp/cartpole-experiment-1', force=True)

	episodes_per_update = 5
	noise_scaling = 0.3
	parameters = np.random.rand(4) * 2 -1
	bestreward = 0
	total_steps = 500

	hist_length = 10
	reward_hist = []
	param_hist = []

	for step in range(total_steps):
		decay = np.exp(-step/total_steps)
		newparams = parameters + (np.random.rand(4) *2 -1) * noise_scaling * decay

		reward, time_steps = run_episode(env, newparams)

		print("Epoch : {step+1}, {time_steps} steps, Average : {bestreward}")

		# keep track of last `n` histories
		reward_hist.append(reward)
		reward_hist = reward_hist[-hist_length:]
		param_hist.append(newparams)
		param_hist = param_hist[-hist_length:]

		if reward > bestreward and step < 10:
			bestreward = reward

		if reward > bestreward and step >= 10:
			bestreward = np.average(reward_hist)
			# average of last `n` params
			parameters = weighed_avg(param_hist, reward_hist)

			# if reward == 200:
			# 	break

	env.close()

	if submit:
		gym.upload('/tmp/cartpole-experiment-1')

	return parameters

r = train(submit=False)

print("Trained params : {r}")


#!/usr/bin/env python

# Imports, sorted alphabetically.

# Python packages
from ConfigParser import NoSectionError, NoOptionError
import argparse
from binascii import hexlify
import datetime
import os
import random
import time

# Third-party packages
import pyglet
# Disable error checking for increased performance
pyglet.options['debug_gl'] = False
from pyglet.gl import *

# Modules from this project
from controllers import *
import globals as G
from timer import Timer

class InvalidChoice(Exception):
    pass

def get_or_update_config(section, option, default_value, conv=str, choices=()):
    try:
        if conv is bool:
            user_value = G.config.getboolean(section, option)
        else:
            user_value = conv(G.config.get(section, option))
    except NoSectionError:
        G.config.add_section(section)
    except NoOptionError:
        pass
    else:
        # If the option is already set:
        if choices and user_value not in choices:
            raise InvalidChoice('%s.%s must be in %s' %
                                (section, option, repr(tuple(choices))))
        return user_value
    G.config.set(section, option, str(default_value))
    return default_value

class InvalidKey(Exception):
    pass

def get_key(key_name):
    key_code = getattr(pyglet.window.key, key_name, None)
    if key_code is None:
        # Handles cases like pyglet.window.key._1
        key_code = getattr(pyglet.window.key, '_' + key_name, None)
        if key_code is None:
            raise InvalidKey('%s is not a valid key.' % key_name)
    return key_code

def initialize_config():
    #
    # General
    #

    general = 'General'

    G.DEBUG = get_or_update_config(
        general, 'debug', G.DEBUG, conv=bool)

    get_or_update_config(
        general, 'save_mode', G.SAVE_MODE, choices=G.SAVE_MODES)

    #
    # Graphics
    #

    graphics = 'Graphics'

    G.FULLSCREEN = get_or_update_config(
        graphics, 'fullscreen', G.FULLSCREEN, conv=bool)
    G.WINDOW_WIDTH = get_or_update_config(
        graphics, 'width', G.WINDOW_WIDTH, conv=int)
    G.WINDOW_HEIGHT = get_or_update_config(
        graphics, 'height', G.WINDOW_HEIGHT, conv=int)

    G.DRAW_DISTANCE_CHOICE = get_or_update_config(
        graphics, 'draw_distance', G.DRAW_DISTANCE_CHOICE,
        choices=G.DRAW_DISTANCE_CHOICES)
    G.DRAW_DISTANCE = G.DRAW_DISTANCE_CHOICES[G.DRAW_DISTANCE_CHOICE]

    G.FOG_ENABLED = get_or_update_config(
        graphics, 'fog_enabled', G.FOG_ENABLED, conv=bool)

    G.MOTION_BLUR = get_or_update_config(
        graphics, 'motion_blur', G.MOTION_BLUR, conv=bool)

    #
    # World
    #

    world = 'World'

    # TODO: This setting must be removed when terrain generation will improve.
    get_or_update_config(world, 'size', 64, conv=int)

    #
    # Controls
    #

    controls = 'Controls'

    # Adds missing keys to configuration file and converts to pyglet keys.
    for control, default_key_name in G.KEY_BINDINGS.items():
        key_name = get_or_update_config(controls, control, default_key_name)
        try:
            pyglet_key = get_key(key_name)
        except InvalidKey:
            pyglet_key = get_key(default_key_name)
            G.config.set(controls, control, default_key_name)
        setattr(G, control.upper() + '_KEY', pyglet_key)

    #
    # Save config file
    #

    with open(G.config_file, 'wb') as handle:
        G.config.write(handle)

class Window(pyglet.window.Window):
    def __init__(self, **kwargs):
        kwargs.update(
            caption=G.APP_NAME,
        )
        super(Window, self).__init__(
            G.WINDOW_WIDTH, G.WINDOW_HEIGHT, **kwargs)
        self.exclusive = False
        self.reticle = None
        self.controller = None
        controller = MainMenuController(self)
        self.switch_controller(controller)
        if G.FULLSCREEN:
            self.set_fullscreen()
        pyglet.clock.schedule_interval(self.update, 1.0 / G.MAX_FPS)

    def set_exclusive_mouse(self, exclusive):
        super(Window, self).set_exclusive_mouse(exclusive)
        self.exclusive = exclusive

    def update(self, dt):
        self.controller.update(dt)

    def switch_controller(self, new_controller):
        if self.controller:
            self.controller.pop_handlers()
        self.controller = new_controller
        self.controller.push_handlers()

    def on_key_press(self, symbol, modifiers):
        if self.exclusive:
            if symbol == G.ESCAPE_KEY and not self.fullscreen:
                self.set_exclusive_mouse(False)
            elif symbol == key.Q and self.fullscreen:  # FIXME: Better fullscreen mode.
                pyglet.app.exit()  # for fullscreen

    def on_draw(self):
        if self.exclusive:
            self.reticle.draw(GL_LINES)
            if G.MOTION_BLUR:
                glAccum(GL_MULT, 0.65)
                glAccum(GL_ACCUM, 0.35)
                glAccum(GL_RETURN, 1.0)

    def on_resize(self, width, height):
        if self.reticle:
            self.reticle.delete()
        x, y = width / 2, height / 2
        n = 10
        self.reticle = pyglet.graphics.vertex_list(
            4,
            ('v2i', (x - n, y, x + n, y, x, y - n, x, y + n))
        )

def main(options):
    G.GAME_MODE = options.game_mode
    G.SAVE_FILENAME = options.save
    G.DISABLE_SAVE = options.disable_save
    for name, val in options._get_kwargs():
        setattr(G.LAUNCH_OPTIONS, name, val)

    G.TERRAIN_CHOICE = options.terrain
    G.TERRAIN = G.TERRAIN_CHOICES[options.terrain]

    G.FLAT_MODE = options.flat

    if options.fast:
        G.TIME_RATE /= 20

    seed = options.seed
    if seed is None:
        # Generates pseudo-random number.
        try:
            seed = long(hexlify(os.urandom(16)), 16)
        except NotImplementedError:
            import time
            seed = long(time.time() * 256)  # use fractional seconds
        # Then convert it to a string so all seeds have the same type.
        seed = str(seed)

        print('Random seed: ' + seed)

    random.seed(seed)

    with open(os.path.join(G.game_dir, 'seeds.txt'), 'a') as seeds:
        seeds.write(datetime.datetime.now().strftime(
            'Seed used the %d %m %Y at %H:%M:%S\n'))
        seeds.write('%s\n\n' % seed)

    # try:
        # window_config = Config(sample_buffers=1, samples=4) #, depth_size=8)  #, double_buffer=True) #TODO Break anti-aliasing/multisampling into an explicit menu option
        # window = Window(resizable=True, config=window_config)
    # except pyglet.window.NoSuchConfigException:
    window = Window(resizable=True, vsync=False)

    G.main_timer = Timer()
    pyglet.clock.schedule_interval(G.main_timer.schedule, G.TIMER_INTERVAL)
    pyglet.app.run()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Play a Python made Minecraft clone.')

    game_group = parser.add_argument_group('Game options')
    game_group.add_argument("--terrain", choices=G.TERRAIN_CHOICES, default=G.DEFAULT_TERRAIN_CHOICE)
    game_group.add_argument("--flat", action="store_true", default=False, help="Generate a flat world.")
    game_group.add_argument("--fast", action="store_true", default=False, help="Makes time progress faster then normal.")
    game_group.add_argument("--game-mode", choices=G.GAME_MODE_CHOICES, default=G.GAME_MODE)

    save_group = parser.add_argument_group('Save options')
    save_group.add_argument("--disable-auto-save", action="store_false", default=True, help="Do not save world on exit.")
    save_group.add_argument("--save", default=G.SAVE_FILENAME, help="Type a name for the world to be saved as.")
    save_group.add_argument("--disable-save", action="store_false", default=True, help="Disables saving.")

    parser.add_argument("--seed", default=None)

    options = parser.parse_args()
    initialize_config()
    main(options)


import tkinter as tk
from PIL import Image, ImageTk
from game_sim import decode_table
from game_sim import encodeCards

root = tk.Tk()
root.geometry('600x400+0+0')

alice_frame = tk.Frame(root, bg="#000000")
alice_frame.place(relx=0.5, rely=1, relwidth=1, relheight=0.3, anchor="s")

# import card pictures
cards = {}
for color_int, color_str in decode_table["colors"].items():
    for value_int, value_str in decode_table["values"].items():
        card_id = (color_int, value_int)
        path = "../card_images/"+color_str.lower()+"_"+value_str.lower()+".png"

        image = Image.open(path).resize((106,142))

        image = ImageTk.PhotoImage(image)
        cards[card_id] = image

bob_frame = tk.Frame(root, bg="#000000")
bob_frame.place(relx=0.5, rely=0, relwidth=1, relheight=0.3, anchor="n")

example_card = tk.Button(bob_frame, image=cards[(0,11)])
example_card.place(relx=0.5, rely=0, relwidth=0.2, relheight=1, anchor="n")

# frame = tk.Frame(root, bg='#00ff00', bd=5)
# frame.place(relx=0.5, rely=0.5 , relwidth=0.5, relheight=0.5)

# button = tk.Button(frame)
# button.place(relx=0.5, relheight=1, relwidth=0.3)

# canvas = tk.Canvas(root, width=40, height=60, bg='#00ff00')
# canvas.place(relx=0, rely=0)
# canvas.create_line(0, 50, 50, 50 )

root.mainloop()


#!usr/bin/env python
# -*- coding:utf-8 _*-
"""
@author:cugxy
@file: jsonify_util
@time: 2018/10/30
"""

from flask import jsonify, current_app

def return_msg(status, msg, data=None, log=None):
    """
    统一服务返回
    :param status:  状态 ERROR_CODE
    :param msg:     返回消息
    :param data:    服务成功时，返回数据
    :param log:     服务异常时，打印日志
    :return: dict
    """
    result = {
        "status": status,
        "msg": msg,
    }
    if data is not None:
        result["data"] = data
    if log is not None:
        current_app.logger.error(log)
    return jsonify(result)


# 建立/断开连接代码
import persional
import socks
from telethon import TelegramClient, sync

def Connect():
    # 机器人id
    api_id = persional.getId()
    # 机器人Hash
    api_hash = persional.getHash()
    # 需要代理用这个
    proxy = (socks.SOCKS5, '127.0.0.1', 1080)
    # proxy = (socks.PROXY_TYPE_HTTP, '127.0.0.1', 1080)
    print("connecting...")
    # 建立连接
    client = TelegramClient('my_session', api_id=api_id, api_hash=api_hash, proxy=proxy).start()
    print("connected!")
    print(client)
    # 函数返回此时的链接
    return client

def disConnect(client):
    # 断开已经建立的连接
    client.disconnect()
    print("disConnect,Done")

if __name__ == '__main__':
    Connect()
    # print("connected!")
    disConnect(Connect())
    # print("disconnected!")


import logging
import os
from sysinternalsupdater.utils import download_file

class Tool():
    def __init__(self, name, last_update_date, url):
        self.name = name
        self.url = url
        self.last_update_date = last_update_date

    def __str__(self):
        return "%s updated on %s" % (self.name, self.last_update_date.strftime("%d/%m/%Y at %H:%M"))

    def is_newer(self, tool):
        return self.last_update_date > tool.last_update_date

def is_tool_updated(tool, have_tools):

    try:
        current_tool = have_tools[tool.name]
        return tool.is_newer(current_tool)
    except KeyError:
        # The tool doesn't already exist.
        pass

    return True

def get_updated_tools(tools, have_tools):

    updated_tools = {}

    for tool_name, tool in tools.items():

        is_updated = is_tool_updated(tool, have_tools)

        if is_updated:
            updated_tools[tool_name] = tool

    return updated_tools

def update_tools_record(tools_record, updated_tools):

    # Update the tools record.
    for tool_name, tool in updated_tools.items():
        tools_record[tool_name] = tool

def update_tools(tools_to_update, destination):

    updated_tools = {}

    for tool_name,tool in tools_to_update.items():
        success = download_tool(tool, destination)

        if success is True:
            updated_tools[tool_name] = tool
        else:
            logging.info("{0} could not be updated.".format(tool_name))

    return updated_tools

def download_tool(tool, destination):

    success = False
    try:
        destination = os.path.join(destination, tool.name)
        success = download_file(tool.url, destination)

    except Exception as e:
        logging.warning(str(e) + " " + tool.url)

    return success


from __future__ import annotations
from uuid import uuid4

from django.db import models

from myapp.application.domain.model.article_vote import ArticleVote
from myapp.application.domain.model.vote import Vote

class ArticleVoteEntity(models.Model):
    VOTE_UP = 1
    VOTE_DOWN = 2

    VOTES_CHOICES = [
        (VOTE_UP, 'UP'),
        (VOTE_DOWN, 'DOWN')
    ]

    id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
    user_id = models.UUIDField()
    article_id = models.UUIDField()
    vote = models.IntegerField(choices=VOTES_CHOICES)

    class Meta:
        unique_together = [['user_id', 'article_id']]
        db_table = 'article_vote'

    @classmethod
    def from_domain_model(cls, article_vote: ArticleVote) -> ArticleVoteEntity:
        vote: int = {
            Vote.UP: cls.VOTE_UP,
            Vote.DOWN: cls.VOTE_DOWN
        }[article_vote.vote]

        return ArticleVoteEntity(
            id=article_vote.id,
            article_id=article_vote.article_id,
            user_id=article_vote.user_id,
            vote=vote
        )

    def to_domain_model(self) -> ArticleVote:
        vote: Vote = {
            self.VOTE_UP: Vote.UP,
            self.VOTE_DOWN: Vote.DOWN
        }[self.vote]

        return ArticleVote(
            id=self.id,
            user_id=self.user_id,
            article_id=self.article_id,
            vote=vote
        )


import h5py

def save_hdf5(filename, datasets, attrs=None, compression="gzip"):

    with h5py.File(filename, "w") as f:
        if attrs is not None:
            for attr_name, attr_data in attrs.items():
                f.attrs[attr_name] = attr_data

        for array_name, array_data in datasets.items():
            f.create_dataset(array_name, data=array_data, compression=compression)

def load_hdf5(filename):
    with h5py.File(filename, 'r') as f:
        attrs = dict(f.attrs)
        array_dict = {key:f[key][()] for key in f.keys()}

    if len(attrs) == 0:
        return array_dict
    else:
        return array_dict, attrs


import discord
from datetime import datetime
__all__ = ('try_delete', 'create_default_embed', 'yes_or_no')

async def try_delete(message):
    try:
        await message.delete()
    except (discord.Forbidden, discord.NotFound, discord.HTTPException):
        pass

def create_default_embed(ctx, **options) -> discord.Embed:
    embed = discord.Embed(color=discord.Color(int('0x2F3136', base=16)), **options)
    bot = ctx.bot
    embed.set_author(name=ctx.message.author.display_name, icon_url=str(ctx.message.author.avatar_url))
    embed.set_footer(text=bot.user.name, icon_url=str(bot.user.avatar_url))
    embed.timestamp = datetime.utcnow()
    return embed

def yes_or_no(content):
    if isinstance(content, bool):
        return content
    if str(content).lower() in ['yes', 'y', 'true', 'on', '1']:
        return True
    if str(content.lower()) in ['no', 'n', 'false', 'off', '0']:
        return False


from django.shortcuts import render , HttpResponse

from django.core.files.storage import FileSystemStorage

import requests

# Create your views here.

def home(request):

    return render(request,'core/home.html')

def carga(request):

    context = {}

    if request.method == 'POST':

        uploaded_file = request.FILES['document']

        fs = FileSystemStorage()

        name = fs.save(uploaded_file.name, uploaded_file)

        uploaded_file_url = fs.url(name)

        context['url'] = fs.url(name)

    return render(request,'core/carga.html',context)

def rxml(request):

    context = {}

    xml_file = open('media/entrada.xml')

    viewxml = xml_file.read()

    context['data'] = viewxml

    return render(request,'core/rxml.html',context)

def prepararxml(request):

    context = {}

    archivo_xml = open("media/entrada.xml","r")

    lectura_xml = archivo_xml.read()

    r = requests.get('http://127.0.0.1:5000/ob_xml',data=lectura_xml)

    #print(r.text)

    #html_response = "
Ejemplo1
"

    #html_response += r.text

    #html_response += "
Eso es el resultado del Ejemplo1
"

    context['ex'] = r.text

    return render(request,'core/preparado.html',context)

def wxml(request):

    context = {}

    #archivo_xml = open("media/entrada.xml","r")

    #lectura_xml = archivo_xml.read()

    r = requests.get('http://127.0.0.1:5000/c_xml')

    #print(r.text)

    #html_response = "
Ejemplo1
"

    #html_response += r.text

    #html_response += "
Eso es el resultado del Ejemplo1
"

    context['data'] = r.text

    return render(request,'core/estadisticas.html',context)

    #return render(request,'core/preparado.html')


# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
import os
import tempfile
import shutil

from nipype.testing import (assert_equal, assert_true,
                            skipif)
import nipype.interfaces.fsl.model as fsl
from nipype.interfaces.fsl import Info
from nipype.interfaces.fsl import no_fsl

tmp_infile = None
tmp_dir = None
cwd = None

@skipif(no_fsl)
def setup_infile():
    global tmp_infile, tmp_dir, cwd
    cwd = os.getcwd()
    ext = Info.output_type_to_ext(Info.output_type())
    tmp_dir = tempfile.mkdtemp()
    tmp_infile = os.path.join(tmp_dir, 'foo' + ext)
    file(tmp_infile, 'w')
    os.chdir(tmp_dir)
    return tmp_infile, tmp_dir

def teardown_infile(tmp_dir):
    os.chdir(cwd)
    shutil.rmtree(tmp_dir)

@skipif(no_fsl)
def test_MultipleRegressDesign():
    _, tp_dir = setup_infile()
    foo = fsl.MultipleRegressDesign()
    foo.inputs.regressors = dict(reg1=[1,1,1],reg2=[0.2,0.4,0.5],reg3=[1,-1,2])
    con1 = ['con1','T',['reg1','reg2'],[0.5,0.5]]
    con2 = ['con2','T',['reg3'],[1]]
    foo.inputs.contrasts = [con1,con2,['con3','F',[con1,con2]]]
    res = foo.run()
    yield assert_equal, res.outputs.design_mat, os.path.join(os.getcwd(),'design.mat')
    yield assert_equal, res.outputs.design_con, os.path.join(os.getcwd(),'design.con')
    yield assert_equal, res.outputs.design_fts, os.path.join(os.getcwd(),'design.fts')
    yield assert_equal, res.outputs.design_grp, os.path.join(os.getcwd(),'design.grp')


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 15 11:22:23 2018

@author: KOGANTI
"""
import csv
import numpy as np
import matplotlib.pyplot as plt
from statsmodels.tsa.stattools import adfuller
from sklearn.metrics import r2_score
from sklearn.metrics import mean_absolute_error
from statsmodels.tsa.arima_model import ARIMA
from statsmodels.graphics.tsaplots import plot_pacf
from statsmodels.graphics.tsaplots import plot_acf
import pandas as pd
import pyflux as pf

my_data = np.genfromtxt('pollution.csv',delimiter=",")[1:,1:]
def nan_helper(y):# from stack overflow
        return np.isnan(y), lambda z: z.nonzero()[0]
def fill_na(data):
    nans, x= nan_helper(data)
    data[nans]= np.interp(x(nans), x(~nans), data[~nans])
    return data
for i in range(4,8):
    modified_pm = fill_na(my_data[:,i])
    my_data[:,i] = np.reshape(modified_pm,modified_pm.size)

stop = 15000
start = 12000
interval = 20
nmbr_predictions = 2
allpredicted_values = np.array([0,0,0,0]).reshape(1,4)
original_values = np.array([0,0,0,0]).reshape(1,4)

for i in range(start,stop,interval):
    i = i+nmbr_predictions
    VAR_data = pd.DataFrame({'pm25':my_data[:,4][:i],'dewp':my_data[:,5][:i],'temp':my_data[:,6][:i],'pres':my_data[:,7][:i]})
    VAR_data = VAR_data[['pm25','dewp','temp','pres']]
    model = pf.VAR(data=VAR_data,lags=4,integ=1)
    x = model.fit()
    print(model.predict_is(nmbr_predictions).values)
    allpredicted_values = np.append(allpredicted_values,np.cumsum(np.append(my_data[i-3,4:8].reshape(1,4),model.predict_is(nmbr_predictions).values,axis=0),axis=0)[1:,:],axis=0)
    original_values = np.append(original_values,my_data[i-2:i,4:8],axis=0)

allpredicted_values = allpredicted_values[1:,]
original_values = original_values[1:,]
np.save("predicted_values_short_term_for_error",allpredicted_values)
np.save("original_values_short_term_for_error",original_values)


import serial   # 아두이노 통신 라이브러리
from time import *
from multiprocessing import Queue, Process
from import_detect import *
import Adafruit_DHT # 온습도 라이브러리
from socket import *
from xml.etree.ElementTree import Element, SubElement, ElementTree, dump    # XML 모듈
from xml.etree.ElementTree import Element, SubElement, ElementTree, dump
import xml.etree.ElementTree as ET

BAUDRATE = 9600
time_flag = 0
last_time = 0

sensor = Adafruit_DHT.DHT11
pin = '4'

clientSock = socket(AF_INET, SOCK_STREAM)
#clientSock.connect(('220.69.249.226', 4000))

# XML 형식의 온습도 문자열
def getTempHumid(temp, humid):
    xmlStr ='''
        <SECS2_XML_MESSAGE>
            <HEAD>
                <SystemByte>00001</SystemByte>
                <CMD>3</CMD>
                <Stream>2</Stream>
                <Function>42</Function>
            </HEAD>
            <BODY>
                <HCACK>0</HCACK>
                <TEMP>'''  + temp  + '''</TEMP>
                <HUMID>''' + humid + '''</HUMID>
            </BODY>
        </SECS2_XML_MESSAGE>
        '''
    return xmlStr

# 온습도 받아오는 함수
def humanity_temp():

    # 온습도 라이브러리
    humidity, temperature = Adafruit_DHT.read_retry(sensor, pin)

    # 온습도 값이 존재할 때 -> 온습도값 리턴
    if humidity is not None and temperature is not None:
        return int(temperature), int(humidity)

    # 온습도 값이 존재하지 않을 때
    else:
        #print('Failed to get reading. Try again!')
        temperature = 0
        humidity = 0
        return temperature, humidity

# 시리얼 객체 생성(ser) - 아두이노 통신
def serial_open():
    ser = serial.Serial('/dev/ttyAMA0', baudrate = BAUDRATE)
    return ser

# 아두이노에게 데이터 전송
def command_arduino(ser, i):
    command = ["go\n", "stop\n", "rgrab\n", "fgrab\n"]

    command[i] = command[i].encode('utf-8')
    ser.write(command[i])

# 아두이노로 부터 데이터 받는 함수
def receive_arduino(ser, q):
    if ser.readable():
        data = ser.readline()

        # decode() : 바이트로 들어온 데이터 해결
        data = str(data[:-1].decode())
        q.put(data)

# 온습도 출력(10초기준)
def get_H_T():
    global time_flag
    global last_time

    if time_flag == 0:
        last_time = time()
        time_flag = 1

    if time() - last_time >= 10:

        # 온습도 받아오는 함수
        temp, humid = humanity_temp()
        time_flag = 0

        # 서버에 전송할 XML 형식의 온습도 문자열
        SendTempHumid = getTempHumid(temp, humid)
        clientSock.send(SendTempHumid.encode('utf-8'))  # 서버로 전송
        print('Temp={0}*C  Humidity={1}%'.format(temp, humid))

# 이미지 처리(카메라)
def image_process(cap, ser, q, state_flag, state_list):
    goods_x, signal, barcode = cam(cap)

    # 정상, 비정상 저장할 변수, 정상 - 1, 비정상 - 2
    global Color

    # 파랑색 제품(정상품) - 1
    if signal == 'P' and (goods_x >= 60 and goods_x <= 272):
        Color = 1
        if state_flag == "SEND_STOP":
            command_arduino(ser, 1)
            state_flag = "SEND_GRAB"
            print("P, SEND_STOP")

        if state_flag == "SEND_GRAB"  and len(barcode) > 5:
            sleep(0.01)
            command_arduino(ser, 2)
            state_flag = "WAIT"
            print("P, SEND_GRAB")

    # 빨강색 제품(불량품) - 2
    elif signal == 'F' and (goods_x >= 60 and goods_x <= 272):
        Color = 2
        if state_flag == "SEND_STOP":
            command_arduino(ser, 1)
            state_flag = "SEND_GRAB"

        if state_flag == "SEND_GRAB"  and len(barcode) > 5:
            sleep(0.01)
            command_arduino(ser, 3)
            state_flag = "WAIT"

    else:
        if q.empty() == False and signal == 'N':
            rx_data = q.get()
            print(rx_data)
            if rx_data == "complete":
                command_arduino(ser, 0)
                state_flag = "SEND_STOP"
    return state_flag

def main_process(ser, q):

    state_list = ["WAIT", "SEND_STOP", "SEND_GRAB" ]

    cap = open_cam()
    command_arduino(ser, 0)
    state_flag = state_list[1]
    q.put("start")
    sleep(1)

    while True:
        state_flag = image_process(cap, ser, q, state_flag, state_list)
        if cv.waitKey(1) & 0xFF == 27:
            break

    cap.release()
    cv.destroyAllWindows()

def serve_process(ser, q):
    while True:
        receive_arduino(ser, q)

def temp_huminity_process(q):
    factory_state = ""
    temp_state = 1

    while True:
        if q.empty() == False and temp_state == 1:
            factory_state = q.get()
            temp_state = 2

        if factory_state == "start" and temp_state == 2:
            get_H_T()
try:
    if __name__ == "__main__":
        print("start \n")
        q = Queue()
        ser = serial_open()
        p1 = Process(target = main_process, args = (ser,q))
        p2 = Process(target = serve_process, args = (ser,q))
        p3 = Process(target = temp_huminity_process, args = (q, ))
        p1.start()
        p2.start()
        p3.start()
        print("ggggg")

except KeyboardInterrupt:
    print("exit() \n")
    p1.join()
    p2.join()
    p3.join()


# Imports required for CLI Script
import click
import os

from msslib.utils import *
from msslib.prepare import *
from msslib.page_prima.page import PrimaPage
from msslib.evaluate import *
from scipy import misc
import numpy as np
import functools as f

def open_and_flatten(page_paths:[str]):
    page_path, label_path, result_path = page_paths
    page_opener = page_img_opener(PrimaPage(page_path))
    scaler = img_resizer((1200,900))
    label = scaler(page_opener(label_path))[0]
    results = misc.imread(result_path)
    return label.flatten(), results.flatten()

@click.command()
@click.argument('page_dir', type=click.Path(exists=True, resolve_path=True))
@click.argument('label_dir', type=click.Path(exists=True, resolve_path=True))
@click.argument('results_dir', type=click.Path(exists=True, resolve_path=True))
def evaluate_results(page_dir, label_dir, results_dir):
    # Define a couple of functions to generate and group all the paths.
    get_page_path = compose(f.partial(format_path, page_dir, 'xml'), only_basename)
    get_label_path = compose(f.partial(format_path, label_dir, 'png'), only_basename)
    get_results_path = compose(f.partial(format_path, results_dir, 'png'), only_basename)

    output_name = "%s_confusion_matrices.npy" % only_basename(results_dir)
    # One of these could really be identity
    path_formatter = applier(get_page_path, get_label_path, get_results_path)
    paths = map(path_formatter, listpaths(results_dir))
    label_results_pairs = map(open_and_flatten, paths)
    label, results = f.reduce(lambda x,y: (np.append(x[0], y[0]), np.append(x[1], y[1])), label_results_pairs)
    c_matrix = normalised_confusion_matrix(label, results)
    np.save(output_name, c_matrix)

if __name__ == '__main__':
    evaluate_results()


# -*- coding: utf-8 -*-
import sys
import random
import csv
import ConfigParser
import numpy as np
import pandas as pd
from keras.optimizers import SGD
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.layers.advanced_activations import LeakyReLU
from keras.layers import Dropout
from keras import backend as K

# gan
K.set_image_dim_ordering('th')
INPUT_SIZE = 200
GEN_OUTPUT_SIZE = 5
BATCH_SIZE = 32
NUM_EPOCH = 50

def generator_model():
    model = Sequential()
    model.add(Dense(input_dim=INPUT_SIZE, output_dim=INPUT_SIZE*10, init='glorot_uniform'))
    model.add(LeakyReLU(0.2))
    model.add(Dropout(0.5))
    model.add(Dense(INPUT_SIZE*10, init='glorot_uniform'))
    model.add(LeakyReLU(0.2))
    model.add(Dropout(0.5))
    model.add(Dense(INPUT_SIZE*5, init='glorot_uniform'))
    model.add(LeakyReLU(0.2))
    model.add(Dropout(0.5))
    model.add(Dense(output_dim=GEN_OUTPUT_SIZE, init='glorot_uniform'))
    model.add(Activation('tanh'))
    return model

def discriminator_model():
    model = Sequential()
    model.add(Dense(input_dim=GEN_OUTPUT_SIZE, output_dim=GEN_OUTPUT_SIZE*10, init='glorot_uniform'))
    model.add(LeakyReLU(0.2))
    model.add(Dense(GEN_OUTPUT_SIZE*10, init='glorot_uniform'))
    model.add(LeakyReLU(0.2))
    model.add(Dense(1, init='glorot_uniform'))
    model.add(Activation('sigmoid'))
    return model

def train(df_genes):
    X_train = []
    flt_size = len(df_genes)/2.0
    # load your train data
    with open('.\\signature\\xss_list.csv', 'r') as f:
        reader = csv.reader(f)
        for row in reader:
            X_train.append(row)
    X_train = np.array(X_train)
    X_train = (X_train.astype(np.float32) - flt_size)/flt_size

    discriminator = discriminator_model()
    d_opt = SGD(lr=0.1, momentum=0.1, decay=1e-5)
    discriminator.compile(loss='binary_crossentropy', optimizer=d_opt)

    # generator+discriminator (fixed weight of discriminator)
    discriminator.trainable = False
    generator = generator_model()
    dcgan = Sequential([generator, discriminator])
    g_opt = SGD(lr=0.1, momentum=0.3)
    dcgan.compile(loss='binary_crossentropy', optimizer=g_opt)

    num_batches = int(len(X_train) / BATCH_SIZE)
    lst_scripts = []
    for epoch in range(NUM_EPOCH):
        for batch in range(num_batches):
            noise = np.array([np.random.uniform(-1, 1, INPUT_SIZE) for _ in range(BATCH_SIZE)])
            generated_images = generator.predict(noise, verbose=0)

            # update weight of discriminator
            image_batch = X_train[batch * BATCH_SIZE:(batch + 1) * BATCH_SIZE]
            X = image_batch
            y = [random.uniform(0.7, 1.2) for _ in range(BATCH_SIZE)]
            d_loss = discriminator.train_on_batch(X, y)
            X = generated_images
            y = [random.uniform(0.0, 0.3) for _ in range(BATCH_SIZE)]
            d_loss = discriminator.train_on_batch(X, y)

            # update weight of generator
            noise = np.array([np.random.uniform(-1, 1, INPUT_SIZE) for _ in range(BATCH_SIZE)])
            g_loss = dcgan.train_on_batch(noise, [1]*BATCH_SIZE)
            for generated_image in generated_images:
                str_html = ''
                for gene_num in generated_image:
                    gene_num = (gene_num*flt_size)+flt_size
                    gene_num = np.round(gene_num)
                    if gene_num == len(df_genes):
                        gene_num -= 1
                    str_html += str(df_genes.loc[gene_num].values[0])
                lst_scripts.append(str_html)
                print('{0},{1},{2},{3},{4},{5}'.format(epoch,
                                                       batch,
                                                       g_loss,
                                                       d_loss,
                                                       np.round((generated_image*flt_size)+flt_size),
                                                       str_html))

        generator.save_weights('.\\weight\\generator_' + str(epoch) + '.h5')
        discriminator.save_weights('.\\weight\\discriminator_' + str(epoch) + '.h5')

    return lst_scripts

if __name__ == "__main__":
    # load config
    inifile = ConfigParser.SafeConfigParser()
    try:
        inifile.read('config.ini')
    except:
        print('usage: can\'t read config.ini.')
        exit(1)
    genes = inifile.get('Genetic', 'gene_list')
    signature = inifile.get('GAN', 'xss_list')
    element = inifile.get('GAN', 'elem_list')
    df_genes = pd.read_csv(genes, encoding='utf-8').fillna('')
    df_sigs = pd.read_csv(signature, encoding='utf-8').fillna('')

    # generate injection codes
    lst_scripts = train(df_genes)
    print(lst_scripts)


import pandas as pd
from sklearn.base import TransformerMixin

class Pruner(TransformerMixin):
    """Prune identifier columns, columns with numerous tokens (>100) and columns
    with low information."""

    def __init__(self):
        self.pruned_columns = ['subjuct_id', 'row_id', 'hadm_id', 'cgid', 'itemid', 'icustay_id']

    def fit(self, X):
        X = pd.DataFrame(X)
        for col in X.columns:
            num = X[col].nunique()
            # Remove columns with numerous tokens (>100)
            if X[col].dtype == object and num > 100:
                self.pruned_columns.append(col)
            # Remove columns with low information (unique values <2)
            elif num <= 1 or X[col].dropna().shape[0] == 0:
                self.pruned_columns.append(col)
        return self

    def transform(self, X):
        X = pd.DataFrame(X)
        for column in X.columns:
            if column in self.pruned_columns:
                X.drop(column, axis=1, inplace=True)
        return X


from django.test import SimpleTestCase

import json
import requests
from django.contrib.gis.geos import GEOSGeometry
from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from rest_framework import generics
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework import permissions
from rest_framework import permissions
from django.test.runner import DiscoverRunner

class NoDbTestRunner(DiscoverRunner):
   """ A test runner to test without database creation/deletion """

   def setup_databases(self, **kwargs):
     pass

   def teardown_databases(self, old_config, **kwargs):
     pass

#python manage.py test app --testrunner=app.filename.NoDbTestRunner
#python manage.py test bcim.tests  --testrunner=bcim.tests.NoDbTestRunner
#python manage.py test bcim.test_spatial_functions  --testrunner=bcim.test_spatial_functions.NoDbTestRunner

class EDGVDetailTestCase(SimpleTestCase):
    def setUp(self):
        self.json_type = type('str')
        #self.host_base = 'http://172.30.10.61:8000'
        self.host_base = 'http://127.0.0.1:8001'
        self.featureString = 'Feature'
        self.polygonString = 'Polygon'
        self.pointString = 'Point'
        self.multilineString = 'MultiLineString'
        self.lineString = 'LineString'
    def url_feature(self):
        return ''

    def test_feature(self):
        if len(self.url_feature())==0:
            return True
        an_url = self.host_base + self.url_feature()
        req = requests.get(an_url)
        self.assertEquals(req.json()['type'], self.featureString)

class UnidadeFederacaoDetailSpatialQueryTestCase(EDGVDetailTestCase):
    #testa se uma feature(multipoligon) contém um ponto dado em WKT
    def test_uf_sigla_contains_point(self):
        an_url = self.host_base + '/instituicoes/ibge/bcim/unidades-federativas/RJ/contains/POINT(-42 -21)/'
        req = requests.get(an_url)
        is_true = req.json().values().__iter__().__next__()== True
        self.assertTrue(is_true)
    #testa se uma feature(multipolygon) contém um ponto dado em geojson
    def test_uf_sigla_contains_point_as_geojson(self):
        an_url = self.host_base + '/instituicoes/ibge/bcim/unidades-federativas/RJ/contains/{ "type": "Point", "coordinates": [ -42, -21]}/'
        req = requests.get(an_url)
        is_true = req.json().values().__iter__().__next__()== True
        self.assertTrue(is_true)

    def test_uf_sigla_transform_area(self):
        an_url = self.host_base + '/instituicoes/ibge/bcim/unidades-federativas/RJ/transform/3857&True/area'
        req = requests.get(an_url)
        is_true = isinstance(req.json().values().__iter__().__next__(), float)
        self.assertTrue(is_true)

    #testa se a boundary de uma feature(multipolygon) responde multilinestring
    #testa se a boundary de uma feature(polygon) responde linestring
    def test_uf_sigla_boundery(self):
        an_url = self.host_base + '/instituicoes/ibge/bcim/unidades-federativas/RJ/boundary/'
        req = requests.get(an_url)
        self.assertEquals(req.json()['type'], self.multilineString)
        an_url = self.host_base + '/instituicoes/ibge/bcim/unidades-federativas/MG/boundary/'
        req = requests.get(an_url)
        self.assertEquals(req.json()['type'], self.lineString)

    #testa se o envelope de uma feature(polygon) responde um polygon
    def test_uf_sigla_envelope(self):
        an_url = self.host_base + '/instituicoes/ibge/bcim/unidades-federativas/MG/envelope/'
        req = requests.get(an_url)
        self.assertEquals(req.json()['type'], self.polygonString)

    #testa o centroid de uma feature(polygon)
    #primeiro obtém o centroid no formato geojson
    def test_uf_sigla_centroid(self):
        an_url = self.host_base + '/instituicoes/ibge/bcim/unidades-federativas/MG/centroid/'
        req = requests.get(an_url)
        self.assertEquals(req.json()['type'], self.pointString)
        an_url = self.host_base + '/instituicoes/ibge/bcim/unidades-federativas/MG/contains/' + self.host_base + '/instituicoes/ibge/bcim/unidades-federativas/MG/centroid/'
        req = requests.get(an_url)
        self.assertTrue(req.json().values().__iter__().__next__()== True)

class AldeiaIndigenaDetailTestCase(EDGVDetailTestCase):
    #testa se a url requisitada responde uma feature
    def test_adeia_indigena(self):
        an_url = self.host_base + '/instituicoes/ibge/bcim/aldeias-indigenas/587'
        req = requests.get(an_url)
        self.assertEquals(req.json()['type'], self.featureString)

    #testa se o evenvolpe de um ponto responde um ponto
    def test_adeia_indigena_envelope(self):
        an_url = self.host_base + '/instituicoes/ibge/bcim/aldeias-indigenas/587/envelope/'
        req = requests.get(an_url)
        self.assertEquals(req.json()['type'], self.pointString)

    # testa se um ponto está dentro de um envelope criado a partir de uma url
    def test_adeia_indigena_within_envelope_by_url(self):
        an_url = self.host_base + '/instituicoes/ibge/bcim/aldeias-indigenas/587/within/' + self.host_base + '/instituicoes/ibge/bcim/unidades-federativas/ES/envelope/'
        req = requests.get(an_url)
        self.assertTrue(req.json().values().__iter__().__next__()== True)

class TrechoFerroviarioDetailTestCase(EDGVDetailTestCase):
    #usado no test_feature para verificar se a url requisitada responde uma feature
    def url_feature(self):
        return '/instituicoes/ibge/bcim/trechos-ferroviarios/12711/'

"""
class UnidadeFederacaoListSpatialQueryTestCase(SimpleTestCase):
    def test_uf_sigla_contains_point(self):
        an_url = 'http://172.30.10.120:8000/ibge/bcim/aldeias-indigenas/'
        req = requests.get(an_url)
        value = json.loads(req.json())["type"]
        self.assertEquals(value,'FeatureCollection')
"""


from django.shortcuts import render, render_to_response, redirect
from django.template import RequestContext
from api import VigilanteApi
import json
from django.conf import settings

# env facter with summary compliance result
def env_facter( request, envid ):
    vigilante_api = VigilanteApi( settings.VIGILANTE_HOST, port=settings.VIGILANTE_PORT )
    env_facter = json.loads( vigilante_api.get_collector_env_data_current( envid ) )
    env_query_facter = json.loads( vigilante_api.query_current_env_with_template( envid, envid ) )
    for env_query_host_key, env_query_host_value in env_query_facter['body'].iteritems():
        env_facter['body'][env_query_host_key][0]['current']['summary'] = env_query_host_value[0]['summary']

    params = { "EnvFacter" : sorted( env_facter['body'].iteritems() ),
               "EnvId" : envid }
    return render_to_response('facter_env.html', params,
            context_instance=RequestContext( request ) )

# env host facter with template diff result
def env_host_facter( request, envid, hostname ):
    vigilante_api = VigilanteApi( settings.VIGILANTE_HOST, port=settings.VIGILANTE_PORT )
    fqdn = "%s.%s.com" % ( hostname, envid )

    env_query_facter = json.loads( vigilante_api.query_current_env_with_template( envid, envid ) )
    template_name = env_query_facter['template'][hostname]['meta']['name']
    env_host_facter = json.loads( vigilante_api.get_collector_role_data_current( fqdn ) )
    env_host_query_facter = json.loads( vigilante_api.query_current_role_with_template( template_name, fqdn ) )
    template = json.loads( vigilante_api.get_template( template_name ) )

    for host_query_facter_key, host_query_facter_value in env_host_query_facter['body'].iteritems():
        real_facter_value = env_host_facter['body'][ host_query_facter_key ]
        env_host_facter['body'][ host_query_facter_key ] = {}
        env_host_facter['body'][ host_query_facter_key ]['real'] = real_facter_value
        env_host_facter['body'][ host_query_facter_key ]['template'] = template['body'][ host_query_facter_key ]
    params = { "HostFacter" : sorted( env_host_facter['body'].iteritems() ),
               "Hostname" : hostname }
    return render_to_response('facter_env_host.html', params,
            context_instance=RequestContext( request ) )


import unittest
import api
from api.api_employee import ApiEployee
from tools.get_log import GetLog

from tools.assesrt_common import assert_common
from parameterized import parameterized

from tools.read_yaml import read_yaml

log = GetLog.get_logger()

class TestEmployee(unittest.TestCase):
    # 1.初始化方法
    @classmethod
    def setUpClass(cls):
        # 获取ApiEmployee对象
        cls.login = ApiEployee()

    # 2.新增员工 接口测试方法
    @parameterized.expand(read_yaml("employee_post.yaml"))
    def test01_post(self, username, mobile, workNumber):
        r = self.login.api_post_employee(username, mobile, workNumber)
        # 断言
        assert_common(self, r)
        print("新增员工结果:", r.json())
        log.info("新增员工结果：{}".format(r.json()))
        # 提取user_id
        api.user_id = r.json().get("data").get("id")
        print("员工user_id值为：", api.user_id)
        log.info("员工user_id值为：{}".format(api.user_id))

    # 3.更新员工 接口测试方法
    def test02_put(self):
        # 1.调用更新接口
        r = self.login.api_put_employee()
        # 2.断言
        assert_common(self, r)
        log.info("更新员工结果：{}".format(r.json()))

    # 4.查询员工 接口测试方法
    def test03_get(self):
        # 1.调用查询接口
        r = self.login.api_get_employee()
        # 2.断言
        assert_common(self, r)
        log.info("查询员工结果：{}".format(r.json()))

    # 5.删除员工 接口测试方法
    def test04_delete(self):
        r = self.login.api_delete_employee()
        print("删除结果为：", r.json())
        # 断言
        assert_common(self, r)
        log.info("删除员工结果：{}".format(r.json()))


from collections import defaultdict
import re
import ldap
ldap.set_option(ldap.OPT_TIMEOUT, 10)
ldap.set_option(ldap.OPT_NETWORK_TIMEOUT, 10)
import asn1
import hashlib

__oid_map = {
   "DC": "0.9.2342.19200300.100.1.25",
   "OU": "2.5.4.11",
   "CN": "2.5.4.3",
   "O": "2.5.4.10",
   "ST": "2.5.4.8",
   "C": "2.5.4.6",
   "L": "2.5.4.7",
   "postalCode": "2.5.4.17",
   "street": "2.5.4.9",
   "emailAddress": "1.2.840.113549.1.9.1",
   }

__dn_split_re = re.compile("/([A-Za-z]+)=")

class DataError(Exception):
    """Raised when there is a problem in the topology or VO data"""

def _generate_ligo_dns():
    """
    Query the LIGO LDAP server for all grid DNs in the LVC collab.

    Returns a list of DNs.
    """
    ldap_obj = ldap.initialize("ldaps://ldap.ligo.org")
    query = "(&(isMemberOf=Communities:LSCVirgoLIGOGroupMembers)(gridX509subject=*))"
    results = ldap_obj.search_s("ou=people,dc=ligo,dc=org", ldap.SCOPE_ONELEVEL,
                                query, ["gridX509subject"])
    all_dns = []
    for result in results:
        user_dns = result[1].get('gridX509subject', [])
        for dn in user_dns:
            if dn.startswith(b"/"):
                all_dns.append(dn.replace(b"\n", b" ").decode("utf-8"))

    return all_dns

def _generate_dn_hash(dn: str):
    """
    Given a DN one-liner as commonly encoded in the grid world
    (e.g., output of `openssl x509 -in $FILE -noout -subject`), run
    the OpenSSL subject hash generation algorithm.

    This is done by calculating the SHA-1 sum of the canonical form of the
    X509 certificate's subject.  Formatting is a bit like this:

    SEQUENCE:
       SET:
         SEQUENCE:
           OID
           UTF8String

    All the UTF-8 values should be converted to lower-case and multiple
    spaces should be replaced with a single space.  That is, "Foo  Bar"
    should be substituted with "foo bar" for the canonical form.
    """
    encoder = asn1.Encoder()
    encoder.start()
    info = __dn_split_re.split(dn)[1:]
    for attr, val in zip(info[0::2], info[1::2]):
        oid = __oid_map.get(attr)
        if not oid:
            raise ValueError("OID for attribute {} is not known.".format(attr))
        encoder.enter(0x11)
        encoder.enter(0x10)
        encoder.write(oid, 0x06)
        encoder.write(val.lower().encode("utf-8"), 0x0c)
        encoder.leave()
        encoder.leave()
    output = encoder.output()
    hash_obj = hashlib.sha1()
    hash_obj.update(output)
    digest = hash_obj.digest()
    int_summary = digest[0] | digest[1] << 8 | digest[2] << 16 | digest[3] << 24
    return "%08lx.0" % int_summary

def _get_resource_by_fqdn(fqdn, resource_groups):
    for group in resource_groups:
        for resource in group.resources:
            if fqdn.lower() == resource.fqdn.lower():
                return resource

def _get_cache_resource(fqdn, resource_groups, suppress_errors):
    resource = None
    if fqdn:
        resource = _get_resource_by_fqdn(fqdn, resource_groups)
        if not resource:
            if suppress_errors:
                return None
            else:
                raise DataError("{} is not a registered resource.".format(fqdn))
        if "XRootD cache server" not in resource.service_names:
            if suppress_errors:
                return None
            else:
                raise DataError("{} (resource name {}) does not provide an XRootD cache server.".format(fqdn, resource.name))
    return resource

def _cache_is_allowed(resource, vo_name, stashcache_data, public, suppress_errors):
    allowed_vos = resource.data.get("AllowedVOs")
    if allowed_vos is None:
        if suppress_errors:
            return False
        else:
            raise DataError("Cache server at {} (resource name {}) does not provide an AllowedVOs list.".format(resource.fqdn, resource.name))

    if ('ANY' not in allowed_vos and
            vo_name not in allowed_vos and
            (not public or 'ANY_PUBLIC' not in allowed_vos)):
        return False

    # For public data, caching is one-way: we OK things as long as the
    # cache is interested in the data.
    if public:
        return True

    allowed_caches = stashcache_data.get("AllowedCaches")
    if allowed_caches is None:
        if suppress_errors:
            return False
        else:
            raise DataError("VO {} in StashCache does not provide an AllowedCaches list.".format(vo_name))

    return 'ANY' in allowed_caches or resource.name in allowed_caches

def generate_cache_authfile(vo_data, resource_groups, fqdn=None, legacy=True, suppress_errors=True):
    """
    Generate the Xrootd authfile needed by a StashCache cache server.
    """
    authfile = ""
    id_to_dir = defaultdict(set)

    resource = _get_cache_resource(fqdn, resource_groups, suppress_errors)
    if fqdn and not resource:
        return ""

    for vo_name, vo_data in vo_data.vos.items():
        stashcache_data = vo_data.get('DataFederations', {}).get('StashCache')
        if not stashcache_data:
            continue

        namespaces = stashcache_data.get("Namespaces")
        if not namespaces:
            if suppress_errors:
                continue
            else:
                raise DataError("VO {} in StashCache does not provide a Namespaces list.".format(vo_name))

        has_non_public = False
        for namespace, authz_list in namespaces.items():
            if not authz_list:
                if suppress_errors:
                    continue
                else:
                    raise DataError("Namespace {} (VO {}) does not provide any authorizations.".format(namespace, vo_name))
            if authz_list != ["PUBLIC"]:
                has_non_public = True
                break
        if not has_non_public:
            continue

        if resource and not _cache_is_allowed(resource, vo_name, stashcache_data, False, suppress_errors):
            continue

        for namespace, authz_list in namespaces.items():
            for authz in authz_list:
                if authz.startswith("FQAN:"):
                    id_to_dir["g {}".format(authz[5:])].add(namespace)
                elif authz.startswith("DN:"):
                    hash = _generate_dn_hash(authz[3:])
                    id_to_dir["u {}".format(hash)].add(namespace)

    if legacy:
        for dn in _generate_ligo_dns():
            hash = _generate_dn_hash(dn)
            id_to_dir["u {}".format(hash)].add("/user/ligo")

    for id, dir_list in id_to_dir.items():
        if dir_list:
            authfile += "{} {}\n".format(id,
                " ".join([i + " rl" for i in sorted(dir_list)]))

    return authfile

def generate_public_cache_authfile(vo_data, resource_groups, fqdn=None, legacy=True, suppress_errors=True):
    """
    Generate the Xrootd authfile needed for public caches
    """
    if legacy:
        authfile = "u * /user/ligo -rl \\\n"
    else:
        authfile = "u * \\\n"

    resource = _get_cache_resource(fqdn, resource_groups, suppress_errors)
    if fqdn and not resource:
        return ""

    public_dirs = set()
    for vo_name, vo_data in vo_data.vos.items():
        stashcache_data = vo_data.get('DataFederations', {}).get('StashCache')
        if not stashcache_data:
            continue
        if resource and not _cache_is_allowed(resource, vo_name, stashcache_data, True, suppress_errors):
            continue

        for dirname, authz_list in stashcache_data.get("Namespaces", {}).items():
            if "PUBLIC" in authz_list:
                public_dirs.add(dirname)

    for dirname in sorted(public_dirs):
        authfile += "    {} rl \\\n".format(dirname)

    if authfile.endswith("\\\n"):
        authfile = authfile[:-2] + "\n"

    return authfile

def _origin_is_allowed(origin_hostname, vo_name, stashcache_data, resource_groups, suppress_errors=True):
    origin_resource = _get_resource_by_fqdn(origin_hostname, resource_groups)
    if not origin_resource:
        if suppress_errors:
            return False
        else:
            raise DataError("{} is not a registered resource.".format(origin_hostname))
    if 'XRootD origin server' not in origin_resource.service_names:
        if suppress_errors:
            return False
        else:
            raise DataError("{} (resource name {}) does not provide an XRootD origin server.".format(origin_hostname, origin_resource.name))
    allowed_vos = origin_resource.data.get("AllowedVOs")
    if allowed_vos is None:
        if suppress_errors:
            return False
        else:
            raise DataError("Origin server at {} (resource name {}) does not provide an AllowedVOs list.".format(origin_hostname, origin_resource.name))

    if 'ANY' not in allowed_vos and vo_name not in allowed_vos:
        return False

    allowed_origins = stashcache_data.get("AllowedOrigins")
    if allowed_origins is None:
        if suppress_errors:
            return False
        else:
            raise DataError("VO {} in StashCache does not provide an AllowedOrigins list.".format(vo_name))

    return origin_resource.name in allowed_origins

def _get_allowed_caches(vo_name, stashcache_data, resource_groups, suppress_errors=True):
    allowed_caches = stashcache_data.get("AllowedCaches")
    if allowed_caches is None:
        if suppress_errors:
            return []
        else:
            raise DataError("VO {} in StashCache does not provide an AllowedCaches list.".format(vo_name))

    resources = []
    for group in resource_groups:
        for resource in group.resources:
            # First, does this provide a cache service?
            if 'XRootD cache server' not in resource.service_names:
                continue

            # Next, does it allow this VO?  Unlike the StashCache origin case requiring the origin to list AllowedVOs,
            # we do not consider the lack of AllowedVOs an error as the cache doesn't
            # explicitly record *which* data federation it is participating in (might not be SC!).
            allowed_vos = resource.data.get("AllowedVOs", [])
            if 'ANY' not in allowed_vos and (vo_name != "ANY_PUBLIC" and vo_name not in allowed_vos):
                continue
            if 'ANY' not in allowed_caches and resource.name not in allowed_caches:
                continue
            resources.append(resource)
    return resources

def generate_origin_authfile(origin_hostname, vo_data, resource_groups, suppress_errors=True, public_only=False):
    public_namespaces = set()
    id_to_namespaces = defaultdict(set)
    for vo_name, vo_data in vo_data.vos.items():
        stashcache_data = vo_data.get('DataFederations', {}).get('StashCache')
        if not stashcache_data:
            continue

        if not _origin_is_allowed(origin_hostname, vo_name, stashcache_data, resource_groups, suppress_errors=suppress_errors):
            continue

        namespaces = stashcache_data.get("Namespaces")
        if not namespaces:
            if suppress_errors:
                continue
            else:
                raise DataError("VO {} in StashCache does not provide a Namespaces list.".format(vo_name))

        for namespace, authz_list in namespaces.items():
            if not authz_list:
                if suppress_errors:
                    continue
                else:
                    raise DataError("Namespace {} (VO {}) does not provide any authorizations.".format(namespace, vo_name))

            if authz_list == ["PUBLIC"]:
                public_namespaces.add(namespace)
                continue

            if public_only:
                continue

            allowed_caches = stashcache_data.get("AllowedCaches")
            if allowed_caches is None:
                if suppress_errors:
                    continue
                else:
                    raise DataError("VO {} in StashCache does not provide an AllowedCaches list.".format(vo_name))

            for resource in _get_allowed_caches(vo_name, stashcache_data, resource_groups, suppress_errors=suppress_errors):
                dn = resource.data.get("DN")
                if not dn:
                    if suppress_errors:
                        continue
                    else:
                        raise DataError("Resource {} is an allowed cache for VO {} but does not provide a DN.".format(resource.name, vo_name))
                dn_hash = _generate_dn_hash(dn)
                id_to_namespaces[dn_hash].add(namespace)

    results = ""
    for id, namespaces in id_to_namespaces.items():
        results += "u {} {}\n".format(id, " ".join("{} lr".format(i) for i in sorted(namespaces)))
    if public_namespaces:
        results += "\nu * {}\n".format(" ".join("{} lr".format(i) for i in sorted(public_namespaces)))
    return results


# -*- coding: utf-8 -*-
"""

"""

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

__author__ = 'timmyliang'
__email__ = '820472580@qq.com'
__date__ = '2020-10-30 19:42:26'

import inspect
import threading
from PySide2 import QtCore

class SingletonType(type):
    _instance_lock = threading.Lock()

    def __init__(cls, name, bases, dic):
        super(SingletonType, cls).__init__(name, bases, dic)
        # print(cls,name,bases,dic)

    def __call__(cls, *args, **kwargs):
        # print(args)

        if not hasattr(cls, "_instance"):
            with SingletonType._instance_lock:
                if not hasattr(cls, "_instance"):
                    cls._instance = super(SingletonType,cls).__call__(*args, **kwargs)

        cls._instance.__addvar__(*args)
        return cls._instance

    # def __new__(cls, name, bases, attrs):
    #     # print(name,bases,attrs)
    #     return super(SingletonType,cls).__new__(cls, name, bases, attrs)

class Binding(object):

    def __init__(self,val):
        self.__val = val

    # def __get__(self, instance, owner):
    #     return self.__val

    # def __set__(self, instance, val):
    #     self.__val = val

class GBinding(Binding):
    pass

class DataBinding(type):

    def __init__(cls, name, bases, attrs):
        super(DataBinding, cls).__init__(name, bases, attrs)
        for member,val in inspect.getmembers(cls):
            if isinstance(val,Binding):
                print(member,val)

def connect_binding(cls):
    """ https://stackoverflow.com/questions/11091609/setting-a-class-metaclass-using-a-decorator """
    __dict = dict(cls.__dict__)
    __dict["__metaclass__"] = DataBinding
    __dict["__wrapped__"] = cls
    return(DataBinding(cls.__name__, cls.__bases__, __dict))

class StateDescriptor(QtCore.QObject):

    def __getitem__(self,key):
        return self.__dict__[key]

    def __setitem__(self,key,value):
        self.__dict__[key] = value

    def __setattr__(self, key, value):
        print("attr",key,value)
        self.__dict__[key] = value

@connect_binding
class Component(object):

    state = StateDescriptor()
    state.number = 1
    state.string = "1"
    state.loc = True

    def __init__(self,*args,**kwargs):
        super(Component, self).__init__(*args,**kwargs)
        number = self.state.number
        string = self.state.string
        print(number,string)

comp = Component()

# print(dir(comp.temple))


# -*- coding: utf-8 -*-
"""
Created on Tue Oct  1 08:42:55 2019
simple_queue_read.py
@author: martidav
"""

#import urlparse
import os
import pika
import config

def callback(ch, method, properties, body):
    print(" [x] Received %r" % body)

# Parse CLODUAMQP_URL (fallback to localhost)
url = os.environ.get('CLOUDAMQP_URL',config.amqp_url)
params = pika.URLParameters(url)
params.socket_timeout = 5

connection = pika.BlockingConnection(params) # Connect to CloudAMQP
channel = connection.channel()
channel.queue_declare(queue='hello')

def read():
    """
    Permet de lire les messages en attente sur CLOUDAMQP
    """
    channel.basic_consume(queue='hello',
                          on_message_callback=callback,
                          auto_ack=True)
    print(' [*] Waiting for messages. To exit press CTRL+C')
    channel.start_consuming()

#test
#read()

"""
Compteur de messages
"""


"""
Author: Tessa Wagenaar
"""

from bisect import bisect_left
from pathlib import Path
from torch.utils.data import Dataset
from utils.bbox import crop_to_bbox
from utils.image_readers import read_image
from utils.io import read_object, save_object

import numpy as np

class CBCTDataset(Dataset):
    def __init__(self, files, n_slices=21, transform=None, cachedir="/cache"):
        self.n_slices = n_slices
        self.image_shapes = {tup[0]: (tup[1][0], tup[1][1], tup[1][2]) for tup in files}
        self.data = {tup[0]: (tup[2], tup[3]) for tup in files}
        self.patients = list(self.image_shapes.keys())
        self.transform = transform
        print(self.image_shapes[self.patients[0]], self.image_shapes[self.patients[0]][-3])
        self.cumulative_imshapes = [0] + list(
            np.cumsum([self.image_shapes[patient][-3] for patient in self.patients])
        )
        self.cachedir = Path(cachedir)
        self.current_patient = None

    def __len__(self):
        return sum([self.image_shapes[patient][-3] for patient in self.patients])

    def _get_segmentation(self, segmentations):
        if len(segmentations) == 2:
            seg_bladder = read_image(segmentations[0], no_meta=True)
            seg_cervix_uterus = read_image(segmentations[1], no_meta=True)

        seg_bladder = crop_to_bbox(seg_bladder, (0, 0, 0, seg_bladder.shape[0], 512, 512))
        seg_cervix_uterus = crop_to_bbox(seg_cervix_uterus, (0, 0, 0, seg_cervix_uterus.shape[0], 512, 512))
        all_segs = seg_bladder + seg_cervix_uterus
        other = all_segs < 1
        segs = [seg_bladder, seg_cervix_uterus, other]
        segmentation = np.stack(segs).astype(int)
        return segmentation

    def _load_image(self, patient):
        cache_fn = self.cachedir / f"{patient}_CT1"
        cache_fn_seg = self.cachedir / f"{patient}_CT1_seg"
        if cache_fn.exists() and cache_fn_seg.exists():
            image = read_object(cache_fn)
            segmentation = read_object(cache_fn_seg)
        else:
            image_path, segmentation_paths = self.data[patient]
            image = read_image(image_path, no_meta=True)
            segmentation = self._get_segmentation(segmentation_paths)
            if len(image.shape) == 3:
                # add "channels" dimension if it is not present
                image = np.expand_dims(image, axis=0)
        return image, segmentation

    def _find_image_from_index(self, i):
        """
        Finds the image containing slice "idx", given that
        indices acumulate over images
        """
        # -1 to compensate for the added [0] to cumulative_imshapes in __init__()
        p_idx = bisect_left(self.cumulative_imshapes, i + 1) - 1
        assert p_idx < len(
            self.patients
        ), f"Illegal slice accessed in dataset! Index {i} exceeds size of dataset."
        return self.patients[p_idx], i - self.cumulative_imshapes[p_idx]

    def __getitem__(self, i):
        patient, slice_idx = self._find_image_from_index(i)
        if self.current_patient != patient:
            self.image, self.segmentation = self._load_image(patient)
            self.current_patient = patient

        start = 0
        middle_slice = self.n_slices // 2

        im_slice = crop_to_bbox(self.image, (0, slice_idx - middle_slice, start, start, 1, self.n_slices, 512, 512))
        seg_slice = crop_to_bbox(self.segmentation, (0, slice_idx, 0, 0, 3, 1, 512, 512))

        sample = {"image": im_slice.squeeze(0), "target": seg_slice.squeeze(1)}
        sample = self.transform(sample)
        im_slice = np.expand_dims(sample["image"],0)
        seg_slice = np.expand_dims(sample["target"],1)

        assert (
            0 not in seg_slice.shape
        ), f"Segmentation slice has dimension of size 0: {seg_slice.shape}"

        return im_slice, seg_slice


#!/usr/bin/env python2
# pylint: disable=missing-docstring,invalid-name,redefined-outer-name
# XXX: Refactor to a comand line tool and remove pylint disable
"""Principal components analysis."""
from __future__ import absolute_import, division, print_function

import argparse
import json
import sys

import numpy as np  # pylint: disable=import-error
from sklearn.decomposition import PCA  # pylint: disable=import-error

import utils

parser = argparse.ArgumentParser(description="PCA")
parser.add_argument('--sample-files', nargs='+', help="All samples")
parser.add_argument('--sample-ids', nargs='+', help="Sample IDs")
parser.add_argument('--genes', nargs='+', help='filter genes')
parser.add_argument('--filter', help="Filter genes with low expression", action="store_true")
args = parser.parse_args()

def isfloat(value):
    """Check if value is float."""
    try:
        float(value)
        return True
    except ValueError:
        return False

def isgzipped(f):
    """Check if file f is gzipped."""
    with open(f, 'rb') as rpkm_file:
        magic = rpkm_file.read(2)

    return magic == '\037\213'

def component_top_factors(component):
    """Return top 20 absolute factors."""
    # 10x faster, but not supported in current numpy:
    #   abs_component = np.abs(component)
    #   unordered_ixs = np.argpartition(abs_component, -20)[-20:]
    #   ixs = unordered_ixs[np.argsort(abs_component[unordered_ixs])[::-1]]
    ixs = np.argsort(np.abs(component))[:-21:-1]
    if len(ixs) == 0:
        return []
    return zip(allgenes_array[ixs].tolist(), component[ixs].tolist())

def save_pca(coordinates, explained_variance_ratios=[0, 0], components=[[], []], warning=None):
    """Save json."""
    data = {
        'pca': {
            'flot': {
                'data': coordinates,
                'xlabel': 'PC 1',
                'ylabel': 'PC 2',
                'sample_ids': sample_ids,
            },
            'zero_gene_symbols': list(zero_genes),
        }
    }
    data['pca']['all_components'] = [component_top_factors(component) for component in components]
    data['pca']['all_explained_variance_ratios'] = explained_variance_ratios
    data['pca']['components'] = data['pca']['all_components'][:10]
    data['pca']['explained_variance_ratios'] = data['pca']['all_explained_variance_ratios'][:10]
    if warning:
        data['proc.warning'] = warning
    print(json.dumps(data, separators=(',', ':'), allow_nan=False))

sample_files = args.sample_files
sample_ids = args.sample_ids

if len(sample_files) != len(sample_ids):
    print('{"rc":"1"}')
    exit(1)

exp = []
allgenes = set()
zero_genes = set()

for fname in sample_files:
    myopen = utils.gzopen if isgzipped(fname) else open

    with myopen(fname) as f:
        exp.append({gene_exp[0]: float(gene_exp[1]) for gene_exp in
                    (l.split('\t') for l in f) if len(gene_exp) == 2 and isfloat(gene_exp[1])})

        allgenes.update(exp[-1].keys())

if args.genes:
    genes = set(args.genes)
    zero_genes = genes.difference(allgenes)
    allgenes = allgenes.intersection(args.genes)

# Default expression value is 0.
allgenes_array = np.array(list(allgenes))
exp = np.array([[genemap.get(g, 0.) for g in allgenes_array] for genemap in exp])

if args.filter:
    exp = np.transpose(exp)
    f_exp = exp[np.sum(exp, axis=1) > exp.shape[1]]
    exp = np.transpose(f_exp)

if exp.shape[1] == 0:
    save_pca(
        coordinates=[[0, 0] for i in range(exp.shape[0])],
        warning='Expressions of all selected genes are 0. Please select different samples or genes.',
    )
    sys.exit(0)

# select the number of components so that explained variance
# is greater than the fraction specified by n_components
pca = PCA(n_components=1.0 - sys.float_info.epsilon, whiten=True)
transformed_data = pca.fit_transform(exp)

coordinates = [[t[0], t[1]] if len(t) > 1 else [t[0], 0] for t in transformed_data]

if any(np.isnan(pca.explained_variance_ratio_)):
    save_pca(coordinates=coordinates)
    sys.exit(0)

save_pca(
    coordinates=coordinates,
    explained_variance_ratios=pca.explained_variance_ratio_.tolist(),
    components=pca.components_,
)


"""
Take (gzipped) input MAF file(s) and write fasta files, for one non-focal species,
with the same coordinates as the focal species. Capital letters in the non-focal
species' fasta files represent standard 5' -> 3' direction of nucleotides. Lower-
case letters represent the reverse synteny of the 5' -> 3' strand (necessary
when 5'/3' context is required, eg. when defining CpG sites). Run once for each
non-focal alignment required (because this program eats up a lot of RAM by keeping
the new data in memory)
"""

import argparse, gzip, sys, Bio, csv, glob, string, copy
from Bio import SeqIO, AlignIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.Alphabet import generic_dna

parser = argparse.ArgumentParser(description="""make reference sequences populated by outgroup alleles using
                                                pairwise alignments in MAF format""")

parser.add_argument('MAF_file_directory', help='the directories with the MAF format files in from EBI')
parser.add_argument('reference_fai_file', help='the index for the reference file, in fasta format - to get the chromosome lengths')
parser.add_argument('reference_species', help='the name of the species that you want to get an outgroup alignment FOR, as defined in the MAF file(s)')
parser.add_argument('outgroup_species', help='the name of the species that you want to get outgroup alleles FROM, as defined in the MAF file(s)')

args = parser.parse_args()

# arguments specified on the command line:
reference_file_index = args.reference_fai_file
ref_species = args.reference_species
out_species = args.outgroup_species
MAFdir = args.MAF_file_directory

# initiate an empty dict:
reference_dict = dict()

# read in the index file and fill the dictionary above with the contig lengths
with open(reference_file_index, 'r') as f:
    textreader = csv.reader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
    for line in textreader:
        if not line[0] in reference_dict:
            reference_dict[line[0]] = line[1]

# make a dict of empty contigs, each the length of the original ref sequence's, for the non-focal species
contigs = {name: ['.'] * int(length) for (name, length) in reference_dict.items()}

i=0
for MAFFILE in glob.glob(MAFdir + '*maf.gz'):

    # this bit is specific to the 1000 genomes maf filenames that I've made, and I
    # include it as a sanity check: get the chromosome of the ref sequence from the filename
    # filename format: # Compara.6_primates_EPO.chr1_1.maf.gz
    # chr_check = MAFFILE.split('/')[-1].split('.')[2].split('_')[0].split('chr')[1]

    with gzip.open(MAFFILE, mode = 'rt') as f:
        for multiple_alignment in AlignIO.parse(f, "maf"):
            nogap = None
            contig = None
            strt = None
            stp = None
            in_strand = None
            out_strand = None
            contig_check = True

            # First thing: There may be paralogs in the multiple alignment.
            # eg. the focal species' sequence may map to another part of its own genome
            # OR the focal species' sequence may map to multiple regions in the outgroup species' genome
            # In either of these cases, ignore this record?
            # So this is all the species in the record:
            spps = [seqrec.id.split('.')[0] for seqrec in multiple_alignment]
            # and if there is not only one of each of the in- and outgroup, then skip this multiple alignment record:
            if len([x for x in spps if x == ref_species]) != 1:
                continue
                # print(spps)
            if len([x for x in spps if x == out_species]) != 1:
                continue
                # print(spps)

            # now go through the records one by one, and do the relevant things if
            # it's an ingroup or an outgroup record

            # for the ingroup:
            for seqrec in multiple_alignment:
                if seqrec.id.split('.')[0] == ref_species:
                    # check that the contig actually exists in the reference file
                    contig = ('.').join(seqrec.id.split('.')[1:])
                    if not contig in contigs:
                        contig_check = False
                        continue

                    # # This bit is specific to the primate alignment that I used for
                    # # Peters sfs methods paper:
                    # # check the chromosome is the one we're expecting:
                    # if seqrec.id.split('.')[1] != chr_check:
                    #     print('chromosome from the filename doesn\'t match the chromosome in the seq record for the ingroup')
                    #     sys.exit()

                    # get the strand that the ingroup sequence is on, because this affects whether we need to reverse complement
                    # the outgroup sequence, and affects the coordinates
                    in_strand = int(copy.copy(seqrec.annotations['strand']))

                    # first, the coordinates:
                    if in_strand < 0:
                        strt = seqrec.annotations['srcSize'] - (seqrec.annotations['start'] + seqrec.annotations['size'])
                        stp = seqrec.annotations['srcSize'] - seqrec.annotations['start']
                        nogap = copy.copy([i for i, item in enumerate(seqrec.seq.reverse_complement()) if item != '-'])
                    if in_strand > 0:
                        strt = seqrec.annotations['start']
                        stp = strt + seqrec.annotations['size']
                        nogap = copy.copy([i for i, item in enumerate(seqrec.seq) if item != '-'])

                    # print(len(nogap))
                    # print(stp - strt)
                    # print()

            if not contig_check:
                continue

            # for the outgroup:
            for seqrec in multiple_alignment:
                if seqrec.id.split('.')[0] == out_species:
                    # this is the strand that the outgroup sequence is on:
                    out_strand = int(copy.copy(seqrec.annotations['strand']))

                    # now carry out the following logic:
                    # if in_strand == out_strand: sequence should be uppercase, because synteny is the same in the in- and outgroups; else sequence should be lowercase
                    # if in_strand < 0: sequence should be reverse complemented to get the alleles that match the human reference sequence
                    tempSeq1 = seqrec.seq

                    if in_strand == out_strand:
                        tempSeq2 = tempSeq1.upper()
                    if in_strand != out_strand:
                        tempSeq2 = tempSeq1.lower()

                    if in_strand < 0:
                        mySeq = tempSeq2.reverse_complement()
                    if in_strand > 0:
                        mySeq = tempSeq2

                    # this is the outgroup sequence for positions which aren't gaps in the ingroup sequence:
                    seq_nogaps = [mySeq[i] for i in nogap]

                    contigs[contig][strt:stp] = seq_nogaps

# there may be gaps in the new reference where there are deletions in the ingroup,
# so replace these with missing data:
for c in contigs.values():
    for idx, item in enumerate(c):
        if item == '-':
            c[idx] = '.'

# Now save the new reference a file

# Make a SeqRecord object with all the sequences in it, by concatenating them
# and assigning their contig names.
new_reference = [SeqRecord(Seq(''.join(s), generic_dna), id = c, description = '') for c, s in contigs.items()]

# Write the record above (with a wrap of 60 characters):
filename = ref_species + '_' + out_species + '_alleles.fa'
Bio.SeqIO.write(new_reference, filename, 'fasta')


# -*- coding: utf-8 -*-

# Define your spider here
#
# Don't forget to add your parse method
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
""" Correo del orinoco

1. urls of news article:
http://www.correodelorinoco.gob.ve/categoria/categorias/venezuela/nacionales/

2. selectors of DOM elments:
archive > div.spoiler > a.left
archive > div.spoiler > div > h3.subtitle
archive > div.spoiler > div > h2.title
archive > div.spoiler > div > div.meta
archive > div.spoiler > div > p

"""
import scrapy
from VeNews.items import OrinocoItem

orinocoUrl1 = 'http://www.correodelorinoco.gob.ve',
orinocoUrl1 = 'http://www.correodelorinoco.gob.ve',

def checkURL(url):
    file_guid = url.split('/')[-1]
    if file_guid[-3:] != 'pdf':
        url = u'{0}print'.format(url)
    return url

class OrinocosSpiderPage(scrapy.Spider):
    name = "orinocoSpiderPage"
    start_urls = (
        'http://www.correodelorinoco.gob.ve/categoria/categorias/venezuela/nacionales',
        'http://www.correodelorinoco.gob.ve/categoria/categorias/venezuela/caracas',
        'http://www.correodelorinoco.gob.ve/categoria/categorias/venezuela/regiones',
        'http://www.correodelorinoco.gob.ve/categoria/categorias/venezuela/politica',
        'http://www.correodelorinoco.gob.ve/categoria/categorias/opinion/',
        'http://espejo3.correodelorinoco.gob.ve/categoria/categorias/venezuela/impacto/',
        'http://espejo3.correodelorinoco.gob.ve/categoria/categorias/venezuela/poder-popular/',
        'http://espejo3.correodelorinoco.gob.ve/categoria/categorias/venezuela/investigacion/',
        'http://espejo3.correodelorinoco.gob.ve/categoria/categorias/venezuela/alimentacion/',
    )

    def parse(self, response):
        for elm in response.css('div#archive > div.spoiler'):
            """
                Creating a item
            """
            item = OrinocoItem()
            """
                Extracting titles
            """
            item['title'] = elm.css('div > h2.title ::text').extract()
            """
                Extracting href
            """
            urls = elm.css('a.left ::attr(href)').extract()
            urls = [checkURL(url) for url in urls if url]
            item['file_urls'] = urls

            yield item

class OrinocoSpiderPdf(scrapy.Spider):
    name = "orinocoSpiderPdf"
    allowed_domains = ["correodelorinoco.gob.ve"]
    start_urls = (
        'http://www.correodelorinoco.gob.ve/edicion-impresa',
    )

    def start_request(self):
        pass

    def log(self, message, level=3, **kw):
        scrapy.Spider.logger.info('msg')

    def parse(self, response):
        """ parse the table
        for elm in response.css('table > tr > td:nth-child(even)')
        for elm in response.css('table > tr > td:nth-child(even) > img ::attr(src)'):
        for elm in response.css('table > tr > td:nth-child(odd) > p.date1')
        for elm in response.css('table > tr > td:nth-child(odd) > p.date2')
        for elm in response.css('table > tr > td:nth-child(odd) > p.numero')
        for elm in response.css('table > tr > td:nth-child(odd) > p.text')
        for elm in response.css('table > tr > td:nth-child(odd) > p.link:nth-child(even) > a.text ::attr(href)')
        for elm in response.css('table > tr > td:nth-child(odd) > p.link:nth-child(odd) > a.text ::attr(href)')
        """
        for elm in response.css('table > tr'):
            item = OrinocoItem()
            """get even table data, which has images link
            """
            item['image_urls'] = elm.css(
                'td:nth-child(odd) > img.attachment-medium::attr(src)').extract()
            """get other meta data from odd table data
            """
            item['date1'] = elm.css(
                'td:nth-child(even) > p.date1 ::text').extract()
            item['date2'] = elm.css(
                'td:nth-child(even) > p.date2 ::text').extract()
            item['numero'] = elm.css(
                'td:nth-child(even) > p.numero ::text').extract()
            item['texto'] = elm.css(
                'td:nth-child(even) > p.texto ::text').extract()

            urls = elm.css(
                'td:nth-child(even) > p.link > a.text ::attr(href)').extract()
            urls = [checkURL(url) for url in urls if url]
            item['file_urls'] = urls
            yield item

        for next_page in response.css('a.page'):
            """
                if you want to go through all pages, using following code
                next_page_url =  next_page.css('::attr(href)').extract_first()
                yield scrapy.Request(response.urljoin(next_page_url),callback=self.parse)
            """
            pass


import json
from datetime import datetime
import graphene
from django.apps import apps
from ..models import Journal
from .types import JournalType, AllModelsType

def get_rus_field_value(data):
	value = str(data)
	if isinstance(data, bool):
		value = 'нет'
		if data:
			value = 'да'
	if isinstance(data, type(None)):
		value = 'неопределенно'
	if isinstance(data, type(datetime.now().date())):
		value = '%02d.%02d.%d' % (data.day, data.month, data.year)
	if isinstance(data, type(datetime.now())):
		value = '%02d.%02d.%d %02d:%02d' % (data.day, data.month, data.year, data.hour, data.minute)
	if isinstance(data, type(datetime.now().time())):
		value = '%02d:%02d' % (data.hour, data.minute)
	return value

def get_rus_string_from_dict(before, after, model):
	string = ''
	if before and after:
		for bf in before:
			for af in after:
				if bf == af:
					field = model._meta.get_field(bf).verbose_name
					before_value = get_rus_field_value(before[bf])
					after_value = get_rus_field_value(after[af])
					string += '%s: %s ➔ %s|=-=|' % (field, before_value, after_value)  # |=-=| - разделитель
	if before and not after:
		for bf in before:
			field = model._meta.get_field(bf).verbose_name
			before_value = get_rus_field_value(before[bf])
			string += '%s: %s ➔ |=-=|' % (field, before_value)

	if not before and after:
		for af in after:
			field = model._meta.get_field(af).verbose_name
			after_value = get_rus_field_value(after[af])
			string += '%s: ➔ %s|=-=|' % (field, after_value)
	return string[0:-5]

# noinspection PyMethodMayBeStatic,PyUnusedLocal
class Query(graphene.ObjectType):
	all_journal = graphene.List(JournalType, model_name=graphene.String(required=True), instance_id=graphene.Int())
	all_models = graphene.List(AllModelsType)

	def resolve_all_models(self, info):
		all_models = Journal.objects.filter().distinct('model').values('model')
		result = []
		for d in all_models:
			app, model = d['model'].split('.')
			model = apps.get_model(app, model)
			result.append(AllModelsType(id=d['model'], name=model._meta.verbose_name))
		return result

	def resolve_all_journal(self, info, model_name, **kwargs):
		instance_id = kwargs.get('instance_id', None)
		all_journal = Journal.objects.select_related('user').filter(model=model_name)
		if instance_id:
			all_journal = all_journal.filter(instance_id=instance_id)
		app, model = model_name.split('.')
		model = apps.get_model(app, model)
		result = []
		# Подготавливаем объекты для отображения их наименования
		ids = (o['instance_id'] for o in all_journal.distinct('instance_id').order_by('instance_id').values('instance_id'))
		select_rel, prefetch_rel = model.history_related()
		instances = model.objects
		if select_rel:
			instances = instances.select_related(*select_rel)
		if prefetch_rel:
			instances = instances.prefetch_related(*prefetch_rel)
		instances = instances.filter(id__in=ids)
		instances = {o.id: o.history_name for o in instances}
		# Запрашиваем журнал
		all_journal = all_journal.order_by('-date')
		for r in all_journal:
			before = ''
			after = ''
			if r.before:
				before = json.loads(r.before)
			if r.after:
				after = json.loads(r.after)
			change = get_rus_string_from_dict(before, after, model)
			result.append(JournalType(date=r.date, user=r.user, instance=instances[r.instance_id], change=change))
		return result


#!/home/ocean/anaconda3/bin/python3
#Right now let's do everything in linear space.
from numpy import cos, arccos, sin, arctan, tan, pi, sqrt; from numpy import array as ary; import numpy as np; tau = 2*pi
from matplotlib import pyplot as plt
import pandas as pd
import sklearn.decomposition as decomp
import os, sys
try:
    n_components_ = int(sys.argv[1])
except:
    print("use program by specifying the number of components that you'd like to use after calling it")
    exit()
for file in [file for file in os.listdir() if (file.endswith(".csv") and not file.startswith("PC"))]: #list all csvs
    list_of_vectors = pd.read_csv(file, header=None, index_col=None).values
    PCA = decomp.PCA(n_components_)#list_of_vectors
    PCA.fit(list_of_vectors)
    # print(decomp.FastICA(list_of_vectors))
    if __name__=="__main__":
        np.savetxt("PCA_PC_"+file, PCA.components_,     delimiter=",")
        np.savetxt("PCA_SV_"+file, PCA.singular_values_,delimiter=",")