import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
import unicodedata

# NBA teams site
url = 'http://espn.go.com/nba/teams'

# Team site categories
team_categories = ('','stats','schedule','roster','depth',
	'transactions','ratings','photos','news','stadium')

def getTeamSiteSoup():
	r = requests.get(url)
	return BeautifulSoup(r.text)

def getTeamNames():
	soup = getTeamSiteSoup()
	team_tags = soup.find_all("div", class_="logo-nba-medium")
	teams = []
	for team_tag in team_tags:
		teams.append(team_tag.a.string)
	return teams

def getTeamUrls():
	soup = getTeamSiteSoup()
	team_tags = soup.find_all("div", class_="logo-nba-medium")
	team_names = getTeamNames()
	team_urls = dict.fromkeys(team_names)
	for team_tag in team_tags:
		team_urls[team_tag.a.string] = team_tag.a.get('href')
	return team_urls

def getTeamUrl(team_name, team_category=''):
	team_urls = getTeamUrls()
	if team_name not in team_urls.keys():
		return 'Invalid team name'
	elif (team_category not in team_categories) or (team_category == ''):
		return team_urls[team_name]
	else:
		a = team_urls[team_name].split('_')
		return a[0] + team_category + '/_' + a[1]

def getTeamRoster(team_name, convert_numerics=True, normalize_unicode=True):
	# get html from NBA team site
	url = getTeamUrl(team_name,'roster')
	r = requests.get(url)

	# pass html text to BeautifulSoup
	soup = BeautifulSoup(r.text)

	# Find player tags
	players = soup.find_all("tr", re.compile("player-"))

	# Roster columns
	roster_cols = ('Number','Name','Position','Age','Height',
		'Weight','College','Salary')
	roster_dict = {}
	for col in roster_cols:
		roster_dict[col] = []

	# Construct dictionary of players
	for player in players:
		n = 0
		for col, n in zip(player.contents, range(len(player.contents))):
			raw_str = col.string
			if normalize_unicode:
				norm_str = unicodedata.normalize('NFKD', raw_str).encode('ascii','ignore')
			else:
				norm_str = raw_str
			roster_dict[roster_cols[n]].append(norm_str)

	roster = pd.DataFrame(roster_dict, columns=roster_cols)

	if convert_numerics:
		roster['Number'] = roster['Number'].astype('int64')
		roster['Age'] = roster['Age'].astype('int64')
		roster['Weight'] = roster['Weight'].astype('int64')
		roster['Height'] = getHeightInt(roster['Height'])
		roster['Salary'] = getSalaryInt(roster['Salary'])
		#roster.insert(5, 'Height (num)', getHeightInt(roster['Height']))
		#roster['Salary (num)'] = getSalaryInt(roster['Salary'])
		return roster
	else:
		return roster

def getHeightInt(height_strs):
	heights = []
	for h in height_strs:
		[feet, inches] = h.split('-')
		height_in = int(feet)*12 + int(inches)
		heights.append(height_in)
	return pd.Series(heights)

def getSalaryInt(salary_strs):
	salaries = []
	for s in salary_strs:
		if re.search('[$,]',s) == None:
			b = 0
		else:
			b = int(re.sub('[$,]','',s))
		salaries.append(b)
	return pd.Series(salaries)

#=============================================================================
# #Tests
# print getTeamNames()
# print getTeamUrls()
# print getTeamUrl('Golden State Warriors','stats')
# print getTeamUrl('Los Angeles Lakers','stadium')
# print getTeamRoster('Charlotte Bobcats')
# print getTeamRoster('Charlotte Bobcats','numerical')
# print getTeamRoster('Charlotte Bobcats','numerical').dtypes


import random

import torch
import torch.nn as nn

from helpers.storage import RolloutStorage
from helpers.utils import parse_geno_log
import torch.npu
import os
NPU_CALCULATE_DEVICE = 0
if os.getenv('NPU_CALCULATE_DEVICE') and str.isdigit(os.getenv('NPU_CALCULATE_DEVICE')):
    NPU_CALCULATE_DEVICE = int(os.getenv('NPU_CALCULATE_DEVICE'))
if torch.npu.current_device() != NPU_CALCULATE_DEVICE:
    torch.npu.set_device(f'npu:{NPU_CALCULATE_DEVICE}')

class REINFORCE(object):
    """REINFORCE gradient estimator
    """

    def __init__(self, controller, lr, baseline_decay, max_grad_norm=2.0):
        """
        Args:
          controller (Controller): RNN architecture generator
          lr (float): learning rate for controller optimizer
          baseline_decay (float): moving average baseline decay
          max_grad_norm (float): controller gradient clip
        """
        self.baseline = None
        self.decay = baseline_decay
        self.controller = controller
        self.optimizer = torch.optim.Adam(controller.parameters(), lr=lr)
        self.max_grad_norm = max_grad_norm

    def update(self, sample):
        """Perform one gradient step for controller and update baseline.
        Args:
          sample (tuple): (reward, action, log_prob)
            reward (float): current reward
            action (list): representation of current architecture
            log_prob (float): log probability of current architecture

        Returns:
          loss (torch.FloatTensor): controller loss
          entropy (torch.FloatTensor): entropy of current architecture
        """
        reward, action, _, _ = sample
        _, _, entropy, log_prob = self.controller.evaluate(action)
        with torch.no_grad():
            if self.baseline is None:
                self.baseline = reward
            else:
                self.baseline = self.decay * self.baseline + (1 - self.decay) * reward

            adv = reward - self.baseline

        loss = -log_prob * adv
        self.optimizer.zero_grad()
        loss.backward()
        nn.utils.clip_grad_norm_(self.controller.parameters(), self.max_grad_norm)
        self.optimizer.step()
        return loss, entropy

    def state_dict(self):
        return {
            "baseline": self.baseline,
            "controller": self.controller.state_dict(),
            "optimizer": self.optimizer.state_dict(),
        }

    def load_state_dict(self, states):
        self.controller.load_state_dict(states["controller"])
        self.baseline = states["baseline"]
        self.optimizer.load_state_dict(states["optimizer"])

class PPO(object):
    """Proximal Policy Optimization with rollout buffer
    part of the update code modified from:

    https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail/blob/master/a2c_ppo_acktr/algo/ppo.py
    """

    def __init__(
        self,
        controller,
        clip_param,
        lr,
        baseline_decay,
        action_size=18,
        ppo_epoch=1,
        num_mini_batch=100,
        max_grad_norm=2.0,
        entropy_coef=0,
        num_steps=100,
        num_processes=1,
    ):
        """
        Args:
          controller (Controller): RNN architecture generator
          clip_param (float): PPO clip parameter epsilon
          lr (float): learning rate for controller optimizer
          baseline_decay (float): moving average baseline decay
          action_size (int): length of architecture representation
          ppo_epoch (int): number of epochs to train
          num_mini_batch (int): number of mini batches in the rollout buffer
          max_grad_norm (float): controller gradient clip
          entropy_coef (float): gradient coefficient for entropy regularization
          num_steps (int): number of steps to train
          num_processes (int): samples per step
        """
        self.ppo_epoch = ppo_epoch
        self.controller = controller
        self.optimizer = torch.optim.Adam(controller.parameters(), lr=lr)
        self.num_mini_batch = num_mini_batch
        self.clip_param = clip_param
        self.max_grad_norm = max_grad_norm
        self.entropy_coef = entropy_coef
        self.rollouts = RolloutStorage(num_steps, num_processes, action_size)
        self.baseline = None
        self.decay = baseline_decay

    def state_dict(self):
        return {
            "baseline": self.baseline,
            "rollouts": self.rollouts,
            "controller": self.controller.state_dict(),
            "optimizer": self.optimizer.state_dict(),
        }

    def load_state_dict(self, states):
        self.controller.load_state_dict(states["controller"])
        self.optimizer.load_state_dict(states["optimizer"])
        self.baseline = states["baseline"]
        if "rollouts" not in states:
            # continue from old checkpoint format
            # fill in rollouts
            with open("genotypes.out") as ro_file:
                lines = ro_file.readlines()
                # randomly pick
                random.shuffle(lines)
                records = lines[: self.rollouts.num_steps]
            for record in records:
                reward, action = parse_geno_log(record)
                with torch.no_grad():
                    _, _, _, log_prob = self.controller.evaluate(action)
                self.update((reward, action, log_prob), is_train=False)
            print(self.rollouts.actions)
        else:
            self.rollouts = states["rollouts"]

    def update(self, sample, is_train=True):
        reward, action, log_prob = sample
        if self.baseline is None:
            self.baseline = reward
        else:
            self.baseline = self.decay * self.baseline + (1 - self.decay) * reward
        self.rollouts.insert(action, log_prob, reward)
        if not is_train:
            return -1, -1
        advantages = self.rollouts.rewards - self.baseline
        loss_epoch = 0
        entropy_epoch = 0

        for _ in range(self.ppo_epoch):
            data_generator = self.rollouts.generator(advantages, self.num_mini_batch)
            for sample in data_generator:
                (
                    actions_batch,
                    rewards_batch,
                    old_actions_log_probs_batch,
                    adv_targ,
                ) = sample

                action_log_probs, entropy = self.controller.evaluate_actions(
                    actions_batch
                )

                ratio = torch.exp(
                    action_log_probs
                    - torch.from_numpy(old_actions_log_probs_batch).float()
                )
                adv_targ_th = torch.from_numpy(adv_targ).float()
                surr1 = ratio * adv_targ_th
                surr2 = (
                    torch.clamp(ratio, 1.0 - self.clip_param, 1.0 + self.clip_param)
                    * adv_targ_th
                )
                action_loss = -torch.min(surr1, surr2).mean()
                self.optimizer.zero_grad()
                dist_entropy = entropy.mean()
                (action_loss - dist_entropy * self.entropy_coef).backward()
                nn.utils.clip_grad_norm_(
                    self.controller.parameters(), self.max_grad_norm
                )
                self.optimizer.step()

                loss_epoch += action_loss.item()
                entropy_epoch += dist_entropy.item()

        num_updates = self.ppo_epoch * self.num_mini_batch

        return loss_epoch / num_updates, entropy_epoch / num_updates


import torch.nn as nn
import os
import torch
from tqdm import tqdm
import torch.optim as optim
from torch.utils.data import DataLoader
from functools import partial
from iRT.model import Transformer
from iRT.data import PRDataset

os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "0"

model = Transformer(
        input_size=22, embed_dim=256, depth=12, num_heads=8, mlp_ratio=4,
        qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6))

model.cuda()

batch_size = 1024
num_workers = 8

train_dataset = PRDataset(phase="train", data_path="/home/ubuntu/proto/data/x.npy",
                          label_path="/home/ubuntu/proto/data/y.npy")

val_dataset = PRDataset(phase="validation", data_path="/home/ubuntu/proto/data/x_val.npy",
                        label_path="/home/ubuntu/proto/data/y_val.npy")

train_dl = DataLoader(train_dataset, batch_size=batch_size,
                    num_workers=num_workers,
                    pin_memory=True,
                    drop_last=True,
                    shuffle=True)

val_dl = DataLoader(val_dataset, batch_size=batch_size*2,
                    num_workers=num_workers,
                    pin_memory=True,
                    drop_last=False,
                    shuffle=False)

criterion = nn.L1Loss()
learning_rate = 1e-4
weight_decay = 1e-3
optimizer = optim.AdamW([
            {'params': model.parameters(),'lr': learning_rate, "weight_decay": weight_decay},
        ], betas=(0.9, 0.999), eps=1e-6)

exp_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.90, last_epoch=-1, verbose=False)

def train():

    model.train()
    optimizer.zero_grad()

    train_loss = 0.0
    cnt = 0

    pbar = tqdm(train_dl)

    for data in pbar:

        cnt += 1

        sample = data
        data, label = sample['data'].cuda(), sample['label'].cuda()

        output = model(data)

        loss = criterion(output.float(), label.float()).float()
        train_loss += loss.item()

        loss.backward()

        optimizer.step()
        optimizer.zero_grad()

    train_epoch_loss = train_loss / cnt

    return train_epoch_loss

def eval():

    cnt = 0
    val_loss = 0.0
    pbar = tqdm(val_dl)
    model.eval()

    for data in pbar:

        cnt += 1
        sample = data
        data, label = sample['data'].cuda(), sample['label'].cuda()

        with torch.no_grad():
            output = model(data)

        loss = criterion(output.float(), label.float()).float()
        val_loss += loss.item()

    val_epoch_loss = val_loss / cnt

    return val_epoch_loss

stats = list()
best_train_loss = 1000.0
best_val_loss = 1000.0
epochs = 100
path = "/home/ubuntu/proto/models/"

for epoch in range(0,epochs):

    train_epoch_loss = train()
    val_epoch_loss = eval()

    exp_scheduler.step()

    if train_epoch_loss < best_train_loss:
        model_name = path + "train_ckpt.pth"
        torch.save(model.state_dict(), model_name)
        best_train_loss = train_epoch_loss

    if val_epoch_loss < best_val_loss:
        model_name = path + "val_ckpt.pth"
        torch.save(model.state_dict(), model_name)
        best_val_loss = val_epoch_loss


from pymongo import MongoClient
client = MongoClient('localhost', 27017)
db = client.dbsparta

## 코딩 할 준비 ##

movieName = list(db.movies.find({'title':'매트릭스'}, {'_id':False}))
print(movieName[0]['star'])

valus = movieName[0]['star']

same_movie = list(db.movies.find({'star' : valus}, {'_id':False}))
for movie_title in same_movie:
    print(movie_title['title'])

db.movies.update_many({'star' : valus}, {'$set' : {'star' : 0}})

count_movie = db.movies.find({ 'star' : {'$gt':'9.38'}}).count()
print(count_movie)

# 생김새
# db.people.update_many(찾을조건,{ '$set': 어떻게바꿀지 })
# same_ages = list(db.users.find({'age':21},{'_id':False}))
# all_users = list(db.users.find())
# db.articles.find( { “likes”: { $lte: 30 } } )


#!/bin/python3

import discord
import sqlite3
import datetime
import requests
import re
import sys
import dice
import os

class DBHelper:
    def __init__(self,dbname='db.sqlite3'):
        self.dbname = dbname
        self.conn = sqlite3.connect(dbname)

    def setup(self):
        tables = [ "crypto_prices", "stock_prices" ]
        for table in tables:
            stmt = 'CREATE TABLE IF NOT EXISTS %s (date text, name text, price real)'%(table)
            self.conn.execute(stmt)
            stmt = 'CREATE INDEX IF NOT EXISTS %s_date_index ON %s (date)'%(table,table)
            self.conn.execute(stmt)
            stmt = 'CREATE INDEX IF NOT EXISTS %s_name_index ON %s (name)'%(table,table)
            self.conn.execute(stmt)
            self.conn.commit()

    def add_item(self,table,values=()):
        stmt = 'INSERT INTO %s (date,name,price) VALUES(?,?,?)'%(table)
        args = values
        if len(args) == 3:
            self.conn.execute(stmt,args)
            self.conn.commit()
        else:
            print("Refusing to insert due to wrong number of args")

    def get_item_by_time(self,table,time):
        stmt = 'SELECT * FROM %s WHERE date > ? ORDER BY date DESC LIMIT 1'%(table)
        args = (time.isoformat(sep=' '),)
        arr = self.conn.execute(stmt,args).fetchall()
        self.conn.commit()
        return arr

class MyClient(discord.Client):
    async def on_ready(self):
        self.db = DBHelper()
        self.db.setup()
        self.coindesk_api = "https://api.coindesk.com/v1/bpi/currentprice.json"
        self.time_delay = 5
        self.commands_msg = "Subcommands supported:\n```btc: display price of BTC in the past 15 min\nroll: with a xdy argument, roll x dice\n```"
        print('Logged on as', self.user)

    def get_cached_btc_price(self):
        now = datetime.datetime.now()
        cache_time = now - datetime.timedelta(minutes=self.time_delay)
        item = self.db.get_item_by_time("crypto_prices",cache_time)
        if len(item) == 0:
            request = requests.get(self.coindesk_api)
            while request.status_code != 200:
                sleep(10)
                request = requests.get(self.coindesk_api)
            time = datetime.datetime.fromisoformat(request.json()['time']['updatedISO'])
            usd_rate = request.json()['bpi']['USD']['rate_float']
            self.db.add_item("crypto_prices",(time,'BTC',usd_rate))
            return time,usd_rate
        else:
            time,name,price = item[0]
            time = datetime.datetime.fromisoformat(time)
            return time,price

    async def on_message(self,message):
        if message.author == self.user:
            return
        elif re.match(r'^gamble\b', message.content) is not None:
            if len(message.content.split()) == 1:
                await message.channel.send(self.commands_msg)
            elif message.content.split()[1] == 'btc':
                time,price =  self.get_cached_btc_price()
                embed = discord.Embed()
                embed.description = "Powered by [CoinDesk](https://www.coindesk.com/price/bitcoin)"
                content = "The price of BTC at %s was %s USD\nThis bot will only query the price if cached data is older than %s minutes to avoid overuse of the API."%(time.ctime(),price,str(self.time_delay))
                await message.channel.send(content=content,embed=embed)
            elif message.content.split()[1] == 'roll':
                if len(message.content.split()) == 2:
                     await message.channel.send("Roll usage:\n```gamble roll xdy: rolls x dice with a range from 1 to y```")
                elif re.match(r'^\d+d\d+$',message.content.split()[2]) is not None:
                    string = message.content.split()[2]
                    x,y = string.split("d")
                    if x == 1:
                        await message.channel.send("Rolling %s die with %s faces each...\n%s!"%(x,y,sum(dice.roll(string))))
                    else:
                        await message.channel.send("Rolling %s dice with %s faces each...\n%s!"%(x,y,sum(dice.roll(string))))
                else:
                    await message.channel.send("Roll usage:\n```gamble roll xdy: rolls x dice with a range from 1 to y```")
            else:
                await message.channel.send(self.commands_msg)
        elif message.content == "ping":
            await message.channel.send('pong')

client = MyClient()
try:
    client.run(os.environ.get('DISCORD_TOKEN'))
except:
    e = sys.exc_info()[0]
    print(e)


import os
import gym
import time
import torch
import numpy as np
import numpy.random as rd
from copy import deepcopy
from elegantrl.replay import ReplayBuffer, ReplayBufferMP
from elegantrl.env import PreprocessEnv

"""[ElegantRL](https://github.com/AI4Finance-LLC/ElegantRL)"""

class Arguments:
    def __init__(self, agent=None, env=None, gpu_id=None, if_on_policy=False):
        self.agent = agent  # Deep Reinforcement Learning algorithm
        self.cwd = None  # current work directory. cwd is None means set it automatically
        self.env = env  # the environment for training
        self.env_eval = None  # the environment for evaluating
        self.gpu_id = gpu_id  # choose the GPU for running. gpu_id is None means set it automatically

        '''Arguments for training (off-policy)'''
        self.net_dim = 2 ** 8  # the network width
        self.batch_size = 2 ** 8  # num of transitions sampled from replay buffer.
        self.repeat_times = 2 ** 0  # repeatedly update network to keep critic's loss small
        self.target_step = 2 ** 10  # collect target_step, then update network
        self.max_memo = 2 ** 17  # capacity of replay buffer
        if if_on_policy:  # (on-policy)
            self.net_dim = 2 ** 9
            self.batch_size = 2 ** 9
            self.repeat_times = 2 ** 4
            self.target_step = 2 ** 12
            self.max_memo = self.target_step
        self.gamma = 0.99  # discount factor of future rewards
        self.reward_scale = 2 ** 0  # an approximate target reward usually be closed to 256
        self.if_per = False  # Prioritized Experience Replay for sparse reward

        self.rollout_num = 2  # the number of rollout workers (larger is not always faster)
        self.num_threads = 8  # cpu_num for evaluate model, torch.set_num_threads(self.num_threads)

        '''Arguments for evaluate'''
        self.break_step = 2 ** 20  # break training after 'total_step > break_step'
        self.if_remove = True  # remove the cwd folder? (True, False, None:ask me)
        self.if_allow_break = True  # allow break training when reach goal (early termination)
        self.eval_gap = 2 ** 5  # evaluate the agent per eval_gap seconds
        self.eval_times1 = 2 ** 2  # evaluation times
        self.eval_times2 = 2 ** 4  # evaluation times if 'eval_reward > max_reward'
        self.random_seed = 0  # initialize random seed in self.init_before_training()

    def init_before_training(self, if_main=True):
        if self.agent is None:
            raise RuntimeError('\n| Why agent=None? Assignment args.agent = AgentXXX please.')
        if not hasattr(self.agent, 'init'):
            raise RuntimeError('\n| There should be agent=AgentXXX() instead of agent=AgentXXX')
        if self.env is None:
            raise RuntimeError('\n| Why env=None? Assignment args.env = XxxEnv() please.')
        if isinstance(self.env, str) or not hasattr(self.env, 'env_name'):
            raise RuntimeError('\n| What is env.env_name? use env=PreprocessEnv(env). It is a Wrapper.')

        '''set gpu_id automatically'''
        if self.gpu_id is None:  # set gpu_id automatically
            import sys
            self.gpu_id = sys.argv[-1][-4]
        else:
            self.gpu_id = str(self.gpu_id)
        if not self.gpu_id.isdigit():  # set gpu_id as '0' in default
            self.gpu_id = '0'

        '''set cwd automatically'''
        if self.cwd is None:
            agent_name = self.agent.__class__.__name__
            self.cwd = f'./{agent_name}/{self.env.env_name}_{self.gpu_id}'

        if if_main:
            print(f'| GPU id: {self.gpu_id}, cwd: {self.cwd}')

            import shutil  # remove history according to bool(if_remove)
            if self.if_remove is None:
                self.if_remove = bool(input("PRESS 'y' to REMOVE: {}? ".format(self.cwd)) == 'y')
            if self.if_remove:
                shutil.rmtree(self.cwd, ignore_errors=True)
                print("| Remove history")
            os.makedirs(self.cwd, exist_ok=True)

        os.environ['CUDA_VISIBLE_DEVICES'] = str(self.gpu_id)
        torch.set_num_threads(self.num_threads)
        torch.set_default_dtype(torch.float32)
        torch.manual_seed(self.random_seed)
        np.random.seed(self.random_seed)

'''single process training'''

def train_and_evaluate(args):
    args.init_before_training()

    '''basic arguments'''
    cwd = args.cwd
    env = args.env
    agent = args.agent
    gpu_id = args.gpu_id  # necessary for Evaluator?

    '''training arguments'''
    net_dim = args.net_dim
    max_memo = args.max_memo
    break_step = args.break_step
    batch_size = args.batch_size
    target_step = args.target_step
    repeat_times = args.repeat_times
    if_break_early = args.if_allow_break
    if_per = args.if_per
    gamma = args.gamma
    reward_scale = args.reward_scale

    '''evaluating arguments'''
    eval_gap = args.eval_gap
    eval_times1 = args.eval_times1
    eval_times2 = args.eval_times2
    if args.env_eval is not None:
        env_eval = args.env_eval
    elif args.env_eval in set(gym.envs.registry.env_specs.keys()):
        env_eval = PreprocessEnv(gym.make(env.env_name))
    else:
        env_eval = deepcopy(env)

    del args  # In order to show these hyper-parameters clearly, I put them above.

    '''init: environment'''
    max_step = env.max_step
    state_dim = env.state_dim
    action_dim = env.action_dim
    if_discrete = env.if_discrete

    '''init: Agent, ReplayBuffer, Evaluator'''
    agent.init(net_dim, state_dim, action_dim, if_per)
    if_on_policy = getattr(agent, 'if_on_policy', False)

    buffer = ReplayBuffer(max_len=max_memo + max_step, state_dim=state_dim, action_dim=1 if if_discrete else action_dim,
                          if_on_policy=if_on_policy, if_per=if_per, if_gpu=True)

    evaluator = Evaluator(cwd=cwd, agent_id=gpu_id, device=agent.device, env=env_eval,
                          eval_gap=eval_gap, eval_times1=eval_times1, eval_times2=eval_times2, )

    '''prepare for training'''
    agent.state = env.reset()
    if if_on_policy:
        steps = 0
    else:  # explore_before_training for off-policy
        with torch.no_grad():  # update replay buffer
            steps = explore_before_training(env, buffer, target_step, reward_scale, gamma)

        agent.update_net(buffer, target_step, batch_size, repeat_times)  # pre-training and hard update
        agent.act_target.load_state_dict(agent.act.state_dict()) if getattr(agent, 'act_target', None) else None
        agent.cri_target.load_state_dict(agent.cri.state_dict()) if getattr(agent, 'cri_target', None) else None
    total_step = steps

    '''start training'''
    if_reach_goal = False
    while not ((if_break_early and if_reach_goal)
               or total_step > break_step
               or os.path.exists(f'{cwd}/stop')):
        steps = agent.explore_env(env, buffer, target_step, reward_scale, gamma)
        total_step += steps

        obj_a, obj_c = agent.update_net(buffer, target_step, batch_size, repeat_times)

        if_reach_goal = evaluator.evaluate_save(agent.act, steps, obj_a, obj_c)
        evaluator.draw_plot()

    print(f'| SavedDir: {cwd}\n| UsedTime: {time.time() - evaluator.start_time:.0f}')

'''multiprocessing training'''

def train_and_evaluate_mp(args):
    act_workers = args.rollout_num
    import multiprocessing as mp  # Python built-in multiprocessing library

    pipe1_eva, pipe2_eva = mp.Pipe()  # Pipe() for Process mp_evaluate_agent()
    pipe2_exp_list = list()  # Pipe() for Process mp_explore_in_env()

    process_train = mp.Process(target=mp_train, args=(args, pipe2_eva, pipe2_exp_list))
    process_evaluate = mp.Process(target=mp_evaluate, args=(args, pipe1_eva))
    process = [process_train, process_evaluate]

    for worker_id in range(act_workers):
        exp_pipe1, exp_pipe2 = mp.Pipe(duplex=True)
        pipe2_exp_list.append(exp_pipe1)
        process.append(mp.Process(target=mp_explore, args=(args, exp_pipe2, worker_id)))

    [p.start() for p in process]
    process_evaluate.join()
    process_train.join()
    [p.terminate() for p in process]

def mp_train(args, pipe1_eva, pipe1_exp_list):
    args.init_before_training(if_main=False)

    '''basic arguments'''
    env = args.env
    cwd = args.cwd
    agent = args.agent
    rollout_num = args.rollout_num

    '''training arguments'''
    net_dim = args.net_dim
    max_memo = args.max_memo
    break_step = args.break_step
    batch_size = args.batch_size
    target_step = args.target_step
    repeat_times = args.repeat_times
    if_break_early = args.if_allow_break
    if_per = args.if_per
    del args  # In order to show these hyper-parameters clearly, I put them above.

    '''init: environment'''
    max_step = env.max_step
    state_dim = env.state_dim
    action_dim = env.action_dim
    if_discrete = env.if_discrete

    '''init: Agent, ReplayBuffer'''
    agent.init(net_dim, state_dim, action_dim, if_per)
    if_on_policy = getattr(agent, 'if_on_policy', False)

    '''send'''
    pipe1_eva.send(agent.act)  # send
    # act = pipe2_eva.recv()  # recv

    buffer_mp = ReplayBufferMP(max_len=max_memo + max_step * rollout_num, if_on_policy=if_on_policy,
                               state_dim=state_dim, action_dim=1 if if_discrete else action_dim,
                               rollout_num=rollout_num, if_gpu=True, if_per=if_per)

    '''prepare for training'''
    if if_on_policy:
        steps = 0
    else:  # explore_before_training for off-policy
        with torch.no_grad():  # update replay buffer
            steps = 0
            for i in range(rollout_num):
                pipe1_exp = pipe1_exp_list[i]

                # pipe2_exp.send((buffer.buf_state[:buffer.now_len], buffer.buf_other[:buffer.now_len]))
                buf_state, buf_other = pipe1_exp.recv()

                steps += len(buf_state)
                buffer_mp.extend_buffer(buf_state, buf_other, i)

        agent.update_net(buffer_mp, target_step, batch_size, repeat_times)  # pre-training and hard update
        agent.act_target.load_state_dict(agent.act.state_dict()) if getattr(env, 'act_target', None) else None
        agent.cri_target.load_state_dict(agent.cri.state_dict()) if getattr(env, 'cri_target', None) in dir(
            agent) else None
    total_step = steps
    '''send'''
    pipe1_eva.send((agent.act, steps, 0, 0.5))  # send
    # act, steps, obj_a, obj_c = pipe2_eva.recv()  # recv

    '''start training'''
    if_solve = False
    while not ((if_break_early and if_solve)
               or total_step > break_step
               or os.path.exists(f'{cwd}/stop')):
        '''update ReplayBuffer'''
        steps = 0  # send by pipe1_eva
        for i in range(rollout_num):
            pipe1_exp = pipe1_exp_list[i]
            '''send'''
            pipe1_exp.send(agent.act)
            # agent.act = pipe2_exp.recv()
            '''recv'''
            # pipe2_exp.send((buffer.buf_state[:buffer.now_len], buffer.buf_other[:buffer.now_len]))
            buf_state, buf_other = pipe1_exp.recv()

            steps += len(buf_state)
            buffer_mp.extend_buffer(buf_state, buf_other, i)
        total_step += steps

        '''update network parameters'''
        obj_a, obj_c = agent.update_net(buffer_mp, target_step, batch_size, repeat_times)

        '''saves the agent with max reward'''
        '''send'''
        pipe1_eva.send((agent.act, steps, obj_a, obj_c))
        # q_i_eva_get = pipe2_eva.recv()

        if_solve = pipe1_eva.recv()

        if pipe1_eva.poll():
            '''recv'''
            # pipe2_eva.send(if_solve)
            if_solve = pipe1_eva.recv()

    buffer_mp.print_state_norm(env.neg_state_avg if hasattr(env, 'neg_state_avg') else None,
                               env.div_state_std if hasattr(env, 'div_state_std') else None)  # 2020-12-12

    '''send'''
    pipe1_eva.send('stop')
    # q_i_eva_get = pipe2_eva.recv()
    time.sleep(4)

def mp_explore(args, pipe2_exp, worker_id):
    args.init_before_training(if_main=False)

    '''basic arguments'''
    env = args.env
    agent = args.agent
    rollout_num = args.rollout_num

    '''training arguments'''
    net_dim = args.net_dim
    max_memo = args.max_memo
    target_step = args.target_step
    gamma = args.gamma
    if_per = args.if_per
    reward_scale = args.reward_scale

    random_seed = args.random_seed
    torch.manual_seed(random_seed + worker_id)
    np.random.seed(random_seed + worker_id)
    del args  # In order to show these hyper-parameters clearly, I put them above.

    '''init: environment'''
    max_step = env.max_step
    state_dim = env.state_dim
    action_dim = env.action_dim
    if_discrete = env.if_discrete

    '''init: Agent, ReplayBuffer'''
    agent.init(net_dim, state_dim, action_dim, if_per)
    agent.state = env.reset()

    if_on_policy = getattr(agent, 'if_on_policy', False)
    buffer = ReplayBuffer(max_len=max_memo // rollout_num + max_step, if_on_policy=if_on_policy,
                          state_dim=state_dim, action_dim=1 if if_discrete else action_dim,
                          if_per=if_per, if_gpu=False)

    '''start exploring'''
    exp_step = target_step // rollout_num
    with torch.no_grad():
        if not if_on_policy:
            explore_before_training(env, buffer, exp_step, reward_scale, gamma)

            buffer.update_now_len_before_sample()

            pipe2_exp.send((buffer.buf_state[:buffer.now_len], buffer.buf_other[:buffer.now_len]))
            # buf_state, buf_other = pipe1_exp.recv()

            buffer.empty_buffer_before_explore()

        while True:
            agent.explore_env(env, buffer, exp_step, reward_scale, gamma)

            buffer.update_now_len_before_sample()

            pipe2_exp.send((buffer.buf_state[:buffer.now_len], buffer.buf_other[:buffer.now_len]))
            # buf_state, buf_other = pipe1_exp.recv()

            buffer.empty_buffer_before_explore()

            # pipe1_exp.send(agent.act)
            agent.act = pipe2_exp.recv()

def mp_evaluate(args, pipe2_eva):
    args.init_before_training(if_main=True)

    '''basic arguments'''
    cwd = args.cwd
    env = args.env
    env_eval = env if args.env_eval is None else args.env_eval
    agent_id = args.gpu_id

    '''evaluating arguments'''
    eval_gap = args.eval_gap
    eval_times1 = args.eval_times1
    eval_times2 = args.eval_times2
    del args  # In order to show these hyper-parameters clearly, I put them above.

    '''init: Evaluator'''
    evaluator = Evaluator(cwd=cwd, agent_id=agent_id, device=torch.device("cpu"), env=env_eval,
                          eval_gap=eval_gap, eval_times1=eval_times1, eval_times2=eval_times2, )  # build Evaluator

    '''act_cpu without gradient for pipe1_eva'''
    # pipe1_eva.send(agent.act)
    act = pipe2_eva.recv()

    act_cpu = deepcopy(act).to(torch.device("cpu"))  # for pipe1_eva
    [setattr(param, 'requires_grad', False) for param in act_cpu.parameters()]

    '''start evaluating'''
    with torch.no_grad():  # speed up running
        act, steps, obj_a, obj_c = pipe2_eva.recv()  # pipe2_eva (act, steps, obj_a, obj_c)

        if_loop = True
        while if_loop:
            '''update actor'''
            while not pipe2_eva.poll():  # wait until pipe2_eva not empty
                time.sleep(1)
            steps_sum = 0
            while pipe2_eva.poll():  # receive the latest object from pipe
                '''recv'''
                # pipe1_eva.send((agent.act, steps, obj_a, obj_c))
                # pipe1_eva.send('stop')
                q_i_eva_get = pipe2_eva.recv()

                if q_i_eva_get == 'stop':
                    if_loop = False
                    break
                act, steps, obj_a, obj_c = q_i_eva_get
                steps_sum += steps
            act_cpu.load_state_dict(act.state_dict())
            if_solve = evaluator.evaluate_save(act_cpu, steps_sum, obj_a, obj_c)
            '''send'''
            pipe2_eva.send(if_solve)
            # if_solve = pipe1_eva.recv()

            evaluator.draw_plot()

    print(f'| SavedDir: {cwd}\n| UsedTime: {time.time() - evaluator.start_time:.0f}')

    while pipe2_eva.poll():  # empty the pipe
        pipe2_eva.recv()

'''utils'''

class Evaluator:
    def __init__(self, cwd, agent_id, eval_times1, eval_times2, eval_gap, env, device):
        self.recorder = [(0., -np.inf, 0., 0., 0.), ]  # total_step, r_avg, r_std, obj_a, obj_c
        self.r_max = -np.inf
        self.total_step = 0

        self.cwd = cwd  # constant
        self.device = device
        self.agent_id = agent_id
        self.eval_gap = eval_gap
        self.eval_times1 = eval_times1
        self.eval_times2 = eval_times2
        self.env = env
        self.target_return = env.target_return

        self.used_time = None
        self.start_time = time.time()
        self.eval_time = -1  # a early time
        print(f"{'ID':>2}  {'Step':>8}  {'MaxR':>8} |"
              f"{'avgR':>8}  {'stdR':>8}   {'objA':>8}  {'objC':>8} |"
              f"{'avgS':>6}  {'stdS':>4}")

    def evaluate_save(self, act, steps, obj_a, obj_c) -> bool:
        self.total_step += steps  # update total training steps

        if time.time() - self.eval_time > self.eval_gap:
            self.eval_time = time.time()

            rewards_steps_list = [get_episode_return(self.env, act, self.device) for _ in range(self.eval_times1)]
            r_avg, r_std, s_avg, s_std = self.get_r_avg_std_s_avg_std(rewards_steps_list)

            if r_avg > self.r_max:  # evaluate actor twice to save CPU Usage and keep precision
                rewards_steps_list += [get_episode_return(self.env, act, self.device)
                                       for _ in range(self.eval_times2 - self.eval_times1)]
                r_avg, r_std, s_avg, s_std = self.get_r_avg_std_s_avg_std(rewards_steps_list)
            if r_avg > self.r_max:  # save checkpoint with highest episode return
                self.r_max = r_avg  # update max reward (episode return)

                '''save actor.pth'''
                act_save_path = f'{self.cwd}/actor.pth'
                torch.save(act.state_dict(), act_save_path)
                print(f"{self.agent_id:<2}  {self.total_step:8.2e}  {self.r_max:8.2f} |")  # save policy and print

            self.recorder.append((self.total_step, r_avg, r_std, obj_a, obj_c))  # update recorder

            if_reach_goal = bool(self.r_max > self.target_return)  # check if_reach_goal
            if if_reach_goal and self.used_time is None:
                self.used_time = int(time.time() - self.start_time)
                print(f"{'ID':>2}  {'Step':>8}  {'TargetR':>8} |"
                      f"{'avgR':>8}  {'stdR':>8}   {'UsedTime':>8}  ########\n"
                      f"{self.agent_id:<2}  {self.total_step:8.2e}  {self.target_return:8.2f} |"
                      f"{r_avg:8.2f}  {r_std:8.2f}   {self.used_time:>8}  ########")

            print(f"{self.agent_id:<2}  {self.total_step:8.2e}  {self.r_max:8.2f} |"
                  f"{r_avg:8.2f}  {r_std:8.2f}   {obj_a:8.2f}  {obj_c:8.2f} |"
                  f"{s_avg:6.0f}  {s_std:4.0f}")
        else:
            if_reach_goal = False
        return if_reach_goal

    def draw_plot(self):
        if len(self.recorder) == 0:
            print("| save_npy_draw_plot() WARNNING: len(self.recorder)==0")
            return None

        '''convert to array and save as npy'''
        np.save('%s/recorder.npy' % self.cwd, self.recorder)

        '''draw plot and save as png'''
        train_time = int(time.time() - self.start_time)
        total_step = int(self.recorder[-1][0])
        save_title = f"plot_step_time_maxR_{int(total_step)}_{int(train_time)}_{self.r_max:.3f}"

        save_learning_curve(self.recorder, self.cwd, save_title)

    @staticmethod
    def get_r_avg_std_s_avg_std(rewards_steps_list):
        rewards_steps_ary = np.array(rewards_steps_list)
        r_avg, s_avg = rewards_steps_ary.mean(axis=0)  # average of episode return and episode step
        r_std, s_std = rewards_steps_ary.std(axis=0)  # standard dev. of episode return and episode step
        return r_avg, r_std, s_avg, s_std

def get_episode_return(env, act, device) -> (float, int):
    episode_return = 0.0  # sum of rewards in an episode
    episode_step = 1
    max_step = env.max_step
    if_discrete = env.if_discrete

    state = env.reset()
    for episode_step in range(max_step):
        s_tensor = torch.as_tensor((state,), device=device)
        a_tensor = act(s_tensor)
        if if_discrete:
            a_tensor = a_tensor.argmax(dim=1)
        action = a_tensor.cpu().numpy()[0]  # not need detach(), because with torch.no_grad() outside
        state, reward, done, _ = env.step(action)
        episode_return += reward
        if done:
            break
    episode_return = getattr(env, 'episode_return', episode_return)
    return episode_return, episode_step + 1

def save_learning_curve(recorder, cwd='.', save_title='learning curve'):
    recorder = np.array(recorder)  # recorder_ary.append((self.total_step, r_avg, r_std, obj_a, obj_c))
    steps = recorder[:, 0]  # x-axis is training steps
    r_avg = recorder[:, 1]
    r_std = recorder[:, 2]
    obj_a = recorder[:, 3]
    obj_c = recorder[:, 4]

    '''plot subplots'''
    import matplotlib as mpl
    mpl.use('Agg')
    """Generating matplotlib graphs without a running X server [duplicate]
    write `mpl.use('Agg')` before `import matplotlib.pyplot as plt`
    https://stackoverflow.com/a/4935945/9293137
    """
    import matplotlib.pyplot as plt
    fig, axs = plt.subplots(2)

    axs0 = axs[0]
    axs0.cla()
    color0 = 'lightcoral'
    axs0.set_xlabel('Total Steps')
    axs0.set_ylabel('Episode Return')
    axs0.plot(steps, r_avg, label='Episode Return', color=color0)
    axs0.fill_between(steps, r_avg - r_std, r_avg + r_std, facecolor=color0, alpha=0.3)

    ax11 = axs[1]
    ax11.cla()
    color11 = 'royalblue'
    axs0.set_xlabel('Total Steps')
    ax11.set_ylabel('objA', color=color11)
    ax11.plot(steps, obj_a, label='objA', color=color11)
    ax11.tick_params(axis='y', labelcolor=color11)

    ax12 = axs[1].twinx()
    color12 = 'darkcyan'
    ax12.set_ylabel('objC', color=color12)
    ax12.fill_between(steps, obj_c, facecolor=color12, alpha=0.2, )
    ax12.tick_params(axis='y', labelcolor=color12)

    '''plot save'''
    plt.title(save_title, y=2.3)
    plt.savefig(f"{cwd}/plot_learning_curve.jpg")
    plt.close('all')  # avoiding warning about too many open figures, rcParam `figure.max_open_warning`
    # plt.show()  # if use `mpl.use('Agg')` to draw figures without GUI, then plt can't plt.show()

def explore_before_training(env, buffer, target_step, reward_scale, gamma) -> int:
    # just for off-policy. Because on-policy don't explore before training.
    if_discrete = env.if_discrete
    action_dim = env.action_dim

    state = env.reset()
    steps = 0

    while steps < target_step:
        action = rd.randint(action_dim) if if_discrete else rd.uniform(-1, 1, size=action_dim)
        next_state, reward, done, _ = env.step(action)
        steps += 1

        scaled_reward = reward * reward_scale
        mask = 0.0 if done else gamma
        other = (scaled_reward, mask, action) if if_discrete else (scaled_reward, mask, *action)
        buffer.append_buffer(state, other)

        state = env.reset() if done else next_state
    return steps


from BuildTree import *
from collections import deque
############### Inorder ###############
# (of a BST it becomes sorted)
arr="4,2,1,N,N,3,N,N,5,N,N" # Preorder Representation
root = deserialize(arr)
stack=[]
while stack or root:
    while root:
        stack.append(root)
        root=root.left
    root = stack.pop()
    print(root.val)
    root=root.right

############### Post Order ###############
print("post order")
arr="3,9,N,N,20,15,N,N,7,N,N"
root = deserialize(arr)
stack=[]
while root or stack:
    if root:
        stack.append(root)
        root=root.left
    else:
        tmp = stack[-1].right
        if tmp:
            root = tmp
        else:
            tmp = stack.pop()
            print(tmp.val)
            while stack and tmp==stack[-1].right:
                tmp=stack.pop()
                print(tmp.val)

############### Level Order ###############
print("level order")
# arr="4,2,1,N,N,3,N,N,5,N,N"
arr="3,9,N,N,20,15,N,N,7,N,N"
root = deserialize(arr)
q=deque([root])
while q:
    print([a.val for a in list(q)])
    for i in range(len(q)):
        node = q.popleft()
        # print(node.val)
        if node.left:
            q.append(node.left)
        if node.right:
            q.append(node.right)


from flask import Flask, request, jsonify
from pydub import AudioSegment
from google.cloud import storage
import os
storage_client = storage.Client()
#bucket_origem = storage_client.get_bucket("catalobyte-output")
bucket_destino = storage_client.get_bucket("catalobyte-play")

'''
trigger: function 'queue-to-speech'
task: 'output-to-cut'

vai receber mensagem de uma fila

vai lê o bucket 'catalobyte-output'
vai escrever no 'catalobyte-play/full/' e 'catalobyte-play/pieces/'
vai apagar do 'catalobyte-output'

um arquivo grande (FLAC com 2 horas e meia) deve durar até uns 2 minutos pra fazer tudo

https://cut-play-iafcvqe73a-ue.a.run.app
'''

app = Flask(__name__)

@app.route("/", methods=["POST"])
def receive():

    data = request.get_json()
    gs_uri = data['gs_uri'] # uri original do arquivo em 'catalobyte-input'
    #gs_uri = 'gs://catalobyte-output/VJaYYVBO4mOb2Yf9y8Nx3VR2Rfz2/a5p7k0r7a8/1158732356/1628633099505.flac'
    path_div = gs_uri.split('/')
    fire_user_uid = path_div[3]
    manticore_id = path_div[4]
    uid_file = path_div[5]
    full_file_name = path_div[6]
    remov_ext = full_file_name.split('.')
    name_file = remov_ext[0]

    tempfile = full_file_name
    #destname = name_file+".aac"
    destname = name_file+".mp3"

    if not os.path.exists(tempfile):
        os.mknod(tempfile)
    with open(tempfile, 'wb+') as file_obj:
        storage_client.download_blob_to_file(gs_uri, file_obj)

    a = AudioSegment.from_file(tempfile)
    a.export(destname, bitrate="30k", format="mp3")
    #a.export(destname, bitrate="30k", format="adts")

    blob = bucket_destino.blob(fire_user_uid+'/'+manticore_id+'/'+uid_file+'/full/'+destname)
    blob.upload_from_filename(destname)

    '''
    gera os 'pices' de audio utilizados na pesquisa Manticore
    dura = int(round(a.duration_seconds))

    def chunks(lst, n):
        """Yield successive n-sized chunks from lst."""
        for i in range(0, len(lst), n):
            yield lst[i:i + n]

    rr = list(chunks(range(0, dura), 10))

    for r in rr:
        rangeList = r
        frs = rangeList[0]
        lst = (rangeList[len(rangeList)-1:][0]+1)
        f_frs = (frs*1000)
        f_lst = (lst*1000)
        slice = a[f_frs:f_lst]
        namef = f'{frs}.aac'
        slice.export(namef, bitrate="30k", format="adts")
        blob_f = bucket_destino.blob(fire_user_uid+'/'+manticore_id+'/'+uid_file+'/pieces/'+namef)
        blob_f.upload_from_filename(namef)

        # catalobyte-output/VJaYYVBO4mOb2Yf9y8Nx3VR2Rfz2/a5p7k0r7a8/1158732356
        #bucket_origem.delete_blob(fire_user_uid+'/'+manticore_id+'/'+uid_file+'/')

    '''
    return "ok"

if __name__ == "__main__":
    # Used when running locally only. When deploying to Cloud Run,
    # a webserver process such as Gunicorn will serve the app.
    app.run(host="localhost", port=8080, debug=True)


import numpy as np
import matplotlib.pyplot as plt
import imageio

#https://medium.com/swlh/python-animated-images-6a85b9b68f86
#pip install imageio
#pip install imageio-ffmpeg
#pip install pygifsicle

image_path = './img_list/'

x=np.arange(0,1,0.001)
for i in range(10):
	plt.clf()
	plt.plot(x,x*float(i),label='y='+str(i)+'*x')
	plt.ylim(0,10)
	plt.legend()
	plt.savefig(image_path+str(i)+'.png')

image_list = []
for i in range(10):
    image_list.append(imageio.imread(image_path+str(i)+'.png'))

imageio.mimwrite(image_path+'animated_from_images.gif', image_list)


"""
This module features a class that extracts a power spectrogram from a
signal.
"""

import logging

import numpy as np

from praudio.transforms.transform import TransformType, Transform
from praudio.transforms.stft import STFT
from praudio.io.signal import Signal

logger = logging.getLogger(__name__)

class PowerSpectrogram(Transform):
    """This class extracts a power spectrogram from a signal. It's a
    concrete Transform.

    Attributes:
        - power: Power to which we raise the spectrum. Default is 2
        - stft: STFT object used to extract Short-Time Fourier Transform
    """

    def __init__(self,
                 frame_length: int = 2048,
                 hop_length: int = 1024,
                 win_length: int = 2048,
                 window: str = "hann",
                 power: int = 2):
        super().__init__(TransformType.POWERSPECTROGRAM)
        self.power = power
        self.stft = STFT(frame_length,
                         hop_length,
                         win_length,
                         window)

    def process(self, signal: Signal) -> Signal:
        """Extract power spectrogram from waveform and modify signal.
        First we extract the STFT. Then, we apply the absolute value and
        raise it to the target power.

        :param signal: Signal object. Note: this transforms works only with
            waveform data

        :return: Modified signal
        """
        signal = self.stft.process(signal)
        signal.name = self._prepend_transform_name(signal.name)
        signal.data = self._raise_to_power(np.abs(signal.data))
        logger.info("Applied %s to %s", self.name.value, signal.file)
        return signal

    def _raise_to_power(self, array: np.ndarray):
        return array ** self.power


import numpy as np
import matplotlib.pyplot as plt
from readdata import read as rd
from pydmd import HODMD
def fuc(x):
    fucresult = np.cos(x)*np.sin(np.cos(x))+np.cos(x*.2)*np.sin(x)+x*(25-x)/100
    return fucresult
import plotcheck
#just test
#path = "/home/bwei/PycharmProjects/data lib/long wind"
#windset = rd(path)
#name = raw_input('the name of data set?')
x = np.linspace(0, 30, 128)
realwindset = fuc(x)
realwindset.shape = (len(realwindset),)
#realwindset = realwindset + np.random.rand(128)
#x = np.linspace(1, len(realwindset), len(realwindset))
hodmd = HODMD(svd_rank=0, exact=True, opt=True, d=58).fit(realwindset)
hodmd.reconstructed_data.shape
hodmd.plot_eigs()
hodmd.original_time['dt'] = hodmd.dmd_time['dt'] = x[1] - x[0]
hodmd.original_time['t0'] = hodmd.dmd_time['t0'] = x[0]
hodmd.original_time['tend'] = hodmd.dmd_time['tend'] = x[-1]

plt.plot(hodmd.original_timesteps, realwindset, '.', label='realwindset')
plt.plot(x, realwindset, '-', label='original function')
plt.plot(hodmd.dmd_timesteps, hodmd.reconstructed_data[0].real, '--', label='DMD output')
plt.legend()
plt.show()

extrap = 100
extrapx = np.linspace(0, extrap, extrap*20)
original_fuc = fuc(extrapx)

hodmd.dmd_time['tend'] = extrap
fig = plt.figure(figsize=(15, 5))
plt.plot(hodmd.original_timesteps, realwindset, '.', label='snapshots')
plt.plot(extrapx, original_fuc, '-', label='original function')
plt.plot(hodmd.dmd_timesteps, hodmd.reconstructed_data[0].real, '--', label='DMD output')
plt.legend()
plt.show()
reconstruct = np.dot(hodmd.modes, hodmd.dynamics)
plotcheck.pl(np.transpose(reconstruct))


#1123.py
import cv2
import numpy as np
import random

#1
WIDTH = 92
HEIGHT = 112
def load_face(filename='./data/faces.csv', test_ratio=0.2):
    file = open(filename, 'r')
    lines = file.readlines()

    N = len(lines)
    faces = np.empty((N, WIDTH*HEIGHT), dtype=np.uint8 )
    labels = np.empty(N, dtype = np.int32)
    for i, line in enumerate(lines):
        filename, label = line.strip().split(';')
        labels[i] = int(label)
        img = cv2.imread(filename, cv2.IMREAD_GRAYSCALE)
        faces[i, :] = img.flatten()

# shuffling and seperate train and test data
    indices = list(range(N))
    random.seed(1) # same random sequences, so the same result
    random.shuffle(indices)
    shuffle_faces = faces[indices]
    shuffle_labels = labels[indices]

    test_size = int(test_ratio*N)

    test_faces = shuffle_faces[:test_size]
    test_labels = shuffle_labels[:test_size]

    train_faces = shuffle_faces[test_size:]
    train_labels = shuffle_labels[test_size:]
    return train_faces, train_labels, test_faces, test_labels

train_faces, train_labels, test_faces, test_labels = load_face()
print('train_faces.shape=',  train_faces.shape)
print('train_labels.shape=', train_labels.shape)
print('test_faces.shape=',   test_faces.shape)
print('test_labels.shape=',  test_labels.shape)

#2
N = 80
mean, eigenFace = cv2.PCACompute(train_faces, mean=None, maxComponents=N)
print('mean.shape = ', mean.shape)
print('eigenFace.shape=',  eigenFace.shape)

Y =cv2.PCAProject(train_faces, mean, eigenFace) # train result
print('Y.shape=', Y.shape)

#3: display eigen Face
dst = np.zeros((8*HEIGHT, 10*WIDTH), dtype=np.uint8)
for i in range(N):
  x = i%10
  y = i//10
  x1 = x*WIDTH
  y1 = y*HEIGHT
  x2 = x1+WIDTH
  y2 = y1+HEIGHT

  img = eigenFace[i].reshape(HEIGHT, WIDTH)
  dst[y1:y2, x1:x2] = cv2.normalize(img,None,0,255,cv2.NORM_MINMAX)
cv2.imshow('eigenFace %s'%N, dst)

#4: predict test_faces using  y = eigenFace(face - mean) and min distance
correct_count = 0
for i, face in enumerate(test_faces):
    y =cv2.PCAProject(face.reshape(1,-1), mean, eigenFace)
    dist=np.sqrt(np.sum((Y-y)**2,axis=1))
    k = np.argmin(dist)
    predict_label = train_labels[k]

    if test_labels[i]== predict_label:
        correct_count+= 1
    print('test_labels={}: predicted:{}'.format(
                     test_labels[i], predict_label))

accuracy = correct_count / float(len(test_faces))
print('accuracy=', accuracy)


import pandas as pd
from sklearn.model_selection import train_test_split
import math
import operator

source_data= pd.read_csv('..\\mydata\\iris.data',header=None)

# print(source_data)

data = source_data[list(range(5))]

# print(data.shape)

train_data,test_data, = train_test_split(data,test_size=0.5)
train_data = train_data.values
test_data = test_data.values

# print(train_data.values[0][0])
# print(test_data.values)

def euclideanDistance(data1,data2,length):
    distance = 0
    for i in range(length):
        distance += pow((data1[i]-data2[i]),2)
    return math.sqrt(distance)

# def test1():
#     data1 = [2, 2, 2, 'a']
#     data2 = [4, 4, 4, 'b']
#     distance = euclideanDistance(data1, data2, 3)
#     print ('Distance: ' + repr(distance))

def findNeighbors(traindata,testdata,k):
    distance = []
    length = len(testdata) -1
    for i in range(len(traindata)):
        dist = euclideanDistance(testdata,traindata[i],length)
        distance.append((traindata[i],dist))
    distance.sort(key = operator.itemgetter(1))
    neighbors = []
    for x in range(k):
        neighbors.append(distance[x][0])
    return neighbors

def getResponse(ngber):
    classVotes = {}
    for x in range(len(ngber)):
        response = ngber[x][-1]
        if response in classVotes:
            classVotes[response] += 1
        else:
            classVotes[response] =1
    sortedVotes = sorted(classVotes.items(),key=operator.itemgetter(1),reverse=True)
    return sortedVotes[0][0]

def getAccuracy(test,predic):
    correct = 0;
    for x in range(len(test)):
        if test[x][-1] is predic[x]:
            correct += 1
        else:
            print("-----------> actul:"+test[x][-1]+" and predic:"+predic[x])
    return (correct/float(len(test)))*100

if __name__ == '__main__':
    prediction = []
    k = 3
    for x in range(len(test_data)):
        neighbors = findNeighbors(train_data,test_data[x],k)
        res = getResponse(neighbors)
        prediction.append(res)
        print('>>>> predicted=' + repr(res) + ', actual=' + repr(test_data[x][-1]))
    accuracy = getAccuracy(test_data, prediction)
    print('Accuracy: ' + repr(accuracy) + '%')


# Warning: May need to be started twice!
import requests
# import os
import time
import subprocess

p = subprocess.Popen(['run_controller.cmd'])
# Testing launch error catching
# if p.poll() is not None:
#     print ("failed to start!")
#     exit(1)
# else:
#     p = subprocess.Popen(['run_controller.cmd'])

time.sleep(5)

def Axis1SetZero():
    payload = {'Axis1SetZero': False}
    SetZero1 = requests.put('http://192.168.0.105/kas/plcvariables?format=text',data=payload)
    payload = {'Axis1SetZero': True}
    SetZero1 = requests.put('http://192.168.0.105/kas/plcvariables?format=text', data=payload)
    return SetZero1.status_code

def Axis2SetZero():
    payload = {'Axis2SetZero': False}
    SetZero2 = requests.put('http://192.168.0.105/kas/plcvariables?format=text',data=payload)
    payload = {'Axis2SetZero': True}
    SetZero2 = requests.put('http://192.168.0.105/kas/plcvariables?format=text', data=payload)
    return SetZero2.status_code

def ReadActualPosition12():
    ActualPos12 = requests.get('http://192.168.0.105/kas/plcvariables?variables=ActPos1,ActPos2&format=text')
    return ActualPos12.text  # two values separated with comma

# Returns only status codes; 200 - normal
def EnableMotors(retry_num = 3):
    retry_attempts = 0
    try:
        payload = "OpenButton=False"
        EnableMotorsS1 = requests.put('http://192.168.0.105/kas/plcvariables?format=text', data=payload)
        payload = "CloseButton=True"
        EnableMotorsS2 = requests.put('http://192.168.0.105/kas/plcvariables?format=text', data=payload)
        Stat1Enabled = requests.get('http://192.168.0.105/kas/plcvariables?variables=Stat1Enabled&format=text')
        Stat2Enabled = requests.get('http://192.168.0.105/kas/plcvariables?variables=Stat2Enabled&format=text')
        return EnableMotorsS1.status_code, EnableMotorsS2.status_code, Stat1Enabled.text, Stat2Enabled.text
    except:
        print("trying again")
        if (retry_attempts <= retry_num):
            time.sleep(retry_attempts)
            retry_attempts += 1
            EnableMotors()
        else:
            print ("Failed to enable motors")
            return False

# Returns only status codes; 200 - normal
def DisableMotors():
    payload = "CloseButton=False"
    EnableMotorsS1 = requests.put('http://192.168.0.105/kas/plcvariables?format=text',data=payload)
    payload = "OpenButton=True"
    EnableMotorsS2 = requests.put('http://192.168.0.105/kas/plcvariables?format=text',data=payload)
    Stat1Enabled = requests.get('http://192.168.0.105/kas/plcvariables?variables=Stat1Enabled&format=text')
    Stat2Enabled = requests.get('http://192.168.0.105/kas/plcvariables?variables=Stat2Enabled&format=text')
    return EnableMotorsS1.status_code, EnableMotorsS2.status_code, Stat1Enabled.text, Stat2Enabled.text

def checkMotorReadiness1():
    Stat1Enabled = requests.get('http://192.168.0.105/kas/plcvariables?variables=Stat1Enabled&format=text')
    #print(Stat1Enabled.text)
    return Stat1Enabled.text  # staus code; 200 - normal

def checkMotorReadiness2():
    Stat2Enabled = requests.get('http://192.168.0.105/kas/plcvariables?variables=Stat1Enabled&format=text')
    #print(Stat2Enabled.text)
    return Stat2Enabled.text  # staus code; 200 - normal

# Change Axis1 parameters
def Axis1Parameters(angle, velocity, acceleration):
    payload1 = {'Axis1RelDistance': angle}
    payload2 = {'Axis1Vel': velocity}
    payload3 = {'Axis1Accel': acceleration}
    r1 = requests.put('http://192.168.0.105/kas/plcvariables?format=text',data=payload1)
    r2 = requests.put('http://192.168.0.105/kas/plcvariables?format=text', data=payload2)
    r2 = requests.put('http://192.168.0.105/kas/plcvariables?format=text', data=payload3)
    s1 = requests.get('http://192.168.0.105/kas/plcvariables?variables=Axis1RelDistance&format=text')
    s2 = requests.get('http://192.168.0.105/kas/plcvariables?variables=Axis1Vel&format=text')
    s3 = requests.get('http://192.168.0.105/kas/plcvariables?variables=Axis1Accel&format=text')
    if angle == float(s1.text):
        s1 = 1
    else:
        s1 = 0
    if velocity == float(s2.text):
        s2 = 1
    else:
        s2 = 0
    if acceleration == float(s3.text):
        s3 = 1
    else:
        s3 = 0
    return s1, s2, s3  # statuses: 1 - normal, 0 - fault

# Change Axis2 parameters
def Axis2Parameters(angle, velocity, acceleration):
    payload1 = {'Axis2RelDistance': angle}
    payload2 = {'Axis2Vel': velocity}
    payload3 = {'Axis2Accel': acceleration}
    r1 = requests.put('http://192.168.0.105/kas/plcvariables?format=text',data=payload1)
    r2 = requests.put('http://192.168.0.105/kas/plcvariables?format=text', data=payload2)
    r2 = requests.put('http://192.168.0.105/kas/plcvariables?format=text', data=payload3)
    s1 = requests.get('http://192.168.0.105/kas/plcvariables?variables=Axis2RelDistance&format=text')
    s2 = requests.get('http://192.168.0.105/kas/plcvariables?variables=Axis2Vel&format=text')
    s3 = requests.get('http://192.168.0.105/kas/plcvariables?variables=Axis2Accel&format=text')
    if angle == float(s1.text):
        s1 = 1
    else:
        s1 = 0
    if velocity == float(s2.text):
        s2 = 1
    else:
        s2 = 0
    if acceleration == float(s3.text):
        s3 = 1
    else:
        s3 = 0
    return s1, s2, s3  # statuses: 1 - normal, 0 - fault

def runMotors(axis):
    if axis == 1:
        payload = {'TestDrive1': True}
        DriveMotors = requests.put('http://192.168.0.105/kas/plcvariables?format=text', data=payload)
    elif axis == 2:
        payload = {'TestDrive2': True}
        DriveMotors = requests.put('http://192.168.0.105/kas/plcvariables?format=text', data=payload)
    return DriveMotors.status_code

def positionWaiter(timeout, threshold, rel_motion, prev_angle):
    # for i in range(0,10):
    print('control = ', float((ReadActualPosition12()).split(",")[0]))
    print ("outcome", ((abs(float((ReadActualPosition12()).split(",")[0])) - (prev_angle + rel_motion))))
    while ((abs(float((ReadActualPosition12()).split(",")[0]) - (prev_angle + rel_motion))) > threshold):
        actual_angle = ReadActualPosition12()
        time.sleep(timeout)
        print(actual_angle)

    print("match!")
    return True

# Motor tests
EnableMotors()
time.sleep(2)
Axis1SetZero()
Axis2SetZero()
time.sleep(2)

or_reading = ReadActualPosition12()
print("Initial position is: ", or_reading)
prev_angle = float((or_reading).split(",")[0])
print(prev_angle)
# Go to actual angle required
angle1 = 360.0  # dgrees
velocity = 150.0  # degree/second
acceleration = 600.0  # degree/second^2
#print(Axis2Parameters(angle2, velocity, acceleration))
print(Axis1Parameters(angle1, velocity, acceleration))

print(runMotors(1))
#print(runMotors(2))
#time.sleep(5)
positionWaiter(0.1, 0.5, angle1, prev_angle)

DisableMotors()


import time, operator, os, sys, argparse, gzip
from subprocess import Popen, PIPE

def rev_comp(s): return s.translate(s.maketrans('ATGC','TACG'))[::-1]

class CrevisSeq:

	def __init__(self, args):

		self.LTR_seq = args.LTR_seq.upper()
		self.target_seq = args.target_seq.upper()
		self.input1 = args.input1
		self.input2 = args.input2
		self.output_name = args.output
		self.ref_dir = args.ref_dir
		self.cut_len = args.r
		self.indi_len = args.i
		self.exin_db = {}
		self.input_dict = {}
		self.indi_dict = {
			'0': [],
			'1': [],
			'2': [],
			'con': []
		}

	def find_target(self):

		print("Find appropriate position of target sequence in LTR sequence...")
		if self.LTR_seq.find(self.target_seq) != -1:
			self.sliced_seq = self.LTR_seq[self.LTR_seq.find(self.target_seq) + 16:]
			self.sliced_len = len(self.LTR_seq) - self.LTR_seq.find(self.target_seq) - 17
		elif self.LTR_seq.find(rev_comp(self.target_seq)) != -1:
			self.sliced_seq = self.LTR_seq[self.LTR_seq.find(rev_comp(self.target_seq)) + 5:]
			self.sliced_len = len(self.LTR_seq) - self.LTR_seq.find(rev_comp(self.target_seq)) - 6
		else:
			print("[Error] find target sequence !!")
			sys.exit()

		self.sliced_LTR = self.sliced_seq[1:]
		self.indi_0 = self.sliced_seq[:self.indi_len]
		self.indi_1 = self.sliced_seq[1:1 + self.indi_len]
		self.indi_2 = self.sliced_seq[2:2 + self.indi_len]
		self.indi_con = self.sliced_seq[2:7]

	def mk_indi_1mm_list(self):

		self.indi_dict['0'] = [self.indi_0]
		self.indi_dict['1'] = [self.indi_1]
		self.indi_dict['2'] = [self.indi_2]
		self.indi_dict['con'] = [self.indi_con]

		for i in range(self.indi_len):
			for nt1 in 'ATGC':
				self.indi_dict['0'].append(self.indi_0[:i] + nt1 + self.indi_0[i + 1:])
				self.indi_dict['1'].append(self.indi_1[:i] + nt1 + self.indi_1[i + 1:])
				self.indi_dict['2'].append(self.indi_2[:i] + nt1 + self.indi_2[i + 1:])

		for i in range(len(self.indi_con)):
			for nt1 in 'ATGC':
				self.indi_dict['con'].append(self.indi_con[:i] + nt1 + self.indi_con[i + 1:])

	def confirm_seq(self, input_name):

		amend_dic = {'0':1, '1':0, '2':2}
		t = time.time()
		pc = 0.1

		with open(input_name) as f:
			line_n = 0
			indi_cnt = 0

			for line in f:
				line_n += 1
				if line_n % 4 != 2: continue
				st = -1
				seq = line.strip()
				confirm_v = False

				for x in self.indi_dict['con']:
					if seq.find(x) != -1:
						confirm_v = True
						break

				if confirm_v == False:
					continue

				for n in ['1','0','2']:
					for indi in self.indi_dict[n]:
						st = seq.find(indi)
						if st != -1:
							indi_cnt += 1
							amend = 1 - amend_dic[n] + self.sliced_len
							subseq = seq[st + amend:st + amend + self.cut_len].replace('\n','')
							if len(subseq) != self.cut_len: break
							if subseq in self.input_dict.keys():
								self.input_dict[subseq] += 1
							else:
								self.input_dict[subseq] = 1
							break
					if st != -1: break

		print("Find appropriate sequence : {0} of {1} in {2}".format(indi_cnt, int(line_n/4), input_name))

	def collect_seq(self):

		print("Read file 1...")
		self.confirm_seq(self.input1)
		if (self.input2):
			print("Read file 2...")
			self.confirm_seq(self.input2)

		print("Sorting sequence...")
		self.sorted_tup = sorted(self.input_dict.items(), key=operator.itemgetter(1), reverse = True)

	def run_bwa(self):

		with open('./bwa/input.txt','w') as fw:
			for x, y in self.sorted_tup:
				fw.write('>{0}\n{1}\n'.format(x,x))

		print('Run bwa fastmap...')
		self.fastmap_res = Popen(['bwa', 'fastmap', '-l', str(self.cut_len-20), self.ref_dir, './bwa/input.txt'], stdout=PIPE).communicate()
		if str(self.fastmap_res[0]) == "b''":
			sys.exit()

	def DB_modeling(self):

		print('Build human gtf3 database...')
		exin_db = {}
		lencnt = 0

		for i in list(range(1, 23))+['X', 'Y']:
			exin_db[str(i)] = {}
		with gzip.open('./DB/exon_intron_v2.txt.gz') as f:
			for line in f:
				line_sp = line.decode("utf-8").strip().split()
				chro = line_sp[0]
				st = int(line_sp[7])
				ed = int(line_sp[8])
				a = st
				i_info = line_sp[0] + '\t' + line_sp[2] + '\t' + line_sp[3] + '\t' + line_sp[4] + '\t' + line_sp[5]
				if str(st)[:-3] == str(ed)[:-3]:
					M = int(a / 1000000)
					K = int(a / 1000)
					if K > 1000: K = int(str(K)[-3:])
					Bst = ('00' + str(st))[-3:]
					Bed = ('00' + str(ed))[-3:]
					if M not in exin_db[chro].keys():
						exin_db[chro][M] = {}
					if K not in exin_db[chro][M].keys():
						exin_db[chro][M][K] = {}
					exin_db[chro][M][K][Bst+';~;'+Bed+';'+str(lencnt)] = i_info
				else:
					while a < ed + 1000:
						M = int(a / 1000000)
						K = int(a / 1000)
						if K > 1000: K = int(str(K)[-3:])
						B = ('00' + str(a))[-3:]
						if M not in exin_db[chro].keys():
							exin_db[chro][M] = {}
						if K not in exin_db[chro][M].keys():
							exin_db[chro][M][K] = {}
						if a == st:
							exin_db[chro][M][K][B + ';+;;' + str(lencnt)] = i_info
						elif a >= ed:
							M = int(ed / 1000000)
							K = int(ed / 1000)
							if K > 1000: K = int(str(K)[-3:])
							B = ('00' + str(ed))[-3:]
							exin_db[chro][M][K][B + ';-;;' + str(lencnt)] = i_info
						else:
							exin_db[chro][M][K][B + ';-;;' + str(lencnt)] = i_info
						a += 1000
						lencnt += 1

		self.exin_db = exin_db

	def read_fastmap_results(self):

		fastmap_list = str(self.fastmap_res[0]).split('//')

		fastmap_dict = {}
		chromomap_dict = {}
		id_chromomap = 0

		for info in fastmap_list[:-1]:
			info = info.split('\\n')
			if info[0] == '': info = info[1:]
			seq = info[0].split('\\t')[1]
			if len(info) == 2:
				fastmap_dict[seq] = 'Can not find position'
				continue
			sub_info_dict = {}
			for sub_info in info[1:-1]:
				sub_info_sp = sub_info.split('\\t')
				if sub_info_sp[1] == '0':
					sub_info_dict[int(sub_info_sp[2]) - int(sub_info_sp[1])] = sub_info
			if len(sub_info_dict) == 0:
				fastmap_dict[seq] = 'Can not find position'
				continue
			info = sorted(sub_info_dict.items(), key=operator.itemgetter(0), reverse=True)[0][1]
			cut_len = int(info.split('\\t')[2])
			if info.split('\\t')[-1] == '*':
				fastmap_dict[seq] = 'fastmap find {0} sites; {1}'.format(info.split('\\t')[-2], seq[:cut_len])
			else:
				fastmap_dict[seq] = []
				chro_c = -1
				if len(info.split('\\t')[4:]) == 1:
					i = info.split('\\t')[4].split(':')
					chromomap_dict[str(id_chromomap)] = [i[0], str(abs(int(i[1]))), str(abs(int(i[1])) + int(info.split('\\t')[2]))]
					id_chromomap += 1
					chro_c = 1
				for i in info.split('\\t')[4:]:
					i = i.split(':')
					fastmap_dict[seq].append([i[0], i[1][:1], i[1][1:], seq[:cut_len], chro_c])

		self.fastmap_dict = fastmap_dict

	def analyze_fastmap_results(self):

		length_sorted_list = []
		for seq, info in self.fastmap_dict.items():
			yl = self.fastmap_dict[seq]
			if type(yl) != str and len(yl[0][3]) == self.cut_len:
				length_sorted_list.append(yl[0][3])
		consolidate_dict = {}
		for x, cnt in self.sorted_tup:
			yl = self.fastmap_dict[x]
			if type(yl) != str and len(yl[0][3]) != self.cut_len:
				c = 0
				seq = yl[0][3]
				for i in length_sorted_list:
					if i.find(seq) != -1:
						if i in consolidate_dict.keys():
							consolidate_dict[i] += cnt
						else:
							consolidate_dict[i] = cnt
						c = 1
						continue
				if c == 0:
					consolidate_dict[x] = cnt
			else:
				if x in consolidate_dict.keys():
					consolidate_dict[x] += cnt
				else:
					consolidate_dict[x] = cnt

		chromomap_dict = {}
		chromomap_id = 0
		with open(self.output_name+'.txt', 'w') as fw, open('{0}_chromomap.txt'.format(self.output_name[:self.output_name.rfind('.')]), 'w') as fwc:
			id_c = 0
			fw.write('ID\tsequence\tchromosome\tstart\tread\tgene\tclass\texon ID\n')
			fwc.write('ID\tsequence\tchromosome\tstart\tread\tgene\tclass\texon ID\n')
			for x, cnt in sorted(consolidate_dict.items(), key=operator.itemgetter(1), reverse=True):
				id_c += 1
				x = x.strip()
				yl = self.fastmap_dict[x]
				if type(yl) == str:
					if yl.find(';') != -1:
						yl = yl.split(';')
						fw.write('{0}\t{1}\t\t{2}\t{3}\n'.format(id_c, y[1], cnt, yl[0]))
					else:
						fw.write('{0}\t{1}\t\t{2}\t{3}\n'.format(id_c, x, cnt, yl))
					continue
				for y in yl:
					chromomap_dict[str(chromomap_id)] = [y[0], int(y[2]), int(y[2])+len(y[3])]
					chromomap_id += 1
					fw.write('{0}\t{1}\t{2}\t{3}\t{4}'.format(id_c, y[3], y[0], y[2], cnt))
					if y[4] == 1:
						fwc.write('{0}\t{1}\t{2}\t{3}\t{4}'.format(id_c, y[3], y[0], y[2], cnt))
					chro = y[0]
					pos = int(y[2])
					B = int(('00' + str(pos))[-3:])
					M = int(pos/ 1000000)
					K = int(('00'+str(int(pos / 1000)))[-3:])
					try:
						db = self.exin_db[chro][M][K]
					except:
						fw.write('\tNothing in database\n')
						if y[4] == 1: fwc.write('\tNothing in database')
						continue

					vv = 0
					sub_db = {}

					for i in db.keys():
						isp = i.split(';')
						v = 0
						if isp[1] == '=': v = 1
						elif int(isp[0]) <= B and isp[1] == '+': v = 1
						elif int(isp[0]) >= B and isp[1] == '-': v = 1
						elif isp[1] == '~':
							if int(isp[0]) <= B <= int(isp[3]): v = 1
						if v == 1:
							vv = 1
							dbl = db[i].split('\t')
							try:
								ssub_db = sub_db[dbl[1]]
								if dbl[2] not in ssub_db['chtype'].split('&'): ssub_db['chtype'] += '&' + dbl[2]
								if ssub_db['exin'] == 'intron' and dbl[3] == 'exon': ssub_db['exin'] = 'exon'
							except:
								sub_db[dbl[1]] = {'chtype': dbl[2], 'exin': dbl[3]}

					for gene, gene_list in sub_db.items():
						fw.write('\t{0}\t{1}\t{2}'.format(gene, gene_list['chtype'], gene_list['exin']))
						if y[4] == 1: fwc.write('\t{0}\t{1}\t{2}'.format(gene, gene_list['chtype'], gene_list['exin']))
					if vv == 0:
						fw.write('\tNothing in database\n')
						if y[4] == 1: fwc.write('\tNothing in datbase\n')
					else:
						fw.write('\n')
						if y[4] == 1: fwc.write('\n')

		self.chromomap_dict = chromomap_dict

	def make_chromomap(self):

		with open('{0}_chromomap.html'.format(self.output_name[:self.output_name.rfind('.')]),'w') as fw:
			s = '\n\t\n'
			s += "\n\n"
			fw.write(s)

def parse_args():

	parser = argparse.ArgumentParser()
	parser.add_argument("LTR_seq", type=str, help="full LTR sequence (5' to 3')")
	parser.add_argument("target_seq", type=str, help="target sequence with PAM (5' to 3')")
	parser.add_argument("input1", type=str, help="NGS result file name 1")
	parser.add_argument("-input2", type=str, help="NGS result file name 2")
	parser.add_argument("output", type=str, help="output file name")
	parser.add_argument("-r", type=int, help="length of expected genomic sequence", default=70)
	parser.add_argument("-i", type=int, help="length of indicator sequence", default=15)
	parser.add_argument("ref_dir", type=str, help="Directory of bwa index")

	return parser.parse_args()

def main():

	tt = time.time()

	c = CrevisSeq(parse_args())

	c.find_target()
	c.mk_indi_1mm_list()
	c.collect_seq()
	c.run_bwa()
	c.DB_modeling()
	c.read_fastmap_results()
	c.analyze_fastmap_results()
	c.make_chromomap()

if __name__ == '__main__':
	main()


from math import sin, cos, pi
from pygame import draw

class Spark:
    def __init__(self, pos, angle, speed, color=(255, 255, 255)):
        self.pos = list(pos)
        self.angle = angle
        self.speed = speed
        self.color = color

    def update(self):
        self.pos[0] += cos(self.angle) * self.speed
        self.pos[1] += sin(self.angle) * self.speed

        self.speed = max(0, self.speed - 0.1)

        return not self.speed

    def render(self, surf, offset=[0, 0]):
        render_points = [
            (self.pos[0] + cos(self.angle) * self.speed * 3 - offset[0], self.pos[1] + sin(self.angle) * self.speed * 3 - offset[1]),
            (self.pos[0] + cos(self.angle + pi * 0.5) * self.speed * 0.5 - offset[0], self.pos[1] + sin(self.angle + pi * 0.5) * self.speed * 0.5 - offset[1]),
            (self.pos[0] + cos(self.angle + pi) * self.speed * 0.5 - offset[0], self.pos[1] + sin(self.angle + pi) * self.speed * 0.5 - offset[1]),
            (self.pos[0] + cos(self.angle - pi * 0.5) * self.speed * 0.5 - offset[0], self.pos[1] + sin(self.angle - pi * 0.5) * self.speed * 0.5 - offset[1]),
        ]

        draw.polygon(surf, self.color, render_points)


import json
import os
import requests
import time
import urllib
import urllib.request
from flask import Flask, Response, request

app = Flask(__name__)

URLSCANIO_API_KEY = os.getenv('URLSCANIO_API_KEY')
SLEEP_TIME = os.getenv('SLEEP_TIME', default=3)
RETRY = os.getenv('RETRY', default=5)
print(URLSCANIO_API_KEY)

@app.route('/')
def hello():
    if 'url' not in request.args:
        return get_error_response("url parametresi verilmedi")

    url = request.args["url"]

    headers = {'API-Key': URLSCANIO_API_KEY, 'Content-Type': 'application/json'}
    data = {"url": url, "visibility": "public"}
    response = requests.post('https://urlscan.io/api/v1/scan/', headers=headers, data=json.dumps(data)).json()

    if 'api' not in response:
        return get_error_response(response['message'])

    data = json.dumps(response['api'])
    api_url = data[1:-1]

    for x in range(RETRY):
        time.sleep(SLEEP_TIME)

        try:
            json_data = urllib.request.urlopen(api_url)
            db = json.loads(json_data.read())

            return get_succes_response(db)
        except:
            pass

    return get_error_response("Server Zaman Aşımına Uğradı")

def get_error_response(msg):
    return Response('{"success": false, "message": "' + msg + '"}', mimetype='application/json')

def get_succes_response(data):
    return Response('{"success": true, "data": ' + json.dumps(data) + '}', mimetype='application/json')

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=8080)


from sklearn.metrics import confusion_matrix
import numpy
#import pdb
import math

class Metrics(object):
    """
    Contains evaluation metrics

    """
    def __init__(self, nclasses):
        self.nclasses= nclasses
        self.fp = 0
        self.fn = 0
        self.tp = 0
        self.cost = 0
        self.error = 0
        self.prec = 0
        self.rec = 0
        self.fmes = 0
        self.conf_mat= numpy.zeros([nclasses, nclasses], dtype= numpy.float32)

    def reset(self):
        self.fp = 0
        self.fn = 0
        self.tp = 0
        self.cost = 0
        self.error = 0
        self.prec = 0
        self.rec = 0
        self.fmes = 0
        self.conf_mat= numpy.zeros([self.nclasses, self.nclasses], dtype= numpy.float32)
        self.mean_iou_index= 0

    def update_metrics_batch(self, preds, y):
        error = 0
        cost = 0
        for i in range(preds.shape[0]):
            self.update_metrics(preds[i],y[i],error,cost)

    def update_metrics(self, preds, y, error, cost):
        self.error += error
        self.cost += cost
        cf_m= confusion_matrix(y.flatten(), preds.flatten(), range(0, self.nclasses))
        self.conf_mat+=cf_m

    def compute_rates(self, nonignore=None):
        perclass_tp = numpy.diagonal(self.conf_mat).astype(numpy.float32)
        if nonignore is not None:
            perclass_fp = self.conf_mat[nonignore,:].sum(axis=0) - perclass_tp #SUM(nji) - nii
        else:
            perclass_fp = self.conf_mat.sum(axis=0) - perclass_tp #SUM(nji) - nii

        perclass_fn = self.conf_mat.sum(axis=1) - perclass_tp

        if self.nclasses == 2:
            self.tp= perclass_tp[1]
            self.fp= perclass_fp[1]
            self.fn= perclass_fn[1]
        self.iou= perclass_tp/(perclass_fp+ perclass_tp+ perclass_fn)
        if nonignore is not None:
            self.iou= self.iou[nonignore]
        else:
            self.iou= self.iou[:-1]

        self.mean_iou_index= self.getScoreAverage(self.iou)

    def getScoreAverage(self, scoreList):
        validScores = 0
        scoreSum    = 0.0
        for score in scoreList:
            if not math.isnan(score):
                validScores += 1
                scoreSum += score
        if validScores == 0:
            return float('nan')
        return scoreSum / validScores

    def compute_final_metrics(self, nmini_batches, nonignore=None):
        self.compute_rates(nonignore)
        if not (self.tp==0 and (self.fp==0 or self.fn==0)):
            self.prec= numpy.true_divide(self.tp,self.tp+self.fp)
            self.rec= numpy.true_divide(self.tp, self.tp+self.fn)
            self.fmes= (2*self.rec*self.prec)/(self.rec+self.prec)
        self.error /= nmini_batches
        self.cost /= nmini_batches
        return self.mean_iou_index


import logging
import logging.config
import sys
from configparser import RawConfigParser
from io import StringIO

DEFAULT_CONFIG_FILE = '/etc/rkivas.conf'

DEFAULTS = """
[sources]

[backup]
filename-format = {source}/{timestamp:%Y-%m}/{source}-{timestamp:%Y%m%d_%H%M%S}-{hash}
hash-algorithm = md5
hash-length = 8

[backup-no-timestamp]
filename-format = {source}/unknown/{source}-{hash}
hash-algorithm = md5
hash-length = 16

[extension-map]
jpeg = jpg
tiff = tif

[extension-handlers]
jpg = exif
tif = exif
"""

class ConfigParser(RawConfigParser):

    def optionxform(self, optionstr):
        return optionstr

def load_config_files(opts):
    cfg = ConfigParser()
    cfg.read_file(StringIO(DEFAULTS))
    cfg.read(opts.config_file)
    return cfg

def add_default_opts(parser):
    parser.add_argument(
        '--config-file', default=DEFAULT_CONFIG_FILE,
        help='load a particular configuration file',
        metavar='FILE')
    parser.add_argument(
        '-L', '--logging',
        choices=['DEBUG', 'WARN', 'WARNING', 'INFO', 'ERROR',
                 'CRITICAL', 'FATAL'],
        help='log to stderr with the given LEVEL', metavar='LEVEL')
    parser.add_argument(
        '--debug-config', action='store_true',
        help='instead of running, output the combined configuration')
    parser.add_argument(
        '--dry-run', action='store_true',
        help="don't affect filesystem, just log what would have been done")

def config_logging(opts, cfg):
    if opts.logging:
        level = getattr(logging, opts.logging)
        logging.basicConfig(
            level=level,
            format='%(asctime)s %(levelname)s %(name)s - %(message)s',
        )
    elif (cfg.has_section('formatters') or
            cfg.has_section('handlers') or
            cfg.has_section('loggers') or
            cfg.has_section('logger_root')):
        tmp = StringIO()
        cfg.write(tmp)
        tmp.seek(0)
        logging.config.fileConfig(tmp, disable_existing_loggers=False)
    else:
        logging.basicConfig(
            level=logging.WARNING,
            format='%(levelname)s %(name)s - %(message)s',
        )
    if hasattr(logging, 'captureWarnings'):
        logging.captureWarnings(True)

def load_opts_into_cfg(opts, cfg, which):
    for section, options in which.items():
        for cfg_key, opt_key in options.items():
            value = getattr(opts, opt_key, None)
            if value is not None:
                cfg.set(section, cfg_key, str(value))

def load_common_config(opts):
    cfg = load_config_files(opts)
    if not opts.debug_config:
        config_logging(opts, cfg)
    load_opts_into_cfg(opts, cfg, {
        'backup': {
            'dry-run': 'dry_run',
        }
    })
    return cfg

def load_config(opts, opts_spec=None):
    cfg = load_common_config(opts)
    if opts_spec:
        load_opts_into_cfg(opts, cfg, opts_spec)
    if opts.debug_config:
        cfg.write(sys.stdout)
        sys.exit(0)
    return cfg


import cv2
import mediapipe as mp
import time

# to check frame rate
time.sleep(20)
cap = cv2.VideoCapture(0)

mpHands = mp.solutions.hands
hands = mpHands.Hands()  # parameters already difined
mpDrawLine = mp.solutions.drawing_utils  # to draw multiple lines
prevTime = 0
currTime = 0

while True:
    success, img = cap.read()
    imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    results = hands.process(imgRGB)  # processing rgb IMG
    # print(results.multi_hand_landmarks)

    # to check if we have multiple hand
    if results.multi_hand_landmarks:
        for handlms in results.multi_hand_landmarks:
            for id, lm in enumerate(handlms.landmark):
                # print(id,lm)
                # c-> channels of img
                height, width, c = img.shape
                # position of centre
                cx, cy = int(lm.x * width), int(lm.y * height)
                print(id, cx, cy)

                if id == 4:  # circle for id 1
                    cv2.circle(img, (cx, cy), 15, (255, 0, 255), cv2.FILLED)

            mpDrawLine.draw_landmarks(img, handlms, mpHands.HAND_CONNECTIONS)  # for single hand
            # mpHands.HAND_CONNECTIONS: for connection with dots

    # frame rate
    currTime = time.time()
    fps = 1 / (currTime - prevTime)
    prevTime = currTime
    # display on screen
    cv2.putText(img, str(int(fps)), (10, 70), cv2.FONT_HERSHEY_PLAIN, 3,
                (255, 0, 255), 3)
    # 3->thickness, (255,0,255)->color,
    # to capture img
    cv2.imshow("Image", img)
    cv2.waitKey(1)


import dash_core_components as dcc
import dash_html_components as html
import dash_bootstrap_components as dbc
from dash.dependencies import Input, Output, State
from dash.exceptions import PreventUpdate
import dash_daq as daq
import dash_leaflet as dl
from dash_extensions.javascript import Namespace
import dash_leaflet.express as dlx
from dash_extensions.javascript import arrow_function
import plotly.express as px
import plotly.graph_objects as go
import pandas as pd

from app import app, db

### ----------------------------- SETUP ---------------------------------- ###

# Load data
cosmos_targets = pd.read_sql_table('view_cosmos_targets', con=db.engine)
project_outcomes = pd.read_sql_table('view_project_outcomes', con=db.engine)

# prep data for chart
df = project_outcomes.merge(
    cosmos_targets[['cpa_name', 'long_form', 'quantity']],
    how='right',
    on=['cpa_name', 'long_form'])

df['contribution'] = df['total_outcomes'] / df['quantity']

def plot_cosmos(df):

    fig = go.Figure()
    bar_width = .7

    fig.add_traces(go.Bar(
            x=df['contribution'],
            y=df['long_form'],
            name='Outcomes',
            orientation='h',
            width=bar_width
        ))

    fig.update_layout(barmode='stack')

    return fig

def get_project_points(cpa: str):
    df_cpa = project_outcomes[project_outcomes['cpa_name'] == cpa] if cpa else project_outcomes
    df_cpa = df_cpa[['project_nickname', 'cpa_name', 'coordinate_x', 'coordinate_y']].dropna().drop_duplicates()
    dicts = df_cpa.to_dict('rows')
    for item in dicts:
        item['tooltip'] = item['project_nickname']
        item['popup'] = item['cpa_name']
    geojson = dlx.dicts_to_geojson(dicts, lat='coordinate_x', lon="coordinate_y")
    return dlx.geojson_to_geobuf(geojson)

### ----------------------------- LAYOUT --------------------------------- ###

### SELECT REGION ###
_regions = cosmos_targets['cpa_name'].unique()
_region_options = [{'label': region, 'value': region} for region in _regions]

select_region = dcc.Dropdown(
    id='select-region',
    placeholder='Select a CPA',
    options=_region_options
)

### COSMOS CHART ###
cosmos_chart = dcc.Graph(
    id='cosmos-chart',
    config = {
        'displayModeBar': 'hover',
        'doubleClick': False,
        'scrollZoom': False,
        'modeBarButtonsToRemove': [
            'zoom2d', 'select2d', 'lasso2d',
            'zoomIn2d', 'zoomOut2d', 'autoScale2d',
            'resetScale2d', 'hoverClosestCartesian',
            'hoverCompareCartesian'
            ],
        'displaylogo': False
    }
)

### DATE RANGE SLIDER ###
date_slider = daq.Slider(
    id='date-slider',
    min=5,
    max=15,
    handleLabel={"showCurrentValue": True, "label": "yr"},
    updatemode='mouseup',
    color='blue',
    size = 265*2,
    marks={'5': '5 yr', '10': '10 yr', '15': 'Beyond'}
)

### MAP ###

# set up scatter points
ns = Namespace('dlx', 'scatter')
geojson = dl.GeoJSON(data=get_project_points(None), id='geojson', format='geobuf',
                     zoomToBounds=True,
                     cluster=True,
                     zoomToBoundsOnClick=True,
                     superClusterOptions=dict(radius=150))

# add cpa boundaries
cpa_boundaries = dl.GeoJSON(id='rfmp',
                            url="/assets/cpa.json",
                            zoomToBounds=True,
                            hoverStyle=arrow_function(dict(weight=5, color='#666', dashArray='')))  # must be in assets folder

# create map
map = dl.Map(id='map',
             children= [dl.TileLayer(), cpa_boundaries, geojson],
             style={'width': '100%', 'height': '50vh', 'margin': "auto", "display": "block"})

### PROJECT WINDOW ###
_projects = [
    'Project A',
    'Project B',
    'Project C',
    'Project D'
]

projects_card = dbc.Card(
    dbc.CardBody(
        dcc.Checklist(
            options = [{'label': "\t" + project, 'value': project} for project in _projects],
            labelStyle={'display': 'block'}
        )
    )
)

### LAYOUT ###
layout = html.Div(
    dbc.Row(
        [
            dbc.Col([
                select_region,
                cosmos_chart,
                dbc.Row(date_slider, justify='center', align='center')
                ], width={'size': 7, 'offset': 1}
            ),
            dbc.Col([
                map
                ], width=3
            )
        ]
    )
)

### ---------------------------- CALLBACKS ------------------------------- ###

@app.callback(
    Output('cosmos-chart', 'figure'),
    [Input('select-region', 'value')]
)
def update_chart(region):
    if region:
        filt = df['cpa_name'] == region
        return plot_cosmos(df[filt])
    else:
        return plot_cosmos(df)


from django.conf import settings
from django.db import models

from .signals import user_logged_in
from .utils import get_client_city_data, get_client_ip

class UserSessionManager(models.Manager):
    def create_new(self, user, session_key=None, ip_address=None, city_data=None):
        session_new = self.model()
        session_new.user = user
        session_new.session_key = session_key
        if ip_address is not None:
            session_new.ip_address = ip_address
            if city_data:
                session_new.city_data = city_data
                try:
                    city = city_data['city']
                except:
                    city = None
                session_new.city = city
                try:
                    country = city_data['country_name']
                except:
                    country = None
            session_new.country = country
            session_new.save()
            return session_new
        return None

class UserSession(models.Model):
    user            = models.ForeignKey(settings.AUTH_USER_MODEL)
    session_key     = models.CharField(max_length=60, null=True, blank=True)
    ip_address      = models.GenericIPAddressField(null=True, blank=True)
    city_data       = models.TextField(null=True, blank=True)
    city            = models.CharField(max_length=120, null=True, blank=True)
    country         = models.CharField(max_length=120, null=True, blank=True)
    active          = models.BooleanField(default=True)
    timestamp       = models.DateTimeField(auto_now_add=True)

    objects = UserSessionManager()

    def __str__(self):
        city = self.city
        country = self.country
        if city and country:
            return f"{city}, {country}"
        elif city and not country:
            return f"{city}"
        elif country and not city:
            return f"{country}"
        return self.user.username

def user_logged_in_receiver(sender, request, *args, **kwargs):
    user = sender
    ip_address = get_client_ip(request)
    city_data = get_client_city_data(ip_address)
    request.session['CITY'] = str(city_data.get('city', 'New York'))
    session_key = request.session.session_key
    UserSession.objects.create_new(
                user=user,
                session_key=session_key,
                ip_address=ip_address,
                city_data=city_data
                )

user_logged_in.connect(user_logged_in_receiver)