from requests_html import HTMLSession
import os
from dotenv import load_dotenv
load_dotenv()

playlist_url = os.getenv('APPLE_MUSIC_PLAYLIST_URL')
storage_directory = os.getenv('STORAGE_DIRECTORY')
session = HTMLSession()
r = session.get(playlist_url)
cwd = os.path.abspath(__file__)

artist_item_data_attribute_name = 'data-test-song-artist-url'

tracklists_items = r.html.find('.tracklist-item--song .tracklist-item__text__headline')
artists_items = r.html.find(f'.tracklist-item--song [{artist_item_data_attribute_name}]')
tracklists_headlines = list(map(lambda tracklist_item: tracklist_item.text, tracklists_items))
artists_headlines = list(map(lambda artist_item: artist_item.attrs[artist_item_data_attribute_name], artists_items))

songs = [f'{artist} {tracklists_headlines[i]}' for i, artist in enumerate(artists_headlines)]

for song in songs:
  download_result = os.popen(f'instantmusic -p -s "{song}"').read()

  if storage_directory:
    file_name = download_result.split('[ffmpeg] Destination: ', 1)[1].split('\n', 1)[0]
    os.system(f'mv "./{file_name}" {storage_directory}/')


from github import Github, RateLimitExceededException
import os

from django.shortcuts import render
from django.http import Http404
from django.http import HttpResponse
from .ghproj import GithubProject

def index(request):
    reponame = os.environ.get("REPO_NAME")
    domain = request.GET.get('domain')
    if domain:
        return status(request, domain)
    try:
        context = { 'domains': GithubProject.list_domains(reponame) }
        return render(request, 'status/index.html', context)
    except RateLimitExceededException:
        raise Http404(f"Rate limit exceeded, sorry!")

def status(request, domain):
    reponame = os.environ.get("REPO_NAME")
    try:
        issues, ok = GithubProject.get_domain(reponame, domain)
        if issues or ok:
            context = {'reponame': reponame, 'domain': domain, 'issues': issues, 'ok': ok }
            return render(request, 'status/status.html', context)
        raise Http404(f"No such domain {domain}")
    except RateLimitExceededException:
        raise Http404(f"Rate limit exceeded, sorry!")


# 创建一个背景为蓝色的pygame窗口
import sys #使用其中工具退出游戏

import pygame

class Game:
    def __init__(self):
        pygame.init() # 初始化背景设置
        self.screen = pygame.display.set_mode((1920,1080)) #设置窗口
        self.bg_color = (0,104,120) #设置背景色
        pygame.display.set_caption("一个练习窗口") #设置标题

    def run_game(self):
        #游戏主循环
        while True:

          for event in pygame.event.get() : #监视事件
              if event.type == pygame.QUIT:  #按x就结束程序
                  sys.exit()

          self.screen.fill(self.bg_color) #绘制屏幕
          pygame.display.flip() #最近绘制的屏幕可见

# 创建实例
my_exercise=Game()
my_exercise.run_game()


import requests
from lxml import html as parser
import random

req = requests.Session()
user_agent_list = [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
]
user_agent = random.choice(user_agent_list)
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
    "Dnt": "1",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": user_agent,
}
req.headers = headers

def getSearchURL(search, page):
    baseURL = 'https://www.amazon.com/s?k={}&s=review-rank&page={}&language=es'
    search = search.replace(' ','+')
    return baseURL.format(search, page)

def getProductData(product_list):

    products = []
    for product in product_list:
        data = {}
        asin = product.get('data-asin')
        data['ASIN'] = asin

        img_list = product.xpath('//div[@data-asin="'+asin+'"]/div/span/div/div/div[2]/div[1]/div/div/span/a/div/img')
        if len(img_list)==0:
            continue
        data['IMG'] = img_list[0].get('src')

        price_list = product.xpath('//div[@data-asin="'+asin+'"]/div/span/div/div/div[2]/div[2]/div/div/div[3]/div[1]/div/div[1]/div/a/span/span[2]')
        if len(price_list)==0:
            continue
        data['PRICE'] = price_list[0].text_content()

        title_list = product.xpath('//div[@data-asin="'+asin+'"]/div/span/div/div/div[2]/div[2]/div/div/div[1]/h2/a/span/text()')
        if len(title_list)==0:
            continue
        data['TITLE'] = title_list[0]

        products += [data]
    return products

def getAllProducts(min_products, search):
    page = 0
    product_list = []
    while len(product_list)<min_products:
        page += 1
        url = getSearchURL(search, str(page))
        res = req.get(url)
        data = res.content
        html = parser.fromstring(data)
        products = html.xpath('//div[@data-asin]')
        product_list += getProductData(products)
        print(len(product_list))

    return product_list

if __name__ == '__main__':

    for product in products:
        print('---------------------------------------------')
        print('ASIM:    '+product['ASIN'])
        print('IMAGEN:  '+product['IMG'])
        print('TITLE:   '+product['TITLE'])
        print('PRICE:   '+product['PRICE'])

import json

def lambda_handler(event, context):
    search = event.get('search')
    count = event.get('count')

    products = getAllProducts(count, search)

    return {
        'statusCode': 200,
        'products': products
    }

    #first_image = asin_data[3].xpath('//span[@data-component-type="s-product-image"]').xpath('//a')


#!/usr/bin/env python
# -*- coding: utf -*-
import sys
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np

def main():
    if len(sys.argv) != 2:
        print('Usage: {} <log file>'.format(sys.argv[0]))
        sys.exit(1)
    throughputs = {}
    delays = {}
    with open(sys.argv[1]) as f:
        for l in f:
            if 'Window size' in l:
                current_size = int(l.split()[2])
                throughputs.setdefault(current_size, [])
                delays.setdefault(current_size, [])
            if 'throughput' in l:
                throughputs[current_size].append(float(l.split()[2]))
            elif 'signal delay' in l:
                delays[current_size].append(float(l.split()[4]))
    for window_size in throughputs:
        plt.scatter(
            delays[window_size],
            throughputs[window_size],
            marker='x',
            label=window_size)
    plt.gca().invert_xaxis()
    plt.ylabel('Throughput (Mbits/s)')
    plt.xlabel('Delay (ms)')
    plt.xscale('log')
    plt.title('Fixed window sizes')
    plt.legend()
    plt.savefig('exercise_a.png', bbox_inches='tight')

if __name__ == '__main__':
    main()


from collections import defaultdict
import heapq

#ADJの方が圧倒的に早い感
#(matの受け渡しが辛い?)

class Dijkstra:
    """
    dijkstraの最短経路問題を解く
    input:
        S: start node
        mat : mat[a][b] = edge_weight(a,b)
    """

    def __init__(self, start, num_node, mat):
        """
        mat: mat[a][b] = edge_weight(a,b)
        num_node: num of nodes
        dist: dist from start_node
        prev: 最短経路において、一つ前のノードを記憶。経路復元に使う?
        """
        self.start_node = start
        self.mat = mat
        self.num_node = num_node
        self.dist = [float('inf')]*num_node
        self.prev = defaultdict(lambda: None)

        # dist form S to S = 0
        self.dist[start] = 0

    def calc_dist(self):
        """
        最短経路 distを計算する
        Q: content (dist_from_start,node)
        Qはheapqとして使う。dist_from_startの小さい順になる
        """
        Q = []
        heapq.heappush(Q, (0, self.start_node))
        while Q:
            dist_to_node, node = heapq.heappop(Q)
            # 現在の値よりも大きくて、更新にならない場合
            if self.dist[node] < dist_to_node:
                continue

            # 次のホップを計算する
            for nex in range(self.num_node):
                weight = self.mat[node][nex]
                cand = dist_to_node + weight

                if self.dist[nex] > cand:  # update
                    self.dist[nex] = cand
                    self.prev[nex] = node
                    heapq.heappush(Q, (cand, nex))

    def get_dist(self, d):
        """
            startからdまでの距離を返す
            これの前にcalc()を実行しておく
            たどり着けない場合は、float('inf')がかえる
        """
        return self.dist[d]
    def get_path(self,dst):
        """
        startからdstまでのpathをlistでreturn
        valifyしてないので怪しい
        """
        cur=dst
        ret=[cur]
        while cur!=self.start_node:
            cur=self.prev[cur]
            ret.append(cur)
        return list(reversed(ret))


import matplotlib.pyplot as plt
import sys
import tensorflow as tf
import os

if len(sys.argv) < 2:
    print("Usage: python main.py <path to log file folder>")
    exit()
logdir = sys.argv[1]

losses = []
directory = os.fsencode(logdir)
for logfile in os.listdir(directory):
    filename = os.fsdecode(logfile)
    file_path = os.path.join(logdir, filename)
    for e in tf.train.summary_iterator(file_path):
        for v in e.summary.value:
            if v.tag == 'loss':
                losses.append(v.simple_value)

plt.plot(losses)
plt.ylabel('Loss')
plt.savefig('loss.png')


import os
import numpy as np
import matplotlib.pyplot as plt

import torch
import torch.nn as nn
from torchvision.utils import draw_bounding_boxes
import torchvision.transforms.functional as F

from clearpose.xu_6dof.networks.references.segmentation.train import train_one_epoch
import clearpose.xu_6dof.networks.references.segmentation.transforms as T

from clearpose.xu_6dof.networks.stage1.surface_normals.deeplabv3 import build_model
from clearpose.xu_6dof.datasets.surface_normal_dataset import SurfaceNormalDataset

class NormalCriterion(nn.Module):
	def __init__(self):
		super(NormalCriterion, self).__init__()
		self.cosine_similarity = torch.nn.CosineSimilarity(dim=1)

	def forward(self, output, target):
		norm = torch.linalg.vector_norm(target, dim=1)
		norm_nonzero = torch.nonzero(norm, as_tuple=True)
		output_nonzero = output[norm_nonzero[0],:,norm_nonzero[1],norm_nonzero[2]]
		target_nonzero = target[norm_nonzero[0],:,norm_nonzero[1],norm_nonzero[2]]

		return torch.mean(1-self.cosine_similarity(output_nonzero, target_nonzero))

def get_transform(train):
	transforms = []
	transforms.append(T.PILToTensor())
	transforms.append(T.ConvertImageDtype(torch.float))
	transforms.append(T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]))
	if train:
		transforms.append(T.RandomHorizontalFlip(0.5))
	return T.Compose(transforms)

def show(imgs):
	if not isinstance(imgs, list):
		imgs = [imgs]
	fix, axs = plt.subplots(ncols=len(imgs), squeeze=False)
	for i, img in enumerate(imgs):
		img = img.detach()
		img = F.to_pil_image(img)
		axs[0, i].imshow(np.asarray(img))
		axs[0, i].set(xticklabels=[], yticklabels=[], xticks=[], yticks=[])
	plt.show()

def main(save_dir=os.path.join("experiments","xu_6dof","stage1","surface_normals","models")):
	# train on the GPU or on the CPU, if a GPU is not available
	device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

	# use our dataset and defined transformations
	dataset = SurfaceNormalDataset(transforms=get_transform(train=True))
	dataset_test = SurfaceNormalDataset(transforms=get_transform(train=False))

	# split the dataset in train and test set
	indices = torch.randperm(len(dataset)).tolist()
	dataset = torch.utils.data.Subset(dataset, indices[:-50])
	dataset_test = torch.utils.data.Subset(dataset_test, indices[-50:])

	# define training and validation data loaders
	data_loader = torch.utils.data.DataLoader(
		dataset, batch_size=2, drop_last=True, shuffle=True, num_workers=4)

	data_loader_test = torch.utils.data.DataLoader(
		dataset_test, batch_size=1, shuffle=False, num_workers=4)

	criterion = NormalCriterion()

	# get the model using our helper function
	model = build_model()

	# move model to the right device
	model.to(device)

	# construct an optimizer
	params = [p for p in model.parameters() if p.requires_grad]
	optimizer = torch.optim.SGD(params, lr=0.005,
								momentum=0.9, weight_decay=0.0005)
	# and a learning rate scheduler
	lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
												   step_size=3,
												   gamma=0.1)

	# let's train it for 10 epochs
	num_epochs = 10

	torch.save(model.state_dict(), os.path.join(save_dir,"deeplabv3_0.pt"))
	for epoch in range(num_epochs):
		# train for one epoch, printing every 10 iterations
		train_one_epoch(model, criterion, optimizer, data_loader, lr_scheduler, device, epoch, print_freq=100)
		# update the learning rate
		torch.save(model.state_dict(), os.path.join(save_dir,"deeplabv3_"+str(epoch)+".pt"))
		lr_scheduler.step()

if __name__=="__main__":
	main()


import pygame
import random
import math
from pygame import mixer

LENGTH = 600
BREADTH = 700
GameOver = False

# Screen Setup
pygame.init()
screen = pygame.display.set_mode((BREADTH, LENGTH))
pygame.display.set_caption("Covid Blaster")

# BackGround
icon = pygame.image.load("./spaceship.png")
pygame.display.set_icon(icon)
bg = pygame.image.load("./SafePlanet.jpg")

# Music
mixer.music.load("./SpaceMusic.mp3")
mixer.music.play(-1)

# Enemy
EnemyImg = []
EnemyX = []
EnemyY = []
EnemyChangeX = []
EnemyChangeY = []
num_of_enemies = 8

for i in range(num_of_enemies):
    EnemyImg.append(pygame.image.load("./alien.png"))
    EnemyX.append(random.randint(0, 636))
    EnemyY.append(random.randint(50, 150))
    EnemyChangeX.append(-3)
    EnemyChangeY.append(20)

# Player
PlayerImg = pygame.image.load("./spaceship.png")
PlayerX = 480
PlayerY = 510
PlayerChangeX = 0

# Bullet
BulletImg = pygame.image.load("./syringe1.png")
BulletX = 0
BulletY = 510
BulletChangeY = 0
BulletChangeY = 10
BulletFire = False

#   SCORE
score_value = 0
font = pygame.font.Font('freesansbold.ttf', 32)
Ammo_value = 0
font = pygame.font.Font('freesansbold.ttf', 32)

textX = 10
textY = 10

# Game Over
over_font = pygame.font.Font('freesansbold.ttf', 32)
over_font1 = pygame.font.Font('freesansbold.ttf', 64)

def Show_score(x,y):
    score = font.render("Virus Remaining : " + str(num_of_enemies),True,(0,0,255))
    screen.blit(score,(x,y))
    # Ammo = font.render("Bullets Fired : " + str(Ammo_value),True,(50,205,50))
    # screen.blit(Ammo,(700,y))

def Game_Over(x,y):
    screen.blit(bg, (0, 0))
    over = over_font1.render("Congratratulations,",True,(0,0,0))
    screen.blit(over,(x,y))
    over2 = over_font.render("You Have Defeated COVID-19 ",True,(0,0,0))
    screen.blit(over2,(x,y+70))

def player(PlayerX, PlayerY):
    screen.blit(PlayerImg, (PlayerX, PlayerY))

def enemy(X, Y,i):
    screen.blit(EnemyImg[i], (X, Y))

def bullet(BulletX, BulletY):
    global BulletFire
    BulletFire = True
    screen.blit(BulletImg, (BulletX+16, BulletY+10))

def collision(X,Y,BulletX,BulletY):
    dis = math.sqrt(math.pow(X-BulletX,2)+math.pow(Y-BulletY,2))

    if dis <30:
        return True
    else:
        return False

while not GameOver:
    screen.blit(bg, (0, 0))
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            GameOver = True
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_LEFT:
                PlayerChangeX = -5
            if event.key == pygame.K_RIGHT:
                PlayerChangeX = 5
            if event.key == pygame.K_SPACE:
                if not BulletFire:
                    # Ammo_value += 1
                    Bullet_sound = mixer.Sound("./laser.wav")
                    Bullet_sound.play()
                    BulletChangeY = -5
                    BulletX = PlayerX
                    bullet(BulletX,BulletY)

        if event.type == pygame.KEYUP:
            if event.key == pygame.K_LEFT or event.key == pygame.K_RIGHT:
                PlayerChangeX = 0

    # Player Motion
    PlayerX = PlayerX + PlayerChangeX
    if PlayerX <= 0:
        PlayerX = 0
    elif PlayerX >= 636:
        PlayerX = 636

    # Enemy Motion
    for i in range(num_of_enemies):

        if num_of_enemies == 0:
            Game_Over(0,0)
            break

        EnemyX[i] = EnemyX[i] + EnemyChangeX[i]
        if EnemyX[i] <= 0:
            EnemyChangeX[i] = 2
            EnemyY[i] += 20
        elif EnemyX[i] >= 636:
            EnemyChangeX[i] = -2
            EnemyY[i] += 20

    # screen.fill((0,0,0))
        col = collision(EnemyX[i],EnemyY[i],BulletX,BulletY)
        if col:
            Contact_sound = mixer.Sound("./explosion.wav")
            Contact_sound.play()
            BulletY = 510
            BulletFire = False
            score_value +=1
            # print(score_value)
            num_of_enemies -=1
            # EnemyX[i] = random.randint(0, 936)
            # EnemyY[i] = random.randint(50, 150)

        enemy(EnemyX[i], EnemyY[i],i)

    if BulletY <= 0:
        BulletY = 510
        BulletFire =False
    if BulletFire:
        bullet(BulletX,BulletY)
        BulletY += BulletChangeY

    player(PlayerX, PlayerY)
    Show_score(textX,textY)
    if num_of_enemies == 0:
        Game_Over(0,0)
    pygame.display.update()


from flask import Flask , render_template, request
import pickle
import numpy as np
import pandas as pd
import joblib
import warnings
import sys
if not sys.warnoptions:
    warnings.simplefilter("ignore")
warnings.filterwarnings("ignore",category=DeprecationWarning)

# model=pickle.load(open('first.h5','rb'))

model=joblib.load('loan_prediction')

app = Flask(__name__,template_folder='template')

@app.route('/')
def main():
    return render_template('home.html')

@app.route('/predict',methods=['POST'])
def home():
    name = request.form['NAME']
    address = request.form['ADDRESS']
    email = request.form['EMAIL']
    mobile1 = request.form['MOBILE']

# form
    gender1 = request.form['GENDER']
    married1 = request.form['MARRIED']
    dependents1 = request.form['DEPENDENTS']
    education1 = request.form['EDUCATION']
    self_emp1 = request.form['SELF_EMP']
    app_income1 = request.form['APP_INCOME']
    coapp_income1 = request.form['COAPP_INCOME']
    loan_amount1 = request.form['LOAN_AMOUNT']
    la_term1 = request.form['LA_TERM']
    cr_history1 = request.form['CR_HISTORY']
    property1 = request.form['PROPERTY']

# valeurs saisies de string vers int / float
    mobile=int(mobile1)
    app_income=int(app_income1)
    coapp_income=float(coapp_income1)
    loan_amount=float(loan_amount1)
    la_term=float(la_term1)

    if(cr_history1=='Clear'):
        cr_history=1.0
    else:
        cr_history=0.0
    if(married1=='YES'):
        married=1
    else:
        married=0
    if(gender1=='MALE'):
        gender=1
    else:
        gender=0

    if(dependents1=='zero'):
        dependents='0'
    if(dependents1=='one'):
        dependents='1'
    if(dependents1=='two'):
        dependents='2'
    if(dependents1=='three_plus'):
        dependents='3+'

    if(education1=='Graduate'):
        education=1
    else:
        education=0
    if(self_emp1=='yes'):
        self_emp=1
    else:
        self_emp=0

    p_rural=0
    p_semi_urban=0
    p_urban=0
    if(property1=="rural"):
        p_a=0
    if(property1=="semi-urban"):
       p_a=2
    if(property1=="urban"):
        p_a=1

    arr=np.array([gender,married,dependents,education,self_emp,app_income,coapp_income,loan_amount,la_term,cr_history,p_a])
    # df=pd.DataFrame(arr)
    pred= model.predict(arr.reshape(1,-1))
    return render_template('prediction_page.html', data=pred,name=name)

if __name__ == "__main__":
    app.run(debug=True)


from multiprocessing.sharedctypes import SynchronizedString, Synchronized
import time

from utils.util import BUFF_SIZE

def write_to_shared_memory(
    data_shm: SynchronizedString, stat_shm: Synchronized, serd_num_seq: str
) -> int or None:

    # write to shared memory
    while stat_shm.value != 0:
        time.sleep(0.01)
    serd_num_seq_bytes = serd_num_seq.encode()
    l = len(serd_num_seq_bytes)
    c = 0

    while c < l:
        h = min(l, c + BUFF_SIZE)
        while stat_shm.value not in (0, 2):
            time.sleep(0.01)
        data_shm.value = serd_num_seq_bytes[c:h]
        stat_shm.value = 1
        c = h
    while stat_shm.value != 2:
        time.sleep(0.01)
    stat_shm.value = -1

    if serd_num_seq == '"quit"':
        return -1


# Import modul dan lib yang diperlukan
from flask import Flask, render_template, request, url_for, redirect, flash
from sklearn.cluster import MiniBatchKMeans
from matplotlib.colors import rgb2hex
from collections import Counter
from PIL import Image
import numpy as np
import os

# Setup flask app
app = Flask(__name__)
app.config['SECRET_KEY'] = 'klasifikasi-warna'
app.config['UPLOAD_PATH'] = "static"
app.config['SESSION_TYPE'] = 'filesystem'

# Set laman berada
@app.route("/", methods=["GET", "POST"])
def home():

    if request.method == "GET":
        return render_template("index.html")

    elif request.method == "POST":
        img = request.files["image_file"]
        if img.filename != "":
            img.save( os.path.join(app.config['UPLOAD_PATH'], img.filename) )

            # Buka citra yang akan diklasififikasi
            img_object = Image.open(fp=f"static/{img.filename}", mode="r")

            # Konversi citra ke 2d array
            img_array = np.array(img_object)

            # Gunakan MiniBatchKMeans algoritma dengan mengatur cluster n sebanyak 7
            k_warna = MiniBatchKMeans(n_clusters=7)
            k_warna.fit(img_array.reshape(-1, 3))

            # Hitung berapa banyak pixel per cluster
            n_pixels = len(k_warna.labels_)
            counter = Counter(k_warna.labels_)

            # 2D array yang berisi nilai pixel RGB sejumlah n cluster = 7
            rgb_int = k_warna.cluster_centers_

            # Konversi ke float 0-1
            rgb_float = np.array(rgb_int / 255, dtype=float)

            # Konversi nilai RGB ke nilai HEXA dan simpan ke dalam list
            hex_values = [rgb2hex(rgb_float[i, :]) for i in range(rgb_float.shape[0])]

            prop_warna = {}

            # Kalkulasi presentase tiap pixel warna
            for i in counter:
                prop_warna[i] = np.round(counter[i] / n_pixels, 4)

            # Konversi dictionary ke dalam list
            prop_warna = dict(sorted(prop_warna.items()))
            props_list = [value for (key, value) in prop_warna.items()]

            def to_dictionary(key, value):
                return dict(zip(key, value))

            # Merge 2 list ke dalam sebuah dictionary
            dict_warna = to_dictionary(props_list, hex_values)

            # Sort/urutkan dict secara descending
            sorted_dict = dict(sorted(dict_warna.items(), reverse=True))

            return render_template("index.html", image=img.filename, colors=sorted_dict)
        else:
            # Tampilkan error ketika belum mengunggah citra
            flash("Kamu belum mengunggah citra!")
            return redirect(url_for("home"))

# Jalankan flask app
if __name__ == "__main__":
    app.run(host='0.0.0.0', port='5000', debug=True)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Apr 12 10:36:28 2019

@author: pageot
"""

import os
import otbApplication
import argparse

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description='SAR_fusion_impose_programme. this programme allows to fusion and cut image')
    parser.add_argument('-out', dest='out')
    parser.add_argument('-inp', dest='path',nargs='+',required = True)
    args = parser.parse_args()
    print (args.path)
    print (args.out)

    list_vv=[]
    list_vh=[]
    for i in  os.listdir("{}".format(str(args.path).strip("['']"))):
        #print (i)
        polar=i[10:12]
        orbite=i[13:16]
        if (polar == "vv" and "s1b_31TCJ_vv" and ".tif" in i and "BorderMask" not in i):
            list_vv.append("{}".format(str(args.path).strip("['']"))+i)
        if (polar == "vh" and "s1b_31TCJ_vv" and ".tif" in i and "BorderMask" not in i):
            list_vh.append("{}".format(str(args.path).strip("['']"))+i)
    print(list_vv)
    list_vv_sorted=[]
    for x in sorted(list_vv):
        list_vv_sorted.append(x)
    list_vh_sorted=[]
    for h in sorted(list_vh):
        list_vh_sorted.append(h)

    for p in ["vv","vh"]:
        if (p == "vv"):
            App2 = otbApplication.Registry.CreateApplication("ConcatenateImages")
            App2.SetParameterStringList("il",list_vv_sorted)
            App2.SetParameterString("out","%sSAR_%s%s.tif"%("{}".format(str(args.out).strip("['']")),'vv',orbite))
            App2.ExecuteAndWriteOutput()
        else:
            App2 = otbApplication.Registry.CreateApplication("ConcatenateImages")
            App2.SetParameterStringList("il",list_vh_sorted)
            App2.SetParameterString("out","%sSAR_%s%s.tif"%("{}".format(str(args.out).strip("['']")),'vh',orbite))
            App2.ExecuteAndWriteOutput()


from pathlib import Path
import pandas as pd
import pytest

@pytest.fixture(scope="function")
def imgfile():
    dirpath = Path("input")
    dirpath.mkdir(exist_ok=True)
    filepath = dirpath.joinpath("test.jpg")
    yield str(filepath)
    filepath.unlink(missing_ok=True)

@pytest.fixture(scope="session")
def c_df():
    GITHUB_URL = "https://raw.githubusercontent.com"
    URL_C = f"{GITHUB_URL}/lisphilar/covid19-sir/master/data/japan/covid_jpn_total.csv"
    path = Path("data", "japan", "covid_jpn_total.csv")
    filepath = path if path.exists() else URL_C
    df = pd.read_csv(filepath, dayfirst=False).rename(columns={"Date": "date"})
    df = df[["date", "Positive", "Tested", "Discharged", "Fatal"]].groupby("date", as_index=False).sum()
    df.insert(0, "Country", "Japan")
    return df

@pytest.fixture(scope="session")
def p_df():
    GITHUB_URL = "https://raw.githubusercontent.com"
    URL_P = f"{GITHUB_URL}/lisphilar/covid19-sir/master/data/japan/covid_jpn_prefecture.csv"
    path = Path("data", "japan", "covid_jpn_prefecture.csv")
    filepath = path if path.exists() else URL_P
    df = pd.read_csv(filepath, dayfirst=False).rename(columns={"Date": "date"})
    df.insert(0, "Country", "Japan")
    return df[["Country", "Prefecture", "date", "Positive", "Tested", "Discharged", "Fatal"]]

@pytest.fixture(scope="session")
def japan_df(c_df, p_df):
    df = pd.concat([c_df, p_df], axis=0)
    assert set(df.columns) == {"Country", "Prefecture", "date", "Positive", "Tested", "Discharged", "Fatal"}
    return df


import lexicon
import synonyms
import game

##import mobs

synonyms = synonyms.synonym_list
tokens = lexicon.tokens

Player = game.Player('yourself',50,[],'house')

def player_move(direction):
    print(Player.move(direction))

def check_commands(s):
    try:
        if s[0] == 'GO' and s[1] in tokens['DIRECTIONS']:
            player_move(s[1])
        elif s[0] in tokens['DIRECTIONS']:
            player_move(s[0])
        elif s[0] == 'LOOK' and 'L':
            print(Player.look())
        elif s[0] == 'QUIT':
            print("Goodbye!")
            exit()

        else:
            print("Can't do that!")
    except IndexError:
        print("Index Error")

def synonymize(s):
    if isinstance(s,str):
        if s in synonyms.keys():
            return synonyms[s]
    if isinstance(s,list):
        for num in range(0,len(s)):
            if s[num] in synonyms.keys():
                s[num] = synonyms[s[num]]
        return s
    else:
        return None

def sanitize_sentence(sentence):
    try:
        while 1:
            if 'xxx' in sentence:
                sentence.remove('xxx')
            else:
                break
        return sentence
    except ValueError:
        return sentence

def parse(words):
    sentence = ['xxx','xxx','xxx']
    for word in words:
        if word.upper() in tokens['DIRECTIONS']:
            if sentence[1] == 'xxx':
                sentence[1] = word.upper()
        if word.upper() in tokens['VERBS']:
            if sentence[0] == 'xxx':
                sentence[0] = word.upper()
        if word.upper() in tokens['NOUNS']:
            if sentence[2] == 'xxx':
                sentence[2] = word.upper()
    return(sentence)

def debug_parse(words):
    sentence = ['xxx','xxx','xxx']
    for word in words:
        if word.upper() in tokens[0][1]:
            print(word,' is a direction')
            print('putting it in the sentence')
            if sentence[1] == 'xxx':
                sentence[1] = word.upper()
                print(sentence)
        if word.upper() in tokens[1][1]:
            print(word,' is a verb')
            print('putting it in the sentence')
            if sentence[0] == 'xxx':
                sentence[0] = word.upper()
                print(sentence)
        if word.upper() in tokens[2][1]:
            print(word,' is a noun')
            print('putting it in the sentence')
            if sentence[2] == 'xxx':
                sentence[2] = word.upper()
                print(sentence)
    return(sentence)


__all__ = ["parse_doc", "from_string"]

import gzip
from math import atan2
import re
from . import wrapped_etree as ET
from . import svg_re
from . parse.parse_util import (
    svg_error,
    Matrix,
    clean_path_definition,
    parse_color,
    parse_color_noref,
    parse_color_stop_offset,
    dict_union,
    center_based_to_rect,
    maybe_id_ref,
    extract_local_xlink_href,
    match_children,
    parse_embedded_image_data,
    ABSOLUTE_UNIT_FACTORS
)

from . util import (
    arrow_line_end,
    deg2rad,
    rad_angle,
)

from . parse import (
    ParseState,
    get_style_dict,
    svg_length_attr,
    svg_length_attr_dumb,
)

import faint
import faint.svg.tag as tag
ns_svg = tag.ns_svg
ns_faint = tag.ns_faint

from . ifaint_util import (mul_matrix_tri,
                           to_faint_cap_str,
                           fill_style)

_DEBUG_SVG = False

def getter_default(node, default_value):
    return lambda attr : node.get(attr, default_value)

def parse_color_stops(nodes, state):
    """Parses the color stop nodes for a gradient node.
    Returns a list of offsets to colors.

    """
    stop_list = []
    for stop in [st for st in nodes if st.tag == ns_svg('stop')]:
        color_str, opacity_str = parse_color_stop_style(stop.get('style', ''))
        color_str = stop.get('stop-color', color_str)
        opacity_str = stop.get('stop-opacity', opacity_str)
        offset = parse_color_stop_offset(stop.get("offset"))

        # Fixme: Parse opacity str instead of just floating it
        faint_color = parse_color_noref(color_str, float(opacity_str), state)
        stop_list.append((offset, faint_color))
    return stop_list

def set_name(node, object_id, props):
    name = node.get('id')
    if name is not None:
        props.set_obj_name(object_id, name)

def parse_defs(node, state, id_to_etree_node=None):
    """Recursively parses an SVG defs-node."""
    if id_to_etree_node is None:
        id_to_etree_node = {}
    for child in node:
        ref_id = child.get('id')
        if ref_id is not None:
            id_to_etree_node[ref_id] = child

    for child, func in match_children(node, svg_defs_content_functions):
        item = func(child, state)

        if item is None or item.__class__ is int:
            # I guess "None" is failed parsing? Could
            # throw instead.
            # Fixme: int is probably an object id.
            # Should be able to reference those as well.
            state.add_warning("Ignored referenced item in <defs>")

        elif item == 'marker':
            # No warning on ignored markers for now, since I add them for
            # arrowhead, arrowtail.
            # Fixme: These should be indexed for referencing.
            pass
        elif item == 'calibration' or item == 'grid':
            # Fully handled by function.
            pass
        else:
            ref_id, obj = item
            state.add_ref_item(ref_id, obj)

class SvgLiteralError(faint.LoadError):
    """Exception raised for invalidly expressed SVG literals."""
    def __init__(self, literal_type, literal):
        super().__init__("Invalid %s: %s" % (literal_type, literal))
        self.literal = literal

class SvgLengthError(SvgLiteralError):
    """Exception raised for invalidly expressed SVG lengths."""
    def __init__(self, literal):
        super().__init__("length", literal)

class SvgCoordError(SvgLiteralError):
    """Exception raised for invalidly expressed SVG coordinates."""
    def __init__(self, literal):
        super().__init__("coordinate", literal)

def parse_points(points_str):
    """Parses the points specification for polyline and polygon
    elements (SVG 1.1, 9.7.1).

    """

    # Treat #-# as # -#
    points_str = points_str.replace('-', ' -')
    return [float(s) for s in re.split("[\x20\x09\x0D\x0A]+|[,]",
      points_str) if s != ""]

def parse_pattern_node(node, state, id_to_etree_node):
    """Parses a pattern element"""

    state = state.updated(node)

    node_id = node.get('id')
    if node_id is None:
        # Ignore patterns without identifiers
        return None

    for child in node:
        if child.tag == tag.image:
            encoding, data = parse_embedded_image_data(child.get(tag.xlink.href))
            if encoding == "png":
                return node_id, faint.Pattern(faint.bitmap_from_png_string(data))
    state.add_warning('Failed parsing pattern with id=%s' % node_id)
    return None

def get_linked_stops(xlink, state, id_to_etree_node):
    """Returns a referenced list of color-stops for a gradient.

    Retrieves the referenced id from the xlink, and retrieves its
    stops, if already available as a faint Gradient object in the state,
    otherwise looks up the target node in id_to_etree node, and tries to
    parse it as a gradient.

    """

    ref_id = extract_local_xlink_href(xlink)
    linked = state.ids.get(ref_id, None)

    if linked is None:
        if ref_id in id_to_etree_node:
            other = id_to_etree_node[ref_id]
            if other.tag == ns_svg('linearGradient'):
                linked_id, linked = parse_linear_gradient_node(other, state, id_to_etree_node)
            elif other.tag == ns_svg('radialGradient'):
                linked_id, linked = parse_radial_gradient_node(other, state, id_to_etree_node)
        if linked is None:
            state.add_warning("Referenced gradient not found: %s " % ref_id)

    if linked is not None:
        stops = []
        for num in range(linked.get_num_stops()):
            stops.append(linked.get_stop(num))
        return stops
    return []

def parse_faint_calibration(node, state, id_to_etree_node=None):
    points = tuple([svg_coord_attr(node.get(s,'0'), state)
              for s in ('x1','y1','x2','y2')])

    length = svg_length_attr(node.get('length'), state)
    unit = node.get('unit')
    state.props.set_calibration((points, length, unit))
    return 'calibration' # Fixme: To avoid warning. Find better method

def parse_faint_grid(node, state, id_to_etree_node=None):
    x = svg_coord_attr(node.get('x'), state)
    y = svg_coord_attr(node.get('y'), state)
    dashed = node.get('dashed') == 'True'
    enabled = node.get('enabled') == 'True'
    spacing = svg_length_attr(node.get('spacing'), state)
    state.set_grid((x, y), dashed, enabled, int(spacing))
    return 'grid' # Fixme: To avoid warning. Find better method

def parse_linear_gradient_node(node, state, id_to_etree_node=None):
    """Parses a linear gradient element"""

    state = state.updated(node)

    node_id = node.get('id')
    if node_id is None:
        # Ignore gradients without identifiers
        return None

    x1 = svg_coord_attr(node.get('x1', '0.0'), state)
    x2 = svg_coord_attr(node.get('x2', '0.0'), state)
    y1 = svg_coord_attr(node.get('y1', '0.0'), state)
    y2 = svg_coord_attr(node.get('y2', '0.0'), state)

    dx = x2 - x1
    dy = y2 - y1
    angle = atan2(dy, dx)

    stops = []
    stops = parse_color_stops(list(node), state)

    if len(stops) == 0:
        # No color stops, follow references
        # FIXME: Can there both be stops and a reference?
        xlink = node.get(tag.xlink.href)
        if xlink is not None:
            stops = get_linked_stops(xlink, state, id_to_etree_node)
        else:
            # Ignore gradients without stops
            # (Note: Need to add handling of referenced gradients)
            state.add_warning("linearGradient with id=%s has no color-stops" % node_id)
            return None
    if len(stops) == 0:
        state.add_warning("linearGradient with id=%s has no color-stops" % node_id)

        return None

    return node_id, faint.LinearGradient(angle, *stops)

def parse_radial_gradient_node(node, state, id_to_etree_node=None):
    """Parses a radial gradient node."""
    # Fixme: Review, from old
    node_id = node.get('id')
    if node_id is None:
        # Ignore gradients without identifiers
        return None

    state = state.updated(node)

    stops = []
    stops = parse_color_stops(list(node), state)

    if len(stops) == 0:
        # No color stops, follow references
        # FIXME: Can there both be stops and a reference?
        xlink = node.get(tag.xlink.href)
        if xlink is not None:
            stops = get_linked_stops(xlink, state, id_to_etree_node)
        else:
            # Ignore gradients without stops
            state.add_warning("radialGradient with id=%s has no color-stops" % node_id)
            return None
    if len(stops) == 0:
        state.add_warning("radialGradient with id=%s has no color-stops" % node_id)
        return None

    return node_id, faint.RadialGradient(stops)

def parse_color_stop_style(style):
    """Return the color and opacity from the style as strings.
    Returns None for color-string and 1.0 for opacity if not present

    """

    style_dict = get_style_dict(style)
    stop_opacity = style_dict.get('stop-opacity', '1.0')
    color_str = style_dict.get('stop-color', None)
    return color_str, stop_opacity

def parse_faint_tri_attr(node):
    """Parses a faint:tri attribute from the node"""
    tri_str = node.get(ns_faint("tri"))
    p0, p1, p2 = pairs(parse_points(tri_str))
    return faint.Tri(p0, p1, p2)

def parse_ellipse(node, state):
    """Parses an SVG <ellipse>-element."""
    state = state.updated(node)

    get = getter_default(node, '0')
    cx, cy = get('cx'), get('cy')
    cx, cy = svg_point_attr(cx, cy, state)

    rx, ry = get('rx'), get('ry')
    rx, ry = svg_point_attr(rx, ry, state)

    bounding_rect = center_based_to_rect(cx, cy, rx, ry)
    return state.props.Ellipse(bounding_rect, state.settings)

def parse_group(node, state, id_to_etree_node=None):
    """Parses an SVG <g>-element."""
    # Fixme: Use id_to_etree_node
    state = state.updated(node)
    # Parse the children of the group
    object_ids = []
    for child, func in match_children(node, included_svg_group_content_functions):
        result = func(child, state)
        if result.__class__ is not int:
            # Expected an object id from the function
            # Fixme: maybe_id_ref uses parentheses which looks weird here.
            state.add_warning("Failed parsing child %s%s of group%s" %
                              (child.tag, maybe_id_ref(child), maybe_id_ref(node)))
        else:
            object_ids.append(result)

    if len(object_ids) == 0:
        # Empty group
        return None

    return state.props.Group(object_ids)

def parse_path(node, state):
    """Parses an SVG <path>-element"""

    state = state.updated(node)
    path_def = node.get('d')
    if path_def is None:
        state.add_warning("Ignored path-element without definition "
                          "attribute%s." % maybe_id_ref(node))
        return

    path_def = clean_path_definition(path_def)
    if len(path_def) == 0:
        state.add_warning("Ignored path-element without definition "
                          "attribute%s." % maybe_id_ref(node))
        return

    try:
        path_id = state.props.Path(path_def, state.settings)
    except ValueError:
        faint.copy_text(node.get('d')) # Fixme: Remove
        state.add_warning("Failed parsing a path definition%s." %
                          maybe_id_ref(node))
        return

    return path_id

def parse_path_as_ellipse(node, state):
    """Parses an ellipse with a faint:tri attribute to an ellipse.

    Faint saves ellipses as paths for some reason (probably to
    skew/transform only the points, not the stroked edge).

    """
    state = state.updated(node)
    tri = parse_faint_tri_attr(node)
    # Fixme: Accept Tri in props.Ellipse
    ellipse_id = state.props.Ellipse((0, 0, 1, 1), state.settings)
    state.props.set_obj_tri(ellipse_id, tri)
    return ellipse_id

def parse_polygon(node, state):
    """Parses an SVG <polygon>-element."""
    state = state.updated(node)
    points = parse_points(node.get('points', ''))
    if len(points) % 2 != 0:
        state.add_warning("Odd number of coordinates for polygon%s." %
                          maybe_id_ref(node))
        # SVG 1.1 F2 "Error processing" says
        # Render polyline and polygons with invalid points up to the
        # erroneous point.
        points = points[:-1]

    return state.props.Polygon(points, state.settings)

def parse_polyline(node, state):
    """Parses an SVG <polyline>-element."""
    state = state.updated(node)

    points = parse_points(node.get('points'))

    if state.settings.arrow == 'front':
        x0, y0, x1, y1 = points[-4:]
        angle = rad_angle(x0,y0,x1,y1)
        x, y = arrow_line_end(x1, y1, angle, state.settings.linewidth)
        points[-2] = x
        points[-1] = y

    if len(points) % 2 != 0:
        state.add_warning("Odd number of coordinates for polyline%s." %
                          maybe_id_ref(node))
        # SVG 1.1 F2 "Error processing" says
        # Render polyline and polygons with invalid points up to the
        # erroneous point.
        points = points[:-1]

    return state.props.Line(points, state.settings)

def parse_faint_text_size_attrs(node, state):
    """Parses the faint-added width, height attributes"""

    # Get name-spaced or un-namespaced width, height
    # (added by Faint regardless)
    width_str = (node.get(ns_faint('width')) or
                  node.get('width'))
    height_str = (node.get(ns_faint('height')) or
                  node.get('height'))
    if width_str is not None:
        w = svg_length_attr(width_str, state)
    else:
        w = 200 # Fixme
    if height_str is not None:
        h = svg_length_attr(height_str, state)
    else:
        h = 200
    return not (width_str is None or height_str is None), w, h

def svg_baseline_to_faint(text_id, state):
    """Convert the position of the text object in props with text_id.

    SVG anchors at baseline, Faint at top of text.
    """
    dy = state.props.get_obj_text_height(text_id)
    tri = state.props.get_obj_tri(text_id)
    tri.translate(0, -dy)
    state.props.set_obj_tri(text_id, tri)

def parse_text(node, state):
    """Parses an SVG <text>-element."""
    state = state.updated(node)

    # Fixme: Should support <coordinate-list>
    get = getter_default(node, '0')
    x, y = svg_point_attr(get('x'), get('y'), state)

    boxed, w, h = parse_faint_text_size_attrs(node, state)
    # Fixme: Parse dx, dy lists etc.

    settings = faint.Settings()
    settings.update_all(state.settings)

    # Fixme: These assignments will overwrite inherited settings
    #        even if node doesn't have the retrieved setting
    default_bound = '1' if boxed else '0'
    settings.bounded = node.get(ns_faint('bounded'), default_bound) == '1'

    settings.valign = node.get(ns_faint('valign'), 'top')

    # TODO: If no unit specified size is processed as a height in the user
    #       coordinate system.
    #       Otherwise converts the length into user coordinate system

    fontsize_str = node.get('font-size')
    if fontsize_str is not None:
        # Fixme: Handle 'inherit' generically somehow
        if fontsize_str != 'inherit':
            settings.fontsize = svg_length_attr(fontsize_str, state)

    style = get_style_dict(node.get('style', ''))
    if 'fill' in style:
        settings.fg = parse_color(style['fill'], "1.0", state)

    anchor = node.get('text-anchor', 'start')
    if anchor == 'start':
        settings.halign = 'left'
    elif anchor == 'middle':
        settings.halign = 'center'
        x -= w / 2 # fixme
    elif anchor == 'end':
        settings.halign = 'right'

    text_str = None
    settings.parsing = node.get(ns_faint("parsing")) == '1'
    if settings.parsing:
        raw_node = node.find(ns_faint('raw'))
        if raw_node is not None:
            text_str = raw_node.text

    if text_str is None:
        text_str = ""
        if node.text:
            text_str += node.text

        for child in node:
            if child.tag == ns_svg('tspan'):
                # Fixme: Parse settings from the tspans
                if len(child.text) > 0:
                    text_str += child.text

                if child.get(ns_faint('hardbreak'), '0') == '1':
                    text_str += "\n"

    text_id = state.props.Text((x, y, w, h), text_str, settings)
    svg_baseline_to_faint(text_id, state)
    return text_id

def is_faint_type(node, expected):
    """True if the node has a faint:type attribute with value matching
    expected. Allows mapping SVG shapes etc. to specific Faint object
    types.

    """
    return node.get(ns_faint('type')) == expected

def pairs(iterable):
    """Return a list which pairs up the neighbouring items in tuples.
    Will lose the last item if uneven item count.
    """
    i = iter(iterable)
    return list(zip(i, i))

def parse_circle(node, state):
    """Parses an SVG <circle>-element."""
    state = state.updated(node)
    cx = svg_coord_attr(node.get('cx', '0'), state)
    cy = svg_coord_attr(node.get('cy', '0'), state)
    r = svg_length_attr(node.get('r', '0'), state)
    rect = center_based_to_rect(cx, cy, r, r)
    return state.props.Ellipse(rect, state.settings)

def parse_polygon_as_rect(node, state):
    """Skewed Faint-rectangles are saved as polygons (marked with
    faint:type="rect").

    """
    state = state.updated(node)

    name = node.get('id')

    # Fixme: Pass Tri to props.rect instead of defaulting like this
    rect_id = state.props.Rect((0, 0, 1, 1), state.settings, name)

    p0, p1, p3, p2 = pairs(parse_points(node.get('points')))
    del p3
    state.props.set_obj_tri(rect_id, faint.Tri(p0, p1, p2))
    return rect_id

def parse_image_as_background(node, state):
    """Parses the embedded content of image node with a faint:background
    attribute and sets it as the background.

    """
    image_str = node.get(tag.xlink.href)
    image_type, data = parse_embedded_image_data(image_str)
    assert image_type == 'png'
    state.props.set_background(faint.bitmap_from_png_string(data))

def parse_image(node, state):
    """Parses an SVG <image>-element."""
    assert node.get(ns_faint('background')) is None
    state = state.updated(node)

    x = svg_length_attr(node.get('x', '0.0'), state)
    y = svg_length_attr(node.get('y', '0.0'), state)
    w = svg_length_attr(node.get('width', '0.0'), state)
    h = svg_length_attr(node.get('height', '0.0'), state)

    image_str = node.get(tag.xlink.href)
    if image_str is None:
        state.add_warning("Ignored image element with no data.")
        return

    image_type, image_data = parse_embedded_image_data(image_str)
    bitmap_func = IMAGE_TYPE_TO_BITMAP.get(image_type)
    if bitmap_func is None:
        state.add_warning('Ignored image element with unsupported type: %s' %
            image_type)
        return

    bmp = bitmap_func(image_data)
    return state.props.Raster((x, y, w, h), bmp, state.settings)

def parse_image_custom(node, state):
    """Parses an SVG <image> element into a faint raster background if
    marked-up as such, otherwise as a Faint raster object.

    """
    if node.get(ns_faint('background')) is not None:
        return parse_image_as_background(node, state)
    else:
        return object_common(parse_image)(node, state)

def parse_line(node, state):
    """Parses an SVG <line> element."""
    state = state.updated(node)

    points = [svg_coord_attr(node.get(s, '0'), state)
           for s in ('x1', 'y1', 'x2', 'y2')]

    if state.settings.arrow == 'front':
        x0, y0, x1, y1 = points
        angle = rad_angle(x0,y0,x1,y1)
        x, y = arrow_line_end(x1, y1, angle, state.settings.linewidth)
        points[-2] = x
        points[-1] = y

    return state.props.Line(points, state.settings)

def parse_path_custom(node, state):
    """Parses an SVG <path> element as a Faint Ellipse if marked-up as
    such, otherwise as a Faint Path.

    """
    return (parse_path_as_ellipse(node, state) if is_faint_type(node, 'ellipse')
            else parse_path(node, state))

def parse_polygon_custom(node, state):
    """Parses an SVG <polygon> element as a Faint Rectangle if marked-up
    as such, otherwise as a Faint Polygon.

    """
    return (parse_polygon_as_rect(node, state) if is_faint_type(node, 'rect')
            else parse_polygon(node, state))

def parse_rect_as_background(node, state):
    """Parses a marked-up rect-node to a background color.

    """
    color = parse_color(node.get('fill'), '1.0', state)
    state.props.set_background((color, state.props.get_size()))

def parse_rect(node, state):
    """Parses an SVG <rect> element."""
    state = state.updated(node)

    get = getter_default(node, '0')

    x, y = svg_point_attr(get('x'), get('y'), state)
    w, h = svg_size_attr(get('width'), get('height'), state)

    state.settings.rx = svg_length_attr(get('rx'), state)
    state.settings.ry = svg_length_attr(get('ry'), state)

    r = state.props.Rect((x, y, w, h), state.settings)
    return r

def parse_rect_custom(node, state):
    """Parses an SVG <rect> element as the Canvas background color if
    marked-up as such, otherwise as a regular Rectangle.

    """
    if node.get(ns_faint('background')) is not None:
        return parse_rect_as_background(node, state)
    else:
        return object_common(parse_rect)(node, state)

def parse_switch(node, state):
    """Parses a switch node"""
    for child in node:
        if child.tag not in svg_switch_element_content_functions:
            continue

        requiredExtensions = child.get('requiredExtensions')
        if requiredExtensions is not None and requiredExtensions != "":
            # Faint supports no extensions.
            continue

        requiredFeatures = child.get('requiredFeatures')
        if requiredFeatures is not None:
            rfs = requiredFeatures.split(" ")
            unsupported = [rf.strip() for rf in rfs if
                           rf not in supported_svg_features]
            if len(unsupported) != 0:
                # At least one required feature was unsupported.
                continue

        systemLanguage = child.get('systemLanguage', state.system_language)
        if systemLanguage == state.system_language:
            return svg_switch_element_content_functions[child.tag](child, state)

    state.add_warning("No supported child node of SVG switch-element")

def _not_implemented(node, state, *args): #pylint:disable=unused-argument
    if _DEBUG_SVG:
        state.add_warning("Support not implemented: %s" % node.tag)

def _not_implemented_marker(node, state, *args): #pylint:disable=unused-argument
    if _DEBUG_SVG:
        state.add_warning("Support not implemented: %s" % node.tag)
    return 'marker'

# Dictionary of functions for converting embedded image-element data
# to a Bitmap. The data must be decoded from e.g. base64 first.
IMAGE_TYPE_TO_BITMAP = {
    "jpeg": faint.bitmap_from_jpg_string,
    "png": faint.bitmap_from_png_string,
}

# pylint:disable=invalid-name

svg_animation_element_functions = {
    ns_svg('animate') : _not_implemented,
    ns_svg('animateColor') : _not_implemented,
    ns_svg('animateMotion') : _not_implemented,
    ns_svg('animateTransform'): _not_implemented,
    ns_svg('set'): _not_implemented
}

svg_descriptive_element_functions = {
    ns_svg('desc'): _not_implemented,
    ns_svg('metadata'): _not_implemented,
    ns_svg('title'): _not_implemented
}

svg_gradient_element_functions = {
    # SVG 1.1
    ns_svg('linearGradient') : parse_linear_gradient_node,
    ns_svg('radialGradient') : parse_radial_gradient_node
}

def object_common(func):
    """Wraps an object-parse function, and adds common functionality,
    instead of scattering it in the individual parse functions.

    """
    def update_object(node, state):
        obj_id = func(node, state)
        if obj_id is not None:
            state.transform_object(obj_id)
            set_name(node, obj_id, state.props)
        return obj_id

    return update_object

svg_shape_element_functions = {
    tag.circle : object_common(parse_circle),
    tag.ellipse : object_common(parse_ellipse),
    tag.line : object_common(parse_line),
    tag.path : object_common(parse_path_custom),
    tag.polygon: object_common(parse_polygon_custom),
    tag.polyline : object_common(parse_polyline),
    tag.rect : parse_rect_custom,
}

svg_structural_element_functions = {
    tag.defs: parse_defs,
    tag.g: object_common(parse_group),
    tag.svg: _not_implemented, # nested SVG
    tag.symbol: _not_implemented,
    tag.use: _not_implemented
}

svg_content_functions = dict_union(
    svg_animation_element_functions,
    svg_descriptive_element_functions,
    svg_shape_element_functions,
    svg_structural_element_functions,
    svg_gradient_element_functions,
    {
        tag.a: _not_implemented,
        ns_svg('altGlyphDef'): _not_implemented,
        ns_svg('clipPath'): _not_implemented,
        ns_svg('color-profile'): _not_implemented,
        ns_svg('cursor'): _not_implemented,
        ns_svg('filter'): _not_implemented,
        ns_svg('font'): _not_implemented,
        ns_svg('font-face'): _not_implemented,
        tag.foreignObject: _not_implemented,
        tag.image : parse_image_custom,
        ns_svg('marker'): _not_implemented_marker,
        ns_svg('mask'): _not_implemented,
        ns_svg('pattern'): _not_implemented,
        ns_svg('script'): _not_implemented,
        ns_svg('style'): _not_implemented,
        tag.switch: parse_switch,
        tag.text: object_common(parse_text),
        tag.view: _not_implemented
    }
)

if _DEBUG_SVG:
    ignored_svg_group_content = []
else:
    ignored_svg_group_content = [
        # Exclude certain elements unsupported by Faint in group nodes,
        # to remove warnings about failing parsing a child for hopefully
        # less important children
        ns_svg('title')
    ]

svg_group_content_functions = dict_union(
    svg_animation_element_functions,
    svg_descriptive_element_functions,
    svg_shape_element_functions,
    svg_structural_element_functions,
    svg_gradient_element_functions,
    {
        tag.a: _not_implemented,
        ns_svg('altGlyphDef'): _not_implemented,
        ns_svg('clipPath'): _not_implemented,
        ns_svg('color-profile') : _not_implemented,
        ns_svg('cursor'): _not_implemented,
        ns_svg('filter'): _not_implemented,
        ns_svg('font'): _not_implemented,
        ns_svg('font-face'): _not_implemented,
        tag.foreignObject: _not_implemented,

        # Fixme: Shouldn't image in group go directly for the
        # non-background parsing?
        tag.image: parse_image_custom,

        ns_svg('marker'): _not_implemented_marker,
        ns_svg('mask'): _not_implemented,
        ns_svg('pattern'): _not_implemented,
        ns_svg('script'): _not_implemented,
        ns_svg('style'): _not_implemented,
        tag.switch: parse_switch,
        tag.text: object_common(parse_text),
        tag.view: _not_implemented,
    })

included_svg_group_content_functions = {key: svg_group_content_functions[key]
                                        for key in svg_group_content_functions
                                        if key not in ignored_svg_group_content}

svg_switch_element_content_functions = dict_union(
    svg_animation_element_functions,
    svg_descriptive_element_functions,
    svg_shape_element_functions,
    {
        tag.a: _not_implemented,
        tag.foreignObject: _not_implemented,
        tag.g: object_common(parse_group),
        tag.image: object_common(parse_image),
        tag.svg: _not_implemented, # Nested SVG
        tag.switch: parse_switch,
        tag.svg: object_common(parse_text),
        tag.use: _not_implemented,
    })

svg_descriptive_element_functions = {
    ns_svg('desc'): _not_implemented,
    ns_svg('metadata'): _not_implemented,
    tag.title: _not_implemented,
}

svg_defs_content_functions = dict_union(
    svg_animation_element_functions,
    svg_descriptive_element_functions,
    # todo: shape
    # todo: structural
    svg_structural_element_functions,
    svg_gradient_element_functions,
    {
        tag.a: _not_implemented,
        ns_svg('altGlyphDef'): _not_implemented,
        ns_svg('clipPath'): _not_implemented,
        ns_svg('colorProfile'): _not_implemented,
        ns_svg('cursor'): _not_implemented,
        ns_svg('filter'): _not_implemented,
        ns_svg('font'): _not_implemented,
        ns_svg('font-face'): _not_implemented,
        tag.foreignObject: _not_implemented,
        tag.image: _not_implemented,
        ns_svg('marker'): _not_implemented_marker,
        ns_svg('mask'): _not_implemented,
        ns_svg('pattern'): parse_pattern_node,
        ns_svg('script'): _not_implemented,
        ns_svg('style'): _not_implemented,
        ns_svg('switch'): parse_switch,
        tag.text: _not_implemented,
        tag.view: _not_implemented,
        tag.faint.calibration: parse_faint_calibration,
        tag.faint.grid: parse_faint_grid,
    })

supported_svg_features = [

    # Not sure, group
    "http://www.w3.org/TR/SVG11/feature#SVG",

    # Not sure
    "http://www.w3.org/TR/SVG11/feature#SVGDOM",

    # Not totally.
    "http://www.w3.org/TR/SVG11/feature#SVG-static",

    # Probably untrue
    "http://www.w3.org/TR/SVG11/feature#SVGDOM-static",

    # No animation support at all
    #"http://www.w3.org/TR/SVG11/feature#SVG-animation",
    #"http://www.w3.org/TR/SVG11/feature#SVGDOM-animation",
    #"http://www.w3.org/TR/SVG11/feature#SVG-dynamic",
    #"http://www.w3.org/TR/SVG11/feature#SVGDOM-dynamic",

    # Support for attributes:
    # id, xml:base, xml:space attributes
    #"http://www.w3.org/TR/SVG11/feature#CoreAttribute",

    # Support for elements:
    # svg, g, defs, desc, title, metadata, symbol, use
    "http://www.w3.org/TR/SVG11/feature#Structure",

    # Support for elements:
    # svg, g, defs, desc, title, metadata, use
    "http://www.w3.org/TR/SVG11/feature#BasicStructure",

    # Support for enable-background property
    #"http://www.w3.org/TR/SVG11/feature#ContainerAttribute",

    # Support for the switch element, and the attributes:
    # requiredFeatures, requiredExtensions, systemLanguage
    "http://www.w3.org/TR/SVG11/feature#ConditionalProcessing",

    # Support for the image element
    "http://www.w3.org/TR/SVG11/feature#Image",

    # Support for the style element
    # (Not true)
    "http://www.w3.org/TR/SVG11/feature#Style",

    # Support for clip and overflow properties of viewport
    #"http://www.w3.org/TR/SVG11/feature#ViewportAttribute",

    # Support for the shape elements:
    # rect, circle, line, polyline, polygon, ellipse, path.
    "http://www.w3.org/TR/SVG11/feature#Shape",

    # Support for text, tspan, tref, textPath, altGlyph, altGlyphDef,
    # altGlyphItem, glyphRef.
    "http://www.w3.org/TR/SVG11/feature#Text",

    # Support for the text element.
    "http://www.w3.org/TR/SVG11/feature#BasicText",

    # Support for the properties:
    # color, fill, fill-rule, stroke, stroke-dasharray,
    # stroke-dashoffset, stroke-linecap, stroke-linejoin,
    # stroke-miterlimit, stroke-width, color-interpolation,
    # color-rendering.
    "http://www.w3.org/TR/SVG11/feature#PaintAttribute",

    # Support for the properties: color, fill, fill-rule, stroke,
    # stroke-dasharray, stroke-dashoffset, stroke-linecap,
    # stroke-linejoin, stroke-miterlimit, stroke-width,
    # color-rendering.
    "http://www.w3.org/TR/SVG11/feature#BasicPaintAttribute",

    # Support for opacity, stroke-opacity, fill-opacity
    # Fixme: 'opacity'? Check that out.
    "http://www.w3.org/TR/SVG11/feature#OpacityAttribute",

    # Support for the properties display, image-rendering,
    # pointer-events, shape-rendering, text-rendering visibility
    "http://www.w3.org/TR/SVG11/feature#GraphicsAttribute",

    # Support for display and visibility properties
    "http://www.w3.org/TR/SVG11/feature#BasicGraphicsAttribute",

    # Support for the marker element
    #"http://www.w3.org/TR/SVG11/feature#Marker",

    # Support for the color-profile element
    #"http://www.w3.org/TR/SVG11/feature#ColorProfile",

    # Support for elements linearGradient, radialGradient and stop
    "http://www.w3.org/TR/SVG11/feature#Gradient",

    # Support for the pattern element
    #"http://www.w3.org/TR/SVG11/feature#Pattern",

    # Support for the clipPath element and clip-path, clip-rule
    # properties
    #"http://www.w3.org/TR/SVG11/feature#Clip",

    # Support for the clipPath element and clip-path property
    #"http://www.w3.org/TR/SVG11/feature#BasicClip",

    # Support for the mask element
    #"http://www.w3.org/TR/SVG11/feature#Mask",

    # Support for the elements filter, feBlend, feColorMatrix,
    # feComponentTransfer, feComposite, feConvolveMatrix,
    # feDiffuseLighting, feDisplacementMap, feFlood, feGaussianBlur,
    # feImage, feMerge, feMergeNode, feMorphology, feOffset,
    # feSpecularLighting, feTile, feDistantLight, fePointLight,
    # feSpotLight, feFuncR, feFuncG, feFuncB, feFuncA.
    #"http://www.w3.org/TR/SVG11/feature#Filter",

    # Support for the elements filter, feBlend, feColorMatrix,
    # feComponentTransfer, feComposite, feFlood, feGaussianBlur,
    # feImage, feMerge, feMergeNode, feOffset, feTile, feFuncR,
    # feFuncG, feFuncB, feFuncA.
    #"http://www.w3.org/TR/SVG11/feature#BasicFilter",

    # Support for attributes onunload, onabort, onerror, onresize,
    # onscroll and onzoom.
    #"http://www.w3.org/TR/SVG11/feature#DocumentEventsAttribute",

    # Support for the attributes onfocusin, onfocusout, onactivate, onclick,
    # onmousedown, onmouseup, onmouseover, onmousemove, onmouseout,
    # onload.
    #"http://www.w3.org/TR/SVG11/feature#GraphicalEventsAttribute",

    # Support for the attributes onbegin, onend, onrepeat, onload
    #"http://www.w3.org/TR/SVG11/feature#AnimationEventsAttribute",

    # Support for the cursor element
    #"http://www.w3.org/TR/SVG11/feature#Cursor",

    # Support for the a element.
    #"http://www.w3.org/TR/SVG11/feature#Hyperlinking",

    # Support for the xlink:type, xlink:href, xlink:role,
    # xlink:arcrole, xlink:title, xlink:show and xlink:actuate
    # elements.
    # (Fixme: Only xlink:href in rare cases)
    "http://www.w3.org/TR/SVG11/feature#XlinkAttribute",

    # Support for the externalResourcesRequired attribute
    #"http://www.w3.org/TR/SVG11/feature#ExternalResourcesRequired",

    # Support for the view element.
    #"http://www.w3.org/TR/SVG11/feature#View",

    # Support for the script element.
    #"http://www.w3.org/TR/SVG11/feature#Script",

    # Support for the animate, set, animateMotion, animateTransform,
    # animateColor and mpath elements.
    #"http://www.w3.org/TR/SVG11/feature#Animation",

    # Support for the font, font-face, glyph, missing-glyph, hkern,
    # vkern, font-face-src, font-face-uri, font-face-format,
    # font-face-name elements.
    # (Fixme: Very limited support)
    "http://www.w3.org/TR/SVG11/feature#Font",

    # Support for the elements font, font-face, glyph, missing-glyph,
    # hkern, font-face-src, font-face-name.
    "http://www.w3.org/TR/SVG11/feature#BasicFont",

    # The foreignObject element
    #"http://www.w3.org/TR/SVG11/feature#Extensibility",
]

# pylint:enable=invalid-name

def svg_size_attr(w_str, h_str, state):
    """Parses a pair of coordinates, scaling percentages."""

    return (svg_length_attr_dumb(w_str, state.props, state.viewPort[2]),
            svg_length_attr_dumb(h_str, state.props, state.viewPort[3]))

def svg_point_attr(x_str, y_str, state):
    """Parses a pair of coordinates, scaling percentages."""

    return (svg_length_attr_dumb(x_str, state.props, state.viewPort[2]),
            svg_length_attr_dumb(y_str, state.props, state.viewPort[3]))

def svg_coord_attr(coord_str, state):
    """Parses an svg coordinate attribute."""
    # Fixme: Duplicates length_attr
    m = re.match(svg_re.coordinate, coord_str)
    if m is None:
        raise SvgCoordError(coord_str)
    value, unit = m.groups()
    if unit == '%':
        # Fixme: How should this work?
        if float(value) == 0:
            return 0
        else:
            return (state.viewPort[2] * 100) / float(value)
    elif unit in ABSOLUTE_UNIT_FACTORS:
        return float(value) * ABSOLUTE_UNIT_FACTORS[unit]
    elif unit in ('em', 'ex'):
        state.add_warning("Unsupported unit: %s" % unit)
        return float(value)
    else:
        state.add_warning("Invalid unit: %s" % unit)
        return float(value)

def check_svg_size(size):
    """Raises exception if invalid size for SVG node."""
    if size[0] <= 0:
        raise faint.LoadError("SVG element has negative width")
    elif size[1] <= 0:
        raise faint.LoadError("SVG element has negative height")
    return size

# Fixme: Use image width, height for default viewbox
def get_viewbox(node):
    """Returns the viewbox from the node as a sequence of four strings,
    x, y, w, h."""

    # Available for: svg, symbol (via use), image, foreignObject
    viewBox = node.get('viewBox') # pylint:disable=invalid-name
    if viewBox is None:
        return None

    # Fixme: Handle preserveAspectRatio
    # Fixme: Can also use commas
    x, y, w, h = [float(v) for v in viewBox.split(' ')]
    return x, y, w, h

def establish_viewPort(svg_node, image_props, view_box):
    # Use the viewBox width and height if width, height not specified
    if view_box is not None:
        default_w = str(view_box[2])
        default_h = str(view_box[3])
    else:
        default_w = "640.0"
        default_h = "480.0"

    svg_w = svg_node.get('width', None)
    svg_h = svg_node.get('height', None)

    if svg_w is None:
        svg_w = default_w
    if svg_h is None:
        svg_h = default_h

    # Fixme: I guess viewbox should be converted, not float:ed
    w2 = svg_length_attr_dumb(svg_w, image_props, full_span=float(default_w))
    h2 = svg_length_attr_dumb(svg_h, image_props, full_span=float(default_h))
    w2, h2 = check_svg_size((w2, h2))

    return [0, 0, w2, h2]

def parse_svg_root_node(svg_node, image_props, system_language):
    """Parses the SVG-root node and recurses into child nodes. Creates the
    frame from the SVG root node properties.

    """

    # Fixme: Check if SVG element can have transform (Probably)
    viewBox = get_viewbox(svg_node)
    viewPort = establish_viewPort(svg_node, image_props, viewBox)
    if viewBox is None:
        viewBox = viewPort

    frame_props = image_props.add_frame((viewPort[2], viewPort[3]))

    # Fixme: What of translation?
    ctm = Matrix.scale(viewPort[2] / viewBox[2], viewPort[3] / viewBox[3])

    state = ParseState(image_props,
                       frame_props,
                       ctm=ctm,
                       viewPort=viewPort,
                       viewBox=viewBox,
                       system_language=system_language)

    for child in svg_node:
        if child.tag in svg_content_functions:
            # Fixme: I guess there could be a transform already?
            svg_content_functions[child.tag](child, state)

def parse_svgz_file(path, image_props, system_language):
    with gzip.open(path, 'rb') as f:
        file_content = f.read()
    return from_string(file_content, image_props)

def _parse_svg_file(path, image_props, system_language):
    ET.register_namespace("svg", "{http://www.w3.org/2000/svg}")
    tree = ET.parse(path)
    root = tree.getroot()
    if root.tag == ns_svg('svg'):
        parse_svg_root_node(root, image_props, system_language)
    else:
        raise faint.LoadError("Root element was not <svg>.")

def parse_doc(path, image_props, system_language="en"):
    """Parses the SVG document at the given file path, using the
    image_props to build an image.

    """

    try:
        if path.endswith(".svgz"):
            parse_svgz_file(path, image_props, system_language)
        else:
            _parse_svg_file(path, image_props, system_language)

    except svg_error as e:
        raise faint.LoadError(str(e))

def from_string(xml_string, image_props, system_language="en"):
    """Parses the SVG document in the given string, using the image_props
    to build an image.

    """

    try:
        ET.register_namespace("svg", "{http://www.w3.org/2000/svg}")
        root = ET.fromstring(xml_string)
        if root.tag == ns_svg("svg"):
            parse_svg_root_node(root, image_props, system_language)
        else:
            raise faint.LoadError("Root element was not <svg>.")
    except svg_error as e:
        raise faint.LoadError(str(e))


#!/usr/bin/env python3
from subprocess import getoutput
from slacker import Slacker

# slack API トークン
slack_api_token = "xoxp-XXX"
# slack で投稿先のチャンネル名
slack_channel_name="seminar_raspi-testspace"

MESSAGE_1 = "メッセージ送信てすと"

# SlackAPIを使ってメッセージをslackに投稿する
print ("Sending message..")
slacker = Slacker(slack_api_token)
channel_name = "#" + slack_channel_name
slacker.chat.post_message(channel_name, MESSAGE_1)

print ("Done! Check slack channel")


import serdespy as sdp
import numpy as np
import matplotlib.pyplot as plt
import skrf as rf
import scipy as sp

h = np.load("./data/h_thru.npy")
h_ctle = np.load("./data/h_ctle.npy")
ffe_tap_weights = np.load("./data/rx_ffe_tap_weights.npy")
dfe_tap_weights = np.load("./data/rx_dfe_tap_weights.npy")
tx_fir_tap_weights = np.load("./data/tx_fir_tap_weights.npy")

nyquist_f = 26.56e9

samples_per_symbol = 64

voltage_levels = np.array([-3,-1,1,3])

data = sdp.prqs10(1)

TX = sdp.Transmitter(data, voltage_levels, nyquist_f)

TX.FIR(tx_fir_tap_weights)

TX.oversample(samples_per_symbol)

TX.gaussian_jitter(stdev_div_UI=0.025)

TX.tx_bandwidth(freq_bw=120e9)

sdp.simple_eye(TX.signal, samples_per_symbol*3, 1000, TX.UI/TX.samples_per_symbol, "TX Bandwidth-Limited Eye Diagram (-3dB frequency at 100GHz)")
#%%
signal_out = sp.signal.fftconvolve(TX.signal, h, mode = "same")

#xtalk = np.load("./data/xt_response.npy")[:384000]

#signal_out_ctle = sp.signal.fftconvolve(signal+xtalk, h_ctle, mode = "same")
signal_out_ctle = sp.signal.fftconvolve(signal_out, h_ctle, mode = "same")
#%%
sdp.simple_eye(signal_out_ctle, samples_per_symbol*3, 1000, TX.UI/TX.samples_per_symbol, "Eye Diagram with CTLE")

RX = sdp.Receiver(signal_out_ctle, samples_per_symbol, nyquist_f, voltage_levels, shift = True, main_cursor = 0.5006377086200736)

RX.slice_signal()
#%%
RX.signal_BR =  RX.signal_BR + np.random.normal(scale=0.1, size = RX.signal_BR.size)

sdp.simple_eye(RX.signal_BR, 3, 1000, TX.UI, "Eye Diagram with CTLE and noise BR")
#%%
RX.FFE_BR(ffe_tap_weights, 3)

sdp.simple_eye(RX.signal_BR, 3, 800, TX.UI/TX.samples_per_symbol, "Eye Diagram after FFE")
#%%
RX.pam4_DFE_BR(dfe_tap_weights)

sdp.simple_eye(RX.signal_BR, 3, 800, TX.UI/TX.samples_per_symbol, "Eye Diagram after DFE")

#%%
err = sdp.prqs_checker(10, data, RX.symbols_out[10:-10])

print("Bits Transmitted =", RX.symbols_out[10:-10].size*2, 'Bit Errors =', err[0])

print("Bit Error Ratio = ", err[0]/(RX.symbols_out[10:-10].size*2))


import pymongo
import redis

class RedisBackend:

	__newUrlKey = 'newUrl'
	__crawledUrlKey = 'CrawledUrl'

	__port = 6379
	__hostName = '54.183.40.123'

	redisOperator = redis.Redis(host=__hostName,port=__port)
	pub = redisOperator.pubsub()

	def __init__(self):
		pass

	# this function only serve for spider
	def pushUrls(self,urls):
		# urls: list of url
		for url in urls:
			# self.redisOperator.sadd(self.__newUrlKey,url)
			self.redisOperator.publish(self.__newUrlKey,url)

	# this function only seve for master
	def pullUrls(self):

		# pipeline as multi, transaction = false
		pipe = self.redisOperator.pipeline(transaction=False)
		pipe.smembers(self.__newUrlKey)
		pipe.delete(self.__newUrlKey)
		urls = pipe.execute()[0]
		urls = list(map(lambda u: u.decode('utf-8'),urls))
		return urls

class MongoBackend:

	def __init__(self):
		self.myClient = pymongo.MongoClient('mongodb://54.193.51.217')

	# insert one json object in USC collection
	def insertUSC(self, content):
		crawlerDB = self.myClient['crawler']
		usc = crawlerDB['USC']
		usc.insert_one(content)


import torch.optim as optim
import torch
import math
import os
import sys
from utils.datasets import get_train_data, get_test_query, get_test_db
from utils.model import MyAlexNet, pretrain_alexnet, SGDWoReplace, AttnNet, copy_mask_MyAlexNet
from utils.misc import load_dict
from utils.train_utils import copy_params, reset_model, schedule_dict
from utils.test_utils import run_test
from utils.hashloss import sim_log_loss_double

sys.path.append(os.path.dirname(__file__))

def train(config):
    autocuda = config["autocuda"]

    test_period = config["test_period"]

    init_scale = 1.0
    scale = init_scale
    scale_step_size = 200
    scale_iter_num = 0
    scale_gamma = 0.005
    scale_power = 0.5

    train_loader_1 = get_train_data(config["data"]["train_set"])
    train_loader_2 = get_train_data(config["data"]["train_set"], seed=3571)

    query_loader = get_test_query(config["data"]["query_set"], config["test_10crop"])
    db_loader = get_test_db(config["data"]["db_set"], config["test_10crop"])

    net = pretrain_alexnet(autocuda(MyAlexNet(num_bits=config["num_bits"])))
    net.train(True)
    net_shadow = autocuda(MyAlexNet(num_bits=config["num_bits"]))
    copy_params(net, net_shadow)
    attn_net = autocuda(AttnNet(hidden_size=100))
    attnnet_optimizer = optim.Adam(attn_net.parameters(), lr=config["attn_lr"])

    # torch.save(net.hash_layer.state_dict(), os.path.join("./saved_model/init/", "hash_layer64".format("model")))
    # torch.save(net.hash_layer.state_dict(), os.path.join("./saved_model/init/", "hash_layer64".format("model")))
    # torch.save(attn_net.state_dict(), os.path.join("./saved_model/init/", "attn".format("model")))

    hash_layer_dict = torch.load(os.path.join("./saved_model/init/", ("hash_layer"
                                                                       + str(config["num_bits"])).format("model"))
                                 , map_location=lambda storage, loc: storage)
    attn_net_dict = torch.load(os.path.join("./saved_model/init/", "attn".format("model")),
                               map_location=lambda storage, loc: storage)

    load_dict(net.hash_layer, hash_layer_dict)
    load_dict(attn_net, attn_net_dict)

    optimizer_config = config["optimizer"]
    parameter_list = [{"params": net_shadow.feature_layers.parameters(), "lr": 1},
                      {"params": net_shadow.hash_layer.parameters(), "lr": 10}]
    optimizer = SGDWoReplace(parameter_list, **(optimizer_config["optim_params"]))

    param_lr = []
    for param_group in optimizer.param_groups:
        param_lr.append(param_group["lr"])

    schedule_param = optimizer_config["lr_param"]
    lr_scheduler = schedule_dict[optimizer_config["lr_type"]]
    j = 0
    running_loss = 0.0
    running_attn_loss = 0.0

    for epoch in range(config["epoch"]):  # loop over the dataset multiple times

        for i, ((data1, labels1), (data2, labels2)) in enumerate(zip(train_loader_1, train_loader_2), 0):

            net.train()
            net_shadow.train()

            data1, labels1 = autocuda(data1), autocuda(labels1)
            data2, labels2 = autocuda(data2), autocuda(labels2)

            # zero the parameter gradients
            optimizer = lr_scheduler(param_lr, optimizer, j + i, **schedule_param)
            net.zero_grad()
            net.clear_dropout_mask()
            reset_model(net_shadow)
            attnnet_optimizer.zero_grad()

            scale_iter_num += 1
            if scale_iter_num % scale_step_size == 0:
                scale = init_scale * math.pow((1. + scale_gamma * scale_iter_num), scale_power)

            # forward + backward + optimize
            batch_size = data1.size(0)
            data = torch.torch.cat((data1, data2), dim=0)
            feat = net(data)
            feats1, feats2 = torch.split(feat, batch_size, dim=0)
            outputs1 = torch.tanh(scale * feats1)
            outputs2 = torch.tanh(scale * feats2)

            attn_net.get_cont_code_1(outputs1)
            attn_net.get_cont_code_2(outputs2)
            h1 = outputs1.register_hook(attn_net.set_code_grad_1)
            h2 = outputs2.register_hook(attn_net.set_code_grad_2)
            loss = sim_log_loss_double(outputs1, labels1, outputs2, labels2,
                                       positive_weight=config["loss"]["positive_weight"],
                                       sigmoid_param=config["loss"]["sigmoid_param"],
                                       hook=attn_net.get_sim_grad)

            loss.backward(create_graph=True)
            optimizer.step(net, net_shadow)

            copy_params(net_shadow, net)

            h1.remove()
            h2.remove()

            if epoch <= 1000:
                copy_mask_MyAlexNet(net, net_shadow)
                feat = net_shadow(data)
                feats1, feats2 = torch.split(feat, batch_size, dim=0)

                outputs3 = torch.tanh(scale * feats1)
                outputs4 = torch.tanh(scale * feats2)

                attn_loss = sim_log_loss_double(outputs3, labels1, outputs4, labels2,
                                                sigmoid_param=config["loss"]["sigmoid_param"],
                                                positive_weight=config["loss"][
                                                    "positive_weight"])

                attn_loss.backward()
                attnnet_optimizer.step()

            # print statistics
            running_loss += loss.item()
            running_attn_loss += attn_loss.item()
            if (i + j) % test_period == (test_period - 1):

                # quickly test the mAP by using only 1000 query instances. For evaluation, num_query should set as None.
                m_a_p= run_test(query_loader, db_loader, net.eval(),autocuda, test_10crop=config["test_10crop"],
                                                                          num_query=1000,
                                                                          num_top_return=config["test"]["top_return"])

                print('[%d, %5d] loss: %.3f; loss: %.3f; MAP: %.4f' %
                      (epoch + 1, i + 1, running_loss / test_period, running_attn_loss / test_period, m_a_p))

                if config["save_model"]:
                    torch.save(net.state_dict(), os.path.join(config["output_path"], "alexnet".format("model")))

                running_loss = 0.0
                running_attn_loss = 0.0
                net.train(True)
        j = j + i + 1

    print('Training is complete')


import os

import numpy as np
from utils import read_csv_in_dict, find_crops, read_csv_in_dict_double, windowing,binary_cam
import shutil
from torch.utils.data import Dataset
import random
import SimpleITK as sitk
from collections import defaultdict
import glob
from pathlib import Path

class COPDGeneSubtypingLobeChunk(Dataset):
    ON_PREMISE_ROOT = None

    @classmethod
    def get_series_uids(cls, csv_file):
        scan_selected, _ = read_csv_in_dict(csv_file, 'uid')
        return sorted(list(scan_selected.keys()))

    @classmethod
    def get_series_uids_unique_scans(cls, csv_file):
        metas, _ = read_csv_in_dict(csv_file, 'uid')
        scan_lobe_map = defaultdict(list)
        for lobe_wise_uid in metas.keys():
            scan_lobe_map[lobe_wise_uid[:-2]].append(lobe_wise_uid[-1])

        selected_uids = [f"{scan_uid}-{random.sample(scan_lobe_map[scan_uid], 1)[0]}" for scan_uid in
                         scan_lobe_map.keys()]
        return sorted(selected_uids)

    def __init__(self, archive_path, uids, keep_sorted=True,
                 transforms=None):
        super(COPDGeneSubtypingLobeChunk, self).__init__()
        self.archive_path = archive_path
        self.meta_csv = archive_path + '/memo.csv'
        self.meta, _ = read_csv_in_dict(self.meta_csv, "uid")
        self.uids = uids
        self.transforms = transforms
        if keep_sorted:
            self.uids = sorted(self.uids)

    def __len__(self):
        return len(self.uids)

    def __getitem__(self, index):
        d = self.get_data(self.uids[index])
        return d

    def read_file(self, uid, path):
        file_path = os.path.join(path, f"{uid}.mha")
        sitk_image = sitk.ReadImage(file_path)
        spacing = sitk_image.GetSpacing()[::-1]
        origin = sitk_image.GetOrigin()[::-1]
        direction = np.asarray(sitk_image.GetDirection()).reshape(3, 3)[::-1].flatten().tolist()
        scan = sitk.GetArrayFromImage(sitk_image)
        return scan, origin, spacing, direction

    def get_data(self, uid):
        scan, origin, spacing, direction = self.read_file(uid, self.archive_path + '/images/')
        lobe, *ignored = self.read_file(uid, self.archive_path + '/lobes/')
        lesion, *ignored = self.read_file(uid, self.archive_path + '/lesions/')

        base_dict = dict(self.meta[uid])
        base_dict.update({"size": scan.shape,
                          "spacing": spacing,
                          'original_spacing': spacing,
                          'original_size': scan.shape,
                          "origin": origin,
                          "direction": direction})
        ret = {
            "#image": scan.astype(np.int16),
            "#lobe_reference": lobe.astype(np.uint8),
            "#lesion_reference": lesion.astype(np.uint8),
            "meta": base_dict
        }
        if self.transforms:
            ret = self.transforms(ret)
        return ret

class COPDGeneSubtyping(Dataset):
    ON_PREMISE_ROOT = None

    @classmethod
    def get_series_uids(cls, csv_file):
        scan_selected, _ = read_csv_in_dict(csv_file, 'SeriesInstanceUID')
        return sorted(list(scan_selected.keys()))

    def __init__(self, archive_path, series_uids,
                 transforms=None, keep_sorted=True, crop_border=5, emphysema_th=-950):
        super(COPDGeneSubtyping, self).__init__()
        self.archive_path = archive_path
        self.keep_sorted = keep_sorted
        self.transforms = transforms
        self.emphysema_th = emphysema_th
        self.lobe_path = archive_path + '/derived/seg-lobes-copdgene-approved_Lobes/mha/'
        meta_csv = archive_path + '/meta/ctss.csv'

        self.meta, _ = read_csv_in_dict(meta_csv, "SeriesInstanceUID")
        self.crop_border = crop_border
        if not self.keep_sorted:
            self.series_uids = random.sample(series_uids, len(series_uids))
        else:
            self.series_uids = sorted(series_uids)

        self.subtyping_labels = {}

        for series_uid in series_uids:
            cle = int(float(self.meta[series_uid]['CT_Visual_Emph_Severity_P1']))
            pse = int(float(self.meta[series_uid]['CT_Visual_Emph_Paraseptal_P1']))

            self.subtyping_labels[series_uid] = {
                "cle": cle,
                "pse": pse
            }

    def __len__(self):
        return len(self.series_uids)

    def __getitem__(self, index):
        d = self.get_data(self.series_uids[index])
        return d

    def read_image(self, path):
        sitk_image = sitk.ReadImage(path)
        spacing = sitk_image.GetSpacing()[::-1]
        origin = sitk_image.GetOrigin()[::-1]
        direction = np.asarray(sitk_image.GetDirection()).reshape(3, 3)[::-1].flatten().tolist()
        scan = sitk.GetArrayFromImage(sitk_image)
        return scan, origin, spacing, direction

    def get_lobe(self, series_uid):
        reference_file_path = self.lobe_path + f"/{series_uid}.mha"
        if self.ON_PREMISE_ROOT is not None:
            target_path = os.path.join(self.ON_PREMISE_ROOT, "lobes")
            if not os.path.exists(target_path):
                os.makedirs(target_path)
            on_premise_path = os.path.join(target_path, "{}.mha".format(series_uid))
            try:
                if not os.path.exists(on_premise_path):
                    on_premise_path = shutil.copyfile(reference_file_path, on_premise_path)
                f_ = self.read_image(on_premise_path)
            except Exception as e:
                print(
                    "loading file or copying  at {} failed with {},"
                    " now read from {}.".format(on_premise_path, e, reference_file_path))
                f_ = self.read_image(reference_file_path)
            return f_
        return self.read_image(reference_file_path)

    def get_scan(self, series_uid):
        series_scan_path = self.archive_path + f"/{series_uid}.mha"
        if self.ON_PREMISE_ROOT is not None:
            if not os.path.exists(self.ON_PREMISE_ROOT):
                os.makedirs(self.ON_PREMISE_ROOT)
            on_premise_path = os.path.join(self.ON_PREMISE_ROOT, "{}.mha".format(series_uid))
            try:
                if not os.path.exists(on_premise_path):
                    # print("copying file to on premise path {}.".format(on_premise_path))
                    on_premise_path = shutil.copyfile(series_scan_path, on_premise_path)
                f_ = self.read_image(on_premise_path)
            except Exception as e:
                print(
                    "loading file or copying  at {} failed with {},"
                    " now read from {}.".format(on_premise_path, e, series_scan_path))
                f_ = self.read_image(series_scan_path)
            return f_

        return self.read_image(series_scan_path)

    def get_data(self, series_uid):
        scan, origin, spacing, direction = self.get_scan(series_uid)
        original_size = scan.shape
        lobe, origin_, spacing_, direction_ = self.get_lobe(series_uid)
        lung = lobe > 0

        assert lobe.shape == scan.shape
        slices = find_crops(lung, spacing, self.crop_border)
        scan = scan[slices]
        lung = lung[slices].astype(np.uint8)
        lobe = lobe[slices].astype(np.uint8)
        es = (np.logical_and(scan < self.emphysema_th, lung > 0)).astype(np.uint8)

        base_dict = {
            'uid': series_uid,
        }
        base_dict.update({"size": scan.shape,
                          "spacing": spacing,
                          "crop_slices": slices,
                          'LAA': self.emphysema_th,
                          'original_spacing': spacing,
                          'original_size': original_size,
                          "origin": origin,
                          "direction": direction,
                          "cle": self.subtyping_labels[series_uid]['cle'],
                          "pse": self.subtyping_labels[series_uid]['pse'],
                          })
        ret = {
            "#image": scan.astype(np.int16),
            "#lobe_reference": lobe.astype(np.uint8),
            "#lesion_reference": es.astype(np.uint8),
            "meta": base_dict
        }
        if self.transforms:
            ret = self.transforms(ret)
        return ret

class TestDataset(Dataset):

    def __init__(self, scan_path, lobe_path, transforms=None, keep_sorted=True,
                 crop_border=5):
        super(TestDataset, self).__init__()
        self.scan_path = scan_path
        self.lobe_path = lobe_path
        self.crop_border = crop_border
        self.transforms = transforms
        all_scans = glob.glob(self.scan_path + '/*.mha', recursive=False)
        if keep_sorted:
            self.series_uids = sorted([Path(scan_file).stem for scan_file in all_scans])
        else:
            self.series_uids = [Path(scan_file).stem for scan_file in all_scans]

    def __len__(self):
        return len(self.series_uids)

    def __getitem__(self, index):
        d = self.get_data(self.series_uids[index])
        return d

    def read_image(self, path):
        sitk_image = sitk.ReadImage(path)
        spacing = sitk_image.GetSpacing()[::-1]
        origin = sitk_image.GetOrigin()[::-1]
        direction = np.asarray(sitk_image.GetDirection()).reshape(3, 3)[::-1].flatten().tolist()
        scan = sitk.GetArrayFromImage(sitk_image)
        return scan, origin, spacing, direction

    def get_data(self, series_uid):
        scan, origin, spacing, direction = self.read_image(self.scan_path + f'/{series_uid}.mha')
        original_size = scan.shape
        lobe, origin_, spacing_, direction_ = self.read_image(self.lobe_path + f'/{series_uid}.mha')
        assert lobe.shape == scan.shape

        base_dict = {
            'uid': series_uid,
        }
        base_dict.update({"size": scan.shape,
                          "spacing": spacing,
                          'original_spacing': spacing,
                          'original_size': original_size,
                          "origin": origin,
                          "direction": direction,
                          })
        ret = {
            "#image": scan.astype(np.int16),
            "#lobe_reference": lobe.astype(np.uint8),
            "meta": base_dict
        }
        if self.transforms:
            ret = self.transforms(ret)
        return ret

class RadboudCOVID(Dataset):
    ON_PREMISE_ROOT = None

    ctss_cut_off = {
        (0.0, 0.01): 0,
        (0.01, 0.05): 1,
        (0.05, 0.25): 2,
        (0.25, 0.5): 3,
        (0.5, 0.75): 4,
        (0.75, 1.00001): 5
    }

    metric_k_mapping = {
        1: "lul [0-5]",
        2: "lll [0-5]",
        3: "rul [0-5]",
        4: "rll [0-5]",
        5: "rml [0-5]"
    }

    @classmethod
    def get_series_uids(cls, csv_file):
        scan_selected, _ = read_csv_in_dict_double(csv_file, ['patientid', 'study'])
        uids = [f"{pid}_{sid}" for pid, sid in scan_selected.keys()]
        return sorted(uids)

    def __init__(self, archive_path, uids,
                 transforms=None, keep_sorted=True, use_masked_scan=True,
                 crop_border=5, task='wss'):
        super(RadboudCOVID, self).__init__()
        self.transforms = transforms
        self.keep_sorted = keep_sorted
        self.archive_path = archive_path
        self.crop_border = crop_border
        self.use_masked_scan = use_masked_scan
        self.lobe_folder = archive_path + f'/{task}/lobes/'
        self.pseudo_vessel_folder = archive_path + f'/{task}/pseudo_vessels/'
        self.lesion_folder = archive_path + f'/{task}/lesion/'
        self.scan_folder = archive_path + f'/{task}/images/'
        csv_file = archive_path + f'/wss_all.csv'
        self.all_metas, _ = read_csv_in_dict_double(csv_file, ['patientid', 'study'])
        self.uids = uids
        print("dataset containing: {} cases.".format(len(self.uids)), flush=True)

        if keep_sorted:
            self.uids = sorted(self.uids)
        else:
            self.uids = random.sample(self.uids, len(self.uids))
        scan_files = glob.glob(self.scan_folder + '/*.mha', recursive=False)
        self.scan_path_map = {Path(scan_file).stem
                              : scan_file for scan_file in scan_files}

        lobe_files = glob.glob(self.lobe_folder + '/*.mha', recursive=False)
        self.lobe_path_map = {Path(ref_file).stem
                              : ref_file for ref_file in lobe_files}

        lesion_files = glob.glob(self.lesion_folder + '/*.mha', recursive=False)
        self.lesion_path_map = {Path(ref_file).stem
                              : ref_file for ref_file in lesion_files}

        pseudo_vessel_files = glob.glob(self.pseudo_vessel_folder + '/*.mha', recursive=False)
        self.pseudo_vessel_path_map = {Path(ref_file).stem
                                : ref_file for ref_file in pseudo_vessel_files}

    def __len__(self):
        return len(self.uids)

    def __getitem__(self, scan_index):
        uid = self.uids[scan_index]
        ret = self.get_data(uid)
        return ret

    def get_scan(self, scan_name):
        scan_path = self.scan_path_map[scan_name]
        if self.ON_PREMISE_ROOT is not None:
            on_premise_scan_path = os.path.join(self.ON_PREMISE_ROOT, "{}".format(self.__class__.__name__))
            if not os.path.exists(on_premise_scan_path):
                os.makedirs(on_premise_scan_path)
            on_premise_path = os.path.join(on_premise_scan_path, "{}.mha".format(scan_name))
            try:
                if not os.path.exists(on_premise_path):
                    on_premise_path = shutil.copyfile(scan_path, on_premise_path)
                f_ = sitk.ReadImage(on_premise_path)
            except Exception as e:
                print(
                    "loading file or copying  at {} failed with {},"
                    " now read from {}.".format(on_premise_path, e, scan_path))
                f_ = sitk.ReadImage(scan_path)
        else:
            f_ = sitk.ReadImage(scan_path)
        spacing = f_.GetSpacing()
        direction = f_.GetDirection()
        origin = f_.GetOrigin()
        return sitk.GetArrayFromImage(f_), origin[::-1], \
               np.asarray(direction).reshape(3, 3)[::-1].flatten().tolist(), \
               spacing[::-1]

    def get_mask(self, path_map, ref_name):
        if (ref_name not in path_map.keys()) or \
                (not os.path.exists(path_map[ref_name])):
            return None
        file_path = path_map[ref_name]
        return sitk.ReadImage(file_path)

    def get_data(self, uid):
        pid, sid = uid.split("_")
        scan, origin, direction, spacing = self.get_scan(uid)
        original_size = scan.shape
        original_spacing = spacing
        series_meta = self.all_metas[(pid, sid)]
        lobe = sitk.GetArrayFromImage(self.get_mask(self.lobe_path_map, uid)).astype(np.uint8)
        lesion = sitk.GetArrayFromImage(self.get_mask(self.lesion_path_map, uid)).astype(np.uint8)
        v_sitk = self.get_mask(self.pseudo_vessel_path_map, uid)
        if v_sitk is None:
            vessel = np.zeros_like(lobe)
        else:
            vessel = sitk.GetArrayFromImage(v_sitk).astype(np.uint8)

        if self.use_masked_scan:
            scan[lobe == 0] = -2048
        base_dict = {
            'uid': uid,
            'patient_meta': series_meta
        }
        base_dict.update({"size": scan.shape,
                          "spacing": spacing,
                          'original_spacing': original_spacing,
                          'original_size': original_size,
                          "original_origin": origin,
                          "original_direction": direction,
                          "origin": origin,
                          "direction": direction,
                          })

        ret = {
            "#image": scan.astype(np.int16),
            "#lobe_reference": lobe.astype(np.uint8),
            "#lesion_reference": lesion.astype(np.uint8),
            "#vessel_reference": vessel.astype(np.uint8),
            "meta": base_dict
        }
        if self.transforms:
            ret = self.transforms(ret)
        return ret

class RadboudCOVIDLobeVesselChunk(RadboudCOVID):
    ON_PREMISE_ROOT = None

    def __init__(self, archive_path, case_uids,
                 transforms=None, keep_sorted=True,
                 crop_border=5, task='wss_chunk'):
        super(RadboudCOVID, self).__init__()
        self.transforms = transforms
        self.keep_sorted = keep_sorted
        self.archive_path = archive_path
        self.crop_border = crop_border
        self.lobe_folder = archive_path + f'/{task}/lobes/'
        self.scan_folder = archive_path + f'/{task}/images/'
        csv_file = archive_path + f'/{task}/memo.csv'
        self.all_metas, _ = read_csv_in_dict(csv_file, "uid")
        self.case_uids = case_uids
        print("dataset containing: {} cases.".format(len(self.case_uids)), flush=True)

        self.uids = [uid for uid in self.all_metas.keys()
                     if f"{self.all_metas[uid]['patientid']}_{self.all_metas[uid]['study']}" in self.case_uids]
        print("dataset containing: {} chunks.".format(len(self.uids)), flush=True)
        if keep_sorted:
            self.uids = sorted(self.uids)
        else:
            self.uids = random.sample(self.uids, len(self.uids))
        scan_files = glob.glob(self.scan_folder + '/*.mha', recursive=False)
        self.scan_path_map = {Path(scan_file).stem
                              : scan_file for scan_file in scan_files}

        lobe_files = glob.glob(self.lobe_folder + '/*.mha', recursive=False)
        self.lobe_path_map = {Path(ref_file).stem
                              : ref_file for ref_file in lobe_files}

        pseudo_vessel_files = glob.glob(self.pseudo_vessel_folder + '/*.mha', recursive=False)
        self.pseudo_vessel_path_map = {Path(ref_file).stem
                                : ref_file for ref_file in pseudo_vessel_files}

    def get_data(self, lobe_wise_uid):
        lobe_id = lobe_wise_uid.rpartition('-')[-1]
        scan, origin, direction, spacing = self.get_scan(lobe_wise_uid)
        original_size = scan.shape
        original_spacing = spacing
        meta = self.all_metas[lobe_wise_uid]
        ctss = int(meta['ctss'])
        lobe = sitk.GetArrayFromImage(self.get_mask(self.lobe_path_map, lobe_wise_uid)).astype(np.uint8)
        vessel = sitk.GetArrayFromImage(self.get_mask(self.pseudo_vessel_folder, lobe_wise_uid)).astype(np.uint8)

        vessel = np.logical_and(vessel > 0, lobe > 0)
        w_scan = windowing(scan, to_span=(0, 1))
        _, th = binary_cam(w_scan[lobe > 0], 0.75)
        lesion_candidate = (w_scan > th) & (lobe > 0)

        base_dict = {
            'uid': lobe_wise_uid,
            'patient_meta': meta
        }
        base_dict.update({"size": scan.shape,
                          "spacing": spacing,
                          'original_spacing': original_spacing,
                          'original_size': original_size,
                          "original_origin": origin,
                          "original_direction": direction,
                          "origin": origin,
                          "direction": direction,
                          })
        ret = {
            "#image": scan.astype(np.int16),
            "#pseudo_lesion_reference": lesion_candidate.astype(np.uint8),
            "#lobe_reference": lobe.astype(np.uint8),
            "#vessel_reference": vessel.astype(np.uint8),
            "ctss": ctss,
            "lobe_id": lobe_id,
            "meta": base_dict
        }

        if self.transforms:
            ret = self.transforms(ret)
        return ret