#!/usr/bin/env python
import gzip
import os
import sys
import re
from util.consolelog import Console
from core.train import MachineLearning
from core.detector import Detector
from time import time
from alert.teleg import Teleg
import configparser

config = configparser.ConfigParser()
config.read("./config/config.ini")

# INPUT_DIR = "nginx-logs"
# def px(dt, ip, method, bytessent, referrer, ua, stat, url):
#     print("[{}] [{}] {} {} {}".format(colored(dt, "blue"), colored(method,"yellow"), colored(ip, "green"), colored(stat, "cyan"), colored(url,"red")))
#     # print(colored("[" + dt + "]", "blue") + " " + colored("[" + method + "]", "yellow") + " " + colored(ip, "green")) + " " + stat + " " + colored(url, "red")

# ML = MachineLearning()
# ML.preps_predict("./core/dataset/sqli.csv", "./core/model/sqli.h5")

sqli_detect = Detector("./core/dataset/sqli.csv", "./core/model/sqli.h5", "utf-16")
sqli_detect.ignite()
xss_detect = Detector("./core/dataset/xss.csv","./core/model/xss.h5", "utf-8")
xss_detect.ignite()

tele = Teleg(config['TELEGRAM']['token'], config['TELEGRAM']['chat_id'])

def parse(INPUT_DIR):
    lineformat = re.compile(r"""(?P<ipaddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - \[(?P<dateandtime>\d{2}\/[a-z]{3}\/\d{4}:\d{2}:\d{2}:\d{2} (\+|\-)\d{4})\] ((\"(GET|POST) )(?P<url>.+)(http\/1\.1")) (?P<statuscode>\d{3}) (?P<bytessent>\d+) (?P<refferer>-|"([^"]+)") (["](?P<useragent>[^"]+)["])""", re.IGNORECASE)
    x = []
    data = re.search(lineformat, INPUT_DIR)
    if data:
        datadict = data.groupdict()
        ip = datadict["ipaddress"]
        datetimestring = datadict["dateandtime"]
        url = datadict["url"]
        bytessent = datadict["bytessent"]
        referrer = datadict["refferer"]
        useragent = datadict["useragent"]
        status = datadict["statuscode"]
        method = data.group(6)

        x.append([ip, \
        datetimestring, \
        url, \
        bytessent, \
        referrer, \
        useragent, \
        status, \
        method])

        print("+--------------------------------------------+")
        Console.px(datetimestring, ip, method, bytessent, referrer, useragent, status, url)
        attack_vector = ""
        sqli_pred = sqli_detect.check(url[1:])
        Console.info("SQLI prediction: " + str(sqli_pred))
        xss_pred = xss_detect.check(url)
        Console.info("XSS prediction: " + str(xss_pred))
        if((sqli_pred) == 1):
            attack_vector += "SQL Injection, "
            Console.warning("SQL Injection attempt")
        if((xss_pred) == 1):
            attack_vector += "XSS Cross-site scripting, "
            Console.warning("XSS Cross-site scripting")

        if(config['TELEGRAM']['enable_telegram'] == "1"):
            # Prepare to send if there's any misc activity
            if xss_pred == 1 or sqli_pred == 1:
                tele.setReporter(ip, status, attack_vector, url)
                tele.fire()
        print("+--------------------------------------------+")

    return x
    logfile.close()


# 导包
import requests

# path = 'https://www.sogou.com/web?query=是小李同学&'
path = 'https://www.sogou.com/web'

# 动态输入参数
kw = input("please input you query word:")
# 处理url携带的参数，封装到字典中
params = {'query':kw}
# 对指定的url发起请求对应的url是携带参数的，并且请求过程中处理了参数
response = requests.get(url=path,params=kw)
content = response.text
print(content)

# 结果存储
filename = kw + '.html'
with open(filename,'w',encoding='utf-8') as f:
    f.write(content)


import paddle
import paddle.fluid as fluid

def get_strategy(args):
    """
    根据配置，返回相应的学习率变化策略
    :return: 学习率变量（Vars）
    """
    STRATEGY = args["learning_rate_strategy"]

    strategies = {
        "exponential_decay": get_exponential_decay_lr,
        "natural_exp_decay": get_natural_exp_decay_lr,
        "noam_worm_up_and_decay": get_noam_decay_lr,
        "piecewise_decay": get_piecewise_decay_lr,
        "linear_warm_up_and_decay": get_linear_warmup_decay,
        "fixed": get_fixed_learning_rate
    }
    strategy = strategies.get(STRATEGY, -1)
    if strategy == -1:
        raise ValueError("Unknown strategies, expect following options:\n{}".format("\n\t".join(strategies.keys())))
    return strategy(args)

def get_fixed_learning_rate(args):
    """
    直接返回默认的学习率
    """
    lr = fluid.layers.fill_constant(shape=[1], value=args["base_learning_rate"], dtype='float32')
    return lr

def get_exponential_decay_lr(args):
    """
    在学习率上运用指数衰减.
    训练模型时，在训练过程中降低学习率。每 decay_steps 步骤中以 decay_rate 衰减学习率。
    if staircase == True:
        decayed_learning_rate = learning_rate * decay_rate ^ floor(global_step / decay_steps)
    else:
        decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps)
    """
    LR = args["base_learning_rate"]
    DECAY_STEPS = args["decay_step"]
    DECAY_RATE = args["decay_rate"]
    STAIR_DECAY = args["stair_decay"]

    return fluid.layers.exponential_decay(learning_rate=LR, decay_steps=DECAY_STEPS,
                                                            decay_rate=DECAY_RATE,
                                                            staircase=STAIR_DECAY)

def get_natural_exp_decay_lr(args):
    """
    将自然指数衰减运用到初始学习率上。
    训练模型时，在训练过程中降低学习率。 自然指数衰减使用自然指数来计算衰减倍率，每 decay_steps 步衰减倍率的自然指数幂次项上增加 decay_rate 。
    if not staircase:
        decayed_learning_rate = learning_rate * exp(- decay_rate * (global_step / decay_steps))
    else:
        decayed_learning_rate = learning_rate * exp(- decay_rate * floor(global_step / decay_steps))
    """
    LR = args["base_learning_rate"]
    DECAY_STEPS = args["decay_step"]
    DECAY_RATE = args["decay_rate"]
    STAIR_DECAY = args["stair_decay"]

    return fluid.layers.natural_exp_decay(learning_rate=LR, decay_steps=DECAY_STEPS,
                                                            decay_rate=DECAY_RATE,
                                                            staircase=STAIR_DECAY)

def get_noam_decay_lr(args):
    """
    使用Noam衰减策略
    d_model = 2
    current_steps = 20
    warmup_steps = 200
    # 计算
    lr_value = np.power(d_model, -0.5) * np.min([
                           np.power(current_steps, -0.5),
                           np.power(warmup_steps, -1.5) * current_steps])
    """
    LR = args["base_learning_rate"]
    WARM_UP_STEP = args["warm_up_step"]

    return fluid.layers.learning_rate_scheduler.noam_decay(1/(WARM_UP_STEP * (LR ** 2)), WARM_UP_STEP)

def get_piecewise_decay_lr(args):
    """
    使用这个策略时，必须指定一系列学习率和对应的步数，阶梯式下降
    boundaries = [10000, 20000]
    values = [1.0, 0.5, 0.1]
    if step < 10000:
        learning_rate = 1.0
    elif 10000 <= step < 20000:
        learning_rate = 0.5
    else:
        learning_rate = 0.1
    """
    BOUND = [int(i) for i in args["decay_bound"].split(",")]
    LR = [float(i) for i in args["base_learning_rate"].split(",")]

    return fluid.layers.piecewise_decay(boundaries=BOUND, values=LR)

def get_linear_warmup_decay(args):
    """
    线性地将linear做升降变化，作为客制化修改的模板
    """
    START_LR = args["start_learning_rate"]
    WARM_UP_STEP = args["warm_up_step"]
    LR = args["base_learning_rate"]
    DECAY_STEP = args["decay_step"]
    END_LR = args["end_learning_rate"]

    with fluid.default_main_program()._lr_schedule_guard():
        lr = fluid.layers.tensor.create_global_var(
            shape=[1],
            value=0.0,
            dtype='float32',
            persistable=True,
            name="scheduled_learning_rate")
        end_lr = fluid.layers.tensor.create_global_var(
            shape=[1],
            value=END_LR,
            dtype='float32',
            persistable=True,
            name="end_learning_rate")
        global_step = fluid.layers.learning_rate_scheduler._decay_step_counter()

        with fluid.layers.control_flow.Switch() as switch:
            with switch.case(global_step < WARM_UP_STEP):
                warmup_lr = START_LR + (LR - START_LR) * (global_step / WARM_UP_STEP)
                fluid.layers.tensor.assign(warmup_lr, lr)
            with switch.case(global_step < WARM_UP_STEP + DECAY_STEP):
                decayed_lr = fluid.layers.learning_rate_scheduler.polynomial_decay(
                    learning_rate=WARM_UP_STEP * (LR - END_LR) / DECAY_STEP + LR,
                    decay_steps=WARM_UP_STEP + DECAY_STEP,
                    end_learning_rate=END_LR,
                    power=1.0,
                    cycle=False)
                fluid.layers.tensor.assign(decayed_lr, lr)
            with switch.default():
                fluid.layers.tensor.assign(end_lr, lr)

        return lr


import os
import requests
import csv
from bs4 import BeautifulSoup
from stock_find import stock_list

def get_stock_price(stock_list):
    for stock_list in stock_list:
        url = f"https://www.twse.com.tw/exchangeReport/STOCK_DAY?response=html&date=&stockNo={stock_list}"

        # 存进的资料夹
        directory = "stock_data"

        # 发送 GET 请求取得网页内容
        response = requests.get(url)

        # 解析 HTML
        soup = BeautifulSoup(response.text, "html.parser")

        # 找到第一个表格
        table = soup.find("table")

        # 找到所有的资料列
        rows = table.find_all("tr")

        # 建立一个空的资料清单
        data = []

        # 循环处理每一列的资料
        for row in rows:
            # 找到每一列中的所有字段
            columns = row.find_all("td")

            # 如果字段数量不正确，则跳过此列
            if len(columns) != 9:
                continue

            # 取得日期、收盘价等资讯
            date = columns[0].text
            closing_price = columns[6].text

            # 将资讯加入资料清单
            data.append([date, closing_price])

        # 建立档案名称
        filename = f"stock_{stock_list}.csv"

        # 建立档案路径
        file_path = os.path.join(directory, filename)

        # 写入 CSV 档案
        with open(file_path, "w", newline="", encoding="utf-8") as file:
            writer = csv.writer(file)
            writer.writerow(["日期", "收盘价"])  # 写入标题字段
            writer.writerows(data)  # 写入资料

        print(f"资料已储存至 {filename} 档案中。")

# 测试程式码
get_stock_price(stock_list)


# -*- coding: utf-8 -*-
# ---------------------

import torch
from conf import Conf
from models import *

def parse_model(conf, rank=None):
	# type: (Conf, int) -> tuple

	if conf.model_str == "Extended_LT":
		model = (ExtendedLiteModel, {})
	elif conf.model_str == "Extended_LTNL":
		model = (ExtendedLiteNLModel, {})
	else:
		raise NotImplementedError(f"Model {conf.model_str} is not supported or valid")

	return model

def parse_optimizer(conf, model):
	# type: (Conf, torch.nn.Module) -> torch.optim

	decay = conf.train_opts.get('DECAY', 0.0)  # type:float
	lr_dict = conf.train_opts.get('LEARNING_RATES', {})

	param_dicts = [
		{"params": [p for n, p in model.named_parameters() if not
					any(k in n.split('.') for k in list(lr_dict.keys())) and p.requires_grad]},
	]

	for k in list(lr_dict.keys()):
		p = {
			"params": [p for n, p in model.named_parameters() if k in n.split('.') and p.requires_grad],
			"lr": lr_dict[k],
		}
		param_dicts.append(p)

	assert sum([len(p["params"]) for p in param_dicts]) == len([p for p in model.parameters()])

	if conf.optim_str == "Adam":

		optimizer = torch.optim.Adam(param_dicts, lr=conf.lr, weight_decay=decay)

	elif conf.optim_str == "AdamW":
		optimizer = torch.optim.AdamW(param_dicts, lr=conf.lr, weight_decay=decay)

	elif conf.optim_str == "SGD":
		momentum = conf.train_opts.get('MOMENTUM', 0.9)
		optimizer = torch.optim.SGD(param_dicts, lr=conf.lr, weight_decay=decay, momentum=momentum)

	elif conf.optim_str == "RMSProp":
		optimizer = torch.optim.RMSprop(param_dicts, lr=conf.lr, weight_decay=decay)
	else:
		raise NotImplementedError(f"Optimizer {conf.optim_str} is not supported or valid,"
								  f"you con configure this option in conf.py@parse_optimizer()")

	return optimizer


# { 'dictionary' : value }
city = {'Seoul' : 0, 'Busan' : 1}
print(city['Seoul'])
print(city['Busan'])
city['Bucheon'] = 2
print(city)

# for loop
for k in [1,2,3,4,5]:
    print("Hello Everyone!")

# Without for loop
print("Hello Everyone!")
print("Hello Everyone!")
print("Hello Everyone!")
print("Hello Everyone!")
print("Hello Everyone!")

# the range function
for k in range(1,6):
    print(k, "squared is", k*k)

for x in range(5, 0, -1):
    print(x)

# cumulative loops : summaion
sum = 0
for i in range(1, 21):
    sum = sum + (i*i)
print("sum of first 20 squares is", sum)  #2870

#DIY 20!
fac = 1
for k in range(2,101):
    fac= fac*k
print('100!=', fac)

# while loop
sum = 0
k = 1
while k < 21:
    sum = sum + k*k
    k = k+1
print(sum)

# if / else
score = 50
if score < 60:
    print("Your credit is F")
else:
    print("Your credit is A+")

# if / elif 효율적

# DIY
import matplotlib.pyplot as plt
a = -2
b = 2
n = 401
dx = (b-a)/(n-1)
x_list = []
y_list = []

for i in range(0,n):
    x = a + i*dx
    y = x**5-5*x**3+4*x
    x_list.append(x)
    y_list.append(y)

plt.plot(x_list, y_list)
plt.show()


from __future__ import annotations

import os
from datetime import datetime
from typing import Any, cast

from airflow import models
from airflow.models.baseoperator import chain
from airflow.models.xcom_arg import XComArg
from airflow.providers.google.cloud.operators.cloud_build import (
    CloudBuildCreateBuildTriggerOperator,
    CloudBuildDeleteBuildTriggerOperator,
    CloudBuildGetBuildTriggerOperator,
    CloudBuildListBuildTriggersOperator,
    CloudBuildRunBuildTriggerOperator,
    CloudBuildUpdateBuildTriggerOperator,
)
from airflow.utils.trigger_rule import TriggerRule

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")

DAG_ID = "example_gcp_cloud_build_trigger"

# Repository with this name is expected created within the project $SYSTEM_TESTS_GCP_PROJECT
# If you'd like to run this system test locally, please
#   1. Create Cloud Source Repository
#   2. Push into a master branch the following file:
#   tests/system/providers/google/cloud/cloud_build/resources/example_cloud_build.yaml
GCP_SOURCE_REPOSITORY_NAME = "test-cloud-build-repo"

TRIGGER_NAME = f"cloud-build-trigger-{ENV_ID}"

# [START howto_operator_gcp_create_build_trigger_body]
create_build_trigger_body = {
    "name": TRIGGER_NAME,
    "trigger_template": {
        "project_id": PROJECT_ID,
        "repo_name": GCP_SOURCE_REPOSITORY_NAME,
        "branch_name": "main",
    },
    "filename": "example_cloud_build.yaml",
}
# [END howto_operator_gcp_create_build_trigger_body]

# [START howto_operator_gcp_update_build_trigger_body]
update_build_trigger_body = {
    "name": TRIGGER_NAME,
    "trigger_template": {
        "project_id": PROJECT_ID,
        "repo_name": GCP_SOURCE_REPOSITORY_NAME,
        "branch_name": "master",
    },
    "filename": "example_cloud_build.yaml",
}
# [END START howto_operator_gcp_update_build_trigger_body]

# [START howto_operator_create_build_from_repo_body]
create_build_from_repo_body: dict[str, Any] = {
    "source": {"repo_source": {"repo_name": GCP_SOURCE_REPOSITORY_NAME, "branch_name": "master"}},
    "steps": [{"name": "ubuntu", "args": ["echo", "Hello world"]}],
}
# [END howto_operator_create_build_from_repo_body]

with models.DAG(
    DAG_ID,
    schedule="@once",
    start_date=datetime(2021, 1, 1),
    catchup=False,
    tags=["example"],
) as dag:

    # [START howto_operator_create_build_trigger]
    create_build_trigger = CloudBuildCreateBuildTriggerOperator(
        task_id="create_build_trigger", project_id=PROJECT_ID, trigger=create_build_trigger_body
    )
    # [END howto_operator_create_build_trigger]

    build_trigger_id = cast(str, XComArg(create_build_trigger, key="id"))

    # [START howto_operator_run_build_trigger]
    run_build_trigger = CloudBuildRunBuildTriggerOperator(
        task_id="run_build_trigger",
        project_id=PROJECT_ID,
        trigger_id=build_trigger_id,
        source=create_build_from_repo_body["source"]["repo_source"],
    )
    # [END howto_operator_run_build_trigger]

    # [START howto_operator_update_build_trigger]
    update_build_trigger = CloudBuildUpdateBuildTriggerOperator(
        task_id="update_build_trigger",
        project_id=PROJECT_ID,
        trigger_id=build_trigger_id,
        trigger=update_build_trigger_body,
    )
    # [END howto_operator_update_build_trigger]

    # [START howto_operator_get_build_trigger]
    get_build_trigger = CloudBuildGetBuildTriggerOperator(
        task_id="get_build_trigger",
        project_id=PROJECT_ID,
        trigger_id=build_trigger_id,
    )
    # [END howto_operator_get_build_trigger]

    # [START howto_operator_delete_build_trigger]
    delete_build_trigger = CloudBuildDeleteBuildTriggerOperator(
        task_id="delete_build_trigger",
        project_id=PROJECT_ID,
        trigger_id=build_trigger_id,
    )
    # [END howto_operator_delete_build_trigger]
    delete_build_trigger.trigger_rule = TriggerRule.ALL_DONE

    # [START howto_operator_list_build_triggers]
    list_build_triggers = CloudBuildListBuildTriggersOperator(
        task_id="list_build_triggers",
        project_id=PROJECT_ID,
        location="global",
        page_size=5,
    )
    # [END howto_operator_list_build_triggers]

    chain(
        create_build_trigger,
        run_build_trigger,
        update_build_trigger,
        get_build_trigger,
        delete_build_trigger,
        list_build_triggers,
    )

    from tests.system.utils.watcher import watcher

    # This test needs watcher in order to properly mark success/failure
    # when "tearDown" task with trigger rule is part of the DAG
    list(dag.tasks) >> watcher()

from tests.system.utils import get_test_run  # noqa: E402

# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
test_run = get_test_run(dag)


from flask import render_template, redirect, request, flash, url_for, jsonify
from . import college_bp
import app.models.college_models as college_models
from app.college.forms import CollegeForm

headings = ("College Code", "College Name", "Options")

@college_bp.route("/college/display")
def college_display():
    college_data = college_models.College.all()
    return render_template('college.html', headings=headings, data=college_data)

@college_bp.route('/college/edit', methods=["GET", "POST"])
def edit_college():
    college_code = request.args.get('college_code')
    form = CollegeForm()
    data_college = college_models.College.get_college_code(college_code)

    if data_college:
        college_data_dict = {
            "college_code": data_college['college_code'],
            "college_name": data_college['college_name']
        }
    else:
        # Handle the case where no college data was found
        flash("College info cannot be edited.", "error")
        return redirect(url_for("college.college_display"))

    if request.method == "POST" and form.validate():
        new_college_name = request.form.get('college_name')  # Corrected variable name

        if college_models.College.update(college_code, new_college_name):
            flash("College information updated successfully!", "success")
            return redirect(url_for("college.college_display"))
        else:
            flash("Failed to update college information.", "error")

    return render_template("edit_college.html", form=form, row=college_data_dict)

@college_bp.route("/college/delete", methods=["POST"])
def delete_college():
    try:
        college_code = request.form.get('college_code')  # Correct the variable name
        if college_models.College.delete(college_code):
            return jsonify(success=True, message="Successfully deleted")
        else:
            return jsonify(success=False, message="Failed since there are enrolled students")
    except Exception as e:
        # Log the error for debugging purposes
        college_bp.logger.error("An error occurred: %s" % str(e))
        return jsonify(success=False, message="Internal Server Error"), 500

@college_bp.route('/college/add', methods=['POST', 'GET'])
def add_college():
    form = CollegeForm(request.form)

    if request.method == 'POST' and form.validate():
        check_code = form.college_code.data
        college_exists = college_models.College.unique_code(check_code)  # Corrected import here

        if college_exists:
            flash("College already exists! Please enter a unique code", 'error')
        else:
            college = college_models.College(
                college_code=check_code,
                college_name=form.college_name.data,
            )
            college.add()
            flash("College added successfully!", 'success')
            return redirect(url_for('college.college_display'))

    return render_template('add_college.html', form=form)

@college_bp.route('/college/search', methods=['POST'])
def search_college():
    try:
        search_query = request.form.get('searchTerm')  # Updated to 'searchTerm'
        filter_by = request.form.get('filterBy')  # Get the filterBy parameter

        if filter_by == 'all':
            search_results = college_models.College.search(search_query)

        else:
            search_results = college_models.College.filter_college(filter_by, search_query)

        return jsonify(search_results)

    except Exception as e:
        # Handle errors and return an error response
        return jsonify(error=str(e)), 500


import re
from this import d
import numpy as np
import matplotlib.pyplot as plt

def read_log(file: str) -> dict[str, float]:
    speedup: dict[str, float] = {}
    dset: str = ""
    ref_time: float = None
    opt_time: float = None
    for line in open(file=file, mode="r"):
        dset_pattern: str = r"dset = \"([\w\.]*)\""
        time_pattern: str = r"time = ([\d\.]*) \(double\)"
        match: re.Match = re.search(dset_pattern, line)
        if match:
            dset = match.group(1)
        match: re.Match = re.search(time_pattern, line)
        if match:
            if ref_time:
                opt_time: float = float(match.group(1))
                speedup[dset] = ref_time / opt_time
                print(f"| {dset} | {ref_time} | {opt_time} | {speedup[dset]} |")
                ref_time: float = None
            else:
                ref_time: float = float(match.group(1))
    return speedup

def main():
    print("kLen = 32")
    speedup_32: dict[str, float] = read_log("32.log")
    print("kLen = 256")
    speedup_256: dict[str, float] = read_log("256.log")
    x: np.ndarray = np.arange(len(speedup_32))
    width: float = 0.35
    fig, ax = plt.subplots(figsize=(len(speedup_32), 4.8), dpi=600)
    bar_32 = ax.bar(x - width / 2, speedup_32.values(), width, label="kLen = 32")
    bar_256 = ax.bar(x + width / 2, speedup_256.values(), width, label="kLen = 256")
    ax.set_xlabel("Dataset")
    ax.set_ylabel("Speedup")
    # ax.set_title('Scores by group and gender')
    ax.set_xticks(x, speedup_32.keys())
    ax.legend()
    ax.bar_label(bar_32, padding=3)
    ax.bar_label(bar_256, padding=3)
    fig.tight_layout()
    plt.savefig("speedup.png")

if __name__ == "__main__":
    main()


from flask import Flask, render_template, send_from_directory, request, redirect
import csv

app = Flask(__name__)
print(__name__)

@app.route("/")
#when we hit the /, the root, define this function and return message
def home_page():
    return render_template('index.html')

@app.route("/<string:page_name>")
def html_page(page_name):
    return render_template(page_name)

def write_to_file(data):
    with open('database.txt', mode='a') as database:
        email = data['email']
        subject = data['subject']
        message = data['message']
        file = database.write(f'\nE-mail: {email}, Subject: {subject},\n Message: {message}\n\n')

def write_to_csv(data):
    with open('database.csv', 'a', newline='') as csvfile:
        email = data['email']
        subject = data['subject']
        message = data['message']
        csv_writer = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
        csv_writer.writerow([email,subject,message])

@app.route('/submit_form', methods=['POST', 'GET'])
def submit_form():
    if request.method == 'POST':
        try:
            data = request.form.to_dict()
            print(data)
            write_to_csv(data)
            return redirect('/thankyou.html')
        except:
            return 'Did not save to database.'
    else:
        return 'Something went wrong :('


import json
import requests
import xmltodict
import time
import gzip

class ClarinService:
    """ Base service for calling clarin backend. """

    def __init__(self):
        """ Initialize service parameters """

        self.lpmn = "any2txt"
        self.user = "wedt_pw_demo"
        self.text = ""

    def set_text(self, text: str) -> str:
        """ Set text to be analyzed by clarin backend

            Arguments:
            text -- text that should be added to request and analyzed
        """

        self.text = text
        return self.text

    def add_lpmn_command(self, lpmn_command) -> str:
        """ Set command for clarin backend, to process text with

            Arguments:
            lpmn_command -- lpmn command thats should be added to request
        """

        self.lpmn += "|" + lpmn_command
        return self.lpmn

class SynchronousClarinService(ClarinService):
    """ Extended service for calling clarin backend api that can
        be called in synchronous way """

    def __init__(self):
        """ Init service with synchronous url """

        ClarinService.__init__(self)
        self.url_base = "http://ws.clarin-pl.eu/nlprest2/base/process"

    def make_request(self) -> dict:
        """ Make request to clarin backend and parse it from xml to python dict """

        request_data = {
            "lpmn": self.lpmn,
            "user": self.user,
            "text": self.text
        }
        call = requests.post(self.url_base, json=request_data)
        if call.status_code != 200:
            raise Exception('Clarin backend response with non 2xx response: {}'.format(call.text))

        response = xmltodict.parse(call.text)
        return response

class AsynchronousClarinService(ClarinService):
    """ Extended service for calling clarin backend api that must be
         called in asynchronous way """

    def __init__(self):
        """ Init service with asynchronous url """

        ClarinService.__init__(self)
        self.url_base = "http://ws.clarin-pl.eu/nlprest2/base/startTask"
        self.url_check = "http://ws.clarin-pl.eu/nlprest2/base/getStatus"
        self.url_download = "http://ws.clarin-pl.eu/nlprest2/base/download"

        self.file_id = ""
        self.compressed_suffix = "ann_morphosyntax.xml.gz"

    def is_document_ready(self) -> bool:
        """ Ask clarin backend whether processed document is ready"""

        raw_response = requests.get(self.url_check + "/" + self.file_id)
        if raw_response.status_code != 200:
            raise Exception('Clarin backend response with non 2xx response: {}'.format(raw_response.text))

        response = json.loads(raw_response.text)
        if response["status"] == "DONE":
            self.file_id = response["value"][0]["fileID"]
            return True
        else:
            return False

    def upload_document(self):
        """ Send document to clarin backend and receive file id """

        request_data = {
            "lpmn": self.lpmn,
            "user": self.user,
            "text": self.text
        }
        call = requests.post(self.url_base, json=request_data)
        self.file_id = call.text
        if call.status_code != 200:
            raise Exception('Clarin backend response with non 2xx response: {}'.format(call.text))

    def request_response(self) -> dict:
        """ Make a request to clarin backend for compressed response """

        result = requests.get(self.url_download + self.file_id + "/" + self.compressed_suffix)
        if result.status_code != 200:
            raise Exception('Clarin backend response with non 2xx response: {}'.format(result.text))

        response = xmltodict.parse(gzip.decompress(result.content))
        return response

    def make_request(self) -> dict:
        """ Make asynchronous request to clarin backend and parse it from xml to python dict """

        self.upload_document()
        while not self.is_document_ready():
            time.sleep(0.001)
        response = self.request_response()
        return response

class MorphologicService(SynchronousClarinService):
    """ Service that is specialized in calling morphological
        analyzer from clarin. Is able to work with morfeusz in
        versions 1 and 2.
        URL: http://ws.clarin-pl.eu/morpho.shtml
    """

    def __init__(self, use_morfeusz2=True):
        """ Initialize service with proper lpmn command. Uses
            mofreusz analyzer in versions 1 or 2.

            Keyword arguments:
            use_morfeusz2 -- if set to True will use version 2, if not
                             falls back to version 1, default True
        """

        SynchronousClarinService.__init__(self)

        settings = { "morfeusz2": use_morfeusz2 }
        self.add_lpmn_command("maca({})".format(json.dumps(settings)))

class TaggerService(SynchronousClarinService):
    """ Service that is specialized in calling morpho-syntactic
        tagger from clarin. Is able to work with wcrft2 tagger
        and with morfoDita tagger.
        URL: http://ws.clarin-pl.eu/tager.shtml
    """

    def __init__(self,
                 use_wcrft2=False,
                 use_morfeusz2=True,
                 use_morfoDita=True,
                 all_forms=False,
                 guesser=True,
                 use_present_language_model=True):
        """ Initialize service with proper lpmn command. Uses wcrft2 and
            morfoDita tagger. By default, uses wcrft2 tagger.

            Keyword arguments:
            use_wcrft2 -- if set to True will use wcrft2 tagger
            use_morfeusz2 -- if set, will use morfeusz in version 2 for
                             wcrft2 tagging

            use_morfoDita -- if set to True will use morfoDiat tagger
            all_forms -- find all forms, only for morfoDita tagger
            guesser -- allow tagger to guess type, only for morfoDita tagger
            use_present_language_model -- if set to False, will use language
                                          from XIX century, only for morfoDita
        """

        SynchronousClarinService.__init__(self)

        settings = {"guesser": guesser}
        if use_wcrft2:
            settings["morfeusz2"] = use_morfeusz2
            self.add_lpmn_command("wcrft2({})".format(json.dumps(settings)))
        elif use_morfoDita:
            settings["allforms"] = all_forms
            settings["model"] = "XXI" if use_present_language_model else "XIX"
            self.add_lpmn_command("morphoDita({})".format(json.dumps(settings)))
        else:
            raise RuntimeError("Tagger service created without selected tagger")

class NERService(SynchronousClarinService):
    """ Service that is specialized in calling named-entity recognition
        tool -- liner2. Can use up to 8 different models.
        URL: https://ws.clarin-pl.eu/ner.shtml
    """

    def __init__(self, model="top9"):
        """ Initialize service with proper lpmn command. Uses liner2 tool.
            By default, uses top9 model.

            Keyword arguments:
            model -- select which language model should be used by the tool.
                     Tool can use one of eight different models:
                     - top9 - 9 kategorii
                     - n82 - 82 kategorie
                     - 5nam - 5 kategorii
                     - timex1
                     - timex4
                     - names - granice nazw
                     - events - atrybuty i relacje zdarzeń
                     - all - grupa modeli
        """

        SynchronousClarinService.__init__(self)

        ner_models = ["top9", "n82", "5nam", "timex1", "timex4", "names", "events", "all"]
        if model not in ner_models:
            raise RuntimeError("NER service created with incorrect model")
        settings = {"model":  model}
        self.add_lpmn_command("wcrft2")
        self.add_lpmn_command("liner2({})".format(json.dumps(settings)))

class SyntaxParserService(AsynchronousClarinService):
    """ Service that is specialized in calling syntax parser
        tool -- spejd.
        URL: https://ws.clarin-pl.eu/spejd.shtml
    """

    def __init__(self):
        """ Initialize service with proper lpmn command. Uses spejd tool. """

        AsynchronousClarinService.__init__(self)

        self.add_lpmn_command('wcrft2({"morfeusz2":false})')
        self.add_lpmn_command("iobber")
        self.add_lpmn_command('liner2({"model":"n82","output":"tei:gz"})')
        self.add_lpmn_command("spejd")

class ChunkerService(SynchronousClarinService):
    """ Service that is specialized in calling chunker service
        called iobber
        URL: https://ws.clarin-pl.eu/chunker.shtml
    """

    def __init__(self):
        """ Initialize service. Iobber tool takes no arguments """

        SynchronousClarinService.__init__(self)
        self.add_lpmn_command("wcrft2")
        self.add_lpmn_command("iobber")


# importing the dataset
import pandas as pd

data = pd.read_csv('C:/Users/Hp/Downloads/Final dataset Attrition.xlsx - Sheet1.csv')

#Remove the column EmployeeNumber
data = data.drop(['date of termination','Status of leaving','absenteeism','Work_accident','Source of Hire','Jobmode','HigherEducation','Date of Hire','modeofwork','leaves','Gender'], axis = 1) # A number assignment

data_copy=data.copy()
target = 'Attrition'
category_col =[ 'BusinessTravel', 'Department', 'JobRole', 'MaritalStatus','OverTime',]

for col in category_col:
    dummy = pd.get_dummies(data_copy[col], prefix=col)
    data_copy = pd.concat([data_copy,dummy], axis=1)
    del data_copy[col]

target_mapper = {'Yes':0, 'No':1}
def target_encode(val):
    return target_mapper[val]

data_copy['Attrition'] = data_copy['Attrition'].apply(target_encode)

#Split the data into independent 'X' and dependent 'Y' variables
X = data_copy.drop('Attrition',axis=1)
Y = data_copy['Attrition']

# Split the dataset into 75% Training set and 25% Testing set
from sklearn.model_selection import train_test_split
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.25, random_state = 0)

#Use Random Forest Classification algorithm
from sklearn.ensemble import RandomForestClassifier
forest = RandomForestClassifier(n_estimators = 10, criterion = 'entropy', random_state = 0)
forest.fit(X, Y)

# Saving the model
import pickle
pickle.dump(forest, open('attrition_clf.pkl', 'wb'))


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Sun Jun 23 15:22:02 2019

@author: dhruv.sharma
"""

"""
Uncertainty measures that explicitly support representative batch-mode sampling for active learning models.
"""

from typing import Callable, Optional, Tuple, Union
from tqdm import tqdm

import numpy as np
import scipy.sparse as sp
from sklearn.cluster import KMeans
from sklearn.metrics.pairwise import pairwise_distances, pairwise_distances_argmin_min
from keras.models import Model
from strategies.uncertainty import *
from sklearn.neighbors import NearestNeighbors
from scipy.spatial.distance import cdist

def select_instance(
        X_training,
        X_pool,
        X_training_feat,
        X_pool_feat,
        n_annotated,
        X_uncertainty: np.ndarray,
        mask: np.ndarray,
        metric: Union[str, Callable],
        n_jobs : Union[int, None] = -1):
    """
    Core iteration strategy for selecting another record from our unlabeled records.

    Given a set of labeled records (X_training) and unlabeled records (X_pool) with uncertainty scores (X_uncertainty),
    we'd like to identify the best instance in X_pool that best balances uncertainty and dissimilarity.

    Refer to Cardoso et al.'s "Ranked batch-mode active learning":
        https://www.sciencedirect.com/science/article/pii/S0020025516313949

    TODO:
        - Add notebook for Active Learning bake-off (passive vs interactive vs batch vs ranked batch)

    Args:
        X_training: Mix of both labeled and unlabeled records.
        X_pool: Unlabeled records to be selected for labeling.
        X_training_feat: feature vectors for the training data
        X_pool_feat: feature vectors for the unlabeld data
        X_uncertainty: Uncertainty scores for unlabeled records to be selected for labeling.
        mask: Mask to exclude previously selected instances from the pool.
        metric: This parameter is passed to :func:`~sklearn.metrics.pairwise.pairwise_distances`.
        n_jobs: This parameter is passed to :func:`~sklearn.metrics.pairwise.pairwise_distances`.

    Returns:
        Index of the best index from X chosen to be labelled; a single record from our unlabeled set that is considered
        the most optimal incremental record for including in our query set.
    """
    # Extract the number of labeled and unlabeled records.
    n_labeled_records = X_training.shape[0] + n_annotated
    n_unlabeled = X_pool[mask].shape[0]

    # Determine our alpha parameter as |U| / (|U| + |D|). Note that because we
    # append to X_training and remove from X_pool within `ranked_batch`,
    # :alpha: is not fixed throughout our model's lifetime.
    alpha = n_unlabeled / (n_unlabeled + n_labeled_records)

    # Compute pairwise distance (and then similarity) scores from every unlabeled record
    # to every record in X_training. The result is an array of shape (n_samples, ).

    ################## TODO: replace this part with a better similarity computation ############################
    '''
    Args:
        X_u: unlabeled data
        X_l: labeled data
    Returns:
        pairwise distance between the two points
    '''
    if X_pool_feat is None or X_training_feat is None:
        X_pool_features = X_pool[mask].reshape((len(X_pool[mask]), -1))
        X_training_features = X_training.reshape((len(X_training), -1))
    else:
        X_pool_features = X_pool_feat[mask]
        X_training_features = X_training_feat

    if n_jobs == 1 or n_jobs is None:
        _, distance_scores = pairwise_distances_argmin_min(X_pool_features, X_training_features, metric=metric)
    else:
        #distance_scores = pairwise_distances(X_pool_features, X_training_features, metric=metric, n_jobs=n_jobs).min(axis=1)
        distance_scores = cdist(X_pool_features, X_training_features, metric=metric).min(axis=1)
    ############################################################################################################

    similarity_scores = 1 / (1 + distance_scores)

    # Compute our final scores, which are a balance between how dissimilar a given record
    # is with the records in X_uncertainty and how uncertain we are about its class.
    scores = alpha * (1 - similarity_scores) + (1 - alpha) * X_uncertainty[mask]

    # Isolate and return our best instance for labeling as the one with the largest score.
    best_instance_index_in_unlabeled = np.argmax(scores)
    n_pool, *rest = X_pool.shape
    unlabeled_indices = [i for i in range(n_pool) if mask[i]]
    best_instance_index = unlabeled_indices[best_instance_index_in_unlabeled]
    mask[best_instance_index] = 0
    return best_instance_index, np.expand_dims(X_pool[best_instance_index], axis=0), mask

#    best_instance_index = np.argmax(scores)
#    mask[best_instance_index] = 0
#    return best_instance_index, X_pool[best_instance_index], mask

def ranked_batch(classifier,
                 labeled,
                 unlabeled,
                 X_training_feat,
                 X_pool_feat,
                 uncertainty_scores: np.ndarray,
                 n_instances: int,
                 metric: Union[str, Callable],
                 n_jobs: Union[int, None]) -> np.ndarray:
    """
    Query our top :n_instances: to request for labeling.

    Refer to Cardoso et al.'s "Ranked batch-mode active learning":
        https://www.sciencedirect.com/science/article/pii/S0020025516313949

    Args:
        classifier: active learner supported active learning models.
        labeled: the labeled dataset
        unlabeled: Set of records to be considered for our active learning model.
        X_training_feat: feature vectors of the labeled dataset
        X_pool_feat: feature vectors of the unlabeled dataset
        uncertainty_scores: Our classifier's predictions over the response variable.
        n_instances: Limit on the number of records to query from our unlabeled set.
        metric: This parameter is passed to :func:`~sklearn.metrics.pairwise.pairwise_distances`.
        n_jobs: This parameter is passed to :func:`~sklearn.metrics.pairwise.pairwise_distances`.

    Returns:
        The indices of the top n_instances ranked unlabelled samples.
    """
    # Clustering the pool data to find the most representative points
    print("getting the cluster centroids")
    n_clusters = int(0.8*n_instances + 0.2*len(X_pool_feat))
    n_clusters = np.minimum(n_clusters, len(X_pool_feat))
    km = KMeans(n_clusters=n_clusters, n_jobs = n_jobs).fit(X_pool_feat)

    centers = np.array(km.cluster_centers_)
    cluster_labels = km.labels_.tolist()
    closest = []

    for i in range(n_clusters):
        center = centers[i]
        this_center_data = [idx for idx, clu_num in enumerate(cluster_labels) if clu_num == i]
        nbrs = NearestNeighbors(n_neighbors=1).fit(X_pool_feat[this_center_data])
        dist, ind = nbrs.kneighbors([center])
        closest.append(this_center_data[ind[0][0]])
#    closest, _ = pairwise_distances_argmin_min(km.cluster_centers_, X_pool_feat)

    # Define our record container and the maximum number of records to sample.
    instance_index_ranking = []
    ceiling = np.minimum(unlabeled.shape[0], n_instances)

    # mask for unlabeled initialized as transparent
    closest = np.array(closest)
    mask = np.zeros(unlabeled.shape[0], np.bool)
    mask[closest] = 1
    query_count = 0
    for _ in tqdm(range(ceiling)):

        # Receive the instance and corresponding index from our unlabeled copy that scores highest.
        instance_index, instance, mask = select_instance(X_training=labeled,
                                                         X_pool=unlabeled,
                                                         X_training_feat=X_training_feat,
                                                         X_pool_feat=X_pool_feat,
                                                         n_annotated = query_count,
                                                         X_uncertainty=uncertainty_scores,
                                                         mask=mask,
                                                         metric=metric,
                                                         n_jobs=n_jobs)

        # Add our instance we've considered for labeling to our labeled set. Although we don't
        # know it's label, we want further iterations to consider the newly-added instance so
        # that we don't query the same instance redundantly.
        #####
#        labeled = np.concatenate((labeled, instance))
#        X_training_feat = np.concatenate((X_training_feat, np.array([X_pool_feat[instance_index]])))
        #####
        # Finally, append our instance's index to the bottom of our ranking.
        instance_index_ranking.append(instance_index)
        query_count += 1

    # Return numpy array, not a list.
    return np.array(instance_index_ranking)

def informative_batch_sampling(model: Model,
                               X_u: Union[np.ndarray, sp.csr_matrix],
                               features_labeled = None,
                               features_unlabeled = None,
                               n_instances: int = 20,
                               metric: Union[str, Callable] = 'euclidean',
                               n_jobs: Optional[int] = None,
                               **uncertainty_measure_kwargs
                               ) -> Tuple[np.ndarray, Union[np.ndarray, sp.csr_matrix]]:
    """
    coreset based ranked Batch sampling query strategy. Selects the least sure instances for labelling.

    Refer to Cardoso et al.'s "Ranked batch-mode active learning":
        https://www.sciencedirect.com/science/article/pii/S0020025516313949

    Args:
        classifier: active learning model.
        X: Set of records to be considered for our active learning model.
        features_labeled: feature vectors of the labeled data to be used for similarity matrix computation.
        features_unlabeled: feature vectors of the unlabeled data to be used for similarity matrix computation.
        n_instances: Number of records to return for labeling from `X`.
        metric: This parameter is passed to :func:`~sklearn.metrics.pairwise.pairwise_distances`
        n_jobs: If not set, :func:`~sklearn.metrics.pairwise.pairwise_distances_argmin_min` is used for calculation of
            distances between samples. Otherwise it is passed to :func:`~sklearn.metrics.pairwise.pairwise_distances`.
        **uncertainty_measure_kwargs: Keyword arguments to be passed for the :meth:`predict_proba` of the classifier.

    Returns:
        Indices of the instances from `X` chosen to be labelled; records from `X` chosen to be labelled.
    """
    uncertainty = segmentation_uncertainty(model, X_u, **uncertainty_measure_kwargs)
    query_indices = ranked_batch(model,
                                 labeled=model.X_training,
                                 unlabeled=X_u,
                                 X_training_feat = features_labeled,
                                 X_pool_feat = features_unlabeled,
                                 uncertainty_scores=uncertainty,
                                 n_instances=n_instances,
                                 metric=metric,
                                 n_jobs=n_jobs)
    return query_indices, X_u[query_indices]


# coding: utf8
""" Implementation of exotic DPP objects:

- Uniform spanning trees :class:`UST`
- Descent procresses :class:`Descent`:

    * :class:`CarriesProcess`
    * :class:`DescentProcess`
    * :class:`VirtualDescentProcess`

- :class:`PoissonizedPlancherel` measure

.. seealso:

    `Documentation on ReadTheDocs <https://dppy.readthedocs.io/en/latest/exotic_dpps/index.html>`_
"""

from abc import ABCMeta, abstractmethod

import numpy as np
from scipy.linalg import qr
import matplotlib.pyplot as plt
from matplotlib import collections as mc  # see plot_diagram

from dppy.exotic_dpps_core import ust_sampler_wilson, ust_sampler_aldous_broder
from dppy.exact_sampling import proj_dpp_sampler_eig, proj_dpp_sampler_kernel

# For DescentProcess
from re import findall  # to convert class names to string in
# from dppy.exotic_dpps_core import wrapper_plot_descent

# For Poissonized Plancherel measure
from dppy.exotic_dpps_core import RSK, xy_young_ru, limit_shape

# For both Descent Processes and Poissonized Plancherel
from dppy.exotic_dpps_core import uniform_permutation

from dppy.utils import check_random_state

#####################
# Descent Processes #
#####################
class Descent(metaclass=ABCMeta):

    def __init__(self):

        self.name = ' '.join(findall('[A-Z][^A-Z]*', self.__class__.__name__))
        self.list_of_samples = []
        self.size = 100

    @property
    @abstractmethod
    def _bernoulli_param(self):
        """Parameter of the corresponding process formed by i.i.d. Bernoulli variables.
        This parameter corresponds to the probability that a descent occurs any index"""

        return 0.5

    @abstractmethod
    def sample(self, random_state=None):
        """Sample from corresponding process"""

    def flush_samples(self):
        """ Empty the :py:attr:`list_of_samples` attribute.
        """
        self.list_of_samples = []

    def plot(self, vs_bernoullis=True, random_state=None):
        """Display the last realization of the process.
        If ``vs_bernoullis=True`` compare it to a sequence of i.i.d. Bernoullis with parameter ``_bernoulli_param``

        .. seealso::

            - :py:meth:`sample`
        """
        rng = check_random_state(random_state)
        title = 'Realization of the {} process'.format(self.name)

        fig, ax = plt.subplots(figsize=(19, 2))

        sampl = self.list_of_samples[-1]
        ax.scatter(sampl,
                   np.zeros_like(sampl) + (1.0 if vs_bernoullis else 0.0),
                   color='b', s=20, label=self.name)

        if vs_bernoullis:
            title += r' vs independent Bernoulli variables with parameter $p$={:.3f}'.format(self._bernoulli_param)

            bern = np.where(rng.rand(self.size) < self._bernoulli_param)[0]
            ax.scatter(bern, -np.ones_like(bern),
                       color='r', s=20, label='Bernoullis')

        plt.title(title)

        # Spine options
        ax.spines['bottom'].set_position('center')
        ax.spines['left'].set_visible(False)
        ax.spines['top'].set_visible(False)
        ax.spines['right'].set_visible(False)

        # Ticks options
        minor_ticks = np.arange(0, self.size + 1)
        major_ticks = np.arange(0, self.size + 1, 10)
        ax.set_xticks(major_ticks)
        ax.set_xticks(minor_ticks, minor=True)
        ax.set_xticklabels(major_ticks, fontsize=15)
        ax.xaxis.set_ticks_position('bottom')

        ax.tick_params(
            axis='y',           # changes apply to the y-axis
            which='both',       # both major and minor ticks are affected
            left=False,         # ticks along the left edge are off
            right=False,        # ticks along the right edge are off
            labelleft=False)    # labels along the left edge are off

        ax.xaxis.grid(True)
        ax.set_xlim([-1, self.size + 1])
        ax.legend(bbox_to_anchor=(0, 0.85),
                  frameon=False,
                  prop={'size': 15})

class CarriesProcess(Descent):
    """ DPP on :math:`\\{1, \\dots, N-1\\}` (with a non symmetric kernel) derived from the cumulative sum of :math:`N` i.i.d. digits in :math:`\\{0, \\dots, b-1\\}`.

    :param base:
        Base/radix

    :type base:
        int, default 10

    .. seealso::

        - :cite:`BoDiFu10`
        - :ref:`carries_process`
    """

    def __init__(self, base=10):
        super().__init__()
        self.base = base

    def __str__(self):

        str_info = ['Carries process in base {}'.format(self.base),
                    'Number of samples = {}'.format(len(self.list_of_samples))]

        return '\n'.join(str_info)

    @property
    def _bernoulli_param(self):
        return 0.5 * (1 - 1 / self.base)

    def sample(self, size=100, random_state=None):
        """ Compute the cumulative sum (in base :math:`b`) of a sequence of i.i.d. digits and record the position of carries.

        :param size:
            size of the sequence of i.i.d. digits in :math:`\\{0, \\dots, b-1\\}`

        :type size:
            int
        """
        rng = check_random_state(random_state)

        self.size = size
        A = rng.randint(0, self.base, self.size)
        B = np.mod(np.cumsum(A), self.base)

        carries = 1 + np.where(B[:-1] > B[1:])[0]

        self.list_of_samples.append(carries.tolist())

class DescentProcess(Descent):
    """ DPP on :math:`\\{1, \\dots, N-1\\}` associated to the descent process on the symmetric group :math:`\\mathfrak{S}_N`.

        .. seealso::

            - :cite:`BoDiFu10`
            - :ref:`descent_process`
    """

    def __init__(self):
        super().__init__()

    def __str__(self):

        str_info = ['Descent process',
                    'Number of samples = {}'.format(len(self.list_of_samples))]

        return '\n'.join(str_info)

    @property
    def _bernoulli_param(self):
        return 0.5

    def sample(self, size=100, random_state=None):
        """ Draw a permutation :math:`\\sigma \\in \\mathfrak{S}_N` uniformly at random and record the descents i.e. :math:`\\{ i ~;~ \\sigma_i > \\sigma_{i+1} \\}`.

        :param size:
            size of the permutation i.e. degree :math:`N` of :math:`\\mathfrak{S}_N`.

        :type size:
            int
        """

        rng = check_random_state(random_state)

        self.size = size
        sigma = uniform_permutation(self.size, random_state=rng)

        descent = 1 + np.where(sigma[:-1] > sigma[1:])[0]

        self.list_of_samples.append(descent.tolist())

class VirtualDescentProcess(Descent):
    """ This is a DPP on :math:`\\{1, \\dots, N-1\\}` with a non symmetric kernel appearing in (or as a limit of) the descent process on the symmetric group :math:`\\mathfrak{S}_N`.

    .. seealso::

        - :cite:`Kam18`
        - :ref:`limiting_descent_process`
        - :class:`DescentProcess`
    """

    def __init__(self, x_0=0.5):

        super().__init__()
        if not (0 <= x_0 <= 1):
            raise ValueError("x_0 must be in [0,1]")
        self.x_0 = x_0

    def __str__(self):

        str_info = ['Limitting Descent process for vitural permutations',
                    'Number of samples = {}'.format(len(self.list_of_samples))]

        return '\n'.join(str_info)

    @property
    def _bernoulli_param(self):
        return 0.5 * (1 - self.x_0**2)

    def sample(self, size=100, random_state=None):
        """ Draw a permutation uniformly at random and record the descents i.e. indices where :math:`\\sigma(i+1) < \\sigma(i)` and something else...

        :param size:
            size of the permutation i.e. degree :math:`N` of :math:`\\mathfrak{S}_N`.

        :type size:
            int

        .. seealso::

            - :cite:`Kam18`, Sec ??

        .. todo::

            ask @kammmoun to complete the docsting and Section in see also
        """

        rng = check_random_state(random_state)

        self.size = size
        sigma = uniform_permutation(self.size + 1, random_state=rng)

        X = sigma[:-1] > sigma[1:]  # Record the descents in permutation

        Y = rng.binomial(n=2, p=self.x_0, size=self.size + 1) != 1

        descent = [i for i in range(self.size)
                   if (~Y[i] and Y[i + 1]) or (~Y[i] and ~Y[i + 1] and X[i])]
        # ~ symbol is equivalent to not on boolean numpy array

        self.list_of_samples.append(descent)

##########################
# Poissonized Plancherel #
##########################
class PoissonizedPlancherel:
    """ DPP on partitions associated to the Poissonized Plancherel measure

    :param theta:
        Poisson parameter i.e. expected length of permutation

    :type theta:
        int, default 10

    .. seealso::

        - :cite:`Bor09` Section 6
        - :ref:`poissonized_plancherel_measure`
    """

    def __init__(self, theta=10):

        self.theta = theta  # Poisson param = expected length of permutation
        self.list_of_young_diag = []
        self.list_of_samples = []

    def __str__(self):

        str_info = ['Poissonized Plancherel measure\
                    with parameter {}'.format(self.theta),
                    'Number of samples = {}'.format(len(self.list_of_samples))]

        return '\n'.join(str_info)

    # def info(self):
    #     """ Print infos about the :class:`UST` object
    #     """
    #     print(self.__str__())

    def sample(self, random_state=None):
        """ Sample from the Poissonized Plancherel measure.

        :param random_state:
        :type random_state:
            None, np.random, int, np.random.RandomState
        """
        rng = check_random_state(random_state)

        N = rng.poisson(self.theta)
        sigma = uniform_permutation(N, random_state=rng)
        P, _ = RSK(sigma)

        # young_diag = [len(row) for row in P]
        young_diag = np.fromiter(map(len, P), dtype=int)
        self.list_of_young_diag.append(young_diag)
        # sampl = [len(row) - i + 0.5 for i, row in enumerate(P, start=1)]
        sampl = young_diag - np.arange(0.5, young_diag.size)
        self.list_of_samples.append(sampl.tolist())

    def plot(self, title=''):
        """Display the process on the real line

        :param title:
            Plot title

        :type title:
            string

        .. seealso::

            - :py:meth:`sample`
        """

        sampl = self.list_of_samples[-1]

        # Display the reparametrized Plancherel sample
        fig, ax = plt.subplots(figsize=(19, 2))

        ax.scatter(sampl, np.zeros_like(sampl), color='blue', s=20)

        # Spine options
        ax.spines['bottom'].set_position('center')
        ax.spines['left'].set_visible(False)
        ax.spines['top'].set_visible(False)
        ax.spines['right'].set_visible(False)

        # Ticks options
        x_max = np.max(np.abs(sampl)) + 0.5
        minor_ticks = np.arange(-x_max, x_max + 1)
        major_ticks = np.arange(-100, 100 + 1, 10)
        ax.set_xticks(major_ticks)
        ax.set_xticks(minor_ticks, minor=True)
        ax.set_xticklabels(major_ticks, fontsize=15)
        ax.xaxis.set_ticks_position('bottom')

        ax.tick_params(
            axis='y',           # changes apply to the y-axis
            which='both',       # both major and minor ticks are affected
            left=False,         # ticks along the left edge are off
            right=False,        # ticks along the right edge are off
            labelleft=False)    # labels along the left edge are off

        ax.xaxis.grid(True)
        ax.set_xlim([-x_max - 2, x_max + 2])
        # ax.legend(bbox_to_anchor=(0,0.85), frameon=False, prop={'size':20})

        str_title = r'Realization of the DPP associated to the Poissonized Plancherel measure with parameter $\theta=${}'.format(self.theta)
        plt.title(title if title else str_title)

    def plot_diagram(self, normalization=False):
        """ Display the Young diagram (russian convention), the associated sample and potentially rescale the two to visualize the limit-shape theorem :cite:`Ker96`.
        The sample corresponds to the projection onto the real line of the descending surface edges.

        :param normalization:
            If ``normalization=True``, the Young diagram and the corresponding sample are scaled by a factor :math:`\\sqrt{\\theta}` and the limiting

        :type normalization:
            bool, default False

        .. seealso::

            - :py:meth:`sample`
            - :py:meth:`plot`
            - :cite:`Ker96`
        """

        y_diag = self.list_of_young_diag[-1]
        sampl = self.list_of_samples[-1].copy()

        x_max = 1.1 * max(y_diag.size, y_diag[0])
        xy_young = xy_young_ru(y_diag)

        if normalization:
            sampl /= np.sqrt(self.theta)
            x_max /= np.sqrt(self.theta)
            xy_young /= np.sqrt(self.theta)

        fig, ax = plt.subplots(figsize=(12, 8))

        # Display corresponding sample
        ax.scatter(sampl, np.zeros_like(sampl),
                   s=3, label='sample')
        # Display absolute value wedge
        ax.plot([-x_max, 0.0, x_max], [x_max, 0.0, x_max],
                c='k', lw=1)
        # Display young diagram in russian notation
        lc = mc.LineCollection(xy_young.reshape((-1, 2, 2)),
                               color='k', linewidths=2)
        ax.add_collection(lc)
        # Display limit shape
        if normalization:
            x_lim_sh = np.linspace(-x_max, x_max, 100)
            ax.plot(x_lim_sh, limit_shape(x_lim_sh),
                    c='r', label='limit shape')

        # Display stems linking sample on real line and descent in young diag
        # xy_y_diag = np.column_stack([y_diag,
        # np.arange(0.5, y_diag.size)]).dot(rot_45_and_scale.T)
        # if normalization:
        #     xy_y_diag /= np.sqrt(theta)
        # plt.scatter(xy_y_diag[:,0], np.zeros_like(y_diag), color='r')
        # plt.stem(xy_y_diag[:,0], xy_y_diag[:,1], linefmt='C0--', basefmt=' ')

        plt.legend(loc='best')
        plt.axis('equal')

        str_title = r'Young diagram associated to Poissonized Plancherel measure with parameter $\theta=${}'.format(self.theta)
        plt.title(str_title)

##########################
# Uniform Spanning Trees #
##########################
class UST:
    """ DPP on edges of a connected graph :math:`G` with correlation kernel the projection kernel onto the span of the rows of the incidence matrix :math:`\\text{Inc}` of :math:`G`.

    This DPP corresponds to the uniform measure on spanning trees (UST) of :math:`G`.

    :param graph:
        Connected undirected graph
    :type graph:
        networkx graph

    .. seealso::

        - :ref:`UST`
        - :ref:`Definition of DPP <finite_dpps_definition>`
    """

    def __init__(self, graph):
        # For Uniform Spanning Trees
        try:
            import networkx as nx
            self.nx = nx
        except ImportError:
            raise ValueError('The networkx package is required to sample spanning trees (see setup.py).')

        if nx.is_connected(graph):
            self.graph = graph
        else:
            raise ValueError('graph not connected')

        self.nodes = list(self.graph.nodes())
        self.nb_nodes = self.graph.number_of_nodes()  # len(self.graph)

        self.edges = list(self.graph.edges())
        self.nb_edges = self.graph.number_of_edges()  # len(self.edges)

        self.edge_labels = {edge: r'$e_{}$'.format(i)
                            for i, edge in enumerate(self.edges)}

        self.neighbors = [list(graph.neighbors(v))
                          for v in range(self.nb_nodes)]

        self.sampling_mode = 'Wilson'  # Default (avoid eig_vecs computation)
        self._sampling_modes = {'markov-chain': ['Wilson', 'Aldous-Broder'],
                                'spectral-method': ['GS'],
                                'projection-K-kernel': ['Schur', 'Chol']}
        self.list_of_samples = []

        self.kernel = None
        self.kernel_eig_vecs = None

    def __str__(self):

        str_info = ['Uniform Spanning Tree measure on a graph with:',
                    '- {} nodes'.format(self.nb_nodes),
                    '- {} edges'.format(self.nb_edges),
                    'Sampling mode = {}'.format(self.sampling_mode),
                    'Number of samples = {}'.format(len(self.list_of_samples))]

        return '\n'.join(str_info)

    # def info(self):
    #     """ Print infos about the :class:`UST` object
    #     """
    #     print(self.__str__())

    def flush_samples(self):
        """ Empty the :py:attr:`list_of_samples` attribute.
        """
        self.list_of_samples = []

    def sample(self, mode='Wilson', root=None, random_state=None):
        """ Sample a spanning of the underlying graph uniformly at random.
        It generates a networkx graph object.

        :param mode:

            Markov-chain-based samplers:

            - ``'Wilson'``, ``'Aldous-Broder'``

            Chain-rule-based samplers:

            - ``'GS'``, ``'GS_bis'``, ``'KuTa12'`` from eigenvectors
            - ``'Schur'``, ``'Chol'``, from :math:`\\mathbf{K}` correlation kernel

        :type mode:
            string, default ``'Wilson'``

        :param root:
            Starting node of the random walk when using Markov-chain-based samplers
        :type root:
            int

        :param random_state:
        :type random_state:
            None, np.random, int, np.random.RandomState

        .. seealso::

            - Wilson :cite:`PrWi98`
            - Aldous-Broder :cite:`Ald90`
            - :py:meth:`~dppy.FiniteDPP.sample`
        """

        rng = check_random_state(random_state)

        self.sampling_mode = mode

        if self.sampling_mode in self._sampling_modes['markov-chain']:
            if self.sampling_mode == 'Wilson':
                sampl = ust_sampler_wilson(self.neighbors,
                                           random_state=rng)

            elif self.sampling_mode == 'Aldous-Broder':
                sampl = ust_sampler_aldous_broder(self.neighbors,
                                                  random_state=rng)

        elif self.sampling_mode in self._sampling_modes['spectral-method']:

            self.compute_kernel_eig_vecs()
            dpp_sample = proj_dpp_sampler_eig(self.kernel_eig_vecs,
                                              mode=self.sampling_mode,
                                              random_state=rng)

            sampl = self.nx.Graph()
            sampl.add_edges_from([self.edges[e] for e in dpp_sample])

        elif self.sampling_mode in self._sampling_modes['projection-K-kernel']:

            self.compute_kernel()
            dpp_sample = proj_dpp_sampler_kernel(self.kernel,
                                                 mode=self.sampling_mode,
                                                 random_state=rng)

            sampl = self.nx.Graph()
            sampl.add_edges_from([self.edges[e] for e in dpp_sample])

        else:
            err_print = '\n'.join(
                'Invalid sampling mode',
                'Chose from: {}'.format(self._sampling_modes.values()),
                'Given {}'.format(mode))
            raise ValueError()

        self.list_of_samples.append(sampl)

    def compute_kernel(self):
        """ Compute the orthogonal projection kernel :math:`\\mathbf{K} = \\text{Inc}^+ \\text{Inc}` i.e. onto the span of the rows of the vertex-edge incidence matrix :math:`\\text{Inc}` of size :math:`|V| \\times |E|`.

        In fact, for a connected graph, :math:`\\text{Inc}` has rank :math:`|V|-1` and any row can be discarded to get an basis of row space. If we note :math:`A` the amputated version of :math:`\\text{Inc}`, then :math:`\\text{Inc}^+ = A^{\\top}[AA^{\\top}]^{-1}`.

        In practice, we orthogonalize the rows of :math:`A` to get the eigenvectors :math:`U` of :math:`\\mathbf{K}=UU^{\\top}`.

        .. seealso::

            - :py:meth:`plot_kernel`
        """

        if self.kernel is None:
            self.compute_kernel_eig_vecs()  # U = QR(Inc[:-1,:].T)
            # K = UU.T
            self.kernel = self.kernel_eig_vecs.dot(self.kernel_eig_vecs.T)

    def compute_kernel_eig_vecs(self):
        """ See explaination in :func:`compute_kernel <compute_kernel>`
        """

        if self.kernel_eig_vecs is None:

            inc_mat = self.nx.incidence_matrix(self.graph, oriented=True)
            # Discard any row e.g. the last one
            A = inc_mat[:-1, :].toarray()
            # Orthonormalize rows of A
            self.kernel_eig_vecs, _ = qr(A.T, mode='economic')

    def plot(self, title=''):
        """ Display the last realization (spanning tree) of the corresponding :class:`UST` object.

        :param title:
            Plot title

        :type title:
            string

        .. seealso::

            - :py:meth:`sample`
        """

        graph_to_plot = self.list_of_samples[-1]

        plt.figure(figsize=(4, 4))

        pos = self.nx.circular_layout(self.graph)
        self.nx.draw_networkx(graph_to_plot,
                         pos=pos,
                         node_color='orange',
                         with_labels=True,
                         width=3)

        edge_labs = {e: self.edge_labels[e if e in self.edges else e[::-1]]
                     for e in graph_to_plot.edges()}
        self.nx.draw_networkx_edge_labels(graph_to_plot,
                                     pos=pos,
                                     edge_labels=edge_labs,
                                     font_size=20)

        plt.axis('off')

        str_title = 'A realization of UST with {} procedure'.format(self.sampling_mode)
        plt.title(title if title else str_title)

    def plot_graph(self, title=''):
        """Display the original graph defining the :class:`UST` object

        :param title:
            Plot title

        :type title:
            string

        .. seealso::

            - :func:`compute_kernel <compute_kernel>`
        """

        # edge_lab = [r'$e_{}$'.format(i) for i in range(self.nb_edges)]
        # edge_labels = dict(zip(self.edges, edge_lab))
        # node_labels = dict(zip(self.nodes, self.nodes))

        plt.figure(figsize=(4, 4))

        pos = self.nx.circular_layout(self.graph)
        self.nx.draw_networkx(self.graph,
                         pos=pos,
                         node_color='orange',
                         with_labels=True,
                         width=3)
        # nx.draw_networkx_labels(self.graph,
        #                         pos,
        #                         node_labels)
        self.nx.draw_networkx_edge_labels(self.graph,
                                     pos=pos,
                                     edge_labels=self.edge_labels,
                                     font_size=20)

        plt.axis('off')

        str_title = 'Original graph'
        plt.title(title if title else str_title)

    def plot_kernel(self, title=''):
        """Display a heatmap of the underlying orthogonal projection kernel :math:`\\mathbf{K}` associated to the DPP underlying the :class:`UST` object

        :param title:
            Plot title

        :type title:
            string

        .. seealso::

            - :func:`compute_kernel <compute_kernel>`
        """

        self.compute_kernel()

        fig, ax = plt.subplots(1, 1, figsize=(4, 4))

        heatmap = ax.pcolor(self.kernel, cmap='jet')

        ax.set_aspect('equal')

        ticks = np.arange(self.nb_edges)
        ticks_label = [r'${}$'.format(tic) for tic in ticks]

        ax.xaxis.tick_top()
        ax.set_xticks(ticks + 0.5, minor=False)

        ax.invert_yaxis()
        ax.set_yticks(ticks + 0.5, minor=False)

        ax.set_xticklabels(ticks_label, minor=False)
        ax.set_yticklabels(ticks_label, minor=False)

        str_title = 'Correlation K kernel: transfer current matrix'
        plt.title(title if title else str_title, y=1.08)

        # Adapt size of colbar to plot
        # https://stackoverflow.com/questions/18195758/set-matplotlib-colorbar-size-to-match-graph
        cax = fig.add_axes([ax.get_position().x1 + 0.02,
                            ax.get_position().y0,
                            0.05,
                            ax.get_position().height])
        plt.colorbar(heatmap, cax=cax)


import time
import requests
import json
import logging

from service.model import parse_tx
from service.db import DBWriter

from service.settings import NODE_EXT_URL, NODE_INT_URL

logging.basicConfig(level=logging.INFO, format='[%(levelname)s]: %(asctime)s: %(message)s')

db = DBWriter()

last_top, _ = db.get_last_block() or (37000, 0)

logging.info("Starting from %s" % last_top)

while True:
    data = requests.post(NODE_INT_URL,
                         json=["top"]).json()
    height = data[1][2]

    size = min(height - last_top, 500)

    if size > 0:
        headers = requests.post(NODE_EXT_URL,
                                json=["headers", size + 1, last_top]).json()[1][1:]

        for idx, block in enumerate(range(last_top, last_top + size + 1)):
            data = requests.post(NODE_INT_URL,
                                 json=["block", 3, block]).json()

            data, hashes = data[1][1][1:], data[1][2][1:]
            txs = zip([x[1] for x in data], hashes)

            timestamp = (headers[idx][5] + 15192951759) // 10

            db.store_block((block, timestamp))

            txns = []
            for tidx, (tx, _hash) in enumerate(txs):
                txi = parse_tx(tx)
                txns.append((
                    block,
                    timestamp,
                    _hash,
                    txi['nonce'],
                    tidx,
                    txi['from'],
                    txi['to'],
                    txi['amount'],
                    txi['fee'],
                    txi['type'],
                    txi['extra'] and json.dumps(txi['extra'])
                ))

            if len(txns):
                db.store_transactions(txns)

            logging.info("Handling block %d, tx count %d" % (block, len(txns)))
            db.commit()

            last_top = block

    time.sleep(20)

db.close()


import numpy as np
import PIL.Image as image
from sklearn import preprocessing
from sklearn.cluster import KMeans

def load_data(filePath):
  f = open(filePath, 'rb')
  img = image.open(f)
  width, height = img.size
  data = []
  for x in range(width):
    for y in range(height):
      c1, c2, c3 = img.getpixel((x, y))
      data.append([c1, c2, c3])
  f.close()

## 为了加快聚类的收敛，进行MM规范化
  mm = preprocessing.MinMaxScaler()
  data = mm.fit_transform(data)
  return np.mat(data), width, height

img, width, height = load_data('./weixin.jpg')

kmeans = KMeans(n_clusters=2)
kmeans.fit(img)
label = kmeans.predict(img)

# convert the predict result to img size matrix
label = label.reshape([width, height])

pic_mark = image.new('L', (width, height))
for x in range(width):
  for y in range(height):
    pic_mark.putpixel((x, y), int(256/(label[x][y]+1))-1)

pic_mark.save("weixin_mark.jpg", "JPEG")


#!/usr/bin/env python3

################################
#         App.py               #
# Main website routing/control #
# Created by Andrew Copas      #
# Modified by:                 #
# Emanuel Saunders(Nov 10,2019)#
################################

from flask import Flask, render_template, request, redirect
from flaskext.mysql import MySQL
from flask_thumbnails import Thumbnail

app = Flask(__name__, static_url_path='/var/www/html/static', static_folder='static', template_folder='templates')

# Thumbnail extension config
thumb = Thumbnail(app)
app.config['THUMBNAIL_MEDIA_ROOT'] = '/var/www/html/static/user_images'
app.config['THUMBNAIL_MEDIA_URL'] = '/user_images/'
app.config['THUMBNAIL_MEDIA_THUMBNAIL_ROOT'] = '/var/www/html/static/user_images/cache'
app.config['THUMBNAIL_MEDIA_THUMBNAIL_URL'] = '/var/www/html/static/user_images/cache/'
app.config['THUMBNAIL_STORAGE_BACKEND'] = 'flask_thumbnails.storage_backends.FilesystemStorageBackend'
app.config['THUMBNAIL_DEFAUL_FORMAT'] = 'JPEG'
# Connect to the MySQL database
app.config['MYSQL_DATABASE_USER'] = 'root'
app.config['MYSQL_DATABASE_PASSWORD'] = 'password'
app.config['MYSQL_DATABASE_DB'] = 'prototypedb'
app.config['MYSQL_DATABASE_HOST'] = 'localhost'

mysql = MySQL()
mysql.init_app(app)

@app.route('/ourTeam')
def outTeam():
    return render_template('about.html')

@app.route('/about_Alex')
def alexProfile():
    return render_template('about_team_members/about_Alex.html')

@app.route('/about_AlexLee')
def alexLeeProfile():
    return render_template('about_team_members/about_AlexLee.html')

@app.route('/about_Andrew')
def andrewProfile():
    return render_template('about_team_members/about_Andrew.html')

@app.route('/about_Emanuel')
def emanuelProfile():
    return render_template('about_team_members/about_Emanuel.html')

@app.route('/about_Kevin')
def kevinProfile():
    return render_template('about_team_members/about_Kevin.html')

@app.route('/about_Tim')
def timProfile():
    return render_template('about_team_members/about_Tim.html')

@app.route('/admin_dash')
def adminDash():
    return render_template('admin_dash.html')

@app.route('/user_dash')
def userDash():
    return render_template('user_dash.html')

@app.route('/vp', methods=['GET', 'POST'])
def verticalproto():
    conn = mysql.connect()
    cursor = conn.cursor()
    cursor.execute("SELECT name from categories ORDER BY name ASC")
    conn.commit()
    cats = cursor.fetchall()
    if request.method == "POST":
        category = request.form['category']
        item = request.form['item']
        query = item
        # all in the search box will return all the tuples
        if len(item) == 0:
            if category == "All":
                cursor.execute("SELECT P.title, P.description, C.name, P.image FROM posts P, categories C WHERE P.category=C.cID")
                conn.commit()
                data = cursor.fetchall()
            else:
                cursor.execute("SELECT P.title, P.description, C.name, P.image FROM posts P, categories C WHERE P.category = C.cID AND C.name = '%s'" %(category))
                conn.commit()
                data = cursor.fetchall()
        # search by category
        else:
            if category == "All":
                cursor.execute("SELECT P.title, P.description, C.name, P.image FROM posts P JOIN categories C ON P.category = C.cID WHERE P.description LIKE '%%%s%%' OR P.title LIKE '%%%s%%'" %(item, item))
                conn.commit()
                data = cursor.fetchall()
            else:
                cursor.execute("SELECT P.title, P.description, C.name, P.image FROM posts P JOIN categories C ON P.category = C.cID WHERE C.name = '%s' AND (P.description LIKE '%%%s%%' OR P.title LIKE '%%%s%%')" %(category, item, item))
                conn.commit()
                data = cursor.fetchall()
            conn.close()
        return render_template('verticalproto.html', query=query, data=data, cats=cats)
    conn.close()
    return render_template('verticalproto.html', cats=cats)

@app.route('/', methods=['GET', 'POST'])
def search():
    conn = mysql.connect()
    cursor = conn.cursor()
    cursor.execute("SELECT name from categories ORDER BY name ASC")
    conn.commit()
    cats = cursor.fetchall()
    if request.method == "POST":
        category = request.form['category']
        item = request.form['item']
        query = item
        # all in the search box will return all the tuples
        if len(item) == 0:
            if category == "All":
                cursor.execute("SELECT P.title, P.description, C.name, P.image FROM posts P, categories C WHERE P.category=C.cID")
                conn.commit()
                data = cursor.fetchall()
            else:
                cursor.execute("SELECT P.title, P.description, C.name, P.image FROM posts P, categories C WHERE P.category = C.cID AND C.name = '%s'" %(category))
                conn.commit()
                data = cursor.fetchall()
        # search by category
        else:
            if category == "All":
                cursor.execute("SELECT P.title, P.description, C.name, P.image FROM posts P JOIN categories C ON P.category = C.cID WHERE P.description LIKE '%%%s%%' OR P.title LIKE '%%%s%%'" %(item, item))
                conn.commit()
                data = cursor.fetchall()
            else:
                cursor.execute("SELECT P.title, P.description, C.name, P.image FROM posts P JOIN categories C ON P.category = C.cID WHERE C.name = '%s' AND (P.description LIKE '%%%s%%' OR P.title LIKE '%%%s%%')" %(category, item, item))
                conn.commit()
                data = cursor.fetchall()
            conn.close()
        return render_template('search.html', query=query, data=data, cats=cats)
    conn.close()
    return render_template('index.html', cats=cats)

if __name__ == '__main__':
    app.run(host="0.0.0.0", port=80, debug=True)


from moviepy.editor import VideoFileClip, TextClip
import os
# 读取文本文件内容
with open(os.getcwd() + "/log/output.txt", 'r',encoding="utf-8") as file:
    text_content = file.read()
# 设置视频速率
rate = 30
# 读取视频文件
video = VideoFileClip(os.getcwd() + "/log/voc_changed.mp4")
# 创建字幕文本对象
text = TextClip(text_content, fontsize=30, color='white', bg_color='black')
# 设置字幕文本的位置和持续时间
text = text.set_position(('center', 'bottom')).set_duration(video.duration)
# 设置视频速率
video = video.set_duration(video.duration/rate)
# 合并视频和字幕
final = video.set_audio(None).set_audio(video.audio).set_fps(rate).set_duration(video.duration)
final = final.set_audio(None).set_audio(video.audio)
# 保存合成后的视频
final.write_videofile("output.mp4", codec="libx264", audio_codec="aac")


from selenium import webdriver
from selenium.webdriver.common.by import By
import re
import time
import numpy as np
from datetime import datetime, timedelta
import argparse

parser = argparse.ArgumentParser()
parser.add_argument("-d", "--day", dest="day", help="Day Value")
parser.add_argument("-m", "--month", dest="month", help="Month Value")
parser.add_argument("-y", "--year", dest="year", help="Year Value")
parser.add_argument("-u", "--utc", dest="time", help="Time Value")
parser.add_argument("-t", "--type", dest="type", help="1 for Wind Speed, Direction and Relative Humidity \n 2 for Current Speed, Direction and Temperature")

options = parser.parse_args()

#utc = ["0000","0600","1200","1800"]
utc = [str(options.time)]

start = datetime(int(options.year), int(options.month), int(options.day))
end = start+timedelta(days=1)

try:
    if(str(options.type)=="1"):
        while(start!=end):
            stringdate = start.strftime('%Y-%m-%d')
            start = start + timedelta(days=1)
            filename = stringdate.replace("-", "_")
            htmldate = stringdate.replace("-", "/")

            for l in utc:
                reset = 1
                driver = webdriver.Firefox()
                driver.get("https://earth.nullschool.net/#"+htmldate+"/"+l+"Z/wind/surface/level/overlay=relative_humidity/loc=0,0")
                time.sleep(4)
                counter=1
                stringdir=""
                stringspeed=""
                stringrelativehumidity=""
                print(htmldate+"_"+l)
                for i in np.arange(30.00,-10.20,-0.20):
                    i = round(i, 1)

                    if (reset == 50):
                        driver.close()
                        driver = webdriver.Firefox()
                        driver.get("https://earth.nullschool.net/#"+htmldate+"/"+l+"Z/wind/surface/currents/overlay=temp/loc=0,0")
                        time.sleep(4)
                        reset=1

                    for j in np.arange(60.00,100.20,0.20):
                        j = round(j, 1)
                        site="https://earth.nullschool.net/#"+htmldate+"/"+l+"Z/wind/surface/level/overlay=relative_humidity/loc="+str(j)+","+str(i)
                        driver.get(site)
                        coord = driver.find_element(By.XPATH, "//div[@id='spotlight-panel']/div[1]/div[1]")
                        spotlight = driver.find_element(By.XPATH, "//div[@id='spotlight-panel']/div[2]/div[1]")
                        humidity = driver.find_element(By.XPATH, "//div[@id='spotlight-panel']/div[3]/div[1]")
                        pattern = r'[^A-Za-z0-9.]+'
                        x = spotlight.text.split("@")
                        angle = re.sub(pattern, '', x[0])
                        speed = re.sub(pattern, '', x[1])

                        rh = re.sub(pattern, '', humidity.text)
                        stringdir = stringdir+angle+","
                        stringspeed=stringspeed+speed+","
                        stringrelativehumidity=stringrelativehumidity+rh+","

                    stringdir = stringdir+angle+"\n"
                    stringspeed=stringspeed+speed+"\n"
                    stringrelativehumidity=stringrelativehumidity+rh+"\n"

                    reset+=1

                    print(counter)
                    counter+=1

                driver.close()
                f = open("rh/rh_"+filename+"_"+l+".csv", "w")
                f.write(stringrelativehumidity)
                f.close()

                f = open("wind_direction/w_d_"+filename+"_"+l+".csv", "w")
                f.write(stringdir)
                f.close()

                f = open("wind_speed/w_s_"+filename+"_"+l+".csv", "w")
                f.write(stringspeed)
                f.close()

    if(str(options.type)=="2"):
        while(start!=end):
            stringdate = start.strftime('%Y-%m-%d')
            start = start + timedelta(days=1)
            filename = stringdate.replace("-", "_")
            htmldate = stringdate.replace("-", "/")

            for l in utc:
                reset = 1
                driver = webdriver.Firefox()
                driver.get("https://earth.nullschool.net/#"+htmldate+"/"+l+"Z/wind/surface/currents/overlay=temp/loc=0,0")
                time.sleep(4)
                counter=1
                stringdir=""
                stringspeed=""
                stringtemperature=""
                print(htmldate+"_"+l)
                for i in np.arange(30.00,-10.20,-0.20):
                    i = round(i, 1)

                    if (reset == 50):
                        driver.close()
                        driver = webdriver.Firefox()
                        driver.get("https://earth.nullschool.net/#"+htmldate+"/"+l+"Z/wind/surface/currents/overlay=temp/loc=0,0")
                        time.sleep(4)
                        reset=1

                    for j in np.arange(60.00,100.20,0.20):
                        j = round(j, 1)
                        site="https://earth.nullschool.net/#"+htmldate+"/"+l+"Z/wind/surface/currents/overlay=temp/loc="+str(j)+","+str(i)
                        driver.get(site)
                        spotlight = driver.find_element(By.XPATH, "//div[@id='spotlight-panel']/div[2]/div[1]")
                        temp = driver.find_element(By.XPATH, "//div[@id='spotlight-panel']/div[3]/div[1]")
                        pattern = r'[^A-Za-z0-9.]+'
                        tpattern = r'[^A-Za-z0-9.-]+'
                        if (len(spotlight.text) == 0):
                            angle = "0"
                            speed = "0"
                        else:
                            x = spotlight.text.split("@")
                            angle = re.sub(pattern, '', x[0])
                            speed = re.sub(pattern, '', x[1])

                        t = re.sub(tpattern, '', temp.text)

                        stringdir = stringdir+angle+","
                        stringspeed=stringspeed+speed+","
                        stringtemperature=stringtemperature+t+","

                    stringdir = stringdir+angle+"\n"
                    stringspeed=stringspeed+speed+"\n"
                    stringtemperature=stringtemperature+t+"\n"

                    reset+=1

                    print(counter)
                    counter+=1

                driver.close()
                f = open("temp/t_"+filename+"_"+l+".csv", "w")
                f.write(stringtemperature)
                f.close()

                f = open("current_direction/c_d_"+filename+"_"+l+".csv", "w")
                f.write(stringdir)
                f.close()

                f = open("current_speed/c_s_"+filename+"_"+l+".csv", "w")
                f.write(stringspeed)
                f.close()
except:
    try:
        driver.close()
    except:
        exit(1)
    exit(1)
else:
    exit(0)


"""empty message

Revision ID: 57305af1bc6e
Revises:
Create Date: 2023-02-16 11:56:32.264329

"""
import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision = "57305af1bc6e"
down_revision = None
branch_labels = None
depends_on = None

def upgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "dummy_model",
        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
        sa.Column("name", sa.String(length=200), nullable=False),
        sa.PrimaryKeyConstraint("id"),
    )
    op.create_table(
        "position",
        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
        sa.Column("name", sa.String(length=10), nullable=False),
        sa.PrimaryKeyConstraint("id"),
        sa.UniqueConstraint("name"),
    )
    op.create_table(
        "team",
        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
        sa.Column("name", sa.String(length=200), nullable=False),
        sa.Column("resolution", sa.Text(), nullable=False),
        sa.Column("max_members", sa.Integer(), nullable=False),
        sa.PrimaryKeyConstraint("id"),
    )
    op.create_table(
        "user",
        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
        sa.Column("username", sa.String(length=50), nullable=False),
        sa.Column("fullname", sa.String(length=20), nullable=False),
        sa.Column("hashed_password", sa.String(length=200), nullable=False),
        sa.Column("token", sa.String(length=200), nullable=False),
        sa.Column("team_id", sa.Integer(), nullable=True),
        sa.ForeignKeyConstraint(
            ["team_id"],
            ["team.id"],
        ),
        sa.PrimaryKeyConstraint("id"),
        sa.UniqueConstraint("username"),
    )
    op.create_table(
        "team_application",
        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
        sa.Column("comment", sa.Text(), nullable=False),
        sa.Column("user_id", sa.Integer(), nullable=False),
        sa.Column("team_id", sa.Integer(), nullable=False),
        sa.ForeignKeyConstraint(
            ["team_id"],
            ["team.id"],
        ),
        sa.ForeignKeyConstraint(
            ["user_id"],
            ["user.id"],
        ),
        sa.PrimaryKeyConstraint("id"),
        sa.UniqueConstraint("user_id", "team_id"),
    )
    op.create_table(
        "users_positions",
        sa.Column("user_id", sa.Integer(), nullable=False),
        sa.Column("position_id", sa.Integer(), nullable=False),
        sa.ForeignKeyConstraint(
            ["position_id"],
            ["position.id"],
        ),
        sa.ForeignKeyConstraint(
            ["user_id"],
            ["user.id"],
        ),
        sa.PrimaryKeyConstraint("user_id", "position_id"),
    )
    # ### end Alembic commands ###

def downgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("users_positions")
    op.drop_table("team_application")
    op.drop_table("user")
    op.drop_table("team")
    op.drop_table("position")
    op.drop_table("dummy_model")
    # ### end Alembic commands ###