"""
Tests SparkML StringIndexer converter.
"""
import unittest
from pyspark.ml.feature import VectorAssembler
from onnxmltools import convert_sparkml
from onnxmltools.convert.common.data_types import FloatTensorType
from onnxmltools.utils import dump_data_and_sparkml_model
from sparkml import SparkMlTestCase

class TestSparkmlStringIndexer(SparkMlTestCase):
    def test_model_vector_assembler(self):
        import numpy
        import pandas
        col_names = ["a", "b", "c"]
        model = VectorAssembler(inputCols=col_names, outputCol='features')
        data = self.spark.createDataFrame([(1., 0., 3.)], col_names)
        model_onnx = convert_sparkml(model, 'Sparkml VectorAssembler',  [
            ('a', FloatTensorType([1, 1])),
            ('b', FloatTensorType([1, 1])),
            ('c', FloatTensorType([1, 1]))
        ])
        self.assertTrue(model_onnx is not None)
        self.assertTrue(model_onnx.graph.node is not None)
        # run the model
        predicted = model.transform(data)
        predicted_np = predicted.select("features").toPandas().features.apply(lambda x: pandas.Series(x.toArray())).values
        data_np = {
            'a': data.select('a').toPandas().values.astype(numpy.float32),
            'b': data.select('b').toPandas().values.astype(numpy.float32),
            'c': data.select('c').toPandas().values.astype(numpy.float32)
        }
        dump_data_and_sparkml_model(data_np, predicted_np, model, model_onnx,
                                    basename="SparkmlVectorAssembler")

if __name__ == "__main__":
    unittest.main()


#coding:utf-8
'''
创建者：liu
作用：从jmeter脚本中提取url，头文件，发送的json数据，发送的方法等
使用方法：实例化的时候传入jmeter脚本。
创建日期：2017/7/20
'''
import os
import re
import json
import sys
from log import *
import logging
reload(sys)
sys.setdefaultencoding('utf8')

__dir__ = os.path.dirname(os.path.abspath(__file__))
jmeter_path = __dir__+'/jmeter/'

class Analysis():

    def __init__(self,file):
        self.file = unicode(jmeter_path+file)

    def __get_value(self,keyvalue):
        '''
        return 正则匹配'>' '<'之间的值。
        '''
        con = self.__open_jmeter()
        patt = re.compile(r"\>(.*?)\<")
        l = [patt.findall(i)[0] for i in con if keyvalue in i]
        return l

    def __open_jmeter(self):
        '''
        :return: jmeter脚本读取的文本
        '''
        try:
            with open(self.file,'r') as f:
                con = f.readlines()
                return con
        except:
            logging.error(u'jmeter脚本打开错误')
            logging.info(u'退出执行py脚本')
            exit()

    def get_headers(self):
        '''

        :return: 请求头信息
        '''
        headers_key = self.__get_value('<stringProp name="Header.name">')
        headers_value = self.__get_value('<stringProp name="Header.value">')
        headers = {}
        for i in range(len(headers_value)):
            headers[headers_key[i]] = headers_value[i]
        logging.info(u'提取头文件')
        return headers

    def get_casename(self):
        '''

        :return: 线程组名字
        '''
        con = self.__open_jmeter()
        for i in con:
            if 'guiclass="ThreadGroupGui"' in i:
                if 'testname' in i:
                    l = i.strip(' ')
        name = l.split(' ')[3].split('=')[1].strip('"')
        logging.info(u'提取名字')
        return name

    def get_method(self):
        '''

        :return: 请求的方法
        '''
        method = self.__get_value('<stringProp name="HTTPSampler.method">')[0]
        logging.info(u'提取方法')
        return method

    def get_url(self):
        '''

        :return: 请求的url
        '''
        IP = self.__get_value('<stringProp name="HTTPSampler.domain">')[0]
        path = self.__get_value('<stringProp name="HTTPSampler.path">')[0]
        protocol = self.__get_value('<stringProp name="HTTPSampler.protocol">')[0]
        url = protocol+'://'+IP+path
        logging.info(u'提取url')
        return url

    def get_post_data(self):
        '''

        :return: 发送的json数据
        '''
        con = self.__open_jmeter()
        for i in con:
            if '<stringProp name="Argument.value">' in i:
                index1 = con.index(i)
            if '<stringProp name="Argument.metadata">' in i:
                index2 = con.index(i)
        p = [i.replace('"','"') for i in con[index1:index2]]
        p1 = ''.join([i.replace('
\n', '') for i in p])
        patt1 = re.compile("\{(.*)\}", )
        data = '{'+patt1.findall(p1)[0]+'}'
        logging.info(u'提取post数据')
        return eval(data)

#a = Analysis('14queryActionCenter_7.jmx')
#print a.get_method()
#print a.get_url()
#print a.get_headers()
#print a.get_casename()
#print a.get_post_data()


import torch
import os

class Policy(torch.nn.Module):
    def __init__(self):
        super().__init__()

        self.hidden1 = torch.nn.Sequential(
            torch.nn.Linear(in_features=110, out_features=256, bias=True),
            torch.nn.Tanh()
        )
        self.hidden2 = torch.nn.Sequential(
            torch.nn.Linear(in_features=256, out_features=256, bias=True),
            torch.nn.Tanh()
        )
        self.logits = torch.nn.Sequential(
            torch.nn.Linear(in_features=256, out_features=4, bias=True),
        )

    def forward(self, x):
        x = self.hidden1(x)
        x = self.hidden2(x)
        return self.logits(x)

class PolicyWrapper:
    def __init__(self, load_path='models/'):
        self.device = torch.device('cuda:0')
        self.policy = Policy()
        self.policy.to(self.device)
        self.policy.hidden1.load_state_dict(torch.load(os.path.join(load_path, 'h1_sd')))
        self.policy.hidden2.load_state_dict(torch.load(os.path.join(load_path, 'h2_sd')))
        self.policy.logits.load_state_dict(torch.load(os.path.join(load_path, 'logit_sd')))

    def get_action(self, observation):
        obs_t = torch.tensor(observation, dtype=torch.float32).to(self.device)
        acts = torch.clip(self.policy(obs_t), min=-1, max=1).detach().cpu().numpy()
        v, th = acts[0], acts[1]

        return v, th


from datetime import datetime

from flask import Flask, render_template, request, redirect
from flask_sqlalchemy import SQLAlchemy

from app.commands import add, subtract, multiply, divide
from app.validations import validate_number

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///database/calculator.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)

class Calculator(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(50), nullable=False)
    firstNumber = db.Column(db.Float, nullable=True)
    secondNumber = db.Column(db.Float, nullable=True)
    operation = db.Column(db.String(50), nullable=True)
    date_created = db.Column(db.DateTime, default=datetime.utcnow)

    def __repr__(self):
        return '<Name %r>' % self.id

@app.route("/")
def index():
    return render_template("app.html")

def save_to_database(record: Calculator) -> str:
    try:
        db.session.add(record)
        db.session.commit()
        return redirect('/')
    except:
        return 'There was an issue adding your task'

def update_database(record: Calculator) -> str:
    try:
        # TODO: update
        # db.session.
        db.session.add(record)
        db.session.commit()
        return redirect('/')
    except:
        return 'There was an issue adding your task'

def get_from_database(id: int) -> Calculator:
    record = Calculator.query.get_or_404(id)
    if record is None:
        pass

    return record

@app.route("/first", methods=["POST"])
def post_first_number():
    print("pirmas")
    number = request.form["number"]
    validate_number(number)
    save_to_database(Calculator(first_number=number))

@app.route("/second", methods=["POST"])
def post_second_number():
    print("antras")
    number = request.form["number"]
    validate_number(number)
    save_to_database(Calculator(second_number=number))

@app.route("/operation", methods=["POST"])
def post_operation_number():
    print("operation")
    operation = request.form.get["operation"]
    save_to_database(Calculator(operation=operation))

@app.route("/calculate")
def get_result(id: int):
    record = get_from_database(id)
    if record.operation == "add":
        result = add(1, 5)
    elif record.operation == "subtract":
        result = subtract(1, 2)
    elif record.operation == "multiply":
        result = multiply(1, 2)
    else:
        result = divide(1, 2)
    return result

@app.errorhandler(404)
def error(error):
    return render_template("index.html", error=error), 404

if __name__ == "__main__":
    app.run(debug=True, port=2000)


from dataclasses import dataclass
from os.path import relpath, abspath

from .const import (
    LIMITED,
    NORMAL,
    FULL,
)
from .utils import formatted_current_time, File, Directory

class Colors:
    DARK_GRAY = "\x1b[38;5;8m"
    LIGHT_GRAY = "\x1b[38;5;249m"
    GREEN = "\x1b[38;5;40m"
    YELLOW = "\x1b[38;5;220m"
    RED = "\x1b[38;5;196m"
    UNDERLINE = "\x1b[4m"
    RESET = "\x1b[0m"

class NoColors:
    DARK_GRAY = ""
    LIGHT_GRAY = ""
    GREEN = ""
    YELLOW = ""
    RED = ""
    UNDERLINE = ""
    RESET = ""

@dataclass
class Operation:
    name: str
    symbol: str
    color: str

class CLI:
    def __init__(self, verbosity: int, no_color: bool):
        self.verbosity = verbosity
        self.colors = NoColors() if no_color else Colors()
        self.add_operation = Operation("added", "+", self.colors.GREEN)
        self.modify_operation = Operation("modified", "~", self.colors.YELLOW)
        self.remove_operation = Operation("removed", "-", self.colors.RED)

    def _print_prefix_timestamp(self):
        print(
            f"{self.colors.LIGHT_GRAY}[{self.colors.DARK_GRAY}{formatted_current_time()}{self.colors.LIGHT_GRAY}]{self.colors.RESET}",
            end=" ",
        )

    def _verbose_type_of_entries(self, entries: "set[str]"):
        nr_of_files = sum(1 for entry in entries if type(entry) == File)
        nr_of_directories = sum(1 for entry in entries if type(entry) == Directory)
        output_parts = []

        if nr_of_files > 0:
            output_parts.append(f'{nr_of_files} file{"" if nr_of_files == 1 else "s"}')
        if nr_of_directories > 0:
            output_parts.append(f'{nr_of_directories} director{"y" if nr_of_directories == 1 else "ies"}')

        return " and ".join(output_parts)

    def watching_entries(self, entries: "set[File | Directory]", max_displayed_entries = 10):
        sorted_entries: "list[File | Directory]" = sorted(entries, key=lambda entry: entry.path)
        types_of_entries = self._verbose_type_of_entries(sorted_entries)

        self._print_prefix_timestamp()
        if self.verbosity == LIMITED:
            print(f'Watching {types_of_entries}')

        elif self.verbosity == NORMAL:
            print(f'Watching {types_of_entries}:')
            for entry in sorted_entries[:max_displayed_entries]:
                print(relpath(entry.path))
            if len(sorted_entries) > max_displayed_entries:
                print(f'and {len(sorted_entries) - max_displayed_entries} more...')

        elif self.verbosity == FULL:
            print(f'Watching {types_of_entries}:')
            for entry in sorted_entries:
                print(abspath(entry.path))

    def _print_entry_operation(self, entry: "File | Directory", operation: Operation):
        self._print_prefix_timestamp()
        if self.verbosity == LIMITED:
            print(f"{operation.color}{operation.symbol}{self.colors.UNDERLINE}{relpath(entry.path)}{self.colors.RESET}")

        elif self.verbosity == NORMAL:
            print(f"{operation.color}{self.colors.UNDERLINE}{relpath(entry.path)}{self.colors.RESET} {operation.name}")

        elif self.verbosity == FULL:
            entry_type = type(entry) == File and "File" or "Directory"
            print(f"{entry_type} {operation.color}{self.colors.UNDERLINE}{abspath(entry.path)}{self.colors.RESET} was {operation.name}")

    def added_entry(self, entry: "File | Directory"):
        self._print_entry_operation(entry, self.add_operation)

    def modified_entry(self, entry: "File | Directory"):
        self._print_entry_operation(entry, self.modify_operation)

    def removed_entry(self, entry: "File | Directory"):
        self._print_entry_operation(entry, self.remove_operation)

    def running_command(self, command: str, background: bool = False):
        if self.verbosity == LIMITED:
            return

        self._print_prefix_timestamp()
        if self.verbosity == NORMAL:
            print(command)

        elif self.verbosity == FULL:
            print(f'Running in {"background" if background else "foreground"}: {command}')


import dash_bootstrap_components as dbc
import dash
import json
import pandas as pd
import requests
import datetime

import plotly.express as px
from dash import html, dcc, Input, Output, callback, State, no_update, dash_table
from auth import authenticate_user, validate_login_session
from flask import session

# Info carrier
from utils.data import Data_carrier
info_carrier = Data_carrier()

CONTENT_STYLE = {
    "margin-left": "18rem",
    "margin-right": "2rem",
    "padding": "2rem 1rem",
}

SIDEBAR_STYLE = {
    "position": "fixed",
    "top": 0,
    "left": 0,
    "bottom": 0,
    "width": "18rem",
    "padding": "2rem 1rem",
    "background-color": "#f8f9fa",
}

user_pages = [
    {
        "name": "Home Page.",
        "path": "/home"
    },
    {
        "name": "Transactions Page.",
        "path": "/transactions"
    },
    {
        "name": "Reports page.",
        "path": "/reports"
    }
]

sidebar = html.Div(
    [
        html.H2("Pages", className="display-4"),
        html.Hr(),
        html.P(
            "Navigate through the page with this menu.", className="lead"
        ),
        dbc.Nav(
            [
                dbc.NavLink(
                    [
                        html.Div(page["name"], className="ms-2"),
                    ],
                    href=page["path"],
                    active="exact",
                )
                for page in user_pages
            ],
            vertical=True,
            pills=True,
        ),
    ],
    style=SIDEBAR_STYLE,
)
df = px.data.gapminder()

def table(df, continent=None):
    dff = df[df.continent==continent]
    return dff

def years_data(years=[]):
    if len(years) == 0:
        return [{"label": "Select an Account", "value": "3", "disabled": True},]
    else:
        return [{'label': str(year), 'value': year} for year in years]

# reports layout content
@validate_login_session
def reports_layout():
    accounts = [account['cbu'] for account in info_carrier.get_specific()]
    months = [{'label': str(i), 'value': i} for i in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]]
    col_name = [{"name": i, "id": i} for i in ['amount', 'date', 'description', 'final_account', 'origin_account', 'type']]
    default_data = [{}]
    return \
        html.Div([
            dcc.Location(id='reports-url',pathname='/reports'),
            dbc.Container(
                [
                    sidebar,
                    dbc.Row(
                        [
                        dbc.Col(
                            [
                                dcc.Dropdown(options=accounts, id='cont-choice'),
                            ],
                        ),
                        dbc.Col(
                            [
                                dcc.Dropdown(options=years_data(), id='year-choice'),
                            ],
                        ),
                        dbc.Col(
                            [
                                dcc.Dropdown(options=months, id='month-choice'),
                            ],
                        ),

                        ],
                    justify='center',
                    style=CONTENT_STYLE),
                    dbc.Row([
                        dash_table.DataTable(id = 'table', columns=col_name, data=default_data),
                        ],
                    justify='center',
                    style=CONTENT_STYLE),

                    html.Br(),

                    dbc.Row(
                        dbc.Col(
                            dbc.Button('Logout',id='logout-button',color='danger',size='sm'),
                            width=4
                        ),
                        justify='center',
                        style=CONTENT_STYLE
                    ),

                    html.Br()
                ],
            )
        ]
    )

@callback(
    Output('reports-url','pathname'),
    [Input('logout-button','n_clicks')]
)
def logout_(n_clicks):
    '''clear the session and send user to login'''
    if n_clicks is None or n_clicks==0:
        return no_update
    session['authed'] = False
    return '/login'

@callback(
    [Output('table', 'data'),
    Output('year-choice', 'options')],
    [Input('cont-choice', 'value'),
    Input('year-choice', 'value'),
    Input('month-choice', 'value')],
    prevent_initial_call=True,
)
def update_graph(value, year, month):
    response = requests.get(f'http://127.0.0.1:9000/account/{value}')
    data = json.loads(response.text)

    date = datetime.date.fromisoformat(data['creation_date'])
    date_list = [date.year]
    for y in range(datetime.date.today().year-date.year):
        date_list.append(date.year + y)

    print(value, year, month)
    print(date_list)

    if year is None and month is None:
        return no_update, years_data(date_list)
    else:
        reports = {
            "year": year,
            "month": month,
            "cbu": value
        }

        transactions_report = requests.get('http://127.0.0.1:9000/report_transactions', json=reports)
        if transactions_report.status_code == 200:
            dict_transaction = json.loads(transactions_report.text)
            print(dict_transaction['transactions'])

            df_dict = {
                'amount': [],
                'date':[],
                'description':[],
                'final_account':[],
                'origin_account':[],
                'type':[]
            }

            for data in dict_transaction['transactions']:
                df_dict['amount'].append(data['amount'])
                df_dict['date'].append(datetime.date.fromisoformat(data['date']))
                df_dict['description'].append(data['description'])
                df_dict['final_account'].append(data['final_account'])
                df_dict['origin_account'].append(data['origin_account'])
                df_dict['type'].append(data['type'])

            return pd.DataFrame.from_dict(df_dict).to_dict('records'), years_data(date_list)
        else:
            return no_update, years_data(date_list)


# 1.keywords 3个都有,   2.按长度排序
import re
import scrapy
from copy import deepcopy
import tldextract
from scrapy.linkextractors import LinkExtractor
import xlrd
import io,sys

class SchoolspiderSpider(scrapy.Spider):
    name = 'schoolspider'
    allowed_domains = ['']

    # sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='gb18030')  # 改变标准输出的默认编码 ubuntu不需要
    while True:
        try:
            # excel = input('提取的关键词保存到当前文件夹下的output.xlsx \n请输入excel文件名(带文件扩展名),如文件不在当前文件夹,需带上路径: \n ')
            # data = xlrd.open_workbook(excel)
            data = xlrd.open_workbook("镜像模板-5-17.xlsx")
        except IOError:
            print('文件不存在或路径不对，请从新输入')
        else:
            break

    table = data.sheets()[0]
    nrows = table.nrows
    ncols = table.ncols
    doc_title = table.row_values(0)

    # ajax
    # start_urls = ['http://www.gdsty.cn']
    # 需要重新拼接
    # start_urls = ['http://www.dandelionschool.org']
    # 重定向
    # start_urls = ['http://www.cqhgzy.com']
    # 被黑
    # start_urls = ['http://www.zhirong.net/']
    # start_urls = ['http://www.gdcvi.net']

    start_urls = []
    for i in range(1, nrows):
        start_urls.append('http://'+table.row_values(i)[1])

    # ---
    # start_urls = [table.row_values(i)[1] for i in range(1, nrows)]
    # print(start_urls)
    # sheet = wb.active
    # start_urls = ['http://'+cell.value for cell in list(sheet.columns)[1]][1:]

    # 要匹配的关键词
    title_keywords = ['新闻', '资讯','学校','学院', '校园', '教师', '招聘', '设置', '师资', '教育', '教学', '校务', '信息','公开', '名师', '学子',
                      '风采', '科研', '概况', '党建', '课程','建设','德育','招生','访客','专业','导航','机构','服务','园地','家长']

    def start_requests(self):
        for url in self.start_urls:
            start_url = url
            # item1={'url':'.'.join(tldextract.extract(url))}
            # headers = {}
            url_index_doc = self.start_urls.index(url)
            item1 = {'url': url, 'url_index_doc': url_index_doc}
            # print(item1)
            yield scrapy.Request(
                url=start_url,
                callback=self.parse,
                meta={'item1': item1}
            )

    def parse(self, response):
        print(response.url, '    <<<<<<  response.url')
        item = deepcopy(response.meta["item1"])
        # 匹配后的关键词
        match_words = []
        try:
            try:
                r = response.text
                # r = response.body.decode()
                # print(r)
            # except UnicodeDecodeError :
            #     r = response.body.decode('gbk')
            except Exception as e:
                print(e, '编码错误')
            # body里面所有内容
            # b = re.findall("(.*?)", r, re.S)
            # b=''.join(b)
            # r = ''.join(r)
            title = response.xpath('//title/text()').extract_first()
            if title is None:
                x = re.findall('location.href=\"(.*?)\"',r,re.S)[0] if len(re.findall('location.href=\"(.*?)\"',r,re.S)) > 0 else None
                # print(x,'<')
                if x is None:
                    x = re.findall('location="/(.*?)\"',r,re.S)[0] if len(re.findall('location="/(.*?)\"',r,re.S)) > 0 else None
                    # print(x,'<<')
                if x is not None:
                    url = response.url+'/'+x
                    yield scrapy.Request(
                        url=url,
                        callback=self.parse,
                        dont_filter=True,
                        meta={'item1':item}
                    )
            else:
                title_list = re.findall("[\u4E00-\u9FA5]+", title, re.S)  # title里面的关键词列表
                # title_list = re.findall("([\u4E00-\u9FA5a-zA-Z0-9]+)", title, re.S)  # title里面的关键词列表
                # print('title_list', title_list)
            if len(title_list) > 0:
                for i in title_list:
                    if len(i) < 2:
                        title_list.pop(i)
                count = 0
                x = None
                for word in title_list:
                    num = r.count(word)
                    if num > count:
                        count = num
                        x = word
                # print(count, 'title最大词数目')
                item['title'] = x
                # print("item['title']", item['title'])
                # max(title_list, key=lambda x: r.count(x))
                # if count > 1:
                keywords = response.xpath('//meta[@name="keywords"]/@content').extract_first()
                if keywords is None:
                    keywords = response.xpath('//meta[@name="Keywords"]/@content').extract_first()
                print('keywords --xpath提取内容: ', keywords)

                if keywords is not None:
                    # print("keywords>>>",keywords)
                    keywords_list = re.findall("[\u4E00-\u9FA5]+", keywords, re.S)  # 词列表,正则取汉字
                    # print('<<<keywords_list<<', keywords_list)
                    keywords_list=list(set(keywords_list))  # 去重
                    if len(keywords_list) > 0:
                        for i in keywords_list:
                            if len(i) < 2:
                                keywords_list.pop(i)
                        # keywords>0,在body中找到频率最高的3个词
                        word_num = 0
                        while 1:
                            if keywords_list ==[]:
                                break
                            count = 0
                            word_num += 1
                            if word_num > 3:
                                break
                            for word in keywords_list:
                                # 找keywords中的词在body中出现的次数排前三
                                num = r.count(word)
                                if num > count:
                                    count = num
                                    x = word
                                key = 'keywords{}'.format(word_num)
                                # item['keywords1']

                                item[key] = x
                            if count > 0: # 排除keywords都查不到的情况
                                try:
                                    numa = keywords_list.index(x)
                                    keywords_list.pop(numa)
                                except Exception as e:
                                    print(e)

                        if item.get('keywords2') is None:
                            if item['title'] is not None:
                                if item['title'] != item.get('keywords1'):
                                    item['keywords2'] = item['title']

                else:
                    item['keywords1'] = None

                description = response.xpath('//head/meta[@name="description"]/@content').extract_first()
                item['description'] = description

                copyrt = response.xpath("//*[contains(text(),'版权所有')]/text()").extract_first()
                item['copyrt'] = copyrt
                # print('版权所有', copyrt)

                # 提取当前页面所有2-6字的标题
                le = LinkExtractor()
                links = le.extract_links(response)
                for link in links:
                    # a = link.text.strip()
                    # 去掉汉字以外的特殊字符
                    a = ''.join(re.findall("([\u4E00-\u9FA5]+)", link.text.strip(), re.S))
                    # print('aaaa',a)
                    if 2 < len(a) < 5:
                        for title_keyword in self.title_keywords:
                            # 如果2-6之间的词组 含有title_keywords中的关键词,则加入match_words中
                            if title_keyword in a:
                                match_words.append(a)

                # print('>>',match_words)
                # 含有关键词的词再去重，不用set
                unique_words = []
                if len(match_words) > 0:
                    for i in match_words:
                        if i not in unique_words:
                            unique_words.append(i)
                    item['unique_words'] = unique_words
                else:
                    item['unique_words'] = None

                # if len(match_words) > 0:
                #     unique_words = set(match_words)
                #     item['unique_words'] = unique_words
                # print('spider item',item)
                yield item
        except Exception as e:
            print(e, '--错误')

            # except :
            #     r = response.body.decode('gbk')
            #     print('gbk')
            #     # print(r)
            #     # body里面所有内容
            #     # b = re.findall("(.*?)", r, re.S)
            #     # b=''.join(b)
            #     r = ''.join(r)
            #     # item = {}
            #     # item['url'] = response.url
            #     title = response.xpath('//head/title/text()').extract_first()
            #     title_list = re.findall("([\u4E00-\u9FA5]+)", title, re.S) # title里面的关键词列表
            #     # 取技术最大的词
            #     count=0
            #     x =0
            #     for word in title_list:
            #         num = r.count(word)
            #         if num > count:
            #             count = num
            #             x = word
            #     print(count,'title最大词数目')
            #     item['title']= x
            #     # max(title_list, key=lambda x: r.count(x))
            #
            #     if count > 1:
            #         keywords = response.xpath('//head/meta[@name="keywords"]/@content').extract_first()
            #         keywords_list = re.findall("([\u4E00-\u9FA5]+)", title, re.S) # 词列表
            #         print(keywords_list, '<<<keywords_list<<')
            #         word_num = 0
            #         while 1:
            #             count = 0
            #             word_num += 1
            #             if word_num>3:
            #                 break
            #             if len(keywords_list) > 0:
            #                 for word in keywords_list:
            #                     num = r.count(word)
            #                     if num > count:
            #                         count = num
            #                         x = word
            #                 key = 'keywords{}'.format(word_num)
            #                 item[key] = x
            #                 numa = keywords_list.index(x)
            #                 keywords_list.pop(numa)
            #             else:
            #                 break
            #
            #
            #
            #
            #         description = response.xpath('//head/meta[@name="description"]/@content').extract_first()
            #         item['description'] = description
            #
            #         copyrt = response.xpath("//*[contains(text(),'版权所有')]/text()").extract_first()
            #         item['copyrt'] = copyrt
            #         print('版权所有', copyrt)
            #
            #         # 提取当前页面所有2-6字的标题
            #         le = LinkExtractor()
            #         links = le.extract_links(response)
            #         for link in links:
            #             a = link.text.strip()
            #             if 2 < len(a) <6:
            #                 for title_keyword in self.title_keywords:
            #                     if title_keyword in a:
            #                         match_words.append(a)
            #
            #         # print('>>',match_words)
            #         # 词去重
            #         unique_words = set(match_words)
            #         item['unique_words'] = unique_words
            #
            #         # print(item)
            #         yield item

            # print(match_words)

        # 1.确定导航栏  len()>0
        # a= response.xpath("//ul[contains(@id,'nav')]")
        # print("ul,nav")
        # if a ==[]:
        #     a = response.xpath("//div[contains(@class,'nav')]")
        #     print('div,nav')
        #     if a == []:
        #         a = response.xpath("//div[@id='menu']")
        #         print("div,menu")
        #         if a==[]:
        #             a = response.xpath("//ul[contains(@class,'nav')]")
        #             print('div,Nav')

        # 2.定位导航栏标题的a标签.
        # b = response.xpath(".//a[contains(text(),'首页')]")
        # if b == []:
        #     b= response.xpath(".//a[contains(text(),'概')]")
        # print('b>>',b)
        # print("-----",len(b))
        #
        # # 3.根据2步a标签的 取到兄弟标签
        # c = b.xpath("./following-sibling::*")
        # print(len(c), "*" * 30)
        #
        # if c ==[] or len(c)<3:
        #     c = b.xpath("../following-sibling::*")
        #     print(len(c),'>'*50+'父')
        #
        #     if c ==[] or len(c)<5:
        #         c = b.xpath("../../following-sibling::*")
        #         print(len(c),"父父")
        #
        #         if c == [] or len(c) < 5:
        #             c = b.xpath("../../../following-sibling::*")
        #             print(len(c),'父父父')

#"""
        # le = LinkExtractor(restrict_xpaths='//div[@id="menu"]') # 提取xpath选中区域下的链接
        # le = LinkExtractor()
        # le = LinkExtractor(allow_domains=['txedu.cn']) #过滤域名
        # links = le.extract_links(response)
        # print([link.url for link in links])
        # print(len(links),'链接总数')

        # for link in links:
        #     print(link.url)

            # if len(re.findall(".*?概况",link.text))>0:
            #     print('>>>',link.url, )
            #

        # item['link_num'] = len(links)
        # print(deepcopy(item))
        # print(item,'+++')
#"""


import pytest
import torch
from torch.utils.data import DataLoader
from torchvision import transforms
from torchvision.datasets import CIFAR10

from modeling.diffusion import DiffusionModel
from modeling.training import train_step
from modeling.unet import UnetModel

@pytest.fixture
def train_dataset():
    tf = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
    dataset = CIFAR10(
        "./data",
        train=True,
        download=True,
        transform=tf,
    )
    return dataset

@pytest.mark.parametrize(["device"], [["cpu"], ["cuda"]])
def test_train_on_one_batch(device, train_dataset):
    # note: you should not need to increase the threshold or change the hyperparameters
    ddpm = DiffusionModel(
        eps_model=UnetModel(3, 3, hidden_size=32),
        betas=(1e-4, 0.02),
        num_timesteps=1000,
    )
    ddpm.to(device)

    optim = torch.optim.Adam(ddpm.parameters(), lr=5e-4)
    dataloader = DataLoader(train_dataset, batch_size=4, shuffle=True)

    x, _ = next(iter(dataloader))
    loss = None
    for i in range(50):
        loss = train_step(ddpm, x, optim, device)
    assert loss < 0.5

def test_training():
    # note: implement and test a complete training procedure (including sampling)
    pass


import os
import time
import json
from google.cloud import datastore

os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "./firstdemo-a214cb88e8b0.json"

dbclient = datastore.Client(project='firstdemo-207206')
kind = "Books"

def get_data(id):
    '''Get data by Id'''
    key = dbclient.key(kind, str(id))
    return (dbclient.get(key))

def get_completedata():
    query = dbclient.query(kind=kind)
    results = list(query.fetch())
    return results

def add_data(id, data):
    key=dbclient.key(kind, id)
    entity = datastore.Entity(key=key,)

    entity["book_name"] = data
    dbclient.put(entity)

if __name__ == "__main__":
    print( get_completedata() )


import pytest

pytestmark = pytest.mark.asyncio

async def test_webhook_flow(test_cli_user):
    guild = await test_cli_user.create_guild()
    channel = await test_cli_user.create_guild_channel(guild_id=guild.id)

    resp = await test_cli_user.post(
        f"/api/v6/channels/{channel.id}/webhooks", json={"name": "awooga"}
    )
    assert resp.status_code == 200
    rjson = await resp.json
    assert rjson["channel_id"] == str(channel.id)
    assert rjson["guild_id"] == str(guild.id)
    assert rjson["name"] == "awooga"

    webhook_id = rjson["id"]
    webhook_token = rjson["token"]

    resp = await test_cli_user.post(
        f"/api/v6/webhooks/{webhook_id}/{webhook_token}",
        json={"content": "test_message"},
        headers={"authorization": ""},
    )
    assert resp.status_code == 204

    refetched_channel = await channel.refetch()
    message = await test_cli_user.app.storage.get_message(
        refetched_channel.last_message_id
    )
    assert message["author"]["id"] == webhook_id


def applyRegexsToDirOfXML(directoryPath,stringPhraseList,fieldsSelect,caseSensitive=False):
    """
    Applies a regex search to the list of inputTexts and a dictionary wherein the keys are a tuple of the string phrase and the file name and the values are booleans indicating whether the string phrase was found in the file.

    Parameters
    ----------
    directoryPath : string
        A string corresponding to the path to the directory containing the xml files to be searched.
    stringPhraseList : list of strings
        A list of strings corresponding to the phrases one is interested in assessing the occurrences of.
    fieldsSelect : list of strings
        A list of strings corresponding to the nested sequence of fields to be searched.  First field is the root tag.  Last field is the field to be searched.  Will throw an error if not specified correctly.
    caseSensitive : bool, optional
        A boolean indicating whether the search should be case sensitive. The default is False.

    Returns
    -------
    outputDict : dict
        A dictionary with a field corresponding to the string phrase, whose value is a dictionary with fields corresponding to the file names, whose values are booleans indicating whether the string phrase was found in the file.
    """
    # determine if dask is available
    try:
        import dask
        import dask.bag as db
        daskAvailable=True
    except:
        daskAvailable=False
    import os
    fileList=os.listdir(directoryPath)
    # create an empty dictionary with the tuples of the string phrase and the file name as the keys
    outputDict={}
    # if dask is available, use it to parallelize the search
    if daskAvailable:
        # iterate across the stringPhraseList and apply the regex search to each file
        for iStringPhrase in stringPhraseList:
            # get the list of files in the directory

            # create a dask bag with the file list
            fileListBag=db.from_sequence(fileList)
            # use the dask bag to apply the regex search to each file and place it in the appropriate tuple dictionary entry
            outputDict[iStringPhrase]=dict(fileListBag.map(lambda x: (iStringPhrase,x,applyRegexToXMLFile(os.path.join(directoryPath,x),iStringPhrase,fieldsSelect,caseSensitive=caseSensitive))).compute())
    # if dask is not available, use a for loop to iterate across the files
    else:
        # iterate across the stringPhraseList and apply the regex search to each file
        for iStringPhrase in stringPhraseList:
            # get the list of files in the directory
            fileList=os.listdir(directoryPath)
            # iterate across the files
            for iFile in fileList:
                try:
                    # apply the regex search to the file and place it in the appropriate tuple dictionary entry
                    outputDict[iStringPhrase][iFile]=applyRegexToXMLFile(os.path.join(directoryPath,iFile),iStringPhrase,fieldsSelect,caseSensitive=caseSensitive)
                except:
                    # throw an error and specify the file that caused the error
                    raise ValueError('Error in file: '+iFile)

    return outputDict

def convertTupleDictToEfficientDict(tupleDict,rowDescription='',colDescription=''):
    """
    Convets a dictionary with tuples as keys to a dictionary with three keys: rowNames, colNames, and dataMatrix.
    The rowName and colName are themselves dictionaries, with two fields:  "nameValues" and "description".
    "nameValues" is a list of the unique names of the rows or columns, respectively.  "description" is a string describing the data in the row or column.
    The dataMatrix is a N by M matrix of reflecting the values associated with the tuples.

    This is done because presumably this is a more efficient way to store the data.

    Parameters
    ----------
    tupleDict : dict
        A dictionary with tuples as keys.
    rowDescription : string, optional
        A string describing the data in the rows. The default is '', a blank string.
    colDescription : string, optional
        A string describing the data in the columns. The default is '', a blank string.

    Returns
    -------
    efficientDict : dict
        A dictionary with three keys: rowName, colName, and dataMatrix.
    """
    import numpy as np
    # get the row and column names
    rowName=[iTuple[0] for iTuple in tupleDict.keys()]
    colName=[iTuple[1] for iTuple in tupleDict.keys()]
    # get the unique row and column names and preserve the order
    uniqueRowName=list(dict.fromkeys(rowName))
    uniqueColName=list(dict.fromkeys(colName))
    # create the efficient dictionary
    efficientDict={}
    efficientDict['rowName']={}
    efficientDict['colName']={}
    efficientDict['dataMatrix']=np.zeros((len(uniqueRowName),len(uniqueColName)))
    # go ahead and create the description field for the row and column names
    efficientDict['rowName']['description']=rowDescription
    efficientDict['colName']['description']=colDescription
    # before iterating across the rows and colums, go ahead and fill in the row and column names
    efficientDict['rowName']['nameValues']=uniqueRowName
    efficientDict['colName']['nameValues']=uniqueColName
    # iterate across the rows and columns and fill in the data matrix
    for iRow in range(len(uniqueRowName)):
        for iCol in range(len(uniqueColName)):
            # get the row and column names
            iRowName=uniqueRowName[iRow]
            iColName=uniqueColName[iCol]
            # get the value
            iValue=tupleDict[(iRowName,iColName)]
            # fill in the data matrix
            efficientDict['dataMatrix'][iRow,iCol]=iValue
    return efficientDict

def searchInputListsForKeywords_dask(inputLists,keywordList):
    """
    Divides up the items in the inputLists--for items whose description includes
    an item from the input keywordList variable--into groups associated by keyword.

    Returns a dictionary wherein the keys are keywords and the values are lists of items
    wherein the the keyword was found in the associated description.

    Uses dask if available to parallelize the search.

    Parameters
    ----------
    inputLists : list of lists
        A list of lists wherein each list contains a set of items to be assessed for keyword occurrences.
    keywordList : list of strings
        A list of strings corresponding to the keywords one is interested in assessing the occurrences of.

    Returns
    -------
    grantFindsOut : dictionary
        A a dictionary wherein the keys are keywords and the values are N long boolean vectors indicating
    whether the keyword was found in the associated description.

    See Also
    --------
    grants_by_Agencies : Divides up the grant IDs ('OpportunityID') from the input grantsDF by top level agency.
    """
    import pandas as pd
    import re
    import time
    import numpy as np
    # if inputLists is a series, then convert it to a list
    if type(inputLists)==pd.core.series.Series:
        inputLists=inputLists.values.tolist()

    # create a dictionary in which each each key is a keyword, and the value is a blank boolean vector of length N, where N is the number of items in the inputLists
    # first create a blank boolean vector of length N that will be placed in all dictionary entries
    blankBoolVec=[False]*len(inputLists)
    # then create the dictionary itself
    grantFindsOut={}
    # then populate the dictionary with the blank boolean vectors
    for iKeywords in keywordList:
        grantFindsOut[iKeywords]=blankBoolVec
    # next we compile all of the regex searches that we will perform
    # we replace dashes with spaces to be insensitive to variations in hyphenation behaviors
    compiledRegexList=[re.compile('\\b'+iKeywords.lower().replace('-',' ')+'\\b') for iKeywords in keywordList]

    # next define the function that will be used to search the input text for the compiled regex
    # we do this so that we can parallelize the search
    def searchInputForCompiledRegex(inputText,compiledRegex):
        """
        Searches the input text for the compiled regex and returns True if found, False if not found.
        Function is used to parallelize the search for keywords in the input text.
        """
        # case insensitive find for the keyword phrase
        # use try except to handle the case where there is no description field
        try:
        # get rid of dashes to be insensitive to variations in hyphenation behaviors
            if bool(compiledRegex.search(inputText.lower().replace('-',' '))):
                    #append the ID if found
                    return True
            else:
                return False
        except:
            # do nothing, if there's no description field, then the word can't be found
            return False

    # check if dask is available
    try:
        import dask
        #import dask.dataframe as dd
        import dask.bag as db
        import dask.multiprocessing
        import dask.distributed as dd
        from dask.diagnostics import ProgressBar
        import multiprocessing
        from dask.distributed import Client, progress
        # if dask is available, then use it to parallelize the search

        # first find the number of cores on the machine
        numCores=multiprocessing.cpu_count()
        # establish a dask client with the number of cores minus 2

        # redefining it here, I guess?

            # next define the function that will be used to search the input text for the compiled regex
        # we do this so that we can parallelize the search

        def searchInputForCompiledRegex(inputText,compiledRegex):
            """
            Searches the input text for the compiled regex and returns True if found, False if not found.
            Function is used to parallelize the search for keywords in the input text.
            """
            # case insensitive find for the keyword phrase
            # use try except to handle the case where there is no description field
            try:
            # get rid of dashes to be insensitive to variations in hyphenation behaviors
                if bool(compiledRegex.search(inputText.lower().replace('-',' '))):
                        #append the ID if found
                        return True
                else:
                    return False
            except:
                # do nothing, if there's no description field, then the word can't be found
                return False

        results=[]
        for iKeywords in keywordList:
            for iInput in inputLists:
                tempResult=dask.delayed(searchInputForCompiledRegex(iInput,iKeywords))
                results.append(tempResult)

        with dd.Client(n_workers=numCores-2) as client:
            searchResults=dd.compute(*results,scheduler='processes')

        # now reshape the searchResults into an array
        searchResults=np.array(searchResults).reshape(len(keywordList),len(inputLists))
        # now convert the array into a dictionary
        for iKeywords in range(len(keywordList)):
            grantFindsOut[keywordList[iKeywords]]=searchResults[iKeywords,:]

        # create a dask dataframe from the inputLists
        daskDF=dd.from_pandas(pd.DataFrame(inputLists),npartitions=4)
        # create a dask dataframe from the compiledRegexList
        daskCompiledRegexList=dd.from_pandas(pd.DataFrame(compiledRegexList))
        # use the daskDF and daskCompiledRegexList to parallelize the search across the inputLists, applying searchInputForCompiledRegex for each daskCompiledRegexList entry
        daskSearch=daskDF.map(lambda x: [searchInputForCompiledRegex(x,iRegex) for iRegex in compiledRegexList])
        testOut=daskSearch.compute()

        # next we will parallelize a nested loop, where the outer loop is across the keywords, and the inner loop is across the inputLists
        # this is because there are ~500000 items in the inputLists, and > 100 keywords, the major slowdown is looping across the inputLists
        # as such, we can do a regular loop across the keywords, and then use dask to parallelize the search across the inputLists
        # include a progress bar
        # create a dask bag from the inputLists
        daskBag=db.from_sequence(inputLists,npartitions=4)

        # because there are ~500000 items in the inputLists, and > 100 keywords, the major slowdown is looping across the inputLists
        # as such, we can do a regular loop across the keywords, and then use dask to parallelize the search across the inputLists
        # include a progress bar
        # create a dask bag from the inputLists
        daskBag=db.from_sequence(inputLists,npartitions=4)

        for iKeywords,iCompiledSearch in zip(keywordList,compiledRegexList):
            print('Searching for keyword: '+iKeywords)
            #start timer
            start=time.time()
            # use the dask bag to parallelize the search across the inputLists
            daskSearch=daskBag.map(lambda x: searchInputForCompiledRegex(x,iCompiledSearch))
            # convert the dask bag to list
            boolVec=daskSearch.compute()
            # store the boolean vector in the output dictionary
            grantFindsOut[iKeywords]=boolVec
            #print the number of items found
            print('Number of items found: '+str(sum(boolVec)))
            #print time for this loop
            print('Time for this keyword: '+iKeywords + ' = '+str(time.time()-start))

        # close the dask client
        client.close()
    except:
        # if dask is not available, then use a regular loop
        for iKeywords,iCompiledSearch in zip(keywordList,compiledRegexList):
            for iRows,iListing in enumerate(inputLists):
                # case insensitive find for the keyword phrase
                # use try except to handle the case where there is no description field
                try:
                    # get rid of dashes to be insensitive to variations in hyphenation behaviors
                    if bool(iCompiledSearch.search(iListing.lower().replace('-',' '))):
                        #append the ID if found
                        grantFindsOut[iKeywords][iRows]=True
                except:
                    # do nothing, if there's no description field, then the word can't be found and the false remains in place
                    pass
    return grantFindsOut

def applyKeywordSearch_NSF(inputDF,keywordList):
    """
    Applies the desired regex based keyword search to the relevant field of the NSF dataframe ('AbstractNarration') and
    outputs a dictionary wherein the keys are keywords and the values the 'AwardID' values for the grants in which the
    keyword was found in the associated description.
    Inputs:
        inputDF: pandas dataframe
            The dataframe containing the NSF grant data

        keywordList:  list of strings
            A list of strings corresponding to the keywords one is interested in assessing the occurrences of
    Outputs:
        grantFindsOut: dictionary
            A dictionary wherein the keys are keywords and the values the 'AwardID' values for the grants in which the
            keyword was found in the associated description.
    """
    import pandas as pd
    from itertools import compress
    # get the abstract narration field
    abstractNarrations=inputDF['AbstractNarration'].values.tolist()
    # apply the keyword search
    boolDictionaryKeywords=searchInputListsForKeywords_dask(abstractNarrations,keywordList)
    # convert the boolean vectors to lists of award IDs for each keyword
    # first get the award IDs
    awardIDs=inputDF['AwardID'].values.tolist()
    # then convert the boolean vectors to lists of award IDs
    awardIDLists={}
    for iKeywords in boolDictionaryKeywords.keys():
        awardIDLists[iKeywords]=list(compress(awardIDs,boolDictionaryKeywords[iKeywords]))
    return awardIDLists

def applyRegexToInput(inputText,stringPhrase,caseSensitive=False):
    """
    Applies a regex search to the inputText and returns a boolean indicating whether the stringPhrase was found.

    Parameters
    ----------
    inputText : string
        A string to be assessed for the presence of the stringPhrase.
    stringPhrase : string
        A string corresponding to the phrase one is interested in assessing the occurrences of.
    caseSensitive : bool, optional
        A boolean indicating whether the search should be case sensitive. The default is False.

    Returns
    -------
    bool
        A boolean indicating whether the stringPhrase was found in the inputText.
    """
    import re
    if caseSensitive:
        compiledSearch=re.compile(stringPhrase)
    else:
        compiledSearch=re.compile(stringPhrase.lower())
    try:
        if bool(compiledSearch.search(inputText.lower())):
            return True
        else:
            return False
    except:
        return False

def applyRegexToXMLFile(xmlFilePath,stringPhrase,fieldsSelect,caseSensitive=False):
    """
    Applies a regex search to the inputText and returns a boolean indicating whether the stringPhrase was found.

    Parameters
    ----------
    xmlFilePath : string
        A string corresponding to the path to the xml file to be searched.  Can also be the file contents as a string.
    stringPhrase : string
        A string corresponding to the phrase one is interested in assessing the occurrences of.
    fieldsSelect : list of strings
        A list of strings corresponding to the nested sequence of fields to be searched.  First field is the root tag.  Last field is the field to be searched.  Will throw an error if not specified correctly.
    caseSensitive : bool, optional
        A boolean indicating whether the search should be case sensitive. The default is False.

    Returns
    -------
    bool
        A boolean indicating whether the stringPhrase was found in the inputText.
    """
    import xmltodict
    import re
    if caseSensitive:
        compiledSearch=re.compile(stringPhrase)
    else:
        compiledSearch=re.compile(stringPhrase.lower())
    # if the xmlFilePath is a string, then assume it's the file path and load the file
    if type(xmlFilePath)==str:
        with open(xmlFilePath) as f:
            xmlDict=xmltodict.parse(f.read())
    # close the file
        f.close()
    else:
        xmlDict=xmltodict.parse(xmlFilePath)
    # iterate through the elements of fieldsSelect list to get to the content of the final field
    for iField in fieldsSelect:
        xmlDict=xmlDict[iField]
    # now that we have the relevant contet, we need to convert the text to nlp-ready text
    # use prepareTextForNLP(inputText,stopwordsList=None,lemmatizer=None)
    xmlDict=prepareAllTextsForNLP(xmlDict)

    # use applyRegexToInput
    outputBool=applyRegexToInput(xmlDict,stringPhrase,caseSensitive=caseSensitive)
    return outputBool


#!/usr/bin/env python3

"""
Usage:
    git-cloner [ -n ] [ --workspace <folder> ] <url>

Options:
    --workspace <folder>  The default folder we extract sites to [default: ~/projects]
    -n                    Dry run - show what you would do

"""

# ~/.gitconfig
# [alias]
#   cloner = !~/bin/git-cloner.py

import sys
import os
import docopt
from os.path import expanduser
from urllib.parse import urlparse
from pathlib import Path
import subprocess

def org_from_url(url) -> str:
     return urlparse(url).path.split("/")[1]

def hostname_from_url(url) -> str:
     return urlparse(url).netloc

def get_site_root_folder():
    os.path.dirname()

def root_folder(workspace_folder):
    if workspace_folder.startswith("~"):
        return expanduser(workspace_folder)

if __name__ == "__main__":
    args = docopt.docopt(__doc__, version='git-cloner')
    url = args["<url>"]

    wf = root_folder(args["--workspace"])
    folder = Path(os.path.join(wf,
           hostname_from_url(url),
           org_from_url(url)))

    print(f":: cloning to {folder}")
    if args["-n"]:
      print(f":: mkdir -p {folder}")
      print(f":: git clone {url}")
    else:
      folder.mkdir(parents=True, exist_ok=True)
      subprocess.run(["git", "clone", url], cwd=folder)


#######################################################################
## (C)Copyright 2023 Hewlett Packard Enterprise Development LP
## Licensed under the Apache License, Version 2.0 (the "License"); you may
## not use this file except in compliance with the License. You may obtain
## a copy of the License at
##
##    http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
## License for the specific language governing permissions and limitations
## under the License.
#######################################################################

from datetime import datetime
from enum import IntEnum
from pathlib import Path
from typing import TypeVar, Union

import os
import stat
import time

import swarmlearning.com.swifrpc_pb2 as spb

MessageBody = Union[
    spb.Status
  , spb.SessionRequest
  , spb.SessionResponse
  , spb.SyncRequest
  , spb.SyncResponse
  , spb.LossRequest
  , spb.LossResponse
]

BridgeLaneDirection = TypeVar("Bridge.LaneDirection")

class Bridge:
    class LaneDirection(IntEnum):
        NorthBound = 1
        SouthBound = 2

    def __init__(self, isSL=False, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

        self.__setAPIVersion()

        # createPipes is set to True for SL container.
        # WARNING: If it is set to True from ML container,
        #          then ML call to mkfifo might fail as ML runs with limited privileges.
        self._createPipes = isSL

        self._lanes = {}
        self._openLanes = {}
        self.__makeBridgeLanes()

        # These values must be non-zero. Zero values get optimized out, giving
        # us a serialized size that is smaller than what we want.
        dummyHeader = spb.Header(apiVersion=self._apiVer, msgType=1, msgSize=1)
        self._serializedHeaderSize = len(
            dummyHeader.SerializeToString(deterministic=True)
        )

        # self.__logIt(f"HEADER SIZE = {self._serializedHeaderSize}")

        return

    def __setAPIVersion(self) -> None:
        # Fetching the api version from the dynamically generated proto code.
        apiVerFrmProto = spb.DESCRIPTOR.GetOptions().Extensions[spb.apiVersion]
        # Building _apiVer as spb.APIVersion structure type. This will be used to
        # set the api version inside the header of _send method.
        self._apiVer = spb.APIVersion(apiVersion=apiVerFrmProto, minVersion=apiVerFrmProto)
        return

    def _verifyAPIVersion(self, verionFromHeader: spb.APIVersion) -> None:
        # Left side is the api version from the header of the _recv method
        # Right side is the api version from the dynamically generated proto file
        if verionFromHeader.apiVersion != self._apiVer.apiVersion:
            raise RuntimeError(f"Mismatch in the Swarm API version between ML and SL containers. Expected version {self._apiVer.apiVersion}, but the version received from the header is {verionFromHeader.apiVersion}")
        return

    def _verifyMessageType(self, msgType: spb.MessageType) -> None:
        if (
            (msgType <= spb.MessageType.MsgTypeFirst) or
            (msgType >= spb.MessageType.MsgTypeLast)
           ):
            raise RuntimeError(f"unsupported message type: {msgType}")

        return

    def _sendRecv(
        self
      , sendLane: BridgeLaneDirection
      , recvLane: BridgeLaneDirection
      , msgType: int
      , request: MessageBody
    ) -> (spb.MessageType, MessageBody):
        self._send(sendLane, msgType, request)
        resp = self._recv(recvLane)

        return resp

    def _send(
        self, sendLane: BridgeLaneDirection, msgType: int, request: MessageBody
    ) -> None:
        body = request.SerializeToString(deterministic=True)
        header = spb.Header(
            apiVersion=self._apiVer, msgType=msgType, msgSize=len(body)
        )
        # message = spb.Message(header=header, body=body)

        # chunk the message into 2GB blocks.
        lane = self._openLanes[sendLane]

        # Send the header and body separately until we understand the protobuf
        # serialization format a little better and figure how to send the whole
        # message but, receive and extract only the header.
        lane.write(header.SerializeToString(deterministic=True))
        lane.write(body)
        # lane.write(message.SerializeToString(deterministic=True))
        lane.flush()

        return

    def _recv(
        self, recvLane: BridgeLaneDirection
    ) -> (spb.MessageType, MessageBody):
        serializedHeader = self._read(recvLane, self._serializedHeaderSize)
        header = spb.Header()
        header.ParseFromString(serializedHeader)
        # self.__logIt(f"Header = <{type(header)}> = <{header}>")

        self._verifyAPIVersion(header.apiVersion)
        self._verifyMessageType(header.msgType)

        if header.msgType == spb.MessageType.MsgTypeStatus:
            body = spb.Status()
        elif header.msgType == spb.MessageType.MsgTypeOpenSessionRequest:
            body = spb.SessionRequest()
        elif header.msgType == spb.MessageType.MsgTypeOpenSessionResponse:
            body = spb.SessionResponse()
        elif header.msgType == spb.MessageType.MsgTypeCloseSessionRequest:
            body = spb.SyncRequest()
        elif header.msgType == spb.MessageType.MsgTypeCloseSessionResponse:
            body = spb.SyncResponse()
        elif header.msgType == spb.MessageType.MsgTypeSyncRequest:
            body = spb.SyncRequest()
        elif header.msgType == spb.MessageType.MsgTypeSyncResponse:
            body = spb.SyncResponse()
        elif header.msgType == spb.MessageType.MsgTypeGetLossRequest:
            body = spb.LossRequest()
        elif header.msgType == spb.MessageType.MsgTypeGetLossResponse:
            body = spb.LossResponse()
        elif header.msgType == spb.MessageType.MsgTypeKillSessionRequest:
            body = spb.KillSessionRequest()
        else:
            # Should not get here. We have already validated the message type.
            raise RuntimeError(f"unsupported message type: {header.msgType}")

        serializedBody = self._read(recvLane, header.msgSize)
        body.ParseFromString(serializedBody)

        return header.msgType, body

    def _read(self, recvLane: BridgeLaneDirection, nBytes: int) -> bytes:
        lane = self._openLanes[recvLane]
        msg = lane.read(nBytes)
        if len(msg) != nBytes:
            raise RuntimeError(
                f"wanted: {nBytes} bytes, got {len(msg)} bytes: pipe closed?"
            )

        return msg

    def _openBridgeLanes(
        self, *, northBoundMode: str, southBoundMode: str
    ) -> None:
        def openLane(direction: BridgeLaneDirection, mode: str) -> None:
            # self.__logIt(f"Opening [{direction}] = <{self._lanes[direction]}> in <{mode}> mode")
            self._openLanes[direction] = open(self._lanes[direction], mode)
            # self.__logIt(f"Opened [{direction}] = <{self._lanes[direction]}> in <{mode}> mode")
            return

        openLane(self.LaneDirection.NorthBound, northBoundMode)
        openLane(self.LaneDirection.SouthBound, southBoundMode)

        return

    def __makeBridgeLanes(self) -> None:
        def getPath(envvar: str) -> str:
            pipePath = os.getenv(envvar)

            if pipePath is None or len(pipePath) == 0:
                raise RuntimeError(
                    f"{envvar}: bad environment variable; cannot create bridge"
                )

            return Path(pipePath).resolve().as_posix()

        def makeLane(lane: str, direction: BridgeLaneDirection) -> None:
            try:
                self._lanes[direction] = lane
                # self.__logIt(f"Creating [{direction}] = <{self._lanes[direction]}>")
                if(self._createPipes):
                    os.mkfifo(lane, 0o777)
                    self.__logIt(f"{lane}: pipe created")
                else:
                    while(not(os.path.exists(lane))):
                        self.__logIt(f"{lane}: waiting for pipes to be created")
                        time.sleep(1)
                # self.__logIt(f"Created [{direction}] = <{self._lanes[direction]}>")

            except FileExistsError:
                self.__logIt(f"{lane}: File exists")
            except BaseException as be:
                self.__logIt(f"{lane}: {be}")

            if not stat.S_ISFIFO(os.stat(lane).st_mode):
                raise RuntimeError(f"{lane}: cannot create pipe")

        # The default umask seems to be 022. It is applied to the mode specified
        # in mkfifo. Therefore, even though we ask for 777, what we actually get
        # is 755. This prevents the two ends of the bridge from having different
        # user IDs - the second party will not have the requisite permissions to
        # open the FIFO is write mode. We workaround this by disabling the mask,
        # creating the pipes, and then, restoring the mask.
        currMask = os.umask(0)

        makeLane(getPath("SL_REQUEST_CHANNEL"), self.LaneDirection.SouthBound)
        makeLane(getPath("SL_RESPONSE_CHANNEL"), self.LaneDirection.NorthBound)

        os.umask(currMask)

        return

    def __logIt(self, *args, **kwargs):
        print(datetime.now().isoformat(), *args, **kwargs, flush=True)
        return


from torch import nn
from torch.autograd import Function
from torch.autograd.function import once_differentiable

from tensormask import _C

class _SwapAlign2Nat(Function):
    @staticmethod
    def forward(ctx, X, lambda_val, pad_val):
        ctx.lambda_val = lambda_val
        ctx.input_shape = X.size()

        Y = _C.swap_align2nat_forward(X, lambda_val, pad_val)
        return Y

    @staticmethod
    @once_differentiable
    def backward(ctx, gY):
        lambda_val = ctx.lambda_val
        bs, ch, h, w = ctx.input_shape

        gX = _C.swap_align2nat_backward(gY, lambda_val, bs, ch, h, w)

        return gX, None, None

swap_align2nat = _SwapAlign2Nat.apply

class SwapAlign2Nat(nn.Module):
    """
    The op `SwapAlign2Nat` described in https://arxiv.org/abs/1903.12174.
    Given an input tensor that predicts masks of shape (N, C=VxU, H, W),
    apply the op, it will return masks of shape (N, V'xU', H', W') where
    the unit lengths of (V, U) and (H, W) are swapped, and the mask representation
    is transformed from aligned to natural.
    Args:
        lambda_val (int): the relative unit length ratio between (V, U) and (H, W),
                            as we always have larger unit lengths for (V, U) than (H, W),
                            lambda_val is always >= 1.
        pad_val (float):    padding value for the values falling outside of the input
                            tensor, default set to -6 as sigmoid(-6) is ~0, indicating
                            that is no masks outside of the tensor.
    """

    def __init__(self, lambda_val, pad_val=-6.0):
        super(SwapAlign2Nat, self).__init__()
        self.lambda_val = lambda_val
        self.pad_val = pad_val

    def forward(self, X):
        return swap_align2nat(X, self.lambda_val, self.pad_val)

    def __repr__(self):
        tmpstr = self.__class__.__name__ + "("
        tmpstr += "lambda_val=" + str(self.lambda_val)
        tmpstr += ", pad_val=" + str(self.pad_val)
        tmpstr += ")"
        return tmpstr


# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

from pymongo import MongoClient

client = MongoClient()
collection = client["douban"]["movie"]

class DoubanPipeline(object):
    def process_item(self, item, spider):
        print("电影信息：",item)
        collection.insert(dict(item))  # 存储于Mongodb数据库中
        return item


#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

from utils import connection_util

class DataCollection(object):
    def __init__(self):
        self._target_url = 'https://www.scrapingbee.com/blog/'
        self._init_connection = connection_util.ProcessConnection()

    def main(self):
        # 连接目标网站，获取内容
        get_content = self._init_connection.init_connection(self._target_url)
        if get_content:
            parent = get_content.findAll("section", {"class": "section-sm"})[0]
            get_row = parent.findAll("div", {"class": "col-lg-12 mb-5 mb-lg-0"})[0]
            get_child_item = get_row.findAll("div", {"class": "col-md-4 mb-4"})
            for item in get_child_item:
                # 获取标题文字
                get_title = item.find("a", {"class": "h5 d-block mb-3 post-title"}).get_text()
                # 获取发布时间
                get_release_date = item.find("div", {"class": "mb-3 mt-2"}).findAll("span")[1].get_text()
                # 获取文章描述
                get_description = item.find("p", {"class": "card-text post-description"}).get_text()
                print(get_title)
                print(get_release_date)
                print(get_description)

if __name__ == '__main__':
    DataCollection().main()


import data
import question_model
import quiz_brain

question_bank = []

for question in data.question_data:
    q = question_model.Question(question["question"], question["correct_answer"])
    question_bank.append(q)

quiz = quiz_brain.QuizBrain(question_bank)

while quiz.still_has_questions():
    quiz.next_question()

print(
    f"You completed the quiz, your final score is: {quiz.score}/{quiz.total_questions}"
)


# coding: utf-8

# In[3]:

import json

# In[4]:

with open('flight.json') as data_file:
    data = json.load(data_file)

# In[5]:

ff = data["items"][0]['trips']["tripOption"][0]["slice"][0]["segment"][0]["flight"]

# In[6]:

str(ff['carrier'])+str(ff['number'])

# In[7]:

ll = data["items"][0]['trips']["tripOption"][0]["slice"][0]["segment"][0]["leg"]
ll

# In[8]:

carrier = dict()
for item in data["items"]:
    for f in item["trips"]["data"]['carrier']:
        carrier[f['code']] = f['name']

carrier.items()

# In[9]:

flights = []

for item in data["items"]:
    for f in item["trips"]["tripOption"]:
        flight = dict()
        flight['id'] = len(flights)
        flight['price'] = f['saleTotal'].split('USD')[1]
        segment = f["slice"][0]["segment"][0]
        ff = segment["flight"]
        flight['carrier'] = carrier[ff['carrier']]
        flight['flightNumber'] =str(ff['carrier'])+str(ff['number'])
        leg = segment["leg"][0]
        flight['arrivalTime'] = leg["arrivalTime"]
        flight['departureTime'] = leg['departureTime']
        flight['origin'] = leg['origin']
        flight['destination'] = leg['destination']

        flights.append(flight)

with open('formatedflight.json','w') as fp:
    json.dump(flights,fp)
fp.close()

# In[10]:

'USD55.3'.split('USD')[1]

# In[ ]:


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('conteudo', '0003_imagem_title'),
    ]

    operations = [
        migrations.AlterField(
            model_name='new',
            name='texto',
            field=models.TextField(null=True, verbose_name='Texto', blank=True),
        ),
    ]


from aiogram import Bot, Dispatcher, types
from config import Config

bot = Bot(Config.TOKEN)
dp = Dispatcher(bot)

@dp.message_handler()
async def echo(message: types.Message):
    await message.answer(message.text)


import numpy as np
import cv2
import matplotlib.pyplot as plt
from timeit import default_timer as timer
# from calibration import load_calibration
from copy import copy

# left_lane = Lane()
# right_lane = Lane()
frame_width = 1920
frame_height = 1080

input_scale = 4
output_frame_scale = 4
N = 4 # buffer previous N lines

# fullsize:1280x720
x = [219, 1676, 1060, 860]
y = [1079, 1079, 452, 452]
X = [435, 1485, 1485, 435]
Y = [1079, 1079, 0, 0]

src = np.floor(np.float32([[x[0], y[0]], [x[1], y[1]],[x[2], y[2]], [x[3], y[3]]]) / input_scale)
dst = np.floor(np.float32([[X[0], Y[0]], [X[1], Y[1]],[X[2], Y[2]], [X[3], Y[3]]]) / input_scale)

M = cv2.getPerspectiveTransform(src, dst)
M_inv = cv2.getPerspectiveTransform(dst, src)

X_b = [861, 1059, 1059, 861]
Y_b = [1079, 1079, 0, 0]
src_ = np.floor(np.float32([[x[0], y[0]], [x[1], y[1]],[x[2], y[2]], [x[3], y[3]]]) / (input_scale*2))
dst_ = np.floor(np.float32([[X_b[0], Y_b[0]], [X_b[1], Y_b[1]],[X_b[2], Y_b[2]], [X_b[3], Y_b[3]]]) / (input_scale*2))
M_b = cv2.getPerspectiveTransform(src_, dst_)

def warper(img, M):
    # Compute and apply perspective transform
    img_size = (img.shape[1], img.shape[0])
    warped = cv2.warpPerspective(img, M, img_size, flags=cv2.INTER_NEAREST)  # keep same size as input image

    return warped

# 读取图片
image = cv2.imread('/home/wqg/data/lane_test/2022.3.22-3_16410.jpg')
image = cv2.resize(image, (0, 0), fx=1 / input_scale, fy=1 / input_scale)

binary_warped = warper(image, M)

cv2.imshow('binary_warped',binary_warped)
# cv2.waitKey(0)

newwarp = cv2.warpPerspective(binary_warped, M_inv, (int(frame_width/input_scale), int(frame_height/input_scale)))
# Combine the result with the original image    # result = cv2.addWeighted(undist, 1, newwarp, 0.3, 0)
newwarp_ = cv2.resize(newwarp,None, fx=input_scale/output_frame_scale, fy=input_scale/output_frame_scale, interpolation = cv2.INTER_LINEAR)

cv2.imshow('newwarp_',newwarp_)
# cv2.waitKey(0)

binary_sub = np.zeros_like(binary_warped)
binary_sub[:, int(150 / input_scale):int(-80 / input_scale)] = binary_warped[:,
                                                                   int(150 / input_scale):int(-80 / input_scale)]

undist_ori = newwarp_

undist_birdview = warper(cv2.resize(undist_ori, (0, 0), fx=1 / 2, fy=1 / 2), M_b)
undist_birdview2 = warper(cv2.resize(image, (0, 0), fx=1 / 2, fy=1 / 2), M_b)
# undist_birdview = warper(undist_ori, M_b)

cv2.imshow('undist_birdview',cv2.resize(undist_birdview, (0, 0), fx= 4, fy=4))
cv2.imshow('undist_birdview2',cv2.resize(undist_birdview2, (0, 0), fx= 4, fy=4))
cv2.waitKey(0)


# coding: utf8
import csv
from datetime import datetime, timedelta
import os

class Plant:
    """Plant actions"""
    def __init__(self, db_entry, log_filename="Default"):
        self.log_filename = log_filename
        self.id = db_entry[0]
        self.type = db_entry[1]
        self.temperature = db_entry[2]
        self.location = db_entry[3]
        self.frequency = int(db_entry[4])
        self.webhook = db_entry[5]

    def log(self, data):
        with open(self.log_filename, "a+") as f:
                f.write("{}\n".format(data))

    def watering_need(self, current_date):
        """ Return last watering time if plant deserve water or None"""
        with open(self.log_filename, 'rb') as fh:
            for line in fh:
                pass
            last_watering = datetime.strptime(line.decode().rstrip(), "%Y-%m-%d")
            if (last_watering + timedelta(days=self.frequency)) < current_date:
                # Here we suppose that plants are gonna be watering ...
                # but we can't think about a better solution to check they are.
                # maybe Qrcode.
                self.log(datetime.strftime(current_date, "%Y-%m-%d"))
                return last_watering
        return None


import wizwalker
import asyncio
from threading import Event
from wizwalker import Keycode, HotkeyListener, ModifierKeys

xyz_list = []
run = True
async def startup():
    listener = HotkeyListener()

    p = wizwalker.WizWalker().get_new_clients()[0]

    async def add_xyz():
        global xyz_list
        print('Added a location')
        xyz_list += [await p.body.position()]
    async def end():
        global run
        print('Ending and writing')
        run = False

    await listener.add_hotkey(Keycode.K, add_xyz, modifiers=ModifierKeys.NOREPEAT)
    await listener.add_hotkey(Keycode.L, end, modifiers=ModifierKeys.NOREPEAT)
    listener.start()
    try:

        await p.activate_hooks()
        filename = await p.zone_name()
        # your program here
        global run
        while run:
            await asyncio.sleep(1)

    finally:
        global xyz_list
        await listener.stop()
        await p.close()
        with open(f'Zones\{filename}.txt'.replace('/','-'), 'w') as file:
            for xyz in xyz_list:
                line = str(xyz).replace('<XYZ (','').replace(')>','')
                file.write(f'{line}\n')

if __name__ == "__main__":
        asyncio.run(startup())


from discord import Embed, message

from SnsInfo import SnsInfo

DOMAIN_TWITTER = "twitter.com"
DOMAIN_X = "x.com"
DOMAIN_INSTAGRAM = "instagram.com"
DOMAIN_WEVERSE = "weverse.io"
DOMAIN_H1KEY = "h1key-official.com"
DOMAIN_YEEUN = "yeeun.bstage.in"
DOMAIN_PURPLE_KISS = "purplekiss.co.kr"
DOMAIN_BSTAGE = [DOMAIN_H1KEY, DOMAIN_YEEUN, DOMAIN_PURPLE_KISS]

def post_source(url: str):
    if DOMAIN_TWITTER in url or DOMAIN_X in url:
        return "X", "https://upload.wikimedia.org/wikipedia/commons/thumb/5/5a/X_icon_2.svg/2048px-X_icon_2.svg.png"
    elif DOMAIN_INSTAGRAM in url:
        return "Instagram", "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a5/Instagram_icon.png/600px-Instagram_icon.png"
    elif DOMAIN_WEVERSE in url:
        return "Weverse", "https://image.winudf.com/v2/image1/Y28uYmVueC53ZXZlcnNlX2ljb25fMTY5NjQwNDE0MF8wMTM/icon.webp?w=140&fakeurl=1&type=.webp"

def generate_embeds(username: str, sns_info: SnsInfo):
    embeds = []
    # 圖片訊息，Embed 的 url 如果一樣，最多可以 4 張以下的合併在一個區塊
    for index, image_url in enumerate(sns_info.images[slice(4)]):
        if index == 0:
            source = post_source(sns_info.post_link)
            embed = (
                Embed(title=sns_info.title, description=sns_info.content, url=sns_info.post_link).set_author(
                    name=sns_info.profile.name, icon_url=sns_info.profile.url)
                .set_image(url=image_url)
                .insert_field_at(index=0, name="使用者", value=username))
            if source is not None:
                embed.set_footer(text=post_source(sns_info.post_link)[0],
                                 icon_url=post_source(sns_info.post_link)[1])
            embeds.append(embed)
        else:
            embeds.append(Embed(url=sns_info.post_link)
                          .set_author(name=sns_info.profile.name, url=sns_info.profile.url)
                          .set_image(url=image_url))
    else:
        embeds.append(Embed(title=sns_info.title, description=sns_info.content, url=sns_info.post_link).set_author(
                    name=sns_info.profile.name, icon_url=sns_info.profile.url)
                .insert_field_at(index=0, name="使用者", value=username))

    return embeds

def mentions(message: message, bot_id: int):
    if bot_id in message.raw_mentions:
        return True
    else:
        for role in message.role_mentions:
            for member in role.members:
                if member.id == bot_id:
                    return True
    return False


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jan 15 13:06:12 2022

@author: rodo
"""
import pickle
import numpy as np
import tokenizer_new as tk
import argparse

# ---------------------- HELPER FUNCTIONS -------------------------------------

# ---------------------- Create N-Grams ------------------
def sent2ngrams(sent, n):
    """
    Split the sentence into n-grams
    """
    # check if sentence is at all full (or not empty)
    tok = sent.split(' ')
    if len(tok) > 0:
        # Empty list to store n-grams
        ng = []
        # Note that for these larger n-grams, we’ll need
        # to assume extra contexts to the left and right of the sentence end.
        # [<SOS> <SOS> <SOS> i hate this <EOS>] for 4-grams
        tokens = ["<SOS>" for _ in range(n-2)]+tok
        # At least n+1 tokens should be there --> n-1 for <SOS>
        # +1 for <EOS> and >=1 for middle context => >(n-1+1)
        for k in range(n, len(tokens)):
            # n-gram sentence
            ng.append(" ".join(tokens[k-n:k]))
        return ng
    return None

def ngrams_constructor(n, texts):
    """
    Parameters
    ----------
    n : INT
        length of words to be considered to predict the next.
    texts : List of Strings
        List containing Sentences.

    Returns
    -------
    Frequency Table (a dictionary) for n-grams or tokens.
    """
    ngrams = dict()
    for sent in texts:
        # Create list of n-grams with above function.
        lst_ngrams = sent2ngrams(sent, n)
        if lst_ngrams is not None:
            for s in lst_ngrams:
                if s not in ngrams:
                    ngrams[s] = 1
                else:
                    ngrams[s] += 1
    return ngrams

def recursiveNgramsConstructor(n, texts, unk_thresh=10):
    """
    Parameters:
        n: Highest Order n-gram required
        texts : List of Strings
            List containing Sentences.
        unk_thresh: Threshold count for a
             word to be in <UNK> group.

    Output:
        Dictionary Containing Tables of all n-grams from 1 to n.
    """
    req_dict = {k:ngrams_constructor(k, texts) for k in range(2, n+1)}
    unigrams = ngrams_constructor(1, texts)
    # here those words/unigrams whose count is less than certain threshold
    # put them in <UNK> group.
    unigrams["<UNK>"] = 0
    for key, val in unigrams.items():
        if val <= unk_thresh:
            unigrams["<UNK>"] += val
            _ = unigrams.pop(key)
    req_dict[1] = unigrams
    return req_dict

# ---------------------- LM = Construct Language Model ------------------------

def Cnt(string, cent_dict):
    """

    Parameters
    ----------
    string : string
        a string of word(s).

    Returns
    -------
    count of that string in corpus
    """
    # count no. of words in the given string.
    if len(string) == 0:
        return 0
    nwords = len(string.split(" "))
    # Retieve the count of that string from a central dict.
    try:
        return cent_dict[nwords][string]
    except KeyError:
        # When string doesn't exist in any table then returns 0 or <UNK>
        return 0

def sum_of_counts(history, cent_dict):
    """
    Parameters
    ----------
    history : string
        the past string of word(s).

    Returns
    -------
    cnt : integer
        sum of counts of history + variable_word.

    """
    # initial val
    cnt = 0
    # which n for ngram
    n = len(history.split(" ")) + 1
    # sum all possible counts of 'current' word with given 'history'
    for key, val in cent_dict[n].items():
        req_str = " ".join(key.split(" ")[:-1])
        if req_str == history:
            cnt += val
    return cnt

def cont_count(string, n, cent_dict):
    """
    Parameters
    ----------
    string : string
        the current word whose past context types we need to estimate.
    n : integer
        n stand for ngrams i.e., no. of words.

    Returns
    -------
    cnt : counts
        Total count of strings whose end word is 'string'.
    """
    cnt = 0
    for key in cent_dict[n].keys():
        if key.split(" ")[-1] == string:
            cnt += 1
    return cnt

def count_of_positives(history, cent_dict):
    """
    Parameters
    ----------
    history : string
        the past string of word(s).

    Returns
    -------
    cnt : integer
        No. of history + variable_word having counts > 0.
    """
    # initial val
    cnt = 0
    # which n for ngram
    n = len(history.split(" ")) + 1
    # sum all possible counts of 'current' word with given 'history'
    for key, val in cent_dict[n].items():
        if val > 0:
            req_str = " ".join(key.split(" ")[:-1])
            if req_str == history:
                cnt += 1
    return cnt

def kneserNey(history, current, recur_step, cent_dict):
    """
    Parameters
    ----------
    history : Tuple
        Tuple consisting of past words.
    current : String
        The word whose likelihood is to be estimated.

    Returns
    -------
    probability of happening of current word given the past.
    """
    # n ==> for n-grams
    n = len(history.split())+1
    # Check if current word is in VOCAB
    if current not in cent_dict[1]:
        # return len(list(filter(lambda x: cent_dict[n][x] == 1, cent_dict[n])))/len(cent_dict[n])
        return 0.75/sum(cent_dict[1]["<UNK>"])
    # base condition --- Empty String
    if n == 1:
        return 0.25/len(cent_dict[1]) + 0.75/sum(cent_dict[1]["<UNK>"])
            # 0.75*len(dict(filter(lambda item: item[1]>0,
            # cent_dict[1].items())))/(tot_unigram_counts*len(cent_dict[1]))
            # As V cancels out with the NUMerator.
    # c_KN = count or continuation_count
    if recur_step == 1:
        try:
            first_term = max(Cnt(" ".join([history, current]), cent_dict)-0.75, 0)/sum_of_counts(history, cent_dict)
        except ZeroDivisionError:
            # Handle 0/0 form
            first_term = 0
    else:
        try:
            first_term = max(cont_count(current, n, cent_dict) - 0.75, 0)/len(cent_dict[n])
        except ZeroDivisionError:
            first_term = 0
    # Define Lambda
    try:
        lamb = (0.75/sum_of_counts(history, cent_dict))*count_of_positives(history, cent_dict)

    except ZeroDivisionError:
        # Handle 0/0 error.
        # no point of doing recursion further. Plus, if lamb=0 then first term
        # is also zero as they have same denom and lamb can be only when denom=0.
        # And since lamb have same def for all recursion steps, then it wont't get
        # 0 at lower order, because higher order n-grams are less frequent than lower
        # orders. If a particular lower n-gram doesn't exist then higher too doesn't
        # exist. So at first step only lamb=0 ==> first_term=0 ==> lamb(eps)/V
        return 0.75/sum(cent_dict[1]["<UNK>"])
    # New history for further step...
    new_hist = " ".join(history.split()[1:])
    sec_term = lamb*kneserNey(new_hist, current, recur_step+1, cent_dict)
    # NOw combine all the terms to get final term.
    return first_term + sec_term

# Witten-Bell discounting.
def wittenBell(history, current, cent_dict):
    """
    Parameters
    ----------
    history : Tuple
        Tuple consisting of past words.
    current : String
        The word whose likelihood is to be estimated.

    Returns
    -------
    probability of happening of current word given the past.
    """
    # Define BASE condition for this recursive function.
    n = len(history.split()) + 1
    # base condition --- Empty String
    if n == 1:
        if current in cent_dict[1]:
            return Cnt(current, cent_dict)/sum(cent_dict[1]["<UNK>"])
        return 1/len(cent_dict[1])

    # define lambda parameter
    try:
        lamb = count_of_positives(history, cent_dict)/(count_of_positives(history, cent_dict) + sum_of_counts(history, cent_dict))
    except ZeroDivisionError:
        # again lambda consists of term in denom which is similar to denom of pML
        # Due to which lamb=ZeroDivisionError iff pML=ZeroDivisionError
        return 1/len(cent_dict[n])

    # The first term in WITTEN-Bell expression and it's exception is handled above.
    pML = Cnt(" ".join([history, current]), cent_dict)/sum_of_counts(history, cent_dict)

    # Now arraning all above expressions.
    new_hist = " ".join(history.split()[1:])
    return (1 - lamb)*pML + lamb*wittenBell(new_hist, current, cent_dict)

def perplexityScore(prob_list):
    """
    Input:
        list containing n-gram probability scores
    """
    return np.power(1/np.prod(prob_list), 1/len(prob_list))

def sent2PPScore(sent, n, smoothing, cent_dict, verbosity=0):
    """
    1. Split sentence into n-grams sentence.
    2. Estimate probability of each n-gram.
    3. Compute Perplexity scores of Whole sentence using ALL sentences.
    Input:
        sent: full sentence string.
        n: no. of grams
        smoothing: Type of smoothing
    Output:
        perplexity score of the given n-gram string.
    """
    # Construct n-grams...
    ngrams_lst = sent2ngrams(sent, n); scores = []
    if smoothing == "k":
        for ng in ngrams_lst:
            lst = ng.split(" ")
            hist, current = " ".join(lst[:-1]), lst[-1]
            scores.append(kneserNey(hist, current, 1, cent_dict))
            if verbosity:
                print("PP Score with KNESER-NEY for sentence: {0} = {1:.3f}".format(ng, scores[-1]))
    elif smoothing == "w":
        for ng in ngrams_lst:
            lst = ng.split(" ")
            hist, current = " ".join(lst[:-1]), lst[-1]
            scores.append(wittenBell(hist, current, cent_dict))
            if verbosity:
                print("PP Score with WITTEN-BELL for sentence: {0} = {1:.3f}".format(ng, scores[-1]))
    else:
        raise ValueError("Please enter a correct smoothing format.")
    # calculate PP scores.
    return perplexityScore(scores)

# ======================================= DRIVER CODE ================================================
if __name__ == '__main__':

    # ----------------------- Parse the Command Line Arguments -------------------
    parser = argparse.ArgumentParser()
    parser.add_argument('n_value', type=int)
    parser.add_argument('smoothing', type=str)
    parser.add_argument('path', type=str)

    # Retrieve Arguments
    n = parser.parse_args().n_value
    corpus_path = parser.parse_args().path
    smoothing = parser.parse_args().smoothing

    # ---------------------- Clean Corpora; Build Vocab ---------------------------
    # Try to load the clean and preprocessed COPRUS created before, else Create one.
    try:
        with open("clean_corpora/"+corpus_path.split("/")[-1][:-4]+".txt", "r") as fp:
            preprocessedTexts = fp.readlines()
    except FileNotFoundError:
        # Load the dataset.
        with open(corpus_path, 'r') as fp:
            texts = fp.readlines()
            # readline() returns single line while readlines() return multiple.
        # Create a Tokenizer Object.
        token = tk.Tokenizer()
        # Create an Empty List
        preprocessedTexts = list()
        for txt in texts:
            # Apply all sorts of pre-processing on imported texts.
            txt1 = token.upperToLower(txt)
            txt1 = token.replaceHashtags(txt1)
            txt1 = token.replaceEMAIL(txt1)
            txt1 = token.replaceURL(txt1)
            txt1 = token.replaceMentions(txt1)
            txt1 = token.removeFootNotes(txt1)
            txt1 = token.replacePunctuation(txt1)
            txt1 = token.replaceDateTime(txt1)
            txt1 = token.replaceMobileNumber(txt1)
            txt1 = token.replaceNumericals(txt1)
            txt1 = token.replaceAlphabets(txt1)
            txt1 = token.replaceApostrophe(txt1)
            txt1 = token.replaceSpecialCharsFromWords(txt1, flag=True)
            txt1 = token.addSOSEOS(txt1)
            preprocessedTexts.append(txt1)
        tk.saveText(preprocessedTexts, "clean_corpora/"+corpus_path.split("/")[-1][:-4])
        _ = tk.vocabBuilder(preprocessedTexts, "vocab/"+corpus_path.split("/")[-1][:-4]+"Vocabulary")
    # ---------------------------- TRAIN and TEST split -----------------------------------------
    np.random.seed(23)
    idx = np.random.choice(len(preprocessedTexts), 1000, replace=False)
    train, test = [], []
    for id in range(len(preprocessedTexts)):
        if id in idx:
            test.append(preprocessedTexts[id])
        else:
            train.append(preprocessedTexts[id])

    # ------------ Create N-grams recursively and store their Frequency table for TRAINING corpus.
    try:
        # Try to read existing Stored Dictionary.
        with open("frequency-table/"+corpus_path.split("/")[-1][:-4]+"CentralDictTrain.pickle", 'rb') as handle:
            cent_dict = pickle.load(handle)
    except FileNotFoundError:
        # when File is not there--> create it
        cent_dict = recursiveNgramsConstructor(n=n, texts=train)
        with open("frequency-table/"+corpus_path.split("/")[-1][:-4]+"CentralDictTrain.pickle", "wb") as handle:
            pickle.dump(cent_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)

    # Add Frequency for an unknown word: <UNK> from the count of Words seen once.
    # cent_dict[1]['<UNK>'] = len(list(filter(lambda x: cent_dict[1][x] == 1, cent_dict[1])))

    # ------------------------ Apply Smoothing; k=Kneser-Ney; w=WittenBell -----------------------------
    # Steps:
    #     1) Split the Sentence into 4-gram Sentences.
    #     2) Compute probability of having that particular 4th word given past 3 words in 4-gram sentences.
    #     3) Take PERPLEXITY scores of probabilities computed for each sentences to know how likely is that sentence.
    #     4) Take Average PERPLEXITY scores for both TRAIN and TEST to know how likely they are.

    # FOR TRAINING LET'S COMPUTE Probability for each sentence and then Perplexity scores.
    pp_score_lst_train, pp_score_lst_test = [], []
    # Open a Document and Save the scores calculated.
    LM = input("Enter the name for score TXT file (for which LM): ")

    # ----------------------------------------------- TRAIN ----------------------------------------------
    print("="*20+"COMPUTATION STARTED FOR TRAINING SET"+"="*20)
    with open("scores/2021701010_"+LM+"_train-perplexity.txt", "w") as flm:
        for j, sent in enumerate(train):
            # Omit the newline character
            sent = sent.strip()
            # If by any chance sentence is of length 0.
            if len(sent) != 0:
                pp_score_lst_train.append(sent2PPScore(sent, n, smoothing, cent_dict=cent_dict))
                flm.write(sent+"    PP Score = {0:.3f}\n".format(pp_score_lst_train[-1]))
                print(f"Sentence {j+1} is PROCESSED!!!")
                if j == 1000:
                    break
    with open("scores/2021701010_"+LM+"_train-perplexity.txt", 'r+') as fll:
        content = fll.read()
        fll.seek(0, 0)
        fll.write("Average Perplexity Score: {0:.3f}\n".format(np.mean(pp_score_lst_train)) + content)
    print("="*50+"DONE!!!"+"="*50)

    # ----------------------------------------------- TEST ----------------------------------------------
    print("="*20+"COMPUTATION STARTED FOR TESTING SET"+"="*20)
    with open("scores/2021701010_"+LM+"_test-perplexity.txt", "w") as flm:
        for l, sent_ in enumerate(test):
            if l == 250:
                break
            # Omit the newline character
            sent_ = sent_.strip()
            # if by any chance sentence is empty
            if len(sent_) != 0:
                pp_score_lst_test.append(sent2PPScore(sent_, n, smoothing, cent_dict=cent_dict))
                flm.write(sent_+"    PP Score = {0:.3f}\n".format(pp_score_lst_test[-1]))
                print(f"Sentence {l+1} is PROCESSED!!!")
    with open("scores/2021701010_"+LM+"_test-perplexity.txt", 'r+') as fll:
        content = fll.read()
        fll.seek(0, 0)
        fll.write("Average Perplexity Score: {0:.3f}\n".format(np.mean(pp_score_lst_test)) + content)
    print("="*50+"DONE!!!"+"="*50)