from firebase_admin import auth, initialize_app
import os
import pathlib

fb_admin_config_path = os.path.join(pathlib.Path(__file__).parent, 'key/fb_admin_config.json')

if not os.path.exists(fb_admin_config_path):
    print(f"Please provide firebase admin config file in the path: {fb_admin_config_path}")
    exit(1)

# we have to set this field before any admin stuff, as it will set firebase admin
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = fb_admin_config_path

app = initialize_app() # initialize firebase admin app

def create_user(email, user_id):
    try:
        # if user is there, return that
        return auth.get_user_by_email(email=email)
    except:
        # else create the user
        return auth.create_user(email=email, uid=user_id) if user_id else auth.create_user(email=email)


import os
import wave
import librosa
import numpy as np
from uuid import uuid1
import speech_recognition as sr

from utils import cprint

def video_to_text(file_path):
    audio_path = f'uploads\\{uuid1()}.wav'
    text_file_path = f'uploads\\{uuid1()}.txt'
    command2wav = f"support\\ffmpeg -i {file_path} -v quiet -vn -acodec pcm_s16le -ar 44100 -ac 2 {audio_path}"
    cprint(command2wav)
    os.system(command2wav)
    r = sr.Recognizer()
    audio = sr.AudioFile(audio_path)

    speech_text = []
    with audio as source:
        audio = r.record(source, duration=300)
        try:
            text = r.recognize_google(audio)
        except sr.UnknownValueError:
            text = ''

        with open(text_file_path, "w+") as file:
            file.write(text)
            speech_text.append(text)

    speech_text = ' '.join(speech_text)
    return audio_path, speech_text

def speech_speed(audio_path, speech_text):
    word_count = len(speech_text.split())
    with wave.open(audio_path, 'r') as f:
        frames = f.getnframes()
        rate = f.getframerate()

    duration_min = frames / (rate * 60)
    wpm = word_count / duration_min
    speed = 'Slow'
    if wpm > 120 and wpm <= 160:
        speed = 'Medium'
    elif (wpm > 160):
        speed = 'Fast'

    return wpm, speed

def pause_detection(audio_path):
    TOP_DB_LEVEL = 30

    x, sr = librosa.load(audio_path)
    y = librosa.amplitude_to_db(abs(x))
    refDBVal = np.max(y)
    n_fft = 2048
    S = librosa.stft(x, n_fft=n_fft, hop_length=n_fft // 2)
    D = librosa.amplitude_to_db(np.abs(S), ref=np.max)

    topDB = TOP_DB_LEVEL

    nonMuteSections = librosa.effects.split(x, topDB)
    total_time = nonMuteSections[-1][-1] / sr

    initial_pause = nonMuteSections[0][0] / sr
    initial_pause_percent = initial_pause * 100 / total_time

    mute = nonMuteSections[0][0]
    for i in range(1, len(nonMuteSections)):
        mute += (nonMuteSections[i][0] - nonMuteSections[i - 1][1])
    mute = mute / sr

    mute_percent = (mute * 100) / total_time

    return initial_pause_percent, mute_percent

def find_filler_words(speech_text):
    speech_text = speech_text.split()

    filler_words = ["like", "um", "might", "aa", "aaa", "mean", "know", "well", "totally",
                    "really", "basically", "no", "oh", "so", "maybe", "somehow", "some", "okay", "very"]

    total_filler_words = 0
    for word in filler_words:
        total_filler_words += speech_text.count(word)
    percent = (total_filler_words / max(len(speech_text), 1)) * 100
    return total_filler_words, percent

def analize_audio(file_path):
    audio_path, speech_text = video_to_text(file_path)

    wpm, speed = speech_speed(audio_path, speech_text)
    initial_pause_percent, mute_percent = pause_detection(audio_path)
    total_filler_words, filler_percent = find_filler_words(speech_text)

    output = {
        'wpm': wpm,
        'speed': speed,
        'initial_pause_percent': initial_pause_percent,
        'mute_percent': mute_percent,
        'total_filler_words': total_filler_words,
        'filler_percent': filler_percent
    }

    return output


import numpy as np
import matplotlib.pyplot as plt

def visualize(path, fitness, best_params_history, x_label="x", y_label="x",
            z_label="fitness", num_points=1000, x_min=0.0, x_max=20.0,
            y_min=0.0, y_max=20.0, color="red", azim=20.0, elev=230.0):
    """
    Produces a 3D visualization of the fitness landscape with points
    indicating the history of the best set of parameters. Only works
    if there are two parameters per individual.

    Parameters
    ----------
    path : string
            Output file path for the visualization.
    fitness : Callable
            The fitness function.
    best_params_history : np.ndarray
            A 2D array of parameters.
    x_label : string (default "x")
            The x-axis label (parameter 0).
    y_label : string (default "y")
            The y-axis label (parameter 1).
    z_label : string (default "fitness")
            The z-axis label (fitness value).
    num_points : int (default 1000)
            Number of points to fit across the x and y axes.
            Determines the precision of the fitness landscape.
    x_min : float (default 0.0)
            Minimum value for the x-axis.
    x_max : float (default 20.0)
            Maximum value for the x-axis.
    y_min : float (default 0.0)
            Minimum value for the y-axis.
    y_max : float (default 20.0)
            Maximum value for the y-axis.
    color : string (default "red")
            Color of the parameter points.
    azim : float (default 20.0)
            Plot azimuth angle in degrees.
    elev : float (default 230.0)
            Plot elevation angle in degrees.
    """
    # create a contour map of possible parameter values and
    # the corresponding fitness function values
    x = np.linspace(x_min, x_max, num_points)
    y = np.linspace(y_min, y_max, num_points)
    z = []
    for j in range(num_points):
        for i in range(num_points):
            z.append(fitness([x[i], y[j]]))
    Z = np.array(z).reshape((num_points, num_points))

    X, Y = np.meshgrid(x, y)

    fig = plt.figure()
    ax = plt.axes(projection='3d')
    ax.view_init(azim, elev)

    ax.contour3D(X, Y, Z, 50)

    # set labels
    ax.set_xlabel(x_label)
    ax.axes.set_xlim(x_min, x_max)
    ax.set_ylabel(y_label)
    ax.axes.set_ylim(y_min, y_max)
    ax.set_zlabel(z_label)

    # add points representing the history of best parameters
    ax.scatter3D(best_params_history[:,0], best_params_history[:,1],
        np.apply_along_axis(fitness, 1, best_params_history),
        color=color, depthshade=0)

    # save figure
    plt.savefig(path)


import subprocess
from pathlib import Path
import pickle

import requests
import urllib3
from bs4 import BeautifulSoup
from tqdm import tqdm

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
folders = ["easy", "medium", "hard"]
language_map = {
    "py": "Python",
    "cpp": "C++",
    "java": "Java",
    "rs": "Rust",
    "go": "Go",
    "hs": "Haskell",
    "js": "JavaScript",
}
summary = {}

try:
    with open("docs/.cache", "rb") as f:
        cache = pickle.load(f)
except:
    cache = {}

for folder in folders:
    uri = f"src/{folder}"
    if not Path(uri).exists():
        Path(uri).mkdir(exist_ok=True)
    subprocess.call(["black", uri])
    files = list(Path(uri).glob("*"))
    solutions = {}

    for file in files:
        if file.stem not in solutions:
            solutions[file.stem] = [file.name]
        else:
            solutions[file.stem].append(file.name)

    total = len(solutions)
    s = "s" if total > 1 else ""
    print()
    print(f"Found {total} solution{s} from {uri}")
    print()

    summary[folder] = total

    text = f"# Difficulty - {folder.capitalize()}\n"
    for solution, languages in tqdm(sorted(solutions.items())):
        url = f"https://open.kattis.com/problems/{solution}"
        if solution in cache:
            name = cache[solution]
        else:
            html = requests.get(url, verify=False)
            soup = BeautifulSoup(html.content, "html.parser")
            name = soup.find("h1").text
            cache[solution] = name
        card = f"""
??? success "{name}"

    === "Problem"

        See detail at [{url}]({url}).
"""
        for language in languages:
            ext = language.split(".")[-1]
            card += f"""
    === "{language_map[ext]}"

        ```{ext} linenums="1"
        --8<-- "{uri}/{language}"
        ```
"""
        text += card

    with open(f"docs/{folder}.md", "w", encoding="utf-8") as f:
        f.write(text)

with open("docs/index.md", "w") as f:
    text = "# Kattis\n"

    for folder, total in summary.items():
        text += f"""
- [{folder.capitalize()} ^{total}^]({folder}.md)
"""
    f.write(text)

with open("docs/.cache", "wb") as f:
    pickle.dump(cache, f)


import time
import datetime
import requests
from io import StringIO

# must be exported VehicleParsedOutput Records filtered to the corresponding vehicle
EXPORT_IDS = [
    {
        "platform_id": "SV3-245",
        #"wgms_record_id": 74383,
        "wgms_record_id": 74446,
    },
    {
        "platform_id": "SV3-247",
        #"wgms_record_id": 74297,
        "wgms_record_id": 74445,
    },
]

LOGINSOAP="""<?xml version="1.0" encoding="utf-8"?>
<soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap=
"http://schemas.xmlsoap.org/soap/envelope/">
  <soap:Body>
    <CreateLoginSession xmlns="http://gliders.liquidr.com/webservicesWebServices">
      <login>{login}</login>
      <password>{password}</password>
      <orgName>{org}</orgName>
    </CreateLoginSession>
  </soap:Body>
</soap:Envelope>
"""

# AVAILABLE FIELS:
#
# TimeStamp
# Structure ID
# Ground Speed(kt)
# Target
# Desired Heading (deg)
# Sub Heading
# POG (deg)
# Float Temp(degC)
# Pressure Sensor Float(kPa)
# Battery (Wh)
# Signal Strength
# User Name
# Lat (deg)
# Lon (deg)
# Distance Over Ground(m)
# Water Speed (kt/1000) WS
# Water Direction
# Current Speed (kt)
# Current Heading (deg)
# Created On

def parse_time(s):
    if len(s) == 0:
        return float("nan")
    return datetime.datetime.strptime(s, "%m/%d/%Y %H:%M:%S")

def parse_knots(s):
    if len(s) == 0:
        return float("nan")
    return float(s) * 1.852 / 3.6 # knots to m/s (exact)

def parse_milliknots(s):
    if len(s) == 0:
        return float("nan")
    return float(s) * 1.852 / 3.6 / 1000. # milliknots to m/s (exact)

def parse_degrees(s):
    if len(s) == 0:
        return float("nan")
    return float(s)

def parse_celsius(s):
    if len(s) == 0:
        return float("nan")
    return float(s)

def parse_kilopascal(s):
    if len(s) == 0:
        return float("nan")
    return float(s) * 1000. # kPa to Pa

def parse_watt_hours(s):
    if len(s) == 0:
        return float("nan")
    return float(s)

def parse_lat(s):
    if len(s) == 0:
        return float("nan")
    return float(s)

def parse_lon(s):
    if len(s) == 0:
        return float("nan")
    return float(s)

FIELDMAP = {
        "TimeStamp": {
            "name": "time",
            "parser": parse_time,
        },
        "Ground Speed(kt)": {
            "name": "speed_over_ground",
            "parser": parse_knots,
        },
        "Desired Heading (deg)": {
            "name": "desired_heading",
            "parser": parse_degrees,
        },
        "Float Temp(degC)": {
            "name": "device_temperature",
            "parser": parse_celsius,
        },
        "Pressure Sensor Float(kPa)": {
            "name": "pressure",
            "parser": parse_kilopascal,
        },
        "Battery (Wh)": {
            "name": "battery_charge",
            "parser": parse_watt_hours,
        },
        "Lat (deg)": {
            "name": "lat",
            "parser": parse_lat,
        },
        "Lon (deg)": {
            "name": "lon",
            "parser": parse_lon,
        },
        "Water Speed (kt/1000) WS": {
            "name": "water_speed",
            "parser": parse_milliknots,
        },
        "Water Direction": {
            "name": "water_direction",
            "parser": parse_degrees,
        },
        "Current Speed (kt)": {
            "name": "speed_in_water",
            "parser": parse_knots,
        },
        "Current Heading (deg)": {
            "name": "heading",
            "parser": parse_degrees,
        },
        "Created On": {
            "name": "dataset_time",
            "parser": parse_time,
        },
}

class GliderApi(object):
    def __init__(self, authinfo, base_url="https://apl-uw.wgms.com"):
        self._base_url = base_url
        #curl -k -H "Content-Type: text/xml; charset=utf-8" --dump-header headers -H "SOAPAction:" -d @loginsoap.xml -X POST https://gliders.wgms.com/webservices/entityapi.asmx
        loginsoap = LOGINSOAP.format(**authinfo)
        headers = {
            "Content-Type": "text/xml; charset=utf-8",
            "SOAPAction": None,
        }
        self._session = requests.Session()
        res = self._session.post(
            self._base_url + "/webservices/entityapi.asmx",
            data=loginsoap,
            headers=headers)
        res.raise_for_status()

    def get_export(self, view, entity):
        #http://apl-uw.wgms.com/pages/exportPage.aspx?viewid=74297&entitytype=42
        res = self._session.get(
                self._base_url + "/pages/exportPage.aspx",
                params={
                    "viewid": view,
                    "entitytype": entity
                })
        lines = iter(StringIO(res.text))
        header = next(lines).strip().split(",")
        for line in lines:
            parts = line.strip().split(",")
            data = {}
            for col, part in zip(header, parts):
                try:
                    colinfo = FIELDMAP[col]
                except KeyError:
                    continue
                data[colinfo["name"]] = colinfo["parser"](part.strip())
            yield data

def _main():
    from mqtt_utils import get_config_dir, EUREC4AMqttPublisher
    import os
    import json
    with open(os.path.join(get_config_dir(), "liquidr_gliders.json")) as configfile:
        config = json.load(configfile)

    api = GliderApi(config)

    with EUREC4AMqttPublisher() as publisher:
        while True:
            for vehicle in EXPORT_IDS:
                entries = list(sorted(api.get_export(vehicle["wgms_record_id"], 42),
                                      key=lambda x: x["time"]))
                latest = entries[-1]
                publisher.publish("platform/{}/location".format(vehicle["platform_id"]),
                    {"time": latest["time"], "lat": latest["lat"], "lon": latest["lon"]},
                    retain=True)
            time.sleep(30)

if __name__ == "__main__":
    _main()


"""
APIs of downloading pVol data from the Minio server

Author: @jiqicn
"""
import ipywidgets as widgets
import minio
import pandas as pd
from pathlib import Path
import os
import io

class Minio(object):
    """
    APIs for navigating and downloading data from the UvA-TCE Minio file server
    """
    def __init__(self):
        self.mc = None  # minio client
        self.bucket = ""  # current bucket
        self.prefix = ""  # current folder
        self.selection = []  # selection of the file paths

    def login(self, endpoint, user_name, password):
        """
        login to the UvA-TCE ninio server
        """
        self.mc = minio.Minio(
            endpoint=endpoint,
            access_key=user_name,
            secret_key=password,
            region="nl",
            secure=True
        )

        # check if successfully login
        try:
            self.mc.list_buckets()
        except Exception as e:
            print("! %s" % e.message)
            return
        print("* Login successfully!")

    def get_bucket_list(self):
        """
        get list of buckets that is accessable with the current permission
        """
        buckets = self.mc.list_buckets()
        bucket_names = []
        for b in buckets:
            bucket_names.append(b.name)
        return bucket_names

    def select_bucket(self, bucket):
        """
        select a bucket for further tasks
        """
        self.bucket = bucket

    def navigate(self, next_step=None):
        """
        navigate within the

        next_step[str or None]:
            if str, must be in format of "XXX/", where "XXX" is the directory name;
            if None, go back to a previous step;
        """
        if next_step is None:
            temp = self.prefix.split("/")
            temp = temp[:len(temp)-2]
            self.prefix = "/".join(temp) + "/"
        else:
            if next_step in self.get_object_list():
                self.prefix += next_step
            else:
                print("! Target directory not exists.")

        if self.prefix == "/":
            self.prefix = ""

    def get_object_list(self):
        """
        get list of objects in a specific bucket and under a given directory

        -parameters-
        bucket[str]
        prefix[str]: path to target directory
        """
        if self.bucket == "":
            return []
        objects = self.mc.list_objects(
            bucket_name=self.bucket,
            prefix=self.prefix
        )
        object_names = []
        for obj in objects:
            if obj.is_dir:
                obj_name = obj.object_name.split("/")[-2] + "/"
            else:
                obj_name = obj.object_name.split("/")[-1]
            object_names.append(obj_name)
        return object_names

    def save_selection(self, object_names):
        """
        accumulate the current selection

        -parameters-
        object_names[list]: list of file names for downloading
        """
        for obj_name in object_names:
            self.selection.append(self.prefix + obj_name)
        print("%d files saved to selection!" % len(object_names))

    def download_to_files(self, target_dir):
        """
        download the selected objects to a user selected folder

        -parameters-
        bucket[str]: name of the target bucket
        target_dir[str]
        """
        if target_dir is None:
            print("! Please choose the folder to keep your files.")
            return

        print("* Start downloading...")
        for obj in self.selection:
            fpath = os.path.join(target_dir, obj.split('/')[-1])
            try:
                self.mc.fget_object(
                    bucket_name=self.bucket,
                    object_name=obj,
                    file_path=fpath
                )
            except:
                print("! Fail to download %s." % obj)
                return

        print("* Download successfully to %s" % target_dir)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import requests
import cv2
import torch
from torch import nn
import torch.nn.functional as F
from torchvision import models
from torchvision import transforms
from PIL import Image

class MyVgg16(nn.Module):
    def __init__(self):
        super(MyVgg16, self).__init__()
        self.vgg = models.vgg16(pretrained=True)
        self.features_conv = self.vgg.features[:30]
        self.max_pool = self.vgg.features[30]
        self.avgpool = self.vgg.avgpool
        self.classifier = self.vgg.classifier
        self.gradients = None

    def activation_hook(self, grad):
        self.gradients = grad

    def forward(self, x):
        x = self.features_conv(x)
        h = x.register_hook(self.activation_hook)
        x = self.max_pool(x)
        x = self.avgpool(x)
        x = x.view((1, -1))
        x = self.classifier(x)
        return x

    def get_activations_gradient(self):
        return self.gradients

    def get_activations(self, x):
        return self.features_conv(x)

activation = {}
def get_activation(name):
    def hook(model, input, output):
        activation[name] = output.detach()
    return hook

def view_image(imarray):
    plt.figure()
    plt.imshow(imarray)
    plt.show()

def view_superfical_featureMap(vgg16):
    vgg16.eval()
    vgg16.features[4].register_forward_hook(get_activation("maxpool1"))
    _ = vgg16(input_im)
    maxpool1 = activation["maxpool1"]
    print("get the feature map size: ", maxpool1.shape)

    plt.figure(figsize=(11, 6))
    for ii in range(maxpool1.shape[1]):
        plt.subplot(6, 11, ii + 1)
        plt.imshow(maxpool1.data.numpy()[0, ii, :, :], cmap="gray")
        plt.axis("off")
    plt.subplots_adjust(wspace=0.1, hspace=0.1)
    plt.show()

def view_deep_featureMap(vgg16):
    vgg16.eval()
    vgg16.features[21].register_forward_hook(get_activation("layer21_conv"))
    _ = vgg16(input_im)
    layer21_conv = activation["layer21_conv"]
    print("get the feature map size: ", layer21_conv.shape)

    plt.figure(figsize=(12, 6))
    for ii in range(72):
        plt.subplot(6, 12, ii + 1)
        plt.imshow(layer21_conv.data.numpy()[0, ii, :, :], cmap="gray")
        plt.axis("off")
    plt.subplots_adjust(wspace=0.1, hspace=0.1)
    plt.show()

if __name__ == '__main__':
    vgg16 = models.vgg16(pretrained=True)
    im = Image.open("../Dataset/image.jpg")
    imarray = np.asarray(im) / 255.0
    # view_image(imarray)

    data_transforms = transforms.Compose([
        transforms.Resize((224, 224)),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])
    input_im = data_transforms(im).unsqueeze(0)
    print("input_im.shape: ", input_im.shape)

    # view_superfical_featureMap(vgg16)
    # view_deep_featureMap(vgg16)

    # LABELS_URL = "https://gist.github.com/yrevar/942d3a0ac09ec9e5eb3a.js"
    # response = requests.get(LABELS_URL)
    # labels = {int(key): value for key, value in response.json().item()}
    LABELS_DIR = "../DataFrame/imagenet1000_clsidx_to_labels.txt"
    labels = {}
    with open(LABELS_DIR, "r") as f:
        for data in f.readlines():
            data = data.strip('{')
            data = data.strip('}')
            data = data.strip('\n')
            (key, value) = data.strip().split(':')
            labels[int(key)] = value.replace("'", '')

    vgg16.eval()
    im_pre = vgg16(input_im)
    softmax = nn.Softmax(dim=1)
    im_pre_prob = softmax(im_pre)
    prob, prelab = torch.topk(im_pre_prob, 5)
    prob = prob.data.numpy().flatten()
    prelab = prelab.numpy().flatten()
    for ii, lab in enumerate(prelab):
        print("index: ", lab, "label: ", labels[lab], "||", prob[ii])

    print('--------------------------------------------------------------------------------------------')
    vggcam = MyVgg16()
    vggcam.eval()
    im_pre = vggcam(input_im)
    softmax = nn.Softmax(dim=1)
    im_pre_prob = softmax(im_pre)
    prob, prelab = torch.topk(im_pre_prob, 5)
    prob = prob.data.numpy().flatten()
    prelab = prelab.numpy().flatten()
    for ii, lab in enumerate(prelab):
        # print(ii)
        print("index: ", lab, "label: ", labels[lab], "||", prob[ii])

    # print('--------------------------------------------------------------------------------------------')
    im_pre[:, prelab[0]].backward()
    gradients = vggcam.get_activations_gradient()
    mean_gradients = torch.mean(gradients, dim=[0, 2, 3])
    activations = vggcam.get_activations(input_im).detach()
    for i in range(len(mean_gradients)):
        activations[:, i, :, :] *= mean_gradients[i]
    heatmap = torch.mean(activations, dim=1).squeeze()
    heatmap = F.relu(heatmap)
    heatmap /= torch.max(heatmap)
    heatmap = heatmap.numpy()
    plt.matshow(heatmap)
    # plt.show()

    img = cv2.imread("../Dataset/image.jpg")
    heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
    heatmap = np.uint8(255 * heatmap)
    heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
    Grad_cam_img = heatmap * 0.4 + img
    Grad_cam_img = Grad_cam_img / Grad_cam_img.max()

    b, g, r = cv2.split(Grad_cam_img)
    Grad_cam_img = cv2.merge([r, g, b])
    plt.figure()
    plt.imshow(Grad_cam_img)
    plt.show()


# -*- coding: utf-8 -*-
# @Time    : 2022/1/15 23:42
# @Author  : zhangweiqi
#! -*- coding: utf-8 -*-

import json
import re
from jieba import analyse
from zhon.hanzi import punctuation as p1
from zhon.pinyin import punctuation as p2
import re
import random
from tqdm import tqdm

re_puncs='['+re.escape(p1+p2+" \r\n")+']'

def cutTextToSent(text:str):
    split_text=re.sub(re_puncs,' ',text).split()#每个句子按标点符号分句
    return split_text

# 此处处理停用词
def getStopWords():
    return set(' ')

def readFromJsonFile(path: str):
    with open(path, "r", encoding="utf-8") as f:
        return json.loads(f.read())

def writeToJsonFile(path: str, obj, indent=None):
    with open(path, "w", encoding="utf-8") as f:
        f.write(json.dumps(obj, ensure_ascii=False, indent=indent))

#有些大型json文件，是每一行自成json格式，按行写入的
def readFromJsonFileForLine(path: str,max_line=float('inf')):
    with open(path, "r", encoding="utf-8") as f:
        res=[]
        for i in tqdm(f,ncols=80):
            data=i.strip()
            if len(data)!=0:
                res.append(json.loads(data))
            if len(res)>=max_line:
                return res
        return res

#适用于需要不断追加写入的json文件，f为a+模式打开
#每写一次都打开关闭文件一次，可确保能基本实时保存，但效率低
def appendJsonLine(path: str, obj):
    with open(path, "a+", encoding="utf-8") as f:
        f.write(json.dumps(obj,ensure_ascii=False)+'\n')

def writeToJsonFileForLine(path: str, ls:list):
    with open(path, "a+", encoding="utf-8") as f:
        for i in ls:
            f.write(json.dumps(i,ensure_ascii=False)+'\n')

#根据百分位间隔计算长度分布
def cal_Len_distribution(Lens,begin=0,end=100,step=5):
    Lens=list(Lens)
    Lens.sort()
    total_len=len(Lens)
    for i in range(begin,end+1,step):
        idx=int(total_len*i*0.01)
        idx=min(idx,total_len-1)
        print(f"百分位：{i}%，\tindex：{idx}，\t长度：{Lens[idx]}")

#删除多余空白符。对文本保留自然段（换行），其他空白符替换成空格且只保留一个
def delSpace(st:str):
    ls=st.split('\n')#只有换行特殊处理
    ls=list(filter(lambda x:len(x)>0,ls))
    for i in range(len(ls)):
        ls[i]=" ".join(ls[i].split())#若有多个分割，统一用空格替代
    return "\n".join(ls)


import matplotlib.pyplot as plt
import numpy as np

standard = np.random.randn(1000)
G1= 10+2*standard
G2 = -6 +3*standard

plt.figure(figsize=(10,6))
plt.hist(standard, bins=50,color='blue', alpha=0.5)
plt.hist(G1,bins=50,color='orange',alpha=0.5)
plt.hist(G2,bins=50,color='green',alpha=0.5)

plt.show()


from model import boutModel
import matplotlib.pyplot as plt
import argparse
import json
import numpy as np
import seaborn as sn
import sys
import random
from alive_progress import alive_bar

'''
Data notes:
Most data is from the Women's world championship in 2018, some bouts are from other tournaments, but using the same fencers around the same time
Data recorded:
names of each fencer
for each point:
    who won the point
    how they won it (offensive,defensive, counterattack)
    what each fencer did (quick, defensive, long)

Things I wish i had recorded:
    did the initial action decide the point, ie. was there a significant right of way switch
    some way to differentiate between the call: "attack is no, attack"
        a defensive manuever resulting in a right of way switch, so an offensive point from a defensive action
        however, it is not clear whether or not this should be offensive since often times the
        second attack could be considered more of a riposte than its own attack
    lefthanded vs righthanded (this is extremely important)
Things I wanted to do if I had the time:
    record every action from each point in a more detailed way, rather than just offense/defense/counter

'''
def covariance(x, y):
    # Finding the mean of the series x and y
    mean_x = sum(x)/float(len(x))
    mean_y = sum(y)/float(len(y))
    # Subtracting mean from the individual elements
    sub_x = [i - mean_x for i in x]
    sub_y = [i - mean_y for i in y]
    numerator = sum([sub_x[i]*sub_y[i] for i in range(len(sub_x))])
    denominator = len(x)-1
    cov = numerator/denominator
    return cov

def isValid(point):
    if point['score'] != 'l' and point['score'] != 'r':
        return False
    if point['type'] != 'd' and point['type'] != 'o' and point['type'] != 'c':
        return False
    if point['leftAction'] != 'q' and point['leftAction'] != 'd' and point['leftAction'] != 'l':
        return False
    if point['rightAction'] != 'q' and point['rightAction'] != 'd' and point['rightAction'] != 'l':
        return False
    return True

def calculateAccuracy(fencers,data,scalars, skip_marked = False, only_marked = False):
    num_correct = 0
    num_total = 0
    for bout in data['bouts']:
        if skip_marked and 'marked' in bout:
            continue
        if only_marked and not 'marked' in bout:
            continue
        model = boutModel(fencers[bout['leftName']]['data_array'],fencers[bout['rightName']]['data_array'],scalars,useRandom = useRandom)
        for point in bout['points']:
            if not isValid(point):
                continue
            num_total+=1
            leftAction = point['leftAction']
            rightAction = point['rightAction']

            predictedWinner = model.predictWinner(leftAction,rightAction)
            actualWinner = point['score']
            if predictedWinner == actualWinner:
                num_correct+=1
    return num_correct/num_total

useRandom = False
skip_calcs = True
if len(sys.argv) > 1:
    for arg in sys.argv[1:]:
        if arg == '-r':
            useRandom = True
        if arg == '-s':
            skip_calcs = False

data = {}
with open('data.json','r') as f:
    data = json.load(f)
fencers = {}
print("Total number of bouts: ", len(data['bouts']))
print("Total number of points: ", sum(len(bout['points']) for bout in data['bouts']))
marked_bouts = random.sample(data['bouts'],3)
for bout in marked_bouts:
    bout['marked'] = True
for bout in data['bouts']:
    leftFencer = bout['leftName']
    rightFencer = bout['rightName']
    if leftFencer not in fencers:
        fencers[leftFencer] = {
            'offensive points': 0,
            'defensive points': 0,
            'total points won': 0,
            'speed points': 0,
            'total points': 0,
            'total points predicted correctly': 0,
            'num bouts': 0,
        }
    if rightFencer not in fencers:
        fencers[rightFencer] = {
            'offensive points': 0,
            'defensive points': 0,
            'total points won': 0,
            'speed points': 0,
            'total points': 0,
            'total points predicted correctly': 0,
            'num bouts': 0,
        }
    fencers[leftFencer]['num bouts']+=1
    fencers[rightFencer]['num bouts']+=1
    for point in bout['points']:
        if not isValid(point):
            continue
        fencers[leftFencer]['total points'] += 1
        fencers[rightFencer]['total points'] += 1
        if point['score'] == 'l':
            fencers[leftFencer]['total points won'] += 1
            if point['type'] == 'o':
                fencers[leftFencer]['offensive points'] += 1
                if point['leftAction'] == 'd':
                    fencers[leftFencer]['defensive points'] += 1
            elif point['type'] == 'd':
                fencers[leftFencer]['defensive points'] += 1
            elif point['type'] == 'c':
                fencers[leftFencer]['speed points'] += 1
                fencers[leftFencer]['defensive points'] += 1

            if point['leftAction'] == point['rightAction']:
                fencers[leftFencer]['speed points'] += 1
        else:
            fencers[rightFencer]['total points won'] += 1
            if point['type'] == 'o':
                fencers[rightFencer]['offensive points'] += 1
                if point['rightAction'] == 'd':
                    fencers[rightFencer]['defensive points'] += 1
            elif point['type'] == 'd':
                fencers[rightFencer]['defensive points'] += 1

            elif point['type'] == 'c':
                fencers[rightFencer]['speed points'] += 1
                fencers[rightFencer]['defensive points'] += 1

            if point['leftAction'] == point['rightAction']:
                fencers[rightFencer]['speed points'] += 1

for fencer in fencers:
    fencers[fencer]['data_array'] = [
        fencers[fencer]['offensive points'] / fencers[fencer]['total points won'],
        fencers[fencer]['defensive points'] / fencers[fencer]['total points won'],
        .5+fencers[fencer]['speed points'] / fencers[fencer]['total points won'],
        fencers[fencer]['total points won'] / fencers[fencer]['total points']
    ]
    print(fencer, ": ", fencers[fencer]['data_array'], ' (',fencers[fencer]['num bouts'],')')

num_correct = 0
num_total = 0
type_correct = {
    'qq': 0,
    'qd': 0,
    'ql': 0,
    'dd': 0,
    'dl': 0,
    'll': 0
}
type_count = {
    'qq': 0,
    'qd': 0,
    'ql': 0,
    'dd': 0,
    'dl': 0,
    'll': 0
}
best_acc = 0
best_scalars = [0.2, 0.1, 0.1, 0.9, 3.9]

if not skip_calcs:
    with alive_bar((19**4)) as bar:
#        for s0 in np.arange(.1,2,.1):
#            for s1 in np.arange(.1,2,.1):
#               for s2 in np.arange(.1,2,.1):
#                    for s3 in np.arange(.1,2,.1):
                        for s4 in np.arange(.1,20,.1):
                            scalars = [0.2, 0.1, 0.1, 0.9,s4]
                            acc = calculateAccuracy(fencers,data,scalars,skip_marked=True)
                            if useRandom:
                                acc = 0
                                for i in range(100):
                                    acc +=.01* calculateAccuracy(fencers,data,scalars,skip_marked=True)

                            bar()
                            if acc > best_acc:
                                best_acc = acc
                                best_scalars = scalars
print("Test Set Accuracy: ", calculateAccuracy(fencers,data,best_scalars,only_marked=True))

#print("Best accuracy: ", best_acc)
row_splits = []
generalized_splits = []
for bout in data['bouts']:
    model = boutModel(fencers[bout['leftName']]['data_array'],fencers[bout['rightName']]['data_array'],best_scalars, useRandom = useRandom)
    for point in bout['points']:
        if not isValid(point):
            continue

        leftAction = point['leftAction']
        rightAction = point['rightAction']
        pair_type = leftAction + rightAction
        if pair_type == 'dq':
            pair_type = 'qd'
        elif pair_type == 'lq':
            pair_type = 'ql'
        elif pair_type == 'ld':
            pair_type = 'dl'

        type_count[pair_type] += 1

        predictedWinner = model.predictWinner(leftAction,rightAction)

        if fencers[bout['leftName']]['num bouts'] > 2 and fencers[bout['rightName']]['num bouts'] > 2:
            num_total+=1
        actualWinner = point['score']
        if predictedWinner == actualWinner:
            if fencers[bout['leftName']]['num bouts'] > 2 and fencers[bout['rightName']]['num bouts'] > 2:
                num_correct+=1
            type_correct[pair_type] += 1

            if point['score'] == 'l':
                fencers[bout['leftName']]['total points predicted correctly'] += 1
            elif point['score'] == 'r':
                fencers[bout['rightName']]['total points predicted correctly'] += 1
        #print("[",leftAction,',',rightAction,'] :',predictedWinner,'/',actualWinner)
        model_gen_splits, model_row_splits = model.get_splits()
        generalized_splits.extend(model_gen_splits)
        row_splits.extend(model_row_splits)
plt.clf()
generalized_splits = [x for x in generalized_splits if x > 0]
row_splits = [x for x in row_splits if x > 0]
plt.hist(generalized_splits, bins = 100)
plt.savefig("graphs/Generalized Splits.png")
plt.clf()

plt.hist(row_splits, bins = 100)
plt.savefig("graphs/RoW Splits.png")
print("**************** Best Accuracy ****************")
print(best_scalars)
print(num_correct/num_total)
print("**************** Type Accuracy ****************")
for pair_type in type_count:
    print(pair_type, ": ", type_correct[pair_type]/type_count[pair_type], "  (", type_count[pair_type],")")
print("**************** Accuracy by Fencer **************** ")
plt.clf()
Y = [fencers[fencer]['total points predicted correctly']/fencers[fencer]['total points won'] for fencer in fencers]
X = [fencers[fencer]['total points'] for fencer in fencers]
plt.scatter(X,Y)
plt.plot(np.unique(X), np.poly1d(np.polyfit(X, Y, 5))(np.unique(X)))
plt.savefig('graphs/Accuracy by Fencer.png')
plt.clf()

X = [covariance([fencers[fencer]['data_array'][i]  for fencer in fencers],Y) for i in range(4)]
labels = ['Offensive', 'Defensive', 'Speed', 'Overall']
fig = plt.figure()
plt.bar(labels,X)
plt.savefig('graphs/Characteristic Covariance.png')

plt.clf()
Y = [type_correct[pair_type]/type_count[pair_type] if pair_type !='dd' else 0 for pair_type in type_count]
X = [type_count[pair_type] if pair_type !='dd' else 0 for pair_type in type_count]
X.remove(0)
Y.remove(0)
plt.scatter(X,Y)
plt.plot(np.unique(X), np.poly1d(np.polyfit(X, Y, 1))(np.unique(X)))
plt.savefig('graphs/Type Accuracy by Number of Type.png')

for fencer in fencers:
    print(fencer, ": ", fencers[fencer]['total points predicted correctly']/fencers[fencer]['total points won'], "  (", fencers[fencer]['total points won'],")")
# Analyze decisions after winning a point
num_same_action = 0
num_different_action = 0
num_same_action_won = 0
num_different_action_won = 0
for bout in data['bouts']:
    last_win = ''
    last_win_action = ''
    for point in bout['points']:
        if last_win == 'l':
            if point['leftAction'] == last_win_action:
                num_same_action+=1
                if point['score'] == 'l':
                    num_same_action_won+=1
            else:
                num_different_action+=1
                if point['score'] == 'l':
                    num_different_action_won+=1
        elif last_win == 'r':
            if point['rightAction'] == last_win_action:
                num_same_action+=1
                if point['score'] == 'r':
                    num_same_action_won+=1
            else:
                num_different_action+=1
                if point['score'] == 'r':
                    num_different_action_won+=1
        last_win = point['score']
        last_win_action = point['leftAction'] if point['score'] == 'l' else point['rightAction']

print("*************** Action analysis after winning point *****************")
print("Same action: ", num_same_action)
print("Different action: ", num_different_action)
print("Same action won: ", num_same_action_won)
print("Different action won: ", num_different_action_won)
print("Same action win percent: ", num_same_action_won/num_same_action)
print("Different action win percent: ", num_different_action_won/num_different_action)

action_pair_loss_loss = {
    'qq': 0,
    'qd': 0,
    'ql': 0,
    'dq': 0,
    'dd': 0,
    'dl': 0,
    'lq': 0,
    'ld': 0,
    'll': 0
}
action_pair_loss_win = {
    'qq': 0,
    'qd': 0,
    'ql': 0,
    'dq': 0,
    'dd': 0,
    'dl': 0,
    'lq': 0,
    'ld': 0,
    'll': 0
}
action_pair_win_loss = {
    'qq': 0,
    'qd': 0,
    'ql': 0,
    'dq': 0,
    'dd': 0,
    'dl': 0,
    'lq': 0,
    'ld': 0,
    'll': 0
}
action_pair_win_win = {
    'qq': 0,
    'qd': 0,
    'ql': 0,
    'dq': 0,
    'dd': 0,
    'dl': 0,
    'lq': 0,
    'ld': 0,
    'll': 0
}
action_pairs = {
    'qq': 0,
    'qd': 0,
    'ql': 0,
    'dq': 0,
    'dd': 0,
    'dl': 0,
    'lq': 0,
    'ld': 0,
    'll': 0
}
for bout in data['bouts']:
    last_action_left = ''
    last_action_right = ''
    last_win = ''
    last_winning_action = ''
    for point in bout['points']:
        if not isValid(point):
            last_action_left = ''
            last_action_right = ''
            last_win = ''
            last_winning_action = ''
            continue
        if last_action_left == '':
            last_action_left = point['leftAction']
            last_action_right = point['rightAction']
            continue
        if last_action_left+point['leftAction'] in action_pairs:
            action_pairs[last_action_left+point['leftAction']]+=1
        if last_action_right+point['rightAction'] in action_pairs:
            action_pairs[last_action_right+point['rightAction']]+=1

        if point['score'] == 'l':
            if last_win == 'r':
                action_pair_loss_win[last_action_left+point['leftAction']]+=1
                action_pair_win_loss[last_action_right+point['rightAction']]+=1
            elif last_win == 'l':
                action_pair_win_win[last_action_left+point['leftAction']]+=1
                action_pair_loss_loss[last_action_right+point['rightAction']]+=1

        elif point['score'] == 'r':
            if last_win == 'r':
                action_pair_win_win[last_action_right+point['rightAction']]+=1
                action_pair_loss_loss[last_action_left+point['leftAction']]+=1

            elif last_win == 'l':
                action_pair_loss_win[last_action_right+point['rightAction']]+=1
                action_pair_win_loss[last_action_left+point['leftAction']]+=1
        winning_action = point['leftAction'] if point['score'] == 'l' else point['rightAction']
        last_winning_action = winning_action
        last_action_left = point['leftAction']
        last_action_right = point['rightAction']
        last_win = point['score']

print("**************** Action pair Counts ****************")
for action_pair in action_pairs:
    print(action_pair, ": ", action_pairs[action_pair])

# P(win_current | last_action, current_action )
print("**************** Action pair 2nd win analysis ****************")
for a in ['q', 'd', 'l']:
    print("------ ", a, " ------")
    for p in ['q', 'd', 'l']:
                print(a+p, ": ", (action_pair_loss_win[a+p]+action_pair_win_win[a+p])/(action_pair_win_win[a+p]+action_pair_win_loss[a+p]+action_pair_loss_loss[a+p]+action_pair_loss_win[a+p]))
    #print(ap, ": ", (action_pair_loss_win[ap] + action_pair_win_win[ap] ) / action_pairs[ap])
aps = ['qq', 'qd', 'ql', 'dq', 'dd', 'dl', 'lq', 'ld', 'll']
X = [action_pairs[ap] for ap in aps]
Y = [(action_pair_loss_win[ap]+action_pair_win_win[ap])/(action_pair_win_win[ap]+action_pair_win_loss[ap]+action_pair_loss_loss[ap]+action_pair_loss_win[ap]) for ap in aps]
plt.clf()
plt.scatter(X,Y)
plt.plot(np.unique(X), np.poly1d(np.polyfit(X, Y, 1))(np.unique(X)))
plt.savefig('graphs/Action Pair Success Rate by Frequency.png')
# P(win_current | last_action, current_action && last_action => win )
print("**************** Action pair win - win analysis ****************")
for a in ['q', 'd', 'l']:
    print("------ ", a, " ------")
    for p in ['q', 'd', 'l']:
        print(a+p, ": ", action_pair_win_win[a+p]/(action_pair_win_win[a+p]+action_pair_win_loss[a+p]))

# P(win_current | last_action, current_action && last_action => loss )
print("**************** Action pair loss - win analysis ****************")
for a in ['q', 'd', 'l']:
    print("------ ", a, " ------")
    for p in ['q', 'd', 'l']:
                print(a+p, ": ", action_pair_loss_win[a+p]/(action_pair_loss_loss[a+p]+action_pair_loss_win[a+p]))


import numpy as np
import rdflib
import re
from collections import defaultdict
import csv
import os

# Get the required entities from the rdf file
def get_entities(file_loc):
	g = rdflib.Graph()
	g.load(file_loc)

	# Query is missing bibliographic ID
	query = """SELECT ?listItem ?seqID ?author ?title ?keyword
			WHERE {
			?listItem j.2:itemContent ?record .
			?record j.7:references ?manifest .
			?manifest j.4:embodimentOf ?expression .

			?listItem j.5:hasSequenceIdentifier ?seqID .
			?manifest j.3:keyword ?keyword .

			OPTIONAL {
				?expression j.0:creator ?author .
				FILTER(datatype(?author) = xsd:string) .
			}
			OPTIONAL {
				?expression j.0:title ?title .
				FILTER(datatype(?title) = xsd:string) .
			}
			}"""

	results = g.query(query)

	print("====== RDF Query ======")
	print("The query returned " + str(len(results)) + " results")
	print()

	g.close()
	return results

# Gather all keyword entities from the rdf graph
def get_keywords(entities):
	result = dict()

	return [entity['keyword'].value for entity in entities]

# Creates a dictionary with entities as keys and the split keyword arrays as values
def clean_and_split(entities):
	result = dict()
	counts = [0, 0, 0, 0, 0, 0, 0, 0]
	for entity in entities:
		keywords = entity['keyword'].value
		keywords = keywords.strip()
		keywords = re.split("[;:]\n\s*", keywords)
		result[entity] = keywords
		if len(keywords) > 6:
			counts[7] += 1
		else:
			counts[len(keywords)] += 1

	print("====== clean_and_split result ======")
	for i in range(7):
		print("Number of entries with " + str(i) + " keywords: " + str(counts[i]))
	print("Number of entries with more than 6 keywords: " + str(counts[7]))
	print()

	return result

# Separates keywords into individual words and adds words to appropriate dictionary
def create_dicts(separated_keywords):
	data_count = 0
	dicts = [defaultdict(int) for x in range(5)]
	for keyword_group in separated_keywords:
		if len(keyword_group) != 5:
			continue
		data_count += 1
		for keyword in keyword_group:
			for num in range(5):
				words = keyword.split()
				for raw in words:
					word = clean_word(raw)
					if word in dicts[num]:
						dicts[num][word] = dicts[num][word] + 1
					else:
						dicts[num][word] = 1

	print("====== create_dicts result ======")
	print("Words from " + str(data_count) + " keyword entries were used to create the dictionaries")
	print()
	if print_dictionaries:
		print_dicts(dicts)

	return dicts

# Removes special characters from words in order to attempt to avoid their influence
def clean_word(word):
	res = word.strip("\",.()[]!?{};’")
	res = re.sub("\w?[’'‘]", "", res)
	return res

# Prints dictionaries
def print_dicts(dicts):
	for dic in dicts:
		for w in sorted(dic, key=dic.get, reverse=True):
			print(w, dic[w])
		print("==============")

# Sorts separated keywords into categories according to the dictionaries.
# Extra word are included to prevent data loss
def sort_words(dicts, keyword_dicts, entities):
	sorted = defaultdict()
	for entity in entities:
		keywords = keyword_dicts.get(entity)
		if len(keywords) >= 5:
			sorted[entity] = keywords
		else:
			last_placed = -1
			indices = []
			order = ["NULL", "NULL", "NULL", "NULL", "NULL"]
			for num in range(len(keywords)):
				# Reduce the potential fields the keyword could be in by eliminating possibilities
				# This relies on the assumption that the data is in the correct order, even if incomplete
				front_lim = max(num, last_placed + 1)
				back_lim = 5 - len(keywords) + num

				words = keywords[num].split()
				scores = []

				for dict_num in range(front_lim, back_lim + 1):
					score = 0
					for raw_word in words:
						word = clean_word(raw_word)
						if word in dicts[num]:
							score = score + dicts[dict_num][word]
					scores.append(score)
				if not len(scores) == 0:
					order[num + np.argmax(scores)] = re.sub("\s{2,}", ", ", keywords[num])  # get rid of whitespace here
					last_placed = num + np.argmax(scores)
			sorted[entity] = order
	return sorted

def to_csv(entities, sorted_dict):
	with open("output.csv", 'w') as output:
		wr = csv.writer(output)
		if os.stat("output.csv").st_size == 0:
			wr.writerow(
				["ID", "Autor", "Titel", "Form", "Handlungsort", "Protagonisten", "Handlung", "Themen", "Extras ->"])
		# seqID author title keywords
		for entity in entities:
			row = []

			if entity['seqID'] is not None:
				row.append(entity['seqID'].value)
			else:
				row.append("NULL")

			if entity['author'] is not None:
				row.append(entity['author'].value)
			else:
				row.append("NULL")

			if entity['title'] is not None:
				row.append(entity['title'].value)
			else:
				row.append("NULL")

			row.extend(sorted_dict[entity])
			wr.writerow(row)

#################
# Config values #
#################
print_dictionaries = False
file_location = "MA_MODEL.rdf"

if __name__ == '__main__':
	print()

	entities = get_entities(file_location)

	entities = sorted(entities, key=lambda row: row['seqID'].value)

	keyword_dicts = clean_and_split(entities)

	dictionaries = create_dicts(keyword_dicts.values())

	sorted_dict = sort_words(dictionaries, keyword_dicts, entities)

	to_csv(entities, sorted_dict)


# -*- coding: utf-8 -*-

import configparser
from Predictor import Predictor
from Scraper import Scraper

class Searcher:

    def __init__(self):
        self.__BHK = 0
        self.__FloorArea = 0
        self.__Bathrooms = 0
        self.__Locality = ""
        self.__FurnishingStatus = ""
        self.__TypeofSale = ""

    def putBHK(self, BHK):
        self.__BHK = BHK

    def putFloorArea(self, FloorArea):
        self.__FloorArea = FloorArea

    def putBathrooms(self, Bathrooms):
        self.__Bathrooms = Bathrooms

    def putLocality(self, Locality):
        self.__Locality = Locality

    def putFurnishingStatus(self, FurnishingStatus):
        self.__FurnishingStatus = FurnishingStatus

    def putTypeofSale(self, TypeofSale):
        self.__TypeofSale = TypeofSale

    def searchProperties(self, BHK, FloorArea, Bathrooms, Locality, FurnishingStatus, TypeofSale):
        configParser = configparser.ConfigParser(allow_no_value = True)
        keys = []
        places = []
        configFilePath = 'config.ini'
        configParser.read(configFilePath)
        if(len(configParser.read(configFilePath)) > 0):
            details = configParser.sections()[0]
            for key in configParser[details]:
                keys.append(key)
            places += configParser[details][keys[0]].split(',')
        places_key=[]
        for place in places:
            places_key +=['Locality_'+place]
        inp={}
        inp = {'BHKs':[BHK],'FloorArea':[FloorArea],'Bathrooms':[Bathrooms]}
        for place in places_key:
            inp[place]= [0]
        inp.update({'Type_of_Sale_New':[0],'Type_of_Sale_Resale':[0],'Furnishing_Status_Furnished':[0],
                    'Furnishing_Status_Semi-Furnished':[0],'Furnishing_Status_Unfurnished':[0]})
        inp['Locality_'+Locality] = [1]
        inp['Furnishing_Status_'+FurnishingStatus] = [1]
        inp['Type_of_Sale_'+TypeofSale] = [1]
        #print(inp)
        predictor = Predictor()
        #print(type(predictor.regressor))
        #print(predictor.price)
        #return 0
        price = predictor.getPredictedPrice(inp)
        if price:
            return price
        else:
            predictor.managePredictor()
            return predictor.getPredictedPrice(inp)

    def viewProperties(self, BHK, Locality, FurnishingStatus, TypeofSale):
        scraper = Scraper()
        PropertyURL_list, ImageURL_list, Area_list, BHK_list = scraper.returnPropertyURLS(BHK, Locality, FurnishingStatus, TypeofSale)
        return PropertyURL_list, ImageURL_list, Area_list, BHK_list

"""
if __name__=='__main__':
    searcher = Searcher()"""


#!/usr/bin/env python3.10

# This is a sample Python script.

# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.

import sys

import itertools
import numpy as np

from xml.dom import minidom
from collections import Counter

from spineigenfunctions import *

###################################################################

def ormas_like():
# finp is the xml file
    finp = sys.argv[1]
#    finp = 'input.xml'
    doc = minidom.parse(finp)

# general info
    general = doc.getElementsByTagName("general")
    outfile = general[0].getElementsByTagName('outfile')[0].childNodes[0].data
    elec = general[0].getElementsByTagName('electron')
    spin = int(general[0].getElementsByTagName('spin')[0].childNodes[0].data)
    na = elec[0].getAttribute("na")
    nb = elec[0].getAttribute("nb")

# Get all the blocks
    blocks = doc.getElementsByTagName("block")

    nCSFs = 0

    for block in blocks:
        spaces = block.getElementsByTagName('space')
        seqs = []
        occspace = []
        for space in spaces:
            ne = int(space.getAttribute("ne"))
            ip = int(space.getAttribute("imo"))
            fp = int(space.getAttribute("fmo"))

            seq = np.arange(ip, fp + 1)
            seqtot = np.concatenate((seq, seq), axis=0)
            seqs = np.concatenate((seqs, seq), axis=0)

            s = []
            ind = []
            for l in list(itertools.combinations(seqtot, ne)):
                count = []
                for i in range(ip, fp + 1):
                    count.append(l.count(i))
                s.append(count)

        # remove duplicates
            occ = []
            for i in s:
                if i not in occ:
                    occ.append(i)

            occspace.append(occ)

        if (len(occspace) == 1):
            seqocc = occspace[0]

        for i in range(len(occspace) - 1):
            if (i == 0):
                seqocc = occspace[i]
            s = []
            for j in seqocc:
                for k in occspace[i + 1]:
                    s.append(j + k)
                seqocc = s

        for occ in seqocc:
            p = []
            unp = []
            for i in range(len(occ)):
                if (occ[i] == 2):
                    p.append(int(seqs[i]))
                if (occ[i] == 1):
                    unp.append(int(seqs[i]))

            nCSFs += printCSF(spin, p, unp)

# print info in CI code format
    fhead = open('header.txt', 'a')
    print(nCSFs, '0.000000000000001', file=fhead)
    print(outfile, file=fhead)
    print(na, nb, file=fhead)
    print(nCSFs, file=fhead)
    fhead.close()

    return 0

# See PyCharm help at https://www.jetbrains.com/help/pycharm/
# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    import os

    if os.path.exists("header.txt"):
        os.remove("header.txt")
    if os.path.exists("list.txt"):
        os.remove("list.txt")
    if(ormas_like()==0):
        os.system("cat header.txt list.txt > input_cicode.txt")
        os.remove("header.txt")
        os.remove("list.txt")
        os.system("/home/nico/Workspace/Progs/Ints_From_gbtolib/MyGBTO_CIcode/myCI.exe input_cicode.txt")
        print('Job Done')


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals, print_function, division
import argparse
import glob
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import time
import math
from io import open
import os
import sys
import numpy as np
from scipy.stats.stats import pearsonr
#import seaborn as sns

# Timing
# ======

def count_params(model):
    total_params = sum(p.numel() for p in model.parameters())
    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
    return total_params, trainable_params

# Timing
# ======

def asMinutes(s):
    m = math.floor(s / 60)
    s -= m * 60
    return '%dm %ds' % (m, s)

def timeSince(since, percent):
    now = time.time()
    s = now - since
    es = s / percent
    rs = es - s
    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))

# Visualizing Attention
# =====================

def plot_attention(input_sentence, output_words, attentions):
    # Set up figure with colorbar
    fig = plt.figure()
    ax = fig.add_subplot(111)
    cax = ax.matshow(attentions.cpu().numpy(), cmap='bone')
    fig.colorbar(cax)

    # Set up axes
    ax.set_xticklabels([''] + input_sentence, rotation=90)
    # hack
    ax.set_yticklabels([''] + output_words)

    # Show label at every tick
    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))
    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))

    return fig

def save_and_plot_attention(input_sentence, output_words, attentions, base_dir, name):
    # Set up figure with colorbar
    fig = plt.figure()
    ax = fig.add_subplot(111)
    cax = ax.matshow(attentions.numpy(), cmap='bone')
    fig.colorbar(cax)

    # Set up axes
    ax.set_xticklabels([''] + input_sentence.split(' ') +
                       ['</s>'], rotation=90)
    # hack
    ax.set_yticklabels([''] + output_words.split(' ') + ['</s>'])

    # Show label at every tick
    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))
    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))

    fig.savefig(base_dir + name + '.png', dpi=fig.dpi)
    plt.gcf().clear()
    plt.close('all')
    # write plain text attention
    att = attentions.numpy()
    src_tokens = input_sentence.split() + ['</s>']
    trg_tokens = output_words.split() + ['</s>']
    with open(base_dir + name + '.txt', encoding='utf-8', mode='w') as attn_file:
        attn_file.write('\t' + '\t'.join(src_tokens) + '\n')
        for i in range(len(trg_tokens)):
            coeffs = '\t'.join(map(str, list(att[i])))
            attn_file.write('{}\t{}\n'.format(trg_tokens[i], coeffs))

def read_attention_file(report_name):
    """Reads a report file and outputs a numpy array.

    :report_name: the full path of a report file representing an attention matrix.
    :returns: a numpy array, src and trg labels

    """
    with open(report_name, encoding='utf-8', mode='r') as f:
        lines = f.readlines()
        b = None
        src = lines[0]
        for line in lines[1:]:
            line = line.strip().split('\t')
            if b is None:
                trg = line[0]
                b = np.array([line[1:]])
            else:
                trg += '\t' + line[0]
                b = np.append(b, [line[1:]], axis=0)
    a = b.astype(np.float)
    return a, src, trg

def write_attention(a, src, trg, new_report_name):
    """Writes a file corresponding to an attention matrix.

    :a: numpy array
    :src: string of source labels
    :trg: string of target labels
    :new_report_name: name of the output file
    :returns: nothing.
    """
    with io.open(new_report_name, encoding='utf-8', mode='w') as f:
        f.write(trg)
        s_list = src.strip().split('\t')
        for i in range(len(a)):
            line = s_list[i] + '\t' + '\t'.join(map(str, list(a[i,:])))
            f.write(line + '\n')

def plot_attention_from_file(attention_file):
    # Set up figure with colorbar
    fig = plt.figure()
    ax = fig.add_subplot(111)
    # get attention, input and output words
    attentions, src, trg = read_attention_file(attention_file)
    # print(src, trg)

    cax = ax.matshow(attentions, cmap='bone')
    fig.colorbar(cax)

    # Set up axes
    ax.set_xticklabels([''] + src.split(), rotation=90)
    # hack
    ax.set_yticklabels([''] + trg.split())

    # Show label at every tick
    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))
    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))

    fig.savefig(os.path.join(os.path.dirname(attention_file), os.path.basename(attention_file) + '.png'), dpi=fig.dpi)
    plt.gcf().clear()
    plt.close('all')

# Plotting results
# ----------------

def save_plot(points, base_dir, run_name, xp_name):
    # fig = plt.figure()
    fig, ax = plt.subplots()
    # this locator puts ticks at regular intervals
    loc = ticker.MultipleLocator(base=0.2)
    ax.yaxis.set_major_locator(loc)
    plt.plot(points)
    fig.savefig(base_dir + run_name + '/' + xp_name + '/loss_curve.png', dpi=fig.dpi)
    plt.gcf().clear()
    plt.close('all')

# To save graphs to tensorboard
# graph
# dummy_input = torch.zeros([5, 1], dtype=torch.long).to(device)
# with SummaryWriter(comment='encoder') as w:
#     with torch.no_grad():
#         w.add_graph(encoder1, dummy_input, verbose=True)

# graph
# with SummaryWriter(comment='decoder') as w:
#     with torch.no_grad():
#         w.add_graph(attn_decoder1, (torch.zeros([1, 1], dtype=torch.long),
#                                None,
#                                None,
#                                torch.randn(5,1,128),
#                                torch.randn(1,1,64),
#                                torch.randn(1,1,64)),
#                     verbose=False)

# Entropy related utils
# ---------------------

def calculate_entropy(attention):
    """Reads a numpy array corresponding to an attention matrix
    and outputs the average entropy.

    :att: a numpy array (src indexing lines and trg indexing columns)
    :returns: a float number corresponding to the average entropy

    """
    h_list = []
    for r in attention:
        h = stats.entropy(r)
        # we normalize by the entropy of the uniform distribution
        # on a sentence of that length so we have a percentage of
        # the max possible noise
        h /= math.log(r.shape[0])
        h_list += [h]
    avg_h = np.mean(np.array(h_list))
    return avg_h

def draw_hist(data, name):
    """ Save histogram from a list of data points.

    :data: a list of floats
    :name: the name for the image
    :returns: nothing
    """
    sns_plot = sns.distplot(data, bins=40)
    # sns_plot = sns.distplot(data, norm_hist=False, kde=False, bins=40)
    sns_plot.set_title(".".join(name.split('.')[1:-3]).split('/')[2])
    sns_plot.set_xlim([0.0,1.0])
    sns_plot.set_xlabel("entropy")
    sns_plot.set_ylabel("density")
    fig = sns_plot.get_figure()
    fig.savefig(f"{name}.png")
    plt.gcf().clear()

def entropy_histogram(scope, output_dir=None):
    """Builds a histogram representing the entropy of the
    matrices corresponding to the scope.

    :scope: a glob-like scope
    :output_dir: destination of the png histogram
    :returns: the global average entropy

    """
    d_list = glob.glob(scope + "**/reports/", recursive=True)
    assert len(d_list) == 1
    d = d_list[0]
    # /vol/work/godard/dat/att-seg/output/TRUE_E50_B1_D64_T10.0_Wfalse_Pfalse/reports/
    entropies = []
    for f in glob.glob(d + "report.*"):
        att, _, _ = read_attention(f)
        h = calculate_entropy(att)
        entropies += [h]
    if output_dir is not None:
        draw_hist(entropies, output_dir + os.path.basename(f))
    return np.mean(np.array(entropies))

def draw_training_curves(scope, output_dir):
    """Draw dev and training loss.

    :scope: a glob-like scope
    :output_dir: destination of the png curves
    :returns: None

    """
    import xnmt
    print('hello')
    if output_dir is not None:
        for f in glob.glob(scope + "**/*.log.yaml", recursive=True):
            with open(f, 'r') as s:
                doc = yaml.load(s)
            dev_loss = []
            train_loss = []
            mle_loss = []
            aux_loss = []
            for item in doc:
                if 'epoch' in item and 'score' in item:
                    dev_loss += [item['score'].loss]
                if 'data' in item and 'loss' in item:
                    if str(item['epoch']).endswith('.0'):
                        train_loss += [item['loss']]
                if 'loss_name' in item and 'loss' in item:
                    if item['loss_name'] == 'mle':
                        mle_loss += [item['loss']]
                    if item['loss_name'] == 'aux_loss':
                        aux_loss += [item['loss']]

            name = output_dir + os.path.basename(f)
            plt.title(".".join(name.split('.')[1:-3]).split('/')[2])
            plt.xlabel('epochs')
            plt.ylabel('loss')
            plt.plot(dev_loss, label='dev')
            plt.plot(train_loss, label='train')
            plt.legend()
            plt.savefig(f"{name}.png")
            plt.gcf().clear()

            if len(mle_loss) > 0:
                plt.xlabel('epochs')
                plt.ylabel('loss')
                plt.plot(mle_loss, label='mle')
                plt.plot(aux_loss, label='aux')
                plt.legend()
                plt.savefig(f"{name}_mle_aux.png")
                plt.gcf().clear()

# Word length bias stats
# ----------------------
def l(w):
    if w == "</s>":
        return 1.0
    else:
        return float(len(w))
vl = np.vectorize(l)

def print_word_length_att_correl(runs):
    """ Computes average correlation between word length and attention
    for each experiment in the list of runs provided.

    """
    print(runs)
    for run in runs:
        correl_runs = []
        p_value_runs = []
        print(f"\nCollecting run {run}")
        for dl in os.listdir(run):
            d = os.path.join(run, dl)
            if os.path.isdir(d):
                # correls = 0
                # p_values = 0
                # count_att = 0
                src_lengths = []
                s = []
                for att_file in glob.glob(d + "/attention*.txt"):
                    a, src, trg = read_attention_file(att_file)
                    src_lengths.extend(vl(src.strip().split('\t')))
                    s.extend(np.sum(a, axis=0))
                    # correls += correl
                    # p_values += p_value
                    # count_att += 1
                    assert len(src_lengths) == len(s)
                correl, p_value = pearsonr(src_lengths, s)
                # print(count_att)
                correl_runs.append(correl)
                p_value_runs.append(p_value)
                print('.', end='', flush=True)

        print(correl_runs)
        print(p_value_runs)

def draw_len_dist(filename):
    # print(filename)
    lengths = []
    with open(filename, 'r') as f:
        lines = f.readlines()
    for line in lines:
        lengths.extend([len(line.strip().split())])
    # print(lengths[0:5])
    bins = np.arange(0, 15 + 1, 1)
    sns_plot = sns.distplot(lengths, hist=True, kde=True, kde_kws={"bw": 0.3}, rug=False, bins=bins-0.5)
    # sns_plot.set_title(filename)
    sns_plot.set_xlim([0.0,15.0])
    plt.xticks(bins)
    plt.yticks([0,0.05,0.1,0.15,0.2,0.25])
    sns_plot.set_xlabel("sentence length", fontsize=15)
    sns_plot.set_ylabel("density", fontsize=15)
    sns_plot.tick_params(labelsize=10)
    fig = sns_plot.get_figure()
    # format = 'eps', dpi = 1000
    # plt.savefig(f"{filename}.eps", format='eps', dpi=1000)
    # fig.savefig(f"{filename}.png", dpi=1000)
    fig.savefig(f"{filename}.png", dpi=300)
    plt.gcf().clear()

# ----------------------

def main(arguments):
    parser = argparse.ArgumentParser( description=__doc__,
                                      formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('--runs', type=str, nargs="*", help='Run paths, e.g. "./run_01/ ./run_02/"')
    parser.add_argument('-v', '--verbose', action='store_true')
    parser.add_argument('--correl', action='store_true', help='Correlation between word length and attention quantity.')
    parser.add_argument('--segmented', type=str, help='Segmentated file.')
    parser.add_argument('--lendist', action='store_true', help='Plot length distribution for a segmentation.')
    args = parser.parse_args(arguments)

    if args.correl:
        print_word_length_att_correl(args.runs)
    elif args.lendist:
        draw_len_dist(args.segmented)

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))


import MDAnalysis as mda
from MDAnalysis.analysis import distances
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
u=mda.Universe("437d.gro")
rna_p=u.select_atoms('(nucleic or resname GTP) and (name O* or name P* or name N* or name C*) ')
print(rna_p)
n_rna_p = len(rna_p)
print(n_rna_p)
indexy=list(u.select_atoms('(nucleic or resname GTP) and (name O* or name P* or name N* or name C*) ').indices+1)
print(indexy)
final=np.zeros(shape= [n_rna_p,n_rna_p])
for ts in u.trajectory:
	rna_p=u.select_atoms('(nucleic or resname GTP) and (name O* or name P* or name N* or name C*) ')
	n_rna_p = len(rna_p)
##print('LID has {} residues and NMP has {} residues'.format(n_rna_p, n_k))

	dist_arr = distances.distance_array(rna_p.positions,rna_p.positions,box=u.dimensions)
	dist_arr[dist_arr<=3.9]=1
	dist_arr[dist_arr>3.9]=0
	final=final+dist_arr
#print(final)
#print(final.shape)
final=final*0.02

list1 = final.tolist()

for i, j in zip(list1,indexy):
    for k,z in zip(i,indexy):
        print(j,z,k)

#plt.imshow(final, cmap ='Greens')
#plt.show()
##list1 = final.tolist()

##for i, j in zip(list1,indexy):
##    for k,z in zip(i,indexy):
##        print(j,z,k)

#tick_interval=1
#fig, ax = plt.subplots()
#im = ax.imshow(dist_arr, cmap ='Greens')
#ax.set_yticks(np.arange(n_k)[::5])
#ax.set_xticks(np.arange(n_rna_p)[::20])
#ax.set_yticklabels([i for i in range(1,62)])
#ax.set_xticklabels(indexy, rotation='vertical')
#plt.ylabel('rna_o1p')
#plt.xlabel('k')
#plt.title('Distance between k and o2p at 3000')
#cbar = fig.colorbar(im)
#cbar.ax.set_ylabel('Distance (Angstrom)')
#plt.show()


import urllib
import urllib.request
import random
import re
import requests
from bs4 import BeautifulSoup
import time
from datetime import datetime, date
# set IP and proxy
def set_ip():
    # ip list
    iplist = ['45.125.32.181', '125.123.64.234', '58.22.212.39', '171.11.179.77','125.123.126.185','125.123.123.114','58.22.213.30']
    # create an opener
    proxy_support = urllib.request.ProxyHandler({'http': iplist[random.randint(0, len(iplist) - 1)]})
    opener = urllib.request.build_opener(proxy_support)

    # create a disguise header
    opener.addheaders = [('User-Agent',
                          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36')]
    return opener

#requst html
def req_html(opener, url):
    # use opener to request url
    response = opener.open(url)
    # read response and decode it
    html = response.read().decode('utf-8')
    return html

def find_title(html):
    pattern = re.compile(r'
(.*?)
')
    title = pattern.findall(html)
    return title

def find_audratio(html):
    pattern = re.compile(r'class="graph_num">(.*?)')
    audr = pattern.findall(html)
    return audr

def find_episode(html):
    pattern = re.compile(r'(.*?)')
    episode = pattern.findall(html)
    return episode

def find_date(html):
    pattern  = re.compile(r'(.*?)')
    date = pattern.findall(html)
    return date

def find_tvstation(html):
    pattern = re.compile(r'(.*?)')
    tvs = pattern.findall(html)
    return tvs[0]

def find_localtime():
    detail = time.strftime("%Y-%m-%d", time.localtime())
    return detail

def decide_day():
    # Sunday == 0
    # Monday == 1
    # Tuesday == 2
    # Wednesday == 3
    # Thursday == 4
    # Friday == 5
    # Saturday == 6
    str = time.strftime("%w", time.localtime())
    return int(str)

def write_drama_model(lctime, tvs, title, dmdate, epsd, audr, arst, defin):
    lcd = lctime.split("-")
    m = " "
    lend = len(dmdate)
    for i in range(0, lend):
        dmd = dmdate[i].split("/")
        if int(lcd[2])-1 == int(dmd[2]) and i != (lend-1):
            # current audience ratio
            thisepsd = re.findall(r'\d+\.?\d', audr[i])
            thisepsd = float(thisepsd[0])
            # last audience ratio
            lastepsd = re.findall(r'\d+\.?\d', audr[i+1])
            lastepsd = float(lastepsd[0])
            diff = str("%.1f"%(thisepsd-lastepsd))
            if thisepsd - lastepsd > 0:
                diff = "+" + diff
            if defin == 1:
                # current episode
                episode = re.findall(r'\d+',epsd[i])[0]
                m = "第" + episode + "集:" + audr[i] + "(" + diff + ") 、" + m
            else:
                m = audr[i] + "(" + diff + ") 、" + m
        elif int(lcd[2])-1 == int(dmd[2]) and i == (lend-1):
            if defin == 1:
                episode = re.findall(r'\d+', epsd[i])[0]
                m = "第" + episode + "集:" + audr[i] + "（—）、" + m
            else:
                m = audr[i] + "（—）、" + m
            break
        else:
            break
    if defin == 1: m = (tvs + " "+ title + arst + m)
    else: m = (tvs + " " + title + m)
    return m

def cable_drama(weekday):
    drama_name = [['《#请融化我#》','《#他人即地狱#》','《#我的国家#》','《#美丽爱情完美人生#》','《#黄金庭院#》'],
                  ['《#请融化我#》','《#他人即地狱#》'],
                  ['《#伟大的show#》','《#花党：朝鲜婚姻介绍所#》','《#绿豆传#》'],
                  ['《#伟大的show#》','《#花党：朝鲜婚姻介绍所#》','《#绿豆传#》'],
                  ['《#碰巧发现的一天#》','《#奔跑的调查官#》','《#青日电子李小姐#》','《#山茶花开时#》'],
                  ['《#碰巧发现的一天#》','《#奔跑的调查官#》','《#青日电子李小姐#》','《#山茶花开时#》'],
                  ['《#很便宜，千里马超市#》', '《#我的国家#》']]
    drama_kr = [['날녹여주오','타인은지옥이다','나의나라','사랑은뷰티풀인생은원더풀','황금정원'],
                ['날녹여주오','타인은지옥이다'],
                ['위대한쇼','조선혼담공작소꽃파당','녹도'],
                ['위대한쇼','조선혼담공작소꽃파당','녹도'],
                ['어쩌다발견한하루','달리는조사관','청일전자미쓰리','동백꽃필무렵'],
                ['어쩌다발견한하루','달리는조사관','청일전자미쓰리','동백꽃필무렵'],
                ['쌉니다천리마마트','나의나라']]
    drama_arst = [['（主演：#池昌旭#、#尹世雅#）','（主演：#任时完#、#李栋旭#）','（主演：#梁世宗#、#禹棹焕#、#张赫#）',
                   '（主演：#金宰英#、#吴珉锡#、#尹博#、#薛仁雅#、#赵允熙#、#金美淑#）',
                   '（主演：#韩智慧#、#李尚禹#、#吴智恩#、#李泰成#）'],
                  ['（主演：#池昌旭#、#尹世雅#）',
                   '（主演：#任时完#、#李栋旭#）'],
                  ['（主演：#宋承宪#、#李先彬#）',
                   '（主演：#金旻载#、#孔升妍#、#朴志训# ）',
                   '（主演：#金所泫#、#张东允#、#宋建熙#）'],
                  ['（主演：#宋承宪#、#李先彬#）',
                   '（主演：#金旻载#、#孔升妍#、#朴志训# ）',
                   '（主演：#金所泫#、#张东允#、#宋建熙#）'],
                  ['（主演：#金惠允#、#金路云#、#李在旭#、#郑乾柱#、#金英大#、#李泰利#、#金智仁#）',
                   '（主演：#李枖原#、#崔奎华#）',
                   '（主演：#李惠利#、#金相庆#）',
                   '（主演：#孔晓振#、#姜河那#）', ],
                  ['（主演：#金惠允#、#金路云#、#李在旭#、#郑乾柱#、#金英大#、#李泰利#、#金智仁#）',
                   '（主演：#李枖原#、#崔奎华#）',
                   '（主演：#李惠利#、#金相庆#）',
                   '（主演：#孔晓振#、#姜河那#）',],
                  ['（主演：#李东辉#、#金炳哲#、#郑惠成#、#朴浩山#、#李顺载#）',
                   '（主演：#梁世宗#、#禹棹焕#、#张赫#）']]
    lend = len(drama_name[weekday])
    lctime = find_localtime()
    print(lctime[5:7]+"月"+str(int(lctime[8:10])-1)+"日电视剧收视率")
    for i in range(0, lend):
        url = 'https://search.naver.com/search.naver?sm=top_hty&fbm=1&ie=utf8&query='
        u = url + urllib.parse.quote(drama_kr[weekday][i])
        opener = set_ip()
        html = req_html(opener, u)
        tvs = find_tvstation(html)
        title = find_title(html)
        dmdate = find_date(html)
        epsd = find_episode(html)
        audr = find_audratio(html)
        print(write_drama_model(lctime, tvs, drama_name[weekday][i], dmdate, epsd, audr, drama_arst[weekday][i], 1)[:-2])

def cable_variety(weekday):
    variety_name = [['《#自然而然#》','《#认识的哥哥#》','《#只因工作才见面#》','《#喂狗粮的男人#》',
                     '《#惊人的星期六#》'],
                    ['《#街头大胃王2#》','《#Camping Club#》'],
                    ['《#拜托冰箱#》','《#眼神交流#》','《#冤大头的排行榜#》'],
                    ['《#最佳的一击#》','《#Idol Room#》','《#You Quiz On The Block2#》'],
                    ['《#请给我一顿饭#》','《#Rewind#》','《#星期三是音乐节目#》'],
                    ['《#西伯利亚先遣队#》','《#Queendom#》','《#Love catcher2#》','《#signhere#》'],
                    ['《柳熙烈的sketchbook》','《#begin again 3#》','《#被歌声迷住#》','《#三时三餐山村篇#》','《#新西游记外传：三时三餐-冰岛三餐#》']]
    variety_kr = [['자연스럽게','아는형님','일로만난사이','개밥주는남자','놀라운토요일도레미마켓'],
                  ['스트이트푸드파이터2','캠핑클럽'],
                  ['냉장고부탁해','아이콘택트','호구의차트'],
                  ['최고의한방','아이돌룸','유퀴즈온더블럭2'],
                  ['한끼줍쇼','리와인드','수요일은음악프로'],
                  ['시베리아신발대','퀸덤','러브캐처2','사인히어'],
                  ['유희열의스케치북','비긴어게인3','노래에반하가','삼시세끼산촌편','신서유기외전삼시세끼']]
    lend = len(variety_name[weekday])
    lctime = find_localtime()
    print(lctime)
    print(lctime[5:7] + "月" + str(int(lctime[8:10])-1) + "日综艺收视率")
    for i in range(0, lend):
        url = 'https://search.naver.com/search.naver?sm=top_hty&fbm=1&ie=utf8&query='
        u = url + urllib.parse.quote(variety_kr[weekday][i])
        opener = set_ip()
        html = req_html(opener, u)
        tvs = find_tvstation(html)
        title = find_title(html)
        dmdate = find_date(html)
        #epsd = find_episode(html)
        # print(epsd)
        audr = find_audratio(html)
        # print(audr)
        print(write_drama_model(lctime, tvs, variety_name[weekday][i], dmdate, tvs, audr, variety_name[weekday][i], 0)[:-2])

if __name__ == '__main__' :
    weekday = decide_day()
    cable_drama(weekday)
    print()
    cable_variety(weekday)


"""The object returned by the `--settings` option."""
from __future__ import annotations

from typing import Any, Dict

import attrs

from civ4save.vanilla import enums as e

@attrs.define(slots=True)
class Settings:
    """Class representing the game's settings (sp only)."""

    # CvInitCore
    game_type: e.GameType
    game_name: str
    map_script: str
    """Name of the map script"""
    world_size: e.WorldType
    climate: e.ClimateType
    sea_level: e.SeaLevelType
    start_era: e.EraType
    game_speed: e.GameSpeedType
    game_options: Dict[str, bool]
    """Each GameOption and whether it's enabled"""
    max_turns: int
    advanced_start_points: int
    victories: Dict[str, bool]
    """Each VictoryType and whether it's enabled"""
    num_civs: int
    """Number of civs (not counting barbs)"""

    # CvGame
    start_turn: int
    start_year: int
    handicap: e.HandicapType
    map_random_seed: int
    soren_random_seed: int
    culture_victory_cities: int
    culture_victory_level: e.CultureLevelType

    # CvMap
    grid_width: int
    grid_height: int
    wrap_x: bool
    wrap_y: bool

    @classmethod
    def from_struct(cls, data: Any) -> Settings:
        """Return `Settings` from the parsed struct."""
        game_type = e.GameType[data.game_type]
        game_speed = e.GameSpeedType[data.game_speed]
        world_size = e.WorldType[data.world_size]
        climate = e.ClimateType[data.climate]
        sea_level = e.SeaLevelType[data.sea_level]
        start_era = e.EraType[data.start_era]

        handicap = e.HandicapType[data.handicap]
        culture_victory_level = e.CultureLevelType[data.culture_victory_level]

        game_options = {
            e.GameOptionType(n).name: v for n, v in enumerate(data.game_options)
        }

        advanced_start_points = 0
        if game_options["GAMEOPTION_ADVANCED_START"]:
            advanced_start_points = data.advanced_start_points

        victories = {e.VictoryType(n).name: v for n, v in enumerate(data.victories)}

        num_civs = len([c for c in data.civs[:-1] if c != "NO_CIVILIZATION"])

        return cls(
            game_type=game_type,
            game_name=data.game_name,
            map_script=data.map_script_name,
            world_size=world_size,
            climate=climate,
            sea_level=sea_level,
            start_era=start_era,
            game_speed=game_speed,
            game_options=game_options,
            max_turns=data.max_turns,
            advanced_start_points=advanced_start_points,
            victories=victories,
            num_civs=num_civs,
            start_turn=data.start_turn,
            start_year=data.start_year,
            handicap=handicap,
            map_random_seed=data.map_random_seed,
            soren_random_seed=data.soren_random_seed,
            culture_victory_cities=data.num_culture_victory_cities,
            culture_victory_level=culture_victory_level,
            grid_width=data.grid_width,
            grid_height=data.grid_height,
            wrap_x=data.wrap_x,
            wrap_y=data.wrap_y,
        )


from flask import Flask, render_template, redirect, request, session
from hashlib import md5

from __main__ import app
from utils import *

#TODO shouldn't need to define constants twice...
ROLE_USER = 1
ROLE_ADMIN = 2
ROLE_TEACHER = 3

@app.route('/eventsChoose', methods = ["GET", "POST"])
def eventsChoose():
    if request.method == "POST":
        #Form has been submitted. Check whether they are adding or removing an entry and process accordingly
        if request.form["submit"] == "Add":
            connection = create_connection()
            with connection.cursor() as cursor:
                sql = "INSERT INTO tblEntries(userID, eventID) VALUES (%s, %s);"
                vals = (session["userID"], request.form["eventID"])
                cursor.execute(sql, vals)
                connection.commit()
                connection.close()
        elif  request.form["submit"] == "Remove":
            connection = create_connection()
            with connection.cursor() as cursor:
                sql = "DELETE FROM tblEntries WHERE userID = %s AND eventID = %s;"
                vals = (session["userID"], request.form["eventID"])
                cursor.execute(sql, vals)
                connection.commit()
                connection.close()

    #Display events. Need to pass all events to template, as well as all events that
    #user is currently entered for.
    connection = create_connection()
    with connection.cursor() as cursor:
        sql = "SELECT tblevents.eventID, eventName FROM tblevents;"
        cursor.execute(sql)
        events = cursor.fetchall()
        sql = "SELECT eventID, userID FROM tblEntries WHERE userID = %s;"
        vals = session["userID"]
        cursor.execute(sql, vals)
        enteredEvents = cursor.fetchall()
        connection.close()

    return render_template("eventsChoose.html", events = events, enteredEvents = enteredEvents)

@app.route('/eventsTickCompleted', methods = ["GET", "POST"])
def eventsTickCompleted():
    if request.method == "POST":
        connection = create_connection()
        with connection.cursor() as cursor:

            #First set all values to False for this user
            sql = "UPDATE tblentries SET completed = False WHERE userID = %s;"
            vals = (session["userID"])
            cursor.execute(sql, vals)
            connection.commit()

            #Then set True for all that were ticked in form
            completedEvents = request.form.getlist("completed")
            for event in completedEvents:
                sql = "UPDATE tblentries SET completed = True WHERE entryID = %s;"
                vals = (event)
                cursor.execute(sql, vals)
                connection.commit()
        connection.close()

    #Display events for this user.
    connection = create_connection()
    with connection.cursor() as cursor:
        sql = "SELECT tblevents.eventID, eventName, completed, entryID FROM tblentries INNER JOIN tblevents ON tblevents.eventID = tblentries.eventID WHERE tblentries.userID = %s;"
        vals = session["userID"]
        cursor.execute(sql, vals)
        enteredEvents = cursor.fetchall()
        connection.close()

    return render_template("eventsTickCompleted.html", enteredEvents = enteredEvents)

@app.route("/eventsAdd", methods=["GET", "POST"])
def eventsAdd():
    #First mke sure we have the rights to do this stuff
    if not session["roleID"] == ROLE_ADMIN:
        return redirect("/")
    else:
        if request.method == "POST":
            #Get values from form, insert into tblEvents.
            eventName = request.form["eventName"]

            connection = create_connection()
            with connection.cursor() as cursor:
                sql = "INSERT INTO tblevents(eventName) VALUES(%s)"
                vals = (eventName)
                cursor.execute(sql, vals)
                connection.commit()
                connection.close()

        return render_template("eventsAdd.html")

@app.route("/eventEdit", methods=["GET", "POST"])
def eventEdit():
    #First mke sure we have the rights to do this stuff
    if not session["roleID"] == ROLE_ADMIN:
        return redirect("/")
    else:
        if request.method == "POST":
            eventID = request.form["eventID"]
            eventName = request.form["eventName"]
            connection = create_connection()
            with connection.cursor() as cursor:
                sql = "UPDATE tblevents SET eventName=%s WHERE eventID=%s"
                vals = (eventName, eventID)
                cursor.execute(sql, vals)
                connection.commit()
                connection.close()
            return redirect("/events")
        else:
            eventID = request.args["eventID"]
            connection = create_connection()
            with connection.cursor() as cursor:
                sql  = "SELECT eventID, eventName FROM tblevents WHERE eventID = %s;"
                vals = (eventID)
                cursor.execute(sql, vals)
                event = cursor.fetchone()
                connection.close()
            return render_template("eventEdit.html", event = event)

@app.route("/events")
def events():
    if session["roleID"] == ROLE_ADMIN:
        connection = create_connection()
        with connection.cursor() as cursor:
            sql  = "SELECT eventID, eventName FROM tblEvents;"
            cursor.execute(sql)
            events = cursor.fetchall()
            connection.close()
        return render_template("events.html", events = events)
    else:
        return redirect("/")


import pandas as pd
import numpy as np
import string
from sklearn.preprocessing import LabelEncoder
import tensorflow as tf
import random

## data set
data = {
    "intents" :
    [
        {
            "tag" : "greeting",
            "inputs" : [
                "hello",
                "hi"
            ],
            "responses" : [
                "Hello There"
            ]
        },
        {
            "tag" : "goodbye",
            "inputs" : [
                "Goodbye"
            ],
            "responses" : [
                "Goodbye!"
            ]
        },
        {
            "tag" : "greeting-jp",
            "inputs" : [
            "おはようございます"
            ],
            "responses" : [
            "おはようございます"
            ]
        }
    ]
}

tags = []
inputs = []
responses = {}
for intent in data['intents']:
    responses[intent['tag']] = intent['responses']
    for lines in intent['inputs']:
        inputs.append(lines)
        tags.append(intent['tag'])

df = pd.DataFrame(
    {
        "inputs" : inputs,
        "tags" : tags
    }
)

df = df.sample(frac=1)

## data pre processing
df['inputs'] = df['inputs'].apply(lambda wrd:[ltrs.lower() for ltrs in wrd if ltrs not in string.punctuation])
df['inputs'] = df['inputs'].apply(lambda wrd: ''.join(wrd))

tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=2000)
tokenizer.fit_on_texts(df['inputs'])
train = tokenizer.texts_to_sequences(df['inputs'])
X_train = tf.keras.preprocessing.sequence.pad_sequences(train)

## encode outputs to binary for the machine to understand
encoder = LabelEncoder()
y_train = encoder.fit_transform(df['tags'])

input_shape = X_train.shape[1]

model = tf.keras.Sequential([
    tf.keras.layers.Input(shape=(input_shape,)),
    tf.keras.layers.Dense(64, activation = 'relu'),
    tf.keras.layers.Dense(64, activation  ='relu'),
    tf.keras.layers.Dense(1, activation = 'sigmoid')
  ]
)

initial_learning_rate = 0.06
decay_steps = 1000
decay_rate = 0.9

learning_rate_schedule = tf.optimizers.schedules.ExponentialDecay(initial_learning_rate, decay_steps, decay_rate)

optimiser = tf.keras.optimizers.Adam(learning_rate = learning_rate_schedule)

model.compile(optimizer = optimiser, loss = 'binary_crossentropy', metrics=['accuracy'])

model.fit(X_train, y_train, epochs=200)

while True:
    texts_p = []
    prediction_input = input('You: ')

    prediction_input = [letters.lower() for letters in prediction_input if letters not in string.punctuation]
    prediction_input = ''.join(prediction_input)
    texts_p.append(prediction_input)

    prediction_input = tokenizer.texts_to_sequences(texts_p)
    prediction_input = np.array(prediction_input).reshape(-1)
    prediction_input = tf.keras.preprocessing.sequence.pad_sequences([prediction_input], input_shape)

    output = model.predict(prediction_input)
    output = output.argmax()

    response_tag = encoder.inverse_transform([output])[0]
    bot_name = 'bob'

    print(f'{bot_name}: ', random.choice(responses[response_tag]))
    if response_tag == "goodbye":
        break


import cv2
import numpy as np

if __name__ == "__main__":
    # Get video from webcam
    cap = cv2.VideoCapture(0)
    # Define cascades
    face_cascade = cv2.CascadeClassifier("haarcascade_frontalface_default.xml")
    eye_casacade = cv2.CascadeClassifier("haarcascade_eye.xml")

    # Infinite loop for processing image
    while True:
        ret, img = cap.read()
        # Convert image to gray
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        # Find faces
        faces = face_cascade.detectMultiScale(gray, 1.3, 5)

        # Draw faces on the image
        for (x, y, w, h) in faces:
            cv2.rectangle(img, (x, y), (x+w, x+h), (255, 255, 255), 2)
            # Find eyes on the face
            roi_gray = gray[y:y+h, x:x+w]
            roi_color = img[y:y+h, x:x+w]
            eyes = eye_casacade.detectMultiScale(roi_gray)
            for (ex, ey, ew, eh) in eyes:
                # Draw rectangles near eyes
                cv2.rectangle(roi_color, (ex, ey), (ex+ew, ey+eh), (0,0,255), 1)

        # Show images
        cv2.imshow("Detect face and eyes", img)

        k = cv2.waitKey(30) & 0xff
        if k == 27:
            break

    cap.release()
    cv2.destroyAllWindows()


from django.urls import path

from . import views

app_name = 'landingpage'

urlpatterns=[
    path('',views.home, name='home'),
    path('page/coffee',views.coffee, name='coffee'),
    path('page/tea',views.tea, name='tea'),
    path('page/milk',views.milk, name='milk')
]


from flask import Flask, render_template, request
import pandas as pd

app = Flask(__name__)

@app.route('/')
def home():
    return render_template("index.html")
@app.route("/data", methods=["POST","GET"])
def index():
    d={"Q No.":[],"Answers":[]}
    if request.method=="POST":
        for i,j in request.form.items():
            d["Q No."].append(i)
            d["Answers"].append(j)
        df=pd.DataFrame(d)
        print(df)
        df.to_excel("D:/SVCE_RESULTS.xlsx")
        return d
    else:
        k=request.args.get("n1")
        return k+"k"
if __name__=="__main__":
    app.run(debug=True)


import json
import os
import threading

COLUMN_ALIASES_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), 'column_aliases.json'))

class ColumnAliases(object):
    """Singleton class that contains the set of known column aliases"""
    __lock = threading.Lock()
    __instance = None

    def __init__(self):
        # Load columns
        with open(COLUMN_ALIASES_FILE, 'r') as f:
            self.columns = json.load(f)

        # Load column map and set default type (string)
        self.column_map = {}
        for col in self.columns:
            if 'group' in col:
                self.column_map[col['name']] = col['group']
            else:
                if 'type' not in col:
                    col['type'] = 'string'
                self.column_map[col['name']] = (col['src'], col.get('type'), col.get('expr'))

    @classmethod
    def instance(cls):
        """Get the singleton instance of ColumnAliases"""
        # Double check lock for singleton instance
        if not cls.__instance:
            with cls.__lock:
                if not cls.__instance:
                    cls.__instance = ColumnAliases()

        return cls.__instance

    @classmethod
    def get_columns(cls, include_hidden=False):
        """Get the full list of column aliases, including type and description"""
        inst = cls.instance()
        if include_hidden:
            return inst.columns
        return [ col for col in inst.columns if not col.get('hidden') ]

    @classmethod
    def get_column_alias(cls, key):
        """Lookup the alias for the given key.

        Arguments:
            key (str): The column key

        Returns:
            tuple: The aliased (key, datatype), group list, or the original (key, None) if there was no alias.
        """
        inst = cls.instance()
        if key in inst.column_map:
            return inst.column_map[key]
        return (key, None, None)


# coding: utf-8

"""
    Fatture in Cloud API v2 - API Reference

    Connect your software with Fatture in Cloud, the invoicing platform chosen by more than 500.000 businesses in Italy.   The Fatture in Cloud API is based on REST, and makes possible to interact with the user related data prior authorization via OAuth2 protocol.

    The version of the OpenAPI document: 2.0.30
    Contact: info@fattureincloud.it
    Generated by OpenAPI Generator (https://openapi-generator.tech)

    Do not edit the class manually.
"""  # noqa: E501

from __future__ import annotations
import pprint
import re  # noqa: F401
import json

from typing import Optional, Union
from pydantic import BaseModel, Field, StrictBool, StrictFloat, StrictInt, StrictStr

class VatType(BaseModel):
    """
    VatType
    """

    id: Optional[StrictInt] = Field(None, description="Vat type id")
    value: Optional[Union[StrictFloat, StrictInt]] = Field(
        None, description="[Read Only] Vat type percentual value"
    )
    description: Optional[StrictStr] = Field(
        None, description="Vat type short description"
    )
    notes: Optional[StrictStr] = Field(
        None, description="Vat type notes shown in documents"
    )
    e_invoice: Optional[StrictBool] = Field(
        None, description="Vat type is usable for e-invoices"
    )
    ei_type: Optional[StrictStr] = Field(
        None, description="Vat type e-invoice type (natura)"
    )
    ei_description: Optional[StrictStr] = Field(
        None, description="Vat type e-invoice description"
    )
    editable: Optional[StrictBool] = Field(
        None, description="[Read Only] Is the vat type is editable."
    )
    is_disabled: Optional[StrictBool] = Field(
        None, description="Is the vat type disabled"
    )
    __properties = [
        "id",
        "value",
        "description",
        "notes",
        "e_invoice",
        "ei_type",
        "ei_description",
        "editable",
        "is_disabled",
    ]

    class Config:
        """Pydantic configuration"""

        allow_population_by_field_name = True
        validate_assignment = True

    def to_str(self) -> str:
        """Returns the string representation of the model using alias"""
        return pprint.pformat(self.dict(by_alias=True))

    def to_json(self) -> str:
        """Returns the JSON representation of the model using alias"""
        return json.dumps(self.to_dict())

    @classmethod
    def from_json(cls, json_str: str) -> VatType:
        """Create an instance of VatType from a JSON string"""
        return cls.from_dict(json.loads(json_str))

    def to_dict(self):
        """Returns the dictionary representation of the model using alias"""
        _dict = self.dict(
            by_alias=True,
            exclude={
                "editable",
            },
            exclude_none=True,
        )
        return _dict

    @classmethod
    def from_dict(cls, obj: dict) -> VatType:
        """Create an instance of VatType from a dict"""
        if obj is None:
            return None

        if not isinstance(obj, dict):
            return VatType.parse_obj(obj)

        _obj = VatType.parse_obj(
            {
                "id": obj.get("id") if obj.get("id") is not None else None,
                "value": float(obj.get("value"))
                if obj.get("value") is not None
                else None,
                "description": obj.get("description")
                if obj.get("description") is not None
                else None,
                "notes": obj.get("notes") if obj.get("notes") is not None else None,
                "e_invoice": obj.get("e_invoice")
                if obj.get("e_invoice") is not None
                else None,
                "ei_type": obj.get("ei_type")
                if obj.get("ei_type") is not None
                else None,
                "ei_description": obj.get("ei_description")
                if obj.get("ei_description") is not None
                else None,
                "editable": obj.get("editable")
                if obj.get("editable") is not None
                else None,
                "is_disabled": obj.get("is_disabled")
                if obj.get("is_disabled") is not None
                else None,
            }
        )
        return _obj