import asyncio
import functools
import time
from colorama import Fore
from XAgentServer.exts.exception_ext import XAgentTimeoutError, XAgentCloseError

from inputimeout import inputimeout, TimeoutOccurred
from XAgentServer.application.global_val import redis
import math

def timer(func):
    """
    Decorator function to time the execution of a function.

    Args:
        func (Function): The function to be timed.

    Returns:
        wrapper (Function): The wrapped function with added timing functionality.
    """
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        try:
            start_time = time.time()
            result = func(*args, **kwargs)
            end_time = time.time()
        except:
            pass
    return wrapper

class CommandLineInput:
    """
    Class for handling command line input.

    This child class extends from BaseInput and implements methods to handle and manage command line input data.

    Attributes:
        do_interrupt (bool): If True, input will be interrupted.
        max_wait_seconds (int): Maximum wait time for input in seconds.
    """
    def __init__(self,
                 do_interrupt: bool = False,
                 max_wait_seconds: int = 600,
                 logger=None):
        self.do_interrupt = do_interrupt
        self.max_wait_seconds = max_wait_seconds
        self.logger = logger

    def run(self, input_data):
        """
        Run the command line input method.

        Args:
            input_data (Any): The original input data to be processed.

        Returns:
            data (Any): The processed input data.
        """
        if self.do_interrupt:
            data = self.interrupt(input_data)
        else:
            data = input_data
        return data

    def get_each_input(self, key, value, res, timeout):
        """
        Returns the input from the command line for a single key-value pair.

        Args:
            key (str): The key for which to get input.
            value (Any): The current value associated with the key.
            res (dict): The result dictionary where inputs collected will be stored.
            timeout (int): Timeout in seconds for the input.

        Returns:
            Any: The input data.
        """
        self.logger.typewriter_log(
            f"Now, ASK For {key}, Origin Input: {value}",
            Fore.RED,
            f""
        )
        self.logger.typewriter_log(
            f"Now, you can modify the current field by entering some information, and then press 'Enter' to continue, if you want to keep the original input, please enter '-1' and then press 'Enter':",
            Fore.GREEN
        )
        temp = inputimeout(prompt=f'You have {timeout} seconds to input:\n', timeout=timeout)
        if temp == "-1":
            return value
        else:
            return temp

    def get_input(self, origin_data):
        """
        Get input for all fields of the original data from the command line.

        Args:
            origin_data (dict): The original data for which to get input.

        Returns:
            dict: The dictionary with updated inputs.
        """
        self.logger.typewriter_log(
                "Next, you can start modifying the original input by typing 'Y/y/yes' or skip this step by typing 'N/n/no' and then press 'Enter' to continue the loop:",
                Fore.RED
            )
        update = inputimeout(prompt=f'You have to make a decision within 60 seconds:\n', timeout=60)
        res = {"args": {}}
        if update in ['y', 'Y', 'yes']:
            execute_time = self.max_wait_seconds
            if isinstance(origin_data, dict):
                args = origin_data.get("args", "")
                self.logger.typewriter_log(
                    f"Next, you will have a total of {self.max_wait_seconds} seconds to modify each option:",
                    Fore.RED,
                )
                for key, value in args.items():
                    if key == "done":
                        res[key] = False
                        continue
                    start_time = time.time()
                    res["args"][key] = self.get_each_input(key, value, res, execute_time)
                    end_time = time.time()
                    execute_time = math.floor(execute_time - (end_time - start_time))
            self.logger.info(f"modify the input, receive the data: {res}")
        else:
            res = origin_data
            self.logger.info("skip this step")
        self.logger.info("continue the loop")
        res["done"] = True
        return res

    def interrupt(self, input_data):
        """
        Interrupts the current input process and returns the current data.

        Args:
            input_data (dict): The original input data.

        Returns:
            dict: The current data collected so far.

        Raises:
            XAgentIOTimeoutError: If the input times out.
        """
        try:
            data = self.get_input(input_data)
            return data
        except TimeoutOccurred:
            self.logger.error(f"Waiting timemout, close connection!")
            raise XAgentTimeoutError("timeout!")


# -*- coding: utf-8 -*-
"""Loading a pre-trained model from disk.

Learn how to load a pre-trained model from disk. We will classify individual images from the
Animals dataset and then display the classified images to our screen.

Example:
    $ python shallownet_load.py --dataset ../datasets/animals --model shallownet_weights.hdf5

Attributes:
    dataset (str):
        The path to where our input image dataset resides on disk.
    model (str):
        The path to the pre-trained model.
"""
import argparse
import cv2
import numpy as np
from keras.models import load_model
from imutils import paths
from pyimagesearch.preprocessing import ImageToArrayPreprocessor
from pyimagesearch.preprocessing import SimplePreprocessor
from pyimagesearch.datasets import SimpleDatasetLoader

def main():
    """Load pre-trained model from disk
    """
    # construct the argument parse and parse the arguments
    args = argparse.ArgumentParser()
    args.add_argument("-d", "--dataset", required=True, help="path to input dataset")
    args.add_argument("-m", "--model", required=True, help="path to pre-trained model")
    args = vars(args.parse_args())

    # initialize the class labels
    class_labels = ["cat", "dog", "panda"]

    # grab the list of images in the dataset then randomly sample indexes into the image paths list
    print("[INFO] sampling images...")
    image_paths = np.array(list(paths.list_images(args["dataset"])))
    idxs = np.random.randint(0, len(image_paths), size=(10,))
    image_paths = image_paths[idxs]

    # initialize the image preprocessors
    simple_preprocessor = SimplePreprocessor(32, 32)
    image_to_array_preprocessor = ImageToArrayPreprocessor()

    # load the dataset from disk then scale the raw pixel intensities to the range [0, 1]
    dataset_loader = SimpleDatasetLoader(preprocessors=[simple_preprocessor, image_to_array_preprocessor])
    (data, _) = dataset_loader.load(image_paths)
    data = data.astype("float") / 255.0

    # load the pre-trained network
    print("[INFO] loading pre-trained network...")
    model = load_model(args["model"])

    # make predictions on the images
    print("[INFO] predicting...")
    preds = model.predict(data, batch_size=32).argmax(axis=1)
    # loop over the sample images
    for (i, image_path) in enumerate(image_paths):
        # load the example image, draw the prediction, and display it to our screen
        image = cv2.imread(image_path)
        cv2.putText(
            image, "Label: {}".format(class_labels[preds[i]]), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2
        )
        cv2.imshow("Image", image)
        cv2.waitKey(0)

if __name__ == "__main__":
    main()


import sys
import time
import random

import os
import shutil

from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

from_dir = "C:/Users/judit/Downloads"

class FileEventHandler(FileSystemEventHandler):

    def on_created(self, event):
        print(f"¡Oye, {event.src_path} ha sido creado!")

    def on_deleted(self, event):
        print(f"¡Lo siento! ¡Alguien borró {event.src_path}!")

    def on_modified(self, event):
        print(f"¡Ve!, {event.src_path} se modificó")

    def on_moved(self, event):
        print(f"Se movió {event.src_path} a {event.dest_path}")

event_handler = FileEventHandler()

observer = Observer()

observer.schedule(event_handler, from_dir, recursive=True)

observer.start()

try:
    while True:
        time.sleep(2)
        print("ejecutando...")
except KeyboardInterrupt:
    print("¡detenido!")
    observer.stop()


from django.contrib import admin

# Register your models here.
from .models import Exam, AttemptExam
# from .models import ExamQuestions

class ExamAdminConfig(admin.ModelAdmin):
    list_display = ('name','number_of_questions','duration','course','teacher')
    list_filter = ('name','duration','teacher')
    search_fields = ('name','teacher','course','duration')
    # ordering = ('-start_date',)

    fieldsets = (
        (None, {
            'fields':('name','description','number_of_questions','duration')
            }),
        ('Details', {
            'fields':('course','teacher','student')
            }),
    )

    add_fieldsets = (
        (None, {
            'classes': ('wide',),
            'fields':('full_name','email','department','qualification','password1','password2','profile_img') # for now we cannot add a username to a student while adding a student. add the username when editing an existing user.
            }),
        ('Permissions', {
            'classes': ('wide',),
            'fields':('is_staff','is_superuser','is_active')
            }),
        ('Groups', {
            'classes': ('wide',),
            'fields':('groups','user_permissions')
            })
    )

admin.site.register(Exam, ExamAdminConfig) # , ExamAdminConfig
# admin.site.register(ExamQuestions)
# admin.site.register(AttemptExam)


import os
import sys
import traceback
import uuid
import time
import copy
from .requester import Requester
from .resources import Resources, Resource
from .writer import Writer
from .window import Window
from .asset import Asset
from .threads import Threader
from .pages import Pages, Page
from .stripper import Stripper
from urlparse import urlparse, urljoin
from threading import *

class Generic(object):
    pass

#TODO NEED TO TREAT DATA LIKE SCRIPTS and replace references

class Ripper(object):

    def __init__(self, logger=None, threads=4):
        self.logger = logger
        self.threads = threads
        self.stats = {}
        self.assets = []
        self.errors = []
        self.pages = []
        self.is_threading = False
        self.extension = ".html"
        self.totals = {"pages":1,"assets":0, "pages_down":0, "assets_down":0}
        self.lock = Lock()

    def rip(self,url,base, top_level_pages=True):
        self.url = url
        self.base = base
        self.index_file = os.path.join(base,"index" + self.extension)
        self.stats = {}
        self.assets = []
        self.pages = []

        index = self._get_page(self.url);
        self._update_totals("pages_down",1)

        if not index:
            raise ValueError('Could not access website. Is the URL correct?')

        self.pages = self._get_page_links(self.url,index)

        if top_level_pages:

            self._update_totals("pages",len(self.pages))

            for p in self.pages:
                if not p.exists:
                    if p.replace_reference:
                        content = self._get_page(p.url);
                        if not content:
                            content = "<html></html>"
                        else:
                            content = self._update_page_links(content,self.extension)
                            pages = self._get_page_links(self.url,content)
                            content = self._update_page_links(content,self.extension)
                            content = self._remove_page_links(content,pages)
                            content = self._remove_trackers(content)
                            p.downloaded = True

                        Writer.write(content,os.path.join(self.base,p.name + self.extension))
                    self.logger(self,p)
                    self._update_totals("pages_down",1)

            index = self._update_page_links(index,self.extension)
        else:
            index = self._remove_page_links(index,self.pages,False)

        index = self._remove_trackers(index)
        Writer.write(index,self.index_file)
        self.logger(self,Page(urlparse(self.url).path,self.url,"index",False))

    def _get_page(self,url,relative_assets=False):

        requester = Requester()
        page = requester.get_source(url)

        if page:

            content = page.text
            marker = str(uuid.uuid4())

            threader = False
            if not self.is_threading:
                threader = Threader(self.threads,self._log_asset)
                threader.start(True)
                self.is_threading = True

            resources = Resources(page)
            self._update_totals("assets",self._get_total_assets_to_download(resources))

            for r in resources:

                if threader:
                    threader.add((self._get_asset,{"resource" : r, "marker" : marker}))
                else:
                    asset = self._get_asset(r,marker)
                    self._log_asset(asset)

            if threader:
                threader.finish()
                if threader.errors:
                    self.errors += threader.errors

            return self._update_page_assets(content,marker,relative_assets)
        else:
            return False

    def _get_page_links(self,url,content):
        pages = []
        for p in Pages(url,content):
            if p.reference in map(lambda x: x.url, self.assets):
                p.replace_reference = False
            pages.append(p)
        return pages

    def _get_asset(self,resource,marker):
        requester = Requester()
        asset_exists = self._find_asset_by_reference(resource,marker)
        if asset_exists:
            return asset_exists
        else:
            asset = Asset(self.base,resource,marker)
            if asset.mime:
                x = requester.get_stream(asset.source.url) if asset.mime.stream else self._get_page(asset.source.url,False if asset.mime.category == "scripts" else True)
                if x:
                    Writer.write(x,asset.path)
                    asset.downloaded = True
            return asset

    def _find_asset_by_reference(self,resource,marker):
        find = self._check_asset_exists(resource)
        if find:
            asset = copy.deepcopy(find)
            asset.resource = resource
            asset.marker = marker
            asset.existing = True
            return asset
        else:
            return False

    def _check_asset_exists(self,resource):
        find = filter(lambda a: a.source.reference == resource.reference or a.source.url == resource.url,list(self.assets))
        if len(find) > 0:
            return find[0]
        else:
            return False

    def _log_asset(self,asset):
        if asset.downloaded:
            self._update_stats(asset.mime)
            if not asset.existing:
                self._update_totals("assets_down",1)
        self.assets.append(asset)
        self.logger(self,asset)

    def _update_page_assets(self,content,marker,relative_assets):
        for asset in self.assets:
            if asset.downloaded == True and asset.marker == marker:
                content = self._find_and_replace(content,asset.source.reference,asset.relative_url if relative_assets else asset.url)
        return content

    def _update_page_links(self,content,extension):
        for page in self.pages:
            if page.replace_reference:
                wrap = "{0}{1}{0}"
                content = self._find_and_replace(content,wrap.format('"',page.reference),wrap.format('"',page.name + extension))
                content = self._find_and_replace(content,wrap.format("'",page.reference),wrap.format("'",page.name + extension))
        return content

    def _remove_page_links(self,content,pages,check_pages=True):

        for page in pages:
            if len(filter(lambda p: p.name == page.name,self.pages)) == 0 or not check_pages:
                if page.replace_reference:
                    wrap = "{0}{1}{0}"
                    content = self._find_and_replace(content,wrap.format('"',page.reference),wrap.format('"#',page.name))
                    content = self._find_and_replace(content,wrap.format("'",page.reference),wrap.format("'#",page.name))
        return content

    def _remove_trackers(self, content):
        return Stripper.Strip(content)

    def _find_and_replace(self,text,find,replace):
        text = text.replace(find,replace)
        return text

    def _update_stats(self,mime):
        if mime.category in self.stats:
            self.stats[mime.category] += 1
        else:
            self.stats[mime.category] = 1

    def _update_totals(self,key,value):
        with self.lock:
            self.totals[key] += value

    def _get_total_assets_to_download(self,resources):
        #TODO this needs to compare both the reference and the url just like the _find_asset_by_reference
        return len(list(filter(lambda r: self._check_asset_exists(r),list(resources))))
        #return len(list(filter(lambda r: r.reference not in map(lambda a: a.source.reference,list(self.assets)),list(resources._resources))))


import os
import flask
from flask import Flask, render_template, request, redirect
from src.components.inference import  Inference

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import nltk
from string import punctuation
import re
from nltk.corpus import stopwords
from flask import jsonify

nltk.download('stopwords')

set(stopwords.words('english'))

app = Flask(__name__, template_folder='template')

inferencing = Inference()

@app.route('/')
def home():
    return render_template('home.html')

@app.get("/image_classifier")
def image_classifier():
    return render_template("./image_classifier.html")

@app.get("/sentiment_analysis")
def sentiment_analysis():
    return render_template("./sentiment_analysis.html")

@app.route('/image_classifier', methods=['GET', 'POST'])
def upload_file():
    if request.method == 'POST':
        if 'file' not in request.files:
            return redirect(request.url)
        file = request.files.get('file')
        if not file:
            return
        img_bytes = file.read()
        class_id, class_name = inferencing.predict_class(image=img_bytes)
        return jsonify(class_name=class_name)
    return render_template('./home.html')

@app.route('/sentiment_analysis', methods=['POST'])
def my_sentiment_analysis_post():
    stop_words = stopwords.words('english')

    text1 = request.form['text1'].lower()

    text_final = ''.join(c for c in text1 if not c.isdigit())

    processed_doc1 = ' '.join([word for word in text_final.split() if word not in stop_words])

    sa = SentimentIntensityAnalyzer()
    dd = sa.polarity_scores(text=processed_doc1)
    compound = round((1 + dd['compound'])/2, 2)

    return render_template('sentiment_analysis.html', final=compound, text1=text_final, text2=dd['pos'], text3=dd['neu'], text4=dd['neg'])

if __name__ == '__main__':
    app.run(debug=True, port=int(os.environ.get('PORT', 5000)))


import PyPDF2
from pathlib import Path
from gtts import gTTS
from googletrans import Translator

def define_language(text):
    """Defines language received text and return it"""
    translator = Translator()
    language = translator.detect(text).lang
    return language

def pdf_to_mp3(pdf_file_path, output_dir_path):
    """Extract text from pdf file and convert it to mp3 file. Return output audiofile path or None in case if passed
    incorrect file path or file is not pdf."""
    if Path(pdf_file_path).is_file() and Path(pdf_file_path).suffix == '.pdf':
        with open(pdf_file_path, 'rb') as pdf:
            reader = PyPDF2.PdfFileReader(pdf, strict=False)
            all_pages = [page.extract_text() for page in reader.pages]
        text_from_pdf = ''.join(all_pages).replace('\n', '')  # line breaks are deleted to sound without long pauses

        output_audio = gTTS(text=text_from_pdf, lang=define_language(text_from_pdf))
        mp3_file_name = f'{Path(pdf_file_path).stem}.mp3'
        output_audio.save(f'{output_dir_path}/{mp3_file_name}')

        return mp3_file_name

pdf_to_mp3('uploads/Test_PDF.pdf', 'downloads')


import os
from collections import defaultdict

from django.apps import apps
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.db import models

class Command(BaseCommand):
    help = "Prints a list of all files in MEDIA_ROOT that are not referenced in the database."

    def handle(self, *args, **options):
        if not getattr(settings, 'MEDIA_ROOT'):
            raise CommandError("MEDIA_ROOT is not set, nothing to do")

        # Get a list of all files under MEDIA_ROOT
        media = set()
        for root, dirs, files in os.walk(settings.MEDIA_ROOT):
            for f in files:
                media.add(os.path.abspath(os.path.join(root, f)))

        # Get list of all fields (value) for each model (key)
        # that is a FileField or subclass of a FileField
        model_dict = defaultdict(list)
        for model in apps.get_models():
            for field in model._meta.fields:
                if issubclass(field.__class__, models.FileField):
                    model_dict[model].append(field)

        # Get a list of all files referenced in the database
        referenced = set()
        for model in model_dict:
            for db_object in model.objects.all().iterator():
                for field in model_dict[model]:
                    target_file = getattr(db_object, field.name)
                    if target_file:
                        referenced.add(os.path.abspath(target_file.path))

        # Delete and print each file in MEDIA_ROOT that is not referenced in the database
        not_referenced = media - referenced
        print(media)
        print(referenced)
        for filename in not_referenced:
            try:
                os.remove(filename)
                print(f"Removed {filename}")
            except OSError as e:
                print(f"Can not remove {filename}! {e}")


import json
import argparse

parser = argparse.ArgumentParser(description='main', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--dataset', default='MAGCS')
parser.add_argument('--model', default='ensemble')
args = parser.parse_args()
dataset = args.dataset
model_name = args.model

label2id = {}
id2label = {}
with open(f'./Sandbox/Data/{dataset}/label2id.txt') as fin:
	for line in fin:
		data = line.strip().split()
		label = data[0]
		idx = data[1]
		label2id[label] = idx
		id2label[idx] = label

papers = []
preds_prev = []
with open(f'../{dataset}/{dataset}_predictions_{model_name}.json') as fin:
	for line in fin:
		data = json.loads(line)
		papers.append(data['paper'])
		pred_prev = [label2id[x[0]] for x in data['predictions']]
		preds_prev.append(pred_prev)

preds = []
with open(f'./Sandbox/Results/{dataset}/score_mat.txt') as fin, \
	 open(f'../{dataset}/{dataset}_predictions_futex.json', 'w') as fout:
	for idx, line in enumerate(fin):
		if idx == 0:
			continue
		data = line.strip().split()
		scores = {}
		for y in data:
			y_tup = y.split(':')
			scores[y_tup[0]] = float(y_tup[1])
		scores_sorted = sorted(scores.items(), key=lambda x:x[1], reverse=True)

		pred_prev = preds_prev[idx-1]
		pred = pred_prev + [y[0] for y in scores_sorted if y[0] not in pred_prev]

		out = {}
		out['paper'] = papers[idx-1]
		out['predictions'] = [[id2label[x], 1] for x in pred if x in id2label]
		fout.write(json.dumps(out)+'\n')


import os
from datetime import datetime, date
import django

# from django.contrib.auth.models import User
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'social_media.settings')
django.setup()
from core.attachement.models import AttachementModel

# AUTH_USER_MODEL = "core.UserModel"
attachement = AttachementModel(
    id=1,
    messagesUserId=1,
    fileUrl='https:k/#@sangnt84',
    thumbUrl='https:k/#@sangnt84',
    createdAt='2016-03-09T03:30:25.1263499Z',
    updatedAt='2016-03-09T03:30:25.1263499Z',
    messengesGroupId=None,
)
attachement.save()


# import python modules
import os

from docxtpl import DocxTemplate, InlineImage

# create a document object
# Документ -шаблон
doc = DocxTemplate("shablon_mnz.docx")

DATE = "25.01.2023"

###################################### ОБРАБОТКА катлога с файлами ############

##УКАзываем путь к директории с файлами
path = "c:/PYTHON/diplom_Viligdanov T/Files"

# Получаем список всех файлов в каталоге
fun = lambda x: os.path.isfile(os.path.join(path, x))

files_list = filter(fun, os.listdir(path))

# Создайте список файлов в каталоге вместе с указанием размера
size_of_file = [
    (f, os.stat(os.path.join(path, f)).st_size)
    for f in files_list
]

# Iterate over list of files along with size
# and print them one by one.
# Выполнить итерацию по списку файлов с указанием размера
# и распечатайте их один за другим

# count = int(0)
# Функция для обработки списка

# format(GH, df, inDx1, f[0:], NumFile, inDx2, s)

# print("df=", GH + df)
# print("inDx1=", inDx1)
# print("inDx2=", inDx2)
# print("f=", f)
# print(type(f))
# print("size_file=", size)
#
# print("NumFile=", NumFile)
# print(size_of_file)
#
# print(len(size_of_file))

############################---Заполнение МНЗ---##########################

# create data for reports
salesTblRows = []
# for k in range(len(size_of_file)):

for f, size in size_of_file:
    k = 0
    g = str(f)
    start = g.find('_')
    end = g.find('.d')

    asg = g.find('сб')  # если нашли буквы вп, то к номеру файла прибавляем ВП
    asg1 = g.find('СБ')

    asg2 = g.find('вп')  # если нашли буквы вп, то к номеру файла прибавляем ВП
    asg22 = g.find('ВП')

    asg3 = g.find('тэ4')  # если нашли буквы вп, то к номеру файла прибавляем ВП
    asg33 = g.find('ТЭ4')

    asg4 = g.find('мэ')  # если нашли буквы мэ, то к номеру файла прибавляем МЭ
    asg44 = g.find('МЭ')

    if asg > 0 or asg1 > 0:
        inDx1 = 'Сборочный чертеж'
        inDx2 = 'СБ'
    elif asg2 > 0 or asg22 > 0:
        inDx1 = 'Ведомость покупных изделий'
        inDx2 = 'ВП'
    elif asg3 > 0 or asg33 > 0:
        inDx1 = 'Таблица соединений'
        inDx2 = 'ТЭ4'
    elif asg4 > 0 or asg44 > 0:
        inDx1 = 'Электромонтажный чертеж'
        inDx2 = 'МЭ'

    else:
        inDx1 = ''
        inDx2 = ''

    # print(end)
    df = g[start + 2:end]
    # print(df)
    GH = g[start + 1].upper()
    NameFile = f[0:15]  # обозначение конструкторского документа состоит из первых 15 символов

    # разделяем размер файла пробелами
    num = int(size)
    num = '{0:,}'.format(num).replace(',', ' ')

    # проверка на тип файла документа (dwg или xls)
    data = str(size_of_file[k])
    flag = data.find("dwg") != -1

    if (flag == True):
        name_program = "AutoCad"
    else:
        name_program = "Exel"

    salesTblRows.append(
        {"sNo": k + 1, "name_chertega": GH + df + "\n" + inDx1, "name_file": f, "program": name_program,
         "oboznach_file": NameFile + " " + inDx2, "size": num})
    k += 1

# create context to pass data to template
context = {

    "salesTblRows": salesTblRows,
    "topItemsRows": " ",
    "date": DATE,
    "kolich_file": len(size_of_file)
}

# render context into the document object
doc.render(context)

# save the document object as a word file
reportWordPath = 'ITOG-MNZ-FILE.docx'
doc.save(reportWordPath)

print("Формирование документа завершено.")


# use Python 3.9
# python3.9 -m venv env
# source new3.9/bin/activate
# pip3.9 install -r requirements.txt

import random
import math
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure
import scipy
from scipy.special import softmax
import numpy as np

# Typing
import typing
from typing import TypeVar, Generic
from collections.abc import Callable

from tqdm import tqdm
from sklearn.cluster import KMeans
import statistics
import dataclasses
from dataclasses import dataclass
from tensorflow.keras.optimizers import Adam
from tensorflow.keras import datasets, layers, models
#import keras.backend as K
import copy
from copy import deepcopy
import tensorflow as tf

from pd_paramters import pop_descent_classes

import time
start_time = time.time()

# FUNCTIONS FOR NN IMPLEMENTATION
def new_NN_individual():

	# # FM Model (small)
	# model = tf.keras.Sequential([
	# tf.keras.layers.Flatten(input_shape=(28, 28)),
 #    tf.keras.layers.Dense(128, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(l=.001)),
 #    tf.keras.layers.Dense(64, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(l=.001)),
 #    tf.keras.layers.Dense(10)
	# ])

	# # FM Model (small --> get equal results with PD and .fit)
	# model = tf.keras.Sequential([
	# tf.keras.layers.Flatten(input_shape=(28, 28)),
 #    tf.keras.layers.Dense(128, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(l=.0001)), # good rate from hyperparameter search = 1e-4, 1e-5
 #    tf.keras.layers.Dense(64, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(l=.0001)),
 #    tf.keras.layers.Dense(10)
	# ])

	# # Keras Tutorial Model --> use for just overfitting rn
	# model = tf.keras.Sequential([
	# tf.keras.layers.Flatten(input_shape=(28, 28)),
	# tf.keras.layers.Dense(2, activation='tanh', kernel_regularizer=tf.keras.regularizers.l2(l=.0001)),
	# tf.keras.layers.Dense(10)
	# ])

	# # model #3: for trying to avoid overfitting, hyperparameter vs PD
	model = tf.keras.Sequential([
	tf.keras.layers.Flatten(input_shape=(28, 28)),
	tf.keras.layers.Dense(1024),
	tf.keras.layers.Dense(512),
	tf.keras.layers.Dense(256),
    tf.keras.layers.Dense(128, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(l=.001)),
    tf.keras.layers.Dense(64, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(l=.001)),
    tf.keras.layers.Dense(10)
	])

	# ## big keras model

	# model = tf.keras.Sequential()
	# model.add(layers.Conv2D(128, kernel_size=(3, 3), activation='relu', input_shape=FM_input_shape))
	# model.add(layers.Activation('relu'))
	# model.add(layers.Conv2D(filters=96, kernel_size=(3,3), strides=2))
	# model.add(layers.Activation('relu'))

	# # model.add(layers.Conv2D(filters=192, kernel_size=(3,3)))
	# # model.add(layers.Activation('relu'))
	# # model.add(layers.Conv2D(filters=192, kernel_size=(3,3), strides=2))
	# # model.add(layers.Activation('relu'))

	# model.add(layers.Flatten())
	# model.add(layers.BatchNormalization())
	# model.add(layers.Dense(128, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(l=.001)))
	# model.add(layers.Dense(256, kernel_regularizer=tf.keras.regularizers.l2(l=.001)))
	# model.add(layers.Dense(512))

	# model.add(layers.Activation('relu'))

	# model.add(layers.Dense(10, activation="softmax"))

	optimizer = tf.keras.optimizers.Adam(learning_rate=1e-4) # 1e-3
	LR_constant = 10**(np.random.normal(-4, 2))
	reg_constant = 10**(np.random.normal(0, 2))
	print(LR_constant, reg_constant)

	# creating NN object with initialized parameters
	NN_object = NN_Individual(model, optimizer, LR_constant, reg_constant)
	return NN_object

def NN_optimizer_manual_loss(NN_object):

	# classification_NN_compiler(NN_object.nn)
	batch_size = 64
	epochs = 1
	normalized_training_loss, normalized_validation_loss = [], []

	optimizer = NN_object.opt_obj
	lossfn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)

	indices = np.random.choice(59999, size = (batch_size*21, ), replace=False)
	vIndices = np.random.choice(4999, size = (batch_size*10, ), replace=False)

	# FM dataset
	random_batch_FM_train_images, random_batch_FM_train_labels = FM_train_images[indices], FM_train_labels[indices]
	random_batch_FM_validation_images, random_batch_FM_validation_labels = FM_validation_images[vIndices], FM_validation_labels[vIndices]

	# NN_object.nn.fit(random_batch_FM_train_images, random_batch_FM_train_labels, validation_data = (random_batch_FM_validation_images, random_batch_FM_validation_labels), epochs=epochs, verbose=1, batch_size = batch_size)

	for e in range(epochs):

		with tf.GradientTape() as tape:

			# make a prediction using the model and then calculate the loss
			model_loss = lossfn(random_batch_FM_train_labels, NN_object.nn(random_batch_FM_train_images))

			# use regularization constant
			regularization_loss = NN_object.nn.losses
			reg_loss = regularization_loss[0]
			# reg_loss = ((regularization_loss[0] + regularization_loss[1]))
			mreg_loss = reg_loss * NN_object.reg_constant
			# mreg_loss = reg_loss * 1

			# total_training_loss = tf.math.multiply(NN_object.LR_constant, model_loss) # LR randomization
			# total_training_loss = model_loss + mreg_loss # REG randomization
			total_training_loss = NN_object.LR_constant * (model_loss + mreg_loss) # LR + REG randomization

			tf.print("training loss: %s" % model_loss)
			# print("mreg los: %s" % mreg_loss), print("")
			# print("total loss: %s" % total_training_loss), print("")

			validation_loss = lossfn(random_batch_FM_validation_labels, NN_object.nn(random_batch_FM_validation_images))
			tf.print("validation loss: %s" % validation_loss)
			print("")
	# 	# print(" %s --> unnormalized training loss: %s" % model_loss), print("")

	# 	# # calculate the gradients using our tape and then update the model weights
		grads = tape.gradient(model_loss, NN_object.nn.trainable_variables)
		# grads = tape.gradient(total_training_loss, NN_object.nn.trainable_variables) ## with LR randomization
		optimizer.apply_gradients(zip(grads, NN_object.nn.trainable_variables))

	normalized_training_loss.append(2/(2+(model_loss)))
	normalized_training_loss = np.array(normalized_training_loss)

	normalized_validation_loss.append(2/(2+(validation_loss)))
	normalized_validation_loss = np.array(normalized_validation_loss)

	print(""), print("normalized training loss: %s" % normalized_training_loss)
	print("normalized validation loss: %s" % normalized_validation_loss)

	#print(model_loss)
	return normalized_training_loss, normalized_validation_loss

def NN_randomizer_manual_loss(NN_object, normalized_amount):
	print(""), print("RANDOMIZING")
	# original: (0, 1e-3), (0, normalized_amount), (0, normalized amount)

	factor = 25

	# randomizing NN weights
	model_clone = tf.keras.models.clone_model(NN_object.nn)
	model_clone.set_weights(np.array(NN_object.nn.get_weights()))

	mu, sigma = 0, (1e-2) #1e-4 for sin
	gNoise = (np.random.normal(mu, sigma))*(normalized_amount)

	weights = np.array((NN_object.nn.get_weights()))
	randomized_weights = weights + gNoise
	model_clone.set_weights(randomized_weights)

	# randomizing regularization rate
	mu, sigma = 0, (normalized_amount*factor) # 0.7, 1 #10 # 0.3
	print(mu, sigma)
	print("")
	randomization = 2**(np.random.normal(mu, sigma))
	new_reg_constant = (NN_object.reg_constant) * randomization

	print("reg randomization: %s" % randomization)
	print("%s NN_object.reg_constant" % NN_object.reg_constant)
	# print(normalized_amount)
	print("%s new_reg_constant" % new_reg_constant), print("")

	# randomizing learning_rates
	mu, sigma = 0, (normalized_amount*factor) # 0.7, 1 #10 # 0.3
	randomization = 2**(np.random.normal(mu, sigma))
	new_LR_constant = (NN_object.LR_constant) * randomization

	print("LR randomization: %s" % randomization)
	print("%s NN_object.LR_constant" % NN_object.LR_constant)
	print(normalized_amount)
	print("%s new_lr_constant" % new_LR_constant)
	print(""), print("factor=%s" % factor)

	new_NN_Individual = NN_Individual(model_clone, NN_object.opt_obj, new_LR_constant, new_reg_constant) # without randoimzed LR

	return new_NN_Individual

# unnormalized
def observer(NN_object, tIndices):
	random_batch_FM_test_images, random_batch_FM_test_labels = FM_test_images[tIndices], FM_test_labels[tIndices]

	lossfn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
	test_loss = lossfn(random_batch_FM_test_labels, NN_object.nn(random_batch_FM_test_images))

	ntest_loss = 1/(1+test_loss)

	return test_loss

def graph_history(history, trial, parameter_string, loss_data_string, best_training_model_string, best_test_model_loss_string):
	integers = [i for i in range(1, (len(history))+1)]
	x = [j * rr for j in integers]
	y = history

	plt.scatter(x, history, s=20)
	# plt.rcParams.update({'font.size': 10})
	# figure(figsize=(3, 2), dpi=80)

	plt.tight_layout()
	plt.title("PD trial #%s" % trial)
	plt.ylabel('unnormalized loss of best model')
	plt.xlabel('iterations')

	plt.xlabel("%s\n\n%s\n\n%s\n\n%s" % (parameter_string, loss_data_string, best_training_model_string, best_test_model_loss_string))

	# for i,j in zip(x,y):
	# 	plt.annotate(str(j),xy=(i,j))
	plt.text(x[(len(x))-1], y[(len(y))-1], y[(len(y))-1])
	plt.axhline(y = y[(len(y))-1])

	plt.tight_layout()
	plt.savefig("TEST_DATA/PD_trial_%s.png" % trial)
	plt.show(block=True), plt.pause(0.5), plt.close()

def evaluator(NN_object):
	# classification_NN_compiler(NN_object) # only if using manual loss optimizer/randomizer
	batch_size = 64

	np.random.seed(0)
	tIndices = np.random.choice(4999, size = (batch_size*25, ), replace=False)
	random_batch_FM_test_images, random_batch_FM_test_labels = FM_test_images[tIndices], FM_test_labels[tIndices]

	print(""), print(""), print("Evaluating models on test data after randomization")

	# test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)
	lossfn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
	test_loss = lossfn(random_batch_FM_test_labels, NN_object.nn(random_batch_FM_test_images))

	# test_loss, test_acc = NN_object.nn.evaluate(random_batch_FM_test_images, random_batch_FM_test_labels, batch_size = batch_size)

	ntest_loss = 1/(1+test_loss)
	print("unnormalized test loss: %s" % test_loss)
	print("normalized (1/1+loss) test loss: %s" % ntest_loss)

	return ntest_loss

# def classification_NN_compiler(NN_object):
# 	NN_object.nn.compile(optimizer=NN_object.opt_obj,
#               loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
#               metrics=['accuracy'])
# 	return

# External Evaluator
def Parameter_class_evaluator(population):
	all_test_loss, all_acc = [], []

	for i in range(len(population)):
		individual_total_loss = evaluator(population[i])

		all_test_loss.append(individual_total_loss)

	avg_total_test_loss = np.mean(all_test_loss)
	best_test_model_loss = np.max(all_test_loss)

	return avg_total_test_loss, best_test_model_loss

# CLASSES
Individual = TypeVar('Individual')

@dataclass
class Parameters(Generic[Individual]):

	population: Callable[[int], np.array]
	randomizer: Callable[[np.array, float], np.array]
	optimizer: Callable[[np.array], np.array]
	observer: Callable[[np.array], np.array] # check this for typing
	randomization: bool
	CV_selection: bool
	rr: int
	history: [np.array]

# updated np typing
	# population: Callable[[int], npt.NDArray[Individual]]
	# randomizer: Callable[[npt.NDArray[Individual], float], np.Array[Individual]]
	# optimizer: Callable[[npt.NDArray[Individual]], np.Array[Individual]]

@dataclass
class NN_Individual:

	nn: models.Sequential()
	opt_obj: Adam()
	LR_constant: np.cfloat
	reg_constant: np.cfloat

def individual_to_params(
	pop_size: int,
	new_individual: Callable[[], Individual],
	individual_randomizer: Callable[[Individual, float], Individual],
	individual_optimizer: Callable[[Individual], Individual],
	observer: Callable[[Individual], float],
	randomization: bool,
	CV_selection: bool,
	rr: int, # randomization rate
	history: [float]
	) -> Parameters[Individual]:

	def Parameter_new_population(pop_size: int) -> np.array(Individual):
		population = []
		for i in range(pop_size):
			individual = new_individual()
			population.append(individual)
		population = np.array(population)

		return population

	def Parameter_class_randomizer(population: np.array(Individual), normalized_amount: float) -> np.array(Individual):
		randomized_population = []
		for i in range(len(population)):
			new_object = individual_randomizer(population[i], normalized_amount[i])
			randomized_population.append(new_object)
		randomized_population = np.array(randomized_population)

		return randomized_population

	def Parameter_class_optimizer(population: np.array(Individual)) -> np.array(Individual):
		lFitnesses, vFitnesses = [], []
		for i in range(len(population)):
			print(""), print("model #%s" % (i+1)), print("")
			normalized_training_loss, normalized_validation_loss = individual_optimizer(population[i])
			lFitnesses.append(normalized_training_loss)
			vFitnesses.append(normalized_validation_loss)

		lFitnesses = np.array(lFitnesses)
		lFitnesses = lFitnesses.reshape([len(lFitnesses), ])

		vFitnesses = np.array(vFitnesses)
		vFitnesses = vFitnesses.reshape([len(vFitnesses), ])

		return lFitnesses, vFitnesses

	# (during optimization)
	def Parameter_class_observer(population, history):

		batch_size = 64
		tIndices = np.random.choice(4999, size = (batch_size*10, ), replace=False)

		all_test_loss = []
		for i in range(len(population)):
			unnormalized_model_loss = observer(population[i], tIndices)
			all_test_loss.append(unnormalized_model_loss)

		avg_test_loss = np.mean(all_test_loss)
		best_test_model_loss = np.min(all_test_loss)

		history.append(best_test_model_loss) ## main action of observer (to graph optimization progress later)

		return

	Parameters_object = Parameters(Parameter_new_population, Parameter_class_randomizer, Parameter_class_optimizer, Parameter_class_observer, randomization, CV_selection, rr, history)
	return Parameters_object

def create_Parameters_NN_object(pop_size, randomization, CV_selection, rr):
	history = []

	object = individual_to_params(pop_size, new_NN_individual, NN_randomizer_manual_loss, NN_optimizer_manual_loss, observer, randomization=randomization, CV_selection=CV_selection, rr=rr, history=history)
	object.population = object.population(pop_size) # initiazling population

	return object

# Fashion-MNIST dataset
fashion_mnist = tf.keras.datasets.fashion_mnist
(FM_train_images, FM_train_labels), (FM_test_images, FM_test_labels) = fashion_mnist.load_data()

sample_shape = FM_train_images[0].shape
img_width, img_height = sample_shape[0], sample_shape[1]
FM_input_shape = (img_width, img_height, 1)

# Reshape data
FM_train_images = FM_train_images.reshape(len(FM_train_images), FM_input_shape[0], FM_input_shape[1], FM_input_shape[2])
FM_test_images  = FM_test_images.reshape(len(FM_test_images), FM_input_shape[0], FM_input_shape[1], FM_input_shape[2])

# normalizing data
FM_train_images, FM_test_images = FM_train_images / 255.0, FM_test_images / 255.0

# FM_validation_images, FM_validation_labels = FM_train_images[50000:59999], FM_train_labels[50000:59999]
# FM_train_images, FM_train_labels = FM_train_images[0:50000], FM_train_labels[0:50000]

FM_validation_images, FM_validation_labels = FM_test_images[0:5000], FM_test_labels[0:5000]
FM_test_images, FM_test_labels = FM_test_images[5000:], FM_test_labels[5000:]

class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

trial = 1

# Parameters
iterations = 125
pop_size = 4
number_of_replaced_individuals = 2
randomization = True
CV_selection = True
rr = 15 # leash for exploration

batch_size = 64
batches = 21

## MAIN RUNNING CODE
if __name__ == "__main__":

	Parameters_object = create_Parameters_NN_object(pop_size, randomization, CV_selection, rr)

	loss_data, acc_data, total_test_loss, batch_test_loss, total_test_acc = [], [], [], [], []

	for i in range(1):

		print(""), print("MAJOR ITERATION %s: " % (i+1)), print("")

		optimized_population, lfitnesses, vfitnesses, history = pop_descent_classes(Parameters_object, number_of_replaced_individuals = number_of_replaced_individuals, iterations = iterations)

		print(""), print(""), print("time:"), print("--- %s seconds ---" % (time.time() - start_time)), print(""), print("")

		best_model = np.max(lfitnesses)

		lmean = statistics.mean(lfitnesses)
		loss_data.append(lmean)

		# evaluate from outside
		total_hist, batch_hist = [], []
		avg_total_loss, best_test_model_loss = Parameter_class_evaluator(optimized_population)

	print(""), print("Title: PD vs Hyperparameter Search")
	parameter_string = "CV_sel: %s, randomize=%s, %s iterations, %s models, %s replaced, rr=%s" % (CV_selection, randomization, iterations, pop_size, number_of_replaced_individuals, rr)
	print(""), print(parameter_string)
	print(""), print("")
	loss_data_string = "avg normalized training loss of population on last epoch: %s" % loss_data
	print(loss_data_string)
	best_training_model_string = "normalized training loss of best model: %s" % best_model
	print(best_training_model_string)

	print("")
	# print("normalized average test loss: %s" % avg_total_loss)
	print("")
	best_test_model_loss_string = "normalized (1/1+loss) best model test loss: %s" % best_test_model_loss
	print(best_test_model_loss_string)
	print("")

	graph_history(history, trial, parameter_string, loss_data_string, best_training_model_string, best_test_model_loss_string)

print("--- %s seconds ---" % (time.time() - start_time))


import gym
import numpy as np
import itertools
from urdfenvs.robots.generic_urdf import GenericUrdfReacher
from MotionPlanningEnv.dynamicSphereObstacle import DynamicSphereObstacle
'''
class steering():
    def right(v,r):

    def left(v,r):

    def straight(v, dist):
'''

def initEnv(goal=False, obstacles=False, maps=0,):
    if maps == 1:
        result = []
        robots = [
            GenericUrdfReacher(urdf="pointRobot.urdf", mode="vel"),
            GenericUrdfReacher(urdf="pointRobot.urdf", mode="vel"),
            GenericUrdfReacher(urdf="pointRobot.urdf", mode="vel"),
            GenericUrdfReacher(urdf="pointRobot.urdf", mode="vel")
        ]

        m = len(robots)
        env = gym.make(
            "urdf-env-v0",
            dt=0.01, robots=robots, render=True
        )

        n = env.n()
        pos0 = np.zeros(n)
        pos0[1] = -0.0
        ns_per_robot = env.ns_per_robot()

        initialPositions = np.array(
            [
                (0,0,0), (0,0,1), (0,0,0), (0,0,np.deg2rad(90))
            ]
        )

        mountPositions = np.array(
            [
                #(0,0,0), (-8,6,0), (8,-6,0), (-8,-8.75,0)
                (0,0,0), (-2,0,0), (-3,0,0), (-4,0,0)
            ]
        )
        env.reset(pos=initialPositions,mount_positions=mountPositions)

        #from env.gym_envs_urdf.steering import steeringInput

        from env.gym_envs_urdf.scene_objects.obstacles import (
            walls1,
            #boxes1,
            #obstacles1

        )

        for wall in walls1:
            env.add_shapes(shape_type=wall[0], dim=wall[1], poses_2d=wall[2])

        for i in range(len(walls1)):
            dimensions = walls1[i][1][0:2]
            coordinates = [coord[:2] for coord in walls1[i][2]]

            result.append([[coordinates[0], coordinates[1], dimensions[0], dimensions[1]] for coordinates in coordinates])

        obstacles = list(itertools.chain(*result))

        '''
        for box in boxes1:
            env.add_shapes(shape_type=box[0], dim=box[1],mass =box[2], poses_2d=box[3] , place_height =box[4])

        for obstacle in obstacles1:
            env.add_obstacle(obstacle)

        '''
    else:
        pass

    # Set goal to follow
    if goal:
        from env.gym_envs_urdf.scene_objects.goal import dynamicGoal

        env.add_goal(dynamicGoal)

    if obstacles:
        from env.gym_envs_urdf.scene_objects.obstacles import (
            sphereObst1,
            sphereObst2,
            urdfObst1,
            dynamicSphereObst3,
        )

        env.add_obstacle(sphereObst1)
        env.add_obstacle(sphereObst2)
        env.add_obstacle(urdfObst1)
        env.add_obstacle(dynamicSphereObst3)

    return env , m, mountPositions[:,:2], obstacles, initialPositions[:,2]+np.deg2rad(90)#, #, steeringInput

def robotMain(pos, vel, current_orientations, omega, otherRobots, env, render=False, dt=0.01):
    # Extract position and orientation from the inputs
    x = pos[0][0]
    y = pos[0][1]
    theta = current_orientations[0]
    # Calculate the velocity vector in the original orientation
    v_vector = np.array([vel, 0])

    # Create a rotation matrix to rotate the velocity vector
    rotation_matrix = np.array([[np.cos(theta), -np.sin(theta)],
                                [np.sin(theta), np.cos(theta)]])
    # Rotate the velocity vector
    v_rotated = rotation_matrix @ v_vector

    # Calculate the new position and orientation
    x_new = x + v_rotated[0] * dt
    y_new = y + v_rotated[1] * dt
    theta_new = theta + omega * dt

    # Create the new position and orientation arrays
    pos_new = np.array([[x_new, y_new]])
    orientation_new = np.array([theta_new])

    # Create the action array for the current robot
    action_new = np.append(v_rotated, omega)

    # Create an action array for the other robots
    action_other_bots = np.zeros(9)

    # Concatenate the action arrays for all robots
    action_new = np.concatenate((action_new, action_other_bots))
    vel_new = np.array([vel])

    # Step the environment and return the new position, velocity, and orientation
    ob, _, _, _ = env.step(action_new)
    return pos_new, vel_new, orientation_new


from core.social_auth.oauth import oauth
from core.social_auth.constants import GITHUB_USERINFO_URL, GITHUB_EMAIL_URL
from werkzeug.exceptions import Unauthorized, BadRequest, InternalServerError
from requests.exceptions import RequestException
from authlib.integrations.flask_client import OAuthError
from flask import request

class GithubAuth:
    NAME = "github"

    def __init__(self):
        self.client = oauth.create_client("github")
        if not self.client:
            raise BadRequest("Github is not register for oauth in the backend.")

    def get_mail(self):
        try:
            response = self.client.get(GITHUB_EMAIL_URL)
        except RequestException as e:
            raise InternalServerError(f"Server failed to fetch detailed from {GITHUB_USERINFO_URL}")

        data = response.json()
        if response.status_code == 401:
            raise Unauthorized(data)
        elif response.status_code != 200:
            raise BadRequest(data)
        return next((item['email'] for item in data if item['primary']), None)

    def get_data(self):
        """
        This function fetches the data from github API.
        Parameter
        ---------
        Return
        ------
        data: dict
        """
        token = request.get_json(force=True, silent=True).get('token')
        if not token:
            raise BadRequest("The Token is not provided.")

        self.client.token = token

        data = token.get('userinfo')
        if not data:
            try:
                response = self.client.get(GITHUB_USERINFO_URL, params={'skip_status': True})
            except RequestException as e:
                raise InternalServerError(f"Server failed to fetch detailed from {GITHUB_USERINFO_URL}")
            except OAuthError as error:
                raise InternalServerError(f"Server failed to fetch detailed from {GITHUB_USERINFO_URL}, errors: {error.error}")
            data = response.json()
            if response.status_code == 401:
                raise Unauthorized(data)
            elif response.status_code != 200:
                raise BadRequest(data)

            if not data.get('email'):
                data['email'] = self.get_mail()

        return data

    def auth(self):
        """
        This function handles social logins for github.
        Parameter
        ---------
        Return
        ------
        """
        return self.get_data()


#!/usr/bin/env python
# -*- coding: utf-8 -*

"""Integration test for `cycle_calendar_generator` package."""

import unittest
import os
import sys
import shutil
import subprocess
from pathlib import Path
import datetime

from cycle_calendar_generator import cycle_calendar_generator

import ics
import arrow

CURRENT_WORKING_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
INTEGRATION_TEST_FOLDER = os.path.join(
    CURRENT_WORKING_DIRECTORY,
    'integration_test'
    )
TEST_FILES_FOLDER = os.path.join(INTEGRATION_TEST_FOLDER, 'testing_files')
TEST_EXPECTED_OUTPUT_FOLDER = os.path.join(INTEGRATION_TEST_FOLDER, 'expected')
TEST_TEMP_FOLDER = os.path.join(INTEGRATION_TEST_FOLDER, 'temp')
TEST_OUTPUT_FOLDER = os.path.join(TEST_TEMP_FOLDER, 'output')
SCRIPT_NAME = 'cycle_calendar_generator'

class Test_integration(unittest.TestCase):
    """Tests function to get folder argument and give default if none given"""

    @classmethod
    def setUpClass(cls):
        if (not os.path.exists(TEST_EXPECTED_OUTPUT_FOLDER)):
            os.mkdir(TEST_EXPECTED_OUTPUT_FOLDER)
        if (not os.path.exists(TEST_TEMP_FOLDER)):
            os.mkdir(TEST_TEMP_FOLDER)

    def setUp(self):
        # copy test input files from TEST_FILES_FOLDER to TEST_TEMP_FOLDER
        for file in cycle_calendar_generator.scandir_with_version_check(
                TEST_FILES_FOLDER,
                cycle_calendar_generator.VERSION_MAJOR,
                cycle_calendar_generator.VERSION_MINOR):
            source_path = file.path
            dest_path = os.path.join(TEST_TEMP_FOLDER, file.name)
            shutil.copy(source_path, dest_path)

    def tearDown(self):
        # delete all files and folders in TEST_TEMP_FOLDER
        for file in cycle_calendar_generator.scandir_with_version_check(
                TEST_TEMP_FOLDER,
                cycle_calendar_generator.VERSION_MAJOR,
                cycle_calendar_generator.VERSION_MINOR):
            if file.is_dir():
                shutil.rmtree(file.path)
            else:
                os.remove(file.path)

    def test_script_works_in_normal_case(self):
        # run script
        exit_code = subprocess.run(['python3', '-m',
                                    SCRIPT_NAME, TEST_TEMP_FOLDER])
        # read output icals into dictionary (user name as key)
        output_files = {}
        for file in cycle_calendar_generator.scandir_with_version_check(
                TEST_OUTPUT_FOLDER,
                cycle_calendar_generator.VERSION_MAJOR,
                cycle_calendar_generator.VERSION_MINOR):
            if file.is_file():
                path, filename = os.path.split(file.path)
                user_name = os.path.splitext(filename)[0]
                with open(file.path) as ical:
                    calendar = ics.Calendar(ical.read())
                sorted_events = sorted(calendar.events,
                                       key=lambda event:event.begin)
                output_files[user_name] = sorted_events
        # read expected output icals into similar dictionary
        expected_files = {}
        for file in cycle_calendar_generator.scandir_with_version_check(
                TEST_EXPECTED_OUTPUT_FOLDER,
                cycle_calendar_generator.VERSION_MAJOR,
                cycle_calendar_generator.VERSION_MINOR):
            if file.is_file():
                path, filename = os.path.split(file.path)
                user_name = os.path.splitext(filename)[0]
                with open(file.path) as ical:
                    calendar = ics.Calendar(ical.read())
                sorted_events = sorted(calendar.events,
                                       key=lambda event:event.begin)
                expected_files[user_name] = sorted_events
        # assert both dicts have same size
        self.assertEqual(len(output_files), len(expected_files))
        # assert each key in output has matching in expected
        for key in output_files.keys():
            self.assertTrue(key in expected_files)
            # assert values from matching keys are the same
            output_events = output_files[key]
            expected_events = expected_files[key]
            for i in range(len(output_events)):
                output_event = output_events[i]
                expected_event = expected_events[i]
                self.assertEqual(output_event.name, expected_event.name)
                # Output times already use local timezone,
                # but expected times use tz in iCal file (China Std. Time)
                # Expected times need to have tz replaced with local tz
                # without changing anything else
                output_event_start = output_event.begin.to('utc')
                expected_event_start = expected_event.begin.replace(
                    tzinfo=cycle_calendar_generator.LOCAL_TIMEZONE
                ).to('utc')
                output_event_end = output_event.end.to('utc')
                expected_event_end = expected_event.end.replace(
                    tzinfo=cycle_calendar_generator.LOCAL_TIMEZONE
                ).to('utc')
                self.assertEqual(output_event_start, expected_event_start)
                self.assertEqual(output_event_end, expected_event_end)


from PySide2 import QtCore, QtGui

from countdata import CountData

class DrawingData:
    def __init__(self, name: str, geom, pen: QtGui.QPen, countData=CountData()):
        """
        Minimum objects required to re-create an item
        drawn in a scene.

        Need name of item (Rect, Ellipse, Line)
        Need geometry of item (QRectF, QLine),
        and the pen used to draw the item.

        The count data is also included.
        """
        self.name = name
        self.geom = geom
        self.pen = pen
        self.countData = countData

    @property
    def args(self):
        """
        Arguments necessary to recreate the graphical
        geometry.
        """
        if self.name == "Rect":
            rect = self.geom
            args = [rect.x(), rect.y(), rect.width(), rect.height()]

        elif self.name == "Ellipse":
            rect = self.geom
            args = [rect.x(), rect.y(), rect.width(), rect.height()]

        elif self.name == "Line":
            line = self.geom
            args = [line.x1(), line.y1(), line.x2(), line.y2()]

        return args

    @property
    def penColor(self):
        """
        Pen color in #RRGGBB format
        """
        return self.pen.color().name()

    @property
    def penWidth(self):
        """
        Pen width as an integer
        """
        return self.pen.width()

    @property
    def center(self):
        """
        Center QPointF of the geometry
        """
        return self.geom.center()

    def toDict(self):
        """
        Returns this drawing data as a serializable dict
        """

        return {
            "Name": self.name,
            "Args": self.args,
            "PenColor": self.penColor,
            "PenWidth": self.penWidth,
            "CountData": self.countData.toDict(),
        }

    @staticmethod
    def fromDict(d):
        """
        Initializes object from a dict (ideally, a dict previously created with `toDict`)
        """

        # Extract data
        name = d["Name"]
        args = d["Args"]
        penColor = d["PenColor"]
        penWidth = d["PenWidth"]
        countData = d["CountData"]

        # Setup pen
        pen = QtGui.QPen(penColor)  # Does this color need to be a QColor?
        pen.setWidth(penWidth)

        if name == "Rect":
            geom = QtCore.QRectF(*args)
        elif name == "Ellipse":
            geom = QtCore.QRectF(*args)
        elif name == "Line":
            geom = QtCore.QLineF(*args)
        else:
            raise ValueError(f"Unrecognized geometry name: {name}")

        return DrawingData(name, geom, pen, CountData.fromDict(countData))

    def offset(self, x, y):
        """
        Offset the geometry of this point by a given
        x and y value.
        """
        self.geom.translate(QtCore.QPointF(x, y))

    def scale(self, sf):
        """
        Scales the geometry of this point by a
        scale factor.
        """
        if self.name in ("Rect", "Ellipse"):
            x = self.geom.x() * sf
            y = self.geom.y() * sf
            width = self.geom.width() * sf
            height = self.geom.height() * sf
            self.geom.setRect(x, y, width, height)

        elif self.name in "Line":
            x1 = self.geom.x1() * sf
            y1 = self.geom.y1() * sf
            x2 = self.geom.x2() * sf
            y2 = self.geom.y2() * sf
            self.geom.setP1(QtCore.QPointF(x1, y1))
            self.geom.setP2(QtCore.QPointF(x2, y2))

        self.pen.setWidth(self.pen.width() * sf)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Feb 21 18:43:37 2019

@author: zhanghuangzhao
"""

import numpy
import random
from mnist import MNIST

class dataset(object):

    def __init__(self, x, y):

        self.__x = numpy.asarray(x).reshape((-1, 28, 28, 1))
        self.__y = numpy.asarray(y)
        self.size = self.__x.shape[0]
        self.__idx = []
        self.__shuffle()
        self.images = self.__x
        self.labels = self.__y

    def __shuffle(self):

        self.__idx = random.sample(range(self.size), self.size)

    def reset_epoch(self):

        self.__shuffle()

    def next_batch(self, batch_size, dtype=numpy.float32):

        if len(self.__idx) < batch_size:
            self.__shuffle()
        assert batch_size <= self.size, \
            "Batch size %d is larger than dataset size %d" % (batch_size, self.size)

        idx = self.__idx[:batch_size]
        self.__idx = self.__idx[batch_size:]
        x, y = ([], [])
        for i in idx:
            x.append(self.__x[i])
            y.append([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
            y[-1][self.__y[i]] = 1
        return numpy.asarray(x, dtype=dtype), numpy.asarray(y, dtype=dtype)

class Fashion_MNIST(object):

    def __init__(self, validation_size=10000, data_dir="../data"):

        fmnist = MNIST(data_dir, return_type="lists")
        train = fmnist.load_training()
        test = fmnist.load_testing()

        assert validation_size >= 0 and validation_size <= len(train[0]), \
            "Invalid validattion ratio %.3f, should be within 0 to %d" \
            % (validation_size, len(train[0]))

        idx = random.sample(range(len(train[0])), len(train[0]))
        tr, va = ([[], []], [[], []])
        for i in idx[:validation_size]:
            va[1].append(train[1][i])
            va[0].append(train[0][i])
        for i in idx[validation_size:]:
            tr[1].append(train[1][i])
            tr[0].append(train[0][i])
        self.train = dataset(tr[0], tr[1])
        self.valid = dataset(va[0], va[1])
        self.test = dataset(test[0], test[1])

        self.__label_dict = ["t-shirt/top", "trouser", "pullover", "dress",
                             "coat", "sandal", "shirt", "sneaker", "bag",
                             "ankle boot"]

    def get_label(self, label_idx):

        return self.__label_dict[label_idx]

    def get_labels(self):

        return self.__label_dict

if __name__ == "__main__":

    fmnist = Fashion_MNIST()

    try:
        import matplotlib.pyplot as plt
        for i in range(10):
            x, y = fmnist.train.next_batch(1)
            while not numpy.argmax(y) == i:
                x, y = fmnist.train.next_batch(1)
            plt.imshow(x.reshape([28, 28]))
            plt.imsave("../images/fmnist_%d.jpg"%i, x.reshape([28, 28]))
    except Exception as e:
        print(e)


import os
from unittest import mock

from absl import flags
from absl.testing import absltest, parameterized

try:
  import tensorflow_model_analysis as tfma
  from ml_metadata.proto import metadata_store_pb2
except ImportError:
  pass

from model_card_toolkit import core, dependencies, model_card

try:
  from model_card_toolkit.utils import tf_graphics, tf_sources
  from model_card_toolkit.utils.testdata import tf_testdata_utils
  from model_card_toolkit.utils.testdata.tfxtest import TfxTest
  from model_card_toolkit.utils.tf_utils import (
      _TFX_METRICS_TYPE, _TFX_STATS_TYPE
  )
except ImportError:
  tf_graphics = None
  TfxTest = absltest.TestCase

_MOCK_TENSORFLOW_EXTRA_MISSING_DEP = {
    dependencies._TENSORFLOW_EXTRA_DEPS[0]: None,
}

_IS_MISSING_OPTIONAL_DEPS = not dependencies.has_tensorflow_extra_deps()

class TfCoreTest(parameterized.TestCase, TfxTest):
  def setUp(self):
    super().setUp()
    if _IS_MISSING_OPTIONAL_DEPS:
      self.skipTest('Missing optional dependencies.')
    test_dir = self.create_tempdir()
    self.tmp_db_path = os.path.join(test_dir, 'test_mlmd.db')
    self.mct_dir = test_dir.mkdir(
        os.path.join(test_dir, 'model_card')
    ).full_path

  @mock.patch.dict('sys.modules', _MOCK_TENSORFLOW_EXTRA_MISSING_DEP)
  def test_init_with_store_and_missing_tensorflow_extra_deps(self):
    store = tf_testdata_utils.get_tfx_pipeline_metadata_store(self.tmp_db_path)
    with self.assertRaises(ImportError):
      core.ModelCardToolkit(
          output_dir=self.mct_dir, mlmd_source=tf_sources.MlmdSource(
              store=store, model_uri=tf_testdata_utils.TFX_0_21_MODEL_URI
          )
      )

  @mock.patch.dict('sys.modules', _MOCK_TENSORFLOW_EXTRA_MISSING_DEP)
  def test_init_with_source_and_missing_tensorflow_extra_deps(self):
    with self.assertRaises(ImportError):
      core.ModelCardToolkit(source=tf_sources.Source())

  def test_init_with_store_model_uri_not_found(self):
    store = tf_testdata_utils.get_tfx_pipeline_metadata_store(self.tmp_db_path)
    unknown_model = 'unknown_model'
    with self.assertRaisesRegex(
        ValueError, f'"{unknown_model}" cannot be found in the `store`'
    ):
      core.ModelCardToolkit(
          mlmd_source=tf_sources.MlmdSource(
              store=store, model_uri=unknown_model
          )
      )  # yapf: disable

  @mock.patch.object(
      tf_graphics, 'annotate_dataset_feature_statistics_plots', autospec=True
  )
  @mock.patch.object(tf_graphics, 'annotate_eval_result_plots', autospec=True)
  def test_scaffold_assets_with_store(
      self, mock_annotate_data_stats, mock_annotate_eval_results
  ):
    num_stat_artifacts = 2
    num_eval_artifacts = 1
    output_dir = self.mct_dir
    store = tf_testdata_utils.get_tfx_pipeline_metadata_store(self.tmp_db_path)
    toolkit = core.ModelCardToolkit(
        output_dir=output_dir, mlmd_source=tf_sources.MlmdSource(
            store=store, model_uri=tf_testdata_utils.TFX_0_21_MODEL_URI
        )
    )
    mc = toolkit.scaffold_assets()
    self.assertIsNotNone(mc.model_details.name)
    self.assertIsNotNone(mc.model_details.version.name)
    self.assertIn(
        'default_template.html.jinja',
        os.listdir(os.path.join(output_dir, 'template/html'))
    )
    self.assertIn(
        'default_template.md.jinja',
        os.listdir(os.path.join(output_dir, 'template/md'))
    )
    self.assertEqual(mock_annotate_data_stats.call_count, num_stat_artifacts)
    self.assertEqual(mock_annotate_eval_results.call_count, num_eval_artifacts)

  @parameterized.parameters(
      ('', True), ('', False), ('tfrecord', True), ('tfrecord', False)
  )
  def test_scaffold_assets_with_source(
      self, output_file_format: str, artifacts: bool
  ):

    train_dataset_name = 'Dataset-Split-train'
    train_features = ['feature_name1']
    eval_dataset_name = 'Dataset-Split-eval'
    eval_features = ['feature_name2', 'feature_name3']

    test_dir = self.create_tempdir()
    tfma_path = os.path.join(test_dir, 'tfma')
    tfdv_path = os.path.join(test_dir, 'tfdv')
    pushed_model_path = os.path.join(test_dir, 'pushed_model')

    add_metrics_callbacks = [
        tfma.post_export_metrics.example_count(),
        tfma.post_export_metrics.calibration_plot_and_prediction_histogram(
            num_buckets=2
        ),
    ]

    if artifacts:
      mlmd_store = self._set_up_mlmd()
      self._write_tfma(
          tfma_path, output_file_format, add_metrics_callbacks, mlmd_store
      )
      self._write_tfdv(
          tfdv_path, train_dataset_name, train_features, eval_dataset_name,
          eval_features, mlmd_store
      )
      model_evaluation_artifacts = mlmd_store.get_artifacts_by_type(
          _TFX_METRICS_TYPE
      )
      example_statistics_artifacts = mlmd_store.get_artifacts_by_type(
          _TFX_STATS_TYPE
      )
      # Use placeholder artifact to avoid introducing tfx as a dependency
      pushed_model_artifact = metadata_store_pb2.Artifact(
          uri=pushed_model_path
      )
      tfma_src = tf_sources.TfmaSource(
          model_evaluation_artifacts=model_evaluation_artifacts,
          metrics_exclude=['average_loss']
      )
      tfdv_src = tf_sources.TfdvSource(
          example_statistics_artifacts=example_statistics_artifacts,
          features_include=['feature_name1', 'feature_name3']
      )
      model_src = tf_sources.ModelSource(
          pushed_model_artifact=pushed_model_artifact
      )
    else:
      self._write_tfma(tfma_path, output_file_format, add_metrics_callbacks)
      self._write_tfdv(
          tfdv_path, train_dataset_name, train_features, eval_dataset_name,
          eval_features
      )
      tfma_src = tf_sources.TfmaSource(
          eval_result_paths=[tfma_path], metrics_exclude=['average_loss']
      )
      tfdv_src = tf_sources.TfdvSource(
          dataset_statistics_paths=[tfdv_path],
          features_include=['feature_name1', 'feature_name3']
      )
      model_src = tf_sources.ModelSource(pushed_model_path=pushed_model_path)

    mc = core.ModelCardToolkit(
        source=tf_sources.Source(
            tfma=tfma_src, tfdv=tfdv_src, model=model_src
        )
    ).scaffold_assets()  # yapf: disable

    with self.subTest(name='quantitative_analysis'):
      list_to_proto = lambda lst: [x.to_proto() for x in lst]
      expected_performance_metrics = [
          model_card.PerformanceMetric(
              type='post_export_metrics/example_count', value='2.0'
          )
      ]
      self.assertCountEqual(
          list_to_proto(mc.quantitative_analysis.performance_metrics),
          list_to_proto(expected_performance_metrics)
      )
      self.assertLen(mc.quantitative_analysis.graphics.collection, 1)

    with self.subTest(name='model_parameters.data'):
      self.assertLen(mc.model_parameters.data, 2)  # train and eval
      for dataset in mc.model_parameters.data:
        for graphic in dataset.graphics.collection:
          self.assertIsNotNone(
              graphic.image,
              msg=f'No image found for graphic: {dataset.name} {graphic.name}'
          )
          graphic.image = None  # ignore graphic.image for below assertions
      self.assertIn(
          model_card.Dataset(
              name=train_dataset_name, graphics=model_card.GraphicsCollection(
                  collection=[
                      model_card.Graphic(name='counts | feature_name1')
                  ]
              )
          ), mc.model_parameters.data
      )
      self.assertIn(
          model_card.Dataset(
              name=eval_dataset_name, graphics=model_card.GraphicsCollection(
                  collection=[
                      model_card.Graphic(name='counts | feature_name3')
                  ]
              )
          ), mc.model_parameters.data
      )
      self.assertNotIn(
          model_card.Dataset(
              name=eval_dataset_name, graphics=model_card.GraphicsCollection(
                  collection=[
                      model_card.Graphic(name='counts | feature_name2')
                  ]
              )
          ), mc.model_parameters.data
      )

    with self.subTest(name='model_details.path'):
      self.assertEqual(mc.model_details.path, pushed_model_path)

  def test_scaffold_assets_with_empty_source(self):
    core.ModelCardToolkit(source=tf_sources.Source()).scaffold_assets()

  def test_scaffold_assets_with_invalid_tfma_source(self):
    with self.assertRaisesWithLiteralMatch(
        ValueError,
        'Only one of TfmaSource.metrics_include and TfmaSource.metrics_exclude '
        'should be set.'
    ):
      core.ModelCardToolkit(
          source=tf_sources.Source(
              tfma=tf_sources.TfmaSource(
                  eval_result_paths=['dummy/path'], metrics_include=[
                      'false_positive_rate'
                  ], metrics_exclude=['false_negative_rate']
              )
          )
      )

  def test_scaffold_assets_with_invalid_tfdv_source(self):
    with self.assertRaisesWithLiteralMatch(
        ValueError, 'Only one of TfdvSource.features_include and '
        'TfdvSource.features_exclude should be set.'
    ):
      core.ModelCardToolkit(
          source=tf_sources.Source(
              tfdv=tf_sources.TfdvSource(
                  dataset_statistics_paths=['dummy/path'], features_include=[
                      'brand_confidence'
                  ], features_exclude=['brand_prominence']
              )
          )
      )

if __name__ == '__main__':
  absltest.main()
else:
  # Manually pass and parse flags to prevent UnparsedFlagAccessError when using
  # pytest or unittest as a runner.
  flags.FLAGS(['--test_tmpdir'])


import requests
import bs4
from bs4 import BeautifulSoup as BS
import re
URL = "https://www.google.com/search?q=chees"
def start_ref(q):
    global data
    data = {}
    headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0"
    }
    req = requests.get(q, headers=headers)
    src = req.text
    with open('indextest.html', 'w') as f:
        f.write(src)
    soup = BS(src, "lxml")
#    menu = soup.find("h3").find_parents("div", id="search").
    menu = soup.find_all("h3")

    for i in menu:

        inmenu = i.find_parent().find_parent().find_parent().find_parent()
        for q in 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12:
            lincs = inmenu.get("href")
            h3 = inmenu.find("h3").text
            span = inmenu.find("span")

            data = {q: [lincs, h3, span]}
            print(data)

    return 0

if __name__ == '__main__':
     start_ref(URL)


import pygame  # Importa a biblioteca pygame
from pygame.locals import *  # Importa de pygame tudo do submódulo locals
from sys import exit  # Importa a função exit do módulo sys que fecha a janela

# Inicializa todas as funções e variáveis da biblioteca pygame
pygame.init()

# Cria o objeto tela
largura = 640  # Medida da tela em pixel
altura = 480  # Medida da tela em pixel
tela = pygame.display.set_mode((largura, altura))  # Objeto que configura display

pygame.display.set_caption('Jogo')  # Configura nome da tela

# loop principal do jogo
while True:
    for event in pygame.event.get():  # Detecta se algum evento ocorreu
        if event.type == QUIT:  # Para a janela fechar ao clicar em fechar
            pygame.quit()
            exit()
    pygame.display.update()  #  Atualiza tela do jogo a cada interação do loop principal


import pandas as pd
import numpy as np

from sklearn.linear_model import LogisticRegression
from sklearn.cross_validation import KFold, StratifiedKFold, cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder, LabelBinarizer

import matplotlib.pyplot as plt
from sklearn_pandas import DataFrameMapper

DATA = '../data/'

mapper = DataFrameMapper([
    ('MSSubClass', None),
    ('LotArea', None),
    ('LotShape', LabelBinarizer()),
])

features = [
    'MSSubClass',
    'LotArea',
    'LotShape',
]

pipeline = Pipeline([
    ('featurize', mapper),
    ('regr', LogisticRegression(fit_intercept=True)),
])

fts = [
    'MSSubClass',
    'LotArea',
    'LotShape',
]

fts1 = fts + ['SalePrice']

def load():
    df_train = pd.read_csv(DATA + 'train.csv', header = 0, index_col = 'Id')
    df_test = pd.read_csv(DATA + 'test.csv', header = 0, index_col = 'Id')

    return df_train, df_test

def main():
    df_train, df_test = load()

    import pdb; pdb.set_trace()

if __name__ == '__main__':
    main()


#coding=utf-8
import requests
import time
import pandas as pd
import numpy as np

def get_data():
    data = pd.DataFrame()
    for i in range(1,98):
        url = f"http://www.lottery.gov.cn/historykj/history_{i}.jspx?_ltype=dlt"
        res = requests.get(url,headers=header).text  #获取网页内容
        df = pd.read_html(res)[0]  #使用pandas快速获取表格数据
        data = data.append([df])
        time.sleep(0.1)
    return data

def remake_data():
    data_list = []
    for list in get_data().values:
        # 将8到15之间的数据两两相加，再插入数组中
        list = np.insert(list, 8,[list[8] + list[10], list[9] + list[11], list[12] + list[14], list[13] + list[15]])
        list = np.delete(list, -4)  # 删除“nan“
        list = np.delete(list, np.s_[12:20])  # 删除“8到15之间的数据“
        # 将1到7之间的数据两两相减，再添加到数组末尾
        list = np.append(list, [list[2] - list[1], list[3] - list[2], list[4] - list[3], list[5] - list[4],list[7] - list[6]])

        data_list.append(list)  #将处理好的数据装入空列表data_list中
    #用new_title 作为key ，data_list作为values 创建一个二维数组
    dataframe = pd.DataFrame(data_list, columns=new_title)
    return dataframe

if __name__ == '__main__':
    #url = f"http://www.lottery.gov.cn/historykj/history_1.jspx?_ltype=dlt"
    header = {"User-Agent": "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1 Trident/5.0;"}
    new_title = ['期号', 'first1', 'first2', 'first3', 'first4', 'first5', 'last1', 'last2', '一等奖注数', '一等奖奖金(元)',
                 '二等奖注数', '二等奖奖金(元)', '销售额(元)', '奖池奖金（元）', '开奖日期', '1~2', '2~3', '3~4', '4~5', 'last1~last2']

    #remake_data().to_excel("lottery1.xls",index=False)
    remake_data().to_csv("lottery1.csv",index=False)


import pandabear as pd
import matplotlib.pyplot as plt

from sklearn.model_selection import  train_test_split
from sklearn.neighbors import KNeighborsClassifier
# KNN 알고리즘 사용을 위한 모듈

 #1. 전처리 하기 전 데이터 확인

data = pd.read_csv ("sample_iris.csv")
print(data.head())

# 지도학습 데이터

X = data.values[:, :data.shape[1] - 1]
Y = data.values[:, data.shape[1] - 1]

# X 트레인과 Y 트레인 train test split : 데이터를 마구 섞는다.

X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size= 0.3,random_state=1)

# 2개의 개체를 다음과 같이 나눈다. 값을 유지시키기 위해서 고정값 1

knn = KNeighborsClassifier(n_neighbors=5)
# 거리값  K  거리값 정의하고 학습하는 것

knn.fit(X_train,Y_train)
#이 트레이닝으로 학습을 하겠다. 라는 의미
print('KNN iris of acc > ' , format(knn.score(X_test, Y_test)*100)) # X Y 트레인을 비교해서 값을 나타낸 것,

train_acc = [] # 트레이닝 정확도
test_acc = []  # 테스트 정확도를 리스트에 담아둠
k_list = range(1, 100) # 1부터 31까지를 / 거리값을 계산시 위치를 바꿔주면됌
for k in k_list:
    clf = KNeighborsClassifier(n_neighbors=k) #1부터 30까지의 모든 거리 값을 다 계산
    clf.fit(X_train, Y_train)
    train_acc.append(clf.score(X_train, Y_train)) # 리스트 증가 clf = knn
    test_acc.append(clf.score(X_test, Y_test)) # 트레이닝한거 테스트한거

# 시각화
plt.plot(k_list, test_acc, label= 'test_acc of ')
plt.plot(k_list, train_acc, label = 'train _ acc of')

plt.xlabel('k')
plt.ylabel('acc')
plt.legend()
plt.show()

 # > 여기까지가 KNN


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 28 16:22:45 2018

@author: thiru
"""

from nltk.corpus import stopwords
from apple.models import *
from nltk.stem.wordnet import WordNetLemmatizer
import string
import gensim
from gensim import corpora
import pandas as pd
import datetime
import re
import pyLDAvis.gensim as gensimvis
import pyLDAvis

words_to_remove = set(['rt','apple','apple_support','applesupport','support'])
stop = set(stopwords.words('english'))
stop = stop.union(words_to_remove)

exclude = set(string.punctuation)
lemma = WordNetLemmatizer()

date_timelines = [1,7,30] #days
explained = ['daily','weekly','monthly']
dates = [datetime.date.today() - datetime.timedelta(days=x) for x in date_timelines]

def run():
    tweets = Tweet.objects.filter(created_at__gte = dates[-1]) # last
    for i in range(len(dates)-1,-1,-1):
        tweets = tweets.filter(created_at__gte = dates[i])
        num_topics = 6

        text = [x.text for x in tweets]
        preprocessed_text = preprocess_text(text)
        texts_cleaned = [(clean(x)).split() for x in preprocessed_text]
        if len(texts_cleaned) == 0:
            texts_cleaned = ['no text'.split()]
        texts_cleaned = pd.Series(texts_cleaned)

        ldamodel,dictionary,doc_term_matrix  = run_lda(texts_cleaned,num_topics)

#        results = pretty_print_results(ldamodel,num_topics,num_words = 7)
#        for entry in results:
#            print(entry)
#        print('\n')

        vis_data = gensimvis.prepare(ldamodel,doc_term_matrix, dictionary )
        pyLDAvis.save_html(vis_data,'apple/static/ldatopics/{}.html'.format(explained[i]))

    print("LDA job complete!")

def clean(doc):
    stop_free = ' '.join([i for i in doc.lower().split() if i not in stop])
    punc_free = ''.join(ch for ch in stop_free if ch not in exclude)
    normalized = " ".join(lemma.lemmatize(word) for word in punc_free.split())
    return normalized

def run_lda(doc_clean,num_topics):
    dictionary = corpora.Dictionary(doc_clean)
    doc_term_matrix = [dictionary.doc2bow(doc) for doc in doc_clean]
    lda = gensim.models.ldamodel.LdaModel
    ldamodel = lda(doc_term_matrix,num_topics=num_topics,id2word = dictionary,passes = 60)
    return ldamodel,dictionary,doc_term_matrix

def pretty_print_results(ldamodel,num_topics,num_words):
    z = ldamodel.print_topics(num_topics=num_topics,num_words=num_words)
    group = 1
    for i in range(len(z)):
        entry = z[i][1]
        entry = ''.join([i for i in entry if not (i.isdigit() or i == '.' or i == '*')])
        entry = entry.replace('"','')
        entry = '{}. {}'.format(group,entry)
        entry = entry.replace(' + ',', ')
        z[i] = entry
        group += 1
    return z

def preprocess_text(text_list):
    words_to_remove = ['rt']
    text_list = [x.lower() for x in text_list]
    # strip urls
    text_list = [re.sub(r'(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*\b', '', x, flags=re.MULTILINE) for x in text_list]
    text_list = [' '.join([i for i in x.lower().split() if i not in stop]) for x in text_list]

    for word in words_to_remove:
        c = re.compile('(\s*){}(\s*)'.format(word))
        text_list = [c.sub('',x) for x in text_list]

    return text_list


import numpy as np
import scipy.io as scio

def ReadData1(data_All1):
    data_cup1_tmp = []
    data_cup1_tmp0 = np.concatenate((data_All1[0],
                                     np.zeros((2000 - data_All1[0].shape[0], 39))))
    for i in range(1, 100):
        data_cup1_tmp = np.concatenate((data_All1[i],
                                        np.zeros((2000 - data_All1[i].shape[0], 39))))
        if i == 1:
            data_cup1_tmp_a = np.array(data_cup1_tmp0)
            data_cup1_tmp_b = np.array(data_cup1_tmp)
            data_cup11 = np.array([data_cup1_tmp_a, data_cup1_tmp_b])
        else:
            data_cup1_com_1 = np.append(data_cup11, data_cup1_tmp)
            data_cup1_dim = data_cup11.shape
            data_cup11 = data_cup1_com_1.reshape(data_cup1_dim[0] + 1,
                                                 data_cup1_dim[1],
                                                 data_cup1_dim[2])

    return data_cup11

def LoadData():
    path_base_cat1 = '/Users/mengxue/Documents/Paper/ChestAuthentication/Material/BigData/Final/GQY/Matlab/HeySiri'
    path_base_cat2 = '/Users/mengxue/Documents/Paper/ChestAuthentication/Material/BigData/Final/DengYangTao/Matlab/HeySiri'
    path_base_cat3 = '/Users/mengxue/Documents/Paper/ChestAuthentication/Material/BigData/Final/WuYuan/Matlab/HeySiri'
    path_base_cat4 = '/Users/mengxue/Documents/Paper/ChestAuthentication/Material/BigData/Final/HuHaiYan/Matlab/HeySiri'
    path_base_cat5 = '/Users/mengxue/Documents/Paper/ChestAuthentication/Material/BigData/Final/OuRunMin/Matlab/HeySiri'

    cupLabels = []

    # Get the data from person one.
    for i in range(1, 101):
        path_tmp = path_base_cat1 + str(i) + '.mat'
        data_tmp = scio.loadmat(path_tmp)
        together_data = np.concatenate((data_tmp['cor1_ccc2'], data_tmp['cor2_ccc2']), axis=0)
        if i == 1:
            data_All1 = [together_data]
            cupLabels.append('A')  # label person 1
        else:
            data_All1.append(together_data)
            cupLabels.append('A')  # label person 1
    data_cup1 = ReadData1(data_All1)
    # print("Person1 shape", data_cup1.shape, '\t', "Person1 data num", len(data_All1))

    # Get the data from person two.
    for i in range(1, 101):
        path_tmp = path_base_cat2 + str(i) + '.mat'
        data_tmp = scio.loadmat(path_tmp)
        together_data = np.concatenate((data_tmp['cor1_ccc2'], data_tmp['cor2_ccc2']), axis=0)
        if i == 1:
            data_All2 = [together_data]
            cupLabels.append('B')  # label person 2
        else:
            data_All2.append(together_data)
            cupLabels.append('B')  # label person 2
    data_cup2 = ReadData1(data_All2)
    # print("Person2 shape", data_cup2.shape, '\t', "Person2 data num", len(data_All2))

    # # Get the data from person Three.
    for i in range(1, 101):
        path_tmp = path_base_cat3 + str(i) + '.mat'
        data_tmp = scio.loadmat(path_tmp)
        together_data = np.concatenate((data_tmp['cor1_ccc2'], data_tmp['cor2_ccc2']), axis=0)

        if i == 1:
            data_All3 = [together_data]
            cupLabels.append('C')  # label person 3
        else:
            data_All3.append(together_data)
            cupLabels.append('C')  # label person 3
    data_cup3 = ReadData1(data_All3)
    # print("Person3 shape", data_cup3.shape, '\t', "Person3 data num", len(data_All3))

    # Get the data from person Four.
    for i in range(1, 101):
        path_tmp = path_base_cat4 + str(i) + '.mat'
        data_tmp = scio.loadmat(path_tmp)
        together_data = np.concatenate((data_tmp['cor1_ccc2'], data_tmp['cor2_ccc2']), axis=0)

        if i == 1:
            data_All4 = [together_data]
            cupLabels.append('D')  # label person 3
        else:
            data_All4.append(together_data)
            cupLabels.append('D')  # label person 3
    data_cup4 = ReadData1(data_All4)
    # print("Person4 shape", data_cup4.shape, '\t', "Person4 data num", len(data_All4))

    # Get the data from person Five.
    for i in range(1, 101):
        path_tmp = path_base_cat5 + str(i) + '.mat'
        data_tmp = scio.loadmat(path_tmp)
        together_data = np.concatenate((data_tmp['cor1_ccc2'], data_tmp['cor2_ccc2']), axis=0)

        if i == 1:
            data_All5 = [together_data]
            cupLabels.append('E')  # label person 3
        else:
            data_All5.append(together_data)
            cupLabels.append('E')  # label person 3
    data_cup5 = ReadData1(data_All5)
    # print("Person5 shape", data_cup5.shape, '\t', "Person5 data num", len(data_All5))

    # Concatenate the data off three people.
    data_cup = np.concatenate((data_cup1, data_cup2, data_cup3, data_cup4, data_cup5), axis=0)  #

    return data_cup, cupLabels


"""
Definition of Base Run Handler
"""

from abc import abstractmethod
from io import BytesIO
from os import environ as env_vars
from os.path import exists
from subprocess import check_call as run_shell_command
from zipfile import ZipFile

from mlflow.tracking import MlflowClient
from shared.logger.job_lifecycle_manager import JobLifecycleManager
from shared.models.mlflow_models import SqlArtifact

from ..base_handler import BaseHandler

__author__: str = "Splice Machine, Inc."
__copyright__: str = "Copyright 2019, Splice Machine Inc. All Rights Reserved"
__credits__: list = ["Amrit Baveja"]

__license__: str = "Proprietary"
__version__: str = "2.0"
__maintainer__: str = "Amrit Baveja"
__email__: str = "abaveja@splicemachine.com"

DOWNLOAD_PATH: str = f'{env_vars["WORKER_HOME"]}/pmml'

class BaseDeploymentHandler(BaseHandler):
    """
    Base class for run handlers--
    handlers that execute jobs (AWS Deployment etc.)
    """

    def __init__(self, task_id: int) -> None:
        """
        :param task_id: (int) id of the task to execute
        """
        BaseHandler.__init__(self, task_id)
        self.downloaded_model_path: str = DOWNLOAD_PATH + str(
            task_id)  # So when we temporarily download the model we don't overwrite other models
        self.mlflow_run: object = None
        self.artifact = None
        self.artifact_buffer: bytearray or None = None

    def retrieve_run_from_mlflow(self) -> None:
        """
        Retrieve the current run from mlflow tracking server
        """
        self.logger.info("Retrieving Run from MLFlow Tracking Server...", send_db=True)
        client: MlflowClient = MlflowClient(tracking_uri=env_vars['MLFLOW_URL'])
        try:
            self.mlflow_run: object = client.get_run(self.task.parsed_payload['run_id'].strip())
        except Exception:
            raise Exception(
                "Error: The Run associated with the ID specified could not be retrieved"
            )

    def _retrieve_model_binary_stream_from_db(self) -> None:
        """
        Use SQLAlchemy to retrieve the model artifact
        from Database with the specified path
        and associated Run UUID
        """
        self.logger.info("Reading Model Artifact Stream from Splice Machine", send_db=True)
        run_id: str = self.mlflow_run.info.run_uuid

        self.logger.info(f"Extracting Model from DB with Name: {self.model_dir}", send_db=True)
        try:
            self.artifact = self.Session.query(SqlArtifact) \
                .filter_by(name=self.model_dir) \
                .filter_by(run_uuid=run_id).one()

        except IndexError:
            self.logger.exception(
                f"No artifact could be found in database with name {self.model_dir} and run_id "
                f"{run_id}", send_db=True
            )
            raise Exception("Model with the specified Run ID and Name could not be found!")

    def _deserialize_artifact_stream(self) -> None:
        """
        Take the BLOB Retrieved from the database.py,
        convert it into a model,
        and then serialize it to the disk for deployment
        """
        self.logger.info("Decoding Model Artifact Binary Stream for Deployment", send_db=True)

        try:
            if exists(self.downloaded_model_path):
                run_shell_command(('rm', '-Rf', self.downloaded_model_path))

            artifact_buffer = BytesIO()
            artifact_buffer.write(self.artifact.binary)
            artifact_buffer.seek(0)
            self.logger.info("Decompressing Model Artifact", send_db=True)
            ZipFile(artifact_buffer).extractall(path=self.downloaded_model_path)

        except KeyError as e:
            self.logger.exception("Unable to find the specified file extension handler", send_db=True)
            raise Exception(f"Unable to find the specified fix extension {self.artifact.file_extension}")

    def _cleanup(self) -> None:
        """
        Cleanup after the model is deployed
        """
        self.logger.info("Cleaning up deployment", send_db=True)
        temp_glob: str = "/tmp/tmp*"  # remove all temp files generated by MLFlow

        run_shell_command(('rm', '-Rf', self.downloaded_model_path))
        run_shell_command(('rm', '-Rf', temp_glob))  # cleanup azure deployment files

    @abstractmethod
    def execute(self) -> None:
        """
        Subclass specific run functionality
        """
        pass

    def exception_handler(self, exc: Exception):
        """
        Function that runs if there is an error
        executing a job
        :param exc: the exception thrown
        """
        self.logger.error(f"Running Exception Callback because of encountered: '{exc}'", send_db=True)

    def _handle(self) -> None:
        """
        We add the MLFlow Run URL as a parameter
        that can be displayed in a GUI for all
        of these Jobs.
        """
        try:
            self.retrieve_run_from_mlflow()
            run_url: str = f"#/experiments/{self.mlflow_run.info.experiment_id}/" \
                           f"runs/{self.mlflow_run.info.run_uuid}"

            self.logger.info(f"Retrieved MLFlow Run", send_db=True)

            self.model_dir: str = self.mlflow_run.data.tags.get('splice.model_name')
            if not self.model_dir:
                self.logger.exception(f"No model was found for run {self.mlflow_run.info.run_uuid}. Ensure that"
                                      f"the splice.model_name tag is available for this model", send_db=True)
                raise Exception(f'No model was found for run {self.mlflow_run.info.run_uuid}')

            # populates a link to the associated Mlflow run that opens in a new tab.
            self.logger.info("Updating MLFlow Run for the UI", send_db=True)
            self.task.mlflow_url = f"Link to Mlflow Run"

            # WHEN UPDATING JOBS, WE *MUST* USE THE MANAGER SESSION, NOT THE RUN SESSION
            # TO PREVENT EARLY COMMITTING OF THE DDL TRANSACTIONS WHEN WE UPDATE STATUS/LOGS
            self.manager.Session.add(self.task)
            self.manager.Session.commit()

            self.execute()
            self._cleanup()
        except Exception as e:
            self.exception_handler(exc=e)  # can be overriden by subclasses
            self._cleanup()  # always run cleanup, regardless of success or failure
            raise e


import pytest

import immoscrapy

@pytest.mark.parametrize(
    "real_estate_type, return_type",
    [
        ('HOUSE_BUY', immoscrapy.HOUSE_BUY),
        ('HOUSE_RENT', immoscrapy.HOUSE_RENT),
        ('APARTMENT_BUY', immoscrapy.APARTMENT_BUY),
        ('APARTMENT_RENT', immoscrapy.APARTMENT_RENT),
     ]
)
def test_query_berlin(real_estate_type, return_type):
    result = immoscrapy.query('de', 'berlin', 'berlin', real_estate_type)
    assert isinstance(result[0], return_type)

def test_query_raises_value_error():
    with pytest.raises(ValueError):
        immoscrapy.query('de', 'berlin', 'berlin', 'foo')

def test_regression_gh_8():
    """Apparently this query raises an 401."""
    # I cannot reproduce it on my machine though
    immoscrapy.query('de', 'chemnitz', 'chemnitz', 'HOUSE_BUY', price=900000)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#  Author: Mauro Soria

import sys

if sys.version_info < (3, 0):
    sys.stdout.write("Sorry, dirsearch requires Python 3.x\n")
    sys.exit(1)

from lib.core import ArgumentParser
from lib.controller import *
from lib.output import *

class Program(object):
    def __init__(self):
        self.script_path = (os.path.dirname(os.path.realpath(__file__)))
        self.arguments = ArgumentParser(self.script_path)
        self.output = CLIOutput()
        self.controller = Controller(self.script_path, self.arguments, self.output)

if __name__ == '__main__':
    main = Program()