#!/usr/bin/python3
import argparse
import configparser
import subprocess
import getpass
import sys

from datetime import datetime
from pathlib import Path
from pprint import pformat

_USER = getpass.getuser()

class RunConfig:
    def __init__(self, distance_metric, t1, t2, k_list):
        self.distance_metric = distance_metric
        self.t1 = t1
        self.t2 = t2
        self.k_list = k_list

    def __str__(self):
        return pformat(vars(self))

    def __repr__(self):
        return str(vars(self))

def run_command(label, command, interact):
    print(f"Running {label}...")
    print(f"Command:\n{command}")
    result = subprocess.run(
        command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT
    )
    output = str(result.stdout, encoding="utf-8")
    if interact and result.returncode:
        input(
            f"{output}\nCommand failed: {label}\nPress enter to continue or ctrl+c to abort.\n"
        )
    elif output:
        print(output)

def is_hdfs_file(filepath):
    result = subprocess.run(f"hdfs dfs -ls {filepath}/".split(), capture_output=True,)
    return not bool(result.returncode)

def make_directory(input_name):
    return f"hdfs dfs -mkdir {input_name}"

def copy_to_hdfs(input_name, input_dir):
    return f"hdfs dfs -put -f {input_dir} {input_name}/raw"

def make_seqfiles(input_name):
    return f"mahout seqdirectory -i {input_name}/raw -o {input_name}/seqfiles -c UTF-8 -chunk 5"

def make_sparse_vecs(input_name):
    return f"mahout seq2sparse -nv -i {input_name}/seqfiles -o {input_name}/vectors"

def clean_hdfs(input_name):
    return f"hdfs dfs -rm -r {input_name}"

def get_final_clusters(kmeans_output):
    result = subprocess.run(
        f"hdfs dfs -ls {kmeans_output}".split(), capture_output=True,
    )
    result_list = result.stdout.split()
    if result_list:
        result_str = str(result_list[-1], encoding="utf-8")
    else:
        print(f"WARN: No clusters found at {kmeans_output}!!")
        result_str = ""
    return result_str

def get_centroids(input_name, distance_metric, t1, t2, output):
    return f"""\
mahout canopy -i {input_name}/vectors/tfidf-vectors -ow \
-o {output} \
-dm org.apache.mahout.common.distance.{distance_metric} \
-t1 {t1} -t2 {t2}
"""

def run_kmeans_canopy(input_name, distance_metric, output):
    final_clusters = get_final_clusters(
        f"{input_name}/canopy-centroids-{distance_metric}"
    )
    return f"""\
mahout kmeans -i {input_name}/vectors/tfidf-vectors \
-c {final_clusters} \
-o hdfs://lena/user/{_USER}/{output} \
-dm org.apache.mahout.common.distance.{distance_metric} \
-cl -cd 0.1 -ow -x 20
"""

def run_kmeans(input_name, k, distance_metric, output):
    return f"""\
mahout kmeans -i {input_name}/vectors/tfidf-vectors \
-c {input_name}/kmeans-centroids-{k}-{distance_metric} \
-o hdfs://lena/user/{_USER}/{output} \
-dm org.apache.mahout.common.distance.{distance_metric} \
-cl -cd 0.1 -ow -x 20 -k {k}
"""

def get_cluster_info(
    input_name, k, distance_metric, kmeans_output, output_dir, output_format
):
    ext = output_format.replace("_", "").lower()
    ext = "txt" if ext == "text" else ext
    final_clusters = get_final_clusters(kmeans_output)
    return f"""\
mahout clusterdump \
-i {final_clusters} \
-d {input_name}/vectors/dictionary.file-0 \
-dt sequencefile \
-b 100 \
-n 20 \
-p {kmeans_output}/clusteredPoints \
-o {output_dir}/{input_name}-eval-{ext}/clusters-{k}-{distance_metric}.{ext} \
-dm org.apache.mahout.common.distance.{distance_metric} \
-of {output_format} \
--evaluate
"""

def main():
    time_start = datetime.now()

    argparser = argparse.ArgumentParser(
        description="Automatically run and manage mahout kmeans with simple user configuration files."
    )
    argparser.add_argument("conf", help="config file for choosing kmeans parameters")
    argparser.add_argument(
        "--force-upload",
        action="store_true",
        help="rerun and overwrite raw files that already exist",
    )
    argparser.add_argument(
        "--force-centroids",
        action="store_true",
        help="rerun and overwrite centroid files that already exist",
    )
    argparser.add_argument(
        "--force-canopy-eval",
        action="store_true",
        help="rerun and overwrite canopy cluster evaluation files that already exist",
    )
    argparser.add_argument(
        "--force-kmeans",
        action="store_true",
        help="rerun and overwrite cluster files that already exist",
    )
    argparser.add_argument(
        "--force-eval",
        action="store_true",
        help="rerun and overwrite evaluation files that already exist",
    )
    argparser.add_argument(
        "--skip-canopy",
        action="store_true",
        help="skip canopy clustering and evaluation",
    )
    argparser.add_argument(
        "-i",
        "--interactive",
        dest="interact",
        action="store_true",
        help="user prompt on task failure",
    )

    try:
        args = argparser.parse_args()
    except:
        argparser.print_help()
        sys.exit(0)

    print(f"Start time: {time_start}")
    config = configparser.ConfigParser()

    with open(args.conf, "r") as f:
        config.read_file(f)

    run_configs = []

    for key in config.sections():
        rc = RunConfig(
            distance_metric=config[key].get("distance_metric").strip(),
            t1=config[key].get("t1"),
            t2=config[key].get("t2"),
            k_list=[x.strip() for x in config[key].get("k_list").split(",")],
        )
        run_configs.append(rc)

    input_dir = config["DEFAULT"]["input_dir"].strip()
    input_name = input_dir.split("/")[-1]

    output_dir = config["DEFAULT"]["output_dir"].strip()

    if not is_hdfs_file(f"{input_name}/vectors") or args.force_upload:
        run_command("clean_hdfs", clean_hdfs(input_name), False)
        run_command("make_directory", make_directory(input_name), args.interact)
        run_command("copy_to_hdfs", copy_to_hdfs(input_name, input_dir), args.interact)
        run_command("make_seqfiles", make_seqfiles(input_name), args.interact)
        run_command("make_sparse_vecs", make_sparse_vecs(input_name), args.interact)

    for conf in run_configs:
        centroid_output = f"{input_name}/canopy-centroids-{conf.distance_metric}"
        canopy_output = f"{input_name}/kmeans-clusters-canopy-{conf.distance_metric}"
        if not is_hdfs_file(centroid_output) or args.force_centroids:
            run_command(
                f"get_centroids, dm: {conf.distance_metric}",
                get_centroids(
                    input_name, conf.distance_metric, conf.t1, conf.t2, centroid_output
                ),
                args.interact,
            )
        if not is_hdfs_file(canopy_output) or args.force_centroids:
            run_command(
                f"run_kmeans_canopy, dm: {conf.distance_metric}",
                run_kmeans_canopy(input_name, conf.distance_metric, canopy_output),
                args.interact,
            )
        canopy_eval_output_csv = Path(
            f"{output_dir}/{input_name}-eval-csv/clusters-canopy-{conf.distance_metric}.csv"
        )
        if (not canopy_eval_output_csv.is_file() or args.force_canopy_eval) and not args.skip_canopy:
            canopy_eval_output_csv.parent.mkdir(parents=True, exist_ok=True)
            run_command(
                f"get_cluster_info: csv, dm: {conf.distance_metric}, k: canopy",
                get_cluster_info(
                    input_name,
                    "canopy",
                    conf.distance_metric,
                    canopy_output,
                    output_dir,
                    "CSV",
                ),
                args.interact,
            )
        canopy_eval_output_txt = Path(
            f"{output_dir}/{input_name}-eval-txt/clusters-canopy-{conf.distance_metric}.txt"
        )
        if (not canopy_eval_output_txt.is_file() or args.force_canopy_eval) and not args.skip_canopy:
            canopy_eval_output_txt.parent.mkdir(parents=True, exist_ok=True)
            run_command(
                f"get_cluster_info: txt, dm: {conf.distance_metric}, k: canopy",
                get_cluster_info(
                    input_name,
                    "canopy",
                    conf.distance_metric,
                    canopy_output,
                    output_dir,
                    "TEXT",
                ),
                args.interact,
            )

        for k in conf.k_list:
            kmeans_output = f"{input_name}/kmeans-clusters-k-{k}-{conf.distance_metric}"
            if not is_hdfs_file(kmeans_output) or args.force_kmeans:
                run_command(
                    f"run_kmeans, dm: {conf.distance_metric}, k: {k}",
                    run_kmeans(input_name, k, conf.distance_metric, kmeans_output),
                    args.interact,
                )

            eval_output_txt = Path(
                f"{output_dir}/{input_name}-eval-txt/clusters-{k}-{conf.distance_metric}.txt"
            )
            if not eval_output_txt.is_file() or args.force_eval:
                eval_output_txt.parent.mkdir(parents=True, exist_ok=True)
                run_command(
                    f"get_cluster_info: txt, dm: {conf.distance_metric}, k: {k}",
                    get_cluster_info(
                        input_name,
                        k,
                        conf.distance_metric,
                        kmeans_output,
                        output_dir,
                        "TEXT",
                    ),
                    args.interact,
                )

            eval_output_csv = Path(
                f"{output_dir}/{input_name}-eval-csv/clusters-{k}-{conf.distance_metric}.csv"
            )
            if not eval_output_csv.is_file() or args.force_eval:
                eval_output_csv.parent.mkdir(parents=True, exist_ok=True)
                run_command(
                    f"get_cluster_info: csv, dm: {conf.distance_metric}, k: {k}",
                    get_cluster_info(
                        input_name,
                        k,
                        conf.distance_metric,
                        kmeans_output,
                        output_dir,
                        "CSV",
                    ),
                    args.interact,
                )

    time_end = datetime.now()
    time_elapsed = time_end - time_start

    print(f"End time: {time_end}")
    print(f"Total job time: {time_elapsed}")

if __name__ == "__main__":
    main()


# Author:Liu
import Planes
from PIL import Image
class HeroPlane(Planes.Plane):
    def __init__(self, screen,name):
        self.x = 230
        self.y = 600

        im = Image.open("./feiji/hero.PNG")
        out = im.resize((50, 50), Image.ANTIALIAS)
        out.save("./feiji/hero2.PNG")
        self.imag_name = "./feiji/hero2.PNG"
        super().__init__(screen,name)

    def move_left(self):
        if self.x>0:
             self.x -=10

    def move_right(self):
        if self.x<350:
            self.x += 10


from flask_restx import Namespace, Resource, fields
from flask_jwt_extended import jwt_required, get_jwt_identity
from http import HTTPStatus
from ..models.courses import Course
from ..models.students import Student

course_namespace = Namespace('course', description='name space for course')

course_model = course_namespace.model(
    'Course', {
        'course_id': fields.Integer(description='An ID'),
        'course_code': fields.String(description='course_code'),
        'course_title': fields.String(description='course title'),
        'credit_unit': fields.Integer(description='course unit')
        }
)

@course_namespace.route('/courses')
class CourseGetCreate(Resource):

    @course_namespace.marshal_with(course_model)
    @jwt_required()
    def get(self):
        """
            Get all courses
        """
        courses = Course.query.all()

        return courses, HTTPStatus.OK

    @course_namespace.expect(course_model)
    @course_namespace.marshal_with(course_model)
    @jwt_required()
    def post(self):
        """
            register a course
        """

        name = get_jwt_identity()

        current_student = Student.query.filter_by(name=name).first()

        data = course_namespace.payload

        new_course = Course (
            course_code = data['course_code'],
            course_title = data['course_title'],
            credit_unit = data['credit_unit'],
            teacher_id = data['teacher_id']
        )

        new_course.student = current_student

        new_course.save()

        return new_course, HTTPStatus.CREATED

@course_namespace.route('/course/<int:course_id>')
class GetUpdateDelete(Resource):

    @course_namespace.expect(course_model)
    @course_namespace.marshal_with(course_model)
    @jwt_required()
    def get(self, course_id):
        """
            Retrieve a course by id
        """
        course = Course.get_by_id(course_id)

        return course, HTTPStatus.OK

@course_namespace.route('/course/<int:student_id>/courses')
class StudentCourses(Resource):

    @course_namespace.marshal_list_with(course_model)
    def get(self, student_id):
        """
        Get all Student courses
        """
        student = Student.get_by_id(student_id)

        courses = student.courses

        return courses, HTTPStatus.OK

@course_namespace.route('/student/<int:student_id>/course/<int:course_id>')
class getSpecificCourseByStudent(Resource):

    @course_namespace.marshal_with(course_model)
    def get (self, student_id, course_id):
        """
        Get a student specific course
        """
        student = Student.get_by_id(student_id)

        course = Course.query.filter_by(id=course_id).filter_by(student=student).first()

        return course, HTTPStatus.OK


from typing import Optional

from pyspark.sql import SparkSession

def save_csv_as_parquet(
    spark_session: SparkSession,
    source_path: str,
    destination_path: str,
    delimiter: str = ",",
    has_header: bool = True,
    infer_schema: bool = True,
    num_partitions: Optional[int] = None,
) -> None:
    """
    Saves csv as parquet

    """
    # noinspection SpellCheckingInspection
    spark_session.read.option("delimiter", delimiter).option(
        "header", str(has_header)
    ).option("inferschema", infer_schema).csv(source_path).repartition(
        num_partitions or 1
    ).write.format(
        "parquet"
    ).mode(
        "overwrite"
    ).save(
        destination_path
    )


import requests

r = requests.get("https://www.baidu.com")

# print(type(r.text))

data = {
    "name": "germey",
    "age": 22
}

res = requests.get("http://httpbin.org/get", params=data)
# print(res.text)

url = "https://www.jianshu.com/p/2fc559e6fa00"

headers = {
    "user-agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36"
}

res1 = requests.get(url, headers=headers)
print(res1.text)


from django.contrib import admin

from philo.admin import EntityAdmin, COLLAPSE_CLASSES
from philo.contrib.julian.models import Location, Event, Calendar, CalendarView

class LocationAdmin(EntityAdmin):
	pass

class EventAdmin(EntityAdmin):
	fieldsets = (
		(None, {
			'fields': ('name', 'slug', 'description', 'tags', 'owner')
		}),
		('Location', {
			'fields': ('location_content_type', 'location_pk')
		}),
		('Time', {
			'fields': (('start_date', 'start_time'), ('end_date', 'end_time'),),
		}),
		('Advanced', {
			'fields': ('parent_event', 'site',),
			'classes': COLLAPSE_CLASSES
		})
	)
	filter_horizontal = ['tags']
	raw_id_fields = ['parent_event']
	related_lookup_fields = {
		'fk': raw_id_fields,
		'generic': [["location_content_type", "location_pk"]]
	}
	prepopulated_fields = {'slug': ('name',)}

class CalendarAdmin(EntityAdmin):
	prepopulated_fields = {'slug': ('name',)}
	filter_horizontal = ['events']
	fieldsets = (
		(None, {
			'fields': ('name', 'description', 'events')
		}),
		('Advanced', {
			'fields': ('slug', 'site', 'language',),
			'classes': COLLAPSE_CLASSES
		})
	)

class CalendarViewAdmin(EntityAdmin):
	fieldsets = (
		(None, {
			'fields': ('calendar',)
		}),
		('Pages', {
			'fields': ('index_page', 'event_detail_page')
		}),
		('General Settings', {
			'fields': ('tag_permalink_base', 'owner_permalink_base', 'location_permalink_base', 'events_per_page')
		}),
		('Event List Pages', {
			'fields': ('timespan_page', 'tag_page', 'location_page', 'owner_page'),
			'classes': COLLAPSE_CLASSES
		}),
		('Archive Pages', {
			'fields': ('location_archive_page', 'tag_archive_page', 'owner_archive_page'),
			'classes': COLLAPSE_CLASSES
		}),
		('Feed Settings', {
			'fields': ( 'feeds_enabled', 'feed_suffix', 'feed_type', 'item_title_template', 'item_description_template',),
			'classes': COLLAPSE_CLASSES
		})
	)
	raw_id_fields = ('index_page', 'event_detail_page', 'timespan_page', 'tag_page', 'location_page', 'owner_page', 'location_archive_page', 'tag_archive_page', 'owner_archive_page', 'item_title_template', 'item_description_template',)
	related_lookup_fields = {'fk': raw_id_fields}

admin.site.register(Location, LocationAdmin)
admin.site.register(Event, EventAdmin)
admin.site.register(Calendar, CalendarAdmin)
admin.site.register(CalendarView, CalendarViewAdmin)


from django.db import models

class Person(models.Model):
    GENDER_CHOICE = [
        ('M', 'Man'),
        ('F', 'Female'),
    ]
    TEST_VARIANT = [
        ("E", "Electro"),
        ("V", "Vibration"),
        ("SR", "StereoSoundRight"),
        ("SL", "StereoSoundLeft"),
        ("M", "MonoSound"),
        ("PP", "PicturePress"),
        ("PR", "PictureRelease"),
    ]
    name = models.CharField("ФИО", default="anonimous", max_length=40)
    gender = models.CharField("Пол", max_length=1, choices=GENDER_CHOICE)
    age = models.IntegerField("Возраст")
    react_time = models.IntegerField("Время Реакции")
    test_variant = models.CharField("Вариант Теста",default="E", max_length=30, choices=TEST_VARIANT)
    e_voltage = models.FloatField("Напряжение", blank=True, default=0)


from django.db import models
from django.contrib.auth.models import User
from datetime import datetime
from django.db.models.signals import post_save

STAFF_CHOICES = [
    ('Web Developer', 'developer'),
    ('Receptionist', 'receptionist'),
    ('Manager', 'manager'),
    ('Designer', 'designer'),
]

class Employee(models.Model):
    user = models.OneToOneField(User, on_delete=models.CASCADE)
    type = models.CharField(max_length=100, choices=STAFF_CHOICES)
    address = models.CharField(max_length=200)
    contact = models.CharField(max_length=200)
    image = models.ImageField(upload_to="media/")

def create_employee(sender, **kwargs):
    if kwargs['created']:
        Employee.objects.create(user=kwargs['instance'])

post_save.connect(create_employee, sender=User)

class Entry(models.Model):
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    start_time = models.DateTimeField(auto_now_add=True, blank=True)

    def __str__(self):
        return self.user

class Exit(models.Model):
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    end_time = models.DateTimeField(auto_now_add=True, blank=True)
    status = models.CharField(max_length=50, default="Full Day")

    def __str__(self):
        return f"{self.user}"

class Leave(models.Model):
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    date = models.DateField()
    reason = models.TextField()
    status = models.CharField(max_length=50, default="pending...", blank=True)

    def __str__(self):
        return f"{self.user}"


'''In this example, we first open an image file using the
Image class from the Pillow library. Then, we resize the image,
rotate it, convert it to grayscale, and apply a sharpen filter
to it. Finally, we crop the image and save the modified image
to a new file. You can modify this code to perform other image
manipulation tasks such as changing the brightness or contrast
, adding text or shapes, and more.'''

from PIL import Image, ImageFilter

# Open an image file
image = Image.open('example.jpg')

# Resize the image
image = image.resize((500, 500))

# Rotate the image
image = image.rotate(90)

# Convert the image to grayscale
image = image.convert('L')

# Apply a filter to the image
image = image.filter(ImageFilter.SHARPEN)

# Crop the image
box = (100, 100, 400, 400)
image = image.crop(box)

# Save the modified image
image.save('modified.jpg')


# -*- coding: utf-8 -*-
#関数計測
import functools
import time
from collections import defaultdict
try:
	import cProfile as profile
	import pstats
except: pass

def profileFunction(sortKey="time", rows=30):
	def _(f):
		@functools.wraps(_)
		def __(*fargs, **fkwargs):
			prof = profile.Profile()
			ret = prof.runcall(f, *fargs, **fkwargs)
			pstats.Stats(prof).strip_dirs().sort_stats(sortKey).print_stats(rows)

			return ret

		return __
	return _

#周回カウンター
class LapCounter():
    lap_times = 0
    lap_list = []
    def __init__(self):
        self.start = time.time()

    def count(self, string=''):
        self.end = time.time()
        lap_str = 'lap_time :', string, self.lap_times, ':', self.end - self.start
        self.lap_list.append(lap_str)
        self.lap_times += 1
        self.start = time.time()

    def lap_print(self, print_flag=True, window=None):
        total_time = time.time()  - self.all_start

        if window:
            out_put_time = '{:.5f}'.format(total_time)
            #ウィンドウに計算時間表示
            try:
                window.time_label.setText('- Calculation Time - '+out_put_time+' sec')
            except Exception as e:
                print('{}'.format(e))
                pass

        if print_flag:#表示するかどうかをグローバル変数で管理
            print('----------------------------------')
            for lap_time in self.lap_list:
                print(lap_time)
            print('total_time :', total_time)

    def reset(self):
        self.all_start = time.time()
        self.start = time.time()
        self.lap_list = []

#積算カウンター,同名文字列の処理時間を積算していく
#for文を細かく処理負荷解析したいときに
class IntegrationCounter():
    def __init__(self):
        self.start = time.time()
        self.integration_dict = defaultdict(lambda : 0)

    def count(self, string=''):
        self.end = time.time()
        self.integration_dict[string] += self.end - self.start
        self.start = time.time()

    def integration_print(self, print_flag=True, window=None):
        for string, integration in self.integration_dict.items():
            print('Integration time :', string, integration)

    def reset(self):
        self.all_start = time.time()
        self.start = time.time()
        self.lap_list = []
        self.integration_dict = {}
        self.integration_dict = defaultdict(lambda : 0)


"""Make a video note"""

# by GitHub.com/code-rgb [TG- @DeletedUser420]

import os
from shutil import rmtree

import ujson

from userge import Message, userge
from userge.utils import runcmd, safe_filename, thumb_from_audio

LOG = userge.getLogger(__name__)

@userge.on_cmd(
    "circle",
    about={
        "header": "Convert video / gif / audio to video note",
        "usage": "{tr}circle [reply to media]",
    },
)
async def video_note(message: Message):
    """Covert to video note"""
    _cache_path = "userge/xcache/circle"
    _vid_path = _cache_path + "/temp_vid.mp4"
    reply = message.reply_to_message
    if not reply:
        await message.err("Reply to supported media", del_in=10)
        return
    if not (reply.video or reply.animation or reply.audio):
        await message.err("Only videos, gifs and audio are Supported", del_in=10)
        return
    if os.path.exists(_cache_path):
        rmtree(_cache_path, ignore_errors=True)
    os.mkdir(_cache_path)
    await message.edit("`Processing ...`")
    if reply.video or reply.animation:
        note = safe_filename(await reply.download())
        await crop_vid(note, _vid_path)
    else:
        thumb_loc = _cache_path + "/thumb.jpg"
        audio_loc = _cache_path + "/music.mp3"
        if reply.audio.thumbs:
            audio_thumb = reply.audio.thumbs[0].file_id
            thumb = await userge.download_media(audio_thumb)
        else:
            thumb = None
        music = await reply.download()
        os.rename(music, audio_loc)
        if thumb:
            os.rename(thumb, thumb_loc)
        else:
            await thumb_from_audio(audio_loc, thumb_loc)
        await runcmd(
            f"""ffmpeg -loop 1 -i {thumb_loc} -i {audio_loc} -c:v libx264 -tune stillimage -c:a aac -b:a 192k -vf \"scale=\'iw-mod (iw,2)\':\'ih-mod(ih,2)\',format=yuv420p\" -shortest -movflags +faststart {_vid_path}"""
        )
    if os.path.exists(_vid_path):
        await message.client.send_video_note(message.chat.id, _vid_path)
    await message.delete()
    rmtree(_cache_path, ignore_errors=True)

async def crop_vid(input_vid: str, final_path: str):
    stdout = (await runcmd(f'mediainfo --Output=JSON "{input_vid}"'))[0]
    if stdout is None:
        return
    try:
        out = ujson.loads(stdout)
    except Exception as e:
        LOG.error(str(e))
        return
    correct_aspect = True
    vid_valid = False
    if not (out and out.get("media") and out["media"].get("track")):
        return
    for stream in out["media"].get("track"):
        if stream["@type"] == "Video":
            vid_valid = True
            height_ = int(stream.get("Height", 0))
            width_ = int(stream.get("Width", 0))
            aspect_ratio_ = stream.get("DisplayAspectRatio")
            if aspect_ratio_:
                if aspect_ratio_ == "1":
                    correct_aspect = False
            elif width_ and height_:
                if width_ == height_ != 0:
                    correct_aspect = False
            else:
                return
            break
    if vid_valid:
        if correct_aspect:
            crop_by = min(width_, height_)
            await runcmd(
                f"""ffmpeg -i \'{input_vid}\' -vf \"crop={crop_by}:{crop_by}\" {final_path}"""
            )
            os.remove(input_vid)
        else:
            os.rename(input_vid, final_path)


import requests
import json
from bs4 import BeautifulSoup
import streamlit as st
import openai
import replicate
import os

BROWSERLESS_API_KEY = 'your key'
openai.api_key = 'your key'
os.environ["REPLICATE_API_TOKEN"] = "your key"

def scrape_website(url: str):
    headers = {'Cache-Control': 'no-cache', 'Content-Type': 'application/json'}
    data = {"url":url}
    data_json = json.dumps(data)
    post_url = f"https://chrome.browserless.io/content?token={BROWSERLESS_API_KEY}"
    response = requests.post(post_url, headers=headers, data=data_json)
    if response.status_code == 200:
        soup = BeautifulSoup(response.content, "html.parser")
        text = soup.get_text()
        if text != "" and text != None:
            return text
        else:
            st.text("Couldn't fetch article")
            return None

    else:
        print(f"HTTP request failed with status code {response.status_code}")

st.markdown("
Summarize Anything
", unsafe_allow_html=True)

# Centered text
st.markdown("
PASTE URL BELOW:
", unsafe_allow_html=True)

def generate_summary(article):
    response = openai.ChatCompletion.create(
        model="gpt-3.5-turbo",
        messages=[{"role": "user",
                   "content": f"summarize the following article only mentioning the main ideas, don't go on a seperate tangent, as if a reporter wrote it, use over 180 words: {article}"}],
        temperature=.88
    )
    total_tokens_used_c = response['usage']['total_tokens']
    price = (total_tokens_used_c / 1000) * .0015
    actual_response = response['choices'][0]['message']['content']
    return actual_response

def generate_images(text):
    response = openai.ChatCompletion.create(
        model="ft:gpt-3.5-turbo-0613:personal::7ytneB65",
        messages=[{"role": "system",
                   "content": "You are a prompt engineer who is tasked to write simple visual descriptions to encapsulate abstract text summaries. You will create 3 visual description prompts that take a summary and generate these prompts."},
                  {"role": "user","content": text}],
        temperature=.88
    )
    total_tokens_used_c = response['usage']['total_tokens']
    price = (total_tokens_used_c / 1000) * .0015
    actual_response = response['choices'][0]['message']['content']
    for i in range(2, 4):
        actual_response = actual_response.replace(f"{i}.", "#")
    p = actual_response.split("#")
    for i in p:
        output = replicate.run(
            "stability-ai/sdxl:d830ba5dabf8090ec0db6c10fc862c6eb1c929e1a194a5411852d25fd954ac82",
            input={
                "prompt": f"{i}, RAW candid cinema, 16mm, color graded portra 400 film, remarkable color, ultra realistic, textured skin",
                "negative_prompt": "scary, evil, horrible, ugly, disfigured, disgusting, destroyed"}
        )
        st.image(output[0])

url = st.text_input("")
if url != "":
    print(f"url: {url}")
    try:
        article_text = scrape_website(url)
        #st.write(article_text)
        if article_text != None:
            summary = generate_summary(article_text)

            st.title("Summary:")

            st.write(summary)
            st.text("Generating Images")
            generate_images(summary)
    except:
        st.text("ERROR: THIS ARTICLE CAN'T BE SUMMARIZED")


import cv2
import numpy
import face_recognition as fr
from time import sleep
from shutil import rmtree
from requests import post
from os import (listdir, mkdir)
from subprocess import (Popen, PIPE)
from check_encodings import get_list_of_encodings
from typing import (Dict, Tuple, BinaryIO, List, Union)

def take_picture(camera_pics_dir_path: str) -> None:
    camera_port: int = 0
    i: int
    camera: cv2.VideoCapture = cv2.VideoCapture(camera_port)

    mkdir(camera_pics_dir_path)
    # This value is subject to change.
    sleep(0.5)
    for i in range(0, 5):
        image: numpy.ndarray = camera.read()[1]
        cv2.imwrite(camera_pics_dir_path + f"/webcam_pic_{i}.png", image)
        sleep(0.05)

def check_person_with_images(camera_pics_dir_path: str, operating_system: str, user_name: str) -> None:
    image_name: str
    known_face_encodings: List[numpy.ndarray] = []
    matches: List[numpy.bool_]
    image_path: str

    if operating_system == "Windows":
        known_faces_dir: str = sys_output(operating_system, f"cd /Users/{user_name} && dir /AD /s /b KNOWN_FACES")
    else:
        known_faces_dir: str = sys_output(operating_system, "find", "/Users", "-name", "KNOWN_FACES")

    for image_name in listdir(known_faces_dir):
        if image_name != ".DS_Store":
            image: numpy.ndarray = fr.load_image_file(f"{known_faces_dir}/{image_name}")
            encoding: numpy.ndarray = fr.face_encodings(image)[0]
            known_face_encodings.append(encoding)

    matches, image_path = get_lowest_tolerance(camera_pics_dir_path, known_face_encodings)

    if not matches:
        print("Can not detect face.")
        rmtree(camera_pics_dir_path)
        return

    if False in matches:
        print("Someone is here...")
        send_notification("Door Opened", "Who was it?", image_path)

    rmtree(camera_pics_dir_path)

def check_person_with_trained_data(camera_pics_dir_path: str) -> None:
    matches: List[numpy.bool_]
    image_path: str
    matches, image_path = get_lowest_tolerance(camera_pics_dir_path)

    if not matches:
        print("Can not detect face.")
        rmtree(camera_pics_dir_path)
        return

    if False in matches:
        print("Someone is here...")
        send_notification("Door Opened", "Who was it?", image_path)

    rmtree(camera_pics_dir_path)

def get_lowest_tolerance(camera_pics_dir_path: str, encodings: List[numpy.ndarray] = None) -> \
        Tuple[List[numpy.bool_], str]:

    if encodings is None:
        encodings = get_list_of_encodings()

    i: int
    webcam_image_path: str = camera_pics_dir_path + "/webcam_pic_{}.png"
    face_encoding: numpy.ndarray
    current_tolerance: float
    lowest_tolerance: float = 0.70

    # This value is subject to change.
    tolerance: float = 0.50

    matches: List[numpy.bool_] = []
    track_index: int = 0

    for i in range(0, 5):
        face_cam_image: numpy.ndarray = fr.load_image_file(webcam_image_path.format(i))
        face_locations: List[Tuple] = fr.face_locations(face_cam_image)

        if not face_locations:
            continue

        face_encodings: List[numpy.ndarray] = fr.face_encodings(face_cam_image, face_locations, model="large")

        for face_encoding in face_encodings:
            current_tolerances: Union[numpy.ndarray, float] = fr.face_distance(encodings, face_encoding)
            for current_tolerance in current_tolerances:
                if current_tolerance < lowest_tolerance:
                    lowest_tolerance = current_tolerance
                    matches = list(current_tolerances <= tolerance)
                    track_index = i

    return matches, webcam_image_path.format(track_index)

def send_notification(title_text_message: str, body_text_message: str, image_path: str = "") -> None:
    user_token: str = ""
    user_key: str = ""
    image_file: Dict[str, Tuple[str, BinaryIO, str]] = {}

    if image_path:
        image_file = {"attachment": ("image.jpg", open(image_path, "rb"), "image/jpeg")}

    post(
        "https://api.pushover.net/1/messages.json",
        data={
            "token": user_token,
            "user": user_key,
            "html": 1,
            "title": title_text_message,
            "message": body_text_message
        },
        files=image_file
    )

    sleep(2)

def sys_output(os: str, *cmd: str) -> Union[str, List[str]]:
    if os == "Windows":
        p: Popen = Popen(cmd[0], shell=True, stdout=PIPE)
    else:
        p: Popen = Popen(cmd, stdout=PIPE)
    output: List[str] = p.communicate()[0].decode("ascii").strip().split("\n")
    if len(output) == 1:
        return output[0]
    else:
        return output


from concurrent import futures
import time
import grpc
import master_pb2
import master_pb2_grpc
from multiprocessing import Process, Manager
import queue
import json
from node import Node
import sys

sched_type = 'FIFO'   # FIFO/FAIR

def recordHeartBeat(q, file):
    with open(file, 'w') as f:
        while True:
            data = q.get()
            print("writing")
            data_str = json.dumps(data)
            f.write(data_str + "\n")
            f.flush()

class Master():
    def __init__(self):
        self.nodes = {}
        self.nodes_weights = {}
        self.nodes_vals = {}
        self.last_index = 0
        self.node_ids = []
        self.recodQ = Manager().Queue()

        self.free_nodes = set()
        self.taskid_nodeid = {}

        self.cur_tasks = []
        self.next_task_id = 1

    def getQueue(self):
        return self.recodQ

    def generateID(self, ip, port):
        return len(self.node_ids)

    def join(self, ip, port, weight):
        id = self.generateID(ip, port)
        if id in self.nodes:
            return False
        self.nodes[id] = Node(ip, port)
        self.nodes_weights[id] = weight
        self.nodes_vals[id] = weight
        self.node_ids.append(id)
        self.free_nodes.add(id)
        return id, True
    def leave(self, id):
        if id not in self.nodes:
            return False
        self.nodes.pop(id)
        return True

    def updateInfo(self, id, info):
        if id not in self.nodes:
            print("invalid ")
            return False
        self.recodQ.put(info)
        self.nodes[id].update(info)
        return True

    def scheduler(self, task_type, task_sz):
        index = self.free_nodes.pop()

        ip = self.nodes[index].getIp()
        port = self.nodes[index].getPort()
        print(ip, port)
        return ip, port,index

    # def scheduler(self, task_type, task_sz):
    #     next_index = -1
    #     wt = 0
    #     for id in self.nodes_vals.keys():
    #         if self.nodes_vals[id] > wt:
    #             next_index = id
    #             wt = self.nodes_vals[id]
    #     if next_index == -1:
    #         for id in self.nodes_vals.keys():
    #             self.nodes_vals[id] = self.nodes_weights[id]
    #         return self.scheduler(task_type, task_sz)
    #     self.nodes_vals[next_index] -= 1
    #
    #     index = self.node_ids[next_index]
    #     ip = self.nodes[index].getIp()
    #     port = self.nodes[index].getPort()
    #     print(ip, port)
    #     return ip, port

    def getNodesNum(self):
        return len(self.nodes)
class MasterService(master_pb2_grpc.MasterServicer):
    ms = Master()
    # worker node join to the cluster
    # send the server port and ip to master
    def Join(self, request, context):
        ip = request.ip
        port = request.port
        weight = request.weight
        print("new join ",ip, port)
        id, res = self.ms.join(ip, port, weight)
        print("join id is ", id)
        print("available nodes num = ", len(self.ms.free_nodes))
        return master_pb2.JoinReply(id = id,success = res)

    def Leave(self, request, context):
        id = request.id
        res = self.ms.leave(id)
        return master_pb2.LeaveReply(res)

    def PushState(self, request, context):
        id = request.id
        print("heartbeat id is ", id)
        info = {}
        info['id'] = request.id
        info['utx'] = request.utx
        info['cpu_usage'] = request.cpu_usage
        info['task_id'] = request.task_id
        info['io_perf'] = request.io_perf
        info['pro_perf'] = request.pro_perf
        info['task_st'] = request.task_st
        info['task_ed'] = request.task_ed

        # if request.task_st != 0:
            # print(request.task_id, request.io_perf, request.pro_perf, request.task_st, request.task_ed)
        # else:
            # print(id, request.utx ,request.cpu_usage)
        # print(id, request.io_perf, request.pro_perf)
        self.ms.updateInfo(id, info)
        return master_pb2.SateReply()

    # make weight high node to front
    def add_task_by_weight(self,task):
        l = len(self.ms.cur_tasks)
        t = 0
        for i in range(l-1,-1,-1):
            if(task[1]>self.ms.cur_tasks[i][1]):
                t = i
            else:
                break
        self.ms.cur_tasks.insert(t,task)
        return

    # send target server info to client
    def RouteGuide(self, request, context):
        # if self.ms.getNodesNum() == 0:
        #     return master_pb2.RouteReply(ip = "", port = 555, thread_num = 2)
        # print("available nodes num = ", len(self.ms.free_nodes))
        flag = request.task_id    # 0 request ,1 task finish

        if flag == 1 :
            task__id = request.task_type
            node_id = self.ms.taskid_nodeid[task__id]

            self.ms.free_nodes.add(node_id)
            print("task=",task__id,"is finished,now free server nums =",self.ms.free_nodes)
            return master_pb2.RouteReply(ip="true", port=0)
        else:

            task_type = request.task_type
            task_sz = request.task_sz
            task_id = self.ms.next_task_id

            self.ms.next_task_id += 1
            print("cur task is task id = ", task_id,", weight = ", task_sz)
            if (len(self.ms.free_nodes) > 0):
                next_ip, next_port,nodeid = self.ms.scheduler(task_type, task_sz)
                self.ms.taskid_nodeid[task_id] = nodeid
            else:
                if sched_type == 'FIFO':
                    self.ms.cur_tasks.append([task_id,task_sz])
                else:
                    self.add_task_by_weight([task_id,task_sz])
                print("Waiting : no available server ,add to the task queue...")
                while True:
                    if len(self.ms.free_nodes) > 0:
                        if self.ms.cur_tasks[0][0] == task_id:
                            next_ip, next_port, nodeid = self.ms.scheduler(task_type, task_sz)
                            self.ms.cur_tasks.pop(0)
                            self.ms.taskid_nodeid[task_id] = nodeid
                            break
                    time.sleep(2)   # wait for available nodes

            print("task id = ", task_id,", weight = ", task_sz," schedle to", next_ip, next_port)
            return master_pb2.RouteReply(ip = next_ip, port = task_id)

    def getQ(self):
        return self.ms.getQueue()

if __name__ == "__main__":
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
    service = MasterService()
    master_pb2_grpc.add_MasterServicer_to_server(service, server)
    #bind a port
    server.add_insecure_port('[::]:50051')
    server.start()
    recordP = Process(target = recordHeartBeat, args = (service.getQ(), "heartbeat"))
    recordP.start()
    try:
        while True:
            time.sleep(60*60*24)
    except KeyboardInterrupt:
        server.stop(0)


# control_backend/executor.py
# Created by Ethan Ramsay, 2017
# Contributors:
#   Ethan Ramsay
#   Matthew Marshall
#
# This file handles initialising the iNEMO inertial module chip.
# This chip contains an accelerometer and a gyroscope.

# Import dependencies
import os
import subprocess
from datetime import datetime

subprocess_handles = {}
all_stop_in_progress = False

def executeAllResume():
    global all_stop_in_progress
    all_stop_in_progress = False

def executeAllStop():
    global subprocess_handles
    global all_stop_in_progress
    all_stop_in_progress = True
    for key, val in subprocess_handles.iteritems():
        val.terminate()

def executeMotorCommand(command_data):
    command = ""
    if (command_data[0] == 1):
        command = "python /home/pi/Robotics/Drive_controller.py -f -v " + str(command_data[1]) + " -d " + str(command_data[2])
    elif (command_data[0] == 2):
        command = "python /home/pi/Robotics/Drive_controller.py -b -v " + str(command_data[1]) + " -d " + str(command_data[2])
    elif (command_data[0] == 3):
        command = "python /home/pi/Robotics/Drive_controller.py -l -a " + str(command_data[1])
    elif (command_data[0] == 4):
        command = "python /home/pi/Robotics/Drive_controller.py -r -a " + str(command_data[1])

    if (command == ""):
        return

    subprocess_handles["motors"] = subprocess.Popen(command, shell=True)

def executeArmCommand(command_data):
    command = ""
    if (command_data[0] == 1):
        command = "python /home/pi/Robotics/Arm_controller.py -s"
    elif (command_data[0] == 2):
        command = "python /home/pi/Robotics/Arm_controller.py -e"
    elif (command_data[0] == 3):
        command = "python /home/pi/Robotics/Arm_controller.py -i"
    elif (command_data[0] == 4):
        command = "python /home/pi/Robotics/Arm_controller.py -g"
    elif (command_data[0] == 5):
        command = "python /home/pi/Robotics/Arm_controller.py -d"
    elif (command_data[0] == 6):
        command = "python /home/pi/Robotics/Arm_controller.py -p " + str(command_data[1])
    elif (command_data[0] == 7):
        command = "python /home/pi/Robotics/Arm_controller.py -o " + str(command_data[1])

    if (command == ""):
        return

    subprocess_handles["arm"] = subprocess.Popen(command, shell=True)

def executeCameraCommand(command_data):
    command = ""
    if (command_data[0] == 1):
        command = "python /home/pi/Robotics/Camera_controller.py -p"
    elif (command_data[0] == 2):
        command = "python /home/pi/Robotics/Camera_controller.py -m -a " + str(command_data[1])

    if (command == ""):
        return

    subprocess_handles["camera"] = subprocess.Popen(command, shell=True)

def marshal_command(command_str):
    command_parts = command_str.split(" ")
    {
        -1: lambda : executeAllResume(),
         0: lambda : executeAllStop(),
         1: lambda : executeMotorCommand(command_parts[1:]),
         2: lambda : executeArmCommand(command_parts[1:]),
         3: lambda : executeCameraCommand(command_parts[1:])
    }[int(command_parts[0])]()

def run():
    while True:
        if os.path.isfile("/var/www/html/command-queue.dat"):
            with open("/var/www/html/command-queue.dat", "r") as handle:
                # Read in next command to be executed and marshal.
                command_str = handle.readline()
                if (command_str == ""):
                    continue
                marshal_command(command_str)

                # Don't change command queue as long as all stop in progress.
                if (all_stop_in_progress):
                    continue

                # Add command to command history.
                with open("/var/www/html/command-history.dat", "a+") as handle:
                    handle.write("{} {}".format(command_str, datetime.now()))

                # Remove command from command queue.
                subprocess.call(["tail -n +2 /var/www/html/command-queue.dat > /tmp/command-queue.dat.tmp && mv -f /tmp/command-queue.dat.tmp /var/www/html/command-queue.dat"], shell=True)

# Main
if __name__ == "__main__":
    run()


#web scrape anichart/MAL for anime
#https://myanimelist.net/anime/season
#https://anichart.net/Spring-2019
from animeClass import Anime
import requests
from bs4 import BeautifulSoup
url = 'https://myanimelist.net/anime/season'
page = requests.get(url)
soup = BeautifulSoup(page.text, 'html.parser')

def title_scraping():
    titlesoup = soup.find(class_='seasonal-anime-list js-seasonal-anime-list js-seasonal-anime-list-key-1 clearfix').find_all(class_='link-title')
    seasonal_titles = []
    for title in titlesoup:
        seasonal_titles.append(title.text)
    #print(seasonal_titles)
    return seasonal_titles

def episode_scraping():
    episodesoup = soup.find(class_='seasonal-anime-list js-seasonal-anime-list js-seasonal-anime-list-key-1 clearfix').find_all(class_='eps')
    #getting error about find vs find all, want the spans that contains episode #, although it seems pointless right now
    #spans = episodesoup.find_all('span')
    print(episodesoup)
    episode_count = []
    for episode in episodesoup:
        episode_count.append(episode.text)
    #print(episode_count)
    return episode_count
episode_scraping()
'''
#imported animeClass above and create an instance of Anime object with the values for one punch man
#testing the Anime object
OPM = Anime("One Punch Man", 12, "Action", 15)
print(OPM.get_Genre())
'''

#use soup to get all tags with titletext maybe?
#then put them in hashmap
#what do i want to do with the titles
#what other data do i want to store
#create an anime object maybe?

#One Punch Man 2nd Season
#class= "seasonal-anime-list js-seasonal-anime-list js-seasonal-anime-list-key-1 clearfix"

#endgame is graphing/visualization


"""
UCSD PHYS/SIO 111:
Ch3 Example of Gaussian Packet IC with d\eta/dt = 0

transaltion:Terrindeep Sandhu WI23
"""

import numpy as np
import matplotlib.pyplot as plt

grav = 9.81
Lx = 2e5        # total domain size here 40 km
nx = 2e6        # total number of grid points
dx = Lx / nx    # grid
xx = np.arange(0, Lx-dx, dx) - Lx/2   # x domain
k1 = 2 * np.pi / Lx
kk = k1 * np.concatenate((np.arange(0, nx/2), np.arange(-nx/2+1, 0)))   # wavenumbers

h = 10   # try h=1000, h=100, h=10 m depth
gaussian_width = 50

# dispersion relationship
omega = np.sqrt(grav * np.abs(kk) * np.tanh(np.abs(kk) * h))

# IC in physical space
Fx = (gaussian_width * np.sqrt(2 * np.pi))**(-1) * np.exp(-xx**2 / (2 * gaussian_width**2))

# IC in Fourier space because d\eta/dt = 0
Ak = np.fft.fft(Fx)

# make the eta IC plot
fig, axs = plt.subplots(nrows=3, ncols=1, figsize=(10, 15))
axs[0].plot(xx, Fx)
axs[0].set_xlabel('x (m)')
axs[0].set_ylabel(r'$\eta$ (m)')
axs[0].set_title(f'Initial Condition of a Gaussian hump, d$\eta$/dt=0: h={h:.2f} (m)')
axs[0].set_xlim([-5*gaussian_width, 5*gaussian_width])
ylims = [-0.5 * np.max(Fx), np.max(Fx)]

print('hit enter to start')
input()
for j in range(100):
    t1 = (j - 1) * 10
    Feta = Ak * np.exp(1j * omega * t1) + Ak * np.exp(-1j * omega * t1)
    etaa = np.fft.ifft(Feta)
    eta = np.real(etaa)

    axs[1].cla()
    axs[1].plot(xx, eta), axs[1].grid()
    axs[1].set_xlabel('x (m)')
    axs[1].set_ylabel(r'$\eta$ (m)')
    axs[1].set_title(f'Gaussian bump IC in deep water: t={t1:04d} (s)')
    axs[1].set_xlim([-Lx/10, Lx/10])
    axs[1].set_ylim(ylims)

    axs[2].cla()
    axs[2].plot(xx, eta), axs[2].grid()
    axs[2].set_xlabel('x (m)')
    axs[2].set_ylabel(r'$\eta$ (m)')
    axs[2].set_title(f'Blowup of middle panel: Gaussian bump IC in deep water: t={t1:04d} (s)')
    axs[2].set_xlim([-Lx/30, Lx/30])
    axs[2].set_ylim(ylims)

    plt.pause(0.25)

plt.show()


import sys
import numpy as np
import meshio
import sys

original_stdout = sys.stdout #Original standard output

#·# PARSE INPUT DATA
inputfile= str(input('\nEnter .msh file name/path (include .msh extension): '))
inputmsh = meshio.read(inputfile)
N_COORD  = inputmsh.points                    #Nodal coordinates
CONECT   = inputmsh.cells_dict["quad9"]  #Conectivities
NPE = 9   #Nodes per element in hexahedron

#·# BUILD CONNECTIVITIES AND COORDINATES MATRICES
# Build Connectivities Matrix
element_nodes = 1 + CONECT #pyth indices start in 0 & nªnod must start in 1
print('Element nodes')
print(element_nodes)
num_elements  = np.shape(element_nodes)[0] #Number of elements
print('\n'+18*'-'+' {0} 9-NODED QUAD ELEMENTS DETECTED '.format(num_elements)+18*'-'+'\n')
Elements = np.zeros([num_elements , NPE+1]) #Add aditional column(mat type)
Elements[:,0]  = 1  #1st Column = Layer /Material number
Elements[:,1:] = element_nodes   #Connectivity

print(Elements)

# Build Coordinates Matrix
Nodes = N_COORD
print(Nodes)

#·# WRITE OUTPUT (.txt file)
with open('Connectivities_and_Coordinates_Eq2D.txt','w') as f:
    sys.stdout = f #Change the standard output to the file created

    print('\nMACRO_MODEL.Conectivity = [\n')
    #np.set_printoptions(threshold=sys.maxsize)
    #print(Elements)
    for row in range(0, np.shape(Elements)[0]):
        print('{0:>6}\t{1:>6}\t{2:>6}\t{3:>6}\t{4:>6}\t{5:>6}\t{6:>6}\t{7:>6}\t{8:>6}\t{9:>6}'.format(int(Elements[row,0]),int    (Elements[row,1]),int(Elements[row,2]),int(Elements[row,3]),int(Elements[row,4]),int(Elements[row,5]),int(Elements[row,6]),int(Elements[row,7]),int(Elements[row,8]),int(Elements[row,9])))

    print('\n];')
    print('\nMACRO_MODEL.Coordinates = [\n')
    for row in range(0, np.shape(Nodes)[0]):
        print('{0:>6}\t{1:>6}'.format(Nodes[row,0],Nodes[row,1]))
    print('\n];')


import json
import numpy as np
import sqlite_utils
from sqlite_utils.db import Table
from typing import cast, List, Tuple, Optional, Union, Dict, Any
from .base_vectordb import VectorDatabase
from nchain.models.core import Entry
from annoy import AnnoyIndex
import struct
import time
import os

def encode(values):
    return struct.pack("<" + "f" * len(values), *values)

def encode_as_numpy(values):
    return np.array(values, dtype=np.float32).tobytes()

def decode(binary):
    return struct.unpack("<" + "f" * (len(binary) // 4), binary)

def decode_from_numpy_to_list(binary):
    numpy_array = np.frombuffer(binary, dtype=np.float32)
    return numpy_array.tolist()

class SQLiteVectorDB(VectorDatabase):

    def __init__(self, dimension: int, indexdb_path: str, db_path=None, metric: str = "angular", collection:str="default"):
        self.dimension = dimension
        self.indexdb_path = indexdb_path
        if db_path:
            self.db = sqlite_utils.Database(db_path)
        else:
            self.db = sqlite_utils.Database(memory=True)
        # Create the 'collections' table
        self.db["collections"].create({
            "id": int,     # unique identifier for each collection
            "name": str,   # name of the collection
            "model": str,   # identifier (or model ID) for the embedding model associated with the collection
            "dimension": int,
            "metric": str,
            "indexed_at":int,
            "stored_at": int,
        }, pk="id", not_null=["name", "model"], if_not_exists=True)
        self.db["collections"].create_index(["name"], unique=True, if_not_exists=True)

        # Create the 'embeddings' table
        self.db["embeddings"].create({
            "collection_id": int,     # foreign key to reference the 'collections' table
            "id": int,                # identifier for the entry
            "embedding": "BLOB",      # embedded vector
            "content": str,           # content in text format
            "content_blob": "BLOB",   # content in binary format
            "content_hash": "BLOB",   # hash of the content for deduplication
            "metadata": str,          # additional metadata
            "updated": int            # timestamp
        }, pk=("id"), foreign_keys=[("collection_id", "collections", "id")], not_null=["collection_id", "id"], if_not_exists=True)
        rows = list(self.db["collections"].rows_where("name = ?", [collection]))
        if len(rows) > 0:
            self.collection = rows[0]
            self.collection["metric"] = metric
            self.db["collections"].update(self.collection["id"], {"metric":metric} )
        else:
            collection_row = {
                "name": collection,
                "model": "SentenceTransformers",
                "dimension": dimension,
                "metric": metric,
                "indexed_at": None,
                "stored_at": None
            }
            self.db["collections"].insert(collection_row, replace=True)
            self.collection = list(self.db["collections"].rows_where("name = ?", [collection]))[0]
        if  os.path.exists(indexdb_path):
            self.index = AnnoyIndex(self.dimension, self.collection["metric"])
            self.index.load(indexdb_path)  # Load the Annoy index from disk
            self.vectors_updated = False
        else:
            self.vectors_updated = True

    def store_vectors(self, vectors: List[List[float]], metadata_list: List[dict], chunks: list[Union[str, bytes]], store:bool) -> None:
        hashes = [self.content_hash(content) for content in chunks]
        # Create a mapping of hash to its index for quick look-up
        hash_to_index = {hash: index for index, hash in enumerate(hashes)}

        # Fetch rows that match the provided hashes
        matching_rows = self.db.query(
            """
            select id, content_hash from embeddings
            where collection_id = ? and content_hash in ({})
            """.format(",".join("?" for _ in hashes)),
            [self.collection["id"]] + hashes
        )

        # Get the hash_index for each matching row
        existing_ids = [hash_to_index[row['content_hash']] for row in matching_rows]
        filtered_items = [item + (hashes[idx], ) for idx, item in enumerate(zip(vectors, metadata_list, chunks)) if idx not in existing_ids]
        if len(filtered_items) > 0:
            with self.db.conn:
                cast(Table, self.db["embeddings"]).insert_all(
                    (
                        {
                            "collection_id": self.collection["id"],
                            "embedding": encode_as_numpy(embedding),
                            "content": content
                            if (store and isinstance(content, str))
                            else None,
                            "content_blob": content
                            if (store and isinstance(content, bytes))
                            else None,
                            "content_hash": content_hash,
                            "metadata": json.dumps(metadata) if metadata else None,
                            "updated": int(time.time()),
                        }
                        for (embedding, metadata, content, content_hash) in filtered_items

                    ),
                    replace=True,
                )

            self.collection["stored_at"] = int(time.time())
            self.db["collections"].update(self.collection["id"], {"stored_at":self.collection["stored_at"]})
            self.vectors_updated = True

    def build_index(self):
        self.index = AnnoyIndex(self.dimension, self.collection["metric"])
        for row in self.db["embeddings"].rows_where():
            numpy_array = np.frombuffer(row["embedding"], dtype=np.float32)
            self.index.add_item(row["id"], numpy_array.tolist())
        self.index.build(10)
        self.collection["indexed_at"] = int(time.time())
        self.db["collections"].update(self.collection["id"], {"indexed_at":self.collection["indexed_at"]})
        print("Index built", self.indexdb_path)
        self.index.save(self.indexdb_path)

    def refresh_index(self):
        if self.collection["stored_at"]:
            if self.collection["indexed_at"] is None or self.collection["stored_at"] > self.collection["indexed_at"]:
                self.build_index()

        self.vectors_updated = False

    def search_vectors(self, query_vector: List[float], top_k: int) -> List[Entry]:
        if self.vectors_updated:
            self.refresh_index()

        indices, distances = self.index.get_nns_by_vector(query_vector, top_k, include_distances=True)
        results = []
        for idx, distance in zip(indices, distances):
            row = self.db["embeddings"].get(idx)
            metadata = json.loads(row["metadata"])
            similar_entry = Entry(id=row["id"], score=distance, content=row["content"], metadata=metadata)
            results.append(similar_entry)
        return results

    def update_vector(self, index: int, new_vector: List[float], new_metadata: Optional[dict] = None) -> None:
        # Convert vector to BLOB
        numpy_array = np.array(new_vector, dtype=np.float32)
        blob = numpy_array.tobytes()
        # Update in SQLite
        self.db["vectors"].update(index, {"vector": blob, "metadata": json.dumps(new_metadata or {})})

        # Update in Annoy
        self.index.add_item(index, new_vector)
        self.index_built = False

    def delete_vector(self, index: int) -> None:
        self.db["vectors"].delete_where("id = ?", [index])
        self.vector_updated = True
        # Note: Deletion in Annoy isn't straightforward. One way is to rebuild the index excluding the deleted vector.
        # However, for simplicity, we're not handling it here. The deleted vector may still appear in search results.


import pyparsing as pp
from operator import mul

def define_numeric_word_range(
    names: str, from_: int, to_: int = None, step: int = 1
) -> pp.MatchFirst:
    """
    Compose a MatchFirst of CaselessKeywords, given their names and values,
    which when parsed, are converted to their value
    """

    def define_numeric_word(nm: str, val: int):
        return pp.CaselessKeyword(nm).add_parse_action(lambda: val)

    names = names.split()
    if to_ is None:
        to_ = from_
    values = range(from_, to_ + 1, step)
    ret = pp.MatchFirst(
        define_numeric_word(name, value) for name, value in zip(names, values)
    )

    if len(names) == 1:
        ret.set_name(names[0])
    else:
        ret.set_name(f"{names[0]}-{names[-1]}")

    return ret

def multiply(t):
    """
    Parse action for hundreds and thousands.
    """
    return mul(*t)

opt_dash = pp.Opt(pp.Suppress("-")).set_name("'-'")
opt_and = pp.Opt((pp.CaselessKeyword("and") | "-").suppress()).set_name("'and/-'")

units = define_numeric_word_range("one two three four five six seven eight nine", 1, 9)
teens_only = define_numeric_word_range(
    "eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen",
    11,
    19,
)
ten = define_numeric_word_range("ten", 10)
teens = ten | teens_only

tens = define_numeric_word_range(
    "twenty thirty forty fifty sixty seventy eighty ninety", 20, 90, 10
)
one_to_99 = (units | teens | (tens + pp.Opt(opt_dash + units))).set_name("1-99")
one_to_99.add_parse_action(sum)

hundred = define_numeric_word_range("hundred", 100)
thousand = define_numeric_word_range("thousand", 1000)

hundreds = (units | teens_only | (tens + opt_dash + units)) + hundred
hundreds.set_name("100s")

one_to_999 = (
    (pp.Opt(hundreds + opt_and) + one_to_99 | hundreds).add_parse_action(sum)
).set_name("1-999")

thousands = one_to_999 + thousand
thousands.set_name("1000s")

# for hundreds and thousands, must scale up (multiply) accordingly
hundreds.add_parse_action(multiply)
thousands.add_parse_action(multiply)

numeric_expression = (
    pp.Opt(thousands + opt_and) + pp.Opt(hundreds + opt_and) + one_to_99
    | pp.Opt(thousands + opt_and) + hundreds
    | thousands
).set_name("numeric_words")

# sum all sub-results into total
numeric_expression.add_parse_action(sum)

if __name__ == "__main__":
    numeric_expression.run_tests(
        """
        one
        seven
        twelve
        twenty six
        forty-two
        two hundred
        twelve hundred
        one hundred and eleven
        ninety nine thousand nine hundred and ninety nine
        nine hundred thousand nine hundred and ninety nine
        nine hundred and ninety nine thousand nine hundred and ninety nine
        nineteen hundred thousand nineteen hundred and ninety nine

        # invalid
        twenty hundred
        """,
        postParse=lambda _, s: "{:,}".format(s[0]),
    )

    # create railroad diagram
    numeric_expression.create_diagram("numeric_words_diagram.html", vertical=5)


import logging.config
import os
import uuid

import pandas as pd
import pyspark.sql.functions as F
from pyspark.ml import PipelineModel
from pyspark.sql import SparkSession

from examples_utils import get_dataset_attrs, prepare_test_and_train, get_spark_session
from lightautoml.spark.automl.presets.tabular_presets import SparkTabularAutoML
from lightautoml.spark.dataset.base import SparkDataset
from lightautoml.spark.tasks.base import SparkTask
from lightautoml.spark.utils import log_exec_timer, logging_config, VERBOSE_LOGGING_FORMAT

logging.config.dictConfig(logging_config(level=logging.INFO, log_filename='/tmp/slama.log'))
logging.basicConfig(level=logging.DEBUG, format=VERBOSE_LOGGING_FORMAT)
logger = logging.getLogger(__name__)

# NOTE! This demo requires datasets to be downloaded into a local folder.
# Run ./bin/download-datasets.sh to get required datasets into the folder.

def main(spark: SparkSession, dataset_name: str, seed: int):
    # Algos and layers to be used during automl:
    # For example:
    # 1. use_algos = [["lgb"]]
    # 2. use_algos = [["lgb_tuned"]]
    # 3. use_algos = [["linear_l2"]]
    # 4. use_algos = [["lgb", "linear_l2"], ["lgb"]]
    use_algos = [["lgb", "linear_l2"], ["lgb"]]
    cv = 5
    path, task_type, roles, dtype = get_dataset_attrs(dataset_name)

    with log_exec_timer("spark-lama training") as train_timer:
        task = SparkTask(task_type)
        train_data, test_data = prepare_test_and_train(spark, path, seed)

        test_data_dropped = test_data

        # optionally: set 'convert_to_onnx': True to use onnx-based version of lgb's model transformer
        automl = SparkTabularAutoML(
            spark=spark,
            task=task,
            general_params={"use_algos": use_algos},
            lgb_params={'use_single_dataset_mode': True, 'convert_to_onnx': False, 'mini_batch_size': 1000},
            reader_params={"cv": cv, "advanced_roles": False}
        )

        oof_predictions = automl.fit_predict(
            train_data,
            roles=roles
        )

    logger.info("Predicting on out of fold")

    score = task.get_dataset_metric()
    metric_value = score(oof_predictions)

    logger.info(f"score for out-of-fold predictions: {metric_value}")

    transformer = automl.make_transformer()

    automl.release_cache()

    with log_exec_timer("spark-lama predicting on test (#1 way)") as predict_timer:
        te_pred = automl.predict(test_data_dropped, add_reader_attrs=True)

        score = task.get_dataset_metric()
        test_metric_value = score(te_pred)

        logger.info(f"score for test predictions: {test_metric_value}")

    with log_exec_timer("spark-lama predicting on test (#2 way)"):
        te_pred = automl.make_transformer().transform(test_data_dropped)

        pred_column = next(c for c in te_pred.columns if c.startswith('prediction'))
        score = task.get_dataset_metric()
        test_metric_value = score(te_pred.select(
            SparkDataset.ID_COLUMN,
            F.col(roles['target']).alias('target'),
            F.col(pred_column).alias('prediction')
        ))

        logger.info(f"score for test predictions: {test_metric_value}")

    base_path = "/tmp/spark_results"
    automl_model_path = os.path.join(base_path, "automl_pipeline")
    os.makedirs(base_path, exist_ok=True)

    with log_exec_timer("saving model") as saving_timer:
        transformer.write().overwrite().save(automl_model_path)

    with log_exec_timer("Loading model time") as loading_timer:
        pipeline_model = PipelineModel.load(automl_model_path)

    with log_exec_timer("spark-lama predicting on test (#3 way)"):
        te_pred = pipeline_model.transform(test_data_dropped)

        pred_column = next(c for c in te_pred.columns if c.startswith('prediction'))
        score = task.get_dataset_metric()
        test_metric_value = score(te_pred.select(
            SparkDataset.ID_COLUMN,
            F.col(roles['target']).alias('target'),
            F.col(pred_column).alias('prediction')
        ))

    logger.info(f"score for test predictions via loaded pipeline: {test_metric_value}")

    logger.info("Predicting is finished")

    result = {
        "seed": seed,
        "dataset": dataset_name,
        "used_algo": str(use_algos),
        "metric_value": metric_value,
        "test_metric_value": test_metric_value,
        "train_duration_secs": train_timer.duration,
        "predict_duration_secs": predict_timer.duration,
        "saving_duration_secs": saving_timer.duration,
        "loading_duration_secs": loading_timer.duration
    }

    print(f"EXP-RESULT: {result}")

    train_data.unpersist()
    test_data.unpersist()

    return result

def multirun(spark: SparkSession, dataset_name: str):
    seeds = [1, 5, 42, 100, 777]
    results = [main(spark, dataset_name, seed) for seed in seeds]

    df = pd.DataFrame(results)

    with pd.option_context('display.max_rows', None, 'display.max_columns', None):
        print(df)

    df.to_csv(f"spark-lama_results_{dataset_name}_{uuid.uuid4()}.csv")

if __name__ == "__main__":
    spark_sess = get_spark_session()
    # One can run:
    # 1. main(dataset_name="lama_test_dataste", seed=42)
    # 2. multirun(spark_sess, dataset_name="lama_test_dataset")
    main(spark_sess, dataset_name="lama_test_dataset", seed=42)

    spark_sess.stop()


import logging

import matplotlib.pyplot as plt

from sklearn.metrics import precision_recall_curve

from code.metrics.binary_f1 import BinaryMetric

class BinaryPRCurve(BinaryMetric):
    def __init__(self):
        super(BinaryPRCurve, self).__init__()

    def get_metric(self, plot: bool = False):
        p, r, threshold = precision_recall_curve(self.goldens, self.predicts)

        format = lambda name, values: "%s: " % name + ",".join(map(str, values.tolist()))
        logging.info(format("p", p))
        logging.info(format("r", r))
        logging.info(format("threshold", threshold))

        if plot:
            plt.figure("P-R Curve")
            plt.title("Precision/Recall Curve")
            plt.xlabel("Recall")
            plt.ylabel("Precision")
            plt.plot(r, p)
            plt.show()
            plt.savefig("P-R Curve.png")

        max_threshold = 0.
        max_f1 = 0.
        for cur_p, cur_r, thre in zip(p, r, threshold):
            f1 = 2*cur_p*cur_r / (cur_p + cur_r)
            if f1 >= max_f1:
                max_f1 = f1
                max_threshold = thre
        logging.info("max_threshold: %f, max_f1: %f" % (max_threshold, max_f1))
        print("max_threshold: %f, max_f1: %f" % (max_threshold, max_f1))


import numpy as np
import tut_basics as tb

from plot_fxns import plotWorld, plotStateValue, plotStateActionValue, plotPolicyPi, plotGreedyPolicyQ
from rl_agents import RLAgent, RLExampleAgent
import importlib
import random

class KnightWorld(tb.GridWorld):
    def __init__(self):
        # array of possible moves
        self.actionlist = np.array(['WSW','SSW','SSE','ESE','ENE','NNE','NNW','WNW'])
        action_dict = {'WSW':0,'SSW':1,'SSE':2,'ESE':3,'ENE':4,'NNE':5,'NNW':6,'WNW':7}
        # boad shape
        shape = (5,5)
        # starting position
        start = (0,2)
        terminals = []
        # previously occupied squares
        self.obstacles = []
        # coordinates:reward of selected squares
        rewards = {(3,3):10}
        jumps = {}
        super(KnightWorld, self).__init__(shape, start, terminals, self.obstacles, rewards, jumps)

# build and plot world
importlib.reload(tb)
world = KnightWorld()
plotWorld(world)

idx = np.arange(len(world.actionlist))[world.get_actions()]
actionlist = np.array(['WSW','SSW','SSE','ESE','ENE','NNE','NNW','WNW'])

class squire_agent(object):

    # remember
    # -- world.get_actions() returns an array of 8 bool values, each meaning whether the ith action is valid

    # -- world.stateFilter(coord) takes a 2d coordinate and returns an array containing 8 integers,
    #    each representing the square the knight would move onto if it took the ith action from state coord.
    #    (-1 means the ith action is invalid)

    # -- world.potMov contains 8 integers, each representing the square the knight would move onto
    #    if it took ith action (-1 if the action is invalid).
    #    It is equivalent to world.stateFilter(tb.one2twoD(agent.state)).

    # -- agent.valid_moves(state, actions) returns an array of actions that are both valid and taht do not move
    #    onto a previously occupied square.

    actionlist = np.array(['WSW','NNW','SSE','ESE','SSW','WNW','ENE','NNE'])
    action_dict = {'WSW':0,'NNW':1,'SSE':2,'ESE':3,'SSW':4,'WNW':5,'ENE':6,'NNE':7}
    pConv = {0:4,1:0,2:1,3:5,4:7,5:3,6:2,7:6}
    myconv = {'ENE': 7, 'ESE': 5, 'NNE': 3, 'NNW': 2, 'SSE': 1, 'SSW': 0, 'WNW': 6, 'WSW': 4}
    def __init__(self, world):
        self.world = world
        #initialize q table to zeros
        self.q = np.zeros((self.world.nstates, len(actionlist)))

    def display_env(self):
        # prints a grid displaying current environment
        # agent location (1), never occupied squares (0) and previously occupied squares (-1)
        grid = np.zeros((5,5))
        grid[np.floor(self.world.get_state()/5).astype(int)][self.world.get_state()%5] = 1
        for obstacle in self.world.obstacles:
            grid[np.floor(obstacle/5).astype(int)][obstacle%5] = -1
        return grid

    def valid_moves(self, state):
        #returns array of valid moves
        valid_actions = [action for action in self.actionlist if self.world.potMov[self.myconv[action]] != -1]
        valid_actions = [action for action in valid_actions if self.world.potMov[self.myconv[action]] not in self.world.obstacles]
        return valid_actions

    def get_valid_actions(self):
        #returns array of bools (used for operations on q)
        arr = [False for i in range(8)]
        idx = [self.action_dict[move] for move in self.valid_moves(self.state)]
        for i in idx:
            arr[i] = True
        return arr

    def warnsdorffPolicy(self, state, actions):
        # replicates Warnsdorff's rule, by returning the action bringing the knight to the square with the
        # fewest onward moves.

        #TO DO: fix bug when the knight reaches square no. 20

        available_actions = self.valid_moves(state, actions)
        print(available_actions)
        pConv = {0:4,1:0,2:1,3:5,4:7,5:3,6:2,7:6}
        if len(available_actions) != 0:
            min_new_moves = 1000
            for i, action in enumerate(available_actions):
                new_cord = self.world.stateFilter(tb.oneD2twoD(state, self.world.shape))[pConv[self.world.action_dict[action]]]
                new_cord_moves = self.world.stateFilter(tb.oneD2twoD(new_cord, self.world.shape))
                new_cord_moves = [move for move in new_cord_moves if move != -1 and move not in self.world.obstacles + [state]]
                print(f"Action {action} would bring us to {tb.oneD2twoD(new_cord, self.world.shape)}, which has {len(new_cord_moves)} possible moves, that is {new_cord_moves}")
                if len(new_cord_moves) < min_new_moves:
                    min_new_moves = len(new_cord_moves)
                    best_action = action
                elif len(new_cord_moves) == min_new_moves:
                    if random.choice([True, False]):
                        min_new_moves = len(new_cord_moves)
                        best_action = action

            return best_action

        else:
            return -1

    def reset(self):
        # resets Knight World and agent state
        self.world.init()
        self.state = self.world.get_state()

    def randPolicy(self, state, actions):
        # returns a random action bringing the knight to a never occupied square

        #np.random.choice() is a uniform distribution over actions
        #print(f"<randpolicy> actions: {actions}")
        #print(f"<randpolicy> actionlist: {actionlist}")
        pConv = {0:4,1:0,2:1,3:5,4:7,5:3,6:2,7:6}
        available_actions = self.actionlist[actions]
        #filAct = []
        print(f"================\n\nNEW MOVE!\n\n-@@BASE INFO\n--CURRENT SQUARE: {self.world.state}\n--POTENTIAL MOVES: {self.world.potMov}\n--OBSTACLES: {self.world.obstacles}\n\n-@@POLICY STAGE\n--AVAILABLE ACTIONS: {available_actions}")
        expAct = {}
        for eachAction in available_actions:
            #print(f"\nCURRENT STATE: {self.world.state}")
            #print(f"ACTION: {eachAction}")
            #print(f"THIS WOULD TAKE US TO: {self.world.potMov[pConv[self.world.action_dict[eachAction]]]}")
            #print(f"{self.world.obstacles}")
            if self.world.potMov[pConv[self.world.action_dict[eachAction]]] < 0:
                expAct[eachAction] = f"Was {self.world.potMov[pConv[self.world.action_dict[eachAction]]]}. Negative value!"
                available_actions = np.delete(available_actions, np.where(available_actions == eachAction))
            for x in self.world.obstacles:
                if self.world.potMov[pConv[self.world.action_dict[eachAction]]] == x:
                    expAct[eachAction] = f"Was {self.world.potMov[pConv[self.world.action_dict[eachAction]]]}. Already used!"
                    available_actions = np.delete(available_actions, np.where(available_actions == eachAction))
        for eachEntry in expAct:
            print(f"--{eachEntry} -> {expAct[eachEntry]}")
        print(f"--FILTERED ACTIONS: {available_actions}\n--BREAKDOWN: ")
        for eachAction in available_actions:
            print(f"--{eachAction}: {self.world.potMov[pConv[self.world.action_dict[eachAction]]]}")
        if len(available_actions) != 0:
            return np.random.choice(available_actions)
        else:
            self.take_action()

    def greedyQPolicy(self, state, actions):
        # returns action with the highest expected reward based on Q table
        idx = np.arange(8)[actions]
        return self.actionlist[idx[np.argmax(self.q[state,actions])]]

    def epsilongreedyQPolicy(self, state, actions, epsilon=0.2):
        # returns with probability epsilon a random (not greedy) action
        # and with probability 1 - epsilon the greedy action
        print("Inside epsilongreedyQPolicy")
        print(f"State = {state}")
        print(f"Actions = {actions}")
        valid_moves = self.valid_moves(state)
        print(f"Valid actions = {valid_moves}")
        print(self.display_env())
        idx = np.arange(len(world.actionlist))[actions]
        _q = self.q[state,actions]
        _max  = np.argmax(_q)
        _idx = idx[_max]
        # greedy_action = self.actionlist[idx[np.argmax(self.q[state,actions])]]
        greedy_action = self.actionlist[_idx]
        nongreedy_actions = np.delete(self.actionlist[actions],np.argwhere(self.actionlist==greedy_action))
        r = np.random.rand()
        for c in range(len(nongreedy_actions)):
            if (r<((c+1)*epsilon/len(nongreedy_actions))):
                return nongreedy_actions[c]
        return greedy_action

    def choose_action(self, off_pol):
        # handles action choice based on whether the agent is supposed to exploit its environment
        # (following the learned Q policy) or also explore it

        state = self.world.get_state()
        actions = self.get_valid_actions()
        pConv = {0:4,1:0,2:1,3:5,4:7,5:3,6:2,7:6}
        if off_pol:
            self.action = self.greedyQPolicy(state,actions)
        else:
            self.action = self.epsilongreedyQPolicy(state, actions)
        #self.action = self.warnsdorffPolicy(state, actions)
        print(f"\n-@@ACTION SELECTION\n--WE CHOSE: {self.action}, WHICH TAKES US TO {self.world.potMov[pConv[self.world.action_dict[self.action]]]}")
        #if self.action
        return self.action

    def take_action(self, action):
        # changes the state by taking action
        if action == -1:
            raise(ValueError("Impossible action taken"))
            return self.state, 0, True
        (self.state, self.reward, terminal) = self.world.move(action)
        #print(self.valid_moves(self.state))
        if self.valid_moves(self.state) == []:
            terminal = True
        return self.state, self.reward, terminal

    def evaluatePolicyQ(self, gamma, alpha, ntrials):
        # based on ntrials attempts, update the Q table with alpha as learning rate and gamma as discount factor
        delta = 1.0 # what is delta for?
        old_q = self.q
        for i in range(ntrials):
            is_terminal = False
            c = 0
            self.reset()
            s = self.state
            #a = self.choose_action(off_pol = False)

            # explore from the epsilon-greedy policy

            while not is_terminal:
                c += 1
                #print(agent.display_env())
                a_prime = self.choose_action(off_pol = False) if not is_terminal else 'D'
                # below we choose a prime from the TRUE policy!
                a_greedy = self.choose_action(off_pol = True) if not is_terminal else 'D'
                # taking an action gives terminality status.
                self.state, self.reward, is_terminal = self.take_action(a_prime)

                #update
                self.q[s, self.action_dict[a_prime]] += alpha*(self.reward + gamma * self.q[self.state, self.action_dict[a_greedy]] - self.q[s, self.action_dict[a_prime]])

                s = self.state

            #delta = np.min(np.max(np.abs(self.q - old_q)),delta)
            old_q = self.q

    def policyIteration(self, gamma, alpha, ntrials):
        # iterates the policy until it is stable
        print("Running TD policy iteration...")
        policyStable = False
        itr = 0
        maxiters = 1000 # catch the while loop.
        oldA = np.zeros((self.world.nstates,))
        while (not policyStable and itr < maxiters):
            self.display_env()
            itr += 1
            self.evaluatePolicyQ(gamma, alpha, ntrials)
            policyStable = np.array_equal(oldA,np.argmax(self.q, axis=1)) # see if policy changes!
            oldA = np.argmax(self.q, axis=1)
        print("Converged after {} iterations.".format(itr))
        print(self.q)

if __name__ == "__main__":
    #create world
    importlib.reload(tb)
    world = KnightWorld()
    agent = squire_agent(world)
    state = world.get_state()
    potential_moves = world.stateFilter(tb.oneD2twoD(state, world.shape))

    #create agent
    agent = squire_agent(world)
    num_trials = 10
    num_steps = 1
    gamma = 0.99
    alpha = 0.2
    reward_tracker = []
    agent.policyIteration(gamma, alpha, num_trials)


#!/usr/bin/env python2

import psycopg2 as pg

# Open Connection
connection = pg.connect(database="news")
cursor = connection.cursor()

# Execute and Fetch Top 3. (Table View top3art)
cursor.execute("SELECT * FROM top3artic")
top3 = cursor.fetchall()

# Check for Empty Results
if top3 is not None:
  print("Top 3 Articles:")
  for elem in top3:
    print("Article: {0}, Views: {1}".format(elem[0], elem[1]))
else:
  print("No Articles have been accessed.")

# Close cursor and connection
cursor.close()
connection.close()


import pickle
import os
import cv2
import numpy as np
data_folder = 'training_data/'
labels = os.listdir(data_folder)
imsize = (85,48,0)
X = np.array([], dtype=np.uint8).reshape(170,96,0)
y = []
temp = np.array([], dtype=np.uint8).reshape(170,96,0)
for label in labels:
	images = os.listdir(data_folder+label)
	for i in range(9000):
		img = cv2.imread(data_folder+label+'/'+str(i)+'.jpg')
		img = cv2.resize(img, (96, 170))
		img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
		temp = np.dstack((temp,img))
		y.append(label)
		if(i%500 == 0):
			print(i)
			X = np.dstack((X,temp))
			temp = np.array([], dtype=np.uint8).reshape(170,96,0)
	X = np.dstack((X,temp))
	temp = np.array([], dtype=np.uint8).reshape(170,96,0)

print(X.shape)
print(len(y))
print(y[0])
f = open('database_mid.pickle','wb')
pickle.dump([X,y],f)
f.close()