import binascii
import enum
import json
import mobiclip
import MySQLdb
import nlzss
import os
import sqlite3
import struct
import sys
import textwrap
from ninfile import NinchDllist
from ninfile2 import GameTDB
from ninfile3 import *

with open("./config.json", "rb") as f:
    config = json.load(f)

def u8(data):
    if not 0 <= data <= 255:
        print("u8 out of range: %s" % data, "INFO")
        data = 0
    return struct.pack(">B", data)

def u16(data):
    if not 0 <= data <= 65535:
        print("u16 out of range: %s" % data, "INFO")
        data = 0
    return struct.pack(">H", data)

def u32(data):
    if not 0 <= data <= 4294967295:
        print("u32 out of range: %s" % data, "INFO")
        data = 0
    return struct.pack(">I", data)

def enc(text, length):
    name_fixes = {
        "Take-Two Interactive / GameTek / Rockstar Games / Global Star Software": "Take-Two Interactive",
        "Sierra Entertainment / Vivendi Games / Universal Interactive Studios": "Sierra Entertainment",
        "Marvelous Entertainment / Victor Entertainment / Pack-In-Video / Rising Star Games": "Marvelous Entertainment",
        "Take-Two Interactive / Global Star Software / Gotham Games / Gathering of Developers": "Take-Two Interactive",
    }

    if text in name_fixes:
        text = name_fixes[text]

    if len(text) > length:
        print(len(text))
        print("Error: Text too long.")
        print(text)
        # sys.exit(1)
    return text.encode("utf-16be").ljust(length, b"\0")[:length]

def enc_utf_8(text, length):
    if len(text) > length:
        print("Error: Text too long.")
        sys.exit(1)
    return text.encode("utf-8").ljust(length, b"\0")[:length]

class MakeDList:
    def __init__(self, databases):
        self.header = {}
        self.databases = databases
        self.make_header()
        self.write_ratings_table()
        self.write_title_types()
        self.write_company_table()
        self.write_title_table()
        self.write_new_title_table()
        self.write_videos()
        self.write_new_video_table()
        self.write_demos()
        self.write_recommendations()
        self.write_recent_recommendation_table()
        self.write_pop_videos()
        self.write_rating_images()
        self.write_detailed_ratings_table()
        self.write_file()

    def offset_count(self):
        """
        This function returns the offset of where the selected table is
        """
        return sum(len(values) for values in list(self.header.values()) if values)

    def make_header(self):
        """Creates the header for the DlList"""
        self.header["unknown"] = u16(0)
        self.header["version"] = u8(6)
        self.header["unknownRegion"] = u8(2)
        self.header["filesize"] = u32(0)
        self.header["crc32"] = u32(0)
        self.header["dllistid"] = u32(0x14454D5B)
        self.header["thumbnailId"] = u32(1)
        self.header["countryCode"] = u32(49)
        self.header["languageCode"] = u32(1)
        self.header["unknown2_0"] = u8(1)
        self.header["unknown2_1"] = u8(0x50)
        self.header["unknown2_2"] = u8(0x3C)
        self.header["unknown2_3"] = u8(0xEF)
        self.header["unknown2_4"] = u8(0)
        self.header["unknown2_5"] = u8(0)
        self.header["unknown2_6"] = u8(0)
        self.header["unknown2_7"] = u8(0)
        self.header["unknown2_8"] = u8(0)
        self.header["ratingsEntryNumber"] = u32(0)
        self.header["ratingsTableOffset"] = u32(0)
        self.header["titleTypesEntryNumber"] = u32(0)
        self.header["titleTypesTableOffset"] = u32(0)
        self.header["companyEntryNumber"] = u32(0)
        self.header["companyTableOffset"] = u32(0)
        self.header["titleEntryNumber"] = u32(0)
        self.header["titleTableOffset"] = u32(0)
        self.header["newTitleEntryNumber"] = u32(0)
        self.header["newTitleTableOffset"] = u32(0)
        self.header["videos1EntryNumber"] = u32(0)
        self.header["videos1TableOffset"] = u32(0)
        self.header["newVideoEntryNumber"] = u32(0)
        self.header["newVideoTableOffset"] = u32(0)
        self.header["demosEntryNumber"] = u32(0)
        self.header["demosTableOffset"] = u32(0)
        self.header["unknown5"] = u32(0)
        self.header["unknown6"] = u32(0)
        self.header["recommendationsEntryNumber"] = u32(0)
        self.header["recommendationsTableOffset"] = u32(0)
        for i in range(4):
            self.header["unknown7_%s" % i] = u32(0)
        self.header["recentRecommendationsEntryNumber"] = u32(0)
        self.header["recentRecommendationsTableOffset"] = u32(0)
        for i in range(2):
            self.header["unknown8_%s" % i] = u32(0)
        self.header["popularVideosEntryNumber"] = u32(0)
        self.header["popularVideosTableOffset"] = u32(0)
        self.header["detailedRatingsEntryNumber"] = u32(0)
        self.header["detailedRatingsTableOffset"] = u32(0)
        self.header["lastUpdate"] = enc("RiiConnect24 Edition 2.0", 62)
        self.header["unknown9_0"] = u8(0)
        self.header["unknown9_1"] = u8(0x2E)
        self.header["unknown9_2"] = u8(1)
        for i in range(5):
            self.header["dlUrlIds_%s" % i] = enc_utf_8(
                "THqOxqSaiDd5bjhSQS6hk6nkYJVdioanD5Lc8mOHkobUkblWf8KxczDUZwY84FIV", 256
            )
        self.header["unknown10"] = u32(285278430)

    def write_ratings_table(self):
        """Writes the ESRB rating scales from EC (Early Childhood) to M (Mature)"""
        self.header["ratingsTableOffset"] = u32(self.offset_count())
        ratings_entry_number = 0
        for rating in nin_ratings_table:
            # This is the ESRB value.
            rating_group = 2

            self.header[f"rating_ratingID{ratings_entry_number}"] = u8(
                nin_ratings_table[rating]["rating_id"]
            )
            self.header[f"rating_ratingGroup{ratings_entry_number}"] = u8(rating_group)
            self.header[f"rating_age{ratings_entry_number}"] = u8(
                nin_ratings_table[rating]["age"]
            )
            self.header[f"rating_unknown{ratings_entry_number}"] = u8(222)
            self.header[f"rating_jpegOffset{ratings_entry_number}"] = u32(
                nin_ratings_table[rating]["jpeg_offset"]
            )
            self.header[f"rating_jpegSize{ratings_entry_number}"] = u32(
                nin_ratings_table[rating]["jpeg_size"]
            )
            self.header[f"rating_title{ratings_entry_number}"] = enc(
                nin_ratings_table[rating]["title"], 22
            )

            ratings_entry_number += 1

        self.header["ratingsEntryNumber"] = u32(ratings_entry_number)

    def write_detailed_ratings_table(self):
        """Writes the detailed ratings for titles.
        Examples are "Alcohol Reference" and "Blood and Gore"
        """
        self.header["detailedRatingsTableOffset"] = u32(self.offset_count())
        detail_entry_number = 0
        for detail in nin_detailed_ratings_table:
            self.header["detailedRatingGroup_%s" % detail_entry_number] = u8(
                nin_detailed_ratings_table[detail]["rating_group"]
            )
            self.header["detailedRatingID_%s" % detail_entry_number] = u8(
                nin_detailed_ratings_table[detail]["rating_id"]
            )
            self.header["detailedTitle_%s" % detail_entry_number] = enc(
                nin_detailed_ratings_table[detail]["title"], 204
            )
            detail_entry_number += 1

        self.header["detailedRatingsEntryNumber"] = u32(detail_entry_number)

    def write_title_types(self):
        """Writes the types of titles that the Nintendo Channel supports
        They are Wii Channels, Virtual Console, Disc Games, WiiWare, 3DS and DS/DSi games.
        """
        self.header["titleTypesTableOffset"] = u32(self.offset_count())
        title_types_entry_number = 0
        for title_type in nin_title_types_table:
            self.header["typeID_%s" % title_types_entry_number] = u8(
                nin_title_types_table[title_type]["type_id"]
            )
            self.header["consoleModel_%s" % title_types_entry_number] = enc_utf_8(
                nin_title_types_table[title_type]["console_model"], 3
            )
            self.header["title_%s" % title_types_entry_number] = enc(
                nin_title_types_table[title_type]["title"], 102
            )
            self.header["groupID_%s" % title_types_entry_number] = u8(
                nin_title_types_table[title_type]["group_id"]
            )
            self.header["unknown_title_type_%s" % title_types_entry_number] = u8(
                nin_title_types_table[title_type]["unknown"]
            )
            title_types_entry_number += 1

        self.header["titleTypesEntryNumber"] = u32(title_types_entry_number)

    def write_rating_images(self):
        """Writes the ESRB rating images to the file. It also updates the JPEG Offset and Size which is handy"""
        rating_names = [
            "EC.jpg",
            "E.jpg",
            "E10.jpg",
            "T.jpg",
            "M.jpg",
            "visitesrb.jpg",
            "visitesrb.jpg",
            "maycontain.jpg",
        ]
        for i, rating in enumerate(nin_ratings_table):
            deadbeef = {0: 0xDE, 1: 0xAD, 2: 0xBE, 3: 0xEF}
            # Write the image to our file then update the rating table's offset to the image
            if i == 8:
                return
            with open(f"./ratings/ESRB/{rating_names[i]}", "rb") as image:
                self.header[f"jpegOffset{i}"] = u32(self.offset_count())
                self.header[f"ratingJPEGData{i}"] = image.read()
                counter = 0
                while self.offset_count() % 32 != 0:
                    self.header[f"deadbeef_{i}_{counter}"] = u8(deadbeef[counter % 4])
                    counter += 1
                # Seek to end of file to set filesize
                image.seek(0, os.SEEK_END)
                self.header[f"jpegSize{i}"] = u32(image.tell())
                image.close()

    def write_company_table(self):
        """Writes the companies that made games on the Wii."""
        self.header["companyTableOffset"] = u32(self.offset_count())
        entry_number = 0
        for s in self.databases["Wii"][1].findall("companies"):
            for w in s.findall("company"):
                id = w.get("code").encode()
                id = int(id.hex(), base=16)
                self.header["company_id_%s" % w] = u32(id)
                self.header["devTitle_%s" % w] = enc(w.get("name"), 62)
                self.header["pubTitle_%s" % w] = enc(w.get("name"), 62)
                entry_number += 1
        self.header["companyEntryNumber"] = u32(entry_number)

    def write_title_table(self):
        self.header["titleTableOffset"] = u32(self.offset_count())
        entry_number = 0
        game_type = {
            "3DS": {
                None: 0x12,
                "3DS": 0x12,
                "3DSWare": 0x13,
                "New3DS": 0x18,
                "New3DSWare": 0x19,
                "VC-NES": 0x1A,
                "VC-GB": 0x14,
                "VC-GBC": 0x1C,
                "VC-GBA": 0x1D,
                "VC-GG": 0x1E,
            },
            "NDS": {
                None: 0x0A,
                "DS": 0x0A,
                "DSi": 0x10,
                "DSiWare": 0x11,
            },
            "Switch": {
                None: 0x17,
                "Switch": 0x17,
                "eShop": 0x27,
            },
            "Wii": {
                None: 0x01,
                "Wii": 0x01,
                "Channel": 0x02,
                "WiiWare": 0x0B,
                "VC-NES": 0x03,
                "VC-SNES": 0x04,
                "VC-N64": 0x05,
                "VC-SMS": 0x0C,
                "VC-MD": 0x07,
                "VC-PCE": 0x06,
                "VC-NEOGEO": 0x08,
                "VC-Arcade": 0x0E,
                "VC-C64": 0x0D,
                "VC-MSX": 0x28,
            },
            "WiiU": {
                None: 0x15,
                "WiiU": 0x15,
                "eShop": 0x16,
                "VC-NES": 0x1F,
                "VC-SNES": 0x20,
                "VC-N64": 0x21,
                "VC-GBA": 0x22,
                "VC-DS": 0x23,
                "VC-PCE": 0x24,
                "VC-MSX": 0x25,
                "Channel": 0x26,
            },
        }  # The XML returns None for disc games when we query the type.

        platforms = ["Wii", "NDS", "3DS", "WiiU"]

        titles = {}

        # import leaderboards from RiiTag
        if config["production"]:
            con = MySQLdb.connect(
                "localhost", config["dbuser"], config["dbpass"], "rc24_RiiTag", charset="utf8mb4"
            )

            cur = con.cursor()

            titles_ranking = []

            cur.execute("SELECT game_id FROM game ORDER BY playcount DESC")

            for row in cur.fetchall():
                titles_ranking.append(row)

        titles_keys = {}

        for platform in platforms:
            for title in self.databases[platform][1].findall("game"):
                if title.find("locale", {"lang": "EN"}) is not None:
                    titles[
                        title.find("locale", {"lang": "EN"}).find("title").text
                        + " - "
                        + title.find("id").text
                        + " - "
                        + platform
                    ] = title

        j = 0
        conflict_number = 0
        for key, s in sorted(titles.items()):
            if (
                s.find("region").text == "NTSC-U"
                or s.find("region").text == "USA"
                or s.find("region").text == "ALL"
            ):
                if (
                    s.find("type").text != "CUSTOM"
                    and s.find("type").text != "GameCube"
                ):
                    title = "lmfao null"
                    if s.find("locale", {"lang": "EN"}) is not None:
                        title = (
                            s.find("locale", {"lang": "EN"})
                            .find("title")
                            .text.replace(" - ", ": ")
                            .replace("Ō", "O")
                            .replace("&", "&")
                        )

                    title_platform = key.split(" - ")[-1]

                    titles_key = (
                        " - ".join(key.split(" - ")[:-2]) + " - " + title_platform
                    )

                    if titles_key in titles_keys.keys():
                        """if titles_keys[titles_key][0] == "ALL" and s.find("region").text == "USA" or len(titles_keys[titles_key][2]) == 4 and len(key.split(" - ")[-2]) == 6:
                            for header_key in self.header.keys():
                                if "title_" in header_key and str(titles_keys[titles_key][1]) in header_key:
                                    conflict_number += 1
                                    del header_key
                        else:
                             continue"""

                        continue

                    titles_keys[titles_key] = [
                        s.find("region").text,
                        entry_number,
                        key.split(" - ")[-2],
                    ]

                    if title_platform == "Switch" and s.find("type").text == "eShop":
                        continue

                    # Create custom ID
                    text_id = s.find("id").text[:4]
                    id = int(text_id.encode().hex(), base=16)

                    # for NDS, 3DS, WiiU, and Switch, we xor the id to avoid conflicts. (for example, Mario Kart 7 for 3DS and Mario Kart 8 for Wii U both have the ID "AMKE")
                    if title_platform == "NDS":
                        id ^= 0x22222222

                    elif title_platform == "3DS":
                        id ^= 0x33333333

                    elif title_platform == "WiiU":
                        id ^= 0x44444444

                    elif title_platform == "Switch":
                        id ^= 0x55555555

                    self.header[f"title_id_{entry_number}"] = u32(id)
                    self.header[f"title_titleId_{entry_number}"] = enc_utf_8(
                        s.find("id").text[:4], 4
                    )

                    try:
                        if s.find("type").text in game_type[title_platform]:
                            self.header[f"title_titleType_{entry_number}"] = u8(
                                game_type[title_platform][s.find("type").text]
                            )
                        else:
                            print(title_platform, s.find("type").text)
                            continue
                            # self.header[f"title_titleType_{entry_number}"] = u8(0)
                    except:
                        print(title_platform, s.find("type").text)
                        continue

                    # Parse the genre's. The XML is a giant mess for this, this will be a giant dictionary.
                    genre_dict = {
                        "arcade": 15,
                        "party": 13,
                        "puzzle": 5,
                        "action": 1,
                        "2D platformer": 1,
                        "3D platformer": 1,
                        "shooter": 12,
                        "first-person shooter": 12,
                        "third-person shooter": 12,
                        "rail shooter": 12,
                        "run and gun": 12,
                        "shoot 'em up": 12,
                        "stealth action": 1,
                        "survival horror": 1,
                        "sports": 4,
                        "adventure": 2,
                        "hidden object": 13,
                        "interactive fiction": 2,
                        "interactive movie": 2,
                        "point-and-click": 13,
                        "music": 10,
                        "rhythm": 10,
                        "dance": 10,
                        "karaoke": 10,
                        "racing": 7,
                        "fighting": 14,
                        "simulation": 9,
                        "role-playing": 6,
                        "strategy": 8,
                        "traditional": 11,
                        "health": 3,
                        "others": 13,
                    }

                    # We will have a static variable to store our genre. This is because I didn't include
                    # many subgenres in the dict since they share the same ID as their parent genre.
                    _genre = 13
                    genre_text = s.find("genre")

                    # Check for None
                    if genre_text is None:
                        self.header[f"title_genre_{entry_number}_0"] = u8(13)
                        self.header[f"title_genre_{entry_number}_1"] = u8(13)
                        self.header[f"title_genre_{entry_number}_2"] = u8(13)
                    else:
                        # This is a mess. Some titles don't have genres, some have only 1 or 2 genres, so
                        # workarounds were made
                        genre_list = genre_text.text.split(",")

                        for i in range(3):
                            try:
                                if genre_list[i] in genre_dict:
                                    _genre = genre_dict[genre_list[i]]
                                    self.header[f"title_genre_{entry_number}_{i}"] = u8(
                                        _genre
                                    )
                                else:
                                    self.header[f"title_genre_{entry_number}_{i}"] = u8(
                                        _genre
                                    )
                            except IndexError:
                                self.header[f"title_genre_{entry_number}_{i}"] = u8(
                                    _genre
                                )

                    # We will default to Nintendo because why not
                    self.header[f"title_companyOffset_{entry_number}"] = self.header[
                        "companyTableOffset"
                    ]
                    company_start = struct.unpack(
                        ">I", self.header["companyTableOffset"]
                    )[0]

                    for c in self.databases["Wii"][1].findall("companies"):
                        for i, company in enumerate(c.findall("company")):
                            # Firstly, we will try to find the company by the company code.
                            # This method will only work for disc games. As such, methods below exist
                            if s.find("id").text[4:] != "":
                                if s.find("id").text[4:] == company.get("code"):
                                    self.header[
                                        f"title_companyOffset_{entry_number}"
                                    ] = u32(company_start + (128 * i))
                                    break

                            # If None we will default to Nintendo as well
                            if s.find("publisher").text is None:
                                self.header[
                                    f"title_companyOffset_{entry_number}"
                                ] = self.header["companyTableOffset"]
                                break
                            if company.get("name") in s.find("publisher").text:
                                self.header[
                                    f"title_companyOffset_{entry_number}"
                                ] = u32(company_start + (128 * i))
                                break

                    if s.find("date").get("year") == "":
                        release_year = 0xFFFF
                    else:
                        release_year = s.find("date").get("year")

                    if s.find("date").get("month") == "":
                        release_month = 0xFF
                    else:
                        release_month = s.find("date").get("month")

                    if s.find("date").get("day") == "":
                        release_day = 0xFF
                    else:
                        release_day = s.find("date").get("day")

                    self.header[f"title_releaseYear_{entry_number}"] = u16(
                        int(release_year)
                    )
                    self.header[f"title_releaseMonth_{entry_number}"] = u8(
                        int(release_month) - 1
                    )  # the month starts at 0, not 1
                    self.header[f"title_releaseDay_{entry_number}"] = u8(
                        int(release_day)
                    )

                    if title_platform == "Switch":
                        try:
                            rating = s.find("rating_ESRB").get("value")
                        except AttributeError:
                            rating = ""

                        rating_group = "ESRB"

                    else:
                        try:
                            rating = s.find("rating").get("value")
                            rating_group = s.find("rating").get("type")
                        except AttributeError:
                            rating = ""
                            rating_group = "ESRB"

                    if rating == "E10+" or rating == "E10 ":
                        # GameTDB has E10 as E10+. As I cannot use that as an enum key, here we are
                        rating = "E10"
                    elif rating == "AO":
                        # The Wii has Adults Only?????
                        rating = "M"
                    elif rating == "3":
                        # I don't know why PEGI ratings are getting mixed in
                        rating = "E"
                    elif rating == "" or rating == "Ratin":
                        # Default to E
                        rating = "E"

                    self.header[f"title_ratingId_{entry_number}"] = u8(
                        self.RatingSystem[rating].value
                    )

                    # Unknown Value
                    self.header[f"title_unknown1_{entry_number}"] = u16(2080)

                    # Up ahead is some extremely lazy code. This should work, but some flags will be incorrect such as
                    # Hardcore and casual
                    self.header[f"title_hardcore_{entry_number}"] = u32(536872960)
                    self.header[f"title_friends_{entry_number}"] = u32(2863311530)
                    self.header[f"title_unknown2_{entry_number}"] = u32(2863311530)
                    self.header[f"title_unknown3_{entry_number}"] = u16(43690)
                    # Unknown4 will be a combo of what in the kaitai is unknown16 and unknown17 and other flags
                    self.header[f"title_unknown4_{entry_number}"] = u16(170)
                    self.header[
                        f"title_unknown5_{entry_number}"
                    ] = 168  # contains unknown17 through multiplayer
                    self.header[f"title_unknown6_{entry_number}"] = u32(50331648)
                    self.header[f"title_unknown7_{entry_number}"] = u32(0)
                    self.header[f"title_medal_{entry_number}"] = u8(0)
                    self.header[f"title_unknown9_{entry_number}"] = u8(222)

                    for feature in s.find("wi-fi").findall("feature"):
                        if "online" not in feature.text:
                            self.header[
                                f"title_unknown5_{entry_number}"
                            ] |= 4  # online flag set to no

                    try:
                        if int(s.find("input").get("players")) < 2:
                            self.header[
                                f"title_unknown5_{entry_number}"
                            ] |= 1  # multiplayer flag set to no
                    except TypeError:
                        pass

                    self.header[f"title_unknown5_{entry_number}"] = u8(
                        self.header[f"title_unknown5_{entry_number}"]
                    )

                    if len(title) > 30:
                        fixed_title = textwrap.wrap(title, width=30)[0]
                        fixed_subtitle = textwrap.wrap(title, width=30)[1]

                    elif ": " in title:
                        fixed_title = title.split(": ")[0] + ": "
                        fixed_subtitle = title.split(": ")[1]

                    elif " - " in title:
                        fixed_title = title.split(" - ")[0] + ": "
                        fixed_subtitle = title.split(" - ")[1]

                    else:
                        fixed_title = title
                        fixed_subtitle = ""

                    if config["production"]:
                        if s.find("id").text in titles_ranking:
                            if titles_ranking.index(s.find("id").text) <= 15:
                                self.header[f"title_medal_{entry_number}"] = u8(
                                    4
                                )  # platinum
                            elif titles_ranking.index(s.find("id").text) <= 30:
                                self.header[f"title_medal_{entry_number}"] = u8(
                                    3
                                )  # gold
                            elif titles_ranking.index(s.find("id").text) <= 50:
                                self.header[f"title_medal_{entry_number}"] = u8(
                                    2
                                )  # silver
                            elif titles_ranking.index(s.find("id").text) <= 100:
                                self.header[f"title_medal_{entry_number}"] = u8(
                                    1
                                )  # bronze

                    self.header[f"title_title_{entry_number}"] = enc(fixed_title, 62)
                    self.header[f"title_subtitle_{entry_number}"] = enc(
                        fixed_subtitle, 62
                    )

                    # print(title)

                    self.header[f"title_shortTitle_{entry_number}"] = enc("", 62)
                    entry_number += 1

                    if config["make_info"]:
                        if not os.path.exists(
                            config["file_path"] + "/soft/US/en/" + str(id) + ".info"
                        ):
                            print(
                                "Making info file for {} {}...".format(
                                    title_platform, text_id
                                )
                            )

                            os.system(
                                "python3.10 info.py {} {}".format(
                                    title_platform, text_id
                                )
                            )

            j += 1

        entry_number -= conflict_number

        self.header["titleEntryNumber"] = u32(entry_number)

    def write_new_title_table(self):
        """Writes the offset where the title is in the title table."""
        self.header["newTitleTableOffset"] = u32(self.offset_count())
        entry_number = 0
        for new_title in nin_new_title_table:
            self.header[f"newTitle_offset_{entry_number}"] = self.header[
                "titleTableOffset"
            ]
            entry_number += 1
        self.header["newTitleEntryNumber"] = u32(entry_number)

    def write_videos(self):
        """Writes the videos"""
        self.header["videos1TableOffset"] = u32(self.offset_count())
        entry_number = 0
        for video in nin_videos_1_table:
            self.header["video_id_%s" % entry_number] = u32(
                nin_videos_1_table[video]["id"]
            )
            print(str(nin_videos_1_table[video]["id"]))
            try:
                with open("/var/www/rc24/wapp.wii.com/nintendo/entus/public_html/6/movie/US/en/" + str(nin_videos_1_table[video]["id"]) + "-h.mo", "rb") as f:
                    f.seek(12)
                    fps = struct.unpack("<I", f.read(4))[0]
                    chunk_count = struct.unpack("<I", f.read(4))[0]
                    f.close()
                self.header["video_timeLength_%s" % entry_number] = u16(int(
                    chunk_count / (fps / 256)
                ))
            except FileNotFoundError:
                print(str(nin_popular_videos_table[pop]["id"]) + "-h.mo not found")
                self.header["video_timeLength_%s" % entry_number] = u16(int(
                    0
                ))
            self.header["video_titleID_%s" % entry_number] = u32(
                nin_videos_1_table[video]["title_id"]
            )
            self.header["video_videoType_%s" % entry_number] = u8(
                nin_videos_1_table[video]["video_type"]
            )
            for i in range(14):
                self.header["bruh_unknown_%s_%s" % (i, entry_number)] = u8(0)
            self.header["video_unknown2_%s" % entry_number] = u8(
                nin_videos_1_table[video]["unknown_2"]
            )
            self.header["video_ratingID_%s" % entry_number] = u8(
                nin_videos_1_table[video]["rating_id"]
            )
            self.header["video_unknown3_%s" % entry_number] = u8(
                nin_videos_1_table[video]["unknown_3"]
            )
            self.header["video_newTag_%s" % entry_number] = u8(
                nin_videos_1_table[video]["new_tag"]
            )
            self.header["videoIndex_%s" % entry_number] = u8(
                nin_videos_1_table[video]["video_index"]
            )
            self.header["unknown4_1_%s" % entry_number] = u8(61 + entry_number)
            self.header["unknown4_2_%s" % entry_number] = u8(222)
            self.header["video_title_%s" % entry_number] = enc(
                nin_videos_1_table[video]["title"], 246
            )
            entry_number += 1

        self.header["videos1EntryNumber"] = u32(entry_number)

    def write_new_video_table(self):
        """Writes the new videos"""
        self.header["newVideoTableOffset"] = u32(self.offset_count())
        entry_number = 0
        for new_video in nin_new_video_table:
            entry_number += 1
            self.header["new_videoID_%s" % entry_number] = u32(
                nin_new_video_table[new_video]["id"]
            )
            try:
                with open("/var/www/rc24/wapp.wii.com/nintendo/entus/public_html/6/movie/US/en/" + str(nin_new_video_table[new_video]["id"]) + "-h.mo", "rb") as f:
                    f.seek(12)
                    fps = struct.unpack("<I", f.read(4))[0]
                    chunk_count = struct.unpack("<I", f.read(4))[0]
                    f.close()
                print(chunk_count / (fps / 256))
                try:
                    self.header["new_video_timeLength_%s" % entry_number] = u16(int(
                        chunk_count / (fps / 256)
                    ))
                except:
                    self.header["new_video_timeLength_%s" % entry_number] = u16(int(
                        0
                    ))
            except FileNotFoundError:
                print(str(nin_new_video_table[new_video]["id"]) + "-h.mo not found")
                self.header["new_video_timeLength_%s" % entry_number] = u16(int(
                    0
                ))
            self.header["new_video_titleID_%s" % entry_number] = u32(
                nin_new_video_table[new_video]["title_id"]
            )
            for i, data in enumerate(nin_new_video_table[new_video]["unknown_2"]):
                if i == 0:
                    self.header["new_video_unknown2_%s_%s" % (i, entry_number)] = u8(8)
                elif i == 1:
                    self.header["new_video_unknown2_%s_%s" % (i, entry_number)] = u8(1)
                else:
                    self.header["new_video_unknown2_%s_%s" % (i, entry_number)] = u8(
                        data
                    )
            self.header["new_video_title_%s" % entry_number] = enc(
                nin_new_video_table[new_video]["title"], 204
            )

        self.header["newVideoEntryNumber"] = u32(entry_number)

    def write_demos(self):
        """Writes the demos"""
        self.header["demosTableOffset"] = u32(self.offset_count())
        entry_number = 0
        for demo in nin_demos_table:
            self.header["demo_id_%s" % entry_number] = u32(nin_demos_table[demo]["id"])
            self.header["demo_title_%s" % entry_number] = enc(
                nin_demos_table[demo]["title"], 62
            )
            self.header["demo_subtitle_%s" % entry_number] = enc(
                nin_demos_table[demo]["subtitle"], 62
            )
            self.header["demo_titleID_%s" % entry_number] = u32(
                nin_demos_table[demo]["titleid"]
            )
            self.header["demo_company_offset_%s" % entry_number] = u32(
                nin_demos_table[demo]["company_offset"]
            )
            self.header["demo_removal_year_%s" % entry_number] = u16(65535)
            self.header["demo_removal_month_%s" % entry_number] = u8(255)
            self.header["demo_removal_day_%s" % entry_number] = u8(255)
            self.header["demo_unknown_%s" % entry_number] = u32(0)
            self.header["demo_rating_id_%s" % entry_number] = u8(
                nin_demos_table[demo]["rating_id"]
            )
            self.header["demo_new_tag_%s" % entry_number] = u8(
                nin_demos_table[demo]["new_tag"]
            )
            self.header["demo_new_tag_index_%s" % entry_number] = u8(
                nin_demos_table[demo]["new_tag_index"]
            )
            for i in range(205):
                self.header["demo_unknown2_%s_%s" % (entry_number, i)] = u8(0)
            entry_number += 1

        self.header["demosEntryNumber"] = u32(entry_number)

    def write_recommendations(self):
        """Writes the recommended games"""
        self.header["recommendationsTableOffset"] = u32(self.offset_count())
        entry_number = 0
        for recommend in nin_recommendations_table:
            self.header[
                "recommend_recommendation_title_offset_%s" % entry_number
            ] = self.header["titleTableOffset"]
            entry_number += 1
        self.header["recommendationsEntryNumber"] = u32(entry_number)

    def write_recent_recommendation_table(self):
        self.header["recentRecommendationsTableOffset"] = u32(self.offset_count())
        entry_number = 0
        for idk in nin_recent_recommendations_table:
            self.header[
                "recent_recommendation_title_offset_%s" % entry_number
            ] = self.header["titleTableOffset"]
            self.header["recent_recommendation_unknown_%s" % entry_number] = u16(
                nin_recent_recommendations_table[idk]["unknown"]
            )
            entry_number += 1

        self.header["recentRecommendationsEntryNumber"] = u32(entry_number)

    def write_pop_videos(self):
        self.header["popularVideosTableOffset"] = u32(self.offset_count())
        entry_number = 0
        for pop in nin_popular_videos_table:
            self.header["pop_video_id_%s" % entry_number] = u32(
                nin_popular_videos_table[pop]["id"]
            )
            try:
                with open("/var/www/rc24/wapp.wii.com/nintendo/entus/public_html/6/movie/US/en/" + str(nin_popular_videos_table[pop]["id"]) + "-h.mo", "rb") as f:
                    f.seek(12)
                    fps = struct.unpack("<I", f.read(4))[0]
                    chunk_count = struct.unpack("<I", f.read(4))[0]
                    f.close()
                self.header["pop_video_time_%s" % entry_number] = u16(int(
                    chunk_count / (fps / 256)
                ))
            except FileNotFoundError:
                print(str(nin_popular_videos_table[pop]["id"]) + "-h.mo not found")
                self.header["pop_video_time_%s" % entry_number] = u16(int(
                    0
                ))
            self.header["pop_video_title_id_%s" % entry_number] = u32(0)
            self.header["pop_video_bar_color_%s" % entry_number] = u8(0)
            for i in range(15):
                self.header["pop_video_unknown_%s_%s" % (i, entry_number)] = u8(0)
            self.header["pop_video_ratingID_%s" % entry_number] = u8(
                nin_popular_videos_table[pop]["rating_id"]
            )
            self.header["pop_video_unknown3_%s" % entry_number] = u8(1)
            self.header["pop_video_videoRank_%s" % entry_number] = u8(1)
            self.header["pop_video_unknown4_%s" % entry_number] = u8(222)
            self.header["pop_video_title_%s" % entry_number] = enc(
                nin_popular_videos_table[pop]["title"], 204
            )
            entry_number += 1
        self.header["popularVideosEntryNumber"] = u32(entry_number)

    class RatingSystem(enum.Enum):
        EC = 8
        E = 9
        E10 = 10
        T = 11
        M = 12

    def write_file(self):
        # Now that all the file contents are written, calculate filesize
        self.header["filesize"] = u32(self.offset_count())

        filename = "testing.bin"

        if os.path.exists(filename + "-1"):
            os.remove(filename + "-1")

        if os.path.exists(filename):
            os.remove(filename)

        self.writef = open(filename + "-1", "ab")

        for values in self.header.values():
            self.writef.write(values)

        self.writef.close()

        self.readf = open(filename + "-1", "rb")

        read = self.readf.read()

        self.writef2 = open(filename, "wb")

        self.writef2.write(read)
        self.writef2.seek(8)
        self.writef2.write(
            binascii.unhexlify(format(binascii.crc32(read) & 0xFFFFFFFF, "08x"))
        )

        os.remove(filename + "-1")

        self.writef2.close()

        nlzss.encode_file(filename, filename.replace(".bin", ".LZ"))

MakeDList(GameTDB(True).parse())

# Get genres
# genre_type = {"action": 1, "adventure": 2, "sports": 4, "puzzle": 5,
# "strategy RPG": 6, "tactical RPG": 6, "action RPG": 6, "MMORPG": 6, "roguelike": 6}
# for s in self.databases["Wii"][1].findall("genres"):
# for k in s.findall("maingenre"):
# print(k.get("name"))


"""
This is used to manage the submission of final papers. Authors enter the site via
an authenticated URL with a paper ID from hotcrp. When a paper is submitted, it
receives an upload of a zip file with at least one tex file. We start a docker instance
in a separate task to compile the output.
"""

from flask import json, Flask, request, render_template, current_app
import config
import os
import sys
from search.search_lib import search

# Make sure we aren't running on an old python.
assert sys.version_info >= (3, 6)

def validate_config():
    if 'DB_PATH' not in app.config:
        raise ValueError(str(app.config))

app = Flask(__name__, static_folder='static/' , static_url_path='/static')
#app.config.from_object(config.Config)
if 'DEBUG' in app.config:
    app.config.from_object(config.Config)
else:
    app.config.from_object(config.ProductionConfig)
validate_config()

@app.route('/')
def home():
    return render_template('index.html')

@app.route('/view/<id>')
def view_funder(id):
    result = search(app.config['DB_PATH'],
                    offset=0,
                    textq='id:' + id,
                    locationq=None)
    if len(result.get('results')) > 0:
        result = {'item': result.get('results')[0]}
    else:
        result = {'error': 'no such item'}
    print(result)
    return render_template('index.html', **result)

@app.route('/search', methods=['GET'])
def get_results():
    args = request.args.to_dict()
    if 'textq' not in args and 'locationq' not in args:
        return json.jsonify({'error': 'missing queries'})
    return json.jsonify(search(app.config['DB_PATH'],
                               offset=args.get('offset', 0),
                               textq=args.get('textq'),
                               locationq=args.get('locationq')))

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0')


from django.shortcuts import render
from django.http import HttpResponse

# Create your views here.

from django.shortcuts import render

# relative import of forms

def create_view(request):
    # dictionary for initial data with
    # field names as keys
    context ={}

    # add the dictionary during initialization
    form = GeeksForm(request.POST or None)
    if form.is_valid():
        form.save()

    context['form']= form
    return render(request, "hello.html", context)


# Use case: Python command line tool that accepts city name as input and returns the current weather for that city
# Usage: python weather.py <city_name>
# Example: python weather.py "New York"
# Output: Current weather in New York, US is 10.0 degrees Celsius with clear sky

# Importing the libraries required for API call, JSON parsing, reading environment variables (for API key) and command line arguments
import requests, json, os, sys
from dotenv import load_dotenv

# Load environment variables from .env file
load_dotenv()

# Class that accepts city name as input and returns the current weather for that city
class Weather:

    #constructor that accepts city name as input, defines the API key and the API URL
    def __init__(self, city_name):

        self.city_name = city_name

        #check if API key is defined as environment variable
        if os.environ.get("OPENWEATHERMAP_API_KEY") is None:
            print("Please set the OPENWEATHERMAP_API_KEY environment variable")
            sys.exit(1)
        self.api_key = os.environ.get("OPENWEATHERMAP_API_KEY")
        self.api_url = "http://api.openweathermap.org/data/2.5/weather?q=" + self.city_name + "&appid=" + self.api_key + "&units=metric"

    #function that calls the API and returns the current weather
    def get_weather(self):

        #call the API and display the weather data
        try:
            response = requests.get(self.api_url)
            response.raise_for_status()
            weather_data = json.loads(response.text)

            #print the weather data for entered city
            print("Current weather in " + self.city_name + ", " + weather_data['sys']['country'] + " is " + str(weather_data['main']['temp']) + " degrees Celsius with " + weather_data['weather'][0]['description'])

        #handle HTTP errors
        except requests.exceptions.HTTPError as e:
            print("Error connecting to API")
            print("Please make sure the city name and provided secret OPENWEATHERMAP_API_KEY is correct and try again.")
            sys.exit(1)

        #handle connection errors
        except requests.exceptions.ConnectionError as e:
            print("Error connecting to API")
            sys.exit(1)

        #handle json parsing errors
        except json.decoder.JSONDecodeError as e:
            print("Error parsing JSON response")
            sys.exit(1)

        #handle other exceptions
        except Exception as e:
            print("Error")
            print(e)
            sys.exit(1)

#main function that accepts city name as command line argument and calls the Weather class
if __name__ == "__main__":

    #check if city name is passed as command line argument
    if len(sys.argv) >= 2:
        city_name = ' '.join(sys.argv[1:])

        #call the Weather class
        weather = Weather(city_name)
        weather.get_weather()
    else:
        print("Please enter a city name")
        print("Usage Example: python weather.py <city>")
        sys.exit(1)


# Contact from 31 October 2018 to now

import os
from dotenv import load_dotenv
import pandas as pd
import requests
import json

print()
print()
print()

load_dotenv()

airtable_key = os.environ.get("DOORMATIC_AIRTABLE_KEY")
baseId = "appga9Mr4rk0qIOy5"
tableIdOrName = "Contacts Table"
airtable_create_record_url = f"https://api.airtable.com/v0/{baseId}/{tableIdOrName}"

def read_and_clean_csv():
    contacts_df = pd.read_csv("assets/contacts-2019.csv", low_memory=False)

    # Filter contacts that have been imported
    contacts_df = contacts_df[contacts_df["Imported"] != True]
    contacts_df[["Business Latitude", "Business Longitude"]] = contacts_df[
        ["Business Latitude", "Business Longitude"]
    ].fillna(0)
    contacts_df = contacts_df.fillna("")

    # #
    # contacts_df["Business Latitude"] = (
    #     contacts_df[["Business Latitude"]]fillna(0)
    # )
    # .apply(pd.to_numeric).

    print(contacts_df["Business Latitude"])

    # replace bools with strings
    mask = contacts_df.applymap(type) != bool
    bool_to_string = {True: "TRUE", False: "FALSE"}
    contacts_df = contacts_df.where(mask, contacts_df.replace(bool_to_string))

    return contacts_df

def create_request_body(start_index, end_index, contacts_df):
    records = []

    for i in range(start_index, end_index):
        contact = contacts_df.iloc[i]
        record = {
            "fields": {
                "Surname": contact["Surname"],
                "Business Latitude": contact["Business Latitude"],
                "Favorite": contact["Favorite"],
                "Mobile Phone": contact["Mobile Phone"],
                "E-mail": contact["E-mail"],
                "Create Date": contact["Create Date"],
                "Access Level": contact["Access Level"],
                "Postcode": contact["Postcode"],
                "AEM Opt Out": contact["AEM Opt Out"],
                "Is User": contact["Is User"],
                "Record Manager": contact["Record Manager"],
                "Contact": contact["Contact"],
                "Address 1": contact["Address 1"],
                "AEM Bounce Back": contact["AEM Bounce Back"],
                "Address 3": contact["Address 3"],
                "Private Contact": contact["Private Contact"],
                "Is Imported": contact["Is Imported"],
                "City": contact["City"],
                "Salutation": contact["Salutation"],
                "Last Reach": contact["Last Reach"],
                "Record Creator": contact["Record Creator"],
                "First Name": contact["First Name"],
                "Edit Date": contact["Edit Date"],
                "Business Longitude": contact["Business Longitude"],
                "Machine Compliance Cert": contact["Machine Compliance Cert"],
                "Last Edited By": contact["Last Edited By"],
            }
        }
        records.append(record)

    print(records)

    data = {"records": records}
    # print(json.dumps(data))
    return data

def post_data_to_airtable(data):
    response = requests.post(
        airtable_create_record_url,
        json=data,
        headers={"Authorization": f"Bearer {airtable_key}"},
    ).json()

    print(response)
    return

def update_csv_row(i):
    contact = contacts_df.loc[i]
    contacts_df.loc[i, "Imported"] = "True"
    # print(contact)

contacts_df = read_and_clean_csv()

for i in range(0, len(contacts_df), 10):
    data = create_request_body(0, 3, contacts_df)
    post_data_to_airtable(data)
    # update_csv_row(i)
    break

# contacts_df.to_csv("assets/contacts-2023.csv")


from django.shortcuts import render
from .models import Product, Company
from django.http import HttpResponseRedirect, HttpResponseNotFound
from .forms import ProductForm

def index(request):
    products = Product.objects.all()
    return render(request, 'index.html', {'products': products})

def create(request):
    create_companies()
    form = ProductForm()
    if request.method == 'POST':
        form = ProductForm(request.POST)
        if form.is_valid():
            form.save()
        return HttpResponseRedirect('/')

    return render(request, 'create.html', {'form': form})

def edit(request, id):
    try:
        product = Product.objects.get(id=id)
        form = ProductForm(instance=product)
        if request.method == "POST":
            form = ProductForm(request.POST, instance=product)
            if form.is_valid():
                form.save()
            return HttpResponseRedirect("/")
        else:
            return render(request, "edit.html", {"form": form})
    except Product.DoesNotExist:
        return HttpResponseNotFound(" <h2›Product not found
")

def delete(request, id):
    try:
        product = Product.objects.get(id=id)
        product.delete()
        return HttpResponseRedirect("/")
    except Product.DoesNotExist:
        return HttpResponseNotFound("
Product not found
")

def create_companies():
    if Company.objects.all().count() == 0:
        Company.objects.create(name="Apple")
        Company.objects.create(name="Asus")
        Company.objects.create(name="MSI")


import os
import glob
import time
from PIL import Image
import tensorflow as tf

def load_model(model_path):
    return tf.keras.models.load_model(model_path)

def classify_image(model, image_path):
    image = Image.open(image_path)
    resized_image = image.resize((224, 224), Image.ANTIALIAS)
    input_data = tf.keras.preprocessing.image.img_to_array(resized_image)
    input_data = input_data / 255.0
    input_data = input_data.reshape((1, *input_data.shape))
    prediction = model.predict(input_data)
    return prediction

def organize_screenshots(screenshot_folder, model):
    screenshot_files = glob.glob(f"{screenshot_folder}/*.jpg")

    for screenshot_file in screenshot_files:
        prediction = classify_image(model, screenshot_file)

        if (
            prediction[0][0] > 0.5
        ):  # Adjust this threshold according to your model's output
            label = "queue_pop"
        else:
            label = "not_queue_pop"

        label_folder = os.path.join(screenshot_folder, label)
        if not os.path.exists(label_folder):
            os.makedirs(label_folder)

        file_name = os.path.join(
            label_folder, os.path.basename(screenshot_file)
        )
        os.rename(screenshot_file, file_name)
        print(f"Moved {screenshot_file} to {file_name}")

def main():
    screenshot_folder = "screenshots"
    model_path = "queue_pop_detector.h5"

    if not os.path.exists(screenshot_folder):
        print(
            f"{screenshot_folder} does not exist. Please provide a valid folder path."
        )
        return

    model = load_model(model_path)
    organize_screenshots(screenshot_folder, model)

if __name__ == "__main__":
    main()


from typing import Any
import numpy as np
from numpy.linalg import norm
from numpy.linalg import pinv
import pinocchio as pin
import time
from pinocchio.utils import rotate
from bezier import Bezier
from config import DT, LEFT_HAND, RIGHT_HAND
from scipy.optimize import fmin_bfgs
import math

# in my solution these gains were good enough for all joints but you might want to tune this.
Kp = 1200  # proportional gain (P of PD)
Ki = 24
i = 0
Kd = 2 * np.sqrt(Kp)

CUBE_PLACEMENT_UP = pin.SE3(rotate("z", 0.0), np.array([0.33, -0.3, 1.13]))

graspForce = 100
cubeweight = 100
graspTorque = 1

def getGraspForces(sim, robot):
    fext = [pin.Force.Zero()] * (robot.nv + 1)

    lid = (
        sim.bulletCtrlJointsInPinOrder.index(sim.bullet_names2indices["LARM_EFF"] - 1)
        + 1
    )
    rid = (
        sim.bulletCtrlJointsInPinOrder.index(sim.bullet_names2indices["RARM_EFF"] - 1)
        + 1
    )

    fext[lid] = pin.Force(np.array([-graspForce, 0, -cubeweight, 0, 0, graspTorque]))
    fext[rid] = pin.Force(np.array([-graspForce, 0, -cubeweight, 0, 0, -graspTorque]))

    fext_wrapper = pin.StdVec_Force()
    for ext_force in fext:
        fext_wrapper.append(ext_force)

    return fext_wrapper

def controllaw(sim, robot, trajs, tcurrent, cube):
    global i
    q, vq = sim.getpybulletstate()
    q_of_t, vq_of_t, vvq_of_t = trajs

    # PID
    e_q = q_of_t(tcurrent) - q
    i += e_q * DT
    e_qd = vq_of_t(tcurrent) - vq
    dvq = Kp * e_q + Kd * e_qd + i * Ki + vvq_of_t(tcurrent)

    # Add grasp force
    graps_ext = getGraspForces(sim, robot)

    b = pin.rnea(robot.model, robot.data, q, vq, dvq, graps_ext)
    M = pin.crba(robot.model, robot.data, q)

    torques = M @ dvq + b

    sim.step(torques)

if __name__ == "__main__":
    from tools import setupwithpybullet, setupwithpybulletandmeshcat, rununtil

    robot, sim, cube = setupwithpybullet()

    from config import CUBE_PLACEMENT, CUBE_PLACEMENT_TARGET
    from inverse_geometry import computeqgrasppose
    from path import computepath

    q0, successinit = computeqgrasppose(robot, robot.q0, cube, CUBE_PLACEMENT, None)
    qe, successend = computeqgrasppose(
        robot,
        robot.q0,
        cube,
        CUBE_PLACEMENT_TARGET,
        None
        # robot, robot.q0, cube, CUBE_PLACEMENT_UP, None
    )
    path = computepath(robot, cube, q0, qe, CUBE_PLACEMENT, CUBE_PLACEMENT_TARGET)
    # path = computepath(robot, cube, q0, qe, CUBE_PLACEMENT, CUBE_PLACEMENT_UP)

    # setting initial configuration
    sim.setqsim(q0)

    # TODO this is just an example, you are free to do as you please.
    # In any case this trajectory does not follow the path
    # 0 init and end velocities

    class curve_wrapper:
        def __init__(self, curves, maxtime=10):
            self.curves = curves
            self.segments = len(curves)
            self.T = maxtime

        def __call__(self, t):
            curveNo = math.floor(((t / self.T) * self.segments))
            if curveNo == self.segments:
                curve = self.curves[self.segments - 1]
                return curve(1)
            else:
                curve = self.curves[curveNo]
                t = t * self.segments / self.T - curveNo
                return curve(t)

    def getControls(path, segments):
        start = path[0]
        end = path[-1]
        noPoints = len(path) - 2
        segmentSize = noPoints // segments
        print(noPoints, segmentSize)
        controls = [None] * segments
        curves = [None] * segments
        for i in range(segments - 1):
            path[(i + 1) * segmentSize] = (
                path[(i + 1) * segmentSize - 1] + path[(i + 1) * segmentSize + 1]
            ) / 2
            controls[i] = path[(i * segmentSize) : (i + 1) * segmentSize + 1]

        controls[0] = [start] * 2 + controls[0]
        controls[-1] = path[(segments - 1) * segmentSize + 1 :] + [end] * 2

        for i in range(segments):
            curves[i] = Bezier(
                controls[i],
                t_max=1,
            )

        return curves

    def maketraj(q0, q1, T):  # TODO compute a real trajectory !
        curves = getControls(path, 4)
        vqs = [curve.derivative(1) for curve in curves]
        vvqs = [vq.derivative(1) for vq in vqs]
        curves = curve_wrapper(curves, T)
        vqs = curve_wrapper(vqs, T)
        vvqs = curve_wrapper(vvqs, T)

        return curves, vqs, vvqs

    # TODO this is just a random trajectory, you need to do this yourself
    total_time = 5.0
    trajs = maketraj(q0, qe, total_time)

    from tools import setupwithmeshcat

    if True:
        robot, cube, viz = setupwithmeshcat()
        for t in np.linspace(0, total_time, 100):
            viz.display(trajs[0](t))

    tcur = 0.0

    while tcur < total_time:
        rununtil(controllaw, DT, sim, robot, trajs, tcur, cube)
        tcur += DT


#!/usr/bin/env python

## 2021/06/21. Levi King. From the blog,
## https://naturallanguagepuzzling.blogspot.com
## This code was written to assist with the 2021/05/30 Sunday Puzzle:
## https://www.npr.org/2021/06/20/1008352915/sunday-puzzle-hidden-cities
## That puzzle:
"""
This week's challenge comes from listener Iva Allen in Canada. Name a make of
car. Write it in all capital letters. Rotate one of the letters 90 degrees and
another letter 180 degrees to make a woman's name. What is it?
"""

import itertools
import string
from slugify import slugify

## mapping upright capital letters to 90degree rotated letters
upper_90d = {
    'C': ['U'],
    'E': ['M', 'W'],
    'H': ['I'],
    'I': ['H'],
    'M': ['E'],
    'N': ['Z'],
    'U': ['C'],
    'W': ['E'],
    'Z': ['N']
    }

## mapping upright capital letters to 180degree rotated letters
upper_180d = {
    'A': 'V',
    'M': 'W',
    'U': 'A',
    'V': 'A',
    'W': 'M',
    }

## You can use the text file lists of cars makers and girls names I've prepared;
## download them from GitHub and store it as I have:
car_makers_file = 'resources/car-makers.txt'
girls_names_file = 'resources/girls-names.txt'

## clean up each name to suit this puzzle: remove any whitespace,
## convert to upppercase, slugify (convert to ASCII), remove any non-letters
def clean_name(somename):
    s = somename.strip()
    s = slugify(s)
    s = "".join([l.upper() for l in s if l.isalpha()])
    return s

## takes in agenda (see get_car_variants); for each pair of indices in the
## agenda, does the rotations to produce variant(s)
def get_variants_by_agenda(car, agenda):
    car = "".join(car)
    variants = []
    for ag in agenda:
        agvars = []
        i90 = ag[0]
        i180 = ag[1]
        v90s = upper_90d[car[i90]]
        v180 = upper_180d[car[i180]]
        for v90 in v90s:
            carlist = list(car)
            carlist[i90] = v90
            carlist[i180] = v180
            agvars.append("".join(carlist))
        variants += agvars
    return variants

## takes a car maker (string), determines which positions in string can be
## rotated 90 or 180 degrees; produces a list (agenda) of pairs of positions
## to rotate in order to produce all variant strings; each pair is:
## (i90, i80), where i90 is index of letter to be rotated 90 degrees; i90 != i80
def get_car_variants(car, keys90, keys180):
    cvs = []
    car = list(car)
    i90s = []  ## index in car string for letters rotatable 90 degrees
    i180s = []  ## index in car string for letters rotatable 180 degrees
    for lx in car:
        if lx in keys90:
            i90s.append(car.index(lx))
        if lx in keys180:
            i180s.append(car.index(lx))
    if 0 not in [len(i90s), len(i180s)]:
        agenda = [ii for ii in itertools.product(i90s, i180s) if ii[0] != ii[1]]
        agenda = list(set(agenda))
        cvs += get_variants_by_agenda(car,agenda)
    else:
        pass
    return cvs

## iterate through car makers, expand each car maker to variants using rotated
## letters; compare variants against list of girl names; print solution
def cars_vs_girls(cars, girls, keys90, keys180):
    for car in cars:
        car_variants = get_car_variants(car, keys90, keys180)
        print(car)
        print(car_variants)
        for cv in car_variants:
            for g in girls:
                if cv.strip() == g.strip():
                    print(car, g)

## prepare lists of 90degree rotatable and 180degree rotatable letters
def solve_puzzle(cars, girls):
    keys90 = list(upper_90d.keys())
    keys180 = list(upper_180d.keys())
    cars_vs_girls(cars, girls, keys90, keys180)

def main():
    girl_file = open(girls_names_file, "r")
    girl_list = girl_file.readlines()
    girl_file.close()
    girls = [clean_name(g) for g in girl_list]
    girls.sort()
    car_file = open(car_makers_file, "r")
    car_list = car_file.readlines()
    car_file.close()
    cars = [clean_name(c) for c in car_list]
    cars.sort()
    solve_puzzle(cars, girls)

if __name__ == "__main__":
    main()

"""
Solution:
MAZDA --> WANDA
"""


from config import Instance
from typing import List
from paths import get_training_metrics_path, get_cross_validation_scores_path
import h5py
import pandas as pd

def get_last_epoch(keys):
    epochs = [int(s.split('epoch')[1]) for s in keys]
    last_epoch = max(epochs)
    return last_epoch

def compute_score_for_each_model(instances: List[Instance]):
    cv_k = instances[0].cv_k
    df = Instance.get_projections(instances, hyperparameter_names=instances[0].get_dependencies_for_resources(
        'cross_validated_model'))
    df_scores = pd.DataFrame(columns=['cross_validated_model_hash', 'average_validation_loss'])
    for _, row in df.iterrows():
        d = row.to_dict()
        filtered_instances = Instance.get_filtered_instances(instances, d)
        assert len(filtered_instances) == cv_k
        assert set(Instance.get_projections(filtered_instances, ['cv_fold']).cv_fold.to_list()) == set(range(cv_k))
        average_validation_loss = 0
        for instance in filtered_instances:
            path = get_training_metrics_path(instance)
            with h5py.File(path, 'r') as f5:
                keys = f5.keys()
                # keys are like ['epoch10', 'epoch20', ...]
                last_epoch = get_last_epoch(keys)
                metrics = f5[f'epoch{last_epoch}']
                validation_loss = metrics['validation_loss'][...].item()
                average_validation_loss += validation_loss
        average_validation_loss /= cv_k
        instance_hash = filtered_instances[0].get_instance_hash(resource_name='cross_validated_model')
        assert len(
            set([instance.get_instance_hash(resource_name='cross_validated_model') for instance in filtered_instances]))
        df_scores = df_scores.append(
            {'cross_validated_model_hash': instance_hash, 'average_validation_loss': average_validation_loss},
            ignore_index=True)
    path = get_cross_validation_scores_path()
    df_scores.to_csv(path)

if __name__ == '__main__':
    from config import instances

    compute_score_for_each_model(instances)


from itune_radio import *
from mutagen.mp3 import MP3
from mutagen.id3 import ID3, APIC, error
import requests
from PIL import Image
from StringIO import StringIO
import os

def get_song_from_file(name_of_song):
  audio = MP3(name_of_song, ID3=ID3)
  return audio

def add_album_to_ID3(song,image_name,image_format):
  # add ID3 tag if it doesn't exist
  try:
      song.add_tags()
  except error:
      pass

  song.tags.add(
      APIC(
          encoding=3, # 3 is for utf-8
          mime='image/'+image_format, # image/jpeg or image/png
          type=3, # 3 is for the cover image
          desc=u'Cover',
          data=open(os.getcwd() + '/' + image_name).read()
      )
  )
  song.save()

def get_album_art_from_web(key):

  #getting track,artist and album using itune_radio.py
  track,artist = get_song_and_artist()
  album = get_album()

  #generating the api url for lastfm
  url = "http://ws.audioscrobbler.com/2.0/?method=album.getinfo&api_key=%s&artist=%s&album=%s&format=json" %(key,artist,album)

  #getting the relevant album art url from lastfm
  r = requests.get(url)
  data = r.json()
  content = data['album']
  all_img = content['image']
  extralarge = all_img[3]
  img_url = extralarge['#text'].encode('ascii','ignore')

  #returning the obtained image
  i = requests.get(img_url)
  img = Image.open(StringIO(i.content))
  img_format = img.format.lower()

  img_name = album + "." + img_format
  img.save(img_name)

  return img_name, img_format

def delete_art_after_link(image_name):
  album_art_path = os.getcwd() + '/' + image_name
  os.remove(album_art_path)


import numpy as np
from tmtools import tm_align
from openfold.np import protein, residue_constants
import os
import argparse
import logging
from tqdm import tqdm
from statistics import mean

def tm_score_from_pdbs(fp1, fp2):
    coords1, seq1 = coord_seq_from_file(fp1)
    coords2, seq2 = coord_seq_from_file(fp2)
    res = tm_align(coords1, coords2, seq1, seq2)
    return res.tm_norm_chain1

def coord_seq_from_file(path):
    with open(path, 'r') as f:
        pdb_str = f.read()
    protein_object = protein.from_pdb_string(pdb_str)
    ca_coords = protein_object.atom_positions[:, 1, :]
    seq = _aatype_to_str_sequence(protein_object.aatype)
    return ca_coords, seq

def _aatype_to_str_sequence(aatype):
    return ''.join([
        residue_constants.restypes_with_x[aatype[i]]
        for i in range(len(aatype))
    ])

def gather_job(gt_dir, predict_dir):
    pdb_paths = []
    for f_path in os.listdir(gt_dir):
        if f_path.endswith('.pdb'):
            pdb_path_to_verify = os.path.join(predict_dir, f_path)
            if os.path.exists(pdb_path_to_verify):
                pdb_paths.append(f_path)
    return pdb_paths

def TMalign_dir(path1, path2):
    jobs = gather_job(path1, path2)
    logging.info(f'got {len(jobs)} jobs...')

    tm_list = []
    tm_dict = {}
    for job in tqdm(jobs):
        fp1 = os.path.join(path1, job)
        fp2 = os.path.join(path2, job)
        name = job.split(".")[0]
        tm = tm_score_from_pdbs(fp1, fp2)
        tm_list.append(tm)
        tm_dict[name] = tm
    return tm_list, tm_dict, mean(tm_list), len(jobs)

def main(args):
    tm_list, tm_dict, mean, num_jobs = TMalign_dir(args.gt_dir, args.predict_dir)
    with open(args.log_dir, 'w') as f:
        f.write(str(num_jobs) + " jobs: ")
        f.write(str(mean))
    # print(mean)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--gt_dir", type=str, default=None,
        help="Path to model parameters."
    )
    parser.add_argument(
        "--predict_dir", type=str, default=os.getcwd(),
        help="Name of the directory in which to output the prediction",
    )
    parser.add_argument(
        "--log_dir", type=str, default=os.getcwd(),
        help="Name of the directory in which to output the prediction",
    )
    args = parser.parse_args()
    main(args)


# TensorFlow2.0 VAE Implementation
# Large amount of credit goes to:
# https://github.com/ChengBinJin/VAE-Tensorflow
# https://github.com/keras-team/keras/blob/master/examples/variational_autoencoder.py
# https://github.com/ShengjiaZhao/MMD-Variational-Autoencoder/blob/master/mmd_vae.py
# which I've used as a reference for this implementation

import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras import Model
import numpy as np
import matplotlib.pyplot as plt
import time

# ------------------ #
#  Basic Parameters  #
# ------------------ #

ImgW, ImgH = 28, 28

BufferSize = 60000
BatchSize = 100

LatentDim = 2

Seed = np.random.randint(0, high=BufferSize, size=4)

# ----------- #
#  Load Data  #
# ----------- #

(train_images, train_labels), (_, _) = tf.keras.datasets.mnist.load_data()

noise_images = train_images + 128 * np.random.randn(*train_images.shape)
noise_images = np.clip(noise_images, 0., 255.)

# ------------ #
#  Preprocess  #
# ------------ #

def preprocess_data(train_images):

    copy_images = np.array(train_images)

    copy_images = train_images.reshape(train_images.shape[0], ImgW, ImgH, 1).astype('float32')

    # Normalizing the images to the range of [0., 1.]
    copy_images /= 255.

    # Binarization
    copy_images[copy_images >= .5] = 1.
    copy_images[copy_images < .5] = 0.

    # Create tf Dataset
    copy_dataset = tf.data.Dataset.from_tensor_slices(copy_images).shuffle(BufferSize).batch(BatchSize)

    return copy_images, copy_dataset

train_images, train_dataset = preprocess_data(train_images)
noise_images, noise_dataset = preprocess_data(noise_images)

plt.imshow(np.array(noise_images[0]).reshape((ImgW, ImgH)), cmap=plt.cm.gray)
plt.colorbar()

plt.imshow(np.array(train_images[0]).reshape(ImgW, ImgH), cmap=plt.cm.gray)
plt.colorbar()

# ------------- #
#  Build Model  #
# ------------- #

def build_encoder(latent_dim):

    i = layers.Input(shape=(ImgW, ImgH, 1))
    d = layers.Dense(1)(i)

    # Not able to connect Input with Conv2D, why?
    x = layers.Conv2D(64,(3, 3), padding='valid',activation='relu')(d)
    x = layers.Conv2D(64,(3, 3), padding='valid',activation='relu')(x)
    x = layers.MaxPooling2D((2, 2))(x)
    x = layers.Conv2D(32,(3, 3), padding='valid',activation='relu')(x)
    x = layers.MaxPooling2D((2, 2))(x)

    x = layers.Flatten()(x)

    z = layers.Dense(latent_dim)(x)

    return Model(i, z, name='encoder')

def build_decoder(latent_dim):

    z = layers.Input(shape=latent_dim)

    x = layers.Reshape([1,1,latent_dim])(z)

    x = layers.Conv2DTranspose(128,(1, 1), padding='valid',activation='relu')(x)
    x = layers.Conv2DTranspose(64,(3, 3), padding='valid',activation='relu')(x)
    x = layers.Conv2DTranspose(64,(3, 3), padding='valid',activation='relu')(x)
    x = layers.Conv2DTranspose(48,(3, 3), strides=(2, 2),padding='same',activation='relu')(x)
    x = layers.Conv2DTranspose(48,(3, 3), padding='valid',activation='relu')(x)
    x = layers.Conv2DTranspose(32,(3, 3), strides=(2, 2),padding='same',activation='relu')(x)
    x = layers.Conv2DTranspose(16,(3, 3), padding='valid',activation='relu')(x)

    output = layers.Conv2DTranspose(1,(3, 3), padding='valid',activation='sigmoid')(x)

    return Model(z, output, name='decoder')

encoder = build_encoder(LatentDim)
decoder = build_decoder(LatentDim)

encoder(tf.random.normal([1, ImgW, ImgH, 1]))

_ = decoder(tf.random.normal([1, LatentDim]))
plt.imshow(np.array(_).reshape((ImgW, ImgH)), cmap=plt.cm.gray)
plt.colorbar()

# ------------- #
#  Define Loss  #
# ------------- #

def compute_kernel(x, y):
    x_size = tf.shape(x)[0]
    y_size = tf.shape(y)[0]
    dim = tf.shape(x)[1]
    tiled_x = tf.tile(tf.reshape(x, tf.stack([x_size, 1, dim])), tf.stack([1, y_size, 1]))
    tiled_y = tf.tile(tf.reshape(y, tf.stack([1, y_size, dim])), tf.stack([x_size, 1, 1]))
    return tf.exp(-tf.reduce_mean(tf.square(tiled_x - tiled_y), axis=2) / tf.cast(dim, tf.float32))

def compute_mmd(x, y):
    x_kernel = compute_kernel(x, x)
    y_kernel = compute_kernel(y, y)
    xy_kernel = compute_kernel(x, y)
    return tf.reduce_mean(x_kernel) + tf.reduce_mean(y_kernel) - 2 * tf.reduce_mean(xy_kernel)

def cal_m_loss(z):
    assert z.shape[0] == BatchSize
    assert z.shape[1] == LatentDim
    true_samples = tf.random.normal(tf.shape(z), dtype=tf.float32)
    loss_mmd = compute_mmd(true_samples, z)
    return loss_mmd

def cal_r_loss(img_input, img_output):
    # Because we have applied 'sigmoid' to our img_output in neural network
    # Therefore, 'from_logits' has to be set as False, which means img_output is not a raw probability
    cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=False)
    return cross_entropy(img_input, img_output)

# ------------------ #
#  Define Optimizer  #
# ------------------ #

optimizer = tf.keras.optimizers.Adam(1e-4)

# ------------ #
#  Train Step  #
# ------------ #

@tf.function
def train_step(img_input):

    # Tune the reconstruction level??
    r_to_m_loss_ratio = 0.1

    with tf.GradientTape() as tape:
        z = encoder(img_input)

        img_output = decoder(z)

        loss = cal_r_loss(img_input, img_output) * ImgW * ImgH * r_to_m_loss_ratio + cal_m_loss(z)

    vae_param = encoder.trainable_variables + decoder.trainable_variables

    gradients_of_vae = tape.gradient(loss, vae_param)
    optimizer.apply_gradients(zip(gradients_of_vae, vae_param))

def latent_map(images, labels):

    for i in range(10):
        z = encoder(images[labels == i], training=False)

        z_distribution = [[], []]
        for j in np.array(z):
            z_distribution[0].append(j[0])
            z_distribution[1].append(j[1])

        plt.scatter(z_distribution[0], z_distribution[1], alpha=0.8, label=i)

    plt.legend()

    return plt.gcf()

def sample_save_img(seed, images, epoch, use_VAE=False):

    # Please create a dir './train_record'
    record_dir_path = './train_record'

    # Probe the decoder progress
    fig, axes = plt.subplots(2, 2)

    for i, ax in zip(seed, axes.flatten()):

        if use_VAE:
            z = encoder(images[i].reshape(1, ImgW, ImgH, 1))
            sample = decoder(z)

        else:
            sample = images[i]

        im = ax.imshow(np.array(sample).reshape(ImgW, ImgH), cmap=plt.cm.gray)

    fig.colorbar(im, ax=axes.ravel().tolist())

    if use_VAE:
        plt.savefig('{}/image_at_epoch_{:04d}.png'.format(record_dir_path, epoch + 1))
    else:
        plt.savefig('{}/ground_truth.png'.format(record_dir_path))
    plt.close()

    if use_VAE:
        # Probe the encoder progress
        _ = latent_map(images[:1000], train_labels[:1000])

        plt.savefig('{}/distri_at_epoch_{:04d}.png'.format(record_dir_path, epoch + 1))
        plt.close()

def train(images, dataset, epochs):

    train_start = time.time()

    for epoch in range(epochs):

        epoch_start = time.time()

        for batch in dataset:
            train_step(batch)

        if (epoch + 1) % 2 == 0:
            sample_save_img(Seed, images, epoch, use_VAE=True)
            print ('Time for epoch {} is {:.2f} sec'.format(epoch + 1, time.time()-epoch_start))

    # Generate after the final epoch
    sample_save_img(Seed, images, epoch)
    print('='*30)
    print ('Time for train {} epochs is {:.2f} sec'.format(epoch + 1, time.time()-train_start))

# ---------------- #
#  Start Training  #
# ---------------- #

Epochs = 30
train(noise_images, noise_dataset, Epochs)

# -----------------------------------------#
#  Training Record                         #
#                                          #
#  Time for epoch 2 is 6.78 sec            #
#  Time for epoch 4 is 6.07 sec            #
#  Time for epoch 6 is 5.95 sec            #
#  Time for epoch 8 is 6.01 sec            #
#  Time for epoch 10 is 5.96 sec           #
#  Time for epoch 12 is 6.19 sec           #
#  Time for epoch 14 is 5.99 sec           #
#  Time for epoch 16 is 6.16 sec           #
#  Time for epoch 18 is 6.11 sec           #
#  Time for epoch 20 is 6.00 sec           #
#  Time for epoch 22 is 6.02 sec           #
#  Time for epoch 24 is 6.00 sec           #
#  Time for epoch 26 is 6.26 sec           #
#  Time for epoch 28 is 6.12 sec           #
#  Time for epoch 30 is 5.95 sec           #
#  ==============================          #
#  Time for train 30 epochs is 178.98 sec  #
# ---------------------------------------- #

def image_map(x_min, x_max, y_min, y_max, x_dim=0, y_dim=1):

    record_dir_path = './train_record'

    fix_param = np.random.rand(LatentDim)

    fig, axes = plt.subplots(20, 20, figsize=(10, 10))
    fig.subplots_adjust(hspace=0, wspace=0)

    count = 0
    x_step = (x_max - x_min) / 20
    y_step = (y_max - y_min) / 20

    for i in np.arange(y_min, y_max, y_step):
        for j in np.arange(x_min, x_max, x_step):
            _ = list(fix_param)
            _[x_dim] = i
            _[y_dim] = j
            _ = decoder(np.array(_).reshape(1, LatentDim), training=False)
            axes.ravel()[count].imshow(np.array(_).reshape(ImgW, ImgH), cmap=plt.cm.gray)
            axes.ravel()[count].axis('off')
            axes.ravel()[count].axis('tight')
            count += 1

    fig.savefig(f'{record_dir_path}/image_map.png')

image_map(-1, 1, -1, 1)


#!/usr/bin/env python
#-*- coding:utf-8 -*-

import sys
import os
import csv
import math
import numpy
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot
import joblib

def calculate_n0(r_e):
	size = int(math.ceil(r_e))
	n0 = 0.0
	for y in range(-size, size+1):
		for x in range(-size, size+1):
			r = math.sqrt(x*x + y*y)
			if((0 < r) and (r < r_e)):
				w = r_e/r - 1
				n0 += w
	return n0

def output(dirname, filename, r_e, beta, R, p_theoretical, dt):
	with open(dirname + "/" + filename, "r") as f:
		data = [[math.sqrt(float(line["x"])**2 + float(line["z"])**2), float(line["p"]), float(line["n"])] for line in csv.DictReader(f, skipinitialspace="True")]

	data = numpy.array(data)

	t = dt * int(filename[10:15])
	pyplot.xlim([0, 0.06])
	pyplot.ylim([0, 600])
	pyplot.xlabel("r [m]")
	pyplot.ylabel("p [Pa]")
	pyplot.title("%4.2f [s]" % t)
	pyplot.grid(which="minor", color="lightgray", linestyle="-")
	pyplot.grid(which="major", color="black", linestyle="-", b=True)
	pyplot.minorticks_on()
	pyplot.plot(data.T[0], data.T[1], ".", markersize=1)
	pyplot.plot([0, R], [p_theoretical, 0], "-")
	pyplot.savefig("png/" + filename + ".png")
	pyplot.clf()

	n0 = calculate_n0(r_e)
	n_surface = n0 * beta
	surface = [d[0] for d in data if d[2] < n_surface]
	r_surface_max = max(surface)
	r_surface_min = min(surface)
	roundness = 1.0-(r_surface_max - r_surface_min)/R

	i = data.T[0].argmin() # 中心に一番近いやつ
	p_center = data[i][1]

	return [roundness*100, p_center]

def main(dirname, r_e, beta, dt, L):
	R = L/math.sqrt(math.pi)
	p_theoretical = 1000*9.8*R

	data = joblib.Parallel(n_jobs=-1, verbose=1)([joblib.delayed(output)(dirname, filename, r_e, beta, R, p_theoretical, dt) for filename in sorted(os.listdir(dirname))])
#	data = [output(dirname, filename, r_e, beta, R, p_theoretical, dt) for filename in sorted(os.listdir(dirname))]

	data = numpy.array(data).T
	n = data.shape[1]
	maxT = n * dt
	t = numpy.array(range(n)) * dt

	pyplot.plot(t, data[0])
	pyplot.xlim([0, maxT])
	pyplot.ylim([0, 100])
	pyplot.xlabel("t [s]")
	pyplot.ylabel("Roundness [%]")
	pyplot.grid(which="minor", color="lightgray", linestyle="-")
	pyplot.grid(which="major", color="black", linestyle="-", b=True)
	pyplot.minorticks_on()
	pyplot.savefig("roundness.svg")
	pyplot.clf()

	pyplot.plot(t, data[1])
	pyplot.plot([0, maxT], [p_theoretical, p_theoretical])
	pyplot.xlim([0, maxT])
	pyplot.ylim([0, 600])
	pyplot.xlabel("t [s]")
	pyplot.ylabel("p [Pa]")
	pyplot.grid(which="minor", color="lightgray", linestyle="-")
	pyplot.grid(which="major", color="black", linestyle="-", b=True)
	pyplot.minorticks_on()
	pyplot.savefig("pressure.svg")
	pyplot.clf()

if __name__ == "__main__":
	main(sys.argv[1], float(sys.argv[2]), float(sys.argv[3]), float(sys.argv[4]), float(sys.argv[5]))


from bs4 import BeautifulSoup
from urllib.request import urlopen
import pyexcel

url = 'http://s.cafef.vn/bao-cao-tai-chinh/VNM/IncSta/2017/3/0/0/ket-qua-hoat-dong-kinh-doanh-cong-ty-co-phan-sua-viet-nam.chn'
conn = urlopen(url)
raw_data = conn.read()
web_text = raw_data.decode('utf-8')
soup = BeautifulSoup(web_text,'html.parser')
table = soup.find('table',{'style':'border-top: solid 1px #e6e6e6;border-left:solid 1px #cccccc;border-bottom:solid 1px #cccccc'})
rows = table.find_all('tr')
list1 =[]

for i in range (0,72):
    row = rows[i]
    data_list = row.find_all('td')
    data = {}
    for td in data_list:
        name = data_list[0].string
        data = {
            'Content':name,
            'Quý 4 - 2016':data_list[1].string,
            'Quý 1 - 2017':data_list[2].string,
            'Quý 2 - 2017':data_list[3].string,
            'Quý 3 - 2017':data_list[4].string
        }
    list1.append(data)

pyexcel.save_as(records = list1, dest_file_name = 'holy.xlsx')


import base64
import os
from io import BytesIO
import time
import datetime

import streamlit as st
import pandas as pd
import openpyxl

from .postxml_ema02 import prettyxmlfile, postxml_ema

def xmldownload(xmlfile):
    buffered = BytesIO()
    file = open(xmlfile,'rb').read()
    buffered.write(file)
    b64 = base64.b64encode(buffered.getvalue()).decode()
    href = f'Download XML file'
    return href

def add_block(checklist,label,inexcel,showlevel=4):
    #checklist.append('NONE')
    st.write(f'## Section of {label}')
    dfindex = len(checklist)-1
    temp = st.selectbox(label,checklist,index=dfindex)
    groups = 'test'
    if temp != 'NONE':
        df = pd.read_excel(inexcel,temp,dtype='object')
        if showlevel==4:
            groups = st.multiselect("Groups",df.columns,key='g_'+temp)
            total = st.multiselect('Total',df.columns,key='t_'+temp)
            sets = st.multiselect('Analysis Sets',df.columns,key='s_'+temp)
        elif showlevel == 3:
            groups = st.multiselect("Groups",df.columns,key='g_'+temp)
            sets = st.multiselect('Analysis Sets',df.columns,key='s_'+temp)
        elif showlevel == 2:
            groups = st.multiselect("Groups",df.columns,key='g_'+temp)
        else:
            pass
        st.write('The data is:')
        st.dataframe(df)
        if showlevel==4:
            return df, groups,total, sets
        elif showlevel==3:
            return df, groups,sets
        elif showlevel==2:
            return df, groups
        else:
            return df
    else:
        return None

def app():
    inxml = st.file_uploader('Upload XML file',key='Upload_xml')
    incountry = st.file_uploader('Upload Eucat contry excel file',key='Upload_country',type='XLSX')
    if incountry:
        country_sheets = openpyxl.load_workbook(incountry).sheetnames
        country=st.selectbox("Country:",country_sheets)
    else:
        incountry='euctid_country.xlsx'
        country='country'

    country_sheets = openpyxl.load_workbook(incountry).sheetnames
    country=st.selectbox("Country:",country_sheets)
    eucat_id=pd.read_excel(incountry,country)
    st.write('The Country EuCAT ID is:')
    st.dataframe(eucat_id)

    inexcel = st.file_uploader('Upload Excel created base on TLF meta',key='Upload_excel',type='XLSX')
    if inexcel:
        insheets = openpyxl.load_workbook(inexcel).sheetnames
        if isinstance(insheets,str):
            insheets = [insheets]
        if 'NONE' not in insheets:
            insheets=insheets+['NONE']
        country_input = add_block(insheets,'Country',inexcel,1)
        age_input = add_block(insheets,'Age',inexcel,2)
        disp_input = add_block(insheets,'Disposition',inexcel,2)
        age_n_input = add_block(insheets,'Age continue',inexcel,3)
        age_c_input = add_block(insheets,'Age category',inexcel)
        gender_input = add_block(insheets,'Gender',inexcel)
        demo_c_input = add_block(insheets,'Demography category',inexcel)
        demo_n_input = add_block(insheets,'Demography continue',inexcel,3)
        endpoints_input = add_block(insheets,'Endpoints',inexcel,3)
        stats_input = add_block(insheets,'Stat. analysis',inexcel,1)
        if st.button('Update XML'):
            test = postxml_ema(inxml)
            if len(country_input)>0:
                test.add_country_count(country_input,incountry,country)
            if age_input:
                test.add_agegroup(age_input[0],age_input[1][0])
            if disp_input:
                test.add_patientflow(disp_input[0],disp_input[1])
            if gender_input:
                test.add_gender_cat_characters(gender_input[0],gender_input[1],gender_input[2],
                gender_input[3])
            if age_n_input:
                test.add_age_cont_characters(age_n_input[0],age_n_input[1],age_n_input[2])
            if age_c_input:
                test.add_age_cat_characters(age_c_input[0],age_c_input[1],age_c_input[2],
                age_c_input[3])
            if demo_c_input:
                test.add_baseline_cat_characters(demo_c_input[0],demo_c_input[1],demo_c_input[2],
                demo_c_input[3])
            if demo_n_input:
                test.add_baseline_cont_characters(demo_n_input[0],demo_n_input[1],demo_n_input[2])
            if endpoints_input:
                test.add_endpoints(endpoints_input[0],endpoints_input[1],endpoints_input[2],
                stats_input)
            filename = 'sp'+datetime.datetime.now().strftime("%Y%m%d%h%m%c").replace(':','').replace(' ','')+'.xml'
            test.tofile(filename,pretty=False)
            prettyxmlfile(filename)
            st.markdown(xmldownload(filename),unsafe_allow_html=True)
            os.remove(filename)


# With acknowledgement to deeplearning.ai
# Coursera courses by Andrew Ng
import numpy as np
import pandas as pd
from sklearn.preprocessing import normalize
from sklearn.model_selection import train_test_split
import h5py
import scipy
import matplotlib.pyplot as plt

class NeuralNetwork:

    def __init__(self, layer_dims, learning_rate = 0.0075):
        self.layer_dims = layer_dims
        self.L = len(self.layer_dims) - 1
        self.model_caches = []
        self.learning_rate = learning_rate
        np.random.seed(42)

    def sigmoid(self, Z):
        return 1/(1+np.exp(-Z))

    def sigmoid_backward(self, dA, Z):
        s = 1/(1+np.exp(-Z))
        dZ = dA * s * (1-s)
        return dZ

    def relu(self, Z):
        return np.maximum(0,Z)

    def relu_backward(self, dA, Z):
        dZ = np.array(dA, copy=True)
        # When z <= 0, set dz to 0 as well.
        dZ[Z <= 0] = 0
        return dZ

    def initialize_parameters_he(self):
        parameters = {}
        for l in range(1, self.L + 1):
            parameters['W' + str(l)] = np.random.randn(self.layer_dims[l], self.layer_dims[l-1]) * np.sqrt(2/self.layer_dims[l-1])
            parameters['b' + str(l)] = np.zeros((self.layer_dims[l], 1))

        return parameters

    def linear_activation_forward(self, A_prev, W, b, activation):
        linear_cache = (A_prev, W, b)
        Z = np.dot(W,A_prev) + b

        if activation == "sigmoid":
            A = self.sigmoid(Z)
        elif activation == "relu":
            A = self.relu(Z)

        activation_cache = Z
        cache = (linear_cache, activation_cache)
        return A, cache

    def model_forward(self, X):
        A = X

        # Implement [LINEAR -> RELU]*(L-1)
        for l in range(1, self.L):
            A_prev = A
            A, cache = self.linear_activation_forward(A_prev, self.parameters['W' + str(l)], self.parameters['b' + str(l)], 'relu')
            self.model_caches.append(cache)

        # Implement LINEAR -> SIGMOID
        AL, cache = self.linear_activation_forward(A, self.parameters['W' + str(self.L)], self.parameters['b' + str(self.L)], 'sigmoid')
        self.model_caches.append(cache)

        return AL

    def compute_cost(self, AL, Y):
        m = Y.shape[1]
        cost = (-1/m)*(np.sum(np.multiply(np.log(AL),Y) + np.multiply((1-Y),np.log(1-AL))))
        cost = np.squeeze(cost)
        return cost

    def linear_backward(self, dZ, cache):
        A_prev, W, b = cache
        m = A_prev.shape[1]

        dW = 1 / m * (np.dot(dZ, A_prev.T))
        db = 1 / m * np.sum(dZ, axis=1, keepdims=True)
        dA_prev = (np.dot(W.T, dZ))

        return dA_prev, dW, db

    def linear_activation_backward(self, dA, cache, activation):
        linear_cache, activation_cache = cache

        if activation == "relu":
            dZ = self.relu_backward(dA, activation_cache)

        elif activation == "sigmoid":
            dZ = self.sigmoid_backward(dA, activation_cache)

        dA_prev, dW, db = self.linear_backward(dZ, linear_cache)

        return dA_prev, dW, db

    def model_backward(self, AL, Y):
        grads = {}
        m = AL.shape[1]
        Y = Y.reshape(AL.shape)

        # Initializing the backpropagation
        dAL = - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))

        # (SIGMOID -> LINEAR) gradients.
        current_cache = self.model_caches[self.L-1]
        grads["dA" + str(self.L)], grads["dW" + str(self.L)], grads["db" + str(self.L)] = self.linear_activation_backward(dAL, current_cache, activation = "sigmoid")

        for l in reversed(range(self.L-1)):
            # (RELU -> LINEAR) gradients.
            current_cache = self.model_caches[l]
            dA_prev_temp, dW_temp, db_temp = self.linear_activation_backward(grads["dA" + str(l + 2)], current_cache, activation = "relu")
            grads["dA" + str(l + 1)] = dA_prev_temp
            grads["dW" + str(l + 1)] = dW_temp
            grads["db" + str(l + 1)] = db_temp

        return grads

    def update_parameters(self, grads):

        for l in range(self.L):
            self.parameters["W" + str(l+1)] = self.parameters["W" + str(l+1)] - self.learning_rate * grads["dW" + str(l+1)]
            self.parameters["b" + str(l+1)] = self.parameters["b" + str(l+1)] - self.learning_rate * grads["db" + str(l+1)]

    def initialize_momentum(self):

        self.momentum_v = {}
        for l in range(self.L):
            self.momentum_v["dW" + str(l + 1)] = np.zeros_like(self.parameters["W" + str(l+1)])
            self.momentum_v["db" + str(l + 1)] = np.zeros_like(self.parameters["b" + str(l+1)])

    def update_parameters_momentum(self, grads, beta = 0.9):

        for l in range(self.L):
            self.momentum_v["dW" + str(l + 1)] = beta * self.momentum_v["dW" + str(l + 1)] + (1 - beta) * grads['dW' + str(l + 1)]
            self.momentum_v["db" + str(l + 1)] = beta * self.momentum_v["db" + str(l + 1)] + (1 - beta) * grads['db' + str(l + 1)]
            self.parameters["W" + str(l + 1)] = self.parameters["W" + str(l + 1)] - self.learning_rate * self.momentum_v["dW" + str(l + 1)]
            self.parameters["b" + str(l + 1)] = self.parameters["b" + str(l + 1)] - self.learning_rate * self.momentum_v["db" + str(l + 1)]

    def train(self, X, Y, use_momentum, num_iterations = 1000, print_cost_n = 100):

        # initialize parameters
        self.parameters = self.initialize_parameters_he()

        self.costs = []

        if use_momentum:
            self.initialize_momentum()

        for i in range(0, num_iterations):
            AL = self.model_forward(X)
            cost = self.compute_cost(AL, Y)
            grads = self.model_backward(AL, Y)
            if use_momentum:
                self.update_parameters_momentum(grads)
            else:
                self.update_parameters(grads)
            self.model_caches = []

            # Print the cost every print_cost_n training example
            if i % print_cost_n == 0:
                print ("Cost after iteration %i: %f" %(i, cost))
                self.costs.append(cost)

    def plot_costs(self):

        plt.plot(np.squeeze(self.costs))
        plt.ylabel('cost')
        plt.xlabel('iterations (per hundreds)')
        plt.title("Learning rate =" + str(self.learning_rate))
        plt.show()

    def predict(self, X, y):
        m = X.shape[1]
        n = len(self.parameters) // 2
        p = np.zeros((1,m))

        probs = self.model_forward(X)

        for i in range(0, probs.shape[1]):
            if probs[0,i] > 0.5:
                p[0,i] = 1
            else:
                p[0,i] = 0

        print("Accuracy: "  + str(np.sum((p == y)/m)))
        return p

if __name__ == "__main__":

    df = pd.read_csv('IPO2609FeatureEngineering.csv')
    ipo_train, ipo_test = train_test_split(df, random_state=42)

    # Columns for training (X), remove label (y)
    features = df.columns.drop(["underpriced"])

    train_x = ipo_train[features.values.tolist()].T.astype(float)
    train_x = normalize(train_x)
    train_y = np.ravel(ipo_train["underpriced"])
    train_y = train_y.reshape(1, train_y.shape[0])

    test_x = ipo_test[features.values.tolist()].T.astype(float)
    test_x = normalize(test_x)
    test_y = np.ravel(ipo_test["underpriced"])
    test_y = test_y.reshape(1, test_y.shape[0])

    ipo_layer_dims = [train_x.shape[0], 15, 7, 5, 1]

    # Train/Predict with Gradient Descent
    nn = NeuralNetwork(ipo_layer_dims, .05)
    nn.train(train_x, train_y, False, 2800)
    nn.plot_costs()
    p = nn.predict(test_x, test_y)

    # Train / Predict with Momentum
    nn = NeuralNetwork(ipo_layer_dims, 0.03)
    nn.train(train_x, train_y, True, 7500)
    nn.plot_costs()
    p = nn.predict(test_x, test_y)


import torch
import numpy as np
from thop import profile
from thop import clever_format
if torch.__version__ >= "1.8":
    import torch_npu

def clip_gradient(optimizer, grad_clip):
    """
    For calibrating misalignment gradient via cliping gradient technique
    :param optimizer:
    :param grad_clip:
    :return:
    """
    for group in optimizer.param_groups:
        for param in group['params']:
            if param.grad is not None:
                param.grad.data.clamp_(-grad_clip, grad_clip)

def adjust_lr(optimizer, init_lr, epoch, decay_rate=0.1, decay_epoch=30):
    decay = decay_rate ** (epoch // decay_epoch)
    for param_group in optimizer.param_groups:
        param_group['lr'] *= decay

class AvgMeter(object):
    def __init__(self, num=40):
        self.num = num
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0
        self.losses = []

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count
        self.losses.append(val)

    def show(self):
        return torch.mean(torch.stack(self.losses[np.maximum(len(self.losses)-self.num, 0):]))

def CalParams(model, input_tensor):
    """
    Usage:
        Calculate Params and FLOPs via [THOP](https://github.com/Lyken17/pytorch-OpCounter)
    Necessarity:
        from thop import profile
        from thop import clever_format
    :param model:
    :param input_tensor:
    :return:
    """
    flops, params = profile(model, inputs=(input_tensor,))
    flops, params = clever_format([flops, params], "%.3f")
    print('[Statistics Information]\nFLOPs: {}\nParams: {}'.format(flops, params))

class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self, name, fmt=':f', start_count_index=2):
        self.name = name
        self.fmt = fmt
        self.reset()
        self.start_count_index = start_count_index

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        if self.count == 0:
            self.N = n

        self.val = val
        self.count += n
        if self.count > (self.start_count_index * self.N):
            self.sum += val * n
            self.avg = self.sum / (self.count - self.start_count_index * self.N)

    def __str__(self):
        fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'
        return fmtstr.format(**self.__dict__)

class Conv2dForAvgPool2d(torch.nn.Conv2d):
    def __init__(self, channels, kernel_size, stride, padding):
        super(Conv2dForAvgPool2d, self).__init__(channels, channels,
                                                 kernel_size, stride,
                                                 padding, groups=channels,
                                                 bias=False)
        self.weight.data = torch.ones_like(self.weight.data) / (kernel_size * kernel_size)
        self.requires_grad_(False)
        self.npu()
    def forward(self, x):
        return super().forward(x)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
Measure the harmonic distortion (HD2 and HD3) of the DAC and ADC combined.
Connect DAC0 to ADC0 to perform the measurement.
'''

import numpy as np
import os
import time
import matplotlib.pyplot as plt
from fft import FFT
from koheron import connect

host = os.getenv('HOST', '192.168.1.16')
client = connect(host, 'fft', restart=False)
driver = FFT(client)

# Use Rectangular FFT window
driver.set_fft_window(0)

n_pts = driver.n_pts
fs = 250e6

fmin = 0.1e6
fmax = 40e6

freqs = np.linspace(fmin, fmax ,num=500)
freqs = np.round(freqs / fs * n_pts) * fs / n_pts

hd1 = 0.0 * freqs
hd2 = 0.0 * freqs
hd3 = 0.0 * freqs

for i, freq in enumerate(freqs):
    n = np.uint32(freq / fs * n_pts)
    driver.set_dds_freq(0, freq)
    time.sleep(0.5)
    psd = driver.read_psd()
    psd_db = 10*np.log10(psd)

    hd1[i] = psd_db[n-1]
    hd2[i] = psd_db[2*n-1] - hd1[i]
    hd3[i] = psd_db[3*n-1] - hd1[i]

    print(freq, hd1[i], hd2[i], hd3[i])

fig = plt.figure(figsize=(6,6))
ax = fig.add_subplot(111)

ax.set_xlabel('Frequency (MHz)')
ax.set_ylabel('Harmonic distortion (dB)')
ax.semilogx(freqs*1e-6, hd2, label='HD2')
ax.semilogx(freqs*1e-6, hd3, label='HD3')
ax.set_xlim([fmin * 1e-6, fmax * 1e-6])
ax.legend(loc=2)

plt.show()

np.save('alpha250_dac_distortion.npy', (freqs, hd2, hd3))


'''
0008/0025
一个HTML文件
找出里面的正文
'''
from bs4 import BeautifulSoup
import requests

def search_body(path):
    page = requests.get(path)
    page.encoding = 'utf-8'
    soup = BeautifulSoup(str(page.text), 'html.parser')
    article = soup.find_all('body')
    print(article)

if __name__ == "__main__":
    Path = 'https://www.runoob.com/python/python-tutorial.html'
    search_body(Path)


from datetime import datetime

# def transDataFormat(datas={},keyname="",dateformat=""):
#     if (keyname in datas.keys()):
#         dt = datetime.fromtimestamp( int(datas[keyname]) / 1000 )
#         datas[keyname] = dt.strftime(dateformat )
#         print( datas[keyname] )

def transDataFormat(dateStr,dateformat=""):
        if(dateStr!=None and str(dateStr)!="0"):
                dt = datetime.fromtimestamp( int(dateStr) / 1000)
                return dt.strftime( dateformat )
        else:
                return ""
if __name__ == '__main__':
        transDataFormat(0,"%Y%m%d")


import json
import time
import random
from datetime import datetime
from parameters import TRANSACTIONS_TOPIC, DELAY
from streaming_kafka.confluent_utils import create_producer

_id = 0
producer = create_producer()

if producer is not None:

    fake_data = ["Noor", "Victor", "Yusuf", "Bayang", 123, 152, 555]
    while True:
        # Generate some wrong good case to simulate normal and abnormal behavior
        for_pred = random.choice(fake_data)

        X_test = for_pred

        current_time = datetime.utcnow().isoformat()

        record = {"id": _id, "data": X_test, "current_time": current_time}
        print(f"producing event_{_id}: {record}")
        record = json.dumps(record).encode("utf-8")

        producer.produce(topic=TRANSACTIONS_TOPIC,
                         value=record)
        producer.flush()

        _id += 1
        time.sleep(DELAY)