import json
import urllib.parse
import urllib.request
import discord
import requests
from discord.ext import commands
import spotipy.util as util
import spotipy
import re
from datetime import datetime
from utils import logger as misolog
from utils import misc as misomisc
import random
import tweepy
from tweepy import OAuthHandler
import os
import html
import main

keys = os.environ
OXFORD_APPID = keys['OXFORD_APPID']
OXFORD_TOKEN = keys['OXFORD_TOKEN']
NAVER_APPID = keys['NAVER_APPID']
NAVER_TOKEN = keys['NAVER_TOKEN']
LASTFM_APPID = keys['LASTFM_APIKEY']
LASTFM_TOKEN = keys['LASTFM_SECRET']
TIMEZONE_API_KEY = keys['TIMEZONEDB_API_KEY']
SPOTIFY_CLIENT_ID = keys['SPOTIFY_CLIENT_ID']
SPOTIFY_CLIENT_SECRET = keys['SPOTIFY_CLIENT_SECRET']
GOOGLE_API_KEY = keys['GOOGLE_KEY']
DARKSKY_API_KEY = keys['DARK_SKY_KEY']
STEAM_API_KEY = keys['STEAM_WEB_API_KEY']
WOLFRAM_APPID = keys['WOLFRAM_APPID']
TWITTER_CKEY = keys['TWITTER_CONSUMER_KEY']
TWITTER_CSECRET = keys['TWITTER_CONSUMER_SECRET']

auth = OAuthHandler(TWITTER_CKEY, TWITTER_CSECRET)
# auth.set_access_token(access_token, access_secret)
twt = tweepy.API(auth)

papago_pairs = ['ko/en', 'ko/ja', 'ko/zh-cn', 'ko/zh-tw', 'ko/vi', 'ko/id', 'ko/de', 'ko/ru', 'ko/es', 'ko/it',
                'ko/fr', 'en/ja', 'ja/zh-cn', 'ja/zh-tw', 'zh-cn/zh-tw', 'en/ko', 'ja/ko', 'zh-cn/ko', 'zh-tw/ko',
                'vi/ko', 'id/ko', 'th/ko', 'de/ko', 'ru/ko', 'es/ko', 'it/ko', 'fr/ko', 'ja/en', 'zh-cn/ja',
                'zh-tw/ja', 'zh-tw/zh-tw']

database = main.database

class Apis(commands.Cog):

    def __init__(self, client):
        self.client = client
        self.logger = misolog.create_logger(__name__)

    @commands.command()
    async def weather(self, ctx, *args):
        """Get weather of a location"""
        await ctx.message.channel.trigger_typing()
        self.logger.info(misolog.format_log(ctx, f""))
        address = "+".join(args)
        url = f"https://maps.googleapis.com/maps/api/geocode/json?address={address}&key={GOOGLE_API_KEY}"
        response = requests.get(url=url)
        if response.status_code == 200:
            json_data = json.loads(response.content.decode('utf-8'))['results'][0]
            # print(json.dumps(json_data, indent=4))
            formatted_name = json_data['formatted_address']
            lat = json_data['geometry']['location']['lat']
            lon = json_data['geometry']['location']['lng']
            country = "N/A"
            for comp in json_data['address_components']:
                if 'country' in comp['types']:
                    country = comp['short_name'].lower()

            # we have lat and lon now, plug them into dark sky
            response = requests.get(url=f"https://api.darksky.net/forecast/{DARKSKY_API_KEY}/{lat},{lon}?units=si")
            if response.status_code == 200:
                json_data = json.loads(response.content.decode('utf-8'))
                # print(json.dumps(json_data, indent=4))
                current = json_data['currently']
                hourly = json_data['hourly']
                daily = json_data['daily']
                time = self.get_timezone({'lat': lat, 'lon': lon})

                message = discord.Embed(color=discord.Color.dark_purple())
                message.set_thumbnail(url=f"http://flagpedia.net/data/flags/w580/{country}.png")
                message.set_author(name=formatted_name, icon_url=ctx.author.avatar_url)
                message.add_field(name=hourly['summary'],
                                  value=f"**{current['temperature']} °C** "
                                  f"({current['temperature'] * (9.0 / 5.0) + 32:.2f} °F) **|** Feels like "
                                  f"**{current['apparentTemperature']} °C** "
                                  f"({current['apparentTemperature'] * (9.0 / 5.0) + 32:.2f} °F)\n"
                                  f"Wind speed: **{current['windSpeed']} m/s**")

                message.add_field(name="Weekly forecast:",
                                  value=" ".join(f"**{x}**" if "°C" in x else x for x in daily['summary'].split(" ")))

                message.set_footer(text=f"Local time: {time}")

                await ctx.send(embed=message)

    @commands.command(aliases=['hs'])
    async def horoscope(self, ctx, setting=None, *args):
        """Get your daily horoscope"""
        await ctx.message.channel.trigger_typing()
        self.logger.info(misolog.format_log(ctx, f""))
        hs = ['aries', 'taurus', 'gemini', 'cancer', 'leo', 'virgo', 'libra', 'scorpio', 'sagittarius', 'capricorn',
              'aquarius', 'pisces']
        if setting == "set":
            if args[0].lower() in hs:
                sign = args[0].lower()
            else:
                await ctx.send(f"`{args[0]}` is not a valid sunsign! use `>horoscope help` for a list of sunsigns.")
                return
            # set sign
            database.set_attr("users", f"{ctx.author.id}.sunsign", sign)
            await ctx.send(f"Sunsign saved as `{sign}`")
            return
        elif setting == "help":
            sign_list = "Aries (Mar 21-Apr 19)\nTaurus (Apr 20-May 20)\nGemini (May 21-June 20)\n" \
                        "Cancer (Jun 21-Jul 22)\nLeo (Jul 23-Aug 22)\nVirgo (Aug 23-Sep 22)\nLibra (Sep 23-Oct 22)\n" \
                        "Scorpio (Oct 23-Nov 21)\nSagittarius (Nov 22-Dec 21)\nCapricorn (Dec 22-Jan 19)\n" \
                        "Aquarius (Jan 20-Feb 18)\nPisces (Feb 19-Mar 20)"
            content = discord.Embed()
            content.title = f"Sunsign list"
            content.description = sign_list
            await ctx.send(embed=content)
            return
        elif setting is not None:
            # chosen sign
            if setting in hs:
                sign = setting
            else:
                await ctx.send(f"`{setting}` is not a valid sunsign! use `>horoscope help` for a list of sunsigns.")
                return
        else:
            # get user's sign
            sign = database.get_attr("users", f"{ctx.author.id}.sunsign")
            if sign is None:
                await ctx.send("Please save your sunsign using `>horoscope set`\n"
                               "use `>horoscope help` if you don't know which one you are.")
                return

        url = f"http://theastrologer-api.herokuapp.com/api/horoscope/{sign}/today"
        response = requests.get(url)
        response_data = json.loads(response.content.decode('utf-8'))
        content = discord.Embed()
        content.title = response_data['sunsign']
        content.set_footer(text=response_data['date'])
        content.add_field(name='Mood', value=response_data['meta']['mood'].capitalize(), inline=True)
        content.add_field(name='Keywords', inline=True,
                          value=", ".join([x.capitalize() for x in response_data['meta']['keywords'].split(", ")]))
        content.add_field(name='Intensity', value=response_data['meta']['intensity'], inline=True)
        content.add_field(name='Horoscope', value=response_data['horoscope'].split("(c)")[0])
        await ctx.send(embed=content)

    @commands.command()
    async def define(self, ctx, *args):
        """Search from oxford dictionary"""
        await ctx.message.channel.trigger_typing()
        self.logger.info(misolog.format_log(ctx, f""))
        search_string = ' '.join(args)
        api_url = 'https://od-api.oxforddictionaries.com:443/api/v1'
        query = f'''/search/en?q={search_string}&prefix=false'''
        response = requests.get(api_url + query, headers={
            'Accept': 'application/json',
            'app_id': OXFORD_APPID,
            'app_key': OXFORD_TOKEN,
        })
        if response.status_code == 200:
            data = json.loads(response.content.decode('utf-8'))
            # searched for word id, now use the word id to get definition
            if data['results']:
                word_id = data['results'][0]['id']
                word_string = data['results'][0]['word']
                response = requests.get(api_url + f"/entries/en/{word_id}", headers={
                    'Accept': 'application/json',
                    'app_id': OXFORD_APPID,
                    'app_key': OXFORD_TOKEN,
                })
                data = json.loads(response.content.decode('utf-8'))
                all_entries = []
                for entry in data['results'][0]['lexicalEntries']:
                    definitions_value = ""
                    name = data['results'][0]['word']

                    for i in range(len(entry['entries'][0]['senses'])):
                        for definition in entry['entries'][0]['senses'][i]['definitions']:
                            definitions_value += f"\n**{i + 1}.** {definition}"
                        try:
                            for y in range(len(entry['entries'][0]['senses'][i]['subsenses'])):
                                for definition in entry['entries'][0]['senses'][i]['subsenses'][y]['definitions']:
                                    definitions_value += f"\n**└{i + 1}.{y + 1}.** {definition}"
                            definitions_value += "\n"
                        except KeyError:
                            pass

                    word_type = entry['lexicalCategory']
                    this_entry = {"id": name, "definitions": definitions_value, "type": word_type}
                    all_entries.append(this_entry)

                definitions_embed = discord.Embed(colour=discord.Colour.blue())
                definitions_embed.set_author(name=word_string.capitalize(), icon_url="https://i.imgur.com/vDvSmF3.png")

                for entry in all_entries:
                    definitions_embed.add_field(name=entry["type"], inline=False,
                                                value=entry["definitions"])

                await ctx.send(embed=definitions_embed)
            else:
                await ctx.send(f"ERROR: no definition found for `{search_string}`")
        else:
            await ctx.send(f"ERROR: status code `{response.status_code}`")

    @commands.command()
    async def urban(self, ctx, *args):
        """Search from urban dictionary"""
        await ctx.message.channel.trigger_typing()
        self.logger.info(misolog.format_log(ctx, f""))
        search_string = " ".join(args)
        url = "http://api.urbandictionary.com/v0/define?term="
        response = requests.get(url + search_string)
        if response.status_code == 200:
            message = discord.Embed(colour=discord.Colour.orange())
            message.set_author(name=search_string.capitalize(), icon_url="https://i.imgur.com/yMwpnBe.png")

            json_data = json.loads(response.content.decode('utf-8'))
            # print(json.dumps(json_data, indent=4))

            if json_data['list']:
                word = json_data['list'][0]
                definition = word['definition'].replace("]", "").replace("[", "")
                example = word['example'].replace("]", "").replace("[", "")
                time = word['written_on'][:9].replace("-", "/")
                message.description = f"{definition}"
                message.add_field(name="Example", value=example)
                message.set_footer(text=f"by {word['author']} on {time}")
                await ctx.send(embed=message)
            else:
                await ctx.send("No definition found for " + search_string)
        else:
            await ctx.send("Error: " + str(response.status_code))

    @commands.command(aliases=['tr', 'trans'])
    async def translate(self, ctx, *text):
        """Translator that uses naver papago when possible, using google translator otherwise"""
        await ctx.message.channel.trigger_typing()
        if text[0] == "help":
            self.logger.info(misolog.format_log(ctx, f"help"))
            await ctx.send('Format: `>translate source/target "text"`\n'
                           'Example: `>translate ko/en 안녕하세요`\n\n'
                           'Leave source empty to detect language automatically.\n'
                           'Example: `>translate /en こんにちは`\n\n'
                           'When no language codes given, defaults to detected -> english.\n'
                           'Example: `>translate ㅋㅋㅋ`')
            return
        if "/" in text[0]:
            source, target = text[0].split("/")
            text = text[1:]
            if source == "":
                source = self.detect_language(" ".join(text))
            if target == "":
                target = "en"
        else:
            source = self.detect_language(" ".join(text))
            target = "en"
        query_text = " ".join(text)
        language_pair = f"{source}/{target}"
        # we have language and query, now choose the appropriate translator

        if language_pair in papago_pairs:
            # use papago
            self.logger.info(misolog.format_log(ctx, f"papago [{language_pair}]"))
            query = f"source={source}&target={target}&text={query_text}"
            api_url = 'https://openapi.naver.com/v1/papago/n2mt'
            request = urllib.request.Request(api_url)
            request.add_header('X-Naver-Client-Id', NAVER_APPID)
            request.add_header('X-Naver-Client-Secret', NAVER_TOKEN)
            response = urllib.request.urlopen(request, data=query.encode('utf-8'))
            data = json.loads(response.read().decode('utf-8'))
            translation = data['message']['result']['translatedText']

        else:
            # use google
            self.logger.info(misolog.format_log(ctx, f"google [{language_pair}]"))
            url = f"https://translation.googleapis.com/language/translate/v2?key={GOOGLE_API_KEY}" \
                  f"&model=nmt&target={target}&source={source}&q={query_text}"
            response = requests.get(url)
            data = json.loads(response.content.decode('utf-8'))
            translation = html.unescape(data['data']['translations'][0]['translatedText'])

        await ctx.send(f"`{source}->{target}` " + translation)

    @commands.command()
    async def spotify(self, ctx, url=None, amount=15):
        """Analyze a spotify playlist from URI"""
        await ctx.message.channel.trigger_typing()
        self.logger.info(misolog.format_log(ctx, f""))
        # noinspection PyBroadException
        try:
            if url.startswith("https://open."):
                # its playlist link
                user_id = re.search(r'user/(.*?)/playlist', url).group(1)
                playlist_id = re.search(r'playlist/(.*?)\?', url).group(1)
            else:
                # its URI (probably)
                data = url.split(":")
                playlist_id = data[4]
                user_id = data[2]
        except Exception:
            await ctx.send("Usage: `>spotify {url/URI} {amount to show}(optional)`\n"
                           "How to get Spotify URI?: Right click playlist -> Share -> Copy Spotify URI")
            return

        if amount > 50:
            amount = 50

        token = util.oauth2.SpotifyClientCredentials(client_id=SPOTIFY_CLIENT_ID, client_secret=SPOTIFY_CLIENT_SECRET)
        cache_token = token.get_access_token()
        spotify = spotipy.Spotify(cache_token)
        tracks_per_request = 100
        results = []

        playlist_data = spotify.user_playlist(user_id, playlist_id)
        playlist_name = playlist_data['name']
        playlist_owner = playlist_data['owner']['display_name']
        playlist_image = playlist_data['images'][0]['url']

        this_request_results = spotify.user_playlist_tracks(user_id, playlist_id, limit=tracks_per_request,
                                                            offset=0)["items"]
        for i in range(len(this_request_results)):
            results.append(this_request_results[i])
        while len(this_request_results) >= tracks_per_request:
            this_request_results = spotify.user_playlist_tracks(user_id, playlist_id, limit=tracks_per_request,
                                                                offset=len(results))["items"]
            for i in range(len(this_request_results)):
                results.append(this_request_results[i])

        artists_dict = {}
        total = 0
        for i in range(len(results)):
            artist = results[i]["track"]["artists"][0]["name"]
            if artist in artists_dict:
                artists_dict[artist] += 1
            else:
                artists_dict[artist] = 1
            total += 1

        count = 0
        description = ""
        for item in sorted(artists_dict.items(), key=lambda v: v[1], reverse=True):
            if count < amount:
                percentage = (item[1] / total) * 100
                description += f"**{item[1]}** tracks ({percentage:.2f}%) — **{item[0]}**\n"
                count += 1
            else:
                break

        message = discord.Embed(colour=discord.Colour.green())
        message.set_author(name=f"{playlist_name} · by {playlist_owner}",
                           icon_url="https://i.imgur.com/tN20ywg.png")
        message.set_thumbnail(url=playlist_image)
        message.title = "Artist distribution:"
        message.set_footer(text=f"Total: {total} tracks from {len(artists_dict)} different artists")
        message.description = description

        await ctx.send(embed=message)

    @commands.command(name="convert")
    async def convert(self, ctx, *args):
        """Converts various units"""
        await ctx.message.channel.trigger_typing()
        source_quantity = args[0]
        source_unit, source_name = self.get_ucum_code(args[1])
        target_unit, target_name = self.get_ucum_code(args[len(args)-1])

        base_url = "https://ucum.nlm.nih.gov/ucum-service/v1/ucumtransform/"
        request_format = f"{source_quantity}/from/{source_unit}/to/{target_unit}"

        response = requests.get(base_url + request_format, headers={"Accept": "application/json"})
        if response.status_code == 200:
            json_data = json.loads(response.content.decode('utf-8'))
            # print(json.dumps(json_data, indent=4))
            try:
                converted_quantity = json_data['UCUMWebServiceResponse']['Response']['ResultQuantity']
                await ctx.send(f"**{source_quantity} {source_name}** is **{converted_quantity} {target_name}**")
                self.logger.info(misolog.format_log(ctx, f"from={source_name}[{source_quantity}], "
                                                         f"to={target_name}[{converted_quantity}]"))
            except TypeError:
                await ctx.send(f"```{json_data['UCUMWebServiceResponse']['Response']}```")
                self.logger.error(misolog.format_log(ctx, json_data['UCUMWebServiceResponse']['Response']))

        else:
            await ctx.send(f"Error {response.status_code}")

    @commands.command(name="currency")
    async def currency(self, ctx, *args):
        """Perform currency conversion"""
        await ctx.message.channel.trigger_typing()
        try:
            source_q = float(args[0])
            source_curr = args[1]
            target_curr = args[3]
        except IndexError:
            await ctx.send("Invalid syntax! example: `>currency 20 usd in eur`")
            self.logger.warning(misolog.format_log(ctx, f"Invalid Syntax"))
            return

        response = requests.get(url=f"https://free.currencyconverterapi.com/api/v6/convert"
                                    f"?q={source_curr}_{target_curr}")
        if response.status_code == 200:
            json_data = json.loads(response.content.decode('utf-8'))
            if json_data['results']:
                conversion = json_data['results'][f'{source_curr}_{target_curr}'.upper()]
                rate = conversion['val']
                target_q = source_q*rate
                await ctx.send(f"**{source_q:.2f} {conversion['fr']}** is **{target_q:.2f} {conversion['to']}**")
                self.logger.info(misolog.format_log(ctx, f""))
            else:
                await ctx.send("Invalid currency code!")
                self.logger.warning(misolog.format_log(ctx, f"Invalid Currency"))

    @commands.command(name="color", aliases=["colour"])
    async def color(self, ctx, *args):
        """Get a hex color, the color of discord user, or a random color."""
        await ctx.message.channel.trigger_typing()
        if not args:
            await ctx.send("Missing color source. Valid color sources are:\n"
                           "`[@mention | @rolemention | hex | image_url | random]`\n"
                           "These can be chained together to create patterns")
            return
        if args[0] == "random":
            colors_input = []
            try:
                amount = int(args[1])
            except IndexError:
                amount = 1
            for i in range(amount):
                colors_input += ["{:06x}".format(random.randint(0, 0xFFFFFF))]
        elif ctx.message.mentions:
            colors_input = [str(x.color).replace("#", "") for x in ctx.message.mentions]
        else:
            colors_input = [x.replace("#", "") for x in args]
        colors = []
        for color in colors_input:
            try:
                content = discord.Embed(colour=int(color, 16))
                this_color = color
            except ValueError:
                this_color = misomisc.get_color(color)
                content = discord.Embed(colour=int(this_color, 16))
            colors.append(this_color)
        if len(colors) == 1:
            color = colors[0]
            url = f"http://thecolorapi.com/id?hex={color}&format=json"
            response = requests.get(url=url)
            if response.status_code == 200:
                data = json.loads(response.content.decode('utf-8'))
                hexvalue = data['hex']['value']
                rgbvalue = data['rgb']['value']
                name = data['name']['value']
                image_url = f"http://www.colourlovers.com/img/{color}/200/200/color.png"
                content.title = name
                content.description = f"{hexvalue} - {rgbvalue}"
            else:
                self.logger.error(misolog.format_log(ctx, f"color={color}&statuscode={response.status_code}"))
                await ctx.send(f"Error {response.status_code} on color {color}")
                return
        else:
            palette = ""
            for color in colors:
                url = f"http://thecolorapi.com/id?hex={color}&format=json"
                response = requests.get(url=url)
                if response.status_code == 200:
                    data = json.loads(response.content.decode('utf-8'))
                    hexvalue = data['hex']['value']
                    # rgbvalue = data['rgb']['value']
                    name = data['name']['value']
                    content.add_field(name=name, value=f"{hexvalue}")
                    palette += color + "/"
                else:
                    self.logger.error(misolog.format_log(ctx, f"color={color}&statuscode={response.status_code}"))
                    await ctx.send(f"Error {response.status_code} on color {color}")
                    return
            image_url = f"https://www.colourlovers.com/paletteImg/{palette}palette.png"

        content.set_image(url=image_url)
        await ctx.send(embed=content)
        self.logger.info(misolog.format_log(ctx, f"colors={colors}"))

    @commands.command()
    async def question(self, ctx, *args):
        """Ask something from wolfram alpha"""
        query = " ".join(args)
        url = f"http://api.wolframalpha.com/v1/result?appid={WOLFRAM_APPID}&i={query}&output=json"
        response = requests.get(url.replace("+", "%2B"))
        if response.status_code == 200:
            result = response.content.decode('utf-8')
            self.logger.info(misolog.format_log(ctx, f""))
        else:
            result = "Sorry I did not understand your question."
            self.logger.warning(misolog.format_log(ctx, f"Invalid question"))

        await ctx.send(f"**{result}**")

    @commands.command()
    async def twitter(self, ctx, tweet_url, delete=None):
        """Get all the images from a tweet"""
        self.logger.info(misolog.format_log(ctx, f""))
        if "status" in tweet_url:
            tweet_url = re.search(r'status/(\d+)', tweet_url).group(1)
        tweet = twt.get_status(tweet_url, tweet_mode='extended')

        media_files = []
        try:
            media = tweet.extended_entities.get('media', [])
        except AttributeError:
            await ctx.send("This tweet appears to contain no media!")
            return
        hashtags = []
        for hashtag in tweet.entities.get('hashtags', []):
            hashtags.append(f"#{hashtag['text']}")
        for i in range(len(media)):
            media_url = media[i]['media_url']
            video_url = None
            if not media[i]['type'] == "photo":
                video_urls = media[i]['video_info']['variants']
                largest_rate = 0
                for x in range(len(video_urls)):
                    if video_urls[x]['content_type'] == "video/mp4":
                        if video_urls[x]['bitrate'] > largest_rate:
                            largest_rate = video_urls[x]['bitrate']
                            video_url = video_urls[x]['url']
                            media_url = video_urls[x]['url']
            media_files.append((" ".join(hashtags), media_url, video_url))

        for file in media_files:
            content = discord.Embed(colour=int(tweet.user.profile_link_color, 16))
            content.set_image(url=file[1] + ":orig")
            content.set_author(icon_url=tweet.user.profile_image_url, name=f"@{tweet.user.screen_name}\n{file[0]}",
                               url=f"https://twitter.com/{tweet.user.screen_name}/status/{tweet.id}")

            await ctx.send(embed=content)

            if file[2] is not None:
                # contains a video/gif, send it separately
                await ctx.send(file[2])

        if delete == "delete":
            await ctx.message.delete()

    @commands.command(name="steam")
    async def steam(self, ctx, steam_id, *args):
        """Get steam profile data"""
        self.logger.info(misolog.format_log(ctx, f""))

        try:
            steam_id = int(steam_id)
        except ValueError:
            response = requests.get(url=f"https://api.steampowered.com/ISteamUser/ResolveVanityURL/v1?"
                                        f"key={STEAM_API_KEY}&vanityurl={steam_id}")
            if response.status_code == 200:
                resolved = json.loads(response.content.decode('utf-8'))['response']
                if resolved['success'] == 1:
                    steam_id = resolved['steamid']
                else:
                    await ctx.send(f"Error: {resolved['message']} (ResolveVanityUrl)")
                    return
            else:
                await ctx.send(f"Error {response.status_code} (ResolveVanityUrl)")
                return

        response = requests.get(url=f"http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v0002/?"
                                    f"key={STEAM_API_KEY}&steamids={steam_id}&format=json")
        if response.status_code == 200:
            profile_json = json.loads(response.content.decode('utf-8'))['response']['players'][0]
        else:
            await ctx.send(f"Error {response.status_code} (GetPlayerSummaries)")
            return
        response = requests.get(url=f"http://api.steampowered.com/IPlayerService/GetRecentlyPlayedGames/v0001/?"
                                    f"key={STEAM_API_KEY}&steamid={steam_id}&format=json")
        if response.status_code == 200:
            recent_games_json = json.loads(response.content.decode('utf-8'))['response']
        else:
            await ctx.send(f"Error {response.status_code} (GetRecentlyPlayedGames)")
            return
        response = requests.get(url=f"http://api.steampowered.com/IPlayerService/GetOwnedGames/v0001/?"
                                    f"key={STEAM_API_KEY}&steamid={steam_id}&format=json")
        if response.status_code == 200:
            owned_games_json = json.loads(response.content.decode('utf-8'))['response']
        else:
            await ctx.send(f"Error {response.status_code} (GetOwnedGames)")
            return

        message = discord.Embed(color=discord.Color.blue())
        try:
            if args[0] in ['top', "mostplayed", "topgames"]:
                print("soontm")
        except IndexError:
            # show profile

            created_at = datetime.utcfromtimestamp(int(profile_json['timecreated'])).strftime('%Y-%m-%d %H:%M:%S')
            profile_states = ["Offline", "Online", "Busy", "Away", "Snooze", "Looking to trade", "Looking to play"]
            recent_games_string = "N/A"
            total_playtime_2weeks = 0

            message.title = f"{profile_json['personaname']} ({profile_json['steamid']})"
            message.set_thumbnail(url=profile_json['avatarfull'])
            try:
                for i in range(recent_games_json['total_count']):
                    if i == 0:
                        recent_games_string = ""
                    this_game = recent_games_json['games'][i]
                    total_playtime_2weeks += this_game['playtime_2weeks']
                    recent_games_string += f"{this_game['name']} - " \
                                           f"**{this_game['playtime_2weeks']/60:.1f}** Hours " \
                                           f"(total **{this_game['playtime_forever']/60:.1f}** hours)\n"

                message.add_field(name=f"Recently played - **{total_playtime_2weeks/60:.1f}** hours past two weeks",
                                  value=recent_games_string)
            except Exception as e:
                message.add_field(name=f"Recently played - **{total_playtime_2weeks/60:.1f}** hours past two weeks",
                                  value=str(e))

            try:
                total_playtime = 0
                # games = []
                for i in range(owned_games_json['game_count']):
                    playtime = owned_games_json['games'][i]['playtime_forever']
                    total_playtime += playtime

                try:
                    state = f"In-Game: **{profile_json['gameextrainfo']}**"
                except KeyError:
                    state = f"State: {profile_states[profile_json['personastate']]}"

                try:
                    countrycode = profile_json['loccountrycode']
                except KeyError:
                    countrycode = "N/A"

                message.description = f"Country: **{countrycode}**\n" \
                                      f"{state}\n" \
                                      f"Owned games: **{owned_games_json['game_count']}**\n" \
                                      f"Total playtime: **{total_playtime/60:.1f} hours**\n" \
                                      f"Created at: {created_at}"
            except Exception as e:
                message.description = f"Error: {str(e)}"

        await ctx.send(embed=message)

    def get_timezone(self, coord):
        url = f"http://api.timezonedb.com/v2.1/get-time-zone?key={TIMEZONE_API_KEY}&format=json&by=position&" \
              f"lat={coord['lat']}&lng={coord['lon']}"
        response = requests.get(url)
        if response.status_code == 200:
            json_data = json.loads(response.content.decode('utf-8'))
            time = json_data['formatted'].split(" ")
            return ":".join(time[1].split(":")[:2])
        else:
            return f"<error{response.status_code}>"

    def detect_language(self, string):
        url = f"https://translation.googleapis.com/language/translate/v2/detect?key={GOOGLE_API_KEY}" \
              f"&q={string}"
        response = requests.get(url)
        if response.status_code == 200:
            data = json.loads(response.content.decode('utf-8'))
            return data['data']['detections'][0][0]['language']
        else:
            self.logger.error(f"language detection error {str(response.status_code)}")
            return None

    def get_ucum_code(self, search_query):
        url = "https://clinicaltables.nlm.nih.gov/api/ucum/v3/search?terms="
        response = requests.get(url + search_query)
        if response.status_code == 200:
            json_data = json.loads(response.content.decode('utf-8'))
            ucum_code = json_data[3][0][0]
            name = json_data[3][0][1]
            return urllib.parse.quote_plus(ucum_code), name
        else:
            self.logger.error(f"query={search_query}")

def n_max_elements(list1, n):
    final_list = []
    for i in range(0, n):
        max1 = 0
        maxtuple = (0, 0)
        for j in range(len(list1)):
            if list1[j][1] > max1:
                max1 = list1[j][1]
                maxtuple = list1[j]

        list1.remove(maxtuple)
        final_list.append(maxtuple)

    return final_list

def setup(client):
    client.add_cog(Apis(client))


import math
import random
from Crypto.Util import number
base = 37
KEY_SIZE = 512

#Divides the input message into packets (5 characters each)
def divide_text(message):
    out = []
    for i in range(0, len(message), 5):
        out.append(message[i:i+5])
    while(len(out[len(out)-1]) < 5):
        out[len(out)-1] += ' '
    return out

#Encode each packet
def encrypt_message(packet):
    multiplicand = int(base**4)
    sum = 0
    for c in packet:
        if(c <= '9' and c >= '0'):
            c = ord(c) - ord('0')
        elif(c >= 'a' and c <= 'z'):
            c = ord(c) - ord('a') + 10
        else:
            c = 36
        sum += multiplicand*int(c)
        multiplicand /= base

    return int(sum)

#Decodes each packet
def decrypt_message(code):
    str1 = ''
    out = []
    for i in range(0,5,1):
        d = int(code % base)
        if(d <= 9 and d >= 0):
            d = chr(d + ord('0'))
        elif(d >= 10 and d <= 35):
            d = chr(d - 10 + ord('a'))
        else:
            d = ' '
        code -= code % base
        code /= base
        out.append(d)
    out.reverse()
    return str1.join(out)

#The higher level function that ciphers the input message
def encode(message):
    encoded_message = []
    packets = divide_text(message)
    for p in packets:
        encoded_message.append(encrypt_message(p))
    return encoded_message

#The higher level function that deciphers the ciphertext
def decode(encoded_message):
    message = ''
    for ct in encoded_message:
        message = message + decrypt_message(ct)
    return message

def generate_key(k):
    p = number.getPrime(k)
    q = number.getPrime(k)
    n = p * q
    phi = (p-1)*(q-1)
    e = number.getPrime(k)
    while math.gcd(e,phi) != 1:
        e = number.getPrime(k)
    d = pow(e, -1, phi)

    return p, q, n, phi, e, d

def cipher(plaintext, e, n):
    m_arr = encode(plaintext)
    c = []
    for x in m_arr:
        c.append(pow(x, e, n))
    return c

def decipher(ciphertext, d, n):
    codes = []
    for x in ciphertext:
        codes.append(pow(x, d, n))
    out = decode(codes)

    return out

# plaintext = "mohabfathisalemku"
# p, q, n, phi, e, d = generate_key()
# c = cipher(plaintext)
# back = decipher(c)

# print(back)


from PIL import Image, ImageFilter
from os import mkdir
filters = [
	i for i in dir(ImageFilter) if not i.startswith('__')
]

def blur(file, out: str) -> bool:
	try:
		file.filter(ImageFilter.BoxBlur(10)).save(out) # Names of the funcs and filters that you can use are in filters list
		# to visualize them u can print the list.
		print("filtered and saved")
	except Exception as e:
		print(e)

def main():
	img = Image.open("girl.png")

	blur(img, "./filtered/Color3DLUTgirl2.png")
if __name__ == '__main__':
	print(filters)
	# main()


# -*- coding: utf-8 -*-
"""
Created on Wed Jul  1 17:55:49 2020

@author: noahk
"""
import numpy as np
from BinLightCurve import BinLightCurveFiveDayInterval as bc
from matplotlib import pyplot as plt
import h5py

data=np.loadtxt(r"C:\Users\noahk\pg1302-research\observation-data\pg1302_data_LINEAR+CRTS+ASASSN.txt")
MJD=data[:,0]
mag =data[:,1]
magErr=data[:,2]
flag = data[:,3]

SortedInd = np.argsort(MJD)
MJD = MJD[SortedInd]
mag = mag[SortedInd]
magErr = magErr[SortedInd]
flag = flag[SortedInd]

ind=np.where(flag==0)
y=mag[ind]
MJDy=MJD[ind]
magErry=magErr[ind]

ind2=np.where(flag==1)
a=mag[ind2] - .03489723624
MJDa=MJD[ind2]
magErra=magErr[ind2]

LC_MJD, LC_mag, LC_magErr = bc(MJDy,y,magErry,180)
A_MJD, A_mag, A_magErr = bc(MJDa,a,magErra,100)

bmjd = np.concatenate((LC_MJD, A_MJD))

ind = np.argsort(bmjd)

bmjd = bmjd[ind]

bmag = np.concatenate((LC_mag, A_mag))
bmag = bmag[ind]

bmagErr = np.concatenate((magErry, magErra))
bmagErr = bmag[ind]

f = h5py.File('pg1302_L+C+A_Binned', "a")
f[str(0)+'/MJD']=bmjd
f[str(0)+'/mag']= bmag
f[str(0)+'/magErr']=bmagErr
# f[str(0)+'/omegaSlope']=omegaSlope
f[str(0)+'/sigma']=.11
f[str(0)+'/tau']=100

plt.errorbar(MJD, mag, yerr=magErr, fmt='o', color='blue',zorder=1)
plt.scatter(LC_MJD, LC_mag, color='red', s=150,zorder=2, edgecolors=['black'])
plt.scatter(A_MJD, A_mag, color='lawngreen', s=150,zorder=2, edgecolors=['black'])
plt.title('PG1302 Binned (L+C 180 and A 100)')
plt.show()

plt.errorbar(bmjd,bmag,yerr=bmagErr,fmt='o')
plt.show()


import argparse
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
from torchvision import datasets, transforms
from torch.autograd import Variable

import numpy as np
import scipy.io
import os
import sys
import pickle
from sklearn.metrics import roc_auc_score

from lib.Tox21_Data import Dataset, read
from lib.utils import readData
from lib.treeConvNet import TreeConvNet
from lib.maskedDAE import MaskedDenoisingAutoencoder

# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size', type=int, default=256, metavar='N',
                    help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N',
                    help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs', type=int, default=10, metavar='N',
                    help='number of epochs to train (default: 10)')
parser.add_argument('--dataset', type=str, default="agnews",
                    help='dataset')
parser.add_argument('--target', type=int, default=0, metavar='N',
                    help='target task 0-11')
parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
                    help='learning rate (default: 0.01)')
parser.add_argument('--name', type=str, default="tree-mnist",
                    help='name for this run')
args = parser.parse_args()

use_cuda = torch.cuda.is_available()

if args.dataset=="agnews":
    label_name = ['World', 'Sports', 'Business', 'Sci/Tech']
    training_num, valid_num, test_num, vocab_size = 110000, 10000, 7600, 10000
    training_file = 'dataset/agnews_training_110K_10K-TFIDF-words.txt'
    valid_file = 'dataset/agnews_valid_10K_10K-TFIDF-words.txt'
    test_file = 'dataset/agnews_test_7600_10K-TFIDF-words.txt'

elif args.dataset=="dbpedia":
    label_name = ['Company','EducationalInstitution','Artist','Athlete','OfficeHolder','MeanOfTransportation','Building','NaturalPlace','Village','Animal','Plant','Album','Film','WrittenWork']
    training_num, valid_num, test_num, vocab_size = 549990, 10010, 70000, 10000
    training_file = 'dataset/dbpedia_training_549990_10K-frequent-words.txt'
    valid_file = 'dataset/dbpedia_valid_10010_10K-frequent-words.txt'
    test_file = 'dataset/dbpedia_test_70K_10K-frequent-words.txt'

elif args.dataset=="sogounews":
    label_name = ['sports','finance','entertainment','automobile','technology']
    training_num, valid_num, test_num, vocab_size = 440000, 10000, 60000, 10000
    training_file = 'dataset/sogounews_training_440K_10K-frequent-words.txt'
    valid_file = 'dataset/sogounews_valid_10K_10K-frequent-words.txt'
    test_file = 'dataset/sogounews_test_60K_10K-frequent-words.txt'

elif args.dataset=="yelp":
    label_name = ['1','2','3','4','5']
    training_num, valid_num, test_num, vocab_size = 640000, 10000, 50000, 10000
    training_file = 'dataset/yelpfull_training_640K_10K-frequent-words.txt'
    valid_file = 'dataset/yelpfull_valid_10K_10K-frequent-words.txt'
    test_file = 'dataset/yelpfull_test_50K_10K-frequent-words.txt'

elif args.dataset=="yahoo":
    label_name = ["Society & Culture","Science & Mathematics","Health","Education & Reference","Computers & Internet","Sports","Business & Finance","Entertainment & Music","Family & Relationships","Politics & Government"]
    training_num, valid_num, test_num, vocab_size = 1390000, 10000, 60000, 10000
    training_file = 'dataset/yahoo_training_1.39M_10K-frequent-words.txt'
    valid_file = 'dataset/yahoo_valid_10K_10K-frequent-words.txt'
    test_file = 'dataset/yahoo_test_60K_10K-frequent-words.txt'

print("Running Dataset %s with [lr=%f, epochs=%d, batch_size=%d]"
        % (args.dataset, args.lr, args.epochs, args.batch_size))

num_classes = len(label_name)

randgen = np.random.RandomState(13)
trainX, trainY = readData(training_file, training_num, vocab_size, randgen)
validX, validY = readData(valid_file, valid_num, vocab_size)
testX, testY = readData(test_file, test_num, vocab_size)

input_dim = trainX.size()[1]
# 10000 -> 1000 -> 500 -> 250
kernel_stride = [(9, 10), (5, 2), (5, 2)]

net = TreeConvNet(args.name)

net.learn_structure(trainX, validX, num_classes, kernel_stride, corrupt=0.5,
        lr=1e-3, batch_size=args.batch_size, epochs=10)
# net.net = torch.load('./checkpoint/ckpt-'+args.name+'-structure.t7')
# use skeleton only, initialize weight randomly
# for l in net.net:
#     if isinstance(l, MaskedDenoisingAutoencoder):
#         l.reset_parameters()
#     if isinstance(l, nn.Dropout):
#         l.p = 0.5

net.fit(trainX, trainY, validX, validY, testX, testY, batch_size=args.batch_size,
    lr=args.lr, epochs=args.epochs)
print("Done.")


#!/usr/bin/env python
# coding: utf-8

#
#
#
# *This notebook contains an excerpt from the [Python Data Science Handbook](http://shop.oreilly.com/product/0636920034919.do) by Jake VanderPlas; the content is available [on GitHub](https://github.com/jakevdp/PythonDataScienceHandbook).*
#
# *The text is released under the [CC-BY-NC-ND license](https://creativecommons.org/licenses/by-nc-nd/3.0/us/legalcode), and code is released under the [MIT license](https://opensource.org/licenses/MIT). If you find this content useful, please consider supporting the work by [buying the book](http://shop.oreilly.com/product/0636920034919.do)!*

#
# < [Customizing Ticks](04.10-Customizing-Ticks.ipynb) | [Contents](Index.ipynb) | [Three-Dimensional Plotting in Matplotlib](04.12-Three-Dimensional-Plotting.ipynb) >
#
#
#

# # Customizing Matplotlib: Configurations and Stylesheets

# Matplotlib's default plot settings are often the subject of complaint among its users.
# While much is slated to change in the 2.0 Matplotlib release in late 2016, the ability to customize default settings helps bring the package inline with your own aesthetic preferences.
#
# Here we'll walk through some of Matplotlib's runtime configuration (rc) options, and take a look at the newer *stylesheets* feature, which contains some nice sets of default configurations.

# ## Plot Customization by Hand
#
# Through this chapter, we've seen how it is possible to tweak individual plot settings to end up with something that looks a little bit nicer than the default.
# It's possible to do these customizations for each individual plot.
# For example, here is a fairly drab default histogram:

# In[1]:

import matplotlib.pyplot as plt
plt.style.use('classic')
import numpy as np

get_ipython().run_line_magic('matplotlib', 'inline')

# In[2]:

x = np.random.randn(1000)
plt.hist(x);

# We can adjust this by hand to make it a much more visually pleasing plot:

# In[3]:

# use a gray background
ax = plt.axes(axisbg='#E6E6E6')
ax.set_axisbelow(True)

# draw solid white grid lines
plt.grid(color='w', linestyle='solid')

# hide axis spines
for spine in ax.spines.values():
    spine.set_visible(False)

# hide top and right ticks
ax.xaxis.tick_bottom()
ax.yaxis.tick_left()

# lighten ticks and labels
ax.tick_params(colors='gray', direction='out')
for tick in ax.get_xticklabels():
    tick.set_color('gray')
for tick in ax.get_yticklabels():
    tick.set_color('gray')

# control face and edge color of histogram
ax.hist(x, edgecolor='#E6E6E6', color='#EE6666');

# This looks better, and you may recognize the look as inspired by the look of the R language's ggplot visualization package.
# But this took a whole lot of effort!
# We definitely do not want to have to do all that tweaking each time we create a plot.
# Fortunately, there is a way to adjust these defaults once in a way that will work for all plots.

# ## Changing the Defaults: ``rcParams``
#
# Each time Matplotlib loads, it defines a runtime configuration (rc) containing the default styles for every plot element you create.
# This configuration can be adjusted at any time using the ``plt.rc`` convenience routine.
# Let's see what it looks like to modify the rc parameters so that our default plot will look similar to what we did before.
#
# We'll start by saving a copy of the current ``rcParams`` dictionary, so we can easily reset these changes in the current session:

# In[4]:

IPython_default = plt.rcParams.copy()

# Now we can use the ``plt.rc`` function to change some of these settings:

# In[5]:

from matplotlib import cycler
colors = cycler('color',
                ['#EE6666', '#3388BB', '#9988DD',
                 '#EECC55', '#88BB44', '#FFBBBB'])
plt.rc('axes', facecolor='#E6E6E6', edgecolor='none',
       axisbelow=True, grid=True, prop_cycle=colors)
plt.rc('grid', color='w', linestyle='solid')
plt.rc('xtick', direction='out', color='gray')
plt.rc('ytick', direction='out', color='gray')
plt.rc('patch', edgecolor='#E6E6E6')
plt.rc('lines', linewidth=2)

# With these settings defined, we can now create a plot and see our settings in action:

# In[6]:

plt.hist(x);

# Let's see what simple line plots look like with these rc parameters:

# In[7]:

for i in range(4):
    plt.plot(np.random.rand(10))

# I find this much more aesthetically pleasing than the default styling.
# If you disagree with my aesthetic sense, the good news is that you can adjust the rc parameters to suit your own tastes!
# These settings can be saved in a *.matplotlibrc* file, which you can read about in the [Matplotlib documentation](http://Matplotlib.org/users/customizing.html).
# That said, I prefer to customize Matplotlib using its stylesheets instead.

# ## Stylesheets
#
# The version 1.4 release of Matplotlib in August 2014 added a very convenient ``style`` module, which includes a number of new default stylesheets, as well as the ability to create and package your own styles. These stylesheets are formatted similarly to the *.matplotlibrc* files mentioned earlier, but must be named with a *.mplstyle* extension.
#
# Even if you don't create your own style, the stylesheets included by default are extremely useful.
# The available styles are listed in ``plt.style.available``—here I'll list only the first five for brevity:

# In[8]:

plt.style.available[:5]

# The basic way to switch to a stylesheet is to call
#
# ``` python
# plt.style.use('stylename')
# ```
#
# But keep in mind that this will change the style for the rest of the session!
# Alternatively, you can use the style context manager, which sets a style temporarily:
#
# ``` python
# with plt.style.context('stylename'):
#     make_a_plot()
# ```
#

# Let's create a function that will make two basic types of plot:

# In[9]:

def hist_and_lines():
    np.random.seed(0)
    fig, ax = plt.subplots(1, 2, figsize=(11, 4))
    ax[0].hist(np.random.randn(1000))
    for i in range(3):
        ax[1].plot(np.random.rand(10))
    ax[1].legend(['a', 'b', 'c'], loc='lower left')

# We'll use this to explore how these plots look using the various built-in styles.

# ### Default style
#
# The default style is what we've been seeing so far throughout the book; we'll start with that.
# First, let's reset our runtime configuration to the notebook default:

# In[10]:

# reset rcParams
plt.rcParams.update(IPython_default);

# Now let's see how it looks:

# In[11]:

hist_and_lines()

# ### FiveThiryEight style
#
# The ``fivethirtyeight`` style mimics the graphics found on the popular [FiveThirtyEight website](https://fivethirtyeight.com).
# As you can see here, it is typified by bold colors, thick lines, and transparent axes:

# In[12]:

with plt.style.context('fivethirtyeight'):
    hist_and_lines()

# ### ggplot
#
# The ``ggplot`` package in the R language is a very popular visualization tool.
# Matplotlib's ``ggplot`` style mimics the default styles from that package:

# In[13]:

with plt.style.context('ggplot'):
    hist_and_lines()

# ### *Bayesian Methods for Hackers( style
#
# There is a very nice short online book called [*Probabilistic Programming and Bayesian Methods for Hackers*](http://camdavidsonpilon.github.io/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/); it features figures created with Matplotlib, and uses a nice set of rc parameters to create a consistent and visually-appealing style throughout the book.
# This style is reproduced in the ``bmh`` stylesheet:

# In[14]:

with plt.style.context('bmh'):
    hist_and_lines()

# ### Dark background
#
# For figures used within presentations, it is often useful to have a dark rather than light background.
# The ``dark_background`` style provides this:

# In[15]:

with plt.style.context('dark_background'):
    hist_and_lines()

# ### Grayscale
#
# Sometimes you might find yourself preparing figures for a print publication that does not accept color figures.
# For this, the ``grayscale`` style, shown here, can be very useful:

# In[16]:

with plt.style.context('grayscale'):
    hist_and_lines()

# ### Seaborn style
#
# Matplotlib also has stylesheets inspired by the Seaborn library (discussed more fully in [Visualization With Seaborn](04.14-Visualization-With-Seaborn.ipynb)).
# As we will see, these styles are loaded automatically when Seaborn is imported into a notebook.
# I've found these settings to be very nice, and tend to use them as defaults in my own data exploration.

# In[17]:

import seaborn
hist_and_lines()

# With all of these built-in options for various plot styles, Matplotlib becomes much more useful for both interactive visualization and creation of figures for publication.
# Throughout this book, I will generally use one or more of these style conventions when creating plots.

#
# < [Customizing Ticks](04.10-Customizing-Ticks.ipynb) | [Contents](Index.ipynb) | [Three-Dimensional Plotting in Matplotlib](04.12-Three-Dimensional-Plotting.ipynb) >
#
#
#


from functools import reduce
import operator

seq = [20,1,-1,2,-2,3,3,5,5,1,2,4,20,4,-1,-2,5]

def find_it(seq):
    counter_arr = {}
    x = None
    for num in seq:
        if num in counter_arr:
            counter_arr[num] += 1
        else:
            counter_arr.update({num: 1})
        # print(num, '->', counter_arr[num])
    for key in counter_arr:
        if counter_arr[key] % 2:
            x = key
    return x

# Запомнить!!!
def find_it2(seq):
    nums = set()
    for num in seq:
        if num in nums:
            nums.remove(num)
            print('remove ', num)
        else:
            nums.add(num)
            print('add ', nums)
    return nums.pop()
# Список Python pop() — это встроенная функция Python, которая удаляет и возвращает последнее значение из списка или заданное значение индекса.

# Элегантное, но не эффективное решение, так как каждый раз за проход цикла count проходит заново по всему массиву
def find_it3(seq):
    for i in seq:
        if seq.count(i) % 2: # .count() возвращает количество вхождений
            return i

# Запомнить!!!
# xor(a, a) = 0 and xor(a, 0) = a.
# So, when you xor all the numbers, every number with even frequency gets cancelled out and only the number with odd frequence remains.
# Итак, когда вы выполняете операцию xor для всех чисел, каждое число с четной частотой отменяется, и остается только число с нечетной частотой.
def find_it4(xs):
    return reduce(operator.xor, xs)

def find_it4_1(xs):
    def xors(x, y):
        print ('x = ', x, 'y = ', y)
        print ("XOR ", operator.xor(x, y))
        return operator.xor(x, y)

    return reduce(xors, xs)

def find_it4_2(seq):
    result = 0
    for x in seq:
        result ^= x
    return result

print(find_it(seq))

Find a number that repeats an odd number of times/odd_array.py


# -*- coding: utf-8 -*-

"""
Using the requests and BeautifulSoup Python libraries, print to the screen the
full text of the article on this website:

http://www.vanityfair.com/society/2014/06/monica-lewinsky-humiliation-culture.

The article is long, so it is split up between 4 pages. Your task is to print
out the text to the screen so that you can read the full article without having
to click any buttons.

(Hint: The post here describes in detail how to use the BeautifulSoup and
requests libraries through the solution of the exercise posted here.)

This will just print the full text of the article to the screen. It will not
make it easy to read, so next exercise we will learn how to write this text to
a .txt file.
"""

from bs4 import BeautifulSoup
import requests

url = 'https://www.vanityfair.com/style/society/2014/06/monica-lewinsky-humiliation-culture'
req = requests.get(url)
data = req.text

soup = BeautifulSoup(data, 'lxml')
tfile_name = raw_input('Enter a file name: ')
file = open('%s.txt' % tfile_name,'w')

article_text = ""
for ele in soup.find_all('p'):
    article_text += ele.text.encode('utf-8')

file.write(article_text)
file.close()


"""Contains CrawlResult class"""
from collections import defaultdict
from pathlib import Path

from typing_extensions import Self

from .sync_status import BackupEntry

class CrawlResult:
    """Result from crawl of a single directory"""

    def __init__(self, path: Path) -> None:
        self.loose_paths: list[Path] = []
        self.denied_paths: list[Path] = []
        self.backups: defaultdict[type[BackupEntry], list[BackupEntry]] = defaultdict(list)
        self.path: Path = path

    def add_backup(self, backup: BackupEntry) -> None:
        """Add backup entry to result"""
        self.backups[type(backup)].append(backup)

    def extend(self, other: Self) -> None:
        """Extend current object with another crawl result"""

        for backup_type in other.backups:
            self.backups[backup_type].extend(other.backups[backup_type])

        self.denied_paths.extend(other.denied_paths)

        # If `other` has no backed up paths, we can just mark the entire tree as not backed up
        if other.backups:
            self.loose_paths.extend(other.loose_paths)
        elif other.loose_paths:
            self.loose_paths.append(other.path)


import sqlite3
import pandas as pd
import zipfile
import os
import quandl

def update_tickers():
    """
    Function that updates the tickers database table.
    """
    quandl.read_key()
    conn = sqlite3.connect("data.db")
    # Fetch most recent updated date from tickers table
    with conn:
        execute = conn.execute("SELECT MAX(lastupdated) FROM tickers;")
        date = execute.fetchone()[0]

    # Get all tickers from different tables since date
    tickers = quandl.get_table(
        "SHARADAR/TICKERS", paginate=True, lastupdated={"gt": date}
    )

    # Get tickers for only tables we are concerned with
    tickers = tickers[tickers["table"] == ("SF1" or "SEP" or "SFP")]

    # Make sure we select from the data in the same order as the database columns
    column_order = [
        "permaticker",
        "ticker",
        "name",
        "exchange",
        "isdelisted",
        "category",
        "cusips",
        "siccode",
        "sicsector",
        "sicindustry",
        "famasector",
        "famaindustry",
        "sector",
        "industry",
        "scalemarketcap",
        "scalerevenue",
        "relatedtickers",
        "currency",
        "location",
        "lastupdated",
        "firstadded",
        "firstpricedate",
        "lastpricedate",
        "firstquarter",
        "lastquarter",
        "secfilings",
        "companysite",
    ]

    tickers = tickers[column_order]
    # Convert dataframe to records for database execution
    tickers_records = tickers.astype("str").to_records(index=False)
    print(f"New ticker records: {len(tickers_records)}")

    with conn:
        conn.executemany(
            """INSERT OR REPLACE INTO tickers VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            tickers_records,
        )

def update_prices():
    """
    Function that updates the prices database table.
    """
    quandl.read_key()
    conn = sqlite3.connect("data.db")
    # Fetch most recent updated date from prices table
    with conn:
        execute = conn.execute("SELECT MAX(lastupdated) FROM prices;")
        date = execute.fetchone()[0]

    # Depending upon how long its been since the last update, the return can be quite large, so I need to mkae
    # a bulk request as a ZIP download
    filename = "recent_updates.zip"
    # Equity prices first
    quandl.export_table("SHARADAR/SEP", filename=filename, lastupdated={"gt": date})
    # Extract the csv data from the downloaded ZIP file
    data_zip = zipfile.ZipFile(filename)
    csv_name = data_zip.namelist()[0]
    csv_data = data_zip.open(csv_name)

    column_order = [
        "ticker",
        "date",
        "open",
        "high",
        "low",
        "close",
        "volume",
        "closeadj",
        "closeunadj",
        "lastupdated",
    ]
    sep_records = pd.read_csv(csv_data)[column_order].to_records(index=False)
    print(f"New equity price records: {len(sep_records)}")

    with conn:
        conn.executemany(
            """INSERT OR REPLACE INTO prices VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            sep_records,
        )

    # Now fund prices
    quandl.export_table("SHARADAR/SFP", filename=filename, lastupdated={"gt": date})
    # Extract the csv data from the downloaded ZIP file
    data_zip = zipfile.ZipFile(filename)
    csv_name = data_zip.namelist()[0]
    csv_data = data_zip.open(csv_name)

    sfp_records = pd.read_csv(csv_data)[column_order].to_records(index=False)
    print(f"New fund price records: {len(sfp_records)}")

    with conn:
        conn.executemany(
            """INSERT OR REPLACE INTO prices VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            sfp_records,
        )

    # Remove downloaded zip file
    os.remove(filename)

def update_fundamentals():
    """
    Function that updates fundamentals table.
    """
    quandl.read_key()
    conn = sqlite3.connect("data.db")
    # Fetch most recent updated date from prices table
    with conn:
        execute = conn.execute("SELECT MAX(lastupdated) FROM fundamentals;")
        date = execute.fetchone()[0]

    filename = "recent_updates.zip"
    quandl.export_table("SHARADAR/SF1", filename=filename, lastupdated={"gt": date})

    data_zip = zipfile.ZipFile(filename)
    csv_name = data_zip.namelist()[0]
    csv_data = data_zip.open(csv_name)

    column_order = [
        "ticker",
        "dimension",
        "calendardate",
        "datekey",
        "reportperiod",
        "lastupdated",
        "accoci",
        "assets",
        "assetsavg",
        "assetsc",
        "assetsnc",
        "assetturnover",
        "bvps",
        "capex",
        "cashneq",
        "cashnequsd",
        "cor",
        "consolinc",
        "currentratio",
        "de",
        "debt",
        "debtc",
        "debtnc",
        "debtusd",
        "deferredrev",
        "depamor",
        "deposits",
        "divyield",
        "dps",
        "ebit",
        "ebitda",
        "ebitdamargin",
        "ebitdausd",
        "ebitusd",
        "ebt",
        "eps",
        "epsdil",
        "epsusd",
        "equity",
        "equityavg",
        "equityusd",
        "ev",
        "evebit",
        "evebitda",
        "fcf",
        "fcfps",
        "fxusd",
        "gp",
        "grossmargin",
        "intangibles",
        "intexp",
        "invcap",
        "invcapavg",
        "inventory",
        "investments",
        "investmentsc",
        "investmentsnc",
        "liabilities",
        "liabilitiesc",
        "liabilitiesnc",
        "marketcap",
        "ncf",
        "ncfbus",
        "ncfcommon",
        "ncfdebt",
        "ncfdiv",
        "ncff",
        "ncfi",
        "ncfinv",
        "ncfo",
        "ncfx",
        "netinc",
        "netinccmn",
        "netinccmnusd",
        "netincdis",
        "netincnci",
        "netmargin",
        "opex",
        "opinc",
        "payables",
        "payoutratio",
        "pb",
        "pe",
        "pe1",
        "ppnenet",
        "prefdivis",
        "price",
        "ps",
        "ps1",
        "receivables",
        "retearn",
        "revenue",
        "revenueusd",
        "rnd",
        "roa",
        "roe",
        "roic",
        "ros",
        "sbcomp",
        "sgna",
        "sharefactor",
        "sharesbas",
        "shareswa",
        "shareswadil",
        "sps",
        "tangibles",
        "taxassets",
        "taxexp",
        "taxliabilities",
        "tbvps",
        "workingcapital",
    ]
    sf1_records = pd.read_csv(csv_data, dtype="str")[column_order].to_records(
        index=False
    )
    print(f"New fundamental records: {len(sf1_records)}")

    with conn:
        conn.executemany(
            """
            INSERT OR REPLACE INTO fundamentals VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
                    ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
                    ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
                    ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
                    ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """,
            sf1_records,
        )

    os.remove(filename)

if __name__ == "__main__":
    update_tickers()
    print("Tickers updated.")
    update_prices()
    print("Prices updated.")
    update_fundamentals()
    print("Fundamentals updated.")


import matplotlib.pyplot as plt
plt.style.use('miles')
import numpy as np
import os

numlines = 7 # number of curves to show in parametric plot
t = np.arange(0,2,0.1)
interval = 2*np.pi/numlines

# Parametric plot
fig, ax = plt.subplots()

for i in range(numlines):
	x = np.cos(i*interval) - np.cos(t+i*interval)
	y = -1*np.sin(i*interval) + np.sin(t+i*interval)
	ax.plot(x, y)
ax.set(xlabel = '$x_n$', ylabel = '$y_n$')
ax.grid()

filename = "coriolis parametric"
fig.savefig(filename + ".pdf")
os.system("open '" + filename + "'.pdf")

# Separate time plots
plt.cla()

t = np.arange(0,5,0.1)
B = np.pi/2
x = np.cos(B) - np.cos(t+B)
y = -1*np.sin(B) + np.sin(t+B)

line_x = ax.plot(t, x, label='$x_n(t_n)$')
line_y = ax.plot(t, y, label='$y_n(t_n)$')
ax.set(xlabel = '$t_n$')
ax.grid()
ax.legend()

filename = "coriolis time"
fig.savefig(filename + ".pdf")
os.system("open '" + filename + "'.pdf")


from collections import deque

# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def minDepth(self, root: TreeNode) -> int:
        depth = 0
        if root is None: return depth
        queue  = deque()
        queue.append(root)

        while queue:
            length = len(queue)
            depth += 1

            for _ in range(length):
                node = queue.popleft()

                if node.left is not None: queue.append(node.left)
                if node.right is not None: queue.append(node.right)
                if node.left is None and node.right is None: return depth

        return depth

111. Minimum Depth of Binary Tree/minimum_depth_of_binary_tree.py


import argparse
import pickle
import torch
import numpy as np
from tqdm import tqdm
from sklearn.metrics import roc_auc_score, f1_score

parser = argparse.ArgumentParser()
# parser.add_argument('--datasets', default='ntu120/xsub', choices={'kinetics', 'ntu/xsub', 'ntu/xview'},
#                     help='the work folder for storing results')
parser.add_argument('--alpha', default=0.5, help='weighted summation')
arg = parser.parse_args()

# def calculate_auc(targets, predictions):
#     """
#     Calculate performance measures on test dataset.
#     :param targets: Target vector to predict.
#     :param predictions: Predictions vector.
#     :param edges: Edges dictionary with number of edges etc.
#     :return auc: AUC value.
#     :return f1: F1-score.
#     """
#     # targets = [0 if target == 1 else 1 for target in targets]
#     auc = roc_auc_score(targets, predictions)
#     pred = [1 if p > 0.5 else 0 for p in predictions]
#     f1 = f1_score(targets, pred)
#     pos_ratio = sum(pred)/len(pred)
#     return auc, f1, pos_ratio

# dataset = arg.datasets
# label = open('/home/home/Desktop/Python/data/' + dataset + '/val_label.pkl', 'rb')
# label = open('/home/home/Desktop/Python/data/ntu120/xsetup/val_label.pkl', 'rb')
# label = np.array(pickle.load(label))
label = open('/home/home/Desktop/Code/SGCN/Slashdot_target.pkl', 'rb')
label = pickle.load(label)
# label = [0 if t == 1 else 1 for t in label]
# label = np.array(list(enumerate(pickle.load(label))))
# label=label.T
# ind = np.argsort(label[0])
# # print(ind
# label = label[:, ind]

r1 = open('/home/home/Desktop/Code/SGCN/Slashdot_prediction1.pkl', 'rb')
r11 = pickle.load(r1)
# r11 = np.array(list(enumerate(pickle.load(r1))))
auc1 = roc_auc_score(label, r11)
pred1 = [1 if p > 0.5 else 0 for p in r11]
f11 = f1_score(label, pred1)
# # r1 = open('./work_dir/' + dataset + '/sgcn_test_joint/epoch1_test_score.pkl', 'rb')
# # r1 = open('./work_dir/ntu120/cset/sgcn_test_joint/epoch1_test_score.pkl', 'rb')
# r1 = list(enumerate(pickle.load(r1)))
# # print(r1[0][1])
# # exit(0)
# r1 = sorted(r1, key=lambda x: x[0])
#
# # r2 = open('./work_dir/' + dataset + '/sgcn_test_bone/epoch1_test_score.pkl', 'rb')
r2 = open('/home/home/Desktop/Code/SGCN/Slashdot_prediction2.pkl', 'rb')
r22 = pickle.load(r2)
auc2 = roc_auc_score(label, r22)
pred2 = [1 if p > 0.5 else 0 for p in r22]
f12 = f1_score(label, pred2)
r = r11*(1-arg.alpha) + r22 * arg.alpha
auc = roc_auc_score(label, r)
pred = [1 if p > 0.5 else 0 for p in r]
f = f1_score(label, pred)
# r2 = list(enumerate(pickle.load(r2)))
# r2 = sorted(r2, key=lambda x: x[0])
# # print(label)
# # exit(0)
# # print(r1)
# # print(r2)
# right_num = right_num1 = right_num2 = total_num = right_num_5 = 0
# for i in tqdm(range(len(label[0]))):
#     _, l = label[:, i]
#     _, r11 = r1[i]
#     _, r22 = r2[i]
#     r = r11*(1-arg.alpha) + r22 * arg.alpha
#     # print(r)
#     #print(arg.alpha)
#     # rank_5 = r.argsort()[-5:]
#     # right_num_5 += int(int(l) in rank_5)
#     #print(r11)
#     # r11 = np.argmax((torch.tensor(r11).numpy()))
#     # r22 = np.argmax((torch.tensor(r22).numpy()))
#     # print(r11)
#     # print(r22)
#     # auc = roc_auc_score(label, r11)
#     right_num1 += int(l == round(r11))
#     right_num2 += int(l == round(r22))
#     #right_num2 += int(r22 == int(l))
#     # r = np.argmax((torch.tensor(r).numpy()))
#     right_num += int(l == round(r))
#     total_num += 1
# label=np.array(label)
# r11=np.array(r1)
# r22=np.array(r2)
# print(label)
# print(r11)
# print(r22)
# print(right_num1)
# print(right_num2)
# acc1 = right_num1 / total_num
# acc2 = right_num2 / total_num
# acc = right_num / total_num
# print(total_num)
# # auc = roc_auc_score(label, r11)
# # acc5 = right_num_5 / total_num
# print(acc, acc1, acc2)
# print(arg.alpha)
print(auc1, auc2, auc, f11, f12, f)


import unittest
import datetime

from app import db
from models import HonorableMention
from strings import category_not_found, invalid_state
from tests.test_utils import clear_database, create_state, create_category

class HonorableMentionTestCase(unittest.TestCase):
    def setUp(self):
        self.state_code = 'AK'
        create_state(code=self.state_code)
        self.category = create_category()
        self.honorable_mention = HonorableMention(
            category_id=self.category.id,
            state=self.state_code,
            text='AS 12.61.017',
            url='http://www.akleg.gov/basis/statutes.asp#12.61.017',
            description='Victims of crimes are granted protections from employer...'
        ).save()

    def tearDown(self):
        clear_database(db)

    def test_init(self):
        self.assertEqual(self.honorable_mention.category_id, self.category.id)
        self.assertEqual(self.honorable_mention.state, self.state_code)
        self.assertEqual(self.honorable_mention.text, 'AS 12.61.017')
        self.assertEqual(
            self.honorable_mention.url,
            'http://www.akleg.gov/basis/statutes.asp#12.61.017'
        )
        self.assertEqual(
            self.honorable_mention.description,
            'Victims of crimes are granted protections from employer...'
        )
        self.assertTrue(self.honorable_mention.active)

    def test_init_invalid_category(self):
        with self.assertRaises(ValueError) as e:
            HonorableMention(
                category_id=0,
                state=self.state_code,
            )
        self.assertEqual(str(e.exception), category_not_found)

    def test_init_invalid_state_code(self):
        with self.assertRaises(ValueError) as e:
            HonorableMention(
                category_id=self.category.id,
                state='fake-state-code',
            )
        self.assertEqual(str(e.exception), invalid_state)

    def test_serialize(self):
        self.assertEqual(
            {
                'id': self.honorable_mention.id,
                'category_id': self.category.id,
                'state': self.state_code,
                'text': 'AS 12.61.017',
                'url': 'http://www.akleg.gov/basis/statutes.asp#12.61.017',
                'description': 'Victims of crimes are granted protections from employer...',
                'active': True,
                'deactivated_at': None,
            },
            self.honorable_mention.serialize()
        )

    def test_deactivate(self):
        self.honorable_mention.deactivate()

        self.assertFalse(self.honorable_mention.active)
        self.assertTrue(isinstance(self.honorable_mention.deactivated_at, datetime.datetime))
        self.assertTrue(self.honorable_mention.deactivated_at < datetime.datetime.utcnow())


import xml.etree.ElementTree as ET
from _elementtree import Element
from typing import Union
from xml.etree.ElementTree import Element

class Pracownik:
    def __init__(self, imie, nazwisko, stanowisko, pensja):
        self.imie = imie
        self.nazwisko = nazwisko
        self.stanowisko = stanowisko
        self.pensja = pensja

class Firma:
    def __init__(self):
        self.lista = []
    def zatrudnij(self, pracownik):
        self.lista.append(pracownik)
    def zwolnij(self, pracownik):
        self.lista.remove(pracownik)
    def listapracownikow(self):
        res = []
        for x in self.lista:
            res.append({'imie' : x.imie, 'nazwisko' : x.nazwisko, 'stanowisko' : x.stanowisko, 'pensja' : x.pensja})
        return res

adam = Pracownik('Adam', 'Kowalski', 'kierownik', "2000")
karol = Pracownik('Karol', 'Iksinski', 'Wyrobnik ciasteczek', "300")

korporacja_ciastkarska = Firma()

korporacja_ciastkarska.zatrudnij(adam)
korporacja_ciastkarska.zatrudnij(karol)

print(korporacja_ciastkarska.listapracownikow())

korporacja_ciastkarska.zwolnij(adam)
print(korporacja_ciastkarska.listapracownikow())

element = ET.Element('pracownicy')
pracownik1 = ET.SubElement(element, 'adam', {"imie" : adam.imie, "nazwisko": adam.nazwisko, "stanowisko": adam.stanowisko, "pensja": adam.pensja} )
pracownik2 = ET.SubElement(element, 'karol', {"imie" : karol.imie, "nazwisko": karol.nazwisko, "stanowisko": karol.stanowisko, "pensja": karol.pensja})

tree = ET.ElementTree(element)
tree.write("pracownicy.xml")

zpliku = ET.parse("pracownicy.xml")
root = zpliku.getroot()

print("Dane odczytane z XML")
for x in root:
    print(x.attrib['imie'], x.attrib['nazwisko'])