from discord.ext import commands, menus
from .utils import config, checks, maps, fuzzy, time, formats
from .utils.formats import plural
from .utils.paginator import RoboPages, SimplePages, FieldPageSource

from urllib.parse import quote as urlquote
from email.utils import parsedate_to_datetime
from collections import namedtuple, defaultdict

import itertools
import datetime
import random
import asyncio
import discord
import logging
import aiohttp
import pathlib
import yarl
import json
import re

log = logging.getLogger(__name__)

GameEntry = namedtuple('GameEntry', ('stage', 'mode'))

def is_valid_entry(result, entry):
    # no dupes
    if entry in result:
        return False

    # make sure the map isn't played in the last 2 games
    last_two_games = result[-2:]
    for prev in last_two_games:
        if prev.stage == entry.stage:
            return False

    return True

def get_random_scrims(modes, maps, count):
    result = []
    current_mode_index = 0
    for index in range(count):
        # only try up to 25 times instead of infinitely
        for i in range(25):
            entry = GameEntry(stage=random.choice(maps), mode=modes[current_mode_index])
            if is_valid_entry(result, entry):
                result.append(entry)
                current_mode_index += 1
                if current_mode_index >= len(modes):
                    current_mode_index = 0
                break

    return result

RESOURCE_TO_EMOJI = {
    # Some of these names can be finnicky.
    'Ability Doubler': '<:abilitydoubler:338814715088338957>',
    'Bomb Defense Up': '<:bombdefup:338814715184807957>',
    'Sub Power Up': '<:subpowerup:338814716669329410>',
    'Swim Speed Up': '<:swimspeedup:338814717009068034>',
    'Cold-Blooded': '<:coldblooded:338814717340418069>',
    'Ink Saver (Main)': '<:inksavermain:338814717353263128>',
    'Ink Recovery Up': '<:inkrecoveryup:338814717411721216>',
    'Ink Saver (Sub)': '<:inksaversub:338814717470703616>',
    'Stealth Jump': '<:stealthjump:338814717483024395>',
    'Last-Ditch Effort': '<:lastditcheffort:338814717483155458>',
    'Special Saver': '<:specialsaver:338814717500063746>',
    'Tenacity': '<:tenacity:338814717525098499>',
    'Opening Gambit': '<:openinggambit:338814717529292802>',
    'Quick Respawn': '<:quickrespawn:338814717537812481>',
    'Respawn Punisher': '<:respawnpunisher:338814717562978304>',
    'Drop Roller': '<:droproller:338814717579755520>',
    'Ink Resistance Up': '<:inkresistance:338814717592076289>',
    'Comeback': '<:comeback:338814717663379456>',
    'Thermal Ink': '<:thermal_ink:338814717663641600>',
    'Ninja Squid': '<:ninjasquid:338814717743202304>',
    'Haunt': '<:haunt:338814717755654144>',
    'Object Shredder': '<:objectshredder:338814717780951050>',
    'Quick Super Jump': '<:quicksuperjump:338814717789208576>',
    'Special Charge Up': '<:specialchargeup:338814717793665024>',
    'Run Speed Up': '<:runspeedup:338814717823025152>',
    'Special Power Up': '<:specialpowerup:338815052234752000>',
    'Money': '<:money:338815193305972736>',
    'Unknown': '<:unknown:338815018101506049>',
}

# There's going to be some code duplication here because it's more
# straightforward than trying to be clever, I guess.
# I hope at one day to fix this and make it not-so-ugly.
# Hopefully by completely dropping Splatoon 1 support in
# the future.

class BrandResults:
    __slots__ = ('ability_name', 'info', 'buffs', 'nerfs')

    def __init__(self, brand=None, ability_name=None):
        self.info = brand
        self.ability_name = ability_name
        self.buffs = []
        self.nerfs = []

    def is_brand(self):
        return self.ability_name is None

class Rotation:
    def __init__(self, data):
        self.mode = data['rule']['name']
        self.stage_a = data['stage_a']['name']
        self.stage_b = data['stage_b']['name']

        self.start_time = datetime.datetime.utcfromtimestamp(data['start_time'])
        self.end_time = datetime.datetime.utcfromtimestamp(data['end_time'])

    @property
    def current(self):
        now = datetime.datetime.utcnow()
        return self.start_time <= now <= self.end_time

    def get_generic_value(self):
        return f'{self.stage_a} and {self.stage_b}'

class Gear:
    __slots__ = ('kind', 'brand', 'name', 'stars', 'main', 'frequent_skill', 'price', 'image')

    def __init__(self, data):
        self.kind = data['kind']
        brand = data['brand']
        self.brand = brand['name']
        self.name = data['name']
        self.stars = data['rarity'] + 1
        self.image = data['image']
        self.main = None
        self.price = None

        try:
            self.frequent_skill = brand['frequent_skill']['name']
        except KeyError:
            self.frequent_skill = None

    @classmethod
    def from_json(cls, data):
        """Load from our JSON file."""
        self = cls.__new__(cls)

        self.kind = data.get('kind')
        self.brand = data['brand']
        self.name = data['name']
        self.price = data['price']
        self.main = data['main']
        self.stars = data['stars']
        self.image = data.get('image')
        self.frequent_skill = data.get('frequent_skill')
        return self

class SalmonRun:
    def __init__(self, data):
        fromutc = datetime.datetime.utcfromtimestamp
        self.start_time = fromutc(data['start_time'])
        self.end_time = fromutc(data['end_time'])

        stage = data.get('stage', {})
        self.stage = stage.get('name')
        self._image = stage.get('image')
        weapons = data.get('weapons')
        self.weapons = []
        for weapon in weapons:
            actual_weapon_data = weapon.get('weapon')
            if actual_weapon_data is None:
                name = 'Rare-only Mystery' if weapon['id'] == '-2' else 'Mystery'
            else:
                name = actual_weapon_data.get('name', 'Mystery')
            self.weapons.append(name)

    @property
    def image(self):
        return self._image and f'https://app.splatoon2.nintendo.net{self._image}'

class SalmonRunPageSource(menus.ListPageSource):
    def __init__(self, entries):
        super().__init__(entries=entries, per_page=1)

    async def format_page(self, menu, salmon):
        e = discord.Embed(colour=0xFF7500, title='Salmon Run')

        if salmon.image:
            e.set_image(url=salmon.image)

        now = datetime.datetime.utcnow()
        if now <= salmon.start_time:
            e.set_footer(text='Starts').timestamp = salmon.start_time
            e.description = f'Starts in {time.human_timedelta(salmon.start_time)}'
        elif now <= salmon.end_time:
            e.set_footer(text='Ends').timestamp = salmon.end_time
            e.description = f'Ends in {time.human_timedelta(salmon.end_time)}'

        e.add_field(name='Weapons', value='\n'.join(salmon.weapons) or 'Unknown')
        e.add_field(name='Map', value=salmon.stage or 'Unknown')

        maximum = self.get_max_pages()
        if maximum > 1:
            e.title = f'Salmon Run {menu.current_page + 1} out of {maximum}'

        return e

class Splatfest:
    def __init__(self, data):
        names = data['names']
        self.alpha = names['alpha_short'] # Pearl
        self.bravo = names['bravo_short'] # Marina
        self.alpha_long = names['alpha_long']
        self.bravo_long = names['bravo_long']

        times = data['times']
        fromutc = datetime.datetime.utcfromtimestamp
        self.start = fromutc(times['start'])
        self.end = fromutc(times['end'])
        self.result = fromutc(times['result'])
        self.announce = fromutc(times['announce'])

        self.image = data['images']['panel']
        self.id = data['festival_id']

        colours = data['colors']

        def to_colour(d):
            return discord.Colour.from_rgb(int(d['r'] * 255), int(d['g'] * 255), int(d['b'] * 255))

        self.alpha_colour = to_colour(colours['alpha'])
        self.bravo_colour = to_colour(colours['bravo'])
        self.colour = to_colour(colours['middle'])

    def embed(self):
        e = discord.Embed(colour=self.colour, title=f'{self.alpha} vs {self.bravo}')
        now = datetime.datetime.utcnow()

        e.add_field(name='Pearl', value=self.alpha_long)
        e.add_field(name='Marina', value=self.bravo_long)

        if self.start > now:
            state = 'Starting'
            value = f'In {time.human_timedelta(self.start, source=now)}'
        elif self.end > now > self.start:
            state = 'Ending'
            value = f'In {time.human_timedelta(self.end, source=now)}'
        elif self.result > now:
            state = 'Waiting For Results'
            value = f'In {time.human_timedelta(self.result, source=now)}'
        else:
            state = 'Ended'
            value = time.human_timedelta(self.end, source=now)

        e.add_field(name=state, value=value, inline=False)
        e.set_image(url=f'https://app.splatoon2.nintendo.net{self.image}')
        return e

class Merchandise:
    def __init__(self, data):
        self.gear = Gear(data['gear'])
        self.skill = data['skill']['name']
        self.price = data['price']
        self.end_time = datetime.datetime.utcfromtimestamp(data['end_time'])

class GearPageSource(menus.ListPageSource):
    def __init__(self, entries):
        super().__init__(entries=entries, per_page=1)

    def prepare_embed(self, menu, merch):
        original_gear = None
        data = menu.ctx.cog.splat2_data

        if isinstance(merch, Merchandise):
            price, gear, skill = merch.price, merch.gear, merch.skill
            description = f'{time.human_timedelta(merch.end_time)} left to buy'
            data = data.get(gear.kind, [])
            for elem in data:
                if elem.name == gear.name:
                    original_gear = elem
                    break
        elif isinstance(merch, Gear):
            price, gear, skill = merch.price, merch, merch.main
            description = discord.Embed.Empty

        e = discord.Embed(colour=0x19D719, title=gear.name, description=description)

        if gear.image:
            e.set_thumbnail(url=f'https://app.splatoon2.nintendo.net{gear.image}')
        else:
            e.set_thumbnail(url='https://cdn.discordapp.com/emojis/338815018101506049.png')

        e.add_field(name='Price', value=f'{RESOURCE_TO_EMOJI["Money"]} {price or "???"}')

        UNKNOWN = RESOURCE_TO_EMOJI['Unknown']

        main_slot = RESOURCE_TO_EMOJI.get(skill, UNKNOWN)
        remaining = UNKNOWN * gear.stars
        e.add_field(name='Slots', value=f'{main_slot} | {remaining}')

        if isinstance(merch, Merchandise):
            if original_gear is not None:
                original = RESOURCE_TO_EMOJI.get(original_gear.main, UNKNOWN)
                original_remaining = UNKNOWN * original_gear.stars
                original_price = original_gear.price
            else:
                original = UNKNOWN
                original_remaining = remaining
                original_price = '???'

            e.add_field(name='Original Price', value=f'{RESOURCE_TO_EMOJI["Money"]} {original_price}')
            e.add_field(name='Original Slots', value=f'{original} | {original_remaining}')

        e.add_field(name='Brand', value=gear.brand)
        if gear.frequent_skill is not None:
            common = gear.frequent_skill
        else:
            brands = data.get('brands', [])
            for brand in brands:
                if brand['name'] == gear.brand:
                    common = brand['buffed']
                    break
            else:
                common = 'Not found...'
        e.add_field(name='Common Gear Ability', value=common)

        maximum = self.get_max_pages()
        if maximum > 1:
            e.set_footer(text=f'Gear {menu.current_page + 1}/{maximum}')

        return e

# JSON stuff

class Splatoon2Encoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, Gear):
            payload = {
                attr: getattr(obj, attr)
                for attr in Gear.__slots__
            }
            payload['__gear__'] = True
            return payload
        if isinstance(obj, SalmonRun):
            return obj.to_dict()
        return super().default(obj)

def splatoon2_decoder(obj):
    if '__gear__' in obj:
        return Gear.from_json(obj)
    return obj

def mode_key(argument):
    lower = argument.lower().strip('"')
    if lower.startswith('rank'):
        return 'Ranked Battle'
    elif lower.startswith('turf') or lower.startswith('regular'):
        return 'Regular Battle'
    elif lower == 'league':
        return 'League Battle'
    else:
        raise commands.BadArgument('Unknown schedule type, try: "ranked", "regular", or "league"')

_iso_regex = re.compile(r'([0-9]{4})-([0-9]{1,2})-([0-9]{1,2})[T\s]([0-9]{1,2})\:([0-9]{1,2})')

def iso8601(argument, *, _re=_iso_regex):
    # YYYY-MM-DDTHH:MM
    m = _re.match(argument)
    if m is None:
        raise commands.BadArgument(f'Bad time provided ({argument})')

    return datetime.datetime(*map(int, m.groups()))

class BrandOrAbility(commands.Converter):
    async def convert(self, ctx, argument):
        query = argument.lower()
        if len(query) < 4:
            raise commands.BadArgument('The query must be at least 5 characters long.')

        data = ctx.cog.splat2_data
        brands = data.get('brands', [])

        result = None

        # check for exact match
        for brand in brands:
            if brand['name'].lower() == query:
                return BrandResults(brand=brand)

        # check for fuzzy match
        for brand in brands:
            name = brand['name']

            # basic case, direct brand name match
            if fuzzy.partial_ratio(query, name.lower()) >= 80:
                return BrandResults(brand=brand)

        # now check if it matches an ability instead
        for brand in brands:
            buffed = brand['buffed']
            nerfed = brand['nerfed']

            # if it's not matched up there, it's definitely not matched here
            if not nerfed or not buffed:
                continue

            if fuzzy.partial_ratio(query, buffed.lower()) >= 60:
                result = BrandResults(ability_name=buffed)
                break

            if fuzzy.partial_ratio(query, nerfed.lower()) >= 60:
                result = BrandResults(ability_name=nerfed)
                break

        if result is None:
            raise commands.BadArgument('Could not find anything.')

        # check the brands that buff or nerf the ability we're looking for:
        for brand in brands:
            buffed = brand['buffed']
            nerfed = brand['nerfed']
            if not nerfed or not buffed:
                continue

            if buffed == result.ability_name:
                result.buffs.append(brand['name'])
            elif nerfed == result.ability_name:
                result.nerfs.append(brand['name'])

        return result

class GearQuery(commands.Converter):
    async def convert(self, ctx, argument):
        import shlex

        # parse our pseudo CLI
        args = shlex.split(argument.lower(), posix=False)
        iterator = iter(args)

        # check if flags is one of --brand, --ability, or --frequent
        info = {
            'query': None,
            '--brand': None,
            '--ability': None,
            '--frequent': None,
            '--type': None
        }

        current = 'query'
        temp = []
        for argument in args:
            if argument[0] == '-':
                if argument not in ('--brand', '--ability', '--frequent', '--type'):
                    msg = f'Invalid flag passed, received {argument} expected --brand, --ability, --frequent, or --type'
                    raise commands.BadArgument(msg)

                info[current] = ' '.join(temp)
                temp = []
                current = argument
                continue

            temp.append(argument)

        if temp:
            info[current] = ' '.join(temp)

        query = info['query']
        if len(query) < 4:
            raise commands.BadArgument('The query must be at least 5 characters long.')

        data = ctx.cog.splat2_data
        importance = []

        frequent_lookup = {
            x['buffed'].lower(): x['name']
            for x in data['brands']
            if x['buffed']
        }

        # search by name, main ability or brand
        # sort by importance
        scorer = fuzzy.partial_ratio
        brand = info['--brand']
        ability = info['--ability']
        frequent = info['--frequent']
        if frequent:
            m = fuzzy.extract_one(frequent, frequent_lookup, scorer=scorer, score_cutoff=70)
            if m is None:
                raise commands.BadArgument('Could not figure out the frequent ability requested.')
            _, _, frequent = m

        kind = info['--type']
        if kind in ('hat', 'hats'):
            kind = 'head'

        if kind in ('shirt', 'shirts'):
            kind = 'clothes'

        if kind == 'shoe':
            kind = 'shoes'

        if kind is None:
            iterator = itertools.chain(data['head'], data['shoes'], data['clothes'])
        else:
            iterator = data[kind]

        for gear in iterator:
            important = max(scorer(query, gear.name), scorer(query, gear.main), scorer(query, gear.brand))
            if important >= 70:
                # apply filters:
                if frequent and frequent != gear.brand:
                    continue

                if brand and scorer(brand, gear.brand) < 70:
                    continue

                if ability and scorer(ability, gear.main) < 70:
                    continue

                importance.append((gear, important))

        importance.sort(key=lambda t: t[1], reverse=True)
        if len(importance) == 0:
            raise commands.BadArgument('Could not find anything.')

        top, score = importance[0]
        if score == 100:
            return [top]

        return [g for g, _ in importance]

class Splatoon(commands.Cog):
    """Splatoon related commands."""

    BASE_URL = yarl.URL('https://app.splatoon2.nintendo.net')

    def __init__(self, bot):
        self.bot = bot
        self.splat2_data = config.Config('splatoon2.json', loop=bot.loop,
                                         object_hook=splatoon2_decoder, encoder=Splatoon2Encoder)
        self.map_data = []

        self._splatnet2 = bot.loop.create_task(self.splatnet2())
        self._authenticator = bot.loop.create_task(self.splatnet2_authenticator())
        self._is_authenticated = asyncio.Event(loop=bot.loop)

        # mode: List[Rotation]
        self.sp2_map_data = {}
        self.sp2_shop = []
        self.sp2_festival = None
        self.sp2_salmonrun = []

    def cog_unload(self):
        self._splatnet2.cancel()
        self._authenticator.cancel()

    async def cog_command_error(self, ctx, error):
        if isinstance(error, commands.BadArgument):
            return await ctx.send(error)

    @property
    def salmon_run(self):
        now = datetime.datetime.utcnow()
        return [s for s in self.sp2_salmonrun if now < s.end_time]

    async def splatnet2_authenticator(self):
        try:
            session_token = self.splat2_data.get('session_token')
            while not self.bot.is_closed():
                iksm = self.splat2_data.get('iksm_session')
                if iksm is not None:
                    self.bot.session.cookie_jar.update_cookies({ 'iksm_session': iksm }, response_url=self.BASE_URL)
                    self._is_authenticated.set()

                expires = datetime.datetime.utcfromtimestamp(self.splat2_data.get('iksm_expires', 0.0))
                now = datetime.datetime.utcnow()

                if now < expires:
                    # our session is still valid, so let's wait a while before authenticating
                    delta = (expires - now).total_seconds()
                    await asyncio.sleep(delta)

                # at this point our session is invalid so let's re-authenticate
                self._is_authenticated.clear()
                session = self.bot.session

                url = 'https://accounts.nintendo.com/connect/1.0.0/api/token'
                data = {
                    'client_id': '71b963c1b7b6d119',
                    'grant_type': 'urn:ietf:params:oauth:grant-type:jwt-bearer-session-token',
                    'session_token': session_token
                }
                headers = {
                    'Content-Type': 'application/json; charset=utf-8',
                    'X-Platform': 'Android',
                    'X-ProductVersion': '1.1.0',
                    'User-Agent': 'com.nintendo.znca/1.1.0 (Android/4.4.2)'
                }

                # first, authenticate into the accounts.nintendo.com for our Bearer token and ID token.
                async with session.post(url, headers=headers, data=json.dumps(data)) as resp:
                    if resp.status != 200:
                        extra = f'SplatNet 2 authentication error: {resp.status} for {url}'
                        await self.bot.get_cog('Stats').log_error(extra=extra)
                        await asyncio.sleep(300.0) # try again in 5 minutes
                        continue

                    # we don't care when the token expires but we need our ID token
                    js = await resp.json()
                    id_token = js['id_token']

                # authenticate to the nintendo.net API
                data = {
                    "parameter": {
                        "language": "en-US",
                        'naCountry': 'US',
                        "naBirthday": "1989-04-06",
                        "naIdToken": id_token
                    }
                }

                url = 'https://api-lp1.znc.srv.nintendo.net/v1/Account/Login'
                headers['Authorization'] = 'Bearer'

                async with session.post(url, headers=headers, data=json.dumps(data)) as resp:
                    if resp.status != 200:
                        extra = f'SplatNet 2 authentication error: {resp.status} for {url}'
                        await self.bot.get_cog('Stats').log_error(extra=extra)
                        await asyncio.sleep(300.0) # try again in 5 minutes
                        continue

                    js = await resp.json()
                    if js['status'] != 0:
                        extra = f'Firebase issue for {url}:\n```js\n{json.dumps(js, indent=4)}\n```'
                        await self.bot.get_cog('Stats').log_error(extra=extra)
                        await asyncio.sleep(300.0) # try again in 5 minutes
                        continue

                    token = js['result'].get('webApiServerCredential', {}).get('accessToken')
                    if token is None:
                        extra = f'SplatNet 2 authentication error: No accessToken for {url}'
                        await self.bot.get_cog('Stats').log_error(extra=extra)
                        await asyncio.sleep(300.0) # try again in 5 minutes
                        continue

                # get the web service token
                data = {
                    "parameter": {
                        "id": 5741031244955648 # SplatNet 2 ID
                    }
                }
                url = 'https://api-lp1.znc.srv.nintendo.net/v1/Game/GetWebServiceToken'
                headers['Authorization'] = f'Bearer {token}'
                async with session.post(url, headers=headers, data=json.dumps(data)) as resp:
                    if resp.status != 200:
                        extra = f'SplatNet 2 authentication error: {resp.status} for {url}'
                        await self.bot.get_cog('Stats').log_error(extra=extra)
                        await asyncio.sleep(300.0) # try again in 5 minutes
                        continue

                    js = await resp.json()
                    if js['status'] != 0:
                        extra = f'Firebase issue for {url}:\n```js\n{json.dumps(js, indent=4)}\n```'
                        await self.bot.get_cog('Stats').log_error(extra=extra)
                        await asyncio.sleep(300.0) # try again in 5 minutes
                        continue

                    access_token = js['result'].get('accessToken')
                    if access_token is None:
                        extra = f'SplatNet 2 authentication error: No accessToken for {url}'
                        await self.bot.get_cog('Stats').log_error(extra=extra)
                        await asyncio.sleep(300.0) # try again in 5 minutes
                        continue

                # Now we can **finally** access SplatNet 2
                headers.pop('Authorization')
                headers['x-gamewebtoken'] = access_token
                headers['x-isappanalyticsoptedin'] = 'false'
                headers['X-Requested-With'] = 'com.nintendo.znca'
                async with session.get(self.BASE_URL, params={'lang': 'en-US'}, headers=headers) as resp:
                    if resp.status != 200:
                        extra = f'SplatNet 2 authentication error: {resp.status} for {url}'
                        await self.bot.get_cog('Stats').log_error(extra=extra)
                        await asyncio.sleep(300.0) # try again in 5 minutes
                        continue

                    # finally our shit
                    # SimpleCookie API is cancer
                    m = re.search(r'iksm_session=(?P<session>.+?);.+?expires=(?P<expires>.+?);', str(resp.cookies))
                    if m is None:
                        extra = f'Regex fucked up. See {resp.cookie}'
                        await self.bot.get_cog('Stats').log_error(extra=extra)
                        await asyncio.sleep(300.0) # try again in 5 minutes
                        continue

                    iksm = m.group('session')
                    try:
                        expires = parsedate_to_datetime(m.group('expires'))
                    except:
                        expires = now.timestamp() + 300.0
                    else:
                        expires = expires.timestamp()

                    await self.splat2_data.put('iksm_session', iksm)
                    await self.splat2_data.put('iksm_expires', expires)

                log.info('Authenticated to SplatNet 2. Session: %s Expires: %s', iksm, expires)
                self._is_authenticated.set()
        except asyncio.CancelledError:
            raise
        except (OSError, discord.ConnectionClosed):
            self._authenticator.cancel()
            self._authenticator = self.bot.loop.create_task(self.splatnet2_authenticator())

    async def parse_splatnet2_schedule(self):
        try:
            self.sp2_map_data = {}
            async with self.bot.session.get(self.BASE_URL / 'api/schedules') as resp:
                if resp.status != 200:
                    await self.bot.get_cog('Stats').log_error(extra=f'Splatnet schedule responded with {resp.status}.')
                    return 300.0 # try again in 5 minutes

                data = await resp.json()
                gachi = data.get('gachi', [])
                self.sp2_map_data['Ranked Battle'] = [
                    Rotation(d) for d in gachi
                ]
                regular = data.get('regular', [])
                self.sp2_map_data['Regular Battle'] = [
                    Rotation(d) for d in regular
                ]

                league = data.get('league', [])
                self.sp2_map_data['League Battle'] = [
                    Rotation(d) for d in league
                ]

                newest = []
                for key, value in self.sp2_map_data.items():
                    value.sort(key=lambda r: r.end_time)
                    try:
                        newest.append(value[0].end_time)
                    except IndexError:
                        pass

                try:
                    new = max(newest)
                except ValueError:
                    return 300.0 # try again in 5 minutes

                now = datetime.datetime.utcnow()
                return 300.0 if now > new else (new - now).total_seconds()
        except Exception as e:
            await self.bot.get_cog('Stats').log_error(extra=f'Splatnet schedule Error')
            return 300.0

    async def parse_splatnet2_onlineshop(self):
        try:
            self.sp2_shop = []
            async with self.bot.session.get(self.BASE_URL / 'api/onlineshop/merchandises') as resp:
                if resp.status != 200:
                    await self.bot.get_cog('Stats').log_error(extra=f'Splatnet Shop responded with {resp.status}.')
                    return 300.0 # try again in 5 minutes

                data = await resp.json()
                merch = data.get('merchandises')
                if not merch:
                    return 300.0

                for elem in merch:
                    try:
                        value = Merchandise(elem)
                    except KeyError:
                        pass
                    else:
                        self.sp2_shop.append(value)

                        # update our image cache
                        kind = self.splat2_data.get(value.gear.kind, [])
                        for gear in kind:
                            if gear.name == value.gear.name:
                                gear.image = value.gear.image
                                break

                await self.splat2_data.save()
                self.sp2_shop.sort(key=lambda m: m.end_time)
                now = datetime.datetime.utcnow()
                try:
                    end = self.sp2_shop[0].end_time
                    return 300.0 if now > end else (end - now).total_seconds()
                except:
                    return 300.0
        except Exception as e:
            await self.bot.get_cog('Stats').log_error(extra=f'Splatnet Shop Error')
            return 300.0

    def scrape_data_from_player(self, player, bulk):
        for kind in ('shoes', 'head', 'clothes'):
            try:
                gear = Gear(player[kind])
            except KeyError:
                continue
            else:
                bulk[kind][gear.name] = gear

    async def scrape_splatnet_stats_and_images(self):
        try:
            bulk_lookup = defaultdict(dict)
            async with self.bot.session.get(self.BASE_URL / 'api/results') as resp:
                if resp.status != 200:
                    await self.bot.get_cog('Stats').log_error(extra=f'Splatnet Stats responded with {resp.status}.')
                    return 300.0 # try again in 5 minutes

                data = await resp.json()
                results = data['results']
                base_path = pathlib.Path('splatoon2_stats')

                newest = base_path / f'{results[0]["battle_number"]}.json'

                # we already scraped, so try again in an hour
                if newest.exists():
                    log.info('No Splatoon 2 result data to scrape, retrying in an hour.')
                    return 3600.0

                # I am sorry papa nintendo
                pre_existing_statistics = sorted([int(p.stem) for p in base_path.iterdir()])
                if pre_existing_statistics:
                    largest = pre_existing_statistics[-1]
                else:
                    largest = 0

                added = 0
                for result in results:
                    try:
                        number = result['battle_number']
                    except KeyError:
                        continue

                    if int(number) <= largest:
                        continue

                    # request the full information:
                    async with self.bot.session.get(resp.url / number) as r:
                        if r.status != 200:
                            extra = f'Splatoon Stat {number} responded with {r.status}.'
                            await self.bot.get_cog('Stats').log_error(extra=extra)
                            continue

                        js = await r.json()

                        # save our statistics
                        path = base_path / f'{number}.json'
                        with path.open('w', encoding='utf-8') as fp:
                            json.dump(js, fp, indent=2)

                        added += 1

                        # add stuff to image cache
                        for enemy in js.get('other_team_members', []):
                            self.scrape_data_from_player(enemy.get('player', {}), bulk_lookup)

                        me = js.get('player_result', {}).get('player', {})
                        self.scrape_data_from_player(me, bulk_lookup)

                        for team in js.get('my_team_members', []):
                            self.scrape_data_from_player(team.get('player', {}), bulk_lookup)

                    await asyncio.sleep(1) # one request a second so we don't abuse

                log.info('Scraped Splatoon 2 results from %s games.', added)

                # done with bulk lookups so actually change and save now:
                for kind, data in bulk_lookup.items():
                    old = self.splat2_data.get(kind, [])
                    for gear in old:
                        try:
                            new_data = data.pop(gear.name)
                        except KeyError:
                            continue
                        else:
                            gear.image = new_data.image

                    # add new data
                    log.info('Scraped %s new pieces of %s gear.', len(data), kind)
                    for value in data.values():
                        old.append(value)

                await self.splat2_data.save()
                return 3600.0 # redo in an hour
        except Exception as e:
            await self.bot.get_cog('Stats').log_error(extra=f'Splatnet Stat Error')
            return 300.0

    async def parse_splatnet2_splatfest(self):
        try:
            self.sp2_festival = None
            async with self.bot.session.get(self.BASE_URL / 'api/festivals/active') as resp:
                if resp.status != 200:
                    await self.bot.get_cog('Stats').log_error(extra=f'Splatnet Splatfest Error')
                    return 300.0

                js = await resp.json()
                festivals = js['festivals']
                if len(festivals) == 0:
                    return 3600.0

                current = festivals[0]
                self.sp2_festival = Splatfest(current)
                return 3600.0
        except Exception as e:
            await self.bot.get_cog('Stats').log_error(extra=f'Splatnet Splatfest Error')
            return 300.0

    async def parse_splatnet2_salmonrun(self):
        try:
            self.sp2_salmonrun = []
            async with self.bot.session.get(self.BASE_URL / 'api/coop_schedules') as resp:
                if resp.status != 200:
                    await self.bot.get_cog('Stats').log_error(extra=f'Splatnet Salmon Run Error')
                    return 3600.0

                js = await resp.json()

                data = js['details']
                if len(data) == 0:
                    return 3600.0

                self.sp2_salmonrun = [SalmonRun(d) for d in data]
                self.sp2_salmonrun.sort(key=lambda m: m.end_time)
                now = datetime.datetime.utcnow()
                end = self.sp2_salmonrun[0].end_time
                return 300.0 if now > end else (end - now).total_seconds()
        except Exception as e:
            await self.bot.get_cog('Stats').log_error(extra=f'Splatnet Salmon Run Error')
            return 3600.0

    async def splatnet2(self):
        try:
            while not self.bot.is_closed():
                seconds = []
                await self._is_authenticated.wait()
                seconds.append(await self.parse_splatnet2_schedule())
                seconds.append(await self.parse_splatnet2_onlineshop())
                seconds.append(await self.parse_splatnet2_salmonrun())
                seconds.append(await self.scrape_splatnet_stats_and_images())
                seconds.append(await self.parse_splatnet2_splatfest())
                await asyncio.sleep(min(seconds))
        except asyncio.CancelledError:
            raise
        except (OSError, discord.ConnectionClosed):
            self._splatnet2.cancel()
            self._splatnet2 = self.bot.loop.create_task(self.splatnet2())

    def get_weapons_named(self, name):
        data = self.splat2_data.get('weapons', [])
        name = name.lower()

        choices = {w['name'].lower(): w for w in data}
        results = fuzzy.extract_or_exact(name, choices, scorer=fuzzy.token_sort_ratio, score_cutoff=60)
        return [v for k, _, v in results]

    async def generate_scrims(self, ctx, maps, games, mode):
        modes = ['Rainmaker', 'Splat Zones', 'Tower Control', 'Clam Blitz']

        if mode is not None:
            mode = mode.lower()

            # shortcuts that can't be detected by fuzzy matching:
            shortcuts = {
                'rm': 'Rainmaker',
                'sz': 'Splat Zones',
                'tc': 'Tower Control',
                'tw': 'Turf War',
                'cb': 'Clam Blitz',
            }

            real_mode = shortcuts.get(mode)
            if real_mode is None:
                real_mode = fuzzy.extract_one(mode, modes + ['Turf War'], scorer=fuzzy.partial_ratio, score_cutoff=50)
                if real_mode is not None:
                    real_mode = real_mode[0]
                else:
                    return await ctx.send('Could not figure out what mode you meant.')

            result = [f'The following games will be played in {real_mode}.']
            game_count = max(min(games, len(maps)), 3)
            for index, stage in enumerate(random.sample(maps, game_count), 1):
                result.append(f'Game {index}: {stage}')
        else:
            game_count = max(min(games, len(maps) * 4), 3)
            random.shuffle(modes)
            scrims = get_random_scrims(modes, maps, game_count)
            result = [f'Game {game}: {scrim.mode} on {scrim.stage}' for game, scrim in enumerate(scrims, 1)]

        await ctx.safe_send('\n'.join(result))

    @commands.command(hidden=True)
    async def marie(self, ctx):
        """A nice little easter egg."""
        await ctx.send('http://i.stack.imgur.com/0OT9X.png')

    @commands.command()
    async def splatwiki(self, ctx, *, title: str):
        """Returns a Inkipedia page."""
        url = f'http://splatoonwiki.org/wiki/Special:Search/{urlquote(title)}'

        async with ctx.session.get(url) as resp:
            if 'Special:Search' in resp.url.path:
                await ctx.send(f'Could not find your page. Try a search:\n{resp.url.human_repr()}')
            elif resp.status == 200:
                await ctx.send(resp.url)
            elif resp.status == 502:
                await ctx.send('It seems that Inkipedia is taking too long to respond. Try again later.')
            else:
                await ctx.send(f'An error has occurred of status code {resp.status} happened.')

    async def generic_splatoon2_schedule(self, ctx):
        e = discord.Embed(colour=0x19D719)
        end_time = None

        for key, value in self.sp2_map_data.items():
            try:
                rotation = value[0]
            except IndexError:
                e.add_field(name=key, value='Nothing found...', inline=False)
                continue
            else:
                end_time = rotation.end_time

            e.add_field(name=f'{key}: {rotation.mode}',
                        value=f'{rotation.stage_a} and {rotation.stage_b}',
                        inline=False)

        if end_time is not None:
            e.title = f'For {time.human_timedelta(end_time)}...'

        await ctx.send(embed=e)

    async def paginated_splatoon2_schedule(self, ctx, mode):
        try:
            data = self.sp2_map_data[mode]
        except KeyError:
            return await ctx.send('Sorry, no map data found...')

        entries = [
            (f'Now: {r.mode}' if r.current else f'In {time.human_timedelta(r.start_time)}: {r.mode}',
            f'{r.stage_a} and {r.stage_b}')
            for r in data
        ]

        try:
            p = FieldPages(ctx, entries=entries, per_page=4)
            p.embed.colour = 0xF02D7D
            await p.paginate()
        except menus.MenuError as e:
            await ctx.send(e)

    @commands.command(aliases=['maps'])
    async def schedule(self, ctx, *, type: mode_key = None):
        """Shows the current Splatoon 2 schedule."""
        if type is None:
            await self.generic_splatoon2_schedule(ctx)
        else:
            await self.paginated_splatoon2_schedule(ctx, type)

    @commands.command()
    async def nextmaps(self, ctx):
        """Shows the next Splatoon 2 maps."""
        e = discord.Embed(colour=0x19D719)

        start_time = None
        for key, value in self.sp2_map_data.items():
            try:
                rotation = value[1]
            except IndexError:
                e.add_field(name=key, value='Nothing found...', inline=False)
                continue
            else:
                start_time = rotation.start_time

            e.add_field(name=f'{key}: {rotation.mode}',
                        value=f'{rotation.stage_a} and {rotation.stage_b}',
                        inline=False)

        if start_time is not None:
            e.title = f'In {time.human_timedelta(start_time)}'

        await ctx.send(embed=e)

    @commands.group(invoke_without_command=True)
    async def salmonrun(self, ctx):
        """Shows the Salmon Run schedule, if any."""
        salmon = self.salmon_run
        if not salmon:
            return await ctx.send('No Salmon Run schedule reported.')

        try:
            await menus.MenuPages(SalmonRunPageSource(salmon), delete_message_after=True).start(ctx)
        except menus.MenuError as e:
            await ctx.send(e)

    @commands.command(aliases=['splatnetshop'])
    async def splatshop(self, ctx):
        """Shows the currently running SplatNet 2 merchandise."""
        if not self.sp2_shop:
            return await ctx.send('Nothing currently being sold...')

        pages = menus.MenuPages(GearPageSource(self.sp2_shop), delete_message_after=True)
        try:
            await pages.start(ctx)
        except menus.MenuError as e:
            await ctx.send(e)

    @commands.command()
    async def splatfest(self, ctx):
        """Shows information about the currently running NA Splatfest, if any."""
        if self.sp2_festival is None:
            return await ctx.send('No Splatfest has been announced.')

        await ctx.send(embed=self.sp2_festival.embed())

    @commands.command()
    async def weapon(self, ctx, *, query: str):
        """Displays Splatoon 2 weapon info from a query.

        The query must be at least 3 characters long, otherwise it'll tell you it failed.
        """
        query = query.strip().lower()
        weapons = self.splat2_data.get('weapons', [])
        if len(query) < 3:
            return await ctx.send('The query must be at least 3 characters long.')

        def predicate(weapon):
            lowered = [weapon.lower() for weapon in weapon.values()]
            return any(query in wep for wep in lowered)

        results = list(filter(predicate, weapons))
        if not results:
            return await ctx.send('No results found.')

        e = discord.Embed(colour=discord.Colour.blurple())
        e.title = f'Found {plural(len(results)):weapon}'

        subs = '\n'.join(w['sub'] for w in results)
        names = '\n'.join(w['name'] for w in results)
        special = '\n'.join(w['special'] for w in results)

        e.add_field(name='Name', value=names)
        e.add_field(name='Sub', value=subs)
        e.add_field(name='Special', value=special)
        await ctx.send(embed=e)

    @commands.command()
    async def gear(self, ctx, *, query: GearQuery):
        """Searches for Splatoon 2 gear that matches your query.

        The query can be a main ability, a brand, or a name.

        For advanced queries to reduce results you can pass some filters:

        `--brand` with the brand name.
        `--ability` with the main ability.
        `--frequent` with the buffed main ability probability
        `--type` with the type of clothing (head, hat, shoes, or clothes)

        For example, a query like `ink resist --brand splash mob` will give all
        gear with Ink Resistance Up and Splash Mob as the brand.

        **Note**: you must pass a query before passing a filter.
        """

        pages = menus.MenuPages(GearPageSource(query), delete_message_after=True)
        try:
            await pages.start(ctx)
        except menus.MenuError as e:
            await ctx.send(e)

    @commands.command()
    async def scrim(self, ctx, games=5, *, mode: str = None):
        """Generates Splatoon 2 scrim map and mode combinations.

        The mode combinations do not have Turf War.

        The mode is rotated unless you pick a mode to play, in which all map
        combinations will use that mode instead.
        """
        maps = self.splat2_data.get('maps', [])
        await self.generate_scrims(ctx, maps, games, mode)

    @commands.group(invoke_without_command=True)
    async def brand(self, ctx, *, query: BrandOrAbility):
        """Shows Splatoon 2 brand info

        This is based on either the name or the ability given.

        If the query is an ability then it attempts to find out what brands
        influence that ability, otherwise it just looks for the brand being given.

        The query must be at least 4 characters long.
        """
        e = discord.Embed(colour=0x19D719)
        if query.is_brand():
            e.add_field(name='Name', value=query.info['name'])
            e.add_field(name='Common', value=query.info['buffed'])
            e.add_field(name='Uncommon', value=query.info['nerfed'])
            return await ctx.send(embed=e)

        e.description = f'The following brands deal with {query.ability_name}.'
        e.add_field(name='Common', value='\n'.join(query.buffs))
        e.add_field(name='Uncommon', value='\n'.join(query.nerfs))
        await ctx.send(embed=e)

    @commands.command(hidden=True)
    @commands.is_owner()
    async def new_weapon(self, ctx, name, sub, special):
        """Add a new Splatoon 2 weapon."""
        weapons = self.splat2_data.get('weapons', [])
        entry = {
            'name': name,
            'sub': sub,
            'special': special
        }
        weapons.append(entry)
        await self.splat2_data.put('weapons', weapons)
        await ctx.send('\N{OK HAND SIGN}')

    @commands.command(hidden=True)
    @commands.is_owner()
    async def new_map(self, ctx, *, name):
        """Add a new Splatoon 2 map."""
        entry = self.splat2_data.get('maps', [])
        entry.append(name)
        await self.splat2_data.put('maps', entry)
        await ctx.send('\N{OK HAND SIGN}')

def setup(bot):
    bot.add_cog(Splatoon(bot))


# volatility.py

import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
from collections import defaultdict
from mpl_toolkits.mplot3d import Axes3D
from scipy.interpolate import interp1d
from QuantLib import *

class VolatilitySurface(object):

    def __init__(self, evaluation_date, maturity_dates, strikes,
        implied_vol_matrix, spot, yield_curve,  dividend_curve):
        self.implied_vol_surface = {}
        self.local_vol_surface = {}
        self.create_surface(evaluation_date, maturity_dates, strikes,
        implied_vol_matrix, spot, yield_curve,  dividend_curve)

    def create_surface(self, evaluation_date, maturity_dates, strikes,
        implied_vol_matrix, spot, yield_curve,  dividend_curve):

        calendar = TARGET()
        day_counter = ActualActual()
        evaluation_date_ql = self._date_to_ql(evaluation_date)
        maturity_dates_ql = np.array([self._date_to_ql(date)
            for date in maturity_dates])

        nrows, ncols = implied_vol_matrix.shape
        implied_vol_matrix_ql = Matrix(nrows, ncols)
        for i in range(0, nrows):
            for j in range(0, ncols):
                implied_vol_matrix_ql[i][j] = implied_vol_matrix[i][j]

        self.implied_vol_surface = BlackVarianceSurface(evaluation_date_ql,
            calendar, maturity_dates_ql, strikes,
            implied_vol_matrix_ql, day_counter)

        self.local_vol_surface = LocalVolSurface(
            BlackVolTermStructureHandle(self.implied_vol_surface),
            YieldTermStructureHandle(yield_curve.curve),
            YieldTermStructureHandle(dividend_curve.curve), spot)

    def surface_plot(self, strikes, time_to_maturities, file_path):

        strike_matrix, ttm_matrix = np.meshgrid(strikes, time_to_maturities)
        implied_vol_matrix = np.zeros((len(time_to_maturities), len(strikes)))
            # (time_to_maturity, strike)
        for i in range(0, len(strikes)):
            for j in range(0, len(time_to_maturities)):
                implied_vol_matrix[j][i] = self.implied_vol_surface.blackVol(
                    time_to_maturities[j], strikes[i])

        fig = plt.figure(figsize = (8, 12))

        ax = fig.add_subplot(2, 1, 1, projection = '3d')
        surf = ax.plot_surface(strike_matrix, ttm_matrix,
            implied_vol_matrix,
            rstride = 1, cstride = 1, alpha = 0.65,
            cmap = plt.cm.jet, #plt.cm.coolwarm,
            linewidth = 0.5, antialiased = True)
        ax.set_xlabel('strike')
        ax.set_ylabel('time-to-maturity')
        ax.set_zlabel('implied volatility')
        fig.colorbar(surf, shrink = 0.6) #aspect = 1)

        local_vol_matrix = np.zeros((len(time_to_maturities), len(strikes)))
        for i in range(0, len(strikes)):
            for j in range(0, len(time_to_maturities)):
                try:
                    local_vol_matrix[j][i] = self.local_vol_surface.localVol(
                        time_to_maturities[j], strikes[i])
                except:
                    print(1111)

        ax = fig.add_subplot(2, 1, 2, projection = '3d')
        surf = ax.plot_surface(strike_matrix, ttm_matrix, local_vol_matrix,
            rstride = 1, cstride = 1, alpha = 0.65,
            cmap = plt.cm.jet, #plt.cm.coolwarm,
            linewidth = 0.5, antialiased = True)
        ax.set_xlabel('strike')
        ax.set_ylabel('time-to-maturity')
        ax.set_zlabel('local volatility')
        fig.colorbar(surf, shrink = 0.6) #aspect = 5)

        plt.tight_layout()
        if os.path.exists(file_path):
          os.remove(file_path)
        plt.savefig(file_path)

    def _date_to_ql(self, date):
        return Date(date.day, date.month, date.year)

class DynamicLocalVolSurface(object):

    def __init__(self, volatility_surface, spot, yield_curve,
        dividend_curve, months = 3, grids_per_month = 1,
        deltax_per_month = 0.3):

        self.spot = spot
        self.yield_curve = yield_curve
        self.dividend_curve = dividend_curve
        self.volatility_surface = volatility_surface

        self.dynamic_vix = {}

        self.create_local_var_tree(months, grids_per_month, deltax_per_month)

    def create_local_var_tree(self, months = 3, grids_per_month = 1,
        deltax_per_month = 0.3):

        ncols = months * grids_per_month + 1
        nrows = months * grids_per_month * 2 + 1
        self.deltat = 1.0 / 12.0 / grids_per_month
        self.deltax = deltax_per_month / grids_per_month
        self.A =  self.deltat / 2.0 / self.deltax ** 2
        self.B = self.deltat / 2.0 / self.deltax
        self.vix_steps = 3 * grids_per_month

        strikes = np.linspace(- (ncols - 1) * self.deltax,
            (ncols - 1) * self.deltax, nrows)
        strikes = np.exp(strikes) * self.spot
        time_to_maturities = np.linspace(0.0, months / 12.0 , ncols)

        self.local_var_tree = pd.DataFrame(np.zeros((nrows, ncols)),
            index = strikes, columns = time_to_maturities)
        for j in range(0, ncols):
            for i in range(ncols - j - 1, ncols + j):
                try:
                    self.local_var_tree.iloc[i, j] = \
                        self.volatility_surface.local_vol_surface.localVol(
                        time_to_maturities[j], strikes[i], True) ** 2
                except:
                    self.local_var_tree.iloc[i, j] = \
                        self.local_var_tree.iloc[i - 1, j]

    def cal_dynamic_vix(self, paras):
        self.dynamic_vix = defaultdict(list)
        self._cal_dynamic_vix(paras, self.local_var_tree, 1.0)

        #print(self.dynamic_vix)

    def _cal_dynamic_vix(self, paras, local_var_tree, prob0):
        time_to_maturities = local_var_tree.columns
        strikes = local_var_tree.index
        ncols = len(time_to_maturities)
        nrows = len(strikes)

        prob_tree = np.zeros((nrows, ncols))
        self._build_prob_tree(prob_tree, local_var_tree)

        #print local_var_tree.iloc[::-1]
        #print

        vix_est = 0.0
        for j in range(0, self.vix_steps):
            for i in range(ncols - j - 1, ncols + j):
                #print prob_tree[i][j], local_var_tree.iloc[i, j]
                vix_est += prob_tree[i][j] * local_var_tree.iloc[i, j]
        vix_est = np.sqrt(vix_est * self.deltat / 0.25) * 100.0
        self.dynamic_vix[time_to_maturities[0]].append((prob0, vix_est))

        #print vix_est

        if (ncols <= self.vix_steps + 1):
           return

        local_var_tree_up = local_var_tree.copy()
        local_var_tree_down = local_var_tree.copy()
        for j in range(0, ncols - 1):
            for i in range(ncols + j - 1, ncols - j - 2, -1):

                #print j ,i
                beta = paras['a'] * np.exp(-paras['b'] \
                    * (time_to_maturities[j] - time_to_maturities[0]) \
                    - paras['c'] * strikes[i])
                drift = (- beta ** 2 / 2.0) * self.deltat
                diffusion = beta * np.sqrt(self.deltat)
                alpha = np.log(2.0 / (np.exp(drift + diffusion) \
                    + np.exp(drift - diffusion))) / self.deltat

                drift += alpha * self.deltat

                local_var_tree_up.iloc[i, j] *= np.exp(drift + diffusion)
                local_var_tree_down.iloc[i, j] *= np.exp(drift - diffusion)

        prob_tree_up = np.zeros((nrows, ncols))
        prob_tree_down = np.zeros((nrows, ncols))
        self._build_prob_tree(prob_tree_up, local_var_tree_up)
        self._build_prob_tree(prob_tree_down, local_var_tree_down)

        local_var_tree_up_up = local_var_tree_up.iloc[2:, 1:]
        self._cal_dynamic_vix(paras, local_var_tree_up_up,
            prob_tree[ncols][1] * prob0 / 2.0)
        local_var_tree_up_mid = local_var_tree_up.iloc[1:(nrows - 1), 1:]
        self._cal_dynamic_vix(paras, local_var_tree_up_mid,
            prob_tree[ncols - 1][1] * prob0 / 2.0)
        local_var_tree_up_down = local_var_tree_up.iloc[:(nrows - 2), 1:]
        self._cal_dynamic_vix(paras, local_var_tree_up_down,
            prob_tree[ncols - 2][1] * prob0 / 2.0)

        local_var_tree_down_up = local_var_tree_down.iloc[2:, 1:]
        self._cal_dynamic_vix(paras, local_var_tree_down_up,
            prob_tree[ncols][1] * prob0 / 2.0)
        local_var_tree_down_mid = local_var_tree_down.iloc[1:(nrows - 1), 1:]
        self._cal_dynamic_vix(paras, local_var_tree_down_mid,
            prob_tree[ncols - 1][1] * prob0 / 2.0)
        local_var_tree_down_down = local_var_tree_down.iloc[:(nrows - 2), 1:]
        self._cal_dynamic_vix(paras, local_var_tree_down_down,
            prob_tree[ncols - 2][1] * prob0 / 2.0)

    def _build_prob_tree(self, prob_tree, local_var_tree):

        time_to_maturities = local_var_tree.columns
        strikes = local_var_tree.index
        ncols = len(time_to_maturities)
        nrows = len(strikes)

        yield_rates = [ self.yield_curve.zero_rate(item)
            for item in time_to_maturities ]
        dividend_rates = [ self.dividend_curve.zero_rate(item)
            for item in time_to_maturities ]

        prob_tree[ncols - 1, 0] = 1.0
        for j in range(0, ncols - 1):
            for i in range(ncols - j - 1, ncols + j):
                rates_diff = yield_rates[j] - dividend_rates[j]
                prob_up = local_var_tree.iloc[i, j] * self.A \
                    + self.B * rates_diff
                prob_down = prob_up - 2.0 * self.B * rates_diff
                prob_mid = 1.0 - prob_up - prob_down
                prob_tree[i + 1][j + 1] += prob_tree[i][j] * prob_up
                prob_tree[i][j + 1] += prob_tree[i][j] * prob_mid
                prob_tree[i - 1][j + 1] += prob_tree[i][j] * prob_down

    def cal_vix_option(self, time_to_maturity, strike):

        time_to_maturities = self.dynamic_vix.keys()
        prices = [ np.sum([item[0] * np.max((item[1] - strike, 0))
            for item in value ])
            for value in self.dynamic_vix.values() ]

        f = interp1d(time_to_maturities, prices)

        return f(time_to_maturity)


import numpy as np
import tensorflow as tf
import keras
import keras.backend as K
from tensorflow.keras import layers

# 判断输入数据格式，是channels_first还是channels_last
channel_axis = 1 if K.image_data_format() == "channels_first" else 3

# CAM
def channel_attention(input_xs, reduction_ratio=0.125):
    # get channel
    channel = int(input_xs.shape[channel_axis])
    maxpool_channel = layers.GlobalMaxPooling2D()(input_xs)
    maxpool_channel = layers.Reshape((1, 1, channel))(maxpool_channel)
    avgpool_channel = layers.GlobalAvgPool2D()(input_xs)
    avgpool_channel = layers.Reshape((1, 1, channel))(avgpool_channel)
    Dense_One = layers.Dense(units=int(channel * reduction_ratio), activation='relu', kernel_initializer='he_normal', use_bias=True, bias_initializer='zeros')
    Dense_Two = layers.Dense(units=int(channel), activation='relu', kernel_initializer='he_normal', use_bias=True, bias_initializer='zeros')
    # max path
    mlp_1_max = Dense_One(maxpool_channel)
    mlp_2_max = Dense_Two(mlp_1_max)
    mlp_2_max = layers.Reshape(target_shape=(1, 1, int(channel)))(mlp_2_max)
    # avg path
    mlp_1_avg = Dense_One(avgpool_channel)
    mlp_2_avg = Dense_Two(mlp_1_avg)
    mlp_2_avg = layers.Reshape(target_shape=(1, 1, int(channel)))(mlp_2_avg)
    channel_attention_feature = layers.Add()([mlp_2_max, mlp_2_avg])
    channel_attention_feature = layers.Activation('sigmoid')(channel_attention_feature)
    return layers.Multiply()([channel_attention_feature, input_xs])

# SAM
def spatial_attention(channel_refined_feature):
    maxpool_spatial = layers.Lambda(lambda x: K.max(x, axis=3, keepdims=True))(channel_refined_feature)
    avgpool_spatial = layers.Lambda(lambda x: K.mean(x, axis=3, keepdims=True))(channel_refined_feature)
    max_avg_pool_spatial = layers.Concatenate(axis=3)([maxpool_spatial, avgpool_spatial])
    return layers.Conv2D(filters=1, kernel_size=(3, 3), padding="same", activation='sigmoid', kernel_initializer='he_normal', use_bias=False)(max_avg_pool_spatial)

def cbam_module(input_xs, reduction_ratio=0.5):
    channel_refined_feature = channel_attention(input_xs, reduction_ratio=reduction_ratio)
    spatial_attention_feature = spatial_attention(channel_refined_feature)
    refined_feature = layers.Multiply()([channel_refined_feature, spatial_attention_feature])
    return layers.Add()([refined_feature, input_xs])


import sys
import os
import socket
import time
import numpy as np
import configparser

from PyQt5.QtCore import QThread
from PyQt5.QtGui import QFont
from PyQt5.QtGui import QImage
from PyQt5.QtGui import QPainter
from PyQt5.QtGui import QPen
from PyQt5.QtGui import QPixmap
from PyQt5.QtWidgets import *
from PyQt5.QtCore import QSize,Qt,pyqtSignal
from sort import Sort

global image_data
global age_info_str
global sk
global persons
global ageWindow
image_data = []
sk = socket.socket()
persons = {}
ageWindow = []

updateAgeTime = 4.0

# # categorized previous faces
# class Person():
#     def __init__(self, x, y, age):
#         self.centerX = x
#         self.centerY = y
#         self.age_list = [age]
#         self.age2show = age
#
#     def distance(self, x, y):
#         return np.square(self.centerX-x)+np.square(self.centerY-y)
#
#     def update(self, x, y, age):
#         self.centerX = x
#         self.centerY = y
#         self.age_list.append(age)
#         if self.age2show == -1:
#             self.age2show = age
#
#     def clear_buffer(self):
#         if len(self.age_list) == 0:
#             self.age2show = -1
#         else:
#             self.age2show = int(sum(self.age_list)/len(self.age_list))
#         self.age_list.clear()
#
# # uncategorized current faces
# class Face():
#     def __init__(self, x1, y1, x2, y2, age):
#         self.x1 = x1
#         self.y1 = y1
#         self.x2 = x2
#         self.y2 = y2
#         self.age = age
#
# def findMin(list, matched):
#     MIN = max(list)+1
#     index = -1
#     for i,val in enumerate(list):
#         if matched[i] == 0 and val < MIN:
#             index = i
#             MIN = val
#     return index
#
#
#
#
# def match_faces_persons(faces):
#     global persons
#     result = []
#     if len(faces) == 0:
#         return []
#     elif len(persons) == 0:
#         for face in faces:
#             persons.append(Person((face.x1+face.x2)/2, (face.y1+face.y2)/2, face.age))
#             result.append((face.x1,face.y1,face.x2,face.y2,face.age))
#     elif len(faces) <= len(persons):
#         matched = [0 for i in range(len(persons))]
#         for face in faces:
#             list = [persons[i].distance((face.x1+face.x2)/2, (face.y1+face.y2)/2) for i in range(len(persons))]
#             index = findMin(list, matched)
#             matched[index] = 1
#             persons[index].update((face.x1+face.x2)/2, (face.y1+face.y2)/2, face.age)
#             result.append((face.x1,face.y1,face.x2,face.y2,persons[index].age2show))
#     else:
#         matched = [0 for i in range(len(faces))]
#         for person in persons:
#             list = [person.distance((faces[i].x1+faces[i].x2)/2, (faces[i].y1+faces[i].y2)/2) for i in range(len(faces))]
#             index = findMin(list, matched)
#             matched[index] = 1
#             person.update((faces[index].x1+faces[index].x2)/2, (faces[index].y1+faces[index].y2)/2, faces[index].age)
#             result.append((faces[index].x1, faces[index].y1, faces[index].x2, faces[index].y2, person.age2show))
#         for i, face in enumerate(faces):
#             if matched[i] == 0:
#                 persons.append(Person((face.x1 + face.x2) / 2, (face.y1 + face.y2) / 2, face.age))
#                 result.append((face.x1, face.y1, face.x2, face.y2, face.age))
#     return result

def addMargin(x1,x2,y1,y2):
    h = y2-y1
    w = x2-x1
    x1_ = int(x1-w*0.2)
    if x1_<0:
        x1_=0
    x2_ = int(x2+w*0.2)
    if x2_>1280:
        x2_=1280
    y1_ = int(y1-h*0.2)
    if y1_<0:
        y1_=0
    y2_ = int(y2 + h * 0.4)
    if y2_ > 720:
        y2_ = 720
    return x1_,x2_,y1_,y2_

class Person():
    def __init__(self, face):
        self.x1 = face[0]
        self.y1 = face[1]
        self.x2 = face[2]
        self.y2 = face[3]
        self.age2show = int(face[4]+0.5)
        self.age_list = [face[4]]

    def update(self, face):
        self.x1 = face[0]
        self.y1 = face[1]
        self.x2 = face[2]
        self.y2 = face[3]
        self.age_list.append(face[4])

    def clear_buffer(self):
        self.age2show = int(sum(self.age_list)/len(self.age_list))
        self.age_list.clear()

class MainWindow(QMainWindow):

    def __init__(self):
        super().__init__()
        self.initUI()
        self.thread = Thread()
        self.thread.sinout.connect(self.slotDisplayImage)
        self.thread.start()
        self.time = 0
        self.timeCount = 0
        self.frame = 0
        self.mot_tracker = Sort(max_age=5)

    def initUI(self):
        self.setWindowTitle('Age Estimation')
        self.setGeometry(0, 0, 1280, 720)
        self.setFixedSize(QSize(1280, 720))

        self.image_label = QLabel(self)
        self.show()

    def slotDisplayImage(self):
        # time_st = time.time()
        self.frame += 1
        global socketEnable, image_data, persons, ageWindow, cnt
        # update time count
        costTime = time.time() - self.time
        self.timeCount += costTime
        #update fps
        self.time = time.time()
        fps = 1.0 / costTime
        # decode the frame
        image = QImage.fromData(image_data)
        pixmap = QPixmap.fromImage(image)
        #get current faces
        dets = []
        faces = []
        if ';' in age_info_str:
            for string in age_info_str.split(';')[0:-1]:
                x1 = int(string.split(',')[0])
                y1 = int(string.split(',')[1])
                x2 = int(string.split(',')[2])
                y2 = int(string.split(',')[3])
                age = float(string.split(',')[4])

                score = float(string.split(',')[5])
                x1, x2, y1, y2 = addMargin(x1, x2, y1, y2)
                dets.append([x1,y1,x2,y2,score])
                faces.append([x1, y1, x2, y2, age])

                # faces.append(Face(x1,y1,x2,y2,age))
        dets = np.asarray(dets)
        #get current frame results
        dets_ids, remove_ids = self.mot_tracker.update(dets)
        for d, id in dets_ids.items():
            if id in persons:
                persons[id].update(faces[d])
            else:
                persons[id] = Person(faces[d])
        for id in remove_ids:
            persons.pop(id)
        ids = dets_ids.values()

        # clear buffer after a period of time
        if self.timeCount > updateAgeTime:
            for id, person in persons.items():
                person.clear_buffer()
            self.timeCount = 0

        # set painter and draw fps
        painter = QPainter(pixmap)
        fpsFont = QFont()
        fpsFont.setPixelSize(20);
        fpsFont.setFamily("SimSun")
        painter.setFont(fpsFont)
        painter.setPen(QPen(Qt.black, 2, Qt.SolidLine))
        painter.drawText(pixmap.width() - 100, 40, "fps: {}".format(int(fps)))

        # set painter to draw age
        painter.setPen(QPen(Qt.yellow, 2, Qt.SolidLine))
        ageFont = QFont()
        ageFont.setPixelSize(60);
        ageFont.setFamily("Microsoft YaHei");
        painter.setFont(ageFont)

        if ids != []:
            # draw results
            # avoid results out of border
            for id in ids:
                x1 = persons[id].x1
                y1 = persons[id].y1
                x2 = persons[id].x2
                y2 = persons[id].y2

                age = persons[id].age2show
                painter.drawRect(x1, y1, x2 - x1, y2 - y1)
                if y1 > 100:
                    painter.drawText(x1, y1 - 10, str(age))
                else:
                    painter.drawText(x1, y2 + 60, str(age))
        painter.end()

        # draw image onto the window
        self.image_label.setGeometry(0, 0, pixmap.width(), pixmap.height())
        self.image_label.setPixmap(pixmap)
        # print('draw time: {}'.format(time.time()-time_st))

class Thread(QThread):
    sinout = pyqtSignal()
    def __init__(self):
        super().__init__()

    def run(self):
        # read config.conf
        # use absolute path only
        parentDirPath = os.path.split(os.path.realpath(__file__))[0]
        cf = configparser.ConfigParser()
        cf.read(os.path.join(parentDirPath, 'config.conf'))
        ip = cf.get("baseconf", "presenter_server_ip")
        port = int(cf.get("baseconf", "presenter_server_port"))
        print('Socket sever listen on {}:{}'.format(ip, port))
        address = (ip, port)
        sk.bind(address)
        while 1:
            # wait for client to establish connection
            sk.listen(1)
            print('waiting for client connection........ ')
            conn, addr = sk.accept()
            print("client connected.")

            while 1:
                received_length = 0
                age_info = b''
                disConnected = False
                while received_length < 1024:
                    r_data = conn.recv(1024)
                    # check for disconnection, the same below
                    if r_data==b'':
                        print('client disconnected.')
                        disConnected = True
                        break
                    received_length += len(r_data)
                    age_info += r_data
                if disConnected:
                    break

                tmp_age_info_str = str(age_info)[2:-1]
                print(tmp_age_info_str)
                image_length = int(tmp_age_info_str.split(':')[0])
                received_length = 0
                received_data = b''
                # print('actual:', image_length)
                conn.send(b'A')

                while received_length < image_length:
                    r_data = conn.recv(1024 * 1024)
                    if r_data==b'':
                        print('client disconnected.')
                        disConnected = True
                        break
                    received_length += len(r_data)
                    received_data += r_data
                if disConnected:
                    break
                # print('received:', received_length)
                conn.send(b'B')
                global image_data, age_info_str, pixmap
                image_data = received_data
                age_info_str = tmp_age_info_str.split(':')[1]
                self.sinout.emit()

if __name__ == '__main__':
    app = QApplication(sys.argv)
    ex = MainWindow()
    sys.exit(app.exec_())


import io
import glob
import hashlib
import json
import math
import os
import re

from flask          import abort, jsonify, redirect, request
from werkzeug.utils import secure_filename
from urllib.parse         import quote

from . import checksum
from . import file
from . import http

from .app            import app, db, models
from .sampleresolver import SampleResolver

PART_EXT = "part"
CHECKSUM_EXT = "checksum"

# ------------------------------------------------------------ api routes --- #

@app.route('/')
def index():
    return redirect('/index.html', http.HTTP_302_FOUND)

@app.route('/projects', methods=['GET'])
def get_projects():
    return jsonize(models.get_projects())

@app.route('/projects/<project>', methods=['GET'])
def get_project(project):
    return jsonize(models.get_project(obfuscated_id=as_id(project)))

@app.route('/projects/<project>/samples', methods=['GET'])
def get_project_samples(project):
    # Allow clients to search for a sample rather than having to retrieve all
    # of the samples for a project (even if the latter is arguably the more
    # RESTful approach).  This enables a URL of the form:
    #
    #   http://.../projects/qwErt/samples?q=P001-B009-...
    #
    # In the absence of the query parameter we simply return the entire
    # collection.
    # FIXME: This should support pagination!
    search_terms = request.args.get('q')
    project_id   = as_id(project)
    if search_terms is not None:
        return jsonize(
            SampleResolver().resolve(search_terms.split(' '), project_id))
    else:
        return jsonize(
            models.get_samples(obfuscated_id=as_id(project_id)))

@app.route('/projects/<project>/samples/<sample>', methods=['GET'])
def get_project_sample(project, sample):
    return jsonize(
        models.get_project_sample(
            {'obfuscated_id': as_id(project)},
            {'obfuscated_id': as_id(sample)}))

@app.route('/projects/<_>/samples/<sample>/stages', methods=['GET'])
def get_project_sample_stages(_, sample):
    (stages, token) = models.get_sample_stages(as_id(sample))
    return jsonize({'sample' : sample,
                    'stages' : stages,
                    'token'  : token})

@app.route('/projects/<_>/samples/<sample>/stages/<stage>', methods=['GET'])
def get_project_sample_stage(_, sample, stage):
    return jsonize(models.get_sample_stages(as_id(sample), as_id(stage)))

@app.route('/projects/<project>/samples/<sample>/stages/<stage>', methods=['PUT'])
def put_project_sample_stage(project, sample, stage):
    method = models.get_method(obfuscated_id=as_id(request.json['method']))
    annotation = request.json['annotation']
    token = stage
    rsp = jsonize(
        models.add_sample_stage(
            as_id(sample), as_id(method.obfuscated_id), annotation, token))
    return (rsp, http.HTTP_201_CREATED)

@app.route('/projects/<project>/samples/<sample>/stages/<stage>/files', methods=['GET'])
def get_sample_stage_files(project, sample, stage):
    return jsonize({'stage-id' : stage,
                    'files'    : models.get_files(sample_stage_id=as_id(stage), status=None)})

@app.route('/methods', methods=['GET'])
def get_methods():
    return jsonize(models.get_methods())

@app.route('/methods/<method>', methods=['GET'])
def get_method(method):
    return jsonize(models.get_method(obfuscated_id=as_id(method)))

@app.route('/prepare-multipart-upload', methods=['POST'])
def prepare_file_upload():
    pass

@app.route('/upload-part', methods=['POST'])
def upload_file():
    part = request.files['file']

    # part_number = request.form['part-number']
    part_number = request.form['resumableChunkNumber']
    # total_number_parts = request.form['total-parts']
    total_number_parts = request.form['resumableTotalChunks']

    # file_identifier = request.form['upload-id']
    file_identifier = request.form['resumableIdentifier']
    part_upload_dir = upload_dir(file_identifier)
    mkdirp(part_upload_dir)

    # Save the part into a file with a name of the form 00.part.  The number of
    # leading zeroes depends on the number of parts.
    fmtstr = "%%0%dd" % len(total_number_parts)
    part_filename = '.'.join([(fmtstr % int(part_number)), PART_EXT])
    part_filepath = os.path.join(part_upload_dir, part_filename)
    part.save(part_filepath)

    return json.dumps(
        {'identifier': file_identifier,
         'part': part_number,
         'total_parts': total_number_parts})

@app.route('/complete-multipart-upload', methods=['POST'])
def complete_file_upload():
    request_data    = json.loads(request.data)
    file_identifier = request_data['upload-id']
    file_name       = secure_filename(request_data['file-name'])
    project         = request_data['project']
    sample          = request_data['sample']
    sample_stage    = as_id(request_data['sample-stage'])

    req_checksum_value  = request_data['checksum-value']
    req_checksum_method = request_data['checksum-method']

    part_upload_dir = upload_dir(file_identifier)
    reconstituted_file_name = os.path.join(part_upload_dir, file_name)
    parts = sorted(glob.glob(os.path.join(part_upload_dir, '*.part')))

    part_digester = checksum.get_digester(req_checksum_method)
    with open(reconstituted_file_name, 'w') as target:
        def consume_part(data):
            target.write(data)
            part_digester.update(data)
        for part_name in parts:
            file.FileProcessor(part_name, consume_part).process()
    computed_checksum_value = part_digester.hexdigest()
    if computed_checksum_value != req_checksum_value:
        raise checksum.ChecksumMismatch(
            reconstituted_file_name,
            req_checksum_method,
            req_checksum_value,
            computed_checksum_value)

    with open('.'.join([reconstituted_file_name, CHECKSUM_EXT]), 'w') as cf:
        json.dump({'method': req_checksum_method,
                   'value' : req_checksum_value},
                  cf)

    logmsg = 'Successfully reconsitituted file: %s (%s=%s)' \
             % (reconstituted_file_name, req_checksum_method, req_checksum_value)
    app.logger.info(logmsg)
    models.add_file(
        os.path.relpath(
            reconstituted_file_name, app.config['UPLOAD_PATH']),
        sample_stage)

    return (json.dumps({'identifier' : file_identifier,
                        'file-name'  : file_name}),
            http.HTTP_202_ACCEPTED)

# -------------------------------------------------------- error handlers --- #

@app.errorhandler(checksum.ChecksumError)
def handle_unsupported_checksum_method(e):
    app.logger.error('Checksum error: %s' % e, exc_info=e)
    return (json.dumps({'status_code': e.status_code,
                        'message'    : e.message}),
            e.status_code)

@app.errorhandler(Exception)
def handle_unhandled_exception(e):
    app.logger.error('Unhandled exception: %s' % e, exc_info=e)
    raise e

# ------------------------------------------------------ data marshalling --- #

def as_id(resource_name):
    return resource_name.split('-')[0]

class DBModelJSONEncoder(json.JSONEncoder):

    BAD_URI_PAT  = re.compile("%.{2}|\/|_")
    COLLAPSE_PAT = re.compile("-{2,}")

    def __init__(self, **kw):
        super(DBModelJSONEncoder, self).__init__(**kw)

        # This is a terrible hack to make it possible to add context to the
        # stages IDs.  The relative ordering of stages is important, and from
        # this we infer the ordinal position values.  Note this that assumes
        # that a new encoder instance is used for each JSON document; do not
        # cache encoders!
        self.sample_stage_count = 0

    def _uri_name(self, obfuscated_id, name):
        """
        Convert the name of a resource (like a project or sample) into a
        URI-friendly form.  This function has strong opinions about what a
        "good" URI ID is, and will complain if the ID cannot be rendered in
        an acceptable form.  In particular, it should not contain any
        characters that need to be escaped; cf. `urllib.quote()`.
        """
        assert obfuscated_id is not None
        assert name is not None
        # URI-escape special characters
        new_name = quote(name.lower().replace(" ", "-"))
        # Convert escapes into underscores; i.e. foo%20bar -> foo-bar
        new_name = re.sub(self.BAD_URI_PAT, '-', new_name)
        # Collapse multiple consecutive hyphens; i.e. foo---bar -> foo-bar
        new_name = re.sub(self.COLLAPSE_PAT, '-', new_name)
        # concat with the ID and return
        return '-'.join([obfuscated_id, new_name])

    def _dictify(self, model, exclude={}):
        """
        Return a model as a dictionary. 'Private' attributes are removed and
        underscores are replaced with more hyphens in attribute names, making
        for more aesthetically pleasing HTTP data maps.
        """
        def tr(kv):
            return (kv[0].replace('_', '-'), kv[1])
        return dict(tr(kv) for kv in iter(list(model.__dict__.items()))
                    if (not (kv[0].startswith('_') or (kv[0] in exclude))))

    def strip_private_fields(self, d):
        del d['obfuscated-id']
        return d

    def _encodeProject(self, p):
        d = self._dictify(p)
        d['id'] = self._uri_name(d['obfuscated-id'], d['name'])
        return self.strip_private_fields(d)

    def _encodeMethod(self, m):
        d = self._dictify(m)
        d['id'] = self._uri_name(d['obfuscated-id'], d['name'])
        return self.strip_private_fields(d)

    def _encodeSample(self, s):
        d = self._dictify(s, {'project'})
        d['id'] = self._uri_name(d['obfuscated-id'], d['name'])
        d['project'] = self._uri_name(s.project.obfuscated_id, s.project.name)
        return self.strip_private_fields(d)

    def _encodeSampleStage(self, ss):
        d = self._dictify(ss, {'sample', 'method'})
        self.sample_stage_count += 1
        d['id'] = self._uri_name(d['obfuscated-id'], str(self.sample_stage_count))
        d['sample'] = self._uri_name(ss.sample.obfuscated_id, ss.sample.name)
        d['method'] = self._uri_name(ss.method.obfuscated_id, ss.method.name)
        return self.strip_private_fields(d)

    def _encodeSampleStageFile(self, ssf):
        def isready(ssf):
            return ssf.status in [models.FileStatus.archived,
                                  models.FileStatus.cleaned,
                                  models.FileStatus.complete]
        if isready(ssf):
            status = 'ready'
        else:
            status = 'processing'
        fname = os.path.basename(ssf.relative_target_path)
        return {'id'           : ssf.obfuscated_id + '-' + fname,
                'file'         : fname,
                'status'       : status,
                'mtime'        : ssf.modified_ts.isoformat(),
                'uri' : '/dl/' + ssf.relative_target_path}

    def _encodeModel(self, m):
        return self.strip_private_fields(self._dictify(m))

    def default(self, thing):
        if isinstance(thing, models.Project):
            return self._encodeProject(thing)
        if isinstance(thing, models.Method):
            return self._encodeMethod(thing)
        if isinstance(thing, models.Sample):
            return self._encodeSample(thing)
        if isinstance(thing, models.SampleStage):
            return self._encodeSampleStage(thing)
        if isinstance(thing, models.SampleStageFile):
            return self._encodeSampleStageFile(thing)
        if isinstance(thing, db.Model):
            return self._encodeModel(thing)

def jsonize(x):
    return json.dumps(x, cls=DBModelJSONEncoder)

# ----------------------------------------------------------- utility fns --- #

def mkdirp(p):
    if os.path.exists(p) and os.path.isdir(p):
        return False
    else:
        os.makedirs(p)
        return True

def upload_dir(p):
    if isinstance(p, str):
        add_path = (p,) # Turn the single element into a tuple that can be
                        # joined into a full path
    elif isinstance(p, (list, tuple)):
        add_path = p
    else:
        msg = 'I Don\'t know how to create an upload directory path from "%s", %s' \
              % (p, type(p))
        raise Exception(msg)
    full_path = (app.config['UPLOAD_PATH'],) + add_path
    return os.path.join(*full_path)

def hashfile(fn, hasher, blocksize=65536):
    with open(fn, 'rb') as afile:
        buf = afile.read(blocksize)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(blocksize)
    return hasher.hexdigest()

def hashstring(s, hasher):
    hasher.update(s)
    return hasher.hexdigest()


# Python bytecode 2.7 (decompiled from Python 2.7)
# Embedded file name: e:\jenkins\workspace\client_SERENITY\branches\release\SERENITY\eve\common\lib\jwt\__init__.py
from __future__ import unicode_literals
import base64
import binascii
import hashlib
import hmac
import sys
from datetime import datetime
from calendar import timegm
from collections import Mapping
try:
    import json
except ImportError:
    import simplejson as json

if sys.version_info >= (3, 0, 0):
    unicode = str
    basestring = str
__version__ = u'0.3.0'
__all__ = [u'encode', u'decode', u'DecodeError']

class DecodeError(Exception):
    pass

class ExpiredSignature(Exception):
    pass

class InvalidAudience(Exception):
    pass

class InvalidIssuer(Exception):
    pass

signing_methods = {u'none': lambda msg, key: '',
   u'HS256': lambda msg, key: hmac.new(key, msg, hashlib.sha256).digest(),u'HS384': lambda msg, key: hmac.new(key, msg, hashlib.sha384).digest(),u'HS512': lambda msg, key: hmac.new(key, msg, hashlib.sha512).digest()}
verify_methods = {u'HS256': lambda msg, key: hmac.new(key, msg, hashlib.sha256).digest(),
   u'HS384': lambda msg, key: hmac.new(key, msg, hashlib.sha384).digest(),u'HS512': lambda msg, key: hmac.new(key, msg, hashlib.sha512).digest()}

def prepare_HS_key(key):
    if not isinstance(key, basestring) and not isinstance(key, bytes):
        raise TypeError(u'Expecting a string- or bytes-formatted key.')
    if isinstance(key, unicode):
        key = key.encode(u'utf-8')
    return key

prepare_key_methods = {u'none': lambda key: None,
   u'HS256': prepare_HS_key,
   u'HS384': prepare_HS_key,
   u'HS512': prepare_HS_key
   }
try:
    from Crypto.Signature import PKCS1_v1_5
    from Crypto.Hash import SHA256
    from Crypto.Hash import SHA384
    from Crypto.Hash import SHA512
    from Crypto.PublicKey import RSA
    signing_methods.update({u'RS256': lambda msg, key: PKCS1_v1_5.new(key).sign(SHA256.new(msg)),
       u'RS384': lambda msg, key: PKCS1_v1_5.new(key).sign(SHA384.new(msg)),u'RS512': lambda msg, key: PKCS1_v1_5.new(key).sign(SHA512.new(msg))})
    verify_methods.update({u'RS256': lambda msg, key, sig: PKCS1_v1_5.new(key).verify(SHA256.new(msg), sig),
       u'RS384': lambda msg, key, sig: PKCS1_v1_5.new(key).verify(SHA384.new(msg), sig),u'RS512': lambda msg, key, sig: PKCS1_v1_5.new(key).verify(SHA512.new(msg), sig)})

    def prepare_RS_key(key):
        if isinstance(key, RSA._RSAobj):
            return key
        if isinstance(key, basestring):
            if isinstance(key, unicode):
                key = key.encode(u'utf-8')
            key = RSA.importKey(key)
        else:
            raise TypeError(u'Expecting a PEM- or RSA-formatted key.')
        return key

    prepare_key_methods.update({u'RS256': prepare_RS_key,
       u'RS384': prepare_RS_key,
       u'RS512': prepare_RS_key
       })
except ImportError:
    pass

try:
    import ecdsa
    from Crypto.Hash import SHA256
    from Crypto.Hash import SHA384
    from Crypto.Hash import SHA512
    signing_methods.update({u'ES256': lambda msg, key: key.sign(msg, hashfunc=hashlib.sha256, sigencode=ecdsa.util.sigencode_der),
       u'ES384': lambda msg, key: key.sign(msg, hashfunc=hashlib.sha384, sigencode=ecdsa.util.sigencode_der),u'ES512': lambda msg, key: key.sign(msg, hashfunc=hashlib.sha512, sigencode=ecdsa.util.sigencode_der)})
    verify_methods.update({u'ES256': lambda msg, key, sig: key.verify(sig, msg, hashfunc=hashlib.sha256, sigdecode=ecdsa.util.sigdecode_der),
       u'ES384': lambda msg, key, sig: key.verify(sig, msg, hashfunc=hashlib.sha384, sigdecode=ecdsa.util.sigdecode_der),u'ES512': lambda msg, key, sig: key.verify(sig, msg, hashfunc=hashlib.sha512, sigdecode=ecdsa.util.sigdecode_der)})

    def prepare_ES_key(key):
        if isinstance(key, ecdsa.SigningKey) or isinstance(key, ecdsa.VerifyingKey):
            return key
        if isinstance(key, basestring):
            if isinstance(key, unicode):
                key = key.encode(u'utf-8')
            try:
                key = ecdsa.VerifyingKey.from_pem(key)
            except ecdsa.der.UnexpectedDER:
                try:
                    key = ecdsa.SigningKey.from_pem(key)
                except:
                    raise

        else:
            raise TypeError(u'Expecting a PEM-formatted key.')
        return key

    prepare_key_methods.update({u'ES256': prepare_ES_key,
       u'ES384': prepare_ES_key,
       u'ES512': prepare_ES_key
       })
except ImportError:
    pass

def constant_time_compare(val1, val2):
    if len(val1) != len(val2):
        return False
    result = 0
    if sys.version_info >= (3, 0, 0):
        for x, y in zip(val1, val2):
            result |= x ^ y

    else:
        for x, y in zip(val1, val2):
            result |= ord(x) ^ ord(y)

    return result == 0

def base64url_decode(input):
    rem = len(input) % 4
    if rem > 0:
        input += '=' * (4 - rem)
    return base64.urlsafe_b64decode(input)

def base64url_encode(input):
    return base64.urlsafe_b64encode(input).replace('=', '')

def header(jwt):
    header_segment = jwt.split('.', 1)[0]
    try:
        header_data = base64url_decode(header_segment).decode(u'utf-8')
        return json.loads(header_data)
    except (
     ValueError, TypeError):
        raise DecodeError(u'Invalid header encoding')

def encode(payload, key, algorithm=u'HS256', headers=None):
    segments = []
    if algorithm is None:
        algorithm = u'none'
    if not isinstance(payload, Mapping):
        raise TypeError(u'Expecting a mapping object, as json web token onlysupport json objects.')
    header = {u'typ': u'JWT',u'alg': algorithm}
    if headers:
        header.update(headers)
    json_header = json.dumps(header, separators=(u',', u':')).encode(u'utf-8')
    segments.append(base64url_encode(json_header))
    for time_claim in [u'exp', u'iat', u'nbf']:
        if isinstance(payload.get(time_claim), datetime):
            payload[time_claim] = timegm(payload[time_claim].utctimetuple())

    json_payload = json.dumps(payload, separators=(u',', u':')).encode(u'utf-8')
    segments.append(base64url_encode(json_payload))
    signing_input = '.'.join(segments)
    try:
        key = prepare_key_methods[algorithm](key)
        signature = signing_methods[algorithm](signing_input, key)
    except KeyError:
        raise NotImplementedError(u'Algorithm not supported')

    segments.append(base64url_encode(signature))
    return '.'.join(segments)

def decode(jwt, key=u'', verify=True, **kwargs):
    payload, signing_input, header, signature = load(jwt)
    if verify:
        verify_expiration = kwargs.pop(u'verify_expiration', True)
        leeway = kwargs.pop(u'leeway', 0)
        verify_signature(payload, signing_input, header, signature, key, verify_expiration, leeway, **kwargs)
    return payload

def load(jwt):
    if isinstance(jwt, unicode):
        jwt = jwt.encode(u'utf-8')
    try:
        signing_input, crypto_segment = jwt.rsplit('.', 1)
        header_segment, payload_segment = signing_input.split('.', 1)
    except ValueError:
        raise DecodeError(u'Not enough segments')

    try:
        header_data = base64url_decode(header_segment)
    except (TypeError, binascii.Error):
        raise DecodeError(u'Invalid header padding')

    try:
        header = json.loads(header_data.decode(u'utf-8'))
    except ValueError as e:
        raise DecodeError(u'Invalid header string: %s' % e)

    try:
        payload_data = base64url_decode(payload_segment)
    except (TypeError, binascii.Error):
        raise DecodeError(u'Invalid payload padding')

    try:
        payload = json.loads(payload_data.decode(u'utf-8'))
    except ValueError as e:
        raise DecodeError(u'Invalid payload string: %s' % e)

    try:
        signature = base64url_decode(crypto_segment)
    except (TypeError, binascii.Error):
        raise DecodeError(u'Invalid crypto padding')

    return (payload, signing_input, header,
     signature)

def verify_signature(payload, signing_input, header, signature, key=u'', verify_expiration=True, leeway=0, **kwargs):
    try:
        algorithm = header[u'alg'].upper()
        key = prepare_key_methods[algorithm](key)
        if algorithm.startswith(u'HS'):
            expected = verify_methods[algorithm](signing_input, key)
            if not constant_time_compare(signature, expected):
                raise DecodeError(u'Signature verification failed')
        elif not verify_methods[algorithm](signing_input, key, signature):
            raise DecodeError(u'Signature verification failed')
    except KeyError:
        raise DecodeError(u'Algorithm not supported')

    if u'nbf' in payload and verify_expiration:
        utc_timestamp = timegm(datetime.utcnow().utctimetuple())
        if payload[u'nbf'] > utc_timestamp + leeway:
            raise ExpiredSignature(u'Signature not yet valid')
    if u'exp' in payload and verify_expiration:
        utc_timestamp = timegm(datetime.utcnow().utctimetuple())
        if payload[u'exp'] < utc_timestamp - leeway:
            raise ExpiredSignature(u'Signature has expired')
    audience = kwargs.get(u'audience')
    if audience:
        if isinstance(audience, list):
            audiences = audience
        else:
            audiences = [audience]
        if payload.get(u'aud') not in audiences:
            raise InvalidAudience(u'Invalid audience')
    issuer = kwargs.get(u'issuer')
    if issuer:
        if payload.get(u'iss') != issuer:
            raise InvalidIssuer(u'Invalid issuer')


import pymqr

import datetime
@pymqr.documents.EmbededDocument()
class EmpWorking(object):
    """
    The information of an Employee's Working
    """
    def __init__(self):
        self.rec_id = (str, True)
        self.employee_code = (str, True)
        self.appoint = (int, True)
        self.effect_date = (datetime.datetime, True)
        self.begin_date = (datetime.datetime, True)
        self.end_date = (datetime.datetime)
        self.decision_no = (str, True)
        self.signed_date = (datetime.datetime)
        self.signer_code = (str)
        self.note = (str)
        self.task = (str)
        self.reason = (str)
        self.department_code = (str)
        self.job_pos_code = (str)
        self.job_w_code = (str)
        self.emp_type_code = (str)
        self.region_code = (str)
        self.department_code_old = (str)
        self.job_pos_code_old = (str)
        self.job_w_code_old = (str)
        self.emp_type_code_old = (str)
        self.region_code_old = (str)
        self.province_code = (str)


import asyncio
import random
from asyncio import Future
from typing import Tuple
from wsrpc import rpc, Service

import stephanie
import bob

class AliceService(Service):

    def __init__(self):
        super().__init__("localhost", 6789)

        a = 5  # Secret number here

        self.alice_r = random.randint(0, 100)
        self.a_0 = a - self.alice_r
        self.a_1 = self.alice_r

        self.b_0 = Future()
        self.s_0 = Future()
        self.t_0 = Future()
        self.st_0 = Future()
        self.alpha = Future()
        self.beta = Future()

    @rpc
    async def set_daniel_shares(self, shares: Tuple[int, int, int]) -> None:
        self.s_0.set_result(shares[0])
        self.t_0.set_result(shares[1])
        self.st_0.set_result(shares[2])

    @rpc
    async def set_stephanie_share(self, share: int) -> None:
        self.b_0.set_result(share)

    @rpc
    async def set_alpha_beta(self, alpha_beta: Tuple[int, int]) -> None:
        self.alpha.set_result(alpha_beta[0])
        self.beta.set_result(alpha_beta[1])

    async def z_0(self) -> int:
        await stephanie.StephanieService().set_alice_share(self.a_1)

        alpha_0 = self.a_0 - await self.s_0
        beta_0 = await self.b_0 - await self.t_0

        await stephanie.StephanieService().set_alpha_beta_shares((alpha_0, beta_0))

        return await self.alpha * await self.beta + \
               await self.alpha * await self.t_0 + \
               await self.beta * await self.s_0 + await self.st_0

async def main():
    service = AliceService()
    await service.start()
    await bob.BobService().set_alice_value(await service.z_0())

if __name__ == "__main__":
    asyncio.get_event_loop().run_until_complete(main())


from django.conf.urls import url, include
from rest_framework.routers import DefaultRouter
from api import views

router = DefaultRouter()
router.register(r'matches', views.MatchViewSet)
#router.register(r'users', views.UserViewSet)
router.register(r'players', views.PlayerViewSet)

urlpatterns = [
    url(r'^', include(router.urls)),
]

urlpatterns += [
    url(r'^api-auth/', include('rest_framework.urls')),
]


from sklearn import datasets
from sklearn.svm import SVC
import numpy as np
import matplotlib.pyplot as plt

iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features.
y = iris.target

clf = SVC()
clf.fit(X, y)

test = np.array([[5.0, 3.0], [4.8, 3.2], [6.1, 3.1], [6.9, 2.9]])
#print(clf.predict(test))

x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5

h = .02

plt.figure(2, figsize=(8, 6))
plt.clf()
plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Paired)
plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max)

xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
plt.contourf(xx, yy, Z, cmap=plt.cm.coolwarm, alpha=0.8)

plt.xlabel('Sepal length')
plt.ylabel('Sepal width')
#plt.legend(loc='upper left')

plt.show()

X2 = iris.data[:, [1, 3]]  # we only take the first two features.
y2 = iris.target

clf2= SVC()
clf2.fit(X2, y2)

test2 = np.array([[3.2, 0.3], [2.8, 1.2], [2.1, 2.1], [2.9, 1.9]])
#print(clf2.predict(test2))


import logging
import copy
from django.dispatch import receiver
from hs_core.signals import pre_create_resource
from ref_ts.models import RefTimeSeriesResource
from . import ts_utils

logger = logging.getLogger(__name__)

@receiver(pre_create_resource, sender=RefTimeSeriesResource)
def ref_time_series_describe_resource_trigger(sender, **kwargs):
    url_key = kwargs['url_key']
    if url_key is not None:
        # the call is from UI: redirect to render custom create-resource template
        page_url_dict = kwargs['page_url_dict']
        page_url_dict[url_key] = "pages/create-ref-time-series.html"
    else:
        # the call is from rest api
        metadata = kwargs['metadata']
        try:
            reference_url_dict = None
            site_dict = None
            variable_dict = None
            remove_list = []
            for metadata_dict in metadata:
                if "referenceurl" in metadata_dict:
                    reference_url_dict = copy.copy(metadata_dict["referenceurl"])
                    remove_list.append(metadata_dict)
                elif "site" in metadata_dict:
                    site_dict = copy.copy(metadata_dict["site"])
                    remove_list.append(metadata_dict)
                elif "variable" in metadata_dict:
                    variable_dict = copy.copy(metadata_dict["variable"])
                    remove_list.append(metadata_dict)
            # remove the refts-specific metadata items users pass in
            # as they will be re-extracted later
            # keep core metadata items in "metadata" list unchanged
            for remove_item in remove_list:
                metadata.remove(remove_item)
            if reference_url_dict:
                ref_url = reference_url_dict["value"]
                ref_type = reference_url_dict["type"]
                site_code = None
                variable_code = None
                if site_dict:
                    site_code = site_dict["code"]
                if variable_dict:
                    variable_code = variable_dict["code"]
                if ref_type == "soap" and (site_code is None or variable_code is None):
                    raise Exception("site_code and variable_code should be provided with soap url")
                ts_dict = ts_utils.QueryHydroServerGetParsedWML(service_url=ref_url,
                                                                soap_or_rest=ref_type,
                                                                site_code=site_code,
                                                                variable_code=variable_code)
                ts_utils.prepare_metadata_list(metadata=metadata,
                                               ts_dict=ts_dict,
                                               url=ref_url,
                                               reference_type=ref_type)
            else:
                raise Exception("Missing parameter 'referenceurl' in request")

        except Exception as ex:
            logger.exception("Failed to create refts res from rest api: " + ex.message)
            raise ex