from __future__ import absolute_import

import contextlib
import anydbm
from itertools import chain
import re

from mercurial import (
    bookmarks,
    cmdutil,
    commands,
    dagop,
    error,
    extensions,
    graphmod,
    obsutil,
    phases,
    registrar,
    revlog,
    revset,
    revsetlang,
    scmutil,
    smartset,
    templatekw,
    util,
)
from mercurial import node as nodemod
from mercurial.i18n import _

cmdtable = {}
command = registrar.command(cmdtable)
testedwith = 'ships-with-fb-hgext'
commit_info = False
hiddenchanges = 0

# Remove unsupported --limit option.
logopts = [opt for opt in commands.logopts if opt[1] != "limit"]

@contextlib.contextmanager
def ancestorcache(path):
    # simple cache to speed up revlog.ancestors
    try:
        db = anydbm.open(path, 'c')
    except anydbm.error:
        # database locked, fail gracefully
        yield
    else:
        def revlogancestor(orig, self, a, b):
            key = a + b
            try:
                return db[key]
            except KeyError:
                result = orig(self, a, b)
                db[key] = result
                return result
        extensions.wrapfunction(revlog.revlog, 'ancestor', revlogancestor)
        try:
            yield
        finally:
            extensions.unwrapfunction(revlog.revlog, 'ancestor', revlogancestor)
            try:
                db.close()
            except Exception:
                # database corruption, we just nuke the database
                util.tryunlink(path)

def _drawendinglines(orig, lines, extra, edgemap, seen):
    # if we are going to have only one single column, draw the missing '|'s
    # and restore everything to normal. see comment in 'ascii' below for an
    # example of what will be changed. note: we do not respect 'graphstyle'
    # but always draw '|' here, for simplicity.
    if len(seen) == 1 or any(l[0:2] != [' ', ' '] for l in lines):
        # draw '|' from bottom to top in the 1st column to connect to
        # something, like a '/' in the 2nd column, or a '+' in the 1st column.
        for line in reversed(lines):
            if line[0:2] != [' ', ' ']:
                break
            line[0] = '|'
        # undo the wrapfunction
        extensions.unwrapfunction(graphmod, '_drawendinglines',
                                  _drawendinglines)
        # restore the space to '|'
        for k, v in edgemap.iteritems():
            if v == ' ':
                edgemap[k] = '|'
    orig(lines, extra, edgemap, seen)

def uisetup(ui):
    # Hide output for fake nodes
    def show(orig, self, ctx, copies, matchfn, props):
        if ctx.node() == "...":
            self.ui.write('\n\n\n')
            return
        res = orig(self, ctx, copies, matchfn, props)

        if commit_info and ctx == self.repo['.']:
            changes = ctx.p1().status(ctx)
            prefixes = ['M', 'A', 'R', '!', '?', 'I', 'C']
            for prefix, change in zip(prefixes, changes):
                for fname in change:
                    self.ui.write(' {0} {1}\n'.format(prefix, fname))
            self.ui.write('\n')
        return res

    extensions.wrapfunction(cmdutil.changeset_printer, '_show', show)
    extensions.wrapfunction(cmdutil.changeset_templater, '_show', show)

    def ascii(orig, ui, state, type, char, text, coldata):
        if type == 'F':
            # the fake node is used to move draft changesets to the 2nd column.
            # there can be at most one fake node, which should also be at the
            # top of the graph.
            # we should not draw the fake node and its edges, so change its
            # edge style to a space, and return directly.
            # these are very hacky but it seems to work well and it seems there
            # is no other easy choice for now.
            edgemap = state['edges']
            for k in edgemap.iterkeys():
                edgemap[k] = ' '
            # also we need to hack _drawendinglines to draw the missing '|'s:
            #    (before)      (after)
            #     o draft       o draft
            #    /             /
            #                 |
            #   o             o
            extensions.wrapfunction(graphmod, '_drawendinglines',
                                    _drawendinglines)
            return
        orig(ui, state, type, char, text, coldata)

    extensions.wrapfunction(graphmod, 'ascii', ascii)

    revset.symbols['smartlog'] = smartlogrevset
    revset.safesymbols.add('smartlog')

templatekeyword = registrar.templatekeyword()

@templatekeyword('singlepublicsuccessor')
def singlepublicsuccessor(repo, ctx, templ, **args):
    """String. Get a single public successor for a
    given node.  If there's none or more than one, return empty string.
    This is intended to be used for "Landed as" marking
    in `hg sl` output."""
    successorssets = obsutil.successorssets(repo, ctx.node())
    unfiltered = repo.unfiltered()
    ctxs = (unfiltered[n] for n in chain.from_iterable(successorssets))
    public = (c.hex() for c in ctxs if not c.mutable() and c != ctx)
    first = next(public, '')
    second = next(public, '')

    return '' if first and second else first

@templatekeyword('rebasesuccessors')
def rebasesuccessors(repo, ctx, **args):
    """Return all of the node's successors created as a result of rebase"""
    rsnodes = list(modifysuccessors(ctx, 'rebase'))
    return templatekw.showlist('rebasesuccessor', rsnodes, args)

@templatekeyword('amendsuccessors')
def amendsuccessors(repo, ctx, **args):
    """Return all of the node's successors created as a result of amend"""
    asnodes = list(modifysuccessors(ctx, 'amend'))
    return templatekw.showlist('amendsuccessor', asnodes, args)

@templatekeyword('splitsuccessors')
def splitsuccessors(repo, ctx, **args):
    """Return all of the node's successors created as a result of split"""
    asnodes = list(modifysuccessors(ctx, 'split'))
    return templatekw.showlist('splitsuccessor', asnodes, args)

@templatekeyword('foldsuccessors')
def foldsuccessors(repo, ctx, **args):
    """Return all of the node's successors created as a result of fold"""
    asnodes = list(modifysuccessors(ctx, 'fold'))
    return templatekw.showlist('foldsuccessor', asnodes, args)

@templatekeyword('histeditsuccessors')
def histeditsuccessors(repo, ctx, **args):
    """Return all of the node's successors created as a result of
       histedit
    """
    asnodes = list(modifysuccessors(ctx, 'histedit'))
    return templatekw.showlist('histeditsuccessor', asnodes, args)

@templatekeyword('undosuccessors')
def undosuccessors(repo, ctx, **args):
    """Return all of the node's successors created as a result of undo"""
    asnodes = list(modifysuccessors(ctx, 'undo'))
    return templatekw.showlist('undosuccessor', asnodes, args)

def successormarkers(ctx):
    for data in ctx.repo().obsstore.successors.get(ctx.node(), ()):
        yield obsutil.marker(ctx.repo(), data)

def modifysuccessors(ctx, operation):
    """Return all of the node's successors which were created as a result
    of a given modification operation"""
    repo = ctx.repo().filtered('visible')
    for m in successormarkers(ctx):
        if m.metadata().get('operation') == operation:
            for node in m.succnodes():
                try:
                    repo[node]
                except Exception:
                    # filtered or unknown node
                    pass
                else:
                    yield nodemod.hex(node)

def sortnodes(nodes, parentfunc, masters):
    """Topologically sorts the nodes, using the parentfunc to find
    the parents of nodes.  Given a topological tie between children,
    any node in masters is chosen last."""
    nodes = set(nodes)
    childmap = {}
    parentmap = {}
    roots = []

    # Build a child and parent map
    for n in nodes:
        parents = [p for p in parentfunc(n) if p in nodes]
        parentmap[n] = set(parents)
        for p in parents:
            childmap.setdefault(p, set()).add(n)
        if not parents or (len(parents) == 1 and parents[0] == -1) and n != -1:
            roots.append(n)

    def childsortkey(x):
        # Process children in the master line last. This makes them always
        # appear on the left side of the dag, resulting in a nice straight
        # master line in the ascii output. Otherwise show the oldest first, so
        # the graph is approximately in chronological order.
        return (x in masters, x)

    # Process roots, adding children to the queue as they become roots
    results = []
    while roots:
        n = roots.pop(0)
        results.append(n)
        if n in childmap:
            children = list(childmap[n])
            # reverse=True here because we insert(0) below, resulting
            # in a reversed insertion of the children.
            children = sorted(children, reverse=True, key=childsortkey)
            for c in children:
                childparents = parentmap[c]
                childparents.remove(n)
                if len(childparents) == 0:
                    # insert at the beginning, that way child nodes
                    # are likely to be output immediately after their
                    # parents.
                    roots.insert(0, c)

    return results

def getdag(ui, repo, revs, master):

    # Fake ctx that we stick in the dag so we can special case it later
    class fakectx(object):
        def __init__(self, rev):
            self._rev = rev
        def node(self):
            return "..."
        def obsolete(self):
            return False
        def phase(self):
            return None
        def rev(self):
            return self._rev
        def files(self):
            return []
        def closesbranch(self):
            return False

    knownrevs = set(revs)
    gpcache = {}
    results = []

    # we store parents together with the parent type information
    # but sometimes we need just a list of parents
    # [(a,b), (c,d), (e,f)] => [b, d, f]
    def unzip(parents):
        if parents:
            return list(zip(*parents)[1])
        else:
            return list()

    # For each rev we need to show, compute it's parents in the dag.
    # If we have to reach for a grandparent, insert a fake node so we
    # can show '...' in the graph.
    # Use 'reversed' to start at the lowest commit so fake nodes are
    # placed at their lowest possible positions.
    for rev in reversed(revs):
        ctx = repo[rev]
        # Parents in the dag
        parents = sorted(set([(graphmod.PARENT, p.rev()) for p in ctx.parents()
                              if p.rev() in knownrevs]))
        # Parents not in the dag
        mpars = [p.rev() for p in ctx.parents() if
                 p.rev() != nodemod.nullrev and p.rev() not in unzip(parents)]

        for mpar in mpars:
            gp = gpcache.get(mpar)
            if gp is None:
                gp = gpcache[mpar] = dagop.reachableroots(
                    repo, smartset.baseset(revs), [mpar])
            if not gp:
                parents.append((graphmod.MISSINGPARENT, mpar))
            else:
                gp = [g for g in gp if g not in unzip(parents)]
                for g in gp:
                    parents.append((graphmod.GRANDPARENT, g))

        results.append((ctx.rev(), 'C', ctx, parents))

    # Compute parent rev->parents mapping
    lookup = {}
    for r in results:
        lookup[r[0]] = unzip(r[3])

    def parentfunc(node):
        return lookup.get(node, [])

    # Compute the revs on the master line. We use this for sorting later.
    masters = set()
    queue = [master]
    while queue:
        m = queue.pop()
        if m not in masters:
            masters.add(m)
            queue.extend(lookup.get(m, []))

    # Topologically sort the noderev numbers. Note: unlike the vanilla
    # topological sorting, we move master to the top.
    order = sortnodes([r[0] for r in results], parentfunc, masters)
    order = dict((e[1], e[0]) for e in enumerate(order))

    # Sort the actual results based on their position in the 'order'
    try:
        results.sort(key=lambda x: order[x[0]], reverse=True)
    except ValueError:  # Happened when 'order' is empty
        msg = _('note: smartlog encountered an error\n')
        hint = _('(so the sorting might be wrong.\n\n)')
        ui.warn(msg)
        ui.warn(hint)
        results.reverse()

    # indent the top non-public stack
    if ui.configbool('smartlog', 'indentnonpublic', False):
        rev, ch, ctx, parents = results[0]
        if ctx.phase() != phases.public:
            # find a public parent and add a fake node, so the non-public nodes
            # will be shown in the non-first column
            prev = None
            for i in xrange(1, len(results)):
                pctx = results[i][2]
                if pctx.phase() == phases.public:
                    prev = results[i][0]
                    break
            # append the fake node to occupy the first column
            if prev:
                fakerev = rev + 1
                results.insert(0, (fakerev, 'F', fakectx(fakerev),
                                   [('P', prev)]))

    return results

def _masterrevset(ui, repo, masterstring):
    """
    Try to find the name of ``master`` -- usually a bookmark.

    Defaults to the last public revision, if no suitable local or remote
    bookmark is found.
    """

    if not masterstring:
        masterstring = ui.config('smartlog', 'master')

    if masterstring:
        return masterstring

    names = set(bookmarks.bmstore(repo).keys())
    if util.safehasattr(repo, 'names') and 'remotebookmarks' in repo.names:
        names.update(set(repo.names['remotebookmarks'].listnames(repo)))

    for name in _reposnames(ui):
        if name in names:
            return name

    return 'last(public())'

def _reposnames(ui):
    # '' is local repo. This also defines an order precedence for master.
    repos = ui.configlist('smartlog', 'repos', ['', 'remote/', 'default/'])
    names = ui.configlist('smartlog', 'names', ['@', 'master', 'stable'])

    for repo in repos:
        for name in names:
            yield repo + name

def _masterrev(repo, masterrevset):
    try:
        master = scmutil.revsingle(repo, masterrevset)
    except error.RepoLookupError:
        master = scmutil.revsingle(repo, _masterrevset(repo.ui, repo, ''))
    except error.Abort:  # empty revision set
        return None

    if master:
        return master.rev()
    return None

def smartlogrevset(repo, subset, x):
    """``smartlog([master], [recentdays=N])``
    Changesets relevent to you.

    'master' is the head of the public branch.
    Unnamed heads will be hidden unless it's within 'recentdays'.
    """

    args = revset.getargsdict(x, 'smartlogrevset', 'master recentdays')
    if 'master' in args:
        masterstring = revsetlang.getstring(args['master'],
                                            _('master must be a string'))
    else:
        masterstring = ''

    recentdays = revsetlang.getinteger(args.get('recentdays'),
                                       _("recentdays should be int"), -1)

    revs = set()
    heads = set()

    rev = repo.changelog.rev
    branchinfo = repo.changelog.branchinfo
    ancestor = repo.changelog.ancestor
    node = repo.changelog.node
    parentrevs = repo.changelog.parentrevs

    books = bookmarks.bmstore(repo)
    ignore = re.compile(repo.ui.config('smartlog',
                                       'ignorebookmarks',
                                       '!'))
    for b in books:
        if not ignore.match(b):
            heads.add(rev(books[b]))

    # add 'interesting' remote bookmarks as well
    remotebooks = set()
    if util.safehasattr(repo, 'names') and 'remotebookmarks' in repo.names:
        ns = repo.names['remotebookmarks']
        remotebooks = set(ns.listnames(repo))
        for name in _reposnames(repo.ui):
            if name in remotebooks:
                heads.add(rev(ns.namemap(repo, name)[0]))

    heads.update(repo.revs('.'))

    global hiddenchanges
    headquery = 'head() & branch(.)'
    if remotebooks:
        # When we have remote bookmarks, only show draft heads, since public
        # heads should have a remote bookmark indicating them. This allows us
        # to force push server bookmarks to new locations, and not have the
        # commits clutter the user's smartlog.
        headquery = 'draft() &' + headquery

    allheads = set(repo.revs(headquery))
    if recentdays >= 0:
        recentquery = revsetlang.formatspec('%r & date(-%d)', headquery,
                                            recentdays)
        recentrevs = set(repo.revs(recentquery))
        hiddenchanges += len(allheads - heads) - len(recentrevs - heads)
        heads.update(recentrevs)
    else:
        heads.update(allheads)

    branches = set()
    for head in heads:
        branches.add(branchinfo(head)[0])

    masterrevset = _masterrevset(repo.ui, repo, masterstring)
    masterrev = _masterrev(repo, masterrevset)

    if masterrev is None:
        masterbranch = None
    else:
        masterbranch = branchinfo(masterrev)[0]

    for branch in branches:
        if branch != masterbranch:
            try:
                rs = 'first(reverse(branch("%s")) & public())' % branch
                branchmaster = repo.revs(rs).first()
                if branchmaster is None:
                    # local-only (draft) branch
                    rs = 'branch("%s")' % branch
                    branchmaster = repo.revs(rs).first()
            except Exception:
                branchmaster = repo.revs('tip').first()
        else:
            branchmaster = masterrev

        # Find all draft ancestors and latest public ancestor of heads
        # that are not in master.
        # We don't want to draw all public commits because there can be too
        # many of them.
        # Don't use revsets, they are too slow
        for head in heads:
            if branchinfo(head)[0] != branch:
                continue
            anc = rev(ancestor(node(head), node(branchmaster)))
            queue = [head]
            while queue:
                current = queue.pop(0)
                if current not in revs:
                    revs.add(current)
                    # stop as soon as we find public commit
                    ispublic = repo[current].phase() == phases.public
                    if current != anc and not ispublic:
                        parents = parentrevs(current)
                        for p in parents:
                            if p > anc:
                                queue.append(p)

        # add context: master, current commit, and the common ancestor
        revs.add(branchmaster)

        # get common branch ancestor
        if branch != masterbranch:
            anc = None
            for r in revs:
                if branchinfo(r)[0] != branch:
                    continue
                if anc is None:
                    anc = r
                else:
                    anc = rev(ancestor(node(anc), node(r)))
            if anc:
                revs.add(anc)

    return subset & revs

@command('^smartlog|slog', [
    ('', 'template', '', _('display with template'), _('TEMPLATE')),
    ('', 'master', '', _('master bookmark'), ''),
    ('r', 'rev', [], _('show the specified revisions or range'), _('REV')),
    ('', 'all', False, _('don\'t hide old local changesets'), ''),
    ('', 'commit-info', False, _('show changes in current changeset'), ''),
] + logopts, _('hg smartlog|slog'))
def smartlog(ui, repo, *pats, **opts):
    '''displays the graph of changesets that are relevant to you

Includes:

- Your bookmarks
- The @ or master bookmark (or tip if no bookmarks present).
- Your local heads that don't have bookmarks.

Excludes:

- All changesets under @/master/tip that aren't related to your changesets.
- Your local heads that are older than 2 weeks.
    '''
    if ui.configbool('smartlog', 'useancestorcache'):
        with ancestorcache(repo.vfs.join('cache/smartlog-ancestor')):
            return _smartlog(ui, repo, *pats, **opts)
    else:
        return _smartlog(ui, repo, *pats, **opts)

def _smartlog(ui, repo, *pats, **opts):
    masterstring = opts.get('master')
    masterrevset = _masterrevset(ui, repo, masterstring)

    revs = set()

    global hiddenchanges
    hiddenchanges = 0

    global commit_info
    commit_info = opts.get('commit_info')

    if not opts.get('rev'):
        if opts.get('all'):
            recentdays = -1
        else:
            recentdays = 14
        revstring = revsetlang.formatspec('smartlog(%s, %s)', masterrevset,
                                          recentdays)
        revs.update(scmutil.revrange(repo, [revstring]))
        masterrev = _masterrev(repo, masterrevset)
    else:
        revs.update(scmutil.revrange(repo, opts.get('rev')))
        try:
            masterrev = repo.revs('.').first()
        except error.RepoLookupError:
            masterrev = revs[0]

    if -1 in revs:
        revs.remove(-1)

    # It's important that these function caches come after the revsets above,
    # because the revsets may cause extra nodes to become visible, which in
    # turn invalidates the changelog instance.
    rev = repo.changelog.rev
    ancestor = repo.changelog.ancestor
    node = repo.changelog.node

    # Find lowest common ancestors of revs. If we have multiple roots in the
    # repo the following will find one ancestor per group of revs with the
    # same root.
    ancestors = set()
    for r in revs:
        added = False
        for anc in list(ancestors):
            lca = rev(ancestor(node(anc), node(r)))
            if lca != -1:
                if anc != lca:
                    ancestors.discard(anc)
                    ancestors.add(lca)
                added = True

        if not added:
            ancestors.add(r)

    revs |= ancestors

    revs = sorted(list(revs), reverse=True)

    if len(revs) == 0:
        return

    # Print it!
    overrides = {}
    if ui.config('experimental', 'graphstyle.grandparent') == '|':
        overrides[('experimental', 'graphstyle.grandparent')] = '2.'
    with ui.configoverride(overrides, 'smartlog'):
        revdag = getdag(ui, repo, revs, masterrev)
        displayer = cmdutil.show_changeset(ui, repo, opts, buffered=True)
        ui.pager('smartlog')
        cmdutil.displaygraph(
            ui, repo, revdag, displayer, graphmod.asciiedges, None, None)

    try:
        with open(repo.vfs.join('completionhints'), 'w+') as f:
            for rev in revdag:
                commit_hash = rev[2].node()
                # Skip fakectxt nodes
                if commit_hash != '...':
                    f.write(nodemod.short(commit_hash) + '\n')
    except IOError:
        # No write access. No big deal.
        pass

    if hiddenchanges:
        msg = _(
            "note: hiding %s old heads without bookmarks\n") % hiddenchanges
        hint = _("(use --all to see them)\n")
        ui.warn(msg)
        ui.warn(hint)


import couchdb
import logging
import time

class RqDbAlreadyExistsError(Exception):
    pass
class RqDbCommunicationError(Exception):
    pass
class RqProjectDoesNotExistError(Exception):
    pass
class RqRequestDoesNotExistError(Exception):
    pass

class RqDb(object):
    def __init__(self, db_host):
        self.db_host = db_host
        self.couch_server = couchdb.Server(db_host)
        self.logger = logging.getLogger("RqDb")

    def get_project_by_key(self, key):
        view_result = self.couch_server['rqconfig'].view('doc/by_key', key=key)
        try:
            return list(view_result)[0].value
        except IndexError:
            raise RqProjectDoesNotExistError("Project with that key does not exist.")

    def get_project_by_name(self, name):
        view_result = self.couch_server['rqconfig'].view('doc/by_name', key=name)
        try:
            return list(view_result)[0].value
        except IndexError:
            raise RqProjectDoesNotExistError("Project with that name does not exist.")

    def get_all_projects(self):
        view_result = self.couch_server['rqconfig'].view('doc/by_type', key='project')
        return [project.value for project in list(view_result)]

    def send_message(self, message):
        try:
            self.couch_server['rqmessages'].save(message)
        except Exception as e:
            self.logger.exception(e)
            raise RqDbCommunicationError("Unknown error sending message")

    def add_project_to_config(self, project):
        try:
            project.type = "project"
            self.couch_server["rqconfig"].save(project.__dict__)
        except Exception as e:
            self.logger.exception(e)
            raise RqDbCommunicationError("Unknown error while saving project to config database")

    def add_permission_to_project(self, project_name, perm_name):
        project = self.get_project_by_name(project_name)
        project['permissions'].append(perm_name)

        try:
            self.couch_server["rqconfig"][project['_id']] = project
        except couchdb.ResourceConflict:
            raise RqDbCommunicationError("Project resource conflict. Try again.")

    def create_permission_request(self, request):
        try:
            request.type = "request"
            self.couch_server["rqconfig"].save(request.__dict__)
        except Exception as e:
            self.logger.exception(e)
            raise RqDbCommunicationError("Unknown error while saving permission request to config database")

    def get_perm_request_by_id(self, request_id):
        try:
            request = self.couch_server["rqconfig"].get(request_id)
        except Exception as e:
            self.logger.exception(e)
            raise RqDbCommunicationError("Unknown error while getting permission request from config database")

        if request is None:
            raise RqRequestDoesNotExistError
        else:
            return request

    def update_perm_request(self, request_id, new_status):
        try:
            new_request = self.couch_server["rqconfig"].get(request_id)
        except Exception as e:
            self.logger.exception(e)
            raise RqDbCommunicationError("Unknown error while getting permission request from config database")
        if new_request is None:
            raise RqDbCommunicationError("Could not find request with that id")

        new_request['responded'] = int(time.time())
        new_request['status'] = new_status

        try:
            self.couch_server["rqconfig"][request_id] = new_request
        except couchdb.ResourceConflict:
            raise RqDbCommunicationError("Permission request resource conflict. Try again.")

        updated_request = self.couch_server["rqconfig"].get(request_id)
        if updated_request is None:
            raise RqDbCommunicationError("Could not find the updated request in the database")

        return updated_request


from typing import Dict

import numpy as np
import pandas as pd
from loguru import logger

from preprocessor import sales_feature

def feature_extract(monthly_sales_train: pd.DataFrame):

    feature_collection = {}

    month_count = 12
    total_month_count = 34
    item_count = 22170
    category_count = 84
    shop_count = 60

    logger.info("Statistical average sales with month and item...")
    item_month_avg_sales = sales_feature.evaluate_mean(
        monthly_sales_train,
        ["date_block_num", "item_id"],
        ["item_cnt_day"],
    )
    logger.debug(f"Expected: {month_count * item_count}, Real: {len(item_month_avg_sales)} ({len(item_month_avg_sales) * 100 / (month_count * item_count):.2f}%)")
    # print(item_month_avg_sales.isnull().values.any())
    feature_collection["sales_item_month"] = item_month_avg_sales.fillna(0)

    logger.info("Statistical average sales with month and category of item...")
    category_month_avg_sales = sales_feature.evaluate_mean(
        monthly_sales_train,
        ["date_block_num", "item_category_id"],
        ["item_cnt_day"],
    )
    logger.debug(f"Expected: {month_count * category_count}, Real: {len(category_month_avg_sales)} ({len(category_month_avg_sales) * 100 / (month_count * category_count):.2f}%)")
    # print(category_month_avg_sales.isnull().values.any())
    feature_collection["sales_category_month"] = category_month_avg_sales.fillna(0)

    logger.info("Statistical average sales with shop and item...")
    item_shop_avg_sales = sales_feature.evaluate_mean(
        monthly_sales_train,
        ["shop_id", "item_id"],
        ["item_cnt_day"],
    )
    logger.debug(f"Expected: {shop_count * item_count}, Real: {len(item_shop_avg_sales)} ({len(item_shop_avg_sales) * 100 / (shop_count * item_count):.2f}%)")
    # print(item_shop_avg_sales.isnull().values.any())
    feature_collection["sales_item_shop"] = item_shop_avg_sales.fillna(0)

    logger.info("Statistical average sales with shop and category...")
    category_shop_avg_sales = sales_feature.evaluate_mean(
        monthly_sales_train,
        ["shop_id", "item_category_id"],
        ["item_cnt_day"],
    )
    logger.debug(f"Expected: {shop_count * category_count}, Real: {len(category_shop_avg_sales)} ({len(category_shop_avg_sales) * 100 / (shop_count * category_count):.2f}%)")
    # print(category_shop_avg_sales.isnull().values.any())
    feature_collection["sales_category_shop"] = category_shop_avg_sales.fillna(0)

    return feature_collection

def compose_data(
    data_train: pd.DataFrame,
    feature_collection: Dict["str", pd.DataFrame]
):
    # Copy input data and remove unused columns
    data_compose = data_train[["date_block_num", "month_id", "shop_id", "item_id", "item_category_id", "item_cnt_day"]]

    # print(feature_collection.keys())

    logger.debug("Compose Statistical Features")
    # Join data with extracted feature - item-month average sales
    data_compose = data_compose.merge(
        feature_collection["sales_item_month"],
        how='inner',
        left_on=['month_id', 'item_id'],
        right_on = ['month_id', 'item_id'],
        suffixes = ["", "_item_month"],
    )

    # Join data with extracted feature - category-month average sales
    data_compose = data_compose.merge(
        feature_collection["sales_category_month"],
        how='inner',
        left_on=['month_id', 'item_category_id'],
        right_on = ['month_id', 'item_category_id'],
        suffixes = ["", "_category_month"],
    )

    # Join data with extracted feature - item-shop average sales
    data_compose = data_compose.merge(
        feature_collection["sales_item_shop"],
        how='inner',
        left_on=['shop_id', 'item_id'],
        right_on = ['shop_id', 'item_id'],
        suffixes = ["", "_item_shop"],
    )

    # Join data with extracted feature - category-shop average sales
    data_compose = data_compose.merge(
        feature_collection["sales_category_shop"],
        how='inner',
        left_on=['shop_id', 'item_category_id'],
        right_on = ['shop_id', 'item_category_id'],
        suffixes = ["", "_category_shop"],
    )

    # Filter

    logger.debug("Compose the sales and average price last month")

    columns_in = [
        'month_id',
        'shop_id',
        'item_id',
        'item_category_id',
        'item_cnt_day_item_month',
        'item_cnt_day_category_month',
        'item_cnt_day_item_shop',
        'item_cnt_day_category_shop'
        # 'avg_item_price_weighted',
        # 'item_sales_last_month',
        ]
    data_in = data_compose[columns_in]

    columns_out = [
        'month_id',
        'shop_id',
        'item_id',
        'item_cnt_day',
    ]
    data_out = data_compose[columns_out]

    return data_in, data_out

def feature_join(
    data_inference: pd.DataFrame,
    feature_collection: Dict["str", pd.DataFrame]
):
    data_compose = data_inference

    logger.debug("Compose Statistical Features")
    # Join data with extracted feature - item-month average sales
    data_compose = data_compose.merge(
        feature_collection["sales_item_month"],
        how='left',
        left_on=['month_id', 'item_id'],
        right_on = ['month_id', 'item_id'],
        suffixes = ["", "_item_month"],
    )

    # Join data with extracted feature - category-month average sales
    data_compose = data_compose.merge(
        feature_collection["sales_category_month"],
        how='left',
        left_on=['month_id', 'item_category_id'],
        right_on = ['month_id', 'item_category_id'],
        suffixes = ["", "_category_month"],
    )

    # Join data with extracted feature - item-shop average sales
    data_compose = data_compose.merge(
        feature_collection["sales_item_shop"],
        how='left',
        left_on=['shop_id', 'item_id'],
        right_on = ['shop_id', 'item_id'],
        suffixes = ["", "_item_shop"],
    )

    # Join data with extracted feature - category-shop average sales
    data_compose = data_compose.merge(
        feature_collection["sales_category_shop"],
        how='left',
        left_on=['shop_id', 'item_category_id'],
        right_on = ['shop_id', 'item_category_id'],
        suffixes = ["", "_category_shop"],
    )

    return data_compose

def encode_time_series_data(data_train: pd.DataFrame):

    # Filter unused columns
    data_original: pd.DataFrame = data_train.drop("item_weighted_sales", axis=1)
    logger.debug(f"Original Data Length: {len(data_original)}")

    # Rename columns for better readability
    col_rename_mapper = {
        "item_cnt_day": "item_sales",
        "avg_item_price_weighted": "item_avg_price",
    }
    data_original = data_original.rename(columns=col_rename_mapper)

    # Do Normalization - Remove data with infinite value
    data_original.replace([np.inf, -np.inf], 0, inplace=True)
    logger.debug(f"Replace infintie value with 0, Length: {len(data_original)}")

    # Do Normalization - Filter value smaller than zero
    data_original.query("item_sales >= 0 and item_avg_price > 0", inplace=True)
    logger.debug(f"Remove data with negative value, Length: {len(data_original)}")

    # Do Normalization - Filter sales bigger than a specific value
    # data_original.query("item_sales <= 100", inplace=True)
    # logger.debug(f"Remove outlier of sales, Length: {len(data_original)}")

    # Do Normalization - Filter average price bigger than a specific value
    # data_original.query("item_avg_price <= 10000", inplace=True)
    # logger.debug(f"Remove outlier of sales, Length: {len(data_original)}")

    # Do Normalization - Do range mapping
    data_original["item_sales"] = data_original["item_sales"].apply(np.log10)
    data_original["item_avg_price"] = data_original["item_avg_price"].apply(lambda x: x ** 0.125)

    logger.debug(f"sales data range mapped to ({data_original['item_sales'].min():.2f}, {data_original['item_sales'].max():.2f})")
    logger.debug(f"average price data range mapped to ({data_original['item_avg_price'].min():.2f}, {data_original['item_avg_price'].max():.2f})")
    # distribution_sales = data_original["item_sales"].plot.hist()
    # distribution_sales.figure.savefig("./output/original_data_distributuion_norm_sales.jpg")
    # distribution_sales = data_original["item_avg_price"].plot.hist()
    # distribution_sales.figure.savefig("./output/original_data_distributuion_norm_avg_price.jpg")

    # Filter data from 2013-01 to 2013-06 as original time series data
    data_train_ts = data_original.query("date_block_num >= 6")
    # print(data_train_ts)

    # Join historical sales information - join data i-th month ago
    for i in range(1, 7):
        sales_px = data_original.query(f"date_block_num <= {33 - i}")
        sales_px["date_block_num"].apply(lambda x: x + i)   # shift date_block_num

        data_train_ts = data_train_ts.merge(
            sales_px,
            how="left",
            left_on=['date_block_num', 'shop_id', 'item_id', 'item_category_id'],
            right_on = ['date_block_num','shop_id', 'item_id', 'item_category_id'],
            suffixes=["", f"_p{i}"]
        )

    # Replace NaN with 0
    data_train_ts = data_train_ts.fillna(0)

    # Do data shuffling
    data_train_ts = data_train_ts.sample(frac=1).reset_index(drop=True)
    data_train_ts["month_id"] = data_train_ts["date_block_num"].apply(lambda x: (x % 12) + 1)

    # Compose Input Data
    column_in = [
        # 'date_block_num',
        'month_id',             # Month ID of P0
        'shop_id',
        'item_id',
        'item_category_id',
        # 'item_sales'
        # 'item_avg_price',
        'item_sales_p1', 'item_avg_price_p1',
        'item_sales_p2', 'item_avg_price_p2',
        'item_sales_p3', 'item_avg_price_p3',
        'item_sales_p4', 'item_avg_price_p4',
        'item_sales_p5', 'item_avg_price_p5',
        'item_sales_p6', 'item_avg_price_p6'
    ]
    data_in = data_train_ts[column_in]

    # Compose Output data
    column_out = [
        # 'date_block_num',
        'shop_id',
        # 'item_id',
        # 'item_category_id',
        'item_sales'
        # 'item_avg_price',
    ]
    data_out = data_train_ts[column_out]

    # print(data_in)
    # print(data_in.columns.values)
    # print(data_out)
    # assert False, "Unit Test Assertion"
    return data_in, data_out

def inference_ts_data_join(
    data_inference: pd.DataFrame,
    sales_info: pd.DataFrame,
):

    # Filter unused columns
    sales_info: pd.DataFrame = sales_info.drop("item_weighted_sales", axis=1)

    # Rename columns for better readability
    col_rename_mapper = {
        "item_cnt_day": "item_sales",
        "avg_item_price_weighted": "item_avg_price",
    }
    sales_info = sales_info.rename(columns=col_rename_mapper)

    # Replace +INF, -INF and NaN with 0
    sales_info.replace([np.inf, -np.inf], 0, inplace=True)

    print(sales_info["item_sales"].min(), sales_info["item_sales"].max())
    print(sales_info["item_avg_price"].min(), sales_info["item_avg_price"].max())

    # Do Normalization - Do range mapping
    sales_info["item_sales"] = sales_info["item_sales"].apply(np.log10)
    sales_info["item_avg_price"] = sales_info["item_avg_price"].apply(lambda x: x ** 0.125)

    print(sales_info["item_sales"].min(), sales_info["item_sales"].max())
    print(sales_info["item_avg_price"].min(), sales_info["item_avg_price"].max())
    assert False, "Test"

    # Copy inference data
    data_inference_ts = data_inference

    # Join historical sales information
    for i in range(6):
        sales_px = sales_info.query(f"date_block_num == {33 - i}")

        data_inference_ts = data_inference_ts.merge(
            sales_px,
            how="left",
            left_on=['date_block_num', 'shop_id', 'item_id', 'item_category_id'],
            right_on = ['date_block_num', 'shop_id', 'item_id', 'item_category_id'],
        )
        data_inference_ts = data_inference_ts.rename(
            columns = {
                "item_sales": f"item_sales_p{i + 1}",
                "item_avg_price": f"item_avg_price_p{i + 1}"
            },
        )

    # Replace NaN, -inf and inf with 0
    data_inference_ts = data_inference_ts.fillna(0)

    # Reorder data to fit model input
    column_inf_ts = [
        # 'date_block_num',
        'month_id',             # Month ID of P0
        'shop_id',
        'item_id',
        'item_category_id',
        # 'item_sales'
        # 'item_avg_price',
        'item_sales_p1', 'item_avg_price_p1',
        'item_sales_p2', 'item_avg_price_p2',
        'item_sales_p3', 'item_avg_price_p3',
        'item_sales_p4', 'item_avg_price_p4',
        'item_sales_p5', 'item_avg_price_p5',
        'item_sales_p6', 'item_avg_price_p6'
    ]
    data_inference_ts = data_inference_ts[column_inf_ts]

    return data_inference_ts

def split_test_set():
    pass


import json
from builtins import object

from django.contrib.auth.decorators import login_required
from django.http import HttpResponse
from django.shortcuts import render, redirect

# Create your views here.
from django.utils import timezone

from Course.models import Course, CourseWork, Quiz
from main.models import MyCourse, MyQuiz, MyQuizDetail

@login_required
def index(request):
    # print('Hy')
    # return HttpResponse("This sad be landing page.")
    test = ['1', 2,"three"]

    return render(request, 'index.html', {
        'foo': test,
    })

def CourseSelected(request, courseId=0):
    resp = {}
    resp["course"] = Course.objects.get(pk= courseId)
    resp["mineCourse"] = MyCourse.objects.filter(course__course_id=resp["course"].course_id , user=request.user).first()

    if resp["mineCourse"] is None:
        resp["mineCourse"] = MyCourse(user=request.user, course=resp["course"])
        resp["mineCourse"].save()

    if resp["mineCourse"].current_course_work is None:
        if (resp["course"].coursework_set.count() == 0):
            return render(request, 'extra/error.html', {
                'error_heading': 'No Course Work',
                'error_message': 'There is no course work for ' + resp["course"].__str__()
            })

        courseWork = resp["course"].coursework_set.all()
        resp["mineCourse"].current_course_work = courseWork.first()
        resp["mineCourse"].save()

    if resp["mineCourse"].percentage==100:
        return  redirect('home')

    resp["activeCourseWork"] = resp["mineCourse"].current_course_work
    resp["courseWorkList"] = resp["course"].coursework_set.all()
    if resp["activeCourseWork"].work_type==CourseWork.LECTURE:
        return render(request, 'course/lecture.html', resp)
    else:
        return HandleQuiz(request, resp["activeCourseWork"])

def NextCourseWork(request, CourseId, CurrentCourseWorkId):
    course = Course.objects.get(pk=CourseId)
    mineCourse = MyCourse.objects.filter(course__course_id=course.course_id, user=request.user)
    if mineCourse.count() == 0:
        mineCourse = MyCourse(user=request.user, course=course)
        mineCourse.save()
    else:
        mineCourse = mineCourse.first()

    courseWorkList = course.coursework_set.all()

    nextCourseWorks = []
    for courseWork in courseWorkList:
        if(courseWork.pk  > CurrentCourseWorkId):
            nextCourseWorks.append(courseWork)
    if nextCourseWorks.__len__() == 0:
        mineCourse.percentage = 100
        mineCourse.course_completed = True
    else:
        mineCourse.percentage = (1.00 - (nextCourseWorks.__len__() / courseWorkList.count()))*100
        nextCourseWork = nextCourseWorks[nextCourseWorks.__len__() - 1]
        mineCourse.current_course_work = nextCourseWork

    mineCourse.save()

    if mineCourse.percentage==100:
        return  redirect('home')

    activeCourseWork = mineCourse.current_course_work
    # courseWorkList = course.coursework_set.all()
    nextURL = ''
    # if  is None:
    if activeCourseWork.work_type == CourseWork.LECTURE:
        return render(request, 'course/lecture.html', {
            'activeCourseWork': activeCourseWork,
            'course': course,
            'courseWorkList': courseWorkList,
            'nextURL': nextURL,
            'mineCourse': mineCourse,
        })
    else:
        return HandleQuiz(request, activeCourseWork)
    # return  HttpResponse('
Pending work
')

def HandleQuiz(request, activeCourseWork):
    # quizDetails = activeCourseWork.quiz_set

    mineQuiz = MyQuiz.objects.filter(course_work=activeCourseWork, user=request.user, is_finish=False).first()
    if mineQuiz is None:
        mineQuiz = MyQuiz()
        mineQuiz.course_work  = activeCourseWork
        mineQuiz.user = request.user
        mineQuiz.start_time = timezone.now()
        mineQuiz.is_finish = False
        mineQuiz.save()
    total_questions = json.loads(activeCourseWork.data)['total_questions']
    question_number = 1
    quizQuestion = Quiz.objects.filter(CourseWork=activeCourseWork, question_number=question_number).first()
    question = quizQuestion.quiz_question
    pk = quizQuestion.pk
    options = json.loads(quizQuestion.quiz_options)

    return render(request, 'course/quiz.html', {
        'mineQuiz':mineQuiz,
        'quizQuestion':quizQuestion,
        'question':question,
        'pk':pk,
        'myQuizPk':mineQuiz.pk,
        'activeCourseWork': activeCourseWork,
        'total_questions': total_questions,
        'question_number': question_number,
        'course': activeCourseWork.course,
        'options':options
    })

def SubmitQuizAnswer(request):
    mineQuiz = MyQuiz.objects.get(pk=request.POST.get('myQuizPk'))
    quizQuestion = Quiz.objects.get(pk=request.POST.get('pk'))
    activeCourseWork = quizQuestion.CourseWork
    total_questions = json.loads(activeCourseWork.data)['total_questions']
    quizDetail = MyQuizDetail()
    quizDetail.myQuiz = mineQuiz
    quizDetail.selected_option = request.POST.get('quizanswer')
    quizDetail.right_option = quizQuestion.right_option
    quizDetail.is_right = quizDetail.selected_option==quizDetail.right_option
    quizDetail.save()
    question_number = quizQuestion.question_number + 1
    if question_number>total_questions:
        return  QuizFinished(request, mineQuiz, activeCourseWork)
    quizQuestion = Quiz.objects.filter(CourseWork=activeCourseWork, question_number=question_number).first()
    question = quizQuestion.quiz_question
    pk = quizQuestion.pk
    options = json.loads(quizQuestion.quiz_options)
    return render(request, 'course/quiz.html', {
        'mineQuiz':mineQuiz,
        'quizQuestion':quizQuestion,
        'question':question,
        'pk':pk,
        'myQuizPk':mineQuiz.pk,
        'activeCourseWork': activeCourseWork,
        'total_questions': total_questions,
        'question_number': question_number,
        'course': activeCourseWork.course,
        'options':options})

def QuizFinished(request, myQuiz, activeCourseWork):
    UpdateQuizMarks(myQuiz)
    course = activeCourseWork.course
    mineCourse = MyCourse.objects.filter(course__course_id=course.course_id, user=request.user)
    if mineCourse.count() == 0:
        mineCourse = MyCourse(user=request.user, course=course)
        mineCourse.save()
    else:
        mineCourse = mineCourse.first()
    courseWorkList = course.coursework_set.all()

    nextCourseWorks = []
    for courseWork in courseWorkList:
        if (courseWork.pk > activeCourseWork.pk):
            nextCourseWorks.append(courseWork)
    if nextCourseWorks.__len__() == 0:
        mineCourse.percentage = 100
        mineCourse.course_completed = True
    else:
        mineCourse.percentage = (1.00 - (nextCourseWorks.__len__() / courseWorkList.count())) * 100
        nextCourseWork = nextCourseWorks[nextCourseWorks.__len__() - 1]
        mineCourse.current_course_work = nextCourseWork

    mineCourse.save()

    if mineCourse.percentage==100:
        return redirect('Index')

    activeCourseWork = mineCourse.current_course_work
    nextURL = ''
    # if  is None:
    if activeCourseWork.work_type == CourseWork.LECTURE:
        return render(request, 'course/lecture.html', {
            'activeCourseWork': activeCourseWork,
            'course': course,
            'courseWorkList': courseWorkList,
            'nextURL': nextURL,
            'mineCourse': mineCourse,
        })
    else:
        return HandleQuiz(request, activeCourseWork)

def UpdateQuizMarks(myQuiz):
    quizDetails = MyQuizDetail.objects.filter(myQuiz=myQuiz).all()
    totalMarks = 0
    for quizDetail in quizDetails:
        if quizDetail.is_right:
            totalMarks = totalMarks + 1

    myQuiz.obtained_marks = totalMarks
    myQuiz.total_marks = quizDetails.count()
    myQuiz.is_finish = True
    myQuiz.save()


#编写人：张文君，编写时间5月28号
#对数据进行预处理，pca提取99的特征值，画出散点图。分析数据空间分布的大概

import numpy as np
import scipy as sp
import matplotlib
import matplotlib.pyplot as plt
import sklearn
import IPython
import platform
from sklearn.decomposition import PCA
import pandas as pd
import numpy as np

dataset = pd.read_csv('S1018bf.csv')
dataset = dataset.dropna()
#提取特征和类别
X= dataset.ix[:,'Spectral.Index':'variability_index']
y= dataset.ix[:,'Optical.Class']

y_label=y.copy()

#主成分分析
size_dimention=2
pca=PCA(n_components=size_dimention)     #加载PCA算法，设置降维后主成分数目为
pca.fit(X)
print('方差占比是'+str(pca.explained_variance_ratio_))   #显示特征数值的方差占比，要达到99%以上
reduced_x=pca.fit_transform(X)#对样本进行降维
x_data=reduced_x

#归一化处理和中心化处理
X -= np.mean(X, axis = 0)
X /= np.std(X, axis = 0)

#把类别变成数字
for i in range(1018):
    if y_label[i].strip()=='bll':
        y_label[i]='0'
    else:
        y_label[i]='1'

#看看点的分布图
y_label=np.array(y_label)
x_data=np.array(x_data)
for i in range(1018):
    if y_label[i]=='0':
        plt.scatter(x_data[i,0],x_data[i,1],c="r",label="bell")
    else:
        plt.scatter(x_data[i,0],x_data[i,1],c="b",label="fsrq")

plt.show()


from onlineaccountverifier.network_commands import *
from onlineaccountverifier.network_exceptions import *
from twisted.internet.protocol import Factory
from twisted.python.failure import Failure
import os
from twisted.internet import reactor, defer, endpoints
from twisted.internet.endpoints import TCP4ClientEndpoint, connectProtocol
from twisted.protocols.amp import AMP
import pickle, pprint
import signatures.token_request as token_request
from Crypto.Hash import SHA256

class RequestHandler(amp.AMP):

    def __init__(self):
        super().__init__()
        self.twisted_ballotregulator_port = int(os.environ['TWISTED_BALLOTREGULATOR_PORT'])
        self.twisted_ballotregulator_ip = str(os.environ['TWISTED_BALLOTREGULATOR_IP'])

    @OnlineAccountVerifier_SignBlindToken.responder
    def sign_blind_token(self, user_id, ballot_id, blind_token):
        """
        http://twistedmatrix.com/documents/12.1.0/core/howto/defer.html#class

        :param user_id:
        :param ballot_id:
        :param blind_token:
        :return:
        """
        print('[RequestHandler - request_sign_blind_token] Received request : user_id:%d, ballot_id:%d, blind_token:%s...' % (user_id, ballot_id, str(blind_token)[0:20] ))

        databasequery = self.factory.get_databasequery()

        # First we need to query the OnlineBallotRegulator for the 'user_id'

        def searchuser_onconected(ampProto):
            return ampProto.callRemote(OnlineBallotRegulator_SearchBallotRegisterForUserId, user_id=user_id)

        def searchuser_callremote_errback(failure):
            print("There was an error in the remote call", type(failure))
            raise failure.raiseException()

        print('[RequestHandler - request_sign_blind_token] Connecting to ballotregulator - port=%s, ip=%s'
              % (self.twisted_ballotregulator_ip, self.twisted_ballotregulator_port))

        searchuser_destination      = TCP4ClientEndpoint(reactor, self.twisted_ballotregulator_ip, self.twisted_ballotregulator_port)
        searchuser_connectcall      = connectProtocol(searchuser_destination, AMP())
        searchuser_results          = searchuser_connectcall.addCallback(searchuser_onconected).addErrback(searchuser_callremote_errback)

        # Now we need to format the returned results.

        def format_searchuser_results(pickled_result):

            # First unpickle the results.
            result = pickle.loads(pickled_result['ok'])

            # Transform the list results into a dictionary.
            record_list = []
            for record in result:
                mapper = {
                    'user_id': record[0],
                    'ballot_id': record[1],
                    'timestamp': record[2],
                    'ballot_name': record[3],
                    'ballot_address': record[4]
                }
                # Append each row's dictionary to a list
                record_list.append(mapper)

            # return record_list
            return record_list

        def format_searchuser_results_errback(failure):
            print("[RequestHandler - request_sign_blind_token - format_searchuser_results_errback] There was an error formatting the results", type(failure))
            raise failure.raiseException()

        searchuser_results_format_result  = searchuser_results.addCallback(format_searchuser_results).addErrback(format_searchuser_results_errback)

        # Now, lets check that the voter_id is registered for the ballot_id

        def checkvalid_userid_ballotid(results_list):
            found = False
            for record in results_list:
                if record['ballot_id'] == ballot_id:
                    found = True
                    print("[RequestHandler - request_sign_blind_token - checkvalid_userid_ballotid] user_id=%s is registered for ballot_id=%s"
                          % (user_id, ballot_id))
                    break

            if not found:
                raise UserNotRegisterdForBallot(user_id,ballot_id)

            return found # Return something though we dont need to.

        def checkvalid_userid_ballotid_errback(failure):
            print("[RequestHandler - request_sign_blind_token - checkvalid_userid_ballotid_errback] There was an error when checking the ballot & user id's")
            raise failure.raiseException()

        checkvalid_userid_ballotid_result = searchuser_results_format_result.addCallback(checkvalid_userid_ballotid).addErrback(checkvalid_userid_ballotid_errback)

        # Okay thats good, is this the first time the onlineaccountverifier is seeing this combination of user_id & ballot_id?

        def checkfirsttime_userid_ballotid(userid_found_in_onlineballotregulator):

            query = databasequery.search_token_request_for_user_id(user_id)

            def checkReturnedQuery(pickled_result):
                # First unpickle the results.
                results_list = pickle.loads(pickled_result['ok'])

                 # Transform the list results into a dictionary.
                record_list = []
                for record in results_list:
                    mapper = {
                        'token_request_id': record[0],
                        'blind_token': record[1],
                        'user_id': record[2],
                        'ballot_id': record[3],
                        'timestamp': record[4]
                    }
                    # Append each row's dictionary to a list
                    record_list.append(mapper)

                for record in record_list:
                    if record['ballot_id'] == ballot_id:
                        print("[RequestHandler - request_sign_blind_token - checkfirsttime_userid_ballotid] user_id='%s' is registered for ballot_id='%s'"
                              % (user_id, ballot_id))
                        raise UserAlreadySubmittedTokenForThisBallot(user_id,ballot_id, record['blind_token'] )

                return True # Return something though we dont need to.

            return query.addCallback(checkReturnedQuery)

        def checkfirsttime_userid_ballotid_errback(failure):
            print("[RequestHandler - request_sign_blind_token - checkvalid_userid_ballotid_errback] There was an error checking the ballot & user id's")
            raise failure.raiseException()

        checkfirsttime_userid_ballotid_result   = checkvalid_userid_ballotid_result.addCallback(checkfirsttime_userid_ballotid).addErrback(checkfirsttime_userid_ballotid_errback)

        # All of our checks are complete, lets sign the token.

        def sign_blindtoken(checkfirsttime_userid_ballotid):
            from twisted.internet import defer
            print("[RequestHandler - request_sign_blind_token - sign_blindtoken] Signing the token '%s...'for the ballot '%s'" % (str(blind_token)[0:20], ballot_id))

            d = defer.Deferred()

            signed_blind_token = token_request.sign_blind_token( blind_token, ballot_id )

            d.callback(signed_blind_token)

            return d

        def sign_blindtoken_errback(failure):
            print("[RequestHandler - request_sign_blind_token - sign_blindtoken_errback] There was an error signing the token.")
            raise failure.raiseException()

        sign_blindtoken_result = checkfirsttime_userid_ballotid_result.addCallback(sign_blindtoken).addErrback(sign_blindtoken_errback)

        # Okay, signing worked. Lets save this request.

        def save_token_request(signed_blind_token):

            print("[RequestHandler - request_sign_blind_token - save_token_request] Saving the register token request of \n    "
                  "user_id='%s' for ballot_id='%s' with signed_blind_ballot='%s...'" % (user_id, ballot_id, str(signed_blind_token)[0:20]) )

            # We should save our processed request.
            hash = SHA256.new()
            hash.update(str(blind_token).encode())
            blind_token_hash = hash.hexdigest()

            defer = databasequery.insert_into_register_token_blind_token_hash_user_id_ballot_id(blind_token_hash, user_id, ballot_id)

            # Cool, saving worked. Return singed token to user.

            def return_result(ignored):
                print("[RequestHandler - request_sign_blind_token - returnResult] Computation completed, returning signed token to the client. ")

                # Pickle the signed token to return over the wire
                encoded_signed_blind_token = pickle.dumps(signed_blind_token)

                return { 'ok' : encoded_signed_blind_token }

            def return_result_errback(failure):
                print("[RequestHandler - request_sign_blind_token - return_result_errback] There was an error saving the token.")
                raise failure.raiseException()

            defer.addCallback(return_result).addErrback(return_result_errback)

            return defer

        def save_token_request_errback(failure):
            print("[RequestHandler - request_sign_blind_token - save_token_request_errback] There was an error saving the token.")
            raise failure.raiseException()

        save_token_request = sign_blindtoken_result.addCallback(save_token_request).addErrback(save_token_request_errback)

        return save_token_request

    @OnlineAccountVerifier_SearchTokenRequestForUserId.responder
    def search_token_request_for_user_id(self, user_id):
        databasequery = self.factory.get_databasequery()

        return databasequery.search_token_request_for_user_id(user_id)

    @OnlineAccountVerifier_SearchRegisterVoteForAddress.responder
    def search_register_vote_for_voter_address(self, voter_address):
        databasequery = self.factory.get_databasequery()
        return databasequery.search_register_vote_for_voter_address(voter_address)

    @OnlineAccountVerifier_GetPublicKeyForBallot.responder
    def request_public_key_for_ballot(self, ballot_id):

        d = defer.Deferred()

        def request(ignored):
            public_key_string = token_request.get_public_key_string(ballot_id)
            pickled_public_key_string = pickle.dumps(public_key_string)

            return { 'ok' : pickled_public_key_string }

        def request_errback(failure):
            raise failure.raiseException()

        d.addCallback(request).addErrback(request_errback)
        d.callback(None)

        return d

    @OnlineAccountVerifier_RegisterAddressToBallot.responder
    def register_address_to_ballot(self, ballot_id, pickled_signed_token, pickled_token, pickled_voter_address):
        """
        Called after obtaining a signed_blind_token from `request_sign_blind_token`. This will allow you to
        register an ethereum address to vote on a ballot contract.

        :param ballot_id:
        :param pickled_signed_token:
        :param pickled_token:
        :param pickled_voter_address:
        :return:
        """

        # First lets unpickle
        signed_token = pickle.loads(pickled_signed_token)
        token = pickle.loads(pickled_token)
        voter_address = pickle.loads(pickled_voter_address)

        print('[RequestHandler - register_address_to_ballot] Received request : ballot_id:%d, token:%s, signed_token:%s...' % (ballot_id, token, str(signed_token)[0:20]))

        databasequery = self.factory.get_databasequery()

         # First we need to check that we received a validly signed token.

        def check_token_signed_for_ballot(ignored):
            result = token_request.check_token_signed_for_ballot(signed_token, token, ballot_id)
            return result

        def check_token_signed_for_ballot_errback(failure):
            print("[RequestHandler - request_sign_blind_token - format_searchuser_results_errback] There was an error formatting the results")
            raise failure.raiseException()

        top_defer = defer.Deferred()
        check_token_signed_for_ballot_result = top_defer.addCallback(check_token_signed_for_ballot).addErrback(check_token_signed_for_ballot_errback)
        top_defer.callback(None) # Start our deferred calls

        # Next we need to check that this is the first time we are seeing this token + (voter_address & ballot_id)

        def check_first_time_seeing_token_ballotid_voteraddress(prev_result):

            query = databasequery.search_ballot_register_for_ballot_id(ballot_id, voter_address)

            def checkReturnedQuery(pickled_result):
                # First unpickle the results.
                results_list = pickle.loads(pickled_result['ok'])

                 # Transform the list results into a dictionary.
                record_list = []
                for record in results_list:
                    mapper = {
                        'register_vote_id': record[0],
                        'signed_token': record[1],
                        'voter_address': record[2],
                        'ballot_id': record[3],
                        'timestamp': record[4]
                    }
                    # Append each row's dictionary to a list
                    record_list.append(mapper)

                if len(record_list):
                    raise BallotVoteraddressAlreadyRegistered(record_list[0]['ballot_id'], record_list[0]['voter_address'], record_list[0]['signed_token'])

                return True # Return something though we dont need to.

            def checkReturnedQuery_Errback(failure):
                raise failure.raiseException()

            return query.addCallback(checkReturnedQuery).addErrback(checkReturnedQuery_Errback)

        def check_first_time_seeing_token_ballotid_voteraddress_errback(failure):
            print("[RequestHandler - request_sign_blind_token - check_first_time_seeing_token_ballotid_voteraddress_errback] There was an error when checking not already registered")
            raise failure.raiseException()

        check_first_time_seeing_token_ballotid_voteraddress_result = check_token_signed_for_ballot_result.addCallback(check_first_time_seeing_token_ballotid_voteraddress).addErrback(check_first_time_seeing_token_ballotid_voteraddress_errback)

        # Get the ballot information from the ballot_id

        def get_ballot_information(prev_result):

            destination = TCP4ClientEndpoint(reactor, self.twisted_ballotregulator_ip, self.twisted_ballotregulator_port)
            requestsearchuser_deferred = connectProtocol(destination, AMP())

            def requestsearchuser_connected(ampProto):
                return ampProto.callRemote(OnlineBallotRegulator_SearchBallotsAvailableForAllBallots)
            requestsearchuser_deferred.addCallback(requestsearchuser_connected)

            def done(result):
                '''
                Returns the record ascociated with the ballot_id
                :param result:
                :return:
                '''
                unpickled_result = pickle.loads(result['ok'])

                # Transform the list results into a dictionary.
                record_list = []
                for record in unpickled_result:
                    mapper = {}
                    mapper['ballot_id'] = record[0]
                    mapper['ballot_name'] = record[1]
                    mapper['ballot_address'] = record[2]
                    mapper['timestamp'] = record[3]
                    # Append each row's dictionary to a list
                    record_list.append(mapper)

                # Check the available ballots for *our* ballot
                for record in record_list:
                    if record['ballot_id'] == ballot_id:
                        return record

                # If we reach here we havent found *our* ballot in the list available
                raise BallotNotAvailable(ballot_id)

            def done_errback(failure):
                raise failure.raiseException()

            parsed_results = requestsearchuser_deferred.addCallback(done).addErrback(done_errback)

            return parsed_results

        def get_ballot_information_errback(failure):
            print("[RequestHandler - request_sign_blind_token - get_ballot_information_errback] There was an error when getting the ballot information")
            raise failure.raiseException()

        get_ballot_information_result = check_first_time_seeing_token_ballotid_voteraddress_result.addCallback(get_ballot_information).addErrback(get_ballot_information_errback)

        # Register on the ethereum contract.

        def register_voteraddress_ethereumcontract(prev_result):

            register_voteraddress_ethereumcontract_destination = TCP4ClientEndpoint(reactor, self.twisted_ballotregulator_ip, self.twisted_ballotregulator_port)
            register_voteraddress_ethereumcontract_deferred = connectProtocol(register_voteraddress_ethereumcontract_destination, AMP())

            def add_voter_address_to_contract(ampProto):
                return ampProto.callRemote(OnlineBallotRegulator_RegisterVoterAddressBallotId, voter_addres=voter_address, ballot_id=ballot_id)

            def add_voter_address_to_contract_errback(failure):
                print("There was an error in the remote call to add the voter address to the contract.")
                raise failure.raiseException()

            register_voteraddress_ethereumcontract = register_voteraddress_ethereumcontract_deferred.addCallback(add_voter_address_to_contract).addErrback(add_voter_address_to_contract_errback)

            return register_voteraddress_ethereumcontract

        def register_voteraddress_ethereumcontract_errback(failure):
            print("[RequestHandler - request_sign_blind_token - register_voteraddress_ethereumcontract_errback] There was an error registering address to the ethereum contract")
            raise failure.raiseException()

        register_voteraddress_ethereumcontract_result = get_ballot_information_result.addCallback(register_voteraddress_ethereumcontract).addErrback(register_voteraddress_ethereumcontract_errback)

        # Save our details to register_vote table

        def save_vote_registration(prev_result):

            print("Type prev result (it is a defered?) = ", type(prev_result))

            # We should save our processed request.
            hash = SHA256.new()
            hash.update(str(signed_token).encode())
            signed_token_hash = hash.hexdigest()

            return databasequery.insert_into_register_vote_signed_token_hash_voter_address_ballot_id(signed_token_hash, voter_address, ballot_id)

        def save_vote_registration_errback(failure):
            print("[RequestHandler - request_sign_blind_token - save_vote_registration_errback] There was an error saving the vote registration.")
            raise failure.raiseException()

        save_vote_registration_result = register_voteraddress_ethereumcontract_result.addCallback(save_vote_registration).addErrback(save_vote_registration_errback)

        def return_results_to_client(res):
            print("[RequestHandler - request_sign_blind_token - return_results_to_client] Returning 'ok' to client.")
            return { 'ok' : True }

        return save_vote_registration_result.addCallback(return_results_to_client)

class MyServerFactory(Factory):
    protocol = RequestHandler

    def __init__(self, databasequery):
        self.databasequery = databasequery

    def get_databasequery(self):
        return self.databasequery


import torch
import torchvision
from torch import nn
from diffusers import DDPMScheduler, DDIMScheduler, UNet2DModel
from matplotlib import pyplot as plt
from tqdm import tqdm
from torch.utils.data import DataLoader

# define class-conditioned u-net model from huggingface
class ClassConditionedUNet(nn.module):
    def __init__(self, num_classes, class_emb_size, sample_size, c_in, c_out):
        super().__init__()
        self.class_emb = nn.Embedding(num_classes, class_emb_size)

        self.model = UNet2DModel(
            sample_size=sample_size,
            in_channels=c_in + class_emb_size,
            out_channels=c_out,
            layers_per_block=2,
            block_out_channels=(32, 64, 64),
            down_block_types=(
                "DownBlock2D",
                "AttnDownBlock2D",
                "AttnDownBlock2D",
            ),
            up_block_types=(
                "AttnUpBlock2D",
                "AttnUpBlock2D",
                "UpBlock2D",
            ),
        )

    def forward(self, x, t, class_labels):
        bs, ch, w, h = x.shape
        class_cond = self.class_emb(class_labels)
        class_cond = class_cond.view(bs, class_cond.shape[1], 1, 1).expand(bs, class_cond.shape[1], w, h)
        net_input = torch.cat((x, class_cond), 1)
        return self.model(net_input, t).sample

# define diffusion class from huggingface
class Diffuser():
  def __init__(self, device, dataloader, scheduler='ddpm', num_train_timesteps=1000, beta_schedule='squaredcos_cap_v2', num_classes=10, class_emb_size=4, sample_size=28, c_in=1, c_out=1):
    self.num_classes = num_classes
    self.train_dataloader = dataloader
    self.num_train_timesteps = num_train_timesteps
    self.net = ClassConditionedUNet(num_classes=num_classes, class_emb_size=class_emb_size, sample_size=sample_size, c_in=c_in, c_out=c_out).to(device)
    self.sample_size = sample_size
    self.channels = c_in
    self.device = device

    if scheduler=='ddim':
        self.noise_scheduler = DDIMScheduler(num_train_timesteps=self.num_train_timesteps, beta_schedule=beta_schedule)
        self.noise_scheduler.set_timesteps(self.num_train_timesteps)
    else:
        self.noise_scheduler = DDPMScheduler(num_train_timesteps=self.num_train_timesteps, beta_schedule=beta_schedule)

  def train(self, n_epochs=5, lr=1e-3):
    self.loss_fn = nn.MSELoss()
    self.opt = torch.optim.Adam(self.net.parameters(), lr)
    self.losses = []

    for epoch in range(n_epochs):
      for x,y in tqdm(self.train_dataloader):
        x = x.to(self.device)
        y = y.to(self.device)
        noise = torch.randn_like(x)
        timesteps = torch.randint(0, self.num_train_timesteps-1, (x.shape[0],)).long().to(self.device)
        noisy_x = self.noise_scheduler.add_noise(x,noise,timesteps)

        pred = self.net(noisy_x, timesteps, y)
        loss = self.loss_fn(pred, noise)

        self.opt.zero_grad()
        loss.backward()
        self.opt.step()

        self.losses.append(loss.item())

  def sample(self):
    x = torch.randn(80, self.channels, self.sample_size, self.sample_size).to(self.device)
    y = torch.tensor([[i]*8 for i in range(self.num_classes)]).flatten().to(self.device)

    for i,t in tqdm(enumerate(self.noise_scheduler.timesteps)):
      with torch.no_grad():
        residual = self.net(x,t,y)

      x = self.noise_scheduler.step(residual, t, x).prev_sample

    fig, ax = plt.subplots(1, 1, figsize=(12, 12))
    ax.imshow(torchvision.utils.make_grid(x.detach().cpu().clip(-1, 1), nrow=8)[0], cmap='Greys')
    return x

if __name__ == "__main__":
  device = 'mps' if torch.backends.mps.is_available() else 'cuda' if torch.cuda.is_available() else 'cpu'

  # import dataset
  mnist = torchvision.datasets.MNIST(root="mnist/", train=True, download=True, transform=torchvision.transforms.ToTensor())

  # create dataloaders
  train_dataloader = DataLoader(mnist, batch_size=128, shuffle=True)

  # generate images
  diffuser = Diffuser(dataloader=train_dataloader,
                    num_train_timesteps=1000,
                    beta_schedule='squaredcos_cap_v2',
                    num_classes=10,
                    class_emb_size=4,
                    sample_size=28,
                    c_in=1,
                    c_out=1)
  diffuser.train(n_epochs=150, lr=1e-4)
  samples = diffuser.sample()

  # generate 5040 images
  tests = []
  for _ in range(63):
    samples = diffuser.sample()
    tests.append(samples)

  # save images
  torch.save(tests, 'tests_ddpm.pt')

  # add class to images
  annotated_tests = []
  for batch in tests:
     for i in range(len(batch)):
        cur_num = int(i/8)
        annotated_tests.append((batch[i].cpu(), cur_num))

  # save images + class
  torch.save(annotated_tests, 'annotated_tests_ddpm.pt')


def syn_resolve_references(json_schema):

    """
    Function: syn_resolve_references

    Purpose: Resolves $ref statements referring to schemas registered in
             Synapse.

    Arguments: JSON schema to be de-referenced

    Returns: A de-referenced JSON schema in dictionary form

    """

    import jsonschema

    # The full URL is defined in the Synapse docs.
    syn_url = "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered"

    ref_resolver = jsonschema.RefResolver.from_schema(json_schema)

    for schema_key in json_schema["properties"]:
        if "$ref" in json_schema["properties"][schema_key]:
            ref_url = f"{syn_url}/{json_schema['properties'][schema_key]['$ref']}"
            deref_object = ref_resolver.resolve(ref_url)
            json_schema["properties"][schema_key] = deref_object[1]

    return(json_schema)


from collections import deque

n, x, y = map(int, input().split())

ans = [0] * (n+1)

g = [[i+1, i-1] for i in range(n+1)]
#g[0].clear() pypyにclearなし
g[1].pop(1)
g[n].pop(0)

g[x].append(y)
g[y].append(x)

for i in range(1,n+1):
    q = deque()
    time = 0
    visit = [False] * (n+1)
    q.append((i, time))

    while(len(q) > 0):
        item = q.popleft()

        if visit[item[0]] == False:
            visit[item[0]] = True
            ans[item[1]] += 1
            for k in g[item[0]]:
                q.append((k, item[1]+1))

for i in range(1,n):
    print(ans[i]//2)


#%%
import os
from selenium import webdriver
from bs4 import BeautifulSoup
import re
import argparse
import time
import csv
import pandas as pd
from pprint import pprint

#%%
#game_sites = 'https://www.metacritic.com/browse/' + genres + scores_by + '/year/all/filtered'

if not os.path.exists('./metacritic_review'):
    os.makedirs('./metacritic_review')

#%%
def hasXpath(xpath):
    try:
        driver.find_element_by_xpath(xpath)
        return True
    except:
        return False

#%%

def metacritic_review(genre_name):
    os.chdir('/Users/hyunjaecho/Documents/GitHub/Team-Amazing')
    driver=webdriver.Chrome(executable_path=r"/Users/HyunjaeCho/Documents/Fall2020/DataMining/chromedriver")
    #genre_type = re.findall(r'\w+', str(genre_name))
    genre_type = str(genre_name).replace("['", "")
    genre_type = str(genre_type).replace("']", "")
    genre_type = str(genre_type).replace(",", "")
    genre_type = str(genre_type).replace("('", "")
    genre_type = str(genre_type).replace("')", "")
    site = 'https://www.metacritic.com/browse/games/genre/metascore/' + str(genre_type) + '/all?view=detailed'
    driver.get(site)
    driver.implicitly_wait(10)
    print(site)
    html = driver.page_source
    bsobj = BeautifulSoup(html, 'html.parser')
    last_page = bsobj.find('li', {'class': re.compile('page last_page.*')}).text
    last_page = re.findall('\d+', str(last_page))

    titles = []
    platforms = []
    scores = []
    genres = []
    total_df = pd.DataFrame([])
    for page in range(0, int(last_page[0])):
        os.chdir('/Users/hyunjaecho/Documents/GitHub/Team-Amazing')
        driver=webdriver.Chrome(executable_path=r"/Users/HyunjaeCho/Documents/Fall2020/DataMining/chromedriver")
        sites = site + '&page=' + str(page)
        driver.get(sites)
        print(sites)
        driver.implicitly_wait(5)
        html = driver.page_source
        bsobj = BeautifulSoup(html, 'html.parser')
        title = [i.text for i in bsobj.find_all('a', {'class': 'title'})]
        platform = [i.text for i in bsobj.find_all('div', {'class': 'platform'})]
        platform = platform[1:]
        score = [i.text for i in bsobj.find_all('div', {'class': re.compile('metascore_w large game.*')})]
        score = score[::2] #since it extracts two scores at the same time
        #genre_type = re.findall(r'\w+', str(genre_name))
        #genre_type = str(genre_type).replace("['", "")
        #genre_type = str(genre_type).replace("']", "")
        genre = [genre_type for i in range(len(score))]
        titles += title
        platforms += platform
        scores += score
        genres += genre
        data = {'platform': platforms, 'genre': genres, 'title': titles, 'score': scores}
    total_df = total_df.append(pd.DataFrame(data))
    total_df['platform'] = total_df['platform'].str.extract(r"( .*)")
    total_df['platform'] = total_df['platform'].str.replace('\n', '')
    #total_df['genre'] = total_df['genre'].str.extract(r"(\w+)")
    total_df.to_csv('metacritic_review_' + str(genre_type) + '.csv', encoding = 'utf-8')

if __name__ == '__main__':
    genres = ['adventure', 'fighting', 'first-person', 'flight', 'party', 'platformer', 'puzzle',
    'racing', 'real-time', 'role-playing', 'simulation', 'sports', 'strategy', 'third-person',
    'turn-based', 'wargame', 'wrestling']
    for genre_name in zip(genres):
        metacritic_review(genre_name)

#%%


# coding: utf-8

# In[61]:

import pandas as pd
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
import numpy as np
import os
df=pd.read_excel(r'C:\Users\jyothi\Downloads\Sample - Superstore.xlsx')
df=df[['Order Date','Sales']].sort_values('Order Date',ascending=False)
df= df.rename(columns={'Order Date': 'od', 'Sales': 'sales'})
df.index=df.od
df=df[(df.od>'2018-01-01')]
df=df.drop('od',axis=1)
df
f=plt.figure(figsize=(20,5))
plt.plot(df)
plt.xlabel('Dates')
plt.ylabel('sales')
plt.title('sales by date')

# In[62]:

from pandas.plotting import autocorrelation_plot
autocorrelation_plot(df)

# In[67]:

from statsmodels.tsa.arima_model import ARIMA
model=ARIMA(df,order=(5,1,0))
model_fit = model.fit(disp=0)
print(model_fit.summary())
# plot residual errors
residuals =pd.DataFrame(model_fit.resid)
residuals.plot()

residuals.plot(kind='kde')
print(residuals.describe())

# In[69]:

X = df.values
size = int(len(X) * 0.66)
train, test = X[0:size], X[size:len(X)]
history = [x for x in train]
predictions = list()
for t in range(len(test)):
	model = ARIMA(history, order=(10,1,0))
	model_fit = model.fit(disp=0)
	output = model_fit.forecast()
	yhat = output[0]
	predictions.append(yhat)
	obs = test[t]
	history.append(obs)
	print('predicted=%f, expected=%f' % (yhat, obs))
error = mean_squared_error(test, predictions)
print('Test MSE: %.3f' % error)
# plot
plt.plot(test)
plt.plot(predictions, color='red')