#!/usr/bin/env python3
"""
:Created: 2013-12-30
:Updated: 2018-03-09

- Import old bookmarks from JSON, XML.

Commands:
  - list
  - add | modify | assert | show
  - remove
  - tags
  - urls
  - sync

  Database:
    - info | init | stats | clear
"""
from __future__ import print_function

__description__ = "bookmarks - ..."
__short_description__ = "..."
__version__ = '0.0.4-dev' # script-mpe
__db__ = '~/.bookmarks.sqlite'
__couch__ = 'http://localhost:5984/the-registry'
import os
if os.uname()[0] == 'Darwin':
    chrome_bookmarks_file= '~/Library/Application Support/Google/Chrome/Default/Bookmarks'
    chrome_history_path=   '~/Library/Application Support/Google/Chrome/Default/History'
elif os.uname()[0] == 'Linux':
    chrome_bookmarks_file= '~/.config/google-chrome/Default/Bookmarks'
    chrome_history_path=   '~/.config/google-chrome/Default/History'
#elif os.uname()[0] == 'Win'
    # C:\Users\xyz\AppData\Local\Google\Chrome\User Data\Default

__usage__ = """

Usage:
  bookmarks.py [-v... options] list [NAME] [ --tags=TAG... ]
  bookmarks.py [-v... options] (add|modify|assert|show) REF [ NAME ] [ TAGS... ]
  bookmarks.py [-v... options] remove REF
  bookmarks.py [-v... options] tags [TAGS...]
  bookmarks.py [-v... options] urls REF
  bookmarks.py [-v... options] sync
  bookmarks.py [-v... options] check [NAME]
  bookmarks.py [-v... options] webarchive [NAME]
  bookmarks.py [-v... options] (tag|href|domain) [NAME]
  bookmarks.py [-v... options] x [ARG...]
  bookmarks.py [-v... options] dlcs (parse|import FILE|export)
  bookmarks.py [-v... options] chrome (all|roots|groups|import) [--chrome-bookmarks-root NAME]
  bookmarks.py [-v... options] html (tree|stats|groups|import|export) HTML
  bookmarks.py [-v... options] sql (stats|couch|cleanup)
  bookmarks.py [-v... options] couch (sql|stats|list|update|init)
  bookmarks.py [-v... options] couch (add|modify) REF [ NAME [ TAGS... ] ]
  bookmarks.py [-v... options] info | init | stats | clear | memdebug
  bookmarks.py --background
  bookmarks.py -h|--help
  bookmarks.py help [CMD]
  bookmarks.py --version

Options:
  -s SESSION, --session-name SESSION
                should be bookmarks [default: default].
  -d REF, --dbref=REF
                SQLAlchemy DB URL [default: %s]
  --no-db       Don't initialize SQL DB connection.
  --couch=REF
                Couch DB URL [default: %s]
  --tag-offset INT
                Set import frequency-offset to exclude certain one-to-many
                relations if the usage is below given value.
                This entirely depends on usage.
                0 means to import everything [default: -1]
                Defaults to hiFreq * 0.1.
  --domain-offset INT
                Typical --*-offset, see before.
                Defaults to avgFreq. [default: -1]
  --chrome-bookmarks-file PATH
                [default: %s]
  --chrome-bookmarks-root GROUP
                Ie. bookmark_bar or other. [default: ]
  -O FMT, --output-format FMT
                json, outline, todo.txt, repr [default: rst]
  -i N, --interval N
                Be verbose at least every N records [default: 100]
  --interactive
                Prompt to resolve or override certain warnings.
                XXX: Normally interactive should be enabled if while process has a
                terminal on stdin and stdout.
  --batch
                Overrules `interactive`, exit on errors or strict warnings.
  --auto-commit N
                Auto-commit every N records.
  --no-commit   .
  --commit      [default: true].
  --clear-unknown-keys
                Delete unknown keys from CouchDB documents.
  --newer-only
                Only update entries in target if source update time is more
                recent.
  --no-partial-match
  --partial-match
                Treat input values as partial search values. This adds the
                equivalent of '*' around all the values (without '*').
  --update-hrefids
                Walk over URLs (.href) and rename if found as key.
  --update-tagstype
                Correct couch document 'tags->tag_list' attribute.
  --deleted
  --count       [default: false].
  --status STATUS[,STATUS]
  --days N
  --weeks N
  --from DATE
  --to DATE
  --on DAY
  --older-than SPEC
  --max-age SPEC
                Datetime query window. [Default: 1y] Queries are always on
                updated-time's.
  --added       Use added time instead of update in match with date/time.
  --matched-only
  --exact-match
  --struct-output
  --update
  --delete STATUS[,STATUS]
  --delete-error
  --include-null
  --ignore-last-seen
  --dry-run     Echo but don't make actual changes. This does all the
                document/record operations, but no commit.
  -v, --verbose  Increase verbosity, 3 is maximum.
  -q, --quiet   Turn off verbosity. Overrides verbosity flags.
  --print-memory
                Print memory usage just before program ends.
  -h --help     Show this usage description.
                For a command and argument description use the command 'help'.
  --version     Show version (%s).
""" % ( __db__, __couch__, chrome_bookmarks_file, __version__, )
import sys
import hashlib
import urllib
# import urllib2 TODO: rewrite to Py3
from datetime import datetime, timedelta

import uriref
from pydelicious_xml import dlcs_parse_xml
import bs4

from script_mpe.libhtd import *
from script_mpe.bookmarks_model import *
from script_mpe.res import bm, bm_chrome

ctx = Taxus(version='bookmarks')

cmd_default_settings = dict(
        verbose=1,
        partial_match=True,
        strict=True
    )

# XXX: old rsr bookmarks class
class bookmarks(rsr.Rsr):

    #zope.interface.implements(res.iface.ISimpleCommand)

    NAME = os.path.splitext(os.path.basename(__file__))[0]
    OPT_PREFIX = 'bm'
    OPTS_INHERIT = ('-v',)# '-q', '-i', '-c')

    DEFAULT_CONFIG_KEY = 'bm'

    #TRANSIENT_OPTS = Taxus.TRANSIENT_OPTS + ['']
    DEFAULT_ACTION = 'stats'

    DEPENDS = {
            'stats': ['rsr_session'],
            'list': ['rsr_session'],
            'add': ['rsr_session'],
            'add_lctr': ['rsr_session'],
            'list_lctr': ['rsr_session'],
            'add_lctr_ref_md5': ['rsr_session'],
            'add_ref_md5': ['rsr_session'],
            'moz_js_import': ['rsr_session'],
            'moz_js_group_import': ['rsr_session'],
            'moz_ht_import': ['rsr_session'],
            'dlcs_post_import': ['rsr_session'],
            'dlcs_post_read': ['rsr_session'],
            'dlcs_post_test': ['rsr_session'],
            'dlcs_post_test2': ['rsr_session'],
            'export': ['rsr_session']
        }

    @classmethod
    def get_optspec(Klass, inheritor):

        """
        Return tuples with optparse command-line argument specification.
        """
        p = inheritor.get_prefixer(Klass)
        return (
            # actions
                p(('-s', '--stats',), libcmd.cmddict()),
                p(('-l', '--list',), libcmd.cmddict()),
                p(('-a', '--add',), libcmd.cmddict()),
                p(('--list-lctr',), libcmd.cmddict()),

                (('--moz-js-import',), libcmd.cmddict()),
                (('--dlcs-post-import',), libcmd.cmddict()),
                (('--dlcs-post-test',), libcmd.cmddict()),

                p(('--export',), libcmd.cmddict(help="TODO: bm export")),

                (('--moz-js-group-import',), libcmd.cmddict()),

                (('--add-lctrs',), libcmd.cmddict(
                    help="Add locator records for given Locator's.")),
                (('--add-ref-md5',), libcmd.cmddict(
                    help="Add MD5-refs missing (on all locators). ")),

            # params
                p(('--public',), dict( action='store_true', default=False )),
                p(('--name',), dict( default=None, type="str" )),
                p(('--href',), dict( default=None, type="str" )),
                p(('--ext',), dict( default=None, type="str" )),
                p(('--tags',), dict( default=None, type="str" )),
                p(('--ref-md5',), dict( action='store_true',
                    default=False, help="Calculate MD5 for new locators. " )),
            )

    def assert_locator(self, sa=None, href=None, opts=None):
        lctr = Locator.fetch((Locator.global_id==href,), _sa=sa, exists=False)
        if not lctr:
            if len(href) > 255:
                log.err("Reference too long: %s", href)
                return
            lctr = Locator(
                    global_id=href,
                    date_added=datetime.now() )
            sa.add( lctr )
            if opts.rsr_auto_commit:
                sa.commit()
            if opts.bm_ref_md5:
                self.add_lctr_ref_md5(opts, sa, href)
        yield dict(lctr=lctr)

    def add_lctrs(self, sa=None, opts=None, *refs):
        if refs:
            for ref in refs:
                for ret in self.assert_locator( sa=sa, href=ref, opts=opts ):
                    yield ret['lctr']

    def add(self, sa=None, href=None, name=None, ext=None, public=False,
            tags=None, opts=None):
        "Create or update. alias --update?"
        lctr = [ r['lctr'] for r in self.assert_locator(sa=sa, href=href, opts=opts) ]
        if not lctr:
            yield dict( err="XXX Missed ref" )
        else:
            lctr = lctr.pop()
            assert lctr
            bm = Bookmark.fetch((Bookmark.ref==lctr,), _sa=sa, exists=False)
            if bm:
                # XXX: start local to bean dict
# XXXL name must be unique, must catch problems
                #if name != bm.name:
                #    bm.name = name
                #if lctr != bm.ref:
                #    bm.ref = lctr
                if ext != bm.extended:
                    bm.extended = ext
                if public != bm.public:
                    bm.public = public
                if tags != bm.tags:
                    bm.tags = tags
                bm.last_update = datetime.now()
            else:
                bm = Bookmark.fetch((Bookmark.name==name,), _sa=sa, exists=False)
                if bm:
                    log.err("Duplicate name %s", bm.name)
                    #bm.name = "%s (copy)" % name
                    bm = None
                else:
                    bm = Bookmark(
                            name=name,
                            ref=lctr,
                            extended=ext,
                            public=public,
                            tags=tags,
                        )
                    bm.init_defaults()
            if bm:
                assert bm.ref
                yield dict( bm=bm )
                sa.add(bm)
            if opts.rsr_auto_commit:
                sa.commit()

    def add_lctr_ref_md5(self, opts=None, sa=None, *refs):
        "Add locator and ref_md5 attr for Locators"
        if refs:
            if isinstance( refs[0], str ):
                opts.ref_md5 = True
                lctrs = [ ret['lctr'] for ret in self.add_lctrs(sa, opts, *refs) ]
                return
            else:
                assert isinstance( refs[0], Locator), refs
                lctrs = refs

            for lctr in lctrs:
                ref = lctr.ref or lctr.global_id
                ref_md5 = hashlib.md5( ref ).hexdigest()
                md5 = MD5Digest.fetch(( MD5Digest.digest == ref_md5, ),
                        exists=False)
                if not md5:
                    md5 = MD5Digest( digest=ref_md5,
                            date_added=datetime.now() )
                    sa.add( md5 )
                    log.info("New %s", md5)
                lctr.ref_md5 = md5
                sa.add( lctr )
                if opts.rsr_auto_commit:
                    sa.commit()
                log.note("Updated ref_md5 for %s to %s", lctr, md5)

    def add_ref_md5(self, opts=None, sa=None):
        "Add missing ref_md5 attrs. "
        lctrs = sa.query(Locator).filter(Locator.ref_md5_id==None).all()
        self.add_lctr_ref_md5( opts, sa, *lctrs )

    def moz_js_import(self, opts=None, sa=None, *paths):
        mozbm = res.bm.MozJSONExport()
        for path in paths:
            #for ref in mozbm.read_lctr(path):
            #    list(self.assert_locator(sa=sa, href=ref, opts=opts))
            for node in mozbm.read_bm(path):
                descr = [ a['value'] for a in node.get('annos', [] )
                        if a['name'] == 'bookmarkProperties/description' ]
                if 'uri' not in node or 'title' not in node or not node['title']:
                    log.warn("Illegal %s", node)
                else:
                    list(self.add(sa=sa,
                        href=node['uri'],
                        name=node['title'],
                        ext=descr and descr.pop() or None,
                        opts=opts))

    def moz_js_group_import(self, opts=None, sa=None, *paths):
        "Import groupnodes only. "
        mozbm = res.bm.MozJSONExport()
        for path in paths:
            nodes = {}
            roots = []
            #for ref in mozbm.read_lctr(path):
            #    list(self.assert_locator(sa=sa, href=ref, opts=opts))
            for node in mozbm.read(path):
                if 'title' in node and node['title'] and 'children' in node:
                    nodes[node['id']] = node
                    if 'root' in node:
                        roots.append(node)
            # TODO: store groups, but need to start at the root, sort out struct
            # XXX should need a tree formatter here
            print('Groups')
            for nid, node in nodes.items():
                #print repr(node['title']),
                if 'parent' in node:
                    parent = nodes[node['parent']]
                    self.rsr_add_group( node['title'], parent['title'],
                            sa=sa, opts=opts )
                else:
                    self.rsr_add_group( node['title'], None,
                            sa=sa, opts=opts )
            print('Roots')
            for root in roots:
                print(root['id'], root['title'])

### Commands

def cmd__dlcs_import(FILE, opts, g):

    """
    Import from (old pre-2009) del.icio.us posts XML export using
    pydelicious library.
    """
    global ctx

    importer = res.bm.BmImporter(ctx.sa_session)
    data = dlcs_parse_xml(open(FILE).read())

    # Validate URL, track tag/domain and create records where missing
    for post in data['posts']:
        href = post['href']
        dt = datetime.strptime(post['time'], ISO_8601_DATETIME)
        lctr = importer.init_locator(href, dt)
        if not lctr:
            continue

        tagcsv = unicode(post['tag'].replace(' ', ', '))
        bm = importer.init_bookmark(lctr, dt,
                post['description'], post['extended'], tagcsv )
        if not bm:
            continue

        # commit every x records (if not dry-run)
        importer.batch_flush(g)

    log.std("Checked %i locator references", len(data['posts']))
    # Commit (if not dry-run)
    importer.flush(g)

    # Commit (proc/fetch/init) Domain list, see --domain-offset.
    importer.flush_domains(g)

    # Commit (proc/fetch/init) Tag list, see  --tag-offset
    importer.flush_tags(g)

def cmd__html_groups(HTML, g):
    """
    TODO: work on just the groups (folders) extracted from the bookmark.html.
    """
    soup = bs4.BeautifulSoup(open(HTML), "html5lib")
    items = bm.TxtBmOutline.bm_html_soup_items_gen(soup)
    if not items:
        raise Exception("No definition lists in %s" % HTML)
    paths = []
    for lbl, attr in items:
        if 'href' in attr: continue
        if 'parent' in attr:
            path = attr['parent'] +" / "+ lbl
        else:
            path = lbl
        if path in paths: continue
        paths.append(path)
    print("\n".join(paths))

def cmd__html_tree(HTML, g):
    """
    Extract folder/bookmark item lines from HTML.
    """
    soup = bs4.BeautifulSoup(open(HTML), "html5lib")
    print(bm.html_soup_formatters[g.output_format](soup))

def cmd__html_import(HTML, g):
    """
    Import bookmarks from HTML
    """
    global ctx
    soup = bs4.BeautifulSoup(open(HTML), "html5lib")
    bm.TxtBmOutline.bm_html_soup_parse_to_sa(soup, ctx.sa_session)

def cmd__html_stats(HTML, g):
    """
    Show stats for bookmarks HTML
    """
    global ctx
    soup = bs4.BeautifulSoup(open(HTML), "html5lib")

    ctx.folders_cnt = 0
    def _folder(label, attrs):
        global ctx
        ctx.folders_cnt += 1

    ctx.items_cnt = 0
    def _item(label, attrs):
        global ctx
        ctx.items_cnt += 1

    items = bm.TxtBmOutline.bm_html_soup_items_gen(soup,
            folder_f=_folder, item_f=_item)
    if not items:
        raise Exception("No definition lists in %s" % HTML)
    for it in items: pass

    print("%i items" % ctx.items_cnt)
    print("%i folders" % ctx.folders_cnt)

def cmd__html_export(HTML, g):
    """
    Export bookmarks to HTML
    """
    global ctx

    bms_iter = Bookmark.all()
    for bm in bms_iter:
        print(bm)
        # TODO: save folders

def cmd__chrome_all(g):
    "List Chrome bookmarks (from JSON) in different formats"
    bms = bm_chrome.BookmarksJSON.load(g.chrome_bookmarks_file)
    if g.chrome_bookmarks_root:
        roots = [ g.chrome_bookmarks_root ]
    else:
        roots = dict(bms.roots()).keys()

    for root in roots:
        group = confparse.Values(bms.data['roots'][root])
        bm.moz_json_printer['item'][g.output_format](group)

def cmd__chrome_roots(g):
    bms = bm_chrome.BookmarksJSON.load(g.chrome_bookmarks_file)
    for k, v in bms.data['roots'].items():
        if v['type'] == 'folder': print(k)

def cmd__chrome_groups(g):
    """
    List Chrome bookmarks folder paths (from JSON). See also outline format with
    ``bookmarks.py chrome all``.
    """
    bms = bm_chrome.BookmarksJSON.load(g.chrome_bookmarks_file)
    for it in bm_chrome.flatten_norecurse(list(bms.groups())):
        print(it)

def cmd__chrome_import(g):
    """
    Import Chrome Bookmarks JSON to SQL.
    """

    global ctx

    importer = res.bm.BmImporter(ctx.sa_session)
    bms = bm_chrome.BookmarksJSON.load(g.chrome_bookmarks_file)

    if g.chrome_bookmarks_root:
        datagen = bms.urls_gen(bms.data['roots'][g.chrome_bookmarks_root])
    else:
        datagen = bms.urls_gen()

    # Validate URL, track tag/domain and create records where missing
    urlcnt, bmcnt = 0, 0
    for groups, bm in datagen:
        urlcnt += 1
        dt = res.dt.parse_chrome_microsecondstamp(int(bm['date_added']))
        lctr = importer.init_locator(bm['url'], dt)
        if not lctr:
            continue

        tagcsv = ', '.join(groups[1:])
        bm = importer.init_bookmark(lctr, dt, bm['name'], None, tagcsv)
        if not bm:
            continue
        bmcnt += 1

        # commit every x records (if not dry-run)
        importer.batch_flush(g)

    log.std("Checked %i urls", urlcnt)
    log.std("Added %i bookmarks", bmcnt)
    # Commit (if not dry-run)
    importer.flush(g)

    # Commit (proc/fetch/init) Domain list, see --domain-offset.
    importer.flush_domains(g)

    # Commit (proc/fetch/init) Tag list, see  --tag-offset
    importer.flush_tags(g)

def cmd__sql_stats(g):

    global ctx
    sa = ctx.sa_session

    for stat, label in (
                (sa.query(Locator).count(), "Number of Locators: %s"),
                (sa.query(Bookmark).count(), "Number of bookmarks: %s"),
                (sa.query(Domain).count(), "Number of domains: %s"),
                (sa.query(Tag).count(), "Number of tags: %s"),
            ):
        log.std(label, stat)

def cmd__sql_cleanup(g):

    global ctx
    sa = ctx.sa_session

    for bm in sa.query(Bookmark).filter(Bookmark.locator_id==None).all():
        bm.delete()
        log.note("Deleted bookmark without ref %s", bm)

def cmd__href(NAME, g):
    """List hyper-references"""
    sa = Locator.get_session(g.session_name, g.dbref)
    if NAME:
        rs = Locator.search(ref=NAME)
    else:
        rs = Locator.all()
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
    for r in rs:
        print(r.ref)

def cmd__domain(NAME, g):
    """List domains"""
    sa = Domain.get_session(g.session_name, g.dbref)
    if NAME:
        rs = Domain.search(name=NAME)
    else:
        rs = Domain.all()
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
    for r in rs:
        print(r.name)

def cmd__tag(NAME, g):
    """Get tag"""
    sa = Tag.get_session(g.session_name, g.dbref)
    if NAME:
        rs = Tag.search(name=NAME)
    else:
        rs = Tag.all()
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
    for r in rs:
        print(r.name)

def cmd__tags(TAGS, g):
    """
    Dump all tags found on bookmarks. Filter by given tag, fuzzy by default.
    Print all tags, ie. related tags (on the same record) too.
    """
    global ctx

    tags = Bookmark.unique_tags(TAGS, g, ctx)
    if tags:
        for t in tags:
            if g.matched_only:
                if g.exact_match:
                    if t in TAGS:
                        print(t)
                else:
                    for n in TAGS:
                        if n in t:
                            print(t)
            else:
                print(t)
        ctx.note("%i unique tags found on bookmarks", len(tags))

def cmd__urls(REF, g, opts):
    """List URLs from SQL.

    Unless --exact-match is given, inputs are turned into LIKE expressions for
    partial match.
    """
    global ctx

    filters = ctx.opts_to_filters(Locator)
    if REF:
        filters += ( sql_like_val(Locator.ref, REF, g), )
    if g.count:
        cnt = ctx.sa_session.query(Locator).filter(*filters).count()
        log.std("Records matched: %s", cnt)
        return

    rs = Bookmark.all(filters=filters)
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict: return 1
    for r in rs:
        print(r)
    # FIXME ctx.lines_out(rs)

def cmd__list(NAME, g, opts):

    """
    List bookmarks given NAME, TAGS filter from SQL.

    All matches are partial (delimited by '%' and turned into LIKE expression),
    unless --exact-match is given.
    Asterisks '*' in any of the inputs are also handled as explicit LIKE expression.
    """
    global ctx

    filters = ctx.opts_to_filters(Bookmark)
    for f in filters:
        print(f)
    if g.tags:
        filters += tuple([ sql_like_val(Bookmark.tags, T) for T in g.tags ])
    if NAME:
        filters += ( sql_like_val(Bookmark.name, NAME), )

    if g.count:
        cnt = ctx.sa_session.query(Bookmark).filter(*filters).count()
        log.std("Records matched: %s", cnt)
        return

    rs = Bookmark.all(filters=filters)
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
        return

    render = ctx.get_renderer('bookmark')
    for r in rs:
        print(render(r.to_dict()))
    ctx.flush()
    print('%i found' % len(rs))

def cmd__add(REF, NAME, TAGS, g, sa=None):
    global ctx
    bm = Bookmark.forge(REF, NAME, TAGS, g, ctx.sa_session)
    # TODO: ctx.sa.commit
    print(ctx.get_renderer('bookmark')(bm.to_dict()))
    #if not g.dry_run: sa.commit()

def cmd__modify(REF, g):
    assert 'todo' # TODO: bookmarks.py modify HREF

def cmd__remove(REF, g):
    "Soft-delete Bookmark and Locator by Locator"
    global ctx
    sa = ctx.sa_session

    lctr = sa.query(Locator).filter(Locator.ref == REF).one()
    bm = sa.query(Bookmark).filter(Bookmark.location == lctr).one()

    bm.delete()
    lctr.delete()
    if not g.dry_run:
        sa.add(lctr)
        sa.add(bm)
        if not g.dry_run: sa.commit()
        ctx.note("Deleted %s", REF)
    print(lctr, bm)

def cmd__check(NAME, g):

    """
    Update last-seen time.

    Visit the Locator for each bookmark, and update the last-seen date if
    successful. Options below apply filters and specific actions.

    If no filters are provided, the default is to select Locators not seen in
    the last 52 weeks, and all those without status or last-seen fields.

    --deleted
    --weeks, --days
        Select only locators not seen within the last N weeks or days.
    --status [xxx]
        Select only locators with last recorded status.
    --include-null
        Applies to last-seen, dont exclude null (weeks, days, status)
    --delete [0,4,5,xxx]
        On resolve or HTTP error delete the Locator.

        To delete only on specific
        cases, pass 0 for just name resolution or network errors, a single
        digit for 4xx or 5xxx errors, or any specific HTTP status code.
    --delete-error
        Shortcut to --delete 0,4,5
    """

    sa = Locator.get_session(g.session_name, g.dbref)

    delete = []
    if opts.flags.delete_error:
        delete = [ 0, 4, 5 ]
    elif opts.flags.delete:
        delete = [ int(i) for i in opts.flags.delete.split(',') ]

    f = []

    if opts.flags.status:
        for s in opts.flags.status.split(','):
            if s > 99:
                if opts.flags.include_null:
                    f.append(or_(
                        Resource.status.is_(None),
                        Resource.status == s
                    ))
                else:
                    f.append( Resource.status == s )
            else:
                if opts.flags.include_null:
                    f.append(or_(
                        Resource.status.is_(None),
                        Resource.status >= s*100 and Resource.status < (s+1)*100
                    ))
                else:
                    f.append(
                        Resource.status >= s*100 and Resource.status < (s+1)*100
                    )

    if opts.flags.weeks:
        w = int(opts.flags.weeks)
        if opts.flags.include_null:
            f.append(or_(
                Resource.last_access.is_(None),
                Resource.last_access < (
                    datetime.utcnow() - timedelta(weeks=w))
            ))
        else:
            f.append( Resource.last_access < (
                datetime.utcnow() - timedelta(weeks=w)) )

    elif opts.flags.days:
        d = int(opts.flags.days)
        if opts.flags.include_null:
            f.append(sa.or_(
                Resource.last_access.is_(None),
                Resource.last_access < (datetime.utcnow() - timedelta(days=d))
            ))
        else:
            f.append( Resource.last_access < (
                datetime.utcnow() - timedelta(days=d)) )

    if not f:
        if opts.flags.include_null:
            f.append(or_(
                    Resource.last_access < (
                        datetime.utcnow() - timedelta(weeks=52)),
                    Resource.status.is_(None)
                ))
        else:
            f.append(or_(
                    Resource.last_access < (
                        datetime.utcnow() - timedelta(weeks=52)),
                    Resource.last_access.is_(None),
                    Resource.status.is_(None)
                ))

    if not opts.flags.deleted:
        f.append( Resource.deleted!=True )

    rs = Resource.all(f)
    print('%i Locator\'s to check' % len(rs))

    for i, r in enumerate(rs):
        ref = r.location.href()
        print(i, r.status, r.deleted, r.last_access, ref)
        if not g.dry_run:
            if i and ( i % 10 ) == 0:
                sa.commit()
                print('committed at %s items, %i to go' % ( i, len(rs)-i ))

        try:
            urlinfo = urllib2.urlopen(ref, timeout=9)
        except urllib2.HTTPError as e:
            r.status = e.code
            if e.code in delete:
                r.delete()
            sa.add(r)
            continue
        except urllib2.LocatorError as e:
            r.status = -2
            if 0 in delete or -1 in delete:
                r.delete()
            sa.add(r)
            continue
        except Exception as e:
            print(e, ref)
            r.status = -1
            if 0 in delete or -1 in delete:
                r.delete()
            sa.add(r)
            print(0, ref)
            continue

        info = urlinfo.info()
        status = urlinfo.getcode()
        if r.status == None or status != r.status:
            r.status = status
            sa.add(r)
            print(status, ref)

        if status == 200:
            r.last_access = datetime.now()
            sa.add(r)

        if status and ( status in delete or status/100 in delete ):
            r.delete()
            sa.add(r)

        #print i, r.status, r.deleted, r.last_access, ref
    if not g.dry_run:
        sa.commit()

def cmd__assert(REF, NAME, TAGS, g):

    """
    Create an Locator record if it does not exist yet.

    If NAME and optionally TAGS is given, create a bookmark too but only
    if it does not exist yet.
    """

    sa = Bookmark.get_session(g.session_name, g.dbref)
    bm = Bookmark.forge(REF, NAME, TAGS, g, sa)
    print(bm)
    if g.dry_run:
        log.std("Dry run")
    else:
        sa.commit()

def cmd__show(REF, NAME, TAGS, g):
    global ctx
    sa = ctx.sa_session
    filters = ()
    if not g.deleted:
        filters = ( Bookmark.deleted != True, )

    if REF:
        lctr = sa.query(Locator).filter(Locator.ref == REF).one()
        filters += ( Bookmark.location == lctr, )

    if TAGS:
        filters += tuple([ sql_like_val(Bookmark.tags, T, g) for T in TAGS ])

    if NAME:
        filters += ( sql_like_val(Bookmark.name, NAME, g), )

    if filters:
        rs = Bookmark.all(filters=filters)
    else:
        rs = Bookmark.all()
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
        return

    ctx.out(rs, 'bookmark')
    ctx.note("%i items", len(rs))

def cmd__webarchive(NAME, g):

    """
    Sort out and rewrite web.archive locators.

    http://web.archive.org/web/20030208015752/
    """
    global ctx
    sa = ctx.sa_session

    NS_WA = 'http://web.archive.org/web'
    ns_lctr = Locator.fetch((Locator.ref == NS_WA,), exists=False)
    if not ns_lctr:
        ns_lctr = Locator( ref=NS_WA )
        ns_lctr.init_defaults()
        sa.add(ns_lctr)

    rcres_type = Namespace.fetch(( Namespace.location == ns_lctr ,),
            exists=False)
    if not rcres_type:
        rcres_type = Namespace( location=ns_lctr )
        rcres_type.init_defaults()
        sa.add(rcres_type)

    if g.commit:
        sa.commit()

    rs = sa.query(Locator).filter(
            Locator.ref.like('%/web.archive.org/web/%'),
            Locator.deleted != True).all()
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
        return
    total_records = len(rs)

    i = 0
    for lctr in rs:
        assert 'web.archive.org/web' in lctr.ref

        i += 1
        bm = Bookmark.fetch((Bookmark.location == lctr,), exists=False)
        if not bm:
            continue

        cache_path = lctr.ref.split('web.archive.org/web/')[1]
        p = cache_path.index('/')
        res_ts = cache_path[:p]
        res_url = cache_path[p+1:]
        if not uriref.absoluteURI.match(res_url):
            res_url = 'http://'+res_url

        lctr_new = Locator.fetch((Locator.ref == res_url,), exists=False)
        if not lctr_new:
            lctr_new = Locator( ref=res_url )
            lctr_new.init_defaults()
            log.std("New: %s", lctr_new.href() )

        if res_ts.isdigit():
            DT_FMT='%Y%m%d%H%M%S'
            dt = datetime.strptime(res_ts, DT_FMT)
        else:
            log.err("Unknown date tag: %s", res_ts)
            continue

        rcres = RemoteCachedResource( location=lctr, rcres_type=rcres_type,
                last_access=bm.last_access,
                last_update=bm.last_update,
                last_modified=bm.last_modified,
                status=bm.status,
                date_added=lctr.date_added,
                date_deleted=lctr.date_deleted,
                date_updated=dt )

        if not g.dry_run:
            sa.add(lctr_new)
            sa.add(rcres)
            #sa.delete(lctr)
        if g.auto_commit and i and i % g.auto_commit == 0:
            if g.verbose > 1:
                ctx.note("Auto-commit at %i of %i", i, total_records)
            sa.commit()

        ctx.note('TODO %s', lctr.href())
    if g.commit:
        sa.commit()
    ctx.note("Found %i instances", i)

# TODO: bookmarks.py sync/update
def cmd__sync(g): pass
def cmd__update(g): pass
def cmd__x(g):

    """
    XXX: hacky hack hack
    """
    global ctx
    sa = ctx.sa_session

    bms = bm_chrome.BookmarksJSON.load(g.chrome_bookmarks_file)
    chrome_refs = []
    for groups, href in bms.urls_gen('url'):
        chrome_refs.append(href)

    for lctr in Locator.all():
        if lctr.href in chrome_refs:
            continue

        print(lctr)
        continue

        lctr = Locator.fetch((Locator.ref==href,), sa=sa, exists=False)
        if not lctr:
            print(href)
        continue

# WIP: bookmarks.py couchdb

def cmd__couch_stats(g):
    global ctx
    stats = ctx.couchconn.stats()
    print('# couchdb-stat current max min mean stddev sum description')
    for k in stats['couchdb']:
        print(k, end=' ')
        for k2 in 'current', 'max', 'min', 'mean', 'stddev', 'sum', 'description':
            print(stats['couchdb'][k][k2], end=' ')
        print()

couch_views = {
    '_design/bookmarks': dict( views=dict( list=dict( map= """function(doc) {

    if (doc.type && doc.type == 'bookmark') emit(doc.id, doc.href);

}"""))),
}

def cmd__couch_init(g):

    """
    /<db-name>/_design/<design-doc>/_view/<view-name>
    """
    global ctx

    for k, v in couch_views.items():
        if k in ctx.docs:
            if g.update:
                ctx.note("Updating view %s...", k)
                del ctx.docs[k]
        else:
            ctx.note("Adding view %s...", k)
        if k not in ctx.docs:
            ctx.docs[k] = v

def cmd__couch_list(g):
    global ctx
    for ls in ctx.docs.view('bookmarks/list'):
        print(ls.id, ls.value)

def cmd__couch_add(Locator, TITLE, TAGS, g):
    """
    Add record directly to Couch. NOTE: should be using SQL API instead.
    """
    global ctx
    #[Locator] = {
    #  'type': 'bookmark',
    #  'href': Locator,
    #  'tags': TAGS
    #}
    assert isinstance(TAGS, list), TAGS
    bm = bookmark.Bookmark(id=Bookmark.keyid(Locator), href=Locator, tag_list=TAGS)
    bm.store(ctx.docs)

def cmd__sql_couch(g):

    """
    Update SQL DB Bookmark records from CouchDB.
    """

    global ctx
    total_docs = len(ctx.docs)
    i, c = 0, 0

    sa = Bookmark.get_session(g.session_name, g.dbref)
    for idref in ctx.docs:
        bmdoc = bookmark.Bookmark.load(ctx.docs, idref)
        zope.interface.classImplements(bmdoc.__class__, taxus.iface.IPyDict)
        i += 1
        ctx.note("Processing %i of %i", i, total_docs, num=i)

        lctr = Locator.fetch((Locator.ref == bmdoc.href,), exists=False)
        if not lctr:
            lctr = Locator.forge(bmdoc, g, sa=sa)

        bm = Bookmark.fetch((Bookmark.location == lctr,), exists=False)
        if not bm:
            bm = Bookmark.fetch((Bookmark.name == bmdoc.name,), exists=False)
            if bm:
                ctx.note("Name already exists for other location: %r at %r vs %r",
                        bmdoc.name, bmdoc.href, bmdoc.href )
                continue
            bm = Bookmark.forge(dict(location=lctr, **bmdoc), g, sa=sa)

        if not bm.update_from(bmdoc, location=lctr):
            if g.verbose > 2:
                ctx.note("No-op: %s", bmdoc.href)
            continue

        c += 1
        if not g.dry_run:
            sa.add(bm)
        if not g.quiet:
            ctx.note(bmdoc.href)

        if g.auto_commit and c and c % g.auto_commit == 0:
            if g.verbose > 1:
                ctx.note("Auto-commit at %i of %i", c, total_docs)
            sa.commit()

    if g.commit:
        sa.commit()

def cmd__couch_sql(NAME, opts, g):

    """
    Update CouchDB bookmark-type documents from SQL.
    """

    global ctx
    total_docs = len(ctx.docs)
    c, i = 0, 0

    # Get records to sync
    rs = ctx.get_records(Bookmark, name=NAME)
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
        return
    total_records = len(rs)
    ctx.note('Going to sync %i SQL records to %s...', total_records, ctx.couch[1])

    n, u, e = 0, 0, 0
    for r in rs:
        d = r.to_doc()
        i += 1
        _id = d.id

        map_id = d.href in ctx.docs
        if map_id:
            e += 1
            log.stderr("HREF %s", d.href)
            continue

        if _id not in ctx.docs:

            # New entry
            if not g.dry_run:
                d.store(ctx.docs)
            n += 1
            ctx.note('new %s', d.href)

        else:
            bm = bookmark.Bookmark.load(ctx.docs, _id)
            updated = False

            # NOTE: not doing anything else than bookmarks
            if bm.type != 'bookmark':
                log.stderr((
                    "Document {0} exists but is not of required type: {1}".format(
                    href, bm['type'])))
                continue

            # We can stop right there.
            if r.deleted:
                if bm.deleted:
                    continue
                bm.deleted = true
                bm.date_deleted = r.date_deleted
                # Soft delete, for archival/cleanup by someone else.
                continue

            # Check on update time to things speed up
            if g.clear_unknown_keys:
                c_dt = iso8601_datetime_format(bm.date_updated)
                if c_dt >= r.date_updated:
                    continue

            # Delete missing keys
            #if g.clear_unknown_keys:
            #    for k in bm.keys():
            #        if k in [ '_rev', '_id', 'type' ]:
            #            continue
            #        if k not in d:
            #            del bm[k]
            #            updated = True

            #if r.update_from(bm)
            #raise NotImplementedError
            # Set new or changed values
            for k in d.keys():
              if k not in bm or bm[k] != d[k]:
                  print(k, d[k], bm[k])
                  bm[k] = d[k]
                  updated = True

            if updated:
                # XXX: should I take now? rather not increment on non-descr changes
                bm['date_updated'] = r.date_updated
                if not g.dry_run:
                    bm.store(ctx.docs)
                u += 1
                if g.verbose:
                    print('updated %s' % href)

        if u:
            ctx.note("Updated doc-from-record %i, at %i from %i...", u, i, total_records, num=u)
        elif i:
            ctx.note("No doc-from-record changes at %i from %i...", i, total_records, num=i)

    ctx.note('%i new', n)
    ctx.note('%i updated', u)

def cmd__couch_update(g):

    """
    Update couchdb, preping data and fixing common errors before doing actual
    syncs. Fixes are controlled through flags.

    Unless no-db is on, an initial second run is done using all records from
    SQL. This allows to fix records using data from SQL.
    """

    global ctx
    total_docs = len(ctx.docs)
    # doc num, record num, updated from SQL, updated, skip
    d, i, c, u, e = 0, 0, 0, 0, 0

    if not g.no_db:
        # Starting with records from SQL, check every one with couch
        rs = ctx.get_records(Bookmark)
        if not rs:
            log.stdout("{yellow}Nothing found{default}")
            if g.strict:
                return 1
            return
        total_records = len(rs)
        ctx.note('Going to sync %i SQL records to %s...', total_records, ctx.couch[1])

        for r in rs:
            i += 1

            ### Fix document, or skip further updates by continueing to next doc

            # ID rename cleanup
            if g.update_hrefids:
                map_id = r.href in ctx.docs
                if map_id:
                    u += 1
                    doc = r.to_doc()
                    _id = doc.id
                    if not g.dry_run:
                        del ctx.docs[doc.href]
                        assert doc.href not in ctx.docs, doc.href
                        doc.store(ctx.docs)
                    log.stderr("Renamed href at %i to %s", i, _id)
                    continue

            if g.update_tagstype:
                _id = Bookmark.keyid(r.href)
                doc = ctx.docs[_id]
                if 'tags' in doc and isinstance(doc['tags'], list):
                    tags_split_from_string_error = None
                    for t in doc['tags']:
                        if len(t) == 1:
                            tags_split_from_string_error = True
                        if len(t) != 1:
                            tags_split_from_string_error = False
                    if tags_split_from_string_error:
                        u += 1
                        tags = "".join(doc['tags'])
                        doc['tag_list'] = tags.split(', ')
                        del doc['tags']
                        ctx.docs[doc.id] = doc

                elif 'tags' in doc:
                    u += 1
                    tags = doc['tags']
                    assert isinstance(tags, str), tags
                    assert ', ' in tags, tags
                    doc['tag_list'] = tags.split(', ')
                    del doc['tags']
                    ctx.docs[doc.id] = doc

            # TODO: other generic SQL-to-Couch updates here
            if u:
                ctx.note("Updated doc-from-record %i, at %i from %i...", u, i, total_records, num=u)
            elif i:
                ctx.note("No doc-from-record changes at %i from %i...", i, total_records, num=i)

    if c:
        ctx.note("Updated %i from %i", c, total_records)
    else:
        ctx.note("Nothing to do for %i SQL records", total_records)

    # Now loop over ID's from database again
    log.std("Checking %i docs..", total_docs)
    for idref in ctx.docs:
        d += 1
        doc = ctx.docs[idref]

        ### Fix document, or skip further updates by continueing to next doc

        # ID rename cleanup
        if idref == doc['href']:
            if not g.update_hrefids:
                e += 1
                continue
            c += 1
            bm = bookmark.Bookmark.load(ctx.docs, idref)
            bm.id = bookmark.Bookmark.key(bm)
            if not g.dry_run:
                bm.store(ctx.docs)
        assert not doc['href'] in ctx.docs, doc['href']

        # Migrate tags
        if 'tags' in doc:
            if not g.update_tagstype:
                e += 1
                continue
            c += 1
            bm = bookmark.Bookmark.load(ctx.docs, idref)

            if g.verbose > 1:
                print(idref, doc['href'], bm.date_added )

            if isinstance(doc['tags'], list):
                bm.tag_list = bm.tags
                assert isinstance(bm.tags, list)

            else:
                assert ', ' in bm.tags
                bm.tag_list = bm.tags.split(', ')

            del bm['tags']
            if not g.dry_run:
                #del ctx.docs[idref]
                bm.store(ctx.docs)

        if c:
            ctx.note("Updated %i, at %i from %i...", d, i, total_docs, num=c)
        else:
            ctx.note("Nothing to do at %i from %i...", i, total_docs, num=i)

    if c:
        ctx.note("Updated %i from %i", d, total_docs)
    else:
        ctx.note("Nothing to do for %i documents", total_docs)

"""
TODO: sync shaarli either from SQL or couch.

Also google/firefox bookmarks, chrome outliner.
"""

from shaarli_client.client import ShaarliV1Client, InvalidEndpointParameters

def cmd__shaarli_update(g):
    """
    TODO: Update SQL DB Bookmark records from Shaarli.
    """

def cmd__shaarli_sync(NAME, opts, g):
    """
    TODO: Update Shaarli bookmark-type documents from SQL.
    """

def cmd__stats(g):
    cmd__sql_stats(g)

### Transform cmd__ function names to nested dict

commands = libcmd_docopt.get_cmd_handlers(globals(), 'cmd__')
commands.update(dict(
        help = libcmd_docopt.cmd_help,
        memdebug = libcmd_docopt.cmd_memdebug,
        info = db_sa.cmd_info,
        init = db_sa.cmd_init,
        clear = db_sa.cmd_reset
))

### Util functions to run above functions from cmdline

def defaults(opts, init={}):
    global cmd_default_settings, ctx
    libcmd_docopt.defaults(opts)
    opts.flags.update(cmd_default_settings)
    ctx.settings.update(opts.flags)
    opts.flags.update(ctx.settings)
    opts.flags.update(dict(
        commit = not opts.flags.no_commit and not opts.flags.dry_run,
        verbose = opts.flags.quiet and opts.flags.verbose or 1,
    ))
    if not opts.flags.interactive:
        if os.isatty(sys.stdout.fileno()) and os.isatty(sys.stdout.fileno()):
            opts.flags.interactive = True
    opts.flags.update(dict(
        partial_match = not opts.flags.exact_match,
        auto_commit = not opts.flags.no_commit and opts.flags.auto_commit,
        dbref = taxus.ScriptMixin.assert_dbref(opts.flags.dbref)
    ))
    if opts.flags.interval:
        opts.flags.interval = int(opts.flags.interval)
    if opts.flags.auto_commit:
        opts.flags.auto_commit = int(opts.flags.auto_commit)
    return init

def main(opts):

    """
    Execute using docopt-mpe options.
    """
    global ctx, commands

    ws = Homedir.require()
    ws.yamldoc('bmsync', defaults=dict(
            last_sync=None
        ))
    ctx.ws = ws
    ctx.settings = settings = opts.flags
    ctx.init()
    ret = libcmd_docopt.run_commands(commands, settings, opts)
    if settings.print_memory:
        libcmd_docopt.cmd_memdebug(settings)
    return ret

def get_version():
    return 'bookmarks.mpe/%s' % __version__

if __name__ == '__main__':
    #from importlib import reload
    #reload(sys)
    # sys.setdefaultencoding('utf-8') FIXME: see 'why we should not use ...'

    usage = __description__ +'\n\n'+ __short_description__ +'\n'+ \
            libcmd_docopt.static_vars_from_env(__usage__,
        ( 'BM_DB', __db__ ),
        ( 'COUCH_DB', __couch__ ) )

    db_sa.schema = sys.modules['__main__']
    db_sa.metadata = SqlBase.metadata

    opts = libcmd_docopt.get_opts(usage,
            version=get_version(), defaults=defaults)
    # TODO: mask secrets
    #log.std("Connecting to %s", opts.flags.dbref)
    #log.std("Connecting to %s", opts.flags.couch)
    sys.exit(main(opts))


import gensim
import nltk
import sqlalchemy

import database.tools
import database.models

from configobj import ConfigObj
_config = ConfigObj('settings.cfg')

def database_lyrics():
    """Iterates over the lyrics for each song in the database."""
    with database.tools.session_scope() as session:
        song_query = session.query(database.models.Song)
        song_query = song_query.order_by(database.models.Song.id)

        for song in song_query:
            yield song.lyrics

class LyricCorpus:
    """A corpus that yields the bag-of-words vector for each song's lyrics."""

    def __init__(self):
        """Initialises the corpus dictionary for use when yielding vectors."""
        self.dictionary = gensim.corpora.Dictionary.load(
            _config['DICTIONARY_FILENAME']
        )

    def __iter__(self):
        """Yields the bag-of-words vector for each song's lyrics."""
        for lyrics in database_lyrics():
            lyric_tokens = nltk.word_tokenize(lyrics)
            yield self.dictionary.doc2bow(lyric_tokens)

    def __len__(self):
        """Returns the number of songs in the corpus."""
        with database.tools.session_scope() as session:
            count_query = session.query(
                sqlalchemy.func.count(database.models.Song.id)
            )

            return count_query.scalar()


# coding:utf8
"""
切割大的日志文件为一个个的小文件
"""
import threading
import time
import os
import shutil
import re
import psutil

# 分割日志临时目录
TMP_DIR = '/data/logs/nginx_tmp/'
NGINX_LOG_DIR = '/data/logs/nginx/'

def init():
    """
    初始化所有为临时文件夹
    """
    # 分割日志临时目录
    if os.path.exists(TMP_DIR):
        os.makedirs(TMP_DIR)
    if not os.path.exists(NGINX_LOG_DIR):
        raise Exception(
            'Not found nginx log directory:{}'.format(NGINX_LOG_DIR))

def split():
    """split: 切割日志"""
    # 此数字是按字节进行计算，这里大小为内存除以cpu核数剩以0.5得到的结果为500M
    cpu_count = psutil.cpu_count()
    memory_size = psutil.total
    sizehint = int(memory_size / cpu_count * 0.5 * 1024 * 1024)

    for nginx_log_file in os.listdir(NGINX_LOG_DIR):
        nginx_log_path = NGINX_LOG_DIR + nginx_log_file
        with open(nginx_log_path) as fp:
            position = 0
            file_num = 1
            while True:
                lines = fp.readlines(sizehint)
                # 分割成功的文件名
                split_file_name = TMP_DIR + \
                    os.path.splitext(nginx_log_file)[0] + '_' + file_num + '.log'
                with open(split_file_name, 'w') as split_fp:
                    split_fp.writelines(lines)
                if fp.tell() - position <= 0:
                    break
                # 如果分割的位置大于默认位置就继续执行，否则就退出。
                position = fp.tell()  # 替换位置
                file_num = file_num + 1
        #  os.remove(nginx_log_path)
        time.sleep(5)

if __name__ == '__main__':
    # 执行程序开始时间
    print('Start at:', int(time.strftime('%H%M%S')))
    split()
    print('Stop at:', int(time.strftime('%H%M%S')))


from abc import ABC
from typing import Optional, List

import gym
import numpy as np
from gym.spaces import Box
from ray.rllib.algorithms.ddpg.ddpg_torch_model import DDPGTorchModel
from ray.rllib.utils import try_import_torch
from ray.rllib.utils.typing import ModelConfigDict, TensorType

from src.algorithms.models.models import BatchScaling

torch, nn = try_import_torch()

class IndependentDDPGModel(DDPGTorchModel, ABC):

    def __init__(
            self,
            obs_space: gym.spaces.Space,
            action_space: gym.spaces.Space,
            num_outputs: int,
            model_config: ModelConfigDict,
            name: str,
            actor_hiddens: Optional[List[int]] = None,
            actor_hidden_activation: str = "relu",
            critic_hiddens: Optional[List[int]] = None,
            critic_hidden_activation: str = "relu",
            twin_q: bool = False,
            add_layer_norm: bool = False,
            **kwargs
    ):
        assert isinstance(action_space, Box)
        DDPGTorchModel.__init__(
            self,
            obs_space,
            action_space,
            num_outputs,
            model_config,
            name,
            actor_hiddens,
            actor_hidden_activation,
            critic_hiddens,
            critic_hidden_activation,
            twin_q,
            add_layer_norm
        )

        self.scaling_layer = BatchScaling(self.action_space)

    def max_q_action(self, observation: TensorType, intervals: TensorType) -> TensorType:
        max_action = torch.zeros(observation.size(0), 1)
        max_interval = torch.zeros(intervals.size(0), 2)
        max_q = torch.full((observation.size(0), 1), -np.inf)

        for i in range(0, int(intervals.size(1) / 2), 2):
            new_observation = torch.cat([intervals[:, i:i + 2], observation], 1)
            action = self.policy_model(new_observation)
            q_value = self.q_model(torch.cat([new_observation, action], -1))

            empty = intervals[:, i] != intervals[:, i + 1]
            improved = q_value > max_q
            update = torch.logical_and(empty.reshape(-1, 1), improved)

            max_action[update] = action[update]
            max_interval[torch.cat([update, update], 1)] = intervals[:, i:i+2][torch.cat([update, update], 1)]
            max_q[update] = q_value[update]

        scaled_max_action = self.scaling_layer(max_action, max_interval.reshape(max_interval.size(0), -1, 2))

        return scaled_max_action, max_interval,


from unicodedata import normalize
from collections import defaultdict, OrderedDict

def format_word(word):
    """
    Formate le mot en minuscule et tri les lettres par ordre alphabetique

    :param word: Mot a formater

    :type word: String

    :return: String
    """
    return ''.join(sorted(normalize('NFD', word).encode('ascii', 'ignore').lower().decode('ascii')))

def load_file(fname, encoding):
    """
    Constuit le dictionnaire a partir d'un fichier de mots

    :param fname: Emplacement du fichier
    :param encoding: Encodage du fichier

    :type fname: string
    :type encoding: string

    :return: DefaultDict
    """
    words_dict = defaultdict(lambda: defaultdict(list))

    with open(fname, 'r', encoding=encoding) as f:
        for word in f:
            word = word.rstrip()
            words_dict[len(word)][format_word(word)].append(word)

    return OrderedDict(
        sorted(
            {
                n: OrderedDict(
                    sorted(
                        d.items(),
                        key=lambda t: t[0]
                    )
                ) for n, d in words_dict.items()
            }.items(),
            key=lambda t: t[0]
        )
    )

def est_presque_anagramme(word_1, word_2, gap):
    """
    Verifie si 2 mots sont presques anagrammes

    :param word_1: Premier mot (court)
    :param word_2: Deuxieme mot (long)
    :param gap: Nombre de lettre d'ecart

    :type word_1: String
    :type word_2: String
    :type gap: Integer

    :return: Boolean
    """
    if len(word_1) + gap != len(word_2):
        return False

    word_1 = format_word(word_1)
    word_2 = format_word(word_2)

    for letter in word_1:
        if letter in word_2:
            word_2 = word_2.replace(letter, "", 1)
        else:
            return False

    return True

def presque_anagrammes(words_dict, word, gap):
    """
    Construit la liste des presques anagrammes d'un mot contenu dans un dictionnaire

    :param words_dict: Dictionnaire de mots
    :param word: Mot pour lequel on recherche les presques anagrammes
    :param gap: Nombre de lettre d'ecart

    :type words_dict: OrderedDict
    :type word: String
    :type gap: Integer

    :return: List
    """
    presque_words = []

    n = len(word) + gap
    if n in words_dict.keys():
        for key, words_list in words_dict[n].items():
            if est_presque_anagramme(word, key, gap):
                presque_words += words_list

    return presque_words

if __name__ == "__main__":
    # Chargement du dictionnaire
    FILES = [('../lexiqueODS.txt', 'UTF-8'), ('../liste_de_mots_francais_frgut.txt', 'ISO-8859-1')]

    choix = 0
    words_dict = load_file(FILES[choix][0], FILES[choix][1])

    # Questions
    words = [] # Liste des mots (court) qui ont le plus de presque-anagramme
    sol = [] # Liste des presque-anagrammes
    gap = 1 # Nombre de lettres d'ecart

    for d in words_dict.values():
        for word, l in d.items():
            presque_words = []

            n = len(word) + gap
            if n in words_dict.keys():
                for key, words_list in words_dict[n].items():
                    if est_presque_anagramme(key, word, gap):
                        presque_words += words_list

            if presque_words:
                if sol:
                    if len(sol[0]) < len(presque_words[0]):
                        sol.clear()
                        sol += presque_words
                        words.clear()
                        words += l
                    elif len(sol[0]) == len(presque_words[0]):
                        sol += presque_words
                        words += l
                else:
                    sol += presque_words
                    words += l

    # Reponse 1
    print("Mots ayant le nombre maximal de presque-anagramme :", len(words))

    # Reponse 2
    print("Tous les mots ont la même longueur ? ", len(set([len(w) for w in sol])) == 1)


# -*- coding: utf-8 -*-
"""
This module provides support for converting mime-types and extensions of RDF serializations that are
supported by the LoD app.

Serializations formats that are not in EXTENSION_TO_MIME_TYPE cannot be used during the 303 content negotiation and
data rendering.
"""
from .mimeparse import best_match as lib_best_match

# SPARQL results
JSON_MIME = "application/sparql-results+json"
XML_MIME = "application/sparql-results+xml"

# support RDF mime-types
HTML_MIME = "text/html"
N3_MIME = "text/n3"
TURTLE_MIME = "text/turtle"
RDFXML_MIME = "application/rdf+xml"
NTRIPLES_MIME = "application/n-triples"
JSONLD_MIME = "application/json"
NQUADS_MIME = "application/n-quads"

EXTENSION_TO_MIME_TYPE = {"rdf": RDFXML_MIME, "n3": N3_MIME, "nt": NTRIPLES_MIME, "turtle": TURTLE_MIME,
                          "ttl": TURTLE_MIME, "json-ld": JSONLD_MIME, "json": JSONLD_MIME, 'nquads': NQUADS_MIME}
MIME_TYPE_TO_EXTENSION = dict(list(map(
        reversed,
        [(ext, mime) for ext, mime in list(EXTENSION_TO_MIME_TYPE.items()) if ext not in ['ttl', 'json']]
)))

def mime_to_extension(mime_type):
    """
    Convert mime-type from accept-headings to file extension.
    """
    if mime_type in MIME_TYPE_TO_EXTENSION:
        return MIME_TYPE_TO_EXTENSION[mime_type]
    return "rdf"

def extension_to_mime(extension):
    """Convert file extension to mime-type.

    Some have to be rewritten because the MIME type does not correspond to the file extension.
    """
    if extension == 'ttl':
        extension = 'turtle'
    if extension == 'json':
        extension = 'json-ld'
    if extension == 'nq':
        extension = 'nquads'
    if extension in EXTENSION_TO_MIME_TYPE:
        return extension, EXTENSION_TO_MIME_TYPE[extension]
    return "xml", RDFXML_MIME

def result_extension_to_mime(extension):
    """Convert result extension to MIME type.

     Some have to be rewritten rewritten because the MIME type does not correspond to the file extension.
    """
    if extension == 'xml':
        return XML_MIME
    if extension == 'json':
        return JSON_MIME
    if extension == 'html':
        return HTML_MIME
    return EXTENSION_TO_MIME_TYPE[extension]

def best_match(candidate, header):
    if header and 'text/html' in header:
        return ''
    if header:
        return lib_best_match(candidate, header)
    return None


# -*- codeing = utf-8 -*-
#@Time :2020/11/18 16:08
#@Author :张士澜
#@File :线程池使用.py
#@Software :PyCharm

from multiprocessing.dummy import Pool
import time
import requests

start_time = time.time()
headers = {
        'User-Agent': 'Mozilla / 5.0(Windows NT 10.0;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 86.0.4240.198Safari / 537.36'
    }
urls = [
    'http://downsc.chinaz.net/Files/DownLoad/jianli/202011/jianli13996.rar',
    'http://downsc.chinaz.net/Files/DownLoad/jianli/202011/jianli13987.rar',
    'http://downsc.chinaz.net/Files/DownLoad/jianli/202011/jianli13982.rar'
]

def get_content(url):
    print("正在爬取：",url)
    response = requests.get(url = url,headers = headers)
    if response.status_code == 200:
        return response.content

def parse_content(content):
    print('响应数据长度为：',len(content))

content_list = []
pool = Pool(3)
content_list =pool.map(get_content,urls)
for content in content_list:
    parse_content(content)

end_time = time.time()
print(end_time-start_time)


import os, random
os.environ['CUDA_VISIBLE_DEVICES']="2"

import torch
import argparse
import pytorch_lightning as pl
from pytorch_lightning import Trainer, seed_everything
from transformers import (AdamW, T5Tokenizer, BartTokenizer, BartForConditionalGeneration, T5ForConditionalGeneration, WEIGHTS_NAME,CONFIG_NAME)
from data_loader import prepare_data
from evaluate import evaluate_metrics, compute_acc, compute_prf
from copy import deepcopy
import json
from tqdm import tqdm
from T5 import DST_Seq2Seq

import sys
sys.path.append("..")
# print(sys.path)
import info

def test(args, *more):
    args_1 = deepcopy(args)
    args = vars(args)
    args["model_name"] = args["model_checkpoint"]+ "_slotlang_" +str(args["slot_lang"]) + "_lr_" +str(args["lr"]) + "_epoch_" + str(args["n_epochs"]) + "_seed_" + str(args["seed"]) + "_feedback_" + str(args["train_feedback"])
    save_path = os.path.join(args["saving_dir"], args["dataset"], args["model_name"])

    model = T5ForConditionalGeneration.from_pretrained(save_path)
    tokenizer = T5Tokenizer.from_pretrained(save_path)

    task = DST_Seq2Seq(args, tokenizer, model)

    feedback_model = T5ForConditionalGeneration.from_pretrained(args_1.feedback_model_name)
    feedback_tokenizer = T5Tokenizer.from_pretrained(args_1.feedback_model_name)

    data_test, test_loader, ALL_SLOTS = prepare_data(args, task.tokenizer)

    print("test start...")
    #evaluate model
    _ = evaluate_model(args_1, task.tokenizer, task.model, feedback_model, feedback_tokenizer, data_test, save_path, ALL_SLOTS)

def acquire_state(batch_data, model, tokenizer, device):
    input_batch = tokenizer(batch_data["intput_text"], padding=True, return_tensors="pt", add_special_tokens=False, verbose=False)
    batch_data["encoder_input"] = input_batch["input_ids"]
    batch_data["attention_mask"] = input_batch["attention_mask"]
    dst_outputs = model.generate(input_ids=batch_data["encoder_input"].to(device),
                            attention_mask=batch_data["attention_mask"].to(device),
                            eos_token_id=tokenizer.eos_token_id,
                            max_length=200,
                            )
    value_batch = tokenizer.batch_decode(dst_outputs, skip_special_tokens=True)
    pred_state = {s:v for s, v in zip(batch_data["slot_text"], value_batch) if v != "none"}
    gold_state = batch_data["turn_dialog_state"][0]
    equal = False
    if pred_state == gold_state:
        equal = True
    return pred_state, equal

def acquire_corrected_state(args, batch_data, added_information, deleted_information, model, tokenizer,
    feedback_model, feedback_tokenizer, device):
    feedback_added_information = deepcopy(added_information)
    feedback_deleted_information = deepcopy(deleted_information)
    feedback = info.simulated_negative_feedback(args.dataset, 'test', feedback_added_information, feedback_deleted_information, feedback_model, feedback_tokenizer, device)
    intput_text = [t.replace(batch_data['dialog_history'][i], batch_data['dialog_history'][i] + ' ' + feedback) for i, t in enumerate(batch_data["intput_text"])]
    batch_data["intput_text"] = intput_text
    last_dialog_state_2, equal = acquire_state(batch_data, model, tokenizer, device)
    return last_dialog_state_2, equal

def acquire_pred_gold(args, gold_turn_state, pred_turn_state, last_state, slot_meta):
    if args.feedback_content == 'belief_state':
        pred_state = {s:pred_turn_state.get(s, 'none') for s in slot_meta}
        # print(pred_state)
        gold_state = {s:gold_turn_state.get(s, 'none') for s in slot_meta}
        # print(gold_state)
    elif args.feedback_content == 'turn_label':
        pred_state = {s:pred_turn_state.get(s, 'none') for s in slot_meta if pred_turn_state.get(s, 'none') != last_state.get(s, 'none')}
        # print(pred_state)
        gold_state = {s:gold_turn_state.get(s, 'none') for s in pred_state}
        # print(gold_state)
    else:
        print('select feedback_content in belief_state or turn_label')
        exit()
    return pred_state, gold_state

def evaluate_model(args, tokenizer, model, feedback_model, feedback_tokenizer, data_test, save_path, ALL_SLOTS):
    save_path = os.path.join(save_path,"feedback_results")
    if not os.path.exists(save_path):
        os.makedirs(save_path)
    predictions = {}
    # to gpu
    # gpu = args["GPU"][0]
    device = torch.device("cuda:0")
    model.to(device)
    model.eval()

    feedback_model.to(device)
    feedback_model.eval()

    slot_logger = {slot_name:[0,0,0] for slot_name in ALL_SLOTS}

    count = 0
    max_count = len(data_test)
    num_slot = len(ALL_SLOTS)

    slot_turn_acc, joint_acc, slot_F1_pred, slot_F1_count = 0, 0, 0, 0

    add_acc, add_num, delete_acc, delete_num = 0, 0, 0, 0

    task_acc, task_num, session_acc, session_num = 0, 0, 0, 0

    add_feedback_cost, delete_feedback_cost, all_slot, present_slot = 0, 0, 0, 0

    last_dialog_state = {}
    last_added_information = {}
    last_deleted_information = {}
    count_examples = int(max_count / 10)
    while count < max_count:
        # print(count)
        if (count % count_examples) == 0:
            print(count)
        data = data_test[count: count+num_slot]
        count += num_slot
        batch_data = {}
        for key in data[0]:
            batch_data[key] = [d[key] for d in data]
        if batch_data["turn_id"][0] == 0:
            last_dialog_state = {}
            last_added_information = {}
            last_deleted_information = {}
            last_pred_state = {}

        all_slot += len(ALL_SLOTS)

        feedback_active = False
        if args.feedback_timing == 'turn':
            feedback_active = True
        elif args.feedback_timing == 'task':
            if batch_data["task_final"][0] == 1:
                feedback_active = True
        elif args.feedback_timing == 'session':
            if batch_data["session_final"][0] == 1:
                feedback_active = True
        else:
            print('select feedback_timing in turn, task or session')
            exit()

        turn_dialog_state = batch_data["turn_dialog_state"][0]

        if args.feedback_acquisition_strategy == 'explicit':
            last_dialog_state_1, equal = acquire_state(batch_data, model, tokenizer, device)
            pred_state, gold_state = acquire_pred_gold(args, turn_dialog_state, last_dialog_state_1, last_dialog_state, ALL_SLOTS)
            added_information = {key:gold_state.get(key) for key in gold_state.keys() if pred_state.get(key) != gold_state.get(key) and gold_state.get(key) != 'none'}
            deleted_information = {key:pred_state.get(key) for key in gold_state.keys() if pred_state.get(key) != 'none' and gold_state.get(key) == 'none'}

            if feedback_active == True:
                present_slot += len(pred_state)
                add_feedback_cost += len(added_information)
                delete_feedback_cost += len(deleted_information)

            if (added_information != {} or deleted_information != {}) and feedback_active == True:
                last_dialog_state_2, equal = acquire_corrected_state(args, batch_data, added_information, deleted_information, model, tokenizer,
                                            feedback_model, feedback_tokenizer, device)
                last_dialog_state = last_dialog_state_2
                for s in added_information:
                    add_num += 1
                    if last_dialog_state_2.get(s) == turn_dialog_state.get(s):
                        add_acc += 1
                for s in deleted_information:
                    delete_num += 1
                    if last_dialog_state_2.get(s) == turn_dialog_state.get(s):
                        delete_acc += 1
            else:
                last_dialog_state = last_dialog_state_1

        elif args.feedback_acquisition_strategy == 'implicit':
            if feedback_active == True:
                present_slot += len(last_pred_state)
                add_feedback_cost += len(last_added_information)
                delete_feedback_cost += len(last_deleted_information)
            if (last_added_information != {} or last_deleted_information != {}) and feedback_active == True:
                last_dialog_state_2, equal = acquire_corrected_state(args, batch_data, last_added_information, last_deleted_information, model, tokenizer,
                                            feedback_model, feedback_tokenizer, device)
                pred_state, gold_state = acquire_pred_gold(args, turn_dialog_state, last_dialog_state_2, last_dialog_state, ALL_SLOTS)
                last_dialog_state = last_dialog_state_2
                for s in last_added_information:
                    add_num += 1
                    if last_dialog_state_2.get(s) == turn_dialog_state.get(s):
                        add_acc += 1
                for s in last_deleted_information:
                    delete_num += 1
                    if last_dialog_state_2.get(s) == turn_dialog_state.get(s):
                        delete_acc += 1
            else:
                last_dialog_state_1, equal = acquire_state(batch_data, model, tokenizer, device)
                pred_state, gold_state = acquire_pred_gold(args, turn_dialog_state, last_dialog_state_1, last_dialog_state, ALL_SLOTS)
                last_dialog_state = last_dialog_state_1
            added_information = {key:gold_state.get(key) for key in gold_state.keys() if pred_state.get(key) != gold_state.get(key) and gold_state.get(key) != 'none'}
            deleted_information = {key:pred_state.get(key) for key in gold_state.keys() if pred_state.get(key) != 'none' and gold_state.get(key) == 'none'}
            last_added_information = added_information
            last_deleted_information = deleted_information
            last_pred_state = pred_state
        else:
            print('select feedback_acquisition_strategy in explicit or implicit')
            exit()

        pred_state = []
        for k, v in last_dialog_state.items():
            pred_state.append('-'.join([k, v]))

        if equal:
            joint_acc += 1

        if batch_data["task_final"][0] == 1:
            task_num += 1
            if equal:
                task_acc += 1

        if batch_data["session_final"][0] == 1:
            session_num += 1
            if equal:
                session_acc += 1

        turn_state = batch_data["turn_belief"][0]
        # print('turn_state', turn_state)
        # print('pred_state', pred_state)

        # Compute prediction slot accuracy
        temp_acc = compute_acc(set(turn_state), set(pred_state), ALL_SLOTS)
        slot_turn_acc += temp_acc

        # Compute prediction F1 score
        temp_f1, temp_r, temp_p, count_f1 = compute_prf(turn_state, pred_state)
        slot_F1_pred += temp_f1
        slot_F1_count += count_f1

        value_batch = [last_dialog_state.get(s, 'none') for s in ALL_SLOTS]

        # print(value_batch)
        for idx, value in enumerate(value_batch):
            dial_id = batch_data["ID"][idx]
            if dial_id not in predictions:
                predictions[dial_id] = {}
                predictions[dial_id]["domain"] = batch_data["domains"][idx][0]
                predictions[dial_id]["turns"] = {}
            if batch_data["turn_id"][idx] not in predictions[dial_id]["turns"]:
                predictions[dial_id]["turns"][batch_data["turn_id"][idx]] = {"turn_belief":batch_data["turn_belief"][idx], "pred_belief":[]}

            if value!="none":
                predictions[dial_id]["turns"][batch_data["turn_id"][idx]]["pred_belief"].append(str(batch_data["slot_text"][idx])+'-'+str(value))

            # analyze slot acc:
            if str(value)==str(batch_data["value_text"][idx]):
                slot_logger[str(batch_data["slot_text"][idx])][1]+=1 # hit
            slot_logger[str(batch_data["slot_text"][idx])][0]+=1 # total

        # sum+=1
        # if sum > 50:
        #     exit()
    # exit()

    for slot_log in slot_logger.values():
        slot_log[2] = slot_log[1]/slot_log[0]

    with open(os.path.join(save_path, f"slot_acc.json"), 'w') as f:
        json.dump(slot_logger,f, indent=4)

    with open(os.path.join(save_path, f"prediction.json"), 'w') as f:
        json.dump(predictions,f, indent=4)

    joint_acc_score, F1_score, turn_acc_score = evaluate_metrics(predictions, ALL_SLOTS)

    evaluation_metrics = {"Joint Acc":joint_acc_score, "Turn Acc":turn_acc_score, "Joint F1":F1_score}
    print(f"result:",evaluation_metrics)

    with open(os.path.join(save_path, f"result.json"), 'w') as f:
        json.dump(evaluation_metrics,f, indent=4)

    len_test_data = max_count / num_slot
    joint_acc_score = joint_acc / len_test_data
    turn_acc_score = slot_turn_acc / len_test_data
    slot_F1_score = slot_F1_pred / slot_F1_count
    add_acc_score = add_acc / add_num
    delete_acc_score = delete_acc / delete_num
    task_acc_score = task_acc / task_num
    session_acc_score = session_acc / session_num
    add_feedback_cost_score = add_feedback_cost / all_slot
    delete_feedback_cost_score = delete_feedback_cost / all_slot
    add_feedback_gain_score = add_acc / all_slot
    delete_feedback_gain_score = delete_acc / all_slot
    feedback_slot_acc_score = (add_acc + delete_acc) / (add_num + delete_num)

    print("------------------------------")
    print("feedback_acquisition_strategy : ", args.feedback_acquisition_strategy)
    print("feedback_content : ", args.feedback_content)
    print("feedback_timing : ", args.feedback_timing)
    print("joint accuracy : ", joint_acc_score)
    print("slot turn accuracy : ", turn_acc_score)
    print("slot turn F1: ", slot_F1_score)
    print("add slot accuracy : ", add_acc_score)
    print("delete slot accuracy : ", delete_acc_score)
    print("Task accuracy : ", task_acc_score)
    print("Session accuracy : ", session_acc_score)
    print("add_feedback_cost_score : ", add_feedback_cost_score)
    print("delete_feedback_cost_score : ", delete_feedback_cost_score)
    print("add_feedback_gain_score : ", add_feedback_gain_score)
    print("delete_feedback_gain_score : ", delete_feedback_gain_score)
    print("feedback_slot_acc_score : ", feedback_slot_acc_score)
    print("-----------------------------\n")

    return predictions

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--dataset", type=str, default="MultiWOZ_2.1", help="WOZ_2.0 or MultiWOZ_2.1")
    parser.add_argument("--model_checkpoint", type=str, default="t5-small", help="Path, url or short name of the model")
    parser.add_argument("--saving_dir", type=str, default="save", help="Path for saving")
    parser.add_argument("--train_batch_size", type=int, default=16, help="Batch size for training")
    parser.add_argument("--dev_batch_size", type=int, default=8, help="Batch size for validation")
    parser.add_argument("--test_batch_size", type=int, default=8, help="Batch size for test")
    parser.add_argument("--gradient_accumulation_steps", type=int, default=8, help="Accumulate gradients on several steps")
    parser.add_argument("--lr", type=float, default=1e-4, help="Learning rate")
    parser.add_argument("--max_norm", type=float, default=1.0, help="Clipping gradient norm")
    parser.add_argument("--n_epochs", type=int, default=5, help="Number of training epochs")
    parser.add_argument("--seed", type=int, default=557, help="Random seed")
    parser.add_argument("--GPU", type=int, default=1, help="number of gpu to use")
    parser.add_argument("--slot_lang", type=str, default="slottype", help="use 'none', 'human', 'naive', 'value', 'question', 'slottype' slot description")

    parser.add_argument("--feedback_acquisition_strategy", default='explicit', help='explicit or implicit', type=str)
    parser.add_argument("--feedback_content", default='belief_state', help='belief_state or turn_label', type=str)
    parser.add_argument("--feedback_timing", default='session', help='turn, task or session', type=str)
    parser.add_argument("--train_feedback", default=1.0, type=float)
    args = parser.parse_args()
    args.feedback_model_name = '../' + info.model_file[args.dataset]

    if args.dataset == 'WOZ_2.0':
        data_root = 'data/WOZ_2.0'
        config_root = 'data/dataset_config'
        feedback_data_root = 'feedback_data/WOZ_2.0'
        args.train_data_path = os.path.join(data_root, 'woz_train_en.json')
        args.dev_data_path = os.path.join(data_root, 'woz_validate_en.json')
        args.test_data_path = os.path.join(data_root, 'woz_test_en.json')
        args.config_path = os.path.join(config_root, 'woz2.json')
        args.description_path = os.path.join('utils', 'slot_description_WOZ.json')
        args.feedback_data_path = os.path.join(feedback_data_root, 'add[2]_delete[1]_seed[42].json')
    elif args.dataset == 'MultiWOZ_2.1':
        data_root = 'data/MultiWOZ_2.1'
        config_root = 'data/dataset_config'
        feedback_data_root = 'feedback_data/MultiWOZ_2.1'
        args.train_data_path = os.path.join(data_root, 'train_dials.json')
        args.dev_data_path = os.path.join(data_root, 'dev_dials.json')
        args.test_data_path = os.path.join(data_root, 'test_dials.json')
        args.config_path = os.path.join(config_root, 'multiwoz21.json')
        args.description_path = os.path.join('utils', 'slot_description_MWOZ.json')
        args.feedback_data_path = os.path.join(feedback_data_root, 'add[4]_delete[2]_seed[42].json')
    else:
        print('select dataset in WOZ_2.0 and MultiWOZ_2.1')
        exit()

    args.mode = 'test'
    print(args)
    test(args)


import json

def P(var, cond_and = [], cond_not = [], cond_or1 = [], cond_or2 = [], cond_or3 = [], filter = "", collective = True, exact = False):
    var_array = []
    and_array = []
    not_array = []
    or1_array = []
    or2_array = []
    or3_array = []
    for i in range(len(headers)):
        if exact:
            if var in headers[i]:
            # if var == headers[i]:
                var_array.append(i)
            for cond in cond_and:
                if cond == headers[i]:
                    and_array.append(i)
            for cond in cond_not:
                if cond == headers[i]:
                    not_array.append(i)
            for cond in cond_or1:
                if cond == headers[i]:
                    or1_array.append(i)
            for cond in cond_or2:
                if cond == headers[i]:
                    or2_array.append(i)
            for cond in cond_or3:
                if cond == headers[i]:
                    or3_array.append(i)
        else:
            if var in headers[i]:
                var_array.append(i)
            for cond in cond_and:
                if cond in headers[i]:
                    and_array.append(i)
            for cond in cond_not:
                if cond in headers[i]:
                    not_array.append(i)
            for cond in cond_or1:
                if cond in headers[i]:
                    or1_array.append(i)
            for cond in cond_or2:
                if cond in headers[i]:
                    or2_array.append(i)
            for cond in cond_or3:
                if cond in headers[i]:
                    or3_array.append(i)
    results = {}
    numerator = {}
    denominator = {}
    for row in data:
        if filter != "" and filter not in row[0]:
            continue
        valid = True
        # AND
        for cond in and_array:
            if not row[cond]:
                valid = False
                break
        if not valid:
            continue
        # NOT
        for cond in not_array:
            if row[cond]:
                valid = False
                break
        if not valid:
            continue
        # OR1
        valid = or1_array == []
        for cond in or1_array:
            if row[cond]:
                valid = True
                break
        if not valid:
            continue
        # OR2
        valid = or2_array == []
        for cond in or2_array:
            if row[cond]:
                valid = True
                break
        if not valid:
            continue
        # OR3
        valid = or3_array == []
        for cond in or3_array:
            if row[cond]:
                valid = True
                break
        if not valid:
            continue
        results[row[0]] = sum([row[i] for i in var_array]) / len(var_array)
        numerator[row[0]] = sum([row[i] for i in var_array])
        denominator[row[0]] = len(var_array)
    if len(results) == 0:
        return "N/A"
    if collective:
        return "{:.2f}% ({} / {})".format(sum(numerator.values()) / sum(denominator.values()) * 100, sum(numerator.values()), sum(denominator.values()))
    return results

def search(terms = []):
    if type(terms) != list:
        return "ERROR"
    terms_indices = {}
    results = {}
    for term in terms:
        terms_indices[term] = []
    for i in range(len(headers)):
        for term in terms:
            if term in headers[i]:
                terms_indices[term].append(i)
    for row in data:
        valid = [False] * len(terms)
        valid_indices = [[] for i in range(len(terms))]
        for i in range(len(terms)):
            for index in terms_indices[terms[i]]:
                if row[index]:
                    valid[i] = True
                    valid_indices[i].append(index)
        if all(valid):
            results[row[0]] = []
            for indices in valid_indices:
                results[row[0]] += [headers[index] for index in indices]
    return results

# data = """
# X,AA,B,C,D,D,D
# a,0,0,1,0,0,1
# b,0,1,1,1,1,1
# c,1,0,1,0,0,0
# d,0,0,0,0,0,0
# """
# data = [row.split(',') for row in data.strip().split('\n')]
# headers = data[0]
# data = data[1:]
# for i in range(len(data)-1, -1, -1):
#     for j in range(1, len(data[0])):
#         data[i][j] = int(data[i][j])
# print(P("D", cond_not=["C"], collective=False))

file = open("data.csv", "r")
data = file.read()
data = [row.split(',') for row in data.strip().split('\n')]
headers = data[0]
data = data[1:]
for i in range(len(data)-1, -1, -1):
    for j in range(1, len(data[0])):
        data[i][j] = int(data[i][j])
    if sum(data[i][1:]) == 0:
        data.pop(i)
marks = json.load(open("marks.json", "r"))

""" General Statistics """
# print(P("Correct answer"))
# print(P("Minor error"))
# print(P("Correct answer", cond_and=["Question Type: Multiple-Choice Question"]))
# print(P("Correct answer", cond_and=["Question Type: Programming"]))
# l = P("Correct answer", cond_and=["Question Type: Programming"], collective=False)
# print("{:.2f}%".format(sum([1 for i in l.values() if i > 0]) / len(l) * 100))
# print(P("Correct answer", cond_or1=["Input Type: Image description"]))
# print(P("Correct answer", cond_and=["Input Type: Image description (novice)"]))
# print(P("Correct answer", cond_and=["Input Type: Image description (informed)"]))

""" Novice vs Informed"""
# novice = P("Correct answer", cond_and=["Input Type: Image description (novice)"], collective=False)
# informed = P("Correct answer", cond_and=["Input Type: Image description (informed)"], collective=False)
# for i in novice:
#     print(novice[i])
#     # print(novice[i] * marks[i[:7]])
# print()
# for i in informed:
#     print(informed[i])
#     # print(informed[i] * marks[i[:7]])

""" Inductive vs Deductive """
# print(P("Correct answer", cond_and=["Reasoning: Deductive"]))
# print(P("Correct answer", cond_and=["Reasoning: Inductive"]))

""" Input Type """
# print(P("Correct answer", cond_and=["Input Type: Text"], cond_not=["Input Type: Mathematical formula", "Input Type: Image description (novice)", "Input Type: Image description (informed)", "Input Type: Code"]))
# print(P("Correct answer", cond_and=["Input Type: Text", "Input Type: Mathematical formula"], cond_not=["Input Type: Image description (novice)", "Input Type: Image description (informed)", "Input Type: Code"]))
# print(P("Correct answer", cond_and=["Input Type: Text", "Input Type: Code"], cond_not=["Input Type: Mathematical formula", "Input Type: Image description (novice)", "Input Type: Image description (informed)"]))
# print(P("Correct answer", cond_and=["Input Type: Text", "Input Type: Image description (novice)"], cond_not=["Input Type: Mathematical formula", "Input Type: Image description (informed)", "Input Type: Code"]))
# print(P("Correct answer", cond_and=["Input Type: Text", "Input Type: Image description (informed)"], cond_not=["Input Type: Mathematical formula", "Input Type: Image description (novice)", "Input Type: Code"]))
# print(P("Correct answer", cond_not=["Input Type: Text"]))
# print(P("Correct answer", cond_or1=["Image"]))
# print(P("Correct answer", cond_not=["Image"]))
# print(P("Correct answer", cond_or1=["Programming"]))
# print(P("Minor error", cond_or1=["Programming"]))

""" Output Type """
# print(P("Correct answer", cond_and=["Output Type: Text"]))
# print(P("Correct answer", cond_and=["Output Type: Mathematical formula"]))
# print(P("Correct answer", cond_and=["Output Type: Code"]))

""" Input/Output Type """
# print(P("Correct answer", cond_not=["Math", "Image", "Code"]))
# print(P("Correct answer", cond_or1=["Math"], cond_not=["Image", "Code"]))
# print(P("Correct answer", cond_or1=["Image"]))

""" Topic """
# print(P("Correct answer", cond_and=["Topic: Geometry"]))
# print(P("Correct answer", cond_and=["Topic: Graphics Introduction"]))
# print(P("Correct answer", cond_and=["Topic: Color"]))
# print(P("Correct answer", cond_and=["Topic: Illumination and Shading"]))
# print(P("Correct answer", cond_and=["Topic: 3D Modelling"]))
# print(P("Correct answer", cond_and=["Topic: Texture Mapping"]))
# print(P("Correct answer", cond_and=["Topic: Ray Tracing"]))
# print(P("Correct answer", cond_and=["Topic: Parametric Curves and Surfaces"]))
# print(P("Correct answer", cond_and=["Topic: Image Processing"]))

""" Search """
# print(search(["Color", "Correct answer"]))

""" Marks (old) """
# codes = ["2022a", "2022b", "2023a", "2023b"]
# versions = ["a", "b"]
# for code in codes:
#     for version in versions:
#         print(code, version)
#         for i in range(10):
#             # d = P("Generation {}: Correct answer".format(i + 1), cond_not=["Topic: Image Processing"], filter=code, collective=False)
#             d = P("Generation {}: Correct answer".format(i + 1), filter=code, collective=False)
#             score = 0
#             total = 0
#             for key in d:
#                 if len(key) == 7 or key[-1] == version:
#                     score += d[key] * marks[key[:7]]
#                     total += marks[key[:7]]
#             print("{} {:.2f} ({} / {})".format(i, score / total * 100, score, total))
#             # print("{:.4f}".format(score / total))

""" Bloom's Taxonomy (not categorized well?) """
# print(P("Correct answer", cond_and=["Bloom's Taxonomy: Remember"], collective=False))
# print(P("Correct answer", cond_and=["Bloom's Taxonomy: Understand"]))
# print(P("Correct answer", cond_and=["Bloom's Taxonomy: Apply"]))
# print(P("Correct answer", cond_and=["Bloom's Taxonomy: Analyze"]))
# print(P("Correct answer", cond_and=["Bloom's Taxonomy: Evaluate"]))
# print(P("Correct answer", cond_and=["Bloom's Taxonomy: Create"]))

""" Difficulty Level """
# print(P("Correct answer", cond_and=["Difficulty Level: Easy"], exact=True))
# print(P("Correct answer", cond_and=["Difficulty Level: Easy - Medium"]))
# print(P("Correct answer", cond_and=["Difficulty Level: Medium"], exact=True))
# print(P("Correct answer", cond_and=["Difficulty Level: Medium - Hard"]))
# print(P("Correct answer", cond_and=["Difficulty Level: Hard"]))
# print(P("Correct answer", cond_and=["Difficulty Level: Easy"], exact=True))
# print(P("Correct answer", cond_or1=["Difficulty Level: Easy - Medium", "Difficulty Level: Medium"], exact=True))
# print(P("Correct answer", cond_or1=["Difficulty Level: Medium - Hard", "Difficulty Level: Hard"], exact=True))

""" Marks (new) """
# codes = ["2022a", "2022b", "2023a", "2023b"]
# versions = ["a", "b"]
# mcq = P("Correct answer", cond_not=["Programming"], collective=False)
# pro = P("Correct answer", cond_or1=["Programming"], collective=False)
# for code in codes:
#     for version in versions:
#         print(code, version)
#         score = 0
#         total = 0
#         for q in mcq:
#             if code in q and (len(q) == 7 or q[-1] == version):
#                 score += mcq[q] * marks[q[:7]]
#                 total += marks[q[:7]]
#         for q in pro:
#             if code in q and (len(q) == 7 or q[-1] == version):
#                 score += (pro[q] > 0) * marks[q[:7]]
#                 total += marks[q[:7]]
#         print("{:.2f}% ({} / {})".format(score / total * 100, score, total))

""" Miscellaneous """
# d = P("Input Type: Mathematical formula")
# print(P("Deductive", cond_or1=["Hard"]))
# print(P("Inductive", cond_or1=["Hard"]))
# print(search(["Texture Mapping"]))
# print(P("Input Type: Image description (novice)", cond_or1=["Topic: Ray Tracing", "Topic: Graphics Introduction", "Topic: Image Processing"]))
# print(P("Input Type: Image description (informed)", cond_or1=["Topic: Ray Tracing", "Topic: Graphics Introduction", "Topic: Image Processing"]))
# print(search(["Topic: Ray Tracing"]))
# print(search(["Topic: Graphics Introduction"]))
# print(search(["Topic: Image Processing"]))
# print(search(["Topic: 3D Modelling"]))
# print(search(["Topic: Geometry"]))
# print(search(["Topic: Color"]))
# print(P("Input Type: Image description (novice)", cond_and=["Topic: Color"]))


from typing import Tuple

import numpy as np

from core.filters import DataFilter
from core.filters.kalman import ExtendedKalmanFilter
from core.localization import Localization

__author__ = 'Xomak'

class EkfLocalization(Localization):
    def init_filter(self) -> Tuple[DataFilter, np.array]:
        initial_state = self.robot_model.initial_state
        data_filter = ExtendedKalmanFilter(initial_state.shape[0], self.robot_model.observations_dimension)
        data_filter.set_initial(initial_state, self.robot_model.initial_covariance)
        return data_filter, initial_state

    @property
    def filtered_name(self):
        return "EK filter"

    def get_next_state(self, data_filter: DataFilter, step_data):
        state, covariance = data_filter.predict_update(step_data.transition_function,
                                                       step_data.observations_function,
                                                       step_data.control,
                                                       step_data.observations,
                                                       step_data.process_covariance,
                                                       step_data.observations_covariance)
        return state

loc = EkfLocalization()
loc.visualize()


from django import forms
from .models import Exercise

WEIGHT_TYPE_CHOICES = [
    ('lb', 'lb'),
    ('kg', 'kg'),
]

class ExerciseForm(forms.ModelForm):
    exercise_text = forms.CharField(widget=forms.TextInput(attrs={'placeholder': 'Exercise',"size": "10"}), required=True)
    class Meta:
        model = Exercise
        fields = ('exercise_text',)


import re
import statistics

_fhandler = open('mbox-short.txt')
count = []
for line in _fhandler:
    line = line.rstrip()
    query = re.findall('^New .*: ([0-9]+)', line)
    if len(query) > 0:
        for number in query:
            count.append(float(number))

print(statistics.mean(count))


from pytube import YouTube, Playlist
from threading import Thread

def main():
    format = input('1 - file\n2 - playlist\n')

    if format == 1:
        with open('link.txt', 'r') as file:
            links = file.text
            print(links)
    else:
        link_playlist = input('Enter a playlist link: ')
        playlist = Playlist(link_playlist)
        links = playlist.video_urls

    for index, link in enumerate(links):
        index += 1
        print(f'Total video --- {len(links)}\n')
        Thread(target=download_video, args=(link, index, )).start()

def download_video(link, number):
    video = YouTube(link)
    stream = video.streams.get_highest_resolution()
    stream.download('video/')
    print('\t' + str(number) + ' --- finish')

if __name__ == '__main__':
    main()