import abc
import os
import re

import pandas as pd
from werkzeug.utils import secure_filename

from app_folder import db
from app_folder.models import LinkedInRecord, Contact, EtherealAddress, EtherealContact
from site_config import FConfig
from cytoolz import groupby

ALLOWED_EXTENSIONS = ['.csv', '.xls', '.xlsx']
TRY_ENCODINGS = ['default', 'unicode' 'latin1', 'cp1252', 'iso-8859-1', 'ascii']
UPLOAD_FOLDER = FConfig.UPLOAD_FOLDER

def inject_allowed_ext():
    return dict(allowed_ext=ALLOWED_EXTENSIONS)

class UploadSpreadsheet(object):
    """
    class object to handle heavy lifting of receiving files via upload

    :param request_file: flask request object
    """

    # TODO Subclass, mixins for email providers
    allowed_extensions = ALLOWED_EXTENSIONS
    try_encodings = TRY_ENCODINGS

    def __init__(self, request_file):
        self.request_file = request_file
        self.uploaded_file_path = None
        self.uploaded_file = None
        self.status = None

        self.open_file_()

    def find_file_ext_(self):
        filename = self.request_file.filename
        if not filename:
            return None
        _, file_ext = os.path.splitext(filename)
        return file_ext

    def allowed_file_(self):
        ext = self.find_file_ext_()
        if ext in self.allowed_extensions:
            return True
        else:
            self.status = "Extension {} not permitted".format(ext)
            return False

    def save_upload_(self):

        file = self.request_file
        if not file:
            self.status = "No file received"
            return None
        if file.filename == '':
            self.status = "Your file name is blank"
            return None

        filename = secure_filename(file.filename)

        if self.allowed_file_() is False:
            return None

        # Save file to open later for analysis
        file.save(os.path.join(UPLOAD_FOLDER, filename))

        # Save location
        file_path = os.path.join(UPLOAD_FOLDER, filename)

        return file_path

    def open_file_(self):

        """
        Function that handles opening the spreadsheet with pandas
        Gets file extension and uses appropriate open method with pandas
        Tries several encodings until opening is successful or ends with fail

        :return: pd.DataFrame() or False if fail to open
        """

        self.uploaded_file_path = self.save_upload_()
        if not self.uploaded_file_path:
            print(self.status)
            return None
        file_ext = self.find_file_ext_()
        if not file_ext:
            return None

        if file_ext == '.csv':
            open_method = pd.read_csv
        elif file_ext == '.xls' or file_ext == '.xlsx':
            open_method = pd.read_excel
        else:
            # No file extension found
            self.status = 'No file extension was found'
            return None

        for encoding in self.try_encodings:
            try:
                if encoding == 'default':
                    df = open_method(self.uploaded_file_path)

                else:
                    df = open_method(self.uploaded_file_path, encoding=encoding)
                self.uploaded_file = df
                break
            except Exception as e:
                print(e)
                if encoding == self.try_encodings[-1]:
                    # Tried all encodings, all failed
                    self.status = "Unable to determine file encoding"
                    return None
                else:
                    continue

class OptoutSpreadsheet(UploadSpreadsheet):

    EMAIL_SEARCH = re.compile(r"[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?")
    TARGET_HEADER = 'All emails'

    def __init__(self, request_file):
        self.new_nodes = []
        self.appended_nodes = []
        super().__init__(request_file)
        self.find_new_emails()

    def find_new_emails(self):

        """

        Parses spreadsheet rows to find emails. Each row may contain 0 or more emails

        For each row, if all email addresses are 'new' they are created along with a new EtherealContact

        Else, the 'new' address is created and appended to an existing EtherealContact

        :return:
        """

        new_nodes = []
        appended_nodes = []

        addresses = self.uploaded_file[self.TARGET_HEADER].apply(lambda x: self.EMAIL_SEARCH.findall(x))
        existing_ethereal = EtherealAddress.query.join(EtherealAddress.parent_contact)\
            .filter(EtherealContact._kind == EtherealContact.OPTOUT)
        existing_addresses = set([x.address for x in existing_ethereal])

        def attach_node(duplicated_address_group, xst=existing_addresses):

            # Find the matched EtherealAddress and Attach to .parent
            segmented_data = groupby(lambda x: x in xst, duplicated_address_group)
            is_duplicated, not_duplicated = segmented_data.get(True, []), segmented_data.get(False, [])
            if not is_duplicated:
                return None
            if not not_duplicated:
                return None

            # Query for parent

            parent = EtherealContact.query.filter(EtherealContact.children.any(
                EtherealAddress.address.in_(is_duplicated))).from_self().filter(
                EtherealContact._kind == EtherealContact.OPTOUT).first()

            if not parent:
                return None

            for new_address in not_duplicated:
                new_ethereal = EtherealAddress(is_email=True, is_personal=True, is_phone=False, is_website=False,
                                               address=new_address)
                db.session.add(new_ethereal)
                parent.children.append(new_ethereal)

            return parent

        def create_node(new_address_group):

            new_parent = EtherealContact(_kind=EtherealContact.OPTOUT)
            db.session.add(new_parent)

            for address in new_address_group:
                new_ethereal = EtherealAddress(is_email=True, is_personal=True, is_phone=False, is_website=False,
                                               address=address)
                db.session.add(new_ethereal)
                new_parent.children.append(new_ethereal)

            return new_parent

        for address_group in addresses:
            if not address_group:
                continue
            if any([x in existing_addresses for x in address_group]):
                if not all([x in existing_addresses for x in address_group]):
                    attached_node = attach_node(address_group)
                    if attached_node:
                        appended_nodes.append(attached_node)

            else:
                new_node = create_node(address_group)
                if new_node:
                    new_nodes.append(new_node)

        self.new_nodes = new_nodes
        self.appended_nodes = appended_nodes

        db.session.commit()

class ContactSpreadsheet(abc.ABC, UploadSpreadsheet):

    EMAIL_PERSONAL = 'email_personal'
    EMAIL_NOT_PERSONAL = 'email_work'
    PHONE_PERSONAL = 'phone_personal'
    WEBSITE_PERSONAL = 'website_personal'
    WEBSITE_LINKEDIN = 'website_linkedin'
    MEMBER_ID = 'member_id'

    @property
    @abc.abstractmethod
    def HEADER_SEARCH(self):
        return None

    @property
    @abc.abstractmethod
    def KEEP_PATTERNS(self):
        return []

    @abc.abstractmethod
    def scan_headers_(self):
        relevant_headers = {k: [] for k in self.HEADER_SEARCH.keys()}

        cols = self.uploaded_file.columns.tolist()
        for data_type, pattern in self.HEADER_SEARCH.items():
            relevant_headers[data_type] = [match.group() for match in [pattern.search(c) for c in cols] if match]

        return relevant_headers

    @abc.abstractmethod
    def scan_data_(self):

        relevant_headers = self.scan_headers_()
        file_data = self.uploaded_file.fillna('').to_dict('records')

        data_records = []

        for row_data in file_data:
            td = {}
            for data_type, header_names in relevant_headers.items():
                td[data_type] = [sv for sv in [v for k, v in row_data.items() if k in header_names] if self.filter_data_(sv, data_type)]
            data_records.append(td)
        return data_records

    @abc.abstractmethod
    def filter_data_(self, sv, data_type):
        if sv == '':
            return None
        if not self.KEEP_PATTERNS:
            return True
        for kept in self.KEEP_PATTERNS:
            for kept_type in kept['in']:
                if kept_type not in data_type:
                    continue
                if kept['pattern'] is False:
                    return True
                if kept['pattern'].search(sv) is not None:
                    return True
        return False

class JobJetSpreadsheet(ContactSpreadsheet):

    @property
    def HEADER_SEARCH(self):
        return {self.EMAIL_PERSONAL: re.compile(r"(Personal email \(\d\))", flags=re.IGNORECASE),
                self.EMAIL_NOT_PERSONAL: re.compile(r"(Work email \(\d\))", flags=re.IGNORECASE),
                self.PHONE_PERSONAL: re.compile(r"(Personal phone \(\d\))", flags=re.IGNORECASE),
                self.WEBSITE_PERSONAL: re.compile("(Website url \(\d\))", flags=re.IGNORECASE),
                self.WEBSITE_LINKEDIN: re.compile("(LinkedIn url \(\d\))", flags=re.IGNORECASE),
                self.MEMBER_ID: re.compile(r"(Tag1)", flags=re.IGNORECASE)}

    @property
    def KEEP_PATTERNS(self):

        """
        Fine grained control over filtering incoming data

        Where 'in' refers to the column data type and pattern refers to a regex object
        Specify False to allow all values
        """

        return [{'pattern': False, 'in': ['member_id']},
                {'pattern': False, 'in': ['email']},
                {'pattern': False, 'in': ['phone']},
                {'pattern': re.compile(r"(linkedin)|(github)|(twitter)|(google)|(facebook)", flags=re.IGNORECASE),
                 'in': ['website']}]

    def __init__(self, request):
        super().__init__(request)
        self.data_ = []
        if self.uploaded_file is not None:
            self.scan_data_()

    @property
    def records(self):
        return self.data_

    @records.getter
    def records(self):
        return self.data_

    def scan_data_(self):
        data_records = super().scan_data_()
        self.data_ = data_records

    def scan_headers_(self):
        relevant_headers = super().scan_headers_()
        return relevant_headers

    def filter_data_(self, sv, data_type):
        return super().filter_data_(sv, data_type)

class VsourceSpreadsheet(ContactSpreadsheet):

    @property
    def HEADER_SEARCH(self):
        return {'email_personal': re.compile(r"(home email_\d)", flags=re.IGNORECASE),
                'email_work': re.compile(r"(work email_\d)", flags=re.IGNORECASE),
                'website_personal': re.compile("(website_\d)", flags=re.IGNORECASE),
                'phone_personal': re.compile(r"(home phone_\d)", flags=re.IGNORECASE),
                'member_id': re.compile(r"(Tag1)", flags=re.IGNORECASE)}

    @property
    def KEEP_PATTERNS(self):
        return [{'pattern': False, 'in': ['member_id']},
                {'pattern': False, 'in': ['email']},
                {'pattern': False, 'in': ['phone']},
                {'pattern': re.compile(r"(github)|(twitter)|(google)|(facebook)", flags=re.IGNORECASE),
                 'in': ['website']}]

    def __init__(self, request):
        super().__init__(request)
        self.data_ = []
        if self.uploaded_file is not None:
            self.scan_data_()

    @property
    def records(self):
        return self.data_

    @records.getter
    def records(self):
        return self.data_

    def scan_data_(self):
        data_records = super().scan_data_()
        self.data_ = data_records

    def scan_headers_(self):
        relevant_headers = super().scan_headers_()
        return relevant_headers

    def filter_data_(self, sv, data_type):
        return super().filter_data_(sv, data_type)

class DataMapper(object):

    def __init__(self, mapped_object):
        self.mapped_object = mapped_object
        self.key_values = ['linkedin', 'member_id']

    def fetch_record_(self, row_data):
        member_id = row_data.get('member_id', None)
        linkedin_url = row_data.get('website_linkedin', None)
        if not member_id and not linkedin_url:
            return None
        if member_id:
            record = LinkedInRecord.query.get(member_id)
        else:
            record = None
        if not record and linkedin_url:
            record = LinkedInRecord.query.filter(LinkedInRecord.public_url.in_(linkedin_url)).all()
            if len(record) > 1:  # Avoid duplicating
                return None
            elif len(record) == 1:
                return record[0]
            else:
                return None
        return record

    def contact_factory(self, data_type):

        # Is it email, website or phone?
        if 'email' in data_type:
            is_email, is_website, is_phone = True, False, False
        elif 'website' in data_type:
            is_email, is_website, is_phone = False, True, False
        else:
            is_email, is_website, is_phone = False, False, True

        if 'person' in data_type:
            is_personal = True
        else:
            is_personal = False

        # Make new contact
        crecord = Contact(is_email=is_email, is_personal=is_personal, is_website=is_website, is_phone=is_phone)
        return crecord

    def ethereal_factory(self, data_type):

        # Is it email, website or phone?
        if 'email' in data_type:
            is_email, is_website, is_phone = True, False, False
        elif 'website' in data_type:
            is_email, is_website, is_phone = False, True, False
        else:
            is_email, is_website, is_phone = False, False, True

        if 'person' in data_type:
            is_personal = True
        else:
            is_personal = False

        # Make new contact
        crecord = EtherealAddress(is_email=is_email, is_personal=is_personal, is_website=is_website, is_phone=is_phone)
        crecord._kind = crecord.UNLINKED
        return crecord

    def enrich_record_(self, row_data):

        record = self.fetch_record_(row_data)
        if not record:
            return None, -1  # -1 Indicates no record found
        contact_records = []

        for data_type, data_values in row_data.items():

            if not data_values:
                continue
            if any([data_type in kv for kv in self.key_values]):  # Linkedin, member_id
                continue

            # Check for duplicate values, prevent adding
            if record.contacts:
                current_contacts_addr = [cr.address for cr in record.contacts]
                data_values = list(filter(lambda x: x not in current_contacts_addr, data_values))

            if not data_values:
                continue

            for dv in data_values:
                crecord = self.contact_factory(data_type)
                crecord.address = dv
                db.session.add(crecord)
                contact_records.append(crecord)

        if contact_records:
            record.contacts.extend(contact_records)
            db.session.add(record)
            return record, 1  # 1 is successfully added contact information
        else:
            return None, 0  # 0 indicates found record, no new information

    def enrich(self):

        data_records = self.mapped_object.records
        row_count = len(data_records)
        enriched_records = []
        outcomes = []
        for d in data_records:
            enriched_record = self.enrich_record_(d)
            if enriched_record[0]:
                enriched_records.append(enriched_record[0])
                outcomes.append(enriched_record[1])
            else:
                outcomes.append(enriched_record[1])
        return enriched_records, row_count, outcomes


from setuptools import setup

package_name = "hybridset"
version = '1.0'
url = 'https://gitlab.com/yahya-abou-imran/hybridset'
url_fmt = '/-/archive/v{version}/archive.zip/{package_name}-v{version}.zip'
download_url = (
    url + url_fmt.format(version=version, package_name=package_name)
)
description=('Mutable set storing apart hashables and unhashables values')
try:
    import pypandoc
    long_description = pypandoc.convert_file('README.md', 'rst')
except(IOError, ImportError):
    long_description = open('README.md').read()

setup(
    name=package_name,
    description=description,
    long_description=long_description,
    long_description_content_type='text/markdown',
    version=version,
    author='Yahya Abou Imran',
    author_email='yahya-abou-imran@pm.me',
    license='GPLv3',
    url=url,
    download_url=download_url,
    packages=[package_name],
)


import pymongo

myclient = pymongo.MongoClient("mongodb://localhost:27017/")
mydb = myclient["mydatabase"]#db_name
mycol = mydb["customers"]#tables

mylist = [
  { "_id": 1, "name": "John", "address": "Highway 37"},
  { "_id": 2, "name": "Peter", "address": "Lowstreet 27"},
  { "_id": 3, "name": "Amy", "address": "Apple st 652"},
  { "_id": 4, "name": "Hannah", "address": "Mountain 21"},
  { "_id": 5, "name": "Michael", "address": "Valley 345"},
  { "_id": 6, "name": "Sandy", "address": "Ocean blvd 2"},
  { "_id": 7, "name": "Betty", "address": "Green Grass 1"},
  { "_id": 8, "name": "Richard", "address": "Sky st 331"},
  { "_id": 9, "name": "Susan", "address": "One way 98"},
  { "_id": 10, "name": "Vicky", "address": "Yellow Garden 2"},
  { "_id": 11, "name": "Ben", "address": "Park Lane 38"},
  { "_id": 12, "name": "William", "address": "Central st 954"},
  { "_id": 13, "name": "Chuck", "address": "Main Road 989"},
  { "_id": 14, "name": "Viola", "address": "Sideway 1633"}
]

try:
	print("yo!")
	#uncomment to insert
	x = mycol.insert_many(mylist)
	#print list of the _id values of the inserted documents:
	print(x.inserted_ids)
except:
	print("mylist already exist!")

#find only one
x = mycol.find_one()
print(x)

# #loop and print without id
# for x in mycol.find({},{ "_id": 0, "name": 1, "address": 1 }):
#   print(x)

# #loop and print all without address
# for x in mycol.find({},{ "address": 0 }):
#   print(x)

#search query by address
# myquery = { "address": "Park Lane 38" }
# mydoc = mycol.find(myquery)

# for x in mydoc:
#   print(x)

#Find documents where the address starts with the letter "S" or higher:
# myquery = { "address": { "$gt": "S" } }
# mydoc = mycol.find(myquery)

# for x in mydoc:
#   print(x)

#Find documents where the address starts with the letter "S":
# myquery = { "address": { "$regex": "^S" } }
# mydoc = mycol.find(myquery)

# for x in mydoc:
#   print(x)

# Sort the result alphabetically by name:
#-----> sort("name", 1) #ascending
#-----> sort("name", -1) #descending
# mydoc = mycol.find().sort("name")
# for x in mydoc:
#   print(x)

# #Delete the document with the address "Mountain 21":
# myquery = { "address": "Mountain 21" }
# mycol.delete_one(myquery)

# #Delete all documents were the address starts with the letter S:
# myquery = { "address": {"$regex": "^S"} }
# x = mycol.delete_many(myquery)
# print(x.deleted_count, " documents deleted.")

#Delete all documents in the "customers" collection:
# x = mycol.delete_many({})
# print(x.deleted_count, " documents deleted.")

#Dtop collection
#mycol.drop()

#Change the address from "Valley 345" to "Canyon 123":
myquery = { "address": "Valley 345" }
newvalues = { "$set": { "address": "Canyon 123" } }
mycol.update_one(myquery, newvalues)
#print "customers" after the update:
for x in mycol.find():
  print(x)


import os

import pytest
from _pytest.fixtures import SubRequest
from flask.app import Flask
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy.orm.scoping import scoped_session

from app import create_app
from app.extensions.database import sql as _db

TEST_DB = "tests.db"
TEST_DB_PATH = f"{os.path.dirname(os.path.dirname(__file__))}/{TEST_DB}"
TEST_DATABASE_URI = "sqlite:///" + TEST_DB_PATH

# noinspection PyShadowingNames
@pytest.yield_fixture(scope="session")
def app(request: SubRequest) -> Flask:
    settings = {"TESTING": True, "SQLALCHEMY_DATABASE_URI": TEST_DATABASE_URI}
    app = create_app("testing", settings)
    ctx = app.app_context()
    ctx.push()

    def teardown():
        ctx.pop()

    request.addfinalizer(teardown)
    return app

# noinspection PyShadowingNames
@pytest.fixture(scope="session")
def db(app: Flask, request: SubRequest) -> SQLAlchemy:
    """
    Session-wide tests database.
    http://alexmic.net/flask-sqlalchemy-pytest/
    """

    if os.path.exists(TEST_DB_PATH):
        os.unlink(TEST_DB_PATH)

    def teardown():
        _db.drop_all()
        os.unlink(TEST_DB_PATH)

    _db.app = app
    _db.create_all()

    request.addfinalizer(teardown)
    return _db

@pytest.fixture(scope="function")
def session(db: SQLAlchemy, request: SubRequest) -> scoped_session:
    """
    Creates a new data session for a tests.
    http://alexmic.net/flask-sqlalchemy-pytest/
    """
    connection = db.engine.connect()
    transaction = connection.begin()

    options = dict(bind=connection, binds={})
    session = db.create_scoped_session(options=options)

    db.session = session

    def teardown():
        transaction.rollback()
        connection.close()
        session.remove()

    request.addfinalizer(teardown)
    return session


"""mainProject URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import include, url
from django.contrib import admin,auth
from django.conf.urls.static import static
from django.conf import settings
from accounts.views import login_view,register_view,logout_view,profileedit
#from comments.views import commentView
from comments.views import comment_delete
urlpatterns = [

    url(r'^admin/', admin.site.urls),
      url(r'^', include('bored.urls', namespace="bored")),
      url(r'^', include('bored.api.urls', namespace="bored-api")),
       url(r'^comments', include('comments.api.urls', namespace="comments-api")),
 url(r'^ckeditor/', include('ckeditor_uploader.urls')),
  url(r'^accounts/', include('allauth.urls')),
url(r'^profile/profile-edit/', profileedit,name='profile_edit'),
 url(r'^(?P<id>\d+)/deletecomment/(?P<id1>\d+)',comment_delete,name='comments'),
 url(r'^accounts/login/', login_view,name='login'),
 url(r'^register/', register_view,name='register'),
  url(r'^accounts/logout/', logout_view, name='logout'),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
if settings.DEBUG:
  urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)


import json
import shutil
import logging
from jsonmerge import merge
from airflow.models import BaseOperator
from cwltool.process import relocateOutputs
from cwltool.stdfsaccess import StdFsAccess

class JobCleanup(BaseOperator):

    def __init__(self, *args, **kwargs):
        super(JobCleanup, self).__init__(task_id=self.__class__.__name__, *args, **kwargs)

    def execute(self, context):
        collected_outputs = {}
        for task_outputs in self.xcom_pull(context=context, task_ids=[task.task_id for task in self.upstream_list]):
            collected_outputs = merge(collected_outputs, task_outputs["outputs"])
        logging.debug('Collected outputs: \n{}'.format(json.dumps(collected_outputs, indent=4)))
        tmp_folder = collected_outputs["tmp_folder"]
        output_folder = collected_outputs["output_folder"]
        relocated_outputs = relocateOutputs(outputObj={output_id: collected_outputs[output_src]
                                                       for output_src, output_id in self.dag.get_output_list().items()
                                                       if output_src in collected_outputs},
                                            destination_path=output_folder,
                                            source_directories=[output_folder],
                                            action="copy",
                                            fs_access=StdFsAccess(""))

        relocated_outputs = {key.split("/")[-1]: val for key, val in relocated_outputs.items()}
        shutil.rmtree(tmp_folder, ignore_errors=False)
        logging.debug('Delete temporary output directory: \n{}'.format(tmp_folder))
        logging.info("WORKFLOW RESULTS\n" + json.dumps(relocated_outputs, indent=4))


import os
import argparse
import subprocess

def iterate_over_directory(input_dir,
                           results_file,
                           timeout):
    # get all the files in the directory
    files = os.listdir(input_dir)
    for f in files:
        name, extension = os.path.splitext(f)
        # we only care about the graph files
        if extension != ".graph":
            continue
        # open the graph file
        print("Processing file {}".format(f))
        subprocess.call("python3 toboggan.py {}/{} --skip_truth "
                        "--experiment_info --timeout {} >>"
                        " {}".format(input_dir, f, timeout, results_file),
                        shell=True)

def main(args):
    timeout = -1
    if args.timeout:
        timeout = args.timeout
    iterate_over_directory(args.input_dir, args.results_file_name, timeout)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("input_dir", help="directory containing .graph and"
                        " .truth files", type=str)
    parser.add_argument("results_file_name", help="name of file to store notes"
                        " of instances on which toboggan times out", type=str)
    parser.add_argument("timeout", help="time to run before skipping instance",
                        type=int)

    args = parser.parse_args()
    main(args)


from datetime import datetime

from django.utils.timezone import make_aware

from settings.tasks import exec_command
from systems.commands.index import CommandMixin

import re
import json
import string
import random

class ScheduleMixin(CommandMixin('schedule')):

    def get_schedule_from_representation(self, representation):
        schedule = self.get_interval_schedule(representation)

        if not schedule:
            schedule = self.get_datetime_schedule(representation)
        if not schedule:
            schedule = self.get_crontab_schedule(representation)

        if not schedule:
            self.error("'{}' is not a valid schedule format.  See --help for more information".format(representation))

        return schedule

    def normalize_schedule_time(self, time_string):
        return make_aware(datetime.strptime(time_string, "%Y-%m-%d %H:%M:%S"))

    def set_periodic_task(self):
        schedule = self.schedule

        if schedule:
            begin = self.schedule_begin
            end = self.schedule_end

            schedule_map = {
                'task_interval': 'interval',
                'task_crontab': 'crontab',
                'task_datetime': 'clocked'
            }
            options = self.options.export()
            options['_user'] = self.active_user.name
            task = {
                schedule_map[schedule.facade.name]: schedule,
                'task': 'zimagi.command.exec',
                'user': self.active_user,
                'args': json.dumps([self.get_full_name()]),
                'kwargs': json.dumps(options)
            }
            if begin:
                task['start_time'] = begin
            if end:
                task['expires'] = end

            self._scheduled_task.store(self.get_schedule_name(), **task)

            self.success("Task '{}' has been scheduled to execute periodically".format(self.get_full_name()))
            return True

        return False

    def set_queue_task(self):
        if self.push_queue:
            options = self.options.export()
            options['_user'] = self.active_user.name
            exec_command.delay(self.get_full_name(), **options)
            self.success("Task '{}' has been pushed to the queue to execute in the background".format(self.get_full_name()))
            return True

        return False

    def get_schedule_name(self):
        return "{}:{}{}".format(
            self.get_full_name().replace(' ', '-'),
            datetime.now().strftime("%Y%m%d%H%M%S"),
            random.SystemRandom().choice(string.ascii_lowercase)
        )

    def get_interval_schedule(self, representation):
        interval = self._task_interval.model
        schedule = None
        period_map = {
            'D': interval.DAYS,
            'H': interval.HOURS,
            'M': interval.MINUTES,
            'S': interval.SECONDS
        }

        match = re.match(r'^(\d+)([DHMS])$', representation, flags = re.IGNORECASE)
        if match:
            schedule, created = self._task_interval.store(representation,
                every = match.group(1),
                period = period_map[match.group(2).upper()],
            )
        return schedule

    def get_crontab_schedule(self, representation):
        schedule = None

        match = re.match(r'^([\*\d\-\/\,]+) ([\*\d\-\/\,]+) ([\*\d\-\/\,]+) ([\*\d\-\/\,]+) ([\*\d\-\/\,]+)$', representation)
        if match:
            schedule, created = self._task_crontab.store(representation,
                minute = match.group(1),
                hour = match.group(2),
                day_of_week = match.group(3),
                day_of_month = match.group(4),
                month_of_year = match.group(5)
            )
        return schedule

    def get_datetime_schedule(self, representation):
        schedule = None

        match = re.match(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$', representation)
        if match:
            schedule, created = self._task_datetime.store(representation,
                clocked_time = make_aware(datetime.strptime(representation, "%Y-%m-%d %H:%M:%S")),
            )
        return schedule


import argparse
import string

import pytest

from support import timing

def reacted_length(s: str) -> int:
    s = s.strip()
    ret = [s[0]]
    for c in s[1:]:
        if ret and c == ret[-1].swapcase():
            ret.pop()
        else:
            ret.append(c)
    return len(ret)

def compute(s: str) -> int:
    return min(
        reacted_length(s.replace(c, '').replace(c.upper(), ''))
        for c in string.ascii_lowercase
    )

@pytest.mark.parametrize(
    ('input_s', 'expected'),
    (
        ('dabAcCaCBAcCcaDA', 4),
        ('dabAcCaCBAcCcaDA\n', 4),
    ),
)
def test(input_s: str, expected: int) -> None:
    assert compute(input_s) == expected

def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument('data_file')
    args = parser.parse_args()

    with open(args.data_file) as f, timing():
        print(compute(f.read()))

    return 0

if __name__ == '__main__':
    exit(main())


'''
Utilities for cross-modality hashing experiments, including data loading and
statistics
'''
import numpy as np
import theano
import scipy.spatial
import pickle
import lasagne
import collections

def standardize(X):
    ''' Return column vectors to standardize X, via (X - X_mean)/X_std

    :parameters:
        - X : np.ndarray, shape=(n_filters, n_examples, n_features)
            Data matrix

    :returns:
        - X_mean : np.ndarray, shape=(n_filters, 1, n_features)
            Mean column vector
        - X_std : np.ndarray, shape=(n_filters, 1, n_features)
            Standard deviation column vector
    '''
    std = np.std(X, axis=1)
    return (np.mean(X, axis=1).reshape(X.shape[0], 1, X.shape[2]),
            (std + (std == 0)).reshape(X.shape[0], 1, X.shape[2]))

def load_data(train_list, valid_list):
    '''
    Load in dataset given lists of files in each split.
    Also standardizes (using train mean/std) the data.
    Each file should be a .npz file with a key 'X' for data in the X modality
    and 'Y' for data in the Y modality.

    :parameters:
        - train_list : list of str
            List of paths to files in the training set.
        - valid_list : list of str
            List of paths to files in the validation set.

    :returns:
        - X_train : list
            List of np.ndarrays of X modality features in training set
        - Y_train : list
            List of np.ndarrays of Y modality features in training set
        - X_valid : list
            List of np.ndarrays of X modality features in validation set
        - Y_valid : list
            List of np.ndarrays of Y modality features in validation set
    '''
    # We'll use dicts where key is the data subset, so we can iterate
    X = collections.defaultdict(list)
    Y = collections.defaultdict(list)
    for file_list, key in zip([train_list, valid_list],
                              ['train', 'valid']):
        # Load in all files
        for filename in file_list:
            data = np.load(filename)
            # Shingle and convert to floatX with correct column order
            X[key].append(np.array(
                data['X'], dtype=theano.config.floatX, order='C'))
            Y[key].append(np.array(
                data['Y'], dtype=theano.config.floatX, order='C'))
        # Get mean/std for training set
        if key == 'train':
            X_mean, X_std = standardize(np.concatenate(X[key], axis=1))
            Y_mean, Y_std = standardize(np.concatenate(Y[key], axis=1))
        # Use training set mean/std to standardize
        X[key] = [(x - X_mean)/X_std for x in X[key]]
        Y[key] = [(y - Y_mean)/Y_std for y in Y[key]]

    return X['train'], Y['train'], X['valid'], Y['valid']

def sample_sequences(X, Y, sample_size):
    ''' Given lists of sequences, crop out sequences of length sample_size
    from each sequence with a random offset

    :parameters:
        - X, Y : list of np.ndarray
            List of X/Y sequence matrices, each with shape (n_channels,
            n_time_steps, n_features)
        - sample_size : int
            The size of the cropped samples from the sequences

    :returns:
        - X_sampled, Y_sampled : np.ndarray
            X/Y sampled sequences, shape (n_samples, n_channels, n_time_steps,
            n_features)
    '''
    X_sampled = []
    Y_sampled = []
    for sequence_X, sequence_Y in zip(X, Y):
        # Ignore sequences which are too short
        if sequence_X.shape[1] < sample_size:
            continue
        # Compute a random offset to start cropping from
        offset = np.random.randint(0, sequence_X.shape[1] % sample_size + 1)
        # Extract samples of this sequence at offset, offset + sample_size,
        # offset + 2*sample_size ... until the end of the sequence
        X_sampled += [sequence_X[:, o:o + sample_size] for o in
                      np.arange(offset, sequence_X.shape[1] - sample_size + 1,
                                sample_size)]
        Y_sampled += [sequence_Y[:, o:o + sample_size] for o in
                      np.arange(offset, sequence_Y.shape[1] - sample_size + 1,
                                sample_size)]
    # Combine into new output array
    return np.array(X_sampled), np.array(Y_sampled)

def random_derangement(n):
    '''
    Permute the numbers up to n such that no number remains in the same place

    :parameters:
        - n : int
            Upper bound of numbers to permute from

    :returns:
        - v : np.ndarray, dtype=int
            Derangement indices

    :note:
        From
        http://stackoverflow.com/questions/26554211/numpy-shuffle-with-constraint
    '''
    while True:
        v = np.arange(n)
        for j in np.arange(n - 1, -1, -1):
            p = np.random.randint(0, j+1)
            if v[p] == j:
                break
            else:
                v[j], v[p] = v[p], v[j]
        else:
            if v[0] != 0:
                return v

def get_next_batch(X, Y, batch_size, sample_size, n_iter):
    ''' Randomly generates positive and negative example minibatches

    :parameters:
        - X, Y : list of np.ndarray
            List of data matrices in each modality
        - batch_size : int
            Size of each minibatch to grab
        - sample_size : int
            Size of each sampled sequence
        - n_iter : int
            Total number of iterations to run

    :returns:
        - X_p : np.ndarray
            Positive example minibatch in X modality
        - Y_p : np.ndarray
            Positive example minibatch in Y modality
        - X_n : np.ndarray
            Negative example minibatch in X modality
        - Y_n : np.ndarray
            Negative example minibatch in Y modality
    '''
    # These are dummy values which will force the sequences to be sampled
    current_batch = 1
    # We'll only know the number of batches after we sample sequences
    n_batches = 0
    for n in xrange(n_iter):
        if current_batch >= n_batches:
            X_sampled, Y_sampled = sample_sequences(X, Y, sample_size)
            N = X_sampled.shape[0]
            n_batches = int(np.floor(N/float(batch_size)))
            # Shuffle X_p and Y_p the same
            positive_shuffle = np.random.permutation(N)
            X_p = np.array(X_sampled[positive_shuffle])
            Y_p = np.array(Y_sampled[positive_shuffle])
            # Shuffle X_n and Y_n differently (derangement ensures nothing
            # stays in the same place)
            negative_shuffle = np.random.permutation(N)
            X_n = np.array(X_sampled[negative_shuffle])
            Y_n = np.array(Y_sampled[negative_shuffle][random_derangement(N)])
            current_batch = 0
        batch = slice(current_batch*batch_size, (current_batch + 1)*batch_size)
        yield X_p[batch], Y_p[batch], X_n[batch], Y_n[batch]
        current_batch += 1

def hash_entropy(X):
    ''' Get the entropy of the histogram of hashes.
    We want this to be close to n_bits.

    :parameters:
        - X : np.ndarray, shape=(n_examples, n_bits)
            Boolean data matrix, each column is the hash of an example

    :returns:
        - hash_entropy : float
            Entropy of the hash distribution
    '''
    # Convert bit vectors to ints
    bit_values = np.sum(2**np.arange(X.shape[1])*X, axis=1)
    # Count the number of occurences of each int
    counts = np.bincount(bit_values)
    # Normalize to form a probability distribution
    counts = counts/float(counts.sum())
    # Compute entropy
    return -np.sum(counts*np.log2(counts + 1e-100))

def statistics(X, Y):
    ''' Computes the number of correctly encoded codeworks and the number of
    bit errors made.  Assumes that rows of X should be hashed the same as rows
    of Y

    :parameters:
        - X : np.ndarray, shape=(n_examples, n_features)
            Data matrix of X modality
        - Y : np.ndarray, shape=(n_examples, n_features)
            Codeword matrix of Y modality

    :returns:
        - distance_distribution : int
            Emprical distribution of the codeword distances
        - mean_distance : float
            Mean of distances between corresponding codewords
        - std_distance : float
            Std of distances between corresponding codewords
    '''
    points_equal = (X == Y)
    distances = np.logical_not(points_equal).sum(axis=1)
    counts = np.bincount(distances, minlength=X.shape[1] + 1)
    return counts/float(X.shape[0]), np.mean(distances), np.std(distances)

def mean_reciprocal_rank(X, Y, indices):
    ''' Computes the mean reciprocal rank of the correct match
    Assumes that X[n] should be closest to Y[n]
    Uses hamming distance

    :parameters:
        - X : np.ndarray, shape=(n_examples, n_features)
            Data matrix in X modality
        - Y : np.ndarray, shape=(n_examples, n_features)
            Data matrix in Y modality
        - indices : np.ndarray
            Denotes which rows to use in MRR calculation

    :returns:
        - mrr_pessimist : float
            Mean reciprocal rank, where ties are resolved pessimistically
            That is, rank = # of distances <= dist(X[:, n], Y[:, n])
        - mrr_optimist : float
            Mean reciprocal rank, where ties are resolved optimistically
            That is, rank = # of distances < dist(X[:, n], Y[:, n]) + 1
    '''
    # Compute distances between each codeword and each other codeword
    distance_matrix = scipy.spatial.distance.cdist(X, Y, metric='hamming')
    # Rank is the number of distances smaller than the correct distance, as
    # specified by the indices arg
    n_le = distance_matrix.T <= distance_matrix[np.arange(X.shape[0]), indices]
    n_lt = distance_matrix.T < distance_matrix[np.arange(X.shape[0]), indices]
    return (np.mean(1./n_le.sum(axis=0)),
            np.mean(1./(n_lt.sum(axis=0) + 1)))

def build_network(input_shape, num_filters, filter_size, ds,
                  hidden_layer_sizes, dropout, n_bits):
    '''
    Construct a list of layers of a network given the network's structure.

    :parameters:
        - input_shape : tuple
            In what shape will data be supplied to the network?
        - num_filters : list
            Number of features in each convolutional layer
        - filter_size : list
            Number of features in each convolutional layer
        - ds : dict of list
            Number of features in each convolutional layer
        - hidden_layer_sizes : list
            Size of each hidden layer
        - dropout : bool
            Should dropout be applied between fully-connected layers?
        - n_bits : int
            Output dimensionality

    :returns:
        - layers : list
            List of layer instances for this network.
    '''
    layers = [lasagne.layers.InputLayer(shape=input_shape)]
    # Add each convolutional and pooling layer recursively
    for n in xrange(len(num_filters)):
        # In order to get the same shape out from each convolutional layer, we
        # need to pad the input because Conv2DLayer uses 'valid' convolution by
        # default and we can't rely on Conv2DLayer's mode='same' because it
        # doesn't allow even filter sizes.
        layers.append(lasagne.layers.PadLayer(
            layers[-1], width=((int(np.ceil((filter_size[n][0] - 1) / 2.)),
                                int(np.floor((filter_size[n][0] - 1) / 2.))),
                               (int(np.ceil((filter_size[n][1] - 1) / 2.)),
                                int(np.floor((filter_size[n][1] - 1) / 2.))))))
        # We will initialize weights to \sqrt{2/n_l}
        n_l = num_filters[n]*np.prod(filter_size[n])
        layers.append(lasagne.layers.Conv2DLayer(
            layers[-1], stride=(1, 1), num_filters=num_filters[n],
            filter_size=filter_size[n],
            nonlinearity=lasagne.nonlinearities.rectify,
            W=lasagne.init.Normal(np.sqrt(2./n_l))))
        layers.append(lasagne.layers.MaxPool2DLayer(
            layers[-1], ds[n], ignore_border=False))
    # A dense layer will treat any dimensions after the first as feature
    # dimensions, but the third dimension is really a timestep dimension.
    # We can only squash adjacent dimensions with a ReshapeLayer, so we
    # need to place the time stpe dimension after the batch dimension
    layers.append(lasagne.layers.DimshuffleLayer(
        layers[-1], (0, 2, 1, 3)))
    conv_output_shape = layers[-1].output_shape
    # Reshape to (n_batch*n_time_steps, n_conv_output_features)
    layers.append(lasagne.layers.ReshapeLayer(
        layers[-1], (-1, conv_output_shape[2]*conv_output_shape[3])))
    # Add dense hidden layers and optionally dropout
    for hidden_layer_size in hidden_layer_sizes:
        layers.append(lasagne.layers.DenseLayer(
            layers[-1], num_units=hidden_layer_size,
            nonlinearity=lasagne.nonlinearities.rectify))
        if dropout:
            layers.append(lasagne.layers.DropoutLayer(layers[-1], .5))
    # Add output layer
    layers.append(lasagne.layers.DenseLayer(
        layers[-1], num_units=n_bits,
        nonlinearity=lasagne.nonlinearities.tanh))

    return layers

def save_model(param_list, output_file):
    '''
    Write out a pickle file of a hashing network

    :parameters:
        - param_list : list of np.ndarray
            A list of values, per layer, of the parameters of the network
        - output_file : str
            Path to write the file to
    '''
    with open(output_file, 'wb') as f:
        pickle.dump(param_list, f)

def load_model(layers, param_file):
    '''
    Load in the parameters from a pkl file into a model

    :parameters:
        - layers : list
            A list of layers which define the model
        - param_file : str
            Pickle file of model parameters to load
    '''
    with open(param_file) as f:
        params = pickle.load(f)
    lasagne.layers.set_all_param_values(layers[-1], params)


import json
import random
import sqlite3
import math
import itertools
from scipy.sparse import hstack
from scipy import spatial
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.neural_network import MLPClassifier
from operator import add
import datetime

def get_input_network_feature(input_list, claim_triples, evidence_triples, nlp):
	consistent_count = input_list[0]
	inconsistent_count = input_list[1]
	count = 0
	for i in claim_triples:
		count += 1
		if count > 100:
			break
		claim_entity = i[0]
		claim_noun = i[1]
		claim_verb = i[2]
		for j in evidence_triples:
			evidence_entity = j[0]
			if claim_entity == evidence_entity:
				evidence_noun = j[1]
				evidence_verb = j[2]
				if claim_noun == evidence_noun:
					if nlp(claim_verb).similarity(nlp(evidence_verb)) >= 0.5:
						consistent_count += 1
					else:
						inconsistent_count += 1
			else:
				continue
	return [consistent_count, inconsistent_count]

def get_entity_noun_verb_triples(source_sents, nlp):
	doc = nlp(source_sents)
	triple_list = [] # possible entity-noun-verb triples
	for sentence in doc.sents:
		entity_list = []
		for entity in sentence.ents:
			if entity.label_ == "CARDINAL" or entity.label_ == "ORDINAL" or entity.label_ == "DATE":
				continue
			entity_word = entity.text
			entity_list.append(entity_word.replace(".", "").replace(" ","").replace("`","").replace("\"","").replace("\'",""))

		noun_list = [] # list of nouns in a claim
		verb_list = [] # list of verbs in a claim
		if len(entity_list) == 0:
			continue
		else:
			for token in sentence:
				if "VB" in token.tag_: # extract a list of verbs
					word = token.text
					lemma = lemmatizer(word.replace(".", "").replace(" ","").replace("`","").replace("\"","").replace("\'",""), token.pos_)[0]
					verb_list.append(lemma)
				elif "NN" in token.tag_: # extract a list of nouns
					word = token.text
					lemma = lemmatizer(word.replace(".", "").replace(" ","").replace("`","").replace("\"","").replace("\'",""), token.pos_)[0]
					if nlp.vocab[lemma.lower()].is_stop: # exclude stop words
						continue
					if lemma not in entity_list:
						noun_list.append(lemma)
			if len(noun_list) == 0 or len(verb_list) == 0:
				pass
			else:
				for i in itertools.product(entity_list, noun_list, verb_list):
					if len(triple_list) > 100:
						break
					triple_list.append(i)
		if len(triple_list) > 100:
			break
	return triple_list

# connecting to sqlite3 db where indices for wikipedia pages are stored
# the indices have been stored using "data_processing.py" included in the repository
# these indices will be used when retrieving relevant data for fact checking tasks
conn = sqlite3.connect('fever.db')
cursor = conn.cursor()
# a process for collecing index information of gold standard evidence at a documnet level
# it is for extracting a vocabulary, from the training data, for the word vector (i.e., 5000 most frequently appearing words) used in this algorithm
train_file = open('train.jsonl', "r") # open a file containing training data
train_wiki_page_index_set = set() # a set-typed variable to store the indices of wiki pages in the training data
for line in train_file:
	json_data = json.loads(line) # parse each line (or data point) as a json-typed element
	if json_data["label"] == "NOT ENOUGH INFO": # data points labeled as "NOT ENOUGH INFO" do not have corresponding gold standard documents, and thus no use for extracting the word vector
		continue
	for i in json_data["evidence"]:
		for j in i:
			cursor.execute("SELECT file_num, line_num FROM evidence_index WHERE id = ?", (j[2],)) # from the index db, we select the file number and the line number for correspodning document
			# exception handling for spanish characters with accents
			# this issue will be resolved in a later version
			try:
				temp = cursor.fetchone()
				file_num = temp[0]
				line_num = temp[1]
				train_wiki_page_index_set.add((file_num,line_num))
			except:
				continue
print("done extracting an initial set of vocabulary")

training_data_for_vocabulary = []
for element in train_wiki_page_index_set:
	file_num = element[0]
	line_num = element[1]
	if file_num < 10:
		file = open("wiki-00"+str(element[0])+".jsonl", "r")
	elif file_num < 100:
		file = open("wiki-0"+str(element[0])+".jsonl", "r")
	else:
		file = open("wiki-"+str(element[0])+".jsonl", "r")
	line_num_count = 1
	for line in file:
		if line_num_count == line_num:
			json_data = json.loads(line)
			training_data_for_vocabulary.append(json_data["text"])
			break # break the loop for searching documnets when the targeted one is retrieved
		else:
			pass
		line_num_count += 1

# Since, in the original baseline paper, the authors used both TF and TF-IDF vectors, we estimate them separtely
tfvectorizer = TfidfVectorizer(stop_words='english', max_features=5000, use_idf = False) # vectorizer for TF
tfidfvectorizer = TfidfVectorizer(stop_words='english', max_features=5000, use_idf = True) # vectorizer for TF-IDF
tfvectorizer.fit(training_data_for_vocabulary)
tfidfvectorizer.fit(training_data_for_vocabulary)
print("done word vectorization")

# packages needed for deriving sentiment features
import spacy
from spacy.lemmatizer import Lemmatizer
from spacy.lang.en import LEMMA_INDEX, LEMMA_EXC, LEMMA_RULES
lemmatizer = Lemmatizer(LEMMA_INDEX, LEMMA_EXC, LEMMA_RULES)
nlp = spacy.load('en_core_web_lg')

# count the number of support-labeled, refute-labeled, and not-enough-info-labled claims for a post-anlaysis
support_count = 0
refute_count = 0
not_enough_info_count = 0

# each input contains the TF vectors of a claim and its evidences in a concatenated manner
# and the cosine similarity of the TF-IDF vectors of the claim and the evidences
X_train = []
# each output can be classified into 0=support, 1=refute, and 2=not-enough-info
y_train = []
exception_count = 0
train_file = open('train.jsonl', "r")
process_count = 0
for line in train_file:
	print(process_count, datetime.datetime.now())
	process_count += 1
	json_data = json.loads(line)
	claim_label = json_data["label"]
	# sentiment features of words in the claim_label
	# extract entities, nouns, and verbs from a claim
	claim_triple_list = get_entity_noun_verb_triples(json_data["claim"], nlp)

	network_input = [0,0] #initialize network input

	# vecotrize a claim with the pretrained TF and TF-IDF vectors
	claim_tf = tfvectorizer.transform([json_data["claim"]])
	claim_tfidf = tfidfvectorizer.transform([json_data["claim"]])
	try:
		evidence_text = []
		# for each claim whose gold label is "NOT ENOUGH INFO", we randonly generate three evidences since they do not have gold evidences
		if json_data["label"] == "NOT ENOUGH INFO":
			for i in range(3):
				random_file_num = random.randint(1,109)
				if random_file_num < 10:
					evidence_file = open("wiki-00"+str(random_file_num)+".jsonl", "r")
				elif random_file_num < 100:
					evidence_file = open("wiki-0"+str(random_file_num)+".jsonl", "r")
				else:
					evidence_file = open("wiki-"+str(random_file_num)+".jsonl", "r")
				max_count = 0
				for i in evidence_file:
					max_count += 1
				random_line_num = random.randint(0, max_count)

				if random_file_num < 10:
					evidence_file = open("wiki-00"+str(random_file_num)+".jsonl", "r")
				elif random_file_num < 100:
					evidence_file = open("wiki-0"+str(random_file_num)+".jsonl", "r")
				else:
					evidence_file = open("wiki-"+str(random_file_num)+".jsonl", "r")
				count = 1
				for evidence_line in evidence_file:
					if count == random_line_num:
						evidence_data = json.loads(evidence_line)
						evidence_text.append(evidence_data["text"])
						result_triple_list = get_entity_noun_verb_triples(evidence_data["text"], nlp)
						if len(result_triple_list) == 0:
							continue
						else:
							network_input = get_input_network_feature(network_input, claim_triple_list, result_triple_list, nlp)
					else:
						pass
					count += 1

		# for each claim whose gold label is either "SUPPORT" or "REFUTE", we retreived gold documents
		else:
			for i in json_data["evidence"]:
				evidence_count = 0 # limit the number of gold evidences attached to a claim to three
				for j in i:
					cursor.execute("SELECT file_num, line_num FROM evidence_index WHERE id = ?", (j[2],))
					temp = cursor.fetchone()
					file_num = temp[0]
					line_num = temp[1]
					if file_num < 10:
						evidence_file = open("wiki-00"+str(file_num)+".jsonl", "r")
					elif file_num < 100:
						evidence_file = open("wiki-0"+str(file_num)+".jsonl", "r")
					else:
						evidence_file = open("wiki-"+str(file_num)+".jsonl", "r")
					count = 1
					for evidence_line in evidence_file:
						if count == line_num:
							evidence_data = json.loads(evidence_line)
							evidence_text.append(evidence_data["text"])
							result_triple_list = get_entity_noun_verb_triples(evidence_data["text"], nlp)
							if len(result_triple_list) == 0:
								continue
							else:
								network_input = get_input_network_feature(network_input, claim_triple_list, result_triple_list, nlp)
						else:
							pass
						count += 1
					evidence_count += 1
				if evidence_count == 3:
					break
		# concantenate multiple evidences
		evidence_concat = ""
		for ev in evidence_text:
			evidence_concat += (" " + ev)
		# vecotrize evidences with the pretrained TF and TF-IDF vectors
		evidence_tf = tfvectorizer.transform([evidence_concat])
		evidence_tfidf = tfidfvectorizer.transform([evidence_concat])
		# estimate TF-IDF cosine similarity of the claim and the evidence vetors
		tfidf_cosine_similarity = 1 - spatial.distance.cosine(claim_tfidf.toarray(), evidence_tfidf.toarray())
		if math.isnan(tfidf_cosine_similarity):
			tfidf_cosine_similarity = 0
		# concatenation input vectors into one
		tf_vector = hstack((claim_tf, evidence_tf))
		input_vector = hstack((tf_vector, [tfidf_cosine_similarity], network_input))
		# append the final input vector
		X_train.append(input_vector.toarray()[0]) # take the first element since the "toarray()" returns 2-dim data strucute when 1-dim is needed
		# append the corresponding output
		if claim_label == "SUPPORTS":
			y_train.append(0)
			support_count += 1
		elif claim_label == "REFUTES":
			y_train.append(1)
			refute_count += 1
		else:
			y_train.append(2)
			not_enough_info_count += 1
	except Exception as e:
		print(e,json_data["claim"])
		exception_count += 1

print("done training data preparation")

# train a multi-layer perceptron
mlp = MLPClassifier()
mlp.fit(X_train, y_train)
print("done mlp model fitting")

# prepare test data
# same process for preparing training data
# for understanding the codes, refer to the comments on the codes for training data preparation
test_support_count = 0
test_refute_count = 0
test_not_enough_info_count = 0
X_test = []
y_test = []
test_exception_count = 0
test_file = open('test.jsonl', "r")
for line in test_file:
	json_data = json.loads(line)
	claim_label = json_data["label"]
	# sentiment features of words in the claim_label
	# extract entities, nouns, and verbs from a claim
	claim_triple_list = get_entity_noun_verb_triples(json_data["claim"], nlp)

	network_input = [0,0] #initialize network input

	# vecotrize a claim with the pretrained TF and TF-IDF vectors
	claim_tf = tfvectorizer.transform([json_data["claim"]])
	claim_tfidf = tfidfvectorizer.transform([json_data["claim"]])
	try:
		evidence_text = []
		# for each claim whose gold label is "NOT ENOUGH INFO", we randonly generate three evidences since they do not have gold evidences
		if json_data["label"] == "NOT ENOUGH INFO":
			for i in range(3):
				random_file_num = random.randint(1,109)
				if random_file_num < 10:
					evidence_file = open("wiki-00"+str(random_file_num)+".jsonl", "r")
				elif random_file_num < 100:
					evidence_file = open("wiki-0"+str(random_file_num)+".jsonl", "r")
				else:
					evidence_file = open("wiki-"+str(random_file_num)+".jsonl", "r")
				max_count = 0
				for i in evidence_file:
					max_count += 1
				random_line_num = random.randint(0, max_count)

				if random_file_num < 10:
					evidence_file = open("wiki-00"+str(random_file_num)+".jsonl", "r")
				elif random_file_num < 100:
					evidence_file = open("wiki-0"+str(random_file_num)+".jsonl", "r")
				else:
					evidence_file = open("wiki-"+str(random_file_num)+".jsonl", "r")
				count = 1
				for evidence_line in evidence_file:
					if count == random_line_num:
						evidence_data = json.loads(evidence_line)
						evidence_text.append(evidence_data["text"])
						result_triple_list = get_entity_noun_verb_triples(evidence_data["text"], nlp)
						if len(result_triple_list) == 0:
							continue
						else:
							network_input = get_input_network_feature(network_input, claim_triple_list, result_triple_list, nlp)
					else:
						pass
					count += 1

		# for each claim whose gold label is either "SUPPORT" or "REFUTE", we retreived gold documents
		else:
			for i in json_data["evidence"]:
				evidence_count = 0 # limit the number of gold evidences attached to a claim to three
				for j in i:
					cursor.execute("SELECT file_num, line_num FROM evidence_index WHERE id = ?", (j[2],))
					temp = cursor.fetchone()
					file_num = temp[0]
					line_num = temp[1]
					if file_num < 10:
						evidence_file = open("wiki-00"+str(file_num)+".jsonl", "r")
					elif file_num < 100:
						evidence_file = open("wiki-0"+str(file_num)+".jsonl", "r")
					else:
						evidence_file = open("wiki-"+str(file_num)+".jsonl", "r")
					count = 1
					for evidence_line in evidence_file:
						if count == line_num:
							evidence_data = json.loads(evidence_line)
							evidence_text.append(evidence_data["text"])
							result_triple_list = get_entity_noun_verb_triples(evidence_data["text"], nlp)
							if len(result_triple_list) == 0:
								continue
							else:
								network_input = get_input_network_feature(network_input, claim_triple_list, result_triple_list, nlp)
						else:
							pass
						count += 1
					evidence_count += 1
				if evidence_count == 3:
					break
		evidence_concat = ""
		for ev in evidence_text:
			evidence_concat += (" " + ev)
		evidence_tf = tfvectorizer.transform([evidence_concat])
		evidence_tfidf = tfidfvectorizer.transform([evidence_concat])
		tfidf_cosine_similarity = 1 - spatial.distance.cosine(claim_tfidf.toarray(), evidence_tfidf.toarray())
		if math.isnan(tfidf_cosine_similarity):
			tfidf_cosine_similarity = 0
		tf_vector = hstack((claim_tf, evidence_tf))
		input_vector = hstack((tf_vector, [tfidf_cosine_similarity], network_input))
		X_test.append(input_vector.toarray()[0])

		if claim_label == "SUPPORTS":
			y_test.append(0)
			test_support_count += 1
		elif claim_label == "REFUTES":
			y_test.append(1)
			test_refute_count += 1
		else:
			y_test.append(2)
			test_not_enough_info_count += 1
	except Exception as e:
		print(e, json_data["claim"])
		test_exception_count += 1
print("done test data preparation")

print(exception_count, test_exception_count)
print(support_count, refute_count, not_enough_info_count) # distribution of outcomes in training data
print(test_support_count, test_refute_count, test_not_enough_info_count) # distribution of outcomes in test datda
print(mlp.score(X_test, y_test)) # accuary of the model
# for error analysis, write out prediction restuls in a separate text file
X_predicted = mlp.predict(X_test)
result = open("result_network_fast_part_unnormalized.txt", "w")
for i in range(len(X_predicted)):
	result.write(str(X_predicted[i]) + "\t" + str(y_test[i]) + "\n")


# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.
import scrapy
from TestDemo.items import TestdemoItem
class TestDemo(scrapy.Spider):

    name = "douban"
    allowed_domains = ['movie.douban.com']
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0'}
    # start_urls = ["https://movie.douban.com/top250"]

    def start_requests(self):
        url = 'https://movie.douban.com/top250'
        yield scrapy.Request(url, headers=self.headers)

    def parse(self, response):
        items = TestdemoItem()
        movies = response.xpath("//ol[@class='grid_view']/li")
        for movie in movies:
            items["movie_rank"] = movie.xpath(".//div[@class='pic']/em/text()").extract()[0]
            items["movie_name"]=movie.xpath(".//div[@class='hd']/a/span[1]/text()").extract()[0]
            items["movie_score"] = movie.xpath(".//div[@class='star']/span[@class='rating_num']/text()").extract()[0]
            items["movie_comment"] = movie.xpath(".//div[@class='star']/span/text()").re('\d+人评价')[0]
            print(items["movie_rank"] + ":" + items["movie_name"] + " " + \
                  items["movie_score"] + " " + items["movie_comment"])
            yield items

        next_url = response.xpath("//span[@class='next']/a/@href").extract()
        if next_url:
            next_url = "https://movie.douban.com/top250" + next_url[0]
            yield scrapy.Request(next_url, callback=self.parse,headers=self.headers)


"""This file is used for the visualization of the trends in Greenhouse Gases
and carbon emissions.
Resources:
- https://www.geeksforgeeks.org/box-plot-in-python-using-matplotlib/
- https://www.kite.com/python/answers/how-to-color-a-scatter-plot-by-category-using-matplotlib-in-python
- https://plotly.com/python/box-plots/
- https://towardsdatascience.com/how-to-create-an-animated-choropleth-map-with-less-than-15-lines-of-code-2ff04921c60b
"""
from typing import List

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import plotly.graph_objects as go
import plotly.express as px

from data_manager import Dataset, group_by_values

def data_by_tags_country_year(filepath: str,
                              tags: List[str],
                              countries_or_years: list,
                              country_or_year: int) -> List[List[float]]:
    """The function would take in the filepath of the dataset from which it would extract the data
    convert every variable in the row into its appropriate datatype and then filter out the data
    containing only the tags we want, and only for the countries or years we want, and then return
    emission values separately for all the countries or years.

    @param filepath: the path of the dataset
    @param tags: list of all the categories of emissions we want in the data
    @param countries_or_years: The countries or years for which we want the data
    @param country_or_year: 0 or 1 based on if we want to categorise by country or year
    @return: Return a list containing lists for every countries emission value
    """

    # extract dataset and covert all variables into their respective datatype
    dataset = Dataset(filepath=filepath, types=[str, int, float, str])

    # keeping rows with tags we want
    dataset.filter_by_value(3, tags)

    # splitting by countries
    grouped_data = group_by_values(dataset, country_or_year)

    # Extracting emission values for the countries we want to analyze
    countries_so_far = []

    for county in countries_or_years:
        county_data = grouped_data[county]
        countries_so_far.append(county_data.extract_column(2))

    return countries_so_far

def data_by_tags(filepath: str,
                 tags: List[str],
                 country_filter: List[str] = None,
                 year_filter: List[int] = None) -> List[List]:
    """The function would take in the filepath of the dataset from which it would extract the data
    convert every variable in the row into its appropriate datatype and then filter out the data
    containing only the tags we want, and return the emission values along side the respective
    country and year.

    We can pass in optional lists to only get rows with specific country or year

    @param filepath: the path of the dataset
    @param tags: list of all the categories of emissions we want in the data
    @param country_filter: list of countries we want data for
    @param year_filter: list of years we want data for
    @return: Return a list with emission value alongside the respective country and year
    """

    # extract dataset and covert all variables into thir respective datatype
    dataset = Dataset(filepath=filepath, types=[str, int, float, str])

    # keeping rows with tags we want
    dataset.filter_by_value(3, tags)

    # filtering countries and years
    if country_filter:
        dataset.filter_by_value(0, country_filter)
    if year_filter:
        dataset.filter_by_value(1, year_filter)

    # extract the values, countries and years
    values = dataset.extract_column(2)
    countries = dataset.extract_column(0)
    year = dataset.extract_column(1)

    return [values, countries, year]

def tidy_gdp_data(filepath: str) -> Dataset:
    """Transform the data for plotting purposes. It will remove all the columns with year before 1990 and
    after 2016, and convert columns to their respective data type.

    :param filepath: path of the dataset
    :return: transformed dataset object
    """
    # loading data
    raw_data = Dataset(filepath)

    # creating new dataset object
    data = Dataset(dataset=[])

    # making the data tidy
    for row in raw_data.get():
        for element_index in range(4, len(row)):

            value = row[element_index]
            year = 1960 + element_index - 4

            if 1990 <= year <= 2016:
                data.push([row[0], row[1], row[2], row[3], year, value])

    data.remove_na()

    data.transform([str, str, str, str, int, float])

    return data

# Plot1
def plot_all_countries_peryear(filepath: str) -> None:
    """ Plots 5 boxplots showing amount of nitrogen triflouride (NF3) emissions (kilotonnes) in all
    countries taken together over the years 2010, 2011, 2012, 2013 and 2014.

    Through this visualization, it is easier to understand correlation of how increased development
    has affected Nitrogen Triflouride emissions in recent years.

    @param filepath: the path of the dataset
    """

    years = [2010, 2011, 2012, 2013, 2014]

    tags = ['nitrogen_trifluoride_nf3_emissions_in_kilotonne_co2_equivalent']

    data = data_by_tags_country_year(filepath, tags, years, 1)
    x1 = np.array(data[0])  # NF3 emissions for all countries in 2010
    x2 = np.array(data[1])  # NF3 emissions for all countries in 2011
    x3 = np.array(data[2])  # NF3 emissions for all countries in 2012
    x4 = np.array(data[3])  # NF3 emissions for all countries in 2013
    x5 = np.array(data[4])  # NF3 emissions for all countries in 2014

    plt.boxplot((x1, x2, x3, x4, x5), notch=False, sym="o", labels=["2010",
                                                                    "2011",
                                                                    "2012",
                                                                    "2013",
                                                                    "2014"])
    plt.xlabel("Region/Country", fontsize=15)
    plt.ylabel("Aggregated Emissions of Gases (in kilotonnes) ", fontsize=9)

    plt.title("Aggregated Nitrogen Triflouride emissions for 5 years (from all countries) ",
              fontsize=10,
              fontweight="bold")

    plt.show()

# Plot2
def plot_scatter_n2o(filepath: str) -> None:
    """Plots a scatterplot showing amount of nitrous oxide (N2O) emissions (kilotonnes) in USA,
    Canada and the European Union each year (between 1990 and 2014).

    This helps us to understand trends of nitrous oxide emissions in 3 countries with very
    different demographics. (Canada - Low population Density, EU - Higher Population Density,
    USA - Large scale industrialization)

    @param filepath: the path of the dataset
    """

    countries = ['Canada', 'United States of America', 'European Union']

    tags = [
        'nitrous_oxide_n2o_emissions_without_land_use_land_use_change_and_forestry_lulucf_in_kilotonne_co2_equivalent']

    data = data_by_tags(filepath, tags, country_filter=countries)

    y = data[0]  # emission data
    labels = data[1]  # list of countries
    x = data[2]  # list of years

    data = pd.DataFrame({"X Value": x, "Y Value": y, "Category": labels})

    groups = data.groupby("Category")  # to group the data by region/country

    for name, group in groups:
        plt.plot(group["X Value"], group["Y Value"], marker="o", linestyle="-", label=name)

    plt.xlabel("Years", fontsize=15)
    plt.ylabel("Emissions of Gases (in kilotonnes)", fontsize=15)
    plt.title("Nitrous Oxide emissions each year")
    plt.legend()
    plt.show()

# Plot3
def scatterboxplot_sparsely_pop(filepath: str) -> None:
    """ Plots a scatterplot and a box plot together showing distribution of amount of carbon
    dioxide (CO2) emissions (kilotonnes) in Austria, Belarus and Bulgaria in the 24 year period
    (between 1990 and 2014).

    This plot studies and visualizes details of total carbon dioxide emissions in sparsely
    populated countries. The scatterplot shows each year's emissions data for the 3 aforementioned
    countries and the boxplot aggregates and visualizes this data.

    @param filepath: the path of the dataset
    """
    countries = ['Austria', 'Belarus', 'Bulgaria']

    tags = [
        'carbon_dioxide_co2_emissions_without_land_use_land_use_change_and_forestry_lulucf_in_kilotonne_co2_equivalent']

    data = data_by_tags_country_year(filepath, tags, countries, 0)

    y0 = data[0]
    y1 = data[1]
    y2 = data[2]

    emission_values = [y0, y1, y2]

    colors = ['rgba(93, 164, 214, 0.5)', 'rgba(255, 144, 14, 0.5)', 'rgba(44, 160, 101, 0.5)',
              'rgba(255, 65, 54, 0.5)', 'rgba(207, 114, 255, 0.5)', 'rgba(127, 96, 0, 0.5)']

    fig = go.Figure()

    for xd, yd, cls in zip(countries, emission_values, colors):
        fig.add_trace(go.Box(
            y=yd,
            name=xd,
            boxpoints='all',
            jitter=0.5,
            whiskerwidth=0.2,
            fillcolor=cls))

    fig.update_layout(
        title='Carbon Dioxide (CO2) Emissions in kilotonnes for 3 sparsely populated countries (from 1990 - 2014)',
        margin=dict(
            l=40,
            r=30,
            b=80,
            t=100),
        paper_bgcolor='rgb(243, 243, 243)',
        plot_bgcolor='rgb(243, 243, 243)',
        showlegend=False
    )

    fig.show()

# Plot4
def plot_scatter_boxplot_1990_2014(filepath: str) -> None:
    """Plots a scatterplot and a box plot together showing distribution of hydrofluorocarbon (hfcs)
    emissions in 1990, 2002 and 2014 (in all countries taken together).

    This tells us about trends of changes in hydroflourocarbon emissions in 1990, 2002 and 2014.
    This helps to correlate increased industrialization over the years with release of more
    pollutants. The scatterplot shows each country's emissions data for the 3 aforementioned years
    and the boxplot aggregates and visualizes this data.

    @param filepath: the path of the dataset
    """
    years = [1990, 2002, 2014]

    tags = ['hydrofluorocarbons_hfcs_emissions_in_kilotonne_co2_equivalent']

    data = data_by_tags_country_year(filepath, tags, years, 1)

    y0 = data[0]
    y1 = data[1]
    y2 = data[2]

    emission_values = [y0, y1, y2]

    colors = ['rgba(93, 164, 214, 0.5)', 'rgba(255, 144, 14, 0.5)', 'rgba(44, 160, 101, 0.5)',
              'rgba(255, 65, 54, 0.5)', 'rgba(207, 114, 255, 0.5)', 'rgba(127, 96, 0, 0.5)']

    fig = go.Figure()

    for xd, yd, cls in zip(years, emission_values, colors):
        fig.add_trace(go.Box(
            y=yd,
            name=xd,
            boxpoints='all',
            jitter=0.5,
            whiskerwidth=0.2,
            fillcolor=cls))

    fig.update_layout(
        title='Hydrofluorocarbon (hfcs) Emissions in kilotonnes by all countries (aggregated together) '
              'for 1990, 2002, 2014 ',
        margin=dict(
            l=40,
            r=30,
            b=80,
            t=100),
        paper_bgcolor='rgb(243, 243, 243)',
        plot_bgcolor='rgb(243, 243, 243)',
        showlegend=False
    )

    fig.show()

def plot_emission_per_gdp(filepath: str) -> None:
    """Plots a Choropleth map showing the distribution of CO2 emissions (kg per PPP $ of GDP)
    for all countries in the dataset.

    @param filepath:the path of the dataset
    """
    data = tidy_gdp_data(filepath)

    df = pd.DataFrame({'country': data.extract_column(0),
                       'code': data.extract_column(1),
                       'year': data.extract_column(4),
                       'CO2 emissions (kg per PPP $ of GDP)': data.extract_column(5)})

    fig = px.choropleth(df,
                        locations='country',
                        color='CO2 emissions (kg per PPP $ of GDP)',
                        animation_frame='year',
                        locationmode='country names',
                        scope='world',
                        range_color=(min(data.extract_column(5)), max(data.extract_column(5))),
                        title='CO2 emissions (kg per PPP $ of GDP) by country',
                        height=600
                        )

    fig.show()

def dist_2016(filepath: str) -> None:
    """The function plots a pie chart for percentage contribution of different
    countries for CO2 emissions (kg per PPP $ of GDP) for 2016.

    @param filepath: path of teh dataset
    """
    data = tidy_gdp_data(filepath)

    data.filter_by_value(4, [2016])

    total_in_world = sum(data.extract_column(5))

    values = [value / total_in_world for value in data.extract_column(5)]

    df = pd.DataFrame({'country': data.extract_column(0),
                       'code': data.extract_column(1),
                       'year': data.extract_column(4),
                       'CO2 emissions (kg per PPP $ of GDP)': values})

    fig = px.pie(df,
                 values='CO2 emissions (kg per PPP $ of GDP)',
                 names='country'
                 )

    fig.show()

if __name__ == '__main__':
    import python_ta

    python_ta.check_all(config={
        'extra-imports': ['plotly.express',
                          'plotly.graph_objects',
                          'matplotlib.pyplot',
                          'typing',
                          'data_manager',
                          'numpy',
                          'pandas'],
        'allowed-io': [],
        'max-line-length': 150,
        'disable': ['R1705', 'C0200']
    })


import gzip
import urllib
import urllib2
import hashlib
import time
import chardet

from decorator import decorator
from copy import copy
from urlparse import urlparse
from StringIO import StringIO

from traffgroup.core.model.site import Site
from traffgroup.core.X.mako import render_template
from traffgroup.core.X.xflask import g, request, capp
from traffgroup.core.X.utils import XRedis
from traffgroup.x.util.func import url_append

def url(url, q=None, **kwargs):
    if not q:
        q = {}

    # pass all other params too
    nazavod_url_vars = ['project', 'marker', 'flow', 'id', 'sign', 'gttl']
    for var_name, var_val in filter(lambda var: var[0] not in nazavod_url_vars,
                                    request.values.items()):
        kwargs[var_name] = var_val

    kwargs.update({'marker':    g.marker,
                   'sign':      g.nsign,
                   'gttl':      g.gttl,
                   'marker':    g.marker,
                   'project':   g.project,
                   'flow':      g.flow})
    return url_append(url, q, **kwargs)

def get_footer(host):
    if not host:
        return ''
    xredis = XRedis(db=2)
    footer = xredis.get(host)
    if footer:
        return footer

    req = urllib2.Request(url=host)
    footer = urllib2.urlopen(req).read()
    try:
        if 'footer_begin' in footer:
            offer_parts = footer.split('footer_begin')
            pre_footer = '
'
            footer = pre_footer + offer_parts[1].split('footer_end')[0]           #TODO: shitty shit, find out another way
    except:
        pass

    xredis.setex(host, 86400, footer)
    return footer

def get_terms(host):
    if not host:
        return ''

    xredis = XRedis(db=3)
    terms = xredis.get(host)
    if terms:
        return terms

    req = urllib2.Request(url=host)
    terms = urllib2.urlopen(req).read()
    xredis.setex(host, 86400, terms)
    return terms

def is_member_valid(key=None):
    """
    Validates member by access key
    @param key {string}
    @returns True if valid and False otherwise
    """
    if not key:
        key = (request.args.get('access', '') or
               request.cookies.get('access',''))
    key_parts = key.split(':')
    if len(key_parts) != 4:
        return False

    sub_id, ts_start, duration, signature = key_parts
    if (time.time() > int(ts_start) + int(duration) or
        hashlib.md5(sub_id + ts_start + duration +
                    Site.Globals.SALT).hexdigest() != signature):
        return False
    return True

class Template(object):
    def __init__(self, tmpl, **kwargs):
        self.template = copy(tmpl)
        self.extras = dict(kwargs)

    def __call__(self, action_):
        return decorator(self.wrap_action, action_)

    @staticmethod
    def tmpl_path(tmpl_name):
        return '/paysite/%s/%s/%s' % (g.root_design.templates_folder,
                                      g.scheme, tmpl_name)

    def wrap_action(self, action_, *args, **kwargs):
        g.action_result = action_(*args, **kwargs)
        for k, v in self.extras.iteritems():
            setattr(g, k, v)

        return render_template(Template.tmpl_path(self.template))

def urldecode(url):
    res = {}
    for item in url.split('&'):
        key_val = item.split('=')
        res[urllib.unquote_plus(key_val[0])] = urllib.unquote_plus(key_val[1])
    return res

def fetch(url, post_data=None):
    """
    Get data at url
    @param url: url
    @param post_data: dict
    @return: data, content_type (css and html supported at this moment)
    """

    content_type = '%s; charset=utf-8'

    url = urllib2.unquote(url)
    req = urllib2.Request(url)
    if post_data:
        post_data = urllib.urlencode(request.values)

    # replace urllib2 user agent for prevent browser dependent
    # web site behavior
    req.add_header('User-Agent',
                   'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 '
                   '(KHTML, like Gecko) Chrome/28.0.1468.0 '
                   'Safari/537.36')
    req.add_header('Accept-encoding', 'gzip')
    req.add_header('Accept-Language', request.headers.get('Accept-Language'))

    try:
        resp = urllib2.build_opener().open(req, post_data)
        resp_info = resp.info()
        target_url = resp.geturl()
        content_type = (content_type %
                        ('text/css' if 'text/css' in resp_info.get('Content-Type')
                         else 'text/html'))
        if resp_info.get('Content-Encoding') == 'gzip':
            buf = StringIO(resp.read())
            f = gzip.GzipFile(fileobj=buf)
            page_data = f.read()
        else:
            page_data = resp.read()
        data = unicode(page_data, chardet.detect(page_data)['encoding'])
    except (urllib2.HTTPError, urllib2.URLError, ValueError) as e:
        capp.logger.exception(e)
        data = None
        target_url = url
        content_type = None

    return data, target_url, content_type


import os
import numpy as np
import pandas as pd
from tqdm import tqdm
import multiprocessing
from multiprocessing import  Pool
from utils_preprocess_title import preprocess_title

TRAIN_CSV_DIR = './data/train.csv'
CATEGORIES_OUTPUT_DIR = './output/categories.npy'
TITLES_OUTPUT_DIR = './output/titles_normal.npy'
LABELS_OUTPUT_DIR = './output/labels.npy'

data = pd.read_csv(TRAIN_CSV_DIR)

unreliable = data[data['label_quality'] == 'unreliable']
reliable = data[data['label_quality'] == 'reliable']

merged = pd.concat([
  unreliable,
  reliable,
])

merged = merged.drop('label_quality', axis=1)
categories = merged.category.unique()
categories = np.sort(categories)

np.save(CATEGORIES_OUTPUT_DIR, categories)

categories_list = list(categories)
dictOfCategories = { word : i for i, word in enumerate(categories_list) }

tokens_list_stemmed = []
tokens_list = []
categories = []

def normalize(row):
  row.title = preprocess_title(row.title, row.language)
  row.category = dictOfCategories.get(row.category)
  return row

def normalize_chunk(data):
  tqdm.pandas()
  return data.progress_apply(normalize, axis=1)

def norm(dataframe):
  WORKERS = multiprocessing.cpu_count()
  with Pool(WORKERS) as p:
    df_split = np.array_split(dataframe, WORKERS)
    df = pd.concat(p.map(normalize_chunk, df_split))
    return df

tokens = filtered = norm(merged)
tokens = tokens[tokens.title.str.len() != 0]

tokens_list = np.squeeze(tokens.iloc[:, 0:1].values)
categories = np.squeeze(tokens.iloc[:, 2:3].values)

np.save(TITLES_OUTPUT_DIR, tokens_list)
np.save(LABELS_OUTPUT_DIR, categories)


# local config helper stuff
try:
    import ISStreamer.configutil as configutil
except ImportError:
    import configutil
try:
    import ISStreamer.version as version
except ImportError:
    import version
import uuid
from capture import *

# python 2 and 3 conversion support
import sys
if (sys.version_info < (2,7,0)):
    sys.stderr.write("You need at least python 2.7.0 to use the ISStreamer")
    exit(1)
elif (sys.version_info >= (3,0)):
    import http.client as httplib
else:
    import httplib
import json
# time stuff
import datetime
import time
# performance stuff
import threading
import collections

class Streamer:
    CoreApiBase = ""
    Bucket = ""
    ClientKey = ""
    PubKey = ""
    SubKey = ""
    Channel = ""
    BufferSize = 10
    LogIterations = 0
    StreamApiBase = ""
    LogQueue = None
    DebugLevel = 0
    SessionId = ""
    IsClosed = True
    def __init__(self, bucket="", client_key="", ini_file_location=None, debug_level=0, buffer_size=10):
        config = configutil.getConfig(ini_file_location)

        if (config == None and bucket=="" and client_key == ""):
            raise Exception("config not found and arguments empty")

        if (bucket == ""):
            bucket_name = config["bucket"]
        else:
            bucket_name = bucket
        if (client_key == ""):
            self.ClientKey = config["clientKey"]
        else:
            self.ClientKey = client_key

        #self.LogQueue = Queue.Queue(self.BufferSize)
        self.BufferSize = buffer_size
        self.LogQueue = collections.deque()

        self.CoreApiBase = config["core_api_base"]
        self.StreamApiBase = config["stream_api_base"]
        self.set_bucket(bucket_name)
        self.DebugLevel = debug_level
        self.IsClosed = False

        self.console_message("ClientKey: {clientKey}".format(clientKey=self.ClientKey))
        self.console_message("core_api_base: {api}".format(api=self.CoreApiBase))
        self.console_message("stream_api_base: {api}".format(api=self.StreamApiBase))

    def set_bucket(self, new_bucket, retries=3):

        def __create_bucket(new_bucket, bucket_id, client_key):
            api_base = self.CoreApiBase
            conn = None
            if (self.CoreApiBase.startswith('https://')):
                api_base = self.CoreApiBase[8:]
                self.console_message("core api base domain: {domain}".format(domain=api_base), level=2)
                conn = httplib.HTTPSConnection(api_base)
            else:
                api_base = self.CoreApiBase[7:]
                self.console_message("core api base domain: {domain}".format(domain=api_base), level=2)
                conn = httplib.HTTPConnection(api_base)
            resource = "/api/v1/buckets"
            headers = {
                'Content-Type': 'application/json',
                'User-Agent': 'PyStreamer v' + version.__version__
            }
            body = {
                'bucketId': bucket_id,
                'bucketName': new_bucket,
                'clientKey': client_key
            }

            def ___ship(retry_attempts, wait=0):
                if (retry_attempts <= 0):
                    if (self.DebugLevel >= 2):
                        raise Exception("bucket create failed")
                    else:
                        self.console_message("ISStreamer failed to create or find bucket after a number of attempts", level=0)
                        return

                try:
                    if (wait > 0):
                        time.sleep(wait)
                    conn.request("POST", resource, json.dumps(body), headers)
                    response = conn.getresponse()

                    if (response.status >= 200 and response.status < 300):
                        self.console_message("bucket created successfully!", level=2)
                    elif (response.status == 401 or response.status == 403):
                        self.console_message("ClientKey not authorized")
                        raise Exception("ClientKey not authorized")
                    else:
                        self.console_message("ISStreamer failed to setup the bucket on attempt {atmpt}. StatusCode: {sc}; Reason: {r}".format(sc=response.status, r=response.reason, atmpt=retry_attempts))
                        raise Exception("ship exception")
                except:
                    self.console_message("exception creating bucket on attempt {atmpt}.".format(atmpt=retry_attempts))
                    retry_attempts = retry_attempts - 1
                    ___ship(retry_attempts, 1)

            ___ship(retries)

        self.SessionId = str(uuid.uuid4())
        self.Bucket = new_bucket
        t = threading.Thread(target=__create_bucket, args=(new_bucket, self.SessionId, self.ClientKey))
        t.daemon = False
        t.start()

    def console_message(self, message, level=1):
        if (self.DebugLevel >= level):
            print(message)

    def ship_messages(self, messages, retries=3):
        api_base = self.StreamApiBase
        conn = None
        if (self.StreamApiBase.startswith('https://')):
            api_base = self.StreamApiBase[8:]
            self.console_message("ship messages: stream api base domain: {domain}".format(domain=api_base), level=2)
            conn = httplib.HTTPSConnection(api_base)
        else:
            api_base = self.StreamApiBase[7:]
            self.console_message("ship messages: stream api base domain: {domain}".format(domain=api_base), level=2)
            conn = httplib.HTTPConnection(api_base)
        resource = "/batch_logs"
        headers = {
            'Content-Type': 'application/json',
            'User-Agent': 'PyStreamer v' + version.__version__,
            'X-IS-ClientKey': self.ClientKey
        }

        def __ship(retry_attempts, wait=0):
            self.console_message("ship: beginning message ship!", level=2)
            if (retry_attempts <= 0):

                if (self.DebugLevel >= 2):
                    raise Exception("shipping logs failed.. network issue?")
                print('This is the point to add data control')   #To do catch the data store it then wait for network and send
                if(internet_on() == False):
                    Dump('temp.tmp', json.dumps(messages))   #dumps messages to file
                else:
                    self.console_message("ship: ISStreamer failed to ship the logs after a number of attempts {msgs}".format(msgs=json.dumps(messages)), level=0)

                    return

            try:
                if (wait > 0):
                    time.sleep(wait)
                conn.request('POST', resource, json.dumps(messages), headers)
                response = conn.getresponse()

                if (response.status >= 200 and response.status < 300):
                    self.console_message("ship: success!", level=2)
                else:
                    self.console_message("ship: failed on attempt {atmpt} (StatusCode: {sc}; Reason: {r})".format(sc=response.status, r=response.reason, atmpt=retry_attempts))
                    raise Exception("ship exception")
            except:
                self.console_message("ship: exception shipping logs on attempt {atmpt}.".format(atmpt=retry_attempts))
                retry_attempts = retry_attempts - 1
                __ship(retry_attempts, 1)

        __ship(retries)

    def flush(self):
        messages = []
        self.console_message("flush: checking queue", level=2)
        isEmpty = False
        while not isEmpty:
            try:
                m = self.LogQueue.popleft()
                messages.append(m)
            except IndexError:
                isEmpty = True
                self.console_message("flush: queue empty...", level=2)
        if len(messages) > 0:
            self.console_message("flush: queue not empty, shipping", level=2)
            self.ship_messages(messages)
        self.console_message("flush: finished flushing queue", level=2)

    def log_object(self, obj, signal_prefix=None, epoch=None):
        if (epoch == None):
            epoch = time.time()

        if (signal_prefix == None):
            signal_prefix = str(type(obj).__name__)

        if (type(obj).__name__ == 'list'):
            i = 0
            for val in obj:
                signal_name = "{}_{}".format(signal_prefix, i)
                self.log(signal_name, val, epoch=epoch)
                i += 1
        elif (type(obj).__name__ == 'dict'):
            for key in obj:
                signal_name = "{}_{}".format(signal_prefix, key)
                self.log(signal_name, obj[key], epoch=epoch)
        else:
            for attr in dir(obj):
                if not isinstance(getattr(type(obj), attr, None), property):
                    continue
                signal_name = "{}_{}".format(signal_prefix, attr)
                self.log(signal_name, getattr(obj, attr), epoch=epoch)

    def log(self, signal, value, epoch=None):
        def __ship_ten():
            i = self.BufferSize
            messages = []
            while(i > 0):
                try:
                    m = self.LogQueue.popleft()
                    messages.append(m)
                except IndexError:
                    i = 0
                    self.console_message("ship10: queue empty for now, less than 10")
                i = i - 1

            self.console_message("ship10: shipping", level=2)
            self.ship_messages(messages)
            self.console_message("ship10: finished shipping", level=2)

        timeStamp = time.time()
        gmtime = datetime.datetime.fromtimestamp(timeStamp)

        if epoch != None:
            try:
                gmtime = datetime.datetime.fromtimestamp(epoch)
                timeStamp = epoch
            except:
                self.console_message("epoch was overriden with invalid time, using current timstamp instead")

        formatted_gmTime = gmtime.strftime('%Y-%m-%d %H:%M:%S.%f')
        self.console_message("{time}: {signal} {value}".format(signal=signal, value=value, time=formatted_gmTime))

        if (len(self.LogQueue) >= self.BufferSize):
            self.console_message("log: queue size approximately at or greater than buffer size, shipping!", level=10)
            t = threading.Thread(target=__ship_ten)
            t.daemon = False
            t.start()

        self.console_message("log: queueing log item", level=2)
        log_item = {
            "b": self.Bucket,
            "sn": signal,
            "v": value,
            "e": timeStamp,
            "tid": self.SessionId
        }
        self.LogQueue.append(log_item)

    def close(self):
        self.IsClosed = True
        self.flush()

    def __del__(self):
        """Try to close/flush the cache before destruction"""
        try:
            if (not self.IsClosed):
                self.flush()
        except:
            if (self.DebugLevel >= 2):
                raise Exception("failed to close the buffer, make sure to explicitly call close() on the Streamer")
            else:
                self.console_message("failed to close the buffer, make sure to explicitly call close() on the Streamer", level=1)