import ckanapi
from datetime import datetime
from pprint import pprint

from dateutil import parser

def get_credentials_and_package_id():
    from .credentials import site, ckan_api_key as API_key, transactions_package_id as package_id, resource_name
    #from .credentials import site, ckan_api_key as API_key, debug_package_id as package_id, resource_name
    return site, API_key, package_id

def query_resource(site,query,API_key=None):
    # Use the datastore_search_sql API endpoint to query a CKAN resource.

    # Note that this doesn't work for private datasets.
    # The relevant CKAN GitHub issue has been closed.
    # https://github.com/ckan/ckan/issues/1954

    # In fact, if the dataset is private (the table is inaccessible),
    # the datastore_search_sql function throws a gibberishy, non-helpful error:
    # TypeError: __str__ returned non-string (type dict)

    ckan = ckanapi.RemoteCKAN(site, apikey=API_key)
    response = ckan.action.datastore_search_sql(sql=query)
    # A typical response is a dictionary like this
    #{u'fields': [{u'id': u'_id', u'type': u'int4'},
    #             {u'id': u'_full_text', u'type': u'tsvector'},
    #             {u'id': u'pin', u'type': u'text'},
    #             {u'id': u'number', u'type': u'int4'},
    #             {u'id': u'total_amount', u'type': u'float8'}],
    # u'records': [{u'_full_text': u"'0001b00010000000':1 '11':2 '13585.47':3",
    #               u'_id': 1,
    #               u'number': 11,
    #               u'pin': u'0001B00010000000',
    #               u'total_amount': 13585.47},
    #              {u'_full_text': u"'0001c00058000000':3 '2':2 '7827.64':1",
    #               u'_id': 2,
    #               u'number': 2,
    #               u'pin': u'0001C00058000000',
    #               u'total_amount': 7827.64},
    #              {u'_full_text': u"'0001c01661006700':3 '1':1 '3233.59':2",
    #               u'_id': 3,
    #               u'number': 1,
    #               u'pin': u'0001C01661006700',
    #               u'total_amount': 3233.59}]
    # u'sql': u'SELECT * FROM "d1e80180-5b2e-4dab-8ec3-be621628649e" LIMIT 3'}
    data = response['records']
    return data

def get_package_parameter(site,package_id,parameter=None,API_key=None):
    """Gets a CKAN package parameter. If no parameter is specified, all metadata
    for that package is returned."""
    # Some package parameters you can fetch from the WPRDC with
    # this function are:
    # 'geographic_unit', 'owner_org', 'maintainer', 'data_steward_email',
    # 'relationships_as_object', 'access_level_comment',
    # 'frequency_publishing', 'maintainer_email', 'num_tags', 'id',
    # 'metadata_created', 'group', 'metadata_modified', 'author',
    # 'author_email', 'state', 'version', 'department', 'license_id',
    # 'type', 'resources', 'num_resources', 'data_steward_name', 'tags',
    # 'title', 'frequency_data_change', 'private', 'groups',
    # 'creator_user_id', 'relationships_as_subject', 'data_notes',
    # 'name', 'isopen', 'url', 'notes', 'license_title',
    # 'temporal_coverage', 'related_documents', 'license_url',
    # 'organization', 'revision_id'
    try:
        ckan = ckanapi.RemoteCKAN(site, apikey=API_key)
        metadata = ckan.action.package_show(id=package_id)
        if parameter is None:
            return metadata
        else:
            return metadata[parameter]
    except:
        raise RuntimeError("Unable to obtain package parameter '{}' for package with ID {}".format(parameter,package_id))

def find_resource_id(site,package_id,resource_name,API_key=None):
#def get_resource_id_by_resource_name():
    # Get the resource ID given the package ID and resource name.
    resources = get_package_parameter(site,package_id,'resources',API_key)
    for r in resources:
        if r['name'] == resource_name:
            return r['id']
    return None

def get_resource_id(ref_time,is_a_minizone):
    by_name = True
    if ref_time == 'hybrid':
        from .credentials import transactions_resource_id as resource_id
    elif ref_time == 'purchase_time':
        if by_name:
            site, API_key, package_id = get_credentials_and_package_id()
            from .credentials import resource_name

            if is_a_minizone:
                from .credentials import minizones_resource_name as resource_name
                print("Using data from resource with name {} and package ID {}.".format(resource_name,package_id))

            resource_id = find_resource_id(site,package_id,resource_name,API_key)
            if resource_id is None:
                raise ValueError("No resource found for package ID = {}, resource name = {}.".format(package_id,resource_name))
        else:
            from credentials import transactions_production_resource_id as resource_id
    else:
        raise ValueError("ref_time must specify the reference time to determine the correct resource ID.")
    return resource_id

def get_revenue_and_count_vectorized(ref_time,zone,start_date,end_date,start_hours,end_hours,is_a_minizone=False):
    # Two models for getting data:
    # 1) Do a SQL query for all records in the start_date to end_date
    # range, then pare down to the hour range, then sum the results
    # (getting transaction count and total payments).
    # 2) Do a selective SQL query that plucks out data based both on hour range
    # and date range, and then sum the results.

    # Note that end_date is the last date (NON-INCLUSIVE) of a date range
    # That is, dates = [start_date, end_date)
    # The end_date for October 2018 is 2018-11-01.

    # Caching strategies:
    #   Aggregate by zone and quarter and time range:
    #     A key could be of the form "401 - Downtown | 2018 Q1 | 8am-10am"
    #     With ~60 zones, there would then be
    #       about 60*5*4*3 = 3600 of these quarterly sums, where transaction
    #                   count and total revenue could each be cached separately.
    #   There would be a few more ultimately to account for the extra time slot
    #   for the Southside, once parking became non-free late there on weekends.

    #   Also caching for extra zones would probably be necessary.
    site, API_key, _ = get_credentials_and_package_id()
    resource_id = get_resource_id(ref_time,is_a_minizone)
    #make_datastore_public(site,resource_id,API_key)

    start_string = start_date.strftime("%Y-%m-%d") # This should be the first date in the range.
    end_string = end_date.strftime("%Y-%m-%d")     # This should be the last date in the range.
    #query = 'SELECT SUM(payments) as total_payments, COUNT(_id) as transactions FROM "{}" WHERE zone = \'{}\' AND start >= \'{}\' AND start <= \'{}\''.format(resource_id,zone,start_string,end_string)
    new_approach = True
    # The dataset is split into mobile transactions and meter transactions.

    queries = []
    if zone is None: # This part deviates from the original proto_get_revenue.py, though it's an improvement.
        k = 0
        for start_hour,end_hour in zip(start_hours,end_hours):
            query = 'SELECT {} as col_order, SUM(meter_payments) + SUM(mobile_payments) as total_payments, SUM(meter_transactions) + SUM(mobile_transactions) as transaction_count FROM "{}" WHERE start >= \'{}\' AND start < \'{}\' AND extract(hour from start) >= {} AND extract(hour from start) < {}'.format(k,resource_id,start_string,end_string,start_hour,end_hour)
            # This will definitely fail for extract(hour from start) >= 1 and extract(hour from start) <= 1
            #     though it might work for the designed cases (valet),
            #          but it seems like it doesn't and is somehow giving twice the expected results.
            queries.append(query)
            k += 1
    else:
        k = 0
        for start_hour,end_hour in zip(start_hours,end_hours):
            query = 'SELECT {} as range_order, SUM(meter_payments) + SUM(mobile_payments) as total_payments, SUM(meter_transactions) + SUM(mobile_transactions) as transaction_count FROM "{}" WHERE zone = \'{}\' AND start >= \'{}\' AND start < \'{}\' AND extract(hour from start) >= {} AND extract(hour from start) < {}'.format(k,resource_id,zone,start_string,end_string,start_hour,end_hour)
            queries.append(query)
            k += 1

    # [ ] Verify that hour extraction extracts to a 24-hour clock.
    # 'SELECT *, extract(hour from start) as hour FROM "<resource ID>" WHERE zone = \'S. Craig\' AND start >= \'2018-01-01\' AND start <= \'2018-03-31\' AND extract(hour from start) > 8 LIMIT 5'
    # gives results like this:
    # {'_full_text': "[...]",
    #  '_id': 594455,
    #  'end': '2018-01-02T11:20:00',
    #  'hour': 11.0, <<<<<<<<<<<
    #  'parent_zone': '409 - Oakland 3',
    #  'payments': 2.25,
    #  'start': '2018-01-02T11:10:00',
    #  'transactions': 2,
    #  'utc_start': '2018-01-02T16:10:00',
    #  'zone': 'S. Craig'}

#  '_id': 2950,
#  'end': '2012-09-06T14:10:00',
#  'parent_zone': '409 - Oakland 3',
#  'payments': 2.5,
#  'start': '2012-09-06T14:00:00',
#  'transactions': 2,
#  'utc_start': '2012-09-06T18:00:00',
#  'zone': 'S. Craig'

    #query = """SELECT SUM(meter_payments) + SUM(mobile_payments) as total_payments, SUM(meter_transactions) + SUM(mobile_transactions) as transaction_count FROM "29f42621-33a7-4122-915d-d4e10d6d83f9" WHERE start >= '2018-06-25' AND start < '2018-06-26'"""
    query = ' UNION '.join(queries)
    query += ' ORDER BY range_order ASC'
    data = query_resource(site,query,API_key)

    if end_date > datetime.now().date():
        print("The end_date {} is in the future.".format(end_date))
        print("... though maybe flagging things as too far in the future or past or in an incomplete quarter or month should be handled in a separate function.")

    payments = []
    transaction_counts = []
    for k,group in enumerate(data):
        if group['total_payments'] is not None:
            assert k == group['range_order']
            payments.append(float(group['total_payments']))
            transaction_counts.append(int(group['transaction_count']))
        else:
            payments.append(0.0)
            transaction_counts.append(0)
    return payments, transaction_counts

def get_package_metadata(site,package_id,API_key=None):
    ckan = ckanapi.RemoteCKAN(site, apikey=API_key)
    metadata = ckan.action.package_show(id=package_id)
    return metadata

def source_time_range(ref_time):
    site, API_key, package_id = get_credentials_and_package_id()
    metadata = get_package_metadata(site,package_id,API_key)
    if 'temporal_coverage' in metadata:
        begin_date, end_date = metadata['temporal_coverage'].split('/')
        source_begin_date = parser.parse(begin_date).date()
        source_end_date = parser.parse(end_date).date()
        return "from {} to {}".format(begin_date, end_date), source_begin_date, source_end_date
    return None, None, None


# coding=utf-8
import time
import unittest

from appium import webdriver
from lib import find_element
from AppTestGR2.keyword.appmarket.HiApp import HiApp
from common import util

# define APPCenter test class
class APPCenter(unittest.TestCase):
    # define setup method
    def setUp(self):
        desired_caps = {
            'platformName': 'Android',
            #此处填写机器的android版本
            'platformVersion': '4.4.4',
            #填写adb devices -l 中显示的第一列的设备号
            'deviceName': '192.168.146.101:5555',
            'appPackage': 'com.huawei.appmarket',
            'appActivity': '.MainActivity',
            'appWaitActivity': '.MarketActivity',
            'noReset':True,
            'unicodeKeyboard': True,
            'resetKeyboard': True
        }
        self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps)
        self.verificationErrors = []

    #点击我的，校验微信公众号和客服
    def test_wode01(self):
        try:
            logs = util.createMainLog('E:\\worksplace\\AppTestGR2\\tests\\results\\')
            logs.debug("test_wode01----kaishi zgxing")
            testdata = util.getTestDataInput("Wode",self._testMethodName)
            hiapp = HiApp(self.driver)
            hiapp.enter_wode()
            hiapp.check_msg(**testdata)
            logs = util.createMainLog('E:\\worksplace\\AppTestGR2\\tests\\results\\')

            # hiapp = HiApp(self.driver)
            # hiapp.enter_wode()
            # hiapp.check_msg('//android.widget.TextView[@text="hiapps"]','hiapps')

            # time.sleep(3)
            # el1 = self.driver.find_element_by_xpath('//android.widget.TextView[@text="我的"]')
            # el1.click()
            #
            # time.sleep(1)
            #
            # e12 = self.driver.find_element_by_xpath('//android.widget.TextView[@text="关于"]')
            # time.sleep(1)
            # e12.click()
            #
            # el3 = self.driver.find_element_by_xpath('//android.widget.TextView[@text="hiapps"]')
            # self.assertTrue(el3.text == 'hiapps')
            # print el3.text
            # # 获取当前方法的名字
            # print self._testMethodName
            # #QQ 4008308300
            # el4 = self.driver.find_element_by_xpath('//android.widget.TextView[@text="QQ 4008308300"]')
            # self.assertTrue(el4.text == 'QQ 40083083')
            # print el4.text

        except:
            logs.debug("test_wode01----jieshu zgxing")
            time.sleep(3)
            path = 'E:\\worksplace\\AppTestGR2\\tests\\results\images\\'
            fileName = path+self._testMethodName+time.strftime('_%Y%m%d%H%M%S')+".png"
            self.driver.get_screenshot_as_file(fileName)
            self.assertFalse(True,'Run'+self._testMethodName+'failed')
            time.sleep(3)

    #设置中，打开消息提醒，并校验
    def test_wode02(self):
        try:
            logs = util.createMainLog('E:\\worksplace\\AppTestGR2\\tests\\results\\')
            logs.debug("test_wode0----kaishi zgxing")
            testdata = util.getTestDataInput("Wode",self._testMethodName)
            hiapp = HiApp(self.driver)
            hiapp.enter_wode()
            hiapp.open_switch(**testdata)
            # keyword封装后的代码
            # hiapp = HiApp(self.driver)
            # hiapp.enter_wode()
            # hiapp.open_switch('//android.widget.RelativeLayout[@index=8]/android.widget.Switch')
            # # time.sleep(3)
            # # el1 = self.driver.find_element_by_xpath('//android.widget.TextView[@text="我的"]')
            # # el1.click()
            # e11 = find_element.find_element(self.driver,'//android.widget.TextView[@text="我的"]')
            # e11.click()
            #
            # time.sleep(1)
            # el2 = self.driver.find_element_by_xpath('//android.widget.TextView[@text="设置"]')
            # el2.click()
            #
            # time.sleep(1)
            # el3 = self.driver.find_element_by_xpath('//android.widget.RelativeLayout[@index=8]/android.widget.Switch')
            # if el3.get_attribute('checked') != u'true':
            #     el3.click()
            #     time.sleep(1)
            # self.assertTrue(el3.get_attribute('checked') == u'true')
            # print el3.text
        except:
            logs.debug("test_wode02----kaishi zgxing")
            time.sleep(3)
            path = 'E:\\worksplace\\AppTestGR2\\tests\\results\images\\'
            fileName = path+self._testMethodName+time.strftime('_%Y%m%d%H%M%S')+".png"
            self.driver.get_screenshot_as_file(fileName)
            self.assertFalse(True,'Run'+self._testMethodName+'failed')
            time.sleep(3)

    def tearDown(self):
        time.sleep(3)
        self.driver.quit()

if __name__ == '__main__':
    unittest.main()
    # ts = unittest.TestSuite()
    # testLoader = unittest.defaultTestLoader
    # t1 = testLoader.loadTestsFromName('Wode.APPCenter.test_wode02')
    # ts.addTest(t1)
    # unittest.TextTestRunner(verbosity=2).run(ts)


import sys
import os
import requests
import logging
import hashlib
import json

if sys.version_info > (3,):
    from configparser import ConfigParser
else:
    from ConfigParser import ConfigParser

DEFAULT_CONFIG_PATH = os.path.join(os.path.expanduser('~'), '.minid')
DEFAULT_CONFIG_FILE = os.path.join(DEFAULT_CONFIG_PATH, 'minid-config.cfg')

logger = logging.getLogger(__name__)

def configure_logging(level=logging.INFO, logpath=None):
    log_format = "%(asctime)s - %(levelname)s - %(message)s"
    if logpath:
        logging.basicConfig(filename=logpath, level=level, format=log_format)
    else:
        logging.basicConfig(level=level, format=log_format)

def parse_config(config_file):
    if config_file == DEFAULT_CONFIG_FILE and not os.path.isfile(config_file):
        logger.info("No default configuration file found, creating one")
        create_default_config()
    config = ConfigParser()
    config.read(config_file)
    return dict(config.items('general'))

def create_default_config():
    if not os.path.isdir(DEFAULT_CONFIG_PATH):
        os.makedirs(DEFAULT_CONFIG_PATH)
    with open(DEFAULT_CONFIG_FILE, 'w') as config_file:
        config_file.writelines(['[general]\n',
                                'minid_server: http://minid.bd2k.org/minid\n',
                                'email: \n',
                                'orcid: \n',
                                'code: \n'])
    config_file.close()

def compute_checksum(file_path, algorithm=hashlib.sha256(), block_size=65536):
    logger.info("Computing checksum for %s using %s" % (file_path, algorithm))
    with open(os.path.abspath(file_path), 'rb') as open_file:
        buf = open_file.read(block_size)
        while len(buf) > 0:
            algorithm.update(buf)
            buf = open_file.read(block_size)
    open_file.close()
    return algorithm.hexdigest()

def get_entities(server, name, test):
    logger.info("Checking if the %sentity %s already exists on the server: %s" %
                ("TEST " if test else "", name, server))
    query = ""
    if test:
        query = "?test=true"
    r = requests.get("%s/%s%s" % (server, name, query), headers={"Accept": "application/json"})
    if r.status_code == 404:
        return None
    return r.json()

def create_entity(server, entity):
    r = requests.post(server, json=entity, headers={"Accept": "application/json"})
    if r.status_code in [200, 201]:
        return r.json()
    else:
        logger.error("Error creating entity (%s) -- check parameters or config file for invalid values" % r.status_code)

def entity_json(email, code, checksum, locations, title, test, content_key):
    entity = {"email":  email, "code": code, "checksum": checksum}
    if test:
        entity["test"] = test
    if locations:
        entity["locations"] = locations
    if title:
        entity["title"] = title
    if content_key:
        entity["content_key"] = content_key
    return entity

def print_entity(entity, as_json):
    if as_json:
        print(json.dumps(entity))
    else:
        print("Identifier: %s" % entity["identifier"])
        print("Created by: %s (%s)" % (entity["creator"], entity["orcid"]))
        print("Created: %s" % entity["created"])
        print("Checksum: %s" % entity["checksum"])

        if entity["content_key"]:
            print ("Content Key: %s" % entity["content_key"])

        print("Status: %s" % entity["status"])
        if entity["obsoleted_by"]:
            print("Obsoleted by: %s" % entity["obsoleted_by"])
        print("Locations:")
        for l in entity["locations"]:
            print("  %s - %s" % (l["creator"], l["uri"]))
        print("Title:")
        for t in entity["titles"]:
            print("  %s - %s" % (t["creator"], t["title"]))

def print_entities(entities, as_json):
    for i, entity in entities.items():
        print_entity(entity, as_json)
        print("\n")

def register_user(server, email, name, orcid):
    logger.info("Registering new user \"%s\" with email \"%s\"%s" %
                (name, email, format(" and orcid \"%s\"" % orcid) if orcid else ""))
    user = {"name": name, "email": email}
    if orcid:
        user["orcid"] = orcid
    r = requests.post("%s/user" % server, json=user, headers={"Content-Type": "application/json"})
    return r.json()

def register_entity(server, checksum, email, code, url=None, title='', test=False, content_key=None):
    logger.info("Creating new identifier")

    result = create_entity(server, entity_json(email, code, checksum, url, title, test, content_key))

    if result:
        logger.info("Created/updated minid: %s" % result["identifier"])
        return result["identifier"]

def update_entity(server, name, entity, email, code):
    if not entity:
        logger.info("No entity found to update")
        return None

    entity['email'] = email
    entity['code'] = code

    r=requests.put("%s/%s" % (server, name), json=entity, headers={"Accept": "application/json"})

    if r.status_code in [200, 201]:
        return r.json()
    else:
        logger.error("Error updating entity (%s, %s) -- check parameters or config file for invalid values" % (r.status_code, r.text))


# -*- coding: utf-8 -*-
r"""
Created on Mon Aug 20 11:23:26 2018

@author: F. Obersteiner, florian\obersteiner\\kit\edu
"""

import numpy as np
from numba import njit
from scipy.interpolate import interp1d

###############################################################################

def filter_jumps_simple(v, max_delta, no_val=-1, add_v1=None, add_v2=None,
                        delete_vals=False, use_abs_delta=True):

    max_delta = abs(max_delta)

    result = {'filtered': None,
              'ix_del': [],
              'ix_rem': [],
              'add_v1': add_v1,
              'add_v2': add_v2}

    for ix, v_ix in enumerate(v):
        if ix == 0:
            buffer = v_ix
            result['ix_rem'].append(ix)
            continue

        if use_abs_delta:
            delta = abs(v_ix-buffer)
        else:
            delta = v_ix-buffer

        if delta > max_delta or delta < 0.:
            v[ix] = no_val
            result['ix_del'].append(ix)
        else:
            buffer = v_ix
            result['ix_rem'].append(ix)

    if delete_vals:
        result['filtered'] = [element for element in v if element != no_val]

    if delete_vals and add_v1:
        result['add_v1'] = []
        for ix in result['ix_rem']:
            result['add_v1'].append(add_v1[ix])

    if delete_vals and add_v2:
        result['add_v2'] = []
        for ix in result['ix_rem']:
            result['add_v2'].append(add_v2[ix])

    return result

###############################################################################

def mask_repeated_elems(arr):
    """
    use numpy diff to mask repeated (adjacent) element in a numpy 1d array.
    - for small array sizes (<10000), a looped solution with
      numba njit is faster.
    """
    # n_el = arr.shape[0]
    # mask = np.ones(arr.shape).astype(np.bool_)
    # i = 0
    # while i < n_el-1:
    #     if arr[i] == arr[i+1]:
    #         cur = arr[i]
    #         for j in range(i+1, n_el):
    #             i += 1
    #             if cur == arr[j]:
    #                 mask[j] = False
    #             else:
    #                 break
    #     i += 1
    mask = np.ones(arr.shape).astype(np.bool_)
    mask[1:] = np.diff(arr).astype(np.bool_)
    return mask

@njit
def mask_jumps(arr, thrsh, look_ahead, abs_delta=False):
    """
    check the elements of array "arr" if the delta between element and
    following element(s) exceed a threshold "trsh". How many elements to
    look ahead is defined by "look_ahead"
    """
    n_el = arr.shape[0]
    mask = np.ones(arr.shape).astype(np.bool_)
    i = 0
    while i < n_el-1:
        cur, nxt = arr[i], arr[i+1]
        delta_0 = np.absolute(nxt-cur) if abs_delta else nxt-cur
        if delta_0 > thrsh:
            for value in arr[i+1:i+look_ahead+1]:
                delta_1 = np.absolute(value-cur) if abs_delta else value-cur
                if delta_1 > thrsh:
                    mask[i+1] = False
                    i += 1
                else:
                    break
        i += 1
    return mask

def filter_jumps_v2(arr, thrsh, look_ahead,
                    abs_delta=False,
                    vmiss=np.nan,
                    remove_repeated=False,
                    interpol_jumps=False, interpol_kind='linear'):
    """
    wrapper around mask_jumps()
    ! interpolation assumes equidistant spacing of the independent variable of
      which arr depends !
    """
    if not isinstance(arr, np.ndarray):
        raise ValueError("input array must be of class numpy ndarray.")
    if arr.ndim > 1:
        raise ValueError("input array must be numpy 1d array.")
    if not isinstance(look_ahead, int):
        raise ValueError("parameter look_ahead must be an integer.")
    if look_ahead >= arr.shape[0] or look_ahead < 1:
        raise ValueError(f"parameter look_ahead must be >=1 and <{arr.shape[0]}.")

    result = arr.copy() # do not touch the input...
    if not np.isnan(vmiss):
        result[vmiss] = np.nan
    if remove_repeated:
        result[~mask_repeated_elems(result)] = np.nan
    mask = mask_jumps(result, thrsh, look_ahead, abs_delta=abs_delta)
    result[~mask] = np.nan
    if interpol_jumps:
        f_ip = interp1d(np.arange(0, result.shape[0])[mask], result[mask],
                        kind=interpol_kind, fill_value='extrapolate')
        result = f_ip(np.arange(0, result.shape[0]))
        return (result, mask)
    return (result, mask)

###############################################################################

def filter_jumps_np(v, max_delta, no_val=np.nan, use_abs_delta=True,
                    reset_buffer_after=3, remove_doubles=False,
                    # if v is dependent on another variable x (e.g. time),
                    # IF x is not equidistant, do NOT use interpolation:
                    interpol_jumps=False, interpol_kind='linear'):

    len_v = len(v)

    ix_del = np.full(len_v, -1, dtype='int32')  # deletion index
    ix_rem = np.full(len_v, -1, dtype='int32')  # remaining index

    buffer = [False, 0]

    for ix, v_ix in enumerate(v):
        if any([~np.isfinite(v_ix), v_ix == no_val, np.isnan(v_ix)]):
            ix_rem[ix] = ix
            continue  # skip line if value is np.nan

        if not buffer[0]:
            buffer[0] = v_ix
            ix_rem[ix] = ix
            continue  # fill buffer if not done so yet

        if use_abs_delta:
            delta = abs(v_ix-buffer[0])
        else:
            delta = v_ix-buffer[0]

        if delta > max_delta:  # jump found!
            v[ix] = no_val
            ix_del[ix] = ix
            buffer[1] += 1
            if reset_buffer_after:
                if buffer[1] == reset_buffer_after:
                    buffer = [v_ix, 0]
        else:  # no jump,...
            buffer[0] = v_ix
            if remove_doubles:  # check for double values...
                if delta == 0.:  # double found!
                    v[ix] = no_val
                    ix_del[ix] = ix
                else:  # no double
                    ix_rem[ix] = ix
            else:
                ix_rem[ix] = ix

    w_valid = np.where(ix_del != -1)
    ix_del = ix_del[w_valid]

    w_valid = np.where(ix_rem != -1)
    ix_rem = ix_rem[w_valid]

    if interpol_jumps:
        tmp_x = (np.arange(0, len_v))[ix_rem]
        tmp_y = v[ix_rem]
        f_ip = interp1d(tmp_x, tmp_y,
                        kind=interpol_kind, fill_value='extrapolate')
        filtered = f_ip(np.arange(0, len_v))
    else:
        w_valid = np.where(v != no_val)
        filtered = v[w_valid]

    return {'filtered': filtered,
            'ix_del': ix_del,
            'ix_rem': ix_rem}

###############################################################################

def del_at_edge(v, n_cut, add=2, out_len='same'):
    """
    assume v to be a 1D array which contains blocks of NaNs.
    returns: v with "more NaNs", i.e. range of NaN-blocks is extended by n_cut.
    """

    tf = np.isfinite(v)*1.

    mask = np.convolve(tf, np.ones((int(n_cut+add),))/int(n_cut+add),
                       mode=out_len)

    if tf[0] > 0.9:
        mask[0] = 1.
    if tf[-1] > 0.9:
        mask[-1] = 1.

    mask[np.where(mask < 0.999)] = np.nan

    return v * mask

###############################################################################


import simplejson as json
import time
# django
from django.http import HttpResponse
from django.views.generic import View
from django.core.exceptions import ObjectDoesNotExist
from django.utils.decorators import method_decorator
from django.contrib.auth.decorators import login_required
# apps
from .models import Item, Category, SubCategory
from .forms import ItemForm
from utils.helper import render_to_json, get_limit_offset
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import IsAuthenticated
from utils.decorators import check_auth_api

def add_category(item_as_json, data, sql=True):
    """
    Add category dict to main dict
    Args:
        item_as_json: main_dict
        data: Item, or object from sql query
        sql: if true then data is sql request else Item

    Returns: main dict

    """
    if sql:
        item_as_json["category"] = {"id": data.category_id,
                                    "name": data.category_name,
                                    "subcategory": {"id": data.subcategory_id,
                                                    "name": data.subcategory_name}}
    else:
        item_as_json["category"] = {"id": data.subcategory.id,
                                    "name": data.subcategory.name,
                                    "subcategory": {"id": data.subcategory.category.id,
                                                    "name": data.subcategory.category.name}}
    return item_as_json

def item_json(p, sql=True):
    """
    Make form Item json
    Args:
        p: Item
        sql: if true then p is sql request

    Returns: json

    """
    item_as_json = {"id": p.id,
                    "name": p.name,
                    "manufacturer": p.manufacturer,
                    "code": p.code,
                    "tnved": p.tnved,
                    "monitoring": p.monitoring,
                    "guarantee": p.guarantee,
                    "stock": p.stock,
                    "price": p.price,
                    "presence": p.presence,
                    "expected": p.expected,
                    "promo": p.promo,
                    "packaged": p.packaged,
                    "ddp": p.ddp,
                    "with_discount": p.with_discount,
                    "img": p.img}
    return add_category(item_as_json, p, sql)

items_fields = ['items.id', 'items.manufacturer', 'items.name', 'items.code', 'items.tnved', 'items.price',
                'items.monitoring', 'items.guarantee', 'items.ddp', 'items.stock', 'items.presence',
                'items.expected', 'items.promo', 'items.packaged', 'items.with_discount',
                'category.id', 'category.name', 'subcategory.id', 'subcategory.name', 'img']

class Items(View):
    """
      Get - return all items
      POST - create new item
      PATH - update item
    """

    def dispatch(self, request, *args, **kwargs):
        return super(Items, self).dispatch(request, *args, **kwargs)

    @staticmethod
    def get(request):
        """
          Return items as json
        """
        # start_time = time.time()
        order = request.GET.get('order')
        search = request.GET.get('search')
        search_val = request.GET.get('search_val')
        sql_query = """SELECT items.id, items.manufacturer, items.name, items.code, items.tnved,
                               items.price, items.monitoring, items.guarantee, items.ddp, items.stock, items.presence,
                               items.expected, items.promo, items.packaged, items.with_discount,
                               category.id as category_id, category.name as category_name,
                               subcategory.id as subcategory_id , subcategory.name as subcategory_name, items.img
                        FROM items_item items
                        left join items_subcategory subcategory on subcategory.id=items.subcategory_id
                        left join items_category category on category.id=subcategory.category_id;"""
        if search and search in items_fields and search_val:
            sql_query = sql_query[:-1] + " WHERE {key} = {value}".format(key=search, value=search_val)
        if order and order in items_fields:
            sql_query = sql_query[:-1] + " ORDER BY {key};".format(key=order)
        data = [item_json(p) for p in Item.objects.raw(sql_query)]
        # print('Time {}'.format(time.time() - start_time))
        return render_to_json(data)

    def post(self, request):
        """
          Create new Item from POST args
          100 - some of required field is exists
        """
        form = ItemForm(request.POST)
        if form.is_valid():
            item = form.save()
            return render_to_json(item_json(item, False), 201)
        else:
            return render_to_json({'error': True,
                                   'description': form.errors,
                                   'code': 100},
                                  status=403)

def category_json(obj):
    """
      Return category in json
    """
    return {'id': obj.id,
            'name': obj.name}

def subcategory_json(obj):
    """
      Return subcategory is json

    """
    return {'id': obj.id,
            'name': obj.name,
            'category': category_json(obj.category)}

def list_all_categories(request):
    """ Return list all categories """
    data = [category_json(category) for category in Category.objects.all()]
    return render_to_json(data)

def list_all_subcategory(request):
    """ Return list for all subcategories """
    return render_to_json([subcategory_json(subcategory) for subcategory in SubCategory.objects.all()])

class HandItems(View):

    @staticmethod
    def get(request, item_id):
        """
          100 - bad item id
        """
        try:
            item = Item.objects.get(id=item_id)
        except ObjectDoesNotExist:
            return render_to_json({'error': True,
                                   'description': 'bad id',
                                   'code': 100},
                                  status=404)
        else:
            return render_to_json(item_json(item, False))

    @method_decorator(api_view(['POST']))
    @method_decorator(permission_classes((IsAuthenticated,)))
    def post(self, request, item_id):
        try:
            item = Item.objects.get(id=int(item_id))
        except ObjectDoesNotExist:
            return render_to_json({'error': True,
                                   'description': 'bad id',
                                   'code': 100},
                                  status=404)
        form = ItemForm(request.POST, instance=item)
        if form.is_valid():
            item = form.save()
            return render_to_json(item_json(item, False), 200)
        else:
            print('WTF')
            return render_to_json({'error': True,
                                   'description': form.errors,
                                   'code': 100},
                                  status=403)

    @method_decorator(api_view(['DELETE']))
    @method_decorator(permission_classes((IsAuthenticated,)))
    def delete(self, request, item_id):
        if not request.user.is_authenticated():
            return render_to_json({'error': True,
                                   'description': 'Not login',
                                   'code': 101})
        try:
            item = Item.objects.get(id=int(item_id))
        except ObjectDoesNotExist:
            return render_to_json({'error': True,
                                   'description': 'bad id',
                                   'code': 100},
                                  status=404)
        item.delete()
        return render_to_json({'success': True})


import pandas as pd
import matplotlib.pyplot as plt
import matplotlib

# matplotlib 한글 폰트 오류 문제 해결
# 한글 설정
matplotlib.rcParams['font.family'] = 'NanumGothicCoding'  # '맑은 고딕'으로 설정,
matplotlib.rcParams['axes.unicode_minus'] = False

# Excel 데이터를 데이터프레임 변환
df = pd.read_excel('시도별 전출입 인구수.xlsx', fillna=0, header=0)

# 전출지별에서 누락값(NaN)을 앞 데이터로 채움 (엑셀 양식 병합 부분)
df = df.fillna(method='ffill')

print(df.전출지별.value_counts())
# 서울에서 다른 지역으로 이동한 데이터만 추출하여 정리
mask = (df['전출지별'] == '대구광역시') & (df['전입지별'] != '대구광역시')
df_seoul = df[mask]
df_seoul = df_seoul.drop(['전출지별'], axis=1)
df_seoul.rename({'전입지별':'전입지'}, axis=1, inplace=True)
df_seoul.set_index('전입지', inplace=True)

# 서울에서 '충청남도','경상북도', '강원도'로 이동한 인구 데이터 값만 선택
col_years = list(map(str, range(1981, 2018)))
df_3 = df_seoul.loc[['서울특별시','경상북도', '강원도',], col_years]

print(df_3.T)
# 스타일 서식 지정
plt.style.use('ggplot')

# 그래프 객체 생성 (figure에 1개의 서브 플롯을 생성)
fig = plt.figure(figsize=(20, 5))
ax = fig.add_subplot(1, 1, 1)

# axe 객체에 plot 함수로 그래프 출력
ax.plot(col_years, df_3.loc['서울특별시',:], marker='o', markerfacecolor='green',
        markersize=10, color='olive', linewidth=2, label='대구 -> 서울')
ax.plot(col_years, df_3.loc['경상북도',:], marker='o', markerfacecolor='blue',
        markersize=10, color='skyblue', linewidth=2, label='대구 -> 경북')
ax.plot(col_years, df_3.loc['강원도',:], marker='o', markerfacecolor='red',
        markersize=10, color='magenta', linewidth=2, label='대구 -> 강원')

# 범례 표시
ax.legend(loc='best')

# 차트 제목 추가
ax.set_title('대구 -> 서울, 경북, 강원 인구 이동', size=20)

# 축이름 추가
ax.set_xlabel('기간', size=12)
ax.set_ylabel('이동 인구수', size = 12)

# 축 눈금 라벨 지정 및 90도 회전
ax.set_xticklabels(col_years, rotation=90)

# 축 눈금 라벨 크기
ax.tick_params(axis="x", labelsize=10)
ax.tick_params(axis="y", labelsize=10)

plt.show()  # 변경사항 저장하고 그래프 출력구


############################################################################################
#
# Project:       Peter Moss Acute Myeloid/Lymphoblastic Leukemia AI Research Project
# Repository:    AML/ALL Detection System
# Project:       Movidius NCS1 Classifier
#
# Author:        Adam Milton-Barker (AdamMiltonBarker.com)
# Contributors:
# Title:         Movidius NCS1 Classifier Server Class
# Description:   Flask server that hosts an API endpoint for the
#                AML/ALL Detection System Movidius NCS1 Classifier.
# License:       MIT License
# Last Modified: 2019-05-09
#
############################################################################################

import os
import sys
import time
import json
import cv2
import jsonpickle

import numpy as np

from mvnc import mvncapi as mvnc
from datetime import datetime
from skimage.transform import resize

from Classes.Helpers import Helpers
from Classes.OpenCV import OpenCV
from Classes.Movidius import Movidius

from flask import Flask, request, Response

class Server():
    """ AML/ALL Detection System Movidius NCS1 Classifier Class

    API server for the AML/ALL Detection System Movidius NCS1 Classifier.
    """

    def __init__(self):
        """ Initializes the AML/ALL Detection System Movidius NCS1 Classifier Class. """

        self.Helpers = Helpers("ClassifierServer")
        self.confs = self.Helpers.confs

        self.OpenCV = OpenCV()

        self.Movidius = Movidius()
        self.Movidius.checkNCS()
        self.Movidius.loadInception()

        self.Helpers.logger.info(
            "Classifier server class initialization complete.")

app = Flask(__name__)
Server = Server()

@app.route('/Inference', methods=['POST'])
def Inference():

    if len(request.files) != 0:
        img = Server.OpenCV.loadImage(np.fromstring(
            request.files['file'].read(), np.uint8))
    else:
        img = Server.OpenCV.loadImage(np.fromstring(request.data, np.uint8))

    dx, dy, dz = img.shape
    delta = float(abs(dy-dx))

    if dx > dy:
        img = img[int(0.5*delta):dx-int(0.5*delta), 0:dy]
    else:
        img = img[0:dx, int(0.5*delta):dy-int(0.5*delta)]

    img = cv2.cvtColor(cv2.resize(
        img, (Server.Movidius.reqsize, Server.Movidius.reqsize)), cv2.COLOR_BGR2RGB)

    for i in range(3):
        img[:, :, i] = (img[:, :, i] - Server.Movidius.mean) * \
            Server.Movidius.std

    detectionStart, detectionStart = Server.Helpers.timerStart()
    Server.Helpers.logger.info("API detection started.")

    Server.Movidius.ncsGraph.LoadTensor(img.astype(np.float16), 'user object')
    output, userobj = Server.Movidius.ncsGraph.GetResult()

    detectionClockEnd, difference, detectionEnd = Server.Helpers.timerEnd(
        detectionStart)
    Server.Helpers.logger.info("API detection ended taking " + str(difference))

    top_inds = output.argsort()[::-1][:5]

    if output[top_inds[0]] >= Server.confs["Classifier"]["InceptionThreshold"] and Server.Movidius.classes[top_inds[0]] == "1":
        classification = "AML Positive"
        message = "ALL detected with a confidence of " + \
            str(output[top_inds[0]]) + " in " + str(difference)
    elif output[top_inds[0]] >= Server.confs["Classifier"]["InceptionThreshold"] and Server.Movidius.classes[top_inds[0]] == "0":
        classification = "AML Negative"
        message = "ALL not detected with a confidence of " + \
            str(output[top_inds[0]]) + " in " + str(difference)
    elif output[top_inds[0]] <= Server.confs["Classifier"]["InceptionThreshold"] and Server.Movidius.classes[top_inds[0]] == "1":
        classification = "AML Positive"
        message = "ALL detected with a LOW confidence of " + \
            str(output[top_inds[0]]) + " in " + str(difference)
    elif output[top_inds[0]] <= Server.confs["Classifier"]["InceptionThreshold"] and Server.Movidius.classes[top_inds[0]] == "0":
        classification = "AML Negative"
        message = "ALL not detected with a LOW confidence of " + \
            str(output[top_inds[0]]) + " in " + str(difference)

    Server.Helpers.logger.info(message)

    ServerResponse = jsonpickle.encode({
        'Response': 'OK',
        'Classification': classification,
        'aClassification': Server.Movidius.classes[top_inds[0]],
        'Confidence': str(output[top_inds[0]]),
        'Message': message
    })

    return Response(response=ServerResponse, status=200, mimetype="application/json")

if __name__ == "__main__":
    app.run(host=Server.confs["Classifier"]["IP"],
            port=Server.confs["Classifier"]["Port"])


import cv2
import numpy as np
import os

def box_extraction(img_for_box_extraction_path, cropped_dir_path, img_name):

    color_img = cv2.imread(img_for_box_extraction_path)
    img = cv2.imread(img_for_box_extraction_path, 0)  # Read the image
    (thresh, img_bin) = cv2.threshold(img, 128, 255,
                                      cv2.THRESH_BINARY | cv2.THRESH_OTSU)  # Thresholding the image
    img_bin = 255-img_bin  # Invert the image
    #cv2.imwrite("Image_bin.jpg", img_bin)

    # Defining a kernel length
    kernel_length = np.array(img).shape[1]//40

    # A verticle kernel of (1 X kernel_length), which will detect all the verticle lines from the image.
    verticle_kernel = cv2.getStructuringElement(
        cv2.MORPH_RECT, (1, kernel_length))

    # A horizontal kernel of (kernel_length X 1), which will help to detect all the horizontal line from the image.
    hori_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (kernel_length, 1))
    # A kernel of (3 X 3) ones.
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))

    # Morphological operation to detect verticle lines from an image
    img_temp1 = cv2.erode(img_bin, verticle_kernel, iterations=3)
    verticle_lines_img = cv2.dilate(img_temp1, verticle_kernel, iterations=3)
    #cv2.imwrite("verticle_lines.jpg", verticle_lines_img)

    # Morphological operation to detect horizontal lines from an image
    img_temp2 = cv2.erode(img_bin, hori_kernel, iterations=3)
    horizontal_lines_img = cv2.dilate(img_temp2, hori_kernel, iterations=3)
    # cv2.imwrite("horizontal_lines.jpg", horizontal_lines_img)

    # Weighting parameters, this will decide the quantity of an image to be added to make a new image.
    alpha = 0.5
    beta = 1.0 - alpha

    # This function helps to add two image with specific weight parameter to get a third image as summation of two image.
    img_final_bin = cv2.addWeighted(
        verticle_lines_img, alpha, horizontal_lines_img, beta, 0.0)
    img_final_bin = cv2.erode(~img_final_bin, kernel, iterations=2)
    (thresh, img_final_bin) = cv2.threshold(
        img_final_bin, 128, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)

    # Invert the image again, because find countours only works on white objects
    img_final_bin = 255-img_final_bin

    # For Debugging
    # Enable this line to see verticle and horizontal lines in the image which is used to find boxes
    resized_img = cv2.resize(img_final_bin, (960, 720))
    cv2.imshow("img_final_bin.jpg", resized_img)
    cv2.waitKey(0)
    #cv2.imwrite("img_final_bin.jpg", img_final_bin)

    # Find contours for image, which will detect all the boxes
    contours, hierarchy = cv2.findContours(
        img_final_bin, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    # Sort all the contours by top to bottom. Need to actually make sort_contours or some issue with importing function
    # (contours, boundingBoxes) = sort_contours(
    #     contours, method="top-to-bottom")

    idx = 0
    for item in contours:
        # Returns the location and width,height for every contour
        x, y, w, h = cv2.boundingRect(item)

        # If the box height is greater then 20, widht is >80, then only save it as a box in "cropped/" folder.
        if (w > 300 and h > 300 and w/h > 0.86 and w/h < 1.25):
            idx += 1
            new_img = color_img[y:y+h, x:x+w]
            cv2.imwrite(cropped_dir_path+r"\\"+img_name+r"_cropped_" +
                        str(idx) + '.jpg', new_img)

    # Debugging
    # resized_img = cv2.resize(color_img, (960, 720))
    # cv2.imshow("color image", resized_img)
    # cv2.waitKey(0)

    # Draws contours only on good contours, can't move this to first for loop because draws lines over the image
    box_num = 0
    areas = []
    temp_img_drawn = color_img
    for index, item in enumerate(contours):
        area = cv2.contourArea(item)
        if(area>138000 and area <200000): # Problem need to reorient images with Hough lines
            cv2.drawContours(temp_img_drawn, contours, index, (0,255,0), 3)
            box_num +=1
            areas.append(area)

    # See the countours drawn over image
    #cv2.imwrite(cropped_dir_path+r"\\" + "drawn_img.jpg", drawn_img)
    print(img_name + ' ', areas)
    resized_img = cv2.resize(temp_img_drawn, (960, 720))
    cv2.imshow(str(img_name) + " Drawn over image boxes counted: " + str(box_num), resized_img)
    cv2.waitKey(0)

# Single image box detection
box_extraction(r"4x4 hemo grid slanted after 4 hours.jpg", r"Hemo_images\cropped", 'image_11')

# Loop through files in a directory
# rootdir = r'Hemo_images\images_to_crop'
# destination = r'Hemo_images\cropped'
# for subdir, dirs, files in os.walk(rootdir):
#     for file in files:
#         temp_name = os.path.join(subdir, file)
#         box_extraction(temp_name, destination,file[:-4])


from robocode.registration.models import UserProfile, User
from robocode.uploader.views import *
from django.shortcuts import render_to_response
from django.contrib.admin.views.decorators import staff_member_required

## Mostrem els equips i els robots presentats
@staff_member_required
def reportUser(request):
    user_profile_list = []
    for perfil in UserProfile.objects.all():
        perfil.username = User.objects.get(id=perfil.user_id).username
        user_profile_list.append(perfil)

    submit_list = []
    for sub in Submit.objects.all():
        sub.categoria = Category.objects.get(id=sub.category_id).name
        sub.filename = unicode(unicode(sub.file).rsplit('/',1)[1])
        submit_list.append(sub)

    return render_to_response('admin/report.html' ,{'user_profile_list': user_profile_list, 'submit_list': submit_list})


########################################################################################################################
# This program builds regression model for bike forecasting
# Inputs: expects train and test data files with following
#          columns (in order, the names are immaterial)
#         Hour	Weekday	start station id	RideInitiationCount	N_DryBulbTemp	N_RelativeHumidity	N_WindSpeed
# Outputs: pickled/numpy files for regression model and tranforms
#          also prints the R-Sq on Training data and RMSE for test
########################################################################################################################

#############################################
# Import the necessary modules and libraries
#############################################
import numpy as np
from sklearn.tree import DecisionTreeRegressor
from sklearn import preprocessing
from sklearn.preprocessing import Imputer
from sklearn.model_selection import cross_val_score
from sklearn.externals import joblib
import csv
import os
import os.path
import pickle
print(__doc__)

#############################################
# Input parser for the train/test data files
#############################################
def readdata(filepath, filename, labelcolumn, excludedcatcolumns, excludednumcolumns):
    csvfile = open(os.path.join(filepath,filename))
    reader = csv.reader(csvfile, delimiter=',')

    headers = next(reader)

    nanfloat = float('nan')
    nanint = 0

    parsed = ((row[0],
               row[1],
               row[2],
               nanint if (row[3] == 'NA' or row[3] == 'nan') else int(row[3]),
               nanfloat if row[4] == 'NA' else float(row[4]),
               nanfloat if row[5] == 'NA' else float(row[5]),
               nanfloat if row[6] == 'NA' else float(row[6]),
               )
              for row in reader)

    selector = [x for x in range(len(headers)) if (x not in excludedcatcolumns and x not in excludednumcolumns)]

    data = []
    label = []

    for row in parsed:
        data.append([row[i] for i in selector])
        label.append(row[labelcolumn])
    return{'data':data,'label':label}

#############################################
# Preprocessing the features using
# Label Encoding for categorical features and
# Missing value Imputation for Numericals
#############################################
def processfeatures(features, categoricalfeaturestart, categoricalfeaturestartend, testdata = None):

    istest = False
    if not testdata:
        istest = True

    numfeatures = len(features[0])
    processedfeatures = np.empty([len(features),numfeatures])
    featureencoders = np.empty((categoricalfeaturestartend,), dtype=np.object)

    #loading the categorical to numerical encoders if running on test data
    if istest:
        for iter in range(0, categoricalfeaturestartend):
            featureencoders[iter] = preprocessing.LabelEncoder()
            featureencoders[iter].fit(np.array(np.load('./outputs/'+str(iter)+'_labelencoder.npy')))

    #categorical to numerical transform
    for iter in range(0, numfeatures):
        if(iter >= categoricalfeaturestart and iter < categoricalfeaturestartend):
            encoder = preprocessing.LabelEncoder()
            if not istest:
                encoder.fit([row[iter] for row in (features+testdata)])
                np.save('./outputs/'+str(iter)+'_labelencoder.npy', encoder.classes_)
            else:
                encoder = featureencoders[iter]
            processedfeatures[:,iter] = np.array(encoder.transform([row[iter ] for row in features]))
        else:
            processedfeatures[:,iter] = np.array([row[iter] for row in features])

    #impute nans in numerical cols, auto excluding the categorical features as they have been replaced with classes already
    meanimputer = Imputer(missing_values='NaN', strategy='mean', axis=0)

    if not istest:
        meanimputer.fit(processedfeatures)
        f = open("./outputs/meanimputer.pickle", "wb")
        pickle.dump(meanimputer,f )
        f.close()
    else:
        f = open('./outputs/meanimputer.pickle',"rb")
        meanimputer = pickle.load(f)
        f.close()

    processedfeatures_imp = meanimputer.transform(processedfeatures)
    return processedfeatures_imp

#############################################
# This is the main execution
# we read the train and test files
# do feature preprocessing
# build model and do xval on train
# do out of sample val on test
#############################################
if __name__ == '__main__':
    filepath = './'
    trainfilename = 'TrainDataV6.csv'
    testfilename = 'TestDataV6.csv'

    # create the outputs folder
    os.makedirs('./outputs', exist_ok=True)

    labelcolumn = 3
    excludedcatcolumns = [labelcolumn]
    excludednumcolumns = []
    categoricalfeaturestart = 0
    categoricalfeaturestartend = 3
    numtrees = 5

    #read from train data file
    fileread = readdata(filepath,trainfilename, labelcolumn,excludedcatcolumns, excludednumcolumns)
    traindata = fileread['data']
    trainlabel = fileread['label']

    #read from test data file
    fileread = readdata(filepath,testfilename, labelcolumn, excludedcatcolumns, excludednumcolumns)
    testdata = fileread['data']
    testlabel = fileread['label']

    #process the train features
    processedtraindata_imp = processfeatures(traindata, categoricalfeaturestart, categoricalfeaturestartend, testdata)
    processedlabels = trainlabel

    #read in the cmd line argument if passed
    if len(sys.argv) > 1:
        numtrees = int(sys.argv[1])

    print("Number of trees {}.".format(numtrees))
    #perform random forest regression on train data
    from sklearn.ensemble import RandomForestRegressor
    rf_1 = RandomForestRegressor(random_state=0, n_estimators=numtrees)

    #output model
    r4 = rf_1.fit(processedtraindata_imp,processedlabels)
    pickle.dump(r4, open("./outputs/randomforest.pickle", "wb"))

    #output model performance
    score4 = cross_val_score(r4, processedtraindata_imp, processedlabels).mean()
    print("rf R-Sq on xval TRAIN = %.2f" % score4)

    from azureml.sdk import data_collector
    run_logger = data_collector.current_run()

    # log the number of trees
    run_logger.log("Number of Trees", numtrees)

    # log rsquare
    run_logger.log("R-Square on xval train set", score4)

    #process the test features
    processedtestdata_imp = processfeatures(testdata, categoricalfeaturestart, categoricalfeaturestartend)
    processedtestlabels = testlabel

    #use the above model to predict on test data
    testpredictions = r4.predict(processedtestdata_imp)

    #calculate rmse
    testrmse = np.sqrt(np.mean(np.square(np.int64(testlabel) == np.int64(np.ceil(testpredictions)))))
    print('test RMSE = %.2f' % testrmse )

    #############################################
    # This code is used to do the visualization
    # it shows up under details for the particular run
    #############################################
    import pandas
    from azureml.sdk import data_collector
    run_logger = data_collector.current_run()
    metrics = []
    for i in range(len(testlabel)):
        if i % 100 == 0:
            metrics.append({'Actual': testlabel[i]})

    run_logger.log(pandas.DataFrame(metrics))

    metrics2 = []
    for i in range(len(testpredictions)):
        if i % 100 == 0:
           metrics2.append({'Predicted': testpredictions[i]})

    run_logger.log(pandas.DataFrame(metrics2))


# Copyright (C) 2009 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#    * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#    * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import unittest

from modules.commands.commandtest import CommandsTest
from modules.commands.queues import *
from modules.commands.queuestest import QueuesTest
from modules.mock_bugzillatool import MockBugzillaTool
from modules.outputcapture import OutputCapture

class TestQueue(AbstractQueue):
    name = "test-queue"

class AbstractQueueTest(CommandsTest):
    def _assert_log_progress_output(self, patch_ids, progress_output):
        OutputCapture().assert_outputs(self, TestQueue().log_progress, [patch_ids], expected_stderr=progress_output)

    def test_log_progress(self):
        self._assert_log_progress_output([1,2,3], "3 patches in test-queue [1, 2, 3]\n")
        self._assert_log_progress_output(["1","2","3"], "3 patches in test-queue [1, 2, 3]\n")
        self._assert_log_progress_output([1], "1 patch in test-queue [1]\n")

    def _assert_run_bugzilla_tool(self, run_args):
        queue = TestQueue()
        tool = MockBugzillaTool()
        queue.bind_to_tool(tool)

        queue.run_bugzilla_tool(run_args)
        expected_run_args = ["echo", "--status-host=example.com"] + map(str, run_args)
        tool.executive.run_and_throw_if_fail.assert_called_with(expected_run_args)

    def test_run_bugzilla_tool(self):
        self._assert_run_bugzilla_tool([1])
        self._assert_run_bugzilla_tool(["one", 2])

class CommitQueueTest(QueuesTest):
    def test_style_queue(self):
        expected_stderr = {
            "begin_work_queue" : "CAUTION: commit-queue will discard all local changes in \"%s\"\nRunning WebKit commit-queue.\n" % os.getcwd(),
            "next_work_item" : "2 patches in commit-queue [197, 128]\n",
        }
        self.assert_queue_outputs(CommitQueue(), expected_stderr=expected_stderr)

class StyleQueueTest(QueuesTest):
    def test_style_queue(self):
        expected_stderr = {
            "begin_work_queue" : "CAUTION: style-queue will discard all local changes in \"%s\"\nRunning WebKit style-queue.\n" % os.getcwd(),
            "handle_unexpected_error" : "Mock error message\n",
        }
        self.assert_queue_outputs(StyleQueue(), expected_stderr=expected_stderr)


""" ps2_implementation.py

PUT YOUR NAME HERE:
Till Rohrmann, Matrikelnummer: 343756

Write the functions
- kmeans
- kmeans_agglo
- agglo_dendro
Write your implementations in the given functions stubs!

(c) Felix Brockherde, TU Berlin, 2013
    Translated to Python from Paul Buenau's Matlab scripts
"""

from __future__ import print_function
from __future__ import division
import numpy as np
from scipy.cluster.hierarchy import dendrogram # you can use this
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse
import warnings;
import time;

warnings.filterwarnings('ignore', 'invalid value encountered in add');

def kmeans(X, k, max_iter=100):
    """ Performs k-means clustering

    Input:
    X: (d x n) data matrix with each datapoint in one column
    k: number of clusters
    max_iter: maximum number of iterations

    Output:
    mu: (d x k) matrix with each cluster center in one column
    r: assignment vector

    Description:
        This function performs a k-means cluster analysis on the given dataset X.
        It tries to find k clusters and terminates either if the maximum number of
        iterations has been reached or if the cluster membership does not change
        anymore.

    Author:
        Till Rohrmann, till.rohrmann@campus.tu-berlin.de
    """

    n = X.shape[1];

    # select initial clusters
    clusterIndices = np.random.permutation(np.arange(n))[0:k];

    mu = X[:,clusterIndices].copy();
    r = np.ones(n)*-1;

    start = time.clock();
    for i in range(max_iter):
        # rows are indexed by the data points and columns by the cluster centers
        distance = np.sqrt(((X[:,:,np.newaxis] - mu[:,np.newaxis,:])**2).sum(axis=0));
        rold = r;
        # find in each row the index of minimal distance
        r = np.argmin(distance,axis=1);

        # calculate new cluster centers by averaging the assigned points
        for j in range(k):
            mu[:,j] = X[:,r==j].sum(axis=1)/(r==j).sum();

        numChanges = (r != rold).sum();

        if(numChanges == 0):
            break;

        lossValue = np.sqrt(((X - mu[:,r])**2).sum(axis=0)).sum()

        print('Iteration:' + str(i) + ' #changes:' + str(numChanges) + ' loss function value:' + str(lossValue));

    return mu,r, i,time.clock()-start;

def kmeans_crit(X, r):
    """ Computes k-means criterion

    Input:
    X: (d x n) data matrix with each data point in one column
    r: assignment vector

    Output:
    value: scalar for sum of Euclidean distances to cluster centers
    """
    k = max(r)+1;
    result = 0;
    for j in range(k):
        if (r==j).sum() > 0:
            # calculate cluster center based on point assignment (mean of
            # all points which are assigned to the same cluster)
            m = X[:,r==j].sum(axis=1)/(r==j).sum();

            # sum up the distance of the data points to their cluster centers
            result += np.sqrt(((X[:,r==j]-m[:,np.newaxis])**2).sum(axis=0)).sum();

    return result;

def kmeans_agglo(X, r):
    """ Performs agglomerative clustering with k-means criterion

    Input:
    X: (d x n) data matrix, data point is represented by a column vector
    r: assignment vector

    Output:
    R: k x n matrix that contains cluster memberships before each step
    kmloss: k vector with loss after each step
    mergeidx: (k-1) x 2 matrix that contains merge idx for each step

    Description:
        This function performs the agglomerative clustering with k-means.

    Author:
        Till Rohrmann, till.rohrmann@campus.tu-berlin.de
    """

    n = X.shape[1];
    k = max(r)+1;
    R = np.zeros((k,n),dtype='int');
    kmloss = np.zeros(k);
    mergeidx = np.zeros((k-1,2));

    # array which is used to map the used cluster indices to the range
    # 0...(k-1). After each merge operation, the range is decreased by one
    mapping = np.arange(k);

    R[0,:] = r;

    kmloss[0] = kmeans_crit(X,r);

    for idx in range(1,k):
        minError = np.Inf;
        minSet1 = -1;
        minSet2 = -2;

        # find 2 cluster such that merging them has the minimal overall error
        for i in range(k-idx):
            for j in range(i+1,k-idx+1):
                tempR = R[idx-1,:].copy();
                tempR[R[idx-1,:]==mapping[j]] = mapping[i];
                error = kmeans_crit(X,tempR);

                if error < minError:
                    minError = error;
                    minSet1 = i;
                    minSet2 = j;

        kmloss[idx] = minError;
        mergeidx[idx-1,:] = (mapping[minSet1], mapping[minSet2]);
        R[idx,:] = R[idx-1,:].copy();
        # merging cluster minSet1 and minSet2 and giving it the new index k + idx -1
        # This is in accordance with the implementation of the function scipy.cluster.hierarchy.linkage
        R[idx,R[idx-1,:]==mapping[minSet2]] = k+idx-1;
        R[idx,R[idx-1,:]==mapping[minSet1]] = k+idx-1;

        # set the merged set mapping (with smaller index) to the newly introduced index
        mapping[minSet1] = k+idx-1;
        # overwrite the second merged set mapping with the last mapped value, which would
        # otherwise fall out of the range in the next iteration.
        mapping[minSet2] = mapping[k-idx];

    return R, kmloss, mergeidx;

def agglo_dendro(kmloss, mergeidx):
    """ Plots dendrogram for agglomerative clustering

    Input:
    kmloss: k vector with loss after each step
    mergeidx: (k-1) x 2 matrix that contains merge idx for each step

    Author:
        Till Rohrmann, till.rohrmann@campus.tu-berlin.de
    """
    Z = np.hstack((mergeidx,kmloss[1:,np.newaxis],np.zeros((mergeidx.shape[0],1))));
    dendrogram(Z);
    d = max(Z[:,2]) - min(Z[:,2]);
    # set the y range so that the interesting region is highlighted
    plt.ylim([max(0,min(Z[:,2])-0.01*d), max(Z[:,2])+0.01*d]);
    plt.show(block=False);

def log_norm_pdf(X, mu, C):
    """ Computes log probability density function for multivariate gaussian

    Input:
    X: (d x n) data matrix with each data point in one column
    mu: vector for center
    C: covariance matrix

    Output:
    pdf value for each data point

    Author:
        Till Rohrmann, till.rohrmann@campus.tu-berlin.de
    """
    if(X.ndim == 1):
        d = X.shape[0]
    else:
        d = X.shape[0];
    centeredM = X - mu[:,np.newaxis];

    _,logDet = np.linalg.slogdet(C);

    result = (-d/2)*(np.log(2*np.pi))+(-1./2)*logDet+(-1./2*np.sum(centeredM*np.linalg.solve(C,centeredM),0));

    return result;

def norm_pdf(X, mu, C):
    """ Computes probability density function for multivariate gaussian

    Input:
    X: (d x n) data matrix with each data point in one column
    mu: vector for center
    C: covariance matrix

    Output:
    pdf value for each data point

    Author:
        Till Rohrmann, till.rohrmann@campus.tu-berlin.de
    """

    d = X.shape[0];
    centeredM = X - mu[:,np.newaxis];

    determinant = np.linalg.det(C);

    result = (2*np.pi)**(-d/2)*determinant**(-1./2)*np.exp(-1./2*np.sum(centeredM*np.linalg.solve(C,centeredM),0));

    return result;

def log_likelihood(data, pi, mu, sigma):
    ''' This function calculates the log likelihood of the data given
        a mixture Gaussian distribution.

        Input:
            data : (d x n) data matrix with each datapoint in one column
            pi : k data array with each element specifying the probability of
                each Gaussian component
            mu : (d x k) mean vector matrix with each mean vector in one column
            sigma : (k x d x d) covariance matrix list where the ith Gaussian component
                has the covariance matrix sigma[i].

        Output:
            Log likelihood of the parameters given the data points.

        Author:
            Till Rohrmann, till.rohrmann@campus.tu-berlin.de
    '''
    k = mu.shape[1];
    n = data.shape[1];

    temp = np.zeros(n);

    for i in range(k):
            temp += pi[i]*norm_pdf(data,mu[:,i],sigma[i]);

    logL = np.log(temp).sum();

    return logL;

def log_log_likelihood(data, pi, mu, sigma):
    ''' This function calculates the log likelihood of the data given
        a mixture Gaussian distribution. The calculation is performed in
        the log space.

        Input:
            data : (d x n) data matrix with each datapoint in one column
            pi : k data array with each element specifying the probability of
                each Gaussian component
            mu : (d x k) mean vector matrix with each mean vector in one column
            sigma : (k x d x d) covariance matrix list where the ith Gaussian component
                has the covariance matrix sigma[i].

        Output:
            Log likelihood of the parameters given the data points.

        Author:
            Till Rohrmann, till.rohrmann@campus.tu-berlin.de
    '''
    k = mu.shape[1];
    n = data.shape[1];

    temp = np.ones(n)*-np.Inf;

    for i in range(k):
            # using the log functions in order to stay in the log space
            buf = log_norm_pdf(data,mu[:,i],sigma[i]);
            # Addition of two log values in the log space
            temp = log_sum(temp,np.log(pi[i])+buf);

    logL = temp.sum();

    return logL;

def log_sum(a,b):
    ''' This function performs an addition of two logarithmic values in the log space.
        Given two logarithmic values log(x) and log(y) the function computes the value
        log(x+y) in a numerically stable manner. Assuming x > y then the formula is derived
        the following way:
            log(x+y) = log(x) + log(1+y/x) = log(x) + log(1 + exp(log(y)-log(x)))

        Input:
            a : 1st operand in logarithmic space
            b : 2nd operand in logarithmic space

        Output:
            log(exp(a)+exp(b))

        Author:
            Till Rohrmann, till.rohrman@campus.tu-berlin.de
    '''
    assert np.ndim(a) == np.ndim(b), 'Arguments must have the same dimensions.'
    if(np.ndim(a) == 0 and np.ndim(b) == 0):
        if a == -np.Inf:
            return b
        elif b == -np.Inf:
            return a
        elif a>b:
            return a+np.log(1+np.exp(b-a));
        else:
            return b+np.log(1+np.exp(a-b));
    else:
        # computation for vectors is performed element-wise
        condlist = [a== -np.Inf, b==-np.Inf, a>b, b>=a];
        choicelist = [b, a, a+np.log(1+np.exp(b-a)), b+np.log(1+np.exp(a-b))];
        return np.select(condlist,choicelist);

def log_em_mog(X, k, max_iter=100, init_kmeans=False, eps=1e-3):
    """ Performs EM Mixture of Gaussians in logarithmic space in order
        avoid underflows while calculating with very little probabilities.

    Input:
        X: (d x n) data matrix with each datapoint in one column
        k: number of clusters
        max_iter: maximum number of iterations
        init_kmeans: whether kmeans should be used for initialisation
        eps: when log likelihood difference is smaller than eps, terminate loop

    Output:
    pi: 1 x k matrix of priors
    mu: (d x k) matrix with each cluster center in one column
    sigma: list of d x d covariance matrices

    Author:
        Till Rohrmann, till.rohrmann@campus.tu-berlin.de
    """
    n = X.shape[1];
    d = X.shape[0];
    pi = np.ones(k)/k;
    covariances = np.zeros((k,d,d));
    gammas = np.zeros((n,k));
    logL = -np.Inf;
    # regularization constant
    delta = 1e-5

    # initialize covariance matrices with the identity
    for i in range(k):
        covariances[i] = np.identity(d);

    if(init_kmeans == True):
        mu, _,_,_ = kmeans(X, k, max_iter);
    else:
        candidateIndices = np.random.permutation(np.arange(n))[:k];
        mu = X[:,candidateIndices];

    start = time.clock();

    for counter in range(max_iter):

        sGamma = np.ones(n)*-np.Inf;

        # calculate for every Gaussian the probability that the data points belong
        # to its cluster
        for i in range(k):
            gammas[:,i] = np.log(pi[i]) + log_norm_pdf(X,mu[:,i],covariances[i]);
            sGamma = log_sum(sGamma,gammas[:,i]);

        # normalize the probabilities
        for i in range(k):
            gammas[:,i] = np.exp(gammas[:,i]-sGamma);

        nk = gammas.sum(axis=0);
        pi = nk/n;

        for i in range(k):
            # calculate the new cluster centers
            mu[:,i] = np.dot(X,gammas[:,i])/nk[i];

            wX = (X-mu[:,i,np.newaxis]) * np.sqrt(gammas[:,i]);
            # estimating the new covariance matrix with regularization
            covariances[i] = np.dot(wX,wX.T)/nk[i] + np.identity(d)*delta

        oldLogL = logL;

        logL = log_log_likelihood(X, pi, mu, covariances);

        print('Iteration:' +str(counter) + ' log likelihood:' + str(logL));

        if(logL-oldLogL < eps):
            break;

    return pi, mu, covariances, counter, time.clock() - start;

def em_mog(X, k, max_iter=100, init_kmeans=False, eps=1e-3):
    """ Performs EM Mixture of Gaussians

    Input:
    X: (d x n) data matrix with each datapoint in one column
    k: number of clusters
    max_iter: maximum number of iterations
    init_kmeans: whether kmeans should be used for initialisation
    eps: when log likelihood difference is smaller than eps, terminate loop

    Output:
    pi: 1 x k matrix of priors
    mu: (d x k) matrix with each cluster center in one column
    sigma: list of d x d covariance matrices

    Author:
        Till Rohrmann, till.rohrmann@campus.tu-berlin.de
    """
    n = X.shape[1];
    d = X.shape[0];
    pi = np.ones(k)/k;
    covariances = np.zeros((k,d,d));
    gammas = np.zeros((n,k));
    logL = -np.Inf;
    # regularization constant
    delta = 1e-5

    # covariance matrices of the Gaussians is initialized with the identity matrix
    for i in range(k):
        covariances[i] = np.identity(d);

    if(init_kmeans == True):
        mu, _, _, _ = kmeans(X, k, max_iter);
    else:
        candidateIndices = np.random.permutation(np.arange(n))[:k];
        mu = X[:,candidateIndices];

    start = time.clock();

    for counter in range(max_iter):

        sGamma = np.zeros(n);

        # calculate for each cluster center the probability that a datapoint
        # belongs to it
        for i in range(k):
            gammas[:,i] = pi[i]*norm_pdf(X,mu[:,i],covariances[i]);
            sGamma += gammas[:,i];

        # normalize probability
        for i in range(k):
            gammas[:,i] /= sGamma;

        nk = gammas.sum(axis=0);
        pi = nk/n;

        for i in range(k):
            # calculate new cluster centers
            mu[:,i] = np.dot(X,gammas[:,i])/nk[i];

            wX = (X-mu[:,i,np.newaxis]) * np.sqrt(gammas[:,i]);
            # estimate the new covariance matrices with regularization
            covariances[i] = np.dot(wX,wX.T)/nk[i] + np.identity(d)*delta

        oldLogL = logL;

        logL = log_likelihood(X, pi, mu, covariances);

        print('Iteration:' +str(counter) + ' log likelihood:' + str(logL));

        if(logL-oldLogL < eps):
            break;

    return pi, mu, covariances, counter, time.clock()-start;

def plot_ellipse(mean,covariance, nstd):
    ''' This function prints the 2d ellipse representing the covariance matrix.

        Input:
            mean: 2d vector
            covariance: covariance matrix
            nstd: number of standard deviations which shall be included into the ellipse,
                that is to say, the height and the width of the ellipse

        Author:
            Till Rohrmann, till.rohrmann@campus.tu-berlin.de
    '''

    # find axis of ellipse which are the eigenvectors with biggest eigenvalues
    values, vectors = np.linalg.eigh(covariance);

    ordering = np.argsort(values)[::-1];

    # extract eigenvectors with biggest eigenvalues, the eigenvalues denote the variance
    # in the direction of the eigenvector
    values = values[ordering[:2]];
    vectors = vectors[:,ordering[:2]];

    # caculate the rotation angle of the ellipse
    theta = np.degrees(np.arctan2(vectors[1,0],vectors[0,0]));

    width, height = 2*nstd*np.sqrt(values);

    e = Ellipse(mean, width, height, theta);

    ax = plt.gca();

    ax.add_artist(e);
    e.set_facecolor('none');
    e.set_edgecolor('black');

def plot_em_solution(X, mu, sigma):
    """ Plots covariance ellipses for EM MoG solution and the data points.
        The smaller ellipse denotes the area of a single standard deviation and
        the bigger ellipse denotes the area of 2 times the standard deviation.

    Input:
    X: (d x n) data matrix with each datapoint in one column
    mu: (d x k) matrix with each cluster center in one column
    sigma: list of d x d covariance matrices

    Author:
        Till Rohrmann, till.rohrmann@campus.tu-berlin.de
    """
    k = mu.shape[1];

    plt.scatter(X[0,:],X[1,:]);

    plt.scatter(mu[0,:],mu[1,:],c='r',marker='x');

    for i in range(k):
        plot_ellipse(mu[:,i],sigma[i],1)
        plot_ellipse(mu[:,i],sigma[i],2)

def kmean_loss(X,pi, mu, sigma):
    ''' This function calculates the kmeans loss function for the solution
        of the EM-algorithm. For this purpose the distance to each cluster
        center weighted by the probability of membership is summed up for all
        points.

        Input:
            X : (d x n) data matrix
            pi : probability distribution of the mixture components
            mu : (d x k) cluster mean matrix
            sigma : covariance matrices of the Gaussian clusters

        Output:
            loss function value
    '''
    n = X.shape[1];
    k = mu.shape[1];

    gammas = np.zeros((n,k));
    sGamma = np.ones(n)*-np.Inf;

    for i in range(k):
        gammas[:,i] = np.log(pi[i]) + log_norm_pdf(X,mu[:,i],sigma[i]);
        sGamma = log_sum(sGamma,gammas[:,i]);

    for i in range(k):
        gammas[:,i] = np.exp(gammas[:,i]-sGamma);

    D = np.sqrt(((X[:,:,np.newaxis] - mu[:,np.newaxis,:])**2).sum(axis=0));

    return np.sum(np.sum(D*gammas,axis=1));


from bs4 import BeautifulSoup
import requests

def wiki_crawler(target):
    req = requests.get('https://en.wikipedia.org/w/index.php?search='+target)
    if req.status_code == 404:
        return None, None
    soup = BeautifulSoup(req.content,'html.parser')
    isSearch=None
    if soup.find('h1') is not None:
        isSearch = soup.find('h1').getText()
    if isSearch is not None and isSearch.find("Search results")!=-1:
        req = requests.get('https://en.wikipedia.org/wiki/'+target)
        soup = BeautifulSoup(req.content,'html.parser')
        isSearch = soup.find('h1').getText()
        if req.status_code ==404:
            return None,None
        if isSearch is not None and isSearch.find("Bad title")!=-1:
            return None,None
    if 'refer to' in soup.find('p').getText() and soup.find('ul') is not None:
        return req.url, None
    words = soup.find('p').getText().split(' ')
    if len(words) > 30:
        words = words[0:30]
    summary =  " ".join(words)
    return req.url, summary


import numpy as np
from keras.datasets import imdb
from keras import models, layers

(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)

print(train_data[0])
print(max([max(s) for s in train_data]))
word_index = imdb.get_word_index()
print(type(word_index))
# print(word_index)
reverse_word_index = dict([(v, k) for k, v in word_index.items()])
decoded_review = ' '.join([reverse_word_index.get(i - 3, '?') for i in train_data[2]])
print(decoded_review)

def vectorize_sequences(sequences, dimension=10000):
    results = np.zeros((len(sequences), dimension))
    for i, seq in enumerate(sequences):
        results[i, seq] = 1.
    return results

x_train = vectorize_sequences(train_data)
x_test = vectorize_sequences(test_data)
y_train = np.asarray(train_labels).astype('float32')
y_test = np.asarray(test_labels).astype('float32')
print(train_data[2])
print(x_train[2])

model1 = models.Sequential()
model1.add(layers.Dense(32, activation='relu', input_shape=(10000,)))
model1.add(layers.Dense(16, activation='relu'))
model1.add(layers.Dense(8, activation='relu'))
model1.add(layers.Dense(1, activation='sigmoid'))
print(model1.summary())
model1.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

history = model1.fit(x_train, y_train, epochs=15, batch_size=512, validation_data=(x_test, y_test))


import argparse
from scholarly import scholarly
import json

def search_author(name: str):
    return scholarly.search_author(name)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Get author's info.")
    parser.add_argument('--name', type=str, help="Author's name")
    parser.add_argument('--limit', type=int, default=10)
    parser.add_argument('--offset', type=int, default=0)

    args = parser.parse_args()
    authors = search_author(args.name)
    count = 0
    res = []
    for a in authors:
        res.append(dict(
            container_type=a['container_type'],
            scholar_id=a['scholar_id'],
            url_picture=a['url_picture'],
            name=a['name'],
            affiliation=a['affiliation'],
            email_domain=a['email_domain'],
            interests=a['interests'],
            citedby=a['citedby']
        ))
        count += 1
        if count >= args.limit:
            break
    print(json.dumps(res))