from cassandra import cqlengine
class TableIterator(object):
    """
    Iterates over a Cassandra table defined by a cqlengine model class using query paging in order to pull back chunks
    of data that have `blocksize` number of records.
    Can optionally provide kwargs which are used as where clause filters. These kwargs must be columns on the model
    which are indexed.
    :param model_class: The cqlengine model object that defines the table you want to iterate over.
    :type model_class: An instance of cqlengine.Model.
    :param blocksize: The number of results you want to pull back with each paging query. Can be used to tune the
                        performance of the iteration depending on the characteristics of the table.
    :type blocksize: integer
    :param where_filters: Keyword arguments can be passed to the iterator and these will be used as filter parameters
                        when querying the database for pages of data.
    :type where_filters: **kwargs
    :return: an iterator over the a collection of objects of type model_class.
    """

    def __init__(self, model_class, blocksize=10000, **where_filters):
        self.model_class = model_class

        # Pull the keys of the model class for convenient reference.
        self.partition_keys = model_class._partition_keys
        self.clustering_keys = model_class._clustering_keys
        self.blocksize = blocksize
        self.where_filters = where_filters

    @classmethod
    def generate_where_clause_key(cls, column_name, clause_condition):
        """
        Takes the name of a primary key column and a condition ('gt', 'lt', 'eq') and creates a where clause key that
        can be used with a cqlengine .objects() descriptor to filter based on that condition.
        :param key_name: The name of the model column (primary key) for which you want to generate a where clause.
        :type key_name: The string representation of the column name.
        :param clause_condition: The conditional operator you want to filter by.
        :type clause_condition: A string that is a valid cqlengine conditional operator ('gt', 'lt', 'eq')
        :return: A string of the form "{my_column_name}__{my_clause_condition}".
        """
        return "{}__{}".format(column_name, clause_condition)

    @classmethod
    def get_paging_where_clause_key(cls, primary_key_column):
        """
        Get a where clause key value that can be used to page through the primary key column.
        :param primary_key_column: A primary key column class you want a key to page over.
        :type primary_key_column:  An class that inherits from cqlengine.Column.
        :return: A string of the format "{column_name}__{where_condition}" which will page data from that column in the
                direction defined by the clustering order of that column.
                For example, if I have a clustering key named `my_cluster_key` which has a descending clustering order,
                this function will return the key 'my_cluster_key__lt' which can be used to page over the value of
                my_cluster_key in it's default order.
        """
        condition_lookup = {"asc": "gt", "desc": "lt"}
        clustering_order = getattr(primary_key_column, "clustering_order") or "asc"
        clause_condition = condition_lookup[clustering_order.lower()]
        return cls.generate_where_clause_key(primary_key_column.column_name, clause_condition)

    def get_next_query_set(self, previous_object):
        """
        Takes a cqlengine model object instance and treats that object as the current cursor into a Cassandra table
        generating a cqlengine query object which will page in the set of results immediately following
        `previous_object` according to Cassandra partition tokens and clustering order.
        :param previous_object: The last object fetched by the previous paging query. Can also be viewed as the cursor
                    location for this table iteration.
        :type previous_object: A cqlengine model object instance.
        :return: A cqlengine QuerySet object that will return the objects immediately following `previous_object` in the
                Cassandra table.
        """
        prev_partition_key_vals = {}
        prev_clustering_key_vals = {}

        # Pull all of the key values off of previous_object
        for p_key_name, _ in self.partition_keys.items():
            prev_partition_key_vals[p_key_name] = getattr(previous_object, p_key_name)
        for c_key_name, _ in self.clustering_keys.items():
            prev_clustering_key_vals[c_key_name] = getattr(previous_object, c_key_name)

        # Copy the clustering keys dict since we want to use the values it contains as **kwargs to a QuerySet. We need
        # to alter the values without clobbering the original values.
        cluster_where_clause = dict(prev_clustering_key_vals.items())

        # Iterator over the ordered clustering keys in reverse order.
        for c_key_name, c_key_col in reversed(self.clustering_keys.items()):
            # Drop the equals clause for the current clustering key because we want a paging conditional ('gt' or 'lt').
            del cluster_where_clause[c_key_name]

            # Generate a paging clause for this clustering key that we will use as a where clause filter.
            new_where_key = self.get_paging_where_clause_key(c_key_col)
            cluster_where_clause[new_where_key] = prev_clustering_key_vals[c_key_name]

            # Generate our new where clause consisting of the current partition, our paging clustering conditions and
            # any where_filters that were originally handed to TableIterator.
            where_clause = dict(prev_partition_key_vals.items() + cluster_where_clause.items() + self.where_filters.items())

            current_query = self.model_class.objects(**where_clause).limit(self.blocksize)

            # TODO: Can we optimize to return results from this function rather than doing garbage query round trip?
            if current_query.first():
                # This query returns objects, so it's a valid page and we want to use it.
                return current_query
            else:
                # This query returned nothing so we have exhausted the clustering key we are currently looking at.
                # Drop the clause for this clustering key and continue to the next one.
                del cluster_where_clause[new_where_key]

        # We made it through testing all of the clustering key values and got no results so we have exhausted the
        # current partition we are looking at.

        # Generate the partition key token for the last seen object.
        token = cqlengine.functions.Token(previous_object.pk)

        # Create a where clause for the partition key token.
        pk_token_where = self.generate_where_clause_key('pk__token', 'gt')
        partition_key_clause = {pk_token_where: token}

        where_clause = dict(partition_key_clause.items() + self.where_filters.items())

        query = self.model_class.objects(**where_clause).limit(self.blocksize)

        return query

    def __iter__(self):
        """
        Returns an iterator over the objects that exist in the table passed into __init__.
        """

        done_iterating = False
        query = self.model_class.objects(**self.where_filters).limit(self.blocksize)

        while not done_iterating:
            previous_object = None

            for obj in query:
                previous_object = obj
                yield obj

            if not previous_object is None:
                query = self.get_next_query_set(previous_object)
            else:
                done_iterating = True


#!/usr/bin/env python
"""QuTiP: The Quantum Toolbox in Python

QuTiP is open-source software for simulating the dynamics of closed and open
quantum systems. The QuTiP library depends on the excellent Numpy, Scipy, and
Cython numerical packages. In addition, graphical output is provided by
Matplotlib.  QuTiP aims to provide user-friendly and efficient numerical
simulations of a wide variety of quantum mechanical problems, including those
with Hamiltonians and/or collapse operators with arbitrary time-dependence,
commonly found in a wide range of physics applications. QuTiP is freely
available for use and/or modification on all common platforms. Being free of
any licensing fees, QuTiP is ideal for exploring quantum mechanics in research
as well as in the classroom.
"""

DOCLINES = __doc__.split('\n')

CLASSIFIERS = """\
Development Status :: 4 - Beta
Intended Audience :: Science/Research
License :: OSI Approved :: BSD License
Programming Language :: Python
Programming Language :: Python :: 3
Topic :: Scientific/Engineering
Operating System :: MacOS
Operating System :: POSIX
Operating System :: Unix
Operating System :: Microsoft :: Windows
"""

# import statements
import os
import sys
# The following is required to get unit tests up and running.
# If the user doesn't have, then that's OK, we'll just skip unit tests.
try:
    from setuptools import setup, Extension
    TEST_SUITE = 'nose.collector'
    TESTS_REQUIRE = ['nose']
    EXTRA_KWARGS = {
        'test_suite': TEST_SUITE,
        'tests_require': TESTS_REQUIRE
    }
except:
    from distutils.core import setup
    from distutils.extension import Extension
    EXTRA_KWARGS = {}

try:
    import numpy as np
except:
    np = None

from Cython.Build import cythonize
from Cython.Distutils import build_ext

# all information about QuTiP goes here
MAJOR = 4
MINOR = 2
MICRO = 0
ISRELEASED = False
VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO)
REQUIRES = ['numpy (>=1.8)', 'scipy (>=0.15)', 'cython (>=0.21)']
INSTALL_REQUIRES = ['numpy>=1.8', 'scipy>=0.15', 'cython>=0.21']
PACKAGES = ['qutip', 'qutip/ui', 'qutip/cy', 'qutip/cy/src',
            'qutip/qip', 'qutip/qip/models',
            'qutip/qip/algorithms', 'qutip/control', 'qutip/nonmarkov',
            'qutip/_mkl', 'qutip/tests', 'qutip/legacy',
            'qutip/cy/openmp', 'qutip/cy/openmp/src']
PACKAGE_DATA = {
    '.': ['README.md', 'LICENSE.txt'],
    'qutip': ['configspec.ini'],
    'qutip/tests': ['*.ini'],
    'qutip/cy': ['*.pxi', '*.pxd', '*.pyx'],
    'qutip/cy/src': ['*.cpp', '*.hpp'],
    'qutip/control': ['*.pyx'],
    'qutip/cy/openmp': ['*.pxd', '*.pyx'],
    'qutip/cy/openmp/src': ['*.cpp', '*.hpp']
}
# If we're missing numpy, exclude import directories until we can
# figure them out properly.
INCLUDE_DIRS = [np.get_include()] if np is not None else []
# ajgpitch Mar 2017:
# This HEADERS did not work, but I will leave it in anyway, as it is supposed to.
# I had to do the nasty thing with PACKAGES and PACKAGE_DATA above.
HEADERS = ['qutip/cy/src/zspmv.hpp', 'qutip/cy/openmp/src/zspmv_openmp.hpp']
NAME = "qutip"
AUTHOR = ("Alexander Pitchford, Paul D. Nation, Robert J. Johansson, "
          "Chris Granade, Arne Grimsmo")
AUTHOR_EMAIL = ("alex.pitchford@gmail.com, nonhermitian@gmail.com, "
                "jrjohansson@gmail.com, cgranade@cgranade.com, "
                "arne.grimsmo@gmail.com")
LICENSE = "BSD"
DESCRIPTION = DOCLINES[0]
LONG_DESCRIPTION = "\n".join(DOCLINES[2:])
KEYWORDS = "quantum physics dynamics"
URL = "http://qutip.org"
CLASSIFIERS = [_f for _f in CLASSIFIERS.split('\n') if _f]
PLATFORMS = ["Linux", "Mac OSX", "Unix", "Windows"]

def git_short_hash():
    try:
        git_str = "+" + os.popen('git log -1 --format="%h"').read().strip()
    except:
        git_str = ""
    else:
        if git_str == '+': #fixes setuptools PEP issues with versioning
            git_str = ''
    return git_str

FULLVERSION = VERSION
if not ISRELEASED:
    FULLVERSION += '.dev'+str(MICRO)+git_short_hash()

# NumPy's distutils reads in versions differently than
# our fallback. To make sure that versions are added to
# egg-info correctly, we need to add FULLVERSION to
# EXTRA_KWARGS if NumPy wasn't imported correctly.
if np is None:
    EXTRA_KWARGS['version'] = FULLVERSION

def write_version_py(filename='qutip/version.py'):
    cnt = """\
# THIS FILE IS GENERATED FROM QUTIP SETUP.PY
short_version = '%(version)s'
version = '%(fullversion)s'
release = %(isrelease)s
"""
    a = open(filename, 'w')
    try:
        a.write(cnt % {'version': VERSION, 'fullversion':
                FULLVERSION, 'isrelease': str(ISRELEASED)})
    finally:
        a.close()

local_path = os.path.dirname(os.path.abspath(sys.argv[0]))
os.chdir(local_path)
sys.path.insert(0, local_path)
sys.path.insert(0, os.path.join(local_path, 'qutip'))  # to retrive _version

# always rewrite _version
if os.path.exists('qutip/version.py'):
    os.remove('qutip/version.py')

write_version_py()

# Add Cython extensions here
cy_exts = ['spmatfuncs', 'stochastic', 'sparse_utils', 'graph_utils', 'interpolate',
        'spmath', 'heom', 'math', 'spconvert', 'ptrace', 'testing']

# If on Win and Python version >= 3.5 (i.e. Visual studio compile)
if sys.platform == 'win32' and int(str(sys.version_info[0])+str(sys.version_info[1])) >= 35:
    _compiler_flags = ['/w', '/Ox']
# Everything else
else:
    _compiler_flags = ['-w', '-O3', '-march=native', '-funroll-loops']

EXT_MODULES =[]
# Add Cython files from qutip/cy
for ext in cy_exts:
    _mod = Extension('qutip.cy.'+ext,
            sources = ['qutip/cy/'+ext+'.pyx', 'qutip/cy/src/zspmv.cpp'],
            include_dirs = [np.get_include()],
            extra_compile_args=_compiler_flags,
            extra_link_args=[],
            language='c++')
    EXT_MODULES.append(_mod)

# Add Cython files from qutip/control
_mod = Extension('qutip.control.cy_grape',
            sources = ['qutip/control/cy_grape.pyx'],
            include_dirs = [np.get_include()],
            extra_compile_args=_compiler_flags,
            extra_link_args=[],
            language='c++')
EXT_MODULES.append(_mod)

# Add optional ext modules here
if "--with-openmp" in sys.argv:
    sys.argv.remove("--with-openmp")
    if (sys.platform == 'win32'
            and int(str(sys.version_info[0])+str(sys.version_info[1])) >= 35):
        omp_flags = ['/openmp']
        omp_args = []
    else:
        omp_flags = ['-fopenmp']
        omp_args = omp_flags
    _mod = Extension('qutip.cy.openmp.parfuncs',
            sources = ['qutip/cy/openmp/parfuncs.pyx',
                       'qutip/cy/openmp/src/zspmv_openmp.cpp'],
            include_dirs = [np.get_include()],
            extra_compile_args=_compiler_flags+omp_flags,
            extra_link_args=omp_args,
            language='c++')
    EXT_MODULES.append(_mod)
    # Add benchmark pyx
    _mod = Extension('qutip.cy.openmp.benchmark',
            sources = ['qutip/cy/openmp/benchmark.pyx'],
            include_dirs = [np.get_include()],
            extra_compile_args=_compiler_flags,
            extra_link_args=[],
            language='c++')
    EXT_MODULES.append(_mod)

# Setup commands go here
setup(
    name = NAME,
    version = FULLVERSION,
    packages = PACKAGES,
    include_dirs = INCLUDE_DIRS,
    headers = HEADERS,
    ext_modules = cythonize(EXT_MODULES),
    cmdclass = {'build_ext': build_ext},
    author = AUTHOR,
    author_email = AUTHOR_EMAIL,
    license = LICENSE,
    description = DESCRIPTION,
    long_description = LONG_DESCRIPTION,
    keywords = KEYWORDS,
    url = URL,
    classifiers = CLASSIFIERS,
    platforms = PLATFORMS,
    requires = REQUIRES,
    package_data = PACKAGE_DATA,
    zip_safe = False,
    install_requires=INSTALL_REQUIRES,
    **EXTRA_KWARGS
)


import yaml

class ReadYaml(object):
    def __init__(self, file_path, encoding):
        with open(file_path, "r", encoding=encoding) as file:
            self.configMap = yaml.load(file, Loader=yaml.FullLoader)

    def get(self, key_path):
        key_arr = key_path.split(".")
        temp_map = self.configMap
        for index in range(len(key_arr)):
            if index != len(key_arr) - 1:
                temp_map = self.configMap[key_arr[index]]
            else:
                return temp_map[key_arr[index]]

    def write(self, data, path):
        with open(path, 'w', encoding='utf-8') as file:
            yaml.dump(data=data, stream=file, allow_unicode=True)

if __name__ == "__main__":
    res = ReadYaml("E:\\SubjectNetwork-Download\\resources\\application.yml", "utf-8")
    username = res.get("base-url")
    print(username)

    apiData = {"base-url": "http://www.zxxk.com11"}
    res.configMap.update(apiData)
    with open("E:\\SubjectNetwork-Download\\resources\\application.yml", 'w', encoding='utf-8') as f:
        yaml.dump(data=res.configMap, stream=f, allow_unicode=True)

# 获取当前脚本所在文件夹路径
# curPath = os.path.dirname(os.path.realpath(__file__))
# 获取yaml文件路径
# yamlPath = os.path.join(curPath, "cfgyaml.yaml")


import logging
from datetime import datetime, timedelta
from typing import Dict, List

import pandas as pd

from business.helpers import getTimeBlocks
from business.model.asset import Asset
from business.model.timeframe import TimeFrame
from business.modules.asset_bl import AssetBL

# create logger
log = logging.getLogger("CellarLogger")

def updateFutures(
    asset: Asset, timeframe: TimeFrame, bl: AssetBL
) -> List[Dict]:
    result = []
    for cd in asset.contractDetails:

        lastTradeDateTime = datetime.strptime(
            cd.contract.lastTradeDateOrContractMonth, "%Y%m%d"
        )
        now = datetime.now()

        if lastTradeDateTime > now:

            # localSymbol = cd.contract.localSymbol
            localSymbol = f"{cd.contract.localSymbol}-{cd.contract.lastTradeDateOrContractMonth}"
            symbolData = bl.getHistoricalDataFromDB(localSymbol, timeframe)

            if symbolData is not None:

                lastDateTime = symbolData.tail(1).index[0]

                if now > lastDateTime:
                    result.append(
                        {
                            "contract": cd.contract,
                            "from": lastDateTime,
                            "to": now,
                        }
                    )
            else:
                result.append(
                    {
                        "contract": cd.contract,
                        "from": datetime.strptime("19860101", "%Y%m%d"),
                        "to": now,
                    }
                )

        else:
            log.info(
                f" SKIPPED - {cd.contract.localSymbol} - {cd.contract.lastTradeDateOrContractMonth}"
            )

    return result

def downloadFutures(asset: Asset, blockSize: int) -> List[Dict]:
    result = []

    for cd in asset.contractDetails:

        lastTradeDateTime = datetime.strptime(
            cd.contract.lastTradeDateOrContractMonth, "%Y%m%d"
        )
        now = datetime.now()

        if (
            lastTradeDateTime > datetime.strptime("19860101", "%Y%m%d")
            and lastTradeDateTime < now
        ):
            result.append(
                {
                    "contract": cd.contract,
                    "from": lastTradeDateTime - timedelta(days=blockSize),
                    "to": lastTradeDateTime,
                }
            )
        elif lastTradeDateTime >= now:
            result.append(
                {
                    "contract": cd.contract,
                    "from": now - timedelta(days=blockSize),
                    "to": now,
                }
            )

    return result

def updateStock(
    asset: Asset, timeframe: TimeFrame, histData: pd.DataFrame
) -> List[Dict]:
    result = []

    contract = asset.contractDetails[0].contract
    # symbolData = bl.getHistoricalDataFromDB(asset.symbol, timeframe)

    if histData is not None:

        lastDateTime = histData.tail(1).index[0]
        now = datetime.now()

        if now > lastDateTime:
            result.append(
                {"contract": contract, "from": lastDateTime, "to": now}
            )

    return result

def downloadStock(asset: Asset, blockSize: int) -> List[Dict]:
    result = []

    contract = asset.contractDetails[0].contract

    timeBlocks = getTimeBlocks(
        datetime.strptime("19860101", "%Y%m%d"), datetime.now(), blockSize
    )

    for timeBlock in timeBlocks:
        result.append(
            {"contract": contract, "from": timeBlock[0], "to": timeBlock[1],}
        )

    return result


import csv
import struct as st
import numpy as np
import keyboard
import idx2numpy
from PIL import Image
from numpy import linalg as LA
from numpy import matlib
import sounddevice as sd
import pickle as pkl
import ipdb;
from scipy.io import loadmat;
from IPython.display import Audio
import matplotlib.pyplot as plt
import random
import sys

marker = '15';
marker_x = marker+'_x';
marker_y = marker+'_y';
marker_z = marker+'_z';
marker_c = marker+'_c';
x_t = 'frame'

def exponential_cov(x, y, sigma_f, sigma_l):
    return np.exp(sigma_f) * np.exp( -0.5 * np.exp(sigma_l) * np.subtract.outer(x, y)**2)
#***************Reading data from file********************************************

sigma_l_arr = 0;
sigma_f_arr = 0;
sigma_n_arr = 0;

#********************** Obtaining the data *********************************
counter = 0;
data1 = np.array([0 , 0, 0]);
data2 = np.array([0 , 0, 0]);
data3 = np.array([0 , 0, 0]);
data4 = np.array([0 , 0, 0]);
data5 = np.array([0 , 0, 0]);
with open('./data_GP/AG/block1-UNWEIGHTED-SLOW-NONDOMINANT-RANDOM/20161213203046-59968-right-speed_0.500.csv', newline='') as csvfile:
     reader = csv.DictReader(csvfile)
     for row in reader:
         counter =counter+1;
         # print(row['frame'], row['0_x'])
         # ipdb.set_trace();
         # if(counter>=frame_start and counter<frame_start+window_size):
         data1 = np.vstack([data1, np.array([row[x_t], row[marker_x], row[marker_c]],dtype=float)]);

data1=np.delete(data1,0,0);

counter=0
with open('./data_GP/AG/block2-UNWEIGHTED-SLOW-NONDOMINANT-RANDOM/20161213204004-59968-right-speed_0.500.csv', newline='') as csvfile:
     reader = csv.DictReader(csvfile)
     for row in reader:
         counter =counter+1;
         # print(row['frame'], row['0_x'])
         # if(counter>=frame_start and counter<frame_start+window_size):
         data2 = np.vstack([data2, np.array([row[x_t], row[marker_x], row[marker_c]],dtype=float)]);

data2=np.delete(data2,0,0);

counter=0
with open('./data_GP/AG/block3-UNWEIGHTED-SLOW-NONDOMINANT-RANDOM/20161213204208-59968-right-speed_0.500.csv', newline='') as csvfile:
     reader = csv.DictReader(csvfile)
     for row in reader:
         counter =counter+1;
         # print(row['frame'], row['0_x'])
         # if(counter>=frame_start and counter<frame_start+window_size):
         data3 = np.vstack([data3, np.array([row[x_t], row[marker_x], row[marker_c]],dtype=float)]);
data3=np.delete(data3,0,0);
#
#
#
counter=0
with open('./data_GP/AG/block4-UNWEIGHTED-SLOW-NONDOMINANT-RANDOM/20161213204925-59968-right-speed_0.500.csv', newline='') as csvfile:
     reader = csv.DictReader(csvfile)
     for row in reader:
         counter =counter+1;
         # print(row['frame'], row['0_x'])
         # if(counter>=frame_start and counter<frame_start+window_size):
         data4 = np.vstack([data4, np.array([row[x_t], row[marker_x], row[marker_c]],dtype=float)]);
data4=np.delete(data4,0,0);
#
#
counter=0
with open('./data_GP/AG/block5-UNWEIGHTED-SLOW-NONDOMINANT-RANDOM/20161213210121-59968-right-speed_0.500.csv', newline='') as csvfile:
     reader = csv.DictReader(csvfile)
     for row in reader:
         counter =counter+1;
         # print(row['frame'], row['0_x'])
         # if(counter>=frame_start and counter<frame_start+window_size):
         data5 = np.vstack([data5, np.array([row[x_t], row[marker_x], row[marker_c]],dtype=float)]);
data5=np.delete(data5,0,0);
data = np.zeros((1010,2));

# ************** mixing 5 traces **********************************
counter=0;
flag=0;
for i in range(0,1010):
    r=random.randint(1,5);
    flag=0;
    while flag==0:
        if data1[i,2]<0 and data2[i,2]<0 and data3[i,2]<0 and data4[i,2]<0 and data5[i,2]<0:
            flag=1;
            continue;
        elif r==1 and data1[i,2]>0:
            data[counter,:] = data1[i,:-1];
            counter=counter+1;
            flag=1;
        elif r==2 and data2[i,2]>0:
            data[counter,:] = data2[i,:-1];
            counter=counter+1;
            flag=1;
        elif r==3 and data3[i,2]>0:
            data[counter,:] = data3[i,:-1];
            counter=counter+1;
            flag=1;
        elif r==4 and data4[i,2]>0:
            data[counter,:] = data4[i,:-1];
            counter=counter+1;
            flag=1;
        elif r==5 and data5[i,2]>0:
            data[counter,:] = data5[i,:-1];
            counter=counter+1;
            flag=1;
        else:
            r=random.randint(1,5);
# data_report = pkl.load(open("./GP_hyperparam_15_x_report.p","rb"));
# data=np.zeros((1000,2))
# data[:,0] = data_report['X'].reshape(-1,)
# data[:,1] = data_report['Y'].reshape(-1,)
# ************** mixing 2 traces **********************************
# counter=0;
# flag=0;
# for i in range(0,1010):
#     r=random.randint(1,2);
#     flag=0;
#     while flag==0:
#         if data1[i,2]<0 and data2[i,2]<0:
#             flag=1;
#             continue;
#         elif r==1 and data1[i,2]>0:
#             data[counter,:] = data1[i,:-1];
#             counter=counter+1;
#             flag=1;
#         elif r==2 and data2[i,2]>0:
#             data[counter,:] = data2[i,:-1];
#             counter=counter+1;
#             flag=1;
#         else:
#             r=random.randint(1,2);

counter=0;
for i in range(0,1010):
    if data1[i,2]<=0:
        continue;
    else:
        data[counter,:] = data1[i,:-1];
        counter=counter+1;

# data = data1[:,:-1]
window_size=100
test_pts=1;
window_start=0
window_end=1000-window_size
delta=5
logP_local=[0];
#************************ starting the window process ***********************************************
for frame_start in np.arange(window_start,window_end,delta):#data.shape[0]-100-window_size):
# for frame_start in range(0,1):

    # ipdb.set_trace()
    # L = np.max([data1.shape,data2.shape,data3.shape,data4.shape,data5.shape])

    data_curr = data[frame_start:frame_start+window_size,:]
    # mask = np.zeros(window_size)
    mask_i = np.arange(0,window_size,1)

    sampling = random.choices(mask_i, k=test_pts)
    mask = np.zeros(window_size)>5
    mask[sampling] = True
    # Xpredict = data_curr[mask,0]
    # Ypredict = data_curr[mask,1]
    # XX = data_curr[np.invert(mask),0]
    # YY = data_curr[np.invert(mask),1]
    # XX=XX.reshape(-1,1);
    # YY=YY.reshape(-1,1)
    # data_test = data_curr[]
    # ipdb.set_trace()

    # from sklearn.gaussian_process import GaussianProcessRegressor
    # XX = data[:,0].reshape(-1,1);
    XX = data_curr[:,0].reshape(-1,1);
    XX=XX*1.0;
    # XX = np.array([-2.1, -1.5, -0.7, 0.3, 1.0, 1.8, 2.5]).reshape(-1,1);
    #
    #
    YY = data_curr[:,1].reshape(-1,1);
    # YY=np.array([-1.5128756 , 0.52371713, -0.1382640378102619, -0.13952425, 0.4967141530112327, -0.93665367, -1.29343995]).reshape(-1,1);
    # Xpredict = data[frame_start+window_size:frame_start+window_size+15,0];
    # Ypredict = data[frame_start+window_size:frame_start+window_size+15,1];
    sigma_f = -1;2;#-2.09;-2;
    sigma_l=-8;-6;#-7.6;-4;
    sigma_n = -8;-9;#np.Inf;-1;-6.28#-np.Inf;#1;#-2;
    eta = 1e-2;
    f=1;
    fprev=f/2;

    L = XX.size
    K=np.zeros([L,L]);
    k = np.zeros([L,L]);
    kl = np.zeros([L,L]);
    tol=2;
    dPdf = np.array([[1]]); dPdl = np.array([[1]]); dPdn = np.array([[1e4]]);
    err = 1;#(np.abs(dPdf[0,0])+np.abs(dPdl[0,0])+np.abs(dPdn[0,0]));
    err_prev = 10;
    err_prev_2 = 100;
    err_grad = tol*2;
    count=0;

    Xii = np.multiply(XX,XX);
    Xii = np.matlib.repmat(Xii,1,L);
    Xjj = Xii.transpose();
    # ipdb.set_trace();
    XXi_XXj = Xii+Xjj-2*XX.dot(XX.transpose());

    # ipdb.set_trace()

    # while (err < err_prev and err_prev < err_prev_2) or err_grad>tol:
    # while err < err_prev or err_grad>tol:

    # while f>=fprev:# and count>3:
    while    err_grad > tol:# or np.abs(dPdn[0,0])>500:
        if count>1e4:
            break

        kp = np.exp(sigma_f)*np.exp(-0.5*np.exp(sigma_l)*XXi_XXj);

        kl = -0.5*np.exp(sigma_l)*XXi_XXj;

        Q = kp + np.eye(L)*np.exp(sigma_n);
        Lc=np.linalg.cholesky(Q)

        Qinv = np.linalg.solve(Lc.transpose(),np.linalg.solve(Lc,np.eye(L)))
        beta = np.linalg.solve(Lc.transpose(), np.linalg.solve(Lc,YY))
        # negLogP = -0.5*YY.T.dot(Qinv).dot(YY) - np.sum(np.log(Lc)) - L/2*np.log(np.pi);
        # negLogP = -0.5*YY.T.dot(beta) - np.sum(np.log(Lc)) - L/2*np.log(np.pi);
        # negLogP = -0.5*YY.T.dot(Qinv).dot(YY) - 0.5*(np.log(LA.det(Q)+sys.float_info.min)) - L/2*np.log(2*np.pi);

        dPdf = 0.5*(YY.transpose()).dot(Qinv).dot(kp).dot(Qinv).dot(YY) - 0.5*np.trace(Qinv.dot(kp));

        dPdl = 0.5*(YY.transpose()).dot(Qinv).dot(np.multiply(kp,kl)).dot(Qinv).dot(YY) - 0.5*np.trace(Qinv.dot(np.multiply(kp,kl)));

        dPdn = 0.5*(YY.transpose()).dot(Qinv).dot(Qinv).dot(YY)*np.exp(sigma_n) - 0.5*np.trace(Qinv)*np.exp(sigma_n);#should be -trace(Qinv) but that doesn't give right answer

        sigma_f = sigma_f + eta*dPdf;#-eta*dPdf doesn't converge
        sigma_l = sigma_l + eta*dPdl;#-eta*dPdl doesn't converge
        sigma_n = sigma_n + eta*dPdn;#-eta*dPdn doesn't converge
#
# #************ Calculate prediction error **************************
#         # Xpredict = Xpredict.reshape(-1,1)
#         # Xpredict = Xpredict.transpose()
#         # Lpredict = Xpredict.size
#         # XXpred1 = np.matlib.repmat(XX,1,Lpredict)
#         # XXpred1 = np.multiply(XXpred1,XXpred1)
#         # XXpred2 = np.matlib.repmat(Xpredict,L,1)
#         # XXpred2 = np.multiply(XXpred2,XXpred2)
#         # XXpred = XXpred1+XXpred2-2*XX.dot(Xpredict)
#
#         # Xpred_cov = np.multiply(np.matlib.repmat(Xpredict,Lpredict,1),np.matlib.repmat(Xpredict,Lpredict,1)) + (np.multiply(np.matlib.repmat(Xpredict,Lpredict,1),np.matlib.repmat(Xpredict,Lpredict,1))).transpose() -2*Xpredict.T.dot(Xpredict);
#
#         # KXpredX = np.exp(sigma_f)*np.exp(-0.5*np.exp(sigma_l)*XXpred.transpose())
#         # kp = np.exp(sigma_f)*np.exp(-0.5*np.exp(sigma_l)*XXi_XXj) + np.eye(L)*np.exp(sigma_n);
#         # mXpred = KXpredX.dot(LA.inv(kp)).dot(YY)
#         #
#         # err_prev_2 = err_prev;
#         # err_prev = err;
#         # err = np.sqrt(np.mean(np.multiply(Ypredict-mXpred, Ypredict-mXpred)));
        # err_grad = (np.abs(dPdf[0,0])+np.abs(dPdl[0,0]));#+np.abs(dPdn[0,0]));
        err_grad = (np.abs(dPdf[0,0])+np.abs(dPdl[0,0])+np.abs(dPdn[0,0]));
#
#         fprev=f;
#         f=negLogP;
#         # err_grad = np.abs(dPdl[0,0]);
#         # err_grad=0;
#         # print("count=",count)
        # print("grad=",err_grad)
        count=count+1;
#         # print("count=",count)
#         # print("err=",err)
#         # print("f=",f)
#         # print("sigma_l=",sigma_l)
#         # print("sigma_n=",sigma_n)
#         # print("sigma_f=",sigma_f)
#         # print("det=",LA.det(Q))
#         # ipdb.set_trace();
#
    print("count=",count)
    print("err=",err)
    print("sigma_l=",sigma_l)
    print("sigma_n=",sigma_n)
    print("sigma_f=",sigma_f)
    print("det=",LA.det(Q))
    # if count<5000:
    sigma_n_arr = np.vstack([sigma_n_arr, sigma_n]);
    sigma_f_arr = np.vstack([sigma_f_arr, sigma_f]);
    sigma_l_arr = np.vstack([sigma_l_arr, sigma_l]);
    # negLogP = -0.5*YY.T.dot(Qinv).dot(YY) - 0.5*(np.log(LA.det(Q)+sys.float_info.min)) - L/2*np.log(2*np.pi);
    # logP_local = np.vstack([logP_local, negLogP]);
    #
    # ipdb.set_trace()
#********************** Plotting local kernels, comment if not plotting ***************************
#     Xstar = np.linspace(XX[0],XX[-1],window_size*10);#XX+0.5;
#     # Xstar = np.linspace(-3,3,1000)
#     # Xstar = np.array([1.5, 2.5, 3.5, 4.5, 5.5, 9.5, 15.5, 17.5, 18.5])
#     Xstar = Xstar.reshape(-1,1)
#     Xstar = Xstar.transpose()
#     L1 = Xstar.size
#     XXstar1 = np.matlib.repmat(XX,1,L1)
#     XXstar1 = np.multiply(XXstar1,XXstar1)
#     XXstar2 = np.matlib.repmat(Xstar,L,1)
#     XXstar2 = np.multiply(XXstar2,XXstar2)
#     XXstar = XXstar1+XXstar2-2*XX.dot(Xstar)
#
#     Xstar_cov = np.multiply(np.matlib.repmat(Xstar,L1,1),np.matlib.repmat(Xstar,L1,1)) + (np.multiply(np.matlib.repmat(Xstar,L1,1),np.matlib.repmat(Xstar,L1,1))).transpose() -2*Xstar.T.dot(Xstar);
#
#     KXstarX = np.exp(sigma_f)*np.exp(-0.5*np.exp(sigma_l)*XXstar.transpose())
#     kp = np.exp(sigma_f)*np.exp(-0.5*np.exp(sigma_l)*XXi_XXj) + np.eye(L)*np.exp(sigma_n);
#     Lc_kp=np.linalg.cholesky(kp)
#
#     kpinv = np.linalg.solve(Lc_kp.transpose(),np.linalg.solve(Lc_kp,np.eye(L)))
#     mXstar = KXstarX.dot(kpinv).dot(YY)
#
#     PXstar = np.exp(sigma_f)*np.exp(-0.5*np.exp(sigma_l)*Xstar_cov) + np.eye(L1)*np.exp(sigma_n) - KXstarX.dot(kpinv).dot(KXstarX.transpose())
#
#     plt.errorbar(Xstar.T,mXstar,color=[0,0,0], yerr=np.sqrt(np.diagonal(PXstar))*2, ecolor = [0.7,0.7,0.7], label='GP mean function')
#     plt.plot(Xstar.T,mXstar,color='k')
#
#
# l1,=plt.plot(data[:,0],data[:,1],'r.',label='input data')
#
# plt.title('Local GP kernels fitted to mixture of 5 traces')
# plt.xlabel('Frame number')
# plt.ylabel('Position (m)')
# plt.legend(handles = [l1])
# plt.rcParams.update({'font.size': 25})
# plt.show()

# logP_report_kernels = {"logP": logP_local}
# pkl.dump(logP_report_kernels,open("GP_logP_local.p","wb"))

#***********************************************************************************************

#
#
#
#
# #*********** Predicting new values ***************************
#
# sigma_f = 2; 0;#1;
# sigma_l= -6; np.log(10);#2;
# sigma_n = -9; -np.Inf;#1;#-2;
sigma_l_arr = np.delete(sigma_l_arr,0)
sigma_f_arr = np.delete(sigma_f_arr,0)
sigma_n_arr = np.delete(sigma_n_arr,0)
logP_local = np.delete(logP_local,0)
# ipdb.set_trace();
Xstar = np.linspace(XX[0],XX[-1],window_size*3);#XX+0.5;
# Xstar = np.linspace(-3,3,1000)
# Xstar = np.array([1.5, 2.5, 3.5, 4.5, 5.5, 9.5, 15.5, 17.5, 18.5])
Xstar = Xstar.reshape(-1,1)
Xstar = Xstar.transpose()
L1 = Xstar.size

# B = exponential_cov(Xstar, XX, sigma_f, sigma_l)
# C = exponential_cov(XX, XX, sigma_f, sigma_l)
# A = exponential_cov(Xstar, Xstar, sigma_f, sigma_l)
# B=B.reshape(L1,L)
# C=C.reshape(L,L)
# A=A.reshape(L1,L1)
# #
# # mu = B.dot(np.linalg.inv(C)).dot(YY) #np.linalg.inv(C).dot(B.T).T.dot(YY)
# # mu = np.linalg.inv(C).dot(B.T).T.dot(YY);
# sigma = A - B.dot(np.linalg.inv(C).dot(B.T))

XXstar1 = np.matlib.repmat(XX,1,L1)
XXstar1 = np.multiply(XXstar1,XXstar1)
XXstar2 = np.matlib.repmat(Xstar,L,1)
XXstar2 = np.multiply(XXstar2,XXstar2)
XXstar = XXstar1+XXstar2-2*XX.dot(Xstar)

Xstar_cov = np.multiply(np.matlib.repmat(Xstar,L1,1),np.matlib.repmat(Xstar,L1,1)) + (np.multiply(np.matlib.repmat(Xstar,L1,1),np.matlib.repmat(Xstar,L1,1))).transpose() -2*Xstar.T.dot(Xstar);

KXstarX = np.exp(sigma_f)*np.exp(-0.5*np.exp(sigma_l)*XXstar.transpose())
kp = np.exp(sigma_f)*np.exp(-0.5*np.exp(sigma_l)*XXi_XXj) + np.eye(L)*np.exp(sigma_n);
Lc_kp=np.linalg.cholesky(kp)

kpinv = np.linalg.solve(Lc_kp.transpose(),np.linalg.solve(Lc_kp,np.eye(L)))

mXstar = KXstarX.dot(kpinv).dot(YY)

# mXstar = KXstarX.dot(LA.inv(kp)).dot(YY)

PXstar = np.exp(sigma_f)*np.exp(-0.5*np.exp(sigma_l)*Xstar_cov) + np.eye(L1)*np.exp(sigma_n) - KXstarX.dot(kpinv).dot(KXstarX.transpose())

# PXstar = np.exp(sigma_f)*np.exp(-0.5*np.exp(sigma_l)*Xstar_cov) - KXstarX.dot(kpinv).dot(KXstarX.transpose())

# Ym = gpr.predict(Xstar.T,return_std=True);

c1=np.diagonal(PXstar)#.reshape(-1,1);

l1,l2,l3 = plt.errorbar(Xstar.T,mXstar,color=[0,0,0], yerr=np.sqrt(np.diagonal(PXstar))*2, ecolor = [0.7,0.7,0.7], label='GP mean function')
# plt.errorbar((Xstar).T,mu, yerr=np.sqrt(np.diagonal(sigma)),color='b')
# plt.plot(Xstar.T,Ym[0],'ro');
# plt.plot(XX,YY,'g--')
# plt.plot(Xstar.T,mu,color='g')
l4,=plt.plot(Xstar.T,mXstar,color='k', label='Optimal GP mean function')
l5, = plt.plot(XX,YY,'r.',label='input data')
plt.title('Optimal GP kernel fitted to mixture of 5 traces')
plt.xlabel('Frame number')
plt.ylabel('Position (m)')
plt.legend(handles = [l4,l5])
plt.rcParams.update({'font.size': 25})
plt.savefig('./plots/marker_0_x.png');
plt.show()
ipdb.set_trace();
plt.close();
l1, = plt.plot(np.arange(window_start,window_end,delta),sigma_f_arr,'b', label='sigma_f')

l2, = plt.plot(np.arange(window_start,window_end,delta),sigma_l_arr,'g', label='sigma_l')

l3, = plt.plot(np.arange(window_start,window_end,delta),sigma_n_arr,'r', label='sigma_n')

l4, = plt.plot(data[:,0],data[:,1],'k', label='trajectory')

# l5, = plt.plot(np.arange(window_start,window_end,delta),sigma_f_arr*0-1.79495408,'--b', label='sigma_f_global')
#
# l6, = plt.plot(np.arange(window_start,window_end,delta),sigma_l_arr*0-8.27030516,'--g', label='sigma_l_global')
#
# l7, = plt.plot(np.arange(window_start,window_end,delta),sigma_n_arr*0-6.33156224,'--r', label='sigma_n_global')

# plt.legend(handles = [l1,l2,l3,l4,l5,l6,l7])
plt.legend(handles = [l1,l2,l3,l4])

plt.xlabel('Frame number')
plt.ylabel('hyperparam value')
plt.xlim(-10,1400)
plt.title('Hyperparams Subject:AG, marker:15_x, Single trace')
plt.rcParams.update({'font.size': 17})
plt.savefig('./plots/marker_'+marker+'_x_hyperparams.png');

plt.show()

plt.hist(sigma_f_arr,bins=70)
plt.xlabel('sigma_f_value')
plt.ylabel('frequency')
plt.title('Histogram of sigma_f')
plt.rcParams.update({'font.size': 17})
plt.show()

plt.hist(sigma_l_arr,bins=70)
plt.xlabel('sigma_l_value')
plt.ylabel('frequency')
plt.title('Histogram of sigma_l')
plt.rcParams.update({'font.size': 17})
plt.show()

plt.hist(sigma_n_arr,bins=70)
plt.xlabel('sigma_n_value')
plt.ylabel('frequency')
plt.title('Histogram of sigma_n')
plt.rcParams.update({'font.size': 17})
plt.show()

l1, = plt.plot(data1[data1[:,2]>0,0],data1[data1[:,2]>0,1], label='trial 1',linewidth='2',color='b')
l2, = plt.plot(data2[data2[:,2]>0,0],data2[data2[:,2]>0,1], label='trial 2',linewidth='2',color='g')
l3, = plt.plot(data3[data3[:,2]>0,0],data3[data3[:,2]>0,1], label='trial 3',linewidth='2',color='r')
l4, = plt.plot(data4[data4[:,2]>0,0],data4[data4[:,2]>0,1], label='trial 4',linewidth='2',color='k')
l5, = plt.plot(data5[data5[:,2]>0,0],data5[data5[:,2]>0,1], label='trial 5',linewidth='2',color='c')
plt.legend(handles = [l1,l2,l3,l4,l5])
plt.title('Subject:AG, marker:15_x, trajectory')
plt.xlabel('Frame number')
plt.ylabel('Position (m)')
plt.rcParams.update({'font.size': 22})
plt.show()

# hyper_param_arr = {"f": sigma_f_arr, "l":sigma_l_arr, "n":sigma_n_arr, "X":data[:,0], "Y":data[:,1]}
hyper_param_arr = {"f": sigma_f_arr, "l":sigma_l_arr, "n":sigma_n_arr, "X":XX, "Y":YY}

pkl.dump(hyper_param_arr,open("GP_hyperparam_0_x.p","wb"));
ipdb.set_trace();

# with open('./data_GP/AG/block1-UNWEIGHTED-SLOW-NONDOMINANT-RANDOM/20161213203046-59968-right-speed_0.500.csv') as csv_file:
#
#     csv_reader = csv.reader(csv_file, delimiter=',')
#     line_count = 0
#     for row in csv_reader:
#         if line_count == 0:
#             print(row[0], row[11])
#             line_count += 1
#         else:
#             # print(f'\t{row[0]} works in the {row[1]} department, and was born in {row[2]}.')
#             data = np.vstack([data, np.array([row[0], row[11]],dtype=float)]);
#             line_count += 1
    # csv_dict = csv.DictReader(csv_file);


import re
import json
import logging
import pymongo
from scrapy import Request
from scrapy.spiders import Spider
from house.items import HouseItem
from scrapy_splash import SplashRequest
from house.db import Db

class HouseSpider(Spider):

    name = 'HouseSpider'
    lastId = 0
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36',
    }
    # 当前area
    curArea = {}

    def start_requests(self):
        area_coll = Db.getColl("areas")
        areas = area_coll.find().limit(1)
        for area in areas:
            HouseSpider.curArea =area
            area_key = area["key"]
            url = 'https://www.domain.com.au/sale/' + area_key + '/?sort=dateupdated-desc'
            yield Request(url, headers = self.headers)

    def parse(self, response):
        houses = response.css('.listing-result__address')
        for house in houses:
            url = house.css('::attr(href)').extract_first()
            # 房源详情页
            yield Request(url, callback=self.parseDetail,
                headers=self.headers)
        # 获取下一页
        link_btns = response.css('.paginator .is-outline')
        next_href = link_btns[len(link_btns) - 1].css("::attr(href)").extract_first()
        if next_href:
            next_href = 'https://www.domain.com.au' + next_href
            print(next_href)
            yield Request(next_href, callback=self.parse, headers = self.headers)
    def parseDetail(self, response):
        item = HouseItem()
        dataStr = response.text.split('window.renderizrData["page"] =')[1].split(';</script>')[0]
        data = json.loads(dataStr)
        item["address"] = data.get("address", '')
        item["office_banner"] = data.get("agencyBanner", '')
        item["office_logo"] = data.get("agencyLogo", '')
        item["office_name"] = data.get("agencyName", '')
        item["office_url"] = data.get("agencyProfileUrl", '')
        item["branding_color"] = data.get("brandingColor", '')

        item["agents"] = data.get("agents", '')

        item["created_on"] = data.get("createdOn", '')
        item["description"] = data.get("description", [])
        item["domain_says"] = data.get("domainSays", '')
        item["enable_child_listing_experiment"] = data.get("enableChildListingExperiment", '')
        item["enable_echoice"] = data.get("enableEchoice", '')
        item["enable_header_v2test"] = data.get("enableHeaderV2Test", '')
        item["enable_homepass"] = data.get("enableHomepass", '')
        item["enable_property_features"] = data.get("enablePropertyFeatures", '')
        item["gallery"] = data["gallery"]["slides"]
        item["header"] = data.get("header", '')
        item["headline"] = data.get("headline", '')
        item["house_id"] = data.get("id", '')
        item["inspection"] = data.get("inspection", '')
        item["is_archived"] = data.get("isArchived", '')
        item["is_land_estate"] = data.get("isLandEstate", '')
        item["listing_summary"] = data.get("listingSummary", '')
        item["listing_url"] = data.get("listingUrl", '')
        item["type"] = data.get("listingType", '')
        item["lat"] = data["map"]["latitude"]
        item["lng"] = data["map"]["longitude"]
        item["phone"] = data.get("phone", '')
        item["postcode"] = data.get("postcode", '')
        item["property_type"] = data.get("propertyType", '')
        item["schools"] = data["schoolCatchment"]["schools"]
        item["state_abbreviation"] = data.get("stateAbbreviation", '') #地区缩写
        item["street"] = data.get("street", '')
        item["street_number"] = data.get("streetNumber", '')
        item["suburb"] = data.get("suburb", '')
        item["suburb_insights"] = data.get("suburbInsights", '')
        item["what_isnearby"] = data.get("whatIsNearby", '')
        item["unit_number"] = data.get("unitNumber", '')

        # 自定义数据
        item["refId"] = data.get("id", "")
        item["region"] = HouseSpider.curArea.get("region", "")
        item["region_cn"] = HouseSpider.curArea.get("region_cn", "")
        item["region_id"] = HouseSpider.curArea.get("region_id", "")
        item["region_key"] = HouseSpider.curArea.get("region_key", "")

        item["district"] = HouseSpider.curArea.get("district", "")
        item["district_cn"] = HouseSpider.curArea.get("district_cn", "")
        item["district_id"] = HouseSpider.curArea.get("district_id", "")

        item["area"] = HouseSpider.curArea.get("name", "")
        item["area_cn"] = HouseSpider.curArea.get("cn", "")
        item["area_id"] = HouseSpider.curArea.get("id", "")

        item["property_type_id"] = self.getProperyType(data.get("propertyType", ''))
        item["price"] = data["listingSummary"].get("price", "")
        item["price_val"] = 0
        # 如果price有值
        price = item["price"]

        if "$" in price:
            # price = price.replace('*', '').replace("'", '').replace("+", "").replace("s","")
            # price = price.split('-')[0]
            # price = price.split(' ')[0]
            # a = price[price.find("$") + 1:].split(',')
            # c = ''.join(a)
            # 替换字符串
            price = price.split('-')[0]
            price =  re.sub("\D", "", price)
            item["price_val"] = price

        item["bed"] = data["listingSummary"].get("beds", "")
        item["bath"] = data["listingSummary"].get("baths", "")
        item["car"] = data["listingSummary"].get("parking", "")
        item["status"] = data["listingSummary"].get("status", "")

        # to do landarea & internal area
        return item

    # 获取房源类型id
    def getProperyType(self, key):
        typeDic = {
            "Duplex": 1,
            "House": 1,
            "Semi-Detached": 1,
            "Terrace": 1,
            "Villa": 1,
            "New House & Land": 1,
            "New Home Designs": 1,
            "Block of Units": 2,
            "Penthouse": 2,
            "Studio": 2,
            "Apartment / Unit / Flat": 2,
            "New Apartments / Off the Plan": 2,
            "Townhouse": 3,
            "Development Site": 4,
            "New land": 4,
            "Vacant land": 4,
            "Acreage / Semi-Rural": 5,
            "Farm": 5,
            "Rural": 5
        }
        return typeDic.get(key, 0)


#Import Splinter, BeautifulSoup, and Pandas
from splinter import Browser
from bs4 import BeautifulSoup as soup
import pandas as pd
#from webdriver_manager.chrome import ChromeDriverManager
import datetime as dt
import requests

def scrape_all():
# Set the executable path and initialize Splinter
#executable_path = {'executable_path': ChromeDriverManager().install()}
    browser = Browser('chrome', executable_path="chromedriver",headless=True)

    news_title, news_paragraph= mars_news(browser)
    hemisphere_image_urls= hemisphere(browser)
    # Run all scraping functions and store results in dictionary
    data={
        "news_title": news_title,
        "news_paragraph": news_paragraph,
        "featured_image": featured_image(browser),
        "facts": mars_facts(),
        "hemispheres": hemisphere_image_urls,
        "last_modified": dt.datetime.now()
    }

    # Stop webdriver and return data
    browser.quit()
    return data

def mars_news(browser):
    # Visit the mars nasa news site
    url = 'https://data-class-mars.s3.amazonaws.com/Mars/index.html'
    browser.visit(url)

    # Optional delay for loading the page
    browser.is_element_present_by_css('div.list_text', wait_time=1)

    # Convert the browser html to a soup object and then quit the browser
    html = browser.html
    news_soup = soup(html, 'html.parser')
    try:
        #slide_elem looks for <ul /> tags and descendents <li />
        # the period(.) is used for selecting classes such as item_list
        slide_elem= news_soup.select_one('ul.item_list li.slide')

        # Chained the (.find) to slide_elem which says this variable holds lots of info, so look inside to find this specific entity
        # Get Title
        news_title=slide_elem.find('div', class_= 'content_title').get_text()
        # Get article body
        news_p= slide_elem.find('div', class_='article_teaser_body').get_text()

    except AttributeError:
        return None,None

    return news_title, news_p

def featured_image(browser):
    # ### JPL Space Images Featured Image
    # Visit URL
    url = 'https://data-class-jpl-space.s3.amazonaws.com/JPL_Space/index.html'
    browser.visit(url)

    # Find and click the full image button
    full_image_elem = browser.find_by_tag('button')[1]
    full_image_elem.click()

    # Parse the resulting html with soup
    html = browser.html
    img_soup = soup(html, 'html.parser')
    #img_soup
    try:
        # find the relative image url
        img_url_rel = img_soup.find('img', class_='fancybox-image').get('src')
        #img_url_rel
    except AttributeError:
        return None
    # Use the base url to create an absolute url
    img_url = f'https://data-class-jpl-space.s3.amazonaws.com/JPL_Space/{img_url_rel}'
    return img_url

# ### Mars Facts
def mars_facts():
    try:
        df = pd.read_html('https://data-class-mars-facts.s3.amazonaws.com/Mars_Facts/index.html')[0]
        #df.head()
    except BaseException:
        return None
    df.columns=['Description', 'Mars', 'Earth']
    df.set_index('Description', inplace=True)
    df

    return df.to_html()

# # D1: Scrape High-Resolution Mars’ Hemisphere Images and Titles

# ### Hemispheres
def hemisphere(browser):
    # 1. Use browser to visit the URL
    url = 'https://data-class-mars-hemispheres.s3.amazonaws.com/Mars_Hemispheres/index.html'

    browser.visit(url)

    response = requests.get(url)
    test  = soup(response.text,'html.parser')
    #item1 = test.find_all('div', class_='item')

    #main_url  = 'https://data-class-mars-hemispheres.s3.amazonaws.com/Mars_Hemispheres/'
    #title = test.find('h3').text
    #part_img = test.find('a',class_='itemLink product-item')['href']
    #browser.visit(main_url+part_img)

    #print(main_url+part_img)

    # 2. Create a list to hold the images and titles.
    main_url = 'https://data-class-mars-hemispheres.s3.amazonaws.com/Mars_Hemispheres/'
    hemisphere_image_urls = []
    items = test.find_all('div', class_ ='item')
    # 3. Write code to retrieve the image urls and titles for each hemisphere.
    for item in items:
        #hemispheres = {}
        title = item.find('h3').text
        part_img_url = item.find('a', class_= 'itemLink product-item')['href']
        browser.visit(main_url+part_img_url)
        part_img_html = browser.html
        test = soup(part_img_html,'html.parser')
        img_url = main_url+test.find('img',class_='wide-image')['src']
        hemisphere_image_urls.append({'title':title,'img_url':img_url})

# 4. Print the list that holds the dictionary of each image url and title.
    return hemisphere_image_urls

# 5. Quit the browser
if __name__== "__main__":
    # If running as script, print scrapped data
    print(scrape_all())


import torch
import torch.nn as nn
import numpy as np
from models import PartAlign
from utils.config import Config
from datasets import CUB200
from torchvision import transforms
from utils import array_tool as at
from utils import queue_tool as qt

class PartAlignManager(object):
    def __init__(self, options, path):
        self.options = options
        self.path = path
        self.cam_weight = np.load('cam_weight.npy')

        self.net = PartAlign(
            Config.input_channel,
            Config.roi_size,
            Config.spatial_scale,
            self.cam_weight,
            Config.n_class,
            Config.n_features
        ).to(self.options['device'])

        print(self.net)
        self.cls_criterion = nn.CrossEntropyLoss()
        self.solver = torch.optim.SGD(self.net.parameters(), lr=self.options['base_lr'],
                                     momentum=0.9, weight_decay=self.options['weight_decay'])
        self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
            self.solver, mode='max', factor=0.1, patience=3, verbose=True,
            threshold=1e-4)

        transform_train_list = [
            transforms.Resize(448),
            transforms.RandomHorizontalFlip(),
            transforms.RandomCrop(size=448),
            transforms.ToTensor(),
            transforms.Normalize(mean=(0.485, 0.456, 0.406),
                                 std=(0.229, 0.224, 0.225))
        ]
        transforms_test_list = [
            transforms.Resize(448),
            transforms.CenterCrop(448),
            transforms.ToTensor(),
            transforms.Normalize(mean=(0.485, 0.456, 0.406),
                                 std=(0.229, 0.224, 0.225))
        ]
        train_data = CUB200(
            self.path['cub200'], train=True,
            transform=transforms.Compose(transform_train_list))
        test_data = CUB200(
            self.path['cub200'], train=False,
            transform=transforms.Compose(transforms_test_list))

        self.train_loader = torch.utils.data.DataLoader(
            train_data, batch_size=self.options['batch_size'],
            shuffle=True, num_workers=4, pin_memory=True
        )
        self.test_loader = torch.utils.data.DataLoader(
            test_data, batch_size=16,
            shuffle=False, num_workers=4, pin_memory=True
        )

    def train(self):
        print('Training!')
        if torch.cuda.device_count() > 1:
            print('More than one gpu')
        best_acc = 0.0
        best_epoch = 0
        print('Epoch\tTraining Loss\tTesting Loss:')
        for epoch in range(self.options['epochs']):
            num_total = 0
            num_acc = 0
            for imgs, labels in self.train_loader:
                loss = list()
                imgs = imgs.to(self.options['device'])
                labels = labels.to(self.options['device'])
                print('labels are')
                print(labels)
                feature_maps, part, rois, cam_out, cam_weight, roi_poolings = self.net(imgs)
                order_decode = self._order_decoder(part)
                order_softmax_gt = self._order_softmax_gt(order_decode, roi_poolings, labels)
                order_softmax_loss = self.cls_criterion(part, at.totensor(order_softmax_gt))
                #print(labels)
                cam_out =  cam_out.squeeze()
                cam_loss = self.cls_criterion(cam_out, labels)
                #add roi loc loss diversity cam_iou
                order_softmax_gt_decode = [list(map(int, at.part_dict[order_softmax_gt[i_order]].split(','))) for i_order in range(len(order_softmax_gt))]
                part_fc_list = list()
                roi_poolings = roi_poolings.view(roi_poolings.size(0), 4, -1)
                #print(roi_poolings)
                for i_order in range(len(order_softmax_gt_decode)):
                    #print(roi_poolings[i_order])
                    part_fc_per_img_list = list()
                    order = order_softmax_gt_decode[i_order]
                    for i_idx in range(len(order)):
                        part_fc_per_img_list.append(self.net.part_fc[order[i_idx]](roi_poolings[i_order][i_idx]))
                    assert len(part_fc_per_img_list) == 4
                    part_fc_list.append(torch.cat(tuple(part_fc_per_img_list)))
                part_fc_input = torch.cat((part_fc_list)).contiguous().view(roi_poolings.size(0), -1)
                roi_poolings = roi_poolings.view(roi_poolings.size(0), -1)
                all_part_fc = self.net.all_part_fc(part_fc_input)
                cls_loss = self.cls_criterion(all_part_fc, labels)
                loss = cls_loss+cam_loss+order_softmax_loss
                print(loss)
                loss.backward()
                #_, prediction = torch.max(all_part_fc, 1)

    def _accuracy(self):
        num_total = 0
        num_acc = 0
        self.net.eval()
        with torch.on_grad():
            for imgs, labels in self.test_loader:
                imgs = imgs.to(self.options['device'])
                labels = labels.to(self.options['device'])

    def _order_decoder(self, part):
        _, prediction = torch.max(part, 1)
        #part_numpy = at.tonumpy(part).squeeze()
        prediction_np = at.tonumpy(prediction).squeeze()
        part_decode = list()
        for i in range(len(prediction_np)):
            part_decode.append(at.part_dict[prediction_np[i]])
        return part_decode

    def _order_softmax_gt(self, order_decode, roi_poolings, gt_labels):
        gt_labels_np = at.tonumpy(gt_labels).astype(np.int)
        roi_poolings_np = at.tonumpy(roi_poolings)
        order_gt = np.asarray(gt_labels_np)
        print('gt_labels size(0) is {}'.format(gt_labels.size(0)))
        #print(gt_labels_np)
        for i in range(gt_labels.size(0)):
            #print(i)
            queue_empty, queue_features = self.net.queue.get_features(gt_labels_np[i])
            if queue_empty:
                print('query_empty')
                self.net.queue.put(gt_labels_np[i], roi_poolings_np[i])
                order_gt[i] = at.order_dict[order_decode[i]]
            else:
                roi_per_image = roi_poolings_np[i].reshape(4, -1)
                best_sim = 0.0
                best_order = -1
                for i_order in range(Config.n_order):
                    reorder = list(map(int, at.part_dict[i_order].split(',')))
                    roi_reorder = roi_per_image[reorder, :]     #reorder
                    sim = 0.0
                    for i_feature in range(Config.n_features):
                        queue_feature = queue_features[i_feature].reshape(4, -1)
                        for i_roi in range(4):
                            #print(roi_reorder[i_roi])
                            #print(queue_feature[i_roi])
                            sim += qt.cosine_similarity(roi_reorder[i_roi], queue_feature[i_roi])
                    if best_sim < sim:
                        best_sim = sim
                        best_order = i_order
                assert best_order != -1
                order_gt[i] = best_order
                if best_sim>0.6*4:
                    print(gt_labels_np[i])
                    print(gt_labels_np)
                    print('Execute order upgrating for class {}'.format(gt_labels_np[i]))
                    self.net.queue.put(gt_labels_np[i], roi_poolings_np[i])
        return order_gt

def _smooth_l1_loss(x, t, in_weight, sigma):
    sigma2 = sigma ** 2
    diff = in_weight * (x - t)
    abs_diff = diff.abs()
    flag = (abs_diff.data < (1. / sigma2)).float()
    y = (flag * (sigma2 / 2.) * (diff ** 2) +
         (1 - flag) * (abs_diff - 0.5 / sigma2))
    return y.sum()

def _loc_loss(pred_loc, gt_loc, gt_label, sigma):
    in_weight = torch.zeros(gt_loc.shape).cuda()
    # Localization loss is calculated only for positive rois.
    # NOTE:  unlike origin implementation,
    # we don't need inside_weight and outside_weight, they can calculate by gt_label
    in_weight[(gt_label > 0).view(-1, 1).expand_as(in_weight).cuda()] = 1
    loc_loss = _smooth_l1_loss(pred_loc, gt_loc, in_weight.detach(), sigma)
    # Normalize by total number of negtive and positive rois.
    loc_loss /= ((gt_label >= 0).sum().float()) # ignore gt_label==-1 for rpn_loss
    return loc_loss


#!/usr/bin/env python
"""
 Copyright (c) 2018 Intel Corporation

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

from __future__ import print_function
import sys
import os
from argparse import ArgumentParser
import cv2
import time
import logging as log
from openvino.inference_engine import IENetwork, IEPlugin

def build_argparser():
    parser = ArgumentParser()
    parser.add_argument("-m", "--model", help="Path to an .xml file with a trained model.", required=True, type=str)
    parser.add_argument("-i", "--input",
                        help="Path to video file or image. 'cam' for capturing video stream from camera", required=True,
                        type=str)
    parser.add_argument("-l", "--cpu_extension",
                        help="MKLDNN (CPU)-targeted custom layers.Absolute path to a shared library with the kernels "
                             "impl.", type=str, default=None)
    parser.add_argument("-pp", "--plugin_dir", help="Path to a plugin folder", type=str, default=None)
    parser.add_argument("-d", "--device",
                        help="Specify the target device to infer on; CPU, GPU, FPGA or MYRIAD is acceptable. Demo "
                             "will look for a suitable plugin for device specified (CPU by default)", default="CPU",
                        type=str)
    parser.add_argument("--labels", help="Labels mapping file", default=None, type=str)
    parser.add_argument("-pt", "--prob_threshold", help="Probability threshold for detections filtering",
                        default=0.5, type=float)

    return parser

def main():
    log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
    args = build_argparser().parse_args()
    model_xml = args.model
    model_bin = os.path.splitext(model_xml)[0] + ".bin"
    # Plugin initialization for specified device and load extensions library if specified
    log.info("Initializing plugin for {} device...".format(args.device))
    plugin = IEPlugin(device=args.device, plugin_dirs=args.plugin_dir)
    if args.cpu_extension and 'CPU' in args.device:
        plugin.add_cpu_extension(args.cpu_extension)

    # Read IR
    log.info("Reading IR...")
    net = IENetwork.from_ir(model=model_xml, weights=model_bin)

    if "CPU" in plugin.device:
        supported_layers = plugin.get_supported_layers(net)
        not_supported_layers = [l for l in net.layers.keys() if l not in supported_layers]
        if len(not_supported_layers) != 0:
            log.error("Following layers are not supported by the plugin for specified device {}:\n {}".
                      format(plugin.device, ', '.join(not_supported_layers)))
            log.error("Please try to specify cpu extensions library path in demo's command line parameters using -l "
                      "or --cpu_extension command line argument")
            sys.exit(1)
    assert len(net.inputs.keys()) == 1, "Demo supports only single input topologies"
    assert len(net.outputs) == 1, "Demo supports only single output topologies"
    input_blob = next(iter(net.inputs))
    out_blob = next(iter(net.outputs))
    log.info("Loading IR to the plugin...")
    exec_net = plugin.load(network=net, num_requests=2)
    # Read and pre-process input image
    n, c, h, w = net.inputs[input_blob]
    del net
    if args.input == 'cam':
        input_stream = 0
    else:
        input_stream = args.input
        assert os.path.isfile(args.input), "Specified input file doesn't exist"
    if args.labels:
        with open(args.labels, 'r') as f:
            labels_map = [x.strip() for x in f]
    else:
        labels_map = None

    cap = cv2.VideoCapture(input_stream)

    cur_request_id = 0
    next_request_id = 1

    log.info("Starting inference in async mode...")
    log.info("To switch between sync and async modes press Tab button")
    log.info("To stop the demo execution press Esc button")
    is_async_mode = True
    render_time = 0
    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break
        initial_w = cap.get(3)
        initial_h = cap.get(4)
        in_frame = cv2.resize(frame, (w, h))
        in_frame = in_frame.transpose((2, 0, 1))  # Change data layout from HWC to CHW
        in_frame = in_frame.reshape((n, c, h, w))

        # Main sync point:
        # in the truly Async mode we start the NEXT infer request, while waiting for the CURRENT to complete
        # in the regular mode we start the CURRENT request and immediately wait for it's completion
        inf_start = time.time()
        if is_async_mode:
            exec_net.start_async(request_id=next_request_id, inputs={input_blob: in_frame})
        else:
            exec_net.start_async(request_id=cur_request_id, inputs={input_blob: in_frame})
        if exec_net.requests[cur_request_id].wait(-1) == 0:
            inf_end = time.time()
            det_time = inf_end - inf_start

            # Parse detection results of the current request
            res = exec_net.requests[cur_request_id].outputs[out_blob]
            for obj in res[0][0]:
                # Draw only objects when probability more than specified threshold
                if obj[2] > args.prob_threshold:
                    xmin = int(obj[3] * initial_w)
                    ymin = int(obj[4] * initial_h)
                    xmax = int(obj[5] * initial_w)
                    ymax = int(obj[6] * initial_h)
                    class_id = int(obj[1])
                    # Draw box and label\class_id
                    color = (min(class_id * 12.5, 255), min(class_id * 7, 255), min(class_id * 5, 255))
                    cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), color, 2)
                    det_label = labels_map[class_id] if labels_map else str(class_id)
                    cv2.putText(frame, det_label + ' ' + str(round(obj[2] * 100, 1)) + ' %', (xmin, ymin - 7),
                                cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1)

            # Draw performance stats
            inf_time_message = "Inference time: N\A for async mode" if is_async_mode else \
                "Inference time: {:.3f} ms".format(det_time * 1000)
            render_time_message = "OpenCV rendering time: {:.3f} ms".format(render_time * 1000)
            async_mode_message = "Async mode is on. Processing request {}".format(cur_request_id) if is_async_mode else \
                "Async mode is off. Processing request {}".format(cur_request_id)

            cv2.putText(frame, inf_time_message, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1)
            cv2.putText(frame, render_time_message, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
            cv2.putText(frame, async_mode_message, (10, int(initial_h - 20)), cv2.FONT_HERSHEY_COMPLEX, 0.5,
                        (10, 10, 200), 1)

        #
        render_start = time.time()
        cv2.imshow("Detection Results", frame)
        render_end = time.time()
        render_time = render_end - render_start

        key = cv2.waitKey(1)
        if key == 27:
            break
        if (9 == key):
            is_async_mode = not is_async_mode
            log.info("Switched to {} mode".format("async" if is_async_mode else "sync"))

        if is_async_mode:
            cur_request_id, next_request_id = next_request_id, cur_request_id

    cv2.destroyAllWindows()
    del exec_net
    del plugin

if __name__ == '__main__':
    sys.exit(main() or 0)


import datetime
import time
import curses
import ComPort
import PrintMessage
import CountErrors

PORTNAME = "COM11"
THRESHOLD_ERRORS = 128

COM_WDT = "FFDA8000"
COM_UNRESET_DEVICE = "FFDA6000"
COM_TIMEOUT_SPI = "FFDAB000"
COM_STM_START = "FFDAF000"
COM_BUFFER_FILL = "FFDAF100"
COM_MACHINE = "FFDAD000"

SH_MEM_0 = "F0DA1000"
SH_MEM_1 = "F0DA1001"
SH_START = "F0DA2000"
SH_UART = "F0DA3000"
SH_ALU = "F0DA4000"

print_mes = PrintMessage.PrintMessage()
com_port = ComPort.ComPort(portname=PORTNAME)
errors = CountErrors.CountErrors()

win = curses.initscr()
win.keypad(1)
curses.noecho()
curses.curs_set(0)
win.border(0)
win.nodelay(1)
win.addstr(3, 2, "Errors:")
win.addstr(8, 2, "Events:")
win.addstr(26, 2, " -- To Exit, press ESC -- ")

key = 0
number_errors = 0
count_errors = 0
count_package = 0
error_name = ""

f_number_errors = False
f_errors = False
f_number_errors_alu = False

def flags_reset():
    global number_errors
    global count_errors
    global error_name
    global f_number_errors
    global f_errors
    global f_number_errors_alu

    number_errors = 0
    count_errors = 0
    error_name = ""
    f_number_errors = False
    f_errors = False
    f_number_errors_alu = False

start_time = time.time()
try:
    while key != 27:
        key = win.getch()

        win.addstr(1, 2, datetime.datetime.now().strftime("%d.%m.%Y %H:%M:%S"))
        win.addstr(1, 24, "{0:.0f} seconds".format(time.time() - start_time))

        i = 4
        for name, value in errors.errors_count.items():
            win.addstr(i, 4, "{0:<8s} {1:<4d}".format(name, value))
            i += 1

        i = 9
        for name, value in errors.events_count.items():
            win.addstr(i, 4, "{0:<15s} {1:<4d}".format(name, value))
            i += 1

        win.addstr(19, 2, "{0:<17s} {1:<4d}".format("Packages", count_package))

        data = com_port.read()

        # print(data)

        if data == SH_START:
            print_mes.info(data, "Start CPUi6")
            errors.event_inc(errors.SH_START)
            flags_reset()

        elif data == COM_STM_START:
            print_mes.info(data, "Start STM32")
            errors.event_inc(errors.STM_START)

        elif data == COM_BUFFER_FILL:
            print_mes.error(data, "Buffer STM32 fill")
            errors.event_inc(errors.BUFFER_FILL)
            flags_reset()

        elif data == COM_WDT:
            print_mes.error(data, "WDT is worked")
            errors.event_inc(errors.WDT)
            flags_reset()

        elif data == COM_TIMEOUT_SPI:
            print_mes.error(data, "Timeout SPI")
            errors.event_inc(errors.TIMEOUT_SPI)
            flags_reset()

        elif data == COM_UNRESET_DEVICE:
            print_mes.error(data, "Unreset CPUi6")
            errors.event_inc(errors.UNRESET_DEVICE)
            flags_reset()

        elif data == COM_MACHINE:
            print_mes.error(data, "Machine Error")
            errors.event_inc(errors.MACHINE)
            flags_reset()

        elif data == SH_MEM_0:
            print_mes.info(data, "Memory 0")
            f_number_errors = True
            error_name = errors.MEMORY

        elif data == SH_MEM_1:
            print_mes.info(data, "Memory 1")
            f_number_errors = True
            error_name = errors.MEMORY

        elif data == SH_UART:
            print_mes.info(data, "Uart")
            f_number_errors = True
            error_name = errors.UART

        elif data == SH_ALU:
            print_mes.info(data, "Alu")
            f_number_errors = True
            f_number_errors_alu = True
            error_name = errors.ALU

        elif f_number_errors is True:
            f_number_errors = False
            count_package += 1
            errors.error_inc(error_name, int(data, 16))
            if int(data, 16) == 0:
                print_mes.info(data, "Number errors: 0")
            else:
                print_mes.error(data, "Number errors: {0:d}".format(int(data, 16)))
                if f_number_errors_alu is False:
                    number_errors = int(data, 16) * 2 if int(data, 16) < THRESHOLD_ERRORS else THRESHOLD_ERRORS * 2
                else:
                    number_errors = 1
                f_errors = True
            f_number_errors_alu = False

        elif f_errors is True:
            print_mes.error(data, "Error")
            count_errors += 1
            if count_errors == number_errors:
                count_errors = 0
                f_errors = False

        else:
            print_mes.error(data, "Invalid OPCODE")
            errors.event_inc(errors.INVALID_OPCODE)

finally:
    curses.endwin()


from __future__ import absolute_import, division, print_function

import sys
import os
import argparse
import json
import time
import pdb
import logging

import torch
import torch.quantization
import numpy as np

from tqdm import tqdm
from model import TextGloveGNB, TextGloveCNN, TextGloveDensenetCNN, TextGloveDensenetDSA, TextBertCNN, TextBertCLS
from util import load_checkpoint, load_config, load_label, to_device, to_numpy
from dataset import prepare_dataset, GloveDataset, BertDataset
from sklearn.metrics import classification_report, confusion_matrix

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def set_path(config):
    opt = config['opt']
    if config['emb_class'] == 'glove':
        opt.data_path = os.path.join(opt.data_dir, 'test.txt.ids')
    else:
        if opt.augmented:
            opt.data_path = os.path.join(opt.data_dir, 'augmented.raw.fs')
        else:
            opt.data_path = os.path.join(opt.data_dir, 'test.txt.fs')
    opt.embedding_path = os.path.join(opt.data_dir, 'embedding.npy')
    opt.label_path = os.path.join(opt.data_dir, 'label.txt')
    if opt.augmented:
        opt.test_path = os.path.join(opt.data_dir, 'augmented.raw')
    else:
        opt.test_path = os.path.join(opt.data_dir, 'test.txt')
    opt.vocab_path = os.path.join(opt.data_dir, 'vocab.txt')

def load_model(config, checkpoint):
    opt = config['opt']
    labels = load_label(opt.label_path)
    label_size = len(labels)
    config['labels'] = labels
    if config['emb_class'] == 'glove':
        if config['enc_class'] == 'gnb':
            model = TextGloveGNB(config, opt.embedding_path, label_size)
        if config['enc_class'] == 'cnn':
            model = TextGloveCNN(config, opt.embedding_path, label_size, emb_non_trainable=True)
        if config['enc_class'] == 'densenet-cnn':
            model = TextGloveDensenetCNN(config, opt.embedding_path, label_size, emb_non_trainable=True)
        if config['enc_class'] == 'densenet-dsa':
            model = TextGloveDensenetDSA(config, opt.embedding_path, label_size, emb_non_trainable=True)
    else:
        from transformers import AutoTokenizer, AutoConfig, AutoModel
        bert_config = AutoConfig.from_pretrained(opt.bert_output_dir)
        bert_tokenizer = AutoTokenizer.from_pretrained(opt.bert_output_dir)
        bert_model = AutoModel.from_config(bert_config)
        ModelClass = TextBertCNN
        if config['enc_class'] == 'cls': ModelClass = TextBertCLS
        model = ModelClass(config, bert_config, bert_model, bert_tokenizer, label_size)
    if opt.enable_qat:
        assert opt.device == 'cpu'
        model.qconfig = torch.quantization.get_default_qat_qconfig('fbgemm')
        '''
        # fuse if applicable
        # model = torch.quantization.fuse_modules(model, [['']])
        '''
        model = torch.quantization.prepare_qat(model)
        model.eval()
        model.to('cpu')
        logger.info("[Convert to quantized model with device=cpu]")
        model = torch.quantization.convert(model)
    if opt.enable_qat_fx:
        import torch.quantization.quantize_fx as quantize_fx
        qconfig_dict = {"": torch.quantization.get_default_qat_qconfig('fbgemm')}
        model = quantize_fx.prepare_qat_fx(model, qconfig_dict)
        logger.info("[Convert to quantized model]")
        model = quantize_fx.convert_fx(model)

    model.load_state_dict(checkpoint)
    model = model.to(opt.device)
    '''
    for name, param in model.named_parameters():
        print(name, param.data, param.device, param.requires_grad)
    '''
    logger.info("[model] :\n{}".format(model.__str__()))
    logger.info("[Model loaded]")
    return model

def convert_onnx(config, torch_model, x):
    opt = config['opt']
    import torch.onnx

    if config['emb_class'] == 'glove':
        input_names  = ['input']
        output_names = ['output']
        dynamic_axes = {'input': {0: 'batch', 1: 'sequence'},
                        'output': {0: 'batch', 1: 'sequence'}}
    else:
        input_names  = ['input_ids', 'input_mask', 'segment_ids']
        output_names = ['output']
        dynamic_axes = {'input_ids': {0: 'batch', 1: 'sequence'},
                        'input_mask': {0: 'batch', 1: 'sequence'},
                        'segment_ids': {0: 'batch', 1: 'sequence'},
                        'output': {0: 'batch'}}

    with torch.no_grad():
        torch.onnx.export(torch_model,                  # model being run
                          x,                            # model input (or a tuple for multiple inputs)
                          opt.onnx_path,                # where to save the model (can be a file or file-like object)
                          export_params=True,           # store the trained parameter weights inside the model file
                          opset_version=opt.onnx_opset, # the ONNX version to export the model to
                          do_constant_folding=True,     # whether to execute constant folding for optimization
                          verbose=True,
                          input_names=input_names,      # the model's input names
                          output_names=output_names,    # the model's output names
                          dynamic_axes=dynamic_axes)    # variable length axes

# ------------------------------------------------------------------------------ #
# source code from https://github.com/huggingface/transformers/blob/master/src/transformers/convert_graph_to_onnx.py#L374
# ------------------------------------------------------------------------------ #
def quantize_onnx(onnx_path, quantized_onnx_path):
    """
    Quantize the weights of the model from float32 to in8 to allow very efficient inference on modern CPU.
    """
    import onnx
    from onnxruntime.quantization import QuantizationMode, quantize

    onnx_model = onnx.load(onnx_path)

    quantized_model = quantize(
        model=onnx_model,
        quantization_mode=QuantizationMode.IntegerOps,
        force_fusions=True,
        symmetric_weight=True,
    )

    # Save model
    onnx.save_model(quantized_model, quantized_onnx_path)

def check_onnx(config):
    opt = config['opt']
    import onnx
    onnx_model = onnx.load(opt.onnx_path)
    onnx.checker.check_model(onnx_model)
    print(onnx.helper.printable_graph(onnx_model.graph))

def convert_tvm(config, torch_model, x):
    opt = config['opt']
    import onnx
    import tvm
    from tvm import relay

    onnx_model = onnx.load(opt.onnx_path)
    logger.info("[ONNX model loaded]")
    batch_size = x[0].shape[0]
    seq_len = x[0].shape[1]
    shape_dict = {'input_ids': (batch_size, seq_len),
                  'input_mask': (batch_size, seq_len),
                  'segment_ids': (batch_size, seq_len), }
    model, params = relay.frontend.from_onnx(onnx_model, shape_dict, opset=opt.onnx_opset)
    logger.info("[Converting to TVM done]")

    with open(os.path.join(opt.tvm_dir, 'model.json'), 'w') as fo:
        fo.write(tvm.ir.save_json(model))
    with open(os.path.join(opt.tvm_dir, 'model.params'), 'wb') as fo:
        fo.write(relay.save_param_dict(params))

# ---------------------------------------------------------------------------- #
# Evaluation
# ---------------------------------------------------------------------------- #

def write_prediction(opt, preds, labels):
    # load test data
    tot_num_line = sum(1 for _ in open(opt.test_path, 'r'))
    with open(opt.test_path, 'r', encoding='utf-8') as f:
        data = []
        bucket = []
        for idx, line in enumerate(tqdm(f, total=tot_num_line)):
            line = line.strip()
            sent, label = line.split('\t')
            data.append((sent, label))
    # write prediction
    try:
        pred_path = opt.test_path + '.pred'
        with open(pred_path, 'w', encoding='utf-8') as f:
            for entry, pred in zip(data, preds):
                sent, label = entry
                if opt.augmented:
                    # print logits as label
                    logits = ['%.6f' % p for p in pred]
                    f.write(sent + '\t' + ' '.join(logits) + '\n')
                else:
                    pred_id = np.argmax(pred)
                    pred_label = labels[pred_id]
                    f.write(sent + '\t' + label + '\t' + pred_label + '\n')
    except Exception as e:
        logger.warn(str(e))

def prepare_datasets(config):
    opt = config['opt']
    if config['emb_class'] == 'glove':
        DatasetClass = GloveDataset
    else:
        DatasetClass = BertDataset
    test_loader = prepare_dataset(config, opt.data_path, DatasetClass, sampling=False, num_workers=1)
    return test_loader

def evaluate(opt):
    # set config
    config = load_config(opt)
    if opt.num_threads > 0: torch.set_num_threads(opt.num_threads)
    config['opt'] = opt
    logger.info("%s", config)

    # set path
    set_path(config)

    # prepare test dataset
    test_loader = prepare_datasets(config)

    # load pytorch model checkpoint
    checkpoint = load_checkpoint(opt.model_path, device=opt.device)

    # prepare model and load parameters
    model = load_model(config, checkpoint)
    model.eval()

    # convert to onnx
    if opt.convert_onnx:
        (x, y) = next(iter(test_loader))
        x = to_device(x, opt.device)
        y = to_device(y, opt.device)
        convert_onnx(config, model, x)
        check_onnx(config)
        logger.info("[ONNX model saved] :{}".format(opt.onnx_path))
        # quantize onnx
        if opt.quantize_onnx:
            quantize_onnx(opt.onnx_path, opt.quantized_onnx_path)
            logger.info("[Quantized ONNX model saved] : {}".format(opt.quantized_onnx_path))
        return

    # load onnx model for using onnxruntime
    if opt.enable_ort:
        import onnxruntime as ort
        sess_options = ort.SessionOptions()
        sess_options.inter_op_num_threads = opt.num_threads
        sess_options.intra_op_num_threads = opt.num_threads
        ort_session = ort.InferenceSession(opt.onnx_path, sess_options=sess_options)

    # convert to tvm format
    if opt.convert_tvm:
        (x, y) = next(iter(test_loader))
        x = to_device(x, opt.device)
        y = to_device(y, opt.device)
        convert_tvm(config, model, x)
        logger.info("[TVM model saved] : {}".format(opt.tvm_path))
        return

    # enable to use dynamic quantized model (pytorch>=1.3.0)
    if opt.enable_dqm and opt.device == 'cpu':
        model = torch.quantization.quantize_dynamic(model, {torch.nn.Linear}, dtype=torch.qint8)
        print(model)

    # evaluation
    preds = None
    ys    = None
    correct = 0
    n_batches = len(test_loader)
    total_examples = 0
    whole_st_time = time.time()
    first_time = time.time()
    first_examples = 0
    total_duration_time = 0.0
    with torch.no_grad():
        for i, (x,y) in enumerate(tqdm(test_loader, total=n_batches)):
            start_time = time.time()
            x = to_device(x, opt.device)
            y = to_device(y, opt.device)

            if opt.enable_ort:
                x = to_numpy(x)
                if config['emb_class'] == 'glove':
                    ort_inputs = {ort_session.get_inputs()[0].name: x}
                else:
                    if config['emb_class'] in ['roberta', 'distilbert', 'bart']:
                        ort_inputs = {ort_session.get_inputs()[0].name: x[0],
                                      ort_session.get_inputs()[1].name: x[1]}
                    else:
                        ort_inputs = {ort_session.get_inputs()[0].name: x[0],
                                      ort_session.get_inputs()[1].name: x[1],
                                      ort_session.get_inputs()[2].name: x[2]}
                logits = ort_session.run(None, ort_inputs)[0]
                logits = to_device(torch.tensor(logits), opt.device)
            else:
                logits = model(x)

            if preds is None:
                preds = to_numpy(logits)
                ys = to_numpy(y)
            else:
                preds = np.append(preds, to_numpy(logits), axis=0)
                ys = np.append(ys, to_numpy(y), axis=0)
            predicted = logits.argmax(1)
            correct += (predicted == y).sum().item()
            cur_examples = y.size(0)
            total_examples += cur_examples
            if i == 0: # first one may take longer time, so ignore in computing duration.
                first_time = float((time.time()-first_time)*1000)
                first_examples = cur_examples
            if opt.num_examples != 0 and total_examples >= opt.num_examples:
                logger.info("[Stop Evaluation] : up to the {} examples".format(total_examples))
                break
            duration_time = float((time.time()-start_time)*1000)
            if i != 0: total_duration_time += duration_time
            '''
            logger.info("[Elapsed Time] : {}ms".format(duration_time))
            '''
    # generate report
    labels = config['labels']
    label_names = [v for k, v in sorted(labels.items(), key=lambda x: x[0])]
    preds_ids = np.argmax(preds, axis=1)
    try:
        print(classification_report(ys, preds_ids, target_names=label_names, digits=4))
        print(labels)
        print(confusion_matrix(ys, preds_ids))
    except Exception as e:
        logger.warn(str(e))

    acc  = correct / total_examples
    whole_time = float((time.time()-whole_st_time)*1000)
    avg_time = (whole_time - first_time) / (total_examples - first_examples)
    # write predictions to file
    write_prediction(opt, preds, labels)
    logger.info("[Accuracy] : {:.4f}, {:5d}/{:5d}".format(acc, correct, total_examples))
    logger.info("[Elapsed Time] : {}ms, {}ms on average".format(whole_time, avg_time))
    logger.info("[Elapsed Time(total_duration_time, average)] : {}ms, {}ms".format(total_duration_time, total_duration_time/(total_examples-1)))

# ---------------------------------------------------------------------------- #
# Inference
# ---------------------------------------------------------------------------- #

def load_vocab(vocab_path):
    with open(vocab_path, 'r', encoding='utf-8') as f:
        vocab = {}
        for idx, line in enumerate(f):
            tokens = line.split()
            word = tokens[0]
            word_id = int(tokens[1])
            vocab[word] = word_id
        return vocab

def prepare_tokenizer(config, model):
    from tokenizer import Tokenizer
    opt = config['opt']
    if config['emb_class'] == 'glove':
        vocab = load_vocab(opt.vocab_path)
        tokenizer = Tokenizer(vocab, config)
    else:
        tokenizer = model.bert_tokenizer
    return tokenizer

def encode_text(config, tokenizer, text):
    if config['emb_class'] == 'glove':
        tokens = text.split()
        # kernel size can't be greater than actual input size,
        # we should pad the sequence up to the maximum kernel size + 1.
        min_seq_size = 10
        ids = tokenizer.convert_tokens_to_ids(tokens, pad_sequence=False, min_seq_size=min_seq_size)
        x = torch.tensor([ids])
        # x : [batch_size, variable size]
        # batch size: 1
    else:
        inputs = tokenizer.encode_plus(text, add_special_tokens=True, return_tensors='pt')
        if config['emb_class'] in ['roberta', 'bart', 'distilbert']:
            x = [inputs['input_ids'], inputs['attention_mask']]
            # x[0], x[1] : [batch_size, variable size]
        else:
            x = [inputs['input_ids'], inputs['attention_mask'], inputs['token_type_ids']]
            # x[0], x[1], x[2] : [batch_size, variable size]
        # batch size: 1
    return x

def inference(opt):
    # set config
    config = load_config(opt)
    if opt.num_threads > 0: torch.set_num_threads(opt.num_threads)
    config['opt'] = opt

    # set path: opt.embedding_path, opt.vocab_path, opt.label_path
    set_path(config)

    # load pytorch model checkpoint
    checkpoint = load_checkpoint(opt.model_path, device=opt.device)

    # prepare model and load parameters
    model = load_model(config, checkpoint)
    model.eval()

    # load onnx model for using onnxruntime
    if opt.enable_ort:
        import onnxruntime as ort
        sess_options = ort.SessionOptions()
        sess_options.inter_op_num_threads = opt.num_threads
        sess_options.intra_op_num_threads = opt.num_threads
        ort_session = ort.InferenceSession(opt.onnx_path, sess_options=sess_options)

    # enable to use dynamic quantized model (pytorch>=1.3.0)
    if opt.enable_dqm and opt.device == 'cpu':
        model = torch.quantization.quantize_dynamic(model, {torch.nn.Linear}, dtype=torch.qint8)
        print(model)

    # prepare tokenizer
    tokenizer = prepare_tokenizer(config, model)

    # prepare labels
    labels = config['labels']

    # inference
    f_out = open(opt.test_path + '.inference', 'w', encoding='utf-8')
    total_examples = 0
    total_duration_time = 0.0
    with torch.no_grad(), open(opt.test_path, 'r', encoding='utf-8') as f:
        for i, line in enumerate(f):
            start_time = time.time()
            sent, label = line.strip().split('\t')
            x_raw = sent.split()
            y_raw = label
            text = ' '.join(x_raw)
            x = encode_text(config, tokenizer, text)
            x = to_device(x, opt.device)

            if opt.enable_ort:
                x = to_numpy(x)
                if config['emb_class'] == 'glove':
                    ort_inputs = {ort_session.get_inputs()[0].name: x}
                else:
                    if config['emb_class'] in ['roberta', 'distilbert', 'bart']:
                        ort_inputs = {ort_session.get_inputs()[0].name: x[0],
                                      ort_session.get_inputs()[1].name: x[1]}
                    else:
                        ort_inputs = {ort_session.get_inputs()[0].name: x[0],
                                      ort_session.get_inputs()[1].name: x[1],
                                      ort_session.get_inputs()[2].name: x[2]}
                logits = ort_session.run(None, ort_inputs)[0]
                logits = to_device(torch.tensor(logits), opt.device)
            else:
                logits = model(x)

            predicted = logits.argmax(1)
            predicted = to_numpy(predicted)[0]
            predicted_raw = labels[predicted]
            f_out.write(text + '\t' + y_raw + '\t' + predicted_raw + '\n')
            total_examples += 1
            if opt.num_examples != 0 and total_examples >= opt.num_examples:
                logger.info("[Stop Inference] : up to the {} examples".format(total_examples))
                break
            duration_time = float((time.time()-start_time)*1000)
            if i != 0: total_duration_time += duration_time
            logger.info("[Elapsed Time] : {}ms".format(duration_time))
    f_out.close()
    logger.info("[Elapsed Time(total_duration_time, average)] : {}ms, {}ms".format(total_duration_time, total_duration_time/(total_examples-1)))

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument('--config', type=str, default='configs/config-glove-cnn.json')
    parser.add_argument('--data_dir', type=str, default='data/snips')
    parser.add_argument('--model_path', type=str, default='pytorch-model.pt')
    parser.add_argument('--device', type=str, default='cuda')
    parser.add_argument('--num_threads', type=int, default=0)
    parser.add_argument('--batch_size', type=int, default=1)
    parser.add_argument('--num_examples', default=0, type=int, help="Number of examples to evaluate, 0 means all of them.")
    # for Augmentation
    parser.add_argument('--augmented', action='store_true',
                        help="Set this flag to generate augmented.raw.inference(augmented.txt) for training.")
    # for BERT
    parser.add_argument('--bert_output_dir', type=str, default='bert-checkpoint',
                        help="The checkpoint directory of fine-tuned BERT model.")
    # for ONNX
    parser.add_argument('--convert_onnx', action='store_true',
                        help="Set this flag to convert to ONNX.")
    parser.add_argument('--enable_ort', action='store_true',
                        help="Set this flag to evaluate using ONNXRuntime.")
    parser.add_argument('--onnx_path', type=str, default='pytorch-model.onnx')
    parser.add_argument('--onnx_opset', default=11, type=int, help="ONNX opset version.")
    parser.add_argument('--quantize_onnx', action='store_true',
                        help="Set this flag to quantize ONNX.")
    parser.add_argument('--quantized_onnx_path', type=str, default='pytorch-model.onnx-quantized')
    # for TVM
    parser.add_argument('--convert_tvm', action='store_true',
                        help="Set this flag to convert ONNX to TVM.")
    parser.add_argument('--enable_tvm', action='store_true',
                        help="Set this flag to evaluate using TVM.(not implemented)")
    parser.add_argument('--tvm_dir', type=str, default='tvm-model')
    # for Dynamic Quantization
    parser.add_argument('--enable_dqm', action='store_true',
                        help="Set this flag to use dynamic quantized model.")
    # for Inference
    parser.add_argument('--enable_inference', action='store_true',
                        help="Set this flag to inference for raw input text.")
    # for QAT
    parser.add_argument('--enable_qat', action='store_true',
                        help="Set this flag to use the model by quantization aware training.")
    parser.add_argument('--enable_qat_fx', action='store_true',
                        help="Set this flag for quantization aware training using fx graph mode.")

    opt = parser.parse_args()

    if opt.enable_inference:
        inference(opt)
    else:
        evaluate(opt)

if __name__ == '__main__':
    main()


import numpy as np
import pandas as pd

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.impute import SimpleImputer

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.compose import ColumnTransformer

def process_origin_col(df):
    df['Origin']=df['Origin'].map({1: "India", 2: "USA", 3: "Germany"})
    return df

cyl,hp,acc=0,2,4

class CustomAttrAdder(BaseEstimator, TransformerMixin):
    def __init__(self, acc_on_power=True):
        self.acc_on_power = acc_on_power
    def fit(self, X, y=None):
        return self  # nothing else to do
    def transform(self, X):
        acc_on_cyl = X[:, acc] / X[:, cyl]
        if self.acc_on_power:
            acc_on_power = X[:, acc] / X[:, hp]
            return np.c_[X, acc_on_power, acc_on_cyl]

        return np.c_[X, acc_on_cyl]

def num_pipeline_transformer(data):
    numeric=['int64','float64']
    num_attrs=data.select_dtypes(include=numeric)
    num_pipeline=Pipeline([('imputer',SimpleImputer(strategy='median')),
                       ('attr_adder',CustomAttrAdder())])
    return num_attrs,num_pipeline

def full_pipeline_transformer(data):
    cat_attrs=['Origin']
    # access num_pipeline by calling the respective function
    num_attrs,num_pipeline=num_pipeline_transformer(data)
    full_pipeline=ColumnTransformer([('num',num_pipeline,list(num_attrs))])
    full_pipeline.fit_transform(data)
    return full_pipeline

#Standard scaler
def standard_scaler(data):
    df=pd.read_csv('auto_pre_scaler_num_data.csv')
    scaler=StandardScaler()
    scaler.fit(df)
    scaled_data=scaler.transform(data)
    return scaled_data

## One hot encoder
def onehot_encoder(df):
    data=df['Origin'][0]
    if data=='Germany':
        return np.array([1., 0., 0.])
    elif data=='India':
        return np.array([[0., 1., 0.]])
    else:
        return np.array([[0., 0., 1.]])

def predict_mpg(config,model):
    if type(config)==dict:
        df=pd.DataFrame(config)
    else:
        df=config

    preproc_df=process_origin_col(df)

    pipeline=full_pipeline_transformer(df)
    #returns numerical data after imputation and add_attr
    pre_scaled_num_data=pipeline.transform(preproc_df)

    # give this data to standard scaler ...returns scaled data
    scaled_data=standard_scaler(pre_scaled_num_data)

    ## call one hot encoder for categorical feature encoding
    cat_data=onehot_encoder(df)

    ## now combine the numercal scaled data and categorical encoded data
    combined_data=np.append(scaled_data,cat_data)

    ## reshape combine data from 1d array to ndarray
    prepared_data=combined_data.reshape(1,-1)

    #predict output
    y_pred=model.predict(prepared_data)

    return y_pred