from flask_sqlalchemy import SQLAlchemy
from flask import Flask,url_for,redirect,render_template,request
import os
import webbrowser
import bs4 as bs
import requests

#os.system('database.py')

def log(x):
    filee=open("log.dat","w")
    filee.writeline(x)
    filee.close()

global results
results=[]

#flasking starts here
app=Flask(__name__)
app.config["SQLALCHEMY_DATABASE_URI"] = 'sqlite:///./test.db'
app.config["secret_key"]="very_secret"
app.config['TESTING'] = True
db=SQLAlchemy(app)

class accessdb:
    def __init__(self):
        pass

    def writeindb(self,name,link):
            try:
                link=linkholder(name=name,link=link)
                db.session.add(link)
                db.session.commit()
                return True
            except:
                return False

    def readindb(self):
        try:
            link=linkholder(name=None,link=None)
            links=link.query.all()
            return links
        except:
            return False

class linkholder(db.Model):
    idd = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(80), unique=True, nullable=False)
    link = db.Column(db.String(250), unique=True, nullable=False)

    def __repr__(self):
        return "User \n name "+self.name+" link "+self.link

access=accessdb()

def checkindb(query):

    if "--ignore-database" in query:

        return None

    try:
        links=access.readindb()
        query=query.lower()

        for linked in links:
            linkofsite=linked.link
            nameofsite=linked.name
            if (nameofsite.lower()==query):
                print(linkofsite)
                return linkofsite
            else:
                pass
    except:
        return None

def getlink(query):
    link_predict=checkindb(query)

    if link_predict:
        print("Using database")
        print(link_predict)
        result={}
        result["query"]=query
        result["link"]=link_predict
        results.insert(0,result)
    else:

        try:
            result={}
            q=query
            headers={}
            q=q.replace(" ","+")
            headers['User-Agent'] = 'Mozilla/5.0 (X11; Li5nux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17'
            url="https://www.google.com.np/search?q="+q
            sauce=requests.get(url,headers=headers)
            sauce=sauce.content
            soup=bs.BeautifulSoup(sauce,'html.parser')
            links=soup.find_all('a')
            a=0
            for j in links:
                a+=1
                if a>20:
                    link = (str(j.get('href')))
                    if (link!=None and len(link)>20 and link.startswith("http")):

                        if "--ignore-database" in query:
                            query=query.replace("--ignore-database","")
                        else:
                            access.writeindb(query,link)

                        result["query"]=query
                        result["link"]=link
                        break

            results.insert(0,result)
        except Exception as e:
            print (e)

@app.route("/database")
def database():
    links=access.readindb()
    return render_template("database.html",links=links)

@app.route("/",methods=["POST","GET"])
@app.route("/homepage",methods=["POST","GET"])
def home():

    if request.method=="GET":
        return render_template("home.html",results=results)

    else:
        if request.method=="POST":
            query=request.form["comm"]
            getlink(query)
            return redirect(url_for("home"))

if __name__=='__main__':
    webbrowser.open("http://127.0.0.1:5000/")
    app.run(debug=True,port=5000)


# -*- coding: utf-8 -*-

"""
*------------------------- PageViewVectorField.py ----------------------------*
显示矢量场数据的页面。

部件内容：
    - WidgetPlotVector 用于使用 matplotlib 显示矢量场及背景
    - WidgetPlotBackground 用于使用 matplotlib 显示全域背景

提升部件
    - 提升类名 PageViewVectorField
    - 头文件 bin.Widgets.PageViewVectorField

作者:           胡一鸣
创建日期:       2022年5月18日

The GUI page to view Vector Field dataset.

Contents:
    - WidgetPlotVector, to view the vector fields and backgroud by matplotlib
    - WidgetPlotBackground, to view the full-scale background by matplotlib

Promoted Widget:
    - name of widget class: PageViewVectorField
    - header file: bin.Widgets.PageViewVectorField

author:         Hu Yiming
date:           Mar 30, 2022
*------------------------- PageViewVectorField.py ----------------------------*
"""

from logging import Logger
import os
from PySide6.QtWidgets import QWidget
from PySide6.QtWidgets import QMessageBox
from PySide6.QtWidgets import QInputDialog
from PySide6.QtWidgets import QDialog
# from PySide6.QtWidgets import QMenu
# from PySide6.QtCore import QPoint
from matplotlib.backend_bases import MouseEvent
from matplotlib.backends.backend_qtagg import (
    FigureCanvasQTAgg as FigureCanvas)
from matplotlib.figure import Figure
from matplotlib.axes import Axes
from matplotlib.image import AxesImage
from matplotlib.quiver import Quiver
from matplotlib.patches import Rectangle
from matplotlib.colorbar import Colorbar, make_axes
import h5py
import numpy as np

from bin.BlitManager import BlitManager
from bin.HDFManager import HDFDataNode, HDFHandler
from bin.TaskManager import TaskManager
from bin.Widgets.DialogChooseItem import DialogHDFChoose
from bin.Widgets.PageVirtualImage import DialogSaveImage
from lib.TaskVectorFieldProcess import TaskCurl, TaskFlipVectorField
from lib.TaskVectorFieldProcess import TaskDivergence
from lib.TaskVectorFieldProcess import TaskPotential
from lib.TaskVectorFieldProcess import TaskRotateVectorAngle
from lib.TaskVectorFieldProcess import TaskSliceI
from lib.TaskVectorFieldProcess import TaskSliceJ
from lib.TaskVectorFieldProcess import TaskSubtractVectorOffset
from ui import uiDialogCreateImage
from ui import uiPageViewVectorField
from ui import uiDialogAdjustQuiverEffect
from ui import uiDialogVectorProcessing

class PageViewVectorField(QWidget):
    """
    显示二维矢量场的部件类。使用 Quiver。

    Ui 文件地址：ROOT_PATH/ui/uiPageVieweVectorField.ui

    Widget to view vector fields by quiver plot.

    The path of the ui file: ROOT_PATH/ui/uiPageViewVectorField.ui
    """
    def __init__(self, parent: QWidget = None):
        super().__init__(parent)
        self.ui = uiPageViewVectorField.Ui_Form()
        self.ui.setupUi(self)

        self._data_path = ''
        self._background_path = ''
        self._image_ax = None
        self._colorbar_ax = None
        self._background_ax = None
        self._quiver_object = None
        self._colorbar_object = None
        self._background_object = None
        self._image_object = None
        self._image_max = 0
        self._image_min = 0

        self.ui.lineEdit_vector_path.setReadOnly(True)
        self.ui.lineEdit_background_path.setReadOnly(True)

        self._initUi()
        self._createAxes()

    @property
    def hdf_handler(self) -> HDFHandler:
        global qApp
        return qApp.hdf_handler

    @property
    def data_object(self) -> h5py.Dataset:
        return self.hdf_handler.file[self._data_path]

    @property
    def data_path(self) -> str:
        return self._data_path

    @property
    def logger(self) -> Logger:
        global qApp
        return qApp.logger

    @property
    def image_canvas(self) -> FigureCanvas:
        return self.ui.widget_quiver.canvas

    @property
    def background_canvas(self) -> FigureCanvas:
        return self.ui.widget_background.canvas

    @property
    def image_figure(self) -> Figure:
        return self.ui.widget_quiver.figure

    @property
    def background_figure(self) -> Figure:
        return self.ui.widget_background.figure

    @property
    def image_ax(self) -> Axes:
        return self._image_ax

    @property
    def background_ax(self) -> Axes:
        return self._background_ax

    @property
    def colorbar_ax(self) -> Axes:
        return self._colorbar_ax

    @property
    def image_object(self) -> AxesImage:
        return self._image_object

    @property
    def quiver_object(self) -> Quiver:
        return self._quiver_object

    @property
    def colorbar_object(self) -> Colorbar:
        return self._colorbar_object

    @property
    def background_object(self) -> AxesImage:
        return self._background_object

    @property
    def background_path(self) -> str:
        return self._background_path

    @property
    def image_blit_manager(self) -> BlitManager:
        return self.ui.widget_quiver.blit_manager

    @property
    def background_blit_manager(self) -> BlitManager:
        return self.ui.widget_background.blit_manager

    @property
    def background_visible(self) -> bool:
        return self.ui.checkBox_background_visible.isChecked()

    @property
    def task_manager(self) -> TaskManager:
        global qApp
        return qApp.task_manager

    def setVectorField(self, data_path: str):
        """
        Set the data path in HDF5 file, to show the vector field.

        Will set the data_path attribute. The vector field must be a 3D matrix.

        arguments:
            data_path: (str) the path of the vector field data.

        raises:
            TypeError, KeyError, ValueError
        """
        if not isinstance(data_path, str):
            raise TypeError('data_path must be a str, not '
                '{0}'.format(type(data_path).__name__))

        img_node = self.hdf_handler.getNode(data_path)
        # May raise KeyError if the path does not exist
        if not isinstance(img_node, HDFDataNode):
            raise ValueError('Item {0} must be a Dataset'.format(data_path))

        data_obj = self.hdf_handler.file[data_path]
        if not len(data_obj.shape) == 3:
            raise ValueError('Data must be a 3D matrix (2, i, j)')

        self._data_path = data_path
        self.ui.lineEdit_vector_path.setText(self.data_path)
        self.setWindowTitle('{0} - Vector'.format(img_node.name))

        # We must first render the background image, in order not to
        # cover the vector field.
        if 'background_path' in self.data_object.attrs:
            background_path = self.data_object.attrs['background_path']
            try:
                self.setBackground(background_path)
            except (KeyError, ValueError, TypeError):
                new_background_path = self._createNewBackground()
                self.setBackground(new_background_path)
        else:
            new_background_path = self._createNewBackground()
            self.setBackground(new_background_path)

        self._createQuiver()

        self.image_canvas.draw()
        self.image_canvas.flush_events()

        self.ui.widget_quiver.setProcessingActionItemPath(self.data_path)

    def _createAxes(self):
        """
        Create the axes that contains the quiver, colorbar and the background
        respectively.
        """
        if self._image_ax is None:
            self._image_ax = self.image_figure.add_subplot()
            self.image_blit_manager.addArtist('image_axes', self._image_ax)

        if self._background_ax is None:
            self._background_ax = self.background_figure.add_subplot()
            self.background_blit_manager.addArtist(
                'background_axes',
                self._background_ax,
            )

        if self._colorbar_ax is None:
            self._colorbar_ax, _kw = make_axes(
                self.background_ax,
                location = 'right',
                orientation = 'vertical',
            )
            self._colorbar_ax.xaxis.set_visible(False)
            self._colorbar_ax.yaxis.tick_right()

            self.background_blit_manager['colorbar_axes'] = self._colorbar_ax

    def _createQuiver(self):
        """
        Read the vector field and its attributes, and show it.

        TODO: read and save attributes, like length unit.
        """
        if self._quiver_object in self.image_ax.collections:
            _index = self.image_ax.collections.index(self._quiver_object)
            self.image_ax.collections.pop(_index)

        _, height, width = self.data_object.shape
        array_i = np.linspace(0, height - 1, height)
        array_j = np.linspace(0, width - 1, width)
        coord_i, coord_j = np.meshgrid(array_i, array_j, indexing = 'ij')
        vec_i, vec_j = self.data_object

        if 'quiver_scale' in self.data_object.attrs:
            quiver_scale = self.data_object.attrs['quiver_scale']
        else:
            self.data_object.attrs['quiver_scale'] = 1
            quiver_scale = 1

        if 'quiver_width' in self.data_object.attrs:
            quiver_width = self.data_object.attrs['quiver_width']
        else:
            self.data_object.attrs['quiver_width'] = 0.15
            quiver_width = 0.15

        if 'quiver_color' in self.data_object.attrs:
            quiver_color = self.data_object.attrs['quiver_color']
        else:
            self.data_object.attrs['quiver_color'] = 'black'
            quiver_color = 'black'

        X, Y = coord_j, coord_i
        U, V = vec_j, vec_i

        self._quiver_object = self.image_ax.quiver(
            X, Y, U, V,
            units = 'xy',
            scale = quiver_scale,
            width = quiver_width,
            angles = 'xy',
            pivot = 'mid',
            color = quiver_color,
        )

        self.image_blit_manager['quiver'] = self._quiver_object

    def setBackground(self, background_path: str):
        """
        Set the background path in HDF5 file, to show the background image.

        Will set the background_path attribute. The background image must be
        a 2D matrix, RGB images (3 channels) are not supported here.

        The background image's shape must be equal to one of the channel of
        the vector field. For example, if the shape of the vector field is
            2 x 256 x 256
        the shape of the background image must be 256 x 256 then.

        arguments:
            background_path: (str) the path of the image or data.

        raises:
            TypeError, KeyError, ValueError
        """
        if not isinstance(background_path, str):
            raise TypeError('background_path must be a str, not '
                '{0}'.format(type(background_path).__name__))

        background_node = self.hdf_handler.getNode(background_path)
        # May raise KeyError if the path does not exist
        if not isinstance(background_node, HDFDataNode):
            raise ValueError('Item {0} must be a '
                'Dataset'.format(background_path))

        background_data_obj = self.hdf_handler.file[background_path]
        if not len(background_data_obj.shape) == 2:
            raise ValueError('Data must be a 2D matrix (single channel image)')

        if self.data_path == '':
            raise ValueError('Must set vector field before set background')
        _, height, width = self.data_object.shape
        if (background_data_obj.shape[0] != height
                or background_data_obj.shape[1] != width):
            raise ValueError('The background image\'s shape must be equal to'
                ' one of the channel of the vector field.')

        self._background_path = background_path
        self.ui.lineEdit_background_path.setText(self.background_path)
        self.data_object.attrs['background_path'] = background_path

        self._createBackgroundImage()
        self._createColorbar()

        self.background_canvas.draw()
        self.background_canvas.flush_events()
        self.image_canvas.draw()
        self.image_canvas.flush_events()

    def _createBackgroundImage(self):
        """
        Create the axes and images of the background images.

        TODO: read and save attributes, like norm, cmap, alpha, etc.
        """
        if self._background_object in self.background_ax.images:
            # clear current objects in the axes
            _index = self.background_ax.images.index(self._background_object)
            self.background_ax.images.pop(_index)

        self._background_object = self.background_ax.imshow(
            self.hdf_handler.file[self.background_path]
        )
        self.background_blit_manager['background_image'] = self._background_object

        if self._image_object in self.image_ax.images:
            # clear current objects in the axes
            _index = self.image_ax.images.index(self._image_object)
            self.image_ax.images.pop(_index)

        self._image_object = self.image_ax.imshow(
            self.hdf_handler.file[self.background_path],
            interpolation = 'hermite',
        )
        self._image_object.set_visible(self.background_visible)
        self.image_blit_manager['image'] = self._image_object

    def _createColorbar(self):
        """
        Create the colorbar according to the background image.
        """
        if self._colorbar_object is None:
            self._colorbar_object = Colorbar(
                ax = self.colorbar_ax,
                mappable = self.background_object
            )
        else:
            self.colorbar_object.update_normal(self.background_object)

    def _initUi(self):
        """
        Initialize UI.
        """
        self.ui.checkBox_background_visible.setChecked(False)
        self.ui.checkBox_background_visible.stateChanged.connect(
            self._setImageVisible
        )

        self.ui.pushButton_browse.clicked.connect(
            self._browse
        )
        self.ui.pushButton_browse_background.clicked.connect(
            self._browseBackground
        )
        # self.ui.pushButton_vector_processing.clicked.connect(
        #     self._vectorProcessing
        # )
        self.ui.pushButton_adjust_effects.clicked.connect(
            self._adjustEffects
        )

        self.ui.pushButton_show_color_wheel.setVisible(False)   # TODO
        self.ui.pushButton_show_color_wheel.setVisible(False)   # TODO

    def _adjustEffects(self):
        """
        Open a dialog to adjust quiver display effects.
        """
        dialog = DialogAdjustQuiverEffect(self)
        dialog.setVectorField(self.data_path)
        dialog_code = dialog.exec()
        if dialog_code == DialogAdjustQuiverEffect.Rejected:
            return

        self.data_object.attrs['quiver_scale'] = dialog.getScale()
        self.data_object.attrs['quiver_width'] = dialog.getWidth()
        self.data_object.attrs['quiver_color'] = dialog.getColor()

        self.setVectorField(self.data_path)

    def _browse(self):
        """
        Open a dialog to browse which vector field to be opened.
        """
        dialog = DialogHDFChoose(self)
        dialog_code = dialog.exec()
        if dialog_code == dialog.Accepted:
            current_path = dialog.getCurrentPath()
        else:
            return

        try:
            self.setVectorField(current_path)
        except (KeyError, ValueError, TypeError,) as e:
            self.logger.error('{0}'.format(e), exc_info = True)
            msg = QMessageBox(parent = self)
            msg.setWindowTitle('Warning')
            msg.setIcon(QMessageBox.Warning)
            msg.setStandardButtons(QMessageBox.Ok)
            msg.setText('Cannot open this data: {0}'.format(e))
            msg.exec()

    def _browseBackground(self):
        """
        Open a dialog to browse which image to be opened as the background.
        """
        dialog = DialogHDFChoose(self)
        dialog_code = dialog.exec()
        if dialog_code == dialog.Accepted:
            current_path = dialog.getCurrentPath()
        else:
            return

        try:
            self.setBackground(current_path)
        except (KeyError, ValueError, TypeError,) as e:
            self.logger.error('{0}'.format(e), exc_info = True)
            msg = QMessageBox(parent = self)
            msg.setWindowTitle('Warning')
            msg.setIcon(QMessageBox.Warning)
            msg.setStandardButtons(QMessageBox.Ok)
            msg.setText('Cannot open this data: {0}'.format(e))
            msg.exec()

    def _setImageVisible(self):
        """
        Set whether the background image to be visible or not.
        """
        self.image_object.set_visible(self.background_visible)
        self.image_blit_manager.update()

    def _createNewBackground(self):
        """
        Create a new background image.

        In default, it will be /.../[vector name]_bkgrd.img . If there has been
        a background at this path, try to use it. Otherwise, add an index and
        try again: /.../[vector name]_bkgrd_1.img . The new path will be under
        the same group as the vector field.

        Users can calculate a new background and reset it themselves.

        returns:
            (str) the path of the new background image.
        """

        # Get a valid new name of the new background image
        data_node = self.hdf_handler.getNode(self.data_path)
        if '.' in data_node.name:
            name_array = data_node.name.split('.')
            name_array.pop()
            original_name = '.'.join(name_array)
        else:
            original_name = data_node.name

        bkgrd_name = original_name + '_bkgrd.img'
        _count = 0
        while bkgrd_name in data_node.parent:
            _count += 1
            bkgrd_name = original_name + '_bkgrd_{0}.img'.format(_count)

        # Get the shape of the new background image
        _, height, width = self.data_object.shape

        self.hdf_handler.addNewData(
            parent_path = data_node.parent.path,
            name = bkgrd_name,
            shape = (height, width),
            dtype = 'float64',
        )

        # Get the path of the new background image
        if data_node.parent.path == '/':
            bkgrd_path = '/' + bkgrd_name
        else:
            bkgrd_path = data_node.parent.path + '/' + bkgrd_name

        self.data_object.attrs['background_path'] = bkgrd_path

        return bkgrd_path

    def _vectorProcessing(self):
        """
        Open a dialog to show vector processing methods.

        Including change angles, subtracting mean vector, and calculate curl,
        divergence, potential, ...
        """
        dialog = DialogVectorProcessing(self)
        dialog_code = dialog.exec()
        if not dialog_code == dialog.Accepted:
            return
        result = dialog.getResult()
        if result == 'rotate':
            self._vectorRotateAngle()
        if result == 'subtract':
            self._vectorSubtract()
        if result == 'flip':
            self._vectorFlip()
        if result == 'potential':
            self._vectorPotential()
        if result == 'divergence':
            self._vectorDivergence()
        if result == 'curl':
            self._vectorCurl()
        if result == 'vec_i':
            self._vectorSliceI()
        if result == 'vec_j':
            self._vectorSliceJ()

    def _vectorRotateAngle(self):
        """
        Rotate every vector an angle.

        In practical experiment, there may exist some angular shift between
        the scanning array and the pixelized camera, but we usually cannot
        be aware of this by pure 4D-STEM dataset. Fortunately, we know that
        some vector fields (like Electric field) keeps non-curl, so we use
        this property to get correct vector fields.
        """
        angle, is_accepted = QInputDialog.getDouble(
            self,
            'Input rotation angle',
            'Here input a rotation angle of every vector. Unit: degree',
            0,
            minValue = -360,
            maxValue = 360,
            decimals = 1,
            step = 1,
        )

        if not is_accepted:
            return

        dialog_save = DialogSaveVectorField(self)
        dialog_save.setParentPath(self.data_path)
        dialog_code = dialog_save.exec()
        if not dialog_code == dialog_save.Accepted:
            return
        image_name = dialog_save.getNewName()
        image_parent_path = dialog_save.getParentPath()
        meta = self.data_object.attrs

        self.task = TaskRotateVectorAngle(
            self.data_path,
            image_parent_path,
            image_name,
            angle = angle,
            parent = self,
            **meta,
        )

        self.task_manager.addTask(self.task)

    def _vectorSubtract(self):
        """
        Subtract every vector by their mean vector.

        In practical experiment, there may exist some shift between the center
        of the diffraction pattern and the origin of the diffraction plane. In
        this case, the calculated center of mass will have an global offset.
        Here this function will be able to recover the fine architecture of the
        vector field, by set the vector offset to be zero.
        """
        dialog_save = DialogSaveVectorField(self)
        dialog_save.setParentPath(self.data_path)
        dialog_code = dialog_save.exec()
        if not dialog_code == dialog_save.Accepted:
            return
        image_name = dialog_save.getNewName()
        image_parent_path = dialog_save.getParentPath()
        meta = self.data_object.attrs

        self.task = TaskSubtractVectorOffset(
            self.data_path,
            image_parent_path,
            image_name,
            parent = self,
            **meta,
        )
        self.task_manager.addTask(self.task)

    def _vectorFlip(self):
        """
        Exchange every vector's i, j components.

        In practical experiment, the coordinate of the diffraction plane may
        differ from the convention of 4D-Explorer. For example, both typical
        i-j indexing and x-y coordinates are right-handed system:

            ┌------------> j        ^y
            |                       |
            |                       |
            |                       |
            |                       |
            v                       |
            i                       └----------------> x

        However, in some conventions, the coordinate is left-handed:

            ┌------------> x
            |
            |
            |
            |
            v
            y

        In this case, the calculated vector field's components should be
        exchanged in order to reveal correct electromagnetic field.
        """
        dialog_save = DialogSaveVectorField(self)
        dialog_save.setParentPath(self.data_path)
        dialog_code = dialog_save.exec()
        if not dialog_code == dialog_save.Accepted:
            return
        image_name = dialog_save.getNewName()
        image_parent_path = dialog_save.getParentPath()
        meta = self.data_object.attrs

        self.task = TaskFlipVectorField(
            self.data_path,
            image_parent_path,
            image_name,
            parent = self,
            **meta,
        )
        self.task_manager.addTask(self.task)

    def _vectorPotential(self):
        """
        Calculate potential for the vector field.

        The vector field should be a non-curl field, otherwise the result is
        invalid in physics.
        """
        dialog_save = DialogSaveImage(self)
        dialog_save.setParentPath(self.data_path)
        dialog_code = dialog_save.exec()
        if not dialog_code == dialog_save.Accepted:
            return
        image_name = dialog_save.getNewName()
        image_parent_path = dialog_save.getParentPath()
        meta = self.data_object.attrs

        self.task = TaskPotential(
            self.data_path,
            image_parent_path,
            image_name,
            parent = self,
            **meta,
        )
        self.task_manager.addTask(self.task)

    def _vectorDivergence(self):
        """
        Calculate divergence for the vector field.
        """
        dialog_save = DialogSaveImage(self)
        dialog_save.setParentPath(self.data_path)
        dialog_code = dialog_save.exec()
        if not dialog_code == dialog_save.Accepted:
            return
        image_name = dialog_save.getNewName()
        image_parent_path = dialog_save.getParentPath()
        meta = self.data_object.attrs

        self.task = TaskDivergence(
            self.data_path,
            image_parent_path,
            image_name,
            parent = self,
            **meta,
        )
        self.task_manager.addTask(self.task)

    def _vectorCurl(self):
        """
        Calculate curl for the vector field.
        """
        dialog_save = DialogSaveImage(self)
        dialog_save.setParentPath(self.data_path)
        dialog_code = dialog_save.exec()
        if not dialog_code == dialog_save.Accepted:
            return
        image_name = dialog_save.getNewName()
        image_parent_path = dialog_save.getParentPath()
        meta = self.data_object.attrs

        self.task = TaskCurl(
            self.data_path,
            image_parent_path,
            image_name,
            parent = self,
            **meta,
        )
        self.task_manager.addTask(self.task)

    def _vectorSliceI(self):
        """
        Slice i-component for the vector field.
        """
        dialog_save = DialogSaveImage(self)
        dialog_save.setParentPath(self.data_path)
        dialog_code = dialog_save.exec()
        if not dialog_code == dialog_save.Accepted:
            return
        image_name = dialog_save.getNewName()
        image_parent_path = dialog_save.getParentPath()
        meta = self.data_object.attrs

        self.task = TaskSliceI(
            self.data_path,
            image_parent_path,
            image_name,
            parent = self,
            **meta,
        )
        self.task_manager.addTask(self.task)

    def _vectorSliceJ(self):
        """
        Slice j-component for the vector field.
        """
        dialog_save = DialogSaveImage(self)
        dialog_save.setParentPath(self.data_path)
        dialog_code = dialog_save.exec()
        if not dialog_code == dialog_save.Accepted:
            return
        image_name = dialog_save.getNewName()
        image_parent_path = dialog_save.getParentPath()
        meta = self.data_object.attrs

        self.task = TaskSliceJ(
            self.data_path,
            image_parent_path,
            image_name,
            parent = self,
            **meta,
        )
        self.task_manager.addTask(self.task)

class DialogSaveVectorField(DialogSaveImage):
    """
    选择在 HDF 文件中保存矢量场的路径。

    Dialog to choose where to save the reconstructed image in the HDF file.
    """
    def __init__(self, parent: QWidget = None):
        super().__init__(parent)

    def getNewName(self) -> str:
        """
        returns the new name of the imported dataset.

        Will add '.img' automatically as the extension.
        """
        name = self.ui.lineEdit_name.text()
        if '.' in name:
            if name.split('.')[-1] == 'vec':
                return name
        return name + '.vec'

class DialogAdjustQuiverEffect(QDialog):
    """
    用于调整矢量场箭头图的效果的对话框。

    Dialog to adjust quiver display effect.
    """
    def __init__(self, parent: QWidget = None):
        super().__init__(parent)
        self.ui = uiDialogAdjustQuiverEffect.Ui_Dialog()
        self.ui.setupUi(self)
        self._item_path = None
        self._colors = {
            'blue': 0,
            'cyan': 1,
            'green': 2,
            'black': 3,
            'magenta': 4,
            'red': 5,
            'white': 6,
            'yellow': 7,
        }
        self.ui.doubleSpinBox_scale.setMaximum(1e9)
        self.ui.doubleSpinBox_scale.setMinimum(0.01)
        self.ui.doubleSpinBox_scale.setValue(1)
        self.ui.doubleSpinBox_width.setMinimum(0.01)
        self.ui.doubleSpinBox_width.setValue(0.15)
        self.ui.comboBox_color.setCurrentIndex(3)

        self.ui.pushButton_cancel.clicked.connect(self.reject)
        self.ui.pushButton_ok.clicked.connect(self.accept)

    @property
    def hdf_handler(self) -> HDFHandler:
        global qApp
        return qApp.hdf_handler

    @property
    def data_object(self) -> h5py.Dataset:
        return self.hdf_handler.file[self._item_path]

    @property
    def data_scale(self) -> float:
        return self.data_object.attrs['quiver_scale']

    @property
    def data_width(self) -> float:
        return self.data_object.attrs['quiver_width']

    @property
    def data_color(self) -> str:
        return self.data_object.attrs['quiver_color']

    def setVectorField(self, item_path: str):
        """
        arguments:
            item_path: (str) the vector field's path in hdf5 file.
        """
        self._item_path = item_path

        self.ui.doubleSpinBox_scale.setValue(self.data_scale)
        self.ui.doubleSpinBox_width.setValue(self.data_width)
        self.ui.comboBox_color.setCurrentIndex(
            self._colors[self.data_color]
        )

    def getScale(self) -> float:
        return self.ui.doubleSpinBox_scale.value()

    def getWidth(self) -> float:
        return self.ui.doubleSpinBox_width.value()

    def getColor(self) -> str:
        return self.ui.comboBox_color.currentText()

class DialogVectorProcessing(QDialog):
    """
    用于计算 Vector Field 相关操作的对话框。

    Dialog to calculate some processings for vector fields.
    """
    def __init__(self, parent: QWidget):
        super().__init__(parent)
        self.ui = uiDialogVectorProcessing.Ui_Dialog()
        self.ui.setupUi(self)
        self.ui.pushButton_rotate_vector_angle.clicked.connect(
            self._rotate_vector_angle
        )
        self.ui.pushButton_subtract_mean_vector.clicked.connect(
            self._subtract_mean_vector
        )
        self.ui.pushButton_flip.clicked.connect(
            self._flip
        )
        self.ui.pushButton_calculate_curl.clicked.connect(
            self._calculate_curl
        )
        self.ui.pushButton_calculate_divergence.clicked.connect(
            self._calculate_divergence
        )
        self.ui.pushButton_calculate_potential.clicked.connect(
            self._calculate_potential
        )
        self.ui.pushButton_vec_i.clicked.connect(
            self._slice_vec_i
        )
        self.ui.pushButton_vec_j.clicked.connect(
            self._slice_vec_j
        )

    def getResult(self):
        return self._result

    def _rotate_vector_angle(self):
        self._result = 'rotate'
        self.accept()

    def _subtract_mean_vector(self):
        self._result = 'subtract'
        self.accept()

    def _calculate_curl(self):
        self._result = 'curl'
        self.accept()

    def _calculate_divergence(self):
        self._result = 'divergence'
        self.accept()

    def _calculate_potential(self):
        self._result = 'potential'
        self.accept()

    def _slice_vec_i(self):
        self._result = 'vec_i'
        self.accept()

    def _slice_vec_j(self):
        self._result = 'vec_j'
        self.accept()

    def _flip(self):
        self._result = 'flip'
        self.accept()

# if 'quiver_scale' in self.data_object.attrs:
#             quiver_scale = self.data_object.attrs['quiver_scale']
#         else:
#             self.data_object.attrs['quiver_scale'] = 1
#             quiver_scale = 1

#         if 'quiver_width' in self.data_object.attrs:
#             quiver_width = self.data_object.attrs['quiver_width']
#         else:
#             self.data_object.attrs['quiver_width'] = 0.15
#             quiver_width = 0.15

#         if 'quiver_color' in self.data_object.attrs:


""" Statistics Test file for Multi_Maths Package """
# __doc__ (Statistics Test file for testing modules and packages)

import configparser
from importlib.resources import files
import unittest

from multi_maths.mm_statistics.mm_statistics import MM_Statistics

class TestStatistics(unittest.TestCase):
    """
    Class for unittest Statistics
    """

    def setUp(self):
        """
        Instance the MM_Statistics()
        """

        lang_file = files("multi_maths").joinpath('languages/en.ini')  # noqa: E501  # pylint: disable=C0301
        self.lang = configparser.ConfigParser()
        self.lang.sections()
        self.lang.read(lang_file, 'UTF-8')

        self.statistics = MM_Statistics(self.lang, 'digit')
        self.data_list = [8, 17, 41, 3, 21, 30, 11, 9]

    def test_standard_deviation(self):
        """
        standard_deviation() test for unittest
        """

        test = self.statistics.standard_deviation(self.data_list, 7)
        resolve = 16.978978599601163

        self.assertEqual(test, resolve)

    def test_mean(self):
        """
        mean() test for unittest
        """

        test = self.statistics.mean(self.data_list)
        resolve = 17.5

        self.assertEqual(test, resolve)

    def test_median(self):
        """
        median() test for unittest
        """

        test = self.statistics.median(self.data_list)
        resolve = 14.0

        self.assertEqual(test, resolve)

if __name__ == '__main__':
    unittest.main()


import math
import random
import numpy as np
from E160_config import *
from E160_state import*
from scipy.stats import norm
from scipy.linalg import sqrtm

INITIAL_ERROR = np.array([CONFIG_X_ERROR, CONFIG_Y_ERROR, CONFIG_THETA_ERROR]).reshape((3,1))

CONFIG_ROBOT_RAD_M = 0.147 / 2
CONFIG_WHEEL_RAD_M = 0.034

CONFIG_NUM_STATE_VARS = 3

# from wikipedia page of UKF
CONFIG_ALPHA = 0.001
CONFIG_BETA = 2.0
CONFIG_KAPPA = 0.0

# enums for direction
RIGHT_SENSOR_ID = 2
STRAIGHT_SENSOR_ID = 0
LEFT_SENSOR_ID = 1

print("sensor noise", CONFIG_SENSOR_NOISE_STDEV)
print("initial translation stdev", CONFIG_INIT_TRANSLATION_STDEV)
print("initial angle stdev", CONFIG_INIT_ANGLE_STDEV)

# R_t is the ``prediction noise'' - what is this?
PREDICTION_COVARIANCE = np.eye(CONFIG_NUM_STATE_VARS) * CONFIG_INIT_TRANSLATION_STDEV**2

# Q_t - measurement noise, for converting from sensor to absolute estimation
MEASUREMENT_COVARIANCE = np.eye(CONFIG_NUM_SENSORS) * CONFIG_SENSOR_NOISE_STDEV**2

def normalize_np_angle(ang):
  ang = ang % (2 * math.pi)
  ang[ang > math.pi] -= 2 * math.pi
  return ang

class E160_UKF:

  def __init__(self,
               environment,
               initial_state,
               initial_variance,
               encoder_resolution,
               alpha=CONFIG_ALPHA,
               beta=CONFIG_BETA,
               kappa=CONFIG_KAPPA,
               robot_radius=CONFIG_ROBOT_RAD_M,
               wheel_radius=CONFIG_WHEEL_RAD_M):

    # robot information
    self.robot_radius_m = robot_radius
    self.wheel_radius_m = wheel_radius

    # define the sensor orientations

    self.num_sensors = CONFIG_NUM_SENSORS
    self.sensor_orientation = [0, math.pi/2, -math.pi/2, math.pi/4, -math.pi/4, 3*math.pi/4, -3*math.pi/4, math.pi] # orientations of the sensors on robot
    self.sensor_orientation = self.sensor_orientation[0:self.num_sensors]
    self.walls = environment.walls

    # control signal sensor (process) information
    self.encoder_resolution = encoder_resolution
    self.last_encoder_measurements = [0,0]

    # correction signal sensor (measurement) information
    self.FAR_READING = 1.5
    if CONFIG_IN_HARDWARE_MODE:
      self.IR_sigma_m = 0.2 # Range finder s.d
    else:
      self.IR_sigma_m = 0.3 # Range finder s.d

    # number of state variables
    self.num_state_vars = CONFIG_NUM_STATE_VARS

    # total number of sigma points (includes mean)
    self.numParticles = 2*self.num_state_vars + 1

    # UKF parameters
    self.alpha = alpha
    self.beta = beta
    self.kappa = kappa
    self.lmbda = None
    self.mean_weights = np.zeros(self.numParticles)
    self.cov_weights = np.zeros(self.numParticles)

    # update lambda and weights
    self.lmbda, self.gamma, self.mean_weights, self.cov_weights = self.UpdateWeights()

    # structures that hold data for the sigma points
    self.particles = []
    self.sigma_points = np.zeros((CONFIG_NUM_STATE_VARS, self.numParticles))

    # initialize hypothesis for state and variance
    self.state = np.array(initial_state).reshape(3,1) + INITIAL_ERROR
    self.variance = np.array(initial_variance)

    # initializes sigma_points and particles
    self.particles = self.GenerateParticles(self.state, self.variance)

    self.delay = 0

    # track previous variance of error
    self.expected_measurement_variance = MEASUREMENT_COVARIANCE

  # update weights using rules described in Probabilistic Robotics
  def UpdateWeights(self):
    """
    Create new lambda, update mean and covariance weights.
    """
    lmbda = (self.alpha **2  * (self.num_state_vars + self.kappa)
                  - self.num_state_vars)

    gamma = np.sqrt(self.num_state_vars + lmbda)

    mean_weights = np.zeros(self.numParticles)
    cov_weights = np.zeros(self.numParticles)

    print('num_state_vars', self.num_state_vars)

    # set weights for mean sigma point
    # mean_weights[0] = lmbda / (self.num_state_vars + lmbda)
    cov_weights[0] = (lmbda / (self.num_state_vars + lmbda)
                           + 1.0 - self.alpha**2.0 + self.beta)

    # set other weights in filter
    # mean_weights[1:] = 1.0 / (2 * (self.num_state_vars + lmbda))
    cov_weights[1:]  = 1.0 / (2 * (self.num_state_vars + lmbda))

    mean_weights[0] = 1 / (self.num_state_vars + 1)
    # cov_weights[0] = -2#1 / (self.num_state_vars + 1)

    # set other weights in filter
    mean_weights[1:] = (1.0 - mean_weights[0]) / (2 * (self.num_state_vars))
    # cov_weights[1:]  = (1.0 - cov_weights[0]) / (2 * (self.num_state_vars))

    return lmbda, gamma, mean_weights, cov_weights

  # step 2, 6 in Probabilistic Robotics - generate new sigma points
  def GenerateParticles(self, state, variance):
    """ generate sigma particles """

    numParticles = self.numParticles

    # create sigma points using the gamma offset
    sigma_offsets = np.zeros((numParticles, self.num_state_vars))
    offsets = self.gamma * np.abs(sqrtm(self.variance))

    for i in range(self.num_state_vars):
      sigma_offsets[i + 1,:] = offsets[i,:]
      sigma_offsets[self.num_state_vars + i + 1, :] = -offsets[i,:]

    # create particle object list
    particles = numParticles*[0]

    # get mean state and use it to create sigma points
    x, y, theta = state[0][0], state[1][0], state[2][0]

    for i in range(numParticles):
      particles[i] = self.UKF_Particle(x + sigma_offsets[i][0] * np.cos(theta) - sigma_offsets[i][1] * np.sin(theta),
                                       y + sigma_offsets[i][0] * np.sin(theta) + sigma_offsets[i][1] * np.cos(theta),
                                       self.normalize_angle(theta + sigma_offsets[i][2]),
                                       self.mean_weights[i])
      p = particles[i]
      #print('Particle','i',p.x,p.y,p.heading)
    return particles

  # step 3 in Probabilistic Robotics
  def PropagateSigmaPoints(self, encoder_measurements, last_encoder_measurements):
    """ propagate all sigma points to get next predictions """

    for p in self.particles:
      delta_s, delta_heading = p.update_odometry(encoder_measurements,
                                                 last_encoder_measurements)
      p.update_state(delta_s, delta_heading)

  # steps 4, 5 in Probabilistic Robotics
  def PredictMeanAndCovariance(self):
    """ perform a weighted calculation of the mean and covariance of the
        propagated sigma point predictions """

    # data processing
    particle_data = np.array([[p.x, p.y, math.cos(p.heading), math.sin(p.heading)] for p in self.particles])

    # calculate mean state
    mean_state, sum_avg = np.average(particle_data, axis=0, weights=self.mean_weights, returned=True)
    heading = np.arctan2(mean_state[3], mean_state[2])
    mean_state = np.array([mean_state[0], mean_state[1], heading]).reshape(3,1)

    particle_data[:,2] = np.arctan2(particle_data[:,3], particle_data[:,2])
    particle_data = particle_data[:,0:3]

    # calculate new variance
    variance = np.zeros((self.num_state_vars, self.num_state_vars))
    for i in range(self.numParticles):
      particle_data_vec = particle_data[i,:].reshape((3,1))
      error = (particle_data_vec - mean_state)
      # normalize heading
      error[2] = self.normalize_angle(error[2])
      variance = variance + np.dot(self.cov_weights[i],
                                   np.dot(error,
                                          np.transpose(error)))
    variance = variance + PREDICTION_COVARIANCE

    return mean_state.reshape(3,1), variance

  # step 7 in Probabilistic Robotics
  def CalculateExpectedMeasurements(self):
    """ gets the expected measurements for all the sigma points """

    expected_measurements_m = np.zeros((self.numParticles, self.num_sensors))

    for i in range(self.numParticles):
      # x, y, theta = self.particles[i].x, self.particles[i].y, self.particles[i].heading

      min_dist_right = min(self.FindMinWallDistance(self.particles[i], self.walls, self.sensor_orientation[RIGHT_SENSOR_ID]), self.FAR_READING)
      min_dist_straight = min(self.FindMinWallDistance(self.particles[i], self.walls, self.sensor_orientation[STRAIGHT_SENSOR_ID]), self.FAR_READING)
      min_dist_left = min(self.FindMinWallDistance(self.particles[i], self.walls, self.sensor_orientation[LEFT_SENSOR_ID]), self.FAR_READING)

      expected_measurements_m[i][RIGHT_SENSOR_ID] = min_dist_right
      expected_measurements_m[i][STRAIGHT_SENSOR_ID] = min_dist_straight
      expected_measurements_m[i][LEFT_SENSOR_ID] = min_dist_left
    #print(expected_measurements_m)

    return expected_measurements_m

  # step 8, 9 in Probabilistic Robotics
  def SensorMeanAndCovariance(self, expected_measurements_m, prev_exp_measurement_variance):
    """ get mean and covariance of sensor measurements """
    # calculate mean expected sensor measurements
    expected_measurement_mean = np.average(expected_measurements_m,
                                            axis=0,
                                            weights=self.mean_weights).reshape((self.num_sensors,1))

    # calculate new expected measurement variance
    variance = np.zeros((CONFIG_NUM_SENSORS, CONFIG_NUM_SENSORS))
    for i in range(self.numParticles):
      error = expected_measurements_m[i,:].reshape((self.num_sensors,1)) - expected_measurement_mean
      variance = variance + self.cov_weights[i] * np.dot(error, np.transpose(error))

    expected_measurement_variance = variance + MEASUREMENT_COVARIANCE

    return expected_measurement_mean, expected_measurement_variance

  # step 10 in Probabilistic Robotics
  # TODO: Does not change currently as a function of orientation?
  def CalculateCrossCovariance(self,
                               particles,
                               state,
                               expected_measurements_m,
                               expected_measurement_mean):
    """ calculate cross covariance between measurement estimates and state prediction """

    # calculate cross variance
    cross_covariance = np.zeros((self.num_state_vars, CONFIG_NUM_SENSORS))
    particle_data = np.array([[p.x, p.y, p.heading] for p in self.particles])
    for i in range(self.numParticles):
      state_error = particle_data[i,:].reshape((3,1)) - state
      state_error[2] = self.normalize_angle(state_error[2])

      exp_measurement_error = (expected_measurements_m[i,:].reshape((self.num_sensors,1)) - expected_measurement_mean).reshape((self.num_sensors,1))
      cross_covariance = (cross_covariance
                          + np.dot(self.cov_weights[i],
                                   np.dot((state_error),
                                          (np.transpose(exp_measurement_error)))))
    return cross_covariance

  def CalculateInnovation(self, sensor_readings, exp_measurement_mean,
                          expected_measurements_m):
    # initialize flag matrices for identifying sensors not to use
    far_readings_ndx = np.ones(self.num_sensors, dtype=np.bool_)
    exp_far_readings_ndx = np.zeros(self.num_sensors, dtype=np.bool_)

    far_readings_ndx[sensor_readings[:,0] >= self.FAR_READING] = False

    for i in range(self.numParticles):
      exp_far_readings_ndx[expected_measurements_m[i,:] >= self.FAR_READING] = True
      exp_far_readings_ndx[abs(expected_measurements_m[i,:] - sensor_readings)[:,0] > CONFIG_TOO_BIG_SENSOR_ERROR] = True

    # combines knowledge from sensors and expected measurements
    use_sensor_flags = far_readings_ndx
    use_sensor_flags[exp_far_readings_ndx] = False

    innovation = np.zeros((self.num_sensors,1))

    innovation[use_sensor_flags] = sensor_readings[use_sensor_flags] - exp_measurement_mean[use_sensor_flags]
    return innovation

  def LocalizeEst(self,
                  encoder_measurements,
                  last_encoder_measurements,
                  sensor_readings):
    ''' Localize the robot with particle filters. Call everything
      Args:
        delta_s (float): change in distance as calculated by odometry
        delta_heading (float): change in heading as calculated by odometry
        sensor_readings([float, float, float]): sensor readings from range fingers
      Return:
        None'''
    if(abs(encoder_measurements[0] - last_encoder_measurements[0]) > 1000):
      encoder_measurements = [0,0]
      last_encoder_measurements = [0,0]
    if self.delay % 15 == 16:
      raise Exception('fifth step')
    self.delay += 1

    # convert sensor readings to distances (with max of 1.5m)
    sensor_readings = np.array([min(reading, self.FAR_READING) for reading in sensor_readings]).reshape((self.num_sensors,1))
    #print(sensor_readings)
    # step 2 - identify sigma points at t-1
    self.particles = self.GenerateParticles(self.state, self.variance)

    # step 3 - propagate set of sigma points
    self.PropagateSigmaPoints(encoder_measurements, last_encoder_measurements)

    # step 4, 5 - calculate weighted means and covariance of sigma points
    # this step is already complete in step 1 since all weights are equal
    var1 = self.variance
    #print('pre pre',self.state)
    self.state, self.variance = self.PredictMeanAndCovariance()
    #print('post pre',self.state)
    # step 6 - identify sigma points at time t using predicted mean, covariance
    self.particles = self.GenerateParticles(self.state, self.variance)

    # step 7 - calculate the expected sensor measurements
    expected_measurements_m = self.CalculateExpectedMeasurements()

    # step 8, 9 - calculate mean and variance of expected sensor measurements
    exp_measurement_mean, self.exp_measurement_variance = self.SensorMeanAndCovariance(expected_measurements_m,
                                                                                self.expected_measurement_variance)

    # step 10 - calculate cross-covariance between predicted state and measurements
    cross_covariance = self.CalculateCrossCovariance(self.particles,
                                                     self.state,
                                                     expected_measurements_m,
                                                     exp_measurement_mean)

    # step 11 - calculate Kalman gain
    kalman_gain = np.dot(cross_covariance, np.linalg.inv(self.exp_measurement_variance))

    # step 12, 13 - use actual measurements to calculate new state estimate, covariance
    innovation = self.CalculateInnovation(sensor_readings, exp_measurement_mean, expected_measurements_m)

    var2 = self.variance
    self.state = self.state + np.dot(kalman_gain, innovation)
    self.variance = self.variance - np.dot(kalman_gain, np.dot(self.exp_measurement_variance, np.transpose(kalman_gain)))

    # print('variance before -> after -> after correction \n', var1, '\n', var2, '\n', self.variance)
    # print(self.delay, 'over \n\n\n\n')

    state = E160_state(self.state[0][0], self.state[1][0], self.state[2][0])
    return state

  def FindMinWallDistance(self, particle, walls, sensorT):
    ''' Given a particle position, walls, and a sensor, find
      shortest distance to the wall
      Args:
        particle (E160_Particle): a particle
        walls ([E160_wall, ...]): represents endpoint of the wall
        sensorT: orientation of the sensor on the robot
      Return:
        distance to the closest wall' (float)'''

    #Handle the off centeritude of the horizontal sensors
    sensor_vertical_offset = 0 #straight sensor or in simulation
    if CONFIG_IN_HARDWARE_MODE(CONFIG_ROBOT_MODE):
      if sensorT > 0.1: #Left sensor
        sensor_vertical_offset = CONFIG_LEFT_VERTICAL_OFFSET
      elif sensorT < - 0.1:
        sensor_vertical_offset = CONFIG_RIGHT_VERTICAL_OFFSET

    temp_particle = self.UKF_Particle(0.0,0.0,particle.heading,1.0/self.numParticles)
    temp_particle.x = particle.x + math.cos(particle.heading) * sensor_vertical_offset
    temp_particle.y = particle.y + math.sin(particle.heading) * sensor_vertical_offset

    return min([self.FindWallDistance(particle, wall, sensorT) for wall in walls])

  def FindWallDistance(self, particle, wall, sensorT):
    ''' Given a particle position, a wall, and a sensor, find distance to the wall
      Args:
        particle (E160_Particle): a particle
        wall ([float x4]): represents endpoint of the wall
        sensorT: orientation of the sensor on the robot
      Return:
        distance to the closest wall (float)'''
    sensor_heading = self.normalize_angle(particle.heading + sensorT)
    sensor_slope = math.tan(sensor_heading)
    sensor_intercept = particle.y - sensor_slope * particle.x
    wall_slope, wall_intercept = wall.slope_intercept()

    # if wall slope parallel to sensor slope, sensor will not sense wall
    if wall_slope == sensor_slope:
      return float('inf')

    slope_diff = sensor_slope - wall_slope
    intercept_diff = wall_intercept - sensor_intercept
    x_val = intercept_diff/slope_diff

    # if wall is perfectly vertical, x_vale must be same as wall x_val
    if abs(wall_slope) > 1000:
      x_val, _ =  wall.point1

    # if sensor line is vertical, intersection will be at this x_point
    if abs(sensor_slope) > 1000:
      x_val = particle.x
      y_val = wall_slope * x_val + wall_intercept
    else:
      y_val = sensor_slope * x_val + sensor_intercept
    point = (x_val, y_val)
    # ensure that sensor points towards wall (not other direction)

    if(wall.contains_point(point)):
      distance = math.sqrt((x_val-particle.x)**2 + (y_val-particle.y)**2)
      if abs(math.atan2(particle.y - y_val, particle.x - x_val) - sensor_heading) >= CONFIG_HEADING_TOLERANCE:
        return float('inf')
      return distance
    else:
      return float('inf')

  def normalize_angle(self, ang):
    ''' Wrap angles between -pi and pi'''
    ang = ang % (2 * math.pi)
    while ang < -math.pi:
      ang = ang + 2 * math.pi
    while ang > math.pi:
      ang = ang - 2 * math.pi
    return ang

  class UKF_Particle:
    def __init__(self, x, y, heading, weight):
      self.x = x
      self.y = y
      self.heading = heading
      self.weight = weight
      self.is_first_run = False
      self.weight_memory = 20
      self.recent_weights = [0] * self.weight_memory

    def __str__(self):
      return str(self.x) + " " + str(self.y) + " " + str(self.heading) + " " + str(self.weight)

    # clean up this function
    def update_odometry(self, encoder_measurements, last_encoder_measurements):

      delta_s = 0
      delta_heading = 0

      left_encoder_measurement = encoder_measurements[0]
      right_encoder_measurement = encoder_measurements[1]

      last_left_encoder_measurement = last_encoder_measurements[0]
      last_right_encoder_measurement = last_encoder_measurements[1]

      delta_left =  float(left_encoder_measurement - last_left_encoder_measurement) #* rands[0]
      delta_right = float(right_encoder_measurement - last_right_encoder_measurement) #* rands[1]

      if self.is_first_run:
          delta_right = 0
          delta_left = 0
          self.is_first_run = False

      # cause the lab said so I like my name better
      diffEncoder0 = delta_left
      diffEncoder1 = delta_right

      wheel_circumference = 2 * math.pi * CONFIG_WHEEL_RAD_M

      # TODO: implement calibration from ticks to centimeters
      # left_distance = (delta_left / self.encoder_resolution) * wheel_circumference
      # right_distance = (delta_right / self.encoder_resolution) * wheel_circumference
      left_distance  = delta_left  * CONFIG_CM_TO_M *  CONFIG_LEFT_CM_PER_SEC_TO_TICKS_PER_SEC_MAP[10]
      right_distance = delta_right * CONFIG_CM_TO_M * CONFIG_RIGHT_CM_PER_SEC_TO_TICKS_PER_SEC_MAP[10]

      delta_s = (left_distance + right_distance) / 2
      delta_heading = (right_distance - left_distance) / (2 * CONFIG_ROBOT_RAD_M)

      # set current measurements as the last for next cycle - happens in E160_robot.py
      # self.last_encoder_measurements[0] = left_encoder_measurement
      # self.last_encoder_measurements[1] = right_encoder_measurement

      # keep this to return appropriate changes in distance, angle
      return delta_s, delta_heading

    def update_state(self, delta_s, delta_heading):

      self.x = self.x + math.cos(self.heading + delta_heading / 2) * delta_s

      self.y = self.y + math.sin(self.heading + delta_heading / 2) * delta_s

      self.heading = self.normalize_angle(self.heading + delta_heading)

    def normalize_angle(self, ang):
      ''' Wrap angles between -pi and pi'''
      while ang < -math.pi:
        ang = ang + 2 * math.pi
      while ang > math.pi:
        ang = ang - 2 * math.pi
      return ang


import argparse
import os
import re
import sys
import time
import json
import logging
from collections import defaultdict
#from langchain.llms import OpenAI
#import tiktoken
import openai
import numpy as np
from bs4 import BeautifulSoup
from lib.utils import deserialize_dictionary  #TODO: fix the module import from parsing.py
from lib.utils import load_json_from_file
import ast

import nltk
nltk.data.path.append('./nltk_data/')
from nltk.tokenize import word_tokenize

# Configure logging
logging.basicConfig(
    level=logging.DEBUG,
    format='%(asctime)s - %(levelname)s - %(message)s'
)

# json_string = '{"First Name": "firstName", "Middle Name": "middleName", "Middle Initial": "middleName-initial", "Last Name": "lastName", "Individual Taxpayer Identification Number (itin)": "itin",\
#                "Mailing Addressline1": "mailingAddressLine1", "Mailing Addressline2 (optional)": "mailingAddressLine2", "Zipcode/Postcode": "zip/postalCode",\
#                "State/Province/Region": "state/province/region", "City": "city", "Country": "country", "Primary Phone Number": "primaryPhoneNumber", \
#                "Phone Type": "primaryPhoneType", "Employment Status":"employmentStatus", "Income(annual)": "income-annual-usd", "Housing Status": "housingStatus",\
#                "Monthly Rent/Mortgage": "monthlyRent/Mortgage", "Education Degree": "educationDegree", "Working Industry": "workingIndustry", \
#                "Professional Title": "professionalTitle", "Professional Tenure":"professionalTenure", "Social Security Number (ssn)": "ssn", \
#                "Date of Birth": "dob", "Email Address": "email"}'

json_string = '''{
  "First Name": "firstName",
  "Middle Name": "middleName",
  "Middle Initial": "middleName-initial",
  "Last Name": "lastName",
  "Suffix": "suffix",
  "Tax ID Type": "taxIDType",
  "Individual Taxpayer Identification Number (itin)": "itin",
  "Social Security Number (ssn)": "ssn",
  "Date of Birth (mm/dd/yyyy)": "dob-mm/dd/yyyy",
  "Mother Maiden Name": "motherMaidenName",
  "Address Type": "addressType",
  "Mailing Addressline1": "mailingAddressLine1",
  "Mailing Addressline2 (optional)": "mailingAddressLine2",
  "Zipcode/Postcode": "zip/postalCode",
  "State/Province/Region": "state/province/region",
  "City": "city",
  "Country": "country",
  "Primary Phone Type": "primaryPhoneType",
  "Primary Phone Number": "primaryPhoneNumber",
  "Email Address": "email",
  "Employment Status": "employmentStatus",
  "Income(annual)": "income-annual-usd",
  "Type of residence/Housing Status": "housingStatus",
  "Monthly Rent/Mortgage": "monthlyRent/Mortgage",
  "Education Degree": "educationDegree",
  "Working Industry": "workingIndustry",
  "Professional Title": "professionalTitle",
  "Professional Tenure": "professionalTenure",
  "Linkedin Profile": "linkedinProfile",
  "Personal Website": "personalWebsite",
  "Visa Sponsorship": "visaSponsorship"
}'''

# Convert the JSON string to a Python dictionary
name_maps = json.loads(json_string)

input_string = """First Name: Min
Middle Initial: NA
Last Name: Lu
Mailing Addressline1: 15481 Bristol Ridge Ter
Mailing Addressline2 (optional): #54
State/Province/Region: CA
City: San Diego
Zipcode/Postcode: 92127
Email Address: minlu19@gmail.com
Primary Phone Number: 2176399259
Employment Status: employed
Education Degree: Doctoral Degree (Ph.D, Ed.D, M.D.)
Income(annual): 250000
Monthly Rent/Mortgage: 2500
Date of Birth (mm/dd/yyyy): 12/15/1989
Social Security Number (ssn): 316-14-4952
Middle Name: NA
Suffix: None
Tax ID Type: SOCIAL_SECURITY_NUMBER
Individual Taxpayer Identification Number (itin): 316-88-8888
Country: USA
Primary Phone Type: CELL
Type of residence/Housing Status: OWN
Working Industry: Tech
Professional Title: Engineer
Professional Tenure: 10 Years
Address Type: DOMESTIC
Mother Maiden Name: A
Linkedin Profile: https://www.linkedin.com/in/abc/
Personal Website: https://www.abc.com
Visa Sponsorship: NO"""

label_values_map = {
                         "veteranStatus": {"Filler":["please select"],"Veteran":["Veteran"], "Not Veteran":["Not Veteran"], "Other":["Other", "I don't want to declare"]},
                         "gender": {"Filler":["please select"],"male":["male","m"],"female":["female","f"],"other":["other", "x", "don't want to specify", "decline to self identify"]},
                         "visaSponsorship": {"Filler":["please select"],"Yes":["Yes"], "No":["No"]},
                         "disabilityStatus": {"Filler":["please select"],"Yes":["Yes"], "No":["No"]},
                         "isHispanicLatino": {"Filler":["please select"],"Yes":["Yes"], "No":["No"]},
                         # equalent to "residency status":["US citizen", "Green card (Permanent Resident)", "Foreign (Non-resident)"],
                         "race": {"Filler":["please select"],"American Indian or Alaskan Native":["American Indian or Alaskan Native"], "Asian":["Asian"], "Black or African American":["Black or African American"], "Hispanic or Latino":["Hispanic or Latino"], "White":["White"], "Native Hawaiian or Other Pacific Islander":["Native Hawaiian or Other Pacific Islander"], "Two or More Races":["Two or More Races"], "Decline To Self Identify":["Decline To Self Identify"]},

}

# input_string = """First Name: Xin
# Middle Name: X
# Last Name: Xu
# Suffix: None
# Tax ID Type: SSN
# Individual Taxpayer Identification Number (itin): 316-14-8882
# Social Security Number (ssn): 316-14-8882
# Date of Birth (mm/dd/yyyy): 12/15/1989
# Mother's Maiden Name: A
# Address Type: DOMESTIC
# Mailing Addressline1: 15469 Bristol Ridge Ter
# Mailing Addressline2: #58
# Zipcode/Postcode: 92122
# State/Province/Region: CA
# City: San Diego
# Country: USA
# Primary Phone Type: CELL
# Primary Phone Number: 2176399999
# Email Address: minlu8@gmail.com
# Employment Status: Employed
# Income(annual): 250000
# Type of residence/Housing Status: OWN
# Monthly Rent/Mortgage: 2500
# Education Degree: Doctoral Degree (Ph.D, Ed.D, M.D.)
# Working Industry: Tech
# Professional Title: Engineer
# Professional Tenure: 10 Years
# """

# Split the string into lines
lines = input_string.splitlines()

# Initialize an empty dictionary
pii_dict = {}

# Iterate through the lines and add them to the dictionary
for line in lines:
    key, value = line.split(": ", 1)  # split at the first occurrence of ': '
    pii_dict[key] = value
piikeys = pii_dict.keys()
piistr = str(pii_dict)

print(piistr)

def cosine_similarity_matrix(embeddings1, embeddings2):
    # Normalize the embeddings
    embeddings1_normalized = embeddings1 / np.linalg.norm(embeddings1, axis=1, keepdims=True)
    embeddings2_normalized = embeddings2 / np.linalg.norm(embeddings2, axis=1, keepdims=True)

    # Compute the similarity matrix
    similarity = np.dot(embeddings1_normalized, embeddings2_normalized.T)

    return similarity

def load_html_files_from_directory(directory_path):
    # List all files in the directory
    all_files = [f for f in os.listdir(directory_path) if os.path.isfile(os.path.join(directory_path, f))]

    # Filter for .html files
    html_files = [f for f in all_files if f.endswith('.html')]

    # Load each HTML file into memory
    html_contents = {}
    for html_file in html_files:
        with open(os.path.join(directory_path, html_file), 'r', encoding='utf-8') as file:
            html_contents[html_file] = file.read()

    return html_contents

def nltk_tokenize(input_string):
    tokens_nltk = word_tokenize(input_string.lower())
    tokens_nltk = [token for token in tokens_nltk if not no_alphanumeric_string(token)]
    return ' '.join(tokens_nltk)

def no_alphanumeric_string(s):
    return bool(re.match(r'^[^\w]', s))

def remove_trailing_non_alnum_regex(s):
    return re.sub(r'[^a-zA-Z0-9]*$', '', s)

def map_select_values(name, label_values, form_texts):

    value_tokens = []
    form_texts_list = []
    for form_text in form_texts:
        name_form_text = ' '.join([name.lower(), form_text.lower()])
        value_tokens.append(name_form_text)
        form_texts_list.append(form_text)
    print(value_tokens)
    openai_api_key = os.environ["OPENAI_API_KEY"]
    resp = openai.Embedding.create(
                input=value_tokens,
                engine="text-embedding-ada-002")
    value_token_embedding = dict()
    for i in range(len(value_tokens)):
        #"first name", embeddding vector
        value_token_embedding[value_tokens[i]] = resp['data'][i]['embedding']

    value_token_embedding_array_2d = []
    value_token_labels = []
    for k, v in value_token_embedding.items():
        value_token_embedding_array_2d.append(v)
        value_token_labels.append(k)
    value_token_embedding_array_2d = np.vstack(value_token_embedding_array_2d)

    synonym_label = []
    synonym_label_tokens = []
    label_synonym_embedding_array_2d = []
    label_value_synoyms = label_values_map.get(name)
    if label_value_synoyms:
        for label in label_values:
            synonyms = label_value_synoyms.get(label)
            for synonym in synonyms:
                name_value = ' '.join([name.lower(), synonym.lower()])
                print(name_value)
                synonym_label_tokens.append(name_value)
                synonym_label.append(label)
    print(synonym_label_tokens)
    openai_api_key = os.environ["OPENAI_API_KEY"]
    resp = openai.Embedding.create(
                input=synonym_label_tokens,
                engine="text-embedding-ada-002")

    for i in range(len(synonym_label_tokens)):
        #"first name", embeddding vector
        label_synonym_embedding_array_2d.append(resp['data'][i]['embedding'])
    label_synonym_embedding_array_2d = np.vstack(label_synonym_embedding_array_2d)

    simmatrix = cosine_similarity_matrix(value_token_embedding_array_2d, label_synonym_embedding_array_2d)
    label_dict = dict()
    for ind, value in enumerate(value_tokens):
        name_value = ' '.join([name.lower(), value.lower()])
        dot_products = simmatrix[ind,:]
        max_index = np.argmax(dot_products)
        max_similarity = dot_products[max_index]
        print(f'{value} is mapped to {synonym_label[max_index]} with score: {max_similarity}')
        #if max_similarity > .9:
        label_dict[synonym_label[max_index]] = (form_texts_list[ind], max_similarity)
    return label_dict

def generate_xpath(element):
    path_parts = []
    current = element
    while current is not None and current.name is not None and current.parent is not None:
        siblings = list(current.parent.children)
        tag_siblings = [sibling for sibling in siblings if sibling.name == current.name]
        if len(tag_siblings) > 1:
            index = tag_siblings.index(current) + 1  # XPath is 1-indexed
            path_parts.append(f"{current.name}[{index}]")
        else:
            path_parts.append(current.name)
        current = current.parent
    return '/' + '/'.join(reversed(path_parts))

def get_inverted_index(html_data):
    inverted = dict()
    for filename, content in html_data.items():
        print(f"Contents of {filename}:")
        #print(content)
        page = BeautifulSoup(content, 'html.parser')
        vocab = set()
        # Extracting text strings for all elements in the soup
        for s in page.stripped_strings:
            vocab.add(s.lower())

        # Extracting the 'name' attributes for all elements in the soup
        element_name_attributes = [tag['name'] for tag in page.find_all() if tag.has_attr('name')]
        for s in element_name_attributes:
            vocab.add(s.lower())

        #print(vocab)
        for v in vocab:
            if v not in inverted:
                inverted[v] = [filename]
            else:
                inverted[v].append(filename)
    return inverted

def scan_for_inputs(html_data, label_dict, output_folder):
    for filename, content in html_data.items():
        print(f"Contents of {filename}:")
        #print(content)
        soup, inputs = get_all_inputs(content)
        data = get_label_for_inputs(inputs, soup, label_dict)
        json_output = output_folder + '/' + filename[:-5] + ".json"
        with open(json_output, 'w') as json_file:
            json.dump(data, json_file)

def get_all_inputs(html_content):
    soup = BeautifulSoup(html_content, 'html.parser')
    inputs  = soup.find_all('input')
    selects = soup.find_all('select')
    textareas = soup.find_all('textarea')
    radios = soup.find_all(attrs={'role': 'radio'})
    checkboxes = soup.find_all(attrs={'role': 'checkbox'})
    submit_buttons = soup.find_all(attrs={'role': 'submit'})
    radiogroups = soup.find_all(attrs={'role': 'radiogroup'})
    input_elements = inputs + selects + textareas + radiogroups + radios + checkboxes + submit_buttons
    return soup, input_elements

def check_ground_truth(groundTruth, prediction):
    target_dict = {}
    for ele in groundTruth:
        if len(ele[0]) > 1:
            target_dict[ele[0]] = ele[2]
    pred_dict = {}
    for ele in prediction:
        if len(ele[0]) > 1:
            pred_dict[ele[0]] = ele[2]
    #print(target_dict)
    #print(pred_dict)
    miss = 0
    true_det = 0
    false_det = 0
    match_cat = {}
    for key, value in target_dict.items():
        if value != "no match":
            if key not in pred_dict:
                miss = miss + 1
                match_cat[key] = "miss|" + value
            else:
                pred_label = pred_dict[key]
                if pred_label == value:
                    true_det = true_det + 1
                    match_cat[key] = pred_label + "|" + value
                else:
                    false_det = false_det + 1
        else:
            if key in pred_dict:
                pred_label = pred_dict[key]
                if pred_label != "no match":
                    false_det = false_det + 1
                    match_cat[key] = pred_label + "|" + value
    return true_det, false_det, miss, match_cat

def get_label_for_fields(fields, label_dict):
    data = []
    for field in fields:
        elem_id = ""
        elem_type = ""
        label_text = ""
        label_conf = 0.0
        #if we can find a good match using name, skip
        foundLabel = False
        if field.get('htmlName'):
            s =  field.get('htmlName').lower()
            s = nltk_tokenize(s)
            if s in label_dict:
                label_text = label_dict[s][0] #label category
                label_conf = label_dict[s][1] #confidence score
                foundLabel = True
        if not foundLabel:
            if field.get('label-tag'):
                s =  field.get('label-tag').lower()
                s = nltk_tokenize(s)
                if s in label_dict:
                    label_text = label_dict[s][0] #label category
                    label_conf = label_dict[s][1] #confidence score
                    foundLabel = True
        if field.get('htmlID'):
            elem_id = "#"+field.get('htmlID')

        if field.get('tagName'):
            elem_type = field.get('tagName')

        mappings = dict()
        print(label_text)
        if len(label_text) > 1 and field.get('tagName') == 'select':
            options = field.get('options')
            text_value_map = dict()
            for op in options:
                text_value_map[op.get("text")] = op.get("value")
            form_option_texts = text_value_map.keys()
            label_values = label_values_map[label_text]
            result = map_select_values(label_text, label_values, form_option_texts)
            mappings = result
        xpath = field.get('htmlSelector')
        data_item = (elem_id, xpath, label_text, label_conf, elem_type, mappings)
        data.append(data_item)
    return data

def get_label_for_inputs(non_hidden_inputs, soup, label_dict):

    data = []
    for input_elem in non_hidden_inputs:

        elem_id = ""
        elem_type = ""
        label_text = ""
        label_conf = 0.0

        #if we can find a good match using name, skip
        foundLabel = False
        if input_elem.get('name'):
            s =  input_elem.get('name').lower()
            s = nltk_tokenize(s)
            if s in label_dict:
                label_text = label_dict[s][0] #label category
                label_conf = label_dict[s][1] #confidence score
                foundLabel = True
            #print(f"name is {s}, found label {foundLabel}")

        if not foundLabel:
            # Find associated label using 'for' attribute
            associated_label = soup.find('label', {'for': input_elem.get('id')})

            # If input is a child of label, the parent is the associated label
            if not associated_label and input_elem.find_parent('label'):
                associated_label = input_elem.find_parent('label')

            if associated_label:
                #print(associated_label)

                for s in associated_label.stripped_strings:
                    token = nltk_tokenize(s)
                    #print(f'normalized to {token}')
                    if token in label_dict:
                        print(f'{token} is matched to {label_dict[token]}')
                        label_text = label_dict[token][0] #label category
                        label_conf = round(label_dict[token][1], 2)#confidence score
                        foundLabel = True
                        break
                    #print(f"label is {s}, found label {foundLabel}")

        #print(f'label is {label_text}')
        #if 'id' in input_elem:
        if input_elem.get('id'):
            elem_id = "#"+input_elem.get('id')

        if input_elem.get('type'):
            elem_type = input_elem.get('type')

        xpath = generate_xpath(input_elem)
        data_item = (elem_id, xpath, label_text, label_conf, elem_type)
        data.append(data_item)

    return data

def chunk_list(input_list, chunk_size):
    return [input_list[i:i + chunk_size] for i in range(0, len(input_list), chunk_size)]

def get_embedding(input_tokens):
    print(input_tokens)
    token_embedding = dict()
    sub_tokens_list = chunk_list(input_tokens, 20)
    for sub_tokens in sub_tokens_list:
        print(sub_tokens)
        resp = openai.Embedding.create(
                input=sub_tokens,
                engine="text-embedding-ada-002")
        for i in range(len(sub_tokens)):
            token_embedding[sub_tokens[i]] = resp['data'][i]['embedding']
        print(sub_tokens)
    return token_embedding

def classifier_function(fieldName, sources, targets, model = "gpt-4"):
    messages = [{"role": "user", "content": f"You are domain expert in data schema matching for {fieldName}, you are generating map between two value lists, do not generate python code! Now you are acting like mapper function from a list specified by ```{sources}``` on to one of predefined list specified in: ```{targets}```, meaning the response should be one of the element in ```{targets}```.\n\n Only respond with your `return` python dictionary object. Do not include any other explanatory text or python code in your response."}]

    response = openai.ChatCompletion.create(
        model=model,
        messages=messages,
        temperature=0
    )

    return response.choices[0].message["content"]

def ai_function(function, args, description, model = "gpt-4"):
    # parse args to comma separated string
    args = ", ".join(args)
    messages = [{"role": "system", "content": f"You are now the following python function: ```# {description}\n{function}```\n\nOnly respond with your `return` value. Do not include any other explanatory text in your response."},{"role": "user", "content": args}]

    response = openai.ChatCompletion.create(
        model=model,
        messages=messages,
        temperature=0
    )

    return response.choices[0].message["content"]

def cosine_similarity_matrix(embeddings1, embeddings2):
    # Normalize the embeddings
    embeddings1_normalized = embeddings1 / np.linalg.norm(embeddings1, axis=1, keepdims=True)
    embeddings2_normalized = embeddings2 / np.linalg.norm(embeddings2, axis=1, keepdims=True)

    # Compute the similarity matrix
    similarity = np.dot(embeddings1_normalized, embeddings2_normalized.T)

    return similarity

def matching_simple(formfields, openai_api_key, model = "gpt-3.5-turbo"):
    openai.api_key = openai_api_key
    pii_names = []
    for readable_name, machine_name in name_maps.items():
        pii_names.append((readable_name, machine_name))

    form_names = []
    for element in formfields:
        form_name = ""
        if 'name' in element:
            form_name = form_name + ' ' + element['name']
        if 'label' in element:
            form_name = form_name + ' ' + element['label']
        if 'context' in element:
            form_name = form_name + ' ' + element['context']
        if len(form_name) < 1:
            continue
        form_names.append(form_name)

    function_string = "def map_names(form_names: list, pii_names: list) -> dict:"
    description_string = """Based on the semantic of list elements, build a map from the first list to the second list. please make sure the value of the map are one of tuple in second list or 'no match' if no good match can be found in second list"""
    args = [str(pii_names), str(form_names)]
    result_string = ai_function(function_string, args, description_string, model)
    logging.info(result_string)
    result = ast.literal_eval(result_string)
    print(result)

def matching_simple_bak(formfields, openai_api_key, model = "gpt-3.5-turbo"):
    openai.api_key = openai_api_key

    pii_names = []
    for readable_name, machine_name in name_maps.items():
        pii_names.append(readable_name)

    resp = openai.Embedding.create(
                input=pii_names,
                engine="text-embedding-ada-002")

    pii_name_embedding_array_2d = []
    for i in range(len(pii_names)):
        pii_name_embedding_array_2d.append(resp['data'][i]['embedding'])
    pii_name_embedding_array_2d = np.vstack(pii_name_embedding_array_2d)

    form_names = []
    for element in formfields:
        form_name = ""
        if 'name' in element:
            form_name = form_name + ' ' + element['name']
        elif 'label' in element:
            form_name = form_name + ' ' + element['label']
        elif 'context' in element:
            form_name = form_name + ' ' + element['context']
        if len(form_name) < 1:
            continue
        form_names.append(form_name)

    resp = openai.Embedding.create(
            input=form_names,
            engine="text-embedding-ada-002")

    form_name_embedding_array_2d = []
    for i in range(len(form_names)):
        form_name_embedding_array_2d.append(resp['data'][i]['embedding'])
    form_name_embedding_array_2d = np.vstack(form_name_embedding_array_2d)

    simmatrix = cosine_similarity_matrix(pii_name_embedding_array_2d, form_name_embedding_array_2d)

    form_name_candidates = {}
    for ind, form_name in enumerate(form_names):
        dot_products = simmatrix[:,ind]
        max_index = np.argmax(dot_products)
        max_similarity = dot_products[max_index]
        top_k_indices = np.argsort(dot_products)[-5:]
        candidates = []
        for index in top_k_indices:
            pii_name = pii_names[index]
            similarity = dot_products[index]
            if similarity > .6 and similarity > max_similarity - .05:
                candidates.append(pii_name)
        #verify the match with classification
        form_name_candidates[form_name] = candidates

    form_name_to_pii_name = []
    for element in formfields:
        form_name = ""
        machine_name = "no match"
        sub_dict = ""
        if 'name' in element:
            form_name = form_name + ' ' + element['name']
            sub_dict = element['name']
        elif 'label' in element:
            form_name = form_name + ' ' + element['label']
        elif 'context' in element:
            form_name = form_name + ' ' + element['context']
        if len(form_name) >= 1:
            #print('\n\n')
            #print(form_name)
            sub_dict = sub_dict + ' ' +element['label'] + ' ' + element['context']
            candidates = form_name_candidates[form_name]
            result = classifier_function(sub_dict, candidates,  "gpt-3.5-turbo")
            element['matched'] = result
            if result in name_maps:
                machine_name = name_maps[result]
            else:
                logging.warning(f"Key not found in name_maps: {result}")
            print(sub_dict)
            print(candidates)
            print(result)
        xid = ''
        if 'id' in element:
            xid = '#'+element['id']
        xpath = form_name
        input_type = 'NA'
        if 'type' in element:
            input_type = element['type']
        pii_name = machine_name
        mapped_value = 'N/A'
        form_name_to_pii_name.append((xid, xpath, pii_name, mapped_value, input_type))
    return form_name_to_pii_name

def matching(formfields, openai_api_key, model = "gpt-3.5-turbo"):
    openai.api_key = openai_api_key
    form_key_values = []
    form_key_texts = defaultdict(dict)
    for key, values in formfields.items():
        if (key[1] is None) or (len(key[1]) < 1):
            continue
        form_key_values.append(key[1])
        text_to_value = dict()
        for v in values:
            print(v)
            # key[1] is the actual name / context of the input group
            # v[0], v[1] is the id and xpath of input element,
            # v[2] is the value of the input element
            text_to_value[(key[1], str(v[2]))] = (v[0],v[1],v[3])

        form_key_texts[key[1]] = text_to_value

    unique_form_keys = set(form_key_values)
    if len(unique_form_keys) < len(form_key_values):
        logging.warning('This means some form field names have duplication')
        exit(1)

    #print(formfields['input group 5  Social Security number'])

    pii_key_values = []
    for key, values in pii_dict.items():
        pii_key_values.append(key)

    #print(form_key_values)
    #print(pii_key_values)

    function_string = "def map_names(form_key_values: list, pii_key_values: list) -> dict:"
    description_string = """Based on the semantic of list elements, build a map from the first list to the second list."""
    args = [str(pii_key_values), str(form_key_values)]
    result_string = ai_function(function_string, args, description_string, model)
    logging.info(result_string)
    result = ast.literal_eval(result_string)
    print(result)
    filtered_result = {}
    for pii_name_raw, form_name in result.items():
        if (form_name is None) or (len(form_name) == 0):
            continue
        print(pii_name_raw)
        print(form_name)
        # Compute the cosine similarity
        resp = openai.Embedding.create(
            input=[pii_name_raw, form_name],
            engine="text-similarity-davinci-001")

        embedding_a = resp['data'][0]['embedding']
        embedding_b = resp['data'][1]['embedding']

        similarity = np.dot(embedding_a, embedding_b)
        #similarity = nlp(pii_name_raw).similarity(nlp(form_name))
        print(similarity)
        if similarity >=.6:
            filtered_result[pii_name_raw] = form_name
    result = filtered_result

    form_name_to_pii_name = []

    matched_form_fields = set(result.values())
    for form_name in form_key_values:
        if len(form_name) < 1:
            continue
        if form_name not in matched_form_fields:
            form_value = []
            for (k,v) in form_key_texts[form_name]:
                form_value.append(v)
            if len(form_value) > 1:
                for k, v in form_key_texts[form_name]:
                    xid = v[0]
                    xpath = v[1]
                    input_type = v[2] if len(v) > 2 else 'unknown'
                    print(input_type)
                    form_name_to_pii_name.append((xid, xpath, 'unknown', 'na', input_type))
            if len(form_value) == 1:
                default_value = form_value[0]
                xid, xpath, input_type = form_key_texts[form_name][(form_name, default_value)]
                print(input_type)
                form_name_to_pii_name.append((xid, xpath, 'unknown', 'na', input_type))

    for pii_name_raw, form_name in result.items():

        if pii_name_raw in name_maps:
            pii_name = name_maps[pii_name_raw]
        else:
            logging.warning(f"Key not found in name_maps: {pii_name_raw}")
            continue

        form_value = []
        for (k,v) in form_key_texts[form_name]:
            form_value.append(v)
        print(pii_name_raw)
        print(pii_name)
        print(form_name)
        print(form_value)
        pii_value = pii_dict[pii_name_raw]

        if len(form_value) > 1:
            function_string = "def value_select(values: list, query: str) -> str:"
            args = [str(form_value), pii_value]
            description_string = """Based on the query string to select the best value in the list. Please output the whole result, no abbreviation"""
            mapped_value = ai_function(function_string, args, description_string, model)
            if (form_name, mapped_value) in form_key_texts[form_name]:
                xid, xpath, input_type = form_key_texts[form_name][(form_name, mapped_value)]
                # logging form_name_values
                logging.info((xid, xpath, mapped_value))
                form_name_to_pii_name.append((xid, xpath, pii_name, mapped_value))
            else:
                logging.warning(f"Key not found in form key texts map: {(form_name, mapped_value)}")
            for k, v in form_key_texts[form_name]:
                if (k[0] == form_name) and (k[1] == mapped_value):
                    continue
                xid = v[0]
                xpath = v[1]
                input_type = v[2]
                print(input_type)
                form_name_to_pii_name.append((xid, xpath, 'unselected', 'na', input_type))

        if len(form_value) == 1:
            #check if there is format string availale
            default_value = form_value[0]
            xid, xpath, input_type = form_key_texts[form_name][(form_name, default_value)]
            mapped_value = ''
            if len(default_value) > 1:
                function_string = "def value_norm(format: str, value: str) -> str:"
                args = [default_value, pii_value]
                print(default_value)
                description_string = """Based on the format string, normalized the value string."""
                mapped_value = ai_function(function_string, args, description_string, model)
            else:
                mapped_value = pii_value
            # logging form_name_values
            logging.info((xid, xpath, mapped_value))
            form_name_to_pii_name.append((xid, xpath, pii_name, mapped_value, input_type))

        if len(form_value) == 0:
            mapped_value = pii_value
            # logging form_name_values
            logging.info((xid, xpath, mapped_value))
            form_name_to_pii_name.append((xid, xpath, pii_name, mapped_value, input_type))
    return form_name_to_pii_name

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Matching pii data to extracted form fields.")
    parser.add_argument("--method", type=str, default="simple", help="The pii data json file.")
    parser.add_argument("--formfields", type=str, help="The extraced form fields pickle file.")
    parser.add_argument("--output", type=str, help="The filled information for #id/#xpath of input elements, in json format.")

    args = parser.parse_args()

    if args.method == 'simple':
        formfields = load_json_from_file(args.formfields)
        openai_api_key = os.environ["OPENAI_API_KEY"]
        json_output = args.output
        model = "gpt-3.5-turbo"
        form_name_to_pii_name = matching_simple(formfields, openai_api_key, model)
        with open(json_output, 'w') as json_file:
            json.dump(form_name_to_pii_name, json_file)

    else:

        formfields = deserialize_dictionary(args.formfields)
        openai_api_key = os.environ["OPENAI_API_KEY"]
        json_output = args.output

        model = "gpt-3.5-turbo"
        form_name_to_pii_name = matching(formfields, openai_api_key, model)

        with open(json_output, 'w') as json_file:
            json.dump(form_name_to_pii_name, json_file)


# -*- coding: utf-8 -*-
"""
Utilities for data collection and analysis.

@author: stewart
"""
import os
import re
from pysocialwatcher import watcherAPI
from pysocialwatcher.constants import TOKENS
import json

def query_facebook_audience(access_token, user_id, query_file, extra_auth_data=[], response_file=None):
    """
    Build manual query and execute request.

    access_token :: FB access token
    user_id :: FB user ID
    query_file :: JSON file containing query
    extra_auth_data :: List of auth data pairs.
    response_file :: Name of existing response file, if needed.

    response :: DataFrame with query response(s) => one response per row
    """
    watcher = watcherAPI()
    if(not (access_token, user_id) in TOKENS):
        watcher.add_token_and_account_number(access_token, user_id)
    for (access_token_i, user_id_i) in extra_auth_data:
        if(not (access_token_i, user_id_i) in TOKENS):
            watcher.add_token_and_account_number(access_token_i, user_id_i)
    print('%d FB tokens'%(len(TOKENS)))

    ## execute data collection
    if(response_file is not None and os.path.exists(response_file)):
        print('using response file %s'%(response_file))
        response = watcher.load_data_and_continue_collection(response_file)
    else:
        response = watcher.run_data_collection(query_file)

    ## clean up temporary dataframes
    file_matcher = re.compile('dataframe_.*.csv')
    tmp_files = filter(lambda f: file_matcher.search(f) is not None, os.listdir('.'))
    for f in tmp_files:
        os.remove(f)

    return response

def load_facebook_auth(auth_file='data/facebook_auth.csv'):
    """
    Load Facebook ad API authentication from file.

    auth_file :: File name.

    access_token :: Access token.
    user_id :: User ID.
    app_id :: App ID.
    app_secret :: Secret app ID.
    """
    auth_data = list(open(auth_file))[0].strip().split(',')
    if(len(auth_data) == 2):
        access_token, user_id = auth_data
        app_id = None
        app_secret = None
    elif(len(auth_data) == 4):
        access_token, user_id, app_id, app_secret = auth_data
#    access_token, user_id = list(open(auth_file))[0].strip().split(',')[:2]
    return access_token, user_id, app_id, app_secret

def query_and_write(query_file, out_dir, extra_auth_files=[], response_file=None):
    """
    Query Facebook for specified JSON target and
    write results to out_dir as .tsv.

    query_file :: JSON file containing query
    out_dir :: Output directory.
    extra_auth_files :: Extra FB auth data files.
    response_file :: Name of existing response file, if needed.
    """
    access_token, user_id, _, _ = load_facebook_auth()
    query_base = os.path.basename(query_file).replace('.json', '')

    ## issue query
    extra_auth_data = [load_facebook_auth(auth_file=f)[:2] for f in extra_auth_files]
    results = query_facebook_audience(access_token, user_id, query_file, extra_auth_data=extra_auth_data, response_file=response_file)

    ## clean up JSON cols
    json_cols = filter(lambda x: type(results.loc[:, x].iloc[0]) is dict, results.columns)
    for c in json_cols:
        results.loc[:, c] = results.loc[:, c].apply(json.dumps)

    ## write to file
    out_file = os.path.join(out_dir, '%s.tsv'%(query_base))
    results.to_csv(out_file, sep='\t', index=False)


# encoding: utf-8
"""

"""
__author__ = 'Rhys Evans'
__date__ = '01 Jan 2022'
__copyright__ = 'Copyright 2018 United Kingdom Research and Innovation'
__license__ = 'BSD - see LICENSE file in top-level package directory'
__contact__ = 'rhys.r.evans@stfc.ac.uk'

from setuptools import setup, find_namespace_packages

with open("README.md") as readme_file:
    _long_description = readme_file.read()

setup(
    name='stac_vocab_api',
    description='stac-vocab-api',
    author='Richard Smith',
    url='https://github.com/cedadev/stac-vocab-api/',
    long_description=_long_description,
    long_description_content_type='text/markdown',
    license='BSD - See asset_extractor/LICENSE file for details',
    packages=find_namespace_packages(),
    python_requires='>=3.5',
    package_data={
        'stac_vocab_api': [
            'LICENSE'
        ]
    },
    install_requires=[
        'attrs',
        'fastapi',
    ],
    extras_require={
        'server': ["uvicorn[standard]>=0.12.0,<0.14.0"],
        'dev': [
            'pytest',
            'requests'
        ]
    },
    entry_points={
    }
)


import csv
import json

fileWriteDir = "tinkersConstructMaterials.csv"

fieldNames = [
    "material",
    "level",
    "speed",
    "damage",
    "modifier",
    "durability",
    "traits",
    "part"
]

handlesFile = open("tinkersHandles.json")
headFile = open("tinkersHead.json")
extrasFile = open("tinkersExtras.json")

handles = json.load(handlesFile)
head = json.load(headFile)
extras = json.load(extrasFile)

data = {}

dataPos = 0

for i in extras:
    data[dataPos] = ({k.lower():v for k, v in i.items()})
    data[dataPos]["part"] = "extra"
    dataPos += 1

for i in head:
    data[dataPos] = ({k.lower():v for k, v in i.items()})
    data[dataPos]["part"] = "head"
    dataPos += 1

for i in handles:
    data[dataPos] = ({k.lower():v for k, v in i.items()})
    data[dataPos]["part"] = "handles"
    dataPos += 1

handlesFile.close()
headFile.close()
extrasFile.close()

# csv write
with open(fileWriteDir, 'w', newline = '', encoding = "utf-8") as csvfile:
    writer = csv.DictWriter(csvfile, fieldnames = fieldNames)
    writer.writeheader()
    for i in range(len(data)):
        writer.writerow(data[i])


import torch
model = torch.hub.load('pytorch/vision:v0.10.0', 'alexnet', pretrained=True)
model.eval()

import urllib
url, filename = ("https://github.com/pytorch/hub/raw/master/images/dog.jpg", "dog.jpg")
try: urllib.URLopener().retrieve(url, filename)
except: urllib.request.urlretrieve(url, filename)

from PIL import Image
from torchvision import transforms
input_image = Image.open(filename)
preprocess = transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
input_tensor = preprocess(input_image)
input_batch = input_tensor.unsqueeze(0)

with torch.no_grad():
    output = model(input_batch)

print(output[0])

probabilities = torch.nn.functional.softmax(output[0], dim=0)
print(probabilities)

with open("imagenet_classes.txt", "r") as f:
    categories = [s.strip() for s in f.readlines()]
# Show top categories per image
top5_prob, top5_catid = torch.topk(probabilities, 5)
for i in range(top5_prob.size(0)):
    print(categories[top5_catid[i]], top5_prob[i].item())