import logging
import re

log: logging.Logger = logging.getLogger("bash_declare_parser")

REGEX_BASH_DECLARE_DOUBLE_QUOTE = r"declare (-[-xr]) (.*?)=\"(.*)\""
REGEX_BASH_DECLARE_SINGLE_QUOTE = r"declare (-[-xr]) (.*?)=\$'(.*)'"
REGEX_BASH_DECLARE_ASSOCIATIVE_ARRAY = r"declare (-[A]) (.*?)=\((.*)\)"
REGEX_BASH_DECLARE_SIMPLE_ARRAY = r"declare (-[a]) (.*?)=\((.*)\)"
REGEX_SINGLE_QUOTED_SPLIT = r"'[^']+'|\S+"

class BashDeclareParser:
	def __init__(self, origin: str = 'unknown'):
		self.origin = origin

	def parse_one(self, one_declare):
		all_keys = {}
		count_matches = 0

		# Now parse it with regex-power! it only parses non-array, non-dictionary values, double-quoted.
		for matchNum, match in enumerate(re.finditer(REGEX_BASH_DECLARE_DOUBLE_QUOTE, one_declare, re.DOTALL), start=1):
			count_matches += 1
			value = self.parse_dequoted_value(match.group(2), self.armbian_value_parse_double_quoted(match.group(3)))
			all_keys[match.group(2)] = value

		if count_matches == 0:
			# try for the single-quoted version
			for matchNum, match in enumerate(re.finditer(REGEX_BASH_DECLARE_SINGLE_QUOTE, one_declare, re.DOTALL), start=1):
				count_matches += 1
				value = self.parse_dequoted_value(match.group(2), self.armbian_value_parse_single_quoted(match.group(3)))
				all_keys[match.group(2)] = value

		if count_matches == 0:
			# try for the (A)ssociative Array version
			for matchNum, match in enumerate(re.finditer(REGEX_BASH_DECLARE_ASSOCIATIVE_ARRAY, one_declare, re.DOTALL), start=1):
				count_matches += 1
				all_keys[match.group(2)] = ["@TODO", "bash associative arrays aka dictionaries are not supported yet", match.group(3)]

		if count_matches == 0:
			# try for the simple (a)rray version
			for matchNum, match in enumerate(re.finditer(REGEX_BASH_DECLARE_SIMPLE_ARRAY, one_declare, re.DOTALL), start=1):
				count_matches += 1
				all_keys[match.group(2)] = ["@TODO", "bash simple-arrays are not supported yet", match.group(3)]

		if count_matches == 0:
			log.error(f"** No matches found for Bash declare regex (origin: {self.origin}), line ==>{one_declare}<==")

		return all_keys

	def parse_dequoted_value(self, key, value):
		if ("_LIST" in key) or ("_DIRS" in key) or ("_ARRAY" in key):
			value = self.armbian_value_parse_list(key, value, " ")
		return value

	def armbian_value_parse_double_quoted(self, value: str):
		# replace "\\\\n" with actual newline
		value = value.replace('\\\\n', "\n")
		value = value.replace('\\\\t', "\t")
		value = value.replace('\\\"', '"')
		return value

	def armbian_value_parse_single_quoted(self, value: str):
		value = value.replace('\\n', "\n")
		value = value.replace('\n', "\n")
		value = value.replace('\\t', "\t")
		value = value.replace('\t', "\t")
		return value

	def armbian_value_parse_list(self, key, item_value, delimiter):
		ret = []
		good = re.findall(REGEX_SINGLE_QUOTED_SPLIT, item_value)
		for item in good:
			ret.append(item)
		# trim whitespace out of every value
		ret = list(map(str.strip, ret))
		# filter out empty strings
		ret = list(filter(None, ret))
		return ret


import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

class Customized_Loss(nn.Module):

    def __init__(self, beta = 1000):
        super(Customized_Loss, self).__init__()
        self.beta = beta

    def forward(self, output, target):
        # Ground truth
        xys = target[:, :2]
        cos_sin = target[:, 2:4]

        # Predictions
        xy_preds = output[:, :2]
        cos_sin_preds = output[:, 2:4]
        cos_sin_preds = F.normalize(cos_sin_preds)

        # Position error
        xy_diff = xy_preds - xys
        xy_errors = torch.norm(xy_diff, p = 2, dim = 1)

        # Orientation error
        cos_sin_diff = cos_sin_preds - cos_sin
        cos_sin_errors = torch.norm(cos_sin_diff, p = 2, dim = 1)

        ''' Assertions '''
        inf = float("inf")

        assert not torch.isnan(xy_errors).byte().any() # All elements in the tensor are zero (no NaNs)
        assert not (xy_errors == inf).byte().any() # All elements in the tensor are zero (no infs)

        assert not torch.isnan(cos_sin_errors).byte().any() # All elements in the tensor are zero (no NaNs)
        assert not (cos_sin_errors == inf).byte().any() # All elements in the tensor are zero (no infs)

        ''''''

        # theta0_errors = torch.norm(theta_preds - thetas, 2, dim = 1)
        # theta180_errors = torch.norm(np.pi - torch.abs(theta_preds - thetas), 2, dim = 1)
        # theta_errors_both = torch.stack((theta0_errors, theta180_errors), dim = 1)
        # theta_errors, _ = torch.min(theta_errors_both, dim = 1)

        # total_error = self.beta * torch.mean(xy_errors) + torch.mean(theta_errors)
        angle_error = self.beta * torch.mean(cos_sin_errors)
        position_error = torch.mean(xy_errors)
        # print(position_error, angle_error)
        total_error = position_error + angle_error

        ''' Assertions '''
        assert not torch.isnan(total_error).byte().any() # All elements in the tensor are zero (no NaNs)
        assert not (total_error == inf).byte().any() # All elements in the tensor are zero (no infs)
        ''''''

        # print(torch.mean(xy_errors), torch.mean(cosines_errors), torch.mean(sines_errors), sep = "\n")

        return total_error


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

house = pd.read_csv('train.csv', sep=',',usecols=(62,80))

# print(house.columns)
# print(house.info())
# print(house.head())

plt.scatter(house['GarageArea'],house['SalePrice'])
plt.title("Original Dataframe")
plt.ylabel("Sales Price")
plt.xlabel("Garage Area")
plt.show()

# Q1 = house.quantile(0.25)
# Q3 = house.quantile(0.75)
# IQR = Q3 - Q1
# print(IQR)
# house1 = house[~((house < (Q1 - 1000 * IQR)) |(house > (Q3 + 2.2 * IQR))).any(axis=1)]
# house1.shape
# plt.show()
#
z = np.abs(stats.zscore(house))
# print(z)
threshold = 2.2
print(np.where(z > 2.2))
house1 = house[(z < 2.2).all(axis=1)]
#
plt.scatter(house1['GarageArea'],house1['SalePrice'])
plt.title("Modified Dataframe")
plt.ylabel("Sales Price")
plt.xlabel("Garage Area")
plt.show()

# plt.subplot(1,2,1)
# plt.scatter(house['GarageArea'],house['SalePrice'], 'r--') # More on color options later
# plt.subplot(1,2,2)
# plt.scatter(house1['GarageArea'],house1['SalePrice'], 'g*-');
# plt.show()
# fig, ax = plt.subplots(figsize=(16,8))
# ax.scatter(house['GarageArea'],house['SalePrice'])
# ax.set_xlabel('GarageArea')
# ax.set_ylabel('SalePrice')
# plt.show()


"""
"""
import urllib
from xml.dom import minidom
from optparse import OptionParser
import subprocess
import re
from html.parser import HTMLParser
import requests
from random import randrange
from tenacity import retry

class MyHTMLParser(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self.get_data = False
        self.quotes = []

    def handle_starttag(self, tag, attrs):
        if tag == "dt":
            if attrs[0][0] == "class" and attrs[0][1] == "quote":
                self.get_data = True
        pass

    def handle_endtag(self, data):
        pass

    def handle_data(self, data):
        if self.get_data:
            self.quotes.append(data)
            self.get_data = False

@retry
def get_band_name():
    """
    """
    random_wiki_url = "http://en.wikipedia.org/w/api.php?format=xml&action=query&list=random&rnnamespace=0&rnlimit=1"
    dom = minidom.parse(urllib.request.urlopen(random_wiki_url))

    btitle = []
    for line in dom.getElementsByTagName("page"):
        btitle.append(line.getAttributeNode("title").nodeValue)

    name_parsed = re.sub("[\(\[].*?[\)\]]", "", btitle[0])

    utf_name = bytes(name_parsed, "utf-8").decode("utf-8", "ignore")

    # Eliminate commas, so locations in wikipedia look more like a proper name
    utf_name = utf_name.replace(",", "")

    # Return only the first three words and remove trailing and leading blankspaces
    return " ".join(utf_name.split()[:3]).strip()

@retry
def get_album_title():
    """
    """
    random_quote_url = "http://www.quotationspage.com/random.php"
    page = urllib.request.urlopen(random_quote_url).read()
    parser = MyHTMLParser()
    parser.feed(str(page))

    num_quotes = len(parser.quotes)
    quote = parser.quotes[randrange(0, num_quotes)].rstrip(".")

    last_set = randrange(3, 5)
    words = quote.split()

    if last_set > len(words):
        last_set = len(words)

    title = (" ").join(words[-last_set:])
    parsed_title = re.sub("[\(\[].*?[\)\]]", "", title)

    # Convert to utf8 and remove leading and trailing whitespaces with .strip()
    utf_title = bytes(parsed_title, "utf-8").decode("utf-8", "ignore").strip()

    # Eliminate commas
    utf_title = utf_title.replace(",", "")

    return utf_title


#!/usr/bin/env python3
import argparse
import glob
import os
import subprocess
import sys
from lib.patcher import ProjectConfigPatcher

RNV8_REPO_ROOT = os.path.dirname(os.path.dirname(__file__))

def parse_args():
    arg_parser = argparse.ArgumentParser()

    arg_parser.add_argument(
        "--rn-version", type=str, required=True, help="react-native version"
    )
    arg_parser.add_argument(
        "--npm-source",
        type=str,
        help="react-native-v8 install source from npm",
    )
    arg_parser.add_argument(
        "--v8-android-variant",
        type=str,
        default="v8-android-jit",
        help="v8-android variant to install",
    )

    arg_parser.add_argument("project_dir", action="store", help="Generated project dir")

    args = arg_parser.parse_args()

    has_error = False
    if os.path.exists(args.project_dir):
        print("Project dir existed - {}".format(args.project_dir))
        has_error = True

    if has_error:
        arg_parser.print_help()
        sys.exit(1)

    return args

def main():
    args = parse_args()
    subprocess.run(
        ["npx", "react-native", "init", args.project_dir, "--version", args.rn_version]
    )

    install_tarball = None
    if not args.npm_source:
        subprocess.run(["npm", "pack", "--pack-destination", args.project_dir])
        install_tarball = os.path.abspath(
            glob.glob(os.path.join(args.project_dir, "react-native-v8-*.tgz"))[0]
        )

    os.chdir(args.project_dir)
    subprocess.run(["yarn", "add", args.v8_android_variant])

    if not args.npm_source:
        assert install_tarball
        subprocess.run(["yarn", "cache", "clean", "react-native-v8"])
        subprocess.run(
            [
                "yarn",
                "add",
                "react-native-v8@file:{}".format(install_tarball),
            ]
        )
    else:
        subprocess.run(["yarn", "add", "react-native-v8@{}".format(args.npm_source)])

    patcher = ProjectConfigPatcher()
    patcher.add_v8_support()
    patcher.add_vm_hint()
    subprocess.run(
        ["npx", "react-native", "run-android", "--variant", "release", "--no-packager"]
    )

if __name__ == "__main__":
    main()


import re
from PySide6.QtCore import Qt, QModelIndex, QAbstractListModel
from spinetoolbox.helpers import bisect_chunks

class SimpleFilterCheckboxListModel(QAbstractListModel):
    _SELECT_ALL_STR = '(Select all)'
    _SELECT_ALL_FILTERED_STR = '(Select all filtered)'
    _EMPTY_STR = '(Empty)'
    _ADD_TO_SELECTION_STR = 'Add current selection to filter'

    def __init__(self, parent, show_empty=True):
        """Init class.

        Args:
            parent (QWidget)
        """
        super().__init__(parent)
        self._data = []
        self._data_set = set()
        self._selected = set()
        self._selected_filtered = set()
        self._is_filtered = False
        self._filter_index = []
        self._filter_expression = re.compile("")
        self._all_selected = True
        self._empty_selected = True
        self._add_to_selection = False
        self._action_rows = [self._SELECT_ALL_STR]
        if show_empty:
            self._action_rows.append(self._EMPTY_STR)

    @property
    def _show_empty(self):
        return self._EMPTY_STR in self._action_rows

    @property
    def _show_add_to_selection(self):
        return self._ADD_TO_SELECTION_STR in self._action_rows

    def reset_selection(self):
        self._selected = self._data_set.copy()
        self._all_selected = True
        self._empty_selected = True

    def _handle_select_all_clicked(self):
        if self._all_selected:
            if self._is_filtered:
                self._selected_filtered = set()
            else:
                self._selected = set()
        else:
            if self._is_filtered:
                self._selected_filtered = set(self._data[i] for i in self._filter_index)
            else:
                self._selected = self._data_set.copy()
        self._all_selected = not self._all_selected
        if self._show_empty:
            self._empty_selected = self._all_selected
        self.dataChanged.emit(self.index(0, 0), self.index(self.rowCount(), 0), [Qt.ItemDataRole.CheckStateRole])

    def _check_all_selected(self):
        if self._is_filtered:
            return len(self._selected_filtered) == len(self._filter_index)
        return len(self._selected) == len(self._data_set) and self._empty_selected

    def rowCount(self, parent=QModelIndex()):
        if self._is_filtered:
            if self._filter_index:
                return len(self._filter_index) + len(self._action_rows)
            # no filtered values
            return 0
        return len(self._data) + len(self._action_rows)

    def data(self, index, role=Qt.ItemDataRole.DisplayRole):
        if not index.isValid():
            return None
        row = index.row()
        action_state = [self._all_selected]
        if self._show_empty:
            action_state.append(self._empty_selected)
        if self._show_add_to_selection:
            action_state.append(self._add_to_selection)
        if self._is_filtered:
            selected = self._selected_filtered
            if row >= len(self._action_rows):
                data_row = self._filter_index[row - len(self._action_rows)]
        else:
            selected = self._selected
            if row >= len(self._action_rows):
                data_row = row - len(self._action_rows)
        if role == Qt.ItemDataRole.DisplayRole:
            if row >= len(self._action_rows):
                return self._data[data_row]
            return self._action_rows[row]
        if role == Qt.ItemDataRole.CheckStateRole:
            if row >= len(self._action_rows):
                return Qt.CheckState.Checked if self._data[data_row] in selected else Qt.CheckState.Unchecked
            return Qt.CheckState.Checked if action_state[row] else Qt.CheckState.Unchecked

    def _handle_index_clicked(self, index):
        if index.row() == 0:
            self._handle_select_all_clicked()
        else:
            if index.row() == 1 and self._show_add_to_selection:
                self._add_to_selection = not self._add_to_selection
            elif index.row() == 1 and self._show_empty:
                self._empty_selected = not self._empty_selected
            else:
                if self._is_filtered:
                    i = self._filter_index[index.row() - len(self._action_rows)]
                    item = self._data[i]
                    if item in self._selected_filtered:
                        self._selected_filtered.discard(item)
                    else:
                        self._selected_filtered.add(item)
                else:
                    item = self._data[index.row() - len(self._action_rows)]
                    if item in self._selected:
                        self._selected.discard(item)
                    else:
                        self._selected.add(item)
            self._all_selected = self._check_all_selected()
            self.dataChanged.emit(index, index, [Qt.ItemDataRole.CheckStateRole])
            self.dataChanged.emit(0, 0, [Qt.ItemDataRole.CheckStateRole])

    def set_list(self, data, all_selected=True):
        self.beginResetModel()
        self._data_set = set(data)
        self._data = list(data)
        if all_selected:
            self._selected = self._data_set.copy()
            self._all_selected = True
            self._empty_selected = True
        else:
            self._selected = set()
            self._all_selected = False
            self._empty_selected = False
        self.remove_filter()
        self.endResetModel()

    def filter_by_condition(self, condition):
        """Updates selected items by applying a condition.

        Args:
            condition (function): Filter acceptance condition.
        """
        selected = {item for item in self._data if condition(item)}
        self.set_selected(selected)

    def set_selected(self, selected, select_empty=None):
        self.beginResetModel()
        self._selected = self._data_set.intersection(selected)
        if select_empty is not None:
            self._empty_selected = select_empty
        self._all_selected = self._check_all_selected()
        self.endResetModel()

    def get_selected(self):
        return self._selected.copy()

    def get_not_selected(self):
        if self._all_selected:
            return set()
        return self._data_set.difference(self._selected)

    def set_filter(self, filter_expression):
        if filter_expression and (isinstance(filter_expression, str) and not filter_expression.isspace()):
            self._action_rows[0] = self._SELECT_ALL_STR
            self._filter_expression = re.compile(filter_expression)
            self._filter_index = [i for i, item in enumerate(self._data) if self.search_filter_expression(item)]
            self._selected_filtered = set(self._data[i] for i in self._filter_index)
            self._add_to_selection = False
            self.beginResetModel()
            self._is_filtered = True
            if not self._show_add_to_selection:
                self._action_rows.append(self._ADD_TO_SELECTION_STR)
            self._all_selected = True
            self.endResetModel()
        else:
            self.remove_filter()

    def search_filter_expression(self, item):
        return self._filter_expression.search(item)

    def apply_filter(self):
        if not self._is_filtered:
            return
        if self._add_to_selection:
            self._remove_and_add_filtered()
        else:
            self._remove_and_replace_filtered()

    def _remove_and_add_filtered(self):
        if not self._selected:
            # no previous selected, just replace
            self._selected = set(self._selected_filtered)
        else:
            # add selected
            self._selected.update(self._selected_filtered)
            # remove unselected
            self._selected.difference_update(
                set(item for item in (self._data[i] for i in self._filter_index) if item not in self._selected_filtered)
            )
        self.remove_filter()

    def _remove_and_replace_filtered(self):
        self._selected = set(self._selected_filtered)
        self._empty_selected = False
        self.remove_filter()

    def remove_filter(self):
        if not self._is_filtered:
            return
        self.beginResetModel()
        self._action_rows[0] = self._SELECT_ALL_STR
        self._filter_expression = re.compile("")
        if self._show_add_to_selection:
            self._action_rows.remove(self._ADD_TO_SELECTION_STR)
        self._filter_index = []
        self._is_filtered = False
        self._selected_filtered = set()
        self._all_selected = self._check_all_selected()
        self.endResetModel()

    def _do_add_items(self, data):
        first = len(self._data)
        last = first + len(data) - 1
        self.beginInsertRows(self.index(0, 0), first, last)
        self._data += data
        self.endInsertRows()

    def add_items(self, data, selected=None):
        if selected is None:
            selected = self._all_selected
        data = [x for x in data if x not in self._data_set]
        if not data:
            return
        self._do_add_items(data)
        self._data_set.update(data)
        if selected:
            self._selected.update(data)
            if self._is_filtered:
                self._selected_filtered.update(data)
        if self._is_filtered:
            self._filter_index = [i for i, item in enumerate(self._data) if self.search_filter_expression(item)]
        self._all_selected = self._check_all_selected()

    def remove_items(self, data):
        data = set(data)
        if not data.intersection(self._data_set):
            return
        for k, item in reversed(list(enumerate(self._data))):
            if item in data:
                self.beginRemoveRows(self.index(0, 0), k, k)
                self._data.pop(k)
                self.endRemoveRows()
        self._data_set.difference_update(data)
        self._selected.difference_update(data)
        if self._is_filtered:
            self._filter_index = [i for i, item in enumerate(self._data) if self.search_filter_expression(item)]
            self._selected_filtered.difference_update(data)
        self._all_selected = self._check_all_selected()

class LazyFilterCheckboxListModel(SimpleFilterCheckboxListModel):
    """Extends SimpleFilterCheckboxListModel to allow for lazy loading in synch with another model."""

    def __init__(self, parent, db_mngr, db_maps, fetch_parent, show_empty=True):
        """Init class.

        Args:
            parent (SpineDBEditor)
            fetch_parent (FetchParent)
        """
        super().__init__(parent, show_empty=show_empty)
        self._db_mngr = db_mngr
        self._db_maps = db_maps
        self._fetch_parent = fetch_parent

    def canFetchMore(self, _parent):
        result = False
        for db_map in self._db_maps:
            result |= self._db_mngr.can_fetch_more(db_map, self._fetch_parent)
        return result

    def fetchMore(self, _parent):
        for db_map in self._db_maps:
            self._db_mngr.fetch_more(db_map, self._fetch_parent)

    def _do_add_items(self, data):
        """Adds items so the list is always sorted, while assuming that both existing and new items are sorted."""
        for chunk, pos in bisect_chunks(self._data, data):
            self.beginInsertRows(self.index(0, 0), pos, pos + len(chunk) - 1)
            self._data[pos:pos] = chunk
            self.endInsertRows()

class DataToValueFilterCheckboxListModel(SimpleFilterCheckboxListModel):
    """Extends SimpleFilterCheckboxListModel to allow for translating internal data to a value for display role."""

    def __init__(self, parent, data_to_value, show_empty=True):
        """Init class.

        Args:
            parent (SpineDBEditor)
            data_to_value (method): a method to translate item data to a value for display role
        """
        super().__init__(parent, show_empty=show_empty)
        self.data_to_value = data_to_value

    def data(self, index, role=Qt.ItemDataRole.DisplayRole):
        data = super().data(index, role=role)
        if role == Qt.ItemDataRole.DisplayRole and data not in self._action_rows:
            return self.data_to_value(data)
        return data

    def search_filter_expression(self, item):
        return self._filter_expression.search(self.data_to_value(item))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from gi.repository import Gtk

def create_gridview_from_column_list_and_dict_list(columns_list, dict_list):
    scrolled_window = Gtk.ScrolledWindow()
    scrolled_window.set_hexpand(True)
    scrolled_window.set_vexpand(True)

    types_list = [ type(x) for x in columns_list ]
    liststore = Gtk.ListStore(*types_list)

    for row_as_dict in dict_list:
        row = []
        for col in columns_list:
            row.append(str(row_as_dict[col]))
        liststore.append(row)

    treeview = Gtk.TreeView(model=liststore)

    for i, col in enumerate(columns_list):
        treeviewcolumn = Gtk.TreeViewColumn(col)
        treeview.append_column(treeviewcolumn)
        cellrenderertext = Gtk.CellRendererText()
        treeviewcolumn.pack_start(cellrenderertext, True)
        treeviewcolumn.add_attribute(cellrenderertext, "text", i)

    scrolled_window.add(treeview)
    return scrolled_window

if __name__ == '__main__':
    window = Gtk.Window()
    ''' :type : gtk.Window '''
    window.set_default_size(500, 300)
    window.connect("destroy", lambda q: Gtk.main_quit())

    data = []
    for i in xrange(1,200):
        data.append({'col1':'aa'+str(i), 'col2':'val'+str(i)})
    gv = create_gridview_from_column_list_and_dict_list(['col1', 'col2'], data)
    window.add(gv)

    window.show_all()
    Gtk.main()


import pytest
from pytest_embedded import Dut

CONFIGS = [
    pytest.param('default', marks=[pytest.mark.supported_targets]),
    pytest.param('freertos_options', marks=[pytest.mark.supported_targets]),
    pytest.param('psram', marks=[pytest.mark.esp32]),
    pytest.param('release', marks=[pytest.mark.supported_targets]),
    pytest.param('single_core', marks=[pytest.mark.esp32]),
]

@pytest.mark.generic
@pytest.mark.parametrize('config', CONFIGS, indirect=True)
def test_freertos(dut: Dut) -> None:
    dut.expect_exact('Press ENTER to see the list of tests')
    dut.write('![ignore]')
    # All of the FreeRTOS tests combined take > 60s to run. So we use a 120s timeout
    dut.expect_unity_test_output(timeout=120)


import network
import json
from machine import Pin
import uwebsockets.client
import os
import time
import socket

url = 'ws://185.58.207.148:8888/controller'

def do_connect():
    global sta_if
    if not sta_if.isconnected():
        print('connecting to network...')
        sta_if.active(True)
        sta_if.connect('4elabn', 'nogivjopu')
        while not sta_if.isconnected():
            pass
        print('network config:', sta_if.ifconfig())

def auth_socket(url):
    websocket = uwebsockets.client.connect(url)
    aut_data = json.dumps({'client_id': '321'})
    websocket.send(aut_data)
    response = wait_response(websocket)
    if response == 'You are autorizated':
        return websocket
    return False

def wait_response(websocket):
    return websocket.recv()

def check_websocket(websocket):
    if not websocket.open:
        return auth_socket(url)
    return websocket

def execute_command(command):
    global pin_enable
    lampa_state = not bool(pin_enable.value())
    com = command['command']
    if com == 'enable':
        try:
            value = bool(command['value'])
        except ValueError:
            print('enable error')
            return False
        if value != lampa_state:
            pin_enable.value(lampa_state)

sta_if = network.WLAN(network.STA_IF)
do_connect()
websocket = auth_socket(url)

#pin_enable = Pin(4, Pin.OUT)
pin_enable = Pin(2, Pin.OUT)
pin_enable.value(1)

while(True):
    do_connect()
    websocket = check_websocket(websocket)
    if websocket == False:
        continue
    try:
        response = wait_response(websocket)
        command = json.loads(response)
    except ValueError:
        continue
    print(command)
    execute_command(command)
    print('Cycle')


from bs4 import BeautifulSoup
import urllib.request as req
import urllib.parse as rep
import sys
import io
import os

sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding = 'utf-8')
sys.stderr = io.TextIOWrapper(sys.stderr.detach(), encoding = 'utf-8')
# User-agent : 사용자를 대신해서 인터넷에 접속하는 소프트웨어, 즉 브라우저
# Mozilla/5.0 : 파이썬으로도 사용자 에이젼트 헤더를 바꾸는 게 가능하다.
# 즉 윈도우에서 리눅스 인척 또는 403에러 솔루션
# opener = req.build_opener()
# opener.addheaders = [('User-agent', 'Mozilla/5.0')]
# req.install_opener(opener)

# HTML 가져오기
base = "https://search.naver.com/search.naver?where=image&query="
quote = rep.quote_plus("사자")
url = base + quote

res = req.urlopen(url)
savePath ="C:\\imagedown\\" #c:/imagedown/과 동일 폴더 생성
try:
    if not(os.path.isdir(savePath)): # 디렉토리가 있는지 확인 : 없디면 생성
        os.makedirs(os.path.join(savePath))
except OSError as e: #디렉토리가 있다면
    if e.errno != errno.EEXIST: # EEXIST :file이 존재할 경우에
        print("Failed to create directory!!!!!")
        raise #컴파일 하여 폴더 생성확인

soup = BeautifulSoup(res, "html.parser")

li_list = soup.select("div.img_area _item > a.thumb._thumb > img")
for i, div in enumerate(li_list,1):
    #print(div) #'data-source'확인
    #print(div['src']) #이미지 확인 불가능한 src
    #print("div =", div['data-source']) #사용가능한  url 확인
    fullfilename = os.path.join(savePath, savePath+str(i)+'.jpg')
    # print(fullfilename)
    req.urlretrieve(div['data-source'],fullfilename)
    print(i)

print("다운로드 완료")


import networkx as nx
from tqdm import tqdm

from module.expansion.neighborhood import NeighborExpand
from module.expansion.ppr import PPR
from module.import_options import Options
from module.lfr.helper import LFRHelper

class GMLWriter:

    def __init__(self, save_location, custom_seeders):
        self.save_location = save_location
        self.seeders = custom_seeders

    def lfr_to_gml(self, reader):
        for seeder in self.seeders:
            lfr_dict = reader.read()
            for key, value in lfr_dict.items():

                size, mix, overlap = LFRHelper.extract_key(key)

                graph, communities = value

                for vertex, membership in communities.items():
                    graph.node[vertex]['community'] = membership
                    graph.node[vertex]['roverlap'] = len(membership)

                if seeder:
                    found_communities = self.find_communities(seeder, graph)
                    for vertex, membership in found_communities.items():
                        graph.node[vertex]['bigdiscover'] = max(membership)
                        graph.node[vertex]['smalldiscover'] = min(membership)
                        graph.node[vertex]['foverlap'] = len(membership)

                loc = "%s_%s_%s_%s_%s.gml" % (self.save_location, size, mix, overlap, seeder.name)
                nx.write_gml(graph, loc)

    def graph_to_gml(self, graph, real_communities=None):
        clean_graph = graph.to_undirected()
        for seeder in self.seeders:
            graph = clean_graph.to_undirected()
            found_communities = self.find_communities(seeder, graph)
            for vertex, membership in found_communities.items():
                graph.node[vertex]['bigdiscover'] = max(membership)
                graph.node[vertex]['smalldiscover'] = min(membership)
                graph.node[vertex]['foverlap'] = len(membership)

            if real_communities is not None:
                for membership, vertices in real_communities.items():
                    for vertex in vertices:
                        graph.node[vertex]['community'] = membership

            loc = "%s_%s.gml" % (self.save_location, seeder.name)
            nx.write_gml(graph, loc)

    @staticmethod
    def find_communities(seeder, graph):
        ppr = PPR()
        a_communities = {}

        seeds = seeder.seed(graph)

        expander = NeighborExpand(graph)
        expanded_seeds = expander.expand_seeds(seeds)

        community_count = 0
        size_tuples = []

        for center_seed, neighbor_seeds in tqdm(expanded_seeds.items(), desc="Expanding seeds to form communities",
                                                unit="seed", total=len(seeds)):
            detected = ppr.ppr_rank(graph, neighbor_seeds)
            graph.node[center_seed]['cseed'] = str(community_count)
            for seed in neighbor_seeds:
                graph.node[seed]['nseed'] = str(community_count)

            if len(detected) > 2:
                size_tuples.append((len(detected), detected))
                community_count += 1

        community_count = 0
        size_tuples = sorted(size_tuples)
        for _, detected in size_tuples:
            for v in detected:
                a_communities.setdefault(v, set())
                a_communities[v].add(str(community_count))
            community_count += 1

        return a_communities

if __name__ == '__main__':
    from sys import argv
    import os
    import datetime

    option_import = Options(argv, parameters="smocdt")
    seeders = option_import.select_seeders()

    directory = os.getcwd()
    date = datetime.datetime.now().strftime("%y-%m-%H%S")
    save_name = f"{directory}/network{date}"

    reader = option_import.generate_reader()
    gml_writer = GMLWriter(save_name, seeders)

    if reader is not None:
        gml_writer.lfr_to_gml(reader)
    else:
        graph, communities = option_import.import_real(directory, need_truth=False)
        print("Graph and community imported")
        gml_writer.graph_to_gml(graph, communities)


import random
from openpyxl import Workbook

for i in range(1, 6):

    wb = Workbook()

    for j in range(5):
        ws = wb.create_sheet()

    for x, ws in enumerate(wb, 1):
        ws.title = ('sheet' + str(x).zfill(3))
        for row in range(1, 40):
            ws.append([100*random.random() for _ in range(10)])

    wb.save('book' + str(i).zfill(2) + '.xlsx')


#!/usr/bin/env python3
import os
from collections import defaultdict
from collectRashyUnitIdReplacements import get_unit_type_mapping

replacements = [
	('{~add-ons/Rashy_Era/','{~ERROR_RE_ABSOLUTE_PATH/'),

	("VR_", "AE_RHY_"),
	("vr-", "AE_rhy_"),

	("""
    description= _ ""","""
    description={AE_RHY_UNIT_NOTICE}+ _ """),
	("""
	description= _ ""","""
    description={AE_RHY_UNIT_NOTICE}+ _ """),
	("""
    description=_ ""","""
    description={AE_RHY_UNIT_NOTICE}+ _ """),
	("""
	description=_ ""","""
    description={AE_RHY_UNIT_NOTICE}+ _ """),
	("""
    description=_""","""
    description={AE_RHY_UNIT_NOTICE}+ _ """),
	("""
	description=_""","""
    description={AE_RHY_UNIT_NOTICE}+ _ """),
	("""
    description={NO_DESCR_AVAILABLE}""","""
    description={AE_RHY_UNIT_NOTICE}+{NO_DESCR_AVAILABLE}"""),
	("__DUMMY__","__DUMMY__")
]

unit_type_mapping = get_unit_type_mapping()
for k in sorted(unit_type_mapping, key=lambda x: -len(x)):
	replacements.append((k, unit_type_mapping[k]))

def getAgelessPath(dname, fname):
	if "factions" in dname:
		return os.path.join(".","Ageless_Era","factions","RE",fname)
	if "units" in dname:
		basename = os.path.basename(dname)
		if basename == "units":
			basename = ""
		return os.path.join(".","Ageless_Era","units","RE_units",basename,fname)
	if "macros" in dname:
		return os.path.join(".","Ageless_Era","data","RE_data",fname)
	raise Exception("Unhandled folder {}".format(dname))

eras = defaultdict(str)
for dname, dirs, files in os.walk("."):
	if "factions" not in dname and "units" not in dname and "macros" not in dname:
		continue
	if "Ageless_Era" in dname or "images" in dname:
		continue
	print(dname)
	for fname in files:
		fpath = os.path.join(dname, fname)
		afpath = getAgelessPath(dname, fname)
		# print(fpath, afpath)
		with open(fpath,encoding="utf8") as f:
			s = f.read()
		for (find, replace) in replacements:
			s = s.replace(find, replace)
		if "factions" in dname:
			s = s.replace("{AE_RHY_UNIT_NOTICE}+", "")
			s = s.replace('name= _"', 'name= "RE - " + _"')
			s = s.replace('name=_"', 'name= "RE - " + _"')
			if "default" in fname:
				eras["default"] += s
				eras["default"] += "\n"
			elif "aoh" in fname:
				eras["heroes"] += s
				eras["heroes"] += "\n"
			elif "mrpg" in fname:
				eras["RPG"] += s
				eras["RPG"] += "\n"
			# TODO no rpg era
		else:
			os.makedirs(os.path.dirname(afpath), exist_ok=True)
			with open(afpath, "w", encoding="utf8") as f:
				f.write(s)

if eras["RPG"] == "":
	rpg_era = ""
	for line in eras["default"].split("\n"):
		if "type=" in line or "leader=" in line or "ai]" in line or "recruitment_pattern=" in line:
			continue
		line = line.replace("recruit=","leader=")
		rpg_era += line + "\n"
	eras["RPG"] = rpg_era

with open("Ageless_Era/units/RE_units/units.cfg", "a", encoding="utf8") as f:
	f.write("""

{GET_AE_UNITS RE_units/aquana}
{GET_AE_UNITS RE_units/dwarves}
{GET_AE_UNITS RE_units/elves-dark}
{GET_AE_UNITS RE_units/elyser}
{GET_AE_UNITS RE_units/gnomes}
{GET_AE_UNITS RE_units/human-chevalier}
{GET_AE_UNITS RE_units/human-dardo}
{GET_AE_UNITS RE_units/human-forest}
{GET_AE_UNITS RE_units/human-luz}
{GET_AE_UNITS RE_units/human-mountain}
{GET_AE_UNITS RE_units/human-regis}
{GET_AE_UNITS RE_units/marashy}
{GET_AE_UNITS RE_units/ships}
{GET_AE_UNITS RE_units/trarashy}
{GET_AE_UNITS RE_units/undead}
{GET_AE_UNITS RE_units/vixens}
""")

# factions to file
for era in eras:
	os.makedirs(os.path.dirname("Ageless_Era/factions/{}/".format(era)), exist_ok=True)
	with open("Ageless_Era/factions/{}/{}-RE.cfg".format(era, era), "w", encoding="utf8") as f:
		f.write(eras[era])


# app.py
from flask import Flask
from flask import render_template
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)

app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///trails.sqlite3'
db = SQLAlchemy(app)

db.Model.metadata.reflect(db.engine)

class Trail(db.Model):
    __tablename__ = 'Trails'
    __table_args__ = { 'extend_existing': True }
    TRAIL_ID = db.Column(db.Text, primary_key=True)

@app.route('/')
def hello():
    trailCount = Trail.query.count()

    good_trails = Trail.query.filter_by(CONDITION='Good').all()

    return render_template("index.html", count=trailCount, trails=good_trails)

@app.route('/trails/<slug>')
def detail(slug):
    trail = Trail.query.filter_by(TRAIL_ID=slug).first()

    return render_template("detail.html", trail=trail)

if __name__=='__main__':
    app.run(debug=True)


import sqlite3

def to_transaction_dict(transaction_tuple):
    ''' transaction is a transaction tuple (item_id, amount, category, date, description) '''
    transaction = {'rowid':transaction_tuple[0], 'amount':transaction_tuple[1], 'category':transaction_tuple[2], 'date':transaction_tuple[3], 'desc': transaction_tuple[4]}
    return transaction

def to_transaction_dict_list(transaction_tuples):
    ''' convert a list of transaction tuples into a list of dictionaries'''
    return [to_transaction_dict(transaction) for transaction in transaction_tuples]

class Transaction:

    def __init__(self, dbfile):
        con= sqlite3.connect(dbfile)
        cur = con.cursor()
        cur.execute('''CREATE TABLE IF NOT EXISTS transactions
                    (amount text, category text, date text, desc text)''')
        con.commit()
        con.close()
        self.dbfile = dbfile

    def select_all(self):
        ''' return all of the transactions as a list of dicts.'''
        con= sqlite3.connect(self.dbfile)
        cur = con.cursor()
        cur.execute("SELECT rowid,* from transactions")
        tuples = cur.fetchall()
        con.commit()
        con.close()
        return to_transaction_dict_list(tuples)

    def delete(self,rowid):
        con= sqlite3.connect(self.dbfile)
        cur = con.cursor()
        cur.execute('''DELETE FROM transactions
                       WHERE rowid=(?);
        ''',(rowid,))
        con.commit()
        con.close()

    def add(self,item):
        ''' add a transaction to the transactions table.
            this returns the rowid of the inserted element
        '''
        con= sqlite3.connect(self.dbfile)
        cur = con.cursor()
        cur.execute("INSERT INTO transactions VALUES(?,?,?,?)",(item['amount'], item['category'], item['date'], item['desc']))
        con.commit()
        cur.execute("SELECT last_insert_rowid()")
        last_rowid = cur.fetchone()
        con.commit()
        con.close()
        return last_rowid[0]

    def summarize_by_date(self, date):
        ''' summarizes transactions by date '''
        con= sqlite3.connect(self.dbfile)
        cur = con.cursor()
        cur.execute("SELECT DISTINCT rowid,* FROM transactions WHERE date=(?)",(date,))
        tuples = cur.fetchall()
        con.commit()
        con.close()
        return tuples

    def summarize_by_month(self, month):
        ''' summarize transactions by month'''
        con= sqlite3.connect(self.dbfile)
        cur = con.cursor()
        cur.execute("SELECT rowid,* FROM transactions WHERE SUBSTRING(date, 5,2)=(?)",(month,))
        tuples = cur.fetchall()
        con.commit()
        con.close()
        return tuples

    def summarize_by_year(self, year):
        '''summarizes transactions by year'''
        con= sqlite3.connect(self.dbfile)
        cur = con.cursor()
        cur.execute("SELECT rowid,* FROM transactions WHERE SUBSTRING(date, 1,4)=(?)",(year,))
        tuples = cur.fetchall()
        con.commit()
        con.close()
        return tuples

    def summarize_by_cat(self, cat):
        '''summarizes by category'''
        con= sqlite3.connect(self.dbfile)
        cur = con.cursor()
        cur.execute("SELECT rowid,* FROM transactions WHERE category=(?)", (cat,))
        tuples = cur.fetchall()
        con.commit()
        con.close()
        return tuples


import requests
import bs4
import re
import flask
from flask import request, jsonify

swRegex = re.compile(r'\d{4}-\d{4}-\d{4}')
priceRegex = re.compile(r'\d{3}')
userRegex = re.compile(r'ago by (.*?)')

#return bs4 object
def getPage(url,headers):
    res = requests.get(url, headers=headers)
    try:
        res.raise_for_status()
    except:
        pass

    pageHtml=bs4.BeautifulSoup(res.text,'html.parser')
    return pageHtml

#check if previous scrape already contained the given listing
def alreadyHas(tListings,link):
    for k in tListings:
        if k["url"]==link:
            return True
    return False

#check if a listing is still active, or if it is finished
def checkActivity(link,headers):
    curPost=requests.get(link,headers=headers)
    curPageHtml=bs4.BeautifulSoup(curPost.text,'html.parser')

    words=curPageHtml.find("div",class_="thing")
    cleanPost=words.text.strip('•comentsharvidport 1234567890')

    return cleanPost[:6]=='Active'

#add listing to array if the listing is still active
def addListing(link,listings,headers):
    curPost=requests.get(link,headers=headers)
    curPageHtml=bs4.BeautifulSoup(curPost.text,'html.parser')

    words=curPageHtml.find("div",class_="thing")
    cleanPost=words.text.strip('•comentsharvidport 1234567890')
    FC=swRegex.findall(cleanPost)

    if checkActivity(link,headers):
        if len(FC)>0:
            FC=FC[0]
            cleanPost=cleanPost.replace(FC,'')
            price=priceRegex.findall(cleanPost)
        else:
            FC=''
        if len(price)>0:
            price=price[0]
        else:
            price=''
        curListing = {
                        "price" : price,
                        "FC" : "SW-"+FC,
                        "url" : link
                    }
        listings.append(curListing)
    else:
        pass

# Returns an array of dcitionaries - turnip listings
def getListings(old=[]):
    url = "https://old.reddit.com/r/acturnips/new"
    headers = {'User-Agent': 'Mozilla/5.0'}
    tListings = []
    prev=[]
    for i in range(1):
        pageHtml=getPage(url,headers)
        elems = pageHtml.find_all("a", class_="title may-blank")
        url=pageHtml.find("span",class_="next-button").find("a" , recursive=False)['href']
        for j in range(len(elems)):
            link="https://old.reddit.com"+elems[j]['href']
            if alreadyHas(old,link):
                break
            try:
                addListing(link,tListings,headers)
            except:
                pass
    for i in range(len(old)):
        if checkActivity(old[i]['url'],headers):
            prev.append(old[i])

    tListings=tListings+prev
    print (tListings)
    return tListings

getListings()


import motor.motor_asyncio
import os
from model import Student

# MongoDB driver
mongoHost = os.getenv("MONGODB_HOST", "mongodb")
# url = 'mongodb://mongodb:27017/vdt2023'
url = f'mongodb://{mongoHost}:27017/vdt2023'
client = motor.motor_asyncio.AsyncIOMotorClient(url)

database = client.VDT
collection = database.student

async def fetch_all_students():
    students = []
    cursor = collection.find({})
    async for document in cursor:
        students.append(Student(**document))
    return students


import os
import argparse
import time
import logging
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
from math import log, floor
from datetime import datetime
from concurrent.futures import ProcessPoolExecutor
from concurrent.futures import as_completed

def setup_logging(directory):
    directory_name = os.path.basename(os.path.normpath(directory))
    log_file = f"logs/{directory_name}.log"
    logging.basicConfig(filename=log_file, level=logging.INFO,
                        format="%(asctime)s - %(levelname)s: %(message)s")
    logging.getLogger().addHandler(logging.StreamHandler())

def convert_size(size_bytes):
    if size_bytes == 0:
        return "0B", 0
    size_name = ("B", "KB", "MB", "GB", "TB")
    i = int(floor(log(size_bytes, 1024)))
    p = pow(1024, i)
    s = round(size_bytes / p, 2)
    return f"{s} {size_name[i]}", size_bytes

def human_readable_time(seconds):
    if seconds < 1:
        return f"{seconds * 1000:.2f} ms"
    elif seconds < 60:
        return f"{seconds:.2f} s"
    elif seconds < 3600:
        return f"{seconds / 60:.2f} min"
    else:
        return f"{seconds / 3600:.2f} hr"

memoized_sizes = {}
def get_directory_size(path):
    if path in memoized_sizes:
        return memoized_sizes[path]
    total = 0
    for dirpath, dirnames, filenames in os.walk(path):
        for file in filenames:
            file_path = os.path.join(dirpath, file)
            if not os.path.islink(file_path):
                total += os.path.getsize(file_path)
    memoized_sizes[path] = total
    return total

def get_directory_data(directory, max_depth=5):
    current_time = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
    entries_list = []

    # Added a depth control
    depth = directory.count(os.sep)
    if depth > max_depth:
        return entries_list

    for dirpath, dirnames, filenames in os.walk(directory):
        for dirname in dirnames:
            path = os.path.join(dirpath, dirname)
            size_readable, size_bytes = convert_size(get_directory_size(path))

            # Filtering directories with size < 10MB
            # if size_bytes < 10_485_760:  # 10 MB in bytes
            #     continue

            entries_list.append({
                "path": path,
                "name": dirname,
                "size_readable": size_readable,
                "size_bytes": size_bytes,
                "type": "directory",
                "parent_directory": dirpath,
                "scrape_timestamp": current_time
            })
        for filename in filenames:
            path = os.path.join(dirpath, filename)

            try:
                size_readable, size_bytes = convert_size(os.path.getsize(path))

                # Filtering files with size < 10MB
                # if size_bytes < 10_485_760:  # 10 MB in bytes
                #     continue

                entries_list.append({
                    "path": path,
                    "name": filename,
                    "size_readable": size_readable,
                    "size_bytes": size_bytes,
                    "type": "file",
                    "parent_directory": dirpath,
                    "scrape_timestamp": current_time
                })
            except FileNotFoundError:
                continue  # Just skip if file not found

    # Sort the data by size in descending order before returning
    return sorted(entries_list, key=lambda x: x['size_bytes'], reverse=True)

def parallel_directory_walker(root_directory, max_workers=4):
    directories = [os.path.join(root_directory, dir_name) for dir_name in os.listdir(root_directory) if os.path.isdir(os.path.join(root_directory, dir_name))]
    all_entries = []

    with ProcessPoolExecutor(max_workers=max_workers) as executor:
        futures = {executor.submit(get_directory_data, dir_path): dir_path for dir_path in directories}

        for future in as_completed(futures):
            all_entries.extend(future.result())

    return all_entries

def index_exists(es, index_name):
    return es.indices.exists(index=index_name)

def create_index_with_mapping(es, index_name, mapping):
    return es.indices.create(index=index_name, body=mapping)

def generate_mapping_from_data(data):
    mapping = {}
    if isinstance(data, dict):
        properties = {}
        for key, value in data.items():
            properties[key] = generate_mapping_from_data(value)
        mapping["properties"] = properties
    elif isinstance(data, str):
        try:
            datetime.strptime(data, '%Y-%m-%d %H:%M:%S')
            return {"type": "date", "format": "yyyy-MM-dd HH:mm:ss"}
        except ValueError:
            return {"type": "text"}
    else:
        return {"type": "text"}
    return mapping

def add_to_alias(es, index_name, alias_name):
    if not es.indices.exists_alias(name=alias_name):
        es.indices.put_alias(index=index_name, name=alias_name)
    else:
        alias_data = es.indices.get_alias(name=alias_name)
        if index_name not in alias_data:
            es.indices.update_aliases(body={
                "actions": [
                    {"add": {"index": index_name, "alias": alias_name}}
                ]
            })

def get_alias_name_from_directory(directory):
    return os.path.basename(os.path.normpath(directory))

def delete_index_if_exists(es, index_name):
    if index_exists(es, index_name):
        es.indices.delete(index=index_name)

def upload_to_elasticsearch(directory, data_list):
    es = Elasticsearch(
        [{'host': '192.168.50.99', 'port': 9200, 'scheme': 'http'}],
        basic_auth=('elastic', 'LRvs=smuDyD1b1KcUhYs')
    )
    index_name = f"truenas_directory_data_{get_alias_name_from_directory(directory).lower()}"
    root_alias_name = "truenas_directory_data"

    # Delete the index if it already exists
    delete_index_if_exists(es, index_name)

    if not index_exists(es, index_name):
        mapping = {
            "mappings": generate_mapping_from_data(data_list[0] if data_list else {})
        }
        create_index_with_mapping(es, index_name, mapping)

    # Prepare data for bulk indexing
    actions = [
        {
            "_index": index_name,
            "_source": entry
        }
        for entry in data_list
    ]

    # Use the bulk API
    bulk(es, actions)

    add_to_alias(es, index_name, root_alias_name)
    add_to_alias(es, index_name, get_alias_name_from_directory(directory))

def main():
    parser = argparse.ArgumentParser(description='Scan a directory and create a flat list of file data.')
    parser.add_argument('directory', type=str, help='The directory to scan')
    args = parser.parse_args()
    setup_logging(args.directory)
    start_time = time.time()
    flat_data = parallel_directory_walker(args.directory)

    # Sort by size and slice to keep only top 10,000 entries
    flat_data = sorted(flat_data, key=lambda x: x['size_bytes'], reverse=True)[:10000]

    logging.info(f"{args.directory} has been scanned, uploading data...")
    upload_to_elasticsearch(args.directory, flat_data)
    elapsed_time = time.time() - start_time
    logging.info(f"Data for {args.directory} uploaded to Elasticsearch.")
    logging.info(f"Total scanning time: {human_readable_time(elapsed_time)}")

if __name__ == "__main__":
    main()


import pytest
import os
from pathlib import Path

collect_ignore = ["scrape/test_get_boe_pdfs.py"]

@pytest.fixture(scope="session")
def pdf_dir(tmp_path_factory):
    basetemp = Path.cwd() / "temp_dir"
    os.environ["PYTEST_DEBUG_TEMPROOT"] = str(basetemp)
    basetemp.mkdir(parents=True, exist_ok=True)
    dir = tmp_path_factory.mktemp("pdf_files", numbered=False)
    print(dir)
    return dir


# -*- coding=utf-8
from json import load

from urllib3 import Retry
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
import sys
import logging
import time
import os
from dotenv import load_dotenv

load_dotenv()
now = time.strftime("%Y-%m-%d_%H%M%S", time.localtime())
filename = "halo-" + now + ".tar.gz"

def backup(src, dest):
    command = "tar zc --exclude=logs --exclude=.git " + src + " -f " + dest
    os.system(command)

def upload_cos(bucket, filename):
    # 正常情况日志级别使用INFO，需要定位时可以修改为DEBUG，此时SDK会打印和服务端的通信信息
    logging.basicConfig(level=logging.INFO, stream=sys.stdout)

    # 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在CosConfig中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成
    secret_id = os.getenv("secret_id")
    secret_key = os.getenv("secret_key")
    region = os.getenv("region")
    token = None
    scheme = 'https'

    config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme)
    client = CosS3Client(config, retry=5)

    # # 本地路径 简单上传
    # response = client.put_object_from_local_file(
    #     Bucket=bucket,
    #     LocalFilePath=filename,
    #     Key=filename,
    # )
    # print(response['ETag'])

    # 高级上传接口
    response = client.upload_file(
        Bucket=bucket,
        LocalFilePath=filename,
        Key=filename,
        PartSize=1,
        MAXThread=10,
        EnableMD5=False
    )

def clean_tmp(filename):
    os.remove(filename)

if __name__ == "__main__":
    try:
        backup(os.getenv("backup_dir"), filename)
        upload_cos(os.getenv("bucket"), filename)
    finally:
        clean_tmp(filename)


import torch
import torch.nn as nn
import torch.nn.functional as F

class SoftDotAttention(nn.Module):
    """Soft Dot Attention.

    Ref: http://www.aclweb.org/anthology/D15-1166
    Adapted from PyTorch OPEN NMT.
    """
    def __init__(self, dim, ctx_dim):
        """Initialize layer."""
        super(SoftDotAttention, self).__init__()
        self.linear_in = nn.Linear(dim, ctx_dim, bias=False)
        self.sm = nn.Softmax(dim=-1)
        self.linear_out = nn.Linear(ctx_dim*2, dim, bias=False)
        self.tanh = nn.Tanh()

    def forward(self, input, context, ctx_mask):
        """Propogate input through the network.

        input: batch x dim
        context: batch x sourceL x dim
        """
        # Get attention
        input = self.linear_in(input)
        attn = torch.einsum('bsd,bd->bs', (context, input))
        attn -= (1-ctx_mask) * 100000
        attn = self.sm(attn).clone()#[batch, sourceL]

        weighted_context = torch.einsum('bs,bsd->bd',(attn, context))#[batch, dim]

        h_tilde = torch.cat((weighted_context, input), 1)#[batch, dim*2]
        h_tilde = self.tanh(self.linear_out(h_tilde))#[batch, dim]

        return h_tilde, attn

class LSTMAttentionDot(nn.Module):
    r"""A long short-term memory (LSTM) cell with attention."""

    def __init__(self, input_size, hidden_size, batch_first=True):
        """Initialize params."""
        super(LSTMAttentionDot, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.num_layers = 1
        self.batch_first = batch_first

        self.input_weights = nn.Linear(input_size, 4 * hidden_size)
        self.hidden_weights = nn.Linear(hidden_size, 4 * hidden_size)

        self.attention_layer = SoftDotAttention(hidden_size, input_size)

    def forward(self, steps, hidden, ctx, ctx_mask):
        """Propogate input through the network."""
        def recurrence(hidden):
            """Recurrence helper."""
            hx, cx = hidden  # n_b x hidden_dim
            gates = self.hidden_weights(hx)
            ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)

            ingate = F.sigmoid(ingate)
            forgetgate = F.sigmoid(forgetgate)
            cellgate = F.tanh(cellgate)
            outgate = F.sigmoid(outgate)

            cy = (forgetgate * cx) + (ingate * cellgate)
            hy = outgate * F.tanh(cy)  # n_b x hidden_dim
            h_tilde, _ = self.attention_layer(hy, ctx, ctx_mask)

            return h_tilde, cy

        output = []
        for _ in range(steps):
            hidden = recurrence(hidden)
            output.append(hidden[0] if isinstance(hidden, tuple) else hidden)

        output = torch.cat(output, 0).view(steps, *output[0].size())

        if self.batch_first:
            output = output.transpose(0, 1)

        return output, hidden

class Ctx2SeqAttention(nn.Module):
    """Container module with an encoder, deocder, embeddings."""
    def __init__(
        self,
        ctx_dim,
        num_steps,
        vocab_size,
        src_hidden_dim,
        trg_hidden_dim,
        attention_mode,
        batch_size,
        pad_token_src,
        pad_token_trg,
        bidirectional=True,
        nlayers=2,
        nlayers_trg=2,
        dropout=0.,
    ):
        """Initialize model."""
        super(Ctx2SeqAttention, self).__init__()
        self.num_steps = num_steps
        self.vocab_size = vocab_size
        self.src_hidden_dim = src_hidden_dim
        self.trg_hidden_dim = trg_hidden_dim
        self.attention_mode = attention_mode
        self.batch_size = batch_size
        self.bidirectional = bidirectional
        self.nlayers = nlayers
        self.dropout = dropout
        self.num_directions = 2 if bidirectional else 1
        self.pad_token_src = pad_token_src
        self.pad_token_trg = pad_token_trg

        self.src_hidden_dim = src_hidden_dim//2 if self.bidirectional else src_hidden_dim
        self.decoder = LSTMAttentionDot(ctx_dim, trg_hidden_dim, batch_first=True)

        self.encoder2decoder = nn.Linear(self.src_hidden_dim * self.num_directions, trg_hidden_dim)
        self.decoder2vocab = nn.Linear(trg_hidden_dim, vocab_size)
        self.init_weights()

    def init_weights(self):
        self.encoder2decoder.bias.data.fill_(0)
        self.decoder2vocab.bias.data.fill_(0)

    def forward(self, ctx, state, ctx_mask):
        h_t, c_t = state.chunk(2, -1)
        decoder_init_state = nn.Tanh()(self.encoder2decoder(h_t))

        trg_h, (_, _) = self.decoder(
            self.num_steps,
            (decoder_init_state, c_t),
            ctx,
            ctx_mask
        )

        trg_h_reshape = trg_h.contiguous().view(
            trg_h.size()[0] * trg_h.size()[1],
            trg_h.size()[2]
        )

        decoder_logit = self.decoder2vocab(trg_h_reshape)
        decoder_logit = decoder_logit.view(
            trg_h.size()[0],
            trg_h.size()[1],
            decoder_logit.size()[1]
        )
        return decoder_logit

    def decode(self, logits):
        """Return probability distribution over words."""
        logits_reshape = logits.view(-1, self.vocab_size)
        word_probs = F.softmax(logits_reshape)
        word_probs = word_probs.view(
            logits.size()[0], logits.size()[1], logits.size()[2]
        )
        return word_probs


#!/usr/bin/env python

import rospy
from std_msgs.msg import String
import tf
from tf.transformations import quaternion_from_euler
import geometry_msgs.msg
import roslib
import actionlib
from move_base_msgs.msg import MoveBaseAction, MoveBaseGoal
import numpy as np
from detect_outlier import DetectOutlier
from signal_recover import SignalRecover
import math
class SmartSurvey:
    def __init__(self, survey_time, map_setting, target_list, record_path):
        self.listener = tf.TransformListener()
        self.client = actionlib.SimpleActionClient('move_base',MoveBaseAction)
        self.client.wait_for_server()
        self.survey_time = survey_time
        self.map_setting = map_setting
        self.sojourn_time = 10
        self.survey_fg = []
        self.recover_fg = None
        self.outlier_fg = None
        self.target_list = target_list.tolist()
        self.target_once = target_list.tolist()
        self.record_path = record_path

        self.chatter_sub = None
        self.register_listener()

    def callback_chatter(self, data):
        # record fingerprints
        gettedData = data.data
        (trans, rot) = self.listener.lookupTransform('/map', '/base_link', rospy.Time(0))

        gettedData = str(rospy.Time.now().secs) +","+ str(trans[0]) + "," + str(trans[1]) + gettedData
        print(gettedData)
        gettedData = gettedData.split(',')
        observe = np.zeros(len(gettedData))
        for i in range(len(gettedData)):
            observe[i] = gettedData[i]
        self.survey_fg.append(observe)

    def register_listener(self):
        self.chatter_sub = rospy.Subscriber("chatter", String, self.callback_chatter)

    def unregister_listener(self):
        if self.chatter_sub != None:
            self.chatter_sub.unregister()

    def recover(self, raw_fg):
        tinker = SignalRecover(self.map_setting, raw_fg)
        tinker.recover_process()
        return tinker.GetResult()

    def detect_outlier(self, recover_fg):
        detector = DetectOutlier(recover_fg, self.map_setting, 2, self.target_list)
        detector.process()
        self.outlier_fg = detector.GetOutlierFingerprints()
        return detector.get_target_list()

    def send_goal(self,x,y,rot = 0):
        goal = MoveBaseGoal()
        goal.target_pose.header.frame_id = "map"
        goal.target_pose.header.stamp = rospy.Time.now()
        goal.target_pose.pose.position.x = x
        goal.target_pose.pose.position.y = y
        q = quaternion_from_euler(0, 0, rot)
        goal.target_pose.pose.orientation.x = q[0]
        goal.target_pose.pose.orientation.y = q[1]
        goal.target_pose.pose.orientation.z = q[2]
        goal.target_pose.pose.orientation.w = q[3]
        print("send goal "+ str(x) + "," +str(y))
        self.client.send_goal(goal)
        wait = self.client.wait_for_result()
        if not wait:
            rospy.logerr("Action server not available!")
        else:
            rospy.loginfo("reach goal")
            return self.client.get_result()

    def survey(self):
        last_target = self.target_once[0]
        for target in self.target_once:
            rot = math.atan2(target[1]-last_target[1], target[0]-last_target[0])
            self.send_goal(target[0], target[1], rot)
            last_target = target

    def resurvey(self):
        last_target = self.target_once[0]
        for target in self.target_once:
            rot = math.atan2(target[1]-last_target[1], target[0]-last_target[0])
            self.send_goal(target[0], target[1], rot)
            last_target = target
            rospy.sleep(self.sojourn_time)

    def run(self):
        print("run")
        for i in range(self.survey_time):
            if i == 0:
                self.survey()
            else:
                self.resurvey()
            np.savetxt(self.record_path + "/survey_points" + str(i) + ".csv", self.target_once, fmt='%f', delimiter=',',
                       newline='\r\n')
            np.savetxt(self.record_path+"/survey"+str(i)+".csv", np.array(self.survey_fg), fmt='%f', delimiter=',', newline='\r\n')
            self.recover_fg = self.recover(np.array(self.survey_fg))
            np.savetxt(self.record_path+"/recover"+str(i)+".csv", self.recover_fg, fmt='%f', delimiter=',', newline='\r\n')
            resurvey_target = self.detect_outlier(self.recover_fg)
            np.savetxt(self.record_path + "/outlier" + str(i) + ".csv", self.outlier_fg, fmt='%f', delimiter=',',
                       newline='\r\n')

            if len(resurvey_target) == len(self.target_list):
                break
            else:
                self.target_once = resurvey_target[len(self.target_list):]
                self.target_list = resurvey_target

    def run_sojourn(self):
        print("run")
        for i in range(1):
            self.resurvey()
            np.savetxt(self.record_path + "/survey_points" + str(i) + ".csv", self.target_once, fmt='%f', delimiter=',',
                       newline='\r\n')
            np.savetxt(self.record_path+"/survey"+str(i)+".csv", np.array(self.survey_fg), fmt='%f', delimiter=',', newline='\r\n')
            self.recover_fg = self.recover(np.array(self.survey_fg))
            np.savetxt(self.record_path+"/recover"+str(i)+".csv", self.recover_fg, fmt='%f', delimiter=',', newline='\r\n')
            resurvey_target = self.detect_outlier(self.recover_fg)
            np.savetxt(self.record_path + "/outlier" + str(i) + ".csv", self.outlier_fg, fmt='%f', delimiter=',',
                       newline='\r\n')

            if len(resurvey_target) == len(self.target_list):
                break
            else:
                self.target_once = resurvey_target[len(self.target_list):]
                self.target_list = resurvey_target

if __name__ == '__main__':
    try:
        rospy.init_node('movebase_client_py')
        map_setting = {'ap_num': 19, 'interval': 0.8, 'origin': [0.0, 0.0], 'lost_signal': -100}

        # run sojourn
        # target_list_generated = np.loadtxt('/home/heinrich/catkin_ws/coordinate/list_3F_corridor_static.csv', dtype=float, delimiter=',')
        # survey_sojourn = SmartSurvey(1,map_setting,target_list_generated,"/home/heinrich/survey_results/3F_corridor/sojourn")
        # survey_sojourn.run_sojourn()
        # survey_sojourn.unregister_listener()

        # run test
        target_list_generated = np.loadtxt('/home/heinrich/catkin_ws/coordinate/list_3F_corridor_mobile.csv', dtype=float, delimiter=',')
        survey_test = SmartSurvey(1,map_setting,target_list_generated,"/home/heinrich/survey_results/3F_corridor/test")
        survey_test.run()
        survey_test.unregister_listener()

        # run fast survey
        target_list_generated = np.loadtxt('/home/heinrich/catkin_ws/coordinate/list_3F_corridor_mobile.csv', dtype=float, delimiter=',')
        survey_re = SmartSurvey(2,map_setting,target_list_generated,"/home/heinrich/survey_results/3F_corridor/survey")
        survey_re.run()
        survey_re.unregister_listener()
    except rospy.ROSInterruptException:
        rospy.loginfo("Navigation test finished.")


from abc import ABCMeta, abstractmethod

import datetime as dt
import h5py
import json_serialization as jsons
import logging
import numbers
import numpy as np
import os
import sys
import time

import experiment_config as ec
import hdf5_factory as hfact
import grasp
import graspable_object as go
import obj_file
import sdf_file
import feature_file
import stp_file

import IPython

INDEX_FILE = 'index.db'
HDF5_EXT = '.hdf5'
OBJ_EXT = '.obj'
STL_EXT = '.stl'
SDF_EXT = '.sdf'

READ_ONLY_ACCESS = 'READ_ONLY'
READ_WRITE_ACCESS = 'READ_WRITE'
WRITE_ACCESS = 'WRITE'

# Keys for easy lookups in HDF5 databases
METRICS_KEY = 'metrics'
OBJECTS_KEY = 'objects'
MESH_KEY = 'mesh'
SDF_KEY = 'sdf'
GRASPS_KEY = 'grasps'
GRIPPERS_KEY = 'grippers'
NUM_GRASPS_KEY = 'num_grasps'
LOCAL_FEATURES_KEY = 'local_features'
GLOBAL_FEATURES_KEY = 'global_features'
SHOT_FEATURES_KEY = 'shot'
RENDERED_IMAGES_KEY = 'rendered_images'
SENSOR_DATA_KEY = 'sensor_data'
STP_KEY = 'stable_poses'
CATEGORY_KEY = 'category'
MASS_KEY = 'mass'

CREATION_KEY = 'time_created'
DATASETS_KEY = 'datasets'
DATASET_KEY = 'dataset'

def generate_metric_tag(root, config):
    tag = '%s_f_%f_tg_%f_rg_%f_to_%f_ro_%f' %(root, config['sigma_mu'], config['sigma_trans_grasp'], config['sigma_rot_grasp'],
                                              config['sigma_trans_obj'], config['sigma_rot_obj'])
    return tag

class Database(object):
    """ Abstract class for databases. Main purpose is to wrap individual datasets """
    __metaclass__ = ABCMeta

    def __init__(self, access_level=READ_ONLY_ACCESS):
        self.access_level_ = access_level

    @property
    def access_level(self):
        return self.access_level_

class Hdf5Database(Database):
    def __init__(self, database_filename, access_level=READ_ONLY_ACCESS,
                 cache_dir='.dexnet'):
        Database.__init__(self, access_level)
        self.database_filename_ = database_filename
        if not self.database_filename_.endswith(HDF5_EXT):
            raise ValueError('Must provide HDF5 database')

        self.database_cache_dir_ = cache_dir
        if not os.path.exists(self.database_cache_dir_):
            os.mkdir(self.database_cache_dir_)

        self._load_database()
        self._load_datasets()

    def _create_new_db(self):
        """ Creates a new database """
        self.data_ = h5py.File(self.database_filename_, 'w')

        dt_now = dt.datetime.now()
        creation_stamp = '%s-%s-%s-%sh-%sm-%ss' %(dt_now.month, dt_now.day, dt_now.year, dt_now.hour, dt_now.minute, dt_now.second)
        self.data_.attrs[CREATION_KEY] = creation_stamp
        self.data_.create_group(DATASETS_KEY)

    def _load_database(self):
        """ Loads in the HDF5 file """
        if self.access_level == READ_ONLY_ACCESS:
            self.data_ = h5py.File(self.database_filename_, 'r')
        elif self.access_level == READ_WRITE_ACCESS:
            if os.path.exists(self.database_filename_):
                self.data_ = h5py.File(self.database_filename_, 'r+')
            else:
                self._create_new_db()
        elif self.access_level == WRITE_ACCESS:
            self._create_new_db()
        self.dataset_names_ = self.data_[DATASETS_KEY].keys()

    def _load_datasets(self):
        """ Load in the datasets """
        self.datasets_ = []
        for dataset_name in self.dataset_names_:
            if dataset_name not in self.data_[DATASETS_KEY].keys():
                logging.warning('Dataset %s not in database' %(dataset_name))
            else:
                dataset_cache_dir = os.path.join(self.database_cache_dir_, dataset_name)
                self.datasets_.append(Hdf5Dataset(dataset_name, self.data_[DATASETS_KEY][dataset_name],
                                                  cache_dir=dataset_cache_dir))

    @property
    def datasets(self):
        return self.datasets_

    def dataset(self, dataset_name):
        """ Returns handles to individual datasets """
        if self.datasets is None or dataset_name not in self.dataset_names_:
            return None
        for dataset in self.datasets_:
            if dataset.name == dataset_name:
                return dataset

    def close(self):
        """ Close the HDF5 file """
        self.data_.close()

    def __getitem__(self, dataset_name):
        """ Dataset name indexing """
        return self.dataset(dataset_name)

    # New dataset creation / modification functions
    def create_dataset(self, dataset_name, obj_keys=[]):
        """ Create dataset with obj keys"""
        if dataset_name in self.data_[DATASETS_KEY].keys():
            logging.warning('Dataset %s already exists. Cannot overwrite' %(dataset_name))
            return self.datasets_[self.data_[DATASETS_KEY].keys().index(dataset_name)]
        self.data_[DATASETS_KEY].create_group(dataset_name)
        self.data_[DATASETS_KEY][dataset_name].create_group(OBJECTS_KEY)
        self.data_[DATASETS_KEY][dataset_name].create_group(METRICS_KEY)
        for obj_key in obj_keys:
            self.data_[DATASETS_KEY][dataset_name][OBJECTS_KEY].create_group(obj_key)

        dataset_cache_dir = os.path.join(self.database_cache_dir_, dataset_name)
        self.dataset_names_.append(dataset_name)
        self.datasets_.append(Hdf5Dataset(dataset_name, self.data_[DATASETS_KEY][dataset_name],
                                          cache_dir=dataset_cache_dir))
        return self.datasets_[-1] # return the dataset

    def create_linked_dataset(self, dataset_name, graspable_list, nearest_neighbors):
        """ Creates a new dataset that links to objects physically stored as part of another dataset """
        raise NotImplementedError()

class Dataset(object):
    pass

class Hdf5Dataset(Dataset):
    def __init__(self, dataset_name, data, cache_dir=None,
                 start_index=0, end_index=None):
        self.dataset_name_ = dataset_name
        self.data_ = data
        self.object_keys_ = None
        self.start_index_ = 0
        self.end_index_ = end_index
        if self.end_index_ is None:
            self.end_index_ = len(self.objects.keys())

        self.cache_dir_ = cache_dir
        if self.cache_dir_ is None:
            self.cache_dir_ = os.path.join('.dexnet', self.dataset_name_)
        if not os.path.exists(self.cache_dir_):
            os.mkdir(self.cache_dir_)

    @property
    def name(self):
        return self.dataset_name_

    @property
    def dataset_root_dir(self):
        return self.dataset_root_dir_

    @property
    def objects(self):
        return self.data_[OBJECTS_KEY]

    @property
    def object_keys(self):
        if not self.object_keys_:
            self.object_keys_ = self.objects.keys()[self.start_index_:self.end_index_]
        return self.object_keys_

    @property
    def metrics(self):
        if METRICS_KEY in self.data_.keys():
            return self.data_[METRICS_KEY]
        return None

    # easy data accessors
    def object(self, key):
        return self.objects[key]

    def sdf_data(self, key):
        return self.objects[key][SDF_KEY]

    def mesh_data(self, key):
        return self.objects[key][MESH_KEY]

    def grasp_data(self, key, gripper=None):
        if gripper:
            return self.objects[key][GRASPS_KEY][gripper]
        return self.objects[key][GRASPS_KEY]

    def local_feature_data(self, key):
        return self.objects[key][LOCAL_FEATURES_KEY]

    def shot_feature_data(self, key):
        return self.local_feature_data(key)[SHOT_FEATURES_KEY]

    def stable_pose_data(self, key, stable_pose_id=None):
        if stable_pose_id is not None:
            self.objects[key][STP_KEY][stable_pose_id]
        return self.objects[key][STP_KEY]

    def category(self, key):
        return self.objects[key].attrs[CATEGORY_KEY]

    def rendered_image_data(self, key, stable_pose_id=None, image_type=None):
        if stable_pose_id is not None and image_type is not None:
            return self.stable_pose_data(key)[stable_pose_id][RENDERED_IMAGES_KEY][image_type]
        elif stable_pose_id is not None:
            return self.stable_pose_data(key)[stable_pose_id][RENDERED_IMAGES_KEY]
        elif image_type is not None:
            return self.object(key)[RENDERED_IMAGES_KEY][image_type]
        return self.object(key)[RENDERED_IMAGES_KEY]

    def metric_data(self, metric):
        if metric in self.metrics.keys():
            return self.metrics[metric]
        return None

    # iterators
    def __getitem__(self, index):
        """ Index a particular object in the dataset """
        if isinstance(index, numbers.Number):
            if index < 0 or index >= len(self.object_keys):
                raise ValueError('Index out of bounds. Dataset contains %d objects' %(len(self.object_keys)))
            obj = self.graspable(self.object_keys[index])
            return obj
        elif isinstance(index, (str, unicode)):
            obj = self.graspable(index)
            return obj

    def __iter__(self):
        """ Generate iterator """
        self.iter_count_ = self.start_index_ # NOT THREAD SAFE!
        return self

    def subset(self, start_index, end_index):
        """ Returns a subset of the dataset (should be used for iterating only) """
        return Hdf5Dataset(self.dataset_name_, self.data_, self.cache_dir_,
                           start_index, end_index)

    def next(self):
        """ Read the next object file in the list """
        if self.iter_count_ >= len(self.object_keys) or self.iter_count_ >= self.end_index_:
            raise StopIteration
        else:
            logging.info('Returning datum %s' %(self.object_keys[self.iter_count_]))
            try:
                 obj = self.graspable(self.object_keys[self.iter_count_])
            except:
                logging.warning('Error reading %s. Skipping' %(self.object_keys[self.iter_count_]))
                self.iter_count_ = self.iter_count_ + 1
                return self.next()

            self.iter_count_ = self.iter_count_ + 1
            return obj

    # direct reading / writing
    def graspable(self, key):
        """Read in the GraspableObject3D corresponding to given key."""
        if key not in self.object_keys:
            raise ValueError('Key %s not found in dataset %s' % (key, self.name))

        # read in data (need new interfaces for this....
        sdf = hfact.Hdf5ObjectFactory.sdf_3d(self.sdf_data(key))
        mesh = hfact.Hdf5ObjectFactory.mesh_3d(self.mesh_data(key))
        mass = self.object(key).attrs[MASS_KEY]
        return go.GraspableObject3D(sdf, mesh=mesh, key=key,
                                    model_name=self.obj_mesh_filename(key),
                                    mass=mass)

    def create_graspable(self, key, mesh=None, sdf=None, shot_features=None, stable_poses=None, category='', mass=1.0):
        """ Creates a graspable object """
        # create object tree
        self.objects.create_group(key)
        self.object(key).create_group(MESH_KEY)
        self.object(key).create_group(SDF_KEY)
        self.object(key).create_group(STP_KEY)
        self.object(key).create_group(LOCAL_FEATURES_KEY)
        self.object(key).create_group(GLOBAL_FEATURES_KEY)
        self.object(key).create_group(RENDERED_IMAGES_KEY)
        self.object(key).create_group(SENSOR_DATA_KEY)
        self.object(key).create_group(GRASPS_KEY)

        # add the different pieces if provided
        if sdf:
            hfact.Hdf5ObjectFactory.write_sdf_3d(sdf, self.sdf_data(key))
        if mesh:
            hfact.Hdf5ObjectFactory.write_mesh_3d(mesh, self.mesh_data(key))
        if shot_features:
            hfact.Hdf5ObjectFactory.write_shot_features(shot_features, self.local_feature_data(key))
        if stable_poses:
            hfact.Hdf5ObjectFactory.write_stable_poses(stable_poses, self.stable_pose_data(key))

        # add the attributes
        self.object(key).attrs.create(CATEGORY_KEY, category)
        self.object(key).attrs.create(MASS_KEY, mass)

        # force re-read of keys
        self.object_keys_ = None
        self.end_index_ = len(self.objects.keys())

    def obj_mesh_filename(self, key, scale=1.0, output_dir=None):
        """ Writes an obj file in the database "cache"  directory and returns the path to the file """
        mesh = hfact.Hdf5ObjectFactory.mesh_3d(self.mesh_data(key))
        mesh.rescale(scale)
        if output_dir is None:
            output_dir = self.cache_dir_
        obj_filename = os.path.join(output_dir, key + OBJ_EXT)
        of = obj_file.ObjFile(obj_filename)
        of.write(mesh)
        return obj_filename

    def stl_mesh_filename(self, key, scale=1.0, output_dir=None):
        """ Writes an stl file in the database "cache"  directory and returns the path to the file """
        obj_filename = self.obj_mesh_filename(key, scale=scale, output_dir=output_dir)
        if output_dir is None:
            output_dir = self.cache_dir_
        stl_filename = os.path.join(output_dir, key + STL_EXT)
        meshlabserver_cmd = 'meshlabserver -i \"%s\" -o \"%s\"' %(obj_filename, stl_filename)
        os.system(meshlabserver_cmd)
        return stl_filename

    # mesh data
    def mesh(self, key):
        """ Read the mesh for the given key """
        if key not in self.object_keys:
            raise ValueError('Key %s not found in dataset %s' % (key, self.name))
        return hfact.Hdf5ObjectFactory.mesh_3d(self.mesh_data(key))

    # metric data
    def create_metric(self, metric_name, metric_config):
        """ Creates a metric with the given name for easier access """
        # create metric data if nonexistent
        if self.metrics is None:
            self.data_.create_group(METRICS_KEY)

        if metric_name in self.metrics.keys():
            logging.warning('Metric %s already exists. Aborting...' %(metric_name))
            return False
        self.metrics.create_group(metric_name)

        # add configuration
        metric_group = self.metric_data(metric_name)
        for key in metric_config.keys():
            if isinstance(metric_config[key], dict):
                for k, v in metric_config[key].iteritems():
                    metric_group.attrs.create(key+'_'+k, metric_config[key][k])
            else:
                metric_group.attrs.create(key, metric_config[key])
        return True

    def has_metric(self, metric_name):
        """ Checks if a metric already exists """
        if metric_name in self.metrics.keys():
            return True
        return False

    def delete_metric(self, metric_name):
        """ Deletes a metric from the database """
        if metric_name in self.metric_names:
            del self.metrics[metric_name]

    def available_metrics(self, key, gripper='pr2'):
        """ Returns a list of available metric names """
        grasps = self.grasps(key, gripper=gripper)
        gm = self.grasp_metrics(key, grasps, gripper=gripper)
        metrics = set()
        for grasp in grasps:
            metrics.update(gm[grasp.grasp_id].keys())
        return list(metrics)

    # grasp data
    # TODO: implement handling of stable poses and tasks
    def grasps(self, key, gripper='pr2', stable_pose_id=None):
        """ Returns the list of grasps for the given graspable, optionally associated with the given stable pose """
        if gripper not in self.grasp_data(key).keys():
            logging.warning('Gripper type %s not found. Returning empty list' %(gripper))
            return []
        return hfact.Hdf5ObjectFactory.grasps(self.grasp_data(key, gripper))

    def sorted_grasps(self, key, metric, gripper='pr2', stable_pose_id=None):
        """ Returns the list of grasps for the given graspable sorted by decreasing quality according to the given metric """
        grasps = self.grasps(key, gripper=gripper, stable_pose_id=stable_pose_id)
        if len(grasps) == 0:
            return [], []

        grasp_metrics = self.grasp_metrics(key, grasps, gripper=gripper, stable_pose_id=stable_pose_id)
        if metric not in grasp_metrics[grasp_metrics.keys()[0]].keys():
            raise ValueError('Metric %s not recognized' %(metric))

        grasps_and_metrics = [(g, grasp_metrics[g.grasp_id][metric]) for g in grasps]
        grasps_and_metrics.sort(key=lambda x: x[1], reverse=True)
        sorted_grasps = [g[0] for g in grasps_and_metrics]
        sorted_metrics = [g[1] for g in grasps_and_metrics]
        return sorted_grasps, sorted_metrics

    def has_grasps(self, key, gripper='pr2', stable_pose_id=None):
        """ Checks if grasps exist for a given object """
        if gripper not in self.grasp_data(key).keys():
            return False
        return True

    def delete_grasps(self, key, gripper='pr2', stable_pose_id=None):
        """ Deletes a set of grasps associated with the given gripper """
        if gripper not in self.grasp_data(key).keys():
            logging.warning('Gripper type %s not found. Nothing to delete' %(gripper))
            return False
        del self.grasp_data(key)[gripper]
        return True

    def store_grasps(self, key, grasps, gripper='pr2', stable_pose_id=None, force_overwrite=False):
        """ Associates grasps in list |grasps| with the given object. Optionally associates the grasps with a single stable pose """
        # create group for gripper if necessary
        if gripper not in self.grasp_data(key).keys():
            self.grasp_data(key).create_group(gripper)
            self.grasp_data(key, gripper).attrs.create(NUM_GRASPS_KEY, 0)

        # store each grasp in the database
        return hfact.Hdf5ObjectFactory.write_grasps(grasps, self.grasp_data(key, gripper), force_overwrite)

    def grasp_metrics(self, key, grasps, gripper='pr2', stable_pose_id=None, task_id=None):
        """ Returns a list of grasp metric dictionaries fot the list grasps provided to the database """
        if gripper not in self.grasp_data(key).keys():
            logging.warning('Gripper type %s not found. Returning empty list' %(gripper))
            return {}
        return hfact.Hdf5ObjectFactory.grasp_metrics(grasps, self.grasp_data(key, gripper))

    def store_grasp_metrics(self, key, grasp_metric_dict, gripper='pr2', stable_pose_id=None, task_id=None, force_overwrite=False):
        """ Add grasp metrics in list |metrics| to the data associated with |grasps| """
        return hfact.Hdf5ObjectFactory.write_grasp_metrics(grasp_metric_dict, self.grasp_data(key, gripper), force_overwrite)

    def grasp_features(self, key, grasps, gripper='pr2', stable_pose_id=None, task_id=None):
        """ Returns the list of grasps for the given graspable, optionally associated with the given stable pose """
        if gripper not in self.grasp_data(key).keys():
            logging.warning('Gripper type %s not found. Returning empty list' %(gripper))
            return {}
        return hfact.Hdf5ObjectFactory.grasp_features(grasps, self.grasp_data(key, gripper))

    def store_grasp_features(self, key, grasp_feature_dict, gripper='pr2', stable_pose_id=None, task_id=None, force_overwrite=False):
        """ Add grasp metrics in list |metrics| to the data associated with |grasps| """
        return hfact.Hdf5ObjectFactory.write_grasp_features(grasp_feature_dict, self.grasp_data(key, gripper), force_overwrite)

    # stable pose data
    def stable_poses(self, key, min_p=0.0):
        """ Stable poses for object key """
        stps = hfact.Hdf5ObjectFactory.stable_poses(self.stable_pose_data(key))

        # prune low probability stable poses
        stp_list = []
        for stp in stps:
            if stp.p > min_p:
                stp_list.append(stp)
        return stp_list

    def stable_pose(self, key, stable_pose_id):
        """ Stable pose of stable pose id for object key """
        if stable_pose_id not in self.stable_pose_data(key).keys():
            raise ValueError('Stable pose id %s unknown' %(stable_pose_id))
        return hfact.Hdf5ObjectFactory.stable_pose(self.stable_pose_data(key), stable_pose_id)

    # rendered image data
    def rendered_images(self, key, stable_pose_id=None, image_type="depth"):
        if stable_pose_id is not None and stable_pose_id not in self.stable_pose_data(key).keys():
            logging.warning('Stable pose id %s unknown' %(stable_pose_id))
            return[]
        if stable_pose_id is not None and RENDERED_IMAGES_KEY not in self.stable_pose_data(key)[stable_pose_id].keys():
            logging.warning('No rendered images for stable pose %s' %(stable_pose_id))
            return []
        if stable_pose_id is not None and image_type not in self.rendered_image_data(key, stable_pose_id).keys():
            logging.warning('No rendered images of type %s for stable pose %s' %(image_type, stable_pose_id))
            return []
        if stable_pose_id is None and RENDERED_IMAGES_KEY not in self.object(key).keys():
            logging.warning('No rendered images for object')
            return []
        if stable_pose_id is None and image_type not in self.rendered_image_data(key).keys():
            logging.warning('No rendered images of type %s for object' %(image_type))
            return []

        rendered_images = hfact.Hdf5ObjectFactory.rendered_images(self.rendered_image_data(key, stable_pose_id, image_type))
        for rendered_image in rendered_images:
            rendered_image.obj_key = key
        if stable_pose_id is not None:
            stable_pose = self.stable_pose(key, stable_pose_id)
            for rendered_image in rendered_images:
                rendered_image.stable_pose = stable_pose
        return rendered_images

    def has_rendered_images(self, key, stable_pose_id=None, image_type="depth"):
        """ Checks whether or not a graspable has rendered images for the given stable pose and image type """
        if stable_pose_id is not None and stable_pose_id not in self.stable_pose_data(key).keys():
            return False
        if stable_pose_id is not None and RENDERED_IMAGES_KEY not in self.stable_pose_data(key)[stable_pose_id].keys():
            return False
        if stable_pose_id is not None and image_type not in self.rendered_image_data(key, stable_pose_id).keys():
            return False
        if stable_pose_id is None and RENDERED_IMAGES_KEY not in self.object(key).keys():
            return False
        if stable_pose_id is None and image_type not in self.rendered_image_data(key).keys():
            return False
        return True

    def delete_rendered_images(self, key, stable_pose_id=None, image_type="depth"):
        """ Delete previously rendered images """
        if self.has_rendered_images(key, stable_pose_id, image_type):
            del self.rendered_image_data(key, stable_pose_id)[image_type]
            return True
        return False

    def store_rendered_images(self, key, rendered_images, stable_pose_id=None, image_type="depth", force_overwrite=False):
        """ Store rendered images of the object for a given stable pose """
        if stable_pose_id is not None and stable_pose_id not in self.stable_pose_data(key).keys():
            raise ValueError('Stable pose id %s unknown' %(stable_pose_id))
        if stable_pose_id is not None and RENDERED_IMAGES_KEY not in self.stable_pose_data(key)[stable_pose_id].keys():
            self.stable_pose_data(key)[stable_pose_id].create_group(RENDERED_IMAGES_KEY)
        if stable_pose_id is not None and image_type not in self.rendered_image_data(key, stable_pose_id).keys():
            self.rendered_image_data(key, stable_pose_id).create_group(image_type)
        if stable_pose_id is None and RENDERED_IMAGES_KEY not in self.object(key).keys():
            self.object(key).create_group(RENDERED_IMAGES_KEY)
        if stable_pose_id is None and image_type not in self.rendered_image_data(key).keys():
            self.rendered_image_data(key).create_group(image_type)

        return hfact.Hdf5ObjectFactory.write_rendered_images(rendered_images, self.rendered_image_data(key, stable_pose_id, image_type),
                                                             force_overwrite)


import json
import math

import requests
from scipy.special import ndtri

class Ranking:
    rankingUser = 283230

    user_update_database_address = 'http://127.0.0.1:8080/api/user/{}'
    # user_update_database_address = 'http://157.253.222.182:8080/api/user/{}'
    ratings_file = '../../ml-latest/ratings.csv'
    complete_ranking_file = '../../ml-latest/ranking.json'
    movie_indexes_file = '../../ml-latest/movies-indexes.csv'
    confidence = 0.95

    def __init__(self):
        print('Creating index for movie ids and database ids')
        self.movie_database_id = {}
        with open(self.movie_indexes_file, 'r', encoding='utf-8') as movies_indexes:
            for line in movies_indexes:
                movie_index = line.split(',')
                movie_id = movie_index[0]
                db_id = movie_index[1][:-1]
                self.movie_database_id[movie_id] = db_id

        print('Creating n and total rating for every movie')
        self.movie_totals = {}
        with open(self.ratings_file, 'r', encoding='utf-8') as ratings:
            header = 0
            for line in ratings:
                if header:
                    rating = line.split(',')
                    user_id = rating[0]
                    movie_id = rating[1]
                    stars = float(rating[2])
                    if movie_id not in self.movie_totals:
                        self.movie_totals[movie_id] = (1, stars)
                    else:
                        old_n = self.movie_totals[movie_id][0]
                        old_t = self.movie_totals[movie_id][1]
                        new_n = old_n + 1
                        new_t = old_t + stars
                        self.movie_totals[movie_id] = (new_n, new_t)
                else:
                    header = 1

        print('Creating average for every movie')
        for movie in self.movie_totals:
            n = self.movie_totals[movie][0]
            t = self.movie_totals[movie][1]
            a = t / n
            self.movie_totals[movie] = (n, a)

        print('Creating pos for every movie')
        self.movie_pos = {}
        with open(self.ratings_file, 'r', encoding='utf-8') as ratings:
            header = 0
            for line in ratings:
                if header:
                    rating = line.split(',')
                    user_id = rating[0]
                    movie_id = rating[1]
                    stars = float(rating[2])
                    if movie_id not in self.movie_pos:
                        self.movie_pos[movie_id] = 0
                    average = self.movie_totals[movie_id][1]
                    if stars > average:
                        self.movie_pos[movie_id] += 1
                else:
                    header = 1

        print('Creating a single file with the data from every movie')
        self.movie_parameters = {}
        for movie in self.movie_totals:
            n = self.movie_totals[movie][0]
            p = self.movie_pos[movie]
            self.movie_parameters[movie] = (n, p)

    def ranking(self, top_n=12):
        items_ranking = {}
        print('Getting the complete ranking')
        index = 0
        for movie in self.movie_parameters:
            n = self.movie_parameters[movie][0]
            pos = self.movie_parameters[movie][1]
            if movie in self.movie_database_id:
                items_ranking[self.movie_database_id[movie]] = self.ci_lower_bound(
                    pos, n, self.confidence)

                index += 1
                if index % 1000 == 0:
                    print('Building ranking, {:.2f}% done.'.format(
                        index / 59000 * 100))
        ranking = sorted(items_ranking, key=items_ranking.get, reverse=True)

        with open(self.complete_ranking_file, 'w', encoding='utf-8') as complete_ranking:
            complete_ranking.write(json.dumps(items_ranking))

        return ['{}'.format(r) for r in ranking[:top_n]]

    def ci_lower_bound(self, pos, n, confidence):
        if n == 0:
            return 0
        z = ndtri(1-(1-confidence)/2)
        phat = 1.0*pos/n
        return (phat + z*z/(2*n) - z * math.sqrt((phat*(1-phat)+z*z/(4*n))/n))/(1+z*z/n)

    def put_ranking(self):
        ranking = self.ranking()
        newTop = ''
        for top in ranking:
            newTop += ',{}'.format(top) if newTop else '{}'.format(top)
        print('Ranking: {}'.format(newTop))
        r = requests.put(
            self.user_update_database_address.format(self.rankingUser),
            json={'toponto': newTop})

        if r.status_code >= 300:
            print("Error updating ranking")

if __name__ == '__main__':
    ranking = Ranking()
    ranking.put_ranking()