import json
import logging
import typing as t

from jsonschema import ValidationError, validate
from app.classes.controllers.users_controller import UsersController
from app.classes.models.crafty_permissions import (
    EnumPermissionsCrafty,
    PermissionsCrafty,
)
from app.classes.models.roles import HelperRoles
from app.classes.models.users import HelperUsers
from app.classes.web.base_api_handler import BaseApiHandler

logger = logging.getLogger(__name__)

class ApiUsersUserIndexHandler(BaseApiHandler):
    def get(self, user_id: str):
        auth_data = self.authenticate_user()
        if not auth_data:
            return
        (
            _,
            exec_user_crafty_permissions,
            _,
            _,
            user,
        ) = auth_data

        if user_id in ["@me", user["user_id"]]:
            user_id = user["user_id"]
            res_user = user
        elif EnumPermissionsCrafty.USER_CONFIG not in exec_user_crafty_permissions:
            return self.finish_json(
                400,
                {
                    "status": "error",
                    "error": "NOT_AUTHORIZED",
                },
            )
        else:
            # has User_Config permission and isn't viewing self
            res_user = self.controller.users.get_user_by_id(user_id)
            if not res_user:
                return self.finish_json(
                    404,
                    {
                        "status": "error",
                        "error": "USER_NOT_FOUND",
                    },
                )

        # Remove password and valid_tokens_from from the response
        # as those should never be sent out to the client.
        res_user.pop("password", None)
        res_user.pop("valid_tokens_from", None)
        res_user["roles"] = list(
            map(HelperRoles.get_role, res_user.get("roles", set()))
        )

        self.finish_json(
            200,
            {"status": "ok", "data": res_user},
        )

    def delete(self, user_id: str):
        auth_data = self.authenticate_user()
        if not auth_data:
            return
        (
            _,
            exec_user_crafty_permissions,
            _,
            _,
            user,
        ) = auth_data

        if (user_id in ["@me", user["user_id"]]) and self.helper.get_setting(
            "allow_self_delete", False
        ):
            user_id = user["user_id"]
            self.controller.users.remove_user(user_id)
        elif EnumPermissionsCrafty.USER_CONFIG not in exec_user_crafty_permissions:
            return self.finish_json(
                400,
                {
                    "status": "error",
                    "error": "NOT_AUTHORIZED",
                },
            )
        else:
            # has User_Config permission
            self.controller.users.remove_user(user_id)

        self.controller.management.add_to_audit_log(
            user["user_id"],
            f"deleted the user {user_id}",
            server_id=0,
            source_ip=self.get_remote_ip(),
        )

        self.finish_json(
            200,
            {"status": "ok"},
        )

    def patch(self, user_id: str):
        user_patch_schema = {
            "type": "object",
            "properties": {
                **self.controller.users.user_jsonschema_props,
            },
            "additionalProperties": False,
            "minProperties": 1,
        }
        auth_data = self.authenticate_user()
        if not auth_data:
            return
        (
            _,
            exec_user_crafty_permissions,
            _,
            superuser,
            user,
        ) = auth_data

        try:
            data = json.loads(self.request.body)
        except json.decoder.JSONDecodeError as e:
            return self.finish_json(
                400, {"status": "error", "error": "INVALID_JSON", "error_data": str(e)}
            )

        try:
            validate(data, user_patch_schema)
        except ValidationError as e:
            return self.finish_json(
                400,
                {
                    "status": "error",
                    "error": "INVALID_JSON_SCHEMA",
                    "error_data": str(e),
                },
            )

        if user_id == "@me":
            user_id = user["user_id"]

        if (
            EnumPermissionsCrafty.USER_CONFIG not in exec_user_crafty_permissions
            and str(user["user_id"]) != str(user_id)
        ):
            # If doesn't have perm can't edit other users
            return self.finish_json(
                400,
                {
                    "status": "error",
                    "error": "NOT_AUTHORIZED",
                },
            )

        if "username" in data:
            if data["username"].lower() in ["system", ""]:
                return self.finish_json(
                    400, {"status": "error", "error": "INVALID_USERNAME"}
                )
            if self.controller.users.get_id_by_name(data["username"]) is not None:
                return self.finish_json(
                    400, {"status": "error", "error": "USER_EXISTS"}
                )

        if "superuser" in data:
            if str(user["user_id"]) == str(user_id) and not superuser:
                # Checks if user is trying to change super user status
                # of self without superuser. We don't want that.
                return self.finish_json(
                    400, {"status": "error", "error": "INVALID_SUPERUSER_MODIFY"}
                )
            if not superuser:
                # The user is not superuser so they can't change the superuser status
                data.pop("superuser")

        if "permissions" in data:
            if str(user["user_id"]) == str(user_id) and not superuser:
                # Checks if user is trying to change permissions
                # of self without superuser. We don't want that.
                return self.finish_json(
                    400, {"status": "error", "error": "INVALID_PERMISSIONS_MODIFY"}
                )
            if EnumPermissionsCrafty.USER_CONFIG not in exec_user_crafty_permissions:
                # Checks if user is trying to change permissions of someone
                # else without User Config permission. We don't want that.
                return self.finish_json(
                    400, {"status": "error", "error": "INVALID_PERMISSIONS_MODIFY"}
                )

        if "roles" in data:
            if str(user["user_id"]) == str(user_id) and not superuser:
                # Checks if user is trying to change roles of
                # self without superuser. We don't want that.
                return self.finish_json(
                    400, {"status": "error", "error": "INVALID_ROLES_MODIFY"}
                )
            if EnumPermissionsCrafty.USER_CONFIG not in exec_user_crafty_permissions:
                # Checks if user is trying to change roles of someone
                # else without User Config permission. We don't want that.
                return self.finish_json(
                    400, {"status": "error", "error": "INVALID_ROLES_MODIFY"}
                )

        if "password" in data and str(user["user_id"] == str(user_id)):
            # TODO: edit your own password
            return self.finish_json(
                400, {"status": "error", "error": "INVALID_PASSWORD_MODIFY"}
            )

        user_obj = HelperUsers.get_user_model(user_id)

        if "roles" in data:
            roles: t.Set[str] = set(data.pop("roles"))
            base_roles: t.Set[str] = set(user_obj.roles)
            added_roles = roles.difference(base_roles)
            removed_roles = base_roles.difference(roles)
            logger.debug(
                f"updating user {user_id}'s roles: "
                f"+role:{added_roles} -role:{removed_roles}"
            )

            for role_id in added_roles:
                HelperUsers.get_or_create(user_id, role_id)

            if len(removed_roles) != 0:
                self.controller.users.users_helper.delete_user_roles(
                    user_id, removed_roles
                )

        if "permissions" in data:
            permissions: t.List[UsersController.ApiPermissionDict] = data.pop(
                "permissions"
            )
            permissions_mask = "0" * len(EnumPermissionsCrafty)
            limit_server_creation = 0
            limit_user_creation = 0
            limit_role_creation = 0

            for permission in permissions:
                self.controller.crafty_perms.set_permission(
                    permissions_mask,
                    EnumPermissionsCrafty.__members__[permission["name"]],
                    "1" if permission["enabled"] else "0",
                )

            PermissionsCrafty.add_or_update_user(
                user_id,
                permissions_mask,
                limit_server_creation,
                limit_user_creation,
                limit_role_creation,
            )

        # TODO: make this more efficient
        if len(data) != 0:
            for key in data:
                # If we don't validate the input there could be security issues
                value = data[key]
                if key == "password":
                    value = self.helper.encode_pass(value)
                setattr(user_obj, key, value)
            user_obj.save()

        self.controller.management.add_to_audit_log(
            user["user_id"],
            (
                f"edited user {user_obj.username} (UID: {user_id})"
                f"with roles {user_obj.roles}"
            ),
            server_id=0,
            source_ip=self.get_remote_ip(),
        )

        return self.finish_json(200, {"status": "ok"})


import re
import scrapy
from scrapy.http import TextResponse

from jobparser.items import JobparserItem

TEMPLATE_URL = 'https://hh.ru/search/vacancy?text='

class HhruSpider(scrapy.Spider):
    name = 'hhru'
    allowed_domains = ['hh.ru']

    def __init__(self, query, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.start_urls = [TEMPLATE_URL + query]
        self.site_for_vacancies = 'hh.ru'

    def get_salary(self, salary):
        salary_min = 0
        salary_max = 0
        salary_currency = ''

        new_salary = ''.join(salary)

        if re.search('\d+', new_salary):
            get_salary_text = new_salary.replace('\xa0', '')
            clear_salary_currency = re.sub(r'(\..*)', '', get_salary_text)
            salary_currency = re.sub(r'[дотДОТ\W0-9]', '', clear_salary_currency)
            salary_text = re.sub(r'\b\D*\B\D*', ' ', get_salary_text)
            salaries = salary_text.split(' ')
            if len(salaries) > 3:
                salary_min = int(salaries[1])
                salary_max = int(salaries[2])
            elif 'от' in get_salary_text:
                salary_min = int(salaries[1])
            elif 'до' in get_salary_text:
                salary_max = int(salaries[1])

        return {'salary_min': salary_min,
                'salary_max': salary_max,
                'salary_currency': salary_currency
                }

    def parse_item(self, response: TextResponse, title: str):
        salary = response.xpath('//div[@data-qa="vacancy-salary"]/span//text()').getall()
        salaries = self.get_salary(salary)

        item = JobparserItem()
        item['title'] = title
        item['salary_min'] = salaries['salary_min']
        item['salary_max'] = salaries['salary_max']
        item['salary_currency'] = salaries['salary_currency']
        item['url'] = response.url
        item['vacancy_for_website'] = self.site_for_vacancies
        yield item

    def parse(self, response: TextResponse, *args, **kwargs):
        items = response.xpath('//a[contains(@data-qa, "__vacancy-title")]')
        for item in items:
            url = item.xpath('./@href').get()
            title = item.xpath('.//text()').get()
            data_kwargs = {'title': title}
            yield response.follow(url, callback=self.parse_item, cb_kwargs=data_kwargs)

        next_url = response.xpath('//a[contains(@data-qa, "pager-next")]/@href').get()
        if next_url:
            yield response.follow(next_url, callback=self.parse)


from dataclasses import dataclass
from datetime import datetime, timedelta
import os
import pandas as pd
from pymongo.database import Database
from DbConnector import DbConnector
from pathlib import Path
from collections import OrderedDict, defaultdict
from pprint import pprint
from statistics import mean
from tabulate import tabulate
from haversine import haversine
from heapq import nlargest
import sys

@dataclass
class User:
    _id: str
    has_labels: bool
    activities: list

@dataclass
class Activity:
    _id: int
    user_id: str
    transportation_mode: str
    start_date_time: datetime
    end_date_time: datetime
    # trackpoints: list

@dataclass
class TrackPoint:
    _id: int
    latitude: float
    longitude: float
    altitude: int
    date_days: float
    date_time: datetime
    activity_id: int

class Database_instance:
    def __init__(self):
        self.connection = DbConnector()
        self.client = self.connection.client
        self.db = self.connection.db

    def create_collection(self, collection_name):
        collection = self.db.create_collection(collection_name)
        print('Created collection: ', collection)

    def insert_documents(self, collection_name: str, documents: list):
        """Insert array of documents

        Args:
            collection_name (str): Name of collection
            documents (list): List of documents on json/dict format
        """
        collection = self.db[collection_name]
        collection.insert_many(documents)

    def fetch_documents(self, collection_name: str, criteria={}):
        """Fetch all documents or by criteria

        Args:
            collection_name (str): Name of collection
            criteria (dict, optional): Criteria for find. Defaults to {}.
        """
        collection = self.db[collection_name]
        documents = collection.find(criteria)
        for doc in documents:
            pprint(doc)

    def drop_collection(self, collection_name: str):
        collection = self.db[collection_name]
        collection.drop()

    def show_collections(self):
        collections = self.client[self.db.name].list_collection_names()
        print(collections)

    def _fill_with_data(self):
        db_users: list(User) = []
        db_activities: list(Activity) = []
        db_trackpoints: list(TrackPoint) = []
        base_path = Path(__file__).parents[1]

        # Find all labeled users
        labeled_users = set()
        with open(base_path / "./dataset/dataset/labeled_ids.txt", "r") as f:
            for line in f.readlines():
                labeled_users.add(line.strip())

        # Find all paths to users trackings
        data_folder_path = os.path.join(base_path, "./dataset/dataset/Data")
        activity_files = defaultdict(list)
        for plt_path in Path(data_folder_path).rglob("**/*/*.plt"):
            activity_files[str(os.path.normpath(plt_path)).split(os.sep)[-3]].append(plt_path)

        activity_id = 0
        trackpoint_id = 0
        for key, value in sorted(activity_files.items()):
            is_labeled = True if key in labeled_users else False
            user_activities = []
            if is_labeled:
                labels_df = pd.read_csv(os.path.join(base_path, f"./dataset/dataset/Data/{key}/labels.txt"),
                                        sep="\t", header=1, names=["Start Time", "End Time", "Transportation Mode"])

            # For paths in users data-folder, each plt_path is a .plt file with TrackPoints
            i = 0
            for plt_path in sorted(value):
                n_trackpoints = (sum(1 for _ in open(plt_path)) - 6)  # Ignoring 6 rows of metadata at the top
                if n_trackpoints <= 2500:
                    trackpoints = OrderedDict()
                    with open(plt_path, "r") as f:
                        for index, row in enumerate(f.readlines()[6:]):
                            split_row = row.strip().split(",")
                            helper = [activity_id] + [split_row[k] for k in range(len(split_row) - 2) if k != 2]
                            date_time_str = ",".join([split_row[5], split_row[6]])
                            helper.append(date_time_str)
                            trackpoints[index] = helper

                    plt_start_datetime = datetime.strptime(trackpoints[0][5], '%Y-%m-%d,%H:%M:%S')
                    plt_end_datetime = datetime.strptime(trackpoints[len(trackpoints) - 1][5], '%Y-%m-%d,%H:%M:%S')

                    # Check if a label exactly matches the beginning and ending times of an activity.
                    # If not, don't label the activity.
                    activity_label = None
                    if is_labeled and i < len(labels_df.index):
                        label_start_datetime = datetime.strptime(labels_df["Start Time"].iloc[i], '%Y/%m/%d %H:%M:%S')
                        while plt_start_datetime > label_start_datetime and i < (len(labels_df.index) - 1):
                            label_start_datetime = datetime.strptime(labels_df["Start Time"].iloc[i + 1], '%Y/%m/%d %H:%M:%S')
                            i += 1

                        label_end_datetime = datetime.strptime(labels_df["End Time"].iloc[i], '%Y/%m/%d %H:%M:%S')
                        if label_start_datetime == plt_start_datetime and label_end_datetime == plt_end_datetime:
                            activity_label = labels_df["Transportation Mode"].iloc[i]

                    db_activities.append(Activity(activity_id, key, activity_label, plt_start_datetime, plt_end_datetime).__dict__)
                    user_activities.append(activity_id)

                    for row in trackpoints.values():
                        db_trackpoints.append(TrackPoint(_id=trackpoint_id,
                                                         activity_id=activity_id,
                                                         latitude=row[1],
                                                         longitude=row[2],
                                                         altitude=row[3],
                                                         date_days=row[4],
                                                         date_time=datetime.strptime(row[5], '%Y-%m-%d,%H:%M:%S'),
                                                         ).__dict__)
                        trackpoint_id += 1
                    activity_id += 1
            db_users.append(User(key, is_labeled, user_activities).__dict__)
        self.insert_documents(collection_name="User", documents=db_users)
        self.insert_documents(collection_name="Activity", documents=db_activities)
        self.insert_documents(collection_name="TrackPoint", documents=db_trackpoints)

def tabulate_rows(rows, column_names):
    print(tabulate(rows, headers=column_names, floatfmt=".4f"))

def task_1(db: Database):
    """
    Find the amount of users, db_activities and db_trackpoints.
    """
    # Number of users
    n_users = db.User.count_documents({})

    # Number of activities
    n_activities = db.Activity.count_documents({})

    # Number of trackpoints
    n_trackpoints = db.TrackPoint.count_documents({})

    print("\nTask 1")
    tabulate_rows([[n_users, n_activities, n_trackpoints]], ["n_users", "n_activities", "n_trackpoints"])

def task_2(db: Database):
    """
    Find the average, minimum and maximum number of activities per user.
    """
    # This need the User to have a list of activity_ids, which may be unnecessary
    users: list = db.User.find()
    n_activities = []
    for user in users:
        n_activities.append(len(user["activities"]))
    average = mean(n_activities)
    minimum = min(n_activities)
    maximum = max(n_activities)

    print("\nTask 2")
    tabulate_rows([[average, maximum, minimum]], ["average", "maximum", "minimum"])

def task_2_w_pipeline(db: Database):
    """
    Find the average, minimum and maximum number of activities per user.
    """
    # Should outer left join for including the users without activities
    pipeline = [
        {"$lookup": {
            "from": "Activity",
            "localField": "_id",
            "foreignField": "user_id",
            "as": "activities"},
         },
        {"$group": {"_id": "$_id", "count": {"$sum": {"$size": "$activities"}}}},
        {"$sort": {"_id": 1}}
    ]

    res = db.User.aggregate(pipeline)
    n_activities = []
    for row in res:
        n_activities.append(list(row.values())[1])
    average = mean(n_activities)
    minimum = min(n_activities)
    maximum = max(n_activities)

    print("\nTask 2")
    tabulate_rows([[average, maximum, minimum]], ["average", "maximum", "minimum"])

def task_3(db: Database):
    """
    Find the top 10 users with the highest number of activities.
    """
    pipeline = [
        {"$group": {"_id": "$user_id", "count": {"$sum": 1}}},
        {"$sort": {"count": -1}},
        {"$limit": 10}
    ]

    count_objects = db.Activity.aggregate(pipeline)
    print("\nTask 3")
    tabulate_rows([(obj["_id"], obj["count"]) for obj in count_objects], ["Id", "Count"])

def task_4(db: Database):
    """
    Find the number of users that have started the activity in one day and ended the activity the next day.
    """
    activities = db.Activity.find({})
    user_ids = set()
    for activity in activities:
        if (activity["end_date_time"].date() - activity["start_date_time"].date()) == timedelta(days=1):
            user_ids.add(activity["user_id"])
    print("\nTask 4")
    tabulate_rows([[len(user_ids)]], ["users with multi-day activities"])

def task_5(db: Database):
    """
    Find activities that are registered multiple times. You should find the query even if you get zero results.
    """
    # This query returns no results, so there are no activities fulfilling the requirements.
    pipeline = [
        {
            "$group": {
                "_id": {
                    "uid": "$user_id",
                    "start": "$start_date_time"
                },
                "count": {"$sum": 1}
            },
        },
        {"$match": {"count": {"$gt": 1}}},  # Find entries that have count greater than 1
    ]

    res = db.Activity.aggregate(pipeline)
    print("\nTask 5")
    tabulate_rows([row for row in res], ["activities registred multiple times"])

def task_6(db: Database):
    """
    An infected person has been at position (lat, lon) (39.97548, 116.33031) at time ‘2008-08-24 15:38:00’.
    Find the user_id(s) which have been close to this person in time and space (pandemic tracking).
    Close is defined as the same minute (60 seconds) and space (100 meters).
    """
    infected_coords = (39.97548, 116.33031)
    infected_time = datetime.strptime("2008-08-24 15:38:00", "%Y-%m-%d %H:%M:%S")

    activities_matching = set()
    trackpoints = db.TrackPoint.find({})

    def check_time_window(d1, d2):
        return True if abs(d2 - d1).total_seconds() <= 60 else False

    def check_distance(cord1, cord2):
        dist = haversine(cord1, cord2)
        closeby = True if dist <= 100 else False
        return (closeby, dist)

    for tp in trackpoints:
        if check_time_window(tp["date_time"], infected_time):
            coords = (float(tp["latitude"]), float(tp["longitude"]))
            close, _ = check_distance(coords, infected_coords)
            if close:
                activities_matching.add(tp["activity_id"])

    activities = db.Activity.find({"_id": {"$in": list(activities_matching)}})
    users_in_contact = [a["user_id"] for a in activities]
    print("\nTask 6")
    print(f"Users in contact: {users_in_contact}")

def task_7(db: Database):
    """
    Find all users that have never taken a taxi.
    """
    taxi_activities = db.Activity.distinct("user_id", {"transportation_mode": "taxi"})
    users_not_taxi = list(db.User.find({"_id": {"$nin": taxi_activities}}))

    print("\nTask 7")
    print("users: ", len(users_not_taxi))
    tabulate_rows(sorted([(row["_id"], row["has_labels"]) for row in users_not_taxi],
                         key=lambda x: -int(x[1])), ["Users that have never taken a taxi", "labeled"])

def task_8(db: Database):
    """
    Find all types of transportation modes and count how many distinct users that have used the different transportation modes.
    Do not count the rows where the transportation mode is null.
    """
    activities = db.Activity.find({})
    transportation_res = defaultdict(set)
    for activity in activities:
        transportation_res[activity["transportation_mode"]].add(activity["user_id"])
    del transportation_res[None]
    print("\nTask 8")
    tabulate_rows([(mode, len(count)) for mode, count in sorted(transportation_res.items())], ["transportation_mode", "Count"])

def task_9(db: Database):
    """
    a) Find the year and month with the most activities.
    b) Which user had the most activities this year and month, and how many recorded hours do they have?
        Do they have more hours recorded than the user with the second most activities?
    """
    # a
    pipeline = [
        {
            "$group": {
                "_id": {
                    "y": {"$year": "$start_date_time"},
                    "m": {"$month": "$start_date_time"}
                },
                "count": {"$sum": 1},
            },
        },
        {"$sort": {"count": -1}},
        {"$limit": 1}
    ]

    res = db.Activity.aggregate(pipeline)
    print("\nTask 9a")
    tabulate_rows([[row["_id"]["y"], row["_id"]["m"]] for row in res], ["Year with most activities", "Month with most activities in this year"])

    # b
    p = [
        {
            "$group": {
                "_id": {
                    "uid": "$user_id",
                    "y": {"$year": "$start_date_time"},
                    "m": {"$month": "$start_date_time"}
                },
                "count": {"$sum": 1},
                "timeSpent": {
                    "$sum": {
                        "$dateDiff": {
                            "endDate": "$end_date_time",
                            "startDate": "$start_date_time",
                            "unit": "second"
                        }
                    }
                },
            }
        },
        {"$sort": {"count": -1}},
    ]

    res = db.Activity.aggregate(p)
    top_2 = []
    while len(top_2) < 2:
        el = next(res)
        if el != None:
            if el["_id"]["y"] == 2008 and el["_id"]["m"] == 11:
                el["timeSpent"] = el["timeSpent"] / 3600
                top_2.append(el)

    print("\nTask 9b")
    tabulate_rows([[row["_id"]["uid"], row["count"], row["timeSpent"]] for row in top_2], [
                  "uid", "n_activities", "hours"])

def task_10(db: Database):
    """
    Find the total distance (in km) walked in 2008, by user with id = 112.
    """
    start_2008 = datetime(2008, 1, 1)
    end_2008 = datetime(2008, 12, 31, 23, 59, 59)
    activities_112_walk_2008 = db.Activity.find({"user_id": "112", "transportation_mode": "walk", "start_date_time": {"$gte": start_2008, "$lte": end_2008}})
    trackpoints = db.TrackPoint.find({"activity_id": {'$in': [activity["_id"] for activity in activities_112_walk_2008]}})

    dist = 0
    prev_tp = trackpoints[0]
    for tp in trackpoints[1:]:
        if tp["activity_id"] != prev_tp["activity_id"]:
            continue
        prev_coords = (float(prev_tp["latitude"]), float(prev_tp["longitude"]))
        new_coords = (float(tp["latitude"]), float(tp["longitude"]))
        dist += haversine(prev_coords, new_coords)
        prev_tp = tp
    print("\nTask 10")
    print(f"User_id: 112\nDistance in 2008: {dist}")

def task_11(db: Database):
    """
    Find the top 20 users who have gained the most altitude meters.
        ○ Output should be a table with (id, total meters gained per user).
        ○ Remember that some altitude - values are invalid
        ○ Tip: ∑(tp_{n}.altitude − tp_{n−1}.altitude), tp_{n}.altitude > tp_{n−1}.altitude
    """
    # Short helper function to sum altitude gains
    def gain_alt(alt1, alt2):
        # return (alt2 - alt1) if alt1 < alt2 else 0
        return (alt2 - alt1) * 0.3048 if alt1 < alt2 else 0 # Convert from feet to metres

    num_users = db.User.count_documents({})

    trackpoints = db.TrackPoint.aggregate([
        {"$match": {"altitude": {"$ne": -777}}},
        {"$sort": {"activity_id": 1, "_id": 1}}
    ], allowDiskUse=True)

    activities = defaultdict(dict)
    activities_from_db = db.Activity.find({})
    for a in activities_from_db:
        activities[a["_id"]] = a

    trackpoints = list(trackpoints)

    alt_per_user = [0] * num_users
    prev_tp = trackpoints[0]
    for tp in trackpoints[1:]:
        # Ensure same activity and user
        uid = int(activities[tp["activity_id"]]["user_id"])
        if tp["activity_id"] != prev_tp["activity_id"] or uid != int(activities[prev_tp["activity_id"]]["user_id"]):
            prev_tp = tp
            continue

        alt_gain = gain_alt(float(prev_tp["altitude"]), float(tp["altitude"]))
        alt_per_user[uid] += alt_gain
        prev_tp = tp

    n_highest_gainers = nlargest(20, enumerate(alt_per_user), key=lambda x: x[1])
    print("\nTask 11")
    tabulate_rows(n_highest_gainers, ["uid", "m gained"])

    return n_highest_gainers

def task_12(db: Database):
    """
    Find all users that have invalid activities, and the number of invalid activities per user
        ○ An invalid activity is defined as an activity with consecutive trackpoints
            where the timestamps deviate with at least 5 minutes.
    """
    n_users = db.User.count_documents({})
    trackpoints_activity_fields = db.TrackPoint.aggregate([
        {"$lookup": {
            "from": "Activity",
            "localField": "activity_id",
            "foreignField": "_id",
            "pipeline": [
                {"$project": {"id": 1, "user_id": 1}}
            ],
            "as": "activity_fields"}
         },
        {"$sort": {"activity_id": 1, "date_time": 1}
         },
    ], allowDiskUse=True)
    trackpoints = list(trackpoints_activity_fields)
    invalid_per_user = [0] * n_users

    invalid = False
    prev_tp = trackpoints[0]
    for tp in trackpoints[1:]:
        # Ensure same acitivity and user
        if invalid:
            if tp["activity_fields"][0]["_id"] != prev_tp["activity_fields"][0]["_id"]:
                invalid = False
                prev_tp = tp
            continue

        if tp["activity_fields"][0]["_id"] != prev_tp["activity_fields"][0]["_id"] or tp["activity_fields"][0]["user_id"] != prev_tp["activity_fields"][0]["user_id"]:
            prev_tp = tp
            continue

        time_diff = (tp["date_time"] - prev_tp["date_time"]).total_seconds()
        if time_diff >= 300:
            invalid = True
            invalid_per_user[int(tp["activity_fields"][0]["user_id"])] += 1

        prev_tp = tp

    print("\nTask 12")
    tabulate_rows([(i, invalid_per_user[i]) for i in range(n_users)], ["uid", "Invalid activities"])

def init_db(instance: Database_instance):
    collections = ["User", "Activity", "TrackPoint"]

    for collection in collections:
        instance.drop_collection(collection_name=collection)
        instance.create_collection(collection_name=collection)

    instance.show_collections()
    instance._fill_with_data()

def task_0(db: Database):
    """
    This prints out the first 10 documents in each collection for part 1
    """
    rows_user = [f"\u007b'_id': {x['_id']}, 'has_labels': {x['has_labels']}, 'activities': [...]\u007d" for x in db.User.find({}).limit(10)]
    rows_activity = [x for x in db.Activity.find({}).limit(10)]
    rows_trackpoint = [x for x in db.TrackPoint.find({}).limit(10)]
    print("\nUser:")
    for row in rows_user:
        print(row)
    print("\nActivity:")
    for row in rows_activity:
        print(row)
    print("\nTrackPoint:")
    for row in rows_trackpoint:
        print(row)

if __name__ == '__main__':
    db_instance = None
    try:
        while len(sys.argv) == 1:
            ans = input("Which task you want to run? [i: init db, a: all, 1..12: task 1..12]: ")
            if ans == "a" or ans == "i" or ans.isdigit() and int(ans) in range(1, 13):
                sys.argv.append(ans)

        db_instance = Database_instance()
        tasks = [task_1,
                 task_2,
                 task_3,
                 task_4,
                 task_5,
                 task_6,
                 task_7,
                 task_8,
                 task_9,
                 task_10,
                 task_11,
                 task_12,
                 task_0]

        # db_instance.show_collections()
        # Run tasks
        if sys.argv[1] == "i":
            init_db(db_instance)
        elif sys.argv[1] == "a":
            for task in tasks:
                task(db_instance.db)
        elif int(sys.argv[1]) in range(0, 13):
            tasks[int(sys.argv[1]) - 1](db_instance.db)
        else:
            print(f"Wrong task argument, argv = {sys.argv}")

    finally:
        if db_instance:
            db_instance.connection.close_connection()

TDT4225 - Very Large, Distributed Data Volumes/Assignment 3 w:Jonas 25%/src/GeoLifeProgram.py


#!/usr/bin/python3

#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <https://www.gnu.org/licenses/>.

import sys
import time
import requests
import socket
import threading

from aiohttp import web

IP = '127.0.0.1'
PORT = 5005

#executing_command = False
#
#def exec_ros_cmd_status():
#  global executing_command
#  try:
#    status, addr = s.recvfrom(1024)
#    executing_command = False
#  except:
#    executing_command = False

async def http_handler(request):
  if request.rel_url.path == "/":
    return web.FileResponse("index.html")
  else:
    return web.FileResponse("imgs/" + request.rel_url.path[1:])

async def wshandle(request):
#  global executing_command

  ws = web.WebSocketResponse()

  while(True):
    try:
      await ws.prepare(request)
      async for message in ws:
        print("[HTTPD] Message:", message)
        str_split = message.data.split(',')
        str_split = [x.rstrip() for x in str_split]

#        if executing_command == False:
        if str_split[0] == "start-replay":
          requests.post("http://replay:5000/lastminute", timeout=1)

        elif str_split[0] == "stop-replay":
          requests.delete("http://replay:5000/lastminute", timeout=1)

        elif str_split[0] == "control" or str_split[0] == "moveit" or str_split[0] == "interface" or str_split[0] == "calibration" or str_split[0] == "joystick":
#          executing_command = True
          s.sendto(message.data.encode(), (IP, PORT))
#            # FIXME: Big hammer here...to be fixed properly...and use mutexes
#            threading.Thread(target = exec_ros_cmd_status).start()
    except RuntimeError:
      return ws
    except:
#      executing_command = False
      pass

if __name__=="__main__":
  s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

  app = web.Application()
  if sys.version_info > (3, 5, 3):
    app.add_routes([web.get("/", http_handler),
                    web.get("/arm-topview.png", http_handler),
                    web.get("/bottom-topview.png", http_handler),
                    web.get("/bottom-sideview.png", http_handler),
                    web.get("/middle-sideview.png", http_handler),
                    web.get("/top-sideview.png", http_handler),
                    web.get("/gripper-sideview-open.png", http_handler),
                    web.get("/gripper-sideview-closed.png", http_handler),
                    web.get("/rotate.png", http_handler),
                    web.get("/niryo-one-web", wshandle),
					web.get("/5G-Dive_rgb_horizontal.png",http_handler)])
  else:
    app.router.add_get("/", http_handler)
    app.router.add_get("/arm-topview.png", http_handler)
    app.router.add_get("/bottom-topview.png", http_handler)
    app.router.add_get("/bottom-sideview.png", http_handler)
    app.router.add_get("/middle-sideview.png", http_handler)
    app.router.add_get("/top-sideview.png", http_handler)
    app.router.add_get("/gripper-sideview.png", http_handler)
    app.router.add_get("/rotate.png", http_handler)
    app.router.add_get("/niryo-one-web", wshandle)
    app.router.add_get("/5G-Dive_rgb_horizontal.png",http_handler)

  web.run_app(app)


from csv import reader

from typing import List
from leitor_csv.data_classes import Aluno, Disciplina, Historico, Professor, Turma

class LeitorCSV:

    alunos: List[Aluno] = []
    displinas: List[Disciplina] = []
    historico: List[Historico] = []
    professores: List[Professor] = []
    turmas: List[Turma] = []

    @staticmethod
    def ler_dados():
        LeitorCSV.ler_csv_alunos()
        LeitorCSV.ler_csv_disciplinas()
        LeitorCSV.ler_csv_historico()
        LeitorCSV.ler_csv_professores()
        LeitorCSV.ler_csv_turmas()

    @staticmethod
    def ler_csv_alunos() -> None:
        with open("data/Alunos.csv", encoding="utf-8") as file:
            csv_reader = reader(file)
            csv_reader.__next__()  # pulando a primeira linha do arquivo
            for line in csv_reader:
                matricula, nome, semestre, data_nasc = line[0], line[1], line[2], line[3]
                LeitorCSV.alunos.append(
                    Aluno(matricula, nome, semestre, data_nasc))

    @staticmethod
    def ler_csv_disciplinas() -> None:
        with open("data/Disciplinas.csv", encoding="utf-8") as file:
            csv_reader = reader(file)
            csv_reader.__next__()  # pulando a primeira linha do arquivo
            for line in csv_reader:
                codigo_disc, nome_disc, creditos = line[0], line[1], line[2]
                LeitorCSV.displinas.append(Disciplina(
                    codigo_disc, nome_disc, creditos))

    @staticmethod
    def ler_csv_historico() -> None:
        with open("data/Historico.csv", encoding="utf-8") as file:
            csv_reader = reader(file)
            csv_reader.__next__()  # pulando a primeira linha do arquivo
            for line in csv_reader:
                codigo_historico = line[0]
                matricula_aluno = line[1]
                codigo_turma = line[2]
                codigo_disc = line[3]
                codigo_professor = line[4]
                ano = line[5]
                frequencia = line[6]
                nota = line[7]
                LeitorCSV.historico.append(
                    Historico(
                        codigo_historico, matricula_aluno, codigo_turma,
                        codigo_disc, codigo_professor, ano, frequencia, nota
                    )
                )

    @staticmethod
    def ler_csv_professores() -> None:
        with open("data/Professores.csv", encoding="utf-8") as file:
            csv_reader = reader(file)
            csv_reader.__next__()  # pulando a primeira linha do arquivo
            for line in csv_reader:
                codigo_professor, nome, area_pesquisa = line[0], line[1], line[2]
                LeitorCSV.professores.append(
                    Professor(codigo_professor, nome, area_pesquisa))

    @staticmethod
    def ler_csv_turmas() -> None:
        with open("data/Turmas.csv", encoding="utf-8") as file:
            csv_reader = reader(file)
            csv_reader.__next__()  # pulando a primeira linha do arquivo
            for line in csv_reader:
                codigo_turma, codigo_disc, codigo_professor, ano, horario = line[
                    0], line[1], line[2], line[3], line[4]
                LeitorCSV.turmas.append(
                    Turma(codigo_turma, codigo_disc, codigo_professor, ano, horario))


#!/usr/bin/env python3
import sys
import json
import pprint
from cHash import c_hash

pp = pprint.PrettyPrinter(indent=4)
HASH_COUNT = 6

def csv_tuple_to_dict(csv):
    return dict((a, b) for a,b in [x.split('=') for x in csv.split(',')])

fields = {'saddr', 'daddr', 'sport', 'dport', 'proto'}
def five_tuples_are_equal(flow1, flow2):
    flow1_d, flow2_d = json.loads(flow1), json.loads(flow2)
    return all(flow1_d[field] == flow2_d[field] for field in fields)

def flow_seen_in_flow_set(flow, flows):
    return any(five_tuples_are_equal(flow, seen_flow) for seen_flow in flows)

def hex_flow_counts_to_integers(flow):
    to_modify = json.loads(flow)
    to_modify['packet_count'] = int(to_modify['packet_count'], 16)
    to_modify['flow_count'] = int(to_modify['flow_count'], 16)

    return json.dumps(to_modify)

def merge_flows(flow1, flow2):
    flow1_d, flow2_d = json.loads(flow1), json.loads(flow2)
    flow1_d['packet_count'] += flow2_d['packet_count']
    flow1_d['flow_count'] += flow2_d['flow_count']

    return json.dumps(flow1_d)

def merge_cpu_flows(cpu_flows):
    merged_flows = []
    for flow in cpu_flows:
        matching_flows = list(filter(lambda f: five_tuples_are_equal(f, flow), cpu_flows))
        matching_flows.remove(flow) # The current flow should not match itself
        merged_flow = f'{flow}'
        for matching_flow in matching_flows:
            merged_flow = merge_flows(merged_flow, matching_flow)
        merged_flows.append(merged_flow)

    return set(merged_flows) # Remove duplicate merged flows

def singledecode(flow_info, hosts):
    all_cpus_identified_flows = set()
    for cpu in flow_info.keys():
        identified_flows = set()
        merged_flows = flow_info[str(cpu)]
        usable_flows = {hash_value:  csv_tuple_to_dict(flow) for hash_value, flow in merged_flows.items()}

        for hash_value, flow in usable_flows.items():
            if flow['flow_count'] == '0x1':
                identified_flows.add(json.dumps(flow))
                packet_count = int(flow['packet_count'], 16)
                saddr, daddr = int(flow['saddr'], 16), int(flow['daddr'], 16)
                sport, dport = int(flow['sport'], 16), int(flow['dport'], 16)
                proto, host = int(flow['proto'], 16), int(hosts[int(cpu)], 16)
                fields = {'saddr': saddr, 'daddr': daddr, 'sport': sport, 'dport': dport, 'proto': proto}
                hashes = [
                    c_hash(saddr, daddr, sport, dport, proto, host, k)
                    for k in range(0, HASH_COUNT)
                ]
                print(f'Hashes were: {list(map(hex, hashes))}')
                if not int(hash_value, 16)  in hashes:
                    print(f'The given hash WAS NOT in the calculated hashes.')
                for kth_hash in map(hex, hashes):
                    if usable_flows[kth_hash]:
                        for field, value in fields.items():
                            usable_flows[kth_hash][field] = hex(int(usable_flows[kth_hash][field], 16) ^ value)

                        usable_flows[kth_hash]['flow_count'] = hex(int(str(usable_flows[kth_hash]['flow_count']), 16) - 1)
                        usable_flows[kth_hash]['packet_count'] = hex(int(str(usable_flows[kth_hash]['packet_count']), 16) - packet_count)
                    else:
                        print('Hash not found in merged flow')
            elif flow['flow_count'] == '0x0':
                # Completed removing flow count from a bin, skip over this
                pass
            else:
                # unresolvable conflict (within a CPU) detected
                # We should probably combine info from multiple CPUs here
                # Here we need to handle flows with conflicts
                # Probably repeat the loop with the remaining items?
                print(f'Conflict detected for flow={flow}')
        all_cpus_identified_flows |= identified_flows

    print('All CPUs flows:')
    cpu_flows_integers = list(map(hex_flow_counts_to_integers, all_cpus_identified_flows))
    pp.pprint(cpu_flows_integers)
    print('\nAll merged flows:')
    merged_flows = merge_cpu_flows(cpu_flows_integers)
    pp.pprint(merged_flows)

def parse_flow_info(flow_info, hosts):
    singledecode(flow_info, hosts)

def load_hosts(hosts):
    return dict((int(number), host['0x0'].split('=')[1]) for number, host in hosts.items())

def parse_bloomfilter(bloomfilter):
    print('Tried to parse bloomfilter')
    pass

def main(flowfile, outfile):
    flows = {}
    with open(flowfile, 'r') as f:
        flows = json.load(f)

    if not flows['bloomfilter'] or not flows['flow_info'] or not flows['host_info']:
        print('Bloom filter, host or flow info missing from dumped json')

    # bloomfilter_summary = parse_bloomfilter(flows['bloomfilter'])
    hosts = load_hosts(flows['host_info'])
    parse_flow_info(flows['flow_info'], hosts)

    # Write output in useful format to outfile

if __name__ == '__main__':
    if len(sys.argv) < 2:
       print('usage: %s flow_data_file' % sys.argv[0])
       sys.exit(1)
    main(sys.argv[1], sys.argv[2])


"""
Rename wheelodex_version

Revision ID: 8820a6b34f40
Revises: b835c6894d5a
Create Date: 2018-10-12 20:33:03.002855+00:00
"""

from __future__ import annotations
from alembic import op

# revision identifiers, used by Alembic.
revision = "8820a6b34f40"
down_revision = "b835c6894d5a"
branch_labels: None = None
depends_on: None = None

def upgrade() -> None:
    op.alter_column(
        "wheel_data",
        "wheelodex_version",
        new_column_name="wheel_inspect_version",
    )

def downgrade() -> None:
    op.alter_column(
        "wheel_data",
        "wheel_inspect_version",
        new_column_name="wheelodex_version",
    )


import gensim
import pickle
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
import nltk
import os
import utils
from gensim.models import Word2Vec
import sys
from nltk.tag import pos_tag

data_dir=sys.argv[1]
model_path=sys.argv[2]
pretrained_vectors=sys.argv[3]
context_window=int(sys.argv[4])
print(context_window)

x=[]
for dir,subdir,files in os.walk(data_dir):
    for file in files:
        file_path=os.path.join(dir,file)
        proc=utils.read_data(file_path)
        x+=proc
        print(file_path)
print(len(x))

aux_data={}
def add(w1,w2):
    key=w1+"_"+w2
    key2=w2+"_"+w1
    if key not in aux_data:
        aux_data[key]=0
    if key2 not in aux_data:
        aux_data[key2]=0
    aux_data[key]+=1
    aux_data[key2]+=1

def is_prop_noun(tag):
    if (tag[1]=='NNP' or tag[1]=='NNPS'):
        return True
    return False

counter=0
for line in x:
    if(counter%5000==0):
        print(counter)
    le=len(line)
    pos_tag=nltk.pos_tag(line)
    if(len(line)!=len(pos_tag)):
        print(len(line),len(pos_tag))
        print(line)
        print(pos_tag)
        exit(0)
    counter+=1

    for i in range(le):
        line[i]=line[i].lower()
    for i in range(le):
        w1=line[i]
        for j in range(1,context_window+1,1):
            if i+j<le:
                w2=line[i+j]
                add(w1,w2)
                if is_prop_noun(pos_tag[i]):
                    add('-pro-',w2)
                if is_prop_noun(pos_tag[j]):
                    add(w1,'-pro-')

utils.dump_pickle(dump=aux_data,filename="aux_ds"+str(context_window)+".pkl")
print("Created Aux DS")

def my_train(x,size,epochs,model,model_path,pret=False):
    model_2 = Word2Vec(size=300, min_count=1,workers=1,sg=1)
    model_2.build_vocab(x)
    total_examples = model_2.corpus_count
    print(total_examples)
    print(len(list(model_2.wv.vocab.keys())))
    print(list(model_2.wv.vocab.keys())[0:10])
    total_examples = model_2.corpus_count
    if pret:
        model_2.intersect_word2vec_format(pretrained_vectors, binary=True, lockf=1.0)
    model_2.train(x, total_examples=total_examples, epochs=epochs,report_delay=20)
    utils.dump_pickle(model_2,model_path)

model=None
model = gensim.models.KeyedVectors.load_word2vec_format(pretrained_vectors, binary=True)
print("Loaded Google pretrained_vectors")
my_train(x,300,10,model,model_path,True)


from threading import current_thread

from django.utils.deprecation import MiddlewareMixin

_requests = {}

def get_request(*args, **kwargs):
    t = current_thread()
    if t in _requests:
        return _requests[t]

class UserRequestMiddleware(MiddlewareMixin):
    def process_request(self, request):
        _requests[current_thread()] = request


from numba import njit
from multiprocessing import Process
from time import sleep

@njit
def fun(x, y):
    return x + y

def process(i, xs, ys):
    for x, y in zip(xs, ys):
        print(f"Process [{i}]", x, y, fun(x, y))
        sleep(0.2)

if __name__ == "__main__":
    procs = []
    for i in range(5):
        procs.append(
            Process(target=process, args=(i, list(range(10)), list(range(10))))
        )
        procs[-1].start()

    for proc in procs:
        proc.join()


from django.shortcuts import render
# import requests to get websites
import requests

# Create your views here.
def Movies_list(request):
# to add the base website that we will request information from.
	query = request.GET.get("q")
	url = "http://www.omdbapi.com/?apikey=37c1229d&s=king"
	if query:
		url = "http://www.omdbapi.com/?apikey=37c1229d&s=" + query
	response = requests.get(url)
	print(response.json())

	context = {
		"movies": response.json()
	}

	return render(request, 'movies_list.html', context)

def movie_details(request, movie_id):
	url = "http://www.omdbapi.com/?apikey=37c1229d&i=" + movie_id
	response = requests.get(url)

	context = {
		"movie": response.json()
	}
	return render(request, 'movie_details.html', context)

# def movie_search(requests, movie_id):


import numpy as np
from matplotlib import pyplot as plt
from noiseestimation.playback_sensor import PlaybackSensor
from simple_bicycle_ekf import SimpleBicycleEKF

# Parameters
num_samples = 11650
steering_to_wheel_angle = 0.067
lf = 1.45      # distance center of gravity (cog) <-> front axle
lb = 1.42      # distance cog <-> rear axle [m]
wheelbase = lf + lb
var_vel = 0.001
var_steer = 0.0001
var_measurement = 0.01
var_sim = 0.0001
dt = 0.01

def setup():
    sim = PlaybackSensor("data/vehicle_state_integrated.json",
                         ["fVx", "fStwAng", "pose_x", "pose_y"])
    # set up kalman filter
    tracker = SimpleBicycleEKF(dt, wheelbase, var_vel, var_steer)
    tracker.Q = np.eye(2) * 0.001
    tracker.H = np.array([[1, 0, 0],
                          [0, 1, 0]])
    tracker.R = var_measurement
    tracker.x = np.array([[0, 0, 0]]).T
    tracker.P = np.eye(3) * 500

    return sim, tracker

def filtering(sim, tracker):
    # perform sensor simulation and filtering
    readings = []
    filtered = []
    R = [[var_sim]]
    for _ in range(num_samples):
        time, reading = sim.read(R)
        velocity = reading[0, 0]
        delta = reading[1, 0] * steering_to_wheel_angle
        velocity = velocity if velocity > 0 else 0.1
        tracker.predict(u=np.array([[velocity],
                                    [delta]]))
        tracker.update(reading[2:])
        readings.append(reading)
        filtered.append(tracker.x)

    readings = np.asarray(readings)
    filtered = np.asarray(filtered)
    return readings, filtered

def plot_results(readings, filtered):
    plt.plot(
        filtered[:, 0],
        filtered[:, 1],
        "b-"
    )
    plt.show()

def run_tracker():
    sim, tracker = setup()
    readings, filtered = filtering(sim, tracker)
    plot_results(readings, filtered)

if __name__ == "__main__":
    run_tracker()


from ast import AST
import os
from pathlib import Path
from torch.utils.data import DataLoader
from src.data_utils.teacher_precompute import PrecomputeDataset, get_pretrained_ast, _prepare_dataset_json, _precompute_batch, precompute_teacher_preds
from src.ast.src.dataloader import AudiosetDataset
from src.constants import AST_MODEL_PATH, AUDIOS_PATH, INFERENCE_AUDIO_CONFIG, LABEL_CSV, LABEL_LIST

os.environ['TORCH_HOME'] = 'checkpoints'

def test_get_pretrained_ast():
    pretrained_mdl_path = AST_MODEL_PATH
    audio_model, _ = get_pretrained_ast(pretrained_mdl_path)
    assert audio_model is not None

def test_prepare_dataset_json():
    audios_path = AUDIOS_PATH
    dataset_json_file = _prepare_dataset_json(audios_path, 'testing')

    dataset = AudiosetDataset(dataset_json_file, label_csv=LABEL_CSV, audio_conf=INFERENCE_AUDIO_CONFIG)
    dataloader = DataLoader(dataset, batch_size=2, num_workers=1)

    for audio_inputs, labels in dataloader:
        assert audio_inputs is not None

def test_precompute():
    audios_path = AUDIOS_PATH
    pretrained_mdl_path = AST_MODEL_PATH
    audio_model, device = get_pretrained_ast(pretrained_mdl_path)

    batch_size = 2
    dataset = PrecomputeDataset(audios_path)
    dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=1)

    for audio_input, _, _ in dataloader:
        predictions = _precompute_batch(audio_model, audio_input, device)
        assert predictions.shape == (batch_size, len(LABEL_LIST))
        break


from django.utils import timezone
from rest_framework import serializers
from rest_framework.exceptions import ValidationError

from config.models import Config
from transaction.models import Transaction, Payment
from utils.rest_framework.serializers import Base64ImageField

class TransactionSerializer(serializers.ModelSerializer):
    class Meta:
        model = Transaction
        fields = "__all__"

class TransactionBalance(serializers.ModelSerializer):
    class Meta:
        model = Transaction
        fields = (
            'id',
            'account',
            'desc',
            'method',
            'coin',
            'status',
            'values',
            'thb_values',
            'rate_per_usdt',
            'datetime_create'
        )

class TransactionRetrieveSearializer(serializers.ModelSerializer):
    class Meta:
        model = Transaction
        fields = (
            'id',
            'account',
            'transaction_hash',
            'source_key',
            'destination_key',
            'values',
            'thb_values',
            'paid_by',
            'coin',
            'method',
            'status',
            'datetime_update',
            'datetime_complete'
        )

class PaymentResultSerializer(serializers.ModelSerializer):
    class Meta:
        model = Payment
        fields = "__all__"

class TransactionPaymentSerializer(serializers.ModelSerializer):
    payment = serializers.SerializerMethodField()

    class Meta:
        model = Transaction
        fields = (
            'id',
            'desc',
            'method',
            'coin',
            'status',
            'values',
            'thb_values',
            'rate_per_usdt',
            'datetime_create',
            'payment'
        )

    def get_payment(self, transaction):
        from transaction.models import Payment
        payment = Payment.objects.filter(transaction_id=transaction.id).first()
        return PaymentResultSerializer(payment).data

class TransactionTopUp(serializers.Serializer):
    token_name = serializers.CharField(required=True)
    thb_amount = serializers.FloatField(required=True)
    coin_amount = serializers.FloatField(required=True)
    is_own_account = serializers.BooleanField(default=False)
    is_not_reuse = serializers.BooleanField(default=False)
    is_not_overtime = serializers.BooleanField(default=False)

class PaymentUpdatePaySlip(serializers.ModelSerializer):
    payment_slip = Base64ImageField(allow_empty_file=True, allow_null=True, required=True)
    datetime_stamp = serializers.DateTimeField(required=True)

    class Meta:
        model = Payment
        fields = (
            'id',
            'payment_slip',
            'datetime_stamp'
        )

    def update(self, payment, validated_data):
        if 'payment_slip' in validated_data:
            if validated_data['payment_slip']:
                validated_data['payment_slip_log'] = validated_data['payment_slip']
        payment.datetime_stamp = timezone.now()
        payment.save()
        return super().update(payment, validated_data)

class TransactionActivityCreateSerializer(serializers.Serializer):
    account = serializers.SerializerMethodField()
    source_key = serializers.SerializerMethodField()
    destination_key = serializers.SerializerMethodField()
    values = serializers.SerializerMethodField()
    thb_values = serializers.SerializerMethodField()
    paid_by = serializers.SerializerMethodField()
    coin = serializers.SerializerMethodField()
    method = serializers.SerializerMethodField()
    activity_type = serializers.IntegerField()
    desc = serializers.CharField()
    code = serializers.CharField()

    class Meta:
        model = Transaction
        fields = (
            'id',
            'account',
            'source_key',
            'destination_key',
            'method',
            'coin',
            'status',
            'values',
            'thb_values',
            'rate_per_usdt',
            'datetime_create',
            'datetime_update',
            'activity_type',
            'desc',
            'code'
        )

    def create(self, validated_data):
        from account.models import Account
        from activity.models import CarbonActivity
        if 'account' in validated_data and not validated_data['account']:
            raise ValidationError('account is invalid.')
        is_account = Account.objects.filter(id=validated_data.get('account'), is_active=True).exists()
        if not is_account:
            raise ValidationError('account is invalid.')

        if 'method' in validated_data and not validated_data['method']:
            raise ValidationError('account is invalid.')

        method = validated_data.get('method')
        if method and not isinstance(method, int):
            raise ValidationError('method is invalid.')

        if int(method) != 4:
            raise ValidationError('method is match with type.')

        if 'activity_type' in validated_data and not validated_data['activity_type']:
            raise ValidationError('Activity type is invalid.')

        activity_type = validated_data.get('activity_type')
        if activity_type and not isinstance(activity_type, int):
            raise ValidationError('Activity type is invalid.')

        if int(activity_type) not in CarbonActivity.TYPE_CHOICE:
            raise ValidationError('method is match with type.')
        return super().create(validated_data)

class TransactionUpdateStatusSerializer(serializers.ModelSerializer):
    status = serializers.IntegerField(default=0)
    desc = serializers.CharField()

    class Meta:
        model = Transaction
        fields = (
            'id',
            'status',
            'desc'
        )

    def update(self, transaction, validated_data):
        from transaction.models import Payment
        if 'status' in validated_data:
            # upload new avatar
            if not validated_data['status']:
                raise ValidationError('status is required.')
            if not isinstance(validated_data['status'], int):
                raise ValidationError('Wrong format type status.')
            if int(validated_data['status']) not in dict(Transaction.STATUS_CHOICES):
                raise ValidationError('Status not match with status in system.')
            if int(validated_data['status']) == 2:
                payment_transaction = Payment.objects.filter(transaction_id=transaction.id).first()
                if not payment_transaction.payment_slip:
                    raise ValidationError(
                        'This transaction can not complete, because payment of this transaction does not pay.'
                    )
        return super().update(transaction, validated_data)

class TransactionSwapReceiver(serializers.Serializer):
    sol_amount = serializers.FloatField(required=True)


import json
import csv
from parser import get_people_data, get_book_data
from sorting import get_equals_books, add_books_to_people

people_list = []
with open("users.json", "r") as file:
    people = json.load(file)
for person in people:
    people_list.append(get_people_data(person))

books_list = []
with open("books.csv", "r") as file:
    books = list(csv.DictReader(file))
for book in books:
    books_list.append(get_book_data(book))

equals_books_gen = get_equals_books(books_list, len(people_list))

people_with_books = []
for person in people_list:
    people_with_books.append(add_books_to_people(person, next(equals_books_gen)))

with open("result.json", "w") as outfile:
    json.dump(people_with_books, outfile, indent=4)


import wx

from .defaults import DEFAULT_COLUMN_WIDTH
from .worker import WorkerThread
from ..wallet import Wallet

CONFIRMED = 'Confirmed'
PENDING = 'Pending'
RETRYING = 'Retrying'

class HistoryTab(wx.Panel):  # type: ignore
    def __init__(self,
                 parent: wx.Window,
                 wallet: Wallet) -> None:
        super().__init__(parent)
        self.wallet = wallet
        self.worker = WorkerThread()
        self.init_popup_menu()
        self.create_history_tab()

    def create_history_tab(self) -> None:
        self.transactions = wx.ListCtrl(self, style=wx.LC_REPORT)
        self.transactions.InsertColumn(0, 'Transactions')
        self.transactions.InsertColumn(1, 'Status')

        for bundle in self.wallet.bundles['unconfirmed']:
            self.transactions.Append([str(bundle.hash), PENDING])
        for bundle in self.wallet.bundles['confirmed']:
            self.transactions.Append([str(bundle.hash), CONFIRMED])

        if self.transactions.GetItemCount() > 0:
            column_width = wx.LIST_AUTOSIZE
        else:
            column_width = DEFAULT_COLUMN_WIDTH

        self.transactions.SetColumnWidth(0, column_width)
        self.transactions.SetColumnWidth(1, column_width)

        vbox = wx.BoxSizer(wx.VERTICAL)
        vbox.Add(self.transactions, proportion=1, flag=wx.EXPAND)

        hbox = wx.BoxSizer(wx.HORIZONTAL)
        hbox.Add(vbox, proportion=1, flag=(wx.EXPAND | wx.ALL), border=10)

        self.Bind(wx.EVT_LIST_ITEM_RIGHT_CLICK, self.transactions_right_click, self.transactions)
        self.Bind(wx.EVT_MENU, self.popup_menu_clicked)

        self.SetSizer(hbox)

    def init_popup_menu(self) -> None:
        self.popup_menu = wx.Menu()
        self.popup_menu.Append(0, 'Retry Bundle')

    def popup_menu_clicked(self,
                           event: wx.ContextMenuEvent) -> None:
        index = event.GetId()
        if index == 0:
            # Retry Bundle
            bundle_hashes = []
            list_index = self.transactions.GetFirstSelected()
            while list_index != -1:
                if self.transactions.GetItemText(list_index, col=1) == PENDING:
                    bundle_hashes.append(self.transactions.GetItemText(list_index))
                    list_index = self.transactions.GetNextSelected(list_index)

            print(bundle_hashes)
            bundles = []
            for bundle in self.wallet.bundles['unconfirmed']:
                if bundle.hash in bundle_hashes:
                    bundles.append(bundle)

            self.worker.send(self.wallet.retry_unconfirmed_bundles, args=tuple(bundles))

    def transactions_right_click(self,
                                 event: wx.ListEvent) -> None:
        if event.GetText():
            self.PopupMenu(self.popup_menu)


import numpy as np
import pytest
import torch

from saliency_metrics.metrics import RoarPerturbation

@pytest.fixture
def dummy_img_and_smap():
    # two single-channel images, the first one has channel mean of 5.0, and the second one has channel mean of 15.0
    img_1 = torch.arange(1, 10, dtype=torch.float32).reshape(1, 3, 3)
    img_2 = torch.arange(11, 20, dtype=torch.float32).reshape(1, 3, 3)
    img = torch.stack([img_1, img_2], dim=0)
    # saliency maps
    smap_1 = np.arange(1, 10, dtype=float).reshape(3, 3)
    smap_2 = np.arange(11, 20, dtype=float).reshape(3, 3)
    smap = np.stack([smap_1, smap_2], axis=0)
    yield img, smap

def test_init(dummy_img_and_smap):
    for invalid_fraction in (-0.1, 1.1):
        with pytest.raises(ValueError, match="top_fraction should"):
            _ = RoarPerturbation(top_fraction=invalid_fraction)

    top_fraction = 0.1

    ptb_1 = RoarPerturbation(top_fraction, mean=None)
    assert ptb_1.quantile == 0.9
    assert not ptb_1._has_mean
    with pytest.raises(AttributeError):
        ptb_1.get_buffer("_mean")

    ptb_2 = RoarPerturbation(top_fraction, mean=[0.0, 0.0, 0.0])
    assert ptb_2.quantile == 0.9
    assert ptb_2._has_mean
    torch.testing.assert_allclose(ptb_2.get_buffer("_mean"), torch.zeros((3,), dtype=torch.float32))

@pytest.mark.parametrize("top_fraction", [0.0, 0.2, 1.0])
@pytest.mark.parametrize(
    "mean",
    [
        (100,),
        None,
    ],
)
def test_perturb_and_forward(dummy_img_and_smap, top_fraction, mean):
    img, smap = dummy_img_and_smap

    ptb = RoarPerturbation(top_fraction, mean)

    if mean is not None:
        if top_fraction == 1.0:
            expected_ptb_out = torch.full_like(img, 100.0)
        elif top_fraction == 0.0:
            expected_ptb_out = img
        else:
            out = img.clone()
            out[:, :, 2, 1:] = 100
            expected_ptb_out = out
    else:
        if top_fraction == 1.0:
            expected_ptb_out = torch.stack(
                [torch.full((1, 3, 3), 5.0, dtype=torch.float32), torch.full((1, 3, 3), 15.0, dtype=torch.float32)],
                dim=0,
            )
        elif top_fraction == 0.0:
            expected_ptb_out = img
        else:
            out = img.clone()
            out[0, :, 2, 1:] = 5.0
            out[1, :, 2, 1:] = 15.0
            expected_ptb_out = out

    expected_forward_out = expected_ptb_out
    torch.testing.assert_allclose(ptb.perturb(img, smap), expected_ptb_out)
    torch.testing.assert_allclose(ptb.forward(img, smap), expected_forward_out)


import pygame

from pygame.locals import *

import ui.sftext.resources as resources
from ui.sftext.style.style import Style

class SFText():
    def __init__(self, text, surface=None, font_path='.', style=None):

        if isinstance(text, bytes):
            # print('text is', bytes)
            self.text = text.decode('utf-8')
        elif isinstance(text, str):
            # print('text is', str)
            self.text = text

        self.fonts = resources.Fonts(path=font_path)

        if style:
            Style.set_default(style)

        if surface is None:
            self.screen = pygame.display.get_surface()
        else:
            self.screen = surface
        self.screen_rect = self.screen.get_rect()
        self.bg = self.screen.copy()

        # print('parsing text')
        self.parse_text()
        # print('done parsing')

    def set_font(self, obj):
        if obj['bold'] and obj['italic'] and obj['separate_bolditalic']:
            obj['font_obj'] = self.fonts.load(
                obj['separate_bolditalic'], obj['size'])
        elif obj['separate_bold'] and obj['bold']:
            obj['font_obj'] = self.fonts.load(
                obj['separate_bold'], obj['size'])
        elif obj['separate_italic'] and obj['italic']:
            obj['font_obj'] = self.fonts.load(
                obj['separate_italic'], obj['size'])
        else:
            obj['font_obj'] = self.fonts.load(
                obj['font'], obj['size'])

    def parse_text(self):
        self.parsed = []
        scr_w = self.screen_rect.width

        self.default_style = Style.default_style
        self.default_style['font_obj'] = self.fonts.load(
            self.default_style['font'], self.default_style['size'])
        self.default_style['w'], self.default_style['h'] = (
            self.default_style['font_obj'].size(' '))

        y = 0
        for line in self.text.splitlines():
            x = 0
            for style in line.split("{style}"):

                text, styled_txt = Style.split(style)

                self.set_font(styled_txt)
                font = styled_txt['font_obj']

                w, h = styled_txt['w'], styled_txt['h'] = font.size(' ')
                # determine the amount of space needed to render text

                wraps = self.wrap_text(text, scr_w, x, styled_txt)

                for wrap in wraps:
                    rect = pygame.Rect((0, 0), font.size(wrap['text']))

                    if (x + wrap['w1'] + w * 3) > scr_w:
                        x = 0
                        y += wrap['h']

                    if len(wraps) == 1 and wrap['align'] == 'center':
                        rect.midtop = (
                            self.screen_rect.centerx,
                            self.screen_rect.bottom + y)
                    else:
                        rect.topleft = (
                            x + w * 3,
                            self.screen_rect.bottom + y)
                    wrap['rect'] = rect
                    wrap['x'] = x
                    wrap['y'] = y
                    if False:
                        print("\n{}: {},".format('x', wrap['x']), end='')
                        print("{}: {},".format('y', wrap['y']), end='')
                        print("{}: {},".format('w', wrap['w']), end='')
                        print("{}: {}".format('h', wrap['h']))
                        print(wrap['text'])
                    self.parsed.append(wrap)

                    x += wrap['w1']
            y += wrap['h']
        # exit()
        # print('done parsing')

        self.start_y = 0 - self.screen_rect.h + self.default_style['h']

        self.y = int(self.start_y)

        self.end_y = (
            -sum(p['h'] for p in self.parsed if p['x'] == 0)
            - self.default_style['h'] * 2)

    def wrap_text(self, text, width, _x, styled_txt):
        style = dict(styled_txt)
        x = int(_x)
        wrapped = []
        size = style['font_obj'].size
        c_width = style['w']

        # print(size(text))
        # print(width)
        if size(text)[0] <= (
            width - c_width * 6 - x
        ):
            # print('fits')
            style['text'] = text
            style['w1'] = size(text)[0]
            wrapped.append(style)

            return wrapped
        else:
            # print("doesn't fit")
            # print(text)
            wrapped = [text]
            guessed_length = ((width - c_width * 6 - x) // c_width)
            all_fit = False
            all_fit_iter = 1
            while not all_fit:
                #########
                # DEBUG #
                if False:
                    print("all_fit iteraions: {}".format(all_fit_iter))
                    if all_fit_iter >= guessed_length * 5:
                        exit()
                # DEBUG #
                #########
                for i in range(len(wrapped)):
                    # print('for i in range(len(wrapped))')
                    fit = size(wrapped[i])[0] < width - c_width * 6 - x
                    # print(width - c_width * 6 - x)
                    iter_length = int(guessed_length)
                    # print(iter_length)
                    fit_iter = 0
                    while not fit:
                        #########
                        # DEBUG #
                        if False:
                            fit_iter += 1
                            print("fit iteraions: {}, iter_length: {}".format(
                                fit_iter, iter_length))
                            if fit_iter >= guessed_length * 5:
                                print(wrapped[i])
                                exit()
                        # DEBUG #
                        #########
                        if guessed_length <= 2 or iter_length <= 2:
                            # print('if guessed_length <= 2')
                            x = 0
                            guessed_length = (
                                (width - c_width * 6 - x) // c_width)
                            iter_length = int(guessed_length)
                            continue
                        guess = wrapped[i][:iter_length]
                        # print('while not fit: "{}"'.format(guess))
                        if guess[-1:] not in [" ", ",", ".", "-", "\n"]:
                            # print('if guess[-1:] not in:')
                            iter_length -= 1
                        else:
                            if size(guess)[0] < width - c_width * 6 - x:
                                remains = wrapped[i][iter_length:]
                                wrapped[i] = guess
                                wrapped.append(remains)
                                fit = True
                            else:
                                iter_length -= 1
                    all_fit_iter += 1
                    # print("Cut point: {}".format(iter_length))
                    # print('Guess: ({})"{}"'.format(type(guess), guess))
                    # print('Remains: "{}"'.format(remains))
                    # print("[{}]fit? {}".format(i, fit))
                status = True
                for i in range(len(wrapped)):
                    if size(wrapped[i])[0] >= width:
                        status = False
                all_fit = status

            for i in range(len(wrapped)):
                # print('"{}"'.format(wrapped[i]))
                style['text'] = wrapped[i]
                style['w1'] = size(wrapped[i])[0]
                wrapped[i] = dict(style)

            return wrapped

    def on_update(self):
        for i, p in enumerate(self.parsed[:]):
            rect = p['rect'].move(0, self.y)

            if not isinstance(p['text'], pygame.Surface):
                p['font_obj'].set_bold(False)
                p['font_obj'].set_italic(False)

                if p['bold'] and p['italic'] and not p['separate_bolditalic']:
                    print('pygame-bold', p['text'])
                    p['font_obj'].set_bold(p['bold'])
                    print('pygame-italic', p['text'])
                    p['font_obj'].set_italic(p['italic'])
                elif not p['separate_bold'] and p['bold']:
                    print('pygame-bold', p['text'])
                    p['font_obj'].set_bold(p['bold'])
                elif not p['separate_italic'] and p['italic']:
                    print('pygame-italic', p['text'])
                    p['font_obj'].set_italic(p['italic'])

                p['font_obj'].set_underline(p['underline'])

                p['text'] = p['font_obj'].render(p['text'], 1, p['color'])
            self.screen.blit(p['text'], rect)

            if rect.top >= (
                self.screen_rect.bottom - self.default_style['h']
            ):
                break
        """
        if obj['bold'] and obj['italic'] and obj['separate_bolditalic']:
            obj['font_obj'] = self.fonts.load(
                obj['separate_bold_italic'], obj['size'])
        elif obj['separate_bold'] and obj['bold']:
            obj['font_obj'] = self.fonts.load(
                obj['separate_bold'], obj['size'])
        elif obj['separate_italic'] and obj['italic']:
            obj['font_obj'] = self.fonts.load(
                obj['separate_italic'], obj['size'])
        else:
            obj['font_obj'] = self.fonts.load(
                obj['font'], obj['size'])
        """

    def scroll(self, y=0):
        if isinstance(y, int):
            self.y += y
            if self.y < self.end_y:
                self.y = self.end_y
            elif self.y > self.start_y:
                self.y = self.start_y
        elif isinstance(y, str):
            if y == 'home':
                self.y = self.start_y
            elif y == 'end':
                self.y = self.end_y

    def post_update(self):
        self.screen.blit(self.bg, (0, 0))

    def on_key_press(self, event):
        if event.type == pygame.KEYDOWN and event.key == pygame.K_UP:
            self.scroll(50)
        elif event.type == pygame.KEYDOWN and event.key == pygame.K_PAGEUP:
            self.scroll(500)
        elif event.type == pygame.KEYDOWN and event.key == pygame.K_HOME:
            self.scroll('home')

        elif event.type == pygame.KEYDOWN and event.key == pygame.K_DOWN:
            self.scroll(-50)
        elif event.type == pygame.KEYDOWN and event.key == pygame.K_PAGEDOWN:
            self.scroll(-500)
        elif event.type == pygame.KEYDOWN and event.key == pygame.K_END:
            self.scroll('end')

    def on_mouse_scroll(self, event):
        if event.button == 4:
            self.scroll(50)
        elif event.button == 5:
            self.scroll(-50)


""" Functions for downloading & analyzing wheels """

from __future__ import annotations
from datetime import datetime, timezone
import logging
from pathlib import Path
from tempfile import TemporaryDirectory
import traceback
from typing import Any
import requests
from wheel_inspect import inspect_wheel
from .app import emit_json_log
from .models import Wheel, db
from .util import USER_AGENT

log = logging.getLogger(__name__)

def process_queue(max_wheel_size: int | None = None) -> None:
    """
    Process all of the wheels returned by `Wheel.to_process()` one by one and
    store the results in the database.  If an error occurs, the traceback is
    stored as a `ProcessingError` for the wheel.  The database session is
    committed after each wheel in order to save memory.

    This function requires a Flask application context with a database
    connection to be in effect.

    :param int max_wheel_size: If set, only wheels this size or smaller are
        analyzed
    """
    start_time = datetime.now(timezone.utc)
    wheels_processed = 0
    bytes_processed = 0
    errors = 0
    with TemporaryDirectory() as tmpdir, requests.Session() as s:
        s.headers["User-Agent"] = USER_AGENT
        try:
            # This outer `try` block is so that stats are written to the
            # logfile even when the function is cancelled via Cntrl-C.
            for whl in Wheel.to_process(max_wheel_size=max_wheel_size):
                fpath = Path(tmpdir, whl.filename)
                try:
                    log.info("Downloading %s from %s ...", whl.filename, whl.url)
                    download(s, whl.url, fpath)
                    about = process_wheel(
                        path=fpath,
                        size=whl.size,
                        md5=whl.md5,
                        sha256=whl.sha256,
                    )
                    whl.set_data(about)
                    # Some errors in inserting data aren't raised until we
                    # actually try to insert by calling commit(), so include
                    # the commit() under the `try`.
                    db.session.commit()
                except Exception:
                    # rollback() needs to be called before log.exception() or
                    # else SQLAlchemy gets all complainy.
                    db.session.rollback()
                    log.exception("Error processing %s", whl.filename)
                    whl.add_error(traceback.format_exc())
                    db.session.commit()
                    errors += 1
                finally:
                    fpath.unlink(missing_ok=True)
                wheels_processed += 1
                bytes_processed += whl.size
        finally:
            end_time = datetime.now(timezone.utc)
            emit_json_log(
                "process_queue.log",
                {
                    "op": "process_queue",
                    "start": str(start_time),
                    "end": str(end_time),
                    "wheels": wheels_processed,
                    "bytes": bytes_processed,
                    "errors": errors,
                },
            )

def process_wheel(path: Path, size: int, md5: str | None, sha256: str | None) -> dict:
    """
    Process the wheel at ``path``.  The wheel is analyzed with
    `inspect_wheel()`, and its size & digests are checked against ``size``,
    ``md5``, and ``sha256`` (provided by PyPI) to verify download integrity.

    :return: the results of the call to `inspect_wheel()`
    """
    log.info("Inspecting %s ...", path.name)
    about: dict[str, Any] = inspect_wheel(path)
    if about["file"]["size"] != size:
        log.error(
            "Wheel %s: size mismatch: PyPI reports %d, got %d",
            size,
            about["file"]["size"],
        )
        raise ValueError(
            f'Size mismatch: PyPI reports {size}, got {about["file"]["size"]}'
        )
    for alg, expected in [("md5", md5), ("sha256", sha256)]:
        if expected is not None and expected != about["file"]["digests"][alg]:
            log.error(
                "Wheel %s: %s hash mismatch: PyPI reports %s, got %s",
                alg,
                expected,
                about["file"]["digests"][alg],
            )
            raise ValueError(
                f"{alg} hash mismatch: PyPI reports {expected},"
                f' got {about["file"]["digests"][alg]}'
            )
    log.info("Finished inspecting %s", path.name)
    return about

def download(s: requests.Session, url: str, path: Path) -> None:
    r = s.get(url, stream=True)
    r.raise_for_status()
    with path.open("wb") as fp:
        for chunk in r.iter_content(65535):
            fp.write(chunk)


import numpy as np
import pandas as pd
from scipy.signal import filtfilt

def rolling_mean_by_unit(df, n, verbose=False):
    cols_sensors = [c for c in df.columns if c.startswith('sensor')]
    df_new = df.copy()
    unit_nrs = df['unit_nr'].unique()

    for nr in unit_nrs:
        if verbose:
            print('processing unit nr: {}'.format(nr))
        idx = df['unit_nr'] == nr
        df_new.loc[idx,cols_sensors] = df.loc[idx, cols_sensors].rolling(n, min_periods=1).mean()
    return df_new

def apply_filtfilt_by_unit(df, b, a=1.0):
    cols_data = [col for col in df.columns if col.startswith('sen') or col.startswith('os')]
    df_new = pd.DataFrame(columns=df.columns)

    for unit_nr, df_sub in df.groupby('unit_nr'):
        n = len(df_sub)
        for col in df_sub.columns:
            if col.startswith('s'):
                df_sub[col] = filtfilt(b, a, df_sub[col].copy().values, padlen=5 * len(b))
        df_new = df_new.append(df_sub)

    return df_new

def apply_rolling_by_unit(df, fun, win_size_p=0.05, subsample=0):
    cols_data = [col for col in df.columns if col.startswith('sen') or col.startswith('os')]
    df_new = pd.DataFrame(columns=df.columns)

    for unit_nr, df_sub in df.groupby('unit_nr'):
        n = len(df_sub)
        if isinstance(win_size_p, int):
            win_size_n = int(win_size_p)
        else:
            win_size_n = int(n * win_size_p)

        df_windowed = df_sub.copy()
        df_windowed[cols_data] = df_windowed[cols_data].rolling(win_size_n).apply(fun)
        if subsample > 0:
            df_windowed = df_windowed[::subsample]
        df_new = df_new.append(df_windowed)

    return df_new

def normalize_col_by_unit(df, col):
    df_sub = df.copy()

    rul_abs = df_sub[col].copy().values
    rul_rel = np.array([])
    for unit_nr in df_sub['unit_nr'].unique():
        tmp = np.array(rul_abs[df_sub['unit_nr']==unit_nr], copy=True)
        tmp = tmp - tmp.min()
        tmp = tmp / tmp.max()
        rul_rel = np.append(rul_rel, tmp)

    df_sub[col] = rul_rel
    return df_sub

def resample_fixed(df, n_new):
    n_old, m = df.values.shape
    mat_old = df.values
    mat_new = np.zeros((n_new, m))
    x_old = np.linspace(df.index.min(), df.index.max(), n_old)
    x_new = np.linspace(df.index.min(), df.index.max(), n_new)

    for j in range(m):
        y_old = mat_old[:, j]
        y_new = np.interp(x_new, x_old, y_old)
        mat_new[:, j] = y_new

    return pd.DataFrame(mat_new, index=x_new, columns=df.columns)