#!/usr/bin/env python

import json
import math
import rospy
from time import sleep
from sensor_msgs.msg import Range
from sensor_msgs.msg import Imu
from sensor_msgs.msg import MagneticField
from psi.msg import CardinalDirection
from psi.msg import DirectionError
from psi.msg import MissionStatus

class Navigation:
    def drive_ms_cb(self, data):
        self.drive_mission_data = data.mission_status
        if self.drive_mission_data != self.prev_drive_mission_data:
            temp_mission_data = json.loads(self.drive_mission_data)
            self.direction = temp_mission_data['Direction']
            self.follow_up_direction = temp_mission_data['Follow-up-Direction']
            self.wait = temp_mission_data['Wait']
            self.delay = temp_mission_data['Delay']
            self.prev_drive_mission_data = self.drive_mission_data

    def drive_turn_control_cb(self, data):
        self.direction_error = -float(data.direction_error) / 100

        if self.wait:
            prev_direction = ""
            recorded_time = rospy.Time.now()
            prev_direction = self.direction
            self.direction = "Stop"
            while (rospy.Time.now() - recorded_time < rospy.Duration(self.delay)):
                self.navigate()
                rospy.sleep(1)
            self.wait = False
            self.direction = prev_direction

        self.navigate()

    def imu_cb(self, data):
         # imu data ranges from -180 to 180
         # To return it to positive, we add 360 to the negative values
        if data.orientation.z < 0:
            self.orientation_z = 360 + data.orientation.z
        else:
            self.orientation_z = data.orientation.z

    def navigate(self):
        self.cd_msg.header.stamp = rospy.Time.now()
        # This shoud change in the future when the wheel base link is
        # created and implemented in the urdf
        self.cd_msg.header.frame_id = "base_link"

        # Stop is always the first priority. If a stop is detected, execute the motion immediately.
        if "Stop" in self.direction:
            self.cardinal_direction = "T"
        # If there isn't a stop, determine the decision tree path to execute.
        else:
            # Determine if there is a junction.
            if self.junction:
                # Check if we have already reached the end of the block aisle
                # If the end_block flag has been triggered, we reverse the turn direction.
                if self.end_block:
                    # Execute the direction, which is a turn action, however, we reverse the turn direction.
                    if "Left" in self.direction:
                        # Determine the heading, and check if we have turned 90 degrees
                        self.prev_orientation_z = self.orientation_z
                        self.cardinal_direction = "E"
                        # Publish the message to turn the robot
                        self.cd_msg.cardinal_direction = self.cardinal_direction
                        self.cardinal_direction_pub.publish(self.cd_msg)
                        # Experimentation shows that 60 degree is a right angle turn
                        if self.prev_orientation_z > 270:
                            self.prev_orientation_z = 360 - self.prev_orientation_z
                            self.orientation_z = 360 - self.orientation_z
                        delta_orientation = abs(
                            self.orientation_z - self.prev_orientation_z)
                        while (delta_orientation < 90):
                            self.cardinal_direction = "E"
                            # Recalculate orientation in the while loop
                            delta_orientation = abs(
                                self.orientation_z - self.prev_orientation_z)
                            # Publish as long as the vehicle hasn't reach the 90 degrees
                            self.cd_msg.cardinal_direction = self.cardinal_direction
                            self.cardinal_direction_pub.publish(self.cd_msg)
                            self.rate.sleep()
                        if self.follow_up:
                            if "Forward" in self.follow_up_direction:
                                self.direction = self.follow_up_direction
                                # Publish message to move robot
                                self.cardinal_direction = "N"
                                self.cd_msg.cardinal_direction = self.cardinal_direction
                                self.cardinal_direction_pub.publish(
                                    self.cd_msg)
                                self.end_block = False
                    elif "Right" in self.direction:
                        # Determine the heading, and check if we have turned 90 degrees
                        self.prev_orientation_z = self.orientation_z
                        self.cardinal_direction = "W"
                        # Publish the message to turn the robot
                        self.cd_msg.cardinal_direction = self.cardinal_direction
                        self.cardinal_direction_pub.publish(self.cd_msg)
                        # Experimentation shows that 90 degree is a right angle turn
                        # Attempts to solve the 0/360 heading position error
                        if self.prev_orientation_z < 90:
                            self.prev_orientation_z = 360 + self.prev_orientation_z
                            self.orientation_z = self.orientation_z
                        delta_orientation = abs(
                            self.orientation_z - self.prev_orientation_z)
                        while (delta_orientation < 90):
                            self.cardinal_direction = "W"
                            # Recalculate orientation in while loop
                            delta_orientation = abs(
                                self.orientation_z - self.prev_orientation_z)
                            # Publish the message to turn the vehicle as long as the heading hasn't reach 90 degrees
                            self.cd_msg.cardinal_direction = self.cardinal_direction
                            self.cardinal_direction_pub.publish(self.cd_msg)
                            self.rate.sleep()
                        if self.follow_up:
                            if "Forward" in self.follow_up_direction:
                                self.direction = self.follow_up_direction
                                # Publish the message to move robot
                                self.cardinal_direction = "N"
                                self.cd_msg.cardinal_direction = self.cardinal_direction
                                self.cardinal_direction_pub.publish(
                                    self.cd_msg)
                                self.end_block = False
                else:  # If the end_block flag hasn't been triggered, we follow the motion in the qr code.
                    # Execute the direction, which is a turn action
                    if "Left" in self.direction:
                        # Determine the heading, and check if we have turned 90 degrees
                        self.prev_orientation_z = self.orientation_z
                        self.cardinal_direction = "W"
                        # Publish the message to turn the robot
                        self.cd_msg.cardinal_direction = self.cardinal_direction
                        self.cardinal_direction_pub.publish(self.cd_msg)
                        # Experimentation shows that 90 degree is a right angle turn
                        if self.prev_orientation_z < 90:
                            self.prev_orientation_z = 360 + self.prev_orientation_z
                            self.orientation_z = 360 + self.orientation_z
                        delta_orientation = abs(
                            self.orientation_z - self.prev_orientation_z)
                        while (delta_orientation < 90):
                            self.cardinal_direction = "W"
                            # Recalculate orientation in while loop
                            delta_orientation = abs(
                                self.orientation_z - self.prev_orientation_z)
                            # Publish the message to turn the robot as long as the desired heading hasn't reach
                            self.cd_msg.cardinal_direction = self.cardinal_direction
                            self.cardinal_direction_pub.publish(self.cd_msg)
                            self.rate.sleep()
                        if self.follow_up:
                            if "Forward" in self.follow_up_direction:
                                self.direction = self.follow_up_direction
                                # Publish message to move the robot
                                self.cardinal_direction = "N"
                                self.cd_msg.cardinal_direction = self.cardinal_direction
                                self.cardinal_direction_pub.publish(
                                    self.cd_msg)
                    elif "Right" in self.direction:
                        # Determine the heading, and check if we have turned 90 degrees
                        self.prev_orientation_z = self.orientation_z
                        self.cardinal_direction = "E"
                        # Publish message to turn the robot
                        self.cd_msg.cardinal_direction = self.cardinal_direction
                        self.cardinal_direction_pub.publish(self.cd_msg)
                        # Experimentation shows that 90 degree is a right angle turn
                        delta_orientation = abs(
                            self.orientation_z - self.prev_orientation_z)
                        while (delta_orientation < 90):
                            self.cardinal_direction = "E"
                            # Recalculates orientation in the while loop
                            delta_orientation = abs(
                                self.orientation_z - self.prev_orientation_z)
                            # Publish message as long as the desired heading hasn't been reached
                            self.cd_msg.cardinal_direction = self.cardinal_direction
                            self.cardinal_direction_pub.publish(self.cd_msg)
                            self.rate.sleep()
                        if self.follow_up:
                            if "Forward" in self.follow_up_direction:
                                self.direction = self.follow_up_direction
                                self.cardinal_direction = "N"
                                # Publish message to move the robot
                                self.cd_msg.cardinal_direction = self.cardinal_direction
                                self.cardinal_direction_pub.publish(
                                    self.cd_msg)

            if "Forward" in self.direction:
                if self.direction_error < -0.1:
                    self.cardinal_direction = "NW"
                elif self.direction_error > 0.1:
                    self.cardinal_direction = "NE"
                else:
                    self.cardinal_direction = "N"
            elif "Backward" in self.direction:
                if self.direction_error < -0.1:
                    self.cardinal_direction = "SW"
                elif self.direction_error > 0.1:
                    self.cardinal_direction = "SE"
                else:
                    self.cardinal_direction = "S"
            else:
                self.cardinal_direction = "T"

        self.cd_msg.cardinal_direction = self.cardinal_direction
        self.cardinal_direction_pub.publish(self.cd_msg)

    def status_ms_cb(self, data):
        self.status_mission_data = data.mission_status
        if self.status_mission_data != self.prev_status_mission_data:
            temp_mission_data = json.loads(self.status_mission_data)
            self.junction = temp_mission_data['Junction']
            self.follow_up = temp_mission_data['Follow-up']
            try:
                self.end_block = temp_mission_data['EndBlock']
            except KeyError:
                rospy.loginfo("No End Block value given.")
                pass
            self.end_mission = temp_mission_data['EndMission']

    def start(self):
        rospy.init_node('navigation', anonymous=False)
        self.rate = rospy.Rate(10)

        self.status_mission_data = ""
        self.prev_status_mission_data = ""
        self.junction = False
        self.follow_up = False
        self.end_block = False
        self.end_mission = False
        self.water_mission = False

        self.drive_mission_data = ""
        self.prev_drive_mission_data = ""
        self.direction = ""
        self.follow_up_direction = ""
        self.wait = False
        self.delay = 0.0
        self.direction_error = 0.0
        self.prev_direction_error = 0.0
        self.cardinal_direction = ""

        self.orientation_z = 0.0
        self.prev_orientation_z = 0.0

        self.range = 0

        self.cd_msg = CardinalDirection()
        self.cardinal_direction_pub = rospy.Publisher(
            'drive/cardinal_direction', CardinalDirection, queue_size=1)

        rospy.Subscriber('status/mission_status', MissionStatus,
                         self.status_ms_cb)

        rospy.Subscriber('drive/mission_status', MissionStatus,
                         self.drive_ms_cb)

        rospy.Subscriber('drive/direction_error', DirectionError,
                         self.drive_turn_control_cb)

        rospy.Subscriber('sensors/imu', Imu, self.imu_cb)

        rospy.spin()

if __name__ == "__main__":
    try:
        nav = Navigation()
        nav.start()
    except rospy.ROSInterruptException:
        pass


# Polynomial Regression
# not a linear regression model

# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Importing the dataset
dataset = pd.read_csv('Position_Salaries.csv')
# find the correlations between the level and salary
# to find out if the employee is bluffing about the salary
# position is already encoded in level column
# so it doesn't need to be included in the matrix
# of features X
X = dataset.iloc[:, 1:2].values
y = dataset.iloc[:, 2].values

# Splitting the dataset into the Training set and Test set
# doesn't make sense to split the data into
# training and tests sets because there are only
# 10 observations
'''from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)'''

# Feature Scaling
"""from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_test)
sc_y = StandardScaler()
y_train = sc_y.fit_transform(y_train)"""

# Fitting linear regression to the dataset
# this will be a reference to then be able to
# compare the results of polynomial regression
# to the results of the linear regression reference
# space
from sklearn.linear_model import LinearRegression
lin_reg = LinearRegression()
lin_reg.fit(X,y)

# Fitting Polynomial regression to the dataset
from sklearn.preprocessing import PolynomialFeatures
# used to make a new matrix of features X_poly
# poly_reg = PolynomialFeatures(degree = 2)
# poly_reg = PolynomialFeatures(degree = 3)
poly_reg = PolynomialFeatures(degree = 4)
X_poly = poly_reg.fit_transform(X)
# include this fit into a multiple linear regression model
lin_reg_2 = LinearRegression()
lin_reg_2.fit(X_poly, y)

# Visualising the linear regression results
# actual data
plt.scatter(X, y, color = 'red')
# now plot prediction data
plt.plot(X, lin_reg.predict(X), color = 'blue')
plt.title('Truth or Bluff (Linear Regression)')
plt.xlabel('Position Level')
plt.ylabel('Salary')
plt.savefig('Linear_py.png')

# Visualising the polynomial regression degree 2 results
# actual data
plt.scatter(X, y, color = 'red')
# now plot prediction data
plt.plot(X, lin_reg_2.predict(poly_reg.fit_transform(X)), color = 'blue')
plt.title('Truth or Bluff (Polynomial Degree 2 Regression)')
plt.xlabel('Position Level')
plt.ylabel('Salary')
plt.savefig('Polynomial_2_py.png')

# Visualising the polynomial regression degree 3 results
# actual data
plt.scatter(X, y, color = 'red')
# now plot prediction data
plt.plot(X, lin_reg_2.predict(poly_reg.fit_transform(X)), color = 'blue')
plt.title('Truth or Bluff (Polynomial Degree 3 Regression)')
plt.xlabel('Position Level')
plt.ylabel('Salary')
plt.savefig('Polynomial_3_py.png')

# Visualising the polynomial regression degree 4 results
# perfect model!
# actual data
plt.scatter(X, y, color = 'red')
# now plot prediction data
plt.plot(X, lin_reg_2.predict(poly_reg.fit_transform(X)), color = 'blue')
plt.title('Truth or Bluff (Polynomial Degree 4 Regression)')
plt.xlabel('Position Level')
plt.ylabel('Salary')
plt.savefig('Polynomial_4_py.png')

# Predicting a new result with Linear Regression
# predict one salary of level 6.5
lin_reg.predict(6.5)

# Predicting a new result with Polynomial Regression
lin_reg_2.predict(poly_reg.fit_transform(6.5))

Part 2 - Regression/Section 6 - Polynomial Regression/Polynomial_Regression/polynomial_regression.py


"""Sensor platform for Veolia."""
import datetime

from .const import COORDINATOR, DAILY, DOMAIN, HOURLY
from .entity import VeoliaEntity

async def async_setup_entry(hass, entry, async_add_devices):
    """Set up sensor platform."""
    coordinator = hass.data[DOMAIN][COORDINATOR]
    sensors = [
        VeoliaDailyUsageSensor(coordinator, entry),
        VeoliaMonthlyUsageSensor(coordinator, entry),
    ]
    # HOURLY array is empty when hourly report is not enabled
    if coordinator.data[HOURLY]:
        sensors.append(VeoliaHourlyUsageSensor(coordinator, entry)),

    async_add_devices(sensors)

class VeoliaHourlyUsageSensor(VeoliaEntity):
    """Monitors the hourly water usage."""

    _attr_name = "veolia_hourly_consumption"

    @property
    def state(self):
        """Return the state of the sensor."""
        hour = datetime.datetime.now().hour
        return self.coordinator.data[HOURLY][hour - 1]

class VeoliaDailyUsageSensor(VeoliaEntity):
    """Monitors the daily water usage."""

    _attr_name = "veolia_daily_consumption"

    @property
    def state(self):
        """Return the state of the sensor."""
        state = self.coordinator.data[DAILY][-1]

        if state > 0:
            return state

        return None

class VeoliaMonthlyUsageSensor(VeoliaEntity):
    """Monitors the monthly water usage."""

    _attr_name = "veolia_monthly_consumption"

    @property
    def state(self):
        """Return the state of the sensor."""
        state = sum(self.coordinator.data[DAILY])
        if state > 0:
            return state

        return None


# -*- coding: utf-8 -*-

from odoo import models, fields, api
from datetime import datetime, date, timedelta
import calendar
import time

class AttendanceSheets(models.Model):
    _name = 'berdikari.attendance.sheets'
    _description = 'Berdikari Attendance Sheets'

    name = fields.Many2one('hr.employee', string='Employee')
    period_id = fields.Many2one('berdikari.hr.attendance.period', string='Period Name')

    @api.depends('period_id')
    @api.onchange('period_id')
    def onchange_domain_seq(self):
        domain = {}
        period_id = self.period_id.id
        if period_id:
            domain_seq = [('attendance_period_id', '=', period_id)]
            domain['period_seq_id'] = domain_seq
        hasil = {'domain': domain}
        return hasil

    period_seq_id = fields.Many2one('berdikari.hr.attendance.period.line', string='Period Sequence')

    @api.depends('period_seq_id')
    @api.onchange('period_seq_id')
    def onchange_period_seq(self):
        for rec in self:
            period_seq_id = rec.period_seq_id.id
            model_period_line = self.env['berdikari.hr.attendance.period.line']
            if period_seq_id:
                period_line = model_period_line.search([('id', '=', period_seq_id)], limit=1)
                rec.period_start = period_line.date_from
                rec.period_end = period_line.date_to

    period_start = fields.Date()
    period_end = fields.Date()
    remarks = fields.Char()
    attendance_policy = fields.Char()
    no_of_overtimes = fields.Integer()
    total_overtime = fields.Float()
    ot_meal_provided = fields.Integer()
    no_of_absence_days = fields.Integer()
    total_absence_hour = fields.Float()
    no_of_lates = fields.Integer()
    total_late_in = fields.Float()
    no_of_diff_times = fields.Integer()
    total_diff_time_hours = fields.Float()
    no_of_work_days = fields.Integer()
    total_work_hours = fields.Float()
    no_of_leave_days = fields.Integer()
    calendar_id = fields.Many2one('resource.calendar')
    attendance_line_ids = fields.One2many('berdikari.attendance.sheets.line', 'attendance_sheet_id')

    @api.multi
    def action_get_attendance(self):
        for rec in self:
            model_attendance = self.env['hr.attendance']
            model_contract = self.env['hr.contract']
            model_calendar = self.env['resource.calendar']
            model_calendar_detail = self.env['resource.calendar.attendance']
            model_overtime = self.env['berdikari.overtime.request']
            model_leave = self.env['hr.leave']
            domain = []
            if rec.name:
                domain.append(('employee_id', '=', rec.name.id))
            else:
                domain = domain
            if rec.period_start and rec.period_end:
                domain.append(('check_in', '>=', rec.period_start))
                domain.append(('check_in', '<=', rec.period_end))
                domain.append(('check_out', '>=', rec.period_start))
                domain.append(('check_out', '<=', rec.period_end))
            else:
                domain = domain
            search_attendance_ids = model_attendance.search(domain, order="id asc")
            domain_name = []
            if rec.name:
                domain_name.append(('employee_id', '=', rec.name.id))
            else:
                domain_name = domain_name
            domain_name.append(('active', '=', True))
            contract_id = model_contract.search(domain_name, limit=1)
            calendar_id = model_calendar.search([('id', '=', contract_id.resource_calendar_id.id)], limit=1)
            rec.calendar_id = calendar_id
            # calendar_detail_ids = model_calendar_detail.search([('calendar_id', '=', calendar_id.id)])
            domain_overtime = []
            if rec.name:
                domain_overtime.append(('employee_id', '=', rec.name.id))
            else:
                domain_overtime = domain_overtime
            if rec.period_start and rec.period_end:
                domain_overtime.append(('start_date', '>=', rec.period_start))
                domain_overtime.append(('start_date', '<=', rec.period_end))
                domain_overtime.append(('state', '=', 'done'))
            else:
                domain_overtime = domain_overtime
            overtime_ids = model_overtime.search(domain_overtime)
            # cek leave
            domain_leave = []
            if rec.name:
                domain_leave.append(('employee_id', '=', rec.name.id))
            else:
                domain_leave = domain_leave
            if rec.period_start and rec.period_end:
                domain_leave.append(('date_from', '>=', rec.period_start))
                domain_leave.append(('date_from', '<=', rec.period_end))
                domain_leave.append(('date_to', '>=', rec.period_start))
                domain_leave.append(('date_to', '<=', rec.period_end))
            else:
                domain_leave = domain_leave
            domain_leave.append(('state', '=', 'validate'))
            leave_ids = model_leave.search(domain_leave)
            # hitung berapa banyak overtime
            no_of_overtimes = len(overtime_ids)
            total_overtime = 0
            ot_meal_provided = 0
            for one in overtime_ids:
                total_overtime = total_overtime + one.numbers_of_hour
                if one.meal_provided:
                    ot_meal_provided = ot_meal_provided + 1
            rec.no_of_overtimes = no_of_overtimes
            rec.total_overtime = total_overtime
            rec.ot_meal_provided = ot_meal_provided
            hasil = []
            no_of_absence_days = 0
            total_absence_hour = 0
            no_of_lates = 0
            total_late_in = 0
            no_of_diff_times = 0
            total_diff_time_hours = 0
            no_of_work_days = 0
            total_work_hours = 0
            no_of_leave_days = 0
            numbers_of_day = ((rec.period_end - rec.period_start).days) + 1
            for a in list(range(numbers_of_day)):
                if a == 0:
                    new_date = rec.period_start.day
                else:
                    new_date = (rec.period_start.day) + a
                schedule_date = date(rec.period_start.year, rec.period_start.month, new_date)
                dayNumber = calendar.weekday(rec.period_start.year, rec.period_start.month, new_date)
                day = calendar.day_name[dayNumber]
                day_to_sign_in = day + ' Morning'
                day_to_sign_out = day + ' Evening'
                sign_in = model_calendar_detail.search([('calendar_id', '=', calendar_id.id), ('name', '=', day_to_sign_in)])
                sign_in_time = sign_in.hour_from
                sign_out = model_calendar_detail.search([('calendar_id', '=', calendar_id.id), ('name', '=', day_to_sign_out)])
                sign_out_time = sign_out.hour_to
                data = False
                actual_sign_in_time = 0
                actual_sign_out_time = 0
                actual_sign_in = 0
                actual_sign_out = 0
                total_working_hour = 0
                nett_working_hour = 0
                overtime_hour = 0
                attendance_code = False
                diff_time = 0
                transport_allowance = 0
                meal_allowance = 0
                ot_meal_provided = False

                t1 = 0
                t2 = 0
                if search_attendance_ids:
                    for data in search_attendance_ids:
                        sign_in_date = date(data.check_in.year, data.check_in.month, data.check_in.day)
                        if sign_in_date == schedule_date:
                            # actual time to show
                            t1 = t1 + data.check_in.hour + 7
                            actual_sign_in_time = data.check_in.strftime("%M")
                            actual_sign_in_time = str(t1) + '.' + actual_sign_in_time
                            t2 = t2 + data.check_out.hour + 7
                            actual_sign_out_time = data.check_out.strftime("%M")
                            actual_sign_out_time = str(t2) + '.' + actual_sign_out_time
                            # actual time to calculate
                            actual_sign_in = data.check_in.hour + (data.check_in.minute / 60) + (
                                        data.check_in.second / 3600) + 7
                            actual_sign_out = data.check_out.hour + (data.check_out.minute / 60) + (
                                        data.check_out.second / 3600) + 7
                            total_working_hour = actual_sign_out - actual_sign_in
                            if day == 'Friday':
                                nett_working_hour = total_working_hour - 1.5
                            else:
                                nett_working_hour = total_working_hour - 1
                            diff_time = (sign_out_time - sign_in_time) - total_working_hour
                            attendance_code_id = self.env['berdikari.hr.attendance.code'].search(
                                [('name', '=', 'work')], limit=1)
                            attendance_code = attendance_code_id.code
                            overtime_hour = 0

                            # hitung berapa banyak late
                            if (actual_sign_in - sign_in_time) > 0:
                                no_of_lates = no_of_lates + 1
                                total_late_in = total_late_in + (actual_sign_in - sign_in_time)
                                rec.no_of_lates = no_of_lates
                                rec.total_late_in = total_late_in

                            # hitung berapa banyak diff
                            setup = self.env['jekdoo.setup'].get_setup()
                            min_diff = setup.diff_limit
                            if diff_time > 0 and diff_time >= min_diff:
                                no_of_diff_times = no_of_diff_times + 1
                                total_diff_time_hours = total_diff_time_hours + diff_time
                                rec.no_of_diff_times = no_of_diff_times
                                rec.total_diff_time_hours = total_diff_time_hours

                            # hitung meal dan transport bila employee check in
                            if (actual_sign_in <= sign_in_time) and nett_working_hour > 5:
                                transport_allowance = transport_allowance + 1
                                meal_allowance = meal_allowance + 1

                            #hitung working days and working hour
                            no_of_work_days = no_of_work_days + 1
                            total_work_hours = total_work_hours + nett_working_hour
                            rec.no_of_work_days = no_of_work_days
                            rec.total_work_hours = total_work_hours

                            if overtime_ids:
                                for overtime in overtime_ids:
                                    overtime_date = date(overtime.start_date.year, overtime.start_date.month,
                                                         overtime.start_date.day)
                                    if overtime_date == sign_in_date:
                                        overtime_hour = overtime.numbers_of_hour
                                        ot_meal_provided = overtime.meal_provided
                                        break
                            break
                        else:
                            if overtime_ids:
                                for overtime in overtime_ids:
                                    overtime_date = date(overtime.start_date.year, overtime.start_date.month,
                                                         overtime.start_date.day)
                                    if overtime_date == schedule_date:
                                        overtime_hour = overtime.numbers_of_hour
                                        ot_meal_provided = overtime.meal_provided
                                        break
                # hitung leave
                if leave_ids:
                    for leave in leave_ids:
                        leave_date_from = date(leave.date_from.year, leave.date_from.month, leave.date_from.day)
                        leave_date_to = date(leave.date_to.year, leave.date_to.month, leave.date_to.day)
                        if leave_date_from == schedule_date or leave_date_to == schedule_date:
                            holiday_status_id = leave.holiday_status_id
                            attendance_code_id = holiday_status_id.attendance_code_id
                            attendance_code = attendance_code_id.code
                            no_of_leave_days = no_of_leave_days + 1
                            rec.no_of_leave_days = no_of_leave_days
                            break
                # hitung berapa banyak absen
                if dayNumber in [0, 1, 2, 3, 4]:
                    if t1 <= 0 or t2 <=0:
                        if attendance_code == False:
                            attendance_code_id = self.env['berdikari.hr.attendance.code'].search([('name', '=', 'absence')],limit=1)
                            attendance_code = attendance_code_id.code
                            if dayNumber == 4:
                                total_absence_hour = total_absence_hour + (sign_out_time - sign_in_time - 1.5)
                            else:
                                total_absence_hour = total_absence_hour + (sign_out_time - sign_in_time - 1)
                            no_of_absence_days = no_of_absence_days + 1
                            rec.no_of_absence_days = no_of_absence_days
                            rec.total_absence_hour = total_absence_hour

                baru = [0, 0, {
                    'attendance_sheet_id': data.id if data else False,
                    'date': schedule_date,
                    'day': day,
                    'planned_sign_in': sign_in_time,
                    'planned_sign_out': sign_out_time,
                    'actual_sign_in_time': actual_sign_in_time,
                    'actual_sign_out_time': actual_sign_out_time,
                    'actual_sign_in': actual_sign_in,
                    'actual_sign_out': actual_sign_out,
                    'late_in': (actual_sign_in - sign_in_time) if (actual_sign_in - sign_in_time) > 0 else 0,
                    'total_working_hour': total_working_hour,
                    'nett_working_hour': nett_working_hour,
                    'overtime': overtime_hour,
                    'ot_meal_provided': ot_meal_provided,
                    'diff_time': diff_time,
                    'transport_allowance': transport_allowance,
                    'meal_allowance': meal_allowance,
                    'attendance_code': attendance_code,
                    'note': '',
                }]
                hasil.append(baru)
            if hasil:
                for line in rec.attendance_line_ids:
                    line.unlink()
                rec.attendance_line_ids = hasil

class AttendanceSheetsLine(models.Model):
    _name = 'berdikari.attendance.sheets.line'
    _description = 'Berdikari Attendance Sheets Line'

    attendance_sheet_id = fields.Many2one('berdikari.attendance.sheets')
    date = fields.Date()
    day = fields.Char()
    planned_sign_in = fields.Float()
    planned_sign_out = fields.Float()
    actual_sign_in_time = fields.Char()
    actual_sign_out_time = fields.Char()
    actual_sign_in = fields.Float()
    actual_sign_out = fields.Float()
    late_in = fields.Float()
    overtime = fields.Float()
    ot_meal_provided = fields.Boolean()
    total_working_hour = fields.Float()
    nett_working_hour = fields.Float()
    diff_time = fields.Float()
    transport_allowance = fields.Integer()
    meal_allowance = fields.Integer()
    # attendance_code = fields.Many2one('berdikari.hr.attendance.code')
    attendance_code = fields.Char()
    note = fields.Char()


import os
import sys
import subprocess
from pathlib import Path
import shutil
from src import pipeline

maf = .1
geno = .1
mind = .1
fn = filtered_name = 'filtered'
cur_dir = Path('.')
data_Path = Path('data/')
staging_Path = Path('data/staging')
test_file_dir = Path('test')
pipeline_vcf_dir = Path('data/pipeline/vcf')

if not data_Path.isdir():
    data_path.mkdir()

def clear_staging_area():
    for P in staging_Path.iterdir():
        P.unlink()

def stage(Paths):
    for Path in Paths:
        shutil.copy(str(Path), str(staging_Path))

def process(Paths):
    clear_staging_area()
    stage(Paths)

    os.chdir(str(staging_Path))
    print("Concatenating VCFs...")
    names = [Path.name for Path in Paths]
    subprocess.call(['bcftools', 'concat', *names, '>', 'concatenated'])
    print("%s files concatenated" % len(Paths))
    print("Filtering file...")
    subprocess.call(['plink2', '--vcf', 'concatenated', '--make-bed',
                     '--snps-only', '--maf', str(maf), '--geno', str(geno),
                     '--mind', str(mind), '--out', fn])
    print('Done filtering')
    print('Running PCA')
    subprocess.call(['plink2', '--bed', fn+'.bed', '--bim', fn+'.bim',
                     '--fam', fn+'.fam', '--pca', '2', '--out', 'fin'])
    os.chdir(str(cur_dir))

def main(argv):
    if argv[0] == 'test-project':
        test_file_Paths = [Path for Path in test_file_dir.iterdir()]
        process(test_file_Paths)
    else:
        pipeline.start()
        file_Paths = [Path for Path in pipeline_vcf_dir.iterdir()]
        process(file_Paths)

if __name__ == '__main__':
    main(sys.argv[1:])


from sedona.core.SpatialRDD import SpatialRDD
from sedona.core.spatialOperator.join_params import JoinParams
from sedona.core.spatialOperator.rdd import SedonaPairRDDList, SedonaPairRDD
from sedona.utils.decorators import require

class JoinQueryRaw:

    @classmethod
    @require(["JoinQuery"])
    def SpatialJoinQuery(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool, considerBoundaryIntersection: bool) -> SedonaPairRDDList:
        jvm = spatialRDD._jvm
        sc = spatialRDD._sc

        srdd = jvm.JoinQuery.SpatialJoinQuery(
            spatialRDD._srdd,
            queryRDD._srdd,
            useIndex,
            considerBoundaryIntersection
        )

        return SedonaPairRDDList(srdd, sc)

    @classmethod
    @require(["JoinQuery"])
    def DistanceJoinQuery(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool, considerBoundaryIntersection: bool) -> SedonaPairRDDList:

        jvm = spatialRDD._jvm
        sc = spatialRDD._sc
        srdd = jvm.JoinQuery.DistanceJoinQuery(
            spatialRDD._srdd,
            queryRDD._srdd,
            useIndex,
            considerBoundaryIntersection
        )
        return SedonaPairRDDList(srdd, sc)

    @classmethod
    @require(["JoinQuery"])
    def spatialJoin(cls, queryWindowRDD: SpatialRDD, objectRDD: SpatialRDD, joinParams: JoinParams) -> SedonaPairRDD:

        jvm = queryWindowRDD._jvm
        sc = queryWindowRDD._sc

        jvm_join_params = joinParams.jvm_instance(jvm)

        srdd = jvm.JoinQuery.spatialJoin(queryWindowRDD._srdd, objectRDD._srdd, jvm_join_params)

        return SedonaPairRDD(srdd, sc)

    @classmethod
    @require(["JoinQuery"])
    def DistanceJoinQueryFlat(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool, considerBoundaryIntersection: bool) -> SedonaPairRDD:

        jvm = spatialRDD._jvm
        sc = spatialRDD._sc

        spatial_join = jvm.JoinQuery.DistanceJoinQueryFlat
        srdd = spatial_join(
            spatialRDD._srdd,
            queryRDD._srdd,
            useIndex,
            considerBoundaryIntersection
        )
        return SedonaPairRDD(srdd, sc)

    @classmethod
    @require(["JoinQuery"])
    def SpatialJoinQueryFlat(cls, spatialRDD: SpatialRDD, queryRDD: SpatialRDD, useIndex: bool,
                             considerBoundaryIntersection: bool) -> SedonaPairRDD:

        jvm = spatialRDD._jvm
        sc = spatialRDD._sc

        spatial_join = jvm.JoinQuery.SpatialJoinQueryFlat
        srdd = spatial_join(
            spatialRDD._srdd,
            queryRDD._srdd,
            useIndex,
            considerBoundaryIntersection
        )
        return SedonaPairRDD(srdd, sc)


# -*- coding=UTF-8 -*-

from django import forms
from django.contrib.auth.models import User
from data.models import BrandGroup, Brand, Area
from lib.dynamicforms import Form

def users():
    users = [(x.id, str(x)) for x in User.objects.filter(is_staff=False).order_by('username')]
    users.insert(0, ('', 'выбрать',))
    return users

def brandgroups():
    g = [(x.id, "%s :: %s" % (x.direction, x.title)) \
             for x in BrandGroup.objects.all().order_by('-direction__title')]
    g.insert(0, ('', 'выбрать'))
    return g

class OrderItemForm(Form):
    TEMPLATE = 'cp/ordereditem_form.html'
    CORE = ('part_number',)

    supplier = forms.CharField(widget=forms.Select(choices=(),attrs={'onchange': 'changeDir(this);'}), label=u'DIR', required=True)
    area = forms.CharField(widget=forms.TextInput(attrs={'size':15}), label=u'Поставщик', required=True)
    brand = forms.CharField(widget=forms.TextInput(attrs={'size':15}), label=u'Бренд', required=True)
    part_number = forms.CharField(widget=forms.TextInput(attrs={'size':15}), label=u'PART #',required=True)
    comment_customer = forms.CharField(widget=forms.Textarea(attrs={'cols':15, 'rows':3}), label=u'COMMENT 1', required=False)
    comment_supplier = forms.CharField(widget=forms.Textarea(attrs={'cols':15, 'rows':3}), label=u'COMMENT 2', required=False)
    quantity = forms.IntegerField(min_value=1, widget=forms.TextInput(attrs={'size':5, 'class':'quantity'}), label=u'Q', required=True)
    client = forms.CharField(widget=forms.Select(choices=users()), label=u'CL', required=True)
    description_ru = forms.CharField(widget=forms.Textarea(attrs={'cols':15, 'rows':3}), label=u'RUS', required=False)
    description_en = forms.CharField(widget=forms.Textarea(attrs={'cols':15, 'rows':3}), label=u'ENG', required=False)
    price_base = forms.FloatField(widget=forms.TextInput(attrs={'size':5}), label=u'RETAIL', required=True)
    price_sale = forms.FloatField(widget=forms.TextInput(attrs={'size':5, 'class': 'priceSale'}), label=u'PRICE', required=True)

    def __init__(self, *args, **kwargs):
        super(OrderItemForm, self).__init__(*args, **kwargs)
        self.fields['client'].widget.choices = users()
        self.fields['supplier'].widget.choices = brandgroups()

    def clean_area(self):
        if 'area' in self.cleaned_data and self.cleaned_data['area']:
            area = self.cleaned_data['area']
            try:
                area = Area.objects.get(title__iexact=area)
            except Area.DoesNotExist:
                raise forms.ValidationError(u"Такого поставщика не существует")
            else:
                if 'supplier' in self.cleaned_data and self.cleaned_data['supplier']:
                    brandgroup = BrandGroup.objects.get(id = self.cleaned_data['supplier'])
                    if area not in brandgroup.area.all():
                        raise forms.ValidationError(u"Этот поставщик не входит в выбранное направление")

            return area

    def clean_brand(self):
        if 'brand' in self.cleaned_data and self.cleaned_data['brand']:
            brand = self.cleaned_data['brand']
            try:
                brand = Brand.objects.get(title__iexact=brand)
            except Brand.DoesNotExist:
                raise forms.ValidationError(u"Такого бренда не существует")
            else:
                if 'area' in self.cleaned_data and self.cleaned_data['area']:
                    if brand not in self.cleaned_data['area'].brands.all():
                        raise forms.ValidationError(u"Этот бренд не входит в выбранное направление")

            return brand

    def clean_client(self):
        if 'client' in self.cleaned_data and self.cleaned_data['client']:
            client = self.cleaned_data['client']
            try:
                client = User.objects.get(id = client)
            except User.DoesNotExist:
                raise forms.ValidationError(u"Такого пользователя не существует")

            return client

class SearchForm(forms.Form):
    def __init__(self, *args, **kwargs):
        maker_choices = kwargs.pop("maker_choices")
        super(SearchForm, self).__init__(*args, **kwargs)
        self.fields['maker'].widget.choices = maker_choices
    maker = forms.CharField(widget=forms.Select(choices=()), label=u'MAKE', required=True)
    part_number = forms.CharField(widget=forms.TextInput(attrs={'size':15}), label=u'Part Number', required=True)


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(

     name='sasa_phys',

     version='0.1',

     author="Tim Luca Turan, Max Bräuer",

     author_email="timturan@web.de",

     description="Semi Analytic Stacking Algorithm for Meta Surface stacks",

     long_description=long_description,

   long_description_content_type="text/markdown",

     url="https://github.com/TimLucaTuran/SASA",

     packages=setuptools.find_packages(),

     classifiers=[

         "Programming Language :: Python :: 3",

         "License :: OSI Approved :: MIT License",

         "Operating System :: OS Independent",

     ],

 )


# %%
from typing import List, Tuple

import numpy as np
import pandas as pd

class Processor(object):
    """Class with all parsing and clearning related functions"""

    def __init__(self, logger) -> None:
        self.logger = logger
        pass

    def read_data(self, path: str) -> pd.DataFrame:
        self.logger.info("Loading the data")
        df = pd.read_csv(path, sep=",")
        return df

    def save_data(self, df, path: str, nrows: int) -> None:
        self.logger.info(f"Writing data sample of size {nrows} to {path}")
        df[:nrows].to_csv(path, index=False)
        pass

    def process(self, df) -> pd.DataFrame:
        """DataFrame processing pipeline of basic operations"""
        self.logger.info("Processing the data")
        df_processed = (
            df
            .pipe(self._parse_dates)
            .pipe(self._drop_cols_logic)
            .pipe(self._drop_cols_comp)
            .pipe(self._fill_zeros)
        )
        self.logger.info("Processing finished")
        return df_processed

    def split_data(
        self, df, train_size: float = 0.7, val_size: float = 0.2
    ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
        """Create datasets for train, val, test
        Full dataset contains 200k searches,
        len(df.srch_id.unique()) -> 199,795.
        Searches are on chronological order so split can be done
        by cutting off based on srch_id.

        Args:
            df ([pd.DataFrame]): Full dataset unsplitted

        Returns:
            Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: [
                Train set containing first 70% (if default),
                Validation set containing next 20% (if default),
                Test set containing remaining data (10% default)
            ]
        """
        self.logger.info("Splitting into train, val, test")

        split1, split2 = self._calculate_split_ids(df, train_size, val_size)

        df_train = df[df["srch_id"] <= split1]
        df_val = df[(df["srch_id"] > split1) & (df["srch_id"] <= split2)]
        df_test = df[df["srch_id"] > split2]

        self.logger.info("length of train set: " + str(len(df_train)))
        self.logger.info("length of validation set: " + str(len(df_val)))
        self.logger.info("length of test set: " + str(len(df_test)))

        return df_train, df_val, df_test

    def split_Xy(self, df) -> Tuple[pd.DataFrame, pd.Series]:
        """Split the dataframe into X and y
        In the original kaggle competition the bookings had
        a weight of 4 and clicks of 1 for the ndcg_at_k metric
        These weights are handled in the xgb model
        by setting the label_gain
        """
        self.logger.info("Splitting DataFrame into X, y")
        y = df["srch_id"].to_frame()
        y = y.assign(r=df["click_bool"].add(df["booking_bool"]))
        y = y.drop(columns=["srch_id"])["r"]

        X = df.drop(columns=["click_bool", "booking_bool"])
        return X, y

    def numeric_and_complete_cols(self, df) -> List[str]:
        """create a list of columns which are complete and numeric"""
        missing_values = df.isna().sum()
        no_missing = missing_values[missing_values == 0]
        col_list = [
            x
            for x in no_missing.index
            if not x.endswith("id") and not x.endswith("bool")
        ]
        return col_list

    def _parse_dates(self, df) -> pd.DataFrame:
        """Transforms string to datetime object
        Followed by calculating the day of week,
        month and quarter for seasonal features
        """
        self.logger.info("\tparsing the date_time")
        df_dt = df.assign(date_time=pd.to_datetime(df["date_time"]))
        df_parsed = df_dt.assign(
            day_of_week=df_dt["date_time"].dt.dayofweek + 1,
            month=df_dt["date_time"].dt.month,
        )
        return df_parsed

    def _drop_cols_logic(self, df) -> pd.DataFrame:
        """The competition belonging to this data
        has a formal test set which does not contain
        the position or gross booking.
        Kaggle Expedia reccommender competition.
        """
        self.logger.info("\tdropping columns not usable")

        drop_list = ["date_time", "position", "gross_bookings_usd"]
        df_dropped = df.drop(columns=drop_list)
        return df_dropped

    def _drop_cols_comp(self, df) -> pd.DataFrame:
        """The competitor columns have no added value
        in predictive performance and are half of the
        columns

        The second set of columns is dropped to
        reduce dimensionality without a major
        loss in performance. This saves training
        time and makes shap faster.
        """
        self.logger.info("\tdropping columns competitors")

        for col in df.columns:
            if col.startswith("comp"):
                df = df.drop(columns=col)
        return df

    def _fill_zeros(self, df) -> pd.DataFrame:
        """By defenition the zeros represent missing values in this dataset
        This functions replaces the zeros with np.nan
        """
        self.logger.info("\tFilling zeros with NaNs")

        fill_names = [
            "prop_starrating",
            "prop_review_score",
            "prop_log_historical_price",
        ]
        for col_name in fill_names:
            df[col_name] = df[col_name].replace(to_replace=0, value=np.nan)
        return df

    def _calculate_split_ids(
        self, df, train_size: float, val_size: float
    ) -> Tuple[int, int]:
        srches = df["srch_id"].unique()
        split1 = srches[round(len(srches) * train_size)]
        split2 = srches[round(len(srches) * (train_size + val_size))]
        return split1, split2


import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable

class CNN(nn.Module):

    def __init__(self):
        super().__init__()

        D = 50
        C = 2
        Ci = 1
        Co = 200
        Ks = [2, 3, 4, 5]

        self.convs = nn.ModuleList([nn.Conv2d(Ci, Co, (K, D)) for K in Ks])
        self.dropout = nn.Dropout(0.2)
        self.fc1 = nn.Linear(len(Ks) * Co, Co)
        self.fc2 = nn.Linear(Co, C)

    def forward(self, x):
        #(128, max_sent, 50)
        x = x.unsqueeze(1)  # (N, Ci, W, D)

        x = [F.relu(conv(x)).squeeze(3) for conv in self.convs]  # [(N, Co, W), ...]*len(Ks)

        x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x]  # [(N, Co), ...]*len(Ks)

        x = torch.cat(x, 1)

        x = self.dropout(x)  # (N, len(Ks)*Co)
        x = self.fc1(x)  # (N, C)
        logit = self.fc2(x)
        return logit


import numpy as np
import cv2
import glob
import pickle

# チェスボードの行数、列数、サイズを定義
rows = 5
cols = 7
square_size = 0.1

# キャリブレーション用のオブジェクトを定義
objp = np.zeros((rows * cols, 3), np.float32)
objp[:, :2] = np.mgrid[0:cols, 0:rows].T.reshape(-1, 2) * square_size

print("必要なものをインポートしました。")

# 画像が入っているフォルダのパスを定義
path_list = ['./img0/cam0/.png', './img0/cam1/.png', './img0/cam2/*.png']

for i, path in enumerate(path_list):
    print(f"カメラ{i}の画像を処理します。")

# フォルダから画像を読み込み、画像のリストを作成
images = []
image_points = []
for filename in glob.glob(path):
    img = cv2.imread(filename)
    ret, corners = cv2.findChessboardCorners(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY), (cols, rows), None)
    if ret == True:
        images.append(img)
        image_points.append(corners)
    else:
        print(f"Failed to detect chessboard corners in {filename}")

print(f"カメラ{i}の画像から、画像のリストを作成しました。")

# キャリブレーションを行い、内部パラメータと外部パラメータを求める
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera([objp]*len(images), image_points, cv2.cvtColor(images[0], cv2.COLOR_BGR2GRAY).shape[::-1], None, None)

print(f"カメラ{i}のキャリブレーションを行い、内部パラメータと外部パラメータを求めました。")

# 内部パラメータと外部パラメータをprintする
print(f"カメラ{i}の内部パラメータ:")
print(mtx)
print(f"カメラ{i}の外部パラメータ:")
print(rvecs)
print(tvecs)
print("------------------------")


"""
# HBox
Lays out children in horizontal direction.
"""
import solara

from .common import ColorCard

@solara.component
def Page():
    with solara.VBox() as main:
        colors = "green red orange brown yellow pink".split()
        with solara.HBox():
            for color in colors:
                ColorCard(color, color)
    return main


import argparse
import torch.nn.functional as F
from torch.utils.data import DataLoader
import torch.nn as nn
from Models.dataloader.samplers import CategoriesSampler
from Models.utils import *
from Models.dataloader.data_utils import *
from Models.models.Network import DeepEMD
from torch.utils.tensorboard import SummaryWriter
import tqdm
import time
from datetime import datetime
import matplotlib.pyplot as plt

PRETRAIN_DIR='pretrained_model/'

DATA_DIR='/home/Improved_FSIC_Method/datasets'

parser = argparse.ArgumentParser()
#about dataset and training
parser.add_argument('-dataset', type=str, default='miniimagenet', choices=['miniimagenet','tieredimagenet'])
parser.add_argument('-data_dir', type=str, default=DATA_DIR,help='dir of datasets')

#about training
parser.add_argument('-batchsize', type=int, default=512,help='batch size of generate box')
parser.add_argument('-bs', type=int, default=1,help='batch size of tasks')
parser.add_argument('-max_epoch', type=int, default=30)
parser.add_argument('-lr', type=float, default=0.0005)
parser.add_argument('-temperature', type=float, default=12.5)
parser.add_argument('-step_size', type=int, default=10)
parser.add_argument('-gamma', type=float, default=0.5)
parser.add_argument('-val_frequency',type=int,default=50)
parser.add_argument('-random_val_task', type=int, default=1, help='random samples tasks for validation at each epoch')
#about task
parser.add_argument('-way', type=int, default=5)
parser.add_argument('-shot', type=int, default=1)
parser.add_argument('-query', type=int, default=1,help='number of query image per class')
parser.add_argument('-val_episode', type=int, default=2000, help='number of validation episode')
parser.add_argument('-test_episode', type=int, default=5000, help='number of testing episodes after training')

# about model
parser.add_argument('-pretrain_dir', type=str, default=PRETRAIN_DIR)
parser.add_argument('-metric', type=str, default='ADM', choices=['Cosine', 'L2', 'Dot', 'ADM'])
parser.add_argument('-norm', type=str, default='center', choices=['center'], help='feature normalization')
parser.add_argument('-deepemd', type=str, default='ic', choices=['fcn', 'ic'])
parser.add_argument('-num_patch',type=int,default=9)

# emd_cc setting
parser.add_argument('-outer_num', type=int, default=2)
parser.add_argument('-all_in_num', type=int, default=1)
parser.add_argument('-alpha', type=float, default=0.4)

# ADM setting
parser.add_argument('-cosine_rate', type=float, default=0.8)

# SFC
parser.add_argument('-sfc_lr', type=float, default=0.1, help='learning rate of SFC')
parser.add_argument('-sfc_wd', type=float, default=0, help='weight decay for SFC weight')
parser.add_argument('-sfc_update_step', type=float, default=100, help='number of updating step of SFC')
parser.add_argument('-sfc_bs', type=int, default=4, help='batch size for finetune sfc')

# OTHERS
parser.add_argument('-gpu', default='0')
parser.add_argument('-extra_dir', type=str,default=None,help='extra information that is added to checkpoint dir, e.g. hyperparameters')
parser.add_argument('-seed', type=int, default=1)
parser.add_argument('-train_time', default=datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), type=str, metavar='PATH', help='path to cache (default: none)')

args = parser.parse_args()
pprint(vars(args))

set_seed(args.seed)
num_gpu = set_gpu(args)
Dataset=set_up_datasets(args)

# model
args.pretrain_dir = osp.join(args.pretrain_dir, args.dataset, 'resnet12/max_acc.pth')
print(args.pretrain_dir)
model = DeepEMD(args)
model = load_model(model, args.pretrain_dir)
model = nn.DataParallel(model, list(range(num_gpu)))
model = model.cuda()
model.eval()

args.save_path = '%s/%s/%dquery-%dshot-%dway/'%(args.dataset,args.deepemd,args.query,args.shot,args.way)

args.save_path = osp.join('checkpoint', args.save_path, args.train_time)

ensure_path(args.save_path)

############################################
# generate box
############################################

# Dataset prepare
trainset = Dataset('train', args)
valset = Dataset('val', args)

# Upadate box of dataset
def update_box(cc_loader, model, t=0.5):
    # print('==> Start updating boxes...')
    model.eval()
    boxes = []
    for _, batch in enumerate(cc_loader):
        data, _ = [_.cuda() for _ in batch]
        with torch.no_grad():
            feat_map = model(data)  # (N, C, H, W)
        N, Cf, Hf, Wf = feat_map.shape
        eval_train_map = feat_map.sum(1).view(N, -1)  # (N, Hf*Wf)
        eval_train_map = eval_train_map - eval_train_map.min(1, keepdim=True)[0]
        eval_train_map = eval_train_map / eval_train_map.max(1, keepdim=True)[0]
        eval_train_map = eval_train_map.view(N, 1, Hf, Wf)
        eval_train_map = F.interpolate(eval_train_map, size=data.shape[-2:], mode='bilinear')  # (N, 1, Hi, Wi)

        Hi, Wi = data.shape[-2:]

        for hmap in eval_train_map:
            hmap = hmap.squeeze(0)  # (Hi, Wi)

            h_filter = (hmap.max(1)[0] > t).int()
            w_filter = (hmap.max(0)[0] > t).int()

            h_min, h_max = torch.nonzero(h_filter).view(-1)[[0, -1]] / Hi  # [h_min, h_max]; 0 <= h <= 1
            w_min, w_max = torch.nonzero(w_filter).view(-1)[[0, -1]] / Wi  # [w_min, w_max]; 0 <= w <= 1
            boxes.append(torch.tensor([h_min, w_min, h_max, w_max]))

    all_boxes = torch.stack(boxes, dim=0).cuda()  # (num_iters, 4)
    return all_boxes

# Box generate
train_cc_loader = DataLoader(dataset=trainset, batch_size=args.batchsize, shuffle=False, num_workers=64, pin_memory=True, drop_last=False)
val_cc_loader = DataLoader(dataset=valset, batch_size=args.batchsize, shuffle=False, num_workers=64, pin_memory=True, drop_last=False)

print('==> Start generating train boxes...')
trainset.generate_box = True

len_ds = len(trainset)
all_boxes_outer = update_box(train_cc_loader, model.module.encoder, 0.5)
assert len(all_boxes_outer) == len_ds
trainset.boxes_outer = all_boxes_outer.cpu()

trainset.generate_box = False
print('Train box generated!!!')

print('==> Start generating val boxes...')
valset.generate_box = True

len_ds = len(valset)
all_boxes_outer = update_box(val_cc_loader, model.module.encoder, 0.5)
assert len(all_boxes_outer) == len_ds
valset.boxes_outer = all_boxes_outer.cpu()

valset.generate_box = False
print('Val box generated!!!')
############################################

# meta-train loader
train_sampler = CategoriesSampler(trainset.label, args.val_frequency*args.bs, args.way, args.shot + args.query)
train_loader = DataLoader(dataset=trainset, batch_sampler=train_sampler, num_workers=64, pin_memory=True)

val_sampler = CategoriesSampler(valset.label, args.val_episode, args.way, args.shot + args.query)
val_loader = DataLoader(dataset=valset, batch_sampler=val_sampler, num_workers=64, pin_memory=True)

if args.random_val_task == 0:
    print ('fix val set for all epochs')
    val_loader=[x for x in val_loader]

#label for query set, always in the same pattern
label = torch.arange(args.way, dtype=torch.int8).repeat(args.query)#012340123401234...
label = label.type(torch.LongTensor)
label = label.cuda()

optimizer = torch.optim.SGD([{'params': model.parameters(),'lr':args.lr}], momentum=0.9, nesterov=True, weight_decay=0.0005)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=args.step_size, gamma=args.gamma)

def save_model(name):
    torch.save(dict(params=model.state_dict()), osp.join(args.save_path, name + '.pth'))

trlog = {}
trlog['args'] = vars(args)
trlog['train_loss'] = []
trlog['val_loss'] = []
trlog['train_acc'] = []
trlog['val_acc'] = []
trlog['max_acc'] = 0.0
trlog['max_acc_epoch'] = 0

global_count = 0
writer = SummaryWriter(osp.join(args.save_path,'tf'))

result_list=[args.save_path]
for epoch in range(1, args.max_epoch + 1):
    print (args.save_path)
    start_time=time.time()

    tl = Averager()
    ta = Averager()

    tqdm_gen = tqdm.tqdm(train_loader)
    model.train()
    optimizer.zero_grad()
    for i, batch in enumerate(tqdm_gen, 1):

        global_count = global_count + 1
        data, _ = [_.cuda() for _ in batch]

        k = args.way * args.shot
        model.module.mode = 'encoder'
        data = model(data)
        data_shot, data_query = data[:k], data[k:]
        model.module.mode = 'meta'
        if args.shot > 1:
            data_shot = model.module.get_sfc(data_shot)
        logits = model((data_shot.unsqueeze(0).repeat(num_gpu, 1, 1, 1, 1), data_query))
        loss = F.cross_entropy(logits, label)

        acc = count_acc(logits, label)
        writer.add_scalar('data/loss', float(loss), global_count)
        writer.add_scalar('data/acc', float(acc), global_count)

        total_loss = loss/args.bs#batch of tasks, done by accumulate gradients
        writer.add_scalar('data/total_loss', float(total_loss), global_count)
        tqdm_gen.set_description('epo {}, total loss={:.4f} acc={:.4f}'
              .format(epoch, total_loss.item(), acc))
        tl.add(total_loss.item())
        ta.add(acc)
        total_loss.backward()

        detect_grad_nan(model)
        if i%args.bs==0: #batch of tasks, done by accumulate gradients
            optimizer.step()
            optimizer.zero_grad()

    tl = tl.item()
    ta = ta.item()
    vl = Averager()
    va = Averager()

    #validation
    model.eval()
    with torch.no_grad():
        tqdm_gen = tqdm.tqdm(val_loader)
        for i, batch in enumerate(tqdm_gen, 1):
            data, _ = [_.cuda() for _ in batch]
            k = args.way * args.shot
            model.module.mode = 'encoder'
            data = model(data)
            data_shot, data_query = data[:k], data[k:]
            model.module.mode = 'meta'
            if args.shot > 1:
                data_shot = model.module.get_sfc(data_shot)
            logits = model((data_shot.unsqueeze(0).repeat(num_gpu, 1, 1, 1, 1), data_query))

            loss = F.cross_entropy(logits, label)
            acc = count_acc(logits, label)
            vl.add(loss.item())
            va.add(acc)

    vl = vl.item()
    va = va.item()
    writer.add_scalar('data/val_loss', float(vl), epoch)
    writer.add_scalar('data/val_acc', float(va), epoch)
    tqdm_gen.set_description('epo {}, val, loss={:.4f} acc={:.4f}'.format(epoch, vl, va))
    print('epoch: {}, val_loss={:.4f} acc={:.4f}'.format(epoch, vl, va))

    print ('val acc:%.4f'%va)
    if va >= trlog['max_acc']:
        print ('*********A better model is found*********')
        trlog['max_acc'] = va
        trlog['max_acc_epoch'] = epoch
        save_model('max_acc')

    trlog['train_loss'].append(tl)
    trlog['train_acc'].append(ta)
    trlog['val_loss'].append(vl)
    trlog['val_acc'].append(va)

    result_list.append('epoch:%03d,training_loss:%.5f,training_acc:%.5f,val_loss:%.5f,val_acc:%.5f'%(epoch,tl,ta,vl,va))

    torch.save(trlog, osp.join(args.save_path, 'trlog'))

    print('best epoch {}, best val acc={:.4f}'.format(trlog['max_acc_epoch'], trlog['max_acc']))
    print ('This epoch takes %d seconds'%(time.time()-start_time),'\nstill need %.2f hour to finish'%((time.time()-start_time)*(args.max_epoch-epoch)/3600))
    lr_scheduler.step()

writer.close()

# Test Phase
trlog = torch.load(osp.join(args.save_path, 'trlog'))
print('Test episodes: ', args.test_episode)
test_acc_record = np.zeros((args.test_episode,))
model.load_state_dict(torch.load(osp.join(args.save_path, 'max_acc' + '.pth'))['params'])
model.eval()

# data prepare
testset = Dataset('test', args)
test_cc_loader = DataLoader(dataset=testset, batch_size=args.batchsize, shuffle=False, num_workers=64, pin_memory=True, drop_last=False)
########################################
print('==> Start generating test boxes...')
testset.generate_box = True

len_ds = len(testset)
all_boxes_outer = update_box(test_cc_loader, model.module.encoder, 0.5)
assert len(all_boxes_outer) == len_ds
testset.boxes_outer = all_boxes_outer.cpu()

testset.generate_box = False
print('Test box generated!!!')
########################################

sampler = CategoriesSampler(testset.label, args.test_episode, args.way, args.shot + args.query)
test_loader = DataLoader(testset, batch_sampler=sampler, num_workers=64, pin_memory=True)

ave_acc = Averager()
label = torch.arange(args.way).repeat(args.query)
if torch.cuda.is_available():
    label = label.type(torch.cuda.LongTensor)
else:
    label = label.type(torch.LongTensor)

tqdm_gen = tqdm.tqdm(test_loader)
with torch.no_grad():
    for i, batch in enumerate(tqdm_gen, 1):
        data, _ = [_.cuda() for _ in batch]
        k = args.way * args.shot
        model.module.mode = 'encoder'
        data = model(data)
        data_shot, data_query = data[:k], data[k:]
        model.module.mode = 'meta'
        if args.shot > 1:
            data_shot = model.module.get_sfc(data_shot)
        logits = model((data_shot.unsqueeze(0).repeat(num_gpu, 1, 1, 1, 1), data_query))
        acc = count_acc(logits, label)* 100
        ave_acc.add(acc)
        test_acc_record[i-1] = acc
        tqdm_gen.set_description('batch {}: {:.2f}({:.2f})'.format(i, ave_acc.item(), acc))

m, pm = compute_confidence_interval(test_acc_record)

result_list.append('Test Acc {:.4f} + {:.4f}'.format(m, pm))

print(result_list[-2])
print(result_list[-1])

save_list_to_txt(os.path.join(args.save_path, 'results.txt'), result_list)


# torch
import torch
from torch import nn
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

# utils
import utilities
import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import confusion_matrix

class AutoEncoder(nn.Module):
    def __init__(self, inp_size, hidden_dim, out_dim):
        super().__init__()

        current_dim = inp_size

        self.autoencoder = nn.Sequential()

        i = 0

        for dim in hidden_dim:
            self.autoencoder.add_module(name='Linear_module_' + str(i), module=nn.Linear(current_dim, dim, bias=True))
            self.autoencoder.add_module(name='ReLU_module_' + str(i), module=nn.LeakyReLU(negative_slope=0.4))
            self.autoencoder.add_module(name='Dropout_module' + str(i), module=nn.Dropout(p=0.2))
            current_dim = dim
            i = i + 1

        self.autoencoder.add_module(name='Linear_module_out', module=nn.Linear(current_dim, out_dim))
        self.autoencoder.add_module(name='module_LReLU_out', module=nn.Sigmoid())

        init_weights(self)

    def forward(self, x):
        return self.autoencoder(x)

def init_weights(my_module):
    for sub_module in my_module.modules():
        if isinstance(sub_module, nn.Linear):
            nn.init.kaiming_normal_(sub_module.weight, nonlinearity='leaky_relu')
            nn.init.constant_(sub_module.bias, 0)

def batch_preparation(data, label, batch_size):
    X = data[data.select_dtypes('number').columns.tolist()]
    y = data[label]
    X = X.drop([label], axis=1)

    sc = MinMaxScaler()
    X = pd.DataFrame(data=sc.fit_transform(X), columns=X.columns)

    X_torch = torch.from_numpy(X.values).float()

    data_loader = DataLoader(X_torch, batch_size=batch_size)

    return X_torch, y, data_loader

def train_ae(X_torch, data_loader, model, n_epochs, tb, lr):
    log_interval = 10000

    loss_function = nn.L1Loss(reduction='mean')
    optim = torch.optim.Adam(model.parameters(), lr=lr)

    data = torch.autograd.Variable(X_torch)

    reconstruction_loss_all = 0

    model.train()

    for epoch in range(n_epochs):
        i = 1
        loss_on_epoch = 0
        for mini_batch in data_loader:
            optim.zero_grad()
            batch_recon = model(mini_batch.float())
            loss = loss_function(batch_recon, mini_batch)

            loss_on_epoch += loss.item()

            loss.backward()
            optim.step()

            if (i + 1) % log_interval == 0:
                print("Epoch {} - batch {}. \n The error in this mini_bach was: {:.3f}".format(epoch + 1,
                                                                                i + 1,
                                                                                loss.item()))

            i += 1
        print("Epoch {}. \n The mean error on this epoch was: {:.3f}".format(epoch + 1,
                                                                             loss_on_epoch / len(data_loader)))

        print("... {:.0%} of train completed".format((epoch + 1) / n_epochs))

        tb.add_scalar('Loss', loss_on_epoch, epoch+1)

    model.eval()
    reconstruction = model(data)
    test_loss = nn.L1Loss(reduction='none')
    reconstruction_loss_all = test_loss(reconstruction, data)
    model.train()

    return reconstruction_loss_all

def prediction_quality(reconstruction_loss, y, params, tb):
    reconstruction_loss_np = reconstruction_loss.detach().numpy().sum(axis=1)

    # utilities.plot_outlier_scores(y, reconstruction_loss_np, bw=0.1,
    #                              title='Fraud, Personalized Autoencoder. (epochs={})'.format(params.torch_epochs))

    contaminated_record = int(reconstruction_loss_np.shape[0] * 0.02)

    loss_df = pd.DataFrame(reconstruction_loss_np, columns=['loss'])
    threshold = loss_df.sort_values(ascending=False, by='loss').head(contaminated_record).tail(1).values
    loss_df['label'] = loss_df['loss'].map(lambda x: 1 if x >= threshold[0] else 0)

    report = pd.concat([pd.Series(y, name='true', dtype='int32'),
                        pd.Series(loss_df['label'], name='label', dtype='int32')],
                       axis=1)

    report['true positive'] = np.where((report['true'] == 1) & (report['label'] == 1), 1, 0)
    report['false negative'] = np.where((report['true'] == 1) & (report['label'] == 0), 1, 0)
    report['false positive'] = np.where((report['true'] == 0) & (report['label'] == 1), 1, 0)
    report['true negative'] = np.where((report['true'] == 0) & (report['label'] == 0), 1, 0)

    coverage = report['true positive'].sum() / report['true'].sum()

    print('Checking {} case, the coverage on frauds is about {:.0%}'
          .format(report['label'].sum(),
                  coverage))

    tb.add_text(tag='Report', text_string='Checking {} case, the coverage on frauds is about {:.0%}'
                .format(report['label'].sum(),
                        coverage))

    cm = confusion_matrix(y_true=report['true'].values, y_pred=report['label'].values)

    tb.add_text(tag='Report', text_string='True Negative: {};\n True Positive: {};'
                                          '\n False Positive {};\n False Negative {}.'
                .format(cm[0][0], cm[1][1], cm[0][1], cm[1][0]), global_step=None, walltime=None)

    tb.add_hparams(hparam_dict={'learning rate': params.torch_lr,
                                'batch size': params.torch_batch,
                                'hidden_dimensions': str(params.torch_hidden_dim),
                                'epochs': params.torch_epochs,
                                'loss': 'L1',
                                'activation output': 'sigmoid',
                                'dropout': 'active'
                                }, metric_dict={})

    tb.close()

    return report

def personalized_autoencoder(data, params):
    comment = "Run's ID = {}".format(params.torch_id)
    tb = SummaryWriter(comment=comment)

    X_torch, y,  data_loader = batch_preparation(data, params.label, params.torch_batch)
    model = AutoEncoder(params.torch_input_dim,
                        params.torch_hidden_dim,
                        params.torch_output_dim).float()

    tb.add_graph(model=model, input_to_model=next(iter(data_loader)))

    reconstruction_loss = train_ae(X_torch, data_loader, model, params.torch_epochs, tb, params.torch_lr)
    report = prediction_quality(reconstruction_loss, y, params, tb)

    return report


from unidecode import unidecode

def tratar_caracteres_especiais(palavra: str):
    ## Essa função possui 3 partes:
    # 1 - remover caracteres especiais com unidecode_usage
    # 2 - remover ', ´, `,
    # 3 - remover - e _

    # 1 - Isso aqui remove os acentos da palavra -> ç, á, à, ã, ä
    palavra_tratada = unidecode(palavra)

    # 2 - Aqui vamos remover os caracteres especiais -> ', `, ´
    caracteres_1 = "'´`"
    for i in range(0, len(caracteres_1)):
        palavra_tratada = palavra_tratada.replace(caracteres_1[i], '')  # trocar caractere por "backspace"
    # Caracteres ', `, ´ retirados

    # 3 - Aqui vamos remover os caracteres especiais -> -, _,
    caracteres_2 = "-_"
    for i in range(0, len(caracteres_2)):
        palavra_tratada = palavra_tratada.replace(caracteres_2[i], ' ')  # Trocar caractere por "espaço"

    return palavra_tratada

def upper_case_trim(palavra: str):
    return palavra.upper().strip()

nome = "Pingo-d'Água"

print('Nome original:', nome)
print('Nome UpperCase e "stripped":', upper_case_trim(nome))
print('Nome sem caracteres especiais:', tratar_caracteres_especiais(nome))


import readline  # noqa
from code import InteractiveConsole
from logging import basicConfig, DEBUG

from autoapi.apinode import APINode

if __name__ == '__main__':

    basicConfig(level=DEBUG)
    m = APINode('sphinx')

    for node, leaves in m.walk():
        print(
            '{} node has leaves: {}'.format(
                node.name, ', '.join([l.name for l in leaves])
            )
        )

    InteractiveConsole(globals()).interact()


import os
import argparse
import tensorflow as tf

parser = argparse.ArgumentParser()
parser.add_argument('--records_name', type=str, required=True,
                    help='Name of tfrecords')
parser.add_argument('--output_path', type=str, required=True,
                    help="Folder to store the data")
parser.add_argument('--list_file', type=str, required=False, default='filelist.txt',
                    help='File to record the list of the data')
parser.add_argument('--count', type=int, required=False, default=0,
                    help='Specify the count of the data to extract')
parser.add_argument('--file_type', type=str, required=False, default='data',
                    help='File type')

def read_data_from_tfrecords(records_name, output_path, list_file, file_type, count):
  records_iterator = tf.python_io.tf_record_iterator(records_name)
  count = count if count != 0 else float('Inf')

  with open(os.path.join(output_path, list_file), "w") as f:
    num = 0
    for string_record in records_iterator:
      if num >= count: break

      example = tf.train.Example()
      example.ParseFromString(string_record)
      label  = int(example.features.feature['label'].int64_list.value[0])
      # index  = int(example.features.feature['index'].int64_list.value[0])
      octree = example.features.feature[file_type].bytes_list.value[0]
      if 'filename' in example.features.feature:
        filename = example.features.feature['filename'].bytes_list.value[0] \
                          .decode('utf8').replace('/', '_').replace('\\', '_')
      else:
        filename = '%06d.%s' % (num, file_type)

      num += 1
      with open(os.path.join(output_path, filename), 'wb') as fo:
        fo.write(octree)

      f.write("{} {}\n".format(filename, label))

if __name__ == '__main__':
  args = parser.parse_args()

  if not os.path.exists(args.output_path):
    os.makedirs(args.output_path)

  read_data_from_tfrecords(args.records_name,
                           args.output_path,
                           args.list_file,
                           args.file_type,
                           args.count)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import abc
import collections
import json
import os
import six
import tensorflow.compat.v1 as tf

import configure_finetuning
from finetune import feature_spec
from finetune import task
from finetune.qa import qa_metrics
from model import modeling
from model import tokenization
from util import utils

class QAExample(task.Example):
  """Question-answering example."""

  def __init__(self,
               task_name,
               eid,
               qas_id,
               qid,
               question_text,
               doc_tokens,
               orig_answer_text=None,
               start_position=None,
               end_position=None,
               is_impossible=False):
    super(QAExample, self).__init__(task_name)
    self.eid = eid
    self.qas_id = qas_id
    self.qid = qid
    self.question_text = question_text
    self.doc_tokens = doc_tokens
    self.orig_answer_text = orig_answer_text
    self.start_position = start_position
    self.end_position = end_position
    self.is_impossible = is_impossible

  def __str__(self):
    return self.__repr__()

  def __repr__(self):
    s = ""
    s += "qas_id: %s" % (tokenization.printable_text(self.qas_id))
    s += ", question_text: %s" % (
        tokenization.printable_text(self.question_text))
    s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens))
    if self.start_position:
      s += ", start_position: %d" % self.start_position
    if self.start_position:
      s += ", end_position: %d" % self.end_position
    if self.start_position:
      s += ", is_impossible: %r" % self.is_impossible
    return s

def _check_is_max_context(doc_spans, cur_span_index, position):
  """Check if this is the 'max context' doc span for the token."""

  # Because of the sliding window approach taken to scoring documents, a single
  # token can appear in multiple documents. E.g.
  #  Doc: the man went to the store and bought a gallon of milk
  #  Span A: the man went to the
  #  Span B: to the store and bought
  #  Span C: and bought a gallon of
  #  ...
  #
  # Now the word 'bought' will have two scores from spans B and C. We only
  # want to consider the score with "maximum context", which we define as
  # the *minimum* of its left and right context (the *sum* of left and
  # right context will always be the same, of course).
  #
  # In the example the maximum context for 'bought' would be span C since
  # it has 1 left context and 3 right context, while span B has 4 left context
  # and 0 right context.
  best_score = None
  best_span_index = None
  for (span_index, doc_span) in enumerate(doc_spans):
    end = doc_span.start + doc_span.length - 1
    if position < doc_span.start:
      continue
    if position > end:
      continue
    num_left_context = position - doc_span.start
    num_right_context = end - position
    score = min(num_left_context, num_right_context) + 0.01 * doc_span.length
    if best_score is None or score > best_score:
      best_score = score
      best_span_index = span_index

  return cur_span_index == best_span_index

def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer,
                         orig_answer_text):
  """Returns tokenized answer spans that better match the annotated answer."""

  # The SQuAD annotations are character based. We first project them to
  # whitespace-tokenized words. But then after WordPiece tokenization, we can
  # often find a "better match". For example:
  #
  #   Question: What year was John Smith born?
  #   Context: The leader was John Smith (1895-1943).
  #   Answer: 1895
  #
  # The original whitespace-tokenized answer will be "(1895-1943).". However
  # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match
  # the exact answer, 1895.
  #
  # However, this is not always possible. Consider the following:
  #
  #   Question: What country is the top exporter of electornics?
  #   Context: The Japanese electronics industry is the lagest in the world.
  #   Answer: Japan
  #
  # In this case, the annotator chose "Japan" as a character sub-span of
  # the word "Japanese". Since our WordPiece tokenizer does not split
  # "Japanese", we just use "Japanese" as the annotation. This is fairly rare
  # in SQuAD, but does happen.
  tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text))

  for new_start in range(input_start, input_end + 1):
    for new_end in range(input_end, new_start - 1, -1):
      text_span = " ".join(doc_tokens[new_start:(new_end + 1)])
      if text_span == tok_answer_text:
        return new_start, new_end

  return input_start, input_end

def is_whitespace(c):
  return c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F

class QATask(task.Task):
  """A span-based question answering tasks (e.g., SQuAD)."""

  __metaclass__ = abc.ABCMeta

  def __init__(self, config: configure_finetuning.FinetuningConfig, name,
               tokenizer, v2=False):
    super(QATask, self).__init__(config, name)
    self._tokenizer = tokenizer
    self._examples = {}
    self.v2 = v2

  def _add_examples(self, examples, example_failures, paragraph, split):
    paragraph_text = paragraph["context"]
    doc_tokens = []
    char_to_word_offset = []
    prev_is_whitespace = True
    for c in paragraph_text:
      if is_whitespace(c):
        prev_is_whitespace = True
      else:
        if prev_is_whitespace:
          doc_tokens.append(c)
        else:
          doc_tokens[-1] += c
        prev_is_whitespace = False
      char_to_word_offset.append(len(doc_tokens) - 1)

    for qa in paragraph["qas"]:
      qas_id = qa["id"] if "id" in qa else None
      qid = qa["qid"] if "qid" in qa else None
      question_text = qa["question"]
      start_position = None
      end_position = None
      orig_answer_text = None
      is_impossible = False
      if split == "train":
        if self.v2:
          is_impossible = qa["is_impossible"]
        if not is_impossible:
          if "detected_answers" in qa:  # MRQA format
            answer = qa["detected_answers"][0]
            answer_offset = answer["char_spans"][0][0]
          else:  # SQuAD format
            answer = qa["answers"][0]
            answer_offset = answer["answer_start"]
          orig_answer_text = answer["text"]
          answer_length = len(orig_answer_text)
          start_position = char_to_word_offset[answer_offset]
          if answer_offset + answer_length - 1 >= len(char_to_word_offset):
            utils.log("End position is out of document!")
            example_failures[0] += 1
            continue
          end_position = char_to_word_offset[answer_offset + answer_length - 1]

          # Only add answers where the text can be exactly recovered from the
          # document. If this CAN'T happen it's likely due to weird Unicode
          # stuff so we will just skip the example.
          #
          # Note that this means for training mode, every example is NOT
          # guaranteed to be preserved.
          actual_text = " ".join(
              doc_tokens[start_position:(end_position + 1)])
          cleaned_answer_text = " ".join(
              tokenization.whitespace_tokenize(orig_answer_text))
          actual_text = actual_text.lower()
          cleaned_answer_text = cleaned_answer_text.lower()
          if actual_text.find(cleaned_answer_text) == -1:
            utils.log("Could not find answer: '{:}' in doc vs. "
                      "'{:}' in provided answer".format(
                          tokenization.printable_text(actual_text),
                          tokenization.printable_text(cleaned_answer_text)))
            example_failures[0] += 1
            continue
        else:
          start_position = -1
          end_position = -1
          orig_answer_text = ""

      example = QAExample(
          task_name=self.name,
          eid=len(examples),
          qas_id=qas_id,
          qid=qid,
          question_text=question_text,
          doc_tokens=doc_tokens,
          orig_answer_text=orig_answer_text,
          start_position=start_position,
          end_position=end_position,
          is_impossible=is_impossible)
      examples.append(example)

  def get_feature_specs(self):
    return [
        feature_spec.FeatureSpec(self.name + "_eid", []),
        feature_spec.FeatureSpec(self.name + "_start_positions", []),
        feature_spec.FeatureSpec(self.name + "_end_positions", []),
        feature_spec.FeatureSpec(self.name + "_is_impossible", []),
    ]

  def featurize(self, example: QAExample, is_training, log=False,
                for_eval=False):
    all_features = []
    query_tokens = self._tokenizer.tokenize(example.question_text)

    if len(query_tokens) > self.config.max_query_length:
      query_tokens = query_tokens[0:self.config.max_query_length]

    tok_to_orig_index = []
    orig_to_tok_index = []
    all_doc_tokens = []
    for (i, token) in enumerate(example.doc_tokens):
      orig_to_tok_index.append(len(all_doc_tokens))
      sub_tokens = self._tokenizer.tokenize(token)
      for sub_token in sub_tokens:
        tok_to_orig_index.append(i)
        all_doc_tokens.append(sub_token)

    tok_start_position = None
    tok_end_position = None
    if is_training and example.is_impossible:
      tok_start_position = -1
      tok_end_position = -1
    if is_training and not example.is_impossible:
      tok_start_position = orig_to_tok_index[example.start_position]
      if example.end_position < len(example.doc_tokens) - 1:
        tok_end_position = orig_to_tok_index[example.end_position + 1] - 1
      else:
        tok_end_position = len(all_doc_tokens) - 1
      (tok_start_position, tok_end_position) = _improve_answer_span(
          all_doc_tokens, tok_start_position, tok_end_position, self._tokenizer,
          example.orig_answer_text)

    # The -3 accounts for [CLS], [SEP] and [SEP]
    max_tokens_for_doc = self.config.max_seq_length - len(query_tokens) - 3

    # We can have documents that are longer than the maximum sequence length.
    # To deal with this we do a sliding window approach, where we take chunks
    # of the up to our max length with a stride of `doc_stride`.
    _DocSpan = collections.namedtuple(  # pylint: disable=invalid-name
        "DocSpan", ["start", "length"])
    doc_spans = []
    start_offset = 0
    while start_offset < len(all_doc_tokens):
      length = len(all_doc_tokens) - start_offset
      if length > max_tokens_for_doc:
        length = max_tokens_for_doc
      doc_spans.append(_DocSpan(start=start_offset, length=length))
      if start_offset + length == len(all_doc_tokens):
        break
      start_offset += min(length, self.config.doc_stride)

    for (doc_span_index, doc_span) in enumerate(doc_spans):
      tokens = []
      token_to_orig_map = {}
      token_is_max_context = {}
      segment_ids = []
      tokens.append("[CLS]")
      segment_ids.append(0)
      for token in query_tokens:
        tokens.append(token)
        segment_ids.append(0)
      tokens.append("[SEP]")
      segment_ids.append(0)

      for i in range(doc_span.length):
        split_token_index = doc_span.start + i
        token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]

        is_max_context = _check_is_max_context(doc_spans, doc_span_index,
                                               split_token_index)
        token_is_max_context[len(tokens)] = is_max_context
        tokens.append(all_doc_tokens[split_token_index])
        segment_ids.append(1)
      tokens.append("[SEP]")
      segment_ids.append(1)

      input_ids = self._tokenizer.convert_tokens_to_ids(tokens)

      # The mask has 1 for real tokens and 0 for padding tokens. Only real
      # tokens are attended to.
      input_mask = [1] * len(input_ids)

      # Zero-pad up to the sequence length.
      while len(input_ids) < self.config.max_seq_length:
        input_ids.append(0)
        input_mask.append(0)
        segment_ids.append(0)

      assert len(input_ids) == self.config.max_seq_length
      assert len(input_mask) == self.config.max_seq_length
      assert len(segment_ids) == self.config.max_seq_length

      start_position = None
      end_position = None
      if is_training and not example.is_impossible:
        # For training, if our document chunk does not contain an annotation
        # we throw it out, since there is nothing to predict.
        doc_start = doc_span.start
        doc_end = doc_span.start + doc_span.length - 1
        out_of_span = False
        if not (tok_start_position >= doc_start and
                tok_end_position <= doc_end):
          out_of_span = True
        if out_of_span:
          start_position = 0
          end_position = 0
        else:
          doc_offset = len(query_tokens) + 2
          start_position = tok_start_position - doc_start + doc_offset
          end_position = tok_end_position - doc_start + doc_offset

      if is_training and example.is_impossible:
        start_position = 0
        end_position = 0

      if log:
        utils.log("*** Example ***")
        utils.log("doc_span_index: %s" % doc_span_index)
        utils.log("tokens: %s" % " ".join(
            [tokenization.printable_text(x) for x in tokens]))
        utils.log("token_to_orig_map: %s" % " ".join(
            ["%d:%d" % (x, y) for (x, y) in six.iteritems(token_to_orig_map)]))
        utils.log("token_is_max_context: %s" % " ".join([
            "%d:%s" % (x, y) for (x, y) in six.iteritems(token_is_max_context)
        ]))
        utils.log("input_ids: %s" % " ".join([str(x) for x in input_ids]))
        utils.log("input_mask: %s" % " ".join([str(x) for x in input_mask]))
        utils.log("segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
        if is_training and example.is_impossible:
          utils.log("impossible example")
        if is_training and not example.is_impossible:
          answer_text = " ".join(tokens[start_position:(end_position + 1)])
          utils.log("start_position: %d" % start_position)
          utils.log("end_position: %d" % end_position)
          utils.log("answer: %s" % (tokenization.printable_text(answer_text)))

      features = {
          "task_id": self.config.task_names.index(self.name),
          self.name + "_eid": (1000 * example.eid) + doc_span_index,
          "input_ids": input_ids,
          "input_mask": input_mask,
          "segment_ids": segment_ids,
      }
      if for_eval:
        features.update({
            self.name + "_doc_span_index": doc_span_index,
            self.name + "_tokens": tokens,
            self.name + "_token_to_orig_map": token_to_orig_map,
            self.name + "_token_is_max_context": token_is_max_context,
        })
      if is_training:
        features.update({
            self.name + "_start_positions": start_position,
            self.name + "_end_positions": end_position,
            self.name + "_is_impossible": example.is_impossible
        })
      all_features.append(features)
    return all_features

  def get_prediction_module(self, bert_model, features, is_training,
                            percent_done):
    final_hidden = bert_model.get_sequence_output()

    final_hidden_shape = modeling.get_shape_list(final_hidden, expected_rank=3)
    batch_size = final_hidden_shape[0]
    seq_length = final_hidden_shape[1]

    answer_mask = tf.cast(features["input_mask"], tf.float32)
    answer_mask *= tf.cast(features["segment_ids"], tf.float32)
    answer_mask += tf.one_hot(0, seq_length)

    start_logits = tf.squeeze(tf.layers.dense(final_hidden, 1), -1)

    start_top_log_probs = tf.zeros([batch_size, self.config.beam_size])
    start_top_index = tf.zeros([batch_size, self.config.beam_size], tf.int32)
    end_top_log_probs = tf.zeros([batch_size, self.config.beam_size,
                                  self.config.beam_size])
    end_top_index = tf.zeros([batch_size, self.config.beam_size,
                              self.config.beam_size], tf.int32)
    if self.config.joint_prediction:
      start_logits += 1000.0 * (answer_mask - 1)
      start_log_probs = tf.nn.log_softmax(start_logits)
      start_top_log_probs, start_top_index = tf.nn.top_k(
          start_log_probs, k=self.config.beam_size)

      if not is_training:
        # batch, beam, length, hidden
        end_features = tf.tile(tf.expand_dims(final_hidden, 1),
                               [1, self.config.beam_size, 1, 1])
        # batch, beam, length
        start_index = tf.one_hot(start_top_index,
                                 depth=seq_length, axis=-1, dtype=tf.float32)
        # batch, beam, hidden
        start_features = tf.reduce_sum(
            tf.expand_dims(final_hidden, 1) *
            tf.expand_dims(start_index, -1), axis=-2)
        # batch, beam, length, hidden
        start_features = tf.tile(tf.expand_dims(start_features, 2),
                                 [1, 1, seq_length, 1])
      else:
        start_index = tf.one_hot(
            features[self.name + "_start_positions"], depth=seq_length,
            axis=-1, dtype=tf.float32)
        start_features = tf.reduce_sum(tf.expand_dims(start_index, -1) *
                                       final_hidden, axis=1)
        start_features = tf.tile(tf.expand_dims(start_features, 1),
                                 [1, seq_length, 1])
        end_features = final_hidden

      final_repr = tf.concat([start_features, end_features], -1)
      final_repr = tf.layers.dense(final_repr, 512, activation=modeling.gelu,
                                   name="qa_hidden")
      # batch, beam, length (batch, length when training)
      end_logits = tf.squeeze(tf.layers.dense(final_repr, 1), -1,
                              name="qa_logits")
      if is_training:
        end_logits += 1000.0 * (answer_mask - 1)
      else:
        end_logits += tf.expand_dims(1000.0 * (answer_mask - 1), 1)

      if not is_training:
        end_log_probs = tf.nn.log_softmax(end_logits)
        end_top_log_probs, end_top_index = tf.nn.top_k(
            end_log_probs, k=self.config.beam_size)
        end_logits = tf.zeros([batch_size, seq_length])
    else:
      end_logits = tf.squeeze(tf.layers.dense(final_hidden, 1), -1)
      start_logits += 1000.0 * (answer_mask - 1)
      end_logits += 1000.0 * (answer_mask - 1)

    def compute_loss(logits, positions):
      one_hot_positions = tf.one_hot(
          positions, depth=seq_length, dtype=tf.float32)
      log_probs = tf.nn.log_softmax(logits, axis=-1)
      loss = -tf.reduce_sum(one_hot_positions * log_probs, axis=-1)
      return loss

    start_positions = features[self.name + "_start_positions"]
    end_positions = features[self.name + "_end_positions"]

    start_loss = compute_loss(start_logits, start_positions)
    end_loss = compute_loss(end_logits, end_positions)

    losses = (start_loss + end_loss) / 2.0

    answerable_logit = tf.zeros([batch_size])
    if self.config.answerable_classifier:
      final_repr = final_hidden[:, 0]
      if self.config.answerable_uses_start_logits:
        start_p = tf.nn.softmax(start_logits)
        start_feature = tf.reduce_sum(tf.expand_dims(start_p, -1) *
                                      final_hidden, axis=1)
        final_repr = tf.concat([final_repr, start_feature], -1)
        final_repr = tf.layers.dense(final_repr, 512,
                                     activation=modeling.gelu)
      answerable_logit = tf.squeeze(tf.layers.dense(final_repr, 1), -1)
      answerable_loss = tf.nn.sigmoid_cross_entropy_with_logits(
          labels=tf.cast(features[self.name + "_is_impossible"], tf.float32),
          logits=answerable_logit)
      losses += answerable_loss * self.config.answerable_weight

    return losses, dict(
        loss=losses,
        start_logits=start_logits,
        end_logits=end_logits,
        answerable_logit=answerable_logit,
        start_positions=features[self.name + "_start_positions"],
        end_positions=features[self.name + "_end_positions"],
        start_top_log_probs=start_top_log_probs,
        start_top_index=start_top_index,
        end_top_log_probs=end_top_log_probs,
        end_top_index=end_top_index,
        eid=features[self.name + "_eid"],
    )

  def get_scorer(self, split="dev"):
    return qa_metrics.SpanBasedQAScorer(self.config, self, split, self.v2)

class MRQATask(QATask):
  """Class for finetuning tasks from the 2019 MRQA shared task."""

  def __init__(self, config: configure_finetuning.FinetuningConfig, name,
               tokenizer):
    super(MRQATask, self).__init__(config, name, tokenizer)

  def get_examples(self, split):
    if split in self._examples:
      utils.log("N EXAMPLES", split, len(self._examples[split]))
      return self._examples[split]

    examples = []
    example_failures = [0]
    with tf.io.gfile.GFile(os.path.join(
        self.config.raw_data_dir(self.name), split + ".jsonl"), "r") as f:
      for i, line in enumerate(f):
        if self.config.debug and i > 10:
          break
        paragraph = json.loads(line.strip())
        if "header" in paragraph:
          continue
        self._add_examples(examples, example_failures, paragraph, split)
    self._examples[split] = examples
    utils.log("{:} examples created, {:} failures".format(
        len(examples), example_failures[0]))
    return examples

  def get_scorer(self, split="dev"):
    return qa_metrics.SpanBasedQAScorer(self.config, self, split, self.v2)

class SQuADTask(QATask):
  """Class for finetuning on SQuAD 2.0 or 1.1."""

  def __init__(self, config: configure_finetuning.FinetuningConfig, name,
               tokenizer, v2=False):
    super(SQuADTask, self).__init__(config, name, tokenizer, v2=v2)

  def get_examples(self, split):
    if split in self._examples:
      return self._examples[split]

    with tf.io.gfile.GFile(os.path.join(
        self.config.raw_data_dir(self.name),
        split + ("-debug" if self.config.debug else "") + ".json"), "r") as f:
      input_data = json.load(f)["data"]

    examples = []
    example_failures = [0]
    for entry in input_data:
      for paragraph in entry["paragraphs"]:
        self._add_examples(examples, example_failures, paragraph, split)
    self._examples[split] = examples
    utils.log("{:} examples created, {:} failures".format(
        len(examples), example_failures[0]))
    return examples

  def get_scorer(self, split="dev"):
    return qa_metrics.SpanBasedQAScorer(self.config, self, split, self.v2)

class SQuAD(SQuADTask):
  def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer):
    super(SQuAD, self).__init__(config, "squad", tokenizer, v2=True)

class SQuADv1(SQuADTask):
  def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer):
    super(SQuADv1, self).__init__(config, "squadv1", tokenizer)

class NewsQA(MRQATask):
  def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer):
    super(NewsQA, self).__init__(config, "newsqa", tokenizer)

class NaturalQuestions(MRQATask):
  def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer):
    super(NaturalQuestions, self).__init__(config, "naturalqs", tokenizer)

class SearchQA(MRQATask):
  def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer):
    super(SearchQA, self).__init__(config, "searchqa", tokenizer)

class TriviaQA(MRQATask):
  def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer):
    super(TriviaQA, self).__init__(config, "triviaqa", tokenizer)