import os
import unittest

from spotinst_sdk2 import SpotinstSession
from spotinst_sdk2.models.elastigroup.aws import *

class AwsElastigroupTestCase(unittest.TestCase):

    def setUp(self):
        self.session = SpotinstSession(
            auth_token='dummy-token',
            account_id='dummy-account')

        self.client = self.session.client("elastigroup_aws")

        self.mock_group_json = self.load_group_json()

    def create_formatted_group_request(self, group):
        group_request = ElastigroupCreationRequest(group)
        excluded_group_dict = self.client.exclude_missing(
            json.loads(group_request.toJSON()))
        formatted_group_dict = self.client.convert_json(
            excluded_group_dict, self.client.underscore_to_camel)
        return formatted_group_dict

    @staticmethod
    def load_group_json():
        with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../test_lib/input/elastigroup/aws_group.json')) as group_json:
            return json.load(group_json)

# region Third Party Integrations
class AwsElastigroupTestEcsIntegration(AwsElastigroupTestCase):
    def runTest(self):
        ecs_auto_scale_down = EcsAutoScalerDownConfiguration(
            evaluation_periods=3)
        ecs_auto_scale_attribute = EcsAutoScalerAttributeConfiguration(
            key='the_key', value='the_value')
        ecs_auto_scale_headroom = EcsAutoScalerHeadroomConfiguration(
            cpu_per_unit=4096, memory_per_unit=4096, num_of_units=30)
        ecs_auto_scale = EcsAutoScaleConfiguration(
            is_enabled=True,
            is_auto_config=False,
            cooldown=900,
            headroom=ecs_auto_scale_headroom,
            down=ecs_auto_scale_down,
            attributes=[ecs_auto_scale_attribute])
        ecs = EcsConfiguration(
            cluster_name='test-ecs',
            auto_scale=ecs_auto_scale)
        third_party_integrations = ThirdPartyIntegrations(ecs=ecs)

        group = Elastigroup(third_parties_integration=third_party_integrations)

        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['thirdPartiesIntegration']['ecs']
        expected_request_json = self.mock_group_json['group']['thirdPartiesIntegration']['ecs']

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestKubernetesIntegration(AwsElastigroupTestCase):
    def runTest(self):
        kubernetes_auto_scale_down = KubernetesAutoScalerDownConfiguration(
            evaluation_periods=5)
        kubernetes_auto_scale_headroom = KubernetesAutoScalerHeadroomConfiguration(
            cpu_per_unit=2000, memory_per_unit=4000, num_of_units=2)
        kubernetes_auto_scale = KubernetesAutoScalerConfiguration(
            is_enabled=True,
            cooldown=300,
            headroom=kubernetes_auto_scale_headroom,
            down=kubernetes_auto_scale_down,
            is_auto_config=False)
        kubernetes = KubernetesConfiguration(integration_mode='pod',
                                             cluster_identifier='test-k8s',
                                             auto_scale=kubernetes_auto_scale)
        third_party_integrations = ThirdPartyIntegrations(
            kubernetes=kubernetes)

        group = Elastigroup(third_parties_integration=third_party_integrations)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['thirdPartiesIntegration']['kubernetes']
        expected_request_json = self.mock_group_json['group']['thirdPartiesIntegration']['kubernetes']

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestNomadIntegration(AwsElastigroupTestCase):
    def runTest(self):
        nomad_down = NomadAutoScalerDownConfiguration(evaluation_periods=3)
        nomad_constraints = NomadAutoScalerConstraintsConfiguration(
            key='${node.class}', value='value')
        nomad_scale_headroom = NomadAutoScalerHeadroomConfiguration(
            cpu_per_unit=10, memory_per_unit=1000, num_of_units=2)
        nomad_auto_scale = NomadAutoScalerConfiguration(
            is_enabled=True,
            cooldown=180,
            headroom=nomad_scale_headroom,
            constraints=[nomad_constraints],
            down=nomad_down)
        nomad = NomadConfiguration(
            master_host="https://master.host.com",
            master_port=443,
            acl_token='123',
            auto_scale=nomad_auto_scale)
        third_party_integrations = ThirdPartyIntegrations(nomad=nomad)

        group = Elastigroup(third_parties_integration=third_party_integrations)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['thirdPartiesIntegration']['nomad']
        expected_request_json = self.mock_group_json['group']['thirdPartiesIntegration']['nomad']

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestDockerSwarmIntegration(AwsElastigroupTestCase):
    def runTest(self):
        docker_swarm_down = DockerSwarmAutoScalerDownConfiguration(
            evaluation_periods=4)
        docker_swarm_headroom = DockerSwarmAutoScalerHeadroomConfiguration(
            cpu_per_unit=1000000000, memory_per_unit=800000000, num_of_units=3)
        docker_swarm_auto_scale = DockerSwarmAutoScalerConfiguration(
            is_enabled=True,
            cooldown=300,
            headroom=docker_swarm_headroom,
            down=docker_swarm_down)
        docker_swarm = DockerSwarmConfiguration(
            master_host='10.10.10.10',
            master_port=1234,
            auto_scale=docker_swarm_auto_scale)
        third_party_integrations = ThirdPartyIntegrations(
            docker_swarm=docker_swarm)

        group = Elastigroup(third_parties_integration=third_party_integrations)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['thirdPartiesIntegration']['dockerSwarm']
        expected_request_json = self.mock_group_json['group']['thirdPartiesIntegration']['dockerSwarm']

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestCodeDeployIntegration(AwsElastigroupTestCase):
    def runTest(self):
        code_deploy_deployment_groups = CodeDeployDeploymentGroupsConfiguration(
            application_name='test-app', deployment_group_name='test-grp')
        code_deploy = CodeDeployConfiguration(
            clean_up_on_failure=False,
            terminate_instance_on_failure=False,
            deployment_groups=[code_deploy_deployment_groups])
        third_party_integrations = ThirdPartyIntegrations(
            code_deploy=code_deploy)

        group = Elastigroup(third_parties_integration=third_party_integrations)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['thirdPartiesIntegration']['codeDeploy']
        expected_request_json = self.mock_group_json['group']['thirdPartiesIntegration']['codeDeploy']

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestRoute53Integration(AwsElastigroupTestCase):
    def runTest(self):
        route53_record_set = Route53RecordSetsConfiguration(
            use_public_ip=True, name='test-domain.com')
        route53_domains = Route53DomainsConfiguration(
            hosted_zone_id='Z3UFMBCGJMYLUT', record_sets=[route53_record_set])
        route53 = Route53Configuration(domains=[route53_domains])
        third_party_integrations = ThirdPartyIntegrations(route53=route53)

        group = Elastigroup(third_parties_integration=third_party_integrations)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['thirdPartiesIntegration']['route53']
        expected_request_json = self.mock_group_json['group']['thirdPartiesIntegration']['route53']

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestMlbRuntimeIntegration(AwsElastigroupTestCase):
    def runTest(self):
        mlb_runtime = MlbRuntimeConfiguration(deployment_id='dp-rm0f5b912345')
        third_party_integrations = ThirdPartyIntegrations(
            mlb_runtime=mlb_runtime)

        group = Elastigroup(third_parties_integration=third_party_integrations)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['thirdPartiesIntegration']['mlbRuntime']
        expected_request_json = self.mock_group_json['group']['thirdPartiesIntegration']['mlbRuntime']

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestElasticBeanstalkIntegration(AwsElastigroupTestCase):
    def runTest(self):
        deployment_strategy = BeanstalkDeploymentStrategy(
            action='REPLACE_SERVER', should_drain_instances=True)
        deployment_preferences = DeploymentPreferences(
            automatic_roll=True,
            batch_size_percentage=50,
            grace_period=600,
            strategy=deployment_strategy)
        elastic_beanstalk = ElasticBeanstalk(
            environment_id='123',
            deployment_preferences=deployment_preferences)
        third_party_integrations = ThirdPartyIntegrations(
            elastic_beanstalk=elastic_beanstalk)

        group = Elastigroup(third_parties_integration=third_party_integrations)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['thirdPartiesIntegration']['elasticBeanstalk']
        expected_request_json = self.mock_group_json['group']['thirdPartiesIntegration']['elasticBeanstalk']

        self.assertDictEqual(actual_request_json, expected_request_json)

# endregion

# region Scaling
class AwsElastigroupTestScalingIntegration(AwsElastigroupTestCase):
    def runTest(self):
        scaling_policy_up_action = ScalingPolicyAction(
            type='percentageAdjustment', adjustment=20)
        scaling_policy_up_instance_dimension = ScalingPolicyDimension(
            name='InstanceId')
        step_adjustment_action = ScalingPolicyAction(
            type='setMinTarget', min_target_capacity=3)
        step_adjustment = ScalingPolicyStepAdjustment(
            action=step_adjustment_action, threshold=50)
        scaling_policy_up = ScalingPolicy(
            metric_name='CPUUtilization',
            statistic='average',
            unit='percent',
            namespace='AWS/EC2',
            threshold=90,
            period=300,
            evaluation_periods=1,
            cooldown=300,
            operator='gte',
            action=scaling_policy_up_action,
            dimensions=[scaling_policy_up_instance_dimension],
            step_adjustments=[step_adjustment],
            min_target_capacity=1,
            is_enabled=True,
            should_resume_stateful=False)

        scaling_policy_down_action = ScalingPolicyAction(
            type='adjustment', adjustment=1)
        scaling_policy_down_cluster_dimension = ScalingPolicyDimension(
            name='Cluster', value='M2M')
        scaling_policy_down_env_dimension = ScalingPolicyDimension(
            name='Environment', value='ia-staging')
        scaling_policy_down = ScalingPolicy(
            metric_name='overhead',
            statistic='average',
            unit='milliseconds',
            namespace='Monitoring',
            threshold=0.8,
            period=300,
            evaluation_periods=1,
            cooldown=300,
            operator='lt',
            action=scaling_policy_down_action,
            dimensions=[
                scaling_policy_down_cluster_dimension,
                scaling_policy_down_env_dimension])

        target_tracking = TargetTrackingPolicy(
            policy_name='target_policy_1',
            metric_name='CPUUtilization',
            statistic='average',
            source='cloudWatch',
            unit='percent',
            target=50,
            namespace='AWS/EC2',
            cooldown=300)

        expression = MetricExpression(
            name="e1", expression="metric1+10")
        metric_dimension = ScalingPolicyDimension(
            name="instanceId",
            value="string"
        )
        metric = ScalingPolicyMetric(
            name="metric1",
            metric_name="CPUUtilization",
            namespace="AWS/EC2",
            statistic="average",
            extended_statistic="p1.5",
            unit="percent",
            dimensions=[metric_dimension]
        )
        multiple_metrics = MultipleMetrics(
            metrics=[metric],
            expressions=[expression]
        )

        scaling = Scaling(
            up=[scaling_policy_up],
            down=[scaling_policy_down],
            target=[target_tracking],
            multiple_metrics=multiple_metrics)
        group = Elastigroup(scaling=scaling)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['scaling']
        expected_request_json = self.mock_group_json['group']['scaling']

        self.maxDiff = None
        self.assertDictEqual(actual_request_json, expected_request_json)

# endregion

# region Scheduling
class AwsElastigroupTestSchedulingIntegration(AwsElastigroupTestCase):
    def runTest(self):
        scheduled_ami_backup = ScheduledTask(
            frequency='hourly', task_type='backup_ami')
        scheduled_roll = ScheduledTask(
            cron_expression='00 17 * * 3',
            task_type='roll',
            batch_size_percentage=30)
        scheduled_scale = ScheduledTask(
            cron_expression='00 22 * * 3',
            task_type='scale',
            start_time='2018-05-23T10:55:09Z',
            scale_target_capacity=0,
            scale_min_capacity=0,
            scale_max_capacity=3)

        scheduling = Scheduling(
            tasks=[
                scheduled_ami_backup,
                scheduled_roll,
                scheduled_scale])
        group = Elastigroup(scheduling=scheduling)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['scheduling']
        expected_request_json = self.mock_group_json['group']['scheduling']

        self.maxDiff = None
        self.assertDictEqual(actual_request_json, expected_request_json)
# endregion

# region Rancher
class AwsElastigroupTestRancher(AwsElastigroupTestCase):
    def runTest(self):
        rancher = Rancher(access_key="Access", secret_key="Secret", master_host="https://master.com:8080", version="2")
        third_parties_integration = ThirdPartyIntegrations(rancher=rancher)
        group = Elastigroup(
            name="TestGroup",
            description="Created by the Python SDK",
            third_parties_integration=third_parties_integration)

        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['thirdPartiesIntegration']
        expected_request_json = {
            'rancher': {
                'accessKey': 'Access',
                'masterHost': 'https://master.com:8080',
                'secretKey': 'Secret', 'version': '2'
            }
        }

        self.assertDictEqual(actual_request_json, expected_request_json)
# endregion

# region Compute
class AwsElastigroupTestCompute(AwsElastigroupTestCase):
    def runTest(self):
        compute = Compute(product="Linux/UNIX",
                          preferred_availability_zones=["us-west-2a"])
        group = Elastigroup(
            name="TestGroup",
            description="Created by the Python SDK",
            compute=compute)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['compute']
        expected_request_json = {
            'product': "Linux/UNIX",
            'preferredAvailabilityZones': ["us-west-2a"]}

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestLaunchSpecification(AwsElastigroupTestCase):
    def runTest(self):
        launch_specification = LaunchSpecification(image_id="ami-123",
                                                   health_check_type="type")
        compute = Compute(launch_specification=launch_specification)
        group = Elastigroup(
            name="TestGroup",
            description="Created by the Python SDK",
            compute=compute)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['compute']['launchSpecification']
        expected_request_json = {
            'imageId': "ami-123",
            'healthCheckType': "type"
        }

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestLaunchSpecificationMultipleAMI(AwsElastigroupTestCase):
    def runTest(self):
        image_list = []
        image_list.append(Image(id="ami-08e2d37b6a0129927"))
        image_list.append(Image(id="ami-0d70650c3afa9cf54"))
        image_list.append(Image(id="ami-0f05b297987cf6aff"))

        launch_specification = LaunchSpecification(images=image_list,
                                                   health_check_type="type")
        compute = Compute(launch_specification=launch_specification)
        group = Elastigroup(
            name="TestGroup",
            description="Created by the Python SDK",
            compute=compute)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['compute']['launchSpecification']
        expected_request_json = {
            'images': [ { "id" : "ami-08e2d37b6a0129927" }, { "id" : "ami-0d70650c3afa9cf54" }, { "id" : "ami-0f05b297987cf6aff" } ],
            'healthCheckType': "type"
        }

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestBlockDeviceMapping(AwsElastigroupTestCase):
    def runTest(self):
        block_device_mappings = [BlockDeviceMapping(device_name='device')]
        launch_specification = LaunchSpecification(block_device_mappings=block_device_mappings)
        compute = Compute(launch_specification=launch_specification)
        group = Elastigroup(
            name="TestGroup",
            description="Created by the Python SDK",
            compute=compute)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['compute']['launchSpecification']['blockDeviceMappings'][0]
        expected_request_json = {
            'deviceName': "device"
        }

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestResourceTagSpecification(AwsElastigroupTestCase):
    def runTest(self):
        eni_specifcation = TagSpecification(should_tag=True)
        ami_specification = TagSpecification(should_tag=False)
        resource_tag_specification = ResourceTagSpecification(amis=ami_specification, enis=eni_specifcation)
        launch_specification = LaunchSpecification(resource_tag_specification=resource_tag_specification)
        compute = Compute(launch_specification=launch_specification)
        group = Elastigroup(
            name="TestGroup",
            description="Created by the Python SDK",
            compute=compute)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['compute']['launchSpecification']['resourceTagSpecification']

        expected_request_json = {
            'enis': {
                "shouldTag": True
            },
            'amis': {
                "shouldTag": False
            }
        }

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestEbs(AwsElastigroupTestCase):
    def runTest(self):
        ebs = [EBS(snapshot_id="snp-1", throughput=500)]
        block_device_mappings = [BlockDeviceMapping(ebs=ebs)]
        launch_specification = LaunchSpecification(block_device_mappings=block_device_mappings)
        compute = Compute(launch_specification=launch_specification)
        group = Elastigroup(
            name="TestGroup",
            description="Created by the Python SDK",
            compute=compute)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['compute']['launchSpecification']['blockDeviceMappings'][0]['ebs'][0]
        expected_request_json = {
            'snapshotId': "snp-1",
            'throughput': 500
        }

        self.assertDictEqual(actual_request_json, expected_request_json)

class AwsElastigroupTestDynamicVolumeSize(AwsElastigroupTestCase):
    def runTest(self):
        dynamic_volume_size = DynamicVolumeSize(resource="resource")
        ebs = [EBS(dynamic_volume_size=dynamic_volume_size)]
        block_device_mappings = [BlockDeviceMapping(ebs=ebs)]
        launch_specification = LaunchSpecification(block_device_mappings=block_device_mappings)
        compute = Compute(launch_specification=launch_specification)
        group = Elastigroup(
            name="TestGroup",
            description="Created by the Python SDK",
            compute=compute)
        formatted_group_dict = self.create_formatted_group_request(group)

        actual_request_json = formatted_group_dict['group']['compute']['launchSpecification']['blockDeviceMappings'][0]['ebs'][0]['dynamicVolumeSize']
        expected_request_json = {
            'resource': "resource"
        }

        self.assertDictEqual(actual_request_json, expected_request_json)
# endregion


import hashlib

from .api import DispatcherAPI, UserError
from . import plot_tools

from datetime import datetime
from astropy import units as u
from astropy.coordinates import Angle, SkyCoord
import numpy as np

import logging
import typing
import requests
import os
import tempfile
import shutil
import json
import inspect

logger = logging.getLogger("oda_api.gallery_api")

__all__ = ['GalleryDispatcherAPI']

class GalleryDispatcherAPI(DispatcherAPI):

    def get_list_terms_gallery(self,
                               group: typing.Optional[str] = None,
                               parent: typing.Optional[str] = None,
                               parent_id: typing.Optional[str] = None,
                               token: typing.Optional[str] = None
                               ):
        logger.debug("Getting the list of available instruments on the gallery")
        params = {
            'group': group,
            'parent': parent,
            'parent_id': parent_id,
            'token': token
        }

        res = requests.get("%s/get_list_terms" % self.url,
                           params={**params}
                           )
        response_json = self._decode_res_json(res)

        if res.status_code != 200:
            logger.warning(f"An issue occurred while getting the list of terms from the group {group}, "
                           f"from the product gallery : {res.text}")
        else:
            logger.info(f"List of terms from the group {group} successfully returned")

        return response_json

    def parse_observation_time_arg_product_gallery(self,
                                                   t1=None,
                                                   t2=None,
                                                   observation_time_format: str = 'ISOT'
                                                   ):

        if t1 is not None:
            if observation_time_format is not None and observation_time_format.upper() == 'MJD':
                try:
                    logger.info("The value of T1 has been provided in a difference format from UTC, "
                                "this will be attempted to be converted to UTC before being uploaded over the gallery")
                    t1 = self.convert_mjd_to_utc(float(t1))
                except Exception as e:
                    logger.warning(f'Error during the time conversion: {e}\n'
                                   'please check the time arguments you provided and the relative format')
                    raise UserError('Error during the time conversion, '
                                    'please check the time arguments you provided and the relative format')
            elif observation_time_format is None or observation_time_format.upper() == 'ISOT':
                try:
                    datetime.strptime(t1, '%Y-%m-%dT%H:%M:%S')
                except Exception as e:
                    logger.warning(f'Error during the time conversion: {e}\n'
                                   'please check the time arguments you provided and the relative format')
                    raise UserError(f'Error during the time conversion: {e}\n'
                                    'please check the time arguments you provided and the relative format')

        if t2 is not None:
            if observation_time_format is not None and observation_time_format.upper() == 'MJD':
                try:
                    logger.info("The value of T2 has been provided in a difference format from UTC, "
                                "this will be attempted to be converted to UTC before being uploaded over the gallery")
                    t2 = self.convert_mjd_to_utc(float(t2))
                except Exception as e:
                    logger.warning(f'Error during the time conversion: {e}\n'
                                   'please check the time arguments you provided and the relative format')
                    raise UserError('Error during the time conversion, '
                                    'please check the time arguments you provided and the relative format')
            elif observation_time_format is None or observation_time_format.upper() == 'ISOT':
                try:
                    datetime.strptime(t2, '%Y-%m-%dT%H:%M:%S')
                except Exception as e:
                    logger.warning(f'Error during the time conversion: {e}\n'
                                   'please check the time arguments you provided and the relative format')
                    raise UserError(f'Error during the time conversion: {e}\n'
                                    'please check the time arguments you provided and the relative format')

        return t1, t2

    def get_yaml_files_observation_with_title(self,
                                              observation_title: str = None,
                                              token: str = None):
        params = {
            'title': observation_title,
            'token': token
        }

        res = requests.get(os.path.join(self.url, "get_observation_attachments"),
                           params={**params},
                           )

        if res.status_code != 200:
            response_json = res.json()
            error_message = (f"An issue occurred while performing a request on the product gallery, "
                             f"the following error was returned:\n")
            if 'error_message' in response_json:
                error_message += '\n' + response_json['error_message']
                if 'drupal_helper_error_message' in response_json:
                    error_message += '-' + response_json['drupal_helper_error_message']
            else:
                error_message += res.text
            logger.warning(error_message)
        else:
            response_json = res.json()
            msg = f"Observation with title {observation_title}"
            if 'file_content' in response_json:
                msg += " contains a yaml file\n"

            logger.info(msg)

        return response_json

    def get_list_images_angular_distance(self,
                                         token: str = None,
                                         instrument=None,
                                         e1_kev=None, e2_kev=None,
                                         t1=None, t2=None,
                                         ra_ref=None, dec_ref=None,
                                         r=None
                                         ):

        product_list = self.get_list_images_with_conditions(token=token,
                                                            instrument=instrument,
                                                            e1_kev=e1_kev, e2_kev=e2_kev,
                                                            t1=t1, t2=t2)

        if isinstance(product_list, list) and ra_ref is not None and dec_ref is not None and r is not None:
            source_coord_ref = SkyCoord(ra_ref, dec_ref, unit=(u.deg, u.deg))

            ra_values_prod_list = list(map(lambda x: x['ra'], filter(lambda x: 'ra' in x, product_list)))
            dec_values_prod_list = list(map(lambda x: x['dec'], filter(lambda x: 'dec' in x, product_list)))
            coords_prod_list = SkyCoord(ra_values_prod_list, dec_values_prod_list, unit=(u.deg, u.deg))

            separation = source_coord_ref.separation(coords_prod_list).deg

            ind = (separation <= r)

            product_list = list(np.array(product_list)[ind])

        return product_list

    def get_list_images_with_conditions(self,
                                        token: str = None,
                                        instrument=None,
                                        e1_kev=None, e2_kev=None,
                                        t1=None, t2=None
                                        ):
        rev1_value = None
        if t1 is not None:
            params = {'time_to_convert': t1,
                      'token': token}

            c = requests.get(os.path.join(self.url, "get_revnum"),
                             params={**params}
                             )
            revnum_obj = c.json()
            rev1_value = revnum_obj['revnum']

        rev2_value = None
        if t2 is not None:
            params = {'time_to_convert': t2,
                      'token': token}

            c = requests.get(os.path.join(self.url, "get_revnum"),
                             params={**params}
                             )
            revnum_obj = c.json()
            rev2_value = revnum_obj['revnum']

        return self.get_list_products_with_conditions(token=token,
                                                              instrument_name=instrument,
                                                              product_type='image',
                                                              e1_kev=e1_kev,
                                                              e2_kev=e2_kev,
                                                              rev1_value=rev1_value,
                                                              rev2_value=rev2_value)

    def get_list_lightcurve_with_conditions(self,
                                            token: str = None,
                                            instrument=None,
                                            source_name=None,
                                            e1_kev=None, e2_kev=None,
                                            t1=None, t2=None
                                            ):
        rev1_value = None
        if t1 is not None:
            params = {'time_to_convert': t1,
                      'token': token}

            c = requests.get(os.path.join(self.url, "get_revnum"),
                             params={**params}
                             )
            revnum_obj = c.json()
            rev1_value = revnum_obj['revnum']

        rev2_value = None
        if t2 is not None:
            params = {'time_to_convert': t2,
                      'token': token}

            c = requests.get(os.path.join(self.url, "get_revnum"),
                             params={**params}
                             )
            revnum_obj = c.json()
            rev2_value = revnum_obj['revnum']

        return self.get_list_products_with_conditions(token=token,
                                                      instrument_name=instrument,
                                                      product_type='lightcurve',
                                                      src_name=source_name,
                                                      e1_kev_value=e1_kev,
                                                      e2_kev_value=e2_kev,
                                                      rev1_value=rev1_value,
                                                      rev2_value=rev2_value)

    def get_list_spectra_with_conditions(self,
                                         token: str = None,
                                         instrument=None,
                                         source_name=None,
                                         t1=None, t2=None
                                         ):
        rev1_value = None
        if t1 is not None:
            params = {'time_to_convert': t1,
                      'token': token}

            c = requests.get(os.path.join(self.url, "get_revnum"),
                             params={**params}
                             )
            revnum_obj = c.json()
            rev1_value = revnum_obj['revnum']

        rev2_value = None
        if t2 is not None:
            params = {'time_to_convert': t2,
                      'token': token}

            c = requests.get(os.path.join(self.url, "get_revnum"),
                             params={**params}
                             )
            revnum_obj = c.json()
            rev2_value = revnum_obj['revnum']

        return self.get_list_products_with_conditions(token=token,
                                                      instrument_name=instrument,
                                                      src_name=source_name,
                                                      product_type='spectrum',
                                                      rev1_value=rev1_value,
                                                      rev2_value=rev2_value)

    def get_list_products_with_conditions(self,
                                          token: str = None,
                                          **kwargs):

        """
        :param token: user token
        :param kwargs: keyword arguments representing the main parameters values used to generate the product. Amongst them,
               it is important to mention the following ones:
            * instrument_name: name of the instrument used for the generated product (e.g. isgri, jemx1)
            * product_type: type of product generated (e.g. lightcurve, image)
            * src_name: name of a single, or a list of, known sources (eg Crab, Cyg X-1)
            * others: other parameters used for the product. Not all the parameters are currently supported,
                but the list of the supported ones will be extended. E1_kev=25
        """

        params = {
            'token': token,
            **kwargs
        }

        res = requests.get(os.path.join(self.url, "get_data_product_list_with_conditions"),
                           params=params
                           )

        if res.status_code != 200:
            response_json = res.json()
            error_message = (f"An issue occurred while performing a request on the product gallery, "
                             f"the following error was returned:\n")
            if 'error_message' in response_json:
                error_message += '\n' + response_json['error_message']
                if 'drupal_helper_error_message' in response_json:
                    error_message += '-' + response_json['drupal_helper_error_message']
            else:
                error_message += res.text
            logger.warning(error_message)
        else:
            response_json = self._decode_res_json(res)

        return response_json

    def get_list_products_by_source_name(self,
                                         source_name: str = None,
                                         token: str = None):

        params = {
            'token': token,
            'src_name': source_name
        }

        res = requests.get(os.path.join(self.url, "get_data_product_list_by_source_name"),
                         params=params
                         )

        if res.status_code != 200:
            response_json = res.json()
            error_message = (f"An issue occurred while performing a request on the product gallery, "
                             f"the following error was returned:\n")
            if 'error_message' in response_json:
                error_message += '\n' + response_json['error_message']
                if 'drupal_helper_error_message' in response_json:
                    error_message += '-' + response_json['drupal_helper_error_message']
            else:
                error_message += res.text
            logger.warning(error_message)
        else:
            response_json = self._decode_res_json(res)

        return response_json

    def update_source_with_name(self,
                                 source_name: str = None,
                                 auto_update: bool = False,
                                 token: str = None,
                                 **kwargs):
        copied_kwargs = kwargs.copy()

        copied_kwargs['src_name'] = source_name

        params = {
            'token': token,
            'update_astro_entity': True,
            'auto_update': auto_update,
            **copied_kwargs
        }

        posting_msg = f'Updating an astro entity with title {source_name} on the gallery'

        logger.info(posting_msg)

        res = requests.post(os.path.join(self.url, "post_astro_entity_to_gallery"),
                            params={**params},
                            )
        response_json = self._decode_res_json(res)

        if res.status_code != 200:
            res_obj = res.json()
            error_message = (f"An issue occurred while performing a request on the product gallery, "
                             f"the following error was returned:\n")
            if 'error_message' in res_obj:
                error_message += '\n' + res_obj['error_message']
                if 'drupal_helper_error_message' in res_obj:
                    error_message += ' - ' + res_obj['drupal_helper_error_message']
            else:
                error_message += res.text
            logger.warning(error_message)
        else:
            source_link = response_json['_links']['self']['href'].split("?")[0]
            returned_source_name = response_json['title'][0]['value']
            logger.info(
                f"Source with title {returned_source_name} successfully posted on the gallery, at the link {source_link}\n"
                f"Using the above link you can modify the newly created source in the future.\n")

        return response_json

    def update_observation_with_title(self,
                                      observation_title: str = None,
                                      yaml_file_path=None,
                                      token: str = None,
                                      observation_time_format: str = 'ISOT',
                                      create_new=False,
                                      **kwargs):
        copied_kwargs = kwargs.copy()

        # generate file obj
        files_obj = {}
        if yaml_file_path is not None:
            if isinstance(yaml_file_path, list):
                for yaml_path in yaml_file_path:
                    files_obj['yaml_file_' + str(yaml_file_path.index(yaml_path))] = open(yaml_path, 'rb')
            elif isinstance(yaml_file_path, str):
                files_obj['yaml_file'] = open(yaml_file_path, 'rb')

        copied_kwargs['T1'], copied_kwargs['T2'] = self.parse_observation_time_arg_product_gallery(
            t1=kwargs.get('T1', None), t2=kwargs.get('T2', None),
            observation_time_format=observation_time_format
        )

        obsid_arg = kwargs.get('obsid', None)
        if obsid_arg is not None:
            if isinstance(obsid_arg, list):
                obsid_list = ','.join(map(str, obsid_arg))
            else:
                obsid_list = obsid_arg

            copied_kwargs['obsid'] = obsid_list

        params = {
            'title': observation_title,
            'token': token,
            'update_observation': True,
            'create_new': create_new,
            **copied_kwargs
        }

        posting_msg = f'Posting an observation with title {observation_title} on the gallery'

        logger.info(posting_msg)

        res = requests.post(os.path.join(self.url, "post_observation_to_gallery"),
                            params={**params},
                            files=files_obj
                            )
        response_json = self._decode_res_json(res)

        if res.status_code != 200:
            res_obj = res.json()
            error_message = (f"An issue occurred while performing a request on the product gallery, "
                             f"the following error was returned:\n")
            if 'error_message' in res_obj:
                error_message += '\n' + res_obj['error_message']
                if 'drupal_helper_error_message' in res_obj:
                    error_message += '-' + res_obj['drupal_helper_error_message']
            else:
                error_message += res.text
            logger.warning(error_message)
        else:
            observation_link = response_json['_links']['self']['href'].split("?")[0]
            observation_title = response_json['title'][0]['value']
            logger.info(
                f"Observation with title {observation_title} successfully posted on the gallery, at the link {observation_link}\n"
                f"Using the above link you can modify the newly created observation in the future.\n")

        return response_json

    def post_observation_to_gallery(self,
                                    observation_title: str = None,
                                    yaml_file_path=None,
                                    token: str = None,
                                    observation_time_format: str = 'ISOT',
                                    **kwargs):
        copied_kwargs = kwargs.copy()

        # generate file obj
        files_obj = {}
        if yaml_file_path is not None:
            if isinstance(yaml_file_path, list):
                for yaml_path in yaml_file_path:
                    files_obj['yaml_file_' + str(yaml_file_path.index(yaml_path))] = open(yaml_path, 'rb')
            elif isinstance(yaml_file_path, str):
                files_obj['yaml_file'] = open(yaml_file_path, 'rb')

        copied_kwargs['T1'], copied_kwargs['T2'] = self.parse_observation_time_arg_product_gallery(
            t1=kwargs.get('T1', None), t2=kwargs.get('T2', None),
            observation_time_format=observation_time_format
        )

        obsid_arg = kwargs.get('obsid', None)
        if obsid_arg is not None:
            if isinstance(obsid_arg, list):
                obsid_list = ','.join(map(str, obsid_arg))
            else:
                obsid_list = obsid_arg

            copied_kwargs['obsid'] = obsid_list

        params = {
            'title': observation_title,
            'token': token,
            **copied_kwargs
        }

        posting_msg = f'Posting an observation with title {observation_title} on the gallery'

        logger.info(posting_msg)

        res = requests.post(os.path.join(self.url, "post_observation_to_gallery"),
                            params={**params},
                            files=files_obj
                            )

        if res.status_code != 200:
            error_message = (f"An issue occurred while performing a request on the product gallery, "
                             f"the following error was returned:\n")
            try:
                response_json = res.json()
            except json.decoder.JSONDecodeError:
                error_msg = res.text
                response_json = {'error_message': error_msg}
                logger.debug(response_json)

            if 'error_message' in response_json:
                error_message += '\n' + response_json['error_message']
                if 'drupal_helper_error_message' in response_json:
                    error_message += '-' + response_json['drupal_helper_error_message']
            else:
                error_message += res.text
            logger.warning(error_message)
        else:
            response_json = self._decode_res_json(res)
            observation_link = response_json['_links']['self']['href'].split("?")[0]
            observation_title = response_json['title'][0]['value']
            logger.info(f"Observation with title {observation_title} successfully posted on the gallery, at the link {observation_link}\n"
                        f"Using the above link you can modify the newly created observation in the future.\n")

        return response_json

    def post_data_product_to_gallery(self,
                                     product_title: typing.Optional[str] = None,
                                     product_id: str = None,
                                     observation_id: typing.Optional[str] = None,
                                     gallery_image_path: typing.Optional[str] = None,
                                     fits_file_path=None,
                                     yaml_file_path=None,
                                     token: typing.Optional[str] = None,
                                     insert_new_source: bool = False,
                                     validate_source: bool = False,
                                     force_insert_not_valid_new_source: bool = False,
                                     apply_fields_source_resolution: bool = False,
                                     html_image: str = None,
                                     in_evidence: bool = False,
                                     observation_time_format: str = 'ISOT',
                                     **kwargs):
        """

        :param product_title: title to assign to the product, in case this is not provided, then a title is
               automatically built using the name of the source and the type of product
        :param product_id: identifier of a data product assigned by the user, this can be used during the creation of a new data-product,
               as well as to identify an already existing one and update it with the arguments provided by the user
        :param observation_id: this can be indicated in two different ways
            * by specifying the id of an already present observation (eg 'test observation')
            * by specifying the time range, in particular the value of T1 and T2
        :param observation_time_format: format of the time values for an observation (i.e. T1 and T2), default to ISOT,
               (e.g. '2003-03-15T23:27:40.0'), also the MJD format is supported
        :param in_evidence: a boolean value specifying if the product will be in evidence over thew page of the correspondent
            source
        :param gallery_image_path: path of the generated image and to be uploaded over the gallery
        :param fits_file_path: a list of fits file links used for the generation of the product to upload over the gallery
        :param yaml_file_path: a list of yaml file links to be attached to the observation of the product to upload over the gallery
        :param token: user token
        :param insert_new_source: a boolean value specifying if, in case the sources that are passed as parameters and
               are not available on the product gallery, will be created and then used for the new data product
        :param validate_source: a boolean value to specify if, in case the sources that are passed as parameters
               will be validated against an online service. In case the validation fails the source won't be inserted as
               a parameter for the data product and a warning for the user will be generated (unless this is intentionally
               specified setting to `True` the boolean parameter **force_insert_not_valid_new_sources** described below)
        :param force_insert_not_valid_new_source: a boolean value to specify if, in case the validation of the sources passed as
               parameters fails, those should be in any case provided as a parameter for the data product
        :param apply_fields_source_resolution: a boolean value to specify if, in case only a single source is passed within the
                parameters and then successfully validated, to apply the parameters values returned from the validation
                (an example of these parameters are RA and DEC), default to False
        :param html_image: field used to upload an image encapsulated within an html block generated using external
               tools (e.g. bokeh)
        :param kwargs: keyword arguments representing the main parameters values used to generate the product. Amongst them,
               it is important to mention the following ones:
            * instrument: name of the instrument used for the generated product (e.g. isgri, jemx1)
            * product_type: type of product generated (e.g. isgri_lc, jemx_image)
            * src_name: name of a single, or a list of, known sources (eg Crab, Cyg X-1)
            * others: other parameters used for the product. Not all the parameters are currently supported,
                but the list of the supported ones will be extended. RA=25

        """

        # apply policy for the specific data product
        # use the product_type, if provided, and apply the policy, if applicable
        self.check_gallery_data_product_policy(token=token, **kwargs)

        copied_kwargs = kwargs.copy()

        # generate file obj
        files_obj = {}
        tmp_path_html_folder_path = None
        if gallery_image_path is not None:
            files_obj['img'] = open(gallery_image_path, 'rb')
        if fits_file_path is not None:
            if isinstance(fits_file_path, list):
                for fits_path in fits_file_path:
                    files_obj['fits_file_' + str(fits_file_path.index(fits_path))] = open(fits_path, 'rb')
            elif isinstance(fits_file_path, str):
                files_obj['fits_file'] = open(fits_file_path, 'rb')
        if yaml_file_path is not None:
            if isinstance(yaml_file_path, list):
                for yaml_path in yaml_file_path:
                    files_obj['yaml_file_' + str(yaml_file_path.index(yaml_path))] = open(yaml_path, 'rb')
            elif isinstance(yaml_file_path, str):
                files_obj['yaml_file'] = open(yaml_file_path, 'rb')
        if html_image is not None:
            html_image_hash = hashlib.md5(html_image.encode()).hexdigest()[:8]
            tmp_path_html_folder_path = tempfile.mkdtemp(suffix="gallery_temp_files")
            tmp_path_html_file_path = os.path.join(tmp_path_html_folder_path, f'additional_html_file_{html_image_hash}.html')
            with open(tmp_path_html_file_path, "w") as f_html:
                f_html.write(html_image)

            files_obj['html_file'] = open(tmp_path_html_file_path, 'rb')

        copied_kwargs['T1'], copied_kwargs['T2'] = self.parse_observation_time_arg_product_gallery(
            t1=kwargs.get('T1', None), t2=kwargs.get('T2', None),
            observation_time_format=observation_time_format
        )

        # validate source
        src_name_arg = kwargs.get('src_name', None)
        copied_src_name_arg = None
        entities_portal_link_list = None
        object_ids_list = None
        object_type_list = None
        source_coord_list = None
        if src_name_arg is not None and validate_source:

            if isinstance(src_name_arg, str):
                src_name_list = src_name_arg.split(',')
            else:
                src_name_list = src_name_arg

            for src_name in src_name_list:
                resolved_source = False
                entity_portal_link = None
                object_ids = None
                object_type = None
                source_coord = {}
                # remove any underscore (following the logic of the resolver) and use the edited one
                src_name_edited = src_name.replace('_', ' ')
                resolved_obj = self.resolve_source(src_name=src_name_edited, token=token)
                if resolved_obj is not None:
                    msg = ''
                    if 'message' in resolved_obj:
                        if 'could not be resolved' in resolved_obj['message']:
                            msg = f'\nSource {src_name} could not be validated'
                        elif 'successfully resolved' in resolved_obj['message']:
                            resolved_source = True
                            msg = f'\nSource {src_name} was successfully validated'
                    msg += '\n'
                    logger.info(msg)
                    if 'RA' in resolved_obj:
                        RA = Angle(resolved_obj["RA"], unit='degree')
                        source_coord['source_ra'] = RA.deg
                        if apply_fields_source_resolution:
                            # TODO to be discussed
                            if len(src_name_list) == 1:
                                copied_kwargs['RA'] = RA.deg
                    if 'DEC' in resolved_obj:
                        DEC = Angle(resolved_obj["DEC"], unit='degree')
                        source_coord['source_dec'] = DEC.deg
                        if apply_fields_source_resolution:
                            # TODO to be discussed
                            if len(src_name_list) == 1:
                                copied_kwargs['DEC'] = DEC.deg
                    if 'entity_portal_link' in resolved_obj:
                        entity_portal_link = resolved_obj['entity_portal_link']
                        # copied_kwargs['entity_portal_link'] = resolved_obj['entity_portal_link']
                    if 'object_type' in resolved_obj:
                        object_type = resolved_obj['object_type']
                    if 'object_ids' in resolved_obj:
                        object_ids = resolved_obj['object_ids']
                else:
                    logger.warning(f"{src_name} could not be validated")

                if not resolved_source and not force_insert_not_valid_new_source:
                    # a source won't be added
                    logger.warning(f"the specified source will not be added")
                else:
                    if copied_src_name_arg is None:
                        copied_src_name_arg = []
                    copied_src_name_arg.append(src_name)

                    if entities_portal_link_list is None:
                        entities_portal_link_list = []
                    if entity_portal_link is None:
                        entity_portal_link = ''
                    entities_portal_link_list.append(entity_portal_link)

                    if object_ids_list is None:
                        object_ids_list = []
                    if object_ids is None:
                        object_ids = []
                    object_ids_list.append(object_ids)

                    if object_type_list is None:
                        object_type_list = []
                    if object_type is None:
                        object_type = ''
                    object_type_list.append(object_type)

                    if source_coord_list is None:
                        source_coord_list = []
                    source_coord_list.append(source_coord)

        else:
            copied_src_name_arg = src_name_arg

        if copied_src_name_arg is not None:
            if isinstance(copied_src_name_arg, list):
                copied_kwargs['src_name'] = ','.join(copied_src_name_arg)
            else:
                copied_kwargs['src_name'] = copied_src_name_arg
        else:
            copied_kwargs.pop('src_name', None)

        if entities_portal_link_list is not None:
            if isinstance(entities_portal_link_list, list):
                copied_kwargs['entity_portal_link_list'] = ','.join(entities_portal_link_list)
            else:
                copied_kwargs['entity_portal_link_list'] = entities_portal_link_list

        if source_coord_list is not None:
            copied_kwargs['source_coord_list'] = json.dumps(source_coord_list)

        if object_ids_list is not None:
            copied_kwargs['object_ids_list'] = json.dumps(object_ids_list)

        if object_type_list is not None:
            copied_kwargs['object_type_list'] = json.dumps(object_type_list)

        copied_kwargs['in_evidence'] = 0 if not in_evidence else 1

        params = {
            'content_type': 'data_product',
            'product_title': product_title,
            'observation_id': observation_id,
            'token': token,
            'insert_new_source': insert_new_source,
            'product_id': product_id,
            **copied_kwargs
        }

        posting_msg = 'Posting a product'
        if product_id is not None:
            posting_msg += f' with product_id {product_id}'
        posting_msg += ' on the gallery'

        logger.info(posting_msg)

        res = requests.post("%s/post_product_to_gallery" % self.url,
                            params={**params},
                            files=files_obj
                            )

        if res.status_code != 200:
            error_message = (f"An issue occurred while performing a request on the product gallery, "
                             f"the following error was returned:\n")
            try:
                response_json = res.json()
            except json.decoder.JSONDecodeError:
                error_msg = res.text
                response_json = {'error_message': error_msg}
                logger.debug(response_json)

            if 'error_message' in response_json:
                error_message += '\n' + response_json['error_message']
                if 'drupal_helper_error_message' in response_json:
                    error_message += '-' + response_json['drupal_helper_error_message']

            logger.warning(error_message)
        else:
            response_json = self._decode_res_json(res)
            action = 'posted'
            if product_id is not None and response_json['created'][0]['value'] != response_json['changed'][0]['value']:
                action = 'updated'

            self.check_missing_parameters_data_product(response_json, token=token, **kwargs)

            product_posted_link = response_json['_links']['self']['href'].split("?")[0]
            logger.info(f"Product successfully {action} on the gallery, at the link {product_posted_link}\n"
                        f"Using the above link you can modify the newly created product in the future.\n"
                        f"For example, you will be able to change the instrument as well as the product type.\n")

        if tmp_path_html_folder_path is not None and os.path.exists(tmp_path_html_folder_path):
            logger.info(f'removing tmp_path_html_folder_path={tmp_path_html_folder_path} created for temporary files')
            try:
                shutil.rmtree(tmp_path_html_folder_path)
            except OSError as e:
                logger.error(f'unable to remove temporary directory {tmp_path_html_folder_path} !')

        return response_json

    def resolve_source(self,
                       src_name: typing.Optional[str] = None,
                       token: typing.Optional[str] = None):
        resolved_obj = None
        if src_name is not None and src_name != '':
            params = {
                'name': src_name,
                'token': token
            }

            logger.info(f"Searching the object {src_name}\n")

            res = requests.get("%s/resolve_name" % self.url,
                               params={**params}
                               )
            resolved_obj = self._decode_res_json(res)

            if resolved_obj is not None and 'message' in resolved_obj:
                logger.info(f'{resolved_obj["message"]}')
        else:
            logger.info("Please provide the name of the source\n")

        return resolved_obj

    def convert_ijd_to_utc(self, t_ijd):
        # TODO to reply on a dedicated service in the dispatcher
        res = requests.get(f"https://www.astro.unige.ch/mmoda/dispatch-data/gw/timesystem/api/v1.0/converttime/IJD/{t_ijd}/UTC")
        if res.status_code == 200:
            t_utc = res.text
            return t_utc
        else:
            return None

    def convert_mjd_to_utc(self, t_mjd):
        # TODO to reply on a dedicated service in the dispatcher
        t_utc = self.convert_ijd_to_utc(t_mjd - 51544)
        return t_utc

    def check_gallery_data_product_policy(self,
                                          token: typing.Optional[str] = None,
                                          **kwargs):
        product_type = kwargs.get('product_type', None)
        if product_type is not None and product_type != '':
            params = {
                'term': product_type,
                'group': 'products',
                'token': token
            }

            logger.info(f"Applying the policy for the product {product_type}\n")

            res = requests.get("%s/get_parents_term" % self.url,
                               params={**params}
                               )
            parents_term_list = self._decode_res_json(res)

            if parents_term_list is not None and isinstance(parents_term_list, list):
                # loop over the available ODAProduct from the plot_tools and find the correspondent one
                for name, c in inspect.getmembers(plot_tools, inspect.isclass):
                    if issubclass(c, plot_tools.OdaProduct) \
                            and hasattr(c, 'name') and c.name is not None and c.name in parents_term_list \
                            and hasattr(c, 'check_product_for_gallery'):
                        return c.check_product_for_gallery(**kwargs)
            logger.info(f"A policy for the product_type {product_type} could not be applied\n")
        else:
            logger.info("A product_type has not been provided for the given data product, "
                        "therefore no policy will be verified\n")

        return True

    def check_missing_parameters_data_product(self, response, token: typing.Optional[str] = None, **kwargs):
        missing_instrument = True
        instrument_used = None
        missing_product_type = True
        if '_links' in response:
            for field_link in response['_links']:
                field = field_link.split('/')[-1]
                if field == 'field_instrumentused':
                    missing_instrument = False
                    instrument_used = kwargs.get('instrument', None)
                elif field == 'field_data_product_type':
                    missing_product_type = False

        if missing_instrument:
            list_instruments = self.get_list_terms_gallery(group='instruments', token=token)
            logger.info(f'\nWe noticed no instrument has been specified, the following are available:\n'
                        f'{list_instruments}\n'
                        'Please remember that this can be set at a later stage by editing the newly created data product.\n')

        if missing_product_type:
            if not missing_instrument and instrument_used is not None:
                list_instrument_data_products = self.get_list_terms_gallery(group='products', parent=instrument_used,
                                                                            token=token)
                if list_instrument_data_products is not None:
                    logger.info(f'\nWe noticed no product type has been specified,\n'
                                f'for the instrument {instrument_used}, the following products are available:\n'
                                f'{list_instrument_data_products}\n'
                                'Please remember that this can be set at a later stage by editing the newly created data product.\n')


import time
import cv2
import imutils
import platform
import numpy as np
from threading import Thread
from queue import Queue
from Detection import Detection
from Record import Record

class Camera:

    def __init__(self, video_id):
        if cv2.ocl.haveOpenCL() :
            cv2.ocl.setUseOpenCL(True)
        self.capture = None
        self.width = 640
        self.height = 360
        self.thread = None
        self.stat = False
        self.current_time = 1
        self.preview_time = 1
        self.sec = 0
        self.Q = Queue(maxsize=128)
        self.started = False
        self.detection = Detection(video_id)
        self.soundRecord = Record()
        self.fps = 20.0
        self.isRecord = False
        fourcc = cv2.VideoWriter_fourcc(*'MJPG')
        self.videoWriter = cv2.VideoWriter('test.avi', fourcc, 8, (self.width, 480))

    def run(self, src = 0):

        if platform.system() == 'Windows' :
            self.capture = cv2.VideoCapture(src , cv2.CAP_DSHOW)

        else :
            self.capture = cv2.VideoCapture(src)

        self.capture.set(cv2.CAP_PROP_FRAME_WIDTH, self.width)
        self.capture.set(cv2.CAP_PROP_FRAME_HEIGHT, self.height)
        self.started = True
        self.detection.startTime = time.time()
        self.soundRecord.run()
        self.threadStart()

    def threadStart(self):
        if self.thread is None:
            self.thread = Thread(target=self.update, args=())
            self.thread.daemon = False
            self.thread.start()
        self.started = True

    def stop(self):
        self.started = False
        self.stopRecording()
        # if self.capture is not None:
        #     self.stopRecording()
        #     self.clear()

    def stopRecording(self):
        self.soundRecord.bRecord = False
        self.soundRecord.stopRecording()

    def update(self):
        while self.started:
            self.current_time = time.time() - self.preview_time
            (grabbed, frame) = self.capture.read()
            if grabbed and (self.current_time > 1./ self.fps):
                self.preview_time = time.time()
                self.Q.put(frame)
                self.videoWriter.write(frame)
                self.detection.detection(frame)

    def clear(self):
        with self.Q.mutex:
            self.Q.queue.clear()

    def read(self):
        return self.Q.get()

    def blank(self):
        return np.ones(shape=[self.height, self.width, 3], dtype=np.uint8)

    def bytescode(self):

        if not self.capture.isOpened():
            frame = self.blank()
        else :
            frame = imutils.resize(self.read(), width=int(self.width) )
            self.visualize(frame)
            if self.stat:
                cv2.rectangle(frame, (0,0), (120,30), (0,0,0), -1)
                fps = 'FPS : ' + str(self.fps())
        return cv2.imencode('.jpg', frame )[1].tobytes()

    def visualize(self, frame):
        cv2.putText(frame, "face: " + str(self.detection.face), (30, 45), cv2.FONT_HERSHEY_DUPLEX, 0.7, (147, 58, 31), 1)
        cv2.putText(frame, "head: " + str(self.detection.head), (30, 75), cv2.FONT_HERSHEY_DUPLEX, 0.7, (147, 58, 31), 1)
        cv2.putText(frame, "shoulder: " + str(self.detection.shoulder), (250, 45), cv2.FONT_HERSHEY_DUPLEX, 0.7, (147, 58, 31), 1)
        cv2.putText(frame, "gaze: " + str(self.detection.gaze), (250, 75), cv2.FONT_HERSHEY_DUPLEX, 0.7, (147, 58, 31), 1)
        cv2.line(frame, (self.detection.x_left - 5, self.detection.y_left), (self.detection.x_left + 5, self.detection.y_left), self.detection.gazeColor)
        cv2.line(frame, (self.detection.x_left, self.detection.y_left - 5), (self.detection.x_left, self.detection.y_left + 5), self.detection.gazeColor)
        cv2.line(frame, (self.detection.x_right - 5, self.detection.y_right), (self.detection.x_right + 5, self.detection.y_right), self.detection.gazeColor)
        cv2.line(frame, (self.detection.x_right, self.detection.y_right - 5), (self.detection.x_right, self.detection.y_right + 5), self.detection.gazeColor)
        cv2.rectangle(frame, (self.detection.fX, self.detection.fY), (self.detection.fX + self.detection.fW, self.detection.fY + self.detection.fH), self.detection.faceColor, 2)
        if self.detection.postureClass.results is not None:
            self.detection.mp_drawing.draw_landmarks(frame, self.detection.postureClass.results.pose_landmarks, {(12, 14), (11, 12), (11, 12)},
                                    None,
                                    self.detection.mp_drawing.DrawingSpec(color=self.detection.shoulderColor, thickness=2, circle_radius=4),
                                    )

            self.detection.mp_drawing.draw_landmarks(frame, self.detection.postureClass.results.face_landmarks, self.detection.mp_holistic.FACEMESH_CONTOURS,
                                        None,
                                        self.detection.mp_drawing.DrawingSpec(color=self.detection.headColor, thickness=1, circle_radius=1)
                                    )

    def fps(self):

        self.current_time = time.time()
        self.sec = self.current_time - self.preview_time
        self.preview_time = self.current_time

        if self.sec > 0 :
            fps = round(1/(self.sec),1)

        else :
            fps = 1
        return fps

    def __del__(self) :
        print( '* streamer class exit')
        if self.thread != None :
            self.capture.release()
            self.videoWriter.release()
            self.thread.join()
            self.stopRecording()


import torch

def qort(q, v):
    """
    Rotate vector(s) v about the rotation described by quaternion(s) q.
    Expects a tensor of shape (*, 4) for q and a tensor of shape (*, 3) for v,
    where * denotes any number of dimensions.
    Returns a tensor of shape (*, 3).
    """
    assert q.shape[-1] == 4
    assert v.shape[-1] == 3
    assert q.shape[:-1] == v.shape[:-1]

    qvec = q[..., 1:]
    uv = torch.cross(qvec, v, dim=len(q.shape)-1)
    uuv = torch.cross(qvec, uv, dim=len(q.shape)-1)
    return v + 2 * (q[..., :1] * uv + uuv)

def qinverse(q, inplace=False):
    # We assume the quaternion to be normalized
    """
    The quaternions provided in the code are from the camera coordinate to the world coordinate.
    Therefore, the quaternions from the world coordinate to the camera coordinate is the transpose of quaternions from
    the camera coordinates to the world coordinate.The precondition is that the quaternion is a unit quaternion.
    So the inverse of the quaternions is equal to the transposition of the quaternions.
    """
    if inplace:
        q[..., 1:] *= -1
        return q
    else:
        w = q[..., :1]
        xyz = q[..., 1:]
        return torch.cat((w, -xyz), dim=len(q.shape)-1)


#!/usr/bin/python2

"""
Select 5000 random Spanish words from the Spanish corpa, then selects nine at random to use as anchor points. The anchor point words are printed to a
file so that human can generate their translations
"""

import logging
import random

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    logger = logging.getLogger('select_for_translation')

    # Read in the Spanish file
    logger.info('Opening Spanish file...')
    lines = []
    with open('../../corpa/spanish/model-ascii.w2v') as f:
        for line in f:
            lines.append(line)

    logger.info('Spanish file read in')

    random.shuffle(lines)
    spanish_vocab = lines[:5000]
    logger.info('Shuffled Spanish words and got the first 5K')

    with open('../../corpa/spanish/model-ascii-5k.w2v', 'w') as f:
        file_data = '\n'.join(spanish_vocab)
        f.write(file_data)

    logger.info('Wrote selected lines to a file')


import bs4
from bs4 import BeautifulSoup
import requests
import csv

baseURL = 'http://www.innatthecrossroads.com/home/game-thrones-recipes/recipes-by-meal/'
pages = [
    'main-courses',
    'sides',
    'breakfast',
    'soupsstews',
    'pies',
    'bread',
    'vegetarian',
    'desserts',
    'beverages',
]
outputFile = open('Game of Thrones', 'w')
for page in pages:
    page = requests.get(baseURL + page)
    soup = BeautifulSoup(page.content, 'lxml')
    menus = soup.find('article')
    for item in menus.find_all('li'):
        outputFile.write(item.text + '\n')

outputFile.close()


import os
import time
from random import randint

import discord
import discord.utils
from discord.ext import commands
from discord.ext.commands import Bot
from discord.utils import get

import requests
from PIL import Image, ImageFont, ImageDraw
import io

import youtube_dl

prefix = '!'

zeli_bot = Bot(command_prefix=prefix)

Bot.remove_command(zeli_bot, 'help')

@zeli_bot.event
async def on_ready():
    print('Ок')

    await zeli_bot.change_presence(status=discord.Status.online, activity=discord.Game(" ЧСВшную сучку"))

#@zeli_bot.event
#async def on_command_error(ctx, error):
#    pass

@Bot.command(zeli_bot, aliases=['привет'])
async def hello(ctx):
    author = ctx.message.author
    await ctx.send(f"{author.mention} да-да, Хеллоу, блять... Нахуй иди!")

@Bot.command(zeli_bot, aliases=['сделать_Лупой'])
@commands.has_permissions(administrator=True)
async def set_Lupa(ctx, member: discord.Member):
    zelibobka_role = discord.utils.get(ctx.message.guild.roles, name='Zelibobka')
    lupa_role = discord.utils.get(ctx.message.guild.roles, name='Лупа')
    await member.remove_roles(zelibobka_role)
    await member.add_roles(lupa_role)
    await ctx.send(f"{member.mention} стал Лупой :C")

@Bot.command(zeli_bot, aliases=['сделать_зелибобкой'])
@commands.has_permissions(administrator=True)
async def set_zelibobka(ctx, member: discord.Member):
    zelibobka_role = discord.utils.get(ctx.message.guild.roles, name='Zelibobka')
    lupa_role = discord.utils.get(ctx.message.guild.roles, name='Лупа')
    await member.remove_roles(lupa_role)
    await member.add_roles(zelibobka_role)
    await ctx.send(f"{member.mention} стал зелибобкой :)")

@Bot.command(zeli_bot, aliases=['сделать_лупой'])
@commands.has_role(764468531206422538)
async def set_lupa(ctx, member: discord.Member):
    lupa_role = discord.utils.get(ctx.message.guild.roles, name='Лупа')
    await member.add_roles(lupa_role)
    await ctx.send(f"{member.mention}, {ctx.author.mention} сделал вас Лупой :Э")

@Bot.command(zeli_bot, aliases=['больше_не_лупа'])
@commands.has_role(764467400300429312)
async def del_lupa(ctx, member: discord.Member):
    lupa_role = discord.utils.get(ctx.message.guild.roles, name='Лупа')
    await member.remove_roles(lupa_role)
    await ctx.send(f"{member.mention}, {ctx.author.mention} забрал у вас роль Лупы :C")

@Bot.command(zeli_bot, aliases=['сделать_бот_инвайтером'])
@commands.has_permissions(administrator=True)
async def set_bot_inviter(ctx, member: discord.Member):
    bot_inviter_role = discord.utils.get(ctx.message.guild.roles, name='bot_inviter')
    await member.add_roles(bot_inviter_role)
    await ctx.send(f"{member.mention} стал бот_инвайтером, Аеееее, САСНЫЙ!!!")

@Bot.command(zeli_bot, aliases=['больше_не_бот_инвайтер'])
@commands.has_permissions(administrator=True)
async def del_bot_inviter(ctx, member: discord.Member):
    bot_inviter_role = discord.utils.get(ctx.message.guild.roles, name='bot_inviter')
    await member.remove_roles(bot_inviter_role)
    await ctx.send(f"{member.mention} перестал быть бот_инвайтером :ССССССССС")

@Bot.command(zeli_bot, aliases=['подключить'])
@commands.has_role(764864128383975437)
async def join(ctx):
    global voice
    channel = ctx.message.author.voice.channel
    voice = get(zeli_bot.voice_clients, guild=ctx.guild)
    if voice and voice.is_connected():
        await voice.move_to(channel)
    else:
        voice = await channel.connect()
        await ctx.send(f'Зелибот присоединился к каналу: {channel}')

@Bot.command(zeli_bot, aliases=['кикнуть'])
@commands.has_role(764864128383975437)
async def leave(ctx):
    channel = ctx.message.author.voice.channel
    voice = get(zeli_bot.voice_clients, guild=ctx.guild)
    if voice and voice.is_connected():
        await ctx.send(f'Зелибот решил уйти из канала: {channel}')
        await voice.disconnect()
    else:
        voice = await channel.disconnect()

@Bot.command(zeli_bot, aliases=['включить_музыку', 'вкл_м'])
async def play(ctx, url: str):
    if not discord.opus.is_loaded():
        discord.opus.load_opus('libopus.so')
    song_there = os.path.isfile("song.mp3")
    try:
        if song_there:
            os.remove("song.mp3")
            print("song removed")
    except PermissionError:
        print('it\'s playing')
        await ctx.send('Музыка ещё играет, подождите.')
        return

    await ctx.send('Музыка загружается, ожидайте :)')

    voice = get(zeli_bot.voice_clients, guild=ctx.guild)

    ydl_opts = {
        'format': 'bestaudio/best',
        'postprocessors': [
            {
                'key': 'FFmpegExtractAudio',
                'preferredcodec': 'mp3',
                'preferredquality': '192'
            }
        ]
    }

    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
        print('downloading')
        ydl.download([url])

    for file in os.listdir('./'):
        if file.endswith('.mp3'):
            name = file
            print(f'renamed file: {file}')
            os.rename(file, 'song.mp3')

    voice.play(discord.FFmpegPCMAudio('song.mp3'), after=lambda e: print(f'finish playing {name}'))
    voice.source = discord.PCMVolumeTransformer(voice.source)
    voice.source.volume = 0.07

    nname = name.rsplit('-', 2)
    await ctx.send(f'Играет: {nname[0]} {nname[1]}')
    print(f'playing: {nname[0]}')

@Bot.command(zeli_bot, aliases=['русская_рулетка'])
async def RR(ctx):
    await ctx.send(f'{ctx.author.mention} Крутим барабан...')
    time.sleep(5)
    a = randint(1, 6)
    if a == 1:
        await ctx.send(f'{ctx.author.mention} иди нахуй')
    else:
        await ctx.send(f'{ctx.author.mention}, ну ничего, как-нибудь, в другой раз сходишь нахуй...')

@Bot.command(zeli_bot, aliases=['отчистить'])
async def clear(ctx, amount=100):
    await ctx.channel.purge(limit=amount+1)

@Bot.command(zeli_bot, aliases=['карта'])
async def get_card(ctx):
    img = Image.new('RGBA', (400, 200), '#012136')
    try:
        url = str(ctx.author.avatar_url)[:-10]

        resp = requests.get(url, stream=True)
        resp = Image.open(io.BytesIO(resp.content))
    except:
        resp = Image.open(io.BytesIO(requests.get('https://pp.userapi.com/lLGPiS2Mcnj9OeepZtKRuA5jUG5-3d1I36ow-g/qw5hym27TYE.jpg', stream=True).content))

    resp = resp.convert('RGBA')
    resp = resp.resize((100, 100), Image.ANTIALIAS)
    idraw = ImageDraw.Draw(img)

    name = ctx.author.name
    mention = ctx.message.author.display_name
    tag = ctx.author.discriminator
    role = ctx.author.top_role.name

    if role == 'Zeliboba':
        col = '#46bf36'
    elif role == 'Zelibobka':
        col = '#78ff66'
    elif role == 'Лупа':
        col = '#d466ff'
    elif role == 'Пупа':
        col = '#ff6666'
    else:
        col = '#7cfce7'

    idraw.rectangle(((13, 13), (116, 116)), fill=col)

    img.paste(resp, (15, 15, 115, 115))

    headline = ImageFont.truetype('fonts/3.otf', size=20)
    undertext = ImageFont.truetype('fonts/2.ttf', size=15)
    idraw.text((145, 15), 'Карта участника сервера', font=headline, fill=col)
    idraw.text((145, 50), f'id: {ctx.author.id}', font=undertext, fill=col)
    idraw.text((145, 70), f'Никнейм в discord: {name}#{tag}', font=undertext, fill=col)
    idraw.text((145, 90), f'Никнейм на сервере: {mention}', font=undertext, fill=col)
    idraw.text((145, 110), f'Роль: {role}', font=undertext, fill=col)
    idraw.text((15, 180), f'Сервер: {ctx.message.guild.name}', font=undertext, fill=col)

    img.save('cards/user_card.png')

    await ctx.send(file=discord.File(fp='cards/user_card.png'))

@Bot.command(zeli_bot, aliases=['ПАМАГИТЕ'])
async def help(ctx):
    emb = discord.Embed(title='Список команд Зелибота(Non-admin):', colour=0x2fff00, )
    emb.add_field(name=f'{prefix}help(ПАМАГИТЕ)', value='Выводит список команд Зелибота в личные сообщения.', inline=False)
    emb.add_field(name=f'{prefix}hello(привет)', value='Зелибот поприветствует вас.', inline=False)
    emb.add_field(name=f'{prefix}clear(отчистить) [Кол-во сообщений]', value='Зелибот удалит 100 сообщений/[Кол-во сообщений]', inline=False)
    emb.add_field(name=f'{prefix}join(подключить)', value='Пригласить Зелибота в голосовой чат. (Нужно: быть в голосовом чате; иметь роль bot_inviter)', inline=False)
    emb.add_field(name=f'{prefix}leave(кикнуть)', value='Выгнать Зелибота из голосового чата. (Нужно: быть в голосовом чате; иметь роль bot_inviter)', inline=False)
    emb.add_field(name=f'{prefix}set_lupa(сделать_лупой) [@Пользователь]',
                  value='Вы сделаете [@Пользователя] лупой. (Нужно иметь роль Лупа)', inline=False)
    emb.add_field(name=f'{prefix}del_lupa(больше_не_лупа) [@Пользователь]', value='вы заберёте роль Лупа у [@Пользователя]. (Нужно иметь роль Zelibobka)',
                  inline=False)
    emb.add_field(name=f'{prefix}RR(русская_рулетка)', value='Русская рулетка, шанс 1 к 6 пойти нахуй', inline=False)
    emb.add_field(name=f'{prefix}get_card(карта)', value='Выдаёт карту пользователя серевера, ей можно флексить где угодно.', inline=False)
    emb1 = discord.Embed(title='Список команд Зелибота(admin):', colour=0x2fff00)
    emb1.add_field(name=f'{prefix}help_in(ПАМАГИТЕ_здесь)', value='Выводит список команд Зелибота в данный чат', inline=False)
    emb1.add_field(name=f'{prefix}set_Lupa(сделать_Лупой) [@Пользователь]', value='[@Пользователь] станет лупой.', inline=False)
    emb1.add_field(name=f'{prefix}set_zelibobka(сделать_зелибобкой) [@Пользователь]', value='[@Пользователь] станет зелибобкой', inline=False)
    emb1.add_field(name=f'{prefix}set_bot_inviter(сделать_бот_инвайтером) [@Пользователь]', value='[@Пользователь] станет бот_инвайтером',
                   inline=False)
    emb1.add_field(name=f'{prefix}del_bot_inviter(больше_не_бот_инвайтер) [@Пользователь]', value='[@Пользователь] перестанет быть бот_инвайтером',
                   inline=False)
    await ctx.message.author.send(embed=emb)
    await ctx.message.author.send(embed=emb1)
@Bot.command(zeli_bot, aliases=['ПАМАГИТЕ_здесь'])
@commands.has_permissions(administrator=True)
async def help_in(ctx):
    emb = discord.Embed(title='Список команд Зелибота(Non-admin):', colour=0x2fff00, )
    emb.add_field(name=f'{prefix}help(ПАМАГИТЕ)', value='Выводит список команд Зелибота в личные сообщения.',
                  inline=False)
    emb.add_field(name=f'{prefix}hello(привет)', value='Зелибот поприветствует вас.', inline=False)
    emb.add_field(name=f'{prefix}clear(отчистить) [Кол-во сообщений]',
                  value='Зелибот удалит 100 сообщений/[Кол-во сообщений]', inline=False)
    emb.add_field(name=f'{prefix}join(подключить)',
                  value='Пригласить Зелибота в голосовой чат. (Нужно: быть в голосовом чате; иметь роль bot_inviter)',
                  inline=False)
    emb.add_field(name=f'{prefix}leave(кикнуть)',
                  value='Выгнать Зелибота из голосового чата. (Нужно: быть в голосовом чате; иметь роль bot_inviter)',
                  inline=False)
    emb.add_field(name=f'{prefix}set_lupa(сделать_лупой) [@Пользователь]',
                  value='Вы сделаете [@Пользователя] лупой. (Нужно иметь роль Лупа)', inline=False)
    emb.add_field(name=f'{prefix}del_lupa(больше_не_лупа) [@Пользователь]',
                  value='вы заберёте роль Лупа у [@Пользователя]. (Нужно иметь роль Zelibobka)',
                  inline=False)
    emb.add_field(name=f'{prefix}RR(русская_рулетка)', value='Русская рулетка, шанс 1 к 6 пойти нахуй', inline=False)
    emb.add_field(name=f'{prefix}get_card(карта)',
                  value='Выдаёт карту пользователя серевера, ей можно флексить где угодно.', inline=False)
    emb1 = discord.Embed(title='Список команд Зелибота(admin):', colour=0x2fff00)
    emb1.add_field(name=f'{prefix}help_in(ПАМАГИТЕ_здесь)', value='Выводит список команд Зелибота в данный чат',
                   inline=False)
    emb1.add_field(name=f'{prefix}set_Lupa(сделать_Лупой) [@Пользователь]', value='[@Пользователь] станет лупой.',
                   inline=False)
    emb1.add_field(name=f'{prefix}set_zelibobka(сделать_зелибобкой) [@Пользователь]',
                   value='[@Пользователь] станет зелибобкой', inline=False)
    emb1.add_field(name=f'{prefix}set_bot_inviter(сделать_бот_инвайтером) [@Пользователь]',
                   value='[@Пользователь] станет бот_инвайтером',
                   inline=False)
    emb1.add_field(name=f'{prefix}del_bot_inviter(больше_не_бот_инвайтер) [@Пользователь]',
                   value='[@Пользователь] перестанет быть бот_инвайтером',
                   inline=False)
    await ctx.send(embed=emb)
    await ctx.send(embed=emb1)

@join.error
async def join_error(ctx, error):
    if isinstance(error, commands.MissingRole):
        await ctx.send(f'{ctx.author.mention}, Зелибот не общается с кем попало, для этой команды нужна роль - bot_inviter')

@leave.error
async def leave_error(ctx, error):
    if isinstance(error, commands.MissingRole):
        await ctx.send(f'{ctx.author.mention}, хотел выгнать Зелибота? А может тебя выгнать? Для этой команды нужна роль - bot_inviter')

@set_lupa.error
async def set_lupa_error(ctx, error):
    if isinstance(error, commands.MissingRole):
        await ctx.send(f'{ctx.author.mention}, Нужно быть лупой, чтобы сделать кого-нибудь лупой...')

@del_lupa.error
async def del_lupa_error(ctx, error):
    if isinstance(error, commands.MissingRole):
        await ctx.send(f'{ctx.author.mention}, Нужно быть зелибобкой, чтобы отобрать роль лупы...')

zeli_bot.run(os.environ['TOKEN'])

# webhook added


"""
Split Beverage table into Beverage and BeverageScrape. Separates the scraping (temporal) from the actual beverage.
Beverage table will be renamed to BeverageScrape. Beverage table created with similar columns. Move all unique
BeverageScrapes to Beverages table. Then drop unnecessary columns from BeverageScrape.
"""

import argparse
import logging
import sys
import dateutil.parser
from web.models import MenuScrape, Beverage, Location, BeverageScrape, Brewery, DistinctBeer
from web import db

root_log = logging.getLogger()
root_log.setLevel(logging.INFO)

def link():
    distinct = DistinctBeer.query.all()
    for d in distinct:
        # Technically could have multiple, but pick the first
        beverage = Beverage.query.filter_by(untappd_id=d.untappd_bid).first()
        if beverage:
            d.beverage_id = beverage.id
            db.session.add(d)
    db.session.commit()

if __name__ == '__main__':
    # Setup logging
    sh = logging.StreamHandler(sys.stdout)
    sh.setLevel(logging.INFO)
    sh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
    root_log.addHandler(sh)

    # Command line arguments
    parser = argparse.ArgumentParser(description='Split beverage into scrapes, beverages, and brewery.')

    link()


import torch.nn as nn
import torch.nn.functional as F

class BasicBlock(nn.Module):
    def __init__(self, in_channels, out_channels, stride=1):
        super().__init__()
        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(out_channels)
        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(out_channels)

        self.shortcut = nn.Sequential()
        if stride != 1 or in_channels != out_channels:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(out_channels)
            )

    def forward(self, x):
        residual = x

        out = F.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))

        out += self.shortcut(residual)
        out = F.relu(out)

        return out

class ResNet18(nn.Module):
    def __init__(self, num_classes=141):
        super().__init__()
        self.in_channels = 64

        self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        #Chuyen 64 channel

        self.layer1 = self._make_layer(64, 2, stride=1)
        #Chuyen 128 channel
        self.layer2 = self._make_layer(128, 2, stride=2)
        self.layer3 = self._make_layer(256, 2, stride=2)
        self.layer4 = self._make_layer(512, 2, stride=2)

        # self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        self.fc = nn.Linear(338, num_classes)

    def _make_layer(self, out_channels, num_blocks, stride):
        layers = []
        layers.append(BasicBlock(self.in_channels, out_channels, stride))
        self.in_channels = out_channels
        for _ in range(num_blocks - 1):
            layers.append(BasicBlock(self.in_channels, out_channels))
            self.in_channels = out_channels
        return nn.Sequential(*layers)

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))

        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        # print("preavgpool")
        #batch x channel out x dai x rong neu de padding = 1 thi shape dai rong giu nguyen
        # print(out.shape)
        # out = self.avgpool(out)
        # print("after")
        # print(out.shape)
        # print(out.shape)
        out = out.view(256, out.size(0), -1)
        # print(out.shape)
        out = self.fc(out)

        return out


from PIL import Image
import numpy as np
from collections import Counter

# Ordena a quantidade de ocorrencia da intensidade
def contar_numeros(image):
    pixels = image.flatten()
    counts = Counter(pixels).items()
    return sorted(counts, key=lambda x: x[::-1])

# Constroe árvore de cima para baixo
def construir_arvore(counts):
    nodes = [entry[::-1] for entry in counts] # Reverter cada tupla (símbolo, contagem)
    while len(nodes) > 1:
        leastTwo = tuple(nodes[0:2])  # Obter os 2 para combinar
        theRest = nodes[2:] # Todos os outros
        combFreq = leastTwo[0][0] + leastTwo[1][0] # Frequência do ponto de ramificação
        nodes = theRest + [(combFreq, leastTwo)] # Adicionar ponto de ramificação ao final
        nodes.sort(key=lambda x: x[0])  # Usando x[0] para ordenar pela frequência
    return nodes[0] # Retornar a única árvore dentro da lista

# Função usada para retornar somente os simbolos ordenados por frequencia da arvore
def podar(tree):
    p = tree[1]
    if type(p) is tuple:
        return (podar(p[0]), podar(p[1]))
    return p

def descer_arvore(codes, node, pat):
    if type(node) == tuple: # Ponto de ramificação.
        descer_arvore(codes, node[0], pat + [0]) # Faça o ramo esquerdo.
        descer_arvore(codes, node[1], pat + [1]) # Em seguida, faça o ramo direito.
    else:
        codes[node] = pat # Uma folha

def criar_dicionario(tree): # Faz a inicialização do dicionário que vai ser usado para acessar os códigos na arvore
    codes = {}
    descer_arvore(codes, tree, [])
    return codes

def binario_lista(n): # Converter um inteiro em na menor lista de bits
    return [n] if (n <= 1) else binario_lista(n >> 1) + [n & 1]

def lista_binario(bits):
    result = 0
    for bit in bits: # Converter uma lista de bits em um inteiro
        result = (result << 1) | bit
    return result

def preencher(bits, n):
    # Preenche a lista de bits até que ela tenha um total de n digitos
    assert(n >= len(bits))
    return ([0] * (n - len(bits)) + bits)

class OutputBitStream(object):
    # Inicializa um arquivo que será usado para escrever
    def __init__(self, file_name):
        self.file_name = file_name
        self.file = open(self.file_name, 'wb')
        self.bytes_written = 0
        self.buffer = []

    # Usado para escerver um bit no buffer
    def write_bit(self, value):
        self.write_bits([value])

    # Se o buffer for maior que 8, escrevemos os bits no arquivo
    def write_bits(self, values):
        self.buffer += values
        while len(self.buffer) >= 8:
            self._save_byte()

    def flush(self):
        if len(self.buffer) > 0:  # Adiciona zeros para completar o byte e depois o escreve
            self.buffer += [0] * (8 - len(self.buffer))
            self._save_byte()
        assert(len(self.buffer) == 0)

    # Sava as informações no arquivo
    def _save_byte(self):
        bits = self.buffer[:8]
        self.buffer[:] = self.buffer[8:]

        byte_value = lista_binario(bits)
        if 0 <= byte_value <= 255:
            self.file.write(bytes([byte_value]))
            self.bytes_written += 1
        else:
            raise ValueError(f"Invalid byte value: {byte_value}")

    # garante que tudo seja escrito no arquivo antes de fechá-lo
    def close(self):
        self.flush()
        self.file.close()

def codificar_cabecalho(image, bitstream):
    height_bits = preencher(binario_lista(image.shape[0]), 16)
    bitstream.write_bits(height_bits)
    width_bits = preencher(binario_lista(image.shape[1]), 16)
    bitstream.write_bits(width_bits)

def salvar_size(size, bitstream):
    height_bits = preencher(binario_lista(size[0]), 16)
    bitstream.write_bits(height_bits)
    width_bits = preencher(binario_lista(size[1]), 16)
    bitstream.write_bits(width_bits)

def codificar_arvore(tree, bitstream):
    if type(tree) == tuple:
        bitstream.write_bit(0)
        codificar_arvore(tree[0], bitstream)
        codificar_arvore(tree[1], bitstream)
    else:
        bitstream.write_bit(1)
        symbol_bits = preencher(binario_lista(tree), 16)
        bitstream.write_bits(symbol_bits)

def codificar_pixels(image, codes, bitstream):
    pixels = image.flatten()
    for value in pixels:
        bitstream.write_bits(codes[value])

def compressed_size(counts, codes):
    header_size = 2 * 16

    tree_size = len(counts) * (1 + 16)
    if tree_size % 8 > 0:
        tree_size += 8 - (tree_size % 8)

    pixels_size = sum([count * len(codes[symbol]) for symbol, count in counts])
    if pixels_size % 8 > 0:
        pixels_size += 8 - (pixels_size % 8)

    return (header_size + tree_size + pixels_size) / 8

def compress_array(array, out_file_name,size):
    counts = contar_numeros(array)
    tree = construir_arvore(counts)
    trimmed_tree = podar(tree)
    codes = criar_dicionario(trimmed_tree)
    stream = OutputBitStream(out_file_name)
    salvar_size(size,stream)
    codificar_cabecalho(array, stream)
    stream.flush()
    codificar_arvore(trimmed_tree, stream)
    stream.flush()
    codificar_pixels(array, codes, stream)
    stream.close()

# Ler bits de um arquivo
class InputBitStream(object):
    def __init__(self, file_name):
        self.file_name = file_name
        self.file = open(self.file_name, 'rb')
        self.bytes_read = 0
        self.buffer = []

    def read_bit(self):
        return self.read_bits(1)[0]

    def read_bits(self, count):
        while len(self.buffer) < count:
            self._load_byte()
        result = self.buffer[:count]
        self.buffer[:] = self.buffer[count:]
        return result

    def flush(self):
        assert(not any(self.buffer))
        self.buffer[:] = []

    def _load_byte(self):
        value = ord(self.file.read(1))
        self.buffer += preencher(binario_lista(value), 8)
        self.bytes_read += 1

    def close(self):
        self.file.close()

def decodificar_cabecalho(bitstream):
    height = lista_binario(bitstream.read_bits(16))
    width = lista_binario(bitstream.read_bits(16))
    return (height, width)

def decodificar_size(bitstream):
    height = lista_binario(bitstream.read_bits(16))
    width = lista_binario(bitstream.read_bits(16))
    return (height, width)

def decodificar_arvore(bitstream):
    flag = bitstream.read_bits(1)[0]
    if flag == 1:
        return np.uint16(lista_binario(bitstream.read_bits(16)))
    left = decodificar_arvore(bitstream)
    right = decodificar_arvore(bitstream)
    return (left, right)

def decodificar_valor(tree, bitstream):
    bit = bitstream.read_bits(1)[0]
    node = tree[bit]
    if type(node) == tuple:
        return decodificar_valor(node, bitstream)
    return np.uint16(node)

def decodificar_pixels(height, width, tree, bitstream):
    pixels = np.empty((height, width, 3), dtype=np.uint16)
    for i in range(height):
        for j in range(width):
            for channel in range(3):
                pixels[i, j, channel] = decodificar_valor(tree, bitstream)
    return pixels

def decompress_image(in_file_name):
    stream = InputBitStream(in_file_name)
    size = decodificar_size(stream)
    height, width = decodificar_cabecalho(stream)
    stream.flush()
    trimmed_tree = decodificar_arvore(stream)
    stream.flush()
    pixels = decodificar_pixels(height, width, trimmed_tree, stream)
    stream.close()
    return pixels,size


import os
from typing import List

from domain.contracts.abstract_dataset_preparartion_service import AbstractDatasetPreparationService
from domain.models.evaluation_job_parameters import EvaluationJobParameters
from domain.models.inference_object import InferenceObject
from shared.helpers.dataset_helpers import get_dataset_path
from shared.services.image_classification_formatting_service import ImageClassificationFormattingService

class ImageClassificationDatasetPreparationService(AbstractDatasetPreparationService):

    def __init__(self):
        self.formatter = ImageClassificationFormattingService()

    def _get_classes_folders(self, evaluation_job_parameters: EvaluationJobParameters) -> List[str]:
        classes_folders: List[str] = []
        dataset_path: str = get_dataset_path(evaluation_job_parameters.dataset_name, evaluation_job_parameters.job_type)

        for element in os.listdir(dataset_path):
            if os.path.isdir(os.path.join(dataset_path, element)):
                classes_folders.append(os.path.join(dataset_path, element))

        return classes_folders

    def get_ground_truths(self, evaluation_job_parameters: EvaluationJobParameters) -> List[InferenceObject]:
        dataset_path: str = get_dataset_path(evaluation_job_parameters.dataset_name, evaluation_job_parameters.job_type)

        classes_folders: List[str] = self._get_classes_folders(evaluation_job_parameters)

        ground_truths: List[InferenceObject] = []

        for class_folder in classes_folders:

            class_name = os.path.split(class_folder)[1]

            for image in os.listdir(class_folder):
                image_path: str = os.path.join(class_folder, image)

                label: List[Dict] = [{'ObjectClass': class_name}]

                ground_truths.extend(self.formatter.get_inference_objects_list(label, image_path))

        return ground_truths