#!/cma/g3/wangdp/usr/local/bin/python3
import subprocess
import re
import datetime
import json
from collections import defaultdict

import click

def get_system_from_path(path):
    if path == "/":
        return None
    if not path.startswith('/'):
        return None
    index = path.find('/', 1)
    return path[1: index]

def run_command(command):
    pipe = subprocess.Popen([command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    output, error_output = pipe.communicate()
    output_string = None
    if output is not None:
        output_string = output.decode()
    error_output_string = None
    if error_output is not None:
        error_output_string = error_output.decode()
    return output_string, error_output_string

def parse_error_log(log_string):
    p = re.compile('^\[(.*)\]\[(.*)\](.*): (.*) (.*) (.*) (.*)$')
    m = p.match(log_string)
    date_string = m.group(1)
    command = m.group(2)
    message = m.group(3)
    job_script = m.group(4)
    path = m.group(5)
    total_tries = m.group(6)
    current_try_no = m.group(7)
    record = dict()
    record['date'] = datetime.datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S%Z')
    record['command'] = command
    record['message'] = message
    record['info'] = {
        'job_script': job_script,
        'path': path,
        'total_tries': total_tries,
        'current_try_no': current_try_no
    }
    return record

def get_record_field_value(record, name):
    def get_date_hour(x):
        cur_datetime = x['date']
        cur_date = cur_datetime.date()
        zero_time = datetime.time(cur_datetime.hour)
        cur_hour = datetime.datetime.combine(cur_date, zero_time)
        return cur_hour.strftime("%Y-%m-%d %H:%M:%S")

    field_mapper = {
        'month': lambda x: x['date'].strftime("%Y-%m"),
        'date': lambda x: x['date'].strftime("%Y-%m-%d"),
        'weekday': lambda x: x['date'].weekday(),
        'system': lambda x: get_system_from_path(x['info']['path']),
        'date-hour': get_date_hour,
        'hour': lambda x: x['date'].hour,
    }

    if name in field_mapper:
        return field_mapper[name](record)
    else:
        raise Exception('name unsupported', name)

@click.group()
def cli():
    """
DESCRIPTION
    Analytic llsubmit4 error log."""
    pass

@cli.command('info', help='get log file info')
@click.option('-f', '--file', 'log_file_path', required=True, help="log file path")
@click.option("--pretty-print/--no-pretty-print", default=False, help="print pretty result.")
def info(log_file_path, pretty_print):
    """
    get log file info
    """

    command = "head -n 1 {log_file_path}".format(log_file_path=log_file_path)
    output_string, error_string = run_command(command)
    if len(error_string) > 0:
        result = {
            'app': 'llsubmit4_error_analyzer',
            'type': 'range',
            'timestamp': datetime.datetime.now().timestamp(),
            'error': 'head_command_error',
            'data': {
                'error_message': 'error in executing head command.',
                'output': {
                    'std_out': output_string,
                    'std_err': error_string
                },
                'request': {
                    'log_file_path': log_file_path,
                }
            }
        }

        if pretty_print:
            print(json.dumps(result, indent=4))
        else:
            print(json.dumps(result))
        return

    record = parse_error_log(output_string)
    start_date = record['date']

    command = "tail -n 1 {log_file_path}".format(log_file_path=log_file_path)
    output_string, error_string = run_command(command)
    if len(error_string) > 0:
        result = {
            'app': 'llsubmit4_error_analyzer',
            'type': 'range',
            'timestamp': datetime.datetime.now().timestamp(),
            'error': 'tail_command_error',
            'data': {
                'error_message': 'error in executing tail command.',
                'output': {
                    'std_out': output_string,
                    'std_err': error_string
                },
                'request': {
                    'log_file_path': log_file_path,
                }
            }
        }

        if pretty_print:
            print(json.dumps(result, indent=4))
        else:
            print(json.dumps(result))
        return

    record = parse_error_log(output_string)
    end_date = record['date']

    command = "wc -l {log_file_path}".format(log_file_path=log_file_path)
    output_string, error_string = run_command(command)
    if len(error_string) > 0:
        result = {
            'app': 'llsubmit4_error_analyzer',
            'type': 'range',
            'timestamp': datetime.datetime.now().timestamp(),
            'error': 'tail_command_error',
            'data': {
                'error_message': 'error in executing wc command.',
                'output': {
                    'std_out': output_string,
                    'std_err': error_string
                },
                'request': {
                    'log_file_path': log_file_path,
                }
            }
        }

        if pretty_print:
            print(json.dumps(result, indent=4))
        else:
            print(json.dumps(result))

        return

    line_count = int(output_string.strip().split(' ')[0])

    # with open(log_file_path, 'r') as log_file:
    #     for line in log_file:
    #         record = parse_error_log(line)
    #         print(record['date'])

    result = {
        'app': 'llsubmit4_error_analyzer',
        'type': 'info',
        'timestamp': datetime.datetime.now().timestamp(),
        'data': {
            'range': {
                'start_date_time': start_date.strftime('%Y-%m-%dT%H:%M:%S%Z'),
                'end_date_time': end_date.strftime('%Y-%m-%dT%H:%M:%S%Z'),
                'count': line_count
            },
            'request': {
                'log_file_path': log_file_path,
            }
        }
    }
    if pretty_print:
        print(json.dumps(result, indent=4))
    else:
        print(json.dumps(result))

@cli.command('count', help='count errors in error log file.')
@click.option("-f", "--file", "log_file_path", required=True, help="log file path")
@click.option("--begin-date", help="begin date, YYYY-MM-DD")
@click.option("--end-date", help="end date, YYYY-MM-DD")
@click.option("--begin-time", help="begin time, hh:mm:ss")
@click.option("--end-time", help="end time, hh:mm:ss")
@click.option("--type", "count_type", required=True,
              type=click.Choice(['month', 'date', 'weekday', 'date-hour', 'hour', 'system']),
              help="count type", )
@click.option("--pretty-print/--no-pretty-print", default=False, help="print pretty result.")
def count(log_file_path, begin_date, end_date, begin_time, end_time, count_type, pretty_print):
    """
    count errors in error log file.
    """
    if begin_date:
        begin_date = datetime.datetime.strptime(begin_date, "%Y-%m-%d")
    if end_date:
        end_date = datetime.datetime.strptime(end_date, "%Y-%m-%d")

    count_result = defaultdict(int)
    try:
        with open(log_file_path, 'r') as log_file:
            for line in log_file:
                record = parse_error_log(line)
                # check date
                if begin_date:
                    if record['date'].date() < begin_date.date():
                        continue
                if end_date:
                    if record['date'].date() >= end_date.date():
                        continue

                count_result[get_record_field_value(record, count_type)] += 1

    except FileNotFoundError as e:
        result = {
            'app': 'llsubmit4_error_analyzer',
            'type': 'count',
            'error': 'file_not_found',
            'timestamp': datetime.datetime.now().timestamp(),
            'data': {
                'error_message': 'file is not found',
                'request': {
                    'log_file_path': log_file_path,
                    'begin_date': begin_date.strftime('%Y-%m-%d'),
                    'end_date': end_date.strftime('%Y-%m-%d'),
                    'count_type': count_type
                }
            }
        }
        if pretty_print:
            print(json.dumps(result, indent=4))
        else:
            print(json.dumps(result))
        return

    result = {
        'app': 'llsubmit4_error_analyzer',
        'type': 'count',
        'timestamp': datetime.datetime.now().timestamp(),
        'data': {
            # 'count_type': count_type,
            # 'begin_date': begin_date.strftime('%Y-%m-%d'),
            # 'end_date': end_date.strftime('%Y-%m-%d'),
            # 'count_result': count_result,
            'request': {
                'log_file_path': log_file_path,
                'begin_date': begin_date.strftime('%Y-%m-%d'),
                'end_date': end_date.strftime('%Y-%m-%d'),
                'count_type': count_type
            },
            'response': {
                'count_result': count_result,
            }
        }
    }

    if pretty_print:
        print(json.dumps(result, indent=4))
    else:
        print(json.dumps(result))

@cli.command('grid', help='get grid result.')
@click.option("-f", "--file", "log_file_path", required=True, help="log file path")
@click.option("--begin-date", help="begin date, YYYY-MM-DD")
@click.option("--end-date", help="end date, YYYY-MM-DD")
@click.option("--begin-time", help="begin time, hh:mm:ss")
@click.option("--end-time", help="end time, hh:mm:ss")
@click.option("--x-type", "x_type", required=True,
              type=click.Choice(['hour', 'weekday']),
              help="x axis type")
@click.option("--y-type", "y_type", required=True,
              type=click.Choice(['weekday', 'system', 'date']),
              help="y axis type")
@click.option("--pretty-print/--no-pretty-print", default=False, help="print pretty result.")
def grid(log_file_path, begin_date, end_date, begin_time, end_time, x_type, y_type, pretty_print):
    """
    get grid result.
    """
    if begin_date:
        begin_date = datetime.datetime.strptime(begin_date, "%Y-%m-%d")
    if end_date:
        end_date = datetime.datetime.strptime(end_date, "%Y-%m-%d")

    grid_result = dict()
    try:
        with open(log_file_path, 'r') as log_file:
            for line in log_file:
                record = parse_error_log(line)
                # check date
                if begin_date:
                    if record['date'].date() < begin_date.date():
                        continue
                if end_date:
                    if record['date'].date() >= end_date.date():
                        continue

                x_value = get_record_field_value(record, x_type)
                y_value = get_record_field_value(record, y_type)

                if x_value not in grid_result:
                    grid_result[x_value] = dict()
                if y_value not in grid_result[x_value]:
                    grid_result[x_value][y_value] = 0
                grid_result[x_value][y_value] += 1

    except FileNotFoundError:
        result = {
            'app': 'llsubmit4_error_analyzer',
            'type': 'grid',
            'error': 'file_not_found',
            'timestamp': datetime.datetime.now().timestamp(),
            'data': {
                'error_message': 'file is not found',
                'request': {
                    'log_file_path': log_file_path,
                }
            }
        }
        if pretty_print:
            print(json.dumps(result, indent=4))
        else:
            print(json.dumps(result))
        return

    result = {
        'app': 'llsubmit4_error_analyzer',
        'type': 'grid',
        'timestamp': datetime.datetime.now().timestamp(),
        'data': {
            'response': {
                'grid_result': grid_result,
            },
            'request': {
                'log_file_path': log_file_path,
                'x_type': x_type,
                'y_type': y_type,
                'begin_date': begin_date.strftime('%Y-%m-%d'),
                'end_date': end_date.strftime('%Y-%m-%d'),
            }
        }
    }

    if pretty_print:
        print(json.dumps(result, indent=4))
    else:
        print(json.dumps(result))

if __name__ == "__main__":
    cli()


import os
os.environ['USE_PYGEOS'] = '0'
from owslib.wfs import WebFeatureService
from owslib.wms import WebMapService
import geopandas as gpd
import json
import fiona

import os

# # Get the absolute path of the current file
# absolute_path = os.path.abspath(__file__)

# # Get the directory name of the current file
# directory_name = os.path.dirname(absolute_path)

# # Get the relative path of the current file
# relative_path = os.path.relpath(absolute_path)

# print("Absolute path:", absolute_path)
# print("Directory name:", directory_name)
# print("Relative path:", relative_path)

data_informations_path = "./data/data_informations.json"

def create_data_informations_file():
    """ Fonction to instantiate data_informations.json file """

    data_informations = {
        "services" : {
            "data.grandlyon_wfs": "https://download.data.grandlyon.com/wfs/grandlyon?SERVICE=WFS&VERSION=2.0.0",
            "data.grandlyon_wms" : "https://download.data.grandlyon.com/wms/grandlyon?VERSION=1.3.0&SERVICE=WMS"
        },
        "osm": {
            "network_parameters": {
            }
        },
        "data_wfs" : {
            "fontaines_potables": {
                "name": "Fontaines potables",
                "wfs_key": "ms:epo_eau_potable.epobornefont",
                "service": "data.grandlyon_wfs",
                "marker_option": {
                "iconUrl": "droplet.svg",
                "iconRetinaUrl": "droplet.svg",
                "popupAnchor":  [-0, -0],
                "iconSize": [40,40],
                "clusterCountStyle" : "position:absolute;top:20px;left:-9px;color:white;font-weight:bold;"
                }
            },
            "toilettes_publiques": {
                "name": "Toilettes publiques",
                "wfs_key": "ms:adr_voie_lieu.adrtoilettepublique_latest",
                "service": "data.grandlyon_wfs",
                "marker_option": {
                "iconUrl": "toilet.svg",
                "iconRetinaUrl": "toilet.svg",
                "popupAnchor":  [-0, -0],
                "iconSize": [40,40],
                "clusterCountStyle" : "position:absolute;top:48px;left:-6px;color:black;font-weight:bold;"
                }
            },
            "fontaines_ornementales": {
                "name": "Fontaines ornementales",
                "wfs_key": "ms:adr_voie_lieu.adrfontaineornem_latest",
                "service": "data.grandlyon_wfs",
                "marker_option": {
                "iconUrl": "fountain.svg",
                "iconRetinaUrl": "fountain.svg",
                "popupAnchor":  [-0, -0],
                "iconSize": [40,40],
                "clusterCountStyle" : "position:absolute;top:48px;left:0px;color:black;font-weight:bold;"
            }
            },
            "parcs_jardins_metropole": {
                "name": "Parcs et jardins",
                "wfs_key": "ms:com_donnees_communales.comparcjardin_1_0_0",
                "service": "data.grandlyon_wfs",
            },
            "bancs": {
                "name": "Bancs",
                "wfs_key": "ms:adr_voie_lieu.adrbanc_latest",
                "service": "data.grandlyon_wfs",
                "marker_option": {
                "iconUrl": "bench.svg",
                "iconRetinaUrl": "bench.svg",
                "popupAnchor":  [-0, -0],
                "iconSize": [25,25],
                "clusterCountStyle": "position:absolute;top:0px;left:-10px;color:black;font-weight:bold;"
            }
            },
            # "arbres_alignement": {
            #     "wfs_key": "ms:abr_arbres_alignement.abrarbre",
            #     "service": "data.grandlyon_wfs",
            # },
            # à mettre dans un second temps
            # "hauteur_batiment": {
            #     "wfs_key": "ms:fpc_fond_plan_communaut.fpctoit_2018",
            #     "service": "data.grandlyon_wfs"
            # }

        },
        "data_wms": {
            "vegetation_stratifie" : {
                "wms_key": "MNC_class_2022_INT1U",
                "service" : "data.grandlyon_wms",
                "width" : 3500,
                "height": 3640,
                "format" : "png",
                "transparent" : True,
                "srs": "EPSG:4171"
            }
        },
        "data_raw" : {
            # "vegetation_stratifie_raw" : {
            #     "name": "Végétation stratifiée",
            #     "gpkg_path": "./backend/script_python/data/raw_data/vegetation_stratifie.gpkg"
            # },
            "temp_surface_road_raw": {
                "name": "Température de surface",
                "gpkg_path": "./raw_data/temp_surface.gpkg"
            },
        #     "joined_if": {
        #         "name" : "Calque fraîcheur",
        #         "gpkg_path": "./backend/script_python/data/raw_data/joined_if_3946.gpkg",
        # }
        }
    }

    with open(data_informations_path, "w") as f:
        json.dump(data_informations, f, indent=4)

def create_folder(folder_path):
    exist = os.path.exists(folder_path)
    if not exist:
        os.makedirs(folder_path)
        print(f"{folder_path} created")

def connection_wfs(url, service_name, version):
    """ Return a wfs object after connecting to a service thanks the url provided """
    print(f"Connecting {service_name} WFS ... ")
    wfs=None
    try:
        wfs = WebFeatureService(url, version)
        print(f"SUCCESS : Connected to {service_name}")
    except NameError:
        print(f"Error while connecting to {service_name} ... : {NameError}")

    return wfs

def connection_wms(url, service_name, version):
    """ Return a wms object after connecting to a service thanks the url provided """
    print(f"Connecting {service_name} WMS ... ")
    wms=None
    try:
        wms = WebMapService(url, version)
        print(f"SUCCESS : Connected to {service_name}")
    except NameError:
        print(f"Error while connecting to {service_name} ... : {NameError}")

    return wms

def download_data_wfs(service_name, version, path):
    """ load a list of data from one specific service
        the data are stored in a json file as a dictionnary wich elements have the folowing form :

        "data_name" : {
            "wfs_key" : 'key',
            "service" : 'service_name',
            "download_path": "path/file.gml"
        }
    """

    with open(data_informations_path, "r") as f:
        data_informations = json.load(f)

    data_wfs = data_informations["data_wfs"]
    services = data_informations["services"]

    wfs = connection_wfs(services[f"{service_name}"], service_name, version)

    for d_name, d_info in data_wfs.items():
        if(d_info["service"] == service_name):

            print(f"### {d_name} ###")

            # the boundingBox define the area where the data should belong to
            box = wfs.contents[f"{d_info['wfs_key']}"].boundingBoxWGS84

            print(f"fetching {d_name} ..")
            try:
                new_data = wfs.getfeature(typename=f"{d_info['wfs_key']}", bbox=box)
                print(f"SUCCESS")
            except NameError:
                print(f"Error while fetching {d_name} from {service_name}... : {NameError}")

            print(f"writing {d_name} GML file into {path} \n")

            #write file into given folder (path)
            file = open(f"{path}/{d_name}.gml", "wb")
            file.write(new_data.read())
            file.close()

            #store download path into data_informations.json
            data_informations["data_wfs"][d_name]["gml_path"] = f"{path}/{d_name}.gml"

    # re-write data_informations.json => store all download path
    with open(data_informations_path, "w") as f:
        json.dump(data_informations, f, indent=4)

def download_data_wms(service_name, version, path):
    """ load a list of data from one specific service
        the data are stored in a json file as a dictionnary wich elements have the folowing form :

        "data_name" : {
            "wfs_key" : 'key',
            "service" : 'service_name',
            "download_path": "path/file.gml"
        }
    """

    with open(data_informations_path, "r") as f:
        data_informations = json.load(f)

    data_wms = data_informations["data_wms"]
    services = data_informations["services"]

    wms = connection_wms(services[f"{service_name}"], service_name, version)

    for d_name, d_info in data_wms.items():
        if(d_info["service"] == service_name):

            print(f"### {d_name} ###")

            # the boundingBox define the area where the data should belong to
            box = wms.contents[f"{d_info['wms_key']}"].boundingBoxWGS84

            print(f"fetching {d_name} ..")
            print(d_info['wms_key'])
            try:
                img = wms.getmap(layers=[f"{d_info['wms_key']}"], bbox=box, size=(d_info["width"], d_info["height"]), srs=d_info["srs"], format=f"image/{d_info['format']}" , transparent=d_info["transparent"])
                print(f"SUCCESS")
            except NameError:
                print(f"Error while fetching {d_name} from {service_name}... : {NameError}")

            print(f"writing {d_name} Raster file into {path} \n")

            #write file into given folder (path)
            file = open(f"{path}/{d_name}.{d_info['format']}", "wb")
            file.write(img.read())
            file.close()

            #store download path into data_informations.json
            data_informations["data_wms"][d_name][f"{d_info['format']}_path"] = f"{path}/{d_name}.{d_info['format']}"

    # re-write data_informations.json => store all download path
    with open(data_informations_path, "w") as f:
        json.dump(data_informations, f, indent=4)

def convert_file(input_path, output_path, driver, input_extension = "gml", output_extension='gpkg', folder=False):
    """Convert file type
    If folder = TRUE, all of the files of the given folder are converted to the desired format.
    Careful about the path : either a folder or a file one.
    """
    print("converting ... ")
    if(folder):
        for filename in os.listdir(input_path):
            print(f"filename : {filename}")
            if filename.endswith(input_extension):
                # Read input GML file into a GeoDataFrame
                gdf = gpd.read_file(os.path.join(input_path, filename))

                if(driver == "GeoJSON"):
                    # 4326 is crs of OSM => used to project data on leaflet
                    gdf = gdf.to_crs(epsg=4326)
                    print("CRS of GEOJSON: ", gdf.crs)

                if(driver == "GPKG"):
                    gdf = gdf.to_crs(epsg=3946)

                # Set output file path and name
                output_name = filename.replace(input_extension, output_extension)
                new_output_path = os.path.join(output_path, output_name)

                # Write GeoDataFrame to output Shapefile
                gdf.to_file(new_output_path, driver=driver)

                print("Done, all GML files converted into GPKG")
    else:

        gdf = gpd.read_file(input_path)
        if(driver == "GeoJSON"):
            # 4326 is crs of OSM => used to project data on leaflet
            gdf = gdf.to_crs(epsg=4326)
        if(driver == "GPKG"):
            gdf = gdf.to_crs(epsg=3946)
        gdf.to_file(output_path, driver=driver)
        print(f"Done, {input_path} converted into {output_extension}")

def convert_all(output_path_folder, input_extension, output_extension, driver, connection_type="wfs"):

    """Convert all data stored in data_informations from input format to ouput format"""

    with open(data_informations_path, "r") as f:
        data_informations = json.load(f)

    data = data_informations[f'data_{connection_type}']
    for d_name, d_info in data.items():
        download_path = d_info[f"{input_extension}_path"]
        if(download_path.endswith(input_extension)):
            new_path = f"{output_path_folder}{d_name}.{output_extension}"
            convert_file(download_path, new_path, input_extension=input_extension, output_extension=output_extension, driver=driver)
            if(driver == "GeoJSON"):
                data_informations[f"data_{connection_type}"][d_name][f"geo{output_extension}_path"] = new_path
            else:
                data_informations[f"data_{connection_type}"][d_name][f"{output_extension}_path"] = new_path

    with open(data_informations_path, "w") as f:
        json.dump(data_informations, f, indent=4)

def points_to_polygon(point_path, polygon_path):
    """
        Computes the convex hull of a point shapefile and converts it to a polygon shapefile
    """

    points = gpd.read_file(point_path)

    # change temporarily the CRS (projection system) because buffering need meter
    # points.to_crs(epsg=3857)

    # Create a buffer around each point
    buffered_points = points.buffer(points["buffer_size"])

    # Convert the buffered points to polygons
    polygons = buffered_points.geometry.apply(lambda x: x.convex_hull)

    # Create a new GeoDataFrame with the polygon geometry and any additional attributes from the original points shapefile
    polygon_gdf = gpd.GeoDataFrame(points.drop('geometry', axis=1), geometry=polygons, crs=3946)

    # final_polygon = polygon_gdf.to_crs(epsg=4171)

    polygon_gdf.to_file(polygon_path, driver="GPKG")

def convert_all_points_into_polygons(output_path_folder):
    """ Convert all shapefile with Points Type into Polygons """

    with open(data_informations_path, "r") as f:
        data_informations = json.load(f)

    for d_name, d_info in data_informations["data_wfs"].items():
        original_gpkg_file = gpd.read_file(d_info["cleaned_data_path"])
        #if(original_gpkg_file.geom_type[0] == "Point"):
        buffered_path = f"{output_path_folder}{d_name}_buffered.gpkg"

        print(f"Converting {d_name} into Polygons ... ")

        points_to_polygon(d_info["cleaned_data_path"], buffered_path)
        data_informations["data_wfs"][d_name]["buffered_path"] = buffered_path

    with open(data_informations_path, "w") as f:
        json.dump(data_informations, f, indent=4)

def get_attributes_list(file_path):
    file = gpd.read_file(file_path)
    attribute_list = list(file.columns)
    return(attribute_list)

def write_all_atributes():
    """ Write into data_informations.json all attributes of the layer """

    print("Writing all attributes into data_informations.json file ...")

    with open(data_informations_path, "r") as f:
        data_informations = json.load(f)

    data_wfs = data_informations['data_wfs']
    for d_name, d_info in data_wfs.items():
        #the shape file is the original file containing all attributes
        file_path = data_wfs[d_name]["gpkg_path"]
        attribute_list = get_attributes_list(file_path)
        #print(f"{d_name} = {attribute_list}")
        data_informations["data_wfs"][d_name]["all_attributes"] = attribute_list

    with open(data_informations_path, "w") as f:
        json.dump(data_informations, f, indent=4)

    print("DONE")

def write_attributes_to_add_and_remove(data_name, attributes_to_add, attributes_to_remove):
    """ Write into data_informations.json all attributes to add or remove for one data
        attributes_to_add is a dictionnary with the pair key, values,
        attributes_to_remove is a list of attributes

        attributes_to_add = {
            attribute1 : default_value1
            attribute2 : default_value2
        }
        attributes_to_remove = ["attribute1", "attribute2"]
    """

    with open(data_informations_path, "r") as f:
        data_informations = json.load(f)

    data_informations['data_wfs'][data_name]['attributes_to_remove'] = attributes_to_remove
    data_informations['data_wfs'][data_name]['attributes_to_add'] = attributes_to_add

    with open(data_informations_path, "w") as f:
        json.dump(data_informations, f, indent=4)

def remove_attributes(input_path, output_path, attributes_to_remove):
    file = gpd.read_file(input_path)
    file = file.drop(attributes_to_remove, axis=1)
    file.to_file(output_path)

def add_attributes(input_path, output_path, attributes_to_add):
    """attributes_to_add should be a dictionnary with the following structure :

    attributes_to_add = {
        'new_attribute1': [1, 2, 3],
        'new_attribute2': ['a', 'b', 'c']
    }

    This function can be used to update the attributes

    """
    file = gpd.read_file(input_path)

    for attribute_name, values in attributes_to_add.items():
        file[attribute_name] = values

    file.to_file(output_path)

def remove_and_add_attributes(output_path_folder):
    """ """

    with open(data_informations_path, "r") as f:
        data_informations = json.load(f)

    data_wfs = data_informations["data_wfs"]
    for d_name, d_info in data_wfs.items():
        print(f"Remove and Add attributes for {d_name}")
        input_path = d_info["gpkg_path"]
        output_path = f"{output_path_folder}{d_name}_cleaned.gpkg"
        remove_attributes(input_path, output_path, d_info["attributes_to_remove"])
        add_attributes(output_path, output_path, d_info["attributes_to_add"])

        data_informations["data_wfs"][d_name]["cleaned_data_path"] = output_path

    with open(data_informations_path, "w") as f:
        json.dump(data_informations, f, indent=4)

def print_layers_name(folder_path):
    for filename in os.listdir(folder_path):
        if filename.endswith(".gpkg"):
            file_path = os.path.join(folder_path, filename)
            print(file_path)
            with fiona.open(file_path) as gpkg:
                layer_names = fiona.listlayers(file_path)
                print(f"{filename} : {layer_names}")

def extract_attributes(gdf, index):
    attributes = {}
    for column in gdf.columns:
        if column != "geometry":
            attributes[column] = gdf.loc[index, column]
    return attributes

def filter_dictionnary(dictionnary, filter):
    """the filter is a string contains or not into the dictionnary keys"""
    return {k:v for k,v in dictionnary.items() if filter in k}

def convert_gpkg_into_geojson(input_path, output_path):
    gdf = gpd.read_file(input_path)

    # 4326 is crs of OSM => used to project data on leaflet
    gdf = gdf.to_crs(epsg=4326)

    print("Converting GPKG into GeoJSON")
    gdf.to_file(output_path, driver='GeoJSON')
    print("Done")

# create_folder("./data/geojson")
#convert_gpkg_into_geojson("./data/osm/shortest_path/big_shortest_path_IF_3946.gpkg", "./data/geojson/sp_IF_3946.json")
#convert_gpkg_into_geojson("./data/raw_data/joined_if_3946.gpkg", "./pocwa-init/src/data/joined_if_3946.json")

# convert_gpkg_into_geojson("./data/gpkg/bancs.gpkg", "./data/geojson/bancs.json")

# convert_gpkg_into_geojson("./data/gpkg/fontaines_ornementales.gpkg", "./data/geojson/fontaines_ornementales.json")

# convert_gpkg_into_geojson("./data/gpkg/fontaines_potables.gpkg", "./data/geojson/fontaines_potables.json")

# convert_gpkg_into_geojson("./data/gpkg/parcs_jardins_metropole.gpkg", "./data/geojson/parcs_jardins_metropole.json")

# convert_gpkg_into_geojson("./data/gpkg/toilettes_publiques.gpkg", "./data/geojson/toilettes_publiques.json")


import helpers
import minkit
import os
import pytest

@pytest.mark.core
def test_backend():
    '''
    Test the construction of the backend.
    '''
    with pytest.raises(AttributeError):
        minkit.Backend.DataSet

    bk = minkit.Backend(minkit.backends.core.CPU)

    x = minkit.Parameter('x', bounds=(-1, +1))

    data = helpers.rndm_gen.uniform(0, 1, 1000)

    # Test initialization and constructor methods
    bk.DataSet(minkit.darray.from_ndarray(data, bk), [x])

    dataset = bk.DataSet.from_ndarray(data, x)

    new_bk = minkit.Backend(minkit.backends.core.CPU)

    m = bk.Parameter('m')
    c = bk.Parameter('c')
    s = bk.Parameter('s')
    k = bk.Parameter('k')
    y = bk.Parameter('y')

    g = bk.Gaussian('gauss', m, c, s)
    e = bk.Exponential('exponential', m, k)

    bk.AddPDFs.two_components('pdf', g, e, y)

    # Test the adaption of objects to new backends
    dataset.to_backend(new_bk)

BACKEND = os.environ.get('MINKIT_BACKEND', None)

if minkit.backends.core.is_gpu_backend(BACKEND):

    def test_gpu_backends():
        '''
        Test the change of objects from a CPU to a GPU backend.
        '''
        minkit.Backend()
        minkit.Backend(BACKEND)


import sys
import csv
from PySide2 import QtWidgets as Qtw
from PySide2 import QtGui as Qtg
from PySide2 import QtCore as Qtc

class CSVTableModel(Qtc.QAbstractTableModel):

    def __init__(self, csv_file):
        super(CSVTableModel, self).__init__()

        self.filename = csv_file

        with open(self.filename) as fh:
            csvreader = csv.reader(fh)
            self._headers = next(csvreader)
            self._data = list(csvreader)

if __name__ == '__main__':
    app = Qtw.QApplication(sys.argv)
    mw = CSVTableModel()
    mw.show()
    sys.exit(app.exec_())


import discord
from discord.ext import commands
import json
from discord.ui import Button,View,Select

with open('setting.json', mode = 'r', encoding = 'utf8') as jfile:
    jdata = json.load(jfile)

class React(commands.Cog):
    def __init__(self,bot :commands.Bot) -> None:
        self.bot = bot

    @commands.command()
    async def button_test(self,ctx):
        view = View()
        button = Button(label='點我啊臭ㄐㄐ',style=discord.ButtonStyle.primary,emoji='<:ji:816467849357951058>')

        async def button_callback(interaction):
            user = interaction.user.name
            await interaction.response.send_message(f'{user}點了，{user}是臭ㄐㄐ')
        button.callback = button_callback
        view.add_item(button)
        await ctx.send(view=view)

    @commands.command()
    async def select_test(self,ctx):
        select = Select(
            placeholder='你覺得蔡神有多帥(滿分100)',
            options=[
                discord.SelectOption(
                    label='100分',
                    description='滿分，蔡神超帥'
                ),
                discord.SelectOption(
                    label='200分',
                    description='蔡神怎麼可能被100分侷限住'
                ),
                discord.SelectOption(
                    label='114514分',
                    description='這麼臭的選項有存在的必要嗎(惱'
                )
            ]
        )
        async def callback(interaction):
            user = interaction.user.name
            await interaction.response.send_message(f'{user}覺得蔡神有{select.values[0]}帥')

        select.callback = callback
        view = View()
        view.add_item(select)
        await ctx.send(view = view)

async def setup(bot):
    await bot.add_cog(React(bot))


import warnings
import logging
import bot

chat_commands = {}
default_class = None

class ChatCommand(object):
    '''a command which is run in response to a chat command

    very non-OOP class for representing a chat command
    note: define a sub-class then call SubClass.register(..)
    instances are created through ChatCommand.process_text()
    do not manually create instances of this class or any inheriting
    classes!

    Attributes:
        args: array of arguments for the command executing
        chan: discord.py channel for the context this command executes in
        sender: discord.py user for the context ...
    '''
    # the prefix can be re-defined wherever, possibly per-chanhel-connection?
    command_prefix = "!"
    name = "PLACEHOLDER"

    def __init__(self):
        """setup new instance with placeholder values"""
        self.args = []
        self.chan = None
        self.sender = None

    def bind_args(self, tokens):
        """bind command arguments in an array

        note self.args[0] == registered name (like sys.argv)

        Args:
            tokens:
        """
        for t in tokens:
            self.args.append(t)

    def bind_context(self, channel, sender):
        """bind discord context for execution.

        note: this MUST be called before execute!

        Args:
            channel: discord.py channel object
            sender: ... user object
        """
        self.chan = channel
        self.sender = sender

    def _body(self):
        """override this for commands to implement execution functionality
        """
        warnings.warn("Default body in ChatCommand")
        return

    def execute(self):
        """perform the action associated with this command.

        do not override this in child classes. this is where logic for valid
        state is done because we cannot do that at instance construction!
        """
        if self.chan is None or self.sender is None:
            raise RuntimeError("Invalid ChatCommand state: no context bound")
        self._body()

    def _helptext(self):
        """
        override with explanatory text for a given command. used for generating
        help messages.
        """
        return "PLACEHOLDER"

    @classmethod
    def register(cls, name):
        """ register this CLASS with a given name, to be looked up in parsing

        Args:
            name: string name associating this class to a discord chat command
        """
        global chat_commands
        if name in chat_commands:
            # override for duplicate names, but log it!
            warnings.warn("Duplicate command registered - overriding "+name,
                          stacklevel=3)
        chat_commands[name] = cls
        cls.name = name

    @classmethod
    def process_text(cls, text):
        """parse text and check for a valid registered command call

        Args:
            text: string message from discord

        Returns:
            ChatCommand instance which can be executed, even if the command
                given in text was invalid (see set_default)
        """
        global chat_commands
        global default_class
        # early out for empty strings
        if len(text) < 0:
            return default_class()
        if text[0] == cls.command_prefix:
            # tokenize without prefix character
            tokens = text[1:].split(" ")
            # execute command name if known otherwise log
            if tokens[0] in chat_commands:
                cmd = chat_commands[tokens[0]]()
                cmd.bind_args(tokens)
                if cmd is None:
                    print("Somehow matched none command! {}".format(tokens[0]))
                return cmd
            else:
                logging.info("Unregistered command {} called".format(
                    tokens[0]))
                print("Returning default command handler class")
                return default_class()
        else:
            return default_class()

    @classmethod
    def set_default(cls):
        """set default class for process_text() commands where no command is found
        """
        print("Setting default chat command class globally")
        global default_class
        default_class = cls
        print(default_class)

class CommandNull(ChatCommand):
    """
    no-action "fallthrough" class for commands which do not map to classes
    """
    def _body(self):
        pass

# set this class as the default!
CommandNull.set_default()

class CommandsHelp(ChatCommand):
    """
    command for listing all commands
    """
    def _body(self):
        global chat_commands
        msg = "Commands list: \n```"
        for key in chat_commands:
            # do not print a message for ourself
            if key == self.__class__.name:
                continue
            # get helptext from temp instance of this class
            c = chat_commands[key]()
            msg += "{}: {}\n".format(key, c._helptext())
        # finally send message using bot.client (logged in client)
        bot.client.message_send(self.chan, msg)


#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
@Author  ：fangpf
@Date    ：2022/2/8 15:32
"""
import math
import warnings
from collections import OrderedDict

import cv2
import imgaug
import pyclipper
import torch
import torchvision
from PIL import Image
import numpy as np
from shapely.geometry import Polygon
import imgaug.augmenters as iaa

'''
数据增强变换方法
'''

class BaseAugment():
    '''
    通过 imgaug.augmenters 进行基础变换，包括尺寸调整、翻转、旋转等
    '''

    def __init__(self, only_resize=False, keep_ratio=False, augmenters=None, resize_shape=None):
        self.only_resize = only_resize
        self.keep_ratio = keep_ratio
        self.augmenter = augmenters
        self.resize_shape = resize_shape

    def resize_image(self, image):
        origin_height, origin_width, _ = image.shape
        height = self.resize_shape['height']
        width = self.resize_shape['width']
        if self.keep_ratio:  # 是否保持图像长宽比不变
            width = origin_width * height / origin_height
            N = math.ceil(width / 32)
            width = N * 32
        image = cv2.resize(image, (width, height))
        return image

    def process(self, data):
        image = data['image']
        shape = image.shape

        if self.augmenter:
            aug = self.augmenter.to_deterministic()
            if self.only_resize:
                data['image'] = self.resize_image(image)  # 只进行尺寸调整
            else:
                data['image'] = aug.augment_image(image)  # 图像变换
            self.may_augment_annotation(aug, data, shape)  # 对 polygon 标注进行对应的变换

        filename = data.get('filename', data.get('data_id', ''))
        data.update(filename=filename, shape=shape[:2])
        return data

    def may_augment_annotation(self, aug, data, shape):
        if aug is None:
            return data

        line_polys = []
        for line in data['lines']:
            if self.only_resize:
                new_poly = [(p[0], p[1]) for p in line['poly']]
            else:
                new_poly = self.may_augment_poly(aug, shape, line['poly'])
            line_polys.append({
                'points': new_poly,
                'ignore': line['text'] == '###',  # 图像是否是困难样本（模糊不可辨），本任务数据集中不存在困难样本
                'text': line['text'],
            })
        data['polys'] = line_polys
        return data

    def may_augment_poly(self, aug, img_shape, poly):
        keypoints = [imgaug.Keypoint(p[0], p[1]) for p in poly]
        keypoints = aug.augment_keypoints([imgaug.KeypointsOnImage(keypoints, shape=img_shape)])[0].keypoints
        poly = [(p.x, p.y) for p in keypoints]
        return poly

class ColorJitter():
    '''
    颜色增强，包括亮度、对比度、饱和度、色相变换
    '''

    def __init__(self, b=0.2, c=0.2, s=0.15, h=0.15):
        self.color_jitter = torchvision.transforms.ColorJitter(
            brightness=b, contrast=c, saturation=s, hue=h)

    def process(self, data):
        img = data['image']
        image = Image.fromarray(img.astype('uint8')).convert('RGB')  # 数据类型转换
        img = np.array(self.color_jitter(image)).astype(np.float64)
        data['image'] = img
        return data

class RandomCropData():
    '''
    随机裁剪图像，并保证裁剪时不切割到图像中的文字区域
    '''

    def __init__(self, size=(640, 640)):
        self.size = size
        self.max_tries = 10  # 裁剪尝试的最大次数（因为存在裁剪区域太小等裁剪失败情况）
        self.min_crop_side_ratio = 0.1  # 裁剪区域边长最小比例，即裁剪的图像边长与原始图像边长的比值不能小于 min_crop_side_ratio

    def process(self, data):
        img = data['image']

        ori_img = img
        ori_lines = data['polys']
        all_care_polys = [line['points'] for line in data['polys'] if not line['ignore']]
        crop_x, crop_y, crop_w, crop_h = self.crop_area(img, all_care_polys)  # 裁剪区域的左上角坐标(x, y)以及区域宽高(w, h)

        # 根据裁剪区域参数对图像进行裁剪，并填充空白以得到指定 size 的图像（在右侧或者底侧进行填充）
        scale_w = self.size[0] / crop_w
        scale_h = self.size[1] / crop_h
        scale = min(scale_w, scale_h)
        h = int(crop_h * scale)
        w = int(crop_w * scale)
        padimg = np.zeros((self.size[1], self.size[0], img.shape[2]), img.dtype)
        padimg[:h, :w] = cv2.resize(img[crop_y:crop_y + crop_h, crop_x:crop_x + crop_w], (w, h))
        img = padimg

        # 根据裁剪区域参数对文字位置坐标进行转换
        lines = []
        for line in data['polys']:
            poly = ((np.array(line['points']) - (crop_x, crop_y)) * scale).tolist()
            if not self.is_poly_outside_rect(poly, 0, 0, w, h): lines.append({**line, 'points': poly})  # 不保留裁剪区域之外的文字

        data['polys'] = lines
        data['image'] = img

        return data

    def is_poly_outside_rect(self, poly, x, y, w, h):
        # 判断文字polygon 是否在矩形区域外
        poly = np.array(poly)
        if poly[:, 0].max() < x or poly[:, 0].min() > x + w:
            return True
        if poly[:, 1].max() < y or poly[:, 1].min() > y + h:
            return True
        return False

    def split_regions(self, axis):
        # 返回可划切割线的连续区域
        regions = []
        min_axis = 0
        for i in range(1, axis.shape[0]):
            if axis[i] != axis[i - 1] + 1:
                region = axis[min_axis:i]
                min_axis = i
                regions.append(region)
        return regions

    def random_select(self, axis, max_size):
        # 从一块连续区域中选择两条切割线
        xx = np.random.choice(axis, size=2)
        xmin = np.min(xx)
        xmax = np.max(xx)
        xmin = np.clip(xmin, 0, max_size - 1)
        xmax = np.clip(xmax, 0, max_size - 1)
        return xmin, xmax

    def region_wise_random_select(self, regions, max_size):
        # 从两块连续区域中选择两条切割线
        selected_index = list(np.random.choice(len(regions), 2))
        selected_values = []
        for index in selected_index:
            axis = regions[index]
            xx = int(np.random.choice(axis, size=1))
            selected_values.append(xx)
        xmin = min(selected_values)
        xmax = max(selected_values)
        return xmin, xmax

    def crop_area(self, img, polys):
        # 裁剪区域
        h, w, _ = img.shape
        h_array = np.zeros(h, dtype=np.int32)
        w_array = np.zeros(w, dtype=np.int32)
        for points in polys:
            points = np.round(points, decimals=0).astype(np.int32)
            minx = np.min(points[:, 0])
            maxx = np.max(points[:, 0])
            w_array[minx:maxx] = 1
            miny = np.min(points[:, 1])
            maxy = np.max(points[:, 1])
            h_array[miny:maxy] = 1
        # h_array == 1 的位置表示有文本，h_array == 0 的位置表示无文本；w_array 同理
        h_axis = np.where(h_array == 0)[0]
        w_axis = np.where(w_array == 0)[0]

        if len(h_axis) == 0 or len(w_axis) == 0:
            return 0, 0, w, h

        h_regions = self.split_regions(h_axis)
        w_regions = self.split_regions(w_axis)

        for i in range(self.max_tries):
            if len(w_regions) > 1:
                # 有多块可切割区域时
                xmin, xmax = self.region_wise_random_select(w_regions, w)
            else:
                # 只有一块可切割区域时
                xmin, xmax = self.random_select(w_axis, w)
            if len(h_regions) > 1:
                ymin, ymax = self.region_wise_random_select(h_regions, h)
            else:
                ymin, ymax = self.random_select(h_axis, h)

            if xmax - xmin < self.min_crop_side_ratio * w or ymax - ymin < self.min_crop_side_ratio * h:
                # 切割区域太小，不可取
                continue
            num_poly_in_rect = 0

            # 保证至少有一个文字区域在切割出的区域中即可
            for poly in polys:
                if not self.is_poly_outside_rect(poly, xmin, ymin, xmax - xmin, ymax - ymin):
                    num_poly_in_rect += 1
                    break

            if num_poly_in_rect > 0:
                return xmin, ymin, xmax - xmin, ymax - ymin

        return 0, 0, w, h

class MakeSegDetectionData():
    '''
    构造文本区域二值图（DB论文中的 probability map），以及用于计算loss的mask
    '''

    def __init__(self, min_text_size=8, shrink_ratio=0.4):
        self.min_text_size = min_text_size
        self.shrink_ratio = shrink_ratio  # polygon 收缩比例

    def process(self, data):
        # 数据结构调整统一，方便后续操作
        polygons = []
        ignore_tags = []
        annotations = data['polys']
        for annotation in annotations:
            polygons.append(np.array(annotation['points']))
            ignore_tags.append(annotation['ignore'])
        ignore_tags = np.array(ignore_tags, dtype=np.uint8)
        filename = data.get('filename', data['data_id'])
        shape = np.array(data['shape'])
        data = OrderedDict(image=data['image'],
                           polygons=polygons,
                           ignore_tags=ignore_tags,
                           shape=shape,
                           filename=filename,
                           is_training=data['is_training'])

        image = data['image']
        polygons = data['polygons']
        ignore_tags = data['ignore_tags']
        image = data['image']
        filename = data['filename']

        h, w = image.shape[:2]
        if data['is_training']:
            polygons, ignore_tags = self.validate_polygons(polygons, ignore_tags, h, w)
        gt = np.zeros((h, w), dtype=np.float32)
        mask = np.ones((h, w), dtype=np.float32)
        for i in range(len(polygons)):
            polygon = polygons[i]
            height = max(polygon[:, 1]) - min(polygon[:, 1])
            width = max(polygon[:, 0]) - min(polygon[:, 0])
            if ignore_tags[i] or min(height, width) < self.min_text_size:  # 文本区域太小时，作为困难样本 ignore
                cv2.fillPoly(mask, polygon.astype(np.int32)[np.newaxis, :, :], 0)
                ignore_tags[i] = True
            else:
                # 收缩 polygon 并绘制 probability map
                polygon_shape = Polygon(polygon)
                distance = polygon_shape.area * (1 - np.power(self.shrink_ratio, 2)) / polygon_shape.length
                subject = [tuple(l) for l in polygons[i]]
                padding = pyclipper.PyclipperOffset()
                padding.AddPath(subject, pyclipper.JT_ROUND, pyclipper.ET_CLOSEDPOLYGON)
                shrinked = padding.Execute(-distance)
                if shrinked == []:
                    cv2.fillPoly(mask, polygon.astype(np.int32)[np.newaxis, :, :], 0)
                    ignore_tags[i] = True
                    continue
                shrinked = np.array(shrinked[0]).reshape(-1, 2)
                cv2.fillPoly(gt, [shrinked.astype(np.int32)], 1)

        if filename is None:
            filename = ''
        data.update(image=image,
                    polygons=polygons,
                    gt=gt, mask=mask, filename=filename)
        return data

    def validate_polygons(self, polygons, ignore_tags, h, w):
        '''
        统一polygon坐标顺序，并且ignore面积为0的polygons
        '''
        if len(polygons) == 0:
            return polygons, ignore_tags
        assert len(polygons) == len(ignore_tags)
        for polygon in polygons:
            polygon[:, 0] = np.clip(polygon[:, 0], 0, w - 1)
            polygon[:, 1] = np.clip(polygon[:, 1], 0, h - 1)

        for i in range(len(polygons)):
            area = self.polygon_area(polygons[i])
            if abs(area) < 1:
                ignore_tags[i] = True
            if area > 0:
                polygons[i] = polygons[i][::-1, :]  # 调整坐标顺序
        return polygons, ignore_tags

    def polygon_area(self, polygon):
        edge = 0
        for i in range(polygon.shape[0]):
            next_index = (i + 1) % polygon.shape[0]
            edge += (polygon[next_index, 0] - polygon[i, 0]) * (polygon[next_index, 1] + polygon[i, 1])

        return edge / 2.

class MakeBorderMap():
    '''
    构造文本边界二值图（DB论文中的 threshold map），以及用于计算loss的mask
    '''

    def __init__(self, shrink_ratio=0.4, thresh_min=0.3, thresh_max=0.7):
        self.shrink_ratio = shrink_ratio
        self.thresh_min = thresh_min
        self.thresh_max = thresh_max
        warnings.simplefilter("ignore")

    def process(self, data):
        image = data['image']
        polygons = data['polygons']
        ignore_tags = data['ignore_tags']
        canvas = np.zeros(image.shape[:2], dtype=np.float32)
        mask = np.zeros(image.shape[:2], dtype=np.float32)

        for i in range(len(polygons)):
            if ignore_tags[i]:
                continue
            self.draw_border_map(polygons[i], canvas, mask=mask)  # 绘制 border map
        canvas = canvas * (self.thresh_max - self.thresh_min) + self.thresh_min
        data['thresh_map'] = canvas
        data['thresh_mask'] = mask
        return data

    def draw_border_map(self, polygon, canvas, mask):
        polygon = np.array(polygon)
        assert polygon.ndim == 2
        assert polygon.shape[1] == 2

        polygon_shape = Polygon(polygon)
        distance = polygon_shape.area * (1 - np.power(self.shrink_ratio, 2)) / polygon_shape.length
        subject = [tuple(l) for l in polygon]
        padding = pyclipper.PyclipperOffset()
        padding.AddPath(subject, pyclipper.JT_ROUND, pyclipper.ET_CLOSEDPOLYGON)
        padded_polygon = np.array(padding.Execute(distance)[0])
        cv2.fillPoly(mask, [padded_polygon.astype(np.int32)], 1.0)

        xmin = padded_polygon[:, 0].min()
        xmax = padded_polygon[:, 0].max()
        ymin = padded_polygon[:, 1].min()
        ymax = padded_polygon[:, 1].max()
        width = xmax - xmin + 1
        height = ymax - ymin + 1

        polygon[:, 0] = polygon[:, 0] - xmin
        polygon[:, 1] = polygon[:, 1] - ymin

        xs = np.broadcast_to(np.linspace(0, width - 1, num=width).reshape(1, width), (height, width))
        ys = np.broadcast_to(np.linspace(0, height - 1, num=height).reshape(height, 1), (height, width))

        distance_map = np.zeros((polygon.shape[0], height, width), dtype=np.float32)
        for i in range(polygon.shape[0]):
            j = (i + 1) % polygon.shape[0]
            absolute_distance = self.distance(xs, ys, polygon[i], polygon[j])
            distance_map[i] = np.clip(absolute_distance / distance, 0, 1)
        distance_map = distance_map.min(axis=0)

        xmin_valid = min(max(0, xmin), canvas.shape[1] - 1)
        xmax_valid = min(max(0, xmax), canvas.shape[1] - 1)
        ymin_valid = min(max(0, ymin), canvas.shape[0] - 1)
        ymax_valid = min(max(0, ymax), canvas.shape[0] - 1)
        canvas[ymin_valid:ymax_valid + 1, xmin_valid:xmax_valid + 1] = np.fmax(
            1 - distance_map[
                ymin_valid - ymin:ymax_valid - ymax + height,
                xmin_valid - xmin:xmax_valid - xmax + width],
            canvas[ymin_valid:ymax_valid + 1, xmin_valid:xmax_valid + 1])

    def distance(self, xs, ys, point_1, point_2):
        # 计算图像中的点到 文字polygon 边界的距离
        height, width = xs.shape[:2]
        square_distance_1 = np.square(
            xs - point_1[0]) + np.square(ys - point_1[1])
        square_distance_2 = np.square(
            xs - point_2[0]) + np.square(ys - point_2[1])
        square_distance = np.square(
            point_1[0] - point_2[0]) + np.square(point_1[1] - point_2[1])

        cosin = (square_distance - square_distance_1 - square_distance_2) / (
                    2 * np.sqrt(square_distance_1 * square_distance_2))
        square_sin = 1 - np.square(cosin)
        square_sin = np.nan_to_num(square_sin)
        result = np.sqrt(square_distance_1 * square_distance_2 * square_sin / square_distance)

        result[cosin < 0] = np.sqrt(np.fmin(square_distance_1, square_distance_2))[cosin < 0]
        return result

class NormalizeImage():
    '''
    将图像元素值归一化到[-1, 1]
    '''
    RGB_MEAN = np.array([122.67891434, 116.66876762, 104.00698793])

    def process(self, data):
        assert 'image' in data, '`image` in data is required by this process'
        image = data['image']
        image -= self.RGB_MEAN
        image /= 255.
        image = torch.from_numpy(image).permute(2, 0, 1).float()
        data['image'] = image
        return data

    @classmethod
    def restore(self, image):
        image = image.permute(1, 2, 0).to('cpu').numpy()
        image = image * 255.
        image += self.RGB_MEAN
        image = image.astype(np.uint8)
        return image

class FilterKeys():
    '''
    过滤掉后续不需要的键值对
    '''

    def __init__(self, superfluous):
        self.superfluous_keys = set(superfluous)

    def process(self, data):
        for key in self.superfluous_keys:
            del data[key]
        return data


# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import redis

class PythonscrapyPipeline:

    def process_item(self, item, spider):

        # 建立连接
        pool = redis.ConnectionPool(host='localhost', port=6379, decode_responses=True)
        conn = redis.Redis(connection_pool=pool)

        # 取出标题
        for key, value in item.items():
            value1 = value
            break
        print('目标网站:' + value1)

        for key, value in item.items():
            conn.hset(value1, key, value)

        print(value1 + '爬取成功!')
        # print(conn.hgetall(value1))
        return item


import requests
from flask import Blueprint, render_template, abort, request, session

from src.data_interface import model

bp = Blueprint("you", __name__)

@bp.route("/you")
def index():
    session_id = session["uid"]

    res = requests.get(
        url="http://api:8000/recommend/session/{id}".format(id=session_id),
        params={"page_type": "you"}
    )

    if res.status_code != 200:
        abort(res.status_code)

    res_json = res.json()

    recommendations = res_json["recommendations"]

    return render_template(
        "you/index.html",
        recommendations=recommendations
    )

@bp.route("/you/taste", methods=("GET", "POST"))
def taste():
    if request.method == "POST":
        form_data = request.form.to_dict(flat=False)
        user_genres = form_data.get("genre") or []

        model.User.query\
            .filter_by(username=session.get("username"))\
            .update({"favorite_genres": map(int, user_genres)})

    genres = model.Genre.query.all()
    user_genres = model.User.query\
        .with_entities(model.User.favorite_genres)\
        .filter(model.User.username == session["username"])\
        .one()[0]

    return render_template(
        "you/taste.html",
        genres=genres,
        user_genres=user_genres
    )


import boto3
from pprint import pprint

dynamodb = boto3.resource('dynamodb')

users = dynamodb.Table('Users')

response = users.query(
    KeyConditionExpression='Id = :id',
    # :idで仮の値で初期化。以下で上書き。Prepared Statementっぽい。上記で値を入れるのはNG
    ExpressionAttributeValues={':id': 1}
)

pprint(response)


import http
from functools import wraps

import jwt
from django.conf import settings
from django.http import JsonResponse

def token_required(func):
    @wraps(func)
    def wrapper(request, *args, **kwargs):
        token = request.headers.get("Authorization", "").replace("Bearer", "").strip()
        if not token:
            return JsonResponse(
                {"message": "Invalid Token"}, status=http.HTTPStatus.UNAUTHORIZED
            )
        try:
            payload = jwt.decode(token, settings.JWT_SECRET_KEY, algorithms=["HS256"])
        except (jwt.ExpiredSignatureError, jwt.DecodeError):
            return JsonResponse(
                {"message": "Invalid Token"}, status=http.HTTPStatus.BAD_REQUEST
            )
        request.user_id = payload["sub"]
        request.user_email = payload["user_email"]
        return func(request, *args, **kwargs)

    return wrapper


# levanto el archivo
from openpyxl import load_workbook
import func.som as som
import numpy as np

wb = load_workbook('Ejercicios/Act_4/Drug4.xlsx', read_only=True)
ws = wb.get_sheet_by_name('Sheet1')
ws.max_row
ws.max_column
numerical_data = ws.iter_rows(min_row=1, max_row=ws.max_row, min_col=1, max_col=ws.max_column)
muestra = np.array([[cell.value for cell in row] for row in numerical_data])
# separo los datos de la calse
X = muestra[:, 0:6]
T = muestra[:, 6]
# escalamos los datos
X_escalado = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0))
# Genero matriz de valores esperados
T_matriz = np.concatenate(([T == 0], [T == 1], [T == 2], [T == 3]), axis=0).astype(int)

# Defino Parámetros de la red
P = X_escalado.T
columnas = 4
filas = 4
alfaInicial = 0.2
vecindad = 3
func_vecindad = 1  ####otro nro es sombrero
sigma = 2  ##ancho del sombrero
ite_reduce = 50
dibujar = True

matriz = np.zeros((200, 200))
for func_vecindad in [0, 1]:
    for vecindad in [2, 3]:
        for sigma in [1, 2, 3]:
            for columnas in [2, 3, 4]:
                (w_O) = som.train(P, filas, columnas, alfaInicial, vecindad, func_vecindad, sigma, ite_reduce, dibujar)

                # una columna por cada dimension de mi dataset
                ganador = []
                for p in range(200):
                    distancias = -np.sqrt(
                        np.sum((w_O - (P[:, p][np.newaxis]) * np.ones((filas * columnas, 1))) ** 2, 1))
                    ganadora = np.argmax(distancias)
                    ganador.append(ganadora)

                for i in range(200):
                    for j in range(200):
                        ##if i!=j: opcional.....la funcion dendograma se encarga de esto
                        if ganador[i] == ganador[j]:
                            matriz[i][j] += 1

matriz2 = matriz

som.dendograma(matriz, T)


from app.models.event import *
import datetime

event1 = Event(1, datetime.date(2021, 2, 22), "Judys Seminar on Cats", 15, "Room 401", "Judy talks about her numerous cats")
event2 = Event(2, datetime.date(2021, 4, 15), "Multi-level Marketing Opportunity", 3, "Room 10", "Totally legitimate business, yep")
event3 = Event(3, datetime.date(2021, 3, 2), "Gathering of the Juggalos", 75, "Room 3", "ICP Appreciation Society")

events = [event1, event2, event3]

def add_new_event(event):
    events.append(event)


# coding: utf-8
# Description: 适用于单点服务检测
import socket
import httplib
import subprocess
import traceback
import re
from logger.logger import logger
from config.config_init import config

# Description: private object
class AppCheck(object):
    def __init__(self, name, address='127.0.0.1'):
        self.name = name
        self.address = address
        self.pid = 1
        self.port = 0
        self.port_check_cmd = "ss -tunlp | grep {}".format(self.name)
        self.process_status = False
        self.listen_status = False
        self.connect_status = False
        self.request_status = False

        logger.info('Checking {} status'.format(self.name))

    # Description: 组合检测监控指标 [进程运行，端口监听，端口连接，接口请求]
    def auto_check(self):
        self.process_status = self.process_check()
        if self.process_status is True:
            self.listen_status = self.port_listen_check()
            if self.listen_status is True:
                self.connect_status = self.port_request_check()
                if self.connect_status is True:
                    self.request_status = self.api_check()

    # Description: initial pid
    # ReturnType boolean
    def process_check(self):
        try:
            cmd = 'ps -ef | grep {} | grep -v grep '.format(self.name)
            result = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            line = result.stdout.readline()
            if len(line) > 0:
                self.pid = line.split()[1]
                logger.info('{} is running with pid {}'.format(self.name, self.pid))
                return True
            else:
                logger.error('{} is not running'.format(self.name))
                return False
        except Exception as e:
            logger.error(traceback.format_exc())

    # Description: initial port
    # ReturnType boolean
    def port_listen_check(self):
        try:
            result = subprocess.Popen(self.port_check_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            line = result.stdout.readline()
            if line and 'LISTEN' in line:
                self.port = int(line.split()[4].split(':')[-1])
                logger.info('{} is listening at {}'.format(self.name, self.port))
                return True
            else:
                logger.warning('{} port state is not listen'.format(self.name))
                return False
        except Exception as e:
            logger.error(traceback.format_exc())

    # ReturnType boolean
    def port_request_check(self):
        try:
            s = socket.socket()
            # logger.info('Attempting to connect to %s on port %s' % (self.name, self.port))
            try:
                s.connect((self.address, self.port))
                logger.info("Connected to %s on port %s" % (self.name, self.port))
                return True
            except socket.error as e:
                logger.error("Connected to %s on port %s failed: %s" % (self.name, self.port, e))
                return False
        except Exception as e:
            logger.error(traceback.format_exc())

    def api_check(self):
        print('please override app api')

class Nginx(AppCheck):
    def __init__(self, name='nginx'):
        super(Nginx, self).__init__(name)
        self.auto_check()

    # ReturnType boolean
    def api_check(self):
        try:
            logger.info('sending request to {}'.format(self.name))
            resource = '/'
            if not resource.startswith('/'):
                resource = '/' + resource
            conn = httplib.HTTPConnection(self.address, self.port)
            try:
                logger.info('HTTP connection created successfully')
                conn.request('GET', resource)
                logger.info('sending request for %s successful' % resource)

                response = conn.getresponse()

                logger.info('response status: %s' % response.status)
            except socket.error as e:
                logger.error('HTTP connection failed: %s' % e)
                return False
            finally:
                conn.close()
                logger.info('HTTP connection closed succeesfully')
            if response.status in [200, 301]:
                logger.info('get response from {} successfully'.format(self.name))
                return True
            else:
                return False
        except Exception as e:
            logger.error(traceback.format_exc())

class Mysql(AppCheck):
    def __init__(self, name='mysql'):
        super(Mysql, self).__init__(name)
        self.auto_check()

    def api_check(self):
        try:
            logger.info('sending request to {}'.format(self.name))
            mysql_login = config.mysql_auth()
            cmd = "`which mysql` -P {} -u{} -p{} -e 'select version()'".format(self.port, mysql_login['user'], mysql_login['password'])
            result = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            f = result.stdout.read()
            if 'version' in f:
                logger.info('get response from {} successfully'.format(self.name))
                return True
            else:
                return False
        except Exception as e:
            logger.error(traceback.format_exc())

class Redis(AppCheck):
    def __init__(self, name='redis'):
        super(Redis, self).__init__(name)
        self.auto_check()

    def api_check(self):
        try:
            logger.info('sending request to {}'.format(self.name))
            redis_login = config.redis_auth()
            cmd = "`which redis-cli` -p {} -a {} info".format(self.port, redis_login['password'])
            result = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            f = result.stdout.read()
            if 'Server' in f:
                logger.info('get response from {} successfully'.format(self.name))
                return True
            else:
                return False
        except Exception as e:
            logger.error(traceback.format_exc())

# Description: zk 基于java服务。socket连接无法采用服务名查询，故采用端口查询
class Zookeeper(AppCheck):
    def __init__(self, name='zookeeper'):
        super(Zookeeper, self).__init__(name)
        self.get_port()
        self.port_check_cmd = "ss -tunlp | grep {}".format(self.port)
        self.auto_check()

    def api_check(self):
        try:
            logger.info('sending request to {}'.format(self.name))
            cmd = '`which zkServer.sh` status'
            result = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            f = result.stdout.read()
            if 'Mode' in f:
                logger.info('get response from {} successfully'.format(self.name))
                return True
            else:
                return False
        except Exception as e:
            logger.error(traceback.format_exc())

    def get_port(self):
        try:
            cmd = "`which zkServer.sh` status | grep port"
            result = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            lines = result.stdout.readlines()
            pattern = r"port"
            for line in lines:
                result = re.search(pattern, line)
                if result:
                    self.port = int(line.split('.')[0].split(':')[-1])
                    break
            if result is None:
                raise Exception('port not found in zkServer.sh status')
            return self.port
        except Exception as e:
            logger.error(traceback.format_exc())


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="EgyVoc", # Replace with your own username
    version="0.0.11",
    author="Marwan Kilani",
    author_email="kilani.edu@gmail.com",
    description="Vocalizer for Ancient Egyptian",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/MKilani/EgyVoc",
    include_package_data=True,
    packages=setuptools.find_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    install_requires=["py4j"],
    python_requires='>=3.0',
)


# _*_coding:utf-8 _*_
# from:https://blog.csdn.net/lxg0807/article/details/52960072
# 由于未来得及下测试数据集，所以测试一块做了一定的调整或忽略

# 导入模块
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
import fasttext

import jieba
import os

basedir = "task_3/data"  # 这是我的文件地址，需跟据文件夹位置进行更改
os.chdir(basedir)
dir_list = ['affairs', 'constellation', 'economic', 'edu', 'ent', 'fashion', 'game', 'home', 'house', 'lottery',
            'science', 'sports', 'stock']

##生成fastext的训练和测试数据集

ftrain = open("news_fasttext_train.txt", "w")
ftest = open("news_fasttext_test.txt", "w")

# 第一步获取分类文本，文本直接用的清华大学的新闻分本，可在文本系列的第三篇找到下载地址。
# 输出数据格式： 样本 + 样本标签
# 说明：这一步不是必须的，可以直接从第二步开始，第二步提供了处理好的文本格式。写这一步主要是为了记忆当时是怎么处理原始文本的。
num = -1
for e in dir_list:
    num += 1
    indir = basedir + e + '/'
    files = os.listdir(indir)
    count = 0
    for fileName in files:
        count += 1
        filepath = indir + fileName
        with open(filepath, 'r') as fr:
            text = fr.read()
        text = text.decode("utf-8").encode("utf-8")
        seg_text = jieba.cut(text.replace("\t", " ").replace("\n", " "))
        outline = " ".join(seg_text)
        outline = outline.encode("utf-8") + "\t__label__" + e + "\n"
        #         print outline
        #         break

        if count < 10000:
            ftrain.write(outline)
            ftrain.flush()
            continue
        elif count < 20000:
            ftest.write(outline)
            ftest.flush()
            continue
        else:
            break

ftrain.close()
# ftest.close()

# 训练模型
classifier = fasttext.train_supervised("news_fasttext_train.txt", label_prefix="__label__")

# 保存模型
opt="/home/nicken/NLP_study/nlp_task/task_3/model/news_fasttext.model"
classifier.save_model(opt)
#load训练好的模型
#classifier = fasttext.load_model('news_fasttext.model.bin', label_prefix='__label__')

#测试模型
result = classifier.test("news_fasttext_train.txt")
# print(result.precision)
# print(result.recall)
print(result)


import can
import cantools
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
import pygame
import datetime as dt

CAN_FEEDBACK_NAME = ["APS_Feedback", "Break_ACT_Feedback","Steering_Angle_Feedback","Override_Feedback", "Vehicle_Speed","Turn_Sig_Feed"]
CAN_CONTROL_NAME = ['Accel_CMD','Break_CMD','Steering_CMD','Gear_Shift_CMD','Override_Off','Alive_Count','Angular_Speed_CMD']

class CAN(object):
    def __init__(self):
        # CAN
        self.db = cantools.database.load_file('SantaFe.dbc')
        self.bus = can.interface.Bus(bustype='socketcan', channel='vcan0', bitrate=500000)
        self.vehicle_info_1 = self.db.get_message_by_name(
            'Vehicle_Info_1')  # APS_Feedback, Break_ACT_Feedback, Steering_Angle_Feedback
        self.vehicle_info_2 = self.db.get_message_by_name(
            'Vehicle_Info_2')  # Override_Feedback, Vehicle_Speed, Turn_Sig_Feed

        self.timestamp = 0

        self.feedback_info = {"APS_Feedback":0, "Break_ACT_Feedback":0,"Gear_Shift_Feedback":5,
                              "Steering_Angle_Feedback":0,"Override_Feedback":0, "Vehicle_Speed":0,"Turn_Sig_Feed":0}

        self.command_info = {'Accel_CMD':650,'Break_CMD':0,'Steering_CMD':0,'Gear_Shift_CMD':5,
                             'Override_Off':0,'Alive_Count':1,'Angular_Speed_CMD':30}

    def feedback(self):
        msg = 1
        while msg is not None:
            msg = self.bus.recv()
            data = self.db.decode_message(msg.arbitration_id, msg.data)
            self.timestamp = msg.timestamp

            for feedback_name in CAN_FEEDBACK_NAME:
                if feedback_name in data:
                    self.feedback_info[feedback_name] = data[feedback_name]

            yield self.timestamp, self.feedback_info['Vehicle_Speed'], self.feedback_info['APS_Feedback'], self.feedback_info['Break_ACT_Feedback']

    def command(self):
        cmd = self.calculate_cmd()

    def calculate_cmd(self):
        accel = self.feedback_info['APS_Feedback']
        brk = self.feedback_info['Break_ACT_Feedback']
        speed = self.feedback_info['Vehicle_Speed']
        return

rx_can = CAN()

fig, (ax1, ax2, ax3) = plt.subplots(3,1, sharex=True)  # ax1 : vehicle speed ax2 : APS_Feedback(accel) ax3 : Break_ACT_Feedback(break)

xs, y1, y2, y3 = [], [], [], []

ax1.ylim([0, 255])
ax2.ylim([0, 3800])
ax3.ylim([0, 35000])

def animate(i, xs, y1, y2, y3):
    t, v, a, b = next(rx_can.feedback())
    xs.append(t)
    y1.append(v)
    y2.append(a)
    y3.append(b)

    xs = xs[-1000:]
    y1 = y1[-1000:]
    y2 = y2[-1000:]
    y3 = y3[-1000:]

    ax1.clear()
    ax1.plot(xs, y1)
    ax2.clear()
    ax2.plot(xs, y2)
    ax3.clear()
    ax3.plot(xs,y3)

ani = FuncAnimation(fig, animate, fargs=(xs,y1,y2,y3), interval=20)
plt.show()


import asyncio
import sys
import duckdb
import pandas as pd
import botocore.exceptions
from aiobotocore.session import get_session
from datetime import datetime
from loguru import logger

class ingest:
    def __init__(self) -> None:
        self.queue_name = 'test-queue'
        self.endpoint_url = 'http://interview-localstack:4566'
        self.con = duckdb.connect(database='/home/app/duckdb_artifacts/bayzat')

    async def go(self):
        session = get_session()
        async with session.create_client('sqs', endpoint_url = self.endpoint_url) as client:
            try:
                response = await client.get_queue_url(QueueName=self.queue_name)
            except botocore.exceptions.ClientError as err:
                if (
                    err.response['Error']['Code'] == 'AWS.SimpleQueueService.NonExistentQueue'
                ):
                    logger.error(f"Queue {self.queue_name} does not exist")
                    sys.exit(1)
                else:
                    raise

            queue_url = response['QueueUrl']
            li=[]
            pd_df=pd.DataFrame()
            count = 0
            logger.info('Pulling messages off the queue...')
            start = datetime.now()
            while True:

                try:
                    # poll for messages from the queue at a 2 sec wait
                    response = await client.receive_message(
                        QueueUrl=queue_url,
                        WaitTimeSeconds=2,
                    )

                    if 'Messages' in response:
                        for msg in response['Messages']:
                            li.append(msg)
                            count += 1
                            # Need to remove msg from queue or else it'll reappear
                            await client.delete_message(
                                QueueUrl=queue_url,
                                ReceiptHandle=msg['ReceiptHandle'],
                            )
                            # Flushing messages to database at a 50k record count
                            if count == 50000:
                                pd_df = pd.DataFrame(li)
                                self.con.execute('insert into message_landing select *, current_timestamp from pd_df')
                                logger.info(f'Fetched and flushed {count} messages in {datetime.now()-start}')
                                count = 0
                                li=[]
                                pd_df=pd.DataFrame()
                                start = datetime.now()
                    else:
                        if li:
                            pd_df = pd.DataFrame(li)
                            self.con.execute('insert into message_landing select *, current_timestamp from pd_df')
                            logger.info(f'Fetched and flushed {count} messages in {datetime.now()-start}')
                            count = 0
                            li = []
                            pd_df = pd.DataFrame()
                        logger.info('No messages in queue')
                        break       # remove if more messages are expected
                except KeyboardInterrupt:
                    if li:
                            pd_df = pd.DataFrame(li)
                            self.con.execute('insert into message_landing select *, current_timestamp from pd_df')
                            logger.info(f'Flushed {count} messages...')
                    break

            logger.info('Finished')

    def create_table(self):
        logger.info('Creating landing table in duckdb...')
        self.con.execute("""create table if not exists message_landing (
            message_id varchar,
            receipt_handle varchar,
            md5_of_body varchar,
            body varchar,
            _insert_ts timestamptz
        )""")

if __name__ == '__main__':
    ing = ingest()
    ing.create_table()
    asyncio.run(ing.go())


# import libraries
import csv
import time
from urllib.request import urlopen
from bs4 import BeautifulSoup
from selenium import webdriver
from datetime import datetime

browser = webdriver.PhantomJS()

def scrap_article(url):
  # article_page = urlopen(url)
  # soup = BeautifulSoup(article_page, 'html.parser')
  browser.get(url)
  html = browser.page_source
  soup = BeautifulSoup(html, 'lxml')

  headline = soup.find('h1', attrs={'class': 'story-headline'}).get_text()
  published_date = soup.find('div', attrs={'class': 'story--published-date'}).get_text()
  keywords = soup.find('div', attrs={'class': 'story--keyword'}).get_text().upper()
  tag = soup.find('div', attrs={'class': 'story--web-category'}).get_text()

  # Standfirst
  standfirst = ''
  standfirst_tag = soup.find('h3', attrs={'class': 'standfirst'})
  if (standfirst_tag):
    standfirst = standfirst_tag.get_text()

  # Lead
  lead = ''
  lead_tag = soup.find('p', attrs={'class': 'lead'})
  if lead_tag:
    lead = lead_tag.get_text()

  # Image URL source
  img = soup.select('.group-media-frame > img')
  img_src = ''
  if img and img[0]:
    img_src = 'https://www.tnp.sg' + img[0]['src']

  # Engagements: Shares, Comments, Reactions
  engagements_tag = soup.find('li', attrs={'class': 'share-count'})
  engagement_count = engagements_tag.find('span', attrs={'class': 'share-count-figure'}).get_text()

  engagements_arr = engagements_tag['title'].split(' ')
  if (len(engagements_arr) > 7):
    shares = engagements_arr[0]
    comments = engagements_arr[3]
    reactions = engagements_arr[6]
  else:
    shares = 0
    comments = 0
    reactions = 0

  return [url, headline, published_date, standfirst, lead, keywords, tag, img_src, engagement_count, shares, comments, reactions]

start_time = time.time()

tnp = 'https://tnp.sg'
homepage = urlopen(tnp)
soup = BeautifulSoup(homepage, 'html.parser')
links = soup.select('.card-title > a')

with open('tnp_stories.csv', 'a', newline='') as csv_file:
  for index, link in enumerate(links):
    print(link['href'])
    # story = scrap_article(link['href'])
    story = scrap_article(link['href'])
    if index < 5:
      story.append('True')
    else:
      story.append('False')
    # append to csv file
    writer = csv.writer(csv_file)
    writer.writerow(story)

# idk = scrap_article('https://www.tnp.sg/news/singapore/taxi-passenger-who-alighted-ecp-cabby-appeared-dazed')
# print(idk)

elapsed_time = time.time() - start_time
print('\r\nElapsed time: ' + str(round(elapsed_time, 2)) + 's')


from pydriller import Repository
from github import Github
import json
from fastapi.encoders import jsonable_encoder
from pydantic import BaseModel
from typing import Optional

g = Github("ADD_YOUR_OWN_API_KEY")

# STEP 1: Go to https://github.com/ipfs/go-ipfs

# STEP 2: Decide on granularity of the project
# Files Go files for now as we can identify the modified files easily

# STEP 3:
#  List of all the Entities of the repository:
# repo = g.get_repo("ipfs/go-ipfs")
# contents = repo.get_contents("")
# while contents:
#     file_content = contents.pop(0)
#     if file_content.type == "dir":
#         contents.extend(repo.get_contents(file_content.path))
#     else:
#         print(file_content)

# Modified files refrence
#  https://pydriller.readthedocs.io/en/latest/modifiedfile.html

# STEP 4:
# Decide on the complexity of the project you want to measure
# Cyclomatic complexity has been implemented in pydriller it relies on Lizard: https://github.com/terryyin/lizard
# How is it used in pydriller: https://pydriller.readthedocs.io/en/latest/deltamaintainability.html
# It measures
#  - the number of functions
#  - the nloc (number of lines of code without comments)
# TODO:
#  - token count of the functions ??
#  - parameter count of the functions ??

# STEP 5:
# Timeframe
start_tag =  'v0.5.0'
end_tag = 'v0.10.0'
# Might tweak the timeframe to be more specific if needed
# Futher explanation will be needed to be added

# STEP 6:
# Measure the complexity of the project and number of changes
# Merge this information to identify the hot spots of the project

class Modified_Files(BaseModel):
    Files: dict = None

class File(BaseModel):
    file_name: str
    commits: Optional[list]

class Commit(BaseModel):
    commit_id: Optional[str]
    nloc: Optional[int]
    complexity: Optional[int]
    changed_methods: Optional[str]

commit_count = 0
number_of_changes = 0
# Empty list to store the changes

all_files = Modified_Files(Files={})

for commit in Repository('../go-ipfs', from_tag=start_tag, to_tag=end_tag).traverse_commits():
    commit_count += 1

    for m in commit.modified_files:
        if m.filename.endswith(".go"):
            number_of_changes += 1

            #  Check there isn't a file with the same name in the dictionary already
            if m.filename in all_files.Files:
                #  If there is, add the commit to the list of commits
                if m.complexity is not None or m.nloc is not None:
                    all_files.Files[m.filename].commits.append(Commit(commit_id=commit.hash, nloc=m.nloc, complexity=m.complexity))
            else:
                #  If not, create a new file with the commit

                all_files.Files[m.filename] = File(file_name=m.filename, commits=[Commit(commit_id=commit.hash, nloc=m.nloc, complexity=m.complexity)])

#  Export Files Dictionary to a json file

print_files = jsonable_encoder(all_files)

with open('files.json', 'w') as outfile:
    json.dump(print_files, outfile)

# Import Dictionary string from the json file

with open('files.json') as json_file:
    Files = json.load(json_file)
    modified_Files =Modified_Files(**Files)

# STEP 7: Visualization of the Hot Spots

# STEP 8: Analysis of 6 of the Hot Spots
# a) Complexity trend analysis
# b) Manaul analysis of the entity names and content


#%%
"""
Created on Jan 20 2019
Paths for the CIR process
@author: Lech A. Grzelak
"""
import numpy as np
import matplotlib.pyplot as plt
import scipy.stats as st
import scipy.integrate as integrate
from mpl_toolkits import mplot3d

def GeneratePathsCIREuler(NoOfPaths,NoOfSteps,T,kappa,v0,vbar,gamma):
    Z = np.random.normal(0.0,1.0,[NoOfPaths,NoOfSteps])
    W = np.zeros([NoOfPaths, NoOfSteps+1])
    V = np.zeros([NoOfPaths, NoOfSteps+1])
    V[:,0]=v0
    time = np.zeros([NoOfSteps+1])

    dt = T / float(NoOfSteps)
    for i in range(0,NoOfSteps):

        # Making sure that samples from a normal have mean 0 and variance 1

        if NoOfPaths > 1:
            Z[:,i] = (Z[:,i] - np.mean(Z[:,i])) / np.std(Z[:,i])
        W[:,i+1] = W[:,i] + np.power(dt, 0.5)*Z[:,i]
        V[:,i+1] = V[:,i] + kappa*(vbar - V[:,i]) * dt + gamma* np.sqrt(V[:,i]) * (W[:,i+1]-W[:,i])

        # We apply here the truncated scheme for negative values

        V[:,i+1] = np.maximum(V[:,i+1],0.0)
        time[i+1] = time[i] +dt

    # Outputs

    paths = {"time":time,"V":V}
    return paths

def CIRDensity(kappa,gamma,vbar,s,t,v_s):
    delta = 4.0 *kappa*vbar/gamma/gamma
    c= 1.0/(4.0*kappa)*gamma*gamma*(1.0-np.exp(-kappa*(t-s)))
    kappaBar = 4.0*kappa*v_s*np.exp(-kappa*(t-s))/(gamma*gamma*(1.0-np.exp(-kappa*(t-s))))
    ncx2PDF = lambda x : 1.0/c * st.ncx2.pdf(x/c,delta,kappaBar)
    return ncx2PDF

def CIRMean(kappa,gamma,vbar,v0,T):
    delta = 4.0 *kappa*vbar/gamma/gamma
    c= lambda s,t: 1.0/(4.0*kappa)*gamma*gamma*(1.0-np.exp(-kappa*(t-s)))
    kappaBar = lambda s,t,v_s: 4.0*kappa*v_s*np.exp(-kappa*(t-s))/(gamma*gamma*(1.0-np.exp(-kappa*(t-s))))
    return c(0,T)*(delta + kappaBar(0.0,T,v0))

def mainCalculation():
    NoOfPaths = 25
    NoOfSteps = 500
    T     = 5
    kappa =0.5
    v0    =0.1
    vbar  =0.1
    gamma =0.1

    Paths = GeneratePathsCIREuler(NoOfPaths,NoOfSteps,T,kappa,v0,vbar,gamma)
    timeGrid = Paths["time"]
    V = Paths["V"]

    plt.figure(1)
    plt.plot(timeGrid, np.transpose(V),'b')
    plt.grid()
    plt.xlabel("time")
    plt.ylabel("V(t)")

    # 3D graph for X(t) for paths vs. density

    plt.figure(2)
    ax = plt.axes(projection='3d')
    zline = np.zeros([len(timeGrid),1])

    # Plot paths

    n = 10
    for i in range(0,n,1):
        y1 = np.squeeze(np.transpose(V[i,:]))
        x1 = timeGrid
        z1 = np.squeeze(zline)
        ax.plot3D(x1, y1, z1, 'blue')
    ax.view_init(50, -170)

    Ti = np.linspace(0.5,T,5)

    y1 = np.linspace(0.001,0.4,100)
    for ti in Ti:

        # Density for the V(t) process

        ncx2PDF = CIRDensity(kappa,gamma,vbar,0.0,ti,v0)
        x1 = np.zeros([len(y1),1]) + ti
        z1 = ncx2PDF(y1)
        ax.plot3D(x1, y1, z1, 'red')

    # Compute numerical expectation and compare to analytical expression

    EV_num = integrate.trapz(y1*ncx2PDF(y1) ,y1)
    print("numerical: E[V(t=5)]={0}".format(EV_num))
    print("theoretical: E[V(t=5)]={0}".format(CIRMean(kappa,gamma,vbar,v0,T)))

mainCalculation()