import boto3
import sys
import os
import time
import re
import json
import csv
import random
import string
import utils

from collections import OrderedDict
from botocore.exceptions import ClientError
from six.moves import input as raw_input

def assume_role(aws_account_number, role_name):
    """
    Assumes the provided role in each account and returns a SecurityHub client
    :param aws_account_number: AWS Account Number
    :param role_name: Role to assume in target account
    :param aws_region: AWS Region for the Client call, not required for IAM calls
    :return: SecurityHub client in the specified AWS Account and Region
    """

    # Beginning the assume role process for account
    sts_client = boto3.client('sts')

    # Get the current partition
    partition = sts_client.get_caller_identity()['Arn'].split(":")[1]

    response = sts_client.assume_role(
        RoleArn='arn:{}:iam::{}:role/{}'.format(
            partition,
            aws_account_number,
            role_name
        ),
        RoleSessionName='EnableSecurityHub'
    )

    # Storing STS credentials
    session = boto3.Session(
        aws_access_key_id=response['Credentials']['AccessKeyId'],
        aws_secret_access_key=response['Credentials']['SecretAccessKey'],
        aws_session_token=response['Credentials']['SessionToken']
    )

    print("Assumed session for {}.".format(
        aws_account_number
    ))

    return session

def get_master_members(sh_client, aws_region):
    """
    Returns a list of current members of the SecurityHub master account
    :param aws_region: AWS Region of the SecurityHub master account
    :param detector_id: DetectorId of the SecurityHub master account in the AWS Region
    :return: dict of AwsAccountId:RelationshipStatus
    """

    member_dict = dict()

    results = sh_client.list_members(
        OnlyAssociated=False
    )

    for member in results['Members']:
        member_dict.update({member['AccountId']: member['MemberStatus']})

    while results.get("NextToken"):
        results = sh_client.list_members(
            OnlyAssociated=False,
            NextToken=results['NextToken']
        )

        for member in results['Members']:
            member_dict.update({member['AccountId']: member['MemberStatus']})

    return member_dict

def check_config(session,account, region, s3_bucket_name):
    config = session.client('config', region_name=region)
    iam = session.client('iam')
    s3 = session.client('s3', region_name='eu-west-1')
    default_bucket_avail = False
    default_bucket_exists = False
    try:
        iam.create_service_linked_role(AWSServiceName='config.amazonaws.com', Description='A service-linked role required for AWS Config')
    except ClientError as e:
        if e.response['ResponseMetadata']['HTTPStatusCode'] == 400:
            pass # SLR already exists
        else:
            print(e)
            return False
    # Check if default bucket name is available.
    try:
        s3.list_objects(Bucket='config-bucket-{}'.format(account), MaxKeys=1)
        default_bucket_exists = True
        s3_bucket_name = 'config-bucket-{}'.format(account)
    except ClientError as e:
        if e.response['ResponseMetadata']['HTTPStatusCode'] == 404:
            default_bucket_avail = True
            s3_bucket_name = 'config-bucket-{}'.format(account)
        pass
    if not len(config.describe_configuration_recorders()['ConfigurationRecorders']):
        config.put_configuration_recorder( ConfigurationRecorder={'name':'default','roleARN': 'arn:aws:iam::%s:role/aws-service-role/config.amazonaws.com/AWSServiceRoleForConfig' % account,'recordingGroup': {'allSupported' : True, 'includeGlobalResourceTypes': True}})

    if config.describe_configuration_recorder_status()['ConfigurationRecordersStatus'][0]['recording']:
        return True #config is configured and enabled nothing to do here.
    if len(config.describe_delivery_channels()['DeliveryChannels']):
        try:
            config.start_configuration_recorder(ConfigurationRecorderName=config.describe_configuration_recorder_status()['ConfigurationRecordersStatus'][0]['name'])
            return True
        except ClientError as e:
            print("Error {} starting configuration recorder for account {} in region {}".format(repr(e), account, region))
            return False
    ## Ensure S3 bucket for AWS Config delivery exists
    if default_bucket_avail and not default_bucket_exists:
        try:
            s3.create_bucket(Bucket=s3_bucket_name)
            bucket_policy = {
                "Version": "2012-10-17",
                "Statement": [
                    {
                        "Sid": "AWSConfigBucketPermissionsCheck",
                        "Effect": "Allow",
                        "Principal": {"Service": ["config.amazonaws.com"]},
                        "Action": "s3:GetBucketAcl",
                        "Resource": "arn:aws:s3:::%s" % s3_bucket_name},
                    {
                        "Sid": " AWSConfigBucketDelivery",
                        "Effect": "Allow",
                        "Principal": {"Service": ["config.amazonaws.com"]},
                        "Action": "s3:PutObject",
                        "Resource": "arn:aws:s3:::%s/AWSLogs/%s/Config/*" % (s3_bucket_name, account),
                        "Condition": { "StringEquals": { "s3:x-amz-acl": "bucket-owner-full-control" }}
                    }]
            }
            bucket_policy = json.dumps(bucket_policy)
            s3.put_bucket_policy(Bucket=s3_bucket_name, Policy=bucket_policy)
        except ClientError as e:
            print("Error {} checking bucket for Config delivery in account {}".format(repr(e), account))
            return False
    try:
        config.put_delivery_channel(DeliveryChannel={
            'name': 'config-s3-delivery',
            's3BucketName': s3_bucket_name,
            'configSnapshotDeliveryProperties': {'deliveryFrequency': 'TwentyFour_Hours' }
            })
        config.start_configuration_recorder(ConfigurationRecorderName=config.describe_configuration_recorder_status()['ConfigurationRecordersStatus'][0]['name'])
        return True
    except ClientError as e:
        print("Error {} enabling Config on account {}".format(repr(e), account))
        return False
    return False

def lambda_handler(event, context):

    ###Getting parameters
    master_account = os.environ["master_account"]
    role_name = os.environ["role_name"]
    enabled_regions = os.environ["enabled_regions"]
    enable_standards = os.environ["enable_standards"]
    bucket = 'securityhub-members'
    key = 'enable.csv'

    # Validate master accountId
    if not re.match(r'[0-9]{12}',master_account):
        raise ValueError("Master AccountId is not valid")

    # Generate dict with account & email information
    aws_account_dict = OrderedDict()

    s3_resource = boto3.resource('s3')
    s3_object = s3_resource.Object(bucket, key)

    data = s3_object.get()['Body'].read().decode('utf-8').splitlines()

    lines = csv.reader(data)
    headers = next(lines)
    print('headers: %s' %(headers))

    for line in lines:
      print(line)
      if not re.match(r'[0-9]{12}', str(line[0])):
          print("Invalid account number {}, skipping".format(line[0]))
          continue

      aws_account_dict[line[0]] = line[1]

    # Check length of accounts to be processed
    if len(aws_account_dict.keys()) > 1000:
        raise Exception("Only 1000 accounts can be linked to a single master account")

    # Getting SecurityHub regions
    session = boto3.session.Session()

    securityhub_regions = []
    if enabled_regions:
        securityhub_regions = [str(item) for item in enabled_regions.split(',')]
        print("Enabling members in these regions: {}".format(securityhub_regions))
    else:
        securityhub_regions = session.get_available_regions('securityhub')
        print("Enabling members in all available SecurityHub regions {}".format(securityhub_regions))

    # Check if enable Standards
    standards_arns = []
    if enable_standards:
        standards_arns = [str(item) for item in enable_standards.split(',')]
        print("Enabling the following Security Hub Standards for enabled account(s) and region(s): {}".format(standards_arns))

    # Processing Master account
    master_session = assume_role(master_account, role_name)
    #master_session = boto3.Session()
    master_clients = {}
    members = {}
    for aws_region in securityhub_regions:
        master_clients[aws_region] = master_session.client('securityhub', region_name=aws_region)
        try:
            # Enable Security Hub for the Master Account
            master_clients[aws_region].enable_security_hub()

            # Enable compliance Standards for Master account
            compliance_standards_arns = [utils.get_standard_arn_for_region_and_resource(aws_region, standard) for standard in standards_arns]
            batch_enable_standards_input = [{'StandardsArn': standard_arn} for standard_arn in compliance_standards_arns]
            master_clients[aws_region].batch_enable_standards(StandardsSubscriptionRequests=batch_enable_standards_input)

            # Verify standards get enabled in the Master Account
            standards_to_verify = compliance_standards_arns[:]
            standards_status = {}
            start_time = int(time.time())
            while len(standards_to_verify) > 0:
                if (int(time.time()) - start_time) > 100:
                    print("Timeout waiting for READY state enabling standards {standards} in region {region} for account {account}, last state: {status}"
                          .format(standards=compliance_standards_arns, region=aws_region, account=master_account, status=standards_status))
                    break
                enabled_standards = master_clients[aws_region].get_enabled_standards()
                for enabled_standard in enabled_standards['StandardsSubscriptions']:
                    enabled_standard_arn = enabled_standard['StandardsArn']
                    enabled_standard_status = enabled_standard['StandardsStatus']
                    standards_status[enabled_standard_arn] = enabled_standard_status
                    if enabled_standard_arn in standards_to_verify and enabled_standard_status == 'READY':
                        print("Finished enabling stanard {} on account {} for region {}".format(enabled_standard_arn, master_account, aws_region))
                        standards_to_verify.remove(enabled_standard_arn)

        except ClientError as e:
            if e.response['Error']['Code'] == 'ResourceConflictException':
                pass
            else:
                print("Error: Unable to enable Security Hub on Master account in region {}".format(aws_region))
                raise SystemExit(0)

        members[aws_region] = get_master_members(master_clients[aws_region], aws_region)

    # Processing accounts to be linked
    failed_accounts = []
    for account in aws_account_dict.keys():
        if account == master_account:
            print("Won't try to link master account %s to itself" % account)

        try:

            session = assume_role(account, role_name)
            # Generate unique bucket name for Config delivery channel if default is not avaialable.
            s3_bucket_name = 'config-bucket-{}-{}'.format(''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(5)), account)

            for aws_region in securityhub_regions:
                print('Beginning {account} in {region}'.format(
                    account=account,
                    region=aws_region
                ))

                sh_client = session.client('securityhub', region_name=aws_region)
                #Ensure AWS Config is enabled for the account/region and enable if it not already enabled.
                config_result = check_config(session, account, aws_region, s3_bucket_name)
                print('config_bucket:%s' % s3_bucket_name)
                if not config_result:
                    failed_accounts.append({account: "Error validating or enabling AWS Config for account {} in {} - requested standards not enabled".format(account,aws_region)})
                else:
                    try:
                        sh_client.enable_security_hub()
                    except ClientError as e:
                        if e.response['Error']['Code'] == 'ResourceConflictException':
                            pass

                    if enable_standards:
                        regional_standards_arns = [utils.get_standard_arn_for_region_and_resource(aws_region, standard) for standard in standards_arns]
                        batch_enable_standards_input = [{'StandardsArn': standard_arn} for standard_arn in regional_standards_arns]
                        sh_client.batch_enable_standards(StandardsSubscriptionRequests=batch_enable_standards_input)

                        # Verify standards get enabled
                        standards_to_verify = regional_standards_arns[:]
                        standards_status = {}
                        start_time = int(time.time())
                        while len(standards_to_verify) > 0:
                            if (int(time.time()) - start_time) > 100:
                                print("Timeout waiting for READY state enabling standards {standards} in region {region} for account {account}, last state: {status}"
                                      .format(standards=regional_standards_arns, region=aws_region, account=account, status=standards_status))
                                break
                            enabled_standards = sh_client.get_enabled_standards()
                            for enabled_standard in enabled_standards['StandardsSubscriptions']:
                                enabled_standard_arn = enabled_standard['StandardsArn']
                                enabled_standard_status = enabled_standard['StandardsStatus']
                                standards_status[enabled_standard_arn] = enabled_standard_status
                                if enabled_standard_arn in standards_to_verify and enabled_standard_status == 'READY':
                                    print("Finished enabling stanard {} on account {} for region {}".format(enabled_standard_arn,account, aws_region))
                                    standards_to_verify.remove(enabled_standard_arn)

                if account in members[aws_region]:
                    print('Account {monitored} is already a member of {master} in region {region}'.format(
                        monitored=account,
                        master=master_account,
                        region=aws_region
                    ))
                else:
                    master_clients[aws_region].create_members(
                        AccountDetails=[{
                            "AccountId": account,
                            "Email": aws_account_dict[account]
                        }]
                    )

                    print('Added Account {monitored} to member list in SecurityHub master account {master} for region {region}'.format(
                        monitored=account,
                        master=master_account,
                        region=aws_region
                    ))

                    start_time = int(time.time())
                    while account not in members[aws_region]:
                        if (int(time.time()) - start_time) > 300:
                            print("Membership did not show up for account {}, skipping".format(account))
                            failed_accounts.append({
                                account: "Membership did not show up for account {} in {}".format(
                                    account,
                                    aws_region
                                )
                            })
                            break

                        time.sleep(5)
                        members[aws_region] = get_master_members(master_clients[aws_region], aws_region)

                if account not in members[aws_region]:
                    print("Account {} could not be joined, skipping".format(account))
                    continue

                if members[aws_region][account] == 'Associated' or members[aws_region][account] == 'Enabled':
                    # Member is enabled and already being monitored
                    print('Account {account} is already enabled'.format(account=account))

                else:
                    start_time = int(time.time())
                    while members[aws_region][account] != 'Associated' and members[aws_region][account] != 'Enabled':
                        if (int(time.time()) - start_time) > 300:
                            print("Invitation did not show up for account {}, skipping".format(account))
                            failed_accounts.append({
                                account: "Membership did not show up for account {} in {}".format(
                                    account,
                                    aws_region
                                )
                            })
                            break

                        if members[aws_region][account] == 'Created':
                            # Member has been created in the SecurityHub master account but not invited yet
                            master_clients[aws_region].invite_members(
                                AccountIds=[account]
                            )

                            print('Invited Account {monitored} to SecurityHub master account {master} in region {region}'.format(
                                monitored=account,
                                master=master_account,
                                region=aws_region
                            ))

                        if members[aws_region][account] == 'Invited':
                            # member has been invited so accept the invite

                            response = sh_client.list_invitations()

                            invitation_dict = dict()

                            invitation_id = None
                            for invitation in response['Invitations']:
                                invitation_id = invitation['InvitationId']

                            if invitation_id is not None:
                                sh_client.accept_invitation(
                                    InvitationId=invitation_id,
                                    MasterId=str(master_account)
                                )
                                print('Accepting Account {monitored} to SecurityHub master account {master} in region {region}'.format(
                                    monitored=account,
                                    master=master_account,
                                    region=aws_region
                                ))

                        # Refresh the member dictionary
                        members[aws_region] = get_master_members(master_clients[aws_region], aws_region)

                    print('Finished {account} in {region}'.format(account=account, region=aws_region))

        except ClientError as e:
            print("Error Processing Account {}".format(account))
            failed_accounts.append({
                account: repr(e)
            })

    if len(failed_accounts) > 0:
        print("---------------------------------------------------------------")
        print("Failed Accounts")
        print("---------------------------------------------------------------")
        for account in failed_accounts:
            for account_id, message in account.items():
                print("{}: \n\t{}".format(account_id, message))
        print("---------------------------------------------------------------")


import pymysql

# 打开数据库连接
localdb = pymysql.connect("localhost", "root", "werered", "test")
# webdb = pymysql.connect("localhost", "wangsheng", "cmcc1234", "littlec_user", 33601)
cursor = localdb.cursor()
# webCursor = webdb.cursor()
ONE_STEP = 50000
TOTAL = 1200000
#sql = "SELECT * FROM user_view_info \
#       WHERE event_id = %s" % (eventId)
insertSQL = "insert into t_department(corp_id, count) value(%s,%d)"
insertDepartment = "insert into corp_user_size(corp_id, user_size) value(%s,%s)"
updateUserSize  = "UPDATE test_crop set user_size = %s WHERE corp_id = %s"
try:
    begin = 0
    cursor.execute("SELECT corp_id from test_crop")
    result = cursor.fetchall()
    print("size = ",len(result))
    for corpID in result:
        begin = begin + 1
        cursor.execute("SELECT  user_size from t_department \
        WHERE  `enable` = 1 and corp_id = %s" % corpID)
        corpUserSize = cursor.fetchall()
        if corpUserSize:
            userCount = 0
            for userSize in corpUserSize:
                userCount += userSize[0];
            cursor.execute(updateUserSize, (userCount,corpID))
        if begin % 1000 == 0:
            begin = 0
            print("one setp")

    # 执行sql语句
    localdb.commit()
        # print(end)
        # begin += ONE_STEP
        # end   += ONE_STEP
except Exception as e:
        # 发生错误时回滚
    localdb.rollback()
    print(e)

localdb.close()
# webdb.close()


import sys, os, django
from django.conf import settings
sys.path.append('/home/hu4/lna')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
django.setup()
import csv
from net.models import Equipment
from argus.models import ASTU

def generate_noc_csv():
    all_equipment = Equipment.objects.filter(hostname__isnull=False)

    with open('noc-import.csv', 'w', newline='') as csvfile:
        fieldnames = ['name', 'is_managed', 'container', 'administrative_domain', 'segment', 'pool', 'profile',
                      'vendor', 'platform', 'version', 'next_version', 'object_profile', 'description', 'auth_profile',
                      'scheme', 'address', 'port', 'user', 'password', 'super_password', 'remote_path',
                      'trap_source_type', 'trap_source_ip', 'syslog_source_type', 'syslog_source_ip', 'trap_community',
                      'snmp_ro', 'snmp_rw', 'access_preference', 'vc_domain', 'vrf', 'controller', 'local_cpe_id',
                      'global_cpe_id', 'last_seen', 'termination_group', 'service_terminator', 'shape', 'time_pattern',
                      'config_filter_handler', 'config_diff_filter_handler', 'config_validation_handler', 'max_scripts',
                      'x', 'y', 'default_zoom', 'software_image', 'remote_system', 'remote_id', 'escalation_policy',
                      'box_discovery_alarm_policy', 'periodic_discovery_alarm_policy', 'box_discovery_telemetry_policy',
                      'box_discovery_telemetry_sample', 'periodic_discovery_telemetry_policy',
                      'periodic_discovery_telemetry_sample', 'tt_system', 'tt_queue', 'tt_system_id',
                      'cli_session_policy', 'cli_privilege_policy', 'autosegmentation_policy',
                      'event_processing_policy', 'tags']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()

        counter = 0

        for eq in all_equipment:
            rowdict = get_row_dict_from(eq)
            writer.writerow(rowdict=rowdict)
            counter += 1
            # if counter >=50:
            #     return True

        print(f"total {counter} in generated csv")

def get_row_dict_from(eq):
    row = {}
    actual_ip = str(eq.ne_ip).replace('/32', '')
    row['name'] = eq.hostname
    row['is_managed'] = 'True'
    row['administrative_domain'] = 'default'
    row['segment'] = 'ALL'
    row['pool'] = 'default'
    row['profile'] = get_sa_profile_from(eq)
    row['object_profile'] = get_object_profile_from(eq)
    row['description'] = get_description(actual_ip)
    row['auth_profile'] = 'all-suggest'
    row['scheme'] = 1
    row['address'] = actual_ip
    row['port'] = 0
    row['user'] = ''
    row['password'] = ''
    row['trap_source_type'] = 'a'
    row['syslog_source_type'] = 'a'
    row['trap_community'] = 'nocnbr'
    row['access_preference'] = 'P'
    row['shape'] = 'Cisco/workgroup_switch'
    row['max_scripts'] = 0
    row['escalation_policy'] = 'P'
    row['box_discovery_alarm_policy'] = 'P'
    row['periodic_discovery_alarm_policy'] = 'P'
    row['box_discovery_telemetry_policy'] = 'P'
    row['box_discovery_telemetry_sample'] = 0
    row['periodic_discovery_telemetry_policy'] = 'P'
    row['periodic_discovery_telemetry_sample'] = 0
    row['cli_session_policy'] = 'P'
    row['cli_privilege_policy'] = 'P'
    row['autosegmentation_policy'] = 'p'  # yes, small letter `p`!
    row['event_processing_policy'] = 'P'
    row['tags'] = get_tags_for(eq)

    return row

def get_sa_profile_from(eq):
    """
    Returns managed object SA profile from NOC
    :param eq:
    :return:
    """
    if eq.vendor == 'Alcatel':
        return 'Alcatel.7302'
    if eq.vendor == 'Cisco':
        return 'Cisco.IOS'
    if eq.vendor == 'Eltex':
        return 'Eltex.MES'
    if eq.vendor == 'Huawei':
        return 'Huawei.VRP'
    if eq.vendor == 'Huawei OLT':
        return 'Huawei.MA5600T'
    if eq.vendor == 'Juniper':
        return 'Juniper.JUNOS'
    if eq.vendor == 'SNR':
        return 'NAG.SNR'
    if eq.vendor == 'Zyxel':
        return 'Zyxel.DSLAM'  # Need to check
    print(f"Vendor / Profile not found for {eq.vendor}")
    return None

def get_object_profile_from(eq):
    """
    Returns managed object profile
    :param eq:
    :return:
    """

    router_model_list = ['Cisco 7206VXR', 'cisco ASR1001', 'Cisco C10008', 'mx480', 'mx80-p']

    if eq.model in router_model_list:
        return 'generic_router'
    return 'generic_switch'

def get_description(ip):
    """
    Location from ASTU
    :param ip:
    :return:
    """
    actual_ip = str(ip)
    try:
        astu_obj = ASTU.objects.get(ne_ip=actual_ip)
    except ASTU.DoesNotExist:
        return ''
    return astu_obj.address

def get_tags_for(eq):
    """
    Returns various tags for MO from EQ specifics
    :param eq:
    :return:
    """
    tags = eq.vendor.replace(' ', '')  # without spaces
    try:
        if len(eq.model) < 18:
            tags += ',' + eq.vendor
    except TypeError:  # model can be None
        pass

    # try:
    #     astu_obj = ASTU.objects.get(ne_ip=eq.ne_ip)
    # except ASTU.DoesNotExist:
    #     return tags
    #
    # tags += ',' + astu_obj.segment

    return tags

if __name__ == '__main__':
    generate_noc_csv()


from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle
from reportlab.lib.pagesizes import letter
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
from io import BytesIO
import pandas as pd

def create_pdf(dataframe, filename, chat_log, selected_data):
    # Crie um buffer de bytes para armazenar o PDF
    buffer = BytesIO()

    # Crie um documento PDF com o ReportLab
    doc = SimpleDocTemplate(buffer, pagesize=letter)

    # Adicione os dados selecionados ao PDF
    elements = []

    styles = getSampleStyleSheet()

    # Adicione os dados da tabela
    data = [dataframe.columns.tolist()] + dataframe.values.tolist()
    table = Table(data)

    centered_title_style = ParagraphStyle(
        'CenteredTitle',
        parent=styles['Heading1'],
        alignment=1,  # 1 significa centralizado
    )

    # Adicione "RELATÓRIO FORD" centralizado
    title = Paragraph("RELATÓRIO FORD", centered_title_style)
    elements.append(title)
    elements.append(Spacer(1, 20))

    # Estilize a tabela
    style = TableStyle([('BACKGROUND', (0, 0), (-1, 0), (0.8, 0.8, 0.8)),
                        ('TEXTCOLOR', (0, 0), (-1, 0), (0, 0, 0)),
                        ('ALIGN', (0, 0), (-1, -1), 'CENTER'),
                        ('FONTNAME', (0, 0), (-1, 0), 'Helvetica-Bold'),
                        ('BOTTOMPADDING', (0, 0), (-1, 0), 12),
                        ('BACKGROUND', (0, 1), (-1, -1), (0.95, 0.95, 0.95)),
                        ('GRID', (0, 0), (-1, -1), 1, (0, 0, 0)),
                        ('GRID', (0, 0), (-1, 0), 2, (0, 0, 0))])

    table.setStyle(style)
    elements.append(table)

    # elements.append(Paragraph("Dados Selecionados:", styles['Normal']))
    # for entry in selected_data:
    #     elements.append(Paragraph(entry, styles['Normal']))

    elements.append(Spacer(1, 20))
    # Adicione o log de perguntas e respostas do chat
    elements.append(Paragraph("Chat Log:", styles['Heading1']))
    if chat_log is not None:
        for entry in chat_log:
            question, answer = entry
            elements.append(Paragraph(f"Pergunta: {question}", styles['Normal']))
            elements.append(Spacer(1, 10))
            elements.append(Paragraph(f"Resposta: {answer}", styles['Normal']))
            elements.append(Spacer(1, 30))
    else:
        elements.append(Paragraph("Nenhuma pergunta foi feita", styles['Normal']))

    # Construa o PDF
    doc.build(elements)

    # Salve o PDF no arquivo especificado
    with open(filename, 'wb') as f:
        f.write(buffer.getvalue())


import numba
import numpy as np

from covid_model_seiir_pipeline.lib.ode_mk2.constants import (
    # Indexing tuples
    COMPARTMENT,
    TRACKING_COMPARTMENT,
    TRACKING_COMPARTMENT_TYPE,
    VARIANT,
    VARIANT_GROUP,
    VACCINE_STATUS,
    EPI_MEASURE,
    AGG_INDEX_TYPE,
    # Indexing arrays
    NEW_E,
    RATES,
    EFFECTIVE_SUSCEPTIBLE,
    COMPARTMENTS,
    TRACKING_COMPARTMENTS,
    # Mappings
    BETA_MAP,
    VACCINE_STATUS_MAP,
    EPI_MEASURE_MAP,
    VACCINE_COUNT_MAP,
)
from covid_model_seiir_pipeline.lib.ode_mk2.debug import (
    DEBUG,
    Printer
)
from covid_model_seiir_pipeline.lib.ode_mk2.utils import (
    cartesian_product,
)

@numba.njit
def compute_tracking_compartments(t: float,
                                  dy: np.ndarray,
                                  new_e: np.ndarray,
                                  effective_susceptible: np.ndarray,
                                  beta: np.ndarray,
                                  rates: np.ndarray,
                                  transition_map: np.ndarray) -> np.ndarray:
    for agg_idx, epi_idx in BETA_MAP:
        beta_idx = TRACKING_COMPARTMENTS[TRACKING_COMPARTMENT_TYPE.Beta, VARIANT.none, VARIANT.none, agg_idx]
        dy[beta_idx] = beta[epi_idx]

    for tc, epi_measure in EPI_MEASURE_MAP:
        for variant_from, variant_to in cartesian_product((np.array(VARIANT), np.array(VARIANT))):
            for agg_vaccine_status, vaccine_status in VACCINE_STATUS_MAP:
                infections = new_e[NEW_E[vaccine_status, variant_from, variant_to]]
                if epi_measure == EPI_MEASURE.infection:
                    rate = 1.0
                else:
                    rate = rates[RATES[vaccine_status, variant_from, variant_to, epi_measure]]
                measure = infections * rate
                if variant_from == VARIANT.none:
                    dy[TRACKING_COMPARTMENTS[tc, VARIANT.none, VARIANT_GROUP.all, AGG_INDEX_TYPE.all]] += measure
                    if vaccine_status == VACCINE_STATUS.course_0:
                        dy[TRACKING_COMPARTMENTS[tc, variant_from, VARIANT_GROUP.all, agg_vaccine_status]] += measure
                        dy[TRACKING_COMPARTMENTS[tc, variant_from, variant_to, agg_vaccine_status]] += measure
                dy[TRACKING_COMPARTMENTS[tc, VARIANT_GROUP.all, VARIANT_GROUP.all, AGG_INDEX_TYPE.all]] += measure
                dy[TRACKING_COMPARTMENTS[tc, VARIANT_GROUP.all, VARIANT_GROUP.all, agg_vaccine_status]] += measure
                dy[TRACKING_COMPARTMENTS[tc, VARIANT_GROUP.all, variant_to, AGG_INDEX_TYPE.all]] += measure

    tc_effs = TRACKING_COMPARTMENT.EffectiveSusceptible
    for variant in VARIANT:
        for tc, agg_idx, from_vax, to_vax in VACCINE_COUNT_MAP:
            dy[TRACKING_COMPARTMENTS[tc, VARIANT_GROUP.all, VARIANT_GROUP.all, agg_idx]] += (
                transition_map[COMPARTMENTS[COMPARTMENT.S, from_vax, variant],
                               COMPARTMENTS[COMPARTMENT.S, to_vax, variant]]
            )

        for variant_from, vaccine_status in cartesian_product((np.array(VARIANT), np.array(VACCINE_STATUS))):
            dy[TRACKING_COMPARTMENTS[tc_effs, VARIANT_GROUP.all, variant, AGG_INDEX_TYPE.all]] += (
                effective_susceptible[EFFECTIVE_SUSCEPTIBLE[vaccine_status, variant_from, variant]]
            )

    if DEBUG:
        assert np.all(np.isfinite(dy))
    return dy


import json
import random
import os
from collections import Counter
from glob import glob

from itertools import repeat
from multiprocessing.pool import Pool

from exprimo import get_log_dir
from exprimo.optimizers.base import BaseOptimizer
from exprimo.optimizers.utils import evaluate_placement, apply_placement, generate_random_placement, flatten, \
    get_device_assignment
from exprimo.graph import get_flattened_layer_names, ComputationGraph

import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
from exprimo import PLOT_STYLE, log
import seaborn as sns

from exprimo.plotting import plot_map_elites_archive, plot_archive_animation

sns.set(style=PLOT_STYLE)

def _evaluate(individual, net_string, groups, device_graph, pipeline_batches=1, batches=1,
              simulator_comp_penalty=1, simulator_comm_penalty=1,
              device_memory_utilization=1):
    description, individual = individual

    comp_graph_dict = apply_placement(net_string, individual, groups)

    score = 1 / evaluate_placement(comp_graph_dict, device_graph,
                                   pipeline_batches=pipeline_batches, batches=batches,
                                   comp_penalty=simulator_comp_penalty, comm_penalty=simulator_comm_penalty,
                                   device_memory_utilization=device_memory_utilization)

    return score, description, individual

class MapElitesOptimizer(BaseOptimizer):

    def __init__(self, dimension_sizes=(-1, -1, 10), initial_size=50,
                 steps=1000, allow_cpu=True, mutation_rate=0.05, copy_mutation_rate=0, replace_mutation_rate=0,
                 zone_mutation_rate=0, crossover_rate=0.4, selection='random', tournament_size=10,
                 benchmarking_function=None, benchmarking_steps=0, benchmark_before_selection=False,
                 benchmarking_n_keep=None, benchmarking_time_threshold=None, include_trivial_solutions=True,
                 show_score_plot=False, plot_axes=(0, 2), plot_animation=False, animation_fps=1,
                 archive_log_period=None, **kwargs):
        super().__init__(**kwargs)
        self.dimension_sizes = dimension_sizes
        self.initial_size = initial_size
        self.steps = steps
        self.allow_cpu = allow_cpu
        self.mutation_rate = mutation_rate
        self.copy_mutation_rate = copy_mutation_rate
        self.replace_mutation_rate = replace_mutation_rate
        self.zone_mutation_rate = zone_mutation_rate
        self.crossover_rate = crossover_rate
        self.selection = selection
        self.tournament_size = tournament_size
        self.include_trivial_solutions = include_trivial_solutions
        self.benchmarking_steps = benchmarking_steps
        self.benchmark_before_selection = benchmark_before_selection
        self.benchmarking_n_keep = benchmarking_n_keep
        self.benchmarking_time_threshold = benchmarking_time_threshold
        self.benchmarking_function = benchmarking_function
        self.plot_axes = plot_axes
        self.show_score_plot = show_score_plot
        self.plot_animation = plot_animation
        self.animation_fps = animation_fps
        self.archive_log_period = archive_log_period

        if self.archive_log_period is not None:
            if not os.path.exists(os.path.join(get_log_dir(), 'archive_logs')):
                os.makedirs(os.path.join(get_log_dir(), 'archive_logs'))

        self.worker_pool = None

    def optimize(self, net_string, device_graph, return_full_archive=False):

        if self.n_threads > 1:
            self.worker_pool = Pool(self.n_threads)

        n_devices = len(device_graph.devices)
        groups = self.create_colocation_groups(get_flattened_layer_names(net_string))

        if self.dimension_sizes[0] == -1:
            self.dimension_sizes[0] = n_devices

        if self.dimension_sizes[1] == -1:
            self.dimension_sizes[1] = n_devices

        if self.dimension_sizes[2] == -1:
            comp_graph = ComputationGraph()
            comp_graph.load_from_string(net_string)
            _, max_jumps = comp_graph.get_number_of_jumps(return_max_jumps=True)
            self.dimension_sizes[2] = max_jumps

        archive_scores = np.empty(self.dimension_sizes)
        archive_scores[:] = np.NaN
        archive_individuals = np.zeros(list(self.dimension_sizes) + [len(groups)], dtype=int)

        def evaluate(individual):
            return _evaluate(individual, net_string, groups, device_graph, self.dimension_sizes, self.pipeline_batches,
                             self.batches, self.simulator_comp_penalty, self.simulator_comm_penalty,
                             self.device_memory_utilization)

        def mutate(individual):
            new_individual = []
            if random.random() < self.replace_mutation_rate:
                devices_present = list(set(individual))
                i1 = random.choice(devices_present)
                i2 = random.choice(devices_present)

                new_individual = [i2 if i == i1 else i for i in individual]
            elif random.random() < self.zone_mutation_rate:
                split1 = random.randint(0, len(individual) - 1)
                split2 = split1 + min(np.random.geometric(0.2), len(individual) - split1)
                dev = random.randint(0 if self.allow_cpu else 1, n_devices - 1)
                new_individual = individual[:split1] + [dev] * (split2 - split1) + individual[split2:]
            else:
                for i, gene in enumerate(individual):
                    if random.random() < self.copy_mutation_rate and i > 0:
                        new_individual.append(individual[i - 1])
                    elif random.random() < self.mutation_rate:
                        if self.allow_cpu:
                            new_individual.append(random.randint(0, n_devices - 1))
                        else:
                            new_individual.append(random.randint(1, n_devices - 1))
                    else:
                        new_individual.append(gene)

            return new_individual

        def crossover(parent1, parent2):
            crossover_point = random.randint(1, len(parent1) - 1)
            return parent1[:crossover_point] + parent2[crossover_point:]

        def create_candidates(n, create_random=False, create_trivial=False, selectable_candidates=None):
            if n <= 0:
                return []
            candidates = []
            if create_trivial:
                candidates.extend([
                    [i] * len(groups) for i in range(1, n_devices)
                ])
                n -= n_devices - 1

                if self.allow_cpu:
                    candidates.append([0] * len(groups))
                    n -= 1

            if create_random:
                while len(candidates) < n:
                    candidates.append(generate_random_placement(len(groups), n_devices, allow_device_0=self.allow_cpu))
            else:
                selectable_indices = np.argwhere(np.isfinite(archive_scores))
                # selectable_indices = sorted(selectable_indices, key=lambda x: -archive_scores[x[0], x[1], x[2]])
                while len(candidates) < n:
                    c = []
                    if selectable_candidates:
                        for _ in range(1 + int(random.random() < self.crossover_rate)):
                            c.append(random.choice(selectable_candidates))
                    else:
                        if self.selection == 'random':
                            for _ in range(1 + int(random.random() < self.crossover_rate)):
                                idx = random.choice(selectable_indices)
                                c.append(archive_individuals[idx[0], idx[1], idx[2], :].tolist())
                        elif self.selection == 'tournament':
                            idx = []
                            t = min(self.tournament_size, len(selectable_indices))
                            while len(idx) < 1 + int(random.random() < self.crossover_rate):
                                competitors = random.sample(selectable_indices.tolist(), t)
                                winner = max(competitors, key=lambda x: archive_scores[x[0], x[1], x[2]])
                                idx.append(winner)
                            for i in idx:
                                c.append(archive_individuals[i[0], i[1], i[2], :].tolist())

                    if len(c) == 2:
                        candidate = crossover(*c)
                    else:
                        candidate = c[0]
                    candidate = mutate(candidate)
                    candidates.append(candidate)

            return candidates

        def create_description(individual):
            c = Counter(individual)
            device_mode = c.most_common(1)[0][0]
            device_mode = round((device_mode / len(device_graph.devices)) * self.dimension_sizes[0])

            used_devices = round(((len(set(individual)) - 1) / (len(device_graph.devices))) * self.dimension_sizes[1])

            comp_graph_dict = apply_placement(net_string, individual, groups)
            comp_graph = ComputationGraph()
            comp_graph.load_from_string(json.dumps(comp_graph_dict))

            num_jumps, max_jumps = comp_graph.get_number_of_jumps(return_max_jumps=True)
            num_jumps = round((num_jumps / max_jumps) * (self.dimension_sizes[2] - 1))

            return (device_mode, used_devices, num_jumps)

        def benchmark(individual, benchmarking_function):
            device_assignment = get_device_assignment(apply_placement(net_string, individual, groups))
            time, memory_overflow = benchmarking_function(device_assignment, return_memory_overflow=True)

            description = create_description(individual)

            # Time is set to -1 if memory overflows - but we check with memory_overflow instead
            time = max(time, 0)

            if memory_overflow == -1:
                memory_overflow = 1

            if memory_overflow > 0:
                time += memory_overflow * 10 ** 9 * 1

            return 1 / time, description, individual

        def reevaluate_archive(benchmarking_function=None, n_keep=None, time_threshold=None):
            indices = list(np.argwhere(np.isfinite(archive_scores)))

            if time_threshold:
                indices = [i for i in indices if archive_scores[i[0], i[1], i[2]] >= 1 / time_threshold]

            if n_keep:
                indices = sorted(indices, key=lambda i: -archive_scores[i[0], i[1], i[2]])
                indices = indices[:n_keep]

            assert len(indices), 'No solutions fulfill the specified requirements'

            archive_scores[:] = np.NaN
            if self.verbose:
                if n_keep:
                    log(f'Reevaluating {n_keep} best individuals in archive (and throwing away the rest)')
                else:
                    log('Reevaluating all individuals in archive')
                if time_threshold:
                    log(f'Time threshold: {time_threshold}ms')
            for i in tqdm(indices, disable=not self.verbose):
                individual = archive_individuals[i[0], i[1], i[2], :].tolist()
                if benchmarking_function:
                    archive_scores[i[0], i[1], i[2]] = benchmark(individual, benchmarking_function)[0]
                else:
                    archive_scores[i[0], i[1], i[2]] = evaluate(individual)[0]

        def log_archive(file_name):
            indices = list(np.argwhere(np.isfinite(archive_scores)))
            indices = sorted(indices, key=lambda i: -archive_scores[i[0], i[1], i[2]])

            with open(os.path.join(get_log_dir(), 'archive_logs', file_name), 'w') as f:
                f.write('niche; time; placement\n')
                for i in indices:
                    niche = tuple(i)
                    time = 1 / archive_scores[i[0], i[1], i[2]]
                    placement = archive_individuals[i[0], i[1], i[2]].tolist()

                    f.write(f'{niche}; {time}; {placement}\n')

        def run_optimization(steps, benchmarking_function=None, start_step=0):
            nonlocal archive_individuals, archive_scores

            if self.verbose:
                if benchmarking_function:
                    log('Optimizing with benchmarking...')
                else:
                    log('Optimizing with simulator...')

            step_size = 1 if benchmarking_function else self.n_threads

            for i in tqdm(range(0, steps, step_size), disable=not self.verbose):
                init_number = min(max(0, self.initial_size - i), self.n_threads)

                if self.include_trivial_solutions and i == 0:
                    candidates = create_candidates(init_number, create_trivial=True, create_random=True)
                else:
                    candidates = create_candidates(init_number, create_random=True)
                if init_number > 0:
                    candidates += create_candidates(self.n_threads - init_number, selectable_candidates=candidates[:])
                else:
                    candidates += create_candidates(self.n_threads - init_number)

                if benchmarking_function:
                    eval_results = [benchmark(candidates[0], benchmarking_function)]
                elif self.n_threads == 1:
                    eval_results = [evaluate(candidates[0])]
                else:
                    fn_args = zip(((create_description(c), c) for c in candidates), repeat(net_string), repeat(groups),
                                  repeat(device_graph), repeat(self.pipeline_batches), repeat(self.batches),
                                  repeat(self.simulator_comp_penalty), repeat(self.simulator_comm_penalty),
                                  repeat(self.device_memory_utilization))

                    eval_results = self.worker_pool.starmap(_evaluate, fn_args)

                for result in eval_results:
                    score, description, individual = result

                    previous_elite_score = archive_scores[description[0], description[1], description[2]]
                    if np.isnan(previous_elite_score) or previous_elite_score < score:
                        archive_scores[description[0], description[1], description[2]] = score
                        archive_individuals[description[0], description[1], description[2], :] = individual

                if self.verbose and (i + 1) % self.verbose < step_size:
                    best_time = 1 / np.nanmax(archive_scores)
                    log(f'[{i + 1}/{steps}] Best time: {best_time:.4f}ms')

                if self.score_save_period and (i % self.score_save_period == 0 or steps - i < step_size):
                    best_time = 1 / np.nanmax(archive_scores)
                    with open(os.path.join(get_log_dir(), 'time_history.csv'), 'a') as f:
                        f.write(f'{i + start_step + 1}, {best_time}\n')

                if self.archive_log_period and (i + 1) % self.archive_log_period < step_size:
                    log_archive(f'step_{i + start_step + 1:06}.csv')

        if self.score_save_period:
            with open(os.path.join(get_log_dir(), 'time_history.csv'), 'w') as f:
                f.write('step, time\n')

        run_optimization(self.steps)

        if self.worker_pool:
            self.worker_pool.close()

        if self.archive_log_period is not None:
            log_archive('1_simulation_finished.csv')

        if self.benchmarking_steps > 0 or self.benchmark_before_selection:
            reevaluate_archive(self.benchmarking_function, n_keep=self.benchmarking_n_keep,
                               time_threshold=self.benchmarking_time_threshold)

            if self.archive_log_period is not None:
                log_archive('2_reevaluated.csv')

        if self.benchmarking_steps > 0:
            run_optimization(self.benchmarking_steps, self.benchmarking_function, self.steps)
            log_archive('3_benchmarking_finished.csv')

        if self.show_score_plot:
            if self.verbose:
                log('Plotting archive scores...', end='')
            graph = ComputationGraph()
            graph.load_from_string(net_string)
            _, max_jumps = graph.get_number_of_jumps(return_max_jumps=True)
            plot_map_elites_archive(archive_scores, n_devices, max_jumps, self.plot_axes,
                                    save_path=os.path.join(get_log_dir(), 'archive_plot.pdf'))
            if self.verbose:
                log('Done')

        if self.plot_animation:
            if not self.archive_log_period and self.verbose:
                log('self.plot_animation was set to True, but archive logging was not enabled. '
                    'Skipping animation plot.')
            else:
                if self.verbose:
                    log('Plotting archive animation...', end='')
                paths = glob(os.path.join(get_log_dir(), 'archive_logs', 'step_*.csv'))
                plot_archive_animation(paths, (os.path.join(get_log_dir(), 'archive_animation.mp4'),
                                               os.path.join(get_log_dir(), 'archive_animation.gif')),
                                       self.dimension_sizes,
                                       n_devices=n_devices, max_jumps=max_jumps, axes=self.plot_axes,
                                       fps=self.animation_fps)
                if self.verbose:
                    log('Done')

        if return_full_archive:
            return archive_scores, archive_individuals

        best_index = np.nanargmax(archive_scores)
        best_individual = archive_individuals.reshape((-1, len(groups)))[best_index]

        if self.verbose:
            log(f'Best individual: {best_individual.tolist()}')

        solution = json.dumps(apply_placement(net_string, best_individual.tolist(), groups), indent=4)

        with open(os.path.join(get_log_dir(), 'me_solution.json'), 'w') as f:
            f.write(solution)

        return solution


from secrets import choice
from flask import Flask, flash, redirect ,render_template, request, jsonify, url_for
from database import db,Links

import datetime
from validator_collection import checkers
import pyperclip

app = Flask(__name__,instance_relative_config = False)
app.config.from_object('config.Config')
token_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz"

def init_db():
    db.init_app(app)
    db.app = app
    db.create_all()
init_db()

@app.route('/',methods=['GET','POST'])
@app.route('/home',methods=['GET','POST'])
def home():
    return render_template("home.html")

@app.route('/long',methods=['GET','POST'])
def encode():
    if request.method == 'POST':
        longurl = request.form['long-url']
        if "http" not in longurl:
            longurl = "http://"+longurl
        print(longurl)
        if not checkers.is_url(longurl):
            flash("Enter a Valid URL for eg: www.google.com or google.com","warning")
            return render_template('home.html')

        if request.form['custom-url'] != "":
            token = request.form['custom-url']
        else:
            token = "".join(choice(token_characters) for i in range(7))
        try:
            info = Links(longURL = longurl, shortURL = token)
            db.session.add(info)
            db.session.commit()
            print("Links Recieved")
            flash("Copied to Clipboard","success")
            pyperclip.copy("127.0.0.1:5000/"+token)
        except Exception as e:
            flash("Custom URL already exists.","danger")
            print(e)

    return render_template('home.html',token = token)

@app.route('/<short>',methods = ['GET','POST'])
def toLong(short):
    if request.method == 'GET':
        long = Links.hasShortURL(short)
        print(long)
        if long != None:
            return redirect(long)
    return redirect('/')

if __name__ == '__main__':
    app.run()


import argparse
import sys

class Parser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

def parse_args(args=None):
    parser = Parser(prog='git resolutions',
                    usage='%(prog)s --prepare | --pull | --push | <commit-ish>',
                    description='publish merge resolutions')
    parser.set_defaults(action='merge')
    parser.add_argument('-i', '--install', dest='action', action='store_const',
                        const='install', help='install git resolutions')
    parser.add_argument('-l', '--pull', dest='action', action='store_const',
                        const='fetch', help='pull resolutions from origin')
    parser.add_argument('-p', '--push', dest='action', action='store_const',
                        const='publish', help='push resolutions to origin')
    parser.add_argument('-f', '--force', dest='force', action='store_true',
                        help='force command, even on mixmatch')
    parser.add_argument('--directory')
    parser.add_argument('ref',
                        help=('commit-ish object names to merge'),
                        metavar='commit-ish', nargs='?')
    args = parser.parse_args(args)
    return args, parser


import pickle

import numpy as np
from keras.callbacks import ModelCheckpoint
from keras.layers import Dense, Flatten, Conv2D
from keras.layers import MaxPooling2D, Dropout
from keras.models import Sequential
from keras.utils import np_utils
from keras.utils import print_summary
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle

def loadFromPickle():
    with open("features_numbers_letters", "rb") as f:
        features = np.array(pickle.load(f))
    with open("labels_numbers_letters", "rb") as f:
        labels = np.array(pickle.load(f))

    return features, labels

def keras_model(image_x, image_y):
    num_of_classes = 33
    model = Sequential()
    model.add(Conv2D(32, (5, 5), input_shape=(image_x, image_y, 1), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))
    model.add(Conv2D(64, (5, 5), activation='relu'))
    model.add(MaxPooling2D(pool_size=(5, 5), strides=(5, 5), padding='same'))
    model.add(Flatten())
    model.add(Dense(1024, activation='relu'))
    model.add(Dropout(0.6))
    model.add(Dense(num_of_classes, activation='softmax'))

    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    filepath = "ISL.h5"
    checkpoint1 = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
    callbacks_list = [checkpoint1]

    return model, callbacks_list

def main():
    features, labels = loadFromPickle()
    # loadFromPicklefeatures, labels = augmentData(features, labels)
    features = features / 127.5 - 1.
    features, labels = shuffle(features, labels)
    train_x, test_x, train_y, test_y = train_test_split(features, labels, random_state=0,
                                                        test_size=0.1)
    train_y = np_utils.to_categorical(train_y)
    test_y = np_utils.to_categorical(test_y)
    # train_y = train_y.reshape(train_y.shape[1], train_y.shape[2])

    # test_y = test_y.reshape(test_y.shape[1], test_y.shape[2])
    train_x = train_x.reshape(train_x.shape[0], 50, 50, 1)
    test_x = test_x.reshape(test_x.shape[0], 50, 50, 1)
    model, callbacks_list = keras_model(50,50)
    print_summary(model)
    model.fit(train_x, train_y, validation_data=(test_x, test_y), epochs=1, batch_size=64,
              callbacks=callbacks_list)
    model.save('ISL.h5')

main()


import os
from pathlib import Path
from fastapi import FastAPI, File, UploadFile
from fastapi.middleware.cors import CORSMiddleware
import uvicorn
import numpy as np
from io import BytesIO
from PIL import Image
import tensorflow as tf

app = FastAPI()

origins = [
    "http://192.168.0.108:8080"
]
app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

img = open("./healthy.jpg")

MODEL = tf.keras.models.load_model("./model")

CLASS_NAMES = ["Early Blight", "Late Blight", "Healthy"]

@app.get("/test")
async def test():
    return "Local server is started"

def read_file_as_image(data) -> np.ndarray:
    image = np.array(Image.open(BytesIO(data)))
    return image

@app.post("/predict")
async def predict(
    file: UploadFile = File(...)
):
    image = read_file_as_image(await file.read())
    img_batch = np.expand_dims(image, 0)

    predictions = MODEL.predict(img_batch)

    predicted_index = int(np.argmax(predictions[0]))
    predicted_class = CLASS_NAMES[predicted_index]
    confidence = np.max(predictions[0])
    return {
        'class': predicted_class,
        'confidence': float(confidence),
    }

if __name__ == "__main__":
    uvicorn.run(app, host='localhost', port=8000)


import geopandas as gp
import numpy as np
from shapely import geometry
import networkx as nx
import pyproj as proj
import numexpr as num
import json
import pickle
from workspace import street_name_exists

#bellevue data pickle: bellevue0526
#aus data pickle: sgraphs_aus_bounded.pickle
with open('bellevue0603.pickle', 'rb') as f:
    sgraphs_data = pickle.load(f)

def merge_edges(G):
    starting_nodes = []
    newG = nx.DiGraph()
    # make a list of starting nodes
    # print("SUBGRAPH EDGES \n")
    # print(G.edges(data=True))
    for n in G.nodes():
        if G.degree(nbunch=n) == 2: # in-degree plus out-degree
            starting_nodes.append(n)
        if G.degree(nbunch=n) == 0:
            G.remove_node(n)

    print("starting nodes count: ", len(starting_nodes))
    # iterate through starting nodes to connect sidewalks
    for node in starting_nodes:
        geom_all = []
        first_neigh = next(G.successors(node))
        edge_data_1 = G.get_edge_data(node, first_neigh)
        print(edge_data_1)
        init_geom = edge_data_1['geometry']
        fwd = edge_data_1['forward']
        street = edge_data_1['street']
        geom_all.append(init_geom)
        print("starting node ", node)
        print("first neigh ", first_neigh)
        prev_neigh = node
        while (G.has_node(first_neigh)) and (G.degree(nbunch=first_neigh) != 2): #in-degree plus out-degree. caught in loop here?
            print("start while loop")
            second_neigh = G.successors(first_neigh)  # Returns an iterator over successor nodes of n.
            for neigh in second_neigh:
                if neigh != prev_neigh and neigh != first_neigh: # select node going in correct direction
                    second_neigh = neigh
            edge_data_2 = G.get_edge_data(first_neigh, second_neigh)
            if (edge_data_2 is not None):
                next_geom = edge_data_2['geometry']
                geom_all.append(next_geom)
            prev_neigh = first_neigh
            first_neigh = second_neigh
        dict_info = {
            'forward': fwd,
            'street': street,
            'geometry': geom_all,
        }
        newG.add_edge(node, first_neigh, **dict_info)
    return newG

def main():
    # print(len(sgraphs))
    combined = nx.DiGraph() # giant graph - combination of all subgraphs
    for sg in sgraphs_data:
        # if street_name_exists:
            # sg = merge_edges(sg)
        sg = merge_edges(sg)    # may need to comment out
        sg_edges = sg.edges(data=True)
        combined.add_edges_from(sg_edges)

    # Output as edgelist
    # nx.write_edgelist(combined,'bell_data_edges.edgelist')

    # Output as GeoJSON
    allFeatures = list()
    for e in combined.edges(data=True):
        # get starting node and ending node
        startn = e[0]
        point1 = []
        point2 = []
        edge_geom = []
        endn = e[1]
        # add starting node to coordinates array
        point1.append(startn[0])
        point1.append(startn[1])
        edge_geom.append(point1)

        # get dictionary of data with rest of geom
        ddict = e[2].get("geometry")

        # get forward, street, and visited information
        fwd = e[2].get("forward")
        street = e[2].get("street")
        surface = e[2].get("surface").lower()

        #add next point
        if not isinstance(ddict, list):
            ddict = [ddict]
        nextSeg = ddict[0]
        nextPoint = nextSeg.coords[0]
        point2.append(nextPoint[0])
        point2.append(nextPoint[1])
        edge_geom.append(point2)

        # iterate through linestrings, create and add array of segment with arrays of points
        for linestring in ddict:
            segCoords = linestring.coords
            for point in segCoords:
                newPoint = []
                newPoint.append(point[0])
                newPoint.append(point[1])
                edge_geom.append(newPoint)

        # add second to last point
        ptFinal1 = []
        lastSeg = ddict[len(ddict)-1]
        lastPoint = lastSeg.coords[-1]
        ptFinal1.append(lastPoint[0])
        ptFinal1.append(lastPoint[1])

        # add final point
        ptFinal2 = []
        ptFinal2.append(endn[0])
        ptFinal2.append(endn[1])

        # add last segment of edge
        edge_geom.append(ptFinal1)
        edge_geom.append(ptFinal2)

        info = {"highway": "footway", "surface": surface}
        gmt = {"type": "LineString", "coordinates": edge_geom}

        feat = {"type": "Feature", "geometry": gmt, "properties": info}

        allFeatures.append(feat)

    featCollection = {"type": "FeatureCollection", "features": allFeatures}

    with open('bell_edges_603_v2.json', 'w') as f:  # writing JSON object
        json.dump(featCollection, f)

if __name__ == "__main__":
    main()


from sklearn.metrics import mean_squared_error, mean_absolute_error
from copy import deepcopy
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import scipy
from sklearn.linear_model import LinearRegression

housing = pd.read_csv("../housePrices_train.csv")

print(housing)

# implement multilinear regression from scratch
# predict from multiple features

# 1- get the correlation between each integer values and the label we want to predict

my_label = "SalePrice"
correlations = housing.corr(numeric_only=True)[my_label]
print("the different correlations are: ")
print(correlations)
# after printing correlations, we will choose as features
# feature_one = "TotalBsmtSF"
# feature_two = "1stFlrSF"
# feature_three = "GrLivArea"
# feature_four = "OverallQual"

# represent our features with our label

saleprices = housing["SalePrice"]
# choose some columns in the dataset
housing_second = housing[["TotalBsmtSF", "1stFlrSF",
                          "GrLivArea", "OverallQual", "YearBuilt"]]
# add saleprices to housing_second
housing_second["saleprices"] = saleprices
print(housing_second)

# we are going to work with housing_second

# let plot first what we have

plt.scatter(housing_second["TotalBsmtSF"], housing_second["saleprices"])
# name the axis
plt.xlabel('Total of BsmtSF')
plt.ylabel('Sale Prices')
plt.show()

# transform housing_second into a numpy array

housing_second_np = housing_second.to_numpy()
housing_second_np.shape

X_train, y_train = housing_second_np[:, :5], housing_second_np[:, 5]
print(X_train.shape)
print(y_train.shape)
print(X_train)

model = LinearRegression().fit(X_train, y_train)
predictions = model.predict(X_train)
print(predictions)

# plt.scatter(X_train, predictions)
# plt.plot(X_train, y_train, color="r")
# plt.show()

# mean squared errors functions

mse = mean_squared_error(predictions, y_train)
mae = mean_absolute_error(predictions, y_train)

print(mse, mae)

predictions_housing = pd.DataFrame({'TotalBsmtSF': housing_second["TotalBsmtSF"],
                                    '1stFlrSF': housing_second["1stFlrSF"],
                                    'GrLivArea': housing_second["GrLivArea"],
                                    'OverallQual': housing_second["OverallQual"],
                                    'YearBuilt': housing_second["YearBuilt"],
                                    'Predictions': predictions})

print(predictions_housing)

def get_predictions(model, X):
    pass


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('musico', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='album',
            name='foto',
            field=models.ImageField(upload_to='', blank='True'),
        ),
    ]


import torch
import numpy as np
from collections import OrderedDict
from tqdm import tqdm
from sklearn import metrics
import os
from .postdataset import PostDataset

from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler
from torch.utils.data.dataloader import default_collate

from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold

TRAIN_BATCH_SIZE = 16
VALID_BATCH_SIZE = 4

class KFoldSplit():
    def __init__(self, p_data, k_folds = 5):
        self.k_folds = k_folds
        self.data = np.load(p_data, allow_pickle=True)
        self.kfold = KFold(n_splits=k_folds, shuffle=False)
        self.folds = [f[1] for f in self.split()]
        self.cur_test_fold = 0

        # print(folds)

    def split(self):
        return self.kfold.split(self.data)

    def get_split(self, train_ratio = -1, save_test = True): # one out of four
        # train_ratio == 1: testset for early-stopping  -1: one fold for early-stopping
        i_test = self.folds[self.cur_test_fold]
        x_test = self.data[i_test]
        mask = np.ones(len(self.data), bool)
        mask[i_test] = False
        x_train_val = self.data[mask]
        self.cur_test_fold = self.cur_test_fold+1
        assert (self.cur_test_fold<=self.k_folds), 'Out of boundary.'

        if train_ratio == 1:
            x_train = x_train_val
            x_val= x_test
        else:
            if train_ratio == -1:
                train_ratio = (self.k_folds-2) / (self.k_folds-1)
            x_train, x_val = train_test_split(x_train_val, test_size=1-train_ratio, random_state=0)
        # print(len(x_train), len(x_val), len(x_test))

        if save_test == True:
            output_folder_path = './output/'
            if not os.path.exists(output_folder_path):
                os.makedirs(output_folder_path)
            np.save(os.path.join(output_folder_path, str(self.cur_test_fold-1) + '_testset.npy'), x_test, allow_pickle=True)
        return x_train, x_val, x_test

class DataPreprocessor():
    def __init__(self, x_train, x_val, x_test, tokenizer, max_len, fea_config, device):
        self.x_train, self.x_val, self.x_test = x_train, x_val, x_test
        self.tokenizer = tokenizer
        self.training_set = PostDataset(self.x_train, tokenizer, max_len, fea_config, device)
        self.validation_set = PostDataset(self.x_val, tokenizer, max_len, fea_config, device)
        self.testing_set = PostDataset(self.x_test, tokenizer, max_len, fea_config, device)

    def get_train_dataloader(self):
        params = {'batch_size': TRAIN_BATCH_SIZE,
                        'shuffle': False,
                        'num_workers': 0
                        }
        return DataLoader(self.training_set, **params)

    def get_val_dataloader(self):
        params = {'batch_size': VALID_BATCH_SIZE,
                        'shuffle': False,
                        'num_workers': 0
                        }
        return DataLoader(self.validation_set, **params)

    def get_test_dataloader(self):
        params = {'batch_size': VALID_BATCH_SIZE,
                        'shuffle': False,
                        'num_workers': 0
                        }
        return DataLoader(self.testing_set, **params)

if __name__ == "__main__":
    print("data processor.")
    p_data = '../dataset/all_annotated_post_784.npy'
    k_fold_obj = KFoldSplit(p_data, 5)
    # folds = k_fold_obj.split()
    x_train, x_val, x_test = k_fold_obj.get_split()
    print(x_test[0])
#     from transformers import *
#     from torch import cuda

#     fea_config = {
#         'code_fea' : False,
#         'word_cnt' : False,
#         'readability' : False,
#         'sentiment' : False
#     }
#     device = 'cuda' if cuda.is_available() else 'cpu'
#     tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
#     dp = DataPreprocessor(x_train, x_val, x_test, tokenizer, 128, fea_config, device)

#     vdl = dp.get_val_dataloader()


from flask import Flask, request
import time
import json

app = Flask(__name__)

feedbackArr:list = []

@app.route("/submit", methods=["POST"])
def submit():
    requestJson = request.get_json()
    feedbackArr.append(
        {
            "Feedback":requestJson["feedback"],
            "Time":time.time(),
            "Username":requestJson["username"]
        }
    )
    return "Feedback Submitted Successfully"

@app.route("/get", methods=["GET"])
def get_feedback():
    #the amount of days back the feedback goes
    daysback = request.args.get("daysback")

    if daysback:
        newFeedback = []

        secondsback = int(daysback) * 24 * 60 * 60
        for i in reversed(feedbackArr):
            if time.time()- i["Time"] < secondsback:
                newFeedback.append(i)
            else:
                break
        return json.dumps(newFeedback)

    return json.dumps(feedbackArr)

@app.route("/" , methods=["GET", "POST"])
def index():
    return "null"

if __name__=="__main__":
    app.run(host="0.0.0.0", port=5000, debug=True)


import json
import os
import pickle
import re

import numpy as np
import pandas as pd
import torch
import torchaudio
from sklearn.feature_extraction.text import CountVectorizer
from torch.utils.data import DataLoader, Dataset
from torchsummary import summary
from tqdm.auto import tqdm

device = 'cuda' if torch.cuda.is_available() else 'cpu'

sr = 16000

def loading_fairseq_model(model_file):
    import fairseq

    from torchaudio.models.wav2vec2.utils import import_fairseq_model
    model, _, _ = fairseq.checkpoint_utils.load_model_ensemble_and_task([model_file])
    original = model[0]
    imported = import_fairseq_model(original)
    return imported

def loading_huggingface_model(hf_model):
    from torchaudio.models.wav2vec2.utils import import_huggingface_model
    model = import_huggingface_model(hf_model)
    return model

def walk_librispeech_dirs(librispeech_root, libri_split):
    libri_split_path = os.path.join(librispeech_root,libri_split)
    filelist = []
    for root, dir, file in os.walk(libri_split_path):
        filelist.append(file)
    import itertools
    flatlist_raw = list(itertools.chain(*filelist))
    fileid_pattern = re.compile(r'\d+\-\d+-\d+')
    flatlist = [s for s in flatlist_raw if fileid_pattern.match(s)]
    txtlist = [os.path.join(*[re.split(r'\.|-', x)[0],re.split(r'\.|-', x)[1],x]) for x in flatlist if 'txt' in x]
    flaclist = [os.path.join(*[re.split(r'\.|-', x)[0],re.split(r'\.|-', x)[1],x]) for x in flatlist if 'flac' in x]
    df = pd.DataFrame(flaclist)
    df['fileid'] = df[0].apply(lambda x : os.path.split(x)[1])
    df['fileid'] = df['fileid'].str.replace('.flac', '', regex=True)
    df = df.rename({0:'fullpath'},axis=1)
    txt_out = []
    for txt_file in tqdm(txtlist):
        with open(os.path.join(libri_split_path, txt_file)) as f:
            txt_dict = {}

            sent = f.readline()
            file_id = os.path.split(txt_file)[-1]
            txt_dict['sent'] = sent
            txt_dict['fileid'] = file_id
            txt_out.append(txt_dict)
    df_txt = pd.DataFrame(txt_out)
    df_txt['fileid'] = df_txt['fileid'].str.replace('.txt', '', regex=True)
    merge_df = pd.merge(df,df_txt, on = 'fileid')
    merge_df = merge_df.drop(columns=['fileid'])
    return merge_df

def make_bow(doc):

    doc = list(map(str.lower, doc))
    unique_words = set(' '.join(doc).split())
    print(f'there are {len(unique_words)} unique words')
    index_dict = {}
    for ind, i in enumerate(sorted(unique_words)):
        index_dict[i] = ind
    cv = CountVectorizer(token_pattern=r"(?u)\b\w+\b")
    count_occurrences = cv.fit_transform(doc)
    return count_occurrences.toarray()

def read_json_save_csv(json_path):
    '''Reading the SpokenCOCO json file and extracting the text and wav file pairs'''
    with open(json_path) as json_file:
        data = json.load(json_file)
    text = []
    wav = []
    for image in data['data']:
        for caption in image['captions']:
            text.append(caption['text'])
            wav.append(caption['wav'])

    '''
    Using pandas to combine the two lists extracted in the cells above and
    save the resulting dataframe into a csv file
    '''
    dict_spokencoco = dict(zip(text,wav))
    spokencoco_val = pd.Series(dict_spokencoco, name = 'wav')
    spokencoco_val.index.name = 'text'
    spokencoco_val = spokencoco_val.reset_index()
    column_titles = ['wav', 'text']
    spokencoco_val = spokencoco_val.reindex(columns = column_titles)
    return spokencoco_val

def get_weird_sents(corpus_csv, root_dir):
    from custom_classes import Corpus
    corpus_ = Corpus(corpus_csv, root_dir)
    weird_sents = [x for x in [corpus_.get_depth(i) for i in range(len(corpus_))] if x[1] > len(str.split(x[0]))+2]
    print(f'there are total {len(weird_sents)} sentences')
    print(weird_sents)

def generate_tree(dataset, num_entries = None):
    import stanza
    nlp = stanza.Pipeline(lang='en', processors='tokenize,pos,constituency')
    from nltk.tree import Tree

    text = [x[-2] for x in dataset[:num_entries]]

    tree_list = [str(nlp(sent).sentences[0].constituency) for sent in text]
    nltk_tree_list = [Tree.fromstring(x) for x in tree_list]

    return list(zip((nltk_tree_list,text)))

def recover_from_triu(matrices):

    size_X = matrices[-1]
    X = np.zeros((size_X,size_X))
    full_matrices = []
    for v in matrices[:-1]:
        X[np.triu_indices(X.shape[0], k = 1)] = v
        X = X + X.T
        full_matrices.append(X)
    return full_matrices


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import numpy as np
import yolov5
import odmetrics

###############################################################################
# PARAMETERS
###############################################################################

# Path to the folder containing the test images and labels (ground truth)
PATH_TEST_IMAGES='../../../DATA/BURROWS/DATASET/images/test/'
PATH_TEST_LABELS='../../../DATA/BURROWS/DATASET/labels/test/'

# Path to the trained model
PATH_MODEL='MODELS/BSMALL.pt'

# Image size (required to convert YOLOv5 (xywhn) to ABSLUTE (xyxy))
IMG_WIDTH=640
IMG_HEIGHT=480

# IoU thresholds to use when computing metrics.
IOU_THRESHOLDS=np.round(np.arange(0.1,0.95,0.05),2)

# Classes to evaluate. Must be an iterable.
CLASSES_TO_EVALUATE=[0]

###############################################################################
# MAIN CODE
###############################################################################

# =============================================================================
# OBJECT DETECTION
# =============================================================================

# Install YOLOv5
pathYOLOv5=yolov5.install()

# Perform object detection
theNames,predTimes,predLabels=yolov5.detect_with_torch(pathYOLOv5,PATH_MODEL,PATH_TEST_IMAGES)

# =============================================================================
# STATS NOT REQUIRING ODMETRICS
# =============================================================================

# Let's show some results (not requiring odmetrics)
print('[INFO] AVERAGE PREDICTION TIME PER IMAGE: %.3f ms'%(np.mean(predTimes)))

# Remove outliers from time consumption (usually the first ones are the largest
# due -probably- to startup, caching and stuff)
theMedian=np.median(predTimes)
theStd=np.std(predTimes)
filteredTimes=[[i,t] for i,t in enumerate(predTimes) if abs(t-theMedian)<=3*theStd]

# Get the largest prediction time within the filtered ones
iLargest=np.argmax([x[1] for x in filteredTimes])
iLargest=filteredTimes[iLargest][0]

# Print some related info
print('[INFO] THE MOST TIME CONSUMING IMAGE WAS %s, WHICH REQUIRED %.3f ms TO DETECT %d OBJECTS.'%(theNames[iLargest],predTimes[iLargest],len(predLabels[iLargest])))

# Get the smallest prediction time within the filtered ones
iSmallest=np.argmin([x[1] for x in filteredTimes])
iSmallest=filteredTimes[iSmallest][0]

# Print some related info
print('[INFO] THE LEAST TIME CONSUMING IMAGE WAS %s, WHICH REQUIRED %.3f ms TO DETECT %d OBJECTS.'%(theNames[iSmallest],predTimes[iSmallest],len(predLabels[iSmallest])))

# =============================================================================
# USING ODMETRICS.COMPUTE_METRICS AND RELATED UTILITIES
# =============================================================================

# Load the ground truth labels. Note that "theNames" is necessary to load them
# in the same order than the predLabels.
gtLabels=odmetrics.load_all_yolo_labels(PATH_TEST_LABELS,theNames,IMG_WIDTH,IMG_HEIGHT)

# Compute the main metrics
allResults=odmetrics.compute_metrics(predLabels,gtLabels,IOU_THRESHOLDS,CLASSES_TO_EVALUATE)

# Plot some Recall-Precision curves
odmetrics.plot_rp_curves(allResults, [0.1,0.3,0.5,0.7,0.9])

# Get mAP0.5 and mAP0.5:0.05:0.95, both for class 0.
mAP05=odmetrics.compute_mAP(allResults,[0.5],[0])
mAP05_005_095=odmetrics.compute_mAP(allResults,np.round(np.arange(0.5,0.95,0.05),2),[0])

# Print the results
print('[INFO] HERE ARE SOME STATS FOR YOUR ENJOYMENT:')
print('* mAP@0.5=%.3f'%mAP05)
print('* mAP@0.5:0.05:0.95=%.3f'%mAP05_005_095)

# =============================================================================
# USING ODMETRICS.EXPLORE_PARAMETERS AND RELATED UTILITIES
# =============================================================================

# Let's explore the threshold space
theExploration=odmetrics.explore_parameters(predLabels,gtLabels,np.round(np.arange(0.1,0.9,0.1),2), np.round(np.arange(0.1,0.9,0.1),2), [0])

# Plot the F1-Score for all the explored space
odmetrics.plot_exploration(theExploration,metricToShow=5)

# Search the best threshold combination for class 0 according to the different
# explored metrics.
bestTPConfig=odmetrics.get_best_configuration(theExploration,0)
bestFPConfig=odmetrics.get_best_configuration(theExploration,1)
bestFNConfig=odmetrics.get_best_configuration(theExploration,2)
bestRecallConfig=odmetrics.get_best_configuration(theExploration,3)
bestPrecisionConfig=odmetrics.get_best_configuration(theExploration,4)
bestF1Config=odmetrics.get_best_configuration(theExploration,5)

# Print the configurations
print('[INFO] CONFIGURATION MAXIMIZING TRUE POSITIVES')
odmetrics.print_explored_configuration(bestTPConfig)
print('\n[INFO] CONFIGURATION MAXIMIZING FALSE POSITIVES')
odmetrics.print_explored_configuration(bestFPConfig)
print('\n[INFO] CONFIGURATION MAXIMIZING FALSE NEGATIVES')
odmetrics.print_explored_configuration(bestFNConfig)
print('\n[INFO] CONFIGURATION MAXIMIZING RECALL')
odmetrics.print_explored_configuration(bestRecallConfig)
print('\n[INFO] CONFIGURATION MAXIMIZING PRECISION')
odmetrics.print_explored_configuration(bestPrecisionConfig)
print('\n[INFO] CONFIGURATION MAXIMIZING F1-SCORE')
odmetrics.print_explored_configuration(bestF1Config)


import unittest
import time

from selenium.common.exceptions import (
    InvalidArgumentException,
)

from src.Scraper import Scraper
from src import models as m

URL = "https://www.mtgtop8.com/format?f=ST"

class TestDriver(unittest.TestCase):

    def test_get_mtgtop8(self):
        with Scraper(URL) as scraper:
            page_title = scraper.get_title().lower()
            self.assertIn("mtg", page_title)
            self.assertEqual(scraper.get_url(), URL)

    def test_stable_should_be_in_page_one_body(self):
        with Scraper(URL) as scraper:
            page_source = repr(scraper)
            self.assertIn("Stable", page_source)
            self.assertNotIn("Stttbl", page_source)

    def test_should_return_a_string_for_page_source(self):
        with Scraper(URL) as scraper:
            page_source = repr(scraper)
            self.assertIsInstance(page_source, str)

    def test_is_page_one_with_no_previous_button(self):
        page = 2
        with Scraper(URL) as scraper:
            page_source = repr(scraper)
            stable = scraper.get_by("css", "table.Stable")
        self.assertIn("Nav_PN_no", page_source)
        self.assertIn(f"PageSubmit({page})", page_source)
        self.assertIsNotNone(stable)

    def test_get_button_to_go_to_page_two(self):
        with Scraper(URL) as scraper:
            scraper.execute("PageSubmit", 2)
            stable = scraper.get_by("css", "table.Stable")
        self.assertNotIn("Nav_PN_no", stable)

    @unittest.skip("Out of Date")
    def test_first_page_should_have_this_event_title(self):
        event_title = "FNM @ Deckmaster Games"
        with Scraper(URL) as scraper:
            page_source = repr(scraper)
        self.assertIn(event_title, page_source)

# Skipping because it is gumming up the webdriver access...
@unittest.SkipTest
class TestDriverWithBadInputsDefaultExceptions(unittest.TestCase):

    def test_should_return_with_empty_page(self):
        with self.assertRaises(InvalidArgumentException):
            scraper = Scraper("adfa")
            scraper.quit()

    def test_should_get_mad_if_no_keys_in_selector(self):
        with Scraper(URL) as scraper:
            result = scraper.get_by("css", "table.NotATable")
        self.assertFalse(result)

@unittest.skip("Out of Date")
class TestGettingSpecificElements(unittest.TestCase):

    def test_should_give_all_tables_with_class_Stable(self):
        with Scraper(URL) as scraper:
            table_elements = scraper.get_all_by(
                "css", "table.Stable")
        self.assertTrue(len(table_elements) > 1)
        self.assertIsInstance(table_elements, list)

    def test_should_give_first_table_with_class_stable(self):
        with Scraper(URL) as scraper:
            table_element = scraper.get_by("css", "table.Stable")
        self.assertEqual(len(table_element), 1)

    def test_finding_one_thing_by_name(self):
        with Scraper(URL) as scraper:
            result = scraper.get_by("name", "meta")
        self.assertEqual(len(result), 1)
        self.assertIsInstance(result, list)

    def test_finding_one_thing_by_id(self):
        with Scraper(URL) as scraper:
            result = scraper.get_by("id", "other_tooltip")
        self.assertEqual(len(result), 1)
        self.assertIsInstance(result, list)

    def test_finding_one_thing_by_class_not_css(self):
        with Scraper(URL) as scraper:
            result = scraper.get_by("class", "S14")
            self.assertEqual(len(result), 1)
            self.assertIsInstance(result, list)
            self.assertIn("Players Tour Online", result[0].text)

    def test_finding_all_things_by_class_not_css(self):
        with Scraper(URL) as scraper:
            result = scraper.get_all_by("class", "S14")
            self.assertGreaterEqual(len(result), 2)
            self.assertIn("decks", result[1].text)

    def test_find_one_row_of_stable_with_xpath(self):
        with Scraper(URL) as scraper:
            result = scraper.get_by(
                "xpath", "//table[@class='Stable'][2]//tr[@class='hover_tr']//a")
            self.assertEqual(len(result), 1)
            # self.assertEqual("Japan Championship 2020 Autumn - Last Chance Trial @ BIG Magic", result[0].text)
            # self.assertIn("event?e=27876&f=ST", result[0].get_attribute("href"))

    def test_find_all_rows_in_stable_with_xpath(self):
        with Scraper(URL) as scraper:
            results = scraper.get_all_by(
                "xpath", "//table[@class='Stable'][2]//tr[@class='hover_tr']//a"
            )
            self.assertEqual(len(results), 10)
            # events = [item.text for item in results]
            # links = [item.get_attribute("href") for item in results]
            # self.assertIn("FNM @ Deckmaster Games", events)
            # self.assertIn("https://www.mtgtop8.com/event?e=27847&f=ST", links)

    def test_xpath_finds_things_on_other_pages(self):
        with Scraper(URL) as scraper:
            scraper.execute("PageSubmit", 9)
            time.sleep(2)
            results = scraper.get_all_by(
                "xpath", "//table[@class='Stable'][2]//tr[@class='hover_tr']//a"
            )
            self.assertEqual(len(results), 10)
            # events = [item.text for item in results]
            # links = [item.get_attribute("href") for item in results]
            # self.assertIn("Monday Night Magic 2 @ Plague League", events)
            # self.assertIn("https://www.mtgtop8.com/event?e=27623&f=ST", links)

    def test_xpath_finds_dates(self):
        with Scraper(URL) as scraper:
            result = scraper.get_by(
                "xpath", "//table[@class='Stable'][2]//tr[@class='hover_tr']//td[@class='S10']")
            self.assertEqual(result[0].text, '30/10/20')

class TestRepr(unittest.TestCase):

    def test_should_print_page_source(self):
        with Scraper(URL) as scraper:
            print(scraper)

if __name__ == "__main__":
    unittest.main()


# coding:utf-8
import time
import config as cfg
import requests
from lxml import etree
import pymysql as mdb
import datetime
from selenium import webdriver

class IPFactory:
    """
    代理ip抓取/评估/存储一体化。
    """
    def __init__(self):
        self.page_num = cfg.page_num
        self.round = cfg.examine_round
        self.timeout = cfg.timeout
        self.all_ip = set()
        results = self.get_proxies( )
        self.all_ip.update(results)

    def craw_proxy(self):
        #  创建数据库
        self.create_db()

        # # 抓取全部ip
        current_ips = self.get_all_ip()
        # # 获取有效ip
        valid_ip = self.get_the_best(current_ips, self.timeout, self.round)
        print( valid_ip)
        self.save_to_db( valid_ip)

    def create_db(self):
        """
        创建数据库用于保存有效ip
        """
        # 创建数据库/表语句
        # 创建数据库
        drop_db_str = 'drop database if exists ' + cfg.DB_NAME + ' ;'
        create_db_str = 'create database if not exists ' + cfg.DB_NAME  + ' ;'
        # 选择该数据库
        use_db_str = 'use ' + cfg.DB_NAME + ' ;'
        # 创建表格
        create_table_str = "CREATE TABLE if not exists " + cfg.TABLE_NAME + """(
          `content` varchar(30) NOT NULL,
          `test_times` int(5) NOT NULL DEFAULT '0',
          `failure_times` int(5) NOT NULL DEFAULT '0',
          `success_rate` float(5,2) NOT NULL DEFAULT '0.00',
          `avg_response_time` float NOT NULL DEFAULT '0',
          `score` float(5,2) NOT NULL DEFAULT '0.00'
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8;"""

        # 连接数据库
        conn = mdb.connect(cfg.host, cfg.user, cfg.passwd)
        cursor = conn.cursor()
        try:
            cursor.execute(use_db_str)
            cursor.execute('truncate table ' + cfg.TABLE_NAME )
            cursor.execute(create_db_str)
            cursor.execute(create_table_str)
            conn.commit()
        except OSError:
            print ("无法创建数据库！")
        finally:
            cursor.close()
            conn.close()
    def decode_text(self,html):
        myencoding = ""
        if  html.encoding == 'ISO-8859-1':
            encodings = requests.utils.get_encodings_from_content(html.text)
            if encodings:
                myencoding = encodings[0]
            else:
                myencoding = html.apparent_encoding
        html_text = html.text.encode('ISO-8859-1').decode(myencoding, 'replace')
        return   html_text

    def get_proxy_list(self,url,url_xpath):
        '''
        返回抓取到代理的列表
        整个爬虫的关键
        '''
        options = webdriver.ChromeOptions()

        options.add_argument("--headless")
        browser = webdriver.Chrome(chrome_options = options,executable_path='D:\\chromedriver.exe')
        browser.implicitly_wait(30)
        browser.maximize_window()
        proxy_list = []

        browser.get(url)
        browser.implicitly_wait(3)
        # 找到代理table的位置
        elements = browser.find_elements_by_xpath(url_xpath)
        for element in elements:
            ip = element.find_element_by_xpath('./td[1]').text
            port = element.find_element_by_xpath('./td[2]').text
            ip_addr=ip+':' +port
            print( ip_addr)
            proxy_list.append( ip_addr)

        browser.quit()
        return proxy_list

    def get_proxy_list2(self,url,url_xpath):
        '''
        返回抓取到代理的列表
        整个爬虫的关键
        '''
        options = webdriver.ChromeOptions()

        options.add_argument("--headless")
        browser = webdriver.Chrome(chrome_options = options,executable_path='D:\\chromedriver.exe')
        browser.implicitly_wait(30)
        browser.maximize_window()
        proxy_list = []

        browser.get(url)
        browser.implicitly_wait(3)
        # 找到代理table的位置
        elements = browser.find_elements_by_xpath(url_xpath)
        for element in elements:
            ip = element.find_element_by_xpath('./td[1]').text
            port = element.find_element_by_xpath('./td[2]').text
            ip_addr=ip+':' +port
            print( ip_addr)
            proxy_list.append( ip_addr)

        browser.quit()
        return proxy_list

    def get_content(self, url, url_xpath, port_xpath):
        """
        使用xpath解析网页内容,并返回ip列表。
        """
        # 返回列表
        ip_list = []

        try:
            # 设置请求头信息
            headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko'}

            # 获取页面数据
            results = requests.get(url, headers=headers, timeout=4)
           # html_text =self.decode_text(results)
            tree = etree.HTML(results.text)

            # 提取ip:port
            url_results = tree.xpath(url_xpath)
            port_results = tree.xpath(port_xpath)
            urls = [line.strip() for line in url_results]
            ports = [line.strip() for line in port_results]

            if len(urls) == len(ports):
                for i in range(len(urls)):
                    # 匹配ip:port对
                    full_ip = urls[i]+":"+ports[i]
                    # 此处利用all_ip对过往爬取的ip做了记录，下次再爬时如果发现
                    # 已经爬过，就不再加入ip列表。
                    if full_ip in self.all_ip:
                        continue
                    # 存储
                    ip_list.append(full_ip)
        except Exception as e:
            print ('get proxies error: ', e)

        return ip_list

    def get_all_ip(self):
        """
        各大网站抓取的ip聚合。
        """
        # 有2个概念：all_ip和current_all_ip。前者保存了历次抓取的ip，后者只保存本次的抓取。
        current_all_ip = set()

        ##################################
        # 66ip网
        ###################################
        print(" crawl http://www.66ip.cn/ " )
        url_xpath_66 = '//*[@id="main"]/div/div[1]/table/tbody/tr[position()>1]'
        for i in range(2,self.page_num+1):
            url_66 = 'http://www.66ip.cn/' + str(i+1) + '.html'
            results = self.get_proxy_list(url_66, url_xpath_66)
            self.all_ip.update(results)
            current_all_ip.update(results)
            # 停0.5s再抓取
            time.sleep(0.5)
        url_xpath_66 = '//*[@id="footer"]/div/table/tbody/tr[position()>1]'

        for m in range(1, 35):
            url_66 ='http://www.66ip.cn/areaindex_%s/1.html' % (m)
            results = self.get_proxy_list(url_66, url_xpath_66)
            self.all_ip.update(results)
            current_all_ip.update(results)
            # 停0.5s再抓取
            time.sleep(0.5)
        ##################################
        print(" crawl http://www.kuaidaili.com/free/inha " )
        url_xpath_kuaidaili = '//*[@id="list"]/table/tbody/tr[position()>1]'

        for m in range(1, 3):
            url_kuaidaili ='http://www.kuaidaili.com/free/inha/%s' % (m)
            results = self.get_proxy_list(url_kuaidaili, url_xpath_kuaidaili)
            self.all_ip.update(results)
            current_all_ip.update(results)
            # 停0.5s再抓取
            time.sleep(0.5)
        ##################################
        # xici代理
        ###################################
        print(" crawl http://www.xicidaili.com/nn/ " )
        url_xpath_xici = '//*[@id="footer"]/div/table/tbody/tr[position()>1]'

        for i in range(1,9):
            url_xici = 'http://www.xicidaili.com/nn/' + str(i+1)
            results = self.get_proxy_list2(url_xici, url_xpath_xici)
            self.all_ip.update(results)
            current_all_ip.update(results)
            time.sleep(0.5)

        ##################################
        # mimiip网
        ###################################
# =============================================================================
#         print(" crawl http://www.mimiip.com/gngao/ " )
#         url_xpath_mimi = '//*[@id="middle_wrapper"]/div/table/tbody/tr[position()>1]'
#
#         for i in range(1,2):
#             url_mimi = 'http://www.mimiip.com/gngao/' + str(i+1)
#             results = self.get_proxy_list(url_mimi, url_xpath_mimi)
#             self.all_ip.update(results)
#             current_all_ip.update(results)
#             time.sleep(0.5)
#
# =============================================================================

        return current_all_ip

    def get_valid_ip(self, ip_set, timeout):
        """
        代理ip可用性测试
        """
        # 设置请求地址
        url = 'http://quotes.money.163.com/stock'

        # 可用代理结果
        results = set()

        # 挨个检查代理是否可用
        for p in ip_set:
            proxy = {'http': 'http://'+p}
            try:
                # 请求开始时间
                start = time.time()
                r = requests.get(url, proxies=proxy, timeout=timeout)
                # 请求结束时间
                end = time.time()
                # 判断是否可用
                if r.text is not None:
                    print( 'succeed: ' + p + '\t' + " in " + format(end-start, '0.2f') + 's')
                    # 追加代理ip到返回的set中
                    results.add(p)
            except OSError:
                print ('timeout:', p)

        return results

    def get_the_best(self, valid_ip, timeout, round):
        """
        N轮检测ip列表，避免"辉煌的15分钟"
        """
        # 循环检查次数
        for i in range(round):
            print ("\n>>>>>>>\tRound\t"+str(i+1)+"\t<<<<<<<<<<")
            # 检查代理是否可用
            valid_ip = self.get_valid_ip(valid_ip, timeout)
            # 停一下
            if i < round-1:
                time.sleep(30)

        # 返回可用数据
        return valid_ip

    def save_to_db(self, valid_ips):
        """
        将可用的ip存储进mysql数据库
        """
        if len(valid_ips) == 0:
            print( "本次没有抓到可用ip。")
            return
        # 连接数据库
        print ("\n>>>>>>>>>>>>>>>>>>>> 代理数据入库处理 Start  <<<<<<<<<<<<<<<<<<<<<<\n")
        conn = mdb.connect(cfg.host, cfg.user, cfg.passwd, cfg.DB_NAME)
        cursor = conn.cursor()
        try:
            for item in valid_ips:
                # 检查表中是否存在数据
                item_exist = cursor.execute('SELECT * FROM %s WHERE content="%s"' %(cfg.TABLE_NAME, item))

                # 新增代理数据入库
                if item_exist == 0:
                    # 插入数据
                    n = cursor.execute('INSERT INTO %s VALUES("%s", 1, 0, 0, 1.0, 2.5)' %(cfg.TABLE_NAME, item))
                    conn.commit()

                    # 输出入库状态
                    if n:
                        print (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" "+item+" 插入成功。\n")
                    else:
                        print (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" "+item+" 插入失败。\n")

                else:
                    print (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" "+ item +" 已存在。\n")
        except Exception as e:
            print ("入库失败：" + str(e))
        finally:
            cursor.close()
            conn.close()
        print( "\n>>>>>>>>>>>>>>>>>>>> 代理数据入库处理 End  <<<<<<<<<<<<<<<<<<<<<<\n")

    def get_proxies(self):
        ip_list = []

        # 连接数据库
        conn = mdb.connect(cfg.host, cfg.user, cfg.passwd, cfg.DB_NAME)
        cursor = conn.cursor()

        # 检查数据表中是否有数据
        try:
            cursor.execute('SELECT * FROM %s ' % cfg.TABLE_NAME)

            # 提取数据
            result = cursor.fetchall()

            # 若表里有数据　直接返回，没有则抓取再返回
            if len(result):
                for item in result:
                    ip_list.append(item[0])
            else:
                # 获取代理数据
                current_ips = self.get_all_ip()
                valid_ips = self.get_the_best(current_ips, self.timeout, self.round)
                self.save_to_db(valid_ips)
                ip_list.extend(valid_ips)
        except Exception as e:
            print ("从数据库获取ip失败！")
        finally:
            cursor.close()
            conn.close()

        return ip_list

def main():
    # 抓取搜索关键词
    ip_factory = IPFactory()

    # 抓取数据
    ip_factory. craw_proxy()

if __name__ == '__main__':
    main()


from django.contrib.auth import get_user_model
from rest_framework import serializers
from .models import Blog

class BlogSerializer(serializers.ModelSerializer):
    created_on = serializers.DateTimeField(format="%Y-%m-%d", read_only=True)
    content = serializers.SerializerMethodField()
    class Meta:
        model = Blog
        fields = ('id', 'title', 'img', 'text', 'content', 'created_on')

    def get_content(self, obj):
        return obj.convert_markdown_to_html()