# -*- coding: utf-8 -*-

import re
import json
import pycurl
from copy import copy
from os import path, fsync, getenv
from time import strftime
from io import BytesIO
from urllib.parse import urlencode

CON_TIMEOUT = 300
REQ_TIMEOUT = 3600
headers = {}
host_sistec = 'https://sistec.mec.gov.br'
encoding = 'iso-8859-1'
campus_encoding = 'utf-8'
logDir = u'%s' % getenv('SISTEC_DOWNLOAD_LOG', default='../log')
pydbg_path = u'%s/pycurl_debug_%s.log' % (logDir, strftime('%Y-%m-%d'))

def get_data_sistec(cod_campus, campus, perfil, tipos, downDir, qtdPerfis, cookies, file_log_path=None, level_debug=[False, False, False]):
    global host_sistec, encoding, campus_encoding
    sucesso = False
    campus = campus.decode(campus_encoding)
    campus_csv = path.join(downDir, campus + '.csv')
    file_csv = open(campus_csv, mode='w', encoding=encoding)
    sucesso, mensagem = write_csv(file_csv, campus, cod_campus, perfil, qtdPerfis, tipos, cookies, file_log_path, level_debug)
    file_csv.close()
    if sucesso:
        if level_debug[0]:
            with open(file_log_path, 'a') as file_log:
                file_log.write(u'%s O download da planilha do SISTEC (%s) foi efetuado com sucesso em:\t%s\n' % (
                    strftime('[%Y-%m-%d %H:%M:%S]'), campus.title(), campus_csv))
                file_log.flush()
                fsync(file_log.fileno())
        sucesso = True
    else:
        if level_debug[0]:
            with open(file_log_path, 'a') as file_log:
                file_log.write(u'%s Não foi possível salvar a planilha do SISTEC (%s) em:\t%s\t%s\n' % (
                    strftime('[%Y-%m-%d %H:%M:%S]'), campus.title(), campus_csv, mensagem))
                file_log.flush()
                fsync(file_log.fileno())
        sucesso = False
    return sucesso

def write_csv(file_csv, no_campus, co_campus, perfil, qtdPerfis, tipos, cookies, file_log_path = None, level_debug = [False, False, False]):
    u"""
    Percorre o sistec, pegando os dados dos alunos nos ciclos de matricula,
    do 'campus' e os salva no arquivo 'file_csv'.

    :file_csv:           arquivo csv onde salvar os dados do SISTEC
    :no_campus:          nome do campus no site do SISTEC
    :co_campus:          codigo do campus no site do SISTEC
    :perfil:             perfil do usuário no site do SISTEC
    :tipos:              tipos de cursos [Presencial, EaD, FIC]
    :cookies:            cookies da sessão aberta para acesso ao site do SISTEC
                         [phpsessid, jsessionid, zde, perfil, noticias]
    :http_header_common: cabecalho comum das requisicoes http ao site do SISTEC
    :returns:            Sucesso, Mensagem
    """
    global headers, host_sistec, encoding, CON_TIMEOUT, REQ_TIMEOUT

    url_alterar_perfil = 'https://sistec.mec.gov.br/index/selecionarinstituicao/alterar/perfil'
    url_index          = 'https://sistec.mec.gov.br/index/index'
    url_turmas         = 'https://sistec.mec.gov.br/gridciclo/turmas'
    url_ciclo_common   = 'https://sistec.mec.gov.br/gridciclo/listaralunosacao/periodo//ciclo/'
    url_tempo_sessao   = 'https://sistec.mec.gov.br/tempo-sessao/get-tempo-sessao/'
    url_dados_ciclo    = 'https://sistec.mec.gov.br/admciclomatricula/dadosciclo/'

    header_cache              = 'Cache-Control: max-age=0'
    #header_dnt               = 'DNT: 1'
    header_connection         = 'Connection: keep-alive'
    header_upgrade            = 'Upgrade-Insecure-Requests: 1'
    header_content_type       = 'Content-type: application/x-www-form-urlencoded charset=' + encoding
    header_content_len        = 'Content-Length: 0'
    header_user_agent         = 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36'
    header_accept_text        = 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'
    header_accept_lang        = 'Accept-Language: pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7'
    header_accept_json        = 'Accept: application/json'
    header_accept_enc         = 'Accept-Encoding: gzip, deflate, br'
    header_xreq_xml           = 'X-Requested-With: XMLHttpRequest'
    header_xreq_json          = 'X-Request: JSON'
    header_sec_fetch_site     = 'Sec-Fetch-Site: '
    header_sec_fetch_mode     = 'Sec-Fetch-Mode: '
    header_sec_fetch_user     = 'Sec-Fetch-User: '
    header_sec_fetch_dest     = 'Sec-Fetch-Dest: '
    header_sec_ch_ua          = 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"'
    header_sec_ch_ua_mobile   = 'sec-ch-ua-mobile: ?0'
    header_sec_ch_ua_platform = 'sec-ch-ua-platform: "Linux"'

    cookie_phpsessid = cookies[0]
    cookie_noticias = cookies[1]
    cookie_perfil = u'perfil_cookie=' + perfil
    cookie_usuario = u'co_usuario=' + co_campus

    tipo_presencial = tipos[0]
    tipo_ead = tipos[1]
    tipo_fic = tipos[2]

    info_json   = path.join(getenv('SISTEC_DOWNLOAD_APP', default=path.curdir),   '_info.json')
    turmas_json = path.join(getenv('SISTEC_DOWNLOAD_APP', default=path.curdir), '_turmas.json')
    alunos_json = path.join(getenv('SISTEC_DOWNLOAD_APP', default=path.curdir), '_alunos.json')

    # COLUNAS DO ARQUIVO CSV
    # 00: CO_ALUNO_IDENTIFICADO       # 01: CO_ALUNO                     # 02: NO_ALUNO                     # 03: NO_MAE_ALUNO             # 04: NO_NOME_SOCIAL
    # 05: NU_CPF                      # 06: DS_EMAIL                     # 07: CO_PESSOA_FISICA_ALUNO       # 08: DS_SENHA                 # 09: CO_MATRICULA
    # 10: CO_CICLO_MATRICULA          # 11: CO_STATUS_CICLO_MATRICULA    # 12: CO_CURSO                     # 13: NU_CARGA_HORARIA         # 14: DT_DATA_INICIO
    # 15: DT_DATA_FIM_PREVISTO        # 16: CO_UNIDADE_ENSINO            # 17: CO_PERIODO_CADASTRO          # 18: NO_CICLO_MATRICULA       # 19: ST_ATIVO
    # 20: CO_TIPO_OFERTA_CURSO        # 21: CO_TIPO_INSTITUICAO          # 22: CO_INSTITUICAO               # 23: CO_PORTFOLIO             # 24: CO_TIPO_NIVEL_OFERTA_CURSO
    # 25: CO_TIPO_PROGRAMA_CURSO      # 26: ST_CARGA                     # 27: DT_DATA_FINALIZADO           # 28: NU_VAGAS_OFERTADAS       # 29: NU_TOTAL_INSCRITOS
    # 30: ST_ETEC                     # 31: CO_POLO                      # 32: UAB                          # 33: ST_PREVISTO              # 34: NU_VAGAS_PREVISTAS
    # 35: CO_CURSO_SUPERIOR_CORRELATO # 36: NU_CARGA_HORARIA_ESTAGIO     # 37: NO_ARQUIVO                   # 38: NO_CAMINHO_ARQUIVO       # 39: ST_ESTAGIO
    # 40: ST_EXPERIMENTAL             # 41: NO_STATUS_MATRICULA          # 42: CO_PESSOA_FISICA             # 43: NU_RG                    # 44: SG_SEXO
    # 45: DT_NASCIMENTO               # 46: NO_PESSOA_FISICA             # 47: CO_PESSOA                    # 48: CO_CARGO                 # 49: DS_ORGAO_EXPEDIDOR
    # 50: SG_UF_ORG_EXPED             # 51: DS_CARGO                     # 52: CO_UNIDADE_ENSINO_IMPORTACAO # 53: CO_MATRICULA_RESPONSAVEL # 54: NO_SOCIAL
    # USAREMOS AS SEGUINTES COLUNAS:
    # 02; 05; 12; 14; 15; 18; 20; 31; 41

    file_csv.write('NO_ALUNO;'
                   'NU_CPF;'
                   'CO_CURSO;'
                   'DT_DATA_INICIO;'
                   'DT_DATA_FIM_PREVISTO;'
                   'NO_CICLO_MATRICULA;'
                   'CO_TIPO_OFERTA_CURSO;'
                   'CO_POLO;'
                   'NO_STATUS_MATRICULA'
                   '\n')
    file_csv.flush()
    fsync(file_csv.fileno())

    # =================
    # DOWNLOAD: ETAPA 1
    # =================
    """
    curl 'https://sistec.mec.gov.br/index/index' \
        -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36' \
        -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
        -H 'Accept-Language: pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7' \
        -H 'Cache-Control: max-age=0' \
        -H 'Connection: keep-alive' \
        -H 'Content-Type: application/x-www-form-urlencoded' \
        -H 'Upgrade-Insecure-Requests: 1' \
        -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"' \
        -H 'sec-ch-ua-mobile: ?0' \
        -H 'sec-ch-ua-platform: "Linux"' \
        -H 'Sec-Fetch-Dest: document' \
        -H 'Sec-Fetch-Mode: navigate' \
        -H 'Sec-Fetch-Site: same-origin' \
        -H 'Sec-Fetch-User: ?1' \
        -H 'Origin: https://sistec.mec.gov.br' \
        -H 'Referer: https://sistec.mec.gov.br/index/selecionarinstituicao/' \
        -H 'Cookie: sistecNoticias=0; PHPSESSID=j8fhj595d4irervirp5cv1tombl0kcp7; perfil_cookie=GESTOR+DA+UNIDADE+DE+ENSINO; co_usuario=1660637' \
        --data-raw 'tipo=1660670&acao=&qtdPerfis=14' \
        --compressed
    """

    if level_debug[0]:
        with open(file_log_path, 'a') as file_log:
            file_log.write(u'%s Acessando o peril do %s ...\n' % (strftime('[%Y-%m-%d %H:%M:%S]'), no_campus.title()))
            file_log.flush()
            fsync(file_log.fileno())
    headers = {}
    http_header = [
        header_user_agent,
        header_accept_text,
        header_accept_lang,
        header_accept_enc,
        header_cache,
        header_connection,
        header_content_type,
        header_upgrade,
        header_sec_ch_ua,
        header_sec_ch_ua_mobile,
        header_sec_ch_ua_platform,
        header_sec_fetch_dest + 'document',
        header_sec_fetch_mode + 'navigate',
        header_sec_fetch_site + 'same-origin',
        header_sec_fetch_user + '?1',
        'Origin: ' + host_sistec,
        'Referer: ' + url_alterar_perfil,
        'Cookie: ' + cookie_noticias + '; ' + cookie_phpsessid + '; ' + cookie_perfil + '; ' + cookie_usuario
    ]
    http_header[-1] = http_header[-1].encode(encoding)
    http_header[-2] = http_header[-2].encode(encoding)
    http_header[-3] = http_header[-3].encode(encoding)
    post_data = {
        'tipo': co_campus,
        'acao': '',
        'qtdPerfis': str(qtdPerfis)
    }
    postfields = urlencode(post_data)
    buff = BytesIO()
    c = pycurl.Curl()
    c.setopt(c.URL, url_index)
    c.setopt(c.SSL_VERIFYPEER, 1)
    c.setopt(c.SSL_VERIFYHOST, 2)
    c.setopt(c.TCP_NODELAY, 1)
    c.setopt(c.TCP_FASTOPEN, 1)
    c.setopt(c.CONNECTTIMEOUT, CON_TIMEOUT)
    c.setopt(c.TIMEOUT, REQ_TIMEOUT)
    # c.setopt(c.CAINFO, path.join(path.curdir, 'curl-ca-bundle.crt'))
    c.setopt(c.WRITEFUNCTION, buff.write)
    c.setopt(c.HTTPHEADER, http_header)
    c.setopt(c.HEADERFUNCTION, _pycurl_header)
    c.setopt(c.CUSTOMREQUEST, 'POST')
    c.setopt(c.POSTFIELDS, postfields)
    if level_debug[2]:
        c.setopt(c.VERBOSE, 1)
        c.setopt(c.DEBUGFUNCTION, _pycurl_debug)
    c.perform()
    c.close()
    if level_debug[1]:
        with open(file_log_path, 'a') as file_log:
            file_log.write(u'%s post_data:\t%s\n' % (strftime('[%Y-%m-%d %H:%M:%S]'), post_data))
            file_log.flush()
            fsync(file_log.fileno())

    lines = []
    pagina_ciclos = 1
    total_paginas_ciclos = '#'

    while True:
        # =================
        # DOWNLOAD: ETAPA 2
        # =================
        """
        curl 'https://sistec.mec.gov.br/tempo-sessao/get-tempo-sessao/' \
            -X 'POST' \
            -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36' \
            -H 'Accept: application/json' \
            -H 'Accept-Language: pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7' \
            -H 'Connection: keep-alive' \
            -H 'Content-Length: 0' \
            -H 'Content-type: application/x-www-form-urlencoded; charset=UTF-8' \
            -H 'X-Request: JSON' \
            -H 'X-Requested-With: XMLHttpRequest' \
            -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"' \
            -H 'sec-ch-ua-mobile: ?0' \
            -H 'sec-ch-ua-platform: "Linux"' \
            -H 'Sec-Fetch-Dest: empty' \
            -H 'Sec-Fetch-Mode: cors' \
            -H 'Sec-Fetch-Site: same-origin' \
            -H 'Origin: https://sistec.mec.gov.br' \
            -H 'Referer: https://sistec.mec.gov.br/index/index' \
            -H 'Cookie: sistecNoticias=0; PHPSESSID=j8fhj595d4irervirp5cv1tombl0kcp7; perfil_cookie=GESTOR+DA+UNIDADE+DE+ENSINO; co_usuario=1660670' \
            --compressed
        """
        if level_debug[0]:
            with open(file_log_path, 'a') as file_log:
                file_log.write(u'%s Tempo de sessão do %s ... ' % (strftime('[%Y-%m-%d %H:%M:%S]'), no_campus.title()))
                file_log.flush()
                fsync(file_log.fileno())
        headers = {}
        http_header = [
            header_user_agent,
            header_accept_json,
            header_accept_lang,
            header_accept_enc,
            header_connection,
            header_content_len,
            header_content_type,
            header_xreq_json,
            header_xreq_xml,
            header_sec_ch_ua,
            header_sec_ch_ua_mobile,
            header_sec_ch_ua_platform,
            header_sec_fetch_dest + 'empty',
            header_sec_fetch_mode + 'cors',
            header_sec_fetch_site + 'same-origin',
            'Origin: ' + host_sistec,
            'Referer: ' + url_index,
            'Cookie: ' + cookie_noticias + '; ' + cookie_phpsessid + '; ' + cookie_perfil + '; ' + cookie_usuario
        ]
        http_header[-1] = http_header[-1].encode(encoding)
        http_header[-2] = http_header[-2].encode(encoding)
        http_header[-3] = http_header[-3].encode(encoding)
        data_buffer = BytesIO()
        c = pycurl.Curl()
        c.setopt(c.URL, url_tempo_sessao)
        c.setopt(c.SSL_VERIFYPEER, 1)
        c.setopt(c.SSL_VERIFYHOST, 2)
        c.setopt(c.TCP_NODELAY, 1)
        c.setopt(c.TCP_FASTOPEN, 1)
        c.setopt(c.CONNECTTIMEOUT, CON_TIMEOUT)
        c.setopt(c.TIMEOUT, REQ_TIMEOUT)
        # c.setopt(c.CAINFO, path.join(path.curdir, 'curl-ca-bundle.crt'))
        c.setopt(c.WRITEDATA, data_buffer)
        c.setopt(c.HTTPHEADER, http_header)
        c.setopt(c.HEADERFUNCTION, _pycurl_header)
        c.setopt(c.CUSTOMREQUEST, 'POST')
        if level_debug[2]:
            c.setopt(c.VERBOSE, 1)
            c.setopt(c.DEBUGFUNCTION, _pycurl_debug)
        c.perform()
        c.close()
        tempo_sessao = data_buffer.getvalue().decode(encoding)
        if level_debug[0]:
            with open(file_log_path, 'a') as file_log:
                file_log.write(u'%s\n' % tempo_sessao)
                file_log.flush()
                fsync(file_log.fileno())

        """
        curl 'https://sistec.mec.gov.br/gridciclo/turmas' \
            -X 'POST' \
            -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36' \
            -H 'Accept: application/json' \
            -H 'Accept-Language: pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7' \
            -H 'X-Requested-With: XMLHttpRequest' \
            -H 'X-Request: JSON' \
            -H 'Content-type: application/x-www-form-urlencoded; charset=UTF-8' \
            -H 'Connection: keep-alive' \
            -H 'Content-Length: 0' \
            -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"' \
            -H 'sec-ch-ua-mobile: ?0' \
            -H 'sec-ch-ua-platform: "Linux"' \
            -H 'Sec-Fetch-Dest: empty' \
            -H 'Sec-Fetch-Mode: cors' \
            -H 'Sec-Fetch-Site: same-origin' \
            -H 'Origin: https://sistec.mec.gov.br' \
            -H 'Referer: https://sistec.mec.gov.br/index/index' \
            -H 'Cookie: sistecNoticias=0; PHPSESSID=j8fhj595d4irervirp5cv1tombl0kcp7; perfil_cookie=GESTOR+DA+UNIDADE+DE+ENSINO; co_usuario=1660670' \
            --compressed

        pagina=1
        while (True):
            ...
            if pagina < totalPaginas:
                pagina = pagina + 1
                continue
            break
        """
        if level_debug[0]:
            with open(file_log_path, 'a') as file_log:
                file_log.write(u'%s Acessando a página %s/%s da lista de ciclos do %s ...\n' % (
                    strftime('[%Y-%m-%d %H:%M:%S]'), pagina_ciclos, total_paginas_ciclos, no_campus.title()))
                file_log.flush()
                fsync(file_log.fileno())
        headers = {}
        http_header = [
            header_user_agent,
            header_accept_json,
            header_accept_lang,
            header_accept_enc,
            header_xreq_xml,
            header_xreq_json,
            header_content_type,
            header_connection,
            header_content_len,
            header_sec_ch_ua,
            header_sec_ch_ua_mobile,
            header_sec_ch_ua_platform,
            header_sec_fetch_dest + 'empty',
            header_sec_fetch_mode + 'cors',
            header_sec_fetch_site + 'same-origin'
            'Origin: ' + host_sistec,
            'Referer: ' + url_index,
            'Cookie: ' + cookie_noticias + '; ' + cookie_phpsessid + '; ' + cookie_perfil + '; ' + cookie_usuario
        ]
        http_header[-1] = http_header[-1].encode(encoding)
        http_header[-2] = http_header[-2].encode(encoding)
        http_header[-3] = http_header[-3].encode(encoding)
        post_data = {'pagina': pagina_ciclos}
        postfields = urlencode(post_data)

        count = 0
        while True:
            try:
                file_json = open(turmas_json, mode='wb')
                c = pycurl.Curl()
                c.setopt(c.URL, url_turmas)
                c.setopt(c.SSL_VERIFYPEER, 1)
                c.setopt(c.SSL_VERIFYHOST, 2)
                c.setopt(c.TCP_NODELAY, 1)
                c.setopt(c.TCP_FASTOPEN, 1)
                c.setopt(c.CONNECTTIMEOUT, CON_TIMEOUT)
                c.setopt(c.TIMEOUT, REQ_TIMEOUT)
                # c.setopt(c.CAINFO, path.join(path.curdir, 'curl-ca-bundle.crt'))
                c.setopt(c.WRITEDATA, file_json)
                c.setopt(c.HTTPHEADER, http_header)
                c.setopt(c.HEADERFUNCTION, _pycurl_header)
                c.setopt(c.CUSTOMREQUEST, 'POST')
                c.setopt(c.POSTFIELDS, postfields)
                if level_debug[2]:
                    c.setopt(c.VERBOSE, 1)
                    c.setopt(c.DEBUGFUNCTION, _pycurl_debug)
                c.perform()
                c.close()
                file_json.close()
                file_json_r = open(turmas_json, mode='rb')
                turmas_data = json.load(file_json_r)
                file_json_r.close()
                if level_debug[0]:
                    with open(file_log_path, 'a') as file_log:
                        file_log.write(
                            u'%s %d tentativa(s): ...\n' %
                            (strftime('[%Y-%m-%d %H:%M:%S]'), count + 1))
                        file_log.flush()
                        fsync(file_log.fileno())
                if level_debug[1]:
                    with open(file_log_path, 'a') as file_log:
                        file_log.write(u'%s turmas_data:\t%s\n' % (
                            strftime('[%Y-%m-%d %H:%M:%S]'), turmas_data))
                        file_log.flush()
                        fsync(file_log.fileno())
                ciclos = turmas_data['dados']
            except Exception as erro:
                count = count + 1
                # tenta 10 vezes
                if count > 9:
                    return (False, u"%s" % erro)
                else:
                    continue
            else:
                break

        for i in range(len(ciclos)):

            # PEGA O CODIGO DO CICLO:
            co_ciclo = ciclos[i]['Código do Ciclo']

            # PEGA OS DADOS DO CURSO:
            """
            curl 'https://sistec.mec.gov.br/admciclomatricula/dadosciclo/' \
                -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36' \
                -H 'Accept: application/json' \
                -H 'Accept-Language: pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7' \
                -H 'X-Requested-With: XMLHttpRequest' \
                -H 'X-Request: JSON' \
                -H 'Connection: keep-alive' \
                -H 'Content-type: application/x-www-form-urlencoded; charset=UTF-8' \
                -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"' \
                -H 'sec-ch-ua-mobile: ?0' \
                -H 'sec-ch-ua-platform: "Linux"' \
                -H 'Sec-Fetch-Dest: empty' \
                -H 'Sec-Fetch-Mode: cors' \
                -H 'Sec-Fetch-Site: same-origin' \
                -H 'Origin: https://sistec.mec.gov.br' \
                -H 'Referer: https://sistec.mec.gov.br/index/index' \
                -H 'Cookie: sistecNoticias=0; PHPSESSID=j8fhj595d4irervirp5cv1tombl0kcp7; perfil_cookie=GESTOR+DA+UNIDADE+DE+ENSINO; co_usuario=1660670' \
                --data-raw 'idCiclo=1949843' \
                --compressed
            """
            headers = {}
            http_header = [
                header_user_agent,
                header_accept_json,
                header_accept_lang,
                header_accept_enc,
                header_xreq_xml,
                header_xreq_json,
                header_connection,
                header_content_type,
                header_sec_ch_ua,
                header_sec_ch_ua_mobile,
                header_sec_ch_ua_platform,
                header_sec_fetch_dest + 'empty',
                header_sec_fetch_mode + 'cors',
                header_sec_fetch_site + 'same-origin',
                'Origin: ' + host_sistec,
                'Referer: ' + url_index,
                'Cookie: ' + cookie_noticias + '; ' + cookie_phpsessid + '; ' + cookie_perfil + '; ' + cookie_usuario
            ]
            http_header[-1] = http_header[-1].encode(encoding)
            http_header[-2] = http_header[-2].encode(encoding)
            http_header[-3] = http_header[-3].encode(encoding)
            post_data = {'idCiclo': co_ciclo}
            postfields = urlencode(post_data)
            file_json = open(info_json, mode='wb')
            c = pycurl.Curl()
            c.setopt(c.URL, url_dados_ciclo)
            c.setopt(c.SSL_VERIFYPEER, 1)
            c.setopt(c.SSL_VERIFYHOST, 2)
            c.setopt(c.TCP_NODELAY, 1)
            c.setopt(c.TCP_FASTOPEN, 1)
            c.setopt(c.CONNECTTIMEOUT, CON_TIMEOUT)
            c.setopt(c.TIMEOUT, REQ_TIMEOUT)
            # c.setopt(c.CAINFO, path.join(path.curdir, 'curl-ca-bundle.crt'))
            c.setopt(c.WRITEDATA, file_json)
            c.setopt(c.HTTPHEADER, http_header)
            c.setopt(c.HEADERFUNCTION, _pycurl_header)
            c.setopt(c.CUSTOMREQUEST, 'POST')
            c.setopt(c.POSTFIELDS, postfields)
            if level_debug[2]:
                c.setopt(c.VERBOSE, 1)
                c.setopt(c.DEBUGFUNCTION, _pycurl_debug)
            c.perform()
            c.close()
            file_json.close()
            file_json_r = open(info_json, mode='rb')
            info_data = json.load(file_json_r)
            file_json_r.close()

            try:
                dados = info_data['mensagem']
            except KeyError as erro:
                return (False, u"%s" % erro)

            # PEGA O CODIGO DO CURSO
            co_curso = dados['co_curso']

            # PEGA O TIPO DE CURSO:
            tipo_curso = ciclos[i]['Tipo do Curso']

            # PEGA O NOME DO CURSO:
            nome_curso = ciclos[i]['Nome do Ciclo']

            # VERIFICA SE CURSO EH DO TIPO FIC:
            re_fic = re.compile(u'^.*FORMA[CÇ][AÃ]O\s*(INICIAL|CONTINUADA).*$')
            match_fic = re_fic.search(tipo_curso)

            # VERIFICA SE CURSO EH DO TIPO MULHERES MIL:
            re_mm = re.compile(u'^.*MULHERES\s*MIL.*$')
            match_mm = re_mm.search(tipo_curso)

            # VERIFICA SE CURSO EH DO TIPO EAD:
            re_ead = re.compile(u'^.*\s*DIST[AÂ]NCIA\s*$')
            match_ead = re_ead.search(nome_curso)

            # VERIFICA SE CURSO EH DO TIPO PRESENCIAL:
            re_presencial = re.compile(u'^.*\s*PRESENCIAL\s*$')
            match_presencial = re_presencial.search(nome_curso)

            if match_fic:
                if not tipo_fic:
                    continue
            elif match_mm:
                if not tipo_fic:
                    continue
            elif match_ead:
                co_cursos = [
                    338565, 338569, 338571, 338572, 338573, 338574, 338575, 338578, 338580,
                    338581, 338582, 338583, 362343, 362344, 362345, 362346, 362347
                ]
                if (tipo_presencial and int(co_curso) in co_cursos) or (tipo_ead and int(co_curso) not in co_cursos):
                    pass
                else:
                    continue
            elif match_presencial:
                if not tipo_presencial:
                    continue
            else:
                continue
                # return (
                #    False, u"não foi possível determinar o tipo do curso:\t%s" % nome_curso
                # )

            # PEGA AS DATAS DO INICO E DO FIM PREVISTO:
            dt_data_inicio = dados['dt_data_inicio']
            dt_data_fim_previsto = dados['dt_data_fim_previsto']

            # PEGA O NOME DO CICLO DE MATRICULA:
            no_ciclo_matricula = dados['no_ciclo_matricula']
            no_ciclo_matricula = no_ciclo_matricula.strip()

            # PEGA O CODIGO DO TIPO DA OFERTA DO CURSO:
            co_tipo_oferta_curso = dados['co_tipo_oferta_curso']
            if co_tipo_oferta_curso is None:
                co_tipo_oferta_curso = 0

            # PEGA O CODIGO DO POLO DO CURSO:
            d_campus = {
                u'CÂMPUS ÁGUAS LINDAS': 3647,
                u'CÂMPUS ANÁPOLIS': 699,
                u'CÂMPUS APARECIDA DE GOIÂNIA': 210,
                u'CÂMPUS CIDADE DE GOIÁS': 696,
                u'CÂMPUS FORMOSA': 2012,
                u'CÂMPUS GOIÂNIA': 212,
                u'CÂMPUS GOIÂNIA OESTE': 3646,
                u'CÂMPUS INHUMAS': 238,
                u'CÂMPUS ITUMBIARA': 244,
                u'CÂMPUS JATAÍ': 241,
                u'CÂMPUS LUZIÂNIA': 209,
                u'CÂMPUS SENADOR CANEDO': 3648,
                u'CÂMPUS URUAÇU': 161,
                u'CÂMPUS VALPARAÍSO': 3649
            }
            co_polo = d_campus[no_campus.upper()]

            line_common = []
            line_common.append(str(co_curso) + u';')
            line_common.append(dt_data_inicio + u';')
            line_common.append(dt_data_fim_previsto + u';')
            line_common.append(no_ciclo_matricula + u';')
            line_common.append(str(co_tipo_oferta_curso) + u';')
            line_common.append(str(co_polo) + u';')

            continue_alunos = 0
            pagina_alunos = 1
            while True:
                # =================
                # DOWNLOAD: ETAPA 3
                # =================
                """
                curl 'https://sistec.mec.gov.br/gridciclo/listaralunosacao/periodo//ciclo/2678641/coaluno/' \
                    -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36' \
                    -H 'Accept: application/json' \
                    -H 'Accept-Language: pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7' \
                    -H 'X-Requested-With: XMLHttpRequest' \
                    -H 'X-Request: JSON' \
                    -H 'Connection: keep-alive' \
                    -H 'Content-type: application/x-www-form-urlencoded; charset=UTF-8' \
                    -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"' \
                    -H 'sec-ch-ua-mobile: ?0' \
                    -H 'sec-ch-ua-platform: "Linux"' \
                    -H 'Sec-Fetch-Dest: empty' \
                    -H 'Sec-Fetch-Mode: cors' \
                    -H 'Sec-Fetch-Site: same-origin' \
                    -H 'Origin: https://sistec.mec.gov.br' \
                    -H 'Referer: https://sistec.mec.gov.br/index/index' \
                    -H 'Cookie: sistecNoticias=0; PHPSESSID=j8fhj595d4irervirp5cv1tombl0kcp7; perfil_cookie=GESTOR+DA+UNIDADE+DE+ENSINO; co_usuario=1660637' \
                    --data-raw 'registros=30&pagina=2' \
                    --compressed

                pagina=1
                while (True):
                    ...
                    if pagina < totalPaginas:
                        pagina = pagina + 1
                        continue
                    break
                """
                if level_debug[0]:
                    with open(file_log_path, 'a') as file_log:
                        file_log.write(u'%s Acessando a lista de alunos (%s) do ciclo:\t%s ...\n' % (
                            strftime('[%Y-%m-%d %H:%M:%S]'), no_campus.title(), no_ciclo_matricula.title()))
                        #  file_log.write(u'%s url_ciclo = %s + %s + /coaluno/\n' % (
                            #  strftime('[%Y-%m-%d %H:%M:%S]'), url_ciclo_common, co_ciclo))
                        file_log.flush()
                        fsync(file_log.fileno())
                headers = {}
                url_ciclo = url_ciclo_common + str(co_ciclo) + '/coaluno/'

                http_header = [
                    header_user_agent,
                    header_accept_json,
                    header_accept_lang,
                    header_accept_enc,
                    header_xreq_xml,
                    header_xreq_json,
                    header_connection,
                    header_content_type,
                    header_sec_ch_ua,
                    header_sec_ch_ua_mobile,
                    header_sec_ch_ua_platform,
                    header_sec_fetch_dest + 'empty',
                    header_sec_fetch_mode + 'cors',
                    header_sec_fetch_site + 'same-origin',
                    'Origin: ' + host_sistec,
                    'Referer: ' + url_index,
                    'Cookie: ' + cookie_noticias + '; ' + cookie_phpsessid + '; ' + cookie_perfil + '; ' + cookie_usuario
                ]
                http_header[-1] = http_header[-1].encode(encoding)
                http_header[-2] = http_header[-2].encode(encoding)
                http_header[-3] = http_header[-3].encode(encoding)
                post_data = {'registros': 30, 'pagina': pagina_alunos}
                postfields = urlencode(post_data)
                file_json = open(alunos_json, mode='wb')
                c = pycurl.Curl()
                c.setopt(c.URL, url_ciclo)
                c.setopt(c.SSL_VERIFYPEER, 1)
                c.setopt(c.SSL_VERIFYHOST, 2)
                c.setopt(c.TCP_NODELAY, 1)
                c.setopt(c.TCP_FASTOPEN, 1)
                c.setopt(c.CONNECTTIMEOUT, CON_TIMEOUT)
                c.setopt(c.TIMEOUT, REQ_TIMEOUT)
                # c.setopt(c.CAINFO, path.join(path.curdir, 'curl-ca-bundle.crt'))
                c.setopt(c.WRITEDATA, file_json)
                c.setopt(c.HTTPHEADER, http_header)
                c.setopt(c.HEADERFUNCTION, _pycurl_header)
                c.setopt(c.CUSTOMREQUEST, 'POST')
                c.setopt(c.POSTFIELDS, postfields)
                if level_debug[2]:
                    c.setopt(c.VERBOSE, 1)
                    c.setopt(c.DEBUGFUNCTION, _pycurl_debug)
                c.perform()
                c.close()
                file_json.close()
                file_json_r = open(alunos_json, mode='rb')
                alunos_data = json.load(file_json_r)
                file_json_r.close()
                if level_debug[1]:
                    with open(file_log_path, 'a') as file_log:
                        file_log.write(u'%s url_ciclo:\t%s\n' % (
                            strftime('[%Y-%m-%d %H:%M:%S]'), url_ciclo))
                        file_log.write(u'%s http_header:\t%s\n' % (
                            strftime('[%Y-%m-%d %H:%M:%S]'), http_header))
                        file_log.write(u'%s post_data:\t%s\n' % (
                            strftime('[%Y-%m-%d %H:%M:%S]'), post_data))
                        file_log.write(u'%s alunos_data:\t%s\n' % (
                            strftime('[%Y-%m-%d %H:%M:%S]'), alunos_data))
                        file_log.flush()
                        fsync(file_log.fileno())

                try:
                    alunos = alunos_data['dados']
                except KeyError:
                    try:
                        total_paginas_alunos = alunos_data['totalPaginas']
                    except Exception as erro:
                        return (False, u"%s" % erro)
                    # print(pagina_alunos)
                    # print(total_paginas_alunos)
                    if pagina_alunos < total_paginas_alunos:
                        pagina_alunos = pagina_alunos + 1
                        continue_alunos = 1
                    else:
                        continue_alunos = 2

                if continue_alunos == 1:
                    continue_alunos = 0
                    continue # while alunos
                elif continue_alunos == 2:
                    continue_alunos = 0
                    break # while alunos

                for j in range(len(alunos)):

                    line = copy(line_common)

                    # PEGA O NOME DO ALUNO:
                    nome_aluno = alunos[j]['Aluno']
                    if level_debug[1]:
                        with open(file_log_path, 'a') as file_log:
                            file_log.write(u'%s nome_aluno:\t%s\n' % (
                                strftime('[%Y-%m-%d %H:%M:%S]'), nome_aluno))
                            file_log.flush()
                            fsync(file_log.fileno())
                    nome_aluno = nome_aluno.strip() + ';'
                    line.insert(0, nome_aluno)

                    # PEGA O CPF DO ALUNO:
                    cpf_aluno = alunos[j]['CPF']
                    if level_debug[1]:
                        with open(file_log_path, 'a') as file_log:
                            file_log.write(u'%s cpf_aluno:\t%s\n' % (
                                strftime('[%Y-%m-%d %H:%M:%S]'), cpf_aluno))
                            file_log.flush()
                            fsync(file_log.fileno())
                    cpf_aluno = cpf_aluno.strip() + ';'
                    line.insert(1, cpf_aluno)

                    # PEGA O STATUS DO ALUNO:
                    status_aluno = alunos[j]['Status']
                    if level_debug[1]:
                        with open(file_log_path, 'a') as file_log:
                            file_log.write(u'%s status_aluno:\t%s\n' % (
                                strftime('[%Y-%m-%d %H:%M:%S]'), status_aluno))
                            file_log.flush()
                            fsync(file_log.fileno())
                    status_aluno = status_aluno.strip()
                    line.append(status_aluno)

                    # ADICIONA UMA LINHA DA PLANILHA NO ARRAY "lines":
                    line.append(u'\n')
                    line = u''.join(line)
                    try:
                        new_line = line.encode(encoding)
                    except UnicodeEncodeError:
                        new_line = limpa_linha(line, encoding)
                    line = new_line.decode(encoding)
                    if line not in lines:
                        lines.append(line)

                try:
                    total_paginas_alunos = alunos_data['totalPaginas']
                except Exception as erro:
                    return (False, u"%s" % erro)

                if pagina_alunos < total_paginas_alunos:
                    pagina_alunos = pagina_alunos + 1
                    continue  # while alunos
                else:
                    break  # while alunos

        try:
            total_paginas_ciclos = turmas_data['totalPaginas']
        except Exception as erro:
            return (False, u"%s" % erro)

        if pagina_ciclos < total_paginas_ciclos:
            pagina_ciclos = pagina_ciclos + 1
            continue  # while ciclos
        else:
            break  # while ciclos

    if lines:
        try:
            file_csv.writelines(lines)
            file_csv.flush()
            fsync(file_csv.fileno())
        except Exception as erro:
            return (False, u"%s" % erro)

    return (True, u'OK')

def limpa_linha(line, encoding):
    new_line = []
    for i in range(len(line)):
        try:
            new_line.append(line[i].encode(encoding))
        except UnicodeEncodeError:
            new_line.append(u'_'.encode(encoding))
    for i in range(len(new_line)):
        new_line[i] = new_line[i].decode(encoding)
    new_line = u''.join(new_line)
    new_line = new_line.encode(encoding)
    return (new_line)

def _pycurl_debug(debug_type, debug_msg):
    global pydbg_path
    with open(pydbg_path, 'a') as pydbg_file:
        pydbg_file.write(u'%s DEBUG(%d):\t%s\n' % (strftime('[%Y-%m-%d %H:%M:%S]'), debug_type, debug_msg))
        pydbg_file.flush()
        fsync(pydbg_file.fileno())

def _pycurl_header(header_line):
    global headers, encoding
    header_line = header_line.decode(encoding)
    if ':' not in header_line:
        return
    name, value = header_line.split(':', 1)
    name = name.strip()
    name = name.lower()
    value = value.strip()
    if name == 'set-cookie':
        cookie, dados = value.split(';', 1)
        cookie = cookie.strip()
        cookie_name, cookie_value = cookie.split('=', 1)
        cookie_name = cookie_name.strip()
        cookie_value = cookie_value.strip()
    if name not in headers:
        headers[name] = value
    return


from flask import Flask, request, json
import subprocess

app = Flask(__name__)

@app.route('/githubIssue', methods=['POST'])
def githubIssue():
    data = request.json
    print(f'Issue {data["issue"]["title"]} {data["action"]}')
    print(f'{data["issue"]["body"]}')
    print(f'{data["issue"]["url"]}')
    subprocess.call(['sh', './update.sh'])
    return data

if __name__ == '__main__':
    app.run(debug=True)


# -*- coding: utf-8 -*-

import math
import cairo
from geom_utils import aux_cairo_plot

import matplotlib as mpl
mpl.rc('figure', max_open_warning = 0)
import matplotlib.pyplot as plt
from misc_utils import log_messages as lmsg
from PIL import Image
from pathlib import Path

__author__= "Luis C. Pérez Tato (LCPT) and Ana Ortega (AO_O)"
__copyright__= "Copyright 2015, LCPT and AO_O"
__license__= "GPL"
__version__= "3.0"
__email__= "l.pereztato@ciccp.es" "ana.Ortega@ciccp.es"

def eps2png(inputFileName, outputFileName= None):
    ''' Convert the encapsulated Postscript input image into a PNG image
        with a "decent" resolution.

    :param inputFileName: name of the input file.
    :param outputFileName: name of the output file.
    '''
    eps_image = Image.open(inputFileName)
    # Rasterise onto 4x higher resolution grid
    eps_image.load(scale=4)
    if(outputFileName is None):
        pth= Path(inputFileName)
        outputFileName= str(pth.with_suffix(''))+'.png'
    eps_image.save(outputFileName)

def mplot_section_reinforcement(ax, reinforcement):
    ''' Plot the geometry of the section using matplotlib.

    :param ax: matplotlib subplot.
    :param reinforcement: reinforcement layers to draw.
    '''
    # Plot reinforcement.
    reinfLayersColors= ['black', 'blue', 'darkblue', 'red', 'darkred', 'darkgreen', 'purple']
    numColors= len(reinfLayersColors)
    for idx, reinfLayer in enumerate(reinforcement):
        rebars= reinfLayer.getReinfBars
        rebarColor= reinfLayersColors[idx % numColors]
        for b in rebars:
            ptPlot= b.getPos2d # bar position.
            rPlot= b.diameter/2.0 # bar radius.
            labelPlot= str(int(round(b.diameter*1e3))) # bar label.
            circle= plt.Circle((ptPlot.x, ptPlot.y), rPlot, color= rebarColor)
            ax.add_patch(circle)
            ax.annotate(labelPlot, (ptPlot.x+rPlot, ptPlot.y+rPlot))

def mplot_section_geometry(ax, sectionGeometry):
    ''' Plot the geometry of the section using matplotlib.

    :param ax: matplotlib subplot.
    :param sectionGeometry: geometry of the RC section.
    '''
    ax.axis('equal')
    ax.set_title('Section: '+sectionGeometry.name)
    ax.grid(visible= True, linestyle='dotted')
    # Plot contour.
    contour= sectionGeometry.getRegionsContour()
    x= list(); y= list()
    for p in contour.getVertices():
        x.append(p.x)
        y.append(p.y)
    ax.fill(x,y,'tab:gray')
    # Plot reinforcement.
    mplot_section_reinforcement(ax, sectionGeometry.getReinfLayers)

def plot_reinforcement(reinforcement, ctx):
    '''draw section rebars in a postcript file.

    :param reinforcement: reinforcement to plot.
    :param ctx: cairo context.
    '''
    for reinfLayer in reinforcement:
        rebars= reinfLayer.getReinfBars
        for b in rebars:
            ptPlot= b.getPos2d
            rPlot= b.diameter/2.0
            labelPlot= str(int(round(b.diameter*1e3)))
            ctx.move_to(ptPlot.x+rPlot,ptPlot.y)
            ctx.arc(ptPlot.x,ptPlot.y,rPlot,0.0,2*math.pi)
            ctx.stroke_preserve()
            ctx.fill()
            ctx.move_to(ptPlot.x+1.3*rPlot,ptPlot.y)
            ctx.text_path(labelPlot)
            ctx.stroke()

def plot_section_geometry(geomSection, path):
    ''' draws section geometry in a postscript file.

    :param geomSection: section geometry to draw.
    :param path: output file path.
    '''
    WIDTH, HEIGHT = 256, 256
    surface = cairo.PSSurface(path, WIDTH, HEIGHT)
    ctx = cairo.Context(surface)
    regions= geomSection.getRegions
    bnd= regions.getBnd
    trf= aux_cairo_plot.TransformParams(WIDTH, HEIGHT,aux_cairo_plot.Boundary(bnd.getXMin,bnd.getYMin,bnd.getXMax,bnd.getYMax))
    trf.applyTransform(ctx)# Normalizing the canvas
    ctx.set_line_width(trf.scale/100000)
    ctx.set_source_rgb(0.0, 0.0, 0.0) # Black solid color
    for r in regions:
        trf.plotPolygon(r.getPolygon(),ctx)
    ctx.stroke()
    ctx.set_line_width(trf.scale/300000)
    reinforcement= geomSection.getReinfLayers
    plot_reinforcement(reinforcement,ctx)
    trf.plotYZAxis(ctx)
    surface.set_eps(True)
    ctx.show_page()
    surface.finish()

class FibSectFeaturesToplot(object):
    '''Class to generate python plots of the selected features associated with
    a fiber section.

    :ivar fiberSection: fiber-section to plot
    :ivar colorNeutralAxis: color to display the Neutral axis (defaults to None)
    :ivar colorBendingPlane: color to display the beding plane (defaults to None)
    :ivar colorCompressionPlane: color to display the compression plane (defaults to None)
    :ivar colorTensionPlane: color to display the tension plane (defaults to None)
    :ivar colorIntForcAxis: color to display the internal forces axis (defaults to None)
    :ivar colorLeverArm: color to display the lever arm  (defaults to None)
    :ivar colorEffDepth: color to display the effective depth (defaults to None)
    :ivar colorEffConcrArea: color to display the limit of the effective concrete area (defaults to None)
    :ivar MaxEffHeight: maximum effective height to calculate effective concrete area (defaults to None)
    :ivar colorGrossEffConcrAreaContours: color to display the contours of the gross effective concrete area  (defaults to None)
    '''
    def __init__(self,fiberSection,colorNeutralAxis=None,colorBendingPlane=None,colorCompressionPlane=None,colorTensionPlane=None,colorIntForcAxis=None,colorLeverArm=None,colorEffDepth=None,colorEffConcrArea=None,MaxEffHeight=None,colorGrossEffConcrAreaContours=None):
        self.fiberSection=fiberSection
        self.colorNeutralAxis=colorNeutralAxis
        self.colorBendingPlane=colorBendingPlane
        self.colorCompressionPlane=colorCompressionPlane
        self.colorTensionPlane=colorTensionPlane
        self.colorIntForcAxis=colorIntForcAxis
        self.colorLeverArm=colorLeverArm
        self.colorEffDepth=colorEffDepth
        self.colorEffConcrArea=colorEffConcrArea
        self.MaxEffHeight=MaxEffHeight
        self.colorGrossEffConcrAreaContours=colorGrossEffConcrAreaContours

    def generatePlot(self):
        self.contour=self.fiberSection.getRegionsContour()
        fig = plt.figure()
        ax2d=plt.subplot(111)
        ax2d.axis('equal')
        #Fiber section contour
        (y,z)=data_xcpolygon_to_pyplot(self.contour)
        ax2d.plot(y,z,'k',label='Cross-section')
        for reinfLayer in self.fiberSection.getSectionGeometry.getReinfLayers:
            bars=reinfLayer.getReinfBars
            for b in bars:
                cent= b.getPos2d
                R= b.diameter/2.0
                ax2d.add_artist(plt.Circle((cent[0],cent[1]),R,color='black'))
        #Neutral axis
        if self.colorNeutralAxis is not None:
            (y,z)=data_axis_to_pyplot(self.fiberSection.getNeutralAxis(),self.contour)
            ax2d.plot(y,z,self.colorNeutralAxis,label='Neutral axis')
        #Bending plane
        if self.colorBendingPlane is not None:
            (y,z)=data_axis_to_pyplot(self.fiberSection.getBendingPlaneTrace(),self.contour)
            ax2d.plot(y,z,self.colorBendingPlane,label='Bending plane')
        #Compression plane
        if self.colorCompressionPlane is not None:
            (y,z)=data_axis_to_pyplot(self.fiberSection.getCompressedPlaneTrace(),self.contour)
            ax2d.plot(y,z,self.colorCompressionPlane,label='Compression plane')
        #Tension plane
        if self.colorTensionPlane is not None:
            (y,z)=data_axis_to_pyplot(self.fiberSection.getTensionedPlaneTrace(),self.contour)
            ax2d.plot(y,z,self.colorTensionPlane,label='Tension plane')
        #Internal forces axis
        if self.colorIntForcAxis is not None:
            (y,z)=data_axis_to_pyplot(self.fiberSection.getInternalForcesAxes(),self.contour)
            ax2d.plot(y,z,self.colorIntForcAxis ,label='Internal forces axis')
        #Lever arm
        if self.colorLeverArm is not None:
            (y,z)= data_xcsegment_to_pyplot(self.fiberSection.getLeverArmSegment())
            ax2d.plot(y,z,self.colorLeverArm,label='Lever arm')
        #Effective depth
        if self.colorEffDepth is not None:
            (y,z)= data_xcsegment_to_pyplot(self.fiberSection.getEffectiveDepthSegment())
            ax2d.plot(y,z,self.colorEffDepth,label='Effective depth')
        #Limit of concrete effective area
        if self.colorEffConcrArea is not None:
            if self.MaxEffHeight is None:
                lmsg.error("I can't plot the limit of the effective concrete area, a value to MaxEffHeight must be provided \n")
            else:
                (y,z)=data_axis_to_pyplot(self.fiberSection.getEffectiveConcreteAreaLimitLine(self.MaxEffHeight),self.contour)
                ax2d.plot(y,z,self.colorEffConcrArea,label='Limit of effective concrete area')
        if self.colorGrossEffConcrAreaContours is not None:
            if self.MaxEffHeight is None:
                lmsg.error("I can't plot the contours of the gross effective concrete area, a value to MaxEffHeight must be provided \n")
            else:
                countLst=self.fiberSection.getGrossEffectiveConcreteAreaContour(self.MaxEffHeight)
                label='Contours of gross effective concrete area'
                for pol in countLst:
                    (y,z)=data_xcpolygon_to_pyplot(pol)
                    ax2d.plot(y,z,self.colorGrossEffConcrAreaContours,label=label)
                    label=None
        ax2d.legend()
        return fig,ax2d

def data_xcpolygon_to_pyplot(XCpolygon):
    '''Return the lists [x1,x2,x3,...] and [y1,y2,y3,...] with the coordinates
    of the vertexes of the XCpolygon passed as parameter.
    '''
    nv=XCpolygon.getNumVertices()
    x=[XCpolygon.getVertice(i).x for i in range(nv)]
    x.append(x[0])
    y=[XCpolygon.getVertice(i).y for i in range(nv)]
    y.append(y[0])
    return(x,y)

def data_xcsegment_to_pyplot(XCsegment):
    '''Return the lists [x1,x2] and [y1,y2] with the coordinates
    of the start and end points of the XCsegment passed as parameter.
    '''
    stPt=XCsegment.getFromPoint()
    endPt=XCsegment.getToPoint()
    return ([stPt[0],endPt[0]],[stPt[1],endPt[1]])

def data_axis_to_pyplot(axis,contour):
    '''Return the lists [x1,x2] and [y1,y2] with the coordinates
    of the start and end points of the segment of axis inside the fiberSect contour
    passed as parameters.
    '''
    sg= contour.clip(axis)
    return data_xcsegment_to_pyplot(sg)


import pygame
from pygame.locals import *

import game_loop
import game_level_view

class Application():
    """
    The main application window
    """

    WINDOW_TITLE = "Snake Pit"
    ICON_PATH = "./resources/images/WindowIcon.png"

    def __init__(self):
        pygame.init()
        pygame.display.set_caption(self.WINDOW_TITLE)
        window_icon = pygame.image.load(self.ICON_PATH)
        pygame.display.set_icon(window_icon)

        fov = game_level_view.GameLevelView.FIELD_OF_VIEW
        cell_dim = game_level_view.GameLevelView.GRID_CELL
        display_dim = game_level_view.GameLevelView.DISPLAY_BAR_CELL
        width = fov.width * cell_dim.width
        height = fov.height * cell_dim.height + display_dim.height
        screen_size = (width, height)

        self.screen = pygame.display.set_mode(screen_size)

    def run(self):
        game_loop.GameLoop(self.screen)


import boto3
import pandas as pd
import openai
import json
import numpy as np
from io import StringIO
from botocore.exceptions import ClientError
import os

from dotenv import load_dotenv

# Load environment variables from .env file
load_dotenv('.env')

# Set the OpenAI API key from the environment variable
openai.api_key = os.environ['API_KEY']

s3 = boto3.client('s3')
def get_embedding(text, model):
    embedding = openai.Embedding.create(
    input=text, model=model
    )["data"][0]["embedding"]
    return embedding

def cosine_similarity(a, b):
    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))

def lambda_handler(event, context):
    print("New Event Recieved")
    print(event)

    Body=json.loads(event['body'])
    question = Body['question']
    folder_name = Body['folder_name']

    print(f"Folder Name - {folder_name}")
    print(f"Question - {question}")

    # Get the CSV file from S3
    try:
        print("Loading S3 Bucket")
        obj = s3.get_object(Bucket='csvfiles100', Key=f'{folder_name}/embedding.csv')
        print("S3 Bucket Loaded Successfully")
        print("Decoding Body")
        body = obj['Body'].read().decode('utf-8')
        csv_file = StringIO(body)
        print("Body Decoded Successfully")
    except ClientError as e:
        print(e)
        raise Exception("Error getting S3 object: {}".format(e))

    # Preprocess the data
    emb = pd.read_csv(csv_file)
    print("Dataframe Initialised")
    emb['embedding'] = emb['embedding'].apply(lambda x: '[]' if x is None or not isinstance(x, str) else x)
    emb['embedding'] = emb['embedding'].apply(eval)
    emb['embedding'] = emb['embedding'].apply(lambda x: [float(i) for i in x])
    question_vector = get_embedding(question, model='text-embedding-ada-002')
    emb["similarities"] = emb['embedding'].apply(lambda x: cosine_similarity(x, question_vector) if len(x) > 0 else 0)
    emb = emb.sort_values("similarities", ascending=False).head(4)
    # Generate prompt for OpenAI
    nontext = []
    for i, row in emb.iterrows():
        if isinstance(row['Combined'], str):
            nontext.append(row['Combined'])
    text = "\n".join(nontext)
    nontext = text
    prompt = f"""Answer the following question using only the context below.

    Context:
    {nontext}

    Q: {question}
    A:"""

    # Call OpenAI API to get the answer
    try:
        print("Calling OpenAI API")
        ans = openai.Completion.create(
            prompt=prompt,
            temperature=1,
            max_tokens=200,
            top_p=1,
            frequency_penalty=0,
            presence_penalty=0,
            model="text-davinci-003",
        )["choices"][0]["text"].strip(" \n")
        print(f"Ans -: {ans}")
        res = {"msg": ans, "status": "success"}
        print(res)
        return {
            'statusCode': 200,
            'body': json.dumps(res)
            }
    except Exception as e:
        return {
            'statusCode': 400,
            'body': json.dumps({'message': str(e)})
        }


import pygame
import UI
from funcs import canvas_flood_fill
import funcs
import sys

pygame.init()

screen_size = (1200, 850)
screen_w, screen_h = screen_size
canvas_size = (int(screen_size[0] / 1.2), int(screen_size[1] / 1.2))
flag = True

screen = pygame.display.set_mode(screen_size, 0, 16, pygame.HWACCEL)
clock = pygame.time.Clock()

canv_c, canv_r = 64, 64
canvas_keep_ratio = True
canvas_keep_rc = True
canv_px_w = canvas_size[0] // canv_c
canv_px_h = canvas_size[1] // canv_r
if canvas_keep_ratio:
    mn = min(canv_px_h, canv_px_w)
    canv_px_h, canv_px_w = mn, mn
    if not canvas_keep_rc:
        cw, ch = canvas_size
        cw, ch = cw - cw % mn, ch - ch % mn
        canvas_size = cw, ch
        canv_c, canv_r = cw // mn, ch // mn
        canv_c, canv_r = canv_c - canv_c % mn, canv_r - canv_r % mn
    canvas_size = mn * canv_c, mn * canv_r

main_p_r, main_p_c = 12, 8
p_square = 20
p_title_h = 30
basic_color_tooltip = UI.UITooltip(5, 2, p_square * main_p_c, p_square * main_p_r, main_p_r, main_p_c, title_h=p_title_h, text=('Main palette', UI.black), font=('arial', 15))

for i in range(0, len(UI.colors), main_p_c):
    for col, color in enumerate(UI.colors[i:(i + main_p_c)]):
        b_c = (*color, 255)
        basic_color_tooltip.set_item((i // main_p_c), col, UI.UIButton(0, 0, 20, 20, color=b_c))

custom_color_tooltip = UI.UITooltip(5, basic_color_tooltip.height + 22, 156, 26, 1, 6, title_h=20, text=('Custom palette', UI.black), font=('arial', 15))

current_color_indicator = UI.UITooltip(15, custom_color_tooltip.y + 40, 46, 36, 1, 1, title_h=20, text=('color', UI.black), font=('arial', 15))
current_color_indicator.set_item(0, 0, UI.Rectangle(7, current_color_indicator.y + 2, 42, 32, color=UI.black))

cw_icon = pygame.image.load('icons/pick_color.png')
pick_color_button = UI.UIButton(custom_color_tooltip.width - 10 - current_color_indicator.height, custom_color_tooltip.y + 40, current_color_indicator.height, current_color_indicator.height, image=cw_icon)
pick_color_button.state = False

def new_canvas(in_canv_args):
    return UI.Canvas(*in_canv_args)

size_slider = UI.UISlider(5, 450, 30, 170, min(canv_c // 8, 50), 1, slider_color=UI.red)

tool_square = 40
tool_r, tool_c = 4, 4
paint_tools_tooltip = UI.UITooltip(size_slider.x + size_slider.width + 10, 450, tool_square * tool_c, tool_square * tool_r, tool_r, tool_c, title_h=p_title_h, text=('Tools', UI.black), font=('arial', 15))

for i, item in enumerate(UI.icons):
    paint_tools_tooltip.set_item(i // paint_tools_tooltip.cols, i % paint_tools_tooltip.cols, UI.UIButton(0, 0, 24, 24, image=item))

curr_layer = 0
layer_square = 80
layers_tooltip = UI.UITooltip(screen_w - layer_square - 40, 2, int(layer_square * 1.2), layer_square, 2, 1, title_h=p_title_h, text=('Layers', UI.black), font=('arial', 15))

layers_tooltip.set_item(0, 0, UI.UIButton(0, 0, layer_square, layer_square, image=pygame.image.load('icons/default_image.png')))
layers_tooltip.set_item(1, 0, UI.UITooltip(0, 0, int(1.2 * layer_square) - 4, layer_square // 2 - 4, 1, 2, show_grid=False))
layer_buttons_tooltip = layers_tooltip.get_item(-1, 0)
layer_buttons_tooltip.set_item(0, 0, UI.UIButton(0, 0, 16, 16, image=pygame.image.load('icons/visible.png')))
layer_buttons_tooltip.set_item(0, 1, UI.UIButton(0, 0, 16, 16, image=pygame.image.load('icons/delete.png')))

add_layer_button = UI.UIButton(layers_tooltip.x, layers_tooltip.y + layers_tooltip.height, layer_square + 15, 20, text=('+    add layer', UI.black), font=('arial', 15), image=pygame.image.load('icons/button_1.png'))

old_mouse_x, old_mouse_y, mouse_x, mouse_y = 0, 0, 0, 0
draw_color = UI.black

selected_brush = pygame.image.load('brushes/circle16.png')
brush = selected_brush
selected_layer_button = layers_tooltip.get_item(0, 0)

pick_color_popup_w, pick_color_popup_h = 700, 350
pick_color_popup = UI.Popup((screen_w - pick_color_popup_w) // 2, (screen_h - pick_color_popup_h) // 2, pick_color_popup_w, pick_color_popup_h,
                            text=('  Pick a color', UI.black), font=('courier new', 20), title_h=30)
new_tt_y = pick_color_popup.y + 20
new_tt_square = 20
for i, key in enumerate(UI.colors2.keys()):
    color_lst = UI.colors2[key]
    new_tt = UI.UITooltip(pick_color_popup.x + 20, new_tt_y, new_tt_square * len(color_lst),
                          new_tt_square, 1, len(color_lst), title_h=20, font=('arial', 16),
                          text=(key, UI.black), bg_color=UI.white, border_w=2)
    pick_color_popup.add_item(new_tt, 20, new_tt_y - pick_color_popup.y + 20, name=f'palette{i}')
    for j, clr in enumerate(color_lst):
        new_tt.set_item(0, j, UI.UIButton(0, 0, 20, 20, color=clr))
    new_tt_y += 60

color_wheel_rect = UI.Rectangle(0, 0, 256, 256, image=pygame.image.load('icons/color_picker.png'), color=None)
pick_color_popup.add_item(color_wheel_rect, new_tt.width + 80, pick_color_popup_h - color_wheel_rect.height + 12, name='color_wheel')
old_color_ind = UI.UITooltip(color_wheel_rect.x + 20, color_wheel_rect.y - 84, 54, 64, 1, 1, title_h=20, text=('old', UI.black), font=('arial', 16))
pick_color_popup.add_item(old_color_ind, 20, -84, name='old_color_ind', reference_element=color_wheel_rect)
old_color_ind.set_item(0, 0, UI.Rectangle(0, 0, 62, 62, UI.white))
arrow_rect = UI.Rectangle(0, 0, 64, 64, image=pygame.image.load('icons/arrow_right.png'), color=None)
pick_color_popup.add_item(arrow_rect, old_color_ind.width + 20, 0, name='arrow', reference_element=old_color_ind)
new_color_ind = UI.UITooltip(arrow_rect.x + arrow_rect.width + 15, arrow_rect.y, 54, 64, 1, 1, title_h=20, text=('new', UI.black), font=('arial', 16))
pick_color_popup.add_item(new_color_ind, arrow_rect.width + 15, 0, name='new_color_ind', reference_element=arrow_rect)
new_color_ind.set_item(0, 0, UI.Rectangle(0, 0, 62, 62, UI.red))

sl_x = 40
for i, sl_name in enumerate(('R: ', 'G: ', 'B: ')):
    new_col_slider = UI.UISlider(new_color_ind.x + new_color_ind.width + sl_x + 20,
                                 new_color_ind.y, 40, color_wheel_rect.height + 40,
                                 255, 0, slider_color=UI.red, bg_color=UI.white, color=(80, 80, 80),
                                 text=(sl_name, UI.black), scale_h=color_wheel_rect.height - 15, title_h=20)
    pick_color_popup.add_item(new_col_slider, new_color_ind.width + sl_x + 20, 0,
                              f'slider{i}', reference_element=new_color_ind)
    sl_x += 65

ok_button = UI.UIButton(0, 0, 86, 24, color=UI.gray, text=('Ok', UI.black), font=('arial', 18), frame_col=UI.green, frame_w=2)
pick_color_popup.add_item(ok_button, pick_color_popup_w - ok_button.width - 20, pick_color_popup_h - 15, name='ok_button')
cancel_button = UI.UIButton(0, 0, 86, 24, color=UI.gray, text=('Cancel', UI.black), font=('arial', 18), frame_col=UI.red, frame_w=2)
pick_color_popup.add_item(cancel_button, -(cancel_button.width + 20), 0, name='cancel_button', reference_element=ok_button)

current_popup = None

ui_elements = [basic_color_tooltip, size_slider, paint_tools_tooltip, layers_tooltip, add_layer_button,
               custom_color_tooltip, current_color_indicator, pick_color_button]

canv_x = max(paint_tools_tooltip.x + paint_tools_tooltip.width, basic_color_tooltip.x + basic_color_tooltip.width)
canv_padding = (layers_tooltip.x - canv_x - canv_c * canv_px_w) // 2
canv_x += canv_padding
canv_args = (canv_x, 2, *canvas_size, canv_r, canv_c)
canvas = new_canvas(canv_args)
layers = [[canvas, True]]
selected_tool = None

def draw_layers(layers1, final_layer=-1):
    if final_layer < 0 or final_layer >= len(layers):
        final_layer = len(layers1) - 1
    if layers1:
        pygame.draw.rect(screen, UI.background, layers1[0][0].get_rectangle())
        for canv_obj in layers1[:final_layer + 1]:
            cv, do_draw = canv_obj
            if do_draw:
                cv.draw(screen)

def setup():
    global brush
    screen.fill(UI.white)
    canvas.draw(screen)
    pygame.display.flip()
    brush = selected_brush

def loop():
    global mouse_x
    global mouse_y
    global old_mouse_y
    global old_mouse_x
    global draw_color
    global selected_brush
    global brush
    global curr_layer
    global selected_layer_button
    global current_popup

    pygame.draw.rect(screen, UI.background, (layers_tooltip.x - 5, 0, layers_tooltip.width + 10, add_layer_button.x + 10))
    pygame.draw.rect(screen, UI.background, (paint_tools_tooltip.x - 5, paint_tools_tooltip.y - 5, paint_tools_tooltip.width + 10, paint_tools_tooltip.height + 10))

    draw_canvas = bool(layers)

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
        if event.type == pygame.KEYDOWN:
            pass
        if event.type == pygame.MOUSEBUTTONDOWN:
            if event.button == 4 and layers[curr_layer][1]:
                    layers[curr_layer][0].scale(-1, mouse_x, mouse_y)
                    layers[curr_layer][0].hide(screen, True)
                    draw_layers(layers, curr_layer)
                    pygame.display.update(layers[curr_layer][0].get_rectangle())
            if event.button == 5 and layers[curr_layer][1]:
                    layers[curr_layer][0].scale(1, mouse_x, mouse_y)
                    layers[curr_layer][0].hide(screen, True)
                    draw_layers(layers, curr_layer)
                    pygame.display.update(layers[curr_layer][0].get_rectangle())
            if current_popup is None:
                if basic_color_tooltip.mouse_hover():
                    for row in basic_color_tooltip.table:
                        for button in row:
                            if button.mouse_down() and isinstance(button, UI.UIButton):
                                button.tint(False)
                                draw_color = button.color
                                basic_color_tooltip.clicked = button
                                button.tint(True, amt=20)
                                break

                if paint_tools_tooltip.mouse_hover():
                    for i, row in enumerate(paint_tools_tooltip.table):
                        for j, button in enumerate(row):
                            if button.mouse_down() and isinstance(button, UI.UIButton):
                                paint_tools_tooltip.selected = button
                                ps = i * paint_tools_tooltip.cols + j
                                if ps == 0:
                                    brush = selected_brush
                                if ps == 1:
                                    brush = None
                                if ps == 2:
                                    if draw_canvas:
                                        draw_color = layers[curr_layer][0].bg_col
                                    brush = None
                                if ps == 3:
                                    brush = canvas_flood_fill
                                if ps == 9:
                                    brush = UI.LASSO
                                if ps == 11:
                                    brush = UI.MOVE_TOOL
                                paint_tools_tooltip.clicked = button
                                button.tint(True, amt=20)
                                break

                if layers_tooltip.mouse_hover():
                    for obj_i, obj in enumerate([x[0] for x in layers_tooltip.table]):
                        if obj.mouse_down():
                            layers_tooltip.clicked = obj
                            if isinstance(obj, UI.UIButton):
                                selected_layer_button = obj
                                curr_layer = obj_i // 2
                                draw_layers(layers, curr_layer)
                                obj.color = UI.gray
                                pygame.display.update(layers[curr_layer][0].get_rectangle())
                                break
                            elif isinstance(obj, UI.UITooltip):
                                visibility_toggle_button = obj.table[0][0]
                                delete_button = obj.table[0][1]
                                if visibility_toggle_button.mouse_down():
                                    layers[obj_i // 2][1] = not layers[obj_i // 2][1]
                                    draw_layers(layers, curr_layer)
                                    pygame.display.update(layers[obj_i // 2][0].get_rectangle())
                                    if layers[obj_i // 2][1]:
                                        visibility_toggle_button.image = pygame.image.load('icons/visible.png')
                                    else:
                                        visibility_toggle_button.image = pygame.image.load('icons/invisible.png')
                                    visibility_toggle_button.color = UI.gray
                                    obj.clicked = visibility_toggle_button
                                elif delete_button.mouse_down():
                                    delete_button.color = UI.gray
                                    if layers:
                                        if len(layers_tooltip.table) > 2:
                                            layers_tooltip.pop_item(obj_i - 1, 0)
                                            layers_tooltip.pop_item(obj_i - 1, 0)

                                            layers.pop((obj_i - 1) // 2)
                                            curr_layer = min(curr_layer, len(layers) - 1)

                                            selected_layer_button = layers_tooltip.get_item(curr_layer * 2, 0)

                                            old_add_layer_button_pos = add_layer_button.get_rectangle()
                                            add_layer_button.x, add_layer_button.y = layers_tooltip.x, layers_tooltip.y + layers_tooltip.height
                                            pygame.display.update(old_add_layer_button_pos)

                                            draw_layers(layers, curr_layer)

                                            pygame.display.update(canvas.get_rectangle())
                                    obj.clicked = delete_button
                                    break
                            else:
                                print(obj)

                if add_layer_button.mouse_hover():
                    layers_tooltip.append_item(
                        UI.UIButton(0, 0, layer_square, layer_square,
                                    image=pygame.image.load('icons/default_image.png')))
                    layers_tooltip.append_item(
                        UI.UITooltip(0, 0, int(1.2 * layer_square) - 4, layer_square // 2 - 4, 1, 2, show_grid=False))
                    new_tooltip_element = layers_tooltip.get_item(-1, 0)
                    new_tooltip_element.set_item(0, 0,
                                                 UI.UIButton(0, 0, 16, 16,
                                                             image=pygame.image.load('icons/visible.png')))
                    new_tooltip_element.set_item(0, 1,
                                                 UI.UIButton(0, 0, 16, 16, image=pygame.image.load('icons/delete.png')))
                    layers.append([new_canvas(canv_args), True])
                    curr_layer += 1
                    layers_tooltip.update_item_sizes()
                    old_add_layer_button_pos = add_layer_button.get_rectangle()
                    add_layer_button.x, add_layer_button.y = layers_tooltip.x, layers_tooltip.y + layers_tooltip.height
                    pygame.display.update(old_add_layer_button_pos)
                if pick_color_button.mouse_hover():
                    pick_color_button.onclick()
                    if pick_color_button.state:
                        current_popup = pick_color_popup
                        pick_color_popup.items['old_color_ind'].table[0][0].color = draw_color
                        pick_color_popup.draw(screen)
                    pygame.display.update(pick_color_popup.get_rectangle())
                if custom_color_tooltip.mouse_hover():
                    for item in custom_color_tooltip.table[0]:
                        if item.mouse_down():
                            draw_color = item.color
                            item.tint(True, amt=20)
                            custom_color_tooltip.clicked = item
                            custom_color_tooltip.selected = item

            elif current_popup == pick_color_popup:
                for key in pick_color_popup.items.keys():
                    item = pick_color_popup.items[key]
                    if item.mouse_hover():
                        if 'palette' in key:
                            for color_button in item.table[0]:
                                if color_button.mouse_hover():
                                    color_button.tint(False)
                                    pick_color_popup.items['new_color_ind'].table[0][0].color = color_button.color
                                    r, g, b = color_button.color
                                    pick_color_popup.items['slider0'].set_value(r)
                                    pick_color_popup.items['slider1'].set_value(g)
                                    pick_color_popup.items['slider2'].set_value(b)
                                    color_button.tint(True, amt=20)
                                    item.clicked = color_button
                        elif 'color_wheel' in key:
                            c_w = pick_color_popup.items[key]
                            new_clr = list(c_w.image.get_at((mouse_x - c_w.x, mouse_y - c_w.y)))[:3]
                            pick_color_popup.items['new_color_ind'].table[0][0].color = new_clr
                        elif 'ok_button' in key or 'cancel_button' in key:
                            if 'ok_button' in key:
                                draw_color = pick_color_popup.items['new_color_ind'].table[0][0].color
                                if custom_color_tooltip.selected:
                                    custom_color_tooltip.selected.color = draw_color
                                    custom_color_tooltip.selected.old_color = draw_color
                            screen.fill(UI.white)
                            draw_layers(layers, curr_layer)
                            pygame.display.flip()
                            current_popup = None

        if event.type == pygame.MOUSEBUTTONUP:
            if basic_color_tooltip.clicked:
                basic_color_tooltip.clicked.tint(False)
                basic_color_tooltip.clicked = None
            if paint_tools_tooltip.clicked:
                paint_tools_tooltip.clicked.tint(False)
                paint_tools_tooltip.clicked = None
            if custom_color_tooltip.clicked:
                custom_color_tooltip.clicked.tint(False)
                custom_color_tooltip.clicked = None
            if layers_tooltip.clicked:
                if isinstance(layers_tooltip.clicked, UI.UITooltip):
                    if layers_tooltip.clicked.clicked:
                        layers_tooltip.clicked.clicked.tint(False)
                        layers_tooltip.clicked.clicked = None
                else:
                    layers_tooltip.clicked.tint(False)
            if current_popup == pick_color_popup:
                for key in pick_color_popup.items.keys():
                    if 'palette' in key:
                        if pick_color_popup.items[key].clicked:
                            pick_color_popup.items[key].clicked.tint(False)
                            pick_color_popup.items[key].clicked = None

        if event.type == pygame.MOUSEMOTION:
            old_mouse_x, old_mouse_y = mouse_x, mouse_y
            mouse_x, mouse_y = pygame.mouse.get_pos()
            if basic_color_tooltip.mouse_hover():
                for row in basic_color_tooltip.table:
                    for button in row:
                        button.tint(button.mouse_hover())
                        if button.mouse_hover():
                            basic_color_tooltip.tinted = button
                            break
            elif basic_color_tooltip.tinted:
                basic_color_tooltip.tinted.tint(False)
                basic_color_tooltip.tinted = None
            if size_slider.mouse_down():
                size_slider.move_slider()
            if current_popup == pick_color_popup:
                for key in pick_color_popup.items.keys():
                    if 'palette' in key:
                        for obj in pick_color_popup.items[key].table[0]:
                            obj.tint(obj.mouse_hover())
                    elif 'slider' in key:
                        sl = pick_color_popup.items[key]
                        if sl.mouse_down():
                            sl.move_slider()
                            new_clr = [pick_color_popup.items[key].value for key in pick_color_popup.items.keys() if 'slider' in key]
                            pick_color_popup.items['new_color_ind'].table[0][0].color = new_clr
            if pygame.mouse.get_pressed()[1] and layers[curr_layer][1]:
                layers[curr_layer][0].pan(mouse_x - old_mouse_x, mouse_y - old_mouse_y)
                layers[curr_layer][0].hide(screen, True)
                draw_layers(layers, curr_layer)
                pygame.display.update(layers[curr_layer][0].get_rectangle())

    if not layers:
        draw_canvas = False

    if draw_canvas and current_popup is None:
        now_canvas, do_draw = layers[curr_layer]
        overlay_layers = [x[0] for x in layers if x[0] != now_canvas and x[1]]
        if do_draw:
            brush_size = size_slider.value
            if now_canvas.mouse_down():
                if old_mouse_x != mouse_x or old_mouse_y != mouse_y:
                    now_canvas.line_paint((old_mouse_x, old_mouse_y), (mouse_x, mouse_y), color=draw_color,
                                      size=(brush_size, brush_size), brush=brush, surface=screen, overlays=overlay_layers)
                    if brush in [UI.LASSO, UI.MOVE_TOOL]:
                        now_canvas.show_selected_set(screen)
                    if brush in [UI.MOVE_TOOL]:
                        now_canvas.hide(screen, True)
                        now_canvas.draw(screen)
            else:
                if brush in [UI.LASSO]:
                    if now_canvas.selected_pixel_set:
                        layers[curr_layer][0].lasso_selection()
                        now_canvas.show_selected_set(screen)
                    now_canvas.lasso_clear_flag = True
            pygame.display.update(now_canvas.get_rectangle())
    current_color_indicator.get_item(0, 0).color = draw_color

    for el in ui_elements:
        el.draw(screen)

    selected_layer_button.draw_frame(screen, UI.red, width=5)

    for el in ui_elements:
        pygame.display.update(el.get_rectangle())

    if current_popup is not None:
        current_popup.draw(screen)
        pygame.display.update(current_popup.get_rectangle())

    pygame.display.update((layers_tooltip.x - 5, layers_tooltip.y + layers_tooltip.height,
                           layers_tooltip.width + 10, layers_tooltip.row_size + 20))
    clock.tick(128)
    #print(clock)

setup()
while flag:
    loop()


from torchvision import transforms as T
from torch.utils.data import Dataset
import numpy as np

class BaseDataset(Dataset):
    """
    Define length and sampling method
    """
    def __init__(self, root_dir, split='train', downsample=1.0):
        self.root_dir = root_dir
        self.split = split
        self.downsample = downsample
        self.define_transforms()

    def define_transforms(self):
        self.transform = T.ToTensor()

    def __len__(self):
        if self.split.startswith('train'):
            return 1000
        return len(self.poses)

    def __getitem__(self, idx):
        if self.split.startswith('train'):
            # training pose is retrieved in train.py
            # randomly select images
            img_idxs = np.random.choice(len(self.poses), self.batch_size)
            # randomly select pixels
            pix_idxs = np.random.choice(self.img_wh[0]*self.img_wh[1], self.batch_size)
            sample = {'rgb': self.rays[img_idxs, pix_idxs],
                      'img_idxs': img_idxs,
                      'pix_idxs': pix_idxs}
        else:
            sample = {'pose': self.poses[idx],
                      'img_idxs': idx}
            if len(self.rays)>0: # if ground truth rgb available
                sample['rgb'] = self.rays[idx]

        return sample


import time
import numpy as np
import tensorflow as tf

from models import GAT, HeteGAT, HeteGAT_multi
from utils import process

# 禁用gpu
import os

os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3"

config = tf.ConfigProto()
config.gpu_options.allow_growth = True

dataset = 'acm'
featype = 'fea'
checkpt_file = 'pre_trained/{}/{}_allMP_multi_{}_.ckpt'.format(dataset, dataset, featype)
print('model: {}'.format(checkpt_file))
# training params
batch_size = 1
nb_epochs = 200
patience = 100
lr = 0.005  # learning rate
l2_coef = 0.001  # weight decay
# numbers of hidden units per each attention head in each layer
hid_units = [8]
n_heads = [8, 1]  # additional entry for the output layer
residual = False
nonlinearity = tf.nn.elu
model = HeteGAT_multi

print('Dataset: ' + dataset)
print('----- Opt. hyperparams -----')
print('lr: ' + str(lr))
print('l2_coef: ' + str(l2_coef))
print('----- Archi. hyperparams -----')
print('nb. layers: ' + str(len(hid_units)))
print('nb. units per layer: ' + str(hid_units))
print('nb. attention heads: ' + str(n_heads))
print('residual: ' + str(residual))
print('nonlinearity: ' + str(nonlinearity))
print('model: ' + str(model))

# jhy data
import scipy.io as sio
import scipy.sparse as sp

def sample_mask(idx, l):
    """Create mask."""
    mask = np.zeros(l)
    mask[idx] = 1
    return np.array(mask, dtype=np.bool)

def load_data_dblp(path='/home/jhy/allGAT/acm_hetesim/ACM3025.mat'):
    data = sio.loadmat(path)
    truelabels, truefeatures = data['label'], data['feature'].astype(float)
    N = truefeatures.shape[0]
    rownetworks = [data['PAP'] - np.eye(N), data['PLP'] - np.eye(N)]  # , data['PTP'] - np.eye(N)]

    y = truelabels
    train_idx = data['train_idx']
    val_idx = data['val_idx']
    test_idx = data['test_idx']

    train_mask = sample_mask(train_idx, y.shape[0])
    val_mask = sample_mask(val_idx, y.shape[0])
    test_mask = sample_mask(test_idx, y.shape[0])

    y_train = np.zeros(y.shape)
    y_val = np.zeros(y.shape)
    y_test = np.zeros(y.shape)
    y_train[train_mask, :] = y[train_mask, :]
    y_val[val_mask, :] = y[val_mask, :]
    y_test[test_mask, :] = y[test_mask, :]

    # return selected_idx, selected_idx_2
    print('y_train:{}, y_val:{}, y_test:{}, train_idx:{}, val_idx:{}, test_idx:{}'.format(y_train.shape,
                                                                                y_val.shape,
                                                                                y_test.shape,
                                                                                train_idx.shape,
                                                                                val_idx.shape,
                                                                                test_idx.shape))
    truefeatures_list = [truefeatures, truefeatures, truefeatures]
    return rownetworks, truefeatures_list, y_train, y_val, y_test, train_mask, val_mask, test_mask

# use adj_list as fea_list, have a try~
adj_list, fea_list, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data_dblp()
if featype == 'adj':
    fea_list = adj_list

import scipy.sparse as sp

nb_nodes = fea_list[0].shape[0]
ft_size = fea_list[0].shape[1]
nb_classes = y_train.shape[1]

# adj = adj.todense()

# features = features[np.newaxis]  # [1, nb_node, ft_size]
fea_list = [fea[np.newaxis] for fea in fea_list]
adj_list = [adj[np.newaxis] for adj in adj_list]
y_train = y_train[np.newaxis]
y_val = y_val[np.newaxis]
y_test = y_test[np.newaxis]
train_mask = train_mask[np.newaxis]
val_mask = val_mask[np.newaxis]
test_mask = test_mask[np.newaxis]

biases_list = [process.adj_to_bias(adj, [nb_nodes], nhood=1) for adj in adj_list]

print('build graph...')
with tf.Graph().as_default():
    with tf.name_scope('input'):
        ftr_in_list = [tf.placeholder(dtype=tf.float32,
                                      shape=(batch_size, nb_nodes, ft_size),
                                      name='ftr_in_{}'.format(i))
                       for i in range(len(fea_list))]
        bias_in_list = [tf.placeholder(dtype=tf.float32,
                                       shape=(batch_size, nb_nodes, nb_nodes),
                                       name='bias_in_{}'.format(i))
                        for i in range(len(biases_list))]
        lbl_in = tf.placeholder(dtype=tf.int32, shape=(
            batch_size, nb_nodes, nb_classes), name='lbl_in')
        msk_in = tf.placeholder(dtype=tf.int32, shape=(batch_size, nb_nodes),
                                name='msk_in')
        attn_drop = tf.placeholder(dtype=tf.float32, shape=(), name='attn_drop')
        ffd_drop = tf.placeholder(dtype=tf.float32, shape=(), name='ffd_drop')
        is_train = tf.placeholder(dtype=tf.bool, shape=(), name='is_train')
    # forward
    logits, final_embedding, att_val = model.inference(ftr_in_list, nb_classes, nb_nodes, is_train,
                                                       attn_drop, ffd_drop,
                                                       bias_mat_list=bias_in_list,
                                                       hid_units=hid_units, n_heads=n_heads,
                                                       residual=residual, activation=nonlinearity)

    # cal masked_loss
    log_resh = tf.reshape(logits, [-1, nb_classes])
    lab_resh = tf.reshape(lbl_in, [-1, nb_classes])
    msk_resh = tf.reshape(msk_in, [-1])
    loss = model.masked_softmax_cross_entropy(log_resh, lab_resh, msk_resh)
    accuracy = model.masked_accuracy(log_resh, lab_resh, msk_resh)
    # optimzie
    train_op = model.training(loss, lr, l2_coef)

    saver = tf.train.Saver()

    init_op = tf.group(tf.global_variables_initializer(),
                       tf.local_variables_initializer())

    vlss_mn = np.inf
    vacc_mx = 0.0
    curr_step = 0

    with tf.Session(config=config) as sess:
        sess.run(init_op)

        train_loss_avg = 0
        train_acc_avg = 0
        val_loss_avg = 0
        val_acc_avg = 0

        for epoch in range(nb_epochs):
            tr_step = 0

            tr_size = fea_list[0].shape[0]
            # ================   training    ============
            while tr_step * batch_size < tr_size:

                fd1 = {i: d[tr_step * batch_size:(tr_step + 1) * batch_size]
                       for i, d in zip(ftr_in_list, fea_list)}
                fd2 = {i: d[tr_step * batch_size:(tr_step + 1) * batch_size]
                       for i, d in zip(bias_in_list, biases_list)}
                fd3 = {lbl_in: y_train[tr_step * batch_size:(tr_step + 1) * batch_size],
                       msk_in: train_mask[tr_step * batch_size:(tr_step + 1) * batch_size],
                       is_train: True,
                       attn_drop: 0.6,
                       ffd_drop: 0.6}
                fd = fd1
                fd.update(fd2)
                fd.update(fd3)
                _, loss_value_tr, acc_tr, att_val_train = sess.run([train_op, loss, accuracy, att_val],
                                                                   feed_dict=fd)
                train_loss_avg += loss_value_tr
                train_acc_avg += acc_tr
                tr_step += 1

            vl_step = 0
            vl_size = fea_list[0].shape[0]
            # =============   val       =================
            while vl_step * batch_size < vl_size:
                # fd1 = {ftr_in: features[vl_step * batch_size:(vl_step + 1) * batch_size]}
                fd1 = {i: d[vl_step * batch_size:(vl_step + 1) * batch_size]
                       for i, d in zip(ftr_in_list, fea_list)}
                fd2 = {i: d[vl_step * batch_size:(vl_step + 1) * batch_size]
                       for i, d in zip(bias_in_list, biases_list)}
                fd3 = {lbl_in: y_val[vl_step * batch_size:(vl_step + 1) * batch_size],
                       msk_in: val_mask[vl_step * batch_size:(vl_step + 1) * batch_size],
                       is_train: False,
                       attn_drop: 0.0,
                       ffd_drop: 0.0}

                fd = fd1
                fd.update(fd2)
                fd.update(fd3)
                loss_value_vl, acc_vl = sess.run([loss, accuracy],
                                                 feed_dict=fd)
                val_loss_avg += loss_value_vl
                val_acc_avg += acc_vl
                vl_step += 1
            # import pdb; pdb.set_trace()
            print('Epoch: {}, att_val: {}'.format(epoch, np.mean(att_val_train, axis=0)))
            print('Training: loss = %.5f, acc = %.5f | Val: loss = %.5f, acc = %.5f' %
                  (train_loss_avg / tr_step, train_acc_avg / tr_step,
                   val_loss_avg / vl_step, val_acc_avg / vl_step))

            if val_acc_avg / vl_step >= vacc_mx or val_loss_avg / vl_step <= vlss_mn:
                if val_acc_avg / vl_step >= vacc_mx and val_loss_avg / vl_step <= vlss_mn:
                    vacc_early_model = val_acc_avg / vl_step
                    vlss_early_model = val_loss_avg / vl_step
                    saver.save(sess, checkpt_file)
                vacc_mx = np.max((val_acc_avg / vl_step, vacc_mx))
                vlss_mn = np.min((val_loss_avg / vl_step, vlss_mn))
                curr_step = 0
            else:
                curr_step += 1
                if curr_step == patience:
                    print('Early stop! Min loss: ', vlss_mn,
                          ', Max accuracy: ', vacc_mx)
                    print('Early stop model validation loss: ',
                          vlss_early_model, ', accuracy: ', vacc_early_model)
                    break

            train_loss_avg = 0
            train_acc_avg = 0
            val_loss_avg = 0
            val_acc_avg = 0

        saver.restore(sess, checkpt_file)
        print('load model from : {}'.format(checkpt_file))
        ts_size = fea_list[0].shape[0]
        ts_step = 0
        ts_loss = 0.0
        ts_acc = 0.0

        while ts_step * batch_size < ts_size:
            # fd1 = {ftr_in: features[ts_step * batch_size:(ts_step + 1) * batch_size]}
            fd1 = {i: d[ts_step * batch_size:(ts_step + 1) * batch_size]
                   for i, d in zip(ftr_in_list, fea_list)}
            fd2 = {i: d[ts_step * batch_size:(ts_step + 1) * batch_size]
                   for i, d in zip(bias_in_list, biases_list)}
            fd3 = {lbl_in: y_test[ts_step * batch_size:(ts_step + 1) * batch_size],
                   msk_in: test_mask[ts_step * batch_size:(ts_step + 1) * batch_size],

                   is_train: False,
                   attn_drop: 0.0,
                   ffd_drop: 0.0}

            fd = fd1
            fd.update(fd2)
            fd.update(fd3)
            loss_value_ts, acc_ts, jhy_final_embedding = sess.run([loss, accuracy, final_embedding],
                                                                  feed_dict=fd)
            ts_loss += loss_value_ts
            ts_acc += acc_ts
            ts_step += 1

        print('Test loss:', ts_loss / ts_step,
              '; Test accuracy:', ts_acc / ts_step)

        print('start knn, kmean.....')
        xx = np.expand_dims(jhy_final_embedding, axis=0)[test_mask]

        from numpy import linalg as LA

        # xx = xx / LA.norm(xx, axis=1)
        yy = y_test[test_mask]

        print('xx: {}, yy: {}'.format(xx.shape, yy.shape))
        from jhyexps import my_KNN, my_Kmeans#, my_TSNE, my_Linear

        my_KNN(xx, yy)
        my_Kmeans(xx, yy)

        sess.close()


import xml.etree.ElementTree as ET

page_text = ''

def func_datasource(datasource):
    for elem in datasource:
        if elem.get("name") == "Parameters":
            print("---- Parameters ----")
            key = 1
            parameters = {}
            for datasourceChild in elem:
                if datasourceChild.tag == "column":
                    parameters[key] = {}
                    parameters[key]["caption"] = datasourceChild.get("caption")
                    parameters[key]["value"] = datasourceChild.get("value")
                    parameters[key]["member"] = []
                    for columnChild in datasourceChild:
                        if columnChild.tag == "members":
                            for member in columnChild:
                                parameters[key]["member"].append(member.get("value"))
                    key = key + 1
            print(parameters)
        else:

            # Datasource Definition
            print("---- metadata ----")
            metadatarecords = elem.findall("connection/metadata-records/")
            columns = []
            for metadatarecord in metadatarecords:
                if metadatarecord.get("class") == "column":
                    column = {}
                    for metadata in metadatarecord:
                        tag = metadata.tag
                        if tag == 'local-name':
                            column["local-name"] = metadata.text
                        elif tag == 'local-type':
                            column["localtype"] = metadata.text
                    columns.append(column)
            print(columns)

            # Caliculation Fields
            print("---- Caliculations ----")
            caliculations = []
            for columnChild in elem:
                if columnChild.tag == "column" and columnChild.get("caption") is not None:
                    caliculations.append(columnChild.attrib)

            print(caliculations)

def func_worksheet(worksheet):
    print('---- worksheet ----')

    # sheet name
    sheetName = worksheet.get("name")
    print(sheetName)

    # using datasource
    datasources = worksheet.findall('table/view/datasources/')
    for datasourcesChild in datasources:
        print(datasourcesChild.get('caption'))

    # filter
    view = worksheet.findall('table/view/')
    for viewChild in view:
        if viewChild.tag == 'filter':
            print(viewChild.attrib)

    # marks
    encodings = worksheet.findall('table/panes/pane/encodings/')
    for encodingsChild in encodings:
        if encodingsChild.tag == 'color':
            print('color')
            print(encodingsChild.get("column"))
        elif encodingsChild.tag == 'size':
            print('size')
            print(encodingsChild.get("column"))
        elif encodingsChild.tag == 'lod':
            print('lod')  # Detail
            print(encodingsChild.get("column"))
        elif encodingsChild.tag == 'tooltip':
            print('tooltip')
            print(encodingsChild.get("column"))
        elif encodingsChild.tag == 'text':
            print('text')  # Label
            print(encodingsChild.get("column"))

    # axis
    table = worksheet.findall('table/')
    for tableChild in table:
        if tableChild.tag == 'rows':
            print('horizontal axis')
            print(tableChild.text)
        if tableChild.tag == 'cols':
            print('vertical axis')
            print(tableChild.text)

def func_dashboard(dashboard):
    print('---- dashbord ----')

    # dashbord name
    print(dashboard.get("name"))

    # worksheet names used in the dashboard

    zone = dashboard.findall('.zones//zone[@name]')
    for zoneChild in zone:
        print(zoneChild.get('name'))

def func_datasources(datasources):
    for datasource in datasources:
        func_datasource(datasource)

def func_worksheets(worksheets):
    for worksheet in worksheets:
        func_worksheet(worksheet)

def func_dashboards(dashboards):
    for dashboard in dashboards:
        func_dashboard(dashboard)

# Main process
fp = open("Superstore.twb","r",encoding="utf-8")
content = fp.read()

# root tag attribute
root = ET.fromstring(content)
print(root.tag, root.attrib)

# Child Layer tag attribute
for rootChild in root:
    if rootChild.tag == "datasources":
        func_datasource(rootChild)
    elif rootChild.tag == "worksheets":
        func_worksheets(rootChild)
    elif rootChild.tag == "dashboards":
        func_dashboards(rootChild)


import pyperf
from sample_bench import Benchmark
from macpacking.reader import BinppReader, JburkardtReader
from macpacking.algorithms.online import \
    FirstFit, BestFit, WorstFit
from macpacking.algorithms.offline import \
    FirstFitDecreasing, BestFitDecreasing, WorstFitDecreasing
from helpers import list_case_files

listOfBinppCases = [
    './_datasets/binpp/N1C1W1/N1C1W1_A.BPP.txt',
    './_datasets/binpp/N1C3W4/N1C3W4_A.BPP.txt',
    './_datasets/binpp/N2C1W1/N2C1W1_A.BPP.txt',
    './_datasets/binpp/N2C3W4/N2C3W4_A.BPP.txt',
    './_datasets/binpp/N3C1W1/N3C1W1_A.BPP.txt',
    './_datasets/binpp/N3C3W4/N3C3W4_A.BPP.txt',
    './_datasets/binpp/N4C1W1/N4C1W1_A.BPP.txt',
    './_datasets/binpp/N4C3W4/N4C3W4_A.BPP.txt'
]

binppHardCases = './_datasets/binpp-hard'
jburkardtCases = './_datasets/jburkardt'
listOfBinppHardCases = list_case_files(binppHardCases)
listOfJburkardtCases = list_case_files(jburkardtCases)

def main():

    algorithms = [FirstFit(), BestFit(), WorstFit(), FirstFitDecreasing(), BestFitDecreasing(), WorstFitDecreasing()]  # noqa: E501

    runner = pyperf.Runner()
    benchmark = Benchmark()
    benchmark.execTime(listOfBinppCases, BinppReader, algorithms, runner)
    benchmark.execTime(listOfBinppHardCases, BinppReader, algorithms, runner)
    benchmark.execTime(listOfJburkardtCases, JburkardtReader, algorithms, runner)  # noqa: E501

if __name__ == "__main__":
    main()


"""
breyo.io
========

I/O code.

"""
import os
import astropy.units as u

def get_rawdir(night):
    return os.path.join(os.getenv('BREYO_DATA_DIR'), 'raw', night)

def get_reduxdir(night=None):
    reduxdir = os.path.join(os.getenv('BREYO_DATA_DIR'), 'reduced')
    if night:
        reduxdir = os.path.join(reduxdir, night)
    if not os.path.isdir(reduxdir):
        os.makedirs(reduxdir, exist_ok=True)
    calibdir = os.path.join(reduxdir, 'calib')
    if not os.path.isdir(calibdir):
        os.makedirs(calibdir, exist_ok=True)
    return reduxdir, calibdir

def read_config(verbose=False):
    import yaml

    # not sure why this doesn't work
    #import pkg_resources
    #configfile = pkg_resources.resource_filename('breyo', os.path.join('data', 'SBIG-STL-11000M.yaml'))

    configfile = os.path.join(os.getenv('BREYO_DIR'), 'py', 'breyo', 'data', 'SBIG-STL-11000M.yaml')

    if verbose:
        print('Reading configuration file {}'.format(configfile))
    with open(configfile, 'r') as ff:
        params = yaml.safe_load(ff)

    # attach units (see config file for comments)
    params['ccd']['pixsize'] *= u.micron
    params['ccd']['gain'] *= u.electron / u.adu
    params['ccd']['readnoise'] *= u.electron

    return params