#ohaohaoha

import json
import socket
from base64 import b64decode
from Crypto.Cipher import AES, PKCS1_OAEP
from Crypto.PublicKey import RSA
from Crypto.Util.Padding import unpad
from time import sleep

HOST = '0.0.0.0'
PORT = 50013

TICK = 0.1
c = ''

def rsa(q):
    #print(q)

    try:
        key = RSA.importKey(q['key'])
        cipher = PKCS1_OAEP.new(key)
        s = cipher.decrypt(b64decode(q['data']))
        print(json.loads(s)['extra'])

    except Exception as e:
        print('RSA_PKCS1_OAEP_2048 error', e)

def aes(q):
    #print(q)

    try:
        key = b64decode(q['key'])
        iv = b64decode(q['iv'])
        cipher = AES.new(key, AES.MODE_CBC, iv)
        s = unpad(cipher.decrypt(b64decode(q['data'])), cipher.block_size)
        s = s.decode('utf-8').strip()
        print(json.loads(s)['extra'])

    except Exception as e:
        print('AES_CBC error', e)

def b64(q):
    #print(q)

    try:
        data = b64decode(q['data'])
        print(json.loads(data)['extra'])

    except Exception as e:
        print('b64 error', e)

def plain(q):
    #print(q)

    try:
        print(q['data']['extra'])

    except Exception as e:
        print('plain', e)

def decipher(j):
    try:
        q = json.loads(j)
        m = q['method']
    except:
        print('invalid json:', j)
        return

    if m == 'RSA_PKCS1_OAEP_2048':
        rsa(q)
    elif m == 'AES_CBC':
        aes(q)
    elif m == 'b64':
        b64(q)
    elif m == 'plain':
        plain(q)
    else:
        print('unsupported method')

def work():
    global c
    i = c.find('\n')

    if i == -1:
        try:
            c += s.recv(1024).decode('utf-8')
        except:
            pass # no data yet
    else:
        nextQuery = ''

        if i != len(c) - 1:
            nextQuery = c[i + 1:]

        decipher(c[0:i])
        c = nextQuery

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
s.connect((HOST, PORT))
s.setblocking(0)

try:
    while True:
        work()
        sleep(TICK)

except KeyboardInterrupt:
    print('finished')

except Exception as e:
    print('error:', e)

s.close()


"""Controller program to manage the benchmark."""
import os
import random
import socket
import sys

from controller import Supervisor
try:
    from shapely.geometry import Point
    from shapely.geometry import LineString
except ImportError:
    sys.stderr.write("Error: 'shapely' module not found.\n")
    sys.exit(0)

VEHICLE_DEF_NAME = 'WEBOTS_VEHICLE0'
SUMO_PORTS_RANGE = [1024, 1998]
MAX_TIME = 60.0
sensorVisualizationNodes = []

try:
    sys.path.append(os.path.join(os.path.normpath(os.environ.get("WEBOTS_HOME")), 'projects', 'samples', 'robotbenchmark',
                                 'include'))
    from robotbenchmark import robotbenchmarkRecord
except ImportError:
    sys.stderr.write("Warning: 'robotbenchmark' module not found.\n")
    sys.exit(0)

def apply_spline_subdivision_to_path(path, subdivision):
    """Apply spline subdivision to the list of points."""
    spline = []
    points = list(path)
    pointsNumber = len(points)
    # extend the points array
    points.append((points[pointsNumber - 1][0] + (points[pointsNumber - 1][0] - points[pointsNumber - 2][0]),
                   points[pointsNumber - 1][1] + (points[pointsNumber - 1][1] - points[pointsNumber - 2][1])))
    points.append((points[pointsNumber][0] + (points[pointsNumber][0] - points[pointsNumber - 1][0]),
                   points[pointsNumber][1] + (points[pointsNumber][1] - points[pointsNumber - 1][1])))
    pointMinus1 = (points[0][0] + (points[0][0] - points[1][0]),
                   points[0][1] + (points[0][1] - points[1][1]))
    pointMinus2 = (pointMinus1[0] + (pointMinus1[0] - points[0][0]),
                   pointMinus1[1] + (pointMinus1[1] - points[0][1]))
    # adding to the end is equivalent to index -1 and -2
    points.append(pointMinus2)
    points.append(pointMinus1)

    # interpolation
    spline.append(points[0])  # first point
    for i in range(1, pointsNumber):
        # compute the third order coefficients
        coefficients = []
        for j in range(2):
            coefficients.append([
                (-points[i - 2][j] + 3 * points[i - 1][j] - 3 * points[i][j] + points[i + 1][j]) / 6.0,
                (3 * points[i - 2][j] - 6 * points[i - 1][j] + 3 * points[i][j]) / 6.0,
                (-3 * points[i - 2][j] + 3 * points[i][j]) / 6.0,
                (points[i - 2][j] + 4 * points[i - 1][j] + points[i][j]) / 6.0
            ])
        for j in range(subdivision):
            t = float(j + 1) / subdivision
            spline.append(((coefficients[0][2] + t * (coefficients[0][1] + t * coefficients[0][0])) * t + coefficients[0][3],
                           (coefficients[1][2] + t * (coefficients[1][1] + t * coefficients[1][0])) * t + coefficients[1][3]))
    return spline

def hide_sensors_visualization(supervisor):
    """Hide all the sensors visualization nodes to the camera."""
    vehicleNode = supervisor.getFromDef(VEHICLE_DEF_NAME)
    slotNames = [
        'sensorsSlotFront',
        'sensorsSlotRear',
        'sensorsSlotTop',
        'sensorsSlotCenter'
    ]
    renderingDevicesTypeName = [
        'Camera',
        'RangeFinder',
        'Lidar'
    ]
    renderingDevicesNodes = []
    # look for rendering devices nodes in slot fields
    for slotName in slotNames:
        field = vehicleNode.getField(slotName)
        for i in range(field.getCount()):
            node = field.getMFNode(i)
            if node.getBaseTypeName() in renderingDevicesTypeName:
                renderingDevicesNodes.append(node)
    # look for visualization nodes
    field = supervisor.getFromDef('SENSOR_VISUALIZATION').getField('children')
    for i in range(field.getCount()):
        node = field.getMFNode(i)
        if field.getMFNode(i).getTypeName() == 'DistanceSensorVisualization':
            sensorVisualizationNodes.append(node)
    # hide visualization nodes to rendering devices
    for renderingDevicesNode in renderingDevicesNodes:
        for sensorsVisualizationNode in sensorVisualizationNodes:
            sensorsVisualizationNode.setVisibility(renderingDevicesNode, False)

def enable_sensors_visualization(supervisor, enable):
    """Hide or show the sensors visualization nodes."""
    for node in sensorVisualizationNodes:
        node.setVisibility(supervisor.getFromDef('VIEWPOINT'), enable)

def add_sumo(supervisor):
    """Add the SUMO interface with a free port (in the range [1024;1998]) as argument."""
    freePortFound = False
    port = 8873
    while not freePortFound:
        port = random.randint(SUMO_PORTS_RANGE[0], SUMO_PORTS_RANGE[1])

        # check if this port is already used
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.settimeout(3.0)
        try:
            s.connect(('localhost', port))
        except socket.error:
            freePortFound = True
        s.close()
    nodeString = 'SumoInterface {'
    nodeString += '  gui FALSE'
    nodeString += '  radius 200'
    nodeString += '  laneChangeDelay 4'
    nodeString += '  enableWheelsRotation TRUE'
    nodeString += '  port %d' % port
    nodeString += '  seed 0'  # equivalent to 'random' for SUMO
    nodeString += '}'
    supervisor.getRoot().getField('children').importMFNodeFromString(-1, nodeString)

supervisor = Supervisor()
hide_sensors_visualization(supervisor)
add_sumo(supervisor)

timestep = int(supervisor.getBasicTimeStep())

# get main vehicle node and position
vehicleNode = supervisor.getFromDef(VEHICLE_DEF_NAME)
initialPosition = vehicleNode.getPosition()
initialPoint = Point((initialPosition[0], initialPosition[1]))

# get first SUMO vehicle node and position
sumoFirstVehicle = supervisor.getFromDef("SUMO_VEHICLE0")
sumoFirstVehicleInitialPosition = sumoFirstVehicle.getPosition()
sumoInitializationChecked = False

# parse the road node to extract road path
roadNode = supervisor.getFromDef("ROAD")
coordinates = []
splineSubdivision = roadNode.getField("splineSubdivision").getSFInt32()
waypoints = roadNode.getField("wayPoints")
laneNumber = roadNode.getField("numberOfLanes").getSFInt32()
width = roadNode.getField("width").getSFFloat()
laneWidth = width / laneNumber
for i in range(waypoints.getCount()):
    point = waypoints.getMFVec3f(i)
    coordinates.append((point[1], point[0]))
subdividedCoordinates = coordinates
if splineSubdivision > 0:
    subdividedCoordinates = apply_spline_subdivision_to_path(coordinates, splineSubdivision)
roadPath = LineString(subdividedCoordinates)
emergencyLanePath = roadPath.parallel_offset(1.5 * laneWidth, 'right')
initialDistance = roadPath.project(initialPoint)

# main loop
inEmergencyLane = False
collided = False
sumoFailure = False
time = 0.0
while time < MAX_TIME and not inEmergencyLane and not collided and not sumoFailure and supervisor.step(timestep) != -1:
    # check that SUMO is running
    if time >= 10.0 and not sumoInitializationChecked:
        sumoInitializationChecked = True
        position = sumoFirstVehicle.getPosition()
        # check if first sumo vehicle has moved
        if (position[0] == sumoFirstVehicleInitialPosition[0] and
                position[1] == sumoFirstVehicleInitialPosition[1] and
                position[2] == sumoFirstVehicleInitialPosition[2]):
            sumoFailure = True
    # check if robot is inside the emergency lane
    position = vehicleNode.getPosition()
    positionPoint = Point((position[0], position[1]))
    if emergencyLanePath.distance(positionPoint) < 0.5 * laneWidth:
        inEmergencyLane = True
    # check for collision
    contactPoints = vehicleNode.getContactPoints()
    if len(contactPoints) > 0:
        collided = True
    time = supervisor.getTime()
    distance = roadPath.project(positionPoint) - initialDistance
    supervisor.wwiSendText("update: " + str(time) + " {0:.3f}".format(distance))
    # check if sensors visualization should be enabled/disabled
    message = supervisor.wwiReceiveText()
    while message:
        if message.startswith("sensors visualization:"):
            if message.endswith("false"):
                enable_sensors_visualization(supervisor, False)
            elif message.endswith("true"):
                enable_sensors_visualization(supervisor, True)
        message = supervisor.wwiReceiveText()

supervisor.wwiSendText("stop")

if inEmergencyLane:
    supervisor.setLabel(0, "The vehicle should not enter in the emergency lane.",
                        0.01, 0.85, 0.09, 0xFF0000, 0, "Lucida Console")
elif collided:
    supervisor.setLabel(0, "The vehicle should not collide.",
                        0.01, 0.85, 0.09, 0xFF0000, 0, "Lucida Console")
elif sumoFailure:
    supervisor.setLabel(0, "Problem with traffic generation using SUMO, we can't take this performance into account.",
                        0.01, 0.85, 0.09, 0xFF0000, 0, "Lucida Console")

position = vehicleNode.getPosition()
distance = round(roadPath.project(Point((position[0], position[1]))) - initialDistance, 3)

while supervisor.step(timestep) != -1:
    # wait for record message
    message = supervisor.wwiReceiveText()
    while message:
        if message.startswith("record:"):
            record = robotbenchmarkRecord(message, "highway_driving", distance)
            supervisor.wwiSendText(record)
            break
        elif message == "exit":
            break
        message = supervisor.wwiReceiveText()

supervisor.simulationSetMode(Supervisor.SIMULATION_MODE_PAUSE)


# -*- coding: utf-8 -*-
import argparse
import glob
import logging
import logging.handlers
import os
import platform
import sys
import traceback
import warnings
from datetime import datetime

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
from wordcloud import WordCloud
from collections import Counter
import seaborn as sns
from urllib.parse import quote_plus
import configparser
import pymysql
import zipfile
import requests
import pytz
from sqlalchemy.dialects.mysql import insert
from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, DateTime, create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy import Column, Float
from sqlalchemy.dialects.mysql import DOUBLE
from sqlalchemy import Table, Column, Integer, String, MetaData
from sqlalchemy import Column, Numeric
from sqlalchemy import create_engine, Table, MetaData
from sqlalchemy.dialects.mysql import insert as mysql_insert

# =================================================
# 사용자 매뉴얼
# =================================================
# [소스 코드의 실행 순서]
# 1. 초기 설정 : 폰트 설정
# 2. 유틸리티 함수 : 초기화 함수 (로그 설정, 초기 변수, 초기 전달인자 설정) 또는 자주 사용하는 함수
# 3. 주 프로그램 :부 프로그램을 호출
# 4. 부 프로그램 : 자료 처리를 위한 클래스로서 내부 함수 (초기 변수, 비즈니스 로직, 수행 프로그램 설정)
# 4.1. 환경 변수 설정 (로그 설정) : 로그 기록을 위한 설정 정보 읽기
# 4.2. 환경 변수 설정 (초기 변수) : 입력 경로 (inpPath) 및 출력 경로 (outPath) 등을 설정
# 4.3. 초기 변수 (Argument, Option) 설정 : 파이썬 실행 시 전달인자 설정 (pyhton3 *.py argv1 argv2 argv3 ...)
# 4.4. 비즈니스 로직 수행 : 단위 시스템 (unit 파일명)으로 관리 또는 비즈니스 로직 구현

# =================================================
# 1. 초기 설정
# =================================================
warnings.filterwarnings("ignore")

plt.rc('font', family='Malgun Gothic')
plt.rc('axes', unicode_minus=False)
# sns.set(font="Malgun Gothic", rc={"axes.unicode_minus": False}, style='darkgrid')

# 그래프에서 마이너스 글꼴 깨지는 문제에 대한 대처
mpl.rcParams['axes.unicode_minus'] = False

# =================================================
# 2. 유틸리티 함수
# =================================================
# 로그 설정
def initLog(env=None, contextPath=None, prjName=None):
    if env is None: env = 'local'
    if contextPath is None: contextPath = os.getcwd()
    if prjName is None: prjName = 'test'

    saveLogFile = "{}/{}_{}_{}_{}_{}_{}.log".format(
        contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'log', prjName)
        , platform.system()
        , platform.machine()
        , platform.architecture()[0]
        , platform.node()
        , prjName
        , datetime.now().strftime("%Y%m%d")
    )

    os.makedirs(os.path.dirname(saveLogFile), exist_ok=True)

    # logger instance 생성
    log = logging.getLogger(prjName)

    if len(log.handlers) > 0:
        return log

    # format 생성
    format = logging.Formatter('%(asctime)s [%(name)s | %(lineno)d | %(filename)s] [%(levelname)-5.5s] %(message)s')

    # handler 생성
    streamHandler = logging.StreamHandler()
    fileHandler = logging.FileHandler(saveLogFile)

    # logger instance에 format 설정
    streamHandler.setFormatter(format)
    fileHandler.setFormatter(format)

    # logger instance에 handler 설정
    log.addHandler(streamHandler)
    log.addHandler(fileHandler)

    # logger instance로 log 기록
    log.setLevel(level=logging.INFO)

    return log

#  초기 변수 설정
def initGlobalVar(env=None, contextPath=None, prjName=None):
    if env is None: env = 'local'
    if contextPath is None: contextPath = os.getcwd()
    if prjName is None: prjName = 'test'

    # 환경 변수 (local, 그 외)에 따라 전역 변수 (입력 자료, 출력 자료 등)를 동적으로 설정
    # 즉 local의 경우 현재 작업 경로 (contextPath)를 기준으로 설정
    # 그 외의 경우 contextPath/resources/input/prjName와 같은 동적으로 구성
    globalVar = {
        'prjName': prjName
        , 'sysOs': platform.system()
        , 'contextPath': contextPath
        , 'resPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources')
        , 'cfgPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'config')
        , 'inpPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'input', prjName)
        , 'figPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'fig', prjName)
        , 'outPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'output', prjName)
        , 'movPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'movie', prjName)
        , 'logPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'log', prjName)
        , 'mapPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'mapInfo')
        , 'sysPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'config', 'system.cfg')
        , 'seleniumPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'config', 'selenium')
        , 'fontPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'config', 'fontInfo')
    }

    return globalVar

#  초기 전달인자 설정
def initArgument(globalVar, inParams):
    # 원도우 또는 맥 환경
    if globalVar['sysOs'] in 'Windows' or globalVar['sysOs'] in 'Darwin':
        inParInfo = inParams

    # 리눅스 환경
    if globalVar['sysOs'] in 'Linux':
        parser = argparse.ArgumentParser()

        for i, argv in enumerate(sys.argv[1:]):
            if not argv.__contains__('--'): continue
            parser.add_argument(argv)

        inParInfo = vars(parser.parse_args())

    log.info("[CHECK] inParInfo : {}".format(inParInfo))

    for key, val in inParInfo.items():
        if val is None: continue
        # 전역 변수에 할당
        globalVar[key] = val

    # 전역 변수
    for key, val in globalVar.items():
        if env not in 'local' and key.__contains__('Path') and env and not os.path.exists(val):
            os.makedirs(val)

        globalVar[key] = val.replace('\\', '/')

        log.info("[CHECK] {} : {}".format(key, val))

        # self 변수에 할당
        # setattr(self, key, val)

    return globalVar

# ================================================
# 4. 부 프로그램
# ================================================
class DtaProcess(object):
    # ================================================
    # 요구사항
    # ================================================
    # Python을 이용한 국가별로 탄소 모니터링 비율 계산

    # ================================================================================================
    # 환경변수 설정
    # ================================================================================================
    global env, contextPath, prjName, serviceName, log, globalVar

    # env = 'local'  # 로컬 : 원도우 환경, 작업환경 (현재 소스 코드 환경 시 .) 설정
    env = 'dev'  # 개발 : 원도우 환경, 작업환경 (사용자 환경 시 contextPath) 설정
    # env = 'oper'  # 운영 : 리눅스 환경, 작업환경 (사용자 환경 시 contextPath) 설정

    if (platform.system() == 'Windows'):
        contextPath = os.getcwd() if env in 'local' else 'E:/04. TalentPlatform/Github/TalentPlatform-Python'
    else:
        contextPath = os.getcwd() if env in 'local' else '/SYSTEMS/PROG/PYTHON/PyCharm'

    prjName = 'test'
    serviceName = 'LSH0456'

    # 4.1. 환경 변수 설정 (로그 설정)
    log = initLog(env, contextPath, prjName)

    # 4.2. 환경 변수 설정 (초기 변수)
    globalVar = initGlobalVar(env, contextPath, prjName)

    # ================================================================================================
    # 4.3. 초기 변수 (Argument, Option) 설정
    # ================================================================================================
    def __init__(self, inParams):

        log.info("[START] __init__ : {}".format("init"))

        try:
            # 초기 전달인자 설정 (파이썬 실행 시)
            # pyhton3 *.py argv1 argv2 argv3 ...
            initArgument(globalVar, inParams)

        except Exception as e:
            log.error("Exception : {}".format(e))
            raise e
        finally:
            log.info("[END] __init__ : {}".format("init"))

    # ================================================================================================
    # 4.4. 비즈니스 로직 수행
    # ================================================================================================
    def exec(self):

        log.info('[START] {}'.format("exec"))

        try:

            if platform.system() == 'Windows':
                pass
            else:
                globalVar['inpPath'] = '/DATA/INPUT'
                globalVar['outPath'] = '/DATA/OUTPUT'
                globalVar['figPath'] = '/DATA/FIG'
                globalVar['updPath'] = '/DATA/CSV'

            # 옵션 설정
            sysOpt = {
            }

            # *********************************************************************************
            # 파일 검색
            # *********************************************************************************
            sectorList = ['Coal', 'Oil', 'Gas']
            countryList = ['Japan', 'Russia', 'Australia', 'US', 'Mexico', 'Chile', 'Brazil', 'South Africa', 'UK', 'Germany', 'Italy', 'France', 'Spain', 'EU27 & UK', 'India', 'China']

            # inpFile = '{}/{}/{}'.format(globalVar['inpPath'], serviceName, 'carbon-monitor-Power-maingraphdatas.xlsx')
            inpFile = '{}/{}/{}'.format(globalVar['inpPath'], serviceName, 'energy_global_datas_2023-07-20.csv')
            fileList = sorted(glob.glob(inpFile))

            # data = pd.read_excel(fileList[0])
            data = pd.read_csv(fileList[0])
            dataL1 = data.dropna()

            dataL1['dtDate'] = pd.to_datetime(data['date'], format='%d/%m/%Y')

            dataL2 = dataL1[dataL1['sector'].isin(sectorList)].reset_index(drop=True)

            # statData = dataL2.groupby(['country', 'dtDate'])['GWh'].sum().reset_index().rename(columns={'GWh': 'sumGWh'})
            statData = dataL2.groupby(['country', 'dtDate'])['value'].sum().reset_index().rename(columns={'value': 'sumVal'})

            dataL3 = dataL2.merge(statData, left_on=['country', 'dtDate'], right_on=['country', 'dtDate'], how='left')
            # dataL3['rat'] = dataL3['GWh'] /  dataL3['sumGWh']
            dataL3['rat'] = dataL3['value'] /  dataL3['sumVal']

            # 정렬
            pivotData = dataL3.pivot_table(index='dtDate', columns=['sector', 'country'], values='rat')
            pivotDataL1 = pivotData.reindex(countryList, level=1, axis=1)
            pivotDataL2 = pivotDataL1.reindex(sectorList, level=0, axis=1)

            idx = pd.date_range('2019-01-01', '2023-05-31')
            pivotDataL3 = pivotDataL2.reindex(idx)

            saveFile = '{}/{}/{}.xlsx'.format(globalVar['outPath'], serviceName, '국가별로 탄소 모니터링 비율')
            os.makedirs(os.path.dirname(saveFile), exist_ok=True)
            pivotDataL3.to_excel(saveFile, index=True)
            log.info(f'[CHECK] saveFile : {saveFile}')

        except Exception as e:
            log.error("Exception : {}".format(e))
            raise e
        finally:
            log.info('[END] {}'.format("exec"))

# ================================================
# 3. 주 프로그램
# ================================================
if __name__ == '__main__':

    print('[START] {}'.format("main"))

    try:

        # 파이썬 실행 시 전달인자를 초기 환경변수 설정
        inParams = {}

        print("[CHECK] inParams : {}".format(inParams))

        # 부 프로그램 호출
        subDtaProcess = DtaProcess(inParams)

        subDtaProcess.exec()

    except Exception as e:
        print(traceback.format_exc())
        sys.exit(1)

    finally:
        print('[END] {}'.format("main"))


"""This module contains functions for ImageNet Similarity Search."""

import ipoly
import numpy as np
import pinecone
import tensorflow as tf
import streamlit as st

def text_embedding(text, processor, model):
    """Computes the text embedding for a given text input.

    Args:
        text (str): The text input to be embedded.
        processor (transformers.PreTrainedTokenizer): The tokenizer to process the input text.
        model (transformers.PreTrainedModel): The pretrained model to compute the embeddings.

    Returns:
        tf.Tensor: The normalized text embedding tensor.
    """
    tokens = processor(text=text, padding=True, images=None, return_tensors="tf")
    text_emb = model.get_text_features(**tokens)
    norm_factor = np.linalg.norm(text_emb, axis=1)
    text_emb = tf.transpose(text_emb) / norm_factor
    text_emb = tf.transpose(text_emb)
    return text_emb

def image_embedding(image_batch, processor, model):
    """Computes image embeddings for a batch of images.

    Args:
        image_batch (List[Union[PIL.Image.Image, np.ndarray]]): A list of images to be embedded.
        processor (transformers.PreTrainedTokenizer): The tokenizer to process the input images.
        model (transformers.PreTrainedModel): The pretrained model to compute the embeddings.

    Returns:
        tf.Tensor: The normalized image embedding tensor.
    """
    images = processor(text=None, images=image_batch, return_tensors="tf")[
        "pixel_values"
    ]
    img_emb = model.get_image_features(images)
    norm_factor = np.linalg.norm(img_emb, axis=1)
    img_emb = tf.transpose(img_emb) / norm_factor
    img_emb = tf.transpose(img_emb)
    return img_emb

def connect_pinecone(processor, model, api_key, environment="eu-west1-gcp"):
    """Connects to Pinecone and initializes the index with image embeddings if it doesn't exist.

    Args:
        processor (transformers.PreTrainedTokenizer): The tokenizer to process the input images.
        model (transformers.PreTrainedModel): The pretrained model to compute the embeddings.
        api_key (str): The API key for Pinecone.
        environment (str, optional): The Pinecone environment to connect to. Defaults to "us-central1-gcp".

    Returns:
        pinecone.Index: The Pinecone index containing the image embeddings.
    """
    pinecone.init(api_key=api_key, environment=environment)
    index_name = "imagenet-index"
    if not (index_name in pinecone.list_indexes()):
        dataset = ipoly.load("imagenet-sample/data", keep_3D=True, file_names=True)
        np_dataset = np.array(dataset)
        img_embd = image_embedding(np_dataset[:, 0].tolist(), processor, model)
        index_vectors = list(zip(np_dataset[:, 1].tolist(), img_embd.numpy().tolist()))
        pinecone.create_index(index_name, dimension=512)
        index = pinecone.Index(index_name)
        index.upsert(index_vectors)
    else:
        index = pinecone.Index(index_name)
    return index

def get_images(text_input, processor, model, index):
    """Finds the top 5 most similar images to the input text from the Pinecone index.

    Args:
        text_input (str): The input text to find related images.
        processor (transformers.PreTrainedTokenizer): The tokenizer to process the input text and images.
        model (transformers.PreTrainedModel): The pretrained model to compute the embeddings.
        index (pinecone.Index): The Pinecone index containing the image embeddings.

    Returns:
        List[Tuple[str, float]]: A list of tuples with image names and their similarity scores.
    """
    test_embd = text_embedding(text_input, processor, model).numpy().tolist()
    return index.query(test_embd, top_k=5, include_metadata=True)

if __name__ == "__main__":
    st.title("Image seeker")

    api_token = st.text_input("Enter your API token", type="password")
    text_environment=st.text_input("Enter your cloud environment ex: eu-west1-gcp")
    text_input = st.text_input("Enter a small text")

    model, processor = ipoly.load_transformers("openai/clip-vit-base-patch32")

    if st.button("Validate"):
        if not api_token:
            st.error("Please enter a valid API token.")
        else:
            index = connect_pinecone(processor, model, api_token,text_environment)
            images = get_images(text_input, processor, model, index)
            images = [name["id"] for name in images["matches"]]
            if images:
                for img in images:
                    st.image(img, caption="Image", use_column_width=True)
            else:
                st.error("No images were found.")


import asyncio
from datetime import datetime
from typing import Any

import httpx
from loguru import logger

from app.database.models import Post, Comment, Group
from app.database.services import PostService, CommentService, GroupService
from app.libs.ml_lib import SentimentalAnalysisModel
from app.loader import session
from app.settings.config import VK_TOKEN

class VkParser:
    def __init__(self) -> None:
        self.url = "https://api.vk.com/method/"
        self.wall_get = self.url + "wall.get"
        self.wall_getComments = self.url + "wall.getComments"
        self.groups_getGroup = self.url + "groups.getById"
        self.groups_getMembers = self.url + "groups.getMembers"
        self.vk_version = 5.131

        # Инициализация модели нейросети для анализа тональности текста
        self.sentiment_model = SentimentalAnalysisModel()

    async def get_group_byid(self, group: str) -> bool:
        """
        Получение данных группы
        """
        logger.info(f"Начался сбор групп")

        self.group_metadata: list[dict[str, Any]] = []
        self.group_update_metadata: list[dict[str, Any]] = []
        params_get_group = {
            "group_id": group,
            "access_token": VK_TOKEN,
            "count": 0,
            "v": self.vk_version,
        }

        try:
            get_group = httpx.get(self.groups_getGroup, params=params_get_group).json()["response"][0]
        except KeyError:
            logger.warning("Ошибка парсинга группы (Ошибка get-запроса: невозможно собрать данные)")
            return False

        params_group_members = {
            "group_id": get_group["id"],
            "access_token": VK_TOKEN,
            "count": 0,
            "v": self.vk_version,
        }

        get_group_members = httpx.get(self.groups_getMembers, params=params_group_members).json()["response"]

        group_data = {
            "group_id": get_group["id"],
            "group_name": get_group["name"],
            "screen_name": get_group["screen_name"],
            "group_members": get_group_members["count"],
        }
        await self.insert_group_data_db(group_data)
        return True

    async def insert_group_data_db(self, group_data: dict[str, Any]) -> None:
        """
        Вставляет полученные данные в таблицу groups
        """
        service_group = GroupService(Group)
        if session.query(Group).filter(Group.group_id == group_data["group_id"]).first() is None:
            self.group_metadata.append(group_data)
            service_group.add_all(self.group_metadata)
        else:
            self.group_update_metadata.append(group_data)
            service_group.update_all(self.group_update_metadata)

    async def get_posts(self) -> None:
        """
        Получение постов по группе
        """
        logger.info(f"Начался сбор постов")

        self.posts_metadata: list[dict[str, Any]] = []
        self.posts_update_metadata: list[dict[str, Any]] = []
        groups = self.group_metadata + self.group_update_metadata

        for group in groups:
            params = {
                "domain": group["screen_name"],
                "count": 60,
                "access_token": VK_TOKEN,
                "v": self.vk_version,
            }
            response = httpx.get(self.wall_get, params=params).json()

            for post in response["response"]["items"]:
                post_data = {
                    "post_id": post["id"],
                    "owner_id": post["owner_id"],
                    "group_id": group["group_id"],
                    "likes": post["likes"]["count"],
                    "quantity_comments": post["comments"]["count"],
                    "reposts": post["reposts"]["count"],
                    "views": post["views"]["count"],
                    "photo": True if "attachments" in post else False,
                    "post_text": post["text"],
                    "date": datetime.fromtimestamp(post["date"]),
                }
                if session.query(Post).filter(Post.post_id == post_data["post_id"]).first() is None:
                    self.posts_metadata.append(post_data)
                else:
                    self.posts_update_metadata.append(post_data)
        await self.insert_posts_data_db()

    async def insert_posts_data_db(self):
        """
        Вставляет полученные данные в таблицу posts
        """
        service_post = PostService(Post)
        service_post.add_all(self.posts_metadata)
        service_post.update_all(self.posts_update_metadata)

    async def get_wall_comments(self) -> None:
        """
        Парсинг комментариев поста
        И занесение данных в бд
        """
        logger.info(f"Начался сбор комментариев")

        self.comments_metadata: list[dict[str, Any]] = []
        self.comments_update_metadata: list[dict[str, Any]] = []
        posts = self.posts_metadata + self.posts_update_metadata
        codes = await self.getcomments_methods_vkscript(posts)
        service_post = PostService(Post)

        for num, code in enumerate(codes, start=1):
            params = {
                "count": 100,
                "code": f"return [{code}];",
                "sort": "desc",
                "access_token": VK_TOKEN,
                "v": self.vk_version,
            }

            # Обход ограничение на 3 запроса в секунду
            # Лучше не убирать, если вы хотите собирать больше постов
            if num % 3 == 0:
                await asyncio.sleep(1)

            response = httpx.get(self.url + "execute/", params=params).json()

            for post in response["response"]:
                tones_post = {
                    "post_id": None,
                    "positive_comments": 0,
                    "negative_comments": 0,
                }
                for comment in post["items"]:
                    if len(comment["text"]) > 0:
                        tone = self.sentiment_model.set_tone_comment([comment["text"]])
                        comment_data = {
                            "comment_id": comment["id"],
                            "post_id": comment["post_id"],
                            "text": comment["text"],
                            "tone": tone,
                        }
                        tones_post["post_id"] = comment_data["post_id"]

                        if (
                            session.query(Comment).filter(Comment.comment_id == comment_data["comment_id"]).first()
                        ) is None:
                            if tone == "positive":
                                tones_post["positive_comments"] += 1
                            elif tone == "negative":
                                tones_post["negative_comments"] += 1
                            self.comments_metadata.append(comment_data)
                        else:
                            self.comments_update_metadata.append(comment_data)

                # Обновление количества позитивных/негативных комментариев поста
                if tones_post["post_id"] is not None:
                    service_post.update_tonal_comments(tones_post)
        await self.insert_comments_data_db()

    async def insert_comments_data_db(self):
        """
        Вставляет полученные данные в таблицу comments
        """
        service_comment = CommentService(Comment)
        service_comment.add_all(self.comments_metadata)
        service_comment.update_all(self.comments_update_metadata)

    async def getcomments_methods_vkscript(self, posts):
        """
        Функция возвращает список с методами для получения комментариев
        Методы пишутся на языке VKScript (Документация VK)
        """
        execute_methods = ""
        methods = []
        for number, post in enumerate(posts, start=1):
            if number % 25 != 0:
                execute_methods += f"""API.wall.getComments({{"owner_id": {post['owner_id']}, count: {100}, "post_id": {post['post_id']}}}), """
            else:
                execute_methods += f"""API.wall.getComments({{"owner_id": {post['owner_id']}, count: {100}, "post_id": {post['post_id']}}}), """
                methods.append(execute_methods)
                execute_methods = ""
        methods.append(execute_methods)
        return methods

    async def run_vk_parser(self, group: str) -> None | bool:
        tasks = [self.get_group_byid(group), self.get_posts(), self.get_wall_comments()]
        result_tasks = await asyncio.gather(*tasks)
        if False in result_tasks:
            return False
        else:
            return True


import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
from lists import solutions, queries, dependencies, greetings

def process(message):
	keywords = word_tokenize(message)

	not_flag = 0
	working_flag = 0
	submit_flag = 0
	bid_flag = 0
	confrm_flag = 0
	status_flag = 0
	edit_flag = 0
	one_flag = 0
	two_flag = 0
	price_flag = 0
	attachment_flag = 0
	technical_flag = 0

	for word in keywords:
		word = word.lower() #convert to lowercase

		for key in greetings:
			if key == word:
				return solutions['hi']

		for key in dependencies:
			if key == word:
				return solutions[key]

		for key in queries:
			if key == word:
				if key in solutions.keys():
					return solutions[key]
				if key == 'webpage' or key == 'website':
					return solutions['website']
				if key == 'tenders':
					return solutions['all tenders']
				if key == 'tender':
					return solutions['particular tender']
				if key == 'bid' or key == 'bidding':
					bid_flag = 1
				if key == 'submit' or key == 'submitted' or key == 'submitting':
					submit_flag = 1
				if key == 'confirmation' or key == 'confirm' or key == 'confirmed':
					confrm_flag = 1
				if key == 'status':
					status_flag = 1
				if key == 'not':
					not_flag = 1
				if key == 'working' or key == 'worked' or key == 'work':
					working_flag = 1
				if key == 'edit' or key == 'change':
					edit_flag = 1
				if key == '1' or key == 'one' or key == 'single':
					one_flag = 1
				if key == '2' or key == 'two' or key == 'double':
					two_flag = 1
				if key == 'attachment' or key == 'attach':
					attachment_flag = 1
				if key == 'price' or key == 'cost':
					price_flag = 1
				if key == 'technical' or key == 'technological':
					technical_flag = 1

		if not_flag == 1 and working_flag == 1 :
			return solutions['working']

		if bid_flag == 1 and submit_flag == 1:
			return solutions['submit bid']

		if bid_flag == 1 and confrm_flag == 1 and status_flag == 1:
			return solutions['confirm bid status']

		if bid_flag == 1:
			if one_flag == 1:
				return solutions['1 stage bid']
			elif two_flag == 1:
				return solutions['2 stage bid']
			else:
				return "What type of bid, 2 stage or 1 stage?"

		if bid_flag == 1 and edit_flag == 1:
			return solutions['edit bid']

		if attachment_flag == 1 :
			if price_flag == 1:
				return solutions['price attachment']
			elif technical_flag == 1:
				return solutions['technical attachment']
			else:
				return 'What type of attachment, price of technical?'

		'''for key in vendor_list:
			if key == word:
				return solutions['vendor']

		for key in tender_list:
			if key == word:
				return solutions['tender']'''

	return "Keyword not found!"


# Import the Necessary Packages 🤔🤔🤔
import matplotlib.pyplot as plt
import base64
from io import BytesIO
from wordcloud import WordCloud
from PIL import Image
import numpy as np
import os
import pandas as pd

# the plotly package used in most of the dashboard visualization 😊😊
import plotly_express as px
import plotly.graph_objects as go
import plotly.offline as plt1

# -------------------------------------------------------------------------------------------

# draft template of all the plots to show the watermark
draft_template = go.layout.Template()
# Take the format of each plotting watermark and layout and will use in each plot
draft_template.layout.annotations = [
    dict(
        name="draft watermark",
        text="MYN",
        textangle=-30,
        opacity=0.5,
        font=dict(color="lightblue", size=50),
        xref="paper",
        yref="paper",
        showarrow=False,
    )
]
global fig1
fig1=go.Figure()

# 😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊🍿🍿📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽
# system task to set the graph
def get_graph():
    buffer=BytesIO()
    plt.savefig(buffer,format='png')
    buffer.seek(0)
    image_png=buffer.getvalue()
    graph=base64.b64encode(image_png)
    graph=graph.decode('utf-8')
    buffer.close()
    return graph

# 😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊🍿🍿📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽

# Function to show the barplot of the total stars in a movie
def stars(data,g,y):
    # take the value_count of the top 15 stars in year and genre basis
    bar = data["first_name"].value_counts()[1:16]
    # taking the bar-plot
    fig = px.bar(bar,
       x = bar.index,
       y = bar.values,
       title="<b> Top 15 Stars Acted in movie in {} genre, {} year</b>".format(g, y),
       text_auto='.2s',
       color = "first_name",
       width=800,
       height=600,
       )
    # update the layout and add the watermark
    fig.update_layout(template = draft_template, paper_bgcolor = "black")
    fig.update_traces(textfont_size=12, textangle=0, textposition="outside", cliponaxis=False)
    fig.update_layout(
        font_color="white",
        title_font_color="white",
        xaxis_title="Name of Stars",
        yaxis_title="Total no of Movies"
    )

    gantt_plt = plt1.plot(fig, output_type='div')
    return gantt_plt

# 😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊🍿🍿📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽

# the wordcloud to show the frequency of the directors in movies
def wordcloud(d):
    # taking the required format from system
    mask=np.array(Image.open(r'E:\movies\wordlogo.png'))
    # finding the wordcloud
    wc = WordCloud(mask=mask,min_font_size=10, background_color='black',contour_color='white', contour_width=10, colormap='Paired',width=mask.shape[1],height=mask.shape[0],random_state=42)
    df_wc = wc.generate(d['director'].str.cat(sep=' '))
    figx=plt.figure()
    figx.patch.set_facecolor('black')
    plt.imshow(df_wc,interpolation='None')
    plt.axis('off')
    plt.tight_layout()
    graph=get_graph()
    return graph

# 😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊🍿🍿📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽

# The function to show the histogram of total certificates in movies
def certificate(data,g,y):
    # plotting the histogram
    fig = px.histogram(data, x = "certificate", title="<b> Year wise Certificates (HistPlot with Boxplot)\t <total count : {}> </b>".format(len(data)), marginal="box", template="ggplot2", color_discrete_sequence=["#AB63FA"])
    fig.update_layout(template = draft_template, paper_bgcolor = "black")
    fig.update_layout(
        font_color="white",
        title_font_color="white",
        legend_title_font_color="white"
    )
    gantt_plt=plt1.plot(fig,output_type='div')
    return gantt_plt

# 😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊🍿🍿📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽

# The function to show the pie-plot of the status/tv-shows
def status(df,g,y):
    # taking the values of the tv-shows in 'pie'
    pie = df["status"].value_counts()
    fig = go.Figure(data=[go.Pie(labels=pie.index, values=pie, pull=[0, 0.2], hole = 0.3, title = "<b> MYN </b>")])
    fig.update_traces(textposition="inside", textinfo="percent+label")
    # updating the layout according to the 'MYN' requirments
    fig.update_layout(title="<b> Tv-Shows in {} year and {} genre </b>".format(y, g), height=450, width=450, paper_bgcolor = "black")
    fig.update_layout(
        font_color="white",
        title_font_color="white",
        xaxis_title="Name of Stars",
        yaxis_title="Total no of Movies"
    )
    gantt_plt=plt1.plot(fig,output_type='div')
    return gantt_plt

# 😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊🍿🍿📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽

# The function to show the scatterplot of the avarage rating
def rating(data,g, y):
    # taking the avarage rating of each year in 'rg'
    rg = data.groupby("start_year")["rating"].mean().reset_index()
    # doing the scatterplot
    fig = px.scatter(rg, x="start_year", y="rating", color="start_year", size='rating')
    # altering the layout according to the requiments
    fig.update_layout(template = draft_template, paper_bgcolor = "black", title="<b> Avarage Rating of Movies in {} year, {} Genre </b>".format(y,g))
    fig.update_layout(
        font_color="white",
        title_font_color="white",
        legend_title_font_color="white"
    )
    gantt_plt = plt1.plot(fig, output_type='div')
    return gantt_plt

# 😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊😊🍿🍿📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽📽

# The function to show the lineplot of the total time of the movies
def line_time(data,g,y):
    # taking the total time of movies in each year basis
    line = data.groupby("start_year")["time"].agg("sum").reset_index()
    #  some condition to satisfy the requirmrnts
    if y=='all':
        # doing the lineplot
        fig = px.line(line, x="start_year", y="time")
        fig.update_layout(template = draft_template, paper_bgcolor = "black", title="<b> Total Time of Movies in {} year, {} Genre </b>".format(y, g))
        # altering the layout according to the requiments
        fig.update_layout(
            font_color="white",
            title_font_color="white",
            legend_title_font_color="white"
        )
    else:
        # otherwise do the barplot
        fig = px.bar(line, x="start_year", y="time")
        fig.update_layout(template=draft_template, paper_bgcolor="black",
                          title="<b> Total Time of Movies in {} year, {} Genre </b>".format(y, g))
        # altering the layout according to the requiments
        fig.update_layout(
            font_color="white",
            title_font_color="white",
            legend_title_font_color="white"
        )

    gantt_plt = plt1.plot(fig, output_type='div')
    return gantt_plt

# -----------------------------------------------------------------------------------------------------------------


from datetime import datetime
from dateutil.relativedelta import relativedelta

from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import permissions
from tasks.models import Task

class CountTaskReport(APIView):
    """
    Формируем статистики по месяцам
    """

    permission_classes = [permissions.IsAuthenticated]

    def get(self, request):

        start = request.query_params['start'].rsplit('-', 1)[0]
        end = request.query_params['end'].rsplit('-', 1)[0]

        start_date = datetime.strptime(start, "%Y-%m")
        end_date = datetime.strptime(end, "%Y-%m")

        labels = []

        count_all_tasks = []
        count_executed_tasks = []
        count_expired_tasks = []

        while start_date <= end_date:

            date_str = start_date.strftime("%Y-%m")
            labels.append(date_str)

            count_all_tasks.append(Task.objects.filter(createDate__gte=start_date).filter(
                createDate__lte=start_date + relativedelta(months=1)).count())

            count_executed_tasks.append(Task.objects.filter(createDate__gte=start_date).
                                        filter(createDate__lte=start_date+relativedelta(months=1)).
                                        filter(is_done=True).count())

            count_expired_tasks.append(Task.objects.filter(createDate__gte=start_date).
                                       filter(createDate__lte=start_date + relativedelta(months=1)).
                                       filter(expired=True).filter(is_done=False).count())

            start_date += relativedelta(months=1)

        date_list = {
            'labels': labels,
            'datasets': {
                'count_all_tasks': count_all_tasks,
                'count_executed_tasks': count_executed_tasks,
                'count_expired_tasks': count_expired_tasks,
            }
        }

        return Response(date_list)


import tensorflow as tf

from linear import Linear

class MLP(tf.Module):
    def __init__(self, num_inputs, num_outputs, num_hidden_layers,
                 hidden_layer_width, hidden_activation=tf.identity,
                 output_activation=tf.identity):
        self.num_inputs = num_inputs
        self.num_outputs = num_outputs
        self.num_hidden_layers = num_hidden_layers
        self.hidden_layer_width = hidden_layer_width
        self.hidden_activation = hidden_activation
        self.output_activation = output_activation
        self.hidden_linear = Linear(self.hidden_layer_width,
                                    self.hidden_layer_width)
        self.first_linear = Linear(num_inputs, hidden_layer_width)
        self.final_linear = Linear(self.hidden_layer_width, self.num_outputs)

    def __call__(self, x):
        x = self.hidden_activation(self.first_linear(x))
        for i in range(self.num_hidden_layers):
            x = self.hidden_activation(
                self.hidden_linear(x)
            )
        return self.output_activation(self.final_linear(x))

if __name__ == "__main__":
    import argparse
    from math import pi
    from pathlib import Path

    import matplotlib.pyplot as plt
    import yaml
    from tqdm import trange

    from linear import grad_update

    parser = argparse.ArgumentParser(
        prog="Multi Layer Perceptron",
        description="Uses a multi layer perceptron on some data, \
              given a config",
    )
    parser.add_argument(
        "-c",
        "--config",
        type=Path,
        default=Path("config.yaml")
    )
    args = parser.parse_args()

    config = yaml.safe_load(args.config.read_text())

    rng = tf.random.get_global_generator()
    rng.reset_from_seed(0x43966E87BD57227011B5B03B58785EC1)

    num_samples = config["data"]["num_samples"]
    num_inputs = 2
    num_outputs = 1

    noise_stddev = config["data"]["noise_stddev"]
    e1 = rng.normal(shape=(num_samples, 1), stddev=noise_stddev)
    e2 = rng.normal(shape=(num_samples, 1), stddev=noise_stddev)
    r = tf.linspace(pi/4, 4*pi, num_samples)[:, tf.newaxis]

    x1 = (r + e1)*tf.math.cos(r)
    y1 = (r + e1)*tf.math.sin(r)
    x2 = -(r + e2)*tf.math.cos(r)
    y2 = -(r + e2)*tf.math.sin(r)

    num_hidden_layers = config["mlp"]["num_hidden_layers"]
    hidden_layer_width = config["mlp"]["hidden_layer_width"]

    def relu(x):
        return tf.maximum(x, 0)

    def sigmoid(x):
        return tf.math.sigmoid(x)

    mlp = MLP(num_inputs, num_outputs, num_hidden_layers, hidden_layer_width,
              relu, sigmoid)

    num_iters = config["learning"]["num_iters"]
    step_size = config["learning"]["step_size"]
    decay_rate = config["learning"]["decay_rate"]
    batch_size = config["learning"]["batch_size"]

    refresh_rate = config["display"]["refresh_rate"]

    l2_scale = config["learning"]["l2_scale"]

    bar = trange(num_iters)

    for i in bar:
        batch_indices = rng.uniform(
            shape=[batch_size*2], maxval=num_samples*2, dtype=tf.int32
        )
        with tf.GradientTape() as tape:
            output1 = tf.zeros((num_samples, 1))
            output2 = tf.ones((num_samples, 1))

            input1 = tf.concat((x1, y1), axis=1)
            input2 = tf.concat((x2, y2), axis=1)

            output_combined = tf.concat((output1, output2), axis=0)
            input_combined = tf.concat((input1, input2), axis=0)

            output_batch = tf.gather(output_combined, batch_indices)
            input_batch = tf.gather(input_combined, batch_indices)

            y_hat = mlp(input_batch)

            weightTensor = tf.concat([tf.reshape(weight, [-1]) for weight in
                                      mlp.trainable_variables if weight.name ==
                                      "Linear/w:0"], axis=0)

            l2_norm = tf.norm(weightTensor, ord=2)
            loss = tf.math.reduce_mean(
                -output_batch*tf.math.log(y_hat + 1e-7) -
                (1 - output_batch)*tf.math.log(1 - y_hat + 1e-7)
            ) + l2_scale*l2_norm

        grads = tape.gradient(loss, mlp.trainable_variables)
        grad_update(step_size, mlp.trainable_variables, grads)

        step_size *= decay_rate

        if i % refresh_rate == (refresh_rate - 1):
            bar.set_description(
                f"Step {i}; Loss => {loss.numpy():0.4f}, \
                    step_size => {step_size:0.4f}"
            )
            bar.refresh()

    fig, ax = plt.subplots()

    ax.plot(x1.numpy().squeeze(), y1.numpy().squeeze(), "x", label="Inputs")
    ax.plot(x2.numpy().squeeze(), y2.numpy().squeeze(), "x", label="Inputs")

    x1_quiet = r*tf.math.cos(r)
    y1_quiet = r*tf.math.sin(r)
    ax.plot(
        x1_quiet.numpy().squeeze(),
        y1_quiet.numpy().squeeze(),
        "-",
        label="Clean"
    )

    x2_quiet = -r*tf.math.cos(r)
    y2_quiet = -r*tf.math.sin(r)
    ax.plot(
        x2_quiet.numpy().squeeze(),
        y2_quiet.numpy().squeeze(),
        "-",
        label="Clean"
    )

    ax.set_xlabel("x")
    ax.set_ylabel("y")
    ax.legend()
    ax.set_title("Nonlinear fit using SGD")

    h = ax.set_ylabel("y", labelpad=10)
    h.set_rotation(0)

    import numpy as np
    from sklearn.inspection import DecisionBoundaryDisplay
    xx, yy = np.meshgrid(
        np.linspace(-15., 15., 100),
        np.linspace(-15., 15., 100)
    )

    input_grid = tf.stack(tf.convert_to_tensor((xx.ravel(), yy.ravel()),
                                               dtype=tf.float32), axis=1,)
    zz = mlp(input_grid)
    zz = zz.numpy().reshape(xx.shape)

    dbd = DecisionBoundaryDisplay(xx0=xx, xx1=yy, response=zz)
    dbd.plot(ax=ax, cmap="RdBu", alpha=0.9)

    fig.savefig("artifacts/mlp.pdf")


from abc import ABC, abstractmethod
from typing import List, Any

class Tokenizer(ABC):

    @abstractmethod
    async def encode(self, text: str) -> List[Any]:
        raise NotImplementedError

    @abstractmethod
    def decode(self, raw_text: str, tokens: List[Any]) -> str:
        raise NotImplementedError


# Import streamlit
import streamlit as st
import os
from neuralintents import GenericAssistant
import matplotlib.pyplot as plt
import pandas as pd
import pandas_datareader as pdr
import mplfinance as mpf
import pickle
import datetime as dt

st.title("Financial Assistant Chatbot")

portfolio_path = './data/portfolio.pkl'
portfolio = {}
if os.path.exists(portfolio_path):
    with open(portfolio_path, 'rb') as f:
        portfolio = pickle.load(f)

def greeting():
    st.write("Suuuuuuh Dude!")

def save_portfolio():
    with open('portfolio.pkl', 'wb') as f:
        pickle.dump(portfolio, f)

def add_portfolio(ticker, amount):
    if ticker in portfolio.keys():
        portfolio[ticker] += int(amount)
    else:
        portfolio[ticker] = int(amount)

    save_portfolio()

def remove_portfolio(ticker, amount):
    if ticker in portfolio.keys():
        if int(amount) <= portfolio[ticker]:
            portfolio[ticker] -= int(amount)
            save_portfolio()
        else:
            st.write("You don't have the facilities for that big man!")
    else:
        st.write(f"You don't own any shares of {ticker}")

def show_portfolio():
    st.write("Your Portfolio:")
    for ticker in portfolio.keys():
        st.write(f"You own {portfolio[ticker]} shares of {ticker}")

def portfolio_worth():
    sum = 0
    for ticker in portfolio.keys():
        data = pdr.Datareader(ticker, 'yahoo')
        price = data['Close'].iloc[-1]
        sum += price

    st.write(f"Your portfolio is worth {sum} USD")

def portfolio_gains(starting_date):
    sum_now = 0
    sum_then = 0

    try:
        for ticker in portfolio.keys():
            data = pdr.DataReader(ticker, 'yahoo')
            price_now = data['Close'].iloc[-1]
            price_then = data.loc[data.index == starting_date]['Close'].values[0]
            sum_now += price_now
            sum_then += price_then

        st.write(f"Relative Gains: {((sum_now-sum_then)/sum_then)* 100}%")
        st.write(f"Absolute Gains: {sum_now-sum_then} USD")
    except IndexError:
        st.write("No Trading On This Day")

import yfinance as yf

def plot_chart(ticker, starting_string):
    plt.style.use('dark_background')

    start = dt.datetime.strptime(starting_string, "%d/%m/%Y")
    end = dt.datetime.now()

    data = yf.download(ticker, start=start.strftime('%Y-%m-%d'), end=end.strftime('%Y-%m-%d'))

    colors = mpf.make_marketcolors(up = '#00ff00', down = '#ff0000', wick = 'inherit', edge = 'inherit', volume = 'in')
    mpf_style = mpf.make_mpf_style(base_mpf_style = 'nightclouds', marketcolors = colors)
    mpf.plot(data, type = 'candle', style = mpf_style, volume = True)
    st.pyplot(plt)

mappings = {
    'greeting': greeting,
    'plot_chart' : plot_chart,
    'add_portfolio': add_portfolio,
    'remove_portfolio': remove_portfolio,
    'show_portfolio': show_portfolio,
    'portfolio_worth': portfolio_worth,
    'portfolio_gains': portfolio_gains,
}
assistant = GenericAssistant('intents.json', mappings, "Jafire")

assistant.load_model()

user_message = st.text_input("Your message:")
assistant.request(user_message)


# Functions
import choices
# To work with MySQL
import pymysql
# For the "press any key" in option 1
from getkey import getkey
# To work with months
import datetime
# To work with Neo4j
from neo4j import exceptions

# Main function
def main():
    display_menu()

    while True:
        choice = input("\nChoice: ")

        # Choice ONE
        if (choice == "1"):
            # Set the offset for MySQL cypher (next 2 employees)
            offset = 0

            print ("EMPLOYEES AND DEPARTMENTS")
            print ("--------------------------")

            x = choices.one(offset)
            for i in x:
                print (i["name"], "|", i["d.name"])
            print("-- Quit (q) --")

            # Loop when pressing any key other then "q"
            while True:
                key = getkey()
                # If any key is pressed
                if key != "q":
                    offset += 2
                    x = choices.one(offset)
                    for i in x:
                        print (i["name"], "|", i["d.name"])
                    print("-- Quit (q) --")
                # If "q" is pressed
                else:
                    offset = 0
                    display_menu()
                    break

        # Choice TWO
        elif (choice == "2"):
            print ("SALARY DETAILS")
            print ("--------------------------")

            # Prompt the input
            eid = str(input("Enter Employee ID (EID): "))

            x = choices.two(eid)
            # If non-existing EID is entered
            for i in x:
                if i["eid"] == None:
                    print("Minimum", "|", "Avarage ", "|", "Maximum")
                    print("Non existing EID entered. Please try again.")
            # If existign EID is entered
                else:
                    for i in x:
                        print("\nSalary Details For Employee: " + eid)
                        print("Employee name: " + i["name"])
                        print("----------------------------")
                        print("Minimum", "|", "Avarage ", "|", "Maximum")
                        print("----------------------------")
                        print(i["Minimum"], "|", i["Avarage"], "|", i["Maximum"],"\n")

        # Choice THREE
        elif (choice == "3"):
            print ("EMPLOYEES BY MONTH OF BIRTH")
            print ("---------------------------")

            # Create a list of month names
            month_names = []
            for i in range(1,13):
                month_names.append (((datetime.date(2022, i, 1).strftime('%b'))).upper())
            # Create a list of month numbers
            month_num = []
            for i in range(1,13):
	            month_num.append(str(i)) # Convert to string (because input will be str)
            # Prompt for input
            month = (input("Enter Month of Birth: ")).upper()

            # Loop while the input isn't in either of 2 created lists
            while (month not in month_num) and (month not in month_names):
               month = (input("Enter Month of Birth: ")).upper() # Upper to match lower and upper case
            # If input is a number
            if month in month_num:
                x = choices.three(month)
                for i in x:
                        print(i["eid"], "|", i["name"], "|", i["dob"])
            # If input is a month name
            elif month in month_names:
                x = choices.three(month)
                for i in x:
                        print(i["eid"], "|", i["name"], "|", i["dob"])

        # Choice FOUR
        elif (choice == "4"):

            print ("ADD NEW EMPLOYEE")
            print ("----------------")

            # Prompt for inputs
            employeeID = input ("EID: ")
            name = input ("Name: ")
            employeeDob = input ("DOB (YYYY-MM-DD): ")
            deptID = (input ("Dept ID: ")).upper()

            # If all input is valid run the function
            try:
                choices.four(employeeID, name, employeeDob, deptID)
            # ERROR handling
                # If EID or Dept ID input is invalid
            except pymysql.err.IntegrityError as e:
                if e.args[1] == ("Duplicate entry 'e01' for key 'employee.PRIMARY'"):
                    print("*** ERROR ***:", employeeID, " already exists")
                else:
                    print ("*** ERROR ***: ", deptID, "does not exist")
                # If DOB inout is invalid
            except pymysql.err.OperationalError as e:
                print ("*** ERROR ***: Invalid DOB")
                # If EID or Dept ID inputs are too long (limited in SQL database)
            except pymysql.err.DataError as e:
                if e.args[1] == ("Data too long for column 'did' at row 1"):
                    print("*** ERROR ***: ", deptID, "does not exist")
                else:
                    print("*** ERROR ***: EID too long. Use 3 characters max.")
                # Any other errors
            except Exception as e:
                print ("***ERROR***")

        # Choice FIVE
        elif (choice == "5"):
            print ("DEPARTMENTS MANAGED BY EMPLOYEE")
            print ("-------------------------------")

            # Neo4j function
            n = choices.five()
            print ("\nDepartment   |   Budget")

            # Loop through Neo4j results
            for i in n:
                # MySQL function - pass the results from Neo4j to MySQL
                x = choices.fiveSQL(i)
                for a in x:
                    print (a["did"], " | ", a["budget"])

        # Choice SIX
        elif (choice == "6"):
            print ("ADD MANAGER TO DEPARTMENT")
            print ("-------------------------")
            # Prompt for inputs
            eid = input("Enter EID: ")
            did = input ("Enter DID: ")

            x1 = choices.sqlEID(eid)
            x2 = choices.sqlDID(did)

            # Loop if inputs are not in SQL db
            while not x1 or not x2:
                if not x1 and not x2:
                    print ("EID does not exist.")
                    print ("DID does not exist.")
                elif not x1:
                    print ("EID does not exist.")
                elif not x2:
                    print ("DID does not exist.")

                eid = input("\nEnter EID: ")
                did = input ("Enter DID: ")

                x1 = choices.sqlEID(eid)
                x2 = choices.sqlDID(did)
            # If inputs are valid
            else:
                for i in x1:
                    eid = (i["eid"])
                for i in x2:
                    did = (i["did"])

            # Create Constraint
                # avoids creating a dept managed by multiple managers
            choices.NeoConst()

            # If inputs are valid, call the function
            try:
                choices.six(eid,did)
                print ("Employee", eid, "now manages Department", did,".")
            # If Dept already exists in Neo4j db (constraint)
            except exceptions.ConstraintError as e:
                x = choices.exception(did)
                print ("Department", did, "is already managed by Employee", x, ".")

        # Choice SEVEN
        elif (choice == "7"):
            print ("VIEW DEPARTMENTS")
            print ("----------------")

            # Append the SQL db to an array to save it in memory
                # If array is empty, append results from SQL and output it
                # If it isn't empty, print out the array

            # Output from array (won't run first time when option is chosen)
            try:
                print ("Did    |    Name    |    Location    |    Budget")
                for i in db:
                    print(i["did"], " | ", i["name"], " | ", i["lid"], " | ", i["budget"] )
            # Create an array and append the SQL db (runs when option chosen first time)
            except:
                db = []
                x = choices.seven()
                for i in x:
                    db.append(i)
                for i in db:
                    print(i["did"], " | ", i["name"], " | ", i["lid"], " | ", i["budget"] )

        # Choice EIGHT
        elif (choice == "x"):
            break
        else:
            display_menu()

# Displaying the menu - Menu function
def display_menu():
    print("Employees")
    print("---------")
    print("")
    print("MENU")
    print("====")
    print("1 - View Employees & Departments")
    print("2 - View Salary Details")
    print("3 - View by Month of Birth")
    print("4 - Add New Employee")
    print("5 - View Departments managed by Employee")
    print("6 - Add Manager to Department")
    print("7 - View Departments")
    print("x - Exit application\n")

if __name__ == "__main__":
	main()


from __future__ import print_function
from os.path import join, dirname, abspath
import datetime, xlrd, numpy as np
from xlrd.sheet import ctype_text
import sqlite3

conn = sqlite3.connect('sample_db.sqlite3')
cur = conn.cursor()

xl_workbook = xlrd.open_workbook('2015 ROOMS SEGMENTATION.xlsx')  # location and name of the file
sheet_names = xl_workbook.sheet_names()
xl_sheet = xl_workbook.sheet_by_index(4)
row = xl_sheet.row(4)
ind_or_grp = xl_sheet.row(3)
for iog, cell_obj in enumerate(ind_or_grp):
    if cell_obj.value == 'GROUP':
        group_start = iog

ind_actual = np.zeros((13, 12),
                      dtype=[('subsegment', 'S40'), ('month', 'S40'), ('rns', float), ('arr', float), ('rev', float)])

ind_actual_last_year = np.zeros((13, 12),
                                dtype=[('subsegment', 'S40'), ('month', 'S40'), ('rns', float), ('arr', float),
                                       ('rev', float)])

month_list = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October',
              'November', 'December']
unneeded_columns = ['', 'Barter', 'GRAND TOTAL', 'TOTAL GROUP', 'TOTAL INDIVIDUAL', 'SEGMENT NAME',
                    'Qualified Discount', 'Long Staying']

year = xl_sheet.cell(1, 1).value.split()[1]
last_year = int(year) - 1

subsegment_column = 0
m = 0
actual_row = 7
actual_row_last_year = 9
for idx, cell_obj in enumerate(row):
    # only gets the subsegment
    subsegment = cell_obj.value  # use .value to get the value (duh)

    if subsegment not in unneeded_columns:  # get the Subsegment (rack, group, etc)
        mon = 5
        monx = 0
        for month in month_list:  # get the month (January, February etc)
            ind_actual[subsegment_column, m]['subsegment'] = subsegment
            ind_actual_last_year[subsegment_column, m]['subsegment'] = subsegment
            ind_actual[subsegment_column, m]['month'] = month
            ind_actual_last_year[subsegment_column, m]['month'] = month
            mon = mon + monx
            for x in range(0, 3):  # get the column headers (Room Nights Sold, Average Rm Rate(PHP), Revenue (PHP'000)
                ecolumn = idx + x
                if x == 0:
                    val = 'rns'
                elif x == 1:
                    val = 'arr'
                else:
                    val = 'rev'
                our = xl_sheet.cell(actual_row, ecolumn).value
                our2 = xl_sheet.cell(actual_row_last_year, ecolumn).value
                if isinstance(our, str):
                    our = 0.0
                if isinstance(our2, str):
                    our2 = 0.0
                ind_actual[subsegment_column, m][val] = our
                ind_actual_last_year[subsegment_column, m][val] = our2
            actual_row += 4
            actual_row_last_year += 4
            m += 1
        subsegment_column += 1
        actual_row = 7
        actual_row_last_year = 9
        m = 0

def getDate(month, year):
    thirty_ones = ["January", "March", "May", "July", "August", "October", "December"]
    # thirties = ["February","April","June","September","November"]
    monthMap = {"January": 1, "February": 2, "March": 3, "April": 4, "May": 5, "June": 6, "July": 7, "August": 8,
                "September": 9, "October": 10, "November": 11,
                "December": 12}
    if month in thirty_ones:
        day = 31
    else:
        day = 30
    month = monthMap.get(month)
    date = "%s-%s-%s" % (year, month, day)
    return date

for main in ind_actual:
    for sub in main:
        segment = sub[0].upper().decode('utf-8').strip()
        month = sub[1].decode('utf-8')
        rns = sub[2]
        arr = sub[3]
        rev = sub[4]
        date = getDate(month, year)
        try:
            seg_id_get_query = "select id from seg_list where name like  '%%%s' limit 1" % segment
            cur.execute(seg_id_get_query)
            seg_id = cur.fetchone()[0]
            insert_query = "insert into actual (date,rns,arr,rev,seg_id) values('%s',%s,%s,%s,%s)" % (
                date, rns, arr, rev, seg_id)
            cur.execute(insert_query)
            conn.commit()
        except(Exception):
            pass
conn.close()


import os
import sys
import argparse
import json
from collections import defaultdict
# pylint: disable=unused-import # tool cant handle urlib import
import urllib.request
from urllib.request import Request, urlopen

import copy
import time
from datetime import datetime
import subprocess
import schedule
from github import Github
from aiohttp.web_routedef import options

###########################################################################################
# Global needed configurations, see example setenv.sh file for settings these
############################################################################################

# Github related settings
TOKENFILE = ""  # "KEEP TOKENFILE PRIVATE, NOT TO BE STORED PUBLIC GIT, used to access Github repo for PR observations
TESTREPO = None  # Repo under PR observations
TESTPR = ""  # f"https://api.github.com/repos/{TESTREPO}/pulls"
ORGANIZATION = None  # required Github organization membership before building PR proceeds
# local file to store handled (built) PRs by their Github ID
BUILDPRSFILE = None
# local file to store builds done for open, allready built, but PR having changes being made
BUILDCHANGEDPRSFILE = None

# CLI command (Ghaf inhouse public tool) to manage Hydra operations
HYDRACTL = None
EXT_PORT = None  # Hybdra port dedicated for this POC build server
SERVER = None  # Hydra build server to be commanded
RUNDELAY = None  # minutes to wait before next execution of this script

__version__ = "0.73300"

# HYDRACTL_USERNAME and HYDRACTL_PASSWORD env variables (Hydra admin account). DO NOT STORE TO PUBLIC GIHUB
# Use for example local .sh file which exports them in shell when sourced

# Global commandline options

DRYRUNMODE = None
VERBOSEMODE = None
SERVICEMODE = None
CHERRYPICKEDPR = None

#########################################################################################################
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
def GetChangePRData(pr, counter, myChangedfile, changetime):
    """
    Check if detected change in PR is a new one or has been allready built
    In case a new one, update database file (also if new change for open PR is detected)
    """

    # myChangedfile = ""
    print("")
    pr = str(counter)
    myChangedfile.seek(0)  # we need to read from start
    print(
        f"----- Changed PR numbers and change times from the DB file:{BUILDCHANGEDPRSFILE} ----------")
    foundnewtime = 0
    PRCount = 0
    FilePRNumbers = []
    ContentOfFilePRNumber = []

    myChangedfile.seek(0)
    for one_line in myChangedfile:
        PRCount = PRCount+1

        if (VERBOSEMODE is not None):
            print(one_line)
        # get runtime copy for possible changes
        ContentOfFilePRNumber.append(one_line)
    PRCount = PRCount-1  # fictional first value
    if (VERBOSEMODE is not None):
        print(
            f"------- Existing PRs with done change builds:{PRCount} ----------------")

    myChangedfile.seek(0)
    for one_line in myChangedfile:
        values = one_line.strip().split(",")
        readPR = values[0]
        readChangetime = values[1:]
        if (readPR == pr):
            FilePRNumbers.append(pr)

            if (changetime in readChangetime):
                print("Changetime found from PR built changes list")
            else:
                print("Found new changed time for this PR")
                foundnewtime = 1

# pylint: disable=no-else-return # returning intentionally
    if (pr in FilePRNumbers and foundnewtime == 0):
        print("No new PRs nor existing PRs with new changes found. No actions needed")
        return "NO"
    elif (pr in FilePRNumbers and foundnewtime > 0):
        print("Existing PRs with new changes found")
        print("Going to update changed PRs file!")
        addline = pr+","+changetime

        for index, item in enumerate(ContentOfFilePRNumber):
            if item.startswith(pr):
                matching_index = index
                print(f"Internal matching_index:{matching_index}")
                break
        if (matching_index is not None):
            ContentOfFilePRNumber[matching_index] = addline
        else:
            print("Internal: Can't change PR change time list")
            return "ERROR"

        myChangedfile = open(BUILDCHANGEDPRSFILE, "w", encoding='utf-8')

        for line in ContentOfFilePRNumber:  # write whole "changed PRs build" file again.
            print(f"Writing line:{line}")
            niceline = line+"\n"

            if (DRYRUNMODE is not None):
                print("!!!Dryrun mode, not going to do actions!!!")
            else:
                myChangedfile.write(niceline)

            myChangedfile = open(BUILDCHANGEDPRSFILE, "a", encoding='utf-8')
        return "YES"

    elif (pr not in FilePRNumbers):
        print(
            f"New PR:{pr}with changes found. Cleaned changetime:{changetime}")
        print("Going to update changed PRs file!")
        addline = pr+","+changetime
        ContentOfFilePRNumber.append(addline)

        myChangedfile = open(BUILDCHANGEDPRSFILE, "w", encoding='utf-8')
        for line in ContentOfFilePRNumber:  # write whole "changed PRs build" file again.
            niceline = line+"\n"
            if (DRYRUNMODE is not None):
                print("!!!Dryrun mode, not going to do actions!!!")
            else:
                myChangedfile.write(niceline)

            myChangedfile = open(BUILDCHANGEDPRSFILE, "a", encoding='utf-8')
        return "YES"

    return "DONE"

#########################################################################################
def CheckChangedPR(pr, repo, counter):
    """
    Check if PR (still open) has been changed since creation time
    """
    print(f"Checking PR:{pr}")
    pr = repo.get_pull(counter)
    CREATED = pr.created_at
    commits = pr.get_commits()
    # Sort the commits by the commit timestamp in descending order
    sorted_commits = sorted(
        commits, key=lambda c: c.commit.committer.date, reverse=True)
    if (VERBOSEMODE is not None):
        print(f"Found commits:{sorted_commits}")
    # Get the timestamp of the most recent commit
    CHANGED = sorted_commits[0].commit.committer.date

    print(f"CREATED:{CREATED}")
    print(f"CHANGED:{CHANGED}")

    time_diff_mins = (CHANGED - CREATED).total_seconds() / 60
    print(f"Time difference in minutes:{time_diff_mins}")

# pylint: disable=no-else-return # returning intentionally
    if (time_diff_mins > 10):
        print("Possible change in open PR detected, may require rebuilding")

        return "YES", CHANGED
    else:
        print("No changes for open PR detected")
        return "NO", ""

#########################################################################################
# pylint: disable=too-many-arguments
def PRBuilding(data, ErroCounter, g, org, counter, myfile, tbd_list, timetoken):
    """
    If PR creator is in defined Github membership, initate Hydra build definition actions
    """

    # parse PR info (from Github JSON)
    SOURCE = "NONE"
    TARGET = "NONE"
    SOURCE_REPO = "NONE"
    ErroCounter = 0
    print("")
    try:
        print(f"==> SOURCE PR BRANCH:{data['head']['ref']}")
        SOURCE = data["head"]["ref"]
    except KeyError:
        print("ERROR: no head ref found")
        ErroCounter = ErroCounter+1

    try:
        print(f"==> TARGET BRANCH (like main/master):{data['base']['ref']}")
        TARGET = data["base"]["ref"]
        if (TARGET == "main"):
            print("==> OK target(main) repo")
        else:
            print("ERROR: source repo is not main")
            ErroCounter = ErroCounter+1
    except KeyError:
        print("no base ref found")
    try:
        print(f"==> SOURCE REPO:{data['head']['repo']['html_url']}")
        SOURCE_REPO = data["head"]["repo"]["html_url"]
    except KeyError:
        print("ERROR:no source repo info found")
        ErroCounter = ErroCounter+1

    USER = data["user"]["login"]
    user = g.get_user(USER)

    if (org.has_in_members(user)):
        print(
            f"---> The user '{USER}' is a member of the organization '{ORGANIZATION}'.")
        if (counter in tbd_list):
            print(f"------> Handling PR number:{counter}")
            if (ErroCounter == 0):

                PRActions(SOURCE, counter, TARGET, myfile,
                          USER, SOURCE_REPO, timetoken)

            else:
                print("Errors in PR data from Github, not doing build activities")
    else:
        print(
            f"The user: {USER} is not a member of the organization {ORGANIZATION}")
        print("No build activities done")
    print("--------------------------------------------------------------------------------------------------")

##########################################################################################
# pylint: disable=too-many-locals
def PRActions(SOURCE, PR, TARGET, myfile, USER, SOURCE_REPO, timetoken):
    """
    Construct Hydra build command from PullRequests data (Using Ghaf inhouse CLI command)
    Record handled PR info to local db file
    """

    OK_CMDEXE_COUNTER = 0
    print("")
    print(
        f"Construct Hydra(for project tiiuae/ghaf) build job set for branch:{SOURCE}")

    if (VERBOSEMODE is not None):
        print(f"--> Target main branch:{TARGET}")
        print(f"--> Source branch:{SOURCE}")
        print(f"--> Source repo:{SOURCE_REPO}")
        print(f"--> PR number:{PR}")
        print(f"--> HYDRACTL command location used:{HYDRACTL}")
        print(f"--> Hydra port:{EXT_PORT}")
        print(f"--> Hydra server:{SERVER}")
        print(f"--> User:{USER}")
        print(f"--> Timetoken:{timetoken}")
        print("")
    DESCRIPTION = "\"PR:"+str(PR)+" User:"+USER + \
        " Repo:"+SOURCE_REPO+" Branch:"+SOURCE+"\""  # kept format intentionally

    if (len(timetoken) == 0):
        PROJECT = f"{USER}X{SOURCE}"
    else:
        PROJECT = f"{USER}X{SOURCE}X{timetoken}"

    # two phased convertings got this item usage working
    PROJECT = PROJECT.encode('ascii', errors='ignore')
    # Then convert it from bytes back to a string using:
    PROJECT = PROJECT.decode()

    FLAKE = "git+"+SOURCE_REPO+"/?ref="+SOURCE  # kept format intentionally
    if (len(timetoken) == 0):
        JOBSET = f"{SOURCE}X{PR}"
    else:
        JOBSET = f"{SOURCE}X{PR}X{timetoken}"

    if (VERBOSEMODE is not None):
        print(f"--> Hydra PROJECT:{PROJECT}")
        print(f"--> Hydra DESCRIPTION:{DESCRIPTION}")
        print(f"--> Hydra FLAKE:{FLAKE}")
        print(f"--> Hydra JOBSET:{JOBSET}")
    APCOMMAND = f"python3 {HYDRACTL} {SERVER}  AP --project {PROJECT} --display {DESCRIPTION}"
    AJCOMMAND = f"python3 {HYDRACTL} {SERVER} AJ --description {DESCRIPTION} --check 300 --type flake --flake {FLAKE} -s enabled --jobset {JOBSET} --project {PROJECT}"
    if (VERBOSEMODE is not None):
        print("")
        print(f"Created Hydra CLI APCOMMAND:{APCOMMAND}")
        print("")
        print(f"Created Hydra CLI AJCOMMAND:{AJCOMMAND}")
    DONE = PR  # write PR number to db file if both CMD exections are ok
    DONE = str(DONE)+"\r\n"  # intentionally kept format

    # NOTE: As executing commands from Python file failed (Hydra jobset creation) and using same commands were ok from shell
    # saving commands to file and executing the content from the read file
    # this is temp solution for this POC only

    cmdfile1 = open("cmdfile1", "w", encoding='utf-8')
    # cmdfile1.seek(0) # only if mode a used
    cmdfile2 = open("cmdfile2", "w", encoding='utf-8')
    # cmdfile2.seek(0)
    FIRSTLINE = "#!/bin/bash"+"\r\n"  # intentionally kept format

    cmdfile1.write(FIRSTLINE)
    cmdfile1.write(APCOMMAND)

    cmdfile2.write(FIRSTLINE)
    cmdfile2.write(AJCOMMAND)

    cmdfile1.close()
    cmdfile2.close()

    cmd1 = open("cmdfile1", "r", encoding='utf-8')
    APline1 = cmd1.read()
    rc, _out, err = ExeCMD(APline1)

    if (rc != 0):
        print(f"Command execution error:{rc}")
        print(f"Error message:{err}")
    else:
        print("OK command execution")
        OK_CMDEXE_COUNTER += 1

    time.sleep(2)
    cmd2 = open("cmdfile2", "r", encoding='utf-8')
    AJline2 = cmd2.read()
    rc, _out, err = ExeCMD(AJline2)
    if (rc != 0):
        print(f"Command execution error:{rc}")
        print(f"Error message:{err}")
    else:
        print("OK command execution")
        OK_CMDEXE_COUNTER += 1

    if (OK_CMDEXE_COUNTER == 2):
        if (len(timetoken) == 0):
            print(
                f"2 correct CMD executions,NEW build, going to record PR:{PR} as done deed")
            if (DRYRUNMODE is not None):
                print("!!!Dryrun mode, not going to do actions!!!")
            else:
                myfile.write(DONE)
        else:
            print(
                f"2 correct CMD executions, CHANGED PR build, going to record PR:{PR} with CHANGE time:{timetoken} to own db file")
    else:
        print("-------------------------------------------------------------------------------")
        print(
            f"ERROR ===> CMD executions errors found, NOT marking PR:{PR} as done")
        print("-------------------------------------------------------------------------------")

    # clean temp commandfiles
    os.remove("./cmdfile1")
    os.remove("./cmdfile2")
    time.sleep(2)

    print("*******************************************************************************************************")

##############################################################
def ExeCMD(commandLine):
    """
    Execute given command string and return system feedback
    """

    print("------------------------------------------------------------------------------------------")
    print(f"Executing command:{commandLine}")
    print("------------------------------------------------------------------------------------------")

# pylint: disable=no-else-return # returning intentionally
    if (DRYRUNMODE is not None):
        print("!!! Dryrun mode, not doing actions !!!")
        return (42, "dryrun mode", "dryrun mode")

    else:
        sp = subprocess.Popen(commandLine,
                              shell=True,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE)
        rc = sp.wait()
        out, err = sp.communicate()

        return rc, out, err

##########################################################################################
def Finder():
    """
    Check if any new (not built) pull requests exists in defined repo (fro main branch
    """

    print("------------------------------------------------------------------------------------------")

    #########################################################################
    # Authorization to Github
    # Tokenfile used to provide token for observed Github Repo access
    # DO NOT STORE TOKENFILE TO GITHUB, KEEP IT PRIVATE
    #

    file_exists = os.path.exists(TOKENFILE)
    if (file_exists):
        file = open(TOKENFILE, "r", encoding='utf-8')
        config_array = file.read().split()
        githubtoken = str(config_array[0])
    else:
        print(f"No Github tokenfile found, check:{TOKENFILE}")
        githubtoken = None

    g = Github(githubtoken)
    repo = g.get_repo(TESTREPO)
    org = g.get_organization(ORGANIZATION)
    pulls = repo.get_pulls(state='open', sort='created', base='main')

    ##########################################################################
    # Get all open PRs into a dictionary for data processing
    i = 0
    open_prs = defaultdict(lambda: "OFF")
    print(f"------ All open PullRequest for repo: {TESTREPO} -------")
    for pull in pulls:
        print(pull)
        open_prs.update({pull.number: "ON"})
        i = i+1

    if (i == 0):
        print("No open PRs found, exiting")
        sys.exit(4)
    ####################################################################################
    # Read allready built PRs from disk, initialize possible empty bookkeeping files
    #
    done_prs = defaultdict(lambda: "NONE")
    myfile = open(BUILDPRSFILE, "a+", encoding='utf-8')
    myChangedfile = open(BUILDCHANGEDPRSFILE, "a+", encoding='utf-8')

    # add fictional done PR number to an empty file in order to keep logic running
    if (os.stat(BUILDPRSFILE).st_size == 0):
        print("No done PRs found, adding fictional PR number to an empty file")
        FICTIONALPR = 123456789
        FICTIONALPR = str(FICTIONALPR)+"\r\n"
        myfile.seek(0)
        myfile.write(FICTIONALPR)

    # add fictional changed PR number (and timetoken) to an empty file in order to keep logic running
    if (os.stat(BUILDCHANGEDPRSFILE).st_size == 0):
        print("No done PRs file found, adding fictional PR number and change time to an empty file")
        FICTIONALPR = 123456789
        FICTIONALCHANGETIME = "2020-02-02-23-00-00"
        # kept format intentionally
        FICTIONALCHANGEPR = str(FICTIONALPR)+","+FICTIONALCHANGETIME+"\r\n"
        myChangedfile.seek(0)
        myChangedfile.write(FICTIONALCHANGEPR)
    ##########################################################################################################

    # a+ adds fd to end of file, for appends, we need to read from start
    myfile.seek(0)
    print(f"------Built PR numbers from the DB file: {BUILDPRSFILE}------")
    for one_pr_number in myfile:
        if (VERBOSEMODE is not None):
            print(str(one_pr_number), end='')
        done_prs[int(one_pr_number)] = "DONE"

    print("------ Checking which open PRs are new and require building ------")
    # shallow copies are iterared over as defaultdict changes dict size when default value must be returned
    copy_done_prs = copy.copy(done_prs)
    copy_open_prs = copy.copy(open_prs)
    tbd_list = []
    counter = 1
    ###############################################################################
    # Process all repo's open pull requests
    processed_pr = []
    ErroCounter = 0
    for newPr in copy_open_prs:

        url = str(TESTPR)+"/"+str(newPr)  # kept format intentionally
        with urlopen(url) as response:
            body = response.read()
        data = json.loads(body)
        print("###########################################################################################################################")
        print("==> PR:"+url)

        if (CHERRYPICKEDPR is not None):
            if (CHERRYPICKEDPR == newPr):
                print(f"===> To be cherry picked PR found:{newPr}")
            else:
                print(
                    f"Skipping this one, trying to cherry pick PR:{CHERRYPICKEDPR}")
                continue

        print(f"==> CHECKING ---> SOURCE PR BRANCH: {data['head']['ref']}")

        counter = newPr  # not anymore running counter, but the actual open PR numbers
        for _doneline in copy_done_prs:

            # check duplicate PRs in db file
            # pylint: disable=no-else-break # breaking intentionally
            if (counter in processed_pr):
                print(f"Processed PRs:{processed_pr}")
                print(
                    f"---> Duplicate PR found in db file: {counter} skipping to next one!")
                print(
                    "--------------------------------------------------------------------------------------------------")
                break

            elif (open_prs[counter] == "ON" and done_prs[counter] == "DONE"):
                print(f"==> OLD PR in PR list, has been built:{counter}")

                print("Checking if this (still open) PR has been changed")
                pr = repo.get_pull(counter)
                answer, changetime = CheckChangedPR(pr, repo, counter)
                if (answer == "YES"):
                    print(f"PR changed, timetoken in use:{changetime}")

                    processed_pr.append(counter)
                    if (counter not in tbd_list):
                        tbd_list.append(counter)
                    else:
                        print("not adding")

                    # hydra doesnt like spaces in arguments, nor :
                    changeTimeCleaned = str(changetime).replace(" ", "-")
                    changeTimeCleaned = changeTimeCleaned.replace(":", "-")
                    if (VERBOSEMODE is not None):
                        print("Cleaned (book keeping) timetoken:" +
                              changeTimeCleaned)

                    answer = GetChangePRData(
                        pr, counter, myChangedfile, changeTimeCleaned)
                    if (answer == "YES"):
                        # SOURCE=data["head"]["ref"]
                        print(
                            f"==> CHECKING ---> SOURCE PR BRANCH:{data['head']['ref']}")

                        PRBuilding(data, ErroCounter, g, org, counter,
                                   myfile, tbd_list, changeTimeCleaned)
                    else:
                        print("")
                    break

                else:
                    print("No PR changes, no actions needed")
                    processed_pr.append(counter)
                    break
                print(
                    "--------------------------------------------------------------------------------------------------")
            elif (open_prs[counter] == "ON" and done_prs[counter] == "NONE"):
                print(f"==> NEW PR, going to build this:{counter}")
                processed_pr.append(counter)
                if (counter not in tbd_list):
                    tbd_list.append(counter)
                else:
                    print("not adding")

                timetoken = ""
                PRBuilding(data, ErroCounter, g, org, counter,
                           myfile, tbd_list, timetoken)
                break

            elif (open_prs[counter] == "OFF" and done_prs[counter] == "DONE"):
                print(f"==> OLD done PR:{counter}")
                processed_pr.append(counter)
                break

###################################################################################################################################
def GetEnv(variable):
    """
    Check that needed env variables are defined
    """

    definedVariable = os.getenv(variable)
    if (definedVariable is None and DRYRUNMODE is None):
        print(f"No {variable} env variable defined", file=sys.stderr)
        print("Check example setenv.sh file....Exiting now!!")
        sys.exit(3)
    else:

        definedVariable = str(definedVariable)
        if (VERBOSEMODE is not None):
            print(f"Set env variable:{variable} as {definedVariable}")
        else:
            definedVariable = str(definedVariable)
        return definedVariable

###################################################################################################################################
# pylint: disable=unused-argument
def main(argv):
    """ Main. Check parameters
    """

# pylint: disable=global-statement # used intentionally
    global CHERRYPICKEDPR
    global VERBOSEMODE
    global DRYRUNMODE
    global SERVICEMODE
    global TOKENFILE, TESTREPO, ORGANIZATION, BUILDPRSFILE, HYDRACTL, BUILDCHANGEDPRSFILE, EXT_PORT, SERVER, RUNDELAY, TESTPR

    current_time = datetime.now()
    formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S")
    print("----------------------------------------------------------------------------------------")
    print(f"---------- Execution started:{formatted_time}---------------")
    print("----------------------------------------------------------------------------------------")

    parser = argparse.ArgumentParser(
        description="Github PullRequest Hydra builder activator",

        epilog='''
    "Needs some env variables, see code for comments and setenv.sh as an example setter"
    ''')

    parser.add_argument('-v', help='Check Github open PRs and activate Ghaf Hydra build',
                        action='version', version=f"Version:{__version__}  ")
    parser.add_argument("-d", help='Dry run mode', metavar="dry")
    parser.add_argument('-t', help='Verbose, talking, mode', metavar="verbose")
    parser.add_argument('-s', help='Service mode, runtime delays in secs',
                        metavar="service", type=int, nargs=1)
    parser.add_argument('-p', help='Cherry pick PR number, ignore others',
                        metavar="cherrypick", type=int, nargs=1)

    args = parser.parse_args()

    VERBOSEMODE = args.t or None
    DRYRUNMODE = args.d or None
    SERVICEMODE = args.s or None

    if (args.s):
        SERVICEMODE = args.s[0]
    if (args.p):
        CHERRYPICKEDPR = args.p[0]

    if (VERBOSEMODE is not None):
        print("Verbose mode selected")
    if (DRYRUNMODE is not None):
        print("!!! Dryrun mode selected, no actions taken !!!")
    if (SERVICEMODE):
        print(f"Service mode selected with runtime delay:{SERVICEMODE}")
    if (CHERRYPICKEDPR is not None):
        print(f"Checking only PR:{CHERRYPICKEDPR} skpping other PRs")

    # pylint: disable=unused-variable
    username = GetEnv("HYDRACTL_USERNAME")  # env check only
    # pylint: disable=unused-variable
    passu = GetEnv("HYDRACTL_PASSWORD")  # env check only
    TOKENFILE = GetEnv("TOKENFILE")
    print(f"GOT Tokenfile file:{TOKENFILE}")
    TESTREPO = GetEnv("TESTREPO")
    ORGANIZATION = str(GetEnv("ORGANIZATION"))
    BUILDPRSFILE = GetEnv("BUILDPRSFILE")
    BUILDCHANGEDPRSFILE = GetEnv("BUILDCHANGEDPRSFILE")
    HYDRACTL = GetEnv("HYDRACTL")
    EXT_PORT = GetEnv("EXT_PORT")
    SERVER = f"http://localhost:{EXT_PORT}"
    RUNDELAY = GetEnv("RUNDELAY")
    TESTPR = f"https://api.github.com/repos/{TESTREPO}/pulls"

    print("")
    print(f"TESTPR:{TESTPR}")
    print(f"Tokenfile file:{TOKENFILE}")
    print(f"Build PRs db file:{BUILDPRSFILE}")
    print(f"Build and rebuild PRs due changes db file: {BUILDCHANGEDPRSFILE}")
    print(f"Observing Github repo (main) PRs: {TESTREPO}")
    print(
        f"Organization participation required for PR building:{ORGANIZATION}")
    print(f"Using Hydra build server:{SERVER}")
    print(f"Assuming Hydra CLI command to be:{HYDRACTL}")
    print("Only PRs to main branch are processed")
    print("")

    if (SERVICEMODE):
        print("Service mode starting!")
        schedule.every(RUNDELAY).minutes.do(Finder)
        while True:
            schedule.run_pending()
            time.sleep(SERVICEMODE)
    else:
        print("Running command just once")
        Finder()

#######################################################
# Entry point
if __name__ == "__main__":
    main(sys.argv[1:])