""" Implementation of the CellOT structure.
"""

from pathlib import Path
import torch
from collections import namedtuple
from cellot.networks.icnns import ICNN
from absl import flags
from typing import Optional

FLAGS = flags.FLAGS

# FGPair is a subclass of tuple whose first element is named f
# and second element g.
FGPair = namedtuple("FGPair", "f g")

class CellOT():
    """ Implementation of the CellOT structure.

    Attributes:
        f(ICNN): one of the dual networks.
        g(ICNN): one of the dual networks.
        opts(FGPair): optimizers for f and g.
    """

    def __init__(self,config,restore=None,**kwargs) -> None:
        self.build_networks(config,**kwargs)
        self.build_opts(config)
        self.restore(restore)

    def build_networks(self,config, **kwargs):
        """ Builds dual potential ICNNs with specified configuration."""
        # Function to extract the information for initialization
        def unpack_kernel_init_fxn(name="uniform", **kwargs):
            if name == "normal":
            # Uses normal distribution to initialize.
                def init(*args):
                    return torch.nn.init.normal_(*args, **kwargs)
            elif name == "uniform":
            # Uses uniform distribution to initialize.
                def init(*args):
                    return torch.nn.init.uniform_(*args, **kwargs)
            else:
                raise ValueError
            return init

        # Uses default configuration for hidden layers if not passed by kwargs.
        kwargs.setdefault("hidden_units", [64] * 4)
        # Uploads model information to kwargs.
        # e.g. parameters specific to g are stored in config.model.g.
        kwargs.update(dict(config.get("model", {})))

        # Removes extra information.
        kwargs.pop("name")
        if "latent_dim" in kwargs:
            kwargs.pop("latent_dim")
        # Extracts model information respectively.
        fupd = kwargs.pop("f", {})
        gupd = kwargs.pop("g", {})

        # Parameters for ICNN f.
        fkwargs = kwargs.copy()
        fkwargs.update(fupd)
        fkwargs["kernel_init_fxn"] = unpack_kernel_init_fxn(
            **fkwargs.pop("kernel_init_fxn")
        )

        # Parameters for ICNN g.
        gkwargs = kwargs.copy()
        gkwargs.update(gupd)
        gkwargs["kernel_init_fxn"] = unpack_kernel_init_fxn(
            **gkwargs.pop("kernel_init_fxn")
        )

        self.f = ICNN(**fkwargs)
        self.g = ICNN(**gkwargs)

        # Prints model parameters if required.
        if "verbose" in FLAGS and FLAGS.verbose:
            print(self.g)
            print(kwargs)

        return

    def build_opts(self,config):
        """ Builds optimizers for CELLOT networks."""
        # Loads optimizer's configuration.
        kwargs = dict(config.get("optim", {}))
        assert kwargs.pop("optimizer", "Adam") == "Adam"
        # Model configuration for f and g, empty if not existed in kwargs.
        fupd = kwargs.pop("f", {})
        gupd = kwargs.pop("g", {})
        # Configuration for f.
        fkwargs = kwargs.copy()
        fkwargs.update(fupd)
        fkwargs["betas"] = (fkwargs.pop("beta1", 0.9), fkwargs.pop("beta2", 0.999))
        # Configuration for g.
        gkwargs = kwargs.copy()
        gkwargs.update(gupd)
        gkwargs["betas"] = (gkwargs.pop("beta1", 0.9), gkwargs.pop("beta2", 0.999))
        # Loads optimizers for f and g.
        self.opts = FGPair(
            f=torch.optim.Adam(self.f.parameters(), **fkwargs),
            g=torch.optim.Adam(self.g.parameters(), **gkwargs),
        )
        return

    def restore(self,restore=None):
        """ Restores a previous state. """
        if restore is not None and Path(restore).exists():
            ckpt = torch.load(restore)
            self.f.load_state_dict(ckpt["f_state"])
            self.opts.f.load_state_dict(ckpt["opt_f_state"])

            self.g.load_state_dict(ckpt["g_state"])
            self.opts.g.load_state_dict(ckpt["opt_g_state"])
        return

    def compute_loss_f(
        self,
        source:torch.tensor,
        target:torch.tensor,
        transport:Optional[torch.tensor]=None
        ):
        """ Computes the loss of ICNN f given input features and predicted features."""
        # If no prediction is passed, uses g to predict.
        if transport is None:
            transport = self.g.transport(source)
        # Computes the loss, implemented from part of equation (9) in the CELLOT paper.
        return -self.f(transport) + self.f(target)

    def compute_loss_g(
        self,
        source:torch.tensor,
        transport:Optional[torch.tensor]=None
        ):
        """ Computes the loss of ICNN g given input features and predicted features."""
        # If no prediction is passed, uses g to predict.
        if transport is None:
            transport = self.g.transport(source)
        # Computes the loss, implemented from part of equation (9) in the CellOT paper.
        return self.f(transport) - torch.multiply(source, transport).sum(-1, keepdim=True)

    def compute_g_constraint(self,form=None,beta=0):
        """ Computes the additional penalty for negative weights of g.

        Args:
            form: indicates which type of measure is performed for negative weights. Default is None.
            beta: penalty strength. Default is 0.
        """
        # No penalty is computed
        if form is None or form == "None":
            return 0
        # If clamping is performed, no penalty is computed.
        if form == "clamp":
            self.g.clamp_w()
            return 0
        # Penalizes negative weights.
        elif form == "fnorm":
            if beta == 0:
                return 0
            return beta * sum(map(lambda w: w.weight.norm(p="fro"), self.g.W))
        raise ValueError

    def compute_w2_distance(self,source,target,transport=None):
        """ Computes the 2-norm Wasserstein distance between predictions and targets."""
        # If no prediction is passed, uses g to predict.
        if transport is None:
            transport = self.g.transport(source).squeeze()
        with torch.no_grad():
            Cpq = (source * source).sum(1, keepdim=True) + (target * target).sum(1, keepdim=True)
            Cpq = 0.5 * Cpq

            cost = (
                self.f(transport)
                - torch.multiply(source, transport).sum(-1, keepdim=True)
                - self.f(target)
                + Cpq
            )
            cost = cost.mean()
        return cost

    def predict(self,inputs:torch.tensor):
        """Predicts the cell features after perturbation with CellOT."""
        # Switches to evaluation mode.
        self.g.eval()
        outputs = self.g.transport(inputs.requires_grad_(True))
        return outputs

def numerical_gradient(param:float, fxn, *args, eps=1e-4):
    """ Numerically computes the gradient of a function at some point.

    Args:
        param: float, the parameter to define the input position of the function.
        fxn: the function of which the gradient is computed on.
        *args: the arguments of the function.
        eps: the small value perturbed on the param. Default is 1e-4.

    Returns:
        The gradient of fxn at *args.
    """

    # Computes the function value at param+eps.
    with torch.no_grad():
        param += eps
    plus = float(fxn(*args))

    # Computes the function value at param-eps.
    with torch.no_grad():
        param -= 2 * eps
    minus = float(fxn(*args))

    # Resets param to its original value.
    with torch.no_grad():
        param += eps

    # Computes and returns the gradient.
    return (plus - minus) / (2 * eps)

def load_cellot_model():
    """ Created for other modules' dependencies. """

def compute_loss_f():
    """ Created for other modules' dependencies. """

def compute_loss_g():
    """ Created for other modules' dependencies. """


import pandas as pd
import Levenshtein
import openai
import json

class RegionCreator:
    """
    Assign an economic region for a country in a Dataset

    use RegionCreator(df, country_column).region()
    """
    def __init__(self, df, country_column, region_col_name='Region'):
        self.df=df
        self.country_column = country_column
        self.region_col_name='region_col_name'

        self.regions = pd.read_csv("https://docs.google.com/spreadsheets/d/e/2PACX-1vRmy7Nl3uNIrffyP0MZ2Myzet0x8xrHgPhWWPlD-Bk2K01dAu1Ua1iaGWpbsN6qrA/pub?output=csv")
        self.regions_ = ['EMEA', 'LATAM', 'North America', 'APAC', "CHINA"]

    def region(self):
        self.df[self.country_column] = self.df[self.country_column].str.strip()
        for r in self.regions_:
            self.df.loc[self.df[self.country_column].isin(self.regions[self.regions.Region == r].Country), self.region_col_name] = r
            self.df.loc[self.df[self.country_column].isin(self.regions[self.regions.Region == r].Code), self.region_col_name] = r
        return self.df

class FuzzyComparator:
    def __init__(self, df1, df2, col1, col2, score=0.8,tracking=False):
        """
        Class constructor that initializes the FuzzyComparator object with two dataframes,
        the columns to compare, and a matching score.

        Parameters:
        df1 (pandas DataFrame): first dataframe to compare
        df2 (pandas DataFrame): second dataframe to compare
        col1 (str): column name in first dataframe to compare
        col2 (str): column name in second dataframe to compare
        score (float): matching score threshold (default=0.8)
        tracking (bool): set to True to print progress during comparison (default=False)
        """
        self.df1 = df1
        self.df2 = df2
        self.col1 = col1
        self.col2 = col2
        self.score = score
        self.tracking = tracking
    def compare(self):
        """
        Method that compares the two dataframes based on the selected columns using fuzzy matching.

        Returns:
        pandas DataFrame: dataframe with the matched company names and their scores
        """
        n = 0
        matches = []
        for index1, row1 in self.df1.iterrows():
            for index2, row2 in self.df2.iterrows():
                company1 = str(row1[self.col1])
                company2 = str(row2[self.col2])
                dist = Levenshtein.distance(company1.lower(), company2.lower())
                match_score = 1 - (dist / max(len(company1), len(company2)))
                n+=1
                if match_score >= self.score:
                    matches.append({'company1': company1, 'company2': company2, 'match_score': match_score})
                if self.tracking:
                    print(f'{n / (len(self.df1) * len(self.df2)) * 100:.2f}%')
        result_df = pd.DataFrame(matches)
        return result_df

class GPT:
    def __init__(self, promt):
        """
        Initializes a new instance of the GPT class with a given prompt.

        Parameters:
        prompt (str): the text prompt to send to the OpenAI API
        """
        self.promt = promt

    def run(self):
    # Set up your OpenAI API credentials
        openai.api_key = "Your API Key"

        # Define the prompt to send to the API
        prompt = self.promt

        # Define the parameters for the API request
        params = {
            "model": "text-davinci-002",
            "prompt": prompt,
            "temperature": 0.5,
            "max_tokens": 1024,
            "n": 1,
            "stop": ""
        }

        # Send the request to the OpenAI API
        response = openai.Completion.create(**params)
        # Parse the response to extract the generated text
        generated_text = response.choices[0].text.strip()
        return generated_text


# -*- coding: utf-8 -*-
"""
Created on Tue Mar  5 09:26:33 2019

@author: Tristan O'Hanlon
"""
import matplotlib.pyplot as plt
import math
import h5py
import os
os.chdir('E:/University/University/MSc/Models/climate-analysis/CCCM/reduced_datasets')
f = h5py.File('2010_CCCM_global_latitude.h5', 'r')

tcc = f['Cloud Fraction'][:]
tclw = f['Liquid Water Content'][:] # in g/m^3
tciw = f['Ice Water Content'][:] # in g/m^3

# Plot cloud fraction, liquid and ice water content vs latitude.
plt.figure()
fig, ax = plt.subplots()

#ax2 = ax1.twinx()
ax.plot(cccm_tclw_lat[:,0], cccm_tclw_lat[:,1], '-r', label='Liquid Water Content')
ax.plot(cccm_tciw_lat[:,0], cccm_tciw_lat[:,1], '--r', label='Ice Water Content')
ax.plot(cccm_tclw_av_lat[:,0], cccm_tclw_av_lat[:,1], '-b', label='Mean Liquid Water Content')
ax.plot(cccm_tciw_av_lat[:,0], cccm_tciw_av_lat[:,1], '--b', label='Mean Ice Water Content')

ax.legend(loc='lower center', bbox_to_anchor=(0.5, -0.3),
          ncol=4, fancybox=True, shadow=True);
#ax2.legend(loc='lower center', bbox_to_anchor=(0.5, -0.4),
#          ncol=4, fancybox=True, shadow=True);
ax.set_ylabel('Liquid and Ice Water Content', color='r')
ax.set_xlabel('Latitude')
#ax2.set_ylabel('Liquid and Ice Water Content ($gm^{-3}$)', color='r')

plt.title('Global Cloud Cover and Phase Content vs Latitude - CCCM 2010')

plt.grid(True)
plt.show()


from django.conf.urls import url
from api.views import course
from api.views import degreecourse
from api.views import shoppingcar

urlpatterns = [
    url(r'shoppingcar/$',
        shoppingcar.ShoppingCarView.as_view({'post': 'create', 'get': 'list', 'delete': 'destroy', 'put': 'update'})),
    # url(r'shoppingcar/$', shoppingcar.CoursesCarView.as_view()),
    url(r'courses/$', course.CoursesView.as_view()),
    url(r'courses/(?P<pk>\d+)/$', course.CourseDetailView.as_view()),
    url(r'courses/thematic/$', course.CourseThematicView.as_view()),
    url(r'courses/module/(?P<pk>\d+)/$', course.CourseModuleView.as_view()),
    url(r'courses/faq/(?P<pk>\d+)/$', course.CourseFAQView.as_view()),
    url(r'courses/outline/(?P<pk>\d+)/$', course.CourseOutlineView.as_view()),
    url(r'courses/chapter/(?P<pk>\d+)/$', course.CourseChapterView.as_view()),  # 获取数据失败
    # url(r'degreecourse/(?P<pk>\d+)/$', degreecourse.DegreeCourseTeacherView()),
    url(r'degreecourse/$', degreecourse.DegreeCourseView.as_view()),
    url(r'degreecourse/teachers/$', degreecourse.DegreeCourseTeacherView.as_view()),
    url(r'degreecourse/scholarship/$', degreecourse.DegreeCourseScholarshipView.as_view()),
]


from ugradio import pico as ugp
from numpy import savez, median, diff
from numpy.fft import fftfreq, fftshift, fft
import argparse
import matplotlib.pyplot as plt
parser =argparse.ArgumentParser(description='')
parser.add_argument("--file_name", required=True, help='File name to save')
parser.add_argument("--path", default='/home/maxlee1993/Documents/', help='Path to file')
parser.add_argument("--volt_range", default='1V', help='Voltrange for Pico')
parser.add_argument("--divisor", default=1, help='divisor for sample freq')
parser.add_argument("--nblocks", default=1, help='blocks to take')
parser.add_argument("--dual_mode", default=False, help='take both channels')

args = parser.parse_args()

filepath = args.path+args.file_name
wf = ugp.capture_data(divisor=int(args.divisor), volt_range=args.volt_range, nblocks=int(args.nblocks), dual_mode=args.dual_mode)
savez(filepath, wf)

times = np.linspace(0, len(wf)/((62.5/args.divisor)*1e6), len(wf))
freqs = fftfreq(len(times), median(diff(times)))

power = abs(fft(wf))**2

plt.figure()
plt.plot(shift(freqs/1e6), shift(power))
plt.show()


from threading import Thread
import time
from threading import Thread
import time
import sys
import json
import socket
import select
import math
from xmlrpc.client import MAXINT
import pygame
import contextlib
with contextlib.redirect_stdout(None):
    import pygame
import shared

pygame.font.init()
NAME_FONT = pygame.font.SysFont("consolas", 20)
TIME_FONT = pygame.font.SysFont("consolas", 30)
SCORE_FONT = pygame.font.SysFont("consolas", 26)

pygame.init()

serverIP = None
playerid = None

lastServerTimestamp = 0
gametime = 0
players = {

    # playerid:{
    #   X
    #   Y
    #   score
    #   color
    #   name
    # }
}

foods = [
    #  {
    #   x:
    #   y:
    #   color:
    #   size:
    # }
]

myName = "anonim"

if len(sys.argv) > 1:
    myName = sys.argv[1]
else:
    myName = input("Write your name? ")

def disco():
    global players, serverIP

    while True:
        if serverIP != None:
            break
        DISCOVER_MESSAGE_BYTES = json.dumps(
            {"type": shared.messageTypes.DISCOVER, "game": "agario-clone", "name": myName})
        DISCOVER_MESSAGE_BYTES = str.encode(DISCOVER_MESSAGE_BYTES)

        with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
            sock.bind(("", 0))
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
            for i in range(10):
                sock.sendto(DISCOVER_MESSAGE_BYTES,
                            ('<broadcast>', shared.DISCO_PORT))
        time.sleep(5)

def messagegetterTCP():
    #

    global players, foods, serverIP, playerid
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind(("", shared.DISCO_PORT))
        s.listen()
        while True:
            # Start listenning
            conn, addr = s.accept()
            sender = addr[0]  # ip
            with conn:
                output = conn.recv(shared.RECVSIZE)
                output = output.decode('utf-8')
                print(output, sender)
                # Parse the message
                message = json.loads(output)
                mes_type = message["type"]
                # Received message is type of "Discover Response"
                if mes_type == shared.messageTypes.DISCOVER_RESPONSE:
                    # if message["IP"] == myIP:
                    #     continue
                    # print(message)
                    serverIP = sender
                    playerid = message["playerid"]

def messagegetterUDP():

    global players, foods, playerid, lastServerTimestamp, gametime

    # listener, get messages and print, also DISCOVER_RESPONSE
    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        s.bind(("", shared.MOVE_PORT))
        s.setblocking(0)
        while True:
            result = select.select([s], [], [])
            msg, address = result[0][0].recvfrom(shared.RECVSIZE)
            # print("addd", address[0])
            sender = address[0]
            # print("sender: ", sender)
            # print(msg)
            message = json.loads(msg.decode('utf-8'))
            mesType = message["type"]
            if mesType == shared.messageTypes.CURRENT_STATE:

                if lastServerTimestamp > message["timestamp"]:
                    continue

                localMe = players.get(
                    playerid)
                players = message["players"]
                if localMe != None and message["players"][playerid].get("number_of_deaths") == localMe.get("number_of_deaths"):
                    if ((message["players"][playerid].get("score")) >= (localMe.get("score"))):
                        players[playerid]["X"] = localMe["X"]
                        players[playerid]["Y"] = localMe["Y"]

                foods = message["foods"]

                lastServerTimestamp = message["timestamp"]
                gametime = message["gametime"]

def redraw_window():
    """
    draws new frame
    """
    WIN.fill((255, 255, 255))  # clear previous frame

    # draw all the foods
    for food in foods:
        print(food)
        pygame.draw.circle(WIN, food["color"],
                           (food["X"], food["Y"]), food["size"])

    # draw each player in the list
    for player in sorted(players, key=lambda x: players[x]["score"]):
        p = players[player]
        pygame.draw.circle(WIN, p["color"], (p["X"], p["Y"]),
                           math.sqrt(p["score"]))
        # render and draw name for each player
        text = NAME_FONT.render(p["name"], 1, (0, 0, 0))
        WIN.blit(text, (p["X"] - text.get_width() /
                 2, p["Y"] - text.get_height()/2))

    # draw scoreboard
    sort_players = list(
        reversed(sorted(players, key=lambda x: players[x]["score"])))
    title = TIME_FONT.render("Scoreboard", 1, (0, 0, 0))
    start_y = 25
    x = shared.MAP_WIDTH - title.get_width() - 10
    WIN.blit(title, (x, 5))

    ran = min(len(players), 3)
    for count, i in enumerate(sort_players[:ran]):
        text = SCORE_FONT.render(
            str(count+1) + ". " + str(players[i]["name"]) + " "+str(players[i]["score"]-shared.INITIAL_SCORE), 1, (0, 0, 0))
        WIN.blit(text, (x, start_y + count * 20))

    # draw gametime
    text = TIME_FONT.render("Time: "+str(gametime//(10**6)), 1, (0, 0, 0))
    WIN.blit(text, (10, 10))

    # draw score
    text = TIME_FONT.render(
        "Score: " + str(round(players[playerid]["score"]-shared.INITIAL_SCORE)), 1, (0, 0, 0))
    WIN.blit(text, (10, 15 + text.get_height()))

def getSizeFromScore(score):
    return math.sqrt(score)

def main():
    global players, foods, serverIP, playerid
    while serverIP == None:
        time.sleep(0.01)
        continue
    clock = pygame.time.Clock()
    run = True

    while run:
        clock.tick(60)  # 60 fps max
        player = players.get(playerid)

        if not player:
            redraw_window()
            pygame.display.update()
            continue

        vel = 6 - round(math.sqrt(player["score"])/10)
        if vel <= 1:
            vel = 1

        # print(foods, players)
        # get key presses
        keys = pygame.key.get_pressed()

        # movement based on key presses
        if keys[pygame.K_LEFT] or keys[pygame.K_a]:
            player["X"] = max(player["X"] - vel, 0 +
                              getSizeFromScore(player["score"]))

        if keys[pygame.K_RIGHT] or keys[pygame.K_d]:
            player["X"] = min(player["X"] + vel, shared.MAP_WIDTH -
                              getSizeFromScore(player["score"]))

        if keys[pygame.K_UP] or keys[pygame.K_w]:
            player["Y"] = max(player["Y"] - vel, 0 +
                              getSizeFromScore(player["score"]))

        if keys[pygame.K_DOWN] or keys[pygame.K_s]:
            player["Y"] = min(player["Y"] + vel, shared.MAP_HEIGHT -
                              getSizeFromScore(player["score"]))

        moveMessage(player["X"], player["Y"])

        for event in pygame.event.get():
            # if user hits red x button close window
            if event.type == pygame.QUIT:
                run = False

            if event.type == pygame.KEYDOWN:
                # if user hits a escape key close program
                if event.key == pygame.K_ESCAPE:
                    run = False

        # redraw window then update the frame
        redraw_window()
        pygame.display.update()

def moveMessage(x, y):
    global players, foods, serverIP, playerid

    MOVE_MESSAGE_BYTES = json.dumps(
        {"type": shared.messageTypes.MOVE, "playerid": playerid, "X": x, "Y": y, "timestamp": time.time_ns()})
    # print(MOVE_MESSAGE_BYTES)
    MOVE_MESSAGE_BYTES = str.encode(MOVE_MESSAGE_BYTES)

    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
        sock.bind(("", 0))
        for i in range(2):
            sock.sendto(MOVE_MESSAGE_BYTES,
                        (serverIP, shared.DISCO_PORT))

thread_discovery = Thread(target=disco, args=(), daemon=True)
thread_discovery.start()
thread_TCP_listener = Thread(target=messagegetterTCP, args=(), daemon=True)
thread_TCP_listener.start()
thread_UDP_listener = Thread(target=messagegetterUDP, args=(), daemon=True)
thread_UDP_listener.start()

# setup pygame window
WIN = pygame.display.set_mode((shared.MAP_WIDTH, shared.MAP_HEIGHT))
pygame.display.set_caption("agar.io clone")

main()


from sklearn.ensemble import RandomForestClassifier
import numpy as np
import cross_val
import pre_process
import output_csv
from sklearn.metrics import log_loss

predict_method = 'proba'

def make_best_classifier():
    return RandomForestClassifier(n_jobs = 7, n_estimators = 65, max_depth = 3), predict_method

def train_base_clf(pp):
    clf = make_best_classifier()[0]
    n_estimators_range = np.arange(40, 70, 5)
    max_depth_range = np.arange(2, 4, 1)

    param_grid = dict(n_estimators = n_estimators_range, max_depth = max_depth_range)
    clf, bp, bs = cross_val.fit_clf(clf, pp.X_train, pp.Y_train, param_grid, 'log_loss')
    output_csv.write_test_csv(clf.__class__.__name__, pp.df_output_test, clf.predict_proba(pp.X_test)[:,1])
    output_csv.write_gs_params_base(clf.__class__.__name__, bp, bs,
                                    clf.score(pp_base.X_train, pp_base.Y_train), log_loss(pp.Y_train, clf.predict_proba(pp.X_train)[:,1]))
    return clf, predict_method

if __name__ == '__main__':
    pp_base = pre_process.PreProcessBase(normalize = False)
    clf = train_base_clf(pp_base)[0]


import os
import pickle

from collections import defaultdict
from Bio.Blast import NCBIXML
import numpy as np
import pandas as pd

def getEggNOG_homologousNameUnSortedPP_COG2PP_reversedPP_dict(record):

    CurrentSpe_ID,max_level,EggNOG_groupPath,homologous_COG2PP_path=record

    EggNOG_homologousPP_COG2PP_CurrentSpe_dict_file=homologous_COG2PP_path+"EggNogMaxLevel"+max_level+"_"+CurrentSpe_ID+'_HomologousCOG2PP_NameUnSorted_dict.pickle'

    if not os.path.exists(EggNOG_homologousPP_COG2PP_CurrentSpe_dict_file): # this is not working in nextflow but useful when run seperately
        EggNOG_group_level2=pd.read_csv(EggNOG_groupPath+max_level+".tsv",
                                        header=None,index_col=None,sep="\t")

        EggNOG_group_level2_speID=EggNOG_group_level2.iloc[:,2].values.tolist()
        EggNOG_group_level2_speID=[str(sid) for sid in EggNOG_group_level2_speID]

        EggNOG_group_level2_idx=[idx for idx,sid in  enumerate(EggNOG_group_level2_speID) if (sid==CurrentSpe_ID)]
        EggNOG_group_level2=EggNOG_group_level2.iloc[EggNOG_group_level2_idx,:]
        EggNOG_group_level2=EggNOG_group_level2.sort_values(by=1)

        # create a dictiontionay whose key is eggnog group id/name and keys are speceis and protein belong to this eggnog group
        EggNOG_group_level2_groupDict=defaultdict(list)

        EggNOG_group_level2_list=EggNOG_group_level2.values.tolist()

        for _,group_id,sid,pname in EggNOG_group_level2_list:
            EggNOG_group_level2_groupDict[group_id].append((str(sid),pname))

        print(CurrentSpe_ID,"EggNOG_group_level2_groupDict",len(EggNOG_group_level2_groupDict))

        # for all  eggnog group pairs , get homologous protein pairs belong to these eggnog group pairs
        # and creat two dictinoary to store such information for query and Query species

        # here have to order eggnog group. so each eggnog group pair, only one recored remained
        EggNOG_group_level2_uniqueGroups=sorted(list(EggNOG_group_level2_groupDict.keys()))
        EggNOG_homologousPP_COG2PP_CurrentSpe_dict=defaultdict(list)

        for ig in range(len(EggNOG_group_level2_uniqueGroups)-1):
            ig_eggnogID=EggNOG_group_level2_uniqueGroups[ig]
            ig_values = EggNOG_group_level2_groupDict[ig_eggnogID]
            for jg  in range(ig+1,len(EggNOG_group_level2_uniqueGroups)):
                jg_eggnogID=EggNOG_group_level2_uniqueGroups[jg]
                jg_values = EggNOG_group_level2_groupDict[jg_eggnogID]
                for ig_sid, ig_pname in ig_values:
                    for jg_sid,jg_pname in jg_values:
                        #if (ig_sid == jg_sid):  # actually this step is not neceeeay as of course they are from all species
                        # because we order COG groups before
                        # there we make sure both (protein1, protein2 ) and (protein2, protein1 )  have records
                        EggNOG_homologousPP_COG2PP_CurrentSpe_dict[(ig_eggnogID,jg_eggnogID)].append((ig_pname,jg_pname))
                        EggNOG_homologousPP_COG2PP_CurrentSpe_dict[(jg_eggnogID,ig_eggnogID)].append((jg_pname,ig_pname))
        print("len(EggNOG_homologousPP_COG2PP_CurrentSpe_dict:",len(EggNOG_homologousPP_COG2PP_CurrentSpe_dict))

        #notice even for nextlfow , the folder homologous_COG2PP_path is temperory, it need to be creaed before to avoid file not exist problem
        with open(EggNOG_homologousPP_COG2PP_CurrentSpe_dict_file, 'wb') as handle:
            pickle.dump(EggNOG_homologousPP_COG2PP_CurrentSpe_dict, handle)

def allQuery2SubjectPPIMapping_getNameUnSorted_COGasLink(Query_tuple,Subject_tupleList,PPIInfoBeforeCoEvoComp_csv,homologous_COG2PP_path):

    # use  postive and negative ppi to only use pp that have large Nf90 value, and after removing deep homologs
    currentSpe_allPPIs_beforeCoEvoComp_frame = pd.read_csv(PPIInfoBeforeCoEvoComp_csv,header=0,index_col=None,sep="\t")
    print("currentSpe_allPPIs_beforeCoEvoComp_frame.shape:",currentSpe_allPPIs_beforeCoEvoComp_frame.shape)
    currentSpe_allPPIs_beforeCoEvoComp_info=currentSpe_allPPIs_beforeCoEvoComp_frame.values.tolist()
    allPPIs=[(p1,p2) for p1, p2,_ in currentSpe_allPPIs_beforeCoEvoComp_info]
    print("len(allPPIs):",len(allPPIs))
    Query_allPPI_allInfo_dict={pp:1 for pp in allPPIs}
    print("len(Query_allPPI_allInfo_dict)",len(Query_allPPI_allInfo_dict))

   # this is cog to pp relationship in eggnot level 2 despite spceis
    Query_EggNOG_homologousPP_COG2PP_dict_file=homologous_COG2PP_path+"EggNogMaxLevel2_"+Query_tuple[1]+'_HomologousCOG2PP_NameUnSorted_dict.pickle'
    with open(Query_EggNOG_homologousPP_COG2PP_dict_file, 'rb') as handle:
            Query_EggNOG_homologousPP_COG2PP_dict=pickle.load(handle)
    print("len(Query_EggNOG_homologousPP_COG2PP_dict):",len(Query_EggNOG_homologousPP_COG2PP_dict))

    Query_EggNOG_homologousPP_PP2COG_dict=defaultdict(list)
    for k, vs in Query_EggNOG_homologousPP_COG2PP_dict.items():
        for v in vs:
            Query_EggNOG_homologousPP_PP2COG_dict[v].append(k)
    print(len(Query_EggNOG_homologousPP_PP2COG_dict))

    #keep in mind here "PP2COG_dict" is already contain reversed records
    # here filter cog to pp relation by only keep pp with calculated high dca scores to save computational time later
    Query_EggNOG_homologousPP_PP2COG_dict={ pp: Query_EggNOG_homologousPP_PP2COG_dict[pp] for pp in Query_allPPI_allInfo_dict}
    print("len(Query_EggNOG_homologousPP_PP2COG_dict):",len(Query_EggNOG_homologousPP_PP2COG_dict))

    # for subject data
    Query2Subject_homologous_dict_listDict=dict()
    for Subject_phylum, Subject_speID in Subject_tupleList:
        print("Subject_phylum, Subject_speID:",Subject_phylum, Subject_speID)

        # this is cog to pp relationship in eggnot level 2 despite spceis
        Subject_EggNOG_homologousPP_COG2PP_dict_file=homologous_COG2PP_path+"EggNogMaxLevel2_"+Subject_speID+'_HomologousCOG2PP_NameUnSorted_dict.pickle'
        with open(Subject_EggNOG_homologousPP_COG2PP_dict_file, 'rb') as handle:
                Subject_EggNOG_homologousPP_COG2PP_dict=pickle.load(handle)
        print(len(Subject_EggNOG_homologousPP_COG2PP_dict))

        # combined info
        Query2Subject_homologous_dict=defaultdict(list)
        for Query_pp, cogs in Query_EggNOG_homologousPP_PP2COG_dict.items():
            for cog in cogs:
                if cog in Subject_EggNOG_homologousPP_COG2PP_dict:
                    Subject_pps=Subject_EggNOG_homologousPP_COG2PP_dict[cog]
                    # for each cog , should be a append
                    Query2Subject_homologous_dict[Query_pp].extend(Subject_pps)
                    #Namesorted_Query2Subject_homologous_dict[subject_pp]=query_pps

        #thats because in query speceis, some different  protein pairs can exist  in diffrent cog pairs
        #that can be mapped to same protein pairs in subject species
        # so bacically , one protein pair in sub species and one protein pair in query sepceis share several cog paris
        print(len(Query2Subject_homologous_dict))
        Query2Subject_homologous_dict={k: list(set(v)) for k, v in Query2Subject_homologous_dict.items()}

        print(len(Query2Subject_homologous_dict))

        Query2Subject_homologous_dict_listDict[(Subject_phylum, Subject_speID)]=Query2Subject_homologous_dict

    return(Query2Subject_homologous_dict_listDict)

def allQuery2SubjectSingleProteinMapping(Query2Subject_QueSpeAllPPI_homologous_dict_listDict):

    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict=defaultdict(dict)
    for phylum_speID,temp_Query2Subject_QueSpeAllPPI_homologous_dict in Query2Subject_QueSpeAllPPI_homologous_dict_listDict.items():
        print(phylum_speID)
        for Query_pp, Subject_pps in temp_Query2Subject_QueSpeAllPPI_homologous_dict.items():
            Query_pp1,Query_pp2=Query_pp
            for Subject_pp in Subject_pps:
                Subject_pp1,Subject_pp2=Subject_pp

                # here we should notice one  single protein can have multiple homologous protein pairs
                # and same one singel protein to single protein mapping can appear many times in different homologous protein pair mapping
                if Query_pp1 not in Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict[phylum_speID]:
                    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict[phylum_speID][Query_pp1]={Subject_pp1} #set
                else:
                    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict[phylum_speID][Query_pp1].add(Subject_pp1)

                if Query_pp2 not in Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict[phylum_speID]:
                    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict[phylum_speID][Query_pp2]={Subject_pp2} # set
                else:
                    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict[phylum_speID][Query_pp2].add(Subject_pp2)

    return(Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict)

def allQuery2SubjectSingleProteinMapping_getBlasp(Query_speID,Subject_tupleList,homologous_SeqMappingPath,Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict):
    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Subject2Query_bestHspBits_listDict=defaultdict(dict)
    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Subject2Query_blastp_listDict=defaultdict(dict)
    for phylum, Subject_speID in Subject_tupleList:
        print(phylum, Subject_speID)

        current_homologous_SeqMappingPath=homologous_SeqMappingPath+"EggNogMaxLevel2_QuerySpe_ID"+Query_speID+"and"+"SubjectSpe_ID"+Subject_speID+'/'

        temp_Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_dict=Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict[(phylum,Subject_speID)]

        for Query_proID,Subject_proIDs in temp_Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_dict.items():
            for Subject_proID in Subject_proIDs:
                #print(current_homologous_SeqMappingPath+Query_proID+"and"+Subject_proID+".xml")
                with open(current_homologous_SeqMappingPath+Query_proID+"and"+Subject_proID+".xml") as handle:
                    blast_record = NCBIXML.read(handle)

                    # ???? here there is a problem is that if blast_record.alignments is None
                    # then this for loop will never get started and thus sorted_alignment_hsps and beste_hsp
                    # will just be the values from last for loop

                    if len(blast_record.alignments)>0:
                        for alignment in blast_record.alignments:
                            #print("times")
                            sorted_alignment_hsps=sorted(alignment.hsps,key=lambda hsp: hsp.bits,reverse=True)
                            best_hsp=sorted_alignment_hsps[0]
                            #print("best_hsp.bits:",best_hsp.bits)
                        Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Subject2Query_bestHspBits_listDict[(phylum,Subject_speID)][(Query_proID,Subject_proID)]=best_hsp.bits
                    else:
                        #print("here empty blastp results")
                        Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Subject2Query_bestHspBits_listDict[(phylum,Subject_speID)][(Query_proID,Subject_proID)]=np.nan

                # '6 qseqid  qlen sseqid  slen qstart qend sstart send evalue bitscore pident qcovs qcovhsp'
                try:
                    out_frame=pd.read_csv(current_homologous_SeqMappingPath+Query_proID+"and"+Subject_proID+".txt",header=None,sep="\t")
                    out_frame=out_frame.sort_values(by=[9],ascending=True) # sorted by bit scores
                    out_values=out_frame.values.tolist()
                    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Subject2Query_blastp_listDict[(phylum,Subject_speID)][(Query_proID,Subject_proID)]=out_values

                except:
                    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Subject2Query_blastp_listDict[(phylum,Subject_speID)][(Query_proID,Subject_proID)]=np.nan

    return (Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Subject2Query_bestHspBits_listDict,
           Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Subject2Query_blastp_listDict)

def Query2Subject_AAmapDic(Query_speID,Subject_tupleList,
                           homologous_SeqMappingPath,
                           Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict,
                           overlap_method="remove"):

    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Query2Subject_AAmapDic_listDict=defaultdict(dict)
    for phylum, Subject_speID in Subject_tupleList:

        current_homologous_SeqMappingPath=homologous_SeqMappingPath+"EggNogMaxLevel2_QuerySpe_ID"+Query_speID+"and"+"SubjectSpe_ID"+Subject_speID+'/'

        temp_Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_dict=Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict[(phylum,Subject_speID)]

        for Query_proID,Subject_proIDs in temp_Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_dict.items():
            for Subject_proID in Subject_proIDs:
                with open(current_homologous_SeqMappingPath+Query_proID+"and"+Subject_proID+"subject2query_AAmapDic_overlapMethod"+overlap_method+".pickle", 'rb') as handle:
                    Subject2Query_AAmapDic=pickle.load(handle)

                    # here pos in AAmapDic start with 1
                    # !! we force it to start from 0 , so its consistent with  0-starting pos from other object
                    Subject2Query_AAmapDic={k-1:v-1for k, v in Subject2Query_AAmapDic.items()}

                    Query2Subject_AAmapDic={v:k for k , v in Subject2Query_AAmapDic.items()}

                    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Query2Subject_AAmapDic_listDict[(phylum,Subject_speID)][(Query_proID,Subject_proID)]=Query2Subject_AAmapDic

    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Subject2Query_AAmapDic_listDict=defaultdict(dict)
    for phylum, Subject_speID in Subject_tupleList:

        current_homologous_SeqMappingPath=homologous_SeqMappingPath+"EggNogMaxLevel2_QuerySpe_ID"+Query_speID+"and"+"SubjectSpe_ID"+Subject_speID+'/'

        temp_Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_dict=Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_listDict[(phylum,Subject_speID)]

        for Query_proID,Subject_proIDs in temp_Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_dict.items():
            for Subject_proID in Subject_proIDs:
                with open(current_homologous_SeqMappingPath+Query_proID+"and"+Subject_proID+"subject2query_AAmapDic_overlapMethod"+overlap_method+".pickle", 'rb') as handle:
                    Subject2Query_AAmapDic=pickle.load(handle)

                    # here pos in AAmapDic start with 1
                    # !! we force it to start from 0 , so its consistent with  0-starting pos from other object
                    Subject2Query_AAmapDic={k-1:v-1for k, v in Subject2Query_AAmapDic.items()}

                    Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Subject2Query_AAmapDic_listDict[(phylum,Subject_speID)][(Query_proID,Subject_proID)]=Subject2Query_AAmapDic

    return(Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Query2Subject_AAmapDic_listDict,Query2Subject_QueSpeAllPPI_homologous_singleProteinMaping_Subject2Query_AAmapDic_listDict)

def chooseBestHomologousPP(Query_PP,Subject_PPs,Query2Subject_homologous_ignoreSubjectDCA_singleProteinMaping_Subject2Query_bestHspBits_dict):

    #print(Subject_PPs)
    BestHomologousPP=()
    BestHomologousPP_bestHspBits=()
    # when Subject_PPs is none, for lood dont stat and empty Subject pp returned
    for Subject_PP in Subject_PPs:
        cur_QuerySubjectPP1_bestHspBits=Query2Subject_homologous_ignoreSubjectDCA_singleProteinMaping_Subject2Query_bestHspBits_dict[(Query_PP[0],Subject_PP[0])]
        cur_QuerySubjectPP2_bestHspBits=Query2Subject_homologous_ignoreSubjectDCA_singleProteinMaping_Subject2Query_bestHspBits_dict[(Query_PP[1],Subject_PP[1])]

        if not ((np.isnan(cur_QuerySubjectPP1_bestHspBits)) or (np.isnan(cur_QuerySubjectPP2_bestHspBits))) :

            if len(BestHomologousPP)==0:
                BestHomologousPP=Subject_PP
                BestHomologousPP_bestHspBits=(cur_QuerySubjectPP1_bestHspBits,cur_QuerySubjectPP2_bestHspBits)
            else:
                # compare lower bestHspBits of currrent Subject pp and homologous pp
                # choose one with high lower bestHspbit
                if min(BestHomologousPP_bestHspBits)< min(cur_QuerySubjectPP1_bestHspBits,cur_QuerySubjectPP2_bestHspBits):
                    BestHomologousPP=Subject_PP
                    BestHomologousPP_bestHspBits=(cur_QuerySubjectPP1_bestHspBits,cur_QuerySubjectPP2_bestHspBits)
                # when lower bestHspBits  are same , compare another protein
                elif (min(BestHomologousPP_bestHspBits)== min(cur_QuerySubjectPP1_bestHspBits,cur_QuerySubjectPP2_bestHspBits)) and (max(BestHomologousPP_bestHspBits)< max(cur_QuerySubjectPP1_bestHspBits,cur_QuerySubjectPP2_bestHspBits)):
                    BestHomologousPP=Subject_PP
                    BestHomologousPP_bestHspBits=(cur_QuerySubjectPP1_bestHspBits,cur_QuerySubjectPP2_bestHspBits)

    return(BestHomologousPP)

def getMetaFrame_FullBestHomologousPP(Query_tuple,EggNOG_maxLevel,currentSpe_TaxID,STRING_Version,
                                      benchmark_suffix="STRINPhyPPI_Benchmark/",
                                     smallPhylum="",
                                     BestHomologousPP_filePrefix="BestHomologousPP",
                                        CoEvo_data_folder="CoEvo_data_STRING11.5/",
                                     ):
    '''
    here full BestHomologousPP means all BestHomologous pp compared with Query speceis
    It does not include all pp in origin indepenpendt benchmark dataset
    '''
    input_root_folder=CoEvo_data_folder+smallPhylum+currentSpe_TaxID+"_EggNOGmaxLevel"+EggNOG_maxLevel+"_eggNOGfilteredData/"

    Subect_BestHomologousPP_prefix="BestHomologousPPFor"+Query_tuple[1]+"AtEggNOGmaxLevel"+Query_tuple[0]+"_"
    BestHomologousPP_Benchmark_folder=input_root_folder+Subect_BestHomologousPP_prefix+benchmark_suffix #
    print("BestHomologousPP_Benchmark_folder:",BestHomologousPP_Benchmark_folder)

    allPPI_allInfo_frame=pd.read_csv(BestHomologousPP_Benchmark_folder+BestHomologousPP_filePrefix+"_allPPI_allInfo_frame.csv",
                                 header=0,index_col=None,sep="\t")

    print("allPPI_allInfo_frame.shape:",allPPI_allInfo_frame.shape)

    if benchmark_suffix == "STRINPhyPPI_Benchmark/":
        Pos_allPPI_allInfo_frame=allPPI_allInfo_frame.loc[allPPI_allInfo_frame['benchmark_status']=="P",:]
        #print("Pos_allPPI_allInfo_frame.shape:",Pos_allPPI_allInfo_frame.shape)
        Pos_allPPI_allInfo_frame=Pos_allPPI_allInfo_frame.sort_values(by="maxBetDCA_score",ascending=False)

        Neg_allPPI_allInfo_frame=allPPI_allInfo_frame.loc[allPPI_allInfo_frame['benchmark_status']=="N",:]
        #print("Neg_allPPI_allInfo_frame.shape:",Neg_allPPI_allInfo_frame.shape)

        Neg_allPPI_allInfo_frame=Neg_allPPI_allInfo_frame.sort_values(by="maxBetDCA_score",ascending=False)

        allPPI_allInfo_frame= pd.concat([Pos_allPPI_allInfo_frame,Neg_allPPI_allInfo_frame])
    elif benchmark_suffix == "KEGG_Benchmark/":
        allPPI_allInfo_frame=allPPI_allInfo_frame.sort_values(by="maxBetDCA_score",ascending=False)
    elif benchmark_suffix == "QianCong_Benchmark/":
        allPPI_allInfo_frame=allPPI_allInfo_frame.sort_values(by="maxBetDCA_score",ascending=False)

    else:
        return(allPPI_allInfo_frame)

    return(allPPI_allInfo_frame)

def collect_BestHomologousDCAs_OneSpeOneScore_OnlyTopPosNeg(allPPI_allInfoFrame,
                                                      Query_BestHomologousDCAs_dict,
                                                      ML_inputPath,CoEvo_type="DCA",
                                                           BestHomologousDCAs_dict_withStatus=True):

    X_fileName=ML_inputPath+"OnlyTopPosNeg_NonPara_X_BestHomologous"+CoEvo_type+"s_OneSpeOneScore.npy"
    Y_fileName=ML_inputPath+"OnlyTopPosNeg_NonPara_Y_BestHomologous"+CoEvo_type+"s_OneSpeOneScore.npy"
    #if not os.path.exists(X_fileName):

    NonPara_allPPI_info=allPPI_allInfoFrame.loc[:,["STRING_ID1","STRING_ID2","benchmark_status"]].values.tolist()

    if BestHomologousDCAs_dict_withStatus:
        OnlyTopPosNeg_NonPara_X=np.zeros((len(NonPara_allPPI_info),len(list(Query_BestHomologousDCAs_dict.values())[0])-1))
        OnlyTopPosNeg_NonPara_Y=np.zeros((len(NonPara_allPPI_info),1))
        for idx,l in enumerate(NonPara_allPPI_info):
            p1, p2 , label=l
            numeric_lable=1 if label=="P" else 0

            OnlyTopPosNeg_NonPara_X[idx,:]=Query_BestHomologousDCAs_dict[(p1,p2)][1:]
            OnlyTopPosNeg_NonPara_Y[idx]=numeric_lable
    else:
        OnlyTopPosNeg_NonPara_X=np.zeros((len(NonPara_allPPI_info),len(list(Query_BestHomologousDCAs_dict.values())[0])))
        OnlyTopPosNeg_NonPara_Y=np.zeros((len(NonPara_allPPI_info),1))
        for idx,l in enumerate(NonPara_allPPI_info):
            p1, p2 , label=l
            numeric_lable=1 if label=="P" else 0

            OnlyTopPosNeg_NonPara_X[idx,:]=Query_BestHomologousDCAs_dict[(p1,p2)]
            OnlyTopPosNeg_NonPara_Y[idx]=numeric_lable

    np.save(X_fileName,OnlyTopPosNeg_NonPara_X)
    np.save(Y_fileName,OnlyTopPosNeg_NonPara_Y)

    OnlyTopPosNeg_NonPara_XtopCoEvos=np.load(X_fileName)

    OnlyTopPosNeg_NonPara_YtopCoEvos=np.load(Y_fileName)
    OnlyTopPosNeg_NonPara_YtopCoEvos=np.reshape(OnlyTopPosNeg_NonPara_YtopCoEvos,(OnlyTopPosNeg_NonPara_YtopCoEvos.shape[0],))#

    return(OnlyTopPosNeg_NonPara_XtopCoEvos,OnlyTopPosNeg_NonPara_YtopCoEvos)


"""

"""
import logging
import copy

from django.conf import settings
from django.db import connection, connections

from core.pandajob.models import PandaJob
from core.pandajob.utils import identify_jobtype
from core.schedresource.utils import get_panda_queues
from core.libs.exlib import getPilotCounts
from core.libs.sqlsyntax import interval_to_sec

import core.constants as const

_logger = logging.getLogger('bigpandamon')

def prepare_job_summary_region(jsr_queues_dict, jsr_sites_dict, jsr_regions_dict, **kwargs):
    """
    Convert dict of region job summary to list
    :param jsr_queues_dict: dict of region job summary
    :param jsr_regions_dict: dict of queue job summary
    :return: list of region job summary, list of queue job summary
    """
    split_by = None
    if 'split_by' in kwargs and kwargs['split_by']:
        split_by = kwargs['split_by']
    jsr_queues_list = []
    jsr_sites_list = []
    jsr_regions_list = []
    for pq, params in jsr_queues_dict.items():
        for jt, resourcetypes in params['summary'].items():
            for rt, summary in resourcetypes.items():
                row = list()
                row.append(pq)
                row.append(params['pq_params']['pqtype'])
                row.append(params['pq_params']['region'])
                row.append(params['pq_params']['status'])
                row.append(jt)
                row.append(rt)
                row.append(params['pq_pilots']['count'])
                row.append(params['pq_pilots']['count_nojob'])
                row.append(summary['nwsubmitted'])
                row.append(summary['nwrunning'])
                row.append(summary['rcores'])
                row.append(sum([v for k, v in summary.items() if k in const.JOB_STATES]))
                if summary['failed'] + summary['finished'] > 0:
                    row.append(round(100.0*summary['failed']/(summary['failed'] + summary['finished']), 1))
                else:
                    row.append(0)
                for js in const.JOB_STATES:
                    row.append(summary[js])

                if split_by is None:
                    if jt == 'all' and rt == 'all':
                        jsr_queues_list.append(row)
                elif 'jobtype' in split_by and 'resourcetype' in split_by:
                    if jt != 'all' and rt != 'all':
                        jsr_queues_list.append(row)
                elif 'jobtype' in split_by and 'resourcetype' not in split_by:
                    if jt != 'all' and rt == 'all':
                        jsr_queues_list.append(row)
                elif 'jobtype' not in split_by and 'resourcetype' in split_by:
                    if jt == 'all' and rt != 'all':
                        jsr_queues_list.append(row)

    for site, jobtypes in jsr_sites_dict.items():
        for jt, resourcetypes in jobtypes.items():
            for rt, summary in resourcetypes.items():
                row = list()
                row.append(site)
                row.append(jt)
                row.append(rt)
                row.append(summary['nwsubmitted'])
                row.append(summary['nwrunning'])
                row.append(summary['rcores'])
                row.append(sum([v for k, v in summary.items() if k in const.JOB_STATES]))
                if summary['failed'] + summary['finished'] > 0:
                    row.append(round(100.0 * summary['failed'] / (summary['failed'] + summary['finished']), 1))
                else:
                    row.append(0)
                for js in const.JOB_STATES:
                    row.append(summary[js])

                if split_by is None:
                    if jt == 'all' and rt == 'all':
                        jsr_sites_list.append(row)
                elif 'jobtype' in split_by and 'resourcetype' in split_by:
                    if jt != 'all' and rt != 'all':
                        jsr_sites_list.append(row)
                elif 'jobtype' in split_by and 'resourcetype' not in split_by:
                    if jt != 'all' and rt == 'all':
                        jsr_sites_list.append(row)
                elif 'jobtype' not in split_by and 'resourcetype' in split_by:
                    if jt == 'all' and rt != 'all':
                        jsr_sites_list.append(row)

    for reg, jobtypes in jsr_regions_dict.items():
        for jt, resourcetypes in jobtypes.items():
            for rt, summary in resourcetypes.items():
                row = list()
                row.append(reg)
                row.append(jt)
                row.append(rt)
                row.append(summary['nwsubmitted'])
                row.append(summary['nwrunning'])
                row.append(summary['rcores'])
                row.append(sum([v for k, v in summary.items() if k in const.JOB_STATES]))
                if summary['failed'] + summary['finished'] > 0:
                    row.append(round(100.0 * summary['failed'] / (summary['failed'] + summary['finished']), 1))
                else:
                    row.append(0)
                for js in const.JOB_STATES:
                    row.append(summary[js])

                if split_by is None:
                    if jt == 'all' and rt == 'all':
                        jsr_regions_list.append(row)
                elif 'jobtype' in split_by and 'resourcetype' in split_by:
                    if jt != 'all' and rt != 'all':
                        jsr_regions_list.append(row)
                elif 'jobtype' in split_by and 'resourcetype' not in split_by:
                    if jt != 'all' and rt == 'all':
                        jsr_regions_list.append(row)
                elif 'jobtype' not in split_by and 'resourcetype' in split_by:
                    if jt == 'all' and rt != 'all':
                        jsr_regions_list.append(row)
    return jsr_queues_list, jsr_sites_list, jsr_regions_list

def get_job_summary_region(query, **kwargs):
    """
    :param query: dict of query params for jobs retrieving
    :return: dict of groupings
    """
    jsr_queues_dict = {}
    jsr_sites_dict = {}
    jsr_regions_dict = {}

    job_types = ['analy', 'prod']
    resource_types = ['SCORE', 'MCORE', 'SCORE_HIMEM', 'MCORE_HIMEM']
    worker_metrics = ['nwrunning', 'nwsubmitted']
    extra_metrics = copy.deepcopy(worker_metrics)
    extra_metrics.append('rcores')

    if 'extra' in kwargs and len(kwargs['extra']) > 1:
        extra = kwargs['extra']
    else:
        extra = '(1=1)'
    if 'region' in kwargs and kwargs['region'] != 'all':
        region = kwargs['region']
        regions_list = [region]
    else:
        region = 'all'
    if 'jobtype' in kwargs and kwargs['jobtype'] != 'all':
        jobtype = kwargs['jobtype']
        job_types = [kwargs['jobtype']]
    else:
        jobtype = 'all'
    if 'resourcetype' in kwargs and kwargs['resourcetype'] != 'all':
        resourcetype = kwargs['resourcetype']
        resource_types = [kwargs['resourcetype']]
    else:
        resourcetype = 'all'
    if 'split_by' in kwargs and kwargs['split_by']:
        split_by = kwargs['split_by']
    else:
        split_by = None
    if 'pqs_dict' in kwargs and kwargs['pqs_dict']:
        pqs_dict = kwargs['pqs_dict']
    else:
        # get PQ info
        pqs_dict = get_panda_queues()

    # filter out queues that do not match to provided jobtype (analy/prod)
    if jobtype != 'all':
        pqs_dict = {pqn: params for pqn, params in pqs_dict.items() if is_jobtype_match_queue(jobtype, params)}

    # filter out queues if there is computingsite in the query for jobs
    if 'computingsite' in query:
        pqs_dict = {pqn: params for pqn, params in pqs_dict.items() if pqn == query['computingsite']}
    elif 'computingsite__in' in query:
        pqs_dict = {pqn: params for pqn, params in pqs_dict.items() if pqn in query['computingsite__in']}

    sites_list = list(set([params['atlas_site'] for pq, params in pqs_dict.items() if 'atlas_site' in params]))
    regions_list = list(set([params['cloud'] for pq, params in pqs_dict.items() if 'cloud' in params]))

    # get job summary
    jsq = get_job_summary_split(query, extra=extra)

    # check if there is more values of resourcetype than 4 default ones, if yes -> add them to the list
    if len(jsq) > 0:
        job_resource_types_extra = list(set([row['resourcetype'] for row in jsq]) - set(resource_types))
        if None in job_resource_types_extra or '' in job_resource_types_extra:
            # replace None with 'Not specified'
            resource_types.append('Not specified')
            for row in jsq:
                if row['resourcetype'] is None or row['resourcetype'] == '':
                    row['resourcetype'] = 'Not specified'

        # adding extra job resourcetype to the list
        resource_types.extend([rt for rt in job_resource_types_extra if rt and rt != ''])

    # get workers info
    wsq = []
    if 'core.harvester' in settings.INSTALLED_APPS:
        from core.harvester.utils import get_workers_summary_split
        if 'computingsite__in' not in query:
            # put full list of compitingsites to use index in workers table
            query['computingsite__in'] = list(set([row['computingsite'] for row in jsq]))
        wsq = get_workers_summary_split(query)

    # get PanDA getJob, updateJob request counts
    psq_dict = {}
    if split_by is None and jobtype == 'all' and resourcetype == 'all':
        psq_dict = getPilotCounts('all')

    # create template structure for grouping by queue
    for pqn, params in pqs_dict.items():
        jsr_queues_dict[pqn] = {'pq_params': {}, 'pq_pilots': {},  'summary': {'all': {'all': {}}}}
        jsr_queues_dict[pqn]['pq_params']['pqtype'] = params['type'] if 'type' in params else '-'
        jsr_queues_dict[pqn]['pq_params']['region'] = params['cloud'] if 'cloud' in params else '-'
        jsr_queues_dict[pqn]['pq_params']['atlas_site'] = params['atlas_site'] if 'atlas_site' in params else '-'
        jsr_queues_dict[pqn]['pq_params']['status'] = params['status'] if 'status' in params else '-'
        jsr_queues_dict[pqn]['pq_pilots']['count'] = psq_dict[pqn]['count_abs'] if pqn in psq_dict else -1
        jsr_queues_dict[pqn]['pq_pilots']['count_nojob'] = psq_dict[pqn]['count_nojobabs'] if pqn in psq_dict else -1
        for jt in job_types:
            if is_jobtype_match_queue(jt, params):
                jsr_queues_dict[pqn]['summary'][jt] = {}
                for rt in resource_types:
                    jsr_queues_dict[pqn]['summary'][jt][rt] = {}
                    jsr_queues_dict[pqn]['summary'][jt]['all'] = {}
                    jsr_queues_dict[pqn]['summary']['all'][rt] = {}
                    for js in const.JOB_STATES:
                        jsr_queues_dict[pqn]['summary'][jt][rt][js] = 0
                        jsr_queues_dict[pqn]['summary'][jt]['all'][js] = 0
                        jsr_queues_dict[pqn]['summary']['all'][rt][js] = 0
                        jsr_queues_dict[pqn]['summary']['all']['all'][js] = 0

                    for em in extra_metrics:
                        jsr_queues_dict[pqn]['summary'][jt][rt][em] = 0
                        jsr_queues_dict[pqn]['summary'][jt]['all'][em] = 0
                        jsr_queues_dict[pqn]['summary']['all'][rt][em] = 0
                        jsr_queues_dict[pqn]['summary']['all']['all'][em] = 0

    # create template structure for grouping by region
    for r in regions_list:
        jsr_regions_dict[r] = {}
        for jt in job_types:
            jsr_regions_dict[r][jt] = {}
            jsr_regions_dict[r]['all'] = {}
            for rt in resource_types:
                jsr_regions_dict[r][jt][rt] = {}
                jsr_regions_dict[r][jt]['all'] = {}
                jsr_regions_dict[r]['all'][rt] = {}
                jsr_regions_dict[r]['all']['all'] = {}
                for js in const.JOB_STATES:
                    jsr_regions_dict[r][jt][rt][js] = 0
                    jsr_regions_dict[r][jt]['all'][js] = 0
                    jsr_regions_dict[r]['all'][rt][js] = 0
                    jsr_regions_dict[r]['all']['all'][js] = 0

                for em in extra_metrics:
                    jsr_regions_dict[r][jt][rt][em] = 0
                    jsr_regions_dict[r][jt]['all'][em] = 0
                    jsr_regions_dict[r]['all'][rt][em] = 0
                    jsr_regions_dict[r]['all']['all'][em] = 0

    # create template structure for grouping by site
    for s in sites_list:
        jsr_sites_dict[s] = {}
        for jt in job_types:
            jsr_sites_dict[s][jt] = {}
            jsr_sites_dict[s]['all'] = {}
            for rt in resource_types:
                jsr_sites_dict[s][jt][rt] = {}
                jsr_sites_dict[s][jt]['all'] = {}
                jsr_sites_dict[s]['all'][rt] = {}
                jsr_sites_dict[s]['all']['all'] = {}
                for js in const.JOB_STATES:
                    jsr_sites_dict[s][jt][rt][js] = 0
                    jsr_sites_dict[s][jt]['all'][js] = 0
                    jsr_sites_dict[s]['all'][rt][js] = 0
                    jsr_sites_dict[s]['all']['all'][js] = 0

                for em in extra_metrics:
                    jsr_sites_dict[s][jt][rt][em] = 0
                    jsr_sites_dict[s][jt]['all'][em] = 0
                    jsr_sites_dict[s]['all'][rt][em] = 0
                    jsr_sites_dict[s]['all']['all'][em] = 0

    # fill template with real values of job states counts
    for row in jsq:
        if row['computingsite'] in jsr_queues_dict and row['jobtype'] in jsr_queues_dict[row['computingsite']]['summary'] and row['resourcetype'] in resource_types and row['jobstatus'] in const.JOB_STATES and 'count' in row:
            jsr_queues_dict[row['computingsite']]['summary'][row['jobtype']][row['resourcetype']][row['jobstatus']] += int(row['count'])
            jsr_queues_dict[row['computingsite']]['summary']['all'][row['resourcetype']][row['jobstatus']] += int(row['count'])
            jsr_queues_dict[row['computingsite']]['summary'][row['jobtype']]['all'][row['jobstatus']] += int(row['count'])
            jsr_queues_dict[row['computingsite']]['summary']['all']['all'][row['jobstatus']] += int(row['count'])

            jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']][row['jobtype']][row['resourcetype']][row['jobstatus']] += int(row['count'])
            jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']]['all'][row['resourcetype']][row['jobstatus']] += int(row['count'])
            jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']][row['jobtype']]['all'][row['jobstatus']] += int(row['count'])
            jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']]['all']['all'][row['jobstatus']] += int(row['count'])

            jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']][row['jobtype']][row['resourcetype']][row['jobstatus']] += int(row['count'])
            jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']]['all'][row['resourcetype']][row['jobstatus']] += int(row['count'])
            jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']][row['jobtype']]['all'][row['jobstatus']] += int(row['count'])
            jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']]['all']['all'][row['jobstatus']] += int(row['count'])

            # fill sum of running cores
            if row['jobstatus'] == 'running':
                jsr_queues_dict[row['computingsite']]['summary'][row['jobtype']][row['resourcetype']]['rcores'] += int(row['rcores'])
                jsr_queues_dict[row['computingsite']]['summary']['all'][row['resourcetype']]['rcores'] += int(row['rcores'])
                jsr_queues_dict[row['computingsite']]['summary'][row['jobtype']]['all']['rcores'] += int(row['rcores'])
                jsr_queues_dict[row['computingsite']]['summary']['all']['all']['rcores'] += int(row['rcores'])

                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']][row['jobtype']][row['resourcetype']]['rcores'] += int(row['rcores'])
                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']]['all'][row['resourcetype']]['rcores'] += int(row['rcores'])
                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']][row['jobtype']]['all']['rcores'] += int(row['rcores'])
                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']]['all']['all']['rcores'] += int(row['rcores'])

                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']][row['jobtype']][row['resourcetype']]['rcores'] += int(row['rcores'])
                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']]['all'][row['resourcetype']]['rcores'] += int(row['rcores'])
                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']][row['jobtype']]['all']['rcores'] += int(row['rcores'])
                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']]['all']['all']['rcores'] += int(row['rcores'])

    # fill template with real values of n workers stats
    for row in wsq:
        if row['computingsite'] in jsr_queues_dict and row['jobtype'] in jsr_queues_dict[row['computingsite']]['summary'] and row['resourcetype'] in resource_types:
            for wm in worker_metrics:
                jsr_queues_dict[row['computingsite']]['summary'][row['jobtype']][row['resourcetype']][wm] += int(row[wm])
                jsr_queues_dict[row['computingsite']]['summary']['all'][row['resourcetype']][wm] += int(row[wm])
                jsr_queues_dict[row['computingsite']]['summary'][row['jobtype']]['all'][wm] += int(row[wm])
                jsr_queues_dict[row['computingsite']]['summary']['all']['all'][wm] += int(row[wm])

                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']][row['jobtype']][row['resourcetype']][wm] += int(row[wm])
                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']]['all'][row['resourcetype']][wm] += int(row[wm])
                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']][row['jobtype']]['all'][wm] += int(row[wm])
                jsr_sites_dict[jsr_queues_dict[row['computingsite']]['pq_params']['atlas_site']]['all']['all'][wm] += int(row[wm])

                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']][row['jobtype']][row['resourcetype']][wm] += int(row[wm])
                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']]['all'][row['resourcetype']][wm] += int(row[wm])
                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']][row['jobtype']]['all'][wm] += int(row[wm])
                jsr_regions_dict[jsr_queues_dict[row['computingsite']]['pq_params']['region']]['all']['all'][wm] += int(row[wm])

    return jsr_queues_dict, jsr_sites_dict, jsr_regions_dict

def get_job_summary_split(query, extra):
    """ Query the jobs summary """
    db = connections['default'].vendor
    fields = {field.attname: field.db_column for field in PandaJob._meta.get_fields()}
    extra_str = extra
    for qn, qvs in query.items():
        if '__in' in qn and qn.replace('__in', '') in fields:
            extra_str += " and (" + fields[qn.replace('__in', '')] + " in ("
            if len(qvs) > 0:
                for qv in qvs:
                    extra_str += "'" + str(qv) + "',"
                extra_str = extra_str[:-1]
            else:
                extra_str += "''"
            extra_str += "))"
        elif '__icontains' in qn or '__contains' in qn and qn.replace('__icontains', '').replace('__contains', '') in fields:
            extra_str += " and (" + fields[qn.replace('__icontains', '').replace('__contains', '')] + " LIKE '%%" + qvs + "%%')"
        elif '__startswith' in qn and qn.replace('__startswith', '') in fields:
            extra_str += " and (" + fields[qn.replace('__startswith', '')] + " LIKE '" + qvs + "%%')"
        elif '__endswith' in qn and qn.replace('__endswith', '') in fields:
            extra_str += " and (" + fields[qn.replace('__endswith', '')] + " LIKE '%%" + qvs + "')"
        elif qn in fields:
            extra_str += " and (" + fields[qn] + "= '" + str(qvs) + "' )"

    # get jobs groupings, the jobsactive4 table can keep failed analysis jobs for up to 7 days, so splitting the query
    query_raw = """
        select j.computingsite, j.resource_type as resourcetype, j.prodsourcelabel, j.transform, j.jobstatus,
            count(j.pandaid) as count, sum(j.rcores) as rcores, round(sum(j.walltime)) as walltime
        from  (
        select ja4.pandaid, ja4.resource_type, ja4.computingsite, ja4.prodsourcelabel, ja4.jobstatus, ja4.modificationtime, 0 as rcores,
            case when jobstatus in ('finished', 'failed') then {walltime_sec} else 0 end as walltime,
            case when transformation like 'http%' then 'run' when transformation like '%.py' then 'py' else 'unknown' end as transform
        from {db_scheme}.jobsarchived4 ja4  where modificationtime > TO_DATE('{date_from}', 'YYYY-MM-DD HH24:MI:SS') and {extra_str}
        union
        select jav4.pandaid, jav4.resource_type, jav4.computingsite, jav4.prodsourcelabel, jav4.jobstatus, jav4.modificationtime,
            case when jobstatus = 'running' then actualcorecount else 0 end as rcores, 0 as walltime,
            case when transformation like 'http%' then 'run' when transformation like '%.py' then 'py' else 'unknown' end as transform
        from {db_scheme}.jobsactive4 jav4 where modificationtime > TO_DATE('{date_from}', 'YYYY-MM-DD HH24:MI:SS') and
            jobstatus in ('failed', 'finished', 'closed', 'cancelled') and {extra_str}
        union
        select jav4.pandaid, jav4.resource_type, jav4.computingsite, jav4.prodsourcelabel, jav4.jobstatus, jav4.modificationtime,
            case when jobstatus = 'running' then actualcorecount else 0 end as rcores, 0 as walltime,
            case when transformation like 'http%' then 'run' when transformation like '%.py' then 'py' else 'unknown' end as transform
        from {db_scheme}.jobsactive4 jav4 where jobstatus not in ('failed', 'finished', 'closed', 'cancelled') and {extra_str}
        union
        select jw4.pandaid, jw4.resource_type, jw4.computingsite, jw4.prodsourcelabel, jw4.jobstatus, jw4.modificationtime, 0 as rcores, 0 as walltime,
            case when transformation like 'http%' then 'run' when transformation like '%.py' then 'py' else 'unknown' end as transform
        from {db_scheme}.jobswaiting4 jw4 where {extra_str}
        union
        select jd4.pandaid, jd4.resource_type, jd4.computingsite, jd4.prodsourcelabel, jd4.jobstatus, jd4.modificationtime, 0 as rcores, 0 as walltime,
            case when transformation like 'http%' then 'run' when transformation like '%.py' then 'py' else 'unknown' end as transform
        from {db_scheme}.jobsdefined4 jd4  where {extra_str}
        ) j
        GROUP BY j.computingsite, j.prodsourcelabel, j.transform, j.resource_type, j.jobstatus
        order by j.computingsite, j.prodsourcelabel, j.transform, j.resource_type, j.jobstatus
    """.format(
        date_from=query['modificationtime__castdate__range'][0],
        extra_str=extra_str,
        db_scheme=settings.DB_SCHEMA_PANDA,
        walltime_sec=interval_to_sec('endtime-starttime', db=db)
    )

    cur = connection.cursor()
    cur.execute(query_raw)
    job_summary_tuple = cur.fetchall()
    job_summary_header = ['computingsite', 'resourcetype', 'prodsourcelabel', 'transform', 'jobstatus', 'count', 'rcores', 'walltime']
    summary = [dict(zip(job_summary_header, row)) for row in job_summary_tuple]

    # Translate prodsourcelabel values to descriptive analy|prod job types
    summary = identify_jobtype(summary)

    return summary

def prettify_json_output(jsr_queues_dict, jsr_sites_dict, jsr_regions_dict, **kwargs):
    """
    Remove queues|regions with 0 jobs, add links to jobs page if requested
    :param jsr_queues_dict:
    :param jsr_regions_dict:
    :return:
    """
    region_summary = {}
    site_summary = {}
    queue_summary = {}

    is_add_link = False
    if 'extra' in kwargs and 'links' in kwargs['extra'] and kwargs['extra']['links'] is True:
        is_add_link = True
    if 'hours' in kwargs:
        hours = kwargs['hours']
    else:
        hours = 12

    base_url = 'https://bigpanda.cern.ch'
    jobs_path = '/jobs/?hours=' + str(hours)

    for rn, rdict in jsr_regions_dict.items():
        for jt, jtdict in rdict.items():
            for rt, rtdict in jtdict.items():
                if sum([rtdict[js] for js in const.JOB_STATES if js in rtdict]) > 0:
                    if rn not in region_summary:
                        region_summary[rn] = {}
                    if jt not in region_summary[rn]:
                        region_summary[rn][jt] = {}
                    if rt not in region_summary[rn][jt]:
                        region_summary[rn][jt][rt] = []
                    for js in const.JOB_STATES:
                        temp_dict = {
                            'jobstatus': js,
                            'count': rtdict[js]
                        }
                        if is_add_link:
                            temp_dict['job_link'] = base_url + jobs_path + '&region=' + rn + '&jobstatus=' + js
                            if jt != 'all':
                                temp_dict['job_link'] += '&jobtype=' + jt
                            if rt != 'all':
                                temp_dict['job_link'] += '&resourcetype=' + rt
                        region_summary[rn][jt][rt].append(temp_dict)

    for rn, rdict in jsr_sites_dict.items():
        for jt, jtdict in rdict.items():
            for rt, rtdict in jtdict.items():
                if sum([rtdict[js] for js in const.JOB_STATES if js in rtdict]) > 0:
                    if rn not in site_summary:
                        site_summary[rn] = {}
                    if jt not in site_summary[rn]:
                        site_summary[rn][jt] = {}
                    if rt not in site_summary[rn][jt]:
                        site_summary[rn][jt][rt] = []
                    for js in const.JOB_STATES:
                        temp_dict = {
                            'jobstatus': js,
                            'count': rtdict[js]
                        }
                        if is_add_link:
                            temp_dict['job_link'] = base_url + jobs_path + '&region=' + rn + '&jobstatus=' + js
                            if jt != 'all':
                                temp_dict['job_link'] += '&jobtype=' + jt
                            if rt != 'all':
                                temp_dict['job_link'] += '&resourcetype=' + rt
                        site_summary[rn][jt][rt].append(temp_dict)

    for qn, qdict in jsr_queues_dict.items():
        for jt, jtdict in qdict['summary'].items():
            for rt, rtdict in jtdict.items():
                if sum([rtdict[js] for js in const.JOB_STATES if js in rtdict]) > 0:
                    if qn not in queue_summary:
                        queue_summary[qn] = qdict['pq_params']
                        queue_summary[qn]['job_summary'] = {}
                    if jt not in queue_summary[qn]['job_summary']:
                        queue_summary[qn]['job_summary'][jt] = {}
                    if rt not in queue_summary[qn]['job_summary'][jt]:
                        queue_summary[qn]['job_summary'][jt][rt] = []
                    for js in const.JOB_STATES:
                        temp_dict = {
                            'jobstatus': js,
                            'count': rtdict[js]
                        }
                        if is_add_link:
                            temp_dict['job_link'] = base_url + jobs_path + '&computingsite=' + qn + '&jobstatus=' + js
                            if jt != 'all':
                                temp_dict['job_link'] += '&jobtype=' + jt
                            if rt != 'all':
                                temp_dict['job_link'] += '&resourcetype=' + rt
                        queue_summary[qn]['job_summary'][jt][rt].append(temp_dict)

    return queue_summary, site_summary, region_summary

def is_jobtype_match_queue(jobtype, pq_dict):
    """
    Check if analy/prod job can be run in a PanDA queue
    :param jobtype: str: analy or prod
    :param pq_dict: dict: PQ info
    :return: bool
    """
    pq_type = pq_dict['type'] if 'type' in pq_dict else 'unified'
    pq_jt_matrix = {
        'unified': ('prod', 'analy'),
        'production': ('prod',),
        'analysis': ('analy',),
        'special': ('prod',),
    }
    if pq_type in pq_jt_matrix and jobtype in pq_jt_matrix[pq_type]:
        return True

    return False


from abc import ABC, abstractmethod
from dataclasses import dataclass
from pathlib import Path
from time import perf_counter
from typing import Any, Callable, Mapping, Optional

import pytest
from packaging.version import Version

from check_shapes import Shape
from check_shapes import check_shape as cs
from check_shapes import check_shapes, disable_check_shapes, inherit_check_shapes
from check_shapes.config import ShapeCheckingState
from check_shapes.exceptions import ShapeMismatchError

from ..utils import assert_has_shape

try:
    import numpy as np
    import tensorflow as tf

    requires_tf = lambda f: f
except ImportError:
    from unittest.mock import MagicMock

    np = MagicMock()
    tf = MagicMock()

    requires_tf = pytest.mark.skip("TensorFlow not installed.")

@requires_tf
@pytest.mark.parametrize(
    "shaped,expected_shape",
    [
        (tf.Variable(np.zeros(())), ()),
        (tf.Variable(np.zeros((2, 4))), (2, 4)),
        # pylint: disable=unexpected-keyword-arg
        (tf.Variable(np.zeros((2, 4)), shape=[2, None]), (2, None)),
        (tf.Variable(np.zeros((2, 4)), shape=tf.TensorShape(None)), None),
    ],
)
def test_get_shape(shaped: Any, expected_shape: Shape) -> None:
    assert_has_shape(shaped, expected_shape)

@requires_tf
@pytest.mark.parametrize(
    "state,eager_expected,compiled_expected",
    [
        (ShapeCheckingState.ENABLED, True, True),
        (ShapeCheckingState.EAGER_MODE_ONLY, True, False),
        (ShapeCheckingState.DISABLED, False, False),
    ],
)
def test_shape_checking_state__bool(
    state: ShapeCheckingState, eager_expected: bool, compiled_expected: bool
) -> None:
    enabled = None

    def run() -> None:
        nonlocal enabled
        enabled = bool(state)

    run()
    assert eager_expected == enabled

    tf.function(run)()  # pylint: disable=no-member
    assert compiled_expected == enabled

@requires_tf
def test_check_shapes() -> None:
    N_FEATURES = 10
    N_TRAINING_ROWS = 1_000
    N_TEST_ROWS = 100

    N_ITERATIONS = 1_000
    LEARNING_RATE = 1e-2

    class Model(ABC):
        @abstractmethod
        @check_shapes(
            "training_features: [n_rows, n_features]",
            "training_targets: [n_rows, 1]",
        )
        def train(self, training_features: tf.Tensor, training_targets: tf.Tensor) -> None:
            ...

        @abstractmethod
        @check_shapes(
            "test_features: [n_rows, n_features]",
            "return: [n_rows, 1]",
        )
        def predict(self, test_features: tf.Tensor) -> tf.Tensor:
            ...

    class LinearModel(Model):
        def __init__(self) -> None:
            self._weights: Optional[tf.Tensor] = None  # [n_features]

        @check_shapes(
            "weights: [n_features]",
        )
        def set_weights(self, weights: tf.Tensor) -> None:
            self._weights = weights

        @inherit_check_shapes
        def train(self, training_features: tf.Tensor, training_targets: tf.Tensor) -> None:
            # We intentionally split this into a few more functions than might technically be
            # needed, so that we have something to annotate with type checks:

            @check_shapes(
                "weights: [n_features]",
                "return: []",
            )
            def loss(weights: tf.Tensor) -> tf.Tensor:
                pred = cs(self._predict(weights, training_features), "[n_rows, 1]")
                err = cs(pred - training_targets, "[n_rows, 1]")
                return tf.reduce_mean(err ** 2)

            @tf.function  # type: ignore
            @check_shapes(
                "weights: [n_features]",
                "return: [n_features]",
            )
            def step(weights: tf.Tensor) -> tf.Tensor:
                with tf.GradientTape() as g:
                    g.watch(weights)
                    l = cs(loss(weights), "[]")
                loss_grads = cs(g.gradient(l, weights), "[n_features]")
                return weights - LEARNING_RATE * loss_grads

            n_features = training_features.shape[-1]
            weights = tf.Variable(tf.zeros((n_features,)))
            for _ in range(N_ITERATIONS):
                weights.assign(step(weights))

            self._weights = tf.constant(weights)

        @inherit_check_shapes
        def predict(self, test_features: tf.Tensor) -> tf.Tensor:
            assert self._weights is not None
            return self._predict(self._weights, test_features)

        @staticmethod
        @check_shapes(
            "weights: [n_features]",
            "test_features: [n_rows, n_features]",
            "return: [n_rows, 1]",
        )
        def _predict(weights: tf.Tensor, test_features: tf.Tensor) -> tf.Tensor:
            return test_features @ weights[:, None]

    @dataclass
    class TestData:
        training_features: tf.Tensor
        training_targets: tf.Tensor
        test_features: tf.Tensor
        test_targets: tf.Tensor

        @check_shapes(
            "self.training_features: [n_training_rows, n_features]",
            "self.training_targets: [n_training_rows, 1]",
            "self.test_features: [n_test_rows, n_features]",
            "self.test_targets: [n_test_rows, 1]",
        )
        def __post_init__(self) -> None:
            pass

    def create_data() -> TestData:
        tf.random.set_seed(42)
        model = LinearModel()
        model.set_weights(tf.range(N_FEATURES, dtype=tf.float32))
        training_features = tf.random.uniform([N_TRAINING_ROWS, N_FEATURES], 0.0, 1.0)
        training_targets = model.predict(training_features) + tf.random.normal(
            [N_TRAINING_ROWS, 1], 0.0, 1.0
        )
        test_features = tf.random.uniform([N_TEST_ROWS, N_FEATURES], 0.0, 1.0)
        test_targets = model.predict(test_features)
        return TestData(training_features, training_targets, test_features, test_targets)

    def evaluate(model: Model, data: TestData) -> Mapping[str, float]:
        test_predictions = model.predict(data.test_features)
        err = data.test_targets - test_predictions
        result = {}
        result["rmse"] = tf.sqrt(tf.reduce_mean(err ** 2)).numpy()
        return result

    def main() -> None:
        before = perf_counter()
        data = create_data()
        model = LinearModel()
        model.train(data.training_features, data.training_targets)
        metrics = dict(evaluate(model, data))
        after = perf_counter()
        metrics["time (s)"] = after - before
        for key, value in metrics.items():
            print(key, value)

    main()

@requires_tf
def test_check_shapes__keras() -> None:
    # pylint: disable=arguments-differ,abstract-method,no-value-for-parameter,unexpected-keyword-arg

    @check_shapes(
        "x: [*]",
        "return: [*]",
    )
    def f(x: tf.Tensor) -> tf.Tensor:
        return x + 3

    class SuperLayer(tf.keras.layers.Layer):  # type: ignore[misc]
        def __init__(self) -> None:
            super().__init__()
            self._b = tf.Variable(0.0)

        @check_shapes(
            "x: [batch, input_dim]",
            "y: [batch, 1]",
            "return: [batch, input_dim]",
            tf_decorator=True,
        )
        def call(self, x: tf.Tensor, y: tf.Tensor) -> tf.Tensor:
            return f(x) + y + self._b

    class SubLayer(SuperLayer):
        @inherit_check_shapes
        def call(self, x: tf.Tensor, y: tf.Tensor) -> tf.Tensor:
            return x - y + self._b

    class MyModel(tf.keras.Model):  # type: ignore[misc]
        def __init__(self, join: SuperLayer) -> None:
            super().__init__()
            self._join = join

        @check_shapes(
            "xy: [batch, input_dim_plus_one]",
            "return: [batch, input_dim]",
            tf_decorator=True,
        )
        def call(self, xy: tf.Tensor) -> tf.Tensor:
            x = cs(xy[:, :-1], "[batch, input_dim]")
            y = cs(xy[:, -1:], "[batch, 1]")
            return self._join(x, y)

    x = tf.ones((32, 3))
    y = tf.zeros((32, 1))
    xy = tf.concat([x, y], axis=1)
    y_bad = tf.zeros((32, 2))
    targets = tf.ones((32, 3))

    def test_layer(join: SuperLayer) -> None:
        join(x, y)

        with pytest.raises(ShapeMismatchError):
            join(x, y_bad)

        model = MyModel(join)
        model.compile(
            optimizer=tf.keras.optimizers.SGD(learning_rate=0.25),
            loss="mean_squared_error",
        )
        model.fit(x=xy, y=targets)

    test_layer(SuperLayer())
    test_layer(SubLayer())

_Err = Callable[[tf.Tensor, tf.Tensor], tf.Tensor]
_Loss = Callable[[], tf.Tensor]

_ID_WRAPPER = lambda x: x
_TF_FUNCTION = tf.function
_SHAPED_TF_FUNCTION_ERR = tf.function(
    input_signature=[
        tf.TensorSpec(shape=[None], dtype=tf.float64),
        tf.TensorSpec(shape=[], dtype=tf.float64),
    ]
)

_SHAPED_TF_FUNCTION_LOSS = tf.function(input_signature=[])
_UNSHAPED_TF_FUNCTION_ERR = tf.function(
    input_signature=[
        tf.TensorSpec(shape=None, dtype=tf.float64),
        tf.TensorSpec(shape=None, dtype=tf.float64),
    ]
)
_UNSHAPED_TF_FUNCTION_LOSS = tf.function(input_signature=[])
_RELAXED_TF_FUNCTION = tf.function(experimental_relax_shapes=True)

_NONE_SHAPE = None
_TARGET_SHAPE = tf.TensorShape([])
_V_SHAPE = tf.TensorShape([50])
_UNKNOWN_SHAPE = tf.TensorShape(None)

@requires_tf
@pytest.mark.parametrize(
    "err_wrapper,loss_wrapper",
    [
        (_ID_WRAPPER, _ID_WRAPPER),
        (_TF_FUNCTION, _TF_FUNCTION),
        (_SHAPED_TF_FUNCTION_ERR, _SHAPED_TF_FUNCTION_LOSS),
        (_UNSHAPED_TF_FUNCTION_ERR, _UNSHAPED_TF_FUNCTION_LOSS),
        (_RELAXED_TF_FUNCTION, _RELAXED_TF_FUNCTION),
    ],
)
@pytest.mark.parametrize("target_shape", [_NONE_SHAPE, _TARGET_SHAPE, _UNKNOWN_SHAPE])
@pytest.mark.parametrize("v_shape", [_NONE_SHAPE, _V_SHAPE, _UNKNOWN_SHAPE])
def test_check_shapes_compilation(
    err_wrapper: Callable[[_Err], _Err],
    loss_wrapper: Callable[[_Loss], _Loss],
    target_shape: Optional[tf.TensorShape],
    v_shape: Optional[tf.TensorShape],
) -> None:
    # Yeah, this test seems to be pushing the limits of TensorFlow compilation (which is probably
    # good), but a bunch of this is fragile.
    tf_version = Version(tf.__version__)

    if (target_shape is _UNKNOWN_SHAPE) or (v_shape is _UNKNOWN_SHAPE):
        if (err_wrapper is _TF_FUNCTION) or (err_wrapper is _RELAXED_TF_FUNCTION):
            if Version("2.7.0") <= tf_version < Version("2.8.0"):
                pytest.skip("TensorFlow 2.7.* segfaults when trying to compile this.")
            if Version("2.8.0") <= tf_version < Version("2.9.0"):
                pytest.skip("TensorFlow 2.8.* is missing a TraceType(?) when trying compile this.")

    # See: https://github.com/tensorflow/tensorflow/issues/56414
    if err_wrapper is _RELAXED_TF_FUNCTION:
        if Version("2.9.0") <= tf_version < Version("2.11.0"):
            err_wrapper = _TF_FUNCTION

    if Version(tf.__version__) < Version("2.5.0"):
        # TensorFlow < 2.5.0 doesn't like the optional `z` argument:

        class SqErr:
            @check_shapes(
                "x: [broadcast n...]",
                "y: [broadcast n...]",
                "return: [n...]",
            )
            def __call__(self, x: tf.Tensor, y: tf.Tensor) -> tf.Tensor:
                return (x - y) ** 2

    else:

        class SqErr:  # type: ignore[no-redef]
            @check_shapes(
                "x: [broadcast n...]",
                "y: [broadcast n...]",
                "z: [broadcast n...]",
                "return: [n...]",
            )
            def __call__(
                self, x: tf.Tensor, y: tf.Tensor, z: Optional[tf.Tensor] = None
            ) -> tf.Tensor:
                # z only declared to test the case of `None` arguments.
                return (x - y) ** 2

    sq_err = err_wrapper(SqErr())

    dtype = np.float64
    target = tf.Variable(0.5, dtype=dtype, shape=target_shape)
    v = tf.Variable(np.linspace(0.0, 1.0), dtype=dtype, shape=v_shape)

    @loss_wrapper
    @check_shapes(
        "return: [1]",
    )
    def loss() -> tf.Tensor:
        # keepdims is just to add an extra dimension to make the check more interesting.
        return tf.reduce_sum(sq_err(v, target), keepdims=True)

    optimiser = tf.keras.optimizers.SGD(learning_rate=0.25)
    for _ in range(10):
        optimiser.minimize(loss, var_list=[v])

    np.testing.assert_allclose(target, v.numpy(), atol=0.01)

@requires_tf
@pytest.mark.parametrize("func_wrapper", [lambda x: x, tf.function], ids=["none", "tf.function"])
def test_check_shapes__disable__speed(func_wrapper: Callable[[Any], Any]) -> None:
    if func_wrapper is tf.function:
        pytest.skip(
            "This test is super flaky with tf.function, because the overhead of compiling"
            " seems to dominate any difference caused by check_shapes. However we probably"
            " do want some kind of test of the speed with tf.function, so we keep this"
            " skipped test around to remind us."
        )

    x = tf.zeros((3, 4, 5))
    y = tf.ones((3, 4, 5))

    def time_no_checks() -> float:
        before = perf_counter()

        def f(a: tf.Tensor, b: tf.Tensor) -> tf.Tensor:
            return a + b

        f = func_wrapper(f)
        for _ in range(10):
            f(x, y)

        after = perf_counter()
        return after - before

    def time_disabled_checks() -> float:
        with disable_check_shapes():
            before = perf_counter()

            @check_shapes(
                "a: [d...]",
                "b: [d...]",
                "return: [d...]",
            )
            def f(a: tf.Tensor, b: tf.Tensor) -> tf.Tensor:
                return a + b

            f = func_wrapper(f)
            for _ in range(10):
                f(x, y)

            after = perf_counter()
            return after - before

    def time_with_checks() -> float:
        before = perf_counter()

        @check_shapes(
            "a: [d...]",
            "b: [d...]",
            "return: [d...]",
        )
        def f(a: tf.Tensor, b: tf.Tensor) -> tf.Tensor:
            return a + b

        f = func_wrapper(f)
        for _ in range(10):
            f(x, y)

        after = perf_counter()
        return after - before

    time_no_checks()  # Warm-up.
    t_no_checks = time_no_checks()

    time_disabled_checks()  # Warm-up.
    t_disabled_checks = time_disabled_checks()

    time_with_checks()  # Warm-up.
    t_with_checks = time_with_checks()

    assert t_no_checks < t_with_checks
    assert t_disabled_checks < t_with_checks

@requires_tf
def test_issue_1864() -> None:
    @tf.function
    @check_shapes(
        "x: [*]",
        "return: [*]",
    )
    def f(x: tf.Tensor) -> tf.Tensor:
        for _ in tf.range(3):
            x = x + 1.0
        return x

    x = tf.constant(7.0)
    f(x)

@requires_tf
def test_issue_1936() -> None:
    @tf.function
    @check_shapes(
        "x: [*]",
        "return: [*]",
    )
    def f_if(x: tf.Tensor) -> tf.Tensor:
        if tf.size(x) == 0:
            return x
        else:
            return x + x

    @tf.function
    @check_shapes(
        "x: [*]",
        "return: [*]",
    )
    def f_tf_cond(x: tf.Tensor) -> tf.Tensor:
        return tf.cond(tf.size(x) == 0, lambda: x, lambda: x + x)

    x = tf.constant(7.0)
    f_tf_cond(x)
    f_if(x)

@requires_tf
@pytest.mark.parametrize("model_type", ["SuperModel", "SubModel"])
def test_tf_saved_model(model_type: str, tmp_path: Path) -> None:
    class SuperModel:
        @check_shapes(
            "x: [any...]",
            "return: [any...]",
        )
        def f(self, x: tf.Tensor) -> tf.Tensor:
            return x

    class SubModel(SuperModel):
        @inherit_check_shapes
        def f(self, x: tf.Tensor) -> tf.Tensor:
            return x + 1

    x = np.arange(5)
    model = {
        "SuperModel": SuperModel,
        "SubModel": SubModel,
    }[model_type]()
    out_module = tf.Module()
    out_module.f = tf.function(
        model.f,
        input_signature=[tf.TensorSpec(shape=[None], dtype=tf.float64)],
    )
    tf.saved_model.save(out_module, str(tmp_path))

    in_module = tf.saved_model.load(str(tmp_path))

    np.testing.assert_allclose(
        model.f(x),
        in_module.f(x),
    )


import networkx as nx
import numpy as np
import matplotlib.pyplot as plot
import csv

baci_net_filename = 'net/baci_matching_igo.txt'
igo_net_filename = 'net/IGO_selected_politic_and_military.txt'

BACI_net= nx.read_weighted_edgelist(baci_net_filename,  nodetype=int)
IGO_net= nx.read_weighted_edgelist(igo_net_filename,  nodetype=int)

baci_cc_filename = 'data/BACI/country_codes_V202102.csv'

baci_cc={}
with open(baci_cc_filename) as cc_file:
        csv_reader = csv.DictReader(cc_file)
        line_count = 0

        for row in csv_reader:
                if line_count == 0:
                        line_count += 1

                baci_cc[int(row['country_code'])] = row['country_name_full']

### bridges, hubs, joints

## bridges
baci_bridges = nx.bridges(BACI_net)
igo_bridges = nx.bridges(IGO_net)

print('\nBridges:')
print(f"BACI: {list(baci_bridges)}")
print(f"IGO: {list(igo_bridges)}")

## hubs
# hub_score, authority_score = nx.hits()
baci_h, baci_a = nx.hits(BACI_net)
igo_h, igo_a = nx.hits(IGO_net)

# sort
baci_h = {k: v for k, v in sorted(baci_h.items(), key=lambda el: el[1], reverse=True)}
igo_h = {k: v for k, v in sorted(igo_h.items(), key=lambda el: el[1], reverse=True)}

# top
baci_h_top = {k: v for k, v in list(baci_h.items())[:10]}
baci_h_bot = {k: v for k, v in list(baci_h.items())[-10:]}
igo_h_top = {k: v for k, v in list(igo_h.items())[:10]}
igo_h_bot = {k: v for k, v in list(igo_h.items())[-10:]}

print(f'\nHub score:')
print(f'\nBACI')
print(f'Top countries:')
for cc, v in baci_h_top.items():
    print(f'{baci_cc[cc]} ; {v}')
print(f'\nBottom countries:')
for cc, v in baci_h_bot.items():
    print(f'{baci_cc[cc]} ; {v}')

print(f'\nIGO')
print(f'Top countries:')
for cc, v in igo_h_top.items():
    print(f'{baci_cc[cc]} ; {v}')
print(f'\nBottom countries:')
for cc, v in igo_h_bot.items():
    print(f'{baci_cc[cc]} ; {v}')

## joints (articulation points)
baci_ap = nx.articulation_points(BACI_net)
igo_ap = nx.articulation_points(IGO_net)

baci_bc = nx.is_biconnected(BACI_net)
igo_bc = nx.is_biconnected(IGO_net)

print('\nJoints:')
print(f"BACI biconnected: {baci_bc}, joints: {list(baci_ap)}")
print(f"IGO biconnected: {igo_bc}, joints: {list(igo_ap)}")


import cv2
imagen =cv2.imread('piel.jpg')
gris = cv2.cvtColor(imagen,cv2.COLOR_BGR2GRAY)
_,th = cv2.threshold(gris,240,255,cv2.THRESH_BINARY_INV) #(img in gray, umbral in pixels(2),see the result)

cont,_ = cv2.findContours(th,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE) #function that find the contours of the objects,with three parameters
'''
cv2.drawContours(imagen,cont,-1,(0,0,255),2) #(img,contours,all of them,color,thickness)
print("CONTORNOS: ",len(cont))
cv2.imshow('coins',imagen)
cv2.imshow('th',th)
'''
font = cv2.FONT_HERSHEY_COMPLEX
i=0
for c in cont:
    M = cv2.moments(c)
    if (M["m00"]==0): M["m00"]=1
    x=int(M["m10"]/M["m00"])
    y=int(M["m01"]/M["m00"])

    mensaje = 'Num :' + str(i+1)
    cv2.putText(imagen,mensaje,(x-40,y),font,0.75,(255,0,0),2,cv2.LINE_AA)
    cv2.drawContours(imagen,[c],-1,(0,0,255),2)
    cv2.imshow('ORIGINAL',imagen)
    cv2.waitKey(0)
    i = i+1
cv2.destroyAllWindows()


from typing import TextIO

import numpy as np
import scipy
from numpy.typing import NDArray

from Python.path_stuff import *

def parser(raw_data: TextIO):
    return np.array([[v == '#' for v in line] for line in raw_data.read().splitlines()], dtype=np.uint8)

def part_a_solver(grid: NDArray[np.uint8]):
    kernel = np.ones((3, 3), dtype=np.uint8)
    kernel[1, 1] = 9
    for _ in range(100 if not testing else 4):
        grid = np.isin(scipy.ndimage.convolve(grid, kernel, mode='constant'), (3, 11, 12)).astype(np.uint8)
    return np.count_nonzero(grid)

def part_b_solver(grid: NDArray[bool]):
    kernel = np.ones((3, 3), dtype=np.uint8)
    kernel[1, 1] = 9
    for _ in range(100 if not testing else 4):
        grid = np.isin(scipy.ndimage.convolve(grid, kernel, mode='constant'), (3, 11, 12)).astype(np.uint8)
        grid[[0, 0, -1, -1], [0, -1, 0, -1]] = 1
    return np.count_nonzero(grid)

if __name__ == '__main__':
    testing = False

    with open(test_path if testing else 'input.txt', 'r') as file:
        data = parser(file)

    print(part_a_solver(data))
    print(part_b_solver(data))


# В этой задаче вам необходимо раскодировать текст, находящийся в данном текстовом файле Abracadabra__1_.txt.
# Ключ для декодирования располагается в json-файле.
# В качестве ответа нужно просто отправить получившийся текст.
# И обратите внимание, что раскодировать нужно только лишь буквы
# Все остальные символы(цифры, знаки пунктуации и т.д.) необходимо выводить как есть.

import json

# откроем файл и запишем ему в переменную data
with open('Abracadabra__1_.txt', 'r') as file:
    text = file.read()

with open('Alphabet.json', 'r') as file:
    alphabet = json.load(file)

# получим декодированный текст
decoded_text = ''
for char in text:
    if char in alphabet:
        decoded_text += alphabet[char]
    else:
        decoded_text += char

print(decoded_text)

with open('result.txt', 'w') as file:
    file.write(decoded_text)

########## Ещё вариант
# import json

# with open('Abracadabra.txt', encoding='utf-8') as abra, open('Alphabet.json', encoding='utf-8') as alph:
#     data = json.load(alph)
#     text = abra.read()
#     for i in text:
#         print(data.get(i, i), end='')

# i - это ключ, который вы передаете в функцию get. В вашем случае i - это текущий символ, который вы ищете в словаре.
# i (после запятой) - это значение, которое будет возвращено, если ключ i отсутствует в словаре data.

########## Ещё вариант
# import json

# with open("Alphabet.json") as f:
#     key = json.load(f)
# with open("Abracadabra.txt", encoding="utf-8") as f:
#     txt = f.read()
# print(txt.translate(txt.maketrans(key)))