text:
# Stacking Model

# description -> simple implementation
'''
- the central model must have be able to send requests to the local models -> array of models
- different aggregation methods (to take into account the probability distribution, provide better visualizations)
'''

# procedure
'''
1) send the data to the single models -> in this simple application -> invoque the functions through the array of models (what if a new model wants to be added? -> right now, we just want to check if the stacking procedure is working fine!!)
2) combine the responce
3) predict (aggregating somehow the prediction of the other models)

'''
### ADD CROSS-VALIDATION TO CHOOSE THE BEST AGGREGATIN METHOD

# Import all the useful libraries
from LocalModel import LocalModel
import numpy as np
import pandas as pd
import fancyimpute
from sklearn import model_selection
from sklearn.model_selection import StratifiedKFold

from sklearn.ensemble import AdaBoostClassifier  # PROBABILITY
from sklearn.tree import DecisionTreeClassifier  # PROBABILITY
from sklearn.neighbors import RadiusNeighborsClassifier
from sklearn.linear_model import RidgeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier  # PROBABILITY
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier  # PROBABILITY
from sklearn.linear_model import LogisticRegression  # PROBABILITY
from sklearn.naive_bayes import GaussianNB  # PROBABILITY
from sklearn.ensemble import ExtraTreesClassifier  # PROBABILITY
from sklearn.neighbors import KNeighborsClassifier  # PROBABILITY
from sklearn.ensemble import BaggingClassifier  # PROBABILITY

from imblearn.over_sampling import SMOTE
from imblearn.over_sampling import ADASYN
from imblearn.under_sampling import TomekLinks

# communication
from socket import AF_INET, socket, SOCK_STREAM
from threading import Thread, RLock
import json

class CentralModelCommunication(LocalModel):
    global clientRequesting  # indicate the client who is asking to perform the prediction

    global countResponses
    countResponses = 0
    lock = RLock()

    # initialize
    def __init__(self, data="none", target_name="AFclass", model_name="none", random_state=12345678,
                 imputation_strategy='mice', balance_strategy='SMOTE', local_models=[]):
        LocalModel.__init__(self, data=data, target_name=target_name, model_name=model_name, random_state=random_state,
                            imputation_strategy=imputation_strategy, balance_strategy=balance_strategy)
        self.local_models = local_models
        self.new_dataframe = {}  # the original dataframe (used for trainig) plus the predictions of the local models
        # CENTRAL TRAINING DATA
        self.trainingData = pd.DataFrame()

        self.trainDataPredicted = pd.DataFrame()  # only the X part (no target) # data (predicted) returned from the clients -> used to train the central model
        self.trainDataPredictedChunk = {}  # only the X part (no target) # return of a chunk
        self.chunkNumber = 0  # used to send multiple chunks for prediction
        self.numberOfChunks = 0
        self.splits = []  # list of prediciton splits

        # communication
        self.stackingClients = {}  # clients used to train the central model
        self.clients = {}  ## model that can predict -> gui
        self.appClients = {}  ## no prediction capabilities
        self.clientsAccuracy = {}

        self.addresses = {}
        self.counter = 0  # count the local hospital
        self.HOST = ''
        self.PORT = 33000
        self.BUFSIZ = 102400000
        self.ADDR = (self.HOST, self.PORT)
        self.SERVER = socket(AF_INET, SOCK_STREAM)
        self.SERVER.bind(self.ADDR)
        self.SERVER.listen(10)  ## queue length
        print("Waiting for connection...")
        self.ACCEPT_THREAD = Thread(target=self.accept_incoming_connections)
        self.ACCEPT_THREAD.start()
        self.ACCEPT_THREAD.join()  # TRY TO REVOVE this line and see what happen!!

    # self.SERVER.close()
    # self.lock = Lock()

    """Sets up handling for incoming clients."""

    def accept_incoming_connections(self):
        while True:
            client, client_address = self.SERVER.accept()
            print("%s:%s has connected." % client_address)
            client.sendall(bytes("Greetings from the cave! Now type your name and press enter!", "utf8"))
            self.addresses[client] = client_address
            Thread(target=self.handle_client, args=(client,)).start()

    # establish connection and get responses
    '''
    def recv_basic(self, the_socket):
        total_data=[]
        while True:
            data = the_socket.recv(8192)
            if not data:
                return ''.join(total_data)
            total_data.append(data)
            print ("total_data")
            print (total_data)
    '''

    def handle_client(self, client):  # Takes client socket as argument.
        """Handles a single client connection."""
        data = client.recv(self.BUFSIZ).decode("utf8")
        print(data)
        # welcome = 'Welcome %s! If you ever want to quit, type {quit} to exit.' % name
        client.sendall(bytes("connection OK", "utf8"))  # send(bytes(welcome, "utf8"))
        # self.lock.acquire()

        modelType = data[:3]
        modelAccuracy = data[3:]

        if modelType == "gui":
            self.clients[client] = "Local_hospital_" + str(self.counter)
            self.counter = self.counter + 1
            if len(modelAccuracy) > 0:
                self.clientsAccuracy[self.clients[client]] = float(modelAccuracy)

        if modelType == "app":
            self.appClients[client] = "app_client_" + str(len(self.appClients))

        # self.lock.release()
        print("CONNECTION ON")
        print("number of gui clients -> ", self.counter)
        print("number of stacking clients ->", self.stackingClients)
        print("number of app clients -> ", len(self.appClients))
        '''
        msg = 'receiving begin'
        text = ''
        chunk = ''
        while True:
            try:
                chunk += client.recv()
                if not chunk:
                    print ("message arrived entirely chunk")
                    # Unreliable
                    msg = text.decode("utf8")
                    try :
                        data = json.loads(msg)
                        if data[0] == "train":
                            data_received = pd.read_json(data[1])
                            print (" DATA RECEIVED _> : ", data_received)
                            #self.broadcast(data_received)
                            # perform training
                        elif data[0] == "predict":
                            data_received = pd.read_json(data[1])
                            print ("EOEOE 1")
                            #print (" DATA RECEIVED _> Predict: ", data_received)
                            #self.lock.acquire()
                            #print ("self.countResponses : " +  self.countResponses + " ---  self.counter : " + self.counter)
                            print ("EOEOE 2")
                            print ("counter -> : ", self.counter)

                            global countResponses
                            print ("countResponses -> ",countResponses)

                            #self.broadcast(data_received)
                            if 'prediction' in data_received.columns:
                                self.new_dataframe['prediction ' +self.clients[client]]= data_received['prediction']
                            print ("EOEOE 3")
                            countResponses += 1
                            if countResponses == self.counter :
                                self.predict_on_data()
                            print ("EOEOE 4")
                            print ("countResponses -> ",countResponses)
                            #print ("self.countResponses : " +  self.countResponses + " ---  self.counter : " + self.counter)
                                # perform prediction
                        elif data[0] == "requested_prediction":
                            self.new_dataframe = {}
                            global clientRequesting
                            clientRequesting = client
                            data_received = pd.read_json(data[1])
                            self.prepareDataset_prediction(data_received)
                    except:
                        print ("JSON ERROR")
                        #self.broadcast(msg)
                    text = '' # reset to zero the message
                else:
                    text += chunk
            except Exception:
                if text:
                    #print ("ERROR receiving")
                    pass
                else:
                    #print ("ERROR receiving")
                    pass

        print ("MESSAGE : ", msg)
        '''

        while True:
            try:
                msg = client.recv(self.BUFSIZ).decode("utf8")

                ## TEST
                '''
                msg = 'Something wrong in receiving'
                try:
                    text = ''
                    chunk = ''
                    while True:
                        chunk += client.recv()
                        if not chunk:
                            print ("no chunk")
                            # Unreliable
                            break
                        else:
                            text += chunk
                    msg = text.decode("utf8")
                except Exception:
                    if text:
                        pass
                    else:
                        print ("ERROR receiving")
                '''
                ## END TEST

                print("MESSAGE : ", msg)
                try:
                    global countResponses
                    data = json.loads(msg)
                    if data[0] == "train":
                        data_received = pd.read_json(data[1])
                        print(" DATA RECEIVED _> : ", data_received)
                        self.addDataFrameToTrainSet(data_received)
                        self.sendData(pd.DataFrame(), "training received", client)
                    # self.broadcast(data_received)
                    # perform training

                    elif data[0] == "features":
                        print("features received")
                        data_received = pd.read_json(data[1])
                        print(data_received)
                        self.updateFeatures(data_received)
                        self.sendData(self.important_features, "important features", client)

                    elif data[0] == "trainset predicted":
                        print("trainset predicted")
                        data_received = pd.read_json(data[1])
                        print("countResponses -> ", countResponses)
                        if 'prediction' in data_received.columns:
                            print("qwer 1")
                            self.trainDataPredictedChunk['prediction ' + self.clients[client]] = data_received[
                                'prediction']
                            print("qwer 2")
                        print("qwer 3")
                        countResponses += 1
                        print("qwer 4")
                        if countResponses == len(self.stackingClients):
                            print("qwer 5")
                            countResponses = 0
                            print("qwer 6")
                            self.train_next_chunk()
                            print("qwer 7")

                    elif data[0] == "predict":
                        data_received = pd.read_json(data[1])
                        print("EOEOE 1")
                        # print (" DATA RECEIVED _> Predict: ", data_received)
                        # self.lock.acquire()
                        # print ("self.countResponses : " +  self.countResponses + " ---  self.counter : " + self.counter)
                        print("EOEOE 2")
                        print("counter -> : ", self.counter)
                        print("countResponses -> ", countResponses)

                        # self.broadcast(data_received)

                        if 'prediction' in data_received.columns:
                            self.new_dataframe['prediction ' + self.clients[client]] = data_received['prediction']
                        print("EOEOE 3")
                        countResponses += 1
                        clientSum = len(self.clients)  # used immediately to check when to call the predict_on_data
                        print(" stackingClients len -> : ", len(self.stackingClients))
                        if len(self.stackingClients) > 0:
                            clientSum = len(self.stackingClients)
                        if countResponses == clientSum:
                            self.predict_on_data()
                        print("EOEOE 4")
                        print("countResponses -> ", countResponses)
                    # print ("self.countResponses : " +  self.countResponses + " ---  self.counter : " + self.counter)
                    # perform prediction
                    elif data[0] == "requested_prediction":
                        self.new_dataframe = {}
                        global clientRequesting
                        clientRequesting = client
                        data_received = pd.read_json(data[1])
                        self.prepareDataset_prediction(data_received)

                    elif data[0] == "trigger test":
                        self.train()
                except:
                    print("JSON ERROR")
                # self.broadcast(msg)
                # self.broadcast(msg)
            except OSError:  # Possibly client has left the chat.
                break

    '''
    def broadcast(self, msg):
        """Broadcasts a message to all the clients."""

        for sock in self.clients:
            sock.sendall(bytes(msg, "utf8"))  #(bytes(prefix, "utf8")+msg)
            #sock.send(msg)
    '''

    def broadcast(self, data, label):
        print("BROEADAST 1")
        to_send = data.to_json()
        print("BROEADAST 2")
        message = json.dumps((label, to_send))
        print("BROEADAST 3")
        clients = self.clients
        print("BROEADAST 4")
        print("stacking clients len -> ", len(self.stackingClients))
        if len(self.stackingClients) > 0:
            clients = self.stackingClients
        for sock in clients:
            sock.sendall(bytes(message, "utf8"))
        print("sent to all -> BROEADAST 5")

    def sendData(self, data, label, client):
        to_send = data.to_json()
        message = json.dumps((label, to_send))
        client.sendall(bytes(message, "utf8"))
        print("message sent")

    # add a new local model to the set of local models
    def addLocalModel(self, localModel):
        if type(localModel) is list:
            self.local_models = self.local_models + localModel
        else:
            self.local_models.append(localModel)

    def prepareDataset_prediction(self, train_x):
        # send data to all the models
        print("PREDICTION REQUEST -> PREPARING DATA")
        '''
        to_send = train_x.to_json()
        message = json.dumps(("predict" , to_send))
        self.broadcast(message)
        '''
        self.broadcast(train_x, "predict")
        print("PREDICTION REQUEST -> PREPARING DATA")

    def prepareDataset(self, train_x):

        # send data to all the models
        '''
        to_send = train_x.to_json()
        message = json.dumps(("predict" , to_send))
        self.broadcast(message)
        '''
        self.broadcast(train_x, "predict")

        '''
        local_predictions = {}
        for i in self.local_models :
            train_x_copy = train_x.copy()
            name = i.selected_model_name
            print ("NAME ", name )
            prediction = i.predict(train_x_copy)

            if 'prediction' in prediction.columns:
                local_predictions['prediction ' + name] = prediction['prediction']

            if 'predict_proba_zero' in prediction.columns:
                local_predictions["predict_proba_zero " + name] = prediction['predict_proba_zero']
            if 'predict_proba_uno' in prediction.columns:
                local_predictions["predict_proba_uno " + name] = prediction['predict_proba_uno']

            if 'predict_log_proba_zero' in prediction.columns:
                local_predictions["predict_log_proba_zero " + name] = prediction['predict_log_proba_zero']
            if 'predict_log_proba_uno' in prediction.columns:
                local_predictions["predict_log_proba_uno " + name] = prediction['predict_log_proba_uno']

            ## add other features (or remove them) as necessary
            ## also need to update the local model.predict function , if somehing is modified here
            ## ...
            # 2) combine the prediction in a dataframe

        local_predictions = pd.DataFrame(local_predictions)

        train_x = train_x.reset_index() ## reset the index -> begin from zero -> in this case we can cobine the two datasets (train_x and local_predictions)
        train_x = train_x.drop('index', axis = 1)
        train_dataframe_x =  pd.concat([train_x, local_predictions], axis=1)
        train_dataframe_x = train_dataframe_x.replace(-float('Inf'), -9999999) # cannot deal with -inf
        return train_dataframe_x

        return local_predictions
        '''

    def addDataFrameToTrainSet(self, data):
        # data must be a dataframe -> with the "same" features name
        print("prepare to add DATA")
        try:
            # self.trainingData["h"] = data
            self.trainingData = pd.concat([self.trainingData, data])
            self.trainingData.sort_index(inplace=True)
        except:
            print("CANNOT ADD DATA")
        print("heieie")
        print("current CENTRAL TRAIN DATASET -> dimension : ", self.trainingData.shape)

    # train the central model

    def trainModel(self):
        print("begin model training")
        self.selected_model.fit(self.trainDataPredicted, self.target)
        print("end training")

    def train_next_chunk(self):
        print("send data to be trained")
        # update trainset
        print("wtf 1")
        self.trainDataPredicted = pd.concat([self.trainDataPredicted, pd.DataFrame(
            self.trainDataPredictedChunk)])  # data (predicted) returned from the clients -> used to train the central model
        print("wtf 2")
        self.trainDataPredictedChunk = {}  # reset to get the new data
        print("wtf 3")
        if self.chunkNumber < self.numberOfChunks:
            print("wtf 4")
            # train next chunk
            self.broadcast(self.splits[self.chunkNumber], "train")
            print("wtf 5")
            self.chunkNumber += 1
            print("wtf 6")
        else:
            print("wtf 7")
            self.splits = []
            print("wtf 8")
            self.numberOfChunks = 0
            print("wtf 9")
            self.chunkNumber = 0
            print("wtf 10")
        print("wtf 11")
        self.trainModel()

    def train(self):
        # reset
        print("begin training function 1")
        global countResponses
        countResponses = 0
        self.trainingData = self.trainingData.reset_index().drop('index', axis=1)
        self.trainDataPredicted = pd.DataFrame()  # data (predicted) returned from the clients -> used to train the central model
        self.trainDataPredictedChunk = {}  # return of a chunk
        self.chunkNumber = 0  # used to send multiple chunks for prediction
        self.numberOfChunks = 0
        self.splits = []  # list of prediciton splits
        # split the dataset
        self.stackingClients = self.clients.copy()
        print("self.stackingClients -> len : ", len(self.stackingClients))
        print("stop 1")
        train_x = self.trainingData[self.trainingData.columns[self.trainingData.columns != self.target_name]]
        print("stop 2")
        self.target = self.trainingData[self.target_name]
        print("stop 3")
        # split train_x into chunks to be sent for prediction
        self.splits = self.splitDataframe(train_x)
        print("stop 4")
        self.numberOfChunks = len(self.splits)
        print("begin training function 2")
        print("train shape -> : ", self.trainingData.shape)
        print(self.splits)
        print("self.chunkNumber -> ", self.chunkNumber)
        # send the first chuck to be predicted by the client models
        self.broadcast(self.splits[self.chunkNumber], "train")
        self.chunkNumber += 1
        # UPDATE CLIENTS TRAINED LIST -> to corret prediction phase
        print("UPDATEe CLIENTs LIST")

    '''
    def train(self):
        print ("BEGIN CENTRAL TRAINING")
        # 0) clean and balance the training dataset -> done in the init phase
        train_x = self.data[self.features] ## data to send to the local models
        ## target
        train_y = self.trainingData[self.target_name]

        # 1) send the data to the local models and wait for their predictions
        # if there is at least one local model
        if self.local_models != []:
            train_dataframe_x = self.prepareDataset(train_x)
            # 3) train the central model on the new dataframe

            train_y = train_y.reset_index().drop('index', axis = 1)

            self.cv_x = train_dataframe_x
            self.cv_y = train_y
            # crossValidate
            self.chooseModel_with_crossValidation()

            self.selected_model.fit(train_dataframe_x,train_y)
            #self.features = train_dataframe_x.columns ## update the number of features to mech the new dataset
            self.new_dataframe = pd.concat([train_dataframe_x, train_y], axis=1)
            print ("END CENTRAL TRAINING")
            return
        else :
            super(CentralModel, self).train()
    '''

    # predict once all the responces from the local models have been collected
    def predict_on_data(self):
        print("PREDICTION predict_on_data")
        print("counter : ", self.counter)
        '''
        self.lock.acquire()
        self.countResponses = self.countResponses + 1
        print ("self.countResponses : " +  self.countResponses + " ---  self.counter : " + self.counter)
        self.lock.release()
        '''
        self.new_dataframe = pd.DataFrame(self.new_dataframe)
        print(self.new_dataframe)
        global countResponses
        countResponses = 0
        self.predict(self.new_dataframe)

    def computeWeightedVoting(self, row):
        try:
            print("local hospital -> accuracy AAA ", self.clientsAccuracy)
            model_temp = "Local_hospital_"
            result = 0
            print("local hospital -> accuracy BBB", self.clientsAccuracy)
            accuracySum = sum(self.clientsAccuracy.values)
            for i in [0, self.counter]:
                m = model_temp + str(i)
                result = result + row["prediction " + m] * self.clientsAccuracy[m]
            print("local hospital -> accuracy CCC", self.clientsAccuracy)
            return (round(result / accuracySum, 0))
        except:
            print("local hospital -> accuracy DDD", self.clientsAccuracy)
            for i in [0, self.counter]:
                m = model_temp + str(i)
                result = result + row["prediction " + m]
            print("local hospital -> accuracy EEE", self.clientsAccuracy)
            return (round(result / self.counter, 0))

    def predict(self, test):
        t = test
        if len(self.stackingClients) <= 1:
            print("majority voting")
            # no training data -> no central model -> perform majority voting
            try:
                # t['finalPrediction'] = t[t.columns].mode(axis=1)
                t['finalPrediction'] = t.apply(self.computeWeightedVoting, axis=1)
                print("pred -> majority voting SUCCEED")
            except:
                try:
                    t['finalPrediction'] = t[t.columns].mode(axis=1)
                    print("pred -> majority voting SUCCEED -> MODE ")
                except:
                    print("pred -> majority voting FAILED")
                    t['finalPrediction'] = t[t.columns].min(axis=1)  # if everything fails -> class 0 ;)
                    print("pred -> min SUCCEED")

        else:
            # sofisticated aggregation
            print("sofisticated aggregation")
            print("self.stackingClients -> ", len(self.stackingClients))
            print("traing data columns -> ", self.trainDataPredicted.shape[1])
            try:
                print("sofisticated aggregation begin")
                prediction = self.selected_model.predict(t)
                t['finalPrediction'] = prediction
                print("FINAL PREDICTUION as")
                print(t['finalPrediction'])
                t['finalPrediction'] = t['finalPrediction'].map({'persistierend (>7 Tage, EKV)': 1, 'paroxysmal': 0})
                print("sofisticated aggregation SUCCEED")
            except:
                print("sofisticated aggregation FAILED")
                print("back to majority voting")
                try:
                    t['finalPrediction'] = t[t.columns].mode(axis=1)
                    print("pred -> majority voting SUCCEED")
                except:
                    print("pred -> majority voting FAILED")
                    t['finalPrediction'] = t[t.columns].min(axis=1)  # if everything fails -> class 0 ;)
                    print("pred -> min SUCCEED")
        print(t)
        print("t printed -> send results")
        to_send = t.to_json()
        message = json.dumps(("result", to_send))
        global clientRequesting
        clientRequesting.sendall(bytes(message, "utf8"))

    # return t["finalPrediction to who has asked"]

    '''
    def predict (self, test):
        print ("begin prediction")
        original_x = test
        train = test.copy()
        x_test = self.cleanDataTest(test_x = train)
        x_test = self.recoverMissing(data = x_test)
        result = x_test.copy()
        if self.local_models != []:
            test_dataframe_x = self.prepareDataset(x_test)
            prediction = self.selected_model.predict(test_dataframe_x)
            result['prediction'] = prediction
            return pd.DataFrame(result)
        else :
            prediction = self.selected_model.predict(x_test)
            result['prediction'] = prediction
            return pd.DataFrame(result)
        print ("end prediction")
    '''


text:
"""
 Copyright (c) 2022 Intel Corporation

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""
import math
import numpy as np

from .types import NumericalValue
from .detection_model import DetectionModel
from .utils import Detection, softmax, nms, clip_detections

class NanoDet(DetectionModel):
    __model__ = 'NanoDet'

    def __init__(self, model_adapter, configuration=None, preload=False):
        super().__init__(model_adapter, configuration, preload)
        self._check_io_number(1, 1)
        self.output_blob_name = self._get_outputs()
        self.reg_max = 7
        self.strides = [8, 16, 32]
        self.ad = 0.5

    def _get_outputs(self):
        output_blob_name = next(iter(self.outputs))
        output_size = self.outputs[output_blob_name].shape
        if len(output_size) != 3:
            self.raise_error("Unexpected output blob shape {}. Only 3D output blob is supported".format(output_size))

        return output_blob_name

    @classmethod
    def parameters(cls):
        parameters = super().parameters()
        parameters['resize_type'].update_default_value('fit_to_window')
        parameters['confidence_threshold'].update_default_value(0.5)
        parameters.update({
            'iou_threshold': NumericalValue(default_value=0.6, description="Threshold for NMS filtering"),
            'num_classes': NumericalValue(default_value=80, value_type=int, description="Number of classes")
        })
        return parameters

    def postprocess(self, outputs, meta):
        detections = self._parse_outputs(outputs, meta)
        detections = self.rescale_detections(detections, meta)
        return detections

    def _parse_outputs(self, outputs, meta):
        output = outputs[self.output_blob_name][0]

        cls_scores = output[:, :self.num_classes]
        bbox_preds = output[:, self.num_classes:]
        input_height, input_width = meta['padded_shape'][:2] if meta.get('padded_shape') else meta['resized_shape'][:2]

        bboxes = self.get_bboxes(bbox_preds, input_height, input_width)
        dets = []
        for label, score in enumerate(np.transpose(cls_scores)):
            mask = score > self.confidence_threshold
            filtered_boxes, score = bboxes[mask, :], score[mask]
            if score.size == 0:
                continue
            x_mins, y_mins, x_maxs, y_maxs = filtered_boxes.T
            keep = nms(x_mins, y_mins, x_maxs, y_maxs, score, self.iou_threshold, include_boundaries=True)
            score = score[keep]
            x_mins, y_mins, x_maxs, y_maxs = x_mins[keep], y_mins[keep], x_maxs[keep], y_maxs[keep]
            labels = np.full_like(score, label, dtype=int)
            dets += [Detection(*det) for det in zip(x_mins, y_mins, x_maxs, y_maxs, score, labels)]
        return dets

    @staticmethod
    def distance2bbox(points, distance, max_shape):
        x1 = np.expand_dims(points[:, 0] - distance[:, 0], -1).clip(0, max_shape[1])
        y1 = np.expand_dims(points[:, 1] - distance[:, 1], -1).clip(0, max_shape[0])
        x2 = np.expand_dims(points[:, 0] + distance[:, 2], -1).clip(0, max_shape[1])
        y2 = np.expand_dims(points[:, 1] + distance[:, 3], -1).clip(0, max_shape[0])
        return np.concatenate((x1, y1, x2, y2), axis=-1)

    def get_single_level_center_point(self, featmap_size, stride):
        h, w = featmap_size
        x_range, y_range = (np.arange(w) + self.ad) * stride, (np.arange(h) + self.ad) * stride
        y, x = np.meshgrid(y_range, x_range, indexing='ij')
        return y.flatten(), x.flatten()

    def get_bboxes(self, reg_preds, input_height, input_width):
        featmap_sizes = [(math.ceil(input_height / stride), math.ceil(input_width) / stride) for stride in self.strides]
        list_center_priors = []
        for stride, featmap_size in zip(self.strides, featmap_sizes):
            y, x = self.get_single_level_center_point(featmap_size, stride)
            strides = np.full_like(x, stride)
            list_center_priors.append(np.stack([x, y, strides, strides], axis=-1))
        center_priors = np.concatenate(list_center_priors, axis=0)
        dist_project = np.linspace(0, self.reg_max, self.reg_max + 1)
        x = np.dot(softmax(np.reshape(reg_preds, (*reg_preds.shape[:-1], 4, self.reg_max + 1)), -1, True), dist_project)
        dis_preds = x * np.expand_dims(center_priors[:, 2], -1)
        return self.distance2bbox(center_priors[:, :2], dis_preds, (input_height, input_width))

    @staticmethod
    def rescale_detections(detections, meta):
        input_h, input_w, _ = meta['resized_shape']
        orig_h, orig_w, _ = meta['original_shape']
        w = orig_w / input_w
        h = orig_h / input_h

        for detection in detections:
            detection.xmin *= w
            detection.xmax *= w
            detection.ymin *= h
            detection.ymax *= h

        return clip_detections(detections, meta['original_shape'])

class NanoDetPlus(NanoDet):
    __model__ = 'NanoDet-Plus'

    def __init__(self, model_adapter, configuration=None, preload=False):
        super().__init__(model_adapter, configuration, preload)
        self.ad = 0
        self.strides = [8, 16, 32, 64]

ext: py

reason: import numpy


text:
from __future__ import print_function
import numpy as np
import matplotlib.pyplot as plt
import h5py
from plotstuff import params, colours
reb = params()
cols = colours()
from gatspy.periodic import LombScargle
import sys
import multiprocessing as mp
from multiprocessing import Pool
from GProtation import make_plot, lnprob, Gprob, neglnlike
import emcee
import time
import george
from george.kernels import ExpSquaredKernel, ExpSine2Kernel
import scipy.optimize as spo

def periodograms(id, x, y, yerr, path, plot=False, savepgram=False):
    """
    takes id of the star, returns an array of period measurements and saves the
    results.
    id: star id.
    x, y, yerr: time, flux and error arrays.
    path: path where you want to save the output.
    """
    ps = np.linspace(2, 100, 1000)
    model = LombScargle().fit(x, y, yerr)
    pgram = model.periodogram(ps)

    # find peaks
    peaks = np.array([i for i in range(1, len(ps)-1) if pgram[i-1] <
                     pgram[i] and pgram[i+1] < pgram[i]])
    if len(peaks):
        period = ps[pgram==max(pgram[peaks])][0]
    else: period = 0

    if plot:
        plt.clf()
        plt.plot(ps, pgram)
        plt.axvline(period, color="r")
        plt.savefig("{0}/{1}_pgram".format(path, str(int(id)).zfill(4)))

    if savepgram:
        np.savetxt("{0}/{1}_pgram.txt".format(path, str(int(id)).zfill(4)),
                   np.transpose((ps, pgram)))

    np.savetxt("{0}/{1}_pgram_result.txt".format(path, str(int(id)).zfill(4)),
               np.ones(2).T*period)
    return period

def recover_injections(id, x, y, yerr, path, burnin, run, nwalkers=32,
                       plot=True):
    """
    Take x, y, yerr, calculate ACF period for initialisation and do MCMC.
    npts: number of points per period.
    id: star id.
    x, y, yerr: time, flux and error arrays.
    path: path where you want to save the output.
    burnin: the number of burnin steps.
    run: the number of steps to run for.
    nwalkers: the number of walkers.
    plot: if True then plots of posteriors and chains will be made.
    """

    # initialise with pgram
    try:
        p_init = np.genfromtxt("{0}/{1}_pgramresult.txt".format(path, id))
    except:
        p_init = periodograms(id, x, y, yerr, path, plot=True)

    if p_init < .5:  # small periods raise an error with george.
            p_init = 1.

    # If using lnprob, plims = [pmin, pmax] for a uniform prior.
    # If using Gprob, plims = [mu, sigma] for a Gaussian prior.
#     plims = np.log([p_init - .5 * p_init, p_init + 2 * p_init])
    plims = np.log([p_init - .5 * p_init, p_init + 2 * p_init])
#     plims = np.log([p_init, p_init*.1])  # mean, sigma

    print("Initial period and limits:", p_init, np.exp(plims))

    # assign theta_init
    theta_init = np.log([np.exp(-5), np.exp(7), np.exp(.6), np.exp(-16),
                         p_init])
    print("\n", "log(theta_init) = ", theta_init)
    print("theta_init = ", np.exp(theta_init), "\n")

    # plot initialisation
    t = np.exp(theta_init)
    k = t[0] * ExpSquaredKernel(t[1]) * ExpSine2Kernel(t[2], t[3])
    gp = george.GP(k)
    gp.compute(x, yerr)
    xs = np.linspace(x[0], x[-1], 1000)
    mu, cov = gp.predict(y, xs)
    plt.clf()
    plt.errorbar(x, y, yerr=yerr, **reb)
    plt.plot(xs, mu, color=cols.blue)
    plt.savefig("{0}/{1}_init".format(path, id))

    # set up MCMC
    ndim, nwalkers = len(theta_init), nwalkers
    p0 = [theta_init+1e-4*np.random.rand(ndim) for i in range(nwalkers)]
    args = (x, y, yerr, plims)

    # time the lhf call
    start = time.time()
    print("lnprob = ", lnprob(theta_init, x, y, yerr, plims))
#     print("lnprob = ", Gprob(theta_init, x, y, yerr, plims))
    end = time.time()
    tm = end - start
    print("1 lhf call takes ", tm, "seconds")
    print("burn in will take", tm * nwalkers * burnin, "s")
    print("run will take", tm * nwalkers * run, "s")
    print("total = ", (tm*nwalkers*run + tm*nwalkers*burnin)/60, \
          "mins,", (tm*nwalkers*run + tm*nwalkers*burnin)/3600, "hours")

    # run MCMC
    sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, args=args)
#     sampler = emcee.EnsembleSampler(nwalkers, ndim, Gprob, args=args)
    print("burning in...")
    start = time.time()
    p0, lp, state = sampler.run_mcmc(p0, burnin)
    sampler.reset()
    print("production run...")
    p0, lp, state = sampler.run_mcmc(p0, run)
    end = time.time()
    print("actual time = ", (end - start)/60, "mins")

    # save samples
    f = h5py.File("%s/%s_samples.h5" % (path, id), "w")
    data = f.create_dataset("samples", np.shape(sampler.chain))
    data[:, :] = np.array(sampler.chain)
    f.close()

    # make various plots
    if plot:
        with h5py.File("%s/%s_samples.h5" % (path, id), "r") as f:
            samples = f["samples"][...]
        mcmc_result = make_plot(samples, x, y, yerr, id, path, traces=True,
                                tri=True, prediction=True)

def acf_pgram_GP_LSST(id):
    """
    Run acf, pgram and MCMC recovery on Suzanne's simulations
    """
    id = str(int(id)).zfill(4)
    path = "results"  # where to save results
    x, y, yerr = np.genfromtxt("simulations/{0}.txt".format(id)).T
    yerr = np.ones_like(x) * .001

    periodograms(id, x, y, yerr, path, plot=True)  # pgram
    burnin, run = 500, 1000
    recover_injections(id, x, y, yerr, path, burnin, run, nwalkers=24,
                       plot=True)

if __name__ == "__main__":

#     acf_pgram_GP_LSST(0)

    ids = range(10)
    pool = Pool()
    pool.map(acf_pgram_GP_LSST, ids)

max_stars_repo_name: RuthAngus/LSST-max

max_issues_repo_name: RuthAngus/LSST-max

max_forks_repo_name: RuthAngus/LSST-max

reason: import numpy,import scipy


text:
from nose.tools import assert_equal, assert_true
from nose.tools import assert_false, assert_almost_equal
from Scheduling import DependencyScheduler
from collections import defaultdict
import networkx as nx # Library for displaying graphs.
import matplotlib.pyplot as plt

s = DependencyScheduler()
s.add_task('a', ['b', 'c'])
s.add_task('b', ['c', 'e'])
s._check()
s.show()

# Simple tests
s = DependencyScheduler()
s.add_task('a', [])
assert_equal(s.available_tasks, {'a'})

# Slightly more complicated

s = DependencyScheduler()
s.add_task('a', ['b', 'c'])
s.add_task('b', ['c', 'e'])
assert_equal(s.available_tasks, {'e', 'c'})

s = DependencyScheduler()
s.add_task('a', ['b'])
s.add_task('b', ['a'])
assert_equal(s.available_tasks, set())

# Completed tasks

s = DependencyScheduler()
s.add_task('a', [])
assert_equal(s.available_tasks, {'a'})
r = s.mark_completed('a')
assert_equal(r, set())

s = DependencyScheduler()
s.add_task('a', ['b'])
assert_equal(s.available_tasks, {'b'})
r = s.mark_completed('b')
assert_equal(r, {'a'})

# Complicated tasks
s = DependencyScheduler()
s.add_task('a', ['b', 'c'])
assert_equal(s.available_tasks, {'b', 'c'})
r = s.mark_completed('b')
assert_equal(r, set())
assert_equal(s.available_tasks, {'c'})
r = s.mark_completed('c')
assert_equal(r, {'a'})

s = DependencyScheduler()
s.add_task('a', ['b', 'c'])
s.add_task('b', ['c', 'e'])
s.add_task('c', [])
assert_equal(s.available_tasks, {'c', 'e'})
r = s.mark_completed('e')
assert_equal(r, set())
r = s.mark_completed('c')
assert_equal(r, {'b'})
r = s.mark_completed('b')
assert_equal(r, {'a'})
r = s.mark_completed('a')
assert_equal(r, set())
assert_equal(s.available_tasks, set())

#Redo tests
s = DependencyScheduler()
s.add_task('a', [])
s.add_task('b', ['a'])
s.add_task('c', ['a'])
s.add_task('d', ['b', 'c'])
s.add_task('e', ['a', 'd'])

s.mark_completed('a')
s.mark_completed('b')
s.mark_completed('c')
assert_equal(s.available_tasks, {'d'})
s.redo('b')
assert_equal(s.available_tasks, {'b'})

#Cooking Redo Tests
s = DependencyScheduler()
s.add_task('a', [])
s.add_task('b', [])
s.add_task('c', ['a', 'b'])
s.add_task('d', ['c', 'a'])
s.add_task('e', [])
s.add_task('f', ['e'])
s.add_task('g', ['f', 'd'])

s.mark_completed('a')
s.mark_completed('b')
s.mark_completed('c')
s.mark_completed('d')
assert_equal(s.available_tasks, {'e'})
s.cooking_redo('c')
# When we redo c, both its successor d, and predecessors a, b have to be redone.
assert_equal(s.available_tasks, {'a', 'b', 'e'})
assert_equal(s.completed_tasks, set())

#Complicated cooking  redo test
s = DependencyScheduler()
s.add_task('a', [])
s.add_task('b', [])
s.add_task('c', ['a', 'b'])
s.add_task('d', ['c', 'a'])
s.add_task('e', [])
s.add_task('f', ['e'])
s.add_task('g', ['f', 'd'])

s.mark_completed('a')
s.mark_completed('b')
s.mark_completed('c')
s.mark_completed('d')
s.mark_completed('e')
assert_equal(s.available_tasks, {'f'})
s.cooking_redo('c')
# When we redo c, both its successor d, and predecessors a, b have to be redone.
assert_equal(s.available_tasks, {'a', 'b', 'f'})
assert_equal(s.completed_tasks, {'e'})

reason: import networkx


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/3/27 9:56
# @Author  : Jackokie Zhao
# @Site    : www.jackokie.com
# @File    : file_5_8.py
# @Software: PyCharm
# @contact: jackokie@gmail.com

# ---------------------------------------------------------------------
# Kernel Number Comparing.
# ---------------------------------------------------------------------

import pickle
import numpy as np

import scipy.interpolate as inp
import matplotlib.pyplot as plt
from matplotlib import cm

from mpl_toolkits.mplot3d import Axes3D

def accurs_show_3d(num_layer_1, num_layer_2, accurs, cmap, title, path):
    """ Show the accuracy.
    num_layer_1: The coordinate of the first axes.
    num_layer_2: The coordinate of the second axes.
    accurs: The accuracy of different
    """
    fig = plt.figure(dpi=120, figsize=[8, 7])
    ax = Axes3D(fig)

    fit = inp.interp2d(num_layer_1, num_layer_2, accurs)

    x_n = np.linspace(min(num_layer_1), max(num_layer_1), 10240)
    y_n = np.linspace(min(num_layer_2), max(num_layer_2), 10240)

    accurs_n = fit(x_n, y_n)
    surf = ax.plot_surface(y_n, x_n, accurs_n, cmap=cmap)
    ax.set_xlabel('Second Layer Number')
    ax.set_ylabel('First Layer Number')
    ax.set_zlabel('accuracy')
    fig.colorbar(surf, shrink=0.5, aspect=5)
    # plt.title(title)
    fig.savefig(path)

def kernel_num_hot_map(layers, accuracys, colormap=None, title=None, path=None):
    """ Plot the hot map of data.

    """
    fig = plt.figure(dpi=120, figsize=(8, 7))
    y_n = np.linspace(min(layers), max(layers), 1001*len(layers))
    x_n = np.linspace(min(layers), max(layers), 1001*len(layers))
    fit = inp.interp2d(layers, layers, accuracys)
    accurs_n = fit(x_n, y_n)
    surf = plt.imshow(accurs_n, cmap=colormap)
    fig.colorbar(surf, shrink=0.5, aspect=5)
    plt.xticks([])
    plt.yticks([])
    plt.ylabel('                                        First Layer Kernel Number                    '
               '                         \n\n'
               '256             196              128               96               64               48'
               '              32              16')

    plt.xlabel('16             32              48               64               96               128'
               '              196              256\n\n'
               '                        Second Layer Kernel Number                              ')
    plt.tight_layout()
    plt.savefig(path)

def accurs_show_2d(num_layer_1, num_layer_2, accurs, y_wide, axis_x, linecolor, title=None, mean_show=False, path=None):
    """ Show the accuracy.
    num_layer_1: The coordinate of the first axes.
    num_layer_2: The coordinate of the second axes.
    accurs: The accuracy of different
    """

    fig = plt.figure(dpi=120, figsize=[8, 6])
    p_1_labels = []
    p_1 = []
    if axis_x == 0:
        for i in range(len(num_layer_1)):
            p_1_labels.append('first_layer_num--%d' % num_layer_1[i])

        for i in range(len(num_layer_1)):
            p_1.append(plt.plot(num_layer_1, accurs[i], '--',
                                color=linecolor[i],
                                label=p_1_labels[i]))
        if mean_show:
            means = np.mean(accurs, axis=axis_x)
            plt.plot(num_layer_1, means, '-o', linewidth=2, label='mean_accuray')
        plt.ylim(y_wide)
        plt.xticks(num_layer_1)
        plt.xlabel('second_layer_number')
        plt.ylabel('accuracy')
    else:
        for i in range(len(num_layer_2)):
            p_1_labels.append('second_layer_num--%d' % num_layer_2[i])

        for i in range(len(num_layer_2)):
            p_1.append(plt.plot(num_layer_1, accurs[:, i], '--',
                                color=linecolor[i],
                                label=p_1_labels[i]))
        if mean_show:
            means = np.mean(accurs, axis=axis_x)
            plt.plot(num_layer_2, means, '-ro', linewidth=2, label='mean_accuray')
        plt.ylim(y_wide)
        plt.xticks(num_layer_2)
        plt.xlabel('first_layer_number')
        plt.ylabel('accuracy')

    plt.legend()
    if not title is None:
        plt.title(title)
    plt.tight_layout()
    fig.savefig(path)

def number_compare(kernel_numbers):
    """ Plot the kernel numbers comparing...
    kernel_numbers: the data of kernel numbers.
    """
    x = y = [16, 32, 48, 64, 96, 128, 196, 256]
    linecolor = ['gold', 'red', 'springgreen', 'black', 'blue', 'cyan', 'blueviolet', 'magenta']

    accurs = np.ones([8, 8], dtype=float)
    for m in range(8):
        for n in range(8):
            accurs[m][n] = kernel_numbers[(x[m], y[n])]

    accurs_show_3d(x, y, accurs, plt.cm.rainbow, '不同数目卷积核的分类准确率',
                   'E:/JackokiePapers/figures/chapter_5/fig_5_9.png')

    kernel_num_hot_map(x, accurs, colormap=cm.rainbow,
                  path='E:/JackokiePapers/figures/chapter_5/fig_5_10.png')

    accurs_show_2d(x, y, accurs, [0, 73],
                   axis_x=1,
                   linecolor=linecolor,
                   mean_show=False,
                   path='E:/JackokiePapers/figures/chapter_5/fig_5_11.png')

    accurs_show_2d(x, y, accurs, [66, 72],
                   axis_x=1,
                   linecolor=linecolor,
                   mean_show=True,
                   path='E:/JackokiePapers/figures/chapter_5/fig_5_12.png')

    accurs_show_2d(x, y, accurs, [0, 73],
                   axis_x=0,
                   linecolor=linecolor,
                   mean_show=False,
                   path='E:/JackokiePapers/figures/chapter_5/fig_5_13.png')

    accurs_show_2d(x, y, accurs, [66, 72],
                   axis_x=0,
                   linecolor=linecolor,
                   mean_show=True,
                   path='E:/JackokiePapers/figures/chapter_5/fig_5_14.png')

if __name__ == '__main__':

    kernel_numbers = pickle.load(open('accuracy_kernels_num.pkl', 'rb'))
    # print(kernel_numbers)
    number_compare(kernel_numbers)

    print('The program has finished running...')