import sys
import os
current = os.path.dirname(os.path.realpath(__file__))
parent = os.path.dirname(current)

sys.path.append(parent)

import numpy as np
import matplotlib.pyplot as plt
from autolab_core import Logger, BinaryClassificationResult, YamlConfig
import argparse

from gqcnn.model import get_gqcnn_model
from gqcnn.utils import read_pose_data
from tools.path_settings import EXPER_PATH

"""
Analyse a vgq-cnn network. Will output an analysis over the camera poses by default. Use the flags
to produce extra analysis over psi, beta or objects.
"""

BACKGROUND = True
DEBUG = False

if not BACKGROUND:
    line_color = 'w'
    hist_color = (0.616, 0.773, 0.730)
    transparent = True
else:
    line_color = 'k'
    hist_color = (0.2, 0.6, 1)
    transparent = False

# Matplotlib style setting
plt.rc('axes', edgecolor=line_color)
plt.rc('text', color=line_color)
plt.rc(('xtick', 'ytick'), c=line_color)
plt.rc('font', size=14)
plt.rc('axes', titlesize=14)
plt.rc('axes', labelsize=14)
plt.rc('xtick', labelsize=12)
plt.rc('ytick', labelsize=12)
plt.rc('legend', fontsize=14)
plt.rc('figure', titlesize=20)

class VgqcnnAnalyse:
    def __init__(self, camera_analysis, beta_analysis, psi_analysis, object_analysis, verbose=True):
        """
        Initiates the analysis of a VGQ-CNN model.

        Parameters
        ----------
        data_dir (str): Path to dataset on which the model should be evaluated.
        verbose (bool): Whether or not to log initialization output to `stdout`.
        """
        self.metric_thresh = 0.002  # Change metric threshold here if needed!
        self.analyse_checkpoints = False
        self.verbose = verbose
        self.files = None
        self.logger = None
        self.pose_input = None
        self.gripper_mode = None
        self.psi_analysis = psi_analysis
        self.phi_analysis = camera_analysis
        self.d_analysis = camera_analysis
        self.beta_analysis = beta_analysis
        self.store_in_text = True
        self.plot_grasp_num = False
        self.analyse_objects = object_analysis

        self.dist_step = 0.05
        self.dist_bins = np.arange(0.4, 1.1, self.dist_step)
        self.phi_step = 5
        self.phi_bins = np.arange(0, 70, self.phi_step)
        self.psi_step = 5
        self.psi_bins = np.arange(0, 90, self.psi_step)
        self.beta_step = 5
        self.beta_bins = np.arange(0, 90, self.beta_step)

        # Create output dir if it doesn't exist yet
        if not os.path.exists(output_dir):
            os.mkdir(output_dir)

        # Set up logger
        self.logger = Logger.get_logger(self.__class__.__name__,
                                        log_file=os.path.join(
                                            output_dir, "analysis.log"),
                                        silence=(not self.verbose),
                                        global_log_file=self.verbose)
        self.logger.info("Saving output to %s" % output_dir)
        self.files = self._get_identifiers()

    def _get_identifiers(self):
        """
        Reads the identifiers of all tensors in one dataset.

        Returns
        -------
        numbers (list): List of strings containing the tensor identifiers of the dataset
        """
        self.logger.info("Loading data from {}".format(data_dir))
        files = os.listdir(data_dir)

        numbers = list(set([string.split('_')[-1][:-4] for string in files if '.npz' in string]))
        numbers.sort()
        return numbers

    def _read_model(self, checkpoint):
        """
        Reads in the model parameters from model dir.

        Parameters
        ----------
        checkpoint (str): Checkpoint to be loaded

        Returns
        -------
        models (gqcnn.model): loaded GQCNN model
        """
        # Determine model name
        model_root, model_n = os.path.split(model_dir)
        self.logger.info("Analyzing model %s" % model_n)
        # Load model.

        if checkpoint == 'final':
            model = get_gqcnn_model(verbose=self.verbose).load(
                model_dir, verbose=self.verbose)
        else:
            model = get_gqcnn_model(verbose=self.verbose).load(
                model_dir, verbose=self.verbose, checkpoint_step=checkpoint)
        model.open_session()

        return model

    @staticmethod
    def _plot_heatmap(data, x_bins, x_step, y_bins, y_step, x_axis,
                      y_axis, title, save_name, unit, color="Greens", v_max=None):
        """
        Plots and saves a heatmap with matplotlib.

        Parameters
        ----------
        data (numpy.ndarray): data to be plotted in the heatmap
        x_bins (numpy.array): data bins for the x-axis, e.g. the camera distance d.
        x_step (str): Step size for the x-axis, e.g. 0.05m.
        y_bins (numpy.array): data bins for the y-axis, e.g. the camera elevation angle phi.
        y_step (str): Step size for the y-axis, e.g. 5degree.
        x_axis (str):  x_axis variable, can be 'psi', 'phi', 'beta'
        y_axis (str): y_axis variable, can be 'psi', 'd', 'beta'
        title (str): Title for heatmap.
        save_name (str): Where to save the image (full path)
        unit (str): Unit to label the color bar with.
        color (str): Color of the color bar, defaults to "Greens".
        v_max (float): Maximum value of the heatmap color bar, defaults to None (== maximum value in data).
        """
        sz_x = len(x_bins)
        grid_x = np.arange(0, x_bins[-1] + 2 * x_step, x_step)
        sz_y = len(y_bins)
        grid_y = np.arange(0, y_bins[-1] + 2 * y_step, y_step)
        if x_axis == 'psi':
            x_label = 'Grasp angle psi \u03C8 [\u00b0]'
        elif x_axis == 'd':
            x_label = 'Grasp distance d'
            grid_x = np.arange(0.4, x_bins[-1] + 2 * x_step, x_step)
        elif x_axis == 'beta':
            x_label = 'Grasp angle beta {} [\u00b0]'.format('\u03B2')
        else:
            raise AttributeError("x_axis '{}' cannot be plotted in mesh".format(x_axis))
        if y_axis == 'psi':
            y_label = 'Grasp angle psi \u03C8 [\u00b0]'
        elif y_axis == 'phi':
            y_label = 'Elevation angle \u03C6 [\u00b0]'
        elif y_axis == 'beta':
            y_label = 'Grasp angle beta {} [\u00b0]'.format('\u03B2')
        else:
            raise AttributeError("y_axis '{}' cannot be plotted in mesh".format(y_axis))

        combined_accuracy_map = np.array(data).reshape((sz_y, sz_x))
        x, y = np.meshgrid(grid_x,
                           grid_y)
        fig, ax = plt.subplots(figsize=(5, 6))
        c = ax.pcolormesh(x, y, combined_accuracy_map,
                          cmap=color,
                          vmin=0,
                          vmax=v_max)
        ax.axis([x.min(), x.max(), y.min(), y.max()])
        fig.colorbar(c, ax=ax, label=unit, orientation="horizontal")
        plt.ylabel(y_label)
        plt.xlabel(x_label)
        plt.title(title)
        plt.savefig(save_name)
        plt.close()

    @staticmethod
    def _plot_line(y_data, x_data, title, x_axis, y_axis, save_name, y_lim=None):
        """
        Plots and saves a line with matplotlib.

        Parameters
        ----------
        y_data (list/numpy.array): data points in y-direction.
        x_data (list/np.array): data points in x-direction. If None, the n y-points are plotted along 1, 2, 3, ... n
        title (str): Title for line plot.
        x_axis (str):  can be 'psi', 'phi', 'beta', 'd'; if not one of these, x_axis will be used as x label
        y_axis (str): can be 'pos', 'coll', 'TPR', 'TNR'; if not one of these, y_axis will be used as y label
        save_name (str): Where to save the image (full path)
        y_lim (tuple): y ranges for line plot, can be overwritten by yaxis for TPR and TNR. If None, matplotlib adjusts
                      it automatically.
        """
        if x_axis == 'phi':
            x_label = '{} [\u00b0]'.format('\u03C6')
        elif x_axis == 'psi':
            x_label = '{} [\u00b0]'.format('\u03C8')
        elif x_axis == 'beta':
            x_label = '{} [\u00b0]'.format('\u03B2')
        elif x_axis == 'd':
            x_label = 'Camera distance d'
        else:
            x_label = x_axis
        if y_axis == 'pos':
            y_label = 'Positivity rate [%]'
        elif y_axis == 'coll':
            y_label = 'Collision rate [%]'
        elif y_axis == 'TPR':
            y_label = 'TPR'
            y_lim = (0, 1)
        elif y_axis == 'TNR':
            y_label = 'TNR'
            y_lim = (0, 1)
        else:
            y_label = y_axis

        if x_data is None:
            plt.plot(y_data, '-', color=hist_color)
        else:
            plt.plot(x_data, y_data, 'x-', color=hist_color)
        plt.title(title)
        if y_lim is not None:
            plt.ylim(y_lim)
        plt.xlabel(x_label, color=line_color)
        plt.ylabel(y_label, color=line_color)
        plt.savefig(save_name, transparent=transparent)
        plt.close()

    @staticmethod
    def _balanced_acc(tpr, tnr):
        """
        Calculates the balanced accuracy as (TPR + TNR) / 2 for tpr and tnr values.

        Parameters
        ----------
        tpr (float/list/np.array): True Positive Rate (TP / NUM_P)
        tnr (float/list/np.array): True Negative Rate (TN / NUM_N)

        Returns
        --------
        bal_acc (np.array): Balanced accuracy
        """
        bal_acc = (np.array(tpr) + np.array(tnr)) / 2.0
        return bal_acc

    def _plot_validation_performance(self):
        """
        Plots a range of validation performance plots over the training iterations. The plots include validation loss,
        validation error, validation TPR, validation TNR, validation balanced accuracy and learning rate. Plots are
        stored according to output_dir as defined in the outer scope of this file.
        """
        val_error = []
        val_loss = []
        val_tpr = []
        val_tnr = []
        val_acc = []
        lr = None
        lr_x = [0]
        x = 0
        with open(model_dir + '/training.log', 'r') as reader:
            lines = reader.read().splitlines()
            for line in lines:
                if 'learning rate' in line:
                    rate = float(line.split(':')[-1])
                    if lr is None:
                        lr = [rate]
                    if lr[-1] == rate:
                        x += 1
                    else:
                        lr.append(lr[-1])
                        lr_x.append(x / 100)
                        lr.append(rate)
                        lr_x.append(x / 100)
                if 'alidation error' in line:
                    val_error.append(float(line.split(':')[-1]))
                if 'alidation loss' in line:
                    val_loss.append(float(line.split(':')[-1]))
                if 'alidation tpr' in line:
                    val_tpr.append(float(line.split(':')[-1]) * 100)
                if 'alidation tnr' in line:
                    val_tnr.append(float(line.split(':')[-1]) * 100)
                if 'alidation acc' in line:
                    val_acc.append(float(line.split(':')[-1]) * 100)

        balanced_acc = self._balanced_acc(tpr=val_tpr, tnr=val_tnr)

        self._plot_line(val_error, None, "Validation error {}".format(model_name),
                        "Training iterations [million]", "Validation error",
                        output_dir + '/val_errors.png')

        self._plot_line(val_loss, None, "Validation loss {}".format(model_name),
                        "Training iterations [million]", "Validation loss",
                        output_dir + '/val_loss.png')

        self._plot_line(balanced_acc, None, "Validation balanced accuracy {}".format(model_name),
                        "Training iterations [million]", "Balanced accuracy [%]",
                        output_dir + '/val_bal_acc.png', y_lim=(0, 100))

        self._plot_line(val_tpr, None, "Validation balanced accuracy {}".format(model_name),
                        "Training iterations [million]", "TPR [%]",
                        output_dir + '/val_tpr.png', y_lim=(0, 100))

        self._plot_line(val_tnr, None, "Validation TNR {}".format(model_name),
                        "Training iterations [million]", "TNR [%]",
                        output_dir + '/val_tnr.png', y_lim=(0, 100))

        self._plot_line(val_acc, None, "Validation accuracy {}".format(model_name),
                        "Training iterations [million]", "Accuracy [%]",
                        output_dir + '/val_acc.png', y_lim=(0, 100))

        self._plot_line(lr_x, lr, "Learning rate {}".format(model_name),
                        "Training iterations [million]", "Learning rate",
                        output_dir + '/learning_rate.png', y_lim=(0, lr[0]*1.05))

    @staticmethod
    def _get_accuracies(results, param, bin_1, bin_2=None):
        """
        Calculates the TPR, TNR, balanced accuracy and accuracy based on the prediction results

        Parameters
        ----------
        results (dict): Dictoonary of the prediction results after binary classification
        param (str): defining which parameter bin_1 (and bin_2) relates to. Can be 'phi', 'beta', 'phi_dist', etc.
        bin_1 (float): bin for param, e.g. 5[degree] for 'beta'
        bin_2 (float): 2nd bin for param, e.g. 0.05[m] for the distance in 'phi_dist'

        Returns
        --------
        tpr (np.array): True Positive Rate, calculated as TP / NUM_P
        tnr (np.array): True Negative Rate, calculated as TN / NUM_N
        bal_acc (np.array): Balanced accuracy, calculated as (TPR + TNR) / 2
        acc (np.array): Accuracy, calculated as (TP + TN) / (NUM_P + NUM_N)
        """
        if bin_2 is None:
            tp = sum(results['tp'][param][bin_1])
            tn = sum(results['tn'][param][bin_1])
            num_n = sum(results['num_n'][param][bin_1])
            num_p = sum(results['num_p'][param][bin_1])
        else:
            tp = sum(results['tp'][param][bin_1][bin_2])
            tn = sum(results['tn'][param][bin_1][bin_2])
            num_n = sum(results['num_n'][param][bin_1][bin_2])
            num_p = sum(results['num_p'][param][bin_1][bin_2])

        tpr = tp / num_p
        tnr = tn / num_n
        bal_acc = (tpr + tnr) / 2
        acc = (tn + tp) / (num_p + num_n)
        return tpr, tnr, bal_acc, acc

    def _get_acc_range(self, results, param, bins, bins_2=None):
        """
        Generates an array of accuracy metrics over parameter ranges, e.g. over the grasp-table angle beta.

        Parameters
        ----------
        results (dict): Dictionary of the prediction results after binary classification
        param (str): defining which parameter bins to calculate. Can be 'phi', 'beta', 'psi', 'phi_dist', etc.
        bins (np.array): All bins for the (first) parameter, e.g. 'beta' with [0, 5, 10, 15, ...] degree
        bins_2 (np.array): All bins for the second parameter, e.g. distance d for 'phi_dist'. Defaults to None
                           for single parameter evaluations.

        Returns
        --------
        arr_tpr (np.array): True Positive Rate, calculated as TP / NUM_P
        arr_tnr (np.array): True Negative Rate, calculated as TN / NUM_N
        arr_bal_acc (np.array): Balanced accuracy, calculated as (TPR + TNR) / 2
        arr_acc (np.array): Accuracy, calculated as (TP + TN) / (NUM_P + NUM_N)
        """
        arr_tpr = []
        arr_tnr = []
        arr_bal_acc = []
        arr_acc = []
        for bin1 in bins:
            if bins_2 is None:
                try:
                    tpr, tnr, bal_acc, acc = self._get_accuracies(results, param, bin1)
                    arr_tpr.append(tpr)
                    arr_tnr.append(tnr)
                    arr_bal_acc.append(bal_acc)
                    arr_acc.append(acc)
                except ZeroDivisionError:
                    arr_tpr.append(0.0)
                    arr_tnr.append(0.0)
                    arr_bal_acc.append(0.0)
                    arr_acc.append(0.0)
            else:
                for bin2 in bins_2:
                    try:
                        tpr, tnr, bal_acc, acc = self._get_accuracies(results, param, bin1, bin2)
                        arr_tpr.append(tpr)
                        arr_tnr.append(tnr)
                        arr_bal_acc.append(bal_acc)
                        arr_acc.append(acc)
                    except ZeroDivisionError:
                        arr_tpr.append(0.0)
                        arr_tnr.append(0.0)
                        arr_bal_acc.append(0.0)
                        arr_acc.append(0.0)
        return arr_tpr, arr_tnr, arr_bal_acc, arr_acc

    @staticmethod
    def _run_binary_classification(predictions, labels):
        """
        Runs binary classification on a set of predictions and labels. Returns 0.0 for all metrics
        [tp, tn, num_p, num_n] if predictions is None.

        Parameters
        ----------
        predictions (numpy.array): Predictions of the classification model, can be a vector or a [n, 2] matrix.
                                   If [n, 2] matrix, the second column is assumed to be the prediction, alongside
                                   the definition during GQ-CNN and VGQ-CNN training.
        labels (numpy.array): Array of binary class labels, with 1 for positive grasps and 0 for negative grasps.

        Returns
        --------
        tp (int): Number of true positive grasps, e.g. ground-truth positive grasps that were classified as positive
        tn (int): Number of true negative grasps, e.g. ground-truth negative grasps that were classified as negative
        num_p (int): Number of ground-truth positive grasps
        num_n (int): Number of ground-truth negative grasps
        """
        if predictions is None:
            return 0.0, 0.0, 0.0, 0.0, 0.0
        num_p = sum(labels)
        num_n = len(labels) - sum(labels)
        try:
            results = BinaryClassificationResult(predictions[:, 1], labels)
            tp = len(results.true_positive_indices)
            tn = len(results.true_negative_indices)
        except IndexError:
            results = BinaryClassificationResult(predictions, labels)
            tp = sum(results.predictions[labels == 1])
            tn = len(results.predictions[labels == 0]) - sum(results.predictions[labels == 0])

            if type(tp) is not int:
                tp = tp.squeeze()
            if type(tn) is not int:
                tn = tn.squeeze()

        return tp, tn, num_p, num_n

    @staticmethod
    def _save_results(save_dir, data):
        """
        Generates a text file from an array of results in percentage.

        Parameters
        ----------
        save_dir (str): Full path of the text file.
        data (numpy.array): results stored in the format of an array. Will be converted to [%].
        """
        with open(save_dir, 'w') as f:
            np.savetxt(f, data * 100, '%.1f')

    def _run_prediction(self, model):
        """
        Predicts the data with the loaded model and plots a box-plot to show TPR and TNR performance over the objects.

        Parameters
        ----------
        model (gqcnn.model): Model to be used for prediction.

        Returns
        ----------
        results (dict):             Dictionary with True Positives, True Negatives, Ground-truth Negatives and
                                    Ground-truth Positives after prediction and binary classification
        object_predictions (dict):  Dictionary with the object labels, ground truth grasp labels and model
                                    predictions for object-wise analysis. If self.analyse_objects = False, an empty
                                    dictionary will be returned.
        """
        results = {'tp': {'all': [], 'phi': {}, 'psi': {}, 'beta': {}, 'dist': {}, 'phi_dist': {}},
                   'tn': {'all': [], 'phi': {}, 'psi': {}, 'beta': {}, 'dist': {}, 'phi_dist': {}},
                   'num_n': {'all': [], 'phi': {}, 'psi': {}, 'beta': {}, 'dist': {}, 'phi_dist': {}},
                   'num_p': {'all': [], 'phi': {}, 'psi': {}, 'beta': {}, 'dist': {}, 'phi_dist': {}}}

        res = ['tp', 'tn', 'num_p', 'num_n']
        for result in res:
            results[result]['all'] = []
            for _phi in self.phi_bins:
                results[result]['phi'][_phi] = []
            for _psi in self.psi_bins:
                results[result]['psi'][_psi] = []
            for _beta in self.beta_bins:
                results[result]['beta'][_beta] = []
            for _dist in self.dist_bins:
                results[result]['dist'][_dist] = []
            for _phi in self.phi_bins:
                results[result]['phi_dist'][_phi] = {}
                for _dist in self.dist_bins:
                    results[result]['phi_dist'][_phi][_dist] = []

        object_predictions = {'predictions': [], 'labels': [], 'objects': []}

        step_size = 25
        num_files = len(self.files)
        if DEBUG:
            num_files = 1
            step_size = 1

        # Read and predict data
        for steps in range(0, num_files, step_size):
            self.logger.info("Read in tensors %d to %d of %d" % (steps, steps + step_size, num_files))
            images, poses, all_labels, elev_arr, radii, psi_arr, beta_arr, objects = self._read_data(steps, step_size)
            predictions = model.predict(images, poses)

            tp, tn, num_p, num_n = self._run_binary_classification(predictions, all_labels)
            results['tp']['all'].append(tp)
            results['tn']['all'].append(tn)
            results['num_p']['all'].append(num_p)
            results['num_n']['all'].append(num_n)

            if self.analyse_objects:
                object_predictions['predictions'].extend(predictions)
                object_predictions['labels'].extend(all_labels)
                object_predictions['objects'].extend(list(objects))

            if self.phi_analysis:
                for _phi in self.phi_bins:
                    mask = (elev_arr.squeeze() >= _phi) & (elev_arr.squeeze() < _phi + self.phi_step)
                    phi_predictions = predictions[mask]
                    phi_labels = all_labels[mask]
                    tp, tn, num_p, num_n = self._run_binary_classification(phi_predictions, phi_labels)
                    results['tp']['phi'][_phi].append(tp)
                    results['tn']['phi'][_phi].append(tn)
                    results['num_p']['phi'][_phi].append(num_p)
                    results['num_n']['phi'][_phi].append(num_n)

            if self.beta_analysis:
                for _beta in self.beta_bins:
                    mask = (beta_arr.squeeze() >= _beta) & (beta_arr.squeeze() < _beta + self.beta_step)
                    beta_predictions = predictions[mask]
                    beta_labels = all_labels[mask]
                    tp, tn, num_p, num_n = self._run_binary_classification(beta_predictions, beta_labels)
                    results['tp']['beta'][_beta].append(tp)
                    results['tn']['beta'][_beta].append(tn)
                    results['num_p']['beta'][_beta].append(num_p)
                    results['num_n']['beta'][_beta].append(num_n)

            if self.psi_analysis:
                for _psi in self.psi_bins:
                    mask = (psi_arr.squeeze() >= _psi) & (psi_arr.squeeze() < _psi + self.psi_step)
                    psi_predictions = predictions[mask]
                    psi_labels = all_labels[mask]

                    tp, tn, num_p, num_n = self._run_binary_classification(psi_predictions, psi_labels)
                    results['tp']['psi'][_psi].append(tp)
                    results['tn']['psi'][_psi].append(tn)
                    results['num_p']['psi'][_psi].append(num_p)
                    results['num_n']['psi'][_psi].append(num_n)

            if self.d_analysis:
                for _dist in self.dist_bins:
                    mask = (radii.squeeze() >= _dist) & (radii.squeeze() < _dist + self.dist_step)
                    dist_predictions = predictions[mask]
                    dist_labels = all_labels[mask]

                    tp, tn, num_p, num_n = self._run_binary_classification(dist_predictions, dist_labels)
                    results['tp']['dist'][_dist].append(tp)
                    results['tn']['dist'][_dist].append(tn)
                    results['num_p']['dist'][_dist].append(num_p)
                    results['num_n']['dist'][_dist].append(num_n)

            if self.d_analysis and self.phi_analysis:
                for _phi in self.phi_bins:
                    for _dist in self.dist_bins:
                        mask = (radii.squeeze() >= _dist) & \
                               (radii.squeeze() < _dist + self.dist_step) & \
                               (elev_arr.squeeze() >= _phi) & \
                               (elev_arr.squeeze() < _phi + self.phi_step)
                        phi_dist_predictions = predictions[mask]
                        phi_dist_labels = all_labels[mask]
                        tp, tn, num_p, num_n = self._run_binary_classification(phi_dist_predictions, phi_dist_labels)
                        results['tp']['phi_dist'][_phi][_dist].append(tp)
                        results['tn']['phi_dist'][_phi][_dist].append(tn)
                        results['num_p']['phi_dist'][_phi][_dist].append(num_p)
                        results['num_n']['phi_dist'][_phi][_dist].append(num_n)

        return results, object_predictions

    @staticmethod
    def _analyse_performance_per_object(object_predictions):
        """
        Runs binary classification and plots a box-plot to show TPR and TNR performance over the objects.

        Parameters
        ----------
        object_predictions (dict): Object labels, grasp ground-truth labels and model predictions of model
        """
        object_labels = list(set(list(object_predictions['objects'])))
        object_predictions['objects'] = np.array(object_predictions['objects'])
        object_predictions['labels'] = np.array(object_predictions['labels'])
        object_predictions['predictions'] = np.array(object_predictions['predictions'])[:, 1]
        tpr = []
        tnr = []
        for object_label in object_labels:
            in_bin = (list(object_predictions['objects']) == object_label)
            labels = object_predictions['labels'][in_bin]
            obj_predictions = np.where(object_predictions['predictions'][in_bin] >= 0.5, 1, 0)

            positives = (labels == 1)
            pos_predictions = obj_predictions[positives]
            if len(pos_predictions) != 0:
                tpr.append(sum(pos_predictions) / len(pos_predictions))

            negatives = (labels == 0)
            neg_predictions = obj_predictions[negatives]
            if len(neg_predictions) != 0:
                tnr.append((len(neg_predictions) - sum(neg_predictions)) / len(neg_predictions))

        plt.boxplot([tpr, tnr], labels=["TPR", "TNR"])
        plt.ylim((0.0, 1.0))
        plt.title("TPR and TNR per object")
        plt.savefig(output_dir + '/object_tpr_tnr.png')
        plt.close()

    def _load_model_and_plot_training_stats(self):
        """
        Loads the model and plots the training stats, including learning rate, validation loss, validation errors, ...

        Returns
        --------
        model (gqcnn.model): Loaded model with the highest balanced accuracy on the validation set during training.
        """
        model_config = YamlConfig(model_dir + '/config.json')
        self.gripper_mode = model_config['gqcnn']['gripper_mode']
        if 'pose_input' in model_config['gqcnn']:
            self.pose_input = model_config['gqcnn']['pose_input']
            if self.pose_input['quaternion'] == 0 and self.psi_analysis:
                self.psi_analysis = False

        self._plot_validation_performance()
        model = self._read_model('final')
        return model

    def visualise(self):
        """
        Evaluates the model on the dataset in self.datadir. Plots and saves the resulting classification accuracies.

        """
        model = self._load_model_and_plot_training_stats()
        results, object_predictions = self._run_prediction(model)

        # Calculate prediction accuracy for all models and all elevation (phi) angles

        if sum(results['num_p']['all']) > 0:
            tpr = sum(results['tp']['all']) / sum(results['num_p']['all']) * 100
        else:
            tpr = 0.0
        if sum(results['num_n']['all']) > 0:
            tnr = sum(results['tn']['all']) / sum(results['num_n']['all']) * 100
        else:
            tnr = 0.0
        self.logger.info("TPR: %.1f %%" % tpr)
        self.logger.info("TNR: %.1f %%" % tnr)
        self.logger.info("Balanced accuracy: %.1f %%" % ((tpr + tnr) / 2))

        if self.analyse_objects:
            self._analyse_performance_per_object(object_predictions)

        if self.phi_analysis:
            phi_tpr, phi_tnr, _, _ = self._get_acc_range(results, 'phi', self.phi_bins)
            self._plot_line(x_data=self.phi_bins + self.phi_step / 2, y_data=phi_tpr,
                            title="TPR over varying elevation angle",
                            x_axis='phi',
                            y_axis='TPR',
                            save_name=output_dir + '/phi_tpr.png')

            self._plot_line(x_data=self.phi_bins + self.phi_step / 2, y_data=phi_tnr,
                            title="TNR over varying elevation angle",
                            x_axis='phi',
                            y_axis='TNR',
                            save_name=output_dir + '/phi_tnr.png')
            if self.store_in_text:
                self._save_results("{}/phi_tpr.txt".format(output_dir), phi_tpr)
                self._save_results("{}/phi_tnr.txt".format(output_dir), phi_tnr)

        if self.beta_analysis:
            beta_tpr, beta_tnr, _, _ = self._get_acc_range(results, 'beta', self.beta_bins)
            self._plot_line(x_data=self.beta_bins + self.beta_step / 2, y_data=beta_tpr,
                            title="TPR over varying grasp/table angle",
                            x_axis='beta',
                            y_axis='TPR',
                            save_name=output_dir + '/beta_tpr.png')

            self._plot_line(x_data=self.beta_bins + self.beta_step / 2, y_data=beta_tnr,
                            title="TNR over varying grasp_table angle",
                            x_axis='beta',
                            y_axis='TNR',
                            save_name=output_dir + '/beta_tnr.png')
            if self.store_in_text:
                self._save_results("{}/beta_tpr.txt".format(output_dir), beta_tpr)
                self._save_results("{}/beta_tnr.txt".format(output_dir), beta_tnr)

        if self.d_analysis:
            dist_tpr, dist_tnr, _, _ = self._get_acc_range(results, 'dist', self.dist_bins)
            self._plot_line(x_data=self.dist_bins + self.dist_step / 2, y_data=dist_tpr,
                            title="TPR over varying camera radii",
                            x_axis='d',
                            y_axis='TPR',
                            save_name=output_dir + '/rad_tpr.png')

            self._plot_line(x_data=self.dist_bins + self.dist_step / 2, y_data=dist_tnr,
                            title="TNR over varying camera radii",
                            x_axis='d',
                            y_axis='TNR',
                            save_name=output_dir + '/rad_tnr.png')
            if self.store_in_text:
                self._save_results("{}/d_tpr.txt".format(output_dir), dist_tpr)
                self._save_results("{}/d_tnr.txt".format(output_dir), dist_tnr)

        if self.psi_analysis:
            psi_tpr, psi_tnr, _, _ = self._get_acc_range(results, 'psi', self.psi_bins)
            self._plot_line(x_data=self.psi_bins + self.psi_step / 2, y_data=psi_tpr,
                            title="TPR over varying grasp angle",
                            x_axis='psi',
                            y_axis='TPR',
                            save_name=output_dir + '/psi_tpr.png')

            self._plot_line(x_data=self.psi_bins + self.psi_step / 2, y_data=psi_tnr,
                            title="TNR over varying grasp angle",
                            x_axis='psi',
                            y_axis='TNR',
                            save_name=output_dir + '/psi_tnr.png')
            if self.store_in_text:
                self._save_results("{}/psi_tpr.txt".format(output_dir), psi_tpr)
                self._save_results("{}/psi_tnr.txt".format(output_dir), psi_tnr)

        if self.phi_analysis and self.d_analysis:
            phi_d_tpr, phi_d_tnr, _, _ = self._get_acc_range(results, 'phi_dist', self.phi_bins, self.dist_bins)

            if self.store_in_text:
                self._save_results("{}/phi_d_tpr.txt".format(output_dir), phi_d_tpr)
                self._save_results("{}/phi_d_tnr.txt".format(output_dir), phi_d_tnr)

            tnr = np.array(phi_d_tnr) * 100
            tpr = np.array(phi_d_tpr) * 100
            self.logger.info("Phi-d TNR min: {:.1f}%".format(min(tnr)))
            self.logger.info("Phi-d TNR max: {:.1f}%".format(max(tnr)))
            self.logger.info("Phi-d TNR mean: {:.1f}% +/- {:.1f}%".format(tnr.mean(), tnr.std()))
            self.logger.info("Phi-d TPR min: {:.1f}%".format(min(tpr)))
            self.logger.info("Phi-d TPR max: {:.1f}%".format(max(tpr)))
            self.logger.info("Phi-d TPR mean: {:.1f}% +/- {:.1f}%".format(tpr.mean(), tpr.std()))

            dset_name = data_dir.split('/')[-3]
            title = 'TPR {} on {}'.format(model_name, dset_name)
            save_name = output_dir + '/phi_d_tpr_{}.png'.format(model_name)
            self._plot_heatmap(data=phi_d_tpr, x_bins=self.dist_bins, x_step=self.dist_step,
                               y_bins=self.phi_bins, y_step=self.phi_step, x_axis='d', y_axis='phi', title=title,
                               save_name=save_name, unit="TPR", v_max=1)

            title = 'TNR {} on {}'.format(model_name, dset_name)
            save_name = output_dir + '/phi_d_tnr_{}.png'.format(model_name)
            self._plot_heatmap(data=phi_d_tnr, x_bins=self.dist_bins, x_step=self.dist_step,
                               y_bins=self.phi_bins, y_step=self.phi_step, x_axis='d', y_axis='phi', title=title,
                               save_name=save_name, unit="TNR", v_max=1, color="Blues")

    def _read_data(self, steps, step_size):
        """Reads data from a tensor dataset.

        Parameters
        ----------
            steps (int): Tensor to start reading data from.
            step_size (int): Amount of tensors to read in one batch. All tensors from steps:steps+step_size are read.

        Returns
        ----------
            images (np.array): Depth images.
            poses (np.array): Pose information in format for network-input.
            labels (np.array): Binary ground-truth labels of grasps [0/1].
            phi (np.array): Elevation angle phi of camera for images.
            dist (np.array): Camera distance to T^w origin for images.
            psi (np.array): Angle between grasp approach axis (z) and camera principal ray, psi for all grasps.
            beta (np.array): Angle between grasp approach axis (z) and table normal (T^w z), beta, for all grasps.
            object_labels (np.array): Counter for object identifier for all grasps.
        """
        cam_str = 'camera_poses_'
        depth_str = 'depth_ims_tf_table_'
        pose_str = 'hand_poses_'
        metric_str = 'robust_ferrari_canny_'
        object_str = 'object_labels_'

        phi = None
        object_labels = None
        dist = None
        psi = None
        beta = None

        files = self.files[steps:steps+step_size]

        images = list(np.load(data_dir + depth_str + files[0] + ".npz")['arr_0'])
        pose_arr = list(np.load(data_dir + pose_str + files[0] + ".npz")['arr_0'])
        metric_arr = list(np.load(data_dir + metric_str + files[0] + ".npz")['arr_0'])
        if self.analyse_objects:
            object_labels = list(np.load("{}{}{}.npz".format(data_dir, object_str, files[0]))['arr_0'])
        if self.d_analysis or self.phi_analysis:
            cameras = np.load(data_dir + cam_str + files[0] + ".npz")['arr_0']
            dist = list(cameras[:, 0])
            phi = list(cameras[:, 1])

        for next_file in files[1:]:
            images.extend(list(np.load(data_dir + depth_str + next_file + ".npz")['arr_0']))
            pose_arr.extend(list(np.load(data_dir + pose_str + next_file + ".npz")['arr_0']))
            metric_arr.extend(list(np.load(data_dir + metric_str + next_file + ".npz")['arr_0']))
            if self.analyse_objects:
                object_labels.extend(list(np.load("{}{}{}.npz".format(data_dir, object_str, next_file))['arr_0']))

            if self.d_analysis or self.phi_analysis:
                cameras = np.load(data_dir + cam_str + next_file + ".npz")['arr_0']
                dist.extend(list(cameras[:, 0]))
                phi.extend(list(cameras[:, 1]))

        labels = 1 * (np.array(metric_arr) > self.metric_thresh)
        images = np.array(images)
        pose_arr = np.array(pose_arr)
        poses = read_pose_data(pose_arr, self.gripper_mode, self.pose_input)

        if self.analyse_objects:
            object_labels = np.array(object_labels)
        if self.psi_analysis:
            psi = pose_arr[:, -2].squeeze()
            psi = np.rad2deg(psi)
        if self.beta_analysis:
            beta = np.rad2deg(pose_arr[:, -1].squeeze())
        if self.d_analysis or self.phi_analysis:
            phi = np.rad2deg(np.array(phi))
            dist = np.array(dist)

        return images, poses, labels, phi, dist, psi, beta, object_labels

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Analyse a GQCNN with Tensorflow on single data")
    parser.add_argument("model_name",
                        type=str,
                        default=None,
                        help="name of model to analyse")
    parser.add_argument("data_dir",
                        type=str,
                        default=None,
                        help="path to where the data is stored")
    parser.add_argument("--camera_pose",
                        type=bool,
                        default=True,
                        help="analyse performance over camera pose")
    parser.add_argument("--beta",
                        type=bool,
                        default=False,
                        help="analyse performance over beta")
    parser.add_argument("--psi",
                        type=bool,
                        default=False,
                        help="analyse performance over psi")
    parser.add_argument("--objects",
                        type=bool,
                        default=False,
                        help="analyse performance over objects")

    args = parser.parse_args()
    model_name = args.model_name
    data_dir = args.data_dir
    analyse_pose = args.camera_pose
    analyse_beta = args.beta
    analyse_psi = args.psi
    analyse_objects = args.objects

    # Create model dir.
    model_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../models")
    model_dir = os.path.join(model_dir, model_name)

    # Make the output dir.
    try:
        output_dir = EXPER_PATH + model_name + '_on_{}/'.format(data_dir.split('/')[2])
        if not os.path.exists(output_dir):
            os.mkdir(output_dir)
    except FileNotFoundError:
        output_dir = '/results/' + model_name + '_on_{}/'.format(data_dir.split('/')[2])
        if not os.path.exists(output_dir):
            os.mkdir(output_dir)

    # Initialise analyser and run analysis.
    analyser = VgqcnnAnalyse(analyse_pose, analyse_beta, analyse_psi, analyse_objects)
    analyser.visualise()


import numpy as np
from liboptpy.simulator.bioprocess import bioprocess_simulator
from liboptpy.lib.util import comupute_cumulative_feed
import pandas as pd
from tqdm import tqdm

def compute_reward(cumulative_feed, state, horizon):
    titer = state[1]
    total_CA = state[1] * state[-1]
    oil_consumed = cumulative_feed * 917 - state[2] * state[-1]
    yield_CA = total_CA / oil_consumed
    pv = titer / horizon
    man_cost = (2 / pv + 1 / yield_CA) / 1000  # $/(g h)
    total_reward = - man_cost * horizon  # * state[-1] * state[1]
    return total_reward

def compute_avg_reward(bn, params, policy, time_measurement,  real_measurement, B, isMDP=True):
    reward = np.zeros(B)
    for b in tqdm(range(B)):
        x0 = bn.initial_state_full + np.abs(
            np.random.normal(0, np.array(bn.initial_state_full) / 10 + 0.01))
        # time_measurement = [4 * i for i in range(bn.n_time)]
        # time_measurement[-1] = 135.9
        simulator = bioprocess_simulator(bn, time_measurement, policy, real_measurement)
        states = simulator.g(time_measurement, x0, params)
        feed = pd.DataFrame(simulator.feed_every_second, columns=['time', 'feed_rate'],
                               dtype="float64").drop_duplicates().to_numpy() if isMDP else np.vstack([real_measurement, np.array(policy.measurement)]).T
        total_feed = comupute_cumulative_feed(feed)
        reward[b] = compute_reward(total_feed, states[-1,:], time_measurement[-1])
    return np.mean(reward)


from typing import Callable
import inspect
import sys
import os

def __rgb(r: int = 255, b: int = 255, g: int = 255, mode=False):
    return f'\x1b[{48 if mode else 38};2;{r};{g};{b}m'

__MONOKAI_COLOR_PALLETE = {
    'Background': __rgb(46, 46, 46),  # 2e2e2e
    'Comments': __rgb(121, 121, 121),  # 797979
    'White': __rgb(255, 255, 255),  # fffff only color changed
    'Yellow': __rgb(229, 181, 103),  # e5b567
    'Green': __rgb(180, 210, 115),  # b4d273
    'Orange': __rgb(232, 125, 62),  # e87d3e
    'Purple': __rgb(158, 134, 200),  # 9e86c8
    'Pink': __rgb(176, 82, 121),  # b05279
    'Blue': __rgb(108, 153, 187),  # 6c99bb
}

kwargs_output = ', {kwarg} = {}{kwarg_value}{}'
args_output = '{arg} = {}{value}{}'
dbg_output ='{}[{filename}]{}({lineno}){}{name}{}({args}{kwargs})->{}{result}{}'
def supports_color(color: str) -> str:
    """
    Returns True if the running system's terminal supports color, and False
    otherwise.
    """
    plat = sys.platform
    supported_platform = plat != 'Pocket PC' and (plat != 'win32' or
                                                  'ANSICON' in os.environ)
    is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
    return color if supported_platform and is_a_tty else ''

def __kwargs_str(**kwargs) -> str:
    return ''.join(map(lambda x:
                       kwargs_output.format(supports_color(__MONOKAI_COLOR_PALLETE["Green"]),
                                            supports_color(__MONOKAI_COLOR_PALLETE["White"]), kwarg=x[0],
                                                      kwarg_value=x[1].__repr__()), kwargs.items()))

def __args_str(args: list[str], *values) -> str:
    return ''.join(map(lambda arg: ', ' +
                       args_output.format(__MONOKAI_COLOR_PALLETE["Blue"],
                                          __MONOKAI_COLOR_PALLETE["White"], arg=arg[1],
                                                      value=values[arg[0]]) if arg[0] else
                       args_output.format(__MONOKAI_COLOR_PALLETE["Blue"],
                                         __MONOKAI_COLOR_PALLETE["White"],
                                          arg=arg[1], value=values[arg[0]]),
                       enumerate(args)))

def Dbg(f: Callable):
    fnargs = list(f.__annotations__.keys())[:-1]

    def wrapper(*args, **kwargs):
        stack = inspect.stack()
        result = f(*args, **kwargs)
        print(dbg_output.format(__MONOKAI_COLOR_PALLETE['Yellow'],
                                 __MONOKAI_COLOR_PALLETE['Pink'],
                                 __MONOKAI_COLOR_PALLETE['Blue'],
                                 __MONOKAI_COLOR_PALLETE['White'],
                                 __MONOKAI_COLOR_PALLETE['Green'],
                                 __MONOKAI_COLOR_PALLETE['White'],
                                 filename=stack[1].filename,
                                 lineno=stack[1].lineno, name=f.__qualname__,
                                 args=__args_str(fnargs, *args),
                                 kwargs=__kwargs_str(**kwargs),
                                result=result.__repr__()))
        return result
    return wrapper


import sys
import argparse
import re
from sqlalchemy import create_engine
import numpy as np
import pandas as pd
import nltk
nltk.download(['punkt', 'wordnet','stopwords','averaged_perceptron_tagger'])
from nltk.corpus import stopwords
from nltk import word_tokenize, sent_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.multioutput import MultiOutputClassifier
from sklearn.metrics import classification_report
from sklearn.base import BaseEstimator, TransformerMixin
import pickle

class WordCount(BaseEstimator, TransformerMixin):
    """ We define a WordCount transformer. A custom transformer inherits from BaseEstimator and TransformerMixin

        def __init__(self):
            We have to initialize StandardScaler(). Since the X matrix will be small values, we want to normalize
            the number of words too.

        def fit(self, X, y=None):
            We split the words, obtain the length of the resulting list and fit the StandardScaler.

        def transform(self, X):
            We obtain the normalized word count.

    """

    def __init__(self, active = False):
        # Initialize the standardscaler and wether we plan to use it or no:
        self.standardscaler = StandardScaler()
        self.active = active

    def fit(self, X, y=None):
        # Fit the standardScaler
        if self.active:
            self.standardscaler.fit(pd.Series(X).str.split().str.len().values.reshape(-1,1).astype(np.float))
        return self

    def transform(self, X):
        if self.active:
            # We will always normalize the results using the mean and std obtained in the fit method:
            return pd.DataFrame(self.standardscaler.transform(pd.Series(X).str.split().str.len().values.reshape(-1,1).astype(np.float)))
        return None

class StartingVerbExtractor(BaseEstimator, TransformerMixin):

    """ We define a StartingVerb transformer. A custom transformer inherits from BaseEstimator and TransformerMixin

        def __init__(self):
            We define a special case so we can skip it.

        def fit(self, X, y=None):
            We do not have to fit anything.

        def transform(self, X):
            We convert X to a dataframe and apply the function starting_verb. Returns a Dataframe with
            the results.

        def staring_verb(self, text):
            We go through the sentences of each messages, tokenize them, obtain the part of speech tag
            and if the first word of any of the sentences of the message is either a verb or RT we
            return True (otherwise, return false).
    """
    def __init__(self, active = False):
        self.active = active

    def starting_verb(self, text):
        sentence_list = nltk.sent_tokenize(text)
        for sentence in sentence_list:
            pos_tags = nltk.pos_tag(tokenize(sentence))

            # Since we are removing punctuation, some sentences might be completely empty.
            # In order to prevent an IndexError:
            try:
                first_word, first_tag = pos_tags[0]
                if first_tag in ['VB', 'VBP'] or first_word == 'RT':
                    return True
            except IndexError:
                pass
                # There was an empty sencente (due to punctuation, we want it ignored)
        return False

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        if self.active:
            X_tagged = pd.Series(X).apply(self.starting_verb)
            return pd.DataFrame(X_tagged)
        return None

class KeyWordSearch(BaseEstimator, TransformerMixin):

    """ We define a KeyWordSearch transformer. A custom transformer inherits from BaseEstimator and TransformerMixin

        def __init__(self):
            Define the Keywords we want to be searched

        def fit(self, X, y=None):
            We do not have to fit anything.

        def transform(self, X):
            We convert X to a dataframe and apply the function check_keywords. Returns a Dataframe with
            with whether the keywords are present in the message.

        def check_keywords(self, text):
            We go through the sentences of each messages, tokenize them, obtain the part of speech tag
            and if the first word of any of the sentences of the message is either a verb or RT we
            return True (otherwise, return false).
    """

    def __init__(self, keywords = ['medical', 'doctor', 'injections', 'sick', 'bandage', 'help', 'alone', 'child', 'water','thirst', 'drought'\
           ,'rescue', 'search','trapped','lost', 'food', 'famine','dead','death','money','homeless','accident','floods'\
           ,'fire', 'wildfire','hospital','aid','storm','hail','earthquake','cold','blanket','weather'], active = False):
        self.keywords = keywords
        self.active = active

    def check_keywords(self, row):
        wordset =  set(tokenize(row.values[0]))
        for item in self.keywords:
            row[item] = 0
            if item in wordset:
                row[item] = 1
        return row[self.keywords]

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        if self.active:
            df = pd.DataFrame(X)
            df_tagged = df.apply(self.check_keywords, axis = 1)
            return df_tagged
        return None


from dataclasses import dataclass
from typing import Optional

from utils import load_data
from utils import print_result

@dataclass
class Op:
    instruction: str
    value: Optional[int]

def parse_data(lines: list[str]) -> list[Op]:
    ops = []
    for l in lines:
        l_split = l.split()
        ops.append(Op(l_split[0], int(l_split[1]) if len(l_split) > 1 else None))
    return ops

class System:
    def __init__(self) -> None:
        self._current = 1
        self._cycle_num = 1
        self._snapshots: dict[int, int] = {}
        self._operations: dict[int, int] = {}

    def cycle(self, op: Op) -> None:
        self._log_state()
        if op.instruction == "noop":
            self._cycle_num += 1
            return

        self._cycle_num += 1
        self._log_state()
        self._cycle_num += 1
        self._current += op.value

    def _log_state(self) -> None:
        if self._cycle_num > 0 and (self._cycle_num == 20 or (self._cycle_num - 20) % 40 == 0):
            self._snapshots[self._cycle_num] = self._current

    @property
    def output(self) -> int:
        return sum(k * v for k, v in self._snapshots.items())

def part1(ops: list[Op]) -> int:
    system = System()
    for op in ops:
        system.cycle(op)
    return system.output

def part2(ops: list[Op]) -> int:
    # return sum([1 for pair in pairs if pair[0] & pair[1]])
    pass

if __name__ == "__main__":
    example_data = parse_data(load_data(True))
    data = parse_data(load_data(False))

    assert part1(example_data) == 13140
    print_result(1, part1(data))
    #
    # assert part2(example_data) == 4
    # print_result(2, part2(data))


import numpy as np
import sys
import os
import pyvista as pv
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
from src.mapIO import read_map , greatest_dim
from src.util import pad_map, unpad_map
import matplotlib.pyplot as plt

frag_names = ["Benzene", "Propane", "H-bond Donor", "H-bond Acceptor"]
frag_names_short = ["benc", "prpc", "hbacc", "hbdon"]
path = "../data/maps/"
pdb_id = "1ycr"
tail = ".gfe.map"
path_list = [path+pdb_id+"."+name+tail for name in frag_names_short]

chan_id = 1 # range 0-3
_, _, gfe = read_map(path_list[chan_id])

maxD = greatest_dim("../data/maps/", ["1ycr", "1pw2", "2f6f"])
print(maxD)

######------------- Test padding ----------------#######
pad_gfe, xpad, ypad, zpad = pad_map(gfe, maxD)
print('Original map',gfe.shape)
print('Padded map',pad_gfe.shape)

if np.abs(pad_gfe.sum() - gfe.sum()) > 0.000001:
    print("Error! Zero padding should not affect the sum")
    print("Padded sum = ", pad_gfe.sum())
    print("Map sum = ", gfe.sum())
else:
    print("Padding test passed!")

######------------- Test unpadding ----------------#######
ori_dim = gfe.shape
up_gfe = unpad_map(pad_gfe, xpad, ypad, zpad)
unpad_dim = up_gfe.shape
i = 0; dp = 0
for item in ori_dim:
    dp = item - unpad_dim[i]
    i+=1
    if dp != 0:
        print("Original dim is not same as padded dim!")
        break

#compare original and unpadded map volumes by visialization
p = pv.Plotter(point_smoothing = True, shape=(1, 2))
fs = 16
p.subplot(0, 0)
text = frag_names[chan_id]+" original " + "dim = "+ str(pad_gfe.shape)
p.add_text(text, position = 'upper_left', font_size = fs)
p.add_volume(np.abs(gfe), cmap = "viridis", opacity = "linear")

p.subplot(0, 1)
text = frag_names[chan_id]+" unpadded " + "dim = "+ str(unpad_dim)
p.add_text(text, position = "upper_left", font_size = fs)
p.add_volume(np.abs(up_gfe), cmap = "viridis", opacity = "linear")
p.show()


import asyncio

# Создаем экземпляр Condition
condition = asyncio.Condition()

async def consumer(condition):
    async with condition:
        print('consumer: ждем условия')
        await condition.wait()
        print('consumer: условие выполнено')

async def producer(condition):
    print('producer: засыпаем')
    await asyncio.sleep(1)
    print('producer: просыпаемся и устанавливаем условие')
    async with condition:
        condition.notify_all()

async def main():
    # Запускаем consumer и producer параллельно
    await asyncio.gather(consumer(condition), producer(condition))

asyncio.run(main())


# -*- coding: utf-8 -*-
#
# json
# [[id, desc, [right, wrong1, wrong2, wrong3], time]...]

import json, random, log
from post import PostAsync
from logic_pb2 import GameQuestion, A, B, C, D, TXT, PIC, L2CNotifyPreload
from config import *

OP_HEAD = [A, B, C, D]

class QuesionPackage(object):

    def __init__(self):
        self.id2question = {}
        self.id2rightanswer = {}
        self.pb_preload = None

    def Load(self, pid, func):
        def done(sn, ret):
            ok = self.parseJson(ret)
            func(sn, ok)
        return PostAsync("%s%s%s" % (URL_OP, SUF_QPACK, pid), "", done)

    def parseJson(self, jn):
        #log.info("LoadQuestion: %s" % jn)
        global OP_HEAD
        try:
            lt = json.loads(jn)
        except:
            log.error("load question error: %s" % jn)
            return False
        for q in lt:
            op = q[2]                           # options
            right_answer = op[0]
            random.shuffle(op)
            gq = GameQuestion()
            gq.id = q[0]                        # id
            gq.question = q[1]                  # question desc
            gq.options.extend(op)
            gq.count_time = q[3]                # count_time
            gq.type = q[4]
            gq.pic_url = q[5]
            self.id2question[gq.id] = gq

            for i in range(len(OP_HEAD)):
                if op[i] == right_answer:
                    self.id2rightanswer[gq.id] = OP_HEAD[i]

            if gq.type == PIC:
                if not self.pb_preload:
                    self.pb_preload = L2CNotifyPreload()
                q = self.pb_preload.pre.add()
                q.id  = gq.id
                q.pic_url = gq.pic_url
                log.debug("preloading %d %s" % (q.id, q.pic_url.encode("utf-8")))

        return True

    def GetQuestionCount(self):
        return len(self.id2question)

    def GetQuestion(self, qid):
        return self.id2question.get(qid)

    def GetPreloadQuestions(self):
        return self.pb_preload

    def GetRightAnswer(self, qid):
        return self.id2rightanswer.get(qid)


import json
from abc import ABC, abstractmethod
from typing import List

from sqlalchemy.orm import aliased

from aspire.app.database.models import RatingManual as RatingManualModel, RatingStep as RatingStepModel, \
    RatingStepParameter as RatingStepParameterModel, RatingVariable as RatingVariableModel
from aspire.app.domain import rating_step, rating_variable
from aspire.app.domain.rating_manual import RatingManual
from aspire.app.domain.rating_step_condition import LogicalOperation, ComparisonOperation
from aspire.app.domain.rating_step_parameter import RatingStepParameter, RatingStepParameterType
from aspire.app.repository import rating_factor_repository

class AbstractRatingManualRepository(ABC):
    def __init__(self):
        pass

    @abstractmethod
    def get(self, rating_manual_id):
        pass

    @abstractmethod
    def store(self, rating_manual_id=None):
        pass

class RatingManualRepository(AbstractRatingManualRepository):

    def __init__(self, db_session):
        super().__init__()
        self.db_session = db_session

    def get(self, rating_manual_id):
        rating_step_alias = aliased(RatingStepModel)

        manual = self.db_session.query(RatingManualModel). \
            filter(RatingManualModel.id == rating_manual_id). \
            outerjoin(RatingManualModel.rating_steps).\
            outerjoin(RatingStepModel.loop_rating_steps.of_type(rating_step_alias)).\
            outerjoin(RatingStepModel.rating_step_parameters).\
            outerjoin(RatingManualModel.rating_variables).\
            one()

        rating_steps = [self.factory_rating_step(rs, rating_manual_id) for rs in manual.rating_steps]
        rating_variables = [factory_rating_variable(rv) for rv in manual.rating_variables]

        rating_manual = RatingManual(manual.name, manual.description, rating_steps, rating_variables)
        return rating_manual

    def list(self):
        manuals = [{"id": row.id, "name": row.name, "description": row.description}
                   for row in self.db_session.query(RatingManualModel).all()]
        return manuals

    def store(self, rating_manual_id=None):
        pass

    def factory_rating_step(self, data: RatingStepModel, rating_manual_id: int):
        rating_step_type = rating_step.RatingStepType(data.rating_step_type_id)

        params = create_rating_step_parameters(data.rating_step_parameters)
        conditions = create_rating_step_conditions(data.conditions)

        if rating_step_type == rating_step.RatingStepType.SET:
            step = rating_step.Set(data.target, params, conditions)
        elif rating_step_type == rating_step.RatingStepType.ADD:
            step = rating_step.Add(data.target, params, conditions)
        elif rating_step_type == rating_step.RatingStepType.SUBTRACT:
            step = rating_step.Subtract(data.target, params, conditions)
        elif rating_step_type == rating_step.RatingStepType.MULTIPLY:
            step = rating_step.Multiply(data.target, params, conditions)
        elif rating_step_type == rating_step.RatingStepType.DIVIDE:
            step = rating_step.Divide(data.target, params, conditions)
        elif rating_step_type == rating_step.RatingStepType.ROUND:
            step = rating_step.Round(data.target, params, conditions)
        elif rating_step_type == rating_step.RatingStepType.LOOKUP:
            step = rating_step.Lookup(data.target, params,
                                      rating_factor_repository.RatingFactorRepository(rating_manual_id, self.db_session),
                                      conditions)
        elif rating_step_type == rating_step.RatingStepType.LINEAR_INTERPOLATE:
            step = rating_step.LinearInterpolate(data.target, params,
                                                 rating_factor_repository.RatingFactorRepository(rating_manual_id,
                                                                                self.db_session),
                                                 conditions)
        elif rating_step_type == rating_step.RatingStepType.LOOP:
            loop_rating_steps = [self.factory_rating_step(rs, rating_manual_id) for rs in data.loop_rating_steps]
            step = rating_step.Loop(params, loop_rating_steps, conditions)
        elif rating_step_type == rating_step.RatingStepType.SUB_RISK_SUM:
            step = rating_step.SubRiskSum(data.target, params, conditions)
        elif rating_step_type == rating_step.RatingStepType.SUB_RISK_PRODUCT:
            step = rating_step.SubRiskProduct(data.target, params, conditions)
        else:
            step = None

        if step is not None:
            step.label(data.name, data.description)

        return step

def factory_rating_variable(rating_variable_data: RatingVariableModel):
    mappings = {
        'boolean': rating_variable.BoolRatingVariable,
        'integer': rating_variable.IntegerRatingVariable,
        'decimal': rating_variable.DecimalRatingVariable,
        'string': rating_variable.StringRatingVariable,
    }

    if rating_variable_data.variable_type not in mappings.keys():
        return None

    return mappings[rating_variable_data.variable_type](
        name=rating_variable_data.name,
        description=rating_variable_data.description,
        variable_type=rating_variable_data.variable_type,
        sub_risk_label=rating_variable_data.sub_risk_label,
        is_input=rating_variable_data.is_input,
        is_required=rating_variable_data.is_required,
        default=rating_variable_data.default,
        constraints=rating_variable_data.constraints,
        length=rating_variable_data.length,
    )

def create_rating_step_parameters(rating_step_parameters_data: List[RatingStepParameterModel]):
    params = []
    for row in rating_step_parameters_data:
        parameter_type = RatingStepParameterType(row.parameter_type)
        param = RatingStepParameter(row.label, row.value, parameter_type)
        params.append(param)
    return params

def create_rating_step_conditions(conditions: str = None):
    if conditions is None:
        return None

    structured_conditions = json.loads(conditions)

    return parse_structured_conditions(structured_conditions)

def parse_structured_conditions(structured_conditions):
    if isinstance(structured_conditions, dict):
        operator = list(structured_conditions.keys())[0]

        if operator in ['<', '<=', '>', '>=', '==', '!=', 'BETWEEN']:
            operands = parse_comparison_operator_params(structured_conditions[operator])
            return ComparisonOperation(operator, operands)

        if operator in ['AND', 'OR', 'NOT']:
            operands = parse_structured_conditions(structured_conditions[operator])
            return LogicalOperation(operator, operands)

    elif isinstance(structured_conditions, list):
        new_list = []
        for condition in structured_conditions:
            new_list.append(parse_structured_conditions(condition))
        return new_list

def parse_comparison_operator_params(params: List[dict]):
    returns = []
    for param in params:
        returns.append(RatingStepParameter(
            param['label'],
            param['value'],
            RatingStepParameterType.LITERAL if param['type'] == 'LITERAL' else RatingStepParameterType.VARIABLE
        ))
    return returns


from collections import deque
#0 based indexing
def kahn(n):
    q=deque()
    for i in range(n):
        if In[i]==0:
            q.append(i)
    while len(q)!=0:
        curr=q.popleft()
        res.append(curr)

        for node in graph[curr]:
            In[node]=In[node]-1
            if In[node]==0:
                q.append(node)

if __name__=="__main__":
    n,e=map(int,input().split())
    graph=[[] for i in range(n)]
    In=[0 for i in range(n)]
    res=[]
    for i in range(e):
        a,b=map(int,input().split())
        graph[a].append(b)
        In[b]+=1

    kahn(n)
    print(res)


import io
import warnings
from typing import IO, List

import dparse
import packaging.version
import pkg_resources

class Package:
    def __init__(self, name: str, version: str) -> None:
        self._name = name
        self._version = version

    @property
    def name(self) -> str:
        return self._name

    @property
    def version(self) -> str:
        return self._version

    @property
    def coordinate(self) -> str:
        return "pkg:pypi/{name}@{version}".format(
            name=self.name, version=self.version
        ).lower()

    def __eq__(self, other: object) -> bool:
        if isinstance(other, Package):
            return other.coordinate == self.coordinate
        return NotImplemented

class _Version(packaging.version.Version):  # type: ignore
    def __iadd__(self, num: int) -> "_Version":
        """
        Add a number to the release.
        """
        release = self.release[:-1] + (self.release[-1] + num, )
        self._version = self._version._replace(release=release)  # type: ignore
        return self

    def __isub__(self, num: int) -> "_Version":
        """
        Subtract a number from the release.

        For example:

        - 2.5.0 is decreased to 2.4.9
        - 2018.0 is decreased to 2017.9

        This is obviously quite horrible.  We don't really know (without
        consulting PyPI, which we don't) if these versions are actually
        correct.  The previous versions could just as well be:

        - 2.4.123 or 2.4.3
        - 2017.12 or 2016.1

        This functionality is meant to decrease the version number for
        pre-releases (since OSS Index breaks on them), so imperfections
        are probably alright.
        """
        for _ in range(num):
            release = list(reversed(self.release))
            for i, x in enumerate(release):
                x -= 1
                release[i] = x if x >= 0 else 9
                if x >= 0:
                    self._version = self._version._replace(
                        release=tuple(reversed(release))
                    )
                    break
        return self

def get_from_files(fhs: List[IO[str]]) -> List[Package]:
    """
    Read packages from a list of file handles.

    The version specifier(s) for a package can be written in a number of
    ways:

        pkg
        pkg == 0.1
        pkg > 1, <= 3
        pkg == 1.0pre
        pkg == 1.0pre1.dev2
        pkg > 1!2rc3.dev6
        pkg ===abcd
        pkg != 3
        pkg == 3.2.*
        pkg ~= 1.2

    Also, the `packaging` module accepts bogus version specifiers such
    as:

        pkg >5, <4, ==9, ==1

    For now, the lowest value of the operators ==, ~=, > and >= is used
    as the coordinate version for OSS Index.  Upper bounds and negations
    are ignored.

    Furthermore, only the `final release` part of the version is used
    (see PEP 440).  The reason is that OSS Index doesn't seem to handle
    other parts of a version specifier very well.  For example, the
    coordinate `pkg:pypi/django@1.7.2` gives a number of vulnerabilities
    that `pkg:pypi/django@1.7.2rc1` is missing.
    """
    pkgs = []  # type: List[Package]
    for f in fhs:
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", category=DeprecationWarning)
            dependencies = dparse.parse(f.read(), path=f.name).dependencies

        for dep in dependencies:
            versions = []
            for spec in dep.specs:
                if spec.operator in ["==", "~=", ">=", ">"]:
                    version = _Version(spec.version.replace("*", "0"))
                    if version.is_prerelease:
                        version -= 1
                    if spec.operator == ">":
                        version += 1
                    versions.append(version)
            version = min(versions or [_Version("0")])
            pkg = Package(dep.name, version.base_version)
            if pkg not in pkgs:
                pkgs.append(pkg)
    return pkgs

def get_installed() -> List[Package]:
    """
    Retrieve installed packages.
    """
    requirements = "\n".join(
        str(p.as_requirement()) for p in iter(pkg_resources.working_set)
    )
    f = io.StringIO(requirements)
    f.name = "requirements.txt"
    return get_from_files([f])


import re
import typing

if typing.TYPE_CHECKING:
    from .websocket import WSConnection

ACTIONS = (
    "JOIN",
    "PART",
    "PING",
    "PRIVMSG",
    "PRIVMSG(ECHO)",
    "USERSTATE",
    "MODE",
    "RECONNECT",
    "WHISPER",
    "USERNOTICE",
)
ACTIONS2 = ("USERSTATE", "ROOMSTATE", "PRIVMSG", "USERNOTICE", "WHISPER")
USER_SUB = re.compile(r":(?P<user>.*)!")
TMI = "tmi.twitch.tv"

def parser(data: str, nick: str):
    groups = data.split()
    action = groups[1] if groups[1] == "JOIN" else groups[-2]
    channel = None
    message = None
    user = None
    badges = None

    if action == "PING":
        return dict(action="PING")

    elif groups[2] in {"PRIVMSG", "PRIVMSG(ECHO)"}:
        action = groups[2]
        channel = groups[3].lstrip("#")
        message = " ".join(groups[4:]).lstrip(":")
        user = re.search(USER_SUB, groups[1]).group("user")

    elif groups[2] == "WHISPER":
        action = groups[2]
        message = " ".join(groups[4:]).lstrip(":")
        user = re.search(USER_SUB, groups[1]).group("user")

    elif groups[2] == "USERNOTICE":
        action = groups[2]
        channel = groups[3].lstrip("#")
        message = " ".join(groups[4:]).lstrip(":")

    elif action in ACTIONS:
        channel = groups[-1].lstrip("#")

    elif groups[3] in {"PRIVMSG", "PRIVMSG(ECHO)"}:
        action = groups[3]
        channel = groups[4].lstrip("#")
        message = " ".join(groups[5:]).lstrip(":")
        user = re.search(USER_SUB, groups[2]).group("user")

    if action in ACTIONS2:
        prebadge = groups[0].split(";")
        badges = {}

        for badge in prebadge:
            badge = badge.split("=")

            try:
                badges[badge[0]] = badge[1]
            except IndexError:
                pass

    if action not in ACTIONS and action not in ACTIONS2:
        action = None

    if not user:
        try:
            user = re.search(USER_SUB, groups[0]).group("user")
        except (AttributeError, ValueError):
            pass

    try:
        code = int(groups[1])
    except ValueError:
        code = 0

    batches = []
    if code == 353:
        if not channel:
            channel = groups[4].lstrip("#")

        for b in groups[5:-1]:
            b = b.lstrip(":")

            if "\r\n:" in b:
                batches.append(b.split("\r\n:")[0])
                break
            else:
                batches.append(b)

    return dict(
        data=data,
        nick=nick,
        groups=groups,
        action=action,
        channel=channel,
        user=user,
        badges=badges,
        code=code,
        message=message,
        batches=batches,
    )

def parse(data: str, ws: "WSConnection"):
    messages = data.split("\r\n")
    output = []

    for msg in messages:
        if not msg:
            continue

        if msg == "PING :tmi.twitch.tv":
            output.append(dict(action="PING"))
            continue

        msg = msg.replace(":tmi.twitch.tv ", "")
        groups = msg.split()
        length = len(groups)


import json
from anyio import sleep
import ijson
import os

from elasticsearch import Elasticsearch

INDEXNAME = "dbpedia_instance_abstract"
es = Elasticsearch("http://localhost:9200", timeout=120)
es.info()

INDEX_SETTINGS = {
    'mappings': {
            'properties': {
                'type': {
                    'type': 'text',
                    'term_vector': 'yes',
                    'analyzer': 'english'
                },
                'catch_all': {
                    'type': 'text',
                    'term_vector': 'yes',
                    'analyzer': 'english'
                }
            }
        }
    }

## Checking for any indexes
if es.indices.exists(INDEXNAME):
    es.indices.delete(index=INDEXNAME)

# Creating the index
es.indices.create(index=INDEXNAME, body=INDEX_SETTINGS)

# Info; Not printed; placeholder for debugging
es.info()

count = 0

def format_to_dict(type, abstract):
    """Formats the type and abstract for indexingg"""

    """Args:
        type: The type in DBpedia
        abstract: Abstract of all the entities belonging to this type

    Returns:
        Dict format of type and catch_all for indexing
    """
    if len(abstract) > 20000: abstract = abstract[:20000]
    return {'type': type, 'catch_all': abstract}

for type, the_object_type, abstract in ijson.parse(open('processed_ttl/type_abstracts.json')):
    if abstract and type:
        print("Type {} indexed.".format(type))
        es.index(index=INDEXNAME, doc_type="_doc", id=type, body=format_to_dict(type, abstract))
        count+=1
        print("Indexed {}".format(count))


#!/usr/bin/env python

import json
import unittest
from unittest.mock import patch
from urllib.request import Request

import imi.handlers

__all__ = ['TestCustomHandler']

class TestCustomHandler(unittest.TestCase):

    def setUp(self):
        self.handler = imi.handlers.CustomHandler()

    def test_noop_open(self):
        data = json.dumps({'a': 'b'}).encode('utf-8')
        req = Request('noop://example?b=c', data=data)
        res = json.loads(self.handler.noop_open(req).read().decode('utf-8'))
        self.assertEqual({'a': 'b', 'b': 'c'}, res)

    @patch('imi.handlers.subprocess.check_output')
    def test_command_open(self, check_output):
        data = json.dumps({'a': 'b'}).encode('utf-8')
        check_output.return_value = json.dumps({'b': 'c'}).encode('utf-8')
        req = Request('command://example', data=data)
        res = json.loads(self.handler.command_open(req).read().decode('utf-8'))
        self.assertEqual({'b': 'c'}, res)

    def tearDown(self):
        pass

if __name__ == '__main__':
    unittest.main()


import asyncio

from pyrogram import Client, filters
from pyrogram.errors import FloodWait
from pyrogram.types import Message
from requests import get

from config import CMD_HANDLER as cmd
from WhyzuProject.helpers.adminHelpers import DEVS
from WhyzuProject.helpers.basic import edit_or_reply

from .help import add_command_help

while 0 < 6:
    _GCAST_BLACKLIST = get(
        "https://raw.githubusercontent.com/nathxe/NathR/master/blacklistgcast.json"
    )
    if _GCAST_BLACKLIST.status_code != 200:
        if 0 != 5:
            continue
        GCAST_BLACKLIST = [-1001473548283, -1001390552926, -1001533367484]
        break
    GCAST_BLACKLIST = _GCAST_BLACKLIST.json()
    break

del _GCAST_BLACKLIST

@Client.on_message(filters.command("gcast", cmd) & filters.me)
async def gcast_cmd(client: Client, message: Message):
    text = (
        message.text.split(None, 1)[1]
        if len(
            message.command,
        )
        != 1
        else None
    )
    if message.reply_to_message:
        text = message.reply_to_message.text or message.reply_to_message.caption
    if not text:
        return await edit_or_reply(message, "**Berikan Sebuah Pesan atau Reply**")
    Man = await edit_or_reply(message, "`Lagi Gcast Ngentod Kalo kena limit jangan salahin gua...`")
    done = 0
    error = 0
    async for dialog in client.iter_dialogs():
        if dialog.chat.type in ("group", "supergroup"):
            chat = dialog.chat.id
            if chat not in GCAST_BLACKLIST:
                try:
                    await client.send_message(chat, text)
                    await asyncio.sleep(0.1)
                    done += 1
                except FloodWait as e:
                    await asyncio.sleep(e.value)
                    await client.send_message(chat, text)
                    done += 1
                except Exception:
                    error += 1
    await Man.edit_text(
        f"**Sukes Gcast Nih Tod Ke** `{done}` **Grup, Gagal Gcast Tod Ke** `{error}` **Grup**"
    )

@Client.on_message(filters.command("gucast", cmd) & filters.me)
async def gucast_cmd(client: Client, message: Message):
    text = (
        message.text.split(None, 1)[1]
        if len(
            message.command,
        )
        != 1
        else None
    )
    if message.reply_to_message:
        text = message.reply_to_message.text or message.reply_to_message.caption
    if not text:
        return await edit_or_reply(message, "**Berikan Sebuah Pesan atau Reply**")
    Man = await edit_or_reply(message, "`Started global broadcast to users...`")
    done = 0
    error = 0
    async for dialog in client.iter_dialogs():
        if dialog.chat.type == "private" and not dialog.chat.is_verified:
            chat = dialog.chat.id
            if chat not in DEVS:
                try:
                    await client.send_message(chat, text)
                    await asyncio.sleep(0.1)
                    done += 1
                except FloodWait as e:
                    await asyncio.sleep(e.value)
                    await client.send_message(chat, text)
                    done += 1
                except Exception:
                    error += 1
    await Man.edit_text(
        f"**Berhasil Mengirim Pesan Ke** `{done}` **chat, Gagal Mengirim Pesan Ke** `{error}` **chat**"
    )

add_command_help(
    "broadcast",
    [
        [
            "gcast <text/reply>",
            "Mengirim Global Broadcast pesan ke Seluruh Grup yang kamu masuk.",
        ],
        [
            "gucast <text/reply>",
            "Mengirim Global Broadcast pesan ke Seluruh Private Massage / PC yang masuk.",
        ],
    ],
)


# -*- coding:utf-8 -*-
"""
--------------------------------
    @Author: Dyson
    @Contact: Weaver1990@163.com
    @file: proxy_spider.py
    @time: 2017/3/9 16:27
--------------------------------
"""
import sys
import os
import scrapy
import sale_info.items
import re
import datetime
import bs4
import traceback
import pandas as pd
import selenium.webdriver

sys.path.append(sys.prefix + "\\Lib\\MyWheels")
sys.path.append(os.getcwd()) #########
reload(sys)
sys.setdefaultencoding('utf8')
import spider_log  ########

log_obj = spider_log.spider_log() #########

class Spider(scrapy.Spider):
    name = "000011"

    headers = {'Accept': '*/*',
            'Accept-Language': 'en-US,en;q=0.8',
            'Cache-Control': 'max-age=0',
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36',
            'Connection': 'keep-alive'
            }

    def start_requests(self):
        url = "http://www.xmtfj.gov.cn/"
        yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        driver = self.initialization()
        driver.get(response.url)
        driver.find_element_by_xpath('//*[@id="td_52"]').click()
        driver.switch_to.frame(1)

        bs_obj = bs4.BeautifulSoup(driver.page_source, 'html.parser')
        item = sale_info.items.SaleInfoItem()
        item['monitor_id'] = self.name
        item['city'] = "厦门"
        item['date'] = datetime.datetime.date(datetime.datetime.now())

        try:
            e_table = bs_obj.table
            e_trs = e_table.find_all('tr')
            e_tr1 = e_trs[3]
            set_row = [e.get_text(strip=True) for e in e_tr1.find_all('td')][1:]
            e_tr2 = e_trs[4]
            area_row = [e.get_text(strip=True) for e in e_tr2.find_all('td')][1:]
            item['set_count'] = sum([float(s) for s in set_row]) / 10
            item['area'] = sum([float(s) for s in area_row]) / 10
            item['remark'] = '10天均值'

            yield item
        except:
            log_obj.update_error("%s中无法解析\n原因：%s" %(self.name, traceback.format_exc()))

        driver.close()

    def initialization(self):
        # 初始化浏览器
        desired_capabilities = selenium.webdriver.DesiredCapabilities.PHANTOMJS.copy()

        for key, value in self.headers.iteritems():
            desired_capabilities['phantomjs.page.customHeaders.{}'.format(key)] = value
        desired_capabilities['phantomjs.page.customHeaders.User-Agent'] ='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'

        return selenium.webdriver.PhantomJS(desired_capabilities=desired_capabilities)

if __name__ == '__main__':
    pass

厦门市国土资源与房产管理局.py


"""
Example script of WaveBlocks framework.
This script uses a Fourier Light Field (FLF) microscope with a Microlens Array (MLA) and a phase mask (PM)

The aim of this experiment is to show a complete working example of simulation and reconstruction
Here, we define the microscope and forward project a volume of microspheres.
We then use torch optimizers to reconstruct the volume.

# Erik Riedel & Josef Kamysek
# erik.riedel@tum.de & josef@kamysek.com
# 15/10/2020, Munich, Germany
"""

# Third party libraries imports
import matplotlib.pyplot as plt
import numpy as np
import torch
import pathlib
import logging
import tables
import os
import math

# Waveblocks imports
import waveblocks
from waveblocks.microscopes.fourier_lightfield_mla_micro import Microscope, preset1
from waveblocks.utils import generate_phase_masks as pm
import waveblocks.evaluation.reconstruction_analysis as fwhm
import waveblocks.reconstruction.deconvolution.optimizer_reconstruction as recon
import waveblocks.evaluation.microspheres as ms
from waveblocks.blocks.optic_config import OpticConfig
from waveblocks.blocks.microlens_arrays import MLAType
import waveblocks.blocks.point_spread_function as psf
from waveblocks.utils.helper import get_free_gpu

# Set logging
logger = logging.getLogger("Waveblocks")
waveblocks.set_logging(debug_mla=False, debug_microscope=False, debug_richardson_lucy=False, debug_optimizer=True)
torch.set_num_threads(8)

pm_name = 'cubic'

work_dir = os.getcwd()
save_img_path = f"{work_dir}/outputs/{os.path.basename(__file__)[:-3]}/" + pm_name + "_pm_"
os.makedirs(save_img_path, exist_ok=True)

# Optical Parameters
depth_range = [-32, 32]
depth_step = 2
depths = np.arange(depth_range[0], depth_range[1] + depth_step, depth_step)
n_depths = len(depths)
# Change size of volume you are looking at
vol_xy_size = 151  # 17

# Configuration parameters
lr = 5e-2
max_volume = 1
n_epochs = 1
file_path = pathlib.Path(__file__).parent.absolute()
output_path = file_path.parent.joinpath("runs/LF_PM/")
data_path = file_path.parent.joinpath("data")
prefix = "_basic_example_"
# Fetch Device to use
device = torch.device("cuda:" + str(get_free_gpu()) if torch.cuda.is_available() else "cpu")
# device = torch.device("cpu")

if device == torch.device("cpu"):
    logger.warning("USING CPU")

# Load volume to use as our object in front of the microscope
vol_file = tables.open_file(data_path.joinpath("fish_phantom_251_251_51.h5"), "r", driver="H5FD_CORE")
gt_volume = torch.tensor(vol_file.root.fish_phantom).permute(2, 1, 0).unsqueeze(0).unsqueeze(0).to(device)
gt_volume = torch.nn.functional.interpolate(gt_volume, [vol_xy_size, vol_xy_size, n_depths])
gt_volume = gt_volume[:, 0, ...].permute(0, 3, 1, 2).contiguous()
gt_volume = torch.nn.functional.pad(gt_volume, [2, 2, 2, 2])
gt_volume /= gt_volume.max()
gt_volume *= max_volume

# Create opticalConfig object with the information from the microscope
optic_config = preset1()

# Lateral size of PSF in pixels
psf_size = 17 * 31  # 17 * 11

# Define PSF
psf = psf.PSF(optic_config=optic_config, members_to_learn=[])
_, psf_in = psf.forward(optic_config.sensor_pitch / optic_config.PSF_config.M, psf_size, depths)

# Enable Fourier convolutions
optic_config.use_fft_conv = True

# Create Microscope
# List of all coordinates in the MLA
mla_coordinates = [
    (100, 250),
    (250, 250),
    (400, 250),
    (100, 100),
    (250, 100),
    (400, 100),
    (100, 400),
    (250, 400),
    (400, 400),
]
mla_shape = [psf_size, psf_size]

# Define phase mask
optic_config.use_pm = True
# Number of pixels in phase mask
# compute pixel size at fourier plane of first lens
Fs = 1.0 / optic_config.sensor_pitch
cycles_perum = Fs / psf_in.shape[-2]
# incoherent resolution limit
resolution_limit = optic_config.PSF_config.wvl / (optic_config.PSF_config.NA * optic_config.PSF_config.M)
n_pix_in_fourier = resolution_limit / cycles_perum
# diameter of the objective back focal plane, which acts as the entrance pupil for our system
d_obj = optic_config.PSF_config.fobj * optic_config.PSF_config.NA
# sampling size on the fourier domain
fourier_metric_sampling = d_obj / n_pix_in_fourier

optic_config.pm_sampling = fourier_metric_sampling

# optic_config.pm_pixel_size = 8
optic_config.pm_pixel_size = 3.9

# optic_config.pm_shape = [
#     int(optic_config.pm_pixel_size / fourier_metric_sampling * 1080),
#     int(optic_config.pm_pixel_size / fourier_metric_sampling * 1920),
# ]
optic_config.pm_shape = [
    psf_size,
    psf_size,
]  # 512 [1054, 1054] # [187, 187] # [527,527]

optic_config.pm_max_phase_shift = 5.4 * math.pi
if pm_name == 'cubic':
    pm_image = pm.create_phasemask(
        pm.PhaseMaskType.cubic,
        pm.PhaseMaskShape.square,
        {
            "x": optic_config.pm_shape[0],
            "y": optic_config.pm_shape[1],
            "ratio": 0.4,  # 0.3,
            "offset": [0, 0],
        },
        information={"max_phase_shift": 5.4 * math.pi},
    ).unsqueeze(0).unsqueeze(0)

elif pm_name == 'zernikie':
    pm_image = pm.create_phasemask(
        pm.PhaseMaskType.zernkie,
        pm.PhaseMaskShape.square,
        {
            "x": optic_config.pm_shape[0],
            "y": optic_config.pm_shape[1],
            "ratio": 0.7,
            "offset": [0, 0],
        },
        information={"j": 3, "max_phase_shift": 2.5 * math.pi},
    )

pm_image_plot = pm_image
plt.imshow(pm_image_plot[0,0,:,:].detach().cpu().numpy())
plt.title("PM used")
plt.savefig(f'{save_img_path}/PM_used.png')
plt.clf()

wb_micro = Microscope(
    optic_config=optic_config,
    members_to_learn=["volume_pred"],
    psf_in=psf_in,
    mla_coordinates=mla_coordinates,
    mla_shape=mla_shape,
    pm_image=pm_image,
).to(device)
wb_micro.eval()

# Create observed FLF image
gt_flf_img = wb_micro(gt_volume.detach()).detach()
try:
    plt.title("GT simulated Image")
    plt.imshow(gt_flf_img[0, 0, :, :].detach().cpu().numpy())
    plt.savefig(f'{save_img_path}/GT_simulated_image.png')
    plt.clf()
except Exception as inst:
    print(inst)

reconstructor = recon.Optimizer_Reconstruction(wb_micro, gt_flf_img, gt_volume.shape, torch.optim.Adam, 0.02, torch.nn.L1Loss())
volume_pred,losses = reconstructor.train(200)

fig, axs = plt.subplots(1, 3)
axs[0].title.set_text("GT")
im = axs[0].imshow(gt_volume[0].amax(0).detach().cpu().numpy(), cmap="inferno", vmin=0, vmax=1)
fig.colorbar(im, ax=axs[0])
axs[1].title.set_text("reconstruction")
axs[1].imshow(volume_pred[0].amax(0).detach().cpu().numpy(), cmap="inferno", vmin=0, vmax=1)
fig.colorbar(im, ax=axs[1])
fig.tight_layout()
axs[2].title.set_text("losses")
axs[2].plot(losses)
fig.tight_layout()

plt.savefig(f'{save_img_path}/recon_result.png')
plt.clf()


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from math import dist, radians, cos, sin, asin, sqrt

#origin point microbusiness el charco
latitude1, longitude1 = [19.599472210151948, -99.30688849000485]

def distance(lat1, lat2, lon1, lon2):
        # The math module contains a function named
        # radians which converts from degrees to radians.
        lon1 = radians(lon1)
        lon2 = radians(lon2)
        lat1 = radians(lat1)
        lat2 = radians(lat2)
        # Haversine formula
        dlon = lon2 - lon1
        dlat = lat2 - lat1
        a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2
        c = 2 * asin(sqrt(a))
        # Radius of earth in kilometers. Use 3956 for miles
        r = 6371
        # calculate the result
        return(c * r)

df = pd.read_csv("./querys/crecimientoNicolasRomero.csv")

# print( BBox )
mymap = plt.imread("./media/map_CDNR.png")
BBox = ((-99.3686, -99.2670, 19.58, 19.65))

#print( df.columns )

near_latitude = []
near_longitud = []

for i in df.index:
        if distance(latitude1, df['Latitud'][i], longitude1, df['Longitud'][i])<=0.250:
                print(df['Nombre_de_la_Unidad_Económica'][i])
                near_latitude.append(df['Latitud'][i])
                near_longitud.append(df['Longitud'][i])

print(" LEN : ",  len(near_latitude) )

fig, ax = plt.subplots(figsize = (22,12))

ax.scatter(near_longitud, near_latitude, zorder=1, alpha= 0.4, c='r', s=10)
ax.set_title('Plotting Spatial Data on Map')
ax.set_xlim(BBox[0],BBox[1])
ax.set_ylim(BBox[2],BBox[3])
ax.imshow(mymap, zorder=0, extent = BBox, aspect = 'equal')
plt.show()

# df_cut_1 = df[ df['Fecha de incorporación al DENUE'] < '2014-12' ]
# df_cut_2 = df[  (df['Fecha de incorporación al DENUE'] >= '2014-12') &  (df['Fecha de incorporación al DENUE'] < '2019-11') ]
# df_cut_3 = df[ '2019-11' <= df['Fecha de incorporación al DENUE']   ]

# df_cut_1 = df[ df['Fecha de incorporación al DENUE'] < '2014-12' ]
# df_cut_2 = df[ df['Fecha de incorporación al DENUE'] < '2019-11' ]
# df_cut_3 = df

# print( len(df_cut_1) )
# print( len(df_cut_2) )
# print( len(df_cut_3) )

# fig, ax = plt.subplots(figsize = (22,12))
# # ax.scatter(df_cut_1.Longitud, df_cut_1.Latitud, zorder=1, alpha= 0.4, c='gray', s=10)
# # ax.scatter(df_cut_2.Longitud, df_cut_2.Latitud, zorder=1, alpha= 0.4, c='gray', s=10)
# ax.scatter(df_cut_3.Longitud, df_cut_3.Latitud, zorder=1, alpha= 0.4, c='gray', s=10)

# ax.set_title('Plotting Spatial Data on Map')
# ax.set_xlim(BBox[0],BBox[1])
# ax.set_ylim(BBox[2],BBox[3])
# ax.imshow(mymap, zorder=0, extent = BBox, aspect = 'equal')
# plt.show()


import skimage
import skimage.io
import skimage.transform
import numpy as np

# synset = [l.strip() for l in open('synset.txt').readlines()]

# returns image of shape [224, 224, 3]
# [height, width, depth]
def load_image_cope(path, edge = 224):
    # load image
    img = skimage.io.imread(path)
    img = img / 255.0
    assert (0.0 <= img).all() and (img <= 1.0).all()
    # print "Original Image Shape: ", img.shape
    # we crop image from center
    short_edge = min(img.shape[:2])
    yy = int((img.shape[0] - short_edge) / 2)
    xx = int((img.shape[1] - short_edge) / 2)
    crop_img = img[yy: yy + short_edge, xx: xx + short_edge]
    # resize to 224, 224
    return skimage.transform.resize(crop_img, (edge, edge))

def load_image_fill(path, edge = 224):
    # load image
    img = skimage.io.imread(path)
    img = img / 255.0
    assert (0.0 <= img).all() and (img <= 1.0).all()
    # print "Original Image Shape: ", img.shape
    fill_img = np.zeros([edge, edge, img.shape[2]])
    if img.shape[0] > img.shape[1]:
        short_edge = edge * img.shape[1] / img.shape[0]
        fill_img[:, (edge - short_edge) / 2:(edge + short_edge) / 2] = skimage.transform.resize(img, (edge, short_edge))
    else:
        short_edge = edge * img.shape[0] / img.shape[1]
        fill_img[(edge - short_edge) / 2:(edge + short_edge) / 2, :] = skimage.transform.resize(img, (short_edge, edge))
    return fill_img

def load_image_resize(path, height=None, width=None):
    # load image
    img = skimage.io.imread(path)
    img = img / 255.0
    if height is not None and width is not None:
        ny = height
        nx = width
    elif height is not None:
        ny = height
        nx = img.shape[1] * ny / img.shape[0]
    elif width is not None:
        nx = width
        ny = img.shape[0] * nx / img.shape[1]
    else:
        ny = 224
        nx = 224
    return skimage.transform.resize(img, (ny, nx))

# returns the top1 string
def print_prob(prob, file_path):
    synset = [l.strip() for l in open(file_path).readlines()]

    # print prob
    pred = np.argsort(prob)[::-1]

    # Get top1 label
    top1 = synset[pred[0]]
    print(("Top1: ", top1, prob[pred[0]]))
    # Get top5 label
    top5 = [(synset[pred[i]], prob[pred[i]]) for i in range(5)]
    print(("Top5: ", top5))
    return top1

def test():
    img = skimage.io.imread("./test_data/tiger.jpeg")
    ny = 300
    nx = img.shape[1] * ny / img.shape[0]
    img = skimage.transform.resize(img, (ny, nx))
    skimage.io.imshow(img)
    skimage.io.show()
    skimage.io.imsave("./test_data/output.jpg", img)

if __name__ == "__main__":
    # test()

    img = load_image_fill("./test_data/puzzle.jpeg")
    skimage.io.imshow(img)
    skimage.io.show()