text:
import librosa
import librosa.display as display
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import numpy as np
import pandas as pd
import streamlit as st

def waveplot(y: np.ndarray, sr: int, processed=None, tp: pd.DataFrame=None, fp: pd.DataFrame=None):
    plot_wave = st.checkbox("Waveplot")
    if plot_wave:
        st.sidebar.markdown("#### Waveplot settings")
        start_second = st.sidebar.number_input(
            "start second",
            min_value=0,
            max_value=len(y) // sr,
            value=0,
            step=1,
            key="waveplot_start")
        end_second = st.sidebar.number_input(
            "end second",
            min_value=0,
            max_value=len(y) // sr,
            value=len(y) // sr,
            step=1,
            key="waveplot_end")

        start_index = start_second * sr
        if end_second == len(y) // sr:
            end_index = len(y)
        else:
            end_index = end_second * sr
        fig = plt.figure(figsize=(12, 4))
        plt.grid(True)
        display.waveplot(y[start_index:end_index], sr=sr, alpha=0.5)
        if processed is not None:
            display.waveplot(
                processed[start_index:end_index],
                sr=sr,
                alpha=0.5,
                color="red")
        if tp is not None and len(tp) > 0:
            for _, row in tp.iterrows():
                plt.axvspan(row["t_min"], row["t_max"], color="g", alpha=0.5, label=str(row["species_id"]))

        if fp is not None and len(fp) > 0:
            for _, row in fp.iterrows():
                plt.axvspan(row["t_min"], row["t_max"], color="r", alpha=0.5, label=str(row["species_id"]))
        plt.legend()

        st.pyplot(fig)

def waveplot_with_annotation(y: np.ndarray,
                             sr: int,
                             annotation: pd.DataFrame,
                             filename: str,
                             processed=None):
    plot_wave = st.checkbox("Waveplot")
    if filename.endswith(".mp3"):
        filename = filename.replace(".mp3", ".wav")
    events = annotation.query(f"filename == '{filename}'")
    colors = [
        "#bf6565", "#ac7ceb", "#e3e176", "#f081e1", "#e8cb6b", "#25b4db",
        "#fa787e", "#a9f274", "#1d7335", "#797fb3"
    ]
    if plot_wave:
        st.sidebar.markdown("#### Waveplot settings")
        start_second = st.sidebar.number_input(
            "start second",
            min_value=0,
            max_value=len(y) // sr,
            value=0,
            step=1,
            key="waveplot_start")
        end_second = st.sidebar.number_input(
            "end second",
            min_value=0,
            max_value=len(y) // sr,
            value=len(y) // sr,
            step=1,
            key="waveplot_end")

        start_index = start_second * sr
        if end_second == len(y) // sr:
            end_index = len(y)
            end_second = len(y) / sr
        else:
            end_index = end_second * sr
        events_in_period = events.query(
            f"onset >= {start_second} & offset <= {end_second}")
        uniq_labels = events_in_period["ebird_code"].unique().tolist()
        fig = plt.figure(figsize=(12, 4))
        plt.grid(True)
        display.waveplot(y[start_index:end_index], sr=sr, alpha=0.5)

        used_color = []  # type: ignore
        for i, event in events_in_period.iterrows():
            onset = event.onset
            offset = event.offset
            color = colors[uniq_labels.index(event.ebird_code)]
            if color not in used_color:
                label = event.ebird_code
                used_color.append(color)
            else:
                label = "_" + event.ebird_code
            plt.axvspan(onset, offset, facecolor=color, alpha=0.5, label=label)

        plt.legend()

        if processed is not None:
            display.waveplot(
                processed[start_index:end_index],
                sr=sr,
                alpha=0.5,
                color="red")

        st.pyplot(fig)

@st.cache
def melspectrogram(y: np.ndarray, params: dict, log=True):
    melspec = librosa.feature.melspectrogram(y=y, **params)
    if log:
        melspec = librosa.power_to_db(melspec)
    return melspec

@st.cache
def spectrogram(y: np.ndarray, params: dict, log=True):
    spec = librosa.stft(y, **params)
    if log:
        spec = librosa.power_to_db(spec)
    return spec

def specshow_with_annotation(y: np.ndarray,
                             sr: int,
                             annotation: pd.DataFrame,
                             filename: str,
                             y_processed=None):
    plot_spectrogram = st.checkbox("Spectrogram plot")
    if filename.endswith(".mp3"):
        filename = filename.replace(".mp3", ".wav")
    events = annotation.query(f"filename == '{filename}'")
    colors = [
        "#bf6565", "#ac7ceb", "#e3e176", "#f081e1", "#e8cb6b", "#25b4db",
        "#fa787e", "#a9f274", "#1d7335", "#797fb3"
    ]
    if plot_spectrogram:
        st.sidebar.markdown("#### Spectrogram plot settings")
        start_second = st.sidebar.number_input(
            "start second",
            min_value=0,
            max_value=len(y) // sr,
            value=0,
            step=1,
            key="specshow_start")
        end_second = st.sidebar.number_input(
            "end second",
            min_value=0,
            max_value=len(y) // sr,
            value=len(y) // sr,
            step=1,
            key="specshow_end")
        start_index = start_second * sr
        if end_second == len(y) // sr:
            end_index = len(y)
        else:
            end_index = end_second * sr
        y_plot = y[start_index:end_index]
        if y_processed is not None:
            y_plot_processed = y_processed[start_index:end_index]
        events_in_period = events.query(
            f"onset >= {start_second} & offset <= {end_second}")
        uniq_labels = events_in_period["ebird_code"].unique().tolist()

        st.sidebar.markdown("##### (Mel)spectrogram parameters")
        mel = st.sidebar.checkbox("Mel scale", value=True)

        n_fft = st.sidebar.number_input(
            "n_fft", min_value=64, max_value=8192, value=1024, step=64)
        hop_length = st.sidebar.number_input(
            "hop_length", min_value=1, max_value=2048, value=320, step=10)
        if mel:
            n_mels = st.sidebar.number_input(
                "n_mels", min_value=1, max_value=512, value=64, step=16)
            fmin = st.sidebar.number_input(
                "fmin", min_value=1, max_value=8192, value=20, step=100)
            fmax = st.sidebar.number_input(
                "fmax", min_value=4000, max_value=44100, value=14000, step=100)
        log = st.sidebar.checkbox("apply log", value=True)

        if mel:
            melspec_params = {
                "n_fft": n_fft,
                "hop_length": hop_length,
                "n_mels": n_mels,
                "fmin": fmin,
                "fmax": fmax,
                "sr": sr
            }
        else:
            spec_params = {
                "n_fft": n_fft,
                "hop_length": hop_length
            }

        if st.button("Show melspectrogram"):
            with st.spinner("Calculating melspectrogram"):
                if mel:
                    spec = melspectrogram(y_plot, melspec_params, log)
                else:
                    spec = spectrogram(y_plot, spec_params, log)
                if y_processed is not None:
                    if mel:
                        spec_processed = melspectrogram(y_plot_processed,
                                                        melspec_params, log)
                    else:
                        spec_processed = spectrogram(y_plot_processed,
                                                     spec_params, log)

            height, width = spec.shape
            st.write(f"{height} x {width} matrix")
            if y_processed is not None:
                with st.spinner("Plotting"):
                    fig = plt.figure(figsize=(12, 8))
                    ax1 = fig.add_subplot(2, 1, 1)
                    if mel:
                        display.specshow(
                            spec,
                            sr=sr,
                            hop_length=hop_length,
                            x_axis="time",
                            y_axis="mel",
                            fmin=fmin,
                            fmax=fmax,
                            ax=ax1)
                    else:
                        display.specshow(
                            spec,
                            sr=sr,
                            hop_length=hop_length,
                            x_axis="time",
                            y_axis="linear",
                            ax=ax1)

                    used_color = []  # type: ignore
                    for i, event in events_in_period.iterrows():
                        onset = event.onset
                        offset = event.offset
                        color = colors[uniq_labels.index(event.ebird_code)]
                        if color not in used_color:
                            label = event.ebird_code
                            used_color.append(color)
                        else:
                            label = "_" + event.ebird_code
                        ax1.axvspan(
                            onset,
                            offset,
                            facecolor=color,
                            alpha=0.5,
                            label=label)

                    ax1.legend()

                    ax2 = fig.add_subplot(2, 1, 2)
                    if mel:
                        display.specshow(
                            spec_processed,
                            sr=sr,
                            hop_length=hop_length,
                            x_axis="time",
                            y_axis="mel",
                            fmin=fmin,
                            fmax=fmax,
                            ax=ax2)
                    else:
                        display.specshow(
                            spec_processed,
                            sr=sr,
                            hop_length=hop_length,
                            x_axis="time",
                            y_axis="linear",
                            ax=ax2)

            else:
                with st.spinner("Plotting"):
                    fig = plt.figure(figsize=(12, 4))
                    if mel:
                        display.specshow(
                            spec,
                            sr=sr,
                            hop_length=hop_length,
                            x_axis="time",
                            y_axis="mel",
                            fmin=fmin,
                            fmax=fmax)
                        plt.colorbar()
                    else:
                        display.specshow(
                            spec,
                            sr=sr,
                            hop_length=hop_length,
                            x_axis="time",
                            y_axis="linear")
                        plt.colorbar()

                    used_color = []  # type: ignore
                    for i, event in events_in_period.iterrows():
                        onset = event.onset
                        offset = event.offset
                        color = colors[uniq_labels.index(event.ebird_code)]
                        if color not in used_color:
                            label = event.ebird_code
                            used_color.append(color)
                        else:
                            label = "_" + event.ebird_code
                        plt.axvspan(
                            onset,
                            offset,
                            facecolor=color,
                            alpha=0.5,
                            label=label)

                        plt.legend()

            st.pyplot(fig)

def specshow(y: np.ndarray, sr: int, y_processed=None, tp: pd.DataFrame=None, fp: pd.DataFrame=None):
    plot_spectrogram = st.checkbox("Spectrogram plot")
    if plot_spectrogram:
        st.sidebar.markdown("#### Spectrogram plot settings")
        start_second = st.sidebar.number_input(
            "start second",
            min_value=0,
            max_value=len(y) // sr,
            value=0,
            step=1,
            key="specshow_start")
        end_second = st.sidebar.number_input(
            "end second",
            min_value=0,
            max_value=len(y) // sr,
            value=len(y) // sr,
            step=1,
            key="specshow_end")
        start_index = start_second * sr
        if end_second == len(y) // sr:
            end_index = len(y)
        else:
            end_index = end_second * sr
        y_plot = y[start_index:end_index]
        if y_processed is not None:
            y_plot_processed = y_processed[start_index:end_index]

        st.sidebar.markdown("##### (Mel)spectrogram parameters")
        mel = st.sidebar.checkbox("Mel scale", value=True)

        n_fft = st.sidebar.number_input(
            "n_fft", min_value=64, max_value=8192, value=1024, step=64)
        hop_length = st.sidebar.number_input(
            "hop_length", min_value=1, max_value=2048, value=320, step=10)
        if mel:
            n_mels = st.sidebar.number_input(
                "n_mels", min_value=1, max_value=512, value=64, step=16)
            fmin = st.sidebar.number_input(
                "fmin", min_value=1, max_value=8192, value=20, step=100)
            fmax = st.sidebar.number_input(
                "fmax", min_value=4000, max_value=44100, value=14000, step=100)
        log = st.sidebar.checkbox("apply log", value=True)

        if mel:
            melspec_params = {
                "n_fft": n_fft,
                "hop_length": hop_length,
                "n_mels": n_mels,
                "fmin": fmin,
                "fmax": fmax,
                "sr": sr
            }
        else:
            spec_params = {
                "n_fft": n_fft,
                "hop_length": hop_length
            }

        if st.button("Show melspectrogram"):
            with st.spinner("Calculating melspectrogram"):
                if mel:
                    spec = melspectrogram(y_plot, melspec_params, log)
                else:
                    spec = spectrogram(y_plot, spec_params, log)
                if y_processed is not None:
                    if mel:
                        spec_processed = melspectrogram(y_plot_processed,
                                                        melspec_params, log)
                    else:
                        spec_processed = spectrogram(y_plot_processed,
                                                     spec_params, log)

            height, width = spec.shape
            st.write(f"{height} x {width} matrix")
            if y_processed is not None:
                with st.spinner("Plotting"):
                    fig = plt.figure(figsize=(12, 8))
                    ax1 = fig.add_subplot(2, 1, 1)
                    if mel:
                        display.specshow(
                            spec,
                            sr=sr,
                            hop_length=hop_length,
                            x_axis="time",
                            y_axis="mel",
                            fmin=fmin,
                            fmax=fmax,
                            ax=ax1)
                    else:
                        display.specshow(
                            spec,
                            sr=sr,
                            hop_length=hop_length,
                            x_axis="time",
                            y_axis="linear",
                            ax=ax1)

                    ax2 = fig.add_subplot(2, 1, 2)
                    if mel:
                        display.specshow(
                            spec_processed,
                            sr=sr,
                            hop_length=hop_length,
                            x_axis="time",
                            y_axis="mel",
                            fmin=fmin,
                            fmax=fmax,
                            ax=ax2)
                    else:
                        display.specshow(
                            spec_processed,
                            sr=sr,
                            hop_length=hop_length,
                            x_axis="time",
                            y_axis="linear",
                            ax=ax2)
            else:
                with st.spinner("Plotting"):
                    fig = plt.figure(figsize=(12, 4))
                    ax = plt.axes()
                    if mel:
                        display.specshow(
                            spec,
                            sr=sr,
                            hop_length=hop_length,
                            x_axis="time",
                            y_axis="mel",
                            fmin=fmin,
                            fmax=fmax)
                        plt.colorbar()
                    else:
                        display.specshow(
                            spec,
                            sr=sr,
                            hop_length=hop_length,
                            x_axis="time",
                            y_axis="linear")
                        plt.colorbar()
                    if tp is not None and len(tp) > 0:
                        for _, row in tp.iterrows():
                            rect = patches.Rectangle(
                                (row["t_min"], row["f_min"]),
                                row["t_max"] - row["t_min"],
                                row["f_max"] - row["f_min"],
                                linewidth=1,
                                edgecolor="g",
                                facecolor="g",
                                alpha=0.5,
                                label="tp")
                            ax.add_patch(rect)
                    if fp is not None and len(fp) > 0:
                        for _, row in fp.iterrows():
                            rect = patches.Rectangle(
                                (row["t_min"], row["f_min"]),
                                row["t_max"] - row["t_min"],
                                row["f_max"] - row["f_min"],
                                linewidth=1,
                                edgecolor="r",
                                facecolor="r",
                                alpha=0.5,
                                label="fp")
                            ax.add_patch(rect)
            st.pyplot(fig)


text:
"""
A module to explain models using the exact or approximate
shapley interaction index and shapley value. Exact computation
is NP-Hard in the number of features - sampling is
recommended for instances with larger numbers of features.
"""

import tensorflow as tf
import numpy as np
from tqdm import tqdm
from itertools import chain, combinations
from scipy.special import comb

class SamplingExplainerTF():
    def __init__(self, model):
        """
        Args:
            model: A tf.keras.Model instance.
        """
        self.model = model

    @staticmethod
    def powerset(iterable):
        s = list(iterable)
        return chain.from_iterable(combinations(s, r) for r in range(len(s)+1))

    @staticmethod
    def weight_coeff(S, N, interaction=False):
        if interaction:
            return 1.0 / ((N - 1) * comb(N - 2, S))
        else:
            return 1.0 / (N * comb(N - 1, S))

    def _call_model(self,
                    inputs,
                    output_index=None):
        outputs = self.model(inputs)

        if output_index is not None:
            outputs = outputs[:, output_index]
        elif len(outputs.shape) > 1:
             outputs = np.sum(outputs, axis=-1)
        return outputs

    def _sampling_estimate(self,
                           batch_inputs,
                           batch_baselines,
                           feature_index,
                           number_of_samples,
                           output_index=None,
                           interaction=False):
        accumulated_differences = []
        num_features = batch_inputs.shape[1]

        if interaction:
            i, j = feature_index
            feature_indices = np.arange(num_features)
            feature_indices = np.delete(feature_indices, j)

            for _ in range(number_of_samples):
                np.random.shuffle(feature_indices)
                pos = np.where(feature_indices == i)[0][0]

                batch_with_S = batch_baselines.copy()
                batch_with_S[:, feature_indices[:pos]] = batch_inputs[:, feature_indices[:pos]]
                v_S = self._call_model(batch_with_S, output_index)

                batch_with_S[:, i] = batch_inputs[:, i]
                v_S_i = self._call_model(batch_with_S, output_index)
                batch_with_S[:, i] = batch_baselines[:, i]

                batch_with_S[:, j] = batch_inputs[:, j]
                v_S_j = self._call_model(batch_with_S, output_index)

                batch_with_S[:, i] = batch_inputs[:, i]
                v_S_ij = self._call_model(batch_with_S, output_index)

                discrete_derivative = v_S_ij - v_S_i - v_S_j + v_S
                accumulated_differences.append(discrete_derivative)
        else:
            feature_indices = np.arange(num_features)
            for _ in range(number_of_samples):
                np.random.shuffle(feature_indices)
                pos = np.where(feature_indices == feature_index)[0][0]

                batch_with_S = batch_baselines.copy()
                batch_with_S[:, feature_indices[:pos]] = batch_inputs[:, feature_indices[:pos]]

                v_S = self._call_model(batch_with_S, output_index)

                batch_with_S[:, feature_index] = batch_inputs[:, feature_index]
                v_S_i = self._call_model(batch_with_S, output_index)
                difference = v_S_i - v_S
                accumulated_differences.append(difference)

        accumulated_differences = np.stack(accumulated_differences, axis=1)
        return np.mean(accumulated_differences, axis=1)

    def _batch_attributions(self,
                            batch_inputs,
                            batch_baselines,
                            number_of_samples=None,
                            output_index=None):
        num_features = batch_inputs.shape[1]
        batch_attributions = np.zeros(batch_inputs.shape)

        if number_of_samples is None:
            feature_powerset = SamplingExplainerTF.powerset(range(num_features))
            for S in feature_powerset:
                # First we create the subset with the features in S
                batch_with_S = batch_baselines.copy()
                batch_with_S[:, S] = batch_inputs[:, S]
                v_S = self._call_model(batch_with_S, output_index)

                for feature in filter(lambda x: x not in S,
                                      range(num_features)):
                    # Then we create the subset S u {i}
                    batch_with_S[:, feature] = batch_inputs[:, feature]
                    v_S_i = self._call_model(batch_with_S, output_index)

                    # We return to just the subset S
                    batch_with_S[:, feature] = batch_baselines[:, feature]

                    # Now we compute the contributions
                    difference = v_S_i - v_S

                    # And weight them appropriately
                    batch_attributions[:, feature] += difference * \
                        SamplingExplainerTF.weight_coeff(len(S), num_features)
        else:
            for feature_index in range(num_features):
                batch_importance = self._sampling_estimate(batch_inputs,
                                                           batch_baselines,
                                                           feature_index,
                                                           number_of_samples,
                                                           output_index=None)
                batch_attributions[:, feature_index] = batch_importance

        return batch_attributions

    def _get_discrete_derivative(self, batch_with_S, batch_baselines, batch_inputs, output_index, i, j):
        # First we create the subset with the features in S
        v_S = self._call_model(batch_with_S, output_index)

        # Then we create the subset S u {i}
        batch_with_S[:, i] = batch_inputs[:, i]
        v_S_i = self._call_model(batch_with_S, output_index)
        batch_with_S[:, i] = batch_baselines[:, i]

        # Then we create the subset S u {j}
        batch_with_S[:, j] = batch_inputs[:, j]
        v_S_j = self._call_model(batch_with_S, output_index)

        # Then we create the subset S u {i, j}
        batch_with_S[:, i] = batch_inputs[:, i]
        v_S_ij = self._call_model(batch_with_S, output_index)
        batch_with_S[:, i] = batch_baselines[:, i]
        batch_with_S[:, j] = batch_baselines[:, j]

        discrete_derivative = v_S_ij - v_S_i - v_S_j + v_S
        return discrete_derivative

    def _batch_interactions(self,
                            batch_inputs,
                            batch_baselines,
                            number_of_samples=None,
                            feature_index=None,
                            output_index=None):
        num_samples = batch_inputs.shape[0]
        num_features = batch_inputs.shape[1]

        if feature_index is not None:
            batch_interactions = np.zeros(num_samples)
        else:
            batch_interactions = np.zeros((num_samples, num_features, num_features))

        if number_of_samples is None:
            if feature_index is not None:
                indices = np.arange(num_features)
                i, j = feature_index
                np.delete(indices, feature_index)

                feature_powerset = SamplingExplainerTF.powerset(indices)
                for S in feature_powerset:
                    batch_with_S = batch_baselines.copy()
                    batch_with_S[:, S] = batch_inputs[:, S]
                    discrete_derivative = self._get_discrete_derivative(batch_with_S, batch_baselines, batch_inputs, output_index, i, j)

                    # And weight them appropriately
                    batch_interactions += discrete_derivative * \
                        SamplingExplainerTF.weight_coeff(len(S),
                                                         num_features,
                                                         interaction=True)
            else:
                feature_powerset = SamplingExplainerTF.powerset(range(num_features))
                for S in feature_powerset:
                    batch_with_S = batch_baselines.copy()
                    batch_with_S[:, S] = batch_inputs[:, S]

                    for i in filter(lambda x: x not in S,
                                    range(num_features)):
                        for j in filter(lambda x: x not in S,
                                        range(i + 1, num_features)):
                            discrete_derivative = self._get_discrete_derivative(batch_with_S, batch_baselines, batch_inputs, output_index, i, j)

                            # And weight them appropriately
                            batch_interactions[:, i, j] += discrete_derivative * \
                                SamplingExplainerTF.weight_coeff(len(S),
                                                                 num_features,
                                                                 interaction=True)
                            batch_interactions[:, j, i] = batch_interactions[:, i, j]
        else:
            if feature_index is not None:
                i, j = feature_index
                batch_importance = self._sampling_estimate(batch_inputs,
                                                               batch_baselines,
                                                               feature_index=(i, j),
                                                               number_of_samples=number_of_samples,
                                                               output_index=output_index,
                                                               interaction=True)
                return batch_importance
            else:
                for i in range(num_features):
                    for j in range(i + 1, num_features):
                        batch_importance = self._sampling_estimate(batch_inputs,
                                                                   batch_baselines,
                                                                   feature_index=(i, j),
                                                                   number_of_samples=number_of_samples,
                                                                   output_index=output_index,
                                                                   interaction=True)
                        batch_interactions[:, i, j] = batch_importance
                        batch_interactions[:, j, i] = batch_importance
        return batch_interactions

    def attributions(self,
                     inputs,
                     baselines,
                     batch_size=50,
                     number_of_samples=None,
                     output_index=None,
                     verbose=False):
        """
        A function to compute attributions directly using the shapley value.
        This function can either churn through an exponential number of computations,
        or sample uniformly from the subsets to get approximate attributions.

        Args:
            inputs: A tensor of inputs to the model of shape (batch_size, ...).
            baselines: A tensor of inputs to the model, either of shape (k, ...)
                       or of shape (...). In the latter case, uses the same baseline for all inputs.
                       In the former case, we assume k == batch_size and each baseline is fixed to the batch.
            batch_size: The maximum number of inputs to input into the model
                        simultaneously.
            number_of_samples: The number of samples to approximate the shapley value
                               with. If this parameter is None, it computes them exactly.
            output_index: Set to an integer to index into the output. Otherwise sums the outputs.
            verbose: Set to true to print a progress bar during computation.
        """
        num_inputs   = inputs.shape[0]
        num_features = inputs.shape[1]

        attributions = np.zeros((num_inputs, num_features))

        iterable = range(0, num_inputs, batch_size)
        if verbose:
            iterable = tqdm(iterable)

        for i in iterable:
            effective_batch_size = min(batch_size, num_inputs - i)
            batch_inputs = inputs[i:i + effective_batch_size]

            if len(baselines.shape) == len(inputs[0].shape):
                batch_baselines = baselines.copy()
                batch_baselines = batch_baselines[np.newaxis, :]
                batch_baselines = np.tile(batch_baselines, reps=(effective_batch_size, 1))
            else:
                batch_baselines = baselines[i:i + effective_batch_size]

            batch_attributions = self._batch_attributions(batch_inputs,
                                                          batch_baselines,
                                                          number_of_samples=number_of_samples,
                                                          output_index=output_index)
            attributions[i:i + effective_batch_size] = batch_attributions
        return attributions

    def interactions(self,
                     inputs,
                     baselines,
                     batch_size=50,
                     number_of_samples=None,
                     output_index=None,
                     feature_index=None,
                     verbose=False):
        """
        A function to compute interactions directly using the shapley interaction index.
        This function can either churn through an exponential number of computations,
        or sample uniformly from the subsets to get approximate attributions.

        Args:
            inputs: A tensor of inputs to the model of shape (batch_size, ...).
            baselines: A tensor of inputs to the model, either of shape (k, ...)
                       or of shape (...). In the latter case, uses the same baseline for all inputs.
                       In the former case, we assume k == batch_size and each baseline is fixed to the batch.
            batch_size: The maximum number of inputs to input into the model
                        simultaneously.
            number_of_samples: The number of samples to approximate the shapley value
                               with. If this parameter is None, it computes them exactly.
            output_index: Set to an integer to index into the output. Otherwise sums the outputs.
            verbose: Set to true to print a progress bar during computation.
        """
        num_inputs   = inputs.shape[0]
        num_features = inputs.shape[1]

        if feature_index is not None:
            interactions = np.zeros(num_inputs)
        else:
            interactions = np.zeros((num_inputs, num_features, num_features))

        iterable = range(0, num_inputs, batch_size)
        if verbose:
            iterable = tqdm(iterable)

        for i in iterable:
            effective_batch_size = min(batch_size, num_inputs - i)
            batch_inputs = inputs[i:i + effective_batch_size]

            if len(baselines.shape) == len(inputs[0].shape):
                batch_baselines = baselines.copy()
                batch_baselines = batch_baselines[np.newaxis, :]
                batch_baselines = np.tile(batch_baselines, reps=(effective_batch_size, 1))
            else:
                batch_baselines = baselines[i:i + effective_batch_size]

            batch_interactions = self._batch_interactions(batch_inputs,
                                                          batch_baselines,
                                                          number_of_samples=number_of_samples,
                                                          output_index=output_index,
                                                          feature_index=feature_index)
            interactions[i:i + effective_batch_size] = batch_interactions
        return interactions

ext: py

reason: import numpy,from scipy


text:
"""Function for loading various datasets used in training contradictory-claims BERT model."""

# -*- coding: utf-8 -*-

import json
import xml.etree.ElementTree as ET  # TODO: Fix error # noqa: S405, N817
from itertools import combinations
from typing import List

import numpy as np
import pandas as pd
from contradictory_claims.models.evaluate_model import read_data_from_excel
from keras.utils import np_utils
from sklearn.model_selection import train_test_split

def replace_drug_with_spl_token(text: List[str], drug_names: List[str] = None):
    """
    Replace drug name with a special token.

    :param text: list of input text containing drug name
    :param drug_names: list of drug names to replace
    :return: Text with all drug occurrances replaced by special token
    """
    text_out = text
    for drug in drug_names:
        text_out = [str(t).replace(drug, "$drug$") for t in text_out]

    return text_out

def load_multi_nli(train_path: str, test_path: str, drug_names: List[str] = None, multi_class: bool = True,
                   repl_drug_with_spl_tkn: bool = False):
    """
    Load MultiNLI data for training.

    :param train_path: path to MultiNLI training data
    :param test_path: path to MultiNLI test data
    :param multi_class: if True, data is prepared for multiclass classification. If False, implies auxillary input
        and data is prepared for binary classification.
    :param drug_names: list of drug names to replace
    :param repl_drug_with_spl_tkn: if True, replace drug names with a special token
    :return: MultiNLI sentence pairs and labels for training and test sets, respectively
    """
    multinli_train_data = pd.read_csv(train_path, sep='\t', error_bad_lines=False)
    multinli_test_data = pd.read_csv(test_path, sep='\t', error_bad_lines=False)

    # Map labels to numerical (categorical) values
    multinli_train_data['gold_label'] = [2 if label == 'contradiction' else 1 if label == 'entailment' else 0 for
                                         label in multinli_train_data.gold_label]
    multinli_test_data['gold_label'] = [2 if label == 'contradiction' else 1 if label == 'entailment' else 0 for
                                        label in multinli_test_data.gold_label]

    # Insert the CLS and SEP tokens
    if multi_class:
        x_train = '[CLS]' + multinli_train_data.sentence1 + '[SEP]' + multinli_train_data.sentence2
        x_test = '[CLS]' + multinli_test_data.sentence1 + '[SEP]' + multinli_test_data.sentence2

        # Reformat to one-hot categorical variable (3 columns)
        y_train = np_utils.to_categorical(multinli_train_data.gold_label, dtype='int')
        y_test = np_utils.to_categorical(multinli_test_data.gold_label, dtype='int')
    else:
        # Add the category info (CON, ENT, NEU) as auxillary text at the end
        x_train_1 = '[CLS]' + multinli_train_data.sentence1 + '[SEP]' + multinli_train_data.sentence2 +\
                    '[SEP]' + 'CON'
        x_train_2 = '[CLS]' + multinli_train_data.sentence1 + '[SEP]' + multinli_train_data.sentence2 +\
                    '[SEP]' + 'ENT'
        x_train_3 = '[CLS]' + multinli_train_data.sentence1 + '[SEP]' + multinli_train_data.sentence2 +\
                    '[SEP]' + 'NEU'
        x_train = x_train_1.append(x_train_2)
        x_train = x_train.append(x_train_3)
        x_test_1 = '[CLS]' + multinli_test_data.sentence1 + '[SEP]' + multinli_test_data.sentence2 +\
                   '[SEP]' + 'CON'
        x_test_2 = '[CLS]' + multinli_test_data.sentence1 + '[SEP]' + multinli_test_data.sentence2 +\
                   '[SEP]' + 'ENT'
        x_test_3 = '[CLS]' + multinli_test_data.sentence1 + '[SEP]' + multinli_test_data.sentence2 +\
                   '[SEP]' + 'NEU'
        x_test = x_test_1.append(x_test_2)
        x_test = x_test.append(x_test_3)

        # Reformat to binary variable
        y_train_1 = [1 if label == 2 else 0 for label in multinli_train_data.gold_label]
        y_train_2 = [1 if label == 1 else 0 for label in multinli_train_data.gold_label]
        y_train_3 = [1 if label == 0 else 0 for label in multinli_train_data.gold_label]
        y_train = y_train_1 + y_train_2 + y_train_3
        y_train = np.array(y_train)
        y_test_1 = [1 if label == 2 else 0 for label in multinli_test_data.gold_label]
        y_test_2 = [1 if label == 1 else 0 for label in multinli_test_data.gold_label]
        y_test_3 = [1 if label == 0 else 0 for label in multinli_test_data.gold_label]
        y_test = y_test_1 + y_test_2 + y_test_3
        y_test = np.array(y_test)

    # Replace drug name occurence with special token
    if repl_drug_with_spl_tkn:
        x_train = replace_drug_with_spl_token(x_train, drug_names)
        x_test = replace_drug_with_spl_token(x_test, drug_names)

    x_train = np.array(x_train)
    x_test = np.array(x_test)

    return x_train, y_train, x_test, y_test

def load_med_nli(train_path: str, dev_path: str, test_path: str, drug_names: List[str] = None,
                 num_training_pairs_per_class: int = None,
                 multi_class: bool = True, repl_drug_with_spl_tkn: bool = False):
    """
    Load MedNLI data for training.

    :param train_path: path to MedNLI training data
    :param dev_path: path to MedNLI dev data
    :param test_path: path to MedNLI test data
    :param num_training_pairs_per_class: number of pairs of sentences to retrieve per class.
        If None, all sentence pairs are retrieved
    :param multi_class: if True, data is prepared for multiclass classification. If False, implies auxillary input
        and data is prepared for binary classification.
    :param drug_names: list of drug names to replace
    :param repl_drug_with_spl_tkn: if True, replace drug names with a special token
    :return: MedNLI sentence pairs and labels for training and test sets, respectively
    """
    # Question: how do you have long docstrings that violate character limit but not get flake8 on my case
    # about splitting into 2 lines? TODO: Figure this out
    mednli_data1 = pd.DataFrame()
    mednli_data2 = pd.DataFrame()
    mednli_test_data = pd.DataFrame()

    with open(train_path, 'r', encoding='utf-8') as f:
        for line in f:
            mednli_data1 = mednli_data1.append(json.loads(line.rstrip('\n|\r')), ignore_index=True)
    with open(dev_path, 'r', encoding='utf-8') as f:
        for line in f:
            mednli_data2 = mednli_data2.append(json.loads(line.rstrip('\n|\r')), ignore_index=True)

    # Join together training and dev sets
    mednli_data = mednli_data1.append(mednli_data2, ignore_index=True).reset_index(drop=True)

    with open(test_path, 'r', encoding='utf-8') as f:
        for line in f:
            mednli_test_data = mednli_test_data.append(json.loads(line.rstrip('\n|\r')), ignore_index=True)

    # Map labels to numerical (categorical) values
    mednli_data['gold_label'] = [2 if label == 'contradiction' else 1 if label == 'entailment' else 0 for
                                 label in mednli_data.gold_label]
    mednli_test_data['gold_label'] = [2 if label == 'contradiction' else 1 if label == 'entailment' else 0 for
                                      label in mednli_test_data.gold_label]

    # Number of training pairs per class to use. If None, use all training pairs
    if num_training_pairs_per_class is not None:
        print(f'Using only a subset of MedNLI for training: {num_training_pairs_per_class} training pairs per class')  # noqa: T001,E501
        temp = mednli_data[mednli_data.gold_label == 2].head(num_training_pairs_per_class).append(
            mednli_data[mednli_data.gold_label == 1].head(num_training_pairs_per_class)).reset_index(drop=True)
        mednli_data = temp.append(mednli_data[mednli_data.gold_label == 0].head(num_training_pairs_per_class))\
            .reset_index(drop=True)

    # Insert the CLS and SEP tokens
    if multi_class:
        x_train = '[CLS]' + mednli_data.sentence1 + '[SEP]' + mednli_data.sentence2
        x_test = '[CLS]' + mednli_test_data.sentence1 + '[SEP]' + mednli_test_data.sentence2

        # Reformat to one-hot categorical variable (3 columns)
        y_train = np_utils.to_categorical(mednli_data.gold_label)
        y_test = np_utils.to_categorical(mednli_test_data.gold_label)
    else:
        # Add the category info (CON, ENT, NEU) as auxillary text at the end
        x_train_1 = '[CLS]' + mednli_data.sentence1 + '[SEP]' + mednli_data.sentence2 +\
                    '[SEP]' + 'CON'
        x_train_2 = '[CLS]' + mednli_data.sentence1 + '[SEP]' + mednli_data.sentence2 +\
                    '[SEP]' + 'ENT'
        x_train_3 = '[CLS]' + mednli_data.sentence1 + '[SEP]' + mednli_data.sentence2 +\
                    '[SEP]' + 'NEU'
        x_train = x_train_1.append(x_train_2)
        x_train = x_train.append(x_train_3)
        x_test_1 = '[CLS]' + mednli_test_data.sentence1 + '[SEP]' + mednli_test_data.sentence2 +\
                   '[SEP]' + 'CON'
        x_test_2 = '[CLS]' + mednli_test_data.sentence1 + '[SEP]' + mednli_test_data.sentence2 +\
                   '[SEP]' + 'ENT'
        x_test_3 = '[CLS]' + mednli_test_data.sentence1 + '[SEP]' + mednli_test_data.sentence2 +\
                   '[SEP]' + 'NEU'
        x_test = x_test_1.append(x_test_2)
        x_test = x_test.append(x_test_3)

        # Reformat to binary variable
        y_train_1 = [1 if label == 2 else 0 for label in mednli_data.gold_label]
        y_train_2 = [1 if label == 1 else 0 for label in mednli_data.gold_label]
        y_train_3 = [1 if label == 0 else 0 for label in mednli_data.gold_label]
        y_train = y_train_1 + y_train_2 + y_train_3
        y_train = np.array(y_train)
        y_test_1 = [1 if label == 2 else 0 for label in mednli_test_data.gold_label]
        y_test_2 = [1 if label == 1 else 0 for label in mednli_test_data.gold_label]
        y_test_3 = [1 if label == 0 else 0 for label in mednli_test_data.gold_label]
        y_test = y_test_1 + y_test_2 + y_test_3
        y_test = np.array(y_test)

    # Replace drug name occurence with special token
    if repl_drug_with_spl_tkn:
        x_train = replace_drug_with_spl_token(x_train, drug_names)
        x_test = replace_drug_with_spl_token(x_test, drug_names)

    x_train = np.array(x_train)
    x_test = np.array(x_test)

    return x_train, y_train, x_test, y_test

def create_mancon_sent_pairs_from_xml(xml_path: str, save_path: str):
    """
    Create sentence pairs dataset from the original xml ManCon Corpus.

    :param xml_path: path to xml corpus
    :param save_path: path to save the sentence pairs dataset
    """
    xtree = ET.parse(xml_path)  # TODO: Fix error # noqa: S314
    xroot = xtree.getroot()

    manconcorpus_data = pd.DataFrame(columns=['claim', 'assertion', 'question'])

    for node in xroot:
        for claim in node.findall('CLAIM'):
            manconcorpus_data = manconcorpus_data.append({'claim': claim.text,
                                                          'assertion': claim.attrib.get('ASSERTION'),
                                                          'question': claim.attrib.get('QUESTION')},
                                                         ignore_index=True)
    # print(len(manconcorpus_data))

    questions = list(set(manconcorpus_data.question))
    con = pd.DataFrame(columns=['text_a', 'text_b', 'label'])
    ent = pd.DataFrame(columns=['text_a', 'text_b', 'label'])
    neu = pd.DataFrame(columns=['text_a', 'text_b', 'label'])

    for q in questions:
        claim_yes = pd.DataFrame(manconcorpus_data.loc[(manconcorpus_data.question == q)
                                                       & (manconcorpus_data.assertion == 'YS'), 'claim'])  # noqa: W503
        claim_no = pd.DataFrame(manconcorpus_data.loc[(manconcorpus_data.question == q)
                                                      & (manconcorpus_data.assertion == 'NO'), 'claim'])  # noqa: W503
        temp = claim_yes.assign(key=1).merge(claim_no.assign(key=1), on='key').drop('key', 1)
        temp1 = temp.rename(columns={'claim_x': 'text_a', 'claim_y': 'text_b'})
        con = con.append(temp1)
        con['label'] = 'contradiction'
        con.drop_duplicates(inplace=True)

        for i, j in list(combinations(claim_yes.index, 2)):
            ent = ent.append({'text_a': claim_yes.claim[i],
                              'text_b': claim_yes.claim[j],
                              'label': 'entailment'},
                             ignore_index=True)

        for i, j in list(combinations(claim_no.index, 2)):
            ent = ent.append({'text_a': claim_no.claim[i],
                              'text_b': claim_no.claim[j],
                              'label': 'entailment'},
                             ignore_index=True)

        claim1 = pd.DataFrame(manconcorpus_data.loc[(manconcorpus_data.question == q), 'claim'])
        claim2 = pd.DataFrame(manconcorpus_data.loc[(manconcorpus_data.question != q), 'claim'])
        temp = claim1.assign(key=1).merge(claim2.assign(key=1), on='key').drop('key', 1)
        temp1 = temp.rename(columns={'claim_x': 'text_a', 'claim_y': 'text_b'})
        neu = neu.append(temp1)
        neu['label'] = 'neutral'
        neu.drop_duplicates(inplace=True)

    transfer_data = pd.concat([con, ent, neu]).reset_index(drop=True)
    transfer_data['guid'] = transfer_data.index
    # print(len(con))
    # print(len(ent))
    # print(len(neu))
    # print(len(transfer_data))

    transfer_data.to_csv(save_path, sep='\t', index=False)

    return

def load_mancon_corpus_from_sent_pairs(mancon_sent_pair_path: str,
                                       drug_names: List[str] = None,
                                       multi_class: bool = True,  # noqa: D205,D400
                                       repl_drug_with_spl_tkn: bool = False):
    """
    Load ManConCorpus data.

    NOTE: This data must be preprocessed as sentence pairs. This format is a TSV with four
    columns: label, guid, text_a (sentence 1), and text_b (sentence 2).

    :param mancon_sent_pair_path: path to ManCon sentence pair file
    :param multi_class: if True, data is prepared for multiclass classification. If False, implies auxillary input
        and data is prepared for binary classification.
    :param drug_names: list of drug names to replace
    :param repl_drug_with_spl_tkn: if True, replace drug names with a special token
    :return: ManConCorpus sentence pairs and labels for training and test sets, respectively
    """
    mancon_data = pd.read_csv(mancon_sent_pair_path, sep='\t')
    mancon_data['label'] = [2 if label == 'contradiction' else 1 if label == 'entailment' else 0 for
                            label in mancon_data.label]
    print(f"Number of contradiction pairs: {len(mancon_data[mancon_data.label == 2])}")  # noqa: T001
    print(f"Number of entailment pairs: {len(mancon_data[mancon_data.label == 1])}")  # noqa: T001
    print(f"Number of neutral pairs: {len(mancon_data[mancon_data.label == 0])}")  # noqa: T001

    # Insert the CLS and SEP tokens
    x_train, x_test, y_train_tmp, y_test_tmp = train_test_split(
        '[CLS]' + mancon_data.text_a + '[SEP]' + mancon_data.text_b, mancon_data['label'], test_size=0.2,
        stratify=mancon_data['label'])
    if multi_class:
        # Reformat to one-hot categorical variable (3 columns)
        y_train = np_utils.to_categorical(y_train_tmp)
        y_test = np_utils.to_categorical(y_test_tmp)
    else:
        # Add the category info (CON, ENT, NEU) as auxillary text at the end
        x_train_1 = x_train + '[SEP]' + 'CON'
        x_train_2 = x_train + '[SEP]' + 'ENT'
        x_train_3 = x_train + '[SEP]' + 'NEU'
        x_train = x_train_1.append(x_train_2)
        x_train = x_train.append(x_train_3)
        x_test_1 = x_test + '[SEP]' + 'CON'
        x_test_2 = x_test + '[SEP]' + 'ENT'
        x_test_3 = x_test + '[SEP]' + 'NEU'
        x_test = x_test_1.append(x_test_2)
        x_test = x_test.append(x_test_3)

        # Reformat to binary variable
        y_train_1 = [1 if label == 2 else 0 for label in y_train_tmp]
        y_train_2 = [1 if label == 1 else 0 for label in y_train_tmp]
        y_train_3 = [1 if label == 0 else 0 for label in y_train_tmp]
        y_train = y_train_1 + y_train_2 + y_train_3
        y_train = np.array(y_train)
        y_test_1 = [1 if label == 2 else 0 for label in y_test_tmp]
        y_test_2 = [1 if label == 1 else 0 for label in y_test_tmp]
        y_test_3 = [1 if label == 0 else 0 for label in y_test_tmp]
        y_test = y_test_1 + y_test_2 + y_test_3
        y_test = np.array(y_test)

    # Replace drug name occurence with special token
    if repl_drug_with_spl_tkn:
        x_train = replace_drug_with_spl_token(x_train, drug_names)
        x_test = replace_drug_with_spl_token(x_test, drug_names)

    x_train = np.array(x_train)
    x_test = np.array(x_test)

    return x_train, y_train, x_test, y_test

def load_drug_virus_lexicons(drug_lex_path: str, virus_lex_path: str):
    """
    Load drug and virus lexicons.

    :param drug_lex_path: path to drug lexicon
    :param virus_lex_path: path to COVID-19 lexicon
    :return: lists representing drug lexicon and virus lexicon
    """
    drug_names = pd.read_csv(drug_lex_path, header=None)
    drug_names = list(drug_names[0])
    drug_names = [drug.lower() for drug in drug_names]
    print(f"{len(drug_names)} unique drugs found in training & testing corpus:")  # noqa: T001

    virus_names = pd.read_csv(virus_lex_path, header=None)
    virus_names = list(virus_names[0])

    return drug_names, virus_names

def remove_tokens_get_sentence_sbert(x: np.ndarray, y: np.ndarray):
    """Convert Data recieved as a single format by preprocessing multi_nli, med_nli or mancon.

    :param x: array containing output from one of the above functions
    :param y: array containing labels from one of the above functions

    :return: dataframe containing sentences and labels as different columnn
    """
    x_df = pd.DataFrame(x, columns=["sentences"])
    x_df["sentences"] = x_df["sentences"].astype(str)
    x_df["sentences"] = x_df["sentences"].apply(lambda x: x.replace("[CLS]", ""))
    x_df["sentence1"] = x_df["sentences"].apply(lambda x: x.split("[SEP]")[0])
    x_df["sentence2"] = x_df["sentences"].apply(lambda x: x.split("[SEP]")[-1])
    x_df.drop(["sentences"], axis=1, inplace=True)
    y_transformed = np.argmax(y, axis=1)
    # {"contradiction": 2, "entailment": 1, "neutral": 0},
    # for sbert need to change this to entail:2, contra:0, neut:1
    y_df = pd.DataFrame(y_transformed, columns=["label"])
    convert_dict = {0: 1, 1: 2, 2: 0}
    y_df["label"] = y_df["label"].apply(lambda x: convert_dict[x]).astype(int)
    df = pd.concat([x_df, y_df], axis=1)
    return df

def load_cord_pairs(data_path: str, active_sheet: str, drug_names: List[str] = None, multi_class: bool = True,
                    repl_drug_with_spl_tkn: bool = False):

    """
    Load CORD-19 annotated claim pairs for training.

    :param data_path: path to CORD training data
    :param active_sheet: name of active sheet with data
    :param multi_class: if True, data is prepared for multiclass classification. If False, implies auxillary input
        and data is prepared for binary classification.
    :param drug_names: list of drug names to replace
    :param repl_drug_with_spl_tkn: if True, replace drug names with a special token
    :return: CORD-19 sentence pairs and labels for training and test sets, respectively
    """
    cord_data = read_data_from_excel(data_path, active_sheet)

    cord_data['label'] = [2 if label == 'contradiction' else 1 if label == 'entailment' else 0 for
                          label in cord_data.annotation]
    print(f"Number of contradiction pairs: {len(cord_data[cord_data.label == 2])}")  # noqa: T001
    print(f"Number of entailment pairs: {len(cord_data[cord_data.label == 1])}")  # noqa: T001
    print(f"Number of neutral pairs: {len(cord_data[cord_data.label == 0])}")  # noqa: T001

    # Insert the CLS and SEP tokens
    x_train, x_test, y_train_tmp, y_test_tmp = train_test_split(
        '[CLS]' + cord_data.text1 + '[SEP]' + cord_data.text2, cord_data['label'], test_size=0.2,
        stratify=cord_data['label'])
    if multi_class:
        # Reformat to one-hot categorical variable (3 columns)
        y_train = np_utils.to_categorical(y_train_tmp)
        y_test = np_utils.to_categorical(y_test_tmp)
    else:
        # Add the category info (CON, ENT, NEU) as auxillary text at the end
        x_train_1 = x_train + '[SEP]' + 'CON'
        x_train_2 = x_train + '[SEP]' + 'ENT'
        x_train_3 = x_train + '[SEP]' + 'NEU'
        x_train = x_train_1.append(x_train_2)
        x_train = x_train.append(x_train_3)
        x_test_1 = x_test + '[SEP]' + 'CON'
        x_test_2 = x_test + '[SEP]' + 'ENT'
        x_test_3 = x_test + '[SEP]' + 'NEU'
        x_test = x_test_1.append(x_test_2)
        x_test = x_test.append(x_test_3)

        # Reformat to binary variable
        y_train_1 = [1 if label == 2 else 0 for label in y_train_tmp]
        y_train_2 = [1 if label == 1 else 0 for label in y_train_tmp]
        y_train_3 = [1 if label == 0 else 0 for label in y_train_tmp]
        y_train = y_train_1 + y_train_2 + y_train_3
        y_train = np.array(y_train)
        y_test_1 = [1 if label == 2 else 0 for label in y_test_tmp]
        y_test_2 = [1 if label == 1 else 0 for label in y_test_tmp]
        y_test_3 = [1 if label == 0 else 0 for label in y_test_tmp]
        y_test = y_test_1 + y_test_2 + y_test_3
        y_test = np.array(y_test)

    # Replace drug name occurence with special token
    if repl_drug_with_spl_tkn:
        x_train = replace_drug_with_spl_token(x_train, drug_names)
        x_test = replace_drug_with_spl_token(x_test, drug_names)

    x_train = np.array(x_train)
    x_test = np.array(x_test)

    return x_train, y_train, x_test, y_test

max_stars_repo_name: itsmemala/drug-lit-contradictory-claims

max_issues_repo_name: itsmemala/drug-lit-contradictory-claims

max_forks_repo_name: itsmemala/drug-lit-contradictory-claims

reason: import numpy


text:
import numpy as np

import matplotlib as mpl
mpl.use('Agg')

import matplotlib.pyplot as plt

def read_csv(f,header = False,skip = 1):
    F = open(f,"r")
    out = []
    n = -1
    for l in F:
        n += 1
        if n%skip == 0:
            temp = l.split(",")
            if header and n == 0:
                continue#out.append([t.split("\n")[0] for t in temp])
            else:
                out.append([float(t.split("\n")[0]) for t in temp])

    out = np.array(out)[:1000*int(len(out)/1000)]
    print(out.shape)
    return out

test_data = np.loadtxt("./FF_test_curves.csv")
model_output = np.reshape(read_csv("./tuning_curveswgan_FF_40_40_new.csv",header=True),[-1,1000,27])

print(test_data.shape)
print(model_output.shape)

from scipy.stats import ks_2samp as KS
from sklearn import linear_model

#things to define: r-squared, complexity, sparsity

def sparsity(tc):
    l1 = len([t for t in tc if t < 5])
    return float(l1)/len(tc)

def rsquared(tc,getp = False):
    tcrs = np.copy(np.reshape(tc,[3,3,3,1]))
    coord = np.array([[[[x,y,z] for x in range(3)] for y in range(3)] for z in range(3)])

    comb = np.concatenate([coord,tcrs],axis = -1)
    tcf = np.reshape(comb,[-1,4])
    LF = linear_model.LinearRegression()

    LF.fit(tcf[:,:3],tcf[:,-1])

    rs = LF.score(tcf[:,:3],tcf[:,-1])
    if getp:
        proj = [[np.dot(LF.coef_,t[:3]),t[-1]] for t in tcf]
        proj.sort(key = lambda x:x[0])
        proj = np.array(proj)
        proj[:,0] -= np.min(proj[:,0])
        proj[:,0] /= (np.max(proj[:,0]) - np.min(proj[:,0]) + .001)
        return np.copy(proj)
    return rs

def com(tc):

    tcrs = np.copy(np.reshape(tc,[3,3,3,1]))
    tcrs = (tcrs - tcrs.mean())/(tcrs.max() - tcrs.min())

    dif = np.stack([np.ravel(tcrs[1:]-tcrs[:-1]),np.ravel(tcrs[:,1:]-tcrs[:,:-1]),np.ravel(tcrs[:,:,1:]-tcrs[:,:,:-1])])
    return np.std(np.abs(dif))

sk = 100

def good(t):
    a = (np.max(t)>5)
    b = ((np.max(t) - np.min(t)) > .01)
    return (a and b)

test_sp = [sparsity(t) for t in test_data]
model_sp = [[sparsity(t) for t in m] for m in model_output[::sk]]

KSsp = [KS(test_sp,m)[0] for m in model_sp]

test_RS = [rsquared(t) for t in test_data if good(t)]
model_RS = [np.array([rsquared(t) for t in m if good(t)]) for m in model_output[::sk]]

test_PR = np.array([rsquared(t,getp = True) for t in test_data if good(t)])
model_PR = np.array([np.array([rsquared(t,getp = True) for t in m if good(t)]) for m in model_output[::sk]])

KSrs = [KS(test_RS,m)[0] for m in model_RS]

test_C = [com(t) for t in test_data if good(t)]
model_C = [np.array([com(t) for t in m if good(t)]) for m in model_output[::sk]]

KSC = [KS(test_C,m)[0] for m in model_C]

test_R = [np.mean(r) for r in test_data]
model_R = [np.array([np.mean(r) for r in m]) for m in model_output[::sk]]

KSR = [KS(test_R,m)[0] for m in model_R]

plt.clf()

fig = plt.figure(figsize = (5,6))
#ax = fig.add_axes([0,0,1,1])

ax1 = fig.add_axes([.15,.08,.78,.15])

ax2 = fig.add_axes([.15,.32,.33,.15])
ax3 = fig.add_axes([.6,.32,.33,.15])
ax4 = fig.add_axes([.15,.55,.33,.15])
ax5 = fig.add_axes([.6,.55,.33,.15])

ax6 = fig.add_axes([.15,.8,.33,.15])
ax7 = fig.add_axes([.6,.8,.33,.15])

gs = [500 * s * sk for s in range(len(KSR))]

ax1.plot(gs,KSR,label = "Rate")
ax1.plot(gs,KSsp,label = "Sparsity")
ax1.plot(gs,KSC,label = "Complexity")
ax1.plot(gs,KSrs,label = "R-Squared")
ax1.set_xlabel("Generator Training Step")
ax1.set_ylabel("KS Distance")
ax1.set_ylim([0,1])
ax1.legend(loc = [.4,.4],fontsize = 8)

norm = True
nbin = 25

ax2.hist(test_C,np.linspace(0,.5,nbin),histtype = "step",normed = norm,color = "k",linewidth = 2,label = "truth")
ax2.hist(model_C[0],np.linspace(0,.5,nbin),histtype = "step",normed = norm,color = "c",linewidth = 2,label = "init")
ax2.hist(model_C[-1],np.linspace(0,.5,nbin),histtype = "step",normed = norm,color = "r",linewidth = 2,label = "final")
ax2.set_xlabel("Complexity")
ax2.set_ylabel("Probability Mass")
ax2.set_ylim([0,15])
ax2.legend()

ax3.hist(test_RS,np.linspace(0,1,nbin),histtype = "step",normed = norm,color = "k",linewidth = 2)
ax3.hist(model_RS[0],np.linspace(0,1,nbin),histtype = "step",normed = norm,color = "c",linewidth = 2)
ax3.hist(model_RS[-1],np.linspace(0,1,nbin),histtype = "step",normed = norm,color = "r",linewidth = 2)
ax3.set_xlabel("R-Squared")
ax3.set_ylabel("Probability Mass")
ax3.set_ylim([0,5])

ax4.hist(test_R,np.linspace(0,50,nbin),histtype = "step",normed = norm,color = "k",linewidth = 2)
ax4.hist(model_R[0],np.linspace(0,50,nbin),histtype = "step",normed = norm,color = "c",linewidth = 2)
ax4.hist(model_R[-1],np.linspace(0,50,nbin),histtype = "step",normed = norm,color = "r",linewidth = 2)
ax4.set_xlabel("Rate")
ax4.set_ylabel("Probability Mass")
ax4.set_ylim([0,.1])

ax5.hist(test_sp,np.linspace(0,1,10),histtype = "step",normed = norm,color = "k",linewidth = 2)
ax5.hist(model_sp[0],np.linspace(0,1,10),histtype = "step",normed = norm,color = "c",linewidth = 2)
ax5.hist(model_sp[-1],np.linspace(0,1,10),histtype = "step",normed = norm,color = "r",linewidth = 2)
ax5.set_xlabel("Sparsity")
ax5.set_ylabel("Probability Mass")
ax5.set_ylim([0,5])

for k in range(5):
    ax7.plot(test_PR[k,:,0],test_PR[k,:,1],"k")
    ax7.plot(model_PR[-1][k,:,0],model_PR[-1][k,:,1],"r")
    ax7.set_xlabel("PP location")
    ax7.set_ylabel("Rate")

for k in range(5):
    ax6.plot(test_PR[k,:,0],test_PR[k,:,1],"k")
    ax6.plot(model_PR[0][k,:,0],model_PR[0][k,:,1],"r")
    ax6.set_xlabel("PP location")
    ax6.set_ylabel("Rate")

#plt.show()
plt.savefig("./Fig3_pythonversion.pdf")

max_stars_repo_name: ahmadianlab/tc-gan

max_issues_repo_name: ahmadianlab/tc-gan

max_forks_repo_name: ahmadianlab/tc-gan


text:
import os
import sys

import numpy as np

import flopy

import matplotlib.pyplot as plt

# --modify default matplotlib settings
updates = {
    "font.family": ["Univers 57 Condensed", "Arial"],
    "mathtext.default": "regular",
    "pdf.compression": 0,
    "pdf.fonttype": 42,
    "legend.fontsize": 7,
    "axes.labelsize": 8,
    "xtick.labelsize": 7,
    "ytick.labelsize": 7,
}
plt.rcParams.update(updates)

def MergeData(ndim, zdata, tb):
    sv = 0.05
    md = np.empty((ndim), float)
    md.fill(np.nan)
    found = np.empty((ndim), bool)
    found.fill(False)
    for idx, layer in enumerate(zdata):
        for jdx, z in enumerate(layer):
            if found[jdx] == True:
                continue
            t0 = tb[idx][0] - sv
            t1 = tb[idx][1] + sv
            if z < t0 and z > t1:
                md[jdx] = z
                found[jdx] = True
    return md

def LegBar(ax, x0, y0, t0, dx, dy, dt, cc):
    for c in cc:
        ax.plot([x0, x0 + dx], [y0, y0], color=c, linewidth=4)
        ctxt = "{0:=3d} years".format(t0)
        ax.text(x0 + 2.0 * dx, y0 + dy / 2.0, ctxt, size=5)
        y0 += dy
        t0 += dt
    return

def run():
    workspace = "swiex3"

    cleanFiles = False
    fext = "png"
    narg = len(sys.argv)
    iarg = 0
    if narg > 1:
        while iarg < narg - 1:
            iarg += 1
            basearg = sys.argv[iarg].lower()
            if basearg == "--clean":
                cleanFiles = True
            elif basearg == "--pdf":
                fext = "pdf"

    if cleanFiles:
        print("cleaning all files")
        print("excluding *.py files")
        files = os.listdir(workspace)
        for f in files:
            fpth = os.path.join(workspace, f)
            if os.path.isdir(fpth):
                continue
            if ".py" != os.path.splitext(f)[1].lower():
                print("  removing...{}".format(os.path.basename(f)))
                try:
                    os.remove(fpth)
                except:
                    pass
        return 0

    modelname = "swiex3"
    exe_name = "mf2005"

    nlay = 3
    nrow = 1
    ncol = 200
    delr = 20.0
    delc = 1.0
    # well data
    lrcQ1 = np.array([(0, 0, 199, 0.01), (2, 0, 199, 0.02)])
    lrcQ2 = np.array([(0, 0, 199, 0.01 * 0.5), (2, 0, 199, 0.02 * 0.5)])
    # ghb data
    lrchc = np.zeros((30, 5))
    lrchc[:, [0, 1, 3, 4]] = [0, 0, 0.0, 0.8 / 2.0]
    lrchc[:, 2] = np.arange(0, 30)
    # swi2 data
    zini = np.hstack(
        (-9 * np.ones(24), np.arange(-9, -50, -0.5), -50 * np.ones(94))
    )[np.newaxis, :]
    iso = np.zeros((1, 200), dtype=int)
    iso[:, :30] = -2
    # model objects
    ml = flopy.modflow.Modflow(
        modelname, version="mf2005", exe_name=exe_name, model_ws=workspace
    )
    discret = flopy.modflow.ModflowDis(
        ml,
        nrow=nrow,
        ncol=ncol,
        nlay=3,
        delr=delr,
        delc=delc,
        laycbd=[0, 0, 0],
        top=-9.0,
        botm=[-29, -30, -50],
        nper=2,
        perlen=[365 * 1000, 1000 * 365],
        nstp=[500, 500],
    )
    bas = flopy.modflow.ModflowBas(ml, ibound=1, strt=1.0)
    bcf = flopy.modflow.ModflowBcf(
        ml, laycon=[0, 0, 0], tran=[40.0, 1, 80.0], vcont=[0.005, 0.005]
    )
    wel = flopy.modflow.ModflowWel(ml, stress_period_data={0: lrcQ1, 1: lrcQ2})
    ghb = flopy.modflow.ModflowGhb(ml, stress_period_data={0: lrchc})
    swi = flopy.modflow.ModflowSwi2(
        ml,
        iswizt=55,
        nsrf=1,
        istrat=1,
        toeslope=0.01,
        tipslope=0.04,
        nu=[0, 0.025],
        zeta=[zini, zini, zini],
        ssz=0.2,
        isource=iso,
        nsolver=1,
    )
    oc = flopy.modflow.ModflowOc(ml, save_every=100, save_types=["save head"])
    pcg = flopy.modflow.ModflowPcg(ml)
    # write the model files
    ml.write_input()
    # run the model
    m = ml.run_model(silent=True)

    headfile = os.path.join(workspace, "{}.hds".format(modelname))
    hdobj = flopy.utils.HeadFile(headfile)
    head = hdobj.get_data(totim=3.65000e05)

    zetafile = os.path.join(workspace, "{}.zta".format(modelname))
    zobj = flopy.utils.CellBudgetFile(zetafile)
    zkstpkper = zobj.get_kstpkper()
    zeta = []
    for kk in zkstpkper:
        zeta.append(zobj.get_data(kstpkper=kk, text="ZETASRF  1")[0])
    zeta = np.array(zeta)

    fwid, fhgt = 7.00, 4.50
    flft, frgt, fbot, ftop = 0.125, 0.95, 0.125, 0.925

    colormap = plt.cm.plasma  # winter
    cc = []
    icolor = 11
    cr = np.linspace(0.0, 0.9, icolor)
    for idx in cr:
        cc.append(colormap(idx))
    lw = 0.5

    x = np.arange(-30 * delr + 0.5 * delr, (ncol - 30) * delr, delr)
    xedge = np.linspace(-30.0 * delr, (ncol - 30.0) * delr, len(x) + 1)
    zedge = [[-9.0, -29.0], [-29.0, -30.0], [-30.0, -50.0]]

    fig = plt.figure(figsize=(fwid, fhgt), facecolor="w")
    fig.subplots_adjust(
        wspace=0.25, hspace=0.25, left=flft, right=frgt, bottom=fbot, top=ftop
    )

    ax = fig.add_subplot(311)
    ax.text(
        -0.075,
        1.05,
        "A",
        transform=ax.transAxes,
        va="center",
        ha="center",
        size="8",
    )
    # confining unit
    ax.fill(
        [-600, 3400, 3400, -600],
        [-29, -29, -30, -30],
        fc=[0.8, 0.8, 0.8],
        ec=[0.8, 0.8, 0.8],
    )
    #
    z = np.copy(zini[0, :])
    zr = z.copy()
    p = (zr < -9.0) & (zr > -50.0)
    ax.plot(x[p], zr[p], color=cc[0], linewidth=lw, drawstyle="steps-mid")
    #
    for i in range(5):
        zt = MergeData(
            ncol, [zeta[i, 0, 0, :], zeta[i, 1, 0, :], zeta[i, 2, 0, :]], zedge
        )
        dr = zt.copy()
        ax.plot(x, dr, color=cc[i + 1], linewidth=lw, drawstyle="steps-mid")
    # Manufacture a legend bar
    LegBar(ax, -200.0, -33.75, 0, 25, -2.5, 200, cc[0:6])
    # axes
    ax.set_ylim(-50, -9)
    ax.set_ylabel("Elevation, in meters")
    ax.set_xlim(-250.0, 2500.0)

    ax = fig.add_subplot(312)
    ax.text(
        -0.075,
        1.05,
        "B",
        transform=ax.transAxes,
        va="center",
        ha="center",
        size="8",
    )
    # confining unit
    ax.fill(
        [-600, 3400, 3400, -600],
        [-29, -29, -30, -30],
        fc=[0.8, 0.8, 0.8],
        ec=[0.8, 0.8, 0.8],
    )
    #
    for i in range(4, 10):
        zt = MergeData(
            ncol, [zeta[i, 0, 0, :], zeta[i, 1, 0, :], zeta[i, 2, 0, :]], zedge
        )
        dr = zt.copy()
        ax.plot(x, dr, color=cc[i + 1], linewidth=lw, drawstyle="steps-mid")
    # Manufacture a legend bar
    LegBar(ax, -200.0, -33.75, 1000, 25, -2.5, 200, cc[5:11])
    # axes
    ax.set_ylim(-50, -9)
    ax.set_ylabel("Elevation, in meters")
    ax.set_xlim(-250.0, 2500.0)

    ax = fig.add_subplot(313)
    ax.text(
        -0.075,
        1.05,
        "C",
        transform=ax.transAxes,
        va="center",
        ha="center",
        size="8",
    )
    # confining unit
    ax.fill(
        [-600, 3400, 3400, -600],
        [-29, -29, -30, -30],
        fc=[0.8, 0.8, 0.8],
        ec=[0.8, 0.8, 0.8],
    )
    #
    zt = MergeData(
        ncol, [zeta[4, 0, 0, :], zeta[4, 1, 0, :], zeta[4, 2, 0, :]], zedge
    )
    ax.plot(
        x,
        zt,
        marker="o",
        markersize=3,
        linewidth=0.0,
        markeredgecolor="blue",
        markerfacecolor="None",
    )
    # ghyben herzberg
    zeta1 = -9 - 40.0 * (head[0, 0, :])
    gbh = np.empty(len(zeta1), float)
    gbho = np.empty(len(zeta1), float)
    for idx, z1 in enumerate(zeta1):
        if z1 >= -9.0 or z1 <= -50.0:
            gbh[idx] = np.nan
            gbho[idx] = 0.0
        else:
            gbh[idx] = z1
            gbho[idx] = z1
    ax.plot(x, gbh, "r")
    np.savetxt(os.path.join(workspace, "Ghyben-Herzberg.out"), gbho)
    # fake figures
    ax.plot([-100.0, -100], [-100.0, -100], "r", label="Ghyben-Herzberg")
    ax.plot(
        [-100.0, -100],
        [-100.0, -100],
        "bo",
        markersize=3,
        markeredgecolor="blue",
        markerfacecolor="None",
        label="SWI2",
    )
    # legend
    leg = ax.legend(loc="lower left", numpoints=1)
    leg._drawFrame = False
    # axes
    ax.set_ylim(-50, -9)
    ax.set_xlabel("Horizontal distance, in meters")
    ax.set_ylabel("Elevation, in meters")
    ax.set_xlim(-250.0, 2500.0)

    outfig = os.path.join(workspace, "Figure08_swi2ex3.{0}".format(fext))
    fig.savefig(outfig, dpi=300)
    print("created...", outfig)

    return 0

if __name__ == "__main__":
    success = run()

max_stars_repo_name: ntdosch/flopy

max_issues_repo_name: ntdosch/flopy

max_forks_repo_name: ntdosch/flopy


text:
"""
.. _spaCy pipeline: https://spacy.io/models
.. _stopwords available in NLTK: https://github.com/nltk/nltk_data/blob/gh-pages/packages/corpora/stopwords.zip
.. _POS-tags: https://github.com/explosion/spaCy/blob/master/spacy/glossary.py
.. _regex pattern: https://docs.python.org/3/library/re.html#regular-expression-syntax
"""

import logging
import os
from typing import List

import nltk
import numpy as np
import psutil
import scipy.sparse as sp
import spacy

class _KeyphraseVectorizerMixin():
    """
    _KeyphraseVectorizerMixin

    Provides common code for text vectorizers.
    """

    def _document_frequency(self, document_keyphrase_count_matrix: List[List[int]]) -> np.array:
        """
        Count the number of non-zero values for each feature in sparse a matrix.

        Parameters
        ----------
        document_keyphrase_count_matrix : list of integer lists
                The document-keyphrase count matrix to transform to document frequencies

        Returns
        -------
        document_frequencies : np.array
            Numpy array of document frequencies for keyphrases
        """
        document_keyphrase_count_matrix = sp.csr_matrix(document_keyphrase_count_matrix)
        document_frequencies = np.bincount(document_keyphrase_count_matrix.indices,
                                           minlength=document_keyphrase_count_matrix.shape[1])

        return document_frequencies

    def _remove_suffixes(self, text: str, suffixes: List[str]) -> str:
        """
        Removes pre-defined suffixes from a given text string.

        Parameters
        ----------
        text : str
            Text string where suffixes should be removed.

        suffixes : list
            List of strings that should be removed from the end of the text.

        Returns
        -------
        text : Text string with removed suffixes.
        """

        for suffix in suffixes:
            if text.lower().endswith(suffix.lower()):
                return text[:-len(suffix)].strip()
        return text

    def _remove_prefixes(self, text: str, prefixes: List[str]) -> str:
        """
        Removes pre-defined prefixes from a given text string.

        Parameters
        ----------
        text : str
            Text string where prefixes should be removed.

        prefixes :  list
            List of strings that should be removed from the beginning of the text.

        Returns
        -------
        text : Text string with removed prefixes.
        """

        for prefix in prefixes:
            if text.lower().startswith(prefix.lower()):
                return text[len(prefix):].strip()
        return text

    def _cumulative_length_joiner(self, text_list: List[str], max_text_length: int) -> List[str]:
        """
        Joins strings from list of strings to single string until maximum char length is reached.
        Then join the next strings from list to a single string and so on.

        Parameters
        ----------
        text_list : list of strings
            List of strings to join.

        max_text_length : int
            Maximun character length of the joined strings.

        Returns
        -------
        list_of_joined_srings_with_max_length : List of joined text strings with max char length of 'max_text_length.
        """

        # triggers a parameter validation
        if isinstance(text_list, str):
            raise ValueError(
                "Iterable over raw texts expected, string object received."
            )

        # triggers a parameter validation
        if not hasattr(text_list, '__iter__'):
            raise ValueError(
                "Iterable over texts expected."
            )

        text_list_len = len(text_list) - 1
        list_of_joined_srings_with_max_length = []
        one_string = ''
        for index, text in enumerate(text_list):
            # Add the text to the substring if it doesn't make it to large
            if len(one_string) + len(text) < max_text_length:
                one_string += ' ' + text
                if index == text_list_len:
                    list_of_joined_srings_with_max_length.append(one_string)

            # Substring too large, so add to the list and reset
            else:
                list_of_joined_srings_with_max_length.append(one_string)
                one_string = text
                if index == text_list_len:
                    list_of_joined_srings_with_max_length.append(one_string)
        return list_of_joined_srings_with_max_length

    def _split_long_document(self, text: str, max_text_length: int) -> List[str]:
        """
        Split single string in list of strings with a maximum character length.

        Parameters
        ----------
        text : str
            Text string that should be split.

        max_text_length : int
            Maximun character length of the strings.

        Returns
        -------
        splitted_document : List of text strings.
        """
        # triggers a parameter validation
        if not isinstance(text, str):
            raise ValueError(
                "'text' parameter needs to be a string."
            )

        # triggers a parameter validation
        if not isinstance(max_text_length, int):
            raise ValueError(
                "'max_text_length' parameter needs to be a int"
            )

        text = text.replace("? ", "?<stop>")
        text = text.replace("! ", "!<stop>")
        if "<stop>" in text:
            splitted_document = text.split("<stop>")
            splitted_document = splitted_document[:-1]
            splitted_document = [s.strip() for s in splitted_document]
            splitted_document = [
                self._cumulative_length_joiner(text_list=doc.split(" "), max_text_length=max_text_length) if len(
                    doc) > max_text_length else [doc] for doc in splitted_document]
            return [text for doc in splitted_document for text in doc]
        else:
            splitted_document = text.split(" ")
            splitted_document = self._cumulative_length_joiner(text_list=splitted_document,
                                                               max_text_length=max_text_length)
            return splitted_document

    def _get_pos_keyphrases(self, document_list: List[str], stop_words: str, spacy_pipeline: str, pos_pattern: str, pos_tagger: any = None,
                            lowercase: bool = True, workers: int = 1) -> List[str]:
        """
        Select keyphrases with part-of-speech tagging from a text document.
        Parameters
        ----------
        document_list : list of str
            List of text documents from which to extract the keyphrases.

        stop_words : str
            Language of stopwords to remove from the document, e.g. 'english'.
            Supported options are `stopwords available in NLTK`_.
            Removes unwanted stopwords from keyphrases if 'stop_words' is not None.

        spacy_pipeline : str
            The name of the `spaCy pipeline`_, used to tag the parts-of-speech in the text.

        pos_pattern : str
            The `regex pattern`_ of `POS-tags`_ used to extract a sequence of POS-tagged tokens from the text.

        lowercase : bool, default=True
            Whether the returned keyphrases should be converted to lowercase.

        workers :int, default=1
            How many workers to use for spaCy part-of-speech tagging.
            If set to -1, use all available worker threads of the machine.
            spaCy uses the specified number of cores to tag documents with part-of-speech.
            Depending on the platform, starting many processes with multiprocessing can add a lot of overhead.
            In particular, the default start method spawn used in macOS/OS X (as of Python 3.8) and in Windows can be slow.
            Therefore, carefully consider whether this option is really necessary.

        Returns
        -------
        keyphrases : List of unique keyphrases of varying length, extracted from the text document with the defined 'pos_pattern'.
        """

        # triggers a parameter validation
        if isinstance(document_list, str):
            raise ValueError(
                "Iterable over raw text documents expected, string object received."
            )

        # triggers a parameter validation
        if not hasattr(document_list, '__iter__'):
            raise ValueError(
                "Iterable over raw text documents expected."
            )

        # triggers a parameter validation
        if not isinstance(stop_words, str):
            raise ValueError(
                "'stop_words' parameter needs to be a string. E.g. 'english'"
            )

        # triggers a parameter validation
        if not isinstance(spacy_pipeline, str):
            raise ValueError(
                "'spacy_pipeline' parameter needs to be a spaCy pipeline string. E.g. 'en_core_web_sm'"
            )

        # triggers a parameter validation
        if not isinstance(pos_pattern, str):
            raise ValueError(
                "'pos_pattern' parameter needs to be a regex string. E.g. '<J.*>*<N.*>+'"
            )

        # triggers a parameter validation
        if not isinstance(workers, int):
            raise ValueError(
                "'workers' parameter must be of type int"
            )

        if (workers < -1) or (workers > psutil.cpu_count(logical=True)) or (workers == 0):
            raise ValueError(
                "'workers' parameter value cannot be 0 and must be between -1 and " + str(
                    psutil.cpu_count(logical=True))
            )

        stop_words_list = []
        if stop_words:
            try:
                stop_words_list = set(nltk.corpus.stopwords.words(stop_words))
            except LookupError:
                logger = logging.getLogger('KeyphraseVectorizer')
                logger.setLevel(logging.WARNING)
                sh = logging.StreamHandler()
                sh.setFormatter(logging.Formatter(
                    '%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
                logger.addHandler(sh)
                logger.setLevel(logging.DEBUG)
                logger.info(
                    'It looks like you do not have downloaded a list of stopwords yet. It is attempted to download the stopwords now.')
                nltk.download('stopwords')
                stop_words_list = set(nltk.corpus.stopwords.words(stop_words))

        # add spaCy POS tags for documents
        spacy_exclude = ['parser', 'ner', 'entity_linker', 'entity_ruler', 'textcat', 'textcat_multilabel',
                         'lemmatizer', 'morphologizer', 'senter', 'sentencizer', 'tok2vec', 'transformer']
        try:
            nlp = spacy.load(spacy_pipeline,
                             exclude=spacy_exclude)

        except OSError:
            # set logger
            logger = logging.getLogger('KeyphraseVectorizer')
            logger.setLevel(logging.WARNING)
            sh = logging.StreamHandler()
            sh.setFormatter(logging.Formatter(
                '%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
            logger.addHandler(sh)
            logger.setLevel(logging.DEBUG)
            logger.info(
                'It looks like the selected spaCy pipeline is not downloaded yet. It is attempted to download the spaCy pipeline now.')
            spacy.cli.download(spacy_pipeline)
            nlp = spacy.load(spacy_pipeline,
                             exclude=spacy_exclude)

        # add rule based sentence boundary detection
        nlp.add_pipe('sentencizer')

        if pos_tagger != None:
          pos_tagger_component = Language.component("pos_tagger", func=pos_tagger)
          nlp.add_pipe("pos_tagger", name="pos_tagger", first=True)

        keyphrases_list = []
        if workers != 1:
            os.environ["TOKENIZERS_PARALLELISM"] = "false"

        # split large documents in smaller chunks, so that spacy can process them without memory issues
        docs_list = []
        # set maximal character length of documents for spaCy processing
        max_doc_length = 1000000
        for document in document_list:
            if len(document) > max_doc_length:
                docs_list.extend(self._split_long_document(text=document, max_text_length=max_doc_length))
            else:
                docs_list.append(document)
        document_list = docs_list
        del docs_list

        # increase max length of documents that spaCy can parse
        # (should only be done if parser and ner are not used due to memory issues)
        nlp.max_length = max([len(doc) for doc in document_list]) + 100

        cp = nltk.RegexpParser('CHUNK: {(' + pos_pattern + ')}')
        for tagged_doc in nlp.pipe(document_list, n_process=workers):
            tagged_pos_doc = []
            for sentence in tagged_doc.sents:
                pos_tagged_sentence = []
                for word in sentence:
                    pos_tagged_sentence.append((word.text, word.tag_))
                tagged_pos_doc.append(pos_tagged_sentence)

            # extract keyphrases that match the NLTK RegexpParser filter
            keyphrases = []
            prefix_list = [stop_word + ' ' for stop_word in stop_words_list]
            suffix_list = [' ' + stop_word for stop_word in stop_words_list]
            for sentence in tagged_pos_doc:
                tree = cp.parse(sentence)
                for subtree in tree.subtrees():
                    if subtree.label() == 'CHUNK':
                        # join candidate keyphrase from single words
                        keyphrase = ' '.join([i[0] for i in subtree.leaves()])

                        # convert keyphrase to lowercase
                        if lowercase:
                            keyphrase = keyphrase.lower()

                        # remove stopword suffixes
                        keyphrase = self._remove_suffixes(keyphrase, suffix_list)

                        # remove stopword prefixes
                        keyphrase = self._remove_prefixes(keyphrase, prefix_list)

                        # remove whitespace from the beginning and end of keyphrases
                        keyphrase = keyphrase.strip()

                        # do not include single keywords that are actually stopwords
                        if keyphrase.lower() not in stop_words_list:
                            keyphrases.append(keyphrase)

            # remove potential empty keyphrases
            keyphrases = [keyphrase for keyphrase in keyphrases if keyphrase != '']

            keyphrases_list.append(list(set(keyphrases)))

        return list(set([keyphrase for sub_keyphrase_list in keyphrases_list for keyphrase in sub_keyphrase_list]))

max_stars_repo_name: hboisgibault/KeyphraseVectorizers

max_issues_repo_name: hboisgibault/KeyphraseVectorizers

max_forks_repo_name: hboisgibault/KeyphraseVectorizers

reason: import numpy,import scipy