text:
# Copyright (c) 2019-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
#

import os
import re
import subprocess
from logging import getLogger
from pathlib import Path

import numpy as np
import torch
from tqdm import tqdm

from codegen_sources.model.src.evaluation.evaluator import EncDecEvaluator, EVAL_DATASET_SPLITS, SRC_ST_LANGS, \
    TARGET_ST_LANG, get_l1l2_string, eval_moses_bleu, EVAL_OBF_PROBAS
from codegen_sources.model.src.evaluation.subtoken_score import run_subtoken_score
from codegen_sources.model.src.trainer import get_programming_language_name
from codegen_sources.model.src.utils import (
    to_cuda,
    restore_segmentation,
    show_batch,
    add_noise,
    convert_to_text, TREE_SITTER_ROOT, read_file_lines,
)
from codegen_sources.preprocessing.lang_processors.lang_processor import LangProcessor

from iren.utils.ranking_metrics import run_ranking_scoring

# Adding obf_proba = 0 to evaluation (1 in the bottom not a bug)
EVAL_OBF_PROBAS.append(1)

logger = getLogger()

class DistillationEvaluator(EncDecEvaluator):
    def __init__(self, trainer, data, params):
        """
        Build encoder / decoder evaluator.
        """
        super().__init__(trainer, data, params)
        self.teacher_encoder = trainer.teacher_encoder
        self.teacher_decoder = trainer.teacher_decoder
        self.teacher_scores = {}
        self.t_hyp_paths = {}

    def evaluate_mt(
            self,
            scores,
            data_set,
            lang1,
            lang2,
            eval_bleu,
            eval_computation,
            eval_subtoken_score,
            span,
            deobfuscate=False,
            deobfuscate_probas=None,
    ):
        """
        Evaluate perplexity and next word prediction accuracy.
        """
        params = self.params
        assert data_set in EVAL_DATASET_SPLITS
        assert lang1 in params.langs
        assert lang2 in params.langs
        rng = np.random.RandomState(0)
        torch_rng = torch.Generator().manual_seed(0)
        eval_st = params.eval_st
        if not params.is_master or "cl" in lang1:
            # Computing the accuracy on every node is useful for debugging but
            # no need to evaluate spend too much time on the evaluation when not on master
            eval_bleu = False
            eval_computation = False
            eval_subtoken_score = False
            eval_st = False

        # store hypothesis to compute BLEU score
        if params.eval_bleu_test_only:
            datasets_for_bleu = ["test"]
        else:
            datasets_for_bleu = [s for s in EVAL_DATASET_SPLITS if s != "train"]

        lang1_id = params.lang2id[lang1]
        lang2_id = params.lang2id[lang2]

        self.eval_mode()
        encoder, decoder = self._get_enc_dec(self.encoder, self.decoder, lang2_id, params)
        teacher_encoder, teacher_decoder = self._get_enc_dec(self.teacher_encoder, self.teacher_decoder, lang2_id,
                                                             params)

        for deobfuscation_proba in (
                deobfuscate_probas if deobfuscate_probas is not None else [None]
        ):
            if deobfuscate:
                rng = np.random.RandomState(0)
            l1l2 = get_l1l2_string(lang1, lang2, deobfuscation_proba)
            eval_name = f"{data_set}_{l1l2}"

            n_words = 0
            xe_loss = 0
            n_valid = 0
            t_xe_loss = 0
            t_n_valid = 0
            hypothesis = []
            teacher_hypothesis = []
            sources = []
            references = []
            for i, batch in enumerate(
                    tqdm(self.get_iterator(
                        data_set, lang1, lang2 if lang2 != lang1 else None, span=span
                    ), desc=f"local-rank-{params.local_rank}-obf_proba-{deobfuscation_proba}", mininterval=1.)
            ):
                spans = None
                assert len(batch) >= 2
                if len(batch) == 2:
                    if lang1 == lang2:
                        x2, len2 = batch
                        x1, len1 = add_noise(
                            x2,
                            len2,
                            self.params,
                            len(self.data["dico"]) - 1,
                            rng,
                            torch_rng,
                        )
                    else:
                        (x1, len1, ids1, len_ids1), (x2, len2, ids2, len_ids2) = batch
                        if deobfuscate:
                            (x1, len1, x2, len2) = self.trainer.deobfuscate_by_variable(
                                x1, x2, deobfuscation_proba, params.roberta_mode, rng,
                                obf_type=params.obf_type,
                                shuffle_masks=params.shuffle_dobf_masks
                            )
                            if x1 is None:
                                continue
                else:
                    assert len(batch) == 3
                    (
                        (x1, len1, ids1, len_ids1),
                        (x2, len2, ids2, len_ids2),
                        (spans, len_spans, _, _),
                    ) = batch

                langs1 = x1.clone().fill_(lang1_id)
                langs2 = x2.clone().fill_(lang2_id)

                # target words to predict
                alen = torch.arange(len2.max(), dtype=torch.long, device=len2.device)
                pred_mask = (
                        alen[:, None] < len2[None] - 1
                )  # do not predict anything given the last target word
                y = x2[1:].masked_select(pred_mask[:-1])
                assert len(y) == (len2 - 1).sum().item()

                # cuda
                x1, len1, langs1, x2, len2, langs2, y, spans = to_cuda(
                    x1, len1, langs1, x2, len2, langs2, y, spans
                )
                enc1, _, word_scores, loss = self._model_forward(encoder, decoder, langs1, langs2, len1, len2,
                                                                 pred_mask, spans, x1, x2, y)

                # update stats
                n_words += y.size(0)
                xe_loss += loss.item() * len(y)
                n_valid += (word_scores.max(1)[1] == y).sum().item()

                # generate translation - translate / convert to text
                if (
                        eval_bleu or eval_computation or eval_subtoken_score
                ) and data_set in datasets_for_bleu:
                    generated, lengths = self._generate_hypothesis(data_set, decoder, enc1, i, lang1, lang2, lang2_id,
                                                                   len1, params, x1, x2)

                    hypothesis.extend(
                        convert_to_text(
                            generated,
                            lengths,
                            self.dico,
                            params,
                            generate_several_reps=True,
                        )
                    )
                    references.extend(convert_to_text(x2, len2, self.dico, params))
                    sources.extend(convert_to_text(x1, len1, self.dico, params))

                    if eval_name not in self.t_hyp_paths:
                        t_enc1, _, t_word_scores, t_loss = self._model_forward(teacher_encoder, teacher_decoder,
                                                                               langs1, langs2, len1, len2,
                                                                               pred_mask, spans, x1, x2, y)
                        t_xe_loss += t_loss.item() * len(y)
                        t_n_valid += (t_word_scores.max(1)[1] == y).sum().item()
                        if (
                                eval_bleu or eval_computation or eval_subtoken_score
                        ) and data_set in datasets_for_bleu:
                            t_generated, t_lengths = self._generate_hypothesis(data_set, teacher_decoder, t_enc1, i,
                                                                               lang1, lang2, lang2_id, len1, params, x1,
                                                                               x2, is_teacher=True)
                            teacher_hypothesis.extend(
                                convert_to_text(
                                    t_generated,
                                    t_lengths,
                                    self.dico,
                                    params,
                                    generate_several_reps=True,
                                )
                            )

            # compute perplexity and prediction accuracy
            scores[f"{eval_name}_mt_ppl"] = np.exp(xe_loss / n_words)
            scores[f"{eval_name}_mt_acc"] = 100.0 * n_valid / n_words
            if f"{eval_name}_mt_ppl_teacher" not in self.teacher_scores:
                self.teacher_scores[f"{eval_name}_mt_ppl_teacher"] = np.exp(t_xe_loss / n_words)
                self.teacher_scores[f"{eval_name}_mt_acc_teacher"] = 100.0 * t_n_valid / n_words

            # write hypotheses
            if (
                    eval_bleu or eval_computation or eval_subtoken_score
            ) and data_set in datasets_for_bleu:
                if eval_name not in self.t_hyp_paths:
                    hyp_paths, self.t_hyp_paths[eval_name], ref_path, src_path = self.write_hypo_teacher_hypo_ref_src(
                        data_set,
                        hypothesis,
                        teacher_hypothesis,
                        lang1,
                        lang2,
                        params,
                        references,
                        scores,
                        sources,
                        deobfuscation_proba,
                    )
                else:
                    hyp_paths, ref_path, src_path = self.write_hypo_ref_src(
                        data_set,
                        hypothesis,
                        lang1,
                        lang2,
                        params,
                        references,
                        scores,
                        sources,
                        deobfuscation_proba,
                    )

            # check how many functions compiles + return same output as GT
            if eval_computation and data_set in datasets_for_bleu:
                print("compute_comp_acc")
                self.compute_comp_acc(
                    data_set,
                    hyp_paths,
                    hypothesis,
                    lang1,
                    lang2,
                    params,
                    ref_path,
                    scores,
                    roberta_mode=params.roberta_mode,
                )

            if (
                    eval_st
                    and data_set in datasets_for_bleu
                    and get_programming_language_name(lang1) == SRC_ST_LANGS
                    and get_programming_language_name(lang2) in TARGET_ST_LANG
            ):
                logger.info("Computing ST comp acc")
                self.compute_comp_acc(
                    data_set,
                    hyp_paths,
                    hypothesis,
                    lang1,
                    lang2,
                    params,
                    ref_path,
                    scores,
                    roberta_mode=params.roberta_mode,
                    evosuite_functions=True,
                )
            if eval_subtoken_score and data_set in datasets_for_bleu:
                self.compute_subtoken_metrics(scores, hyp_paths, ref_path, lang1, lang2, data_set, deobfuscation_proba)
                self.compute_subtoken_metrics(self.teacher_scores, self.t_hyp_paths[eval_name], ref_path, lang1, lang2,
                                              data_set, deobfuscation_proba, is_teacher=True)

            # compute BLEU score
            if eval_bleu and data_set in datasets_for_bleu:
                self.evaluate_bleu(scores, hyp_paths, ref_path, deobfuscation_proba, lang1, lang2, eval_computation,
                                   data_set)
                self.evaluate_bleu(self.teacher_scores, self.t_hyp_paths[eval_name], ref_path, deobfuscation_proba,
                                   lang1, lang2, eval_computation, data_set, is_teacher=True)

            if params.eval_ranking_metrics and params.beam_size > 1:
                self.compute_ranking_metrics(scores, hyp_paths, ref_path, deobfuscation_proba, lang1, lang2, data_set)
                self.compute_ranking_metrics(self.teacher_scores, self.t_hyp_paths[eval_name], ref_path,
                                             deobfuscation_proba, lang1, lang2, data_set, is_teacher=True)

            if (
                    deobfuscate
                    and eval_bleu
                    or eval_subtoken_score
                    and data_set in datasets_for_bleu
            ):
                # TODO clean lang1
                vizualize_do_files_student_teacher(lang1.split("_")[0], src_path, ref_path, hyp_paths,
                                                   self.t_hyp_paths[eval_name])
        scores.update(self.teacher_scores)

    @staticmethod
    def compute_ranking_metrics(scores, hyp_paths, ref_path, deobfuscation_proba, lang1, lang2, data_set,
                                is_teacher=False):
        prefix = f"{data_set}_{get_l1l2_string(lang1, lang2, deobfuscation_proba)}_mt_ranking"
        if is_teacher and any(
                key.startswith(prefix) and
                key.endswith("_teacher") for
                key in scores.keys()):
            return
        ranking_metrics = run_ranking_scoring(ref_path, hyp_paths)
        for metric_type, value in ranking_metrics.items():
            logger.info(
                f"Ranking {metric_type} score {hyp_paths} {ref_path} {'teacher' if is_teacher else ''}: {value:f}"
            )
            scores[
                f"{prefix}_{metric_type}" + ("_teacher" if is_teacher else "")
                ] = value

    @staticmethod
    def evaluate_bleu(scores, hyp_paths, ref_path, deobfuscation_proba, lang1, lang2, eval_computation, data_set,
                      is_teacher=False):
        prefix = f"{data_set}_{get_l1l2_string(lang1, lang2, deobfuscation_proba)}_mt_bleu"
        if is_teacher and any(
                key.startswith(prefix) and
                key.endswith("_teacher") for
                key in scores.keys()):
            return
        # evaluate BLEU score
        bleu = eval_moses_bleu(ref_path, hyp_paths[0])
        logger.info(f"BLEU {hyp_paths[0]} {ref_path} {'teacher' if is_teacher else ''}: {bleu:f}")
        scores[
            prefix + (f"_teacher" if is_teacher else "")
            ] = bleu
        if eval_computation:
            for hyp_path in hyp_paths:
                Path(hyp_path).unlink()

    @staticmethod
    def compute_subtoken_metrics(scores, hyp_paths, ref_path, lang1, lang2, data_set, deobfuscation_proba,
                                 is_teacher=False):
        prefix = f"{data_set}_{get_l1l2_string(lang1, lang2, deobfuscation_proba)}_mt_subtoken"
        if is_teacher and any(
                key.startswith(prefix) and
                key.endswith("_teacher") for
                key in scores.keys()):
            return
        subtoken_level_scores = run_subtoken_score(ref_path, hyp_paths)
        for score_type, value in subtoken_level_scores.items():
            logger.info(
                f"Subtoken {score_type} score {hyp_paths} {ref_path} {'teacher' if is_teacher else ''}: {value:f}"
            )
            scores[
                f"{prefix}_{score_type}" + ("_teacher" if is_teacher else "")
                ] = value

    def _generate_hypothesis(self, data_set, decoder, enc1, i, lang1, lang2, lang2_id, len1, params, x1, x2,
                             is_teacher=False):
        len_v = (3 * len1 + 10).clamp(max=params.max_len)
        enc1 = enc1.to(decoder.embeddings.weight.dtype)
        if params.beam_size == 1:
            if params.number_samples > 1:
                assert params.eval_temperature is not None
                generated, lengths = decoder.generate(
                    enc1.repeat_interleave(params.number_samples, dim=0),
                    len1.repeat_interleave(params.number_samples, dim=0),
                    lang2_id,
                    max_len=len_v.repeat_interleave(
                        params.number_samples, dim=0
                    ),
                    sample_temperature=params.eval_temperature,
                )
                generated = generated.T.reshape(
                    -1, params.number_samples, generated.shape[0]
                ).T
                lengths, _ = lengths.reshape(-1, params.number_samples).max(
                    dim=1
                )
            else:
                generated, lengths = decoder.generate(
                    enc1, len1, lang2_id, max_len=len_v
                )
            # print(f'path 1: {generated.shape}')

        else:
            assert params.number_samples == 1
            generated, lengths, _ = decoder.generate_beam(
                enc1,
                len1,
                lang2_id,
                beam_size=params.beam_size,
                length_penalty=params.length_penalty,
                early_stopping=params.early_stopping,
                max_len=len_v,
            )  # (len, beam, batch), (batch, beam)
            # print(f'path 2: {generated.shape}')
        if i == 0:
            # show 1 evaluation example and the corresponding model generation
            show_batch(
                logger,
                [
                    ("source", x1.transpose(0, 1)),
                    ("target", x2.transpose(0, 1)),
                    (
                        "gen",
                        generated.transpose(0, 1)
                        if len(generated.shape) == 2
                        else generated[:, 0, :].transpose(0, 1),
                    ),
                ],
                self.data["dico"],
                self.params.roberta_mode,
                f"{data_set} {lang1}-{lang2}{' teacher' if is_teacher else ''}",
            )
        return generated, lengths

    @staticmethod
    def _get_enc_dec(encoder, decoder, lang2_id, params):
        encoder = encoder[0].module if params.multi_gpu else encoder[0]
        decoder = (
            decoder[lang2_id] if params.separate_decoders else decoder[0]
        )
        decoder = decoder.module if params.multi_gpu else decoder
        return encoder, decoder

    @staticmethod
    def _model_forward(encoder, decoder, langs1, langs2, len1, len2, pred_mask, spans, x1, x2, y):
        # encode source sentence
        enc1 = encoder(
            "fwd", x=x1, lengths=len1, langs=langs1, causal=False, spans=spans
        )
        enc1 = enc1.transpose(0, 1)
        enc1 = enc1.to(decoder.embeddings.weight.dtype)

        # decode target sentence
        dec2 = decoder(
            "fwd",
            x=x2,
            lengths=len2,
            langs=langs2,
            causal=True,
            src_enc=enc1,
            src_len=len1,
            spans=spans,
        )
        # loss
        scores, loss = decoder(
            "predict", tensor=dec2, pred_mask=pred_mask, y=y, get_scores=True
        )
        return enc1, dec2, scores, loss

    @staticmethod
    def write_hypo_teacher_hypo_ref_src(
            data_set,
            hypothesis,
            teacher_hypothesis,
            lang1,
            lang2,
            params,
            references,
            scores,
            sources=None,
            deobfuscation_proba=None,
    ):
        # hypothesis paths
        t_hyp_paths = []
        # export sentences to hypothesis file / restore BPE segmentation
        for beam_number in range(len(teacher_hypothesis[0])):
            hyp_name = f"{get_l1l2_string(lang1, lang2, deobfuscation_proba)}.{data_set}_beam{beam_number}.teacher.txt"
            hyp_path = os.path.join(params.hyp_path, hyp_name)
            t_hyp_paths.append(hyp_path)
            print(f"outputting teacher hypotheses in {hyp_path}")
            with open(hyp_path, "w", encoding="utf-8") as f:
                f.write("\n".join([hyp[beam_number] for hyp in teacher_hypothesis]) + "\n")
            restore_segmentation(
                hyp_path, roberta_mode=params.roberta_mode, single_line=True
            )
        hyp_paths, ref_path, src_path = EncDecEvaluator.write_hypo_ref_src(data_set, hypothesis, lang1, lang2, params,
                                                                           references, scores, sources,
                                                                           deobfuscation_proba)
        return hyp_paths, t_hyp_paths, ref_path, src_path

def vizualize_do_files_student_teacher(lang1, src_file, ref_file, hyp_files, t_hyp_files):
    lang1_processor = LangProcessor.processors[lang1.split("_")[0]](
        root_folder=TREE_SITTER_ROOT
    )
    src_viz = str(Path(src_file).with_suffix(".vizualize.txt"))
    hyp_viz = str(
        Path(re.sub("beam\d", "", hyp_files[0])).with_suffix(".vizualize.txt.tmp")
    )
    t_hyp_viz = str(
        Path(re.sub("beam\d", "", t_hyp_files[0])).with_suffix(".vizualize.txt.tmp")
    )
    ref_viz = str(Path(ref_file).with_suffix(".vizualize.txt"))

    hyp_lines = list(
        zip(*[read_file_lines(path) for path in hyp_files])
    )  # test_size * beam_size
    t_hyp_lines = list(
        zip(*[read_file_lines(path) for path in t_hyp_files])
    )  # test_size * beam_size
    beam_size = len(hyp_lines[0])

    with open(src_file, encoding="utf-8") as f:
        src_lines = f.readlines()  # test_size

    with open(ref_file, encoding="utf-8") as f:
        ref_lines = f.readlines()  # test_size

    with open(src_viz, "w", encoding="utf-8") as src_vizf:
        with open(hyp_viz, "w", encoding="utf-8") as hyp_vizf:
            with open(t_hyp_viz, "w", encoding="utf-8") as t_hyp_vizf:
                with open(ref_viz, "w", encoding="utf-8") as ref_vizf:
                    src_vizf.write(
                        "========================SOURCE============================\n"
                    )
                    hyp_vizf.write(
                        "=====================STUDENT_HYPO=========================\n"
                    )
                    t_hyp_vizf.write(
                        "=====================TEACHER_HYPO=========================\n"
                    )
                    ref_vizf.write(
                        "==========================REF=============================\n"
                    )

                    for src, hyps, t_hyps, ref in zip(src_lines, hyp_lines, t_hyp_lines, ref_lines):
                        try:
                            src = lang1_processor.detokenize_code(src)
                            src_vizf.write(src)
                        except:
                            src = "".join(
                                [
                                    c if (i + 1) % 50 != 0 else c + "\n"
                                    for i, c in enumerate(src)
                                ]
                            )
                            src_vizf.write(src)

                        ref = ref.replace("|", "\n").strip()
                        ref_vizf.write(ref)

                        for i in range(beam_size):
                            hyp = hyps[i]
                            hyp = hyp.replace("|", "\n").strip()
                            hyp_vizf.write(hyp)
                            t_hyp = t_hyps[i]
                            t_hyp = t_hyp.replace("|", "\n").strip()
                            t_hyp_vizf.write(t_hyp)
                            if i == 0:
                                maximum = max(
                                    len(src.split("\n")),
                                    len(hyp.split("\n")),
                                    len(t_hyp.split("\n")),
                                    len(ref.split("\n")),
                                )
                                for i in range(len(src.split("\n")), maximum):
                                    src_vizf.write("\n")
                                for i in range(len(hyp.split("\n")), maximum):
                                    hyp_vizf.write("\n")
                                for i in range(len(t_hyp.split("\n")), maximum):
                                    t_hyp_vizf.write("\n")
                                for i in range(len(ref.split("\n")), maximum):
                                    ref_vizf.write("\n")
                            else:
                                maximum = max(
                                    len(src.split("\n")),
                                    len(hyp.split("\n")),
                                    len(t_hyp.split("\n")),
                                    len(ref.split("\n")),
                                )
                                for i in range(maximum - 1):
                                    src_vizf.write("\n")
                                for i in range(maximum - 1):
                                    ref_vizf.write("\n")
                                for i in range(len(hyp.split("\n")), maximum):
                                    hyp_vizf.write("\n")
                                for i in range(len(t_hyp.split("\n")), maximum):
                                    t_hyp_vizf.write("\n")
                            src_vizf.write("-\n")
                            hyp_vizf.write("-\n")
                            t_hyp_vizf.write("-\n")
                            ref_vizf.write("-\n")

                        src_vizf.write("--\n\n")
                        hyp_vizf.write("--\n\n")
                        t_hyp_vizf.write("--\n\n")
                        ref_vizf.write("--\n\n")

                        src_vizf.write(
                            "==========================================================\n"
                        )
                        hyp_vizf.write(
                            "==========================================================\n"
                        )
                        t_hyp_vizf.write(
                            "==========================================================\n"
                        )
                        ref_vizf.write(
                            "==========================================================\n"
                        )

    command = f"pr -w 320 -m -t {src_viz} {ref_viz} {hyp_viz} {t_hyp_viz} > {hyp_viz[:-4]}"
    subprocess.Popen(
        command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
    ).wait()

    os.remove(src_viz)
    os.remove(ref_viz)
    os.remove(hyp_viz)
    os.remove(t_hyp_viz)

max_stars_repo_name: davidenkoim/CodeGen

max_issues_repo_name: davidenkoim/CodeGen

max_forks_repo_name: davidenkoim/CodeGen


text:
import numpy as np
from matplotlib import pyplot as plt

def plot_gaussian(mu, lmbda, color='b', label='', alpha=1.0, ax=None,
                  artists=None):
    ax = ax if ax else plt.gca()

    # t = np.hstack([np.arange(0, 2 * np.pi, 0.01), 0])
    # circle = np.vstack([np.sin(t), np.cos(t)])
    # ellipse = np.dot(np.linalg.cholesky(lmbda), circle)

    ti = np.linspace(0, 2 * np.pi, 50)
    vec = np.array([np.cos(ti), np.sin(ti)]).transpose()
    eig_val, eig_vec = np.linalg.eig(lmbda)
    ellipse = 1.96 * np.dot(np.dot(vec, np.diag(np.sqrt(eig_val))), eig_vec.transpose())

    if artists is None:
        point = ax.scatter([mu[0]], [mu[1]], marker='D', color=color, s=4,
                           alpha=alpha)

        line, = ax.plot(ellipse[:, 0] + mu[0], ellipse[:, 1] + mu[1],
                        linestyle='-', linewidth=2, color=color, label=label,
                        alpha=alpha)
    else:
        line, point = artists
        point.set_offsets(mu)
        point.set_alpha(alpha)
        point.set_color(color)
        line.set_xdata(ellipse[0, :] + mu[0])
        line.set_ydata(ellipse[1, :] + mu[1])
        line.set_alpha(alpha)
        line.set_color(color)

    return (line, point) if point else (line,)

ext: py

max_stars_repo_name: MichaelLutter/mimo

max_issues_repo_name: MichaelLutter/mimo

max_forks_repo_name: MichaelLutter/mimo

reason: import numpy


text:
"""
Defining standard tensorflow layers as modules.
"""

import tensorflow as tf
from deeplearning import module
from deeplearning import tf_util as U
import numpy as np

class Input(module.Module):
    ninputs=0

class Placeholder(Input):
    def __init__(self, dtype, shape, name, default=None):
        super().__init__(name)
        assert isinstance(shape, (list, tuple))
        self.dtype = dtype
        self.shape = list(shape)
        self.phname = name
        self.default = default

    def _bsz(self):
        return self.nbatch * self.nstep

    def _build(self, inputs):
        bsz = self._bsz()
        shape = [bsz] + self.shape
        if self.default is not None:
            assert list(self.default.shape) == self.shape
            default = np.repeat(self.default[None], bsz, axis=0)
            self.ph = tf.placeholder_with_default(default, shape=shape, name=self.phname)
        else:
            self.ph = tf.placeholder(self.dtype, shape=shape, name=self.phname)
        self.placeholders.append(self.ph)
        return self.ph

class StatePlaceholder(Placeholder):
    def _bsz(self):
        return self.nbatch

class Dense(module.Module):
    ninputs = 1
    def __init__(self, name, *modules, units=1, activation=None, **kwargs):
        super().__init__(name, *modules)
        self.units = units
        self.activation = activation
        self.layer_kwargs = kwargs

    def _build(self, inputs):
        return tf.layers.dense(inputs[0],
                               self.units,
                               kernel_initializer=tf.variance_scaling_initializer(),
                               activation=self.activation,
                               name='dense',
                               **self.layer_kwargs)

class Conv2d(module.Module):
    ninputs = 1
    def __init__(self, name, *modules, filters=1, size=(3,3), strides=(1,1), padding='valid', activation=None, **kwargs):
        super().__init__(name, *modules)
        self.filters = filters
        self.size = size
        self.strides = strides
        self.padding = padding
        self.activation = activation
        self.layer_kwargs = kwargs

    def _build(self, inputs):
        return tf.layers.conv2d(inputs[0],
                                filters=self.filters,
                                strides=self.strides,
                                kernel_size=self.size,
                                kernel_initializer=tf.variance_scaling_initializer(),
                                padding=self.padding,
                                activation=self.activation,
                                name='conv2d',
                                **self.layer_kwargs)

class LSTM(module.RecurrentModule):
    def __init__(self, name, *modules, nlstm=256, nlayers=1, masked=False):
        assert len(modules) == 1, "This LSTM is only designed to work with 1 input"
        modules = list(modules)
        if masked:
            modules.append(Placeholder(tf.float32, [], name=name+'_ph_mask'))

        state_modules = []
        default = np.zeros((nlstm*2,), dtype=np.float32)
        for i in range(nlayers):
            state_modules.append(StatePlaceholder(tf.float32, (nlstm*2,), name+'_ph%d'%i, default))
        super().__init__(name, *modules, state_modules=state_modules)

        self.nlayers = nlayers
        self.nlstm = nlstm
        self.masked = masked

    def _build(self, inputs, state):
        X = inputs[0]
        M = inputs[1] if self.masked else tf.zeros([self.nbatch * self.nstep])
        ms = U.batch_to_seq(M, self.nbatch, self.nstep)
        hs = U.batch_to_seq(X, self.nbatch, self.nstep)
        state_out = []
        for i in range(self.nlayers):
            hs, soi = U.lstm(hs, ms, state[i], 'lstm{}'.format(i), nh=self.nlstm)
            state_out.append(soi)
        h = U.seq_to_batch(hs)
        return h, state_out

class Flatten(module.Module):
    ninputs = 1
    def _build(self, inputs):
        return tf.layers.flatten(inputs[0])

class StopGrad(module.Module):
    def _build(self, inputs):
        return [tf.stop_gradient(i) for i in inputs]

class Softmax(module.Module):
    ninputs = 2 # logits, targets
    def _build(self, inputs):
        logits, target = inputs
        return tf.losses.softmax_cross_entropy(target, logits)

max_stars_repo_name: cbschaff/nlimb

max_issues_repo_name: cbschaff/nlimb

max_forks_repo_name: cbschaff/nlimb


text:
import numpy as np
from nanocut.common import EPSILON
from nanocut.output import error, printstatus

class Geometry:
    """Class for handling crystal structure, containing unit-cell-vectors,
    atom coordinates and names of atoms."""

    def __init__(self, latvecs, basis, basis_names_idx, basis_names,
                 basis_coordsys="lattice", shift=None,
                 shift_coordsys="lattice", bravais_cell=None):
        """Initializes Geometry object.

        Args:
            latvecs: Lattice vectors as (3, 3) array.
            basis: Coordinates of the atoms in the basis.
            basis_names_idx: Type index of every atom in the basis.
            basis_names: List of atom types.
            basis_coordsys: Coordinate system for the basis (should be "lattice"
                or "cartesian").
            bravais_cell: Specifies the Bravais superlattice.
        """
        self.latvecs = np.array(latvecs, dtype=float)
        self.basis_names_idx = basis_names_idx
        self.basis_names = basis_names
        self.basis_coordsys = basis_coordsys
        self.basis = np.array(basis, dtype=float)
        self.basis = self.coord_transform(basis, basis_coordsys)
        if shift is not None:
            shift = self.coord_transform(shift, shift_coordsys)
            self.basis += shift
        self.basis = self.mv_basis_to_prim(self.basis)
        if bravais_cell is not None:
            self.bravais_cell = np.array(bravais_cell)
        else:
            self.bravais_cell = np.eye(3, dtype=float)

    @classmethod
    def fromdict(cls, inidict):
        """Initliazes geometry from dict with type checking.

        Args:
            cls: Class type.
            inidict: Dictionary with settings
        """
        if "lattice_vectors" not in inidict:
            error("latvecs not defined.")
        if "basis" not in inidict:
            error("Basis not defined.")
        try:
            latvecs = np.array(
                [ float(s) for s in inidict["lattice_vectors"].split() ])
            latvecs.shape = (3, 3)
        except ValueError:
            error("Invalid lattice vector specification.")
        if abs(np.linalg.det(latvecs)) < EPSILON:
            error("Linearly dependent lattice vectors.")

        basis = inidict["basis"].split()
        basis_names=[ basis.pop(idx)
                      for idx in range(0, len(basis) * 3 // 4, 3) ]
        try:
            basis = np.array([ float(s) for s in basis ])
            basis.shape= (-1, 3)
        except ValueError:
            error("Invalid basis specification.")
        basis_names_idx = range(len(basis))
        basis_coordsys = inidict.get("basis_coordsys", "lattice")
        if basis_coordsys not in ["lattice", "cartesian"]:
            error("Invalid coordinate system specification.")

        shiftstr = inidict.get("shift_vector", "0.0 0.0 0.0")
        try:
            shift = np.array([ float(ss) for ss in shiftstr.split() ])
            shift.shape = (3, )
        except ValueError:
            error("Invalid shift vector for basis")
        shift_coordsys = inidict.get("shift_vector_coordsys", "lattice")

        bravais_cell = inidict.get("bravais_cell", "1 0 0  0 1 0  0 0 1")
        try:
            bravais_cell = np.array(
                [ float(ss) for ss in bravais_cell.split() ])
            bravais_cell.shape = (3, 3)
        except ValueError:
            error("Invalid Bravais cell definition")

        return cls(latvecs, basis, basis_names_idx, basis_names, basis_coordsys,
                   shift, shift_coordsys, bravais_cell)

    def coord_transform(self, array, array_coordsys):
        """Transforms given vector into cartesian coordinate system.

        Args:
            array: Array with coordinates.
            array_coordsys: Coordinate system ("lattice" or "cartesian")

        Returns:
            Cartesian coordinates.
        """
        if array_coordsys == "lattice":
            return np.dot(array, self.latvecs)
        elif array_coordsys == "cartesian":
            return array
        else:
            raise ValueError("Invalid coodinate system type '{:s}'".format(
                array_coordsys))

    def mv_basis_to_prim(self, basis):
        """Folds vectors into primitive cell.

        Args:
           basis:Coordinates.

        Return:
            Coordintes folded into the central cell.
        """
        invlatvecs = np.linalg.inv(self.latvecs)
        basis = np.dot(basis, invlatvecs) % 1.0
        return self.coord_transform(basis, "lattice")

    def gen_cuboid(self, cuboid, periodicity=None):
        """Generates list of lattice points containing all points in a cuboid.

        Args:
            cuboid: lower and upper ends of the cuboid.
            periodicity: periodicity object.

        Returns:
            Cartesian coordinates of the grid points.
        """

        # Get the 8 corners of the cuboid
        mesh = np.mgrid[0:2,0:2,0:2].reshape(3, -1).transpose()
        abc_corners = [ [ cuboid[mm[0],0], cuboid[mm[1],1], cuboid[mm[2],2] ]
                          for mm in mesh ]

        # Get corners of the containing parallelepiped in relative coordinates
        invlatvecs = np.linalg.inv(self.latvecs)
        nmo_corners = np.dot(abc_corners, invlatvecs)
        nmo_mininds = np.floor(nmo_corners.min(axis=0)).astype(int)
        nmo_maxinds = np.floor(nmo_corners.max(axis=0)).astype(int)

        # Generate mesh indexing all points inside the parallelepiped
        nallpoint = np.abs(np.prod(nmo_maxinds - nmo_mininds))
        printstatus(
            "Number of necessary grid points: {:d}".format(int(nallpoint)),
            indentlevel=1)
        nmo = np.mgrid[nmo_mininds[0]:nmo_maxinds[0]+1,
                       nmo_mininds[1]:nmo_maxinds[1]+1,
                       nmo_mininds[2]:nmo_maxinds[2]+1, ]
        nmo = nmo.reshape(3, -1).transpose()

        # Throw away points in the parallelepiped which are farther away from
        # the cuboid as the maximal size of the unit cell along given direction
        abc = np.dot(nmo, self.latvecs)
        buffer = np.max(np.abs(self.latvecs), axis=0)
        cond1 = np.all(abc < cuboid[0] - buffer, axis=1)
        cond2 = np.all(abc > cuboid[1] + buffer, axis=1)
        inside = np.logical_not(np.logical_or(cond1, cond2))
        return np.dot(nmo[inside], self.latvecs)

    def get_atom_type_names(self):
        return self.basis_names

    def get_name_of_atom(self, index):
        """Returns the name of an atom with given index.

        Args:
            index: Index of the atom.

        Returns:
            Name (type) of the atom.
        """
        return self.basis_names[self.basis_names_idx[index]]

    def gen_atoms(self, lattice_points):
        """Returns the coordinates and index of each atom inside the cells
        corresponding to given lattice points.

        Args:
           lattice_points: Cartesian coordinates of the lattice points as
               (-1,3) shaped array.

        Returns:
            Coordinates of all atoms in the cells with the given coordinates.
        """
        nbasis = len(self.basis)
        nlatpoint = len(lattice_points)
        atoms_coords = np.empty((nlatpoint * nbasis , 3), dtype=float)
        # Taking longer loop (instead over range(len(basis)) to have nicer
        # output with atoms being in the same cell having close indices.
        for ii in range(nlatpoint):
            atoms_coords[ii * nbasis : (ii + 1) * nbasis ] = (self.basis +
                lattice_points[ii])
        atoms_idx = np.resize(np.arange(nbasis), (nlatpoint * nbasis))
        return atoms_coords, atoms_idx


text:
import unittest
import jax
from jax import numpy as jnp
import numpy as np
from flax_extra.util import batch_per_device, originate

class OriginateTest(unittest.TestCase):
    def test(self):
        n_devices = 2
        original_x = jnp.ones((4, 3))
        replicated_x = jnp.ones((n_devices, 4, 3))
        value = dict(leaf=replicated_x)
        expect = dict(leaf=original_x)
        np.testing.assert_array_equal(
            originate(value).get("leaf"),
            expect.get("leaf"),
        )

class BatchPerDeviceTest(unittest.TestCase):
    def test_inputs(self):
        n_devices = 2
        batch_size = 4
        value = jnp.ones((batch_size, 3))
        expect = jnp.ones((n_devices, batch_size // n_devices, 3))
        np.testing.assert_array_equal(
            batch_per_device(value, n_devices=n_devices),
            expect,
        )

    def test_default(self):
        n_devices = jax.local_device_count()
        batch_size = 4
        value = jnp.ones((batch_size, 3))
        expect = jnp.ones((n_devices, batch_size // n_devices, 3))
        np.testing.assert_array_equal(
            batch_per_device(value),
            expect,
        )

max_stars_repo_name: manifest/flax-extra

max_issues_repo_name: manifest/flax-extra

max_forks_repo_name: manifest/flax-extra

reason: import numpy,import jax,from jax


text:
import numpy as np
import networkx as nx
from scipy import sparse
from scipy import optimize
import time

def get_edge_seqs(graph, channels=None):
    """
    Return the edge sequences in each channel for each node.

    Returns
    -------
    dict
        Keys are the node indexes and values are their corresponding edge seqs.
    """
    if channels is None:
        channels = graph.channels

    edge_seqs = {}
    for node_idx in range(graph.n_nodes):
        seq_list = []
        for channel in channels:
            adj = graph.ch_to_adj[channel]
            seq_list.extend([adj[node_idx,:], adj.T[node_idx,:]])
        edge_seqs[node_idx] = sparse.vstack(seq_list, format="csr")

    return edge_seqs

def neighborhood(smp):
    """
    Bound local assignment costs by neighborhood disagreements.

    If u is a node in the template and v is a node in the world, in order for
    v to be a candidate for u there should be a subgraph isomorphism from
    the neighborhood of u to the neighborhood of v. We can check if any such
    subgraph isomorphism exists in which the neighbors of v are candidates for
    the appropriate neighbors of u by looking for a bipartite matching.

    Parameters
    ----------
    smp : MatchingProblem
        A subgraph matching problem on which to compute edgewise cost bounds.
    """
    # TODO: check whether a world node is a candidate for any tmplt node
    # ---> This can be achieved by reduce_world?

    tmplt_edge_seqs = get_edge_seqs(smp.tmplt)
    world_edge_seqs = get_edge_seqs(smp.world, channels=smp.tmplt.channels)

    nbr_counts = smp.tmplt.is_nbr.sum(axis=1).A.flatten()

    # TODO: might want candidates to be sparse in other filters
    sparse_is_cand = sparse.csr_matrix(smp.candidates())
    for tnode_idx, wnode_idx in np.transpose(sparse_is_cand.nonzero()):
        # If the template node has only 1 neighbor, the topology filter is
        # equivalent to the neighborhood filter, so there is no point in
        # using the neighborhood filter since it is more expensive.
        if nbr_counts[tnode_idx] == 1:
            continue

        tmplt_seq = tmplt_edge_seqs[tnode_idx]
        world_seq = world_edge_seqs[wnode_idx]

        lap_mat_rows = []
        for tnbr_idx in range(smp.tmplt.n_nodes):
            # The world nodes that are not candidates to tnbr_idx are np.Inf
            lap_mat_row = smp.global_costs[tnbr_idx]
            lap_mat_row[lap_mat_row != np.Inf] = 0
            # Check if all of the necessary edges are present
            for i, edge_count in enumerate(tmplt_seq[:,tnbr_idx].A.flat):
                if edge_count > 0:
                    lap_mat_row = \
                        lap_mat_row + np.maximum(edge_count-world_seq[i].A, 0)
            lap_mat_rows.append(lap_mat_row.flatten())

        lap_mat = np.stack(lap_mat_rows)
        row_idxs, col_idxs = optimize.linear_sum_assignment(lap_mat)
        # TODO: Figure out the relationship between local_costs and global_costs
        # for neighborhood-local-costs.
        smp.local_costs[tnode_idx, wnode_idx] = lap_mat[row_idxs, col_idxs].sum()

max_stars_repo_name: cfld/uclasm

max_issues_repo_name: cfld/uclasm

max_forks_repo_name: cfld/uclasm

reason: import numpy,from scipy,import networkx


text:
import math
import numpy as np

class OffloadPA:
    def __init__(self, model):
        self.class0 = model.__dict__['classes_'][0]
        self.class1 = model.__dict__['classes_'][1]
        self.w = model.coef_[0]
        self.b = model.intercept_[0]
        self.dim = len(self.w)

    def get_params(self):
        return {'Weight_Vector':self.w, 'Bias_Constant':self.b}

    def get_arduino_code(self):
        str_w = ', '.join([str(x) for x in self.w])

        code = f"""double w[] = {{{str_w}}};
double b = {str(self.b)};

void setup() {{
    Serial.begin(9600);

}}

void loop() {{
    //Data Section: To Be Coded Manually

    float data[{str(self.dim)}]; //This is your feature vector. Retrive your data into this array.

    //ML Inference Section

    double temp = 0.0;
    for(int i=0; i<{str(self.dim)}; i++)
    {{
        temp += data[i]*w[i];
    }}

    if(temp + b >= 0)
    {{
        //Do something for class label {str(self.class1)}.
        Serial.println("{str(self.class1)}");
    }}
    else
    {{
        //Do something for class label {str(self.class0)}.
        Serial.println("{str(self.class0)}");
    }}

    delay(1000);
}}"""
        return code


text:
"""Data streams for processing with a pipeline."""

import warnings
import numpy as np

def segment(data, length, overlap=0):
    """Generate segments of an array.

    Each segment is of a specified length and optional overlap with the
    previous segment. Only segments of the specified length are retrieved (if
    segments don't fit evenly into the data).

    Parameters
    ----------
    data : array, shape (n_channels, n_samples)
        Data to segment.
    length : int
        Number of samples to retrieve in each chunk.
    overlap : int, optional
        Number of overlapping samples in consecutive chunks.

    Yields
    ------
    segment : array (n_channels, length)
        Segment of the input array.

    Examples
    --------
    Segment a 2-channel recording:

    >>> import numpy as np
    >>> from copper.sources import segment
    >>> x = np.arange(8).reshape(2, 4)
    >>> x
    array([[0, 1, 2, 3],
           [4, 5, 6, 7]])
    >>> seg = segment(x, 2)
    >>> next(seg)
    array([[0, 1],
           [4, 5]])
    >>> next(seg)
    array([[2, 3],
           [6, 7]])

    Consecutive segments with overlapping samples agree:

    >>> seg = segment(x, 3, overlap=2)
    >>> next(seg)
    array([[0, 1, 2],
           [4, 5, 6]])
    >>> next(seg)
    array([[1, 2, 3],
           [5, 6, 7]])
    """
    data = np.atleast_2d(data)
    n = data.shape[1]
    for f, t in segment_indices(n, length, overlap=overlap):
        yield data[:, f:t]

def segment_indices(n, length, overlap=0):
    """Generate indices to segment an array.

    Each segment is of a specified length with optional overlap with the
    previous segment. Only segments of the specified length are retrieved if
    they don't fit evenly into the the total length. The indices returned are
    meant to be used for slicing, e.g. ``data[:, from:to]``.

    Parameters
    ----------
    n : int
        Number of samples to segment up.
    length : int
        Length of each segment.
    overlap : int, optional
        Number of overlapping samples in consecutive segments.

    Yields
    ------
    from : int
        Index of the beginning of the segment with respect to the input array.
    to : int
        Index of the end of the segement with respect to the input array.

    Examples
    --------
    Basic usage -- segment a 6-sample recording into segments of length 2:

    >>> import numpy as np
    >>> from copper.sources import segment_indices
    >>> list(segment_indices(6, 2))
    [(0, 2), (2, 4), (4, 6)]

    Overlapping segments:

    >>> list(segment_indices(11, 5, overlap=2))
    [(0, 5), (3, 8), (6, 11)]
    """
    skip = length - overlap

    if (n - length) % skip != 0:
        warnings.warn("Data (length {}) cannot be chunked evenly into "
                      "segments of length {} with overlap {}".format(
                          n, length, overlap),
                      UserWarning)

    for i in range(0, n, skip):
        if i + length <= n:
            yield i, i + length

max_stars_repo_name: ixjlyons/copper

max_issues_repo_name: ixjlyons/copper

max_forks_repo_name: ucdrascal/copper


text:
from copy import deepcopy
from math import *
from pathlib import Path

import numpy as np
import pandas as pd
from sklearn.utils import shuffle

algorithms = ['MultiLayerPerceptron', 'LR', 'RandomForest', 'GaussianNB']
fair_algorithms = ['Feldman-' + algorithm for algorithm in algorithms]
all_algorithms = algorithms + fair_algorithms

metrics = ['predicted_delta', 'true_delta', 'labeled_delta', 'calibrated_delta']

class Dataset:
    def __init__(self, df, dataset_name) -> None:
        self.df = df
        self.df_test = deepcopy(df)  # use the same df_test for all, preserve the same ordering
        self.add_information(dataset_name)

    def __len__(self):
        return self.df.shape[0]

    def add_information(self, dataset_name):
        if dataset_name == 'adult':
            self.dataset_name = 'adult'
            self.class_attr = 'income-per-year'
            self.positive_class_val = 1
            self.sensitive_attrs = ['race', 'sex']  # privilidged: group 0
        elif dataset_name == 'bank':
            self.dataset_name = 'bank'
            self.class_attr = 'y'
            self.positive_class_val = 1
            self.sensitive_attrs = ['age']  # privilidged: group 0
        elif dataset_name == 'propublica-violent-recidivism':
            self.dataset_name = 'propublica-violent-recidivism'
            self.class_attr = 'two_year_recid'
            self.positive_class_val = 1
            self.sensitive_attrs = ['race', 'sex']  # privilidged: group 0
        elif dataset_name == 'propublica-recidivism':
            self.dataset_name = 'propublica-recidivism'
            self.class_attr = 'two_year_recid'
            self.positive_class_val = 1
            self.sensitive_attrs = ['race', 'sex']  # privilidged: group 0
        elif dataset_name == 'german':
            self.dataset_name = 'german'
            self.class_attr = 'credit'
            self.positive_class_val = 1
            self.sensitive_attrs = ['sex', 'age']  # privilidged: group 0
        elif dataset_name == 'ricci':  # might take out later
            self.dataset_name = 'ricci'
            self.class_attr = 'Class'
            self.positive_class_val = 1
            self.sensitive_attrs = ['Race']  # privilidged: group 0
        else:
            raise NotImplementedError

    def shuffle(self, random_state=0, attribute=None) -> None:
        self.df = shuffle(self.df, random_state=random_state)
        # important!!
        self.df.reindex()
        # make sure the self.df[:budget] contains at least one instance for each group
        if attribute is not None:
            list_attribute = self.df[attribute].tolist()
            idx0 = [list_attribute.index(elem) for elem in set(list_attribute)]
            idx1 = [i for i in range(self.__len__()) if i not in idx0]
            self.df = self.df.iloc[idx0 + idx1]

    @classmethod
    def load_from_file(cls, fname: Path, dataset_name: str) -> 'Dataset':
        return cls(pd.read_csv(fname), dataset_name)

    def get_group_frequency(self, attribute, num_groups):
        counts = np.zeros((num_groups,))
        for k in range(num_groups):
            counts[k] = (self.df[attribute] == k).sum()
        return counts

    def get_groupwise_accuracy(self, algorithm_name, attribute, group_id, budget=None):
        if budget is None:
            budget = self.__len__()

        s = self.df['score_' + algorithm_name][:budget]
        predicted_y = s > 0.5
        y = self.df[self.class_attr][:budget]

        return np.mean((predicted_y == y)[self.df[attribute][:budget] == group_id] * 1.0)

    def get_accuracy_difference(self, algorithm_name, attribute, group_idx_i, group_idx_j, budget=None):
        """
        Returns accuracy difference of the algorithm between two groups, computed with the top #budget rows.
        """
        if budget is None:
            budget = self.__len__()

        s = self.df['score_' + algorithm_name][:budget]
        predicted_y = s > 0.5
        y = self.df[self.class_attr][:budget]

        if group_idx_i == -1 and group_idx_j == -1:
            raise ValueError("at least one of group id is not -1")

        if group_idx_i == -1:
            acc_i = np.mean((predicted_y == y)[self.df[attribute][:budget] != group_idx_j] * 1.0)
        else:
            acc_i = np.mean((predicted_y == y)[self.df[attribute][:budget] == group_idx_i] * 1.0)

        if group_idx_j == -1:
            acc_j = np.mean((predicted_y == y)[self.df[attribute][:budget] != group_idx_i] * 1.0)
        else:
            acc_j = np.mean((predicted_y == y)[self.df[attribute][:budget] == group_idx_j] * 1.0)

        return acc_i - acc_j

    def get_conditional_metrics_difference(self, algorithm_name, attribute, group_idx_i, group_idx_j,
                                           metric, budget=None):

        if budget is None:
            budget = self.__len__()

        s = self.df['score_' + algorithm_name][:budget]
        predicted_y = s > 0.5
        y = self.df[self.class_attr][:budget]
        ct = self.df[attribute][:budget]

        if group_idx_i == -1 and group_idx_j == -1:
            raise ValueError("at least one of group id is not -1")

        if group_idx_i == -1:
            y_i = y[ct != group_idx_j] * 1.0
            predicted_y_i = predicted_y[ct != group_idx_j] * 1.0
        else:
            y_i = y[ct == group_idx_i] * 1.0
            predicted_y_i = predicted_y[ct == group_idx_i] * 1.0

        if group_idx_j == -1:
            y_j = y[ct != group_idx_i] * 1.0
            predicted_y_j = predicted_y[ct != group_idx_i] * 1.0
        else:
            y_j = y[ct == group_idx_j] * 1.0
            predicted_y_j = predicted_y[ct == group_idx_j] * 1.0

        if metric == 'tpr':
            tpr_i = np.mean(predicted_y_i[y_i == 1])
            tpr_j = np.mean(predicted_y_j[y_j == 1])
            delta = tpr_i - tpr_j
            if isnan(delta):
                return 0
            return delta

        elif metric == 'fnr':
            fnr_i = np.mean(predicted_y_i[y_i == 0])
            fnr_j = np.mean(predicted_y_j[y_j == 0])
            delta = fnr_i - fnr_j
            if isnan(delta):
                return 0
            return delta
        else:
            raise NotImplementedError

code_llo_calibration = '''
# LLO Calibration model
#
# Observed data / constants
# n     total number of instances
# s     n x 1 vector of model scores
# y     n x 1 vector of true outcomes (the label observed; NOT an indicator of accuracy)
#
# Latent variables
# a,b   calibration scaling parameters (scalars)
# p     estimated p( label=1 | Score )
#
# Note:
# Computational issues arise when there are scores that have exact values of 0 or 1
# To make this code run without JAGS errors, it helps to set the minimum score to 0.001 and the maximum score to 0.999
model{
   # Loop over all instances
   for (i in 1:n) {
      p[i] <- ( exp(b) * ( s[i]^a )) / ( exp(b) * ( s[i]^a )+ (1-s[i])^a )
      # Draw a label according to the recalibrated probability
      y[i] ~ dbern( p[i] )
   }
   # Priors -- these will yield perfect calibration p(label=1|s)=s a priori
   a  <- exp( aa )
   b  ~  dnorm(0,0.1)
   aa ~  dnorm(0,0.1)
}
'''

code_beta_binomial = '''
# Beta-binomial model
#
# Observed data / constants
# nl    number of instances with observed labels
# nc    number of categories
# ct    nl x 1 vector of categories
# s     nl x 1 vector of model scores
# y     nl x 1 vector of labels
#
# Latent variables
# theta nc x 1  accuracy for each category

data
{
    for (i in 1:nl) {
        # Create an indicator for correctness of the i-th instance
        # Assume that a score > 0.5 will lead to a predicted label of "1"
        iscorrect[i] <- ((s[i] > 0.5) == y[i])
    }
}
model
{
    # Loop over all instances with observed labels
    for (i in 1:nl) {
        # Draw indicator of correctness with probability theta according to category
        iscorrect[i] ~ dbern( theta[ct[i]] )
    }
    # Sample the calibration parameters for each category
    for (j in 1:nc) {
        theta[j] ~ dbeta(2, 2)
    }
}
'''

code_hierarchical_beta_binomial = '''
# Hierarchical Beta-binomial model
#
# Observed data / constants
# nl    number of instances with observed labels
# nc    number of categories
# ct    nl x 1 vector of categories
# s     nl x 1 vector of model scores
# y     nl x 1 vector of labels
#
# Latent variables
# theta nc x 1  accuracy for each category
# mu    prior mean accuracy at group level
# c     strength of prior mean at group level

data
{
    for (i in 1:nl) {
        # Create an indicator for correctness of the i-th instance
        # Assume that a score > 0.5 will lead to a predicted label of "1"
        iscorrect[i] <- ((s[i] > 0.5) == y[i])
    }
}
model
{
    # Loop over all instances with observed labels
    for (i in 1:nl) {
        # Draw indicator of correctness with probability theta according to category
        iscorrect[i] ~ dbern( theta[ct[i]] )
    }
    # Sample the calibration parameters for each category
    for (j in 1:nc) {
        theta[j] ~ dbeta(c * mu, c * (1 - mu)) T(0.001,0.999)
    }
    # prior of overall accuracy
    mu ~ dbeta(2, 2)
    c ~ dnorm( 0 , 1 ) T(0.1,)
}
'''

code_hierarchical_beta_calibration = '''
# Hierarchical Beta Calibration model
#
# Observed data / constants
# n     total number of instances
# nl    number of instances with observed labels
# nc    number of categories
# ct    n x 1 vector of categories
# s     n x 1 vector of model scores
# y     nl x 1 vector of labels
# mumua,mumub,mumuc       prior means at group level
# stda,stdb,stdc          standard deviations at category level
# stdstda,stdstdb,stdstdc standard deviations at group level
#
#
# Latent variables
# a,b,c   nc x 1 vector of calibration scaling parameters for each category
# p       n x 1 vector with estimated p( label=1 | Score )
# pc      n x 1 vector with estimated accuracy
# theta   nc x 1 vector with mean of pointwise accuracy over all instances that belong to a category
#
# Note:
# Computational issues arise when there are scores that have exact values of 0 or 1
# To make this code run without JAGS errors, it helps to set the minimum score to 0.001 and the maximum score to 0.999
data {
   for (i in 1:n) {
      # Log transform the scores
      logs1[i] <- log( s[i]   )
      logs0[i] <- log( 1-s[i] )
      # Create a boolean variable for predicted label associated with the i-th instance
      # Assume that a score > 0.5 will lead to a predicted label of "1"
      predl[i] <- ( s[i]>0.5 )
      # Create indicator matrix iscat[j,i] where iscat[j,i]=1 when the i-th instance belongs to category j
      for (j in 1:nc) {
         iscat[j,i] <- ( ct[i] == j )
      }
   }
   for (i in 1:nl) {
      # Create an indicator for correctness of the i-th instance
      iscorrect[i] <- (predl[i] == y[i])
   }
   for (i in (nl+1):n) {
      # We don't need the correctness labels for these instances, but we add them to avoid
      # JAGS from compilation errors
      iscorrect[i] <- 0
   }
}
model{
   # Loop over all instances
   for (i in 1:n) {
      # The Beta calibration step (avoid log odds larger than 15)
      logoddsp[i] <- min( c[ct[i]] + exp( a[ct[i]] )*logs1[i] - exp( b[ct[i]] )*logs0[i] , 15 )
      # p is the model probability of label "1" given this score
      p[i] <- exp( logoddsp[i] ) / ( exp( logoddsp[i] ) + 1 )
      # pc is the pointwise probability that the instance is classified correctly
      pc[i] <- p[i]*predl[i] + (1-p[i])*(1-predl[i])
      # pc2 is the pointwise probability of classifying an instance correctly
      # For the labeled instances, we take observed indicator of correctness
      pc2[i] <- ifelse( i<=nl , iscorrect[i] , pc[i] )
   }
   # Loop over all instances with observed labels
   for (i in 1:nl) {
      # Draw a label according to the LLO probability
      y[i] ~ dbern( p[i] )
   }
   # Sample the calibration parameters for each category
   for (j in 1:nc) {
      a[j] ~ dnorm( mua , 1/(stda^2))
      b[j] ~ dnorm( mub , 1/(stdb^2))
      c[j] ~ dnorm( muc , 1/(stdc^2))
   }
   # Compute the mean accuracy per category
   for (j in 1:nc) {
      # The inner product (inprod) computes the sum of the accuracies for all instances belonging to category j,
      # and the denominator computes the sum of the number of instances belonging to category j
      theta[j] <- inprod( pc2 , iscat[j,] ) / sum( iscat[j,] )
   }
   # Normal priors on the group means
   mua ~  dnorm( mumua , 1/(stdmua^2))
   mub ~  dnorm( mumub , 1/(stdmub^2))
   muc ~  dnorm( mumuc , 1/(stdmuc^2))
   # Half-normal (truncated) priors for the standard deviations
   stda ~ dnorm(0, 1/(stdstda^2) ) T(0,)
   stdb ~ dnorm(0, 1/(stdstdb^2) ) T(0,)
   stdc ~ dnorm(0, 1/(stdstdc^2) ) T(0,)
}
'''

code_hierarchical_llo_calibration = '''
# Hierarchical LLO Calibration model, arrows from p_i to theta
#
# Observed data / constants
# n     total number of instances
# nl    number of instances with observed labels
# nc    number of categories
# ct    n x 1 vector of categories
# s     n x 1 vector of model scores
# y     nl x 1 vector of labels
# mumua,mumub     prior means at group level
# stda,stdb       standard deviations at category level
# stdstda,stdstdb standard deviations at group level
#
#
# Latent variables
# a,b   nc x 1 vector with calibration scaling parameters for each category
# p     n x 1 vector with estimated p( label=1 | Score )
# pc    n x 1 vector with estimated accuracy
# theta nc x 1 vector with mean of pointwise accuracy over all instances that belong to a category
#
# Note:
# Computational issues arise when there are scores that have exact values of 0 or 1
# To make this code run without JAGS errors, it helps to set the minimum score to 0.001 and the maximum score to 0.999
data {
   for (i in 1:n) {
      # Transform the scores to log odds
      logoddss[i] <- log( s[i] / ( 1-s[i] ))
      # Create a boolean variable for predicted label associated with the i-th instance
      # Assume that a score > 0.5 will lead to a predicted label of "1"
      predl[i] <- ( s[i]>0.5 )
      # Create indicator matrix iscat[j,i] where iscat[j,i]=1 when the i-th instance belongs to category j
      for (j in 1:nc) {
         iscat[j,i] <- ( ct[i] == j )
      }
   }
   for (i in 1:nl) {
      # Create an indicator for correctness of the i-th instance
      iscorrect[i] <- (predl[i] == y[i])
   }
   for (i in (nl+1):n) {
      # We don't need the correctness labels for these instances, but we add them to avoid
      # JAGS from compilation errors
      iscorrect[i] <- 0
   }
}
model{
   # Loop over all instances
   for (i in 1:n) {
      # The LLO model is a linear transformation of the log odds
      # Place an upper bound of 15 to avoid numerical problems with conversion to probability
      logoddsp[i] <- min( b[ct[i]] + exp( a[ct[i]] ) * logoddss[i] , 15 )
      # p is the model probability of label "1" given this score
      p[i] <- exp( logoddsp[i] ) / ( exp( logoddsp[i] ) + 1 )
      # pc is the pointwise probability that the instance is classified correctly
      pc[i] <- p[i]*predl[i] + (1-p[i])*(1-predl[i])
      # pc2 is the pointwise probability of classifying an instance correctly
      # For the labeled instances, we take observed indicator of correctness
      pc2[i] <- ifelse( i<=nl , iscorrect[i] , pc[i] )
   }
   # Loop over all instances with observed labels
   for (i in 1:nl) {
      # Draw a label according to the LLO probability
      y[i] ~ dbern( p[i] )
   }
   # Sample the calibration parameters for each category
   for (j in 1:nc) {
      b[j] ~ dnorm( mub , 1/(stdb^2))
      a[j] ~ dnorm( mua , 1/(stda^2))
   }
   # Compute the mean accuracy per category
   for (j in 1:nc) {
      # The inner product (inprod) computes the sum of the accuracies for all instances belonging to category j,
      # and the denominator computes the sum of the number of instances belonging to category j
      theta[j] <- inprod( pc2 , iscat[j,] ) / sum( iscat[j,] )
   }
   # Normal priors on the group means
   mua ~  dnorm( mumua , 1/(stdmua^2))
   mub ~  dnorm( mumub , 1/(stdmub^2))
   # Half-normal (truncated) priors for the standard deviations
   stda ~ dnorm(0, 1/(stdstda^2) ) T(0,)
   stdb ~ dnorm(0, 1/(stdstdb^2) ) T(0,)
}

'''

max_stars_repo_name: disiji/bayesian-fairness-assess

max_issues_repo_name: disiji/bayesian-fairness-assess

max_forks_repo_name: disiji/bayesian-fairness-assess


text:
import torch
import torch.nn as nn
import numpy as np
import pandas as pd
from convnet_feat import get_features, imshow
import matplotlib.pyplot as plt

# Functions for loading SHJ abstract data and images

# Filenames for SGJ stimuli and labels in abstract form
fn_shj_abstract = 'data/shj_stimuli.txt'
fn_shj_labels = 'data/shj_labels.txt'

def get_label_coding(loss_type):
	# Set coding for class A and class B
	POSITIVE = 1.
	if loss_type == 'hinge':
		NEGATIVE = -1.
	elif loss_type == 'll':
		NEGATIVE = 0.
	else:
		assert False
	return POSITIVE,NEGATIVE

def load_shj(loss_type):
	# Loads SHJ data from text files
	#
	# Input
	#   loss_type : either ll or hinge loss
	#
	# Output
	#   X : [ne x dim tensor] stimuli as rows
	#   y_list : list of [ne tensor] labels, with a list element for each shj type
	stimuli = pd.read_csv(fn_shj_abstract, header=None).to_numpy()
	labels = pd.read_csv(fn_shj_labels, header=None).to_numpy()
	stimuli = stimuli.astype(float)
	ntype = labels.shape[0]
	POSITIVE,NEGATIVE = get_label_coding(loss_type)
	labels_float = np.zeros(labels.shape,dtype=float)
	labels_float[labels == 'A'] = POSITIVE
	labels_float[labels == 'B'] = NEGATIVE
	X = torch.tensor(stimuli).float()
	y_list = []
	for mytype in range(ntype):
		y = labels_float[mytype,:].flatten()
		y = torch.tensor(y).float()
		y_list.append(y)
	return X,y_list

def process_shj_images():
	# Return
	#  X : [ne x dim tensor] stimuli as rows
	mydir = 'data/shj_images_set1'
	# mydir = 'data/shj_images_set2'
	print(" Passing SHJ images through ConvNet...")
	# stimuli,images = get_features(mydir,'vgg11')
	stimuli,images = get_features(mydir,'resnet18')

	print(" Done.")
	stimuli = stimuli.cpu().data.numpy().astype(float)
	images = images.cpu().data.numpy()
	X = torch.tensor(stimuli).float()
	return X,images

def load_shj_abstract(loss_type, perm=[0,1,2]):
	# Loads SHJ data in abstract form
	#
	# Input
	#   loss_type : either ll or hinge loss
	#   perm : permutation of abstract feature indices
	#
	# Output
	#   X : [ne x dim tensor] stimuli as rows
	#   y_list : list of [ne tensor] labels, with a list element for each shj type

	# load image and abstract data
	X,y_list = load_shj(loss_type)
	X_abstract = X.data.numpy().astype(int)

	# Apply permutation
	X_perm = X_abstract.copy()
	X_perm = X_perm[:,perm] # permuted features
	perm_idx = []
	for x in X_perm:
		idx = np.where((X_abstract == x).all(axis=1))[0] # get item mapping from original order to perm order
		perm_idx.append(idx[0])
	perm_idx = np.array(perm_idx)
	X = X[perm_idx,:] # permute items from original order to permuted order
	return X,y_list

def load_shj_images(loss_type, perm=[0,1,2], viz_cats=False):
	# Loads SHJ data from images
	#
	# Input
	#   loss_type : either ll or hinge loss
	#   perm : permutation of abstract feature indices
	#
	# Output
	#   X : [ne x dim tensor] stimuli as rows
	#   y_list : list of [ne tensor] labels, with a list element for each shj type

	# load image and abstract data
	X,images = process_shj_images()
	X_abstract,y_list = load_shj(loss_type)
	X_abstract = X_abstract.data.numpy().astype(int)

	# Apply permutation
	X_perm = X_abstract.copy()
	X_perm = X_perm[:,perm] # permuted features
	perm_idx = []
	for x in X_perm:
		idx = np.where((X_abstract == x).all(axis=1))[0] # get item mapping from original order to perm order
		perm_idx.append(idx[0])
	perm_idx = np.array(perm_idx)
	X = X[perm_idx,:] # permute items from original order to permuted order
	images = images[perm_idx] # permute items from original order to permuted order

	if viz_cats:
		for mytype in range(6): # for each type
			y = y_list[mytype].data.numpy()
			images_A = images[y == 1.]
			images_B = images[y != 1.]
			plt.figure(mytype+1)
			for j in range(len(images_A)):
				ax = plt.subplot(2, 4, j+1)
				ax.axis('off')
				imshow(images_A[j])
			for j in range(len(images_B)):
				ax = plt.subplot(2, 4, 5+j)
				ax.axis('off')
				imshow(images_B[j])
			# plt.savefig(str(mytype) + '.pdf')
		plt.show()

	return X,y_list

max_stars_repo_name: brendenlake/alcove

max_issues_repo_name: brendenlake/alcove

max_forks_repo_name: brendenlake/alcove


text:
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt

def crop(image, return_array=False, shape=(3, 3), plot=True, figsize=(10, 10)):
    """
    Slice image into multiple images.
    Params:
        image: str/pathlib object, image directory
        return_array: bool, if true, return sliced images as np.array
        plot: bool, if true, plot sliced images
        shape: tuple (nrows, ncols), number of images the raw input will be sliced into
        figsize: tuple
    Return: list of np.array contained sliced images
    """
    im_np = np.array(Image.open(image))
    M = im_np.shape[0]//shape[0]
    N = im_np.shape[1]//shape[1]

    tiles = [im_np[x:x+M,y:y+N] for x in range(0,im_np.shape[0],M) for y in range(0,im_np.shape[1],N)]
    if plot:
        fig, axs = plt.subplots(nrows=shape[0], ncols=shape[1], figsize=figsize)
        axs = axs.flatten()
        for tile, ax in zip(tiles, axs):
            ax.imshow(tile)
        plt.show()
    if return_array:
        return tiles
    return None

def reconstruct(images, return_array=False, shape=(3, 3), plot=True, figsize=(10, 10)):
    """
    Reversed outputs from crop and combined with to a single image.
    """
    img = np.vstack([np.column_stack(tiles[i: i+shape[1]]) for i in range(0, len(images), shape[1])])
    if plot:
        fig, ax = plt.subplots(figsize=figsize)
        ax.imshow(img)
        plt.show()
    if return_array:
        return img
    return None

max_stars_repo_name: xinrongl/mmdetection

max_issues_repo_name: xinrongl/mmdetection

max_forks_repo_name: xinrongl/mmdetection


text:
import numpy as np
import torch
from torch.utils.data import DataLoader
from torch import nn
from torch.nn import functional as F
import torchvision.transforms as transforms
from sklearn import metrics
import argparse
import timeit
import cv2
import os

import datasets

class RandomGaussianBlur(object):
    def __call__(self, img):
        do_it = np.random.rand() > 0.5
        if not do_it:
            return img
        sigma = np.random.rand() * 1.9 + 0.1
        return cv2.GaussianBlur(np.asarray(img), (23, 23), sigma)

def get_color_distortion(s=1.0):
    # s is the strength of color distortion.
    color_jitter = transforms.ColorJitter(0.8 * s, 0.8 * s, 0.8 * s, 0.2 * s)
    rnd_color_jitter = transforms.RandomApply([color_jitter], p=0.8)
    rnd_gray = transforms.RandomGrayscale(p=0.2)
    color_distort = transforms.Compose([rnd_color_jitter, rnd_gray])
    return color_distort

def main(args):
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    print('Using %s device.' % (device))

    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.228, 0.224, 0.225])
    # build the augmentations
    transform = transforms.Compose([
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(p=0.5),
        transforms.Compose([
            get_color_distortion(),
            RandomGaussianBlur(),
            ]),
        transforms.ToTensor(),
        normalize,
        ])

    # init the dataset and augmentations
    train_dataset = datasets.ODIR5K('train', transform)
    train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=0)

    test_dataset = datasets.ODIR5K('test', transform)
    test_loader = DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False, num_workers=0)

    net = torch.hub.load('facebookresearch/swav', 'resnet50')

    num_features = net.fc.in_features
    net.fc = nn.Sequential(
            nn.Linear(num_features, args.classes),
            nn.Sigmoid())

    if args.model_path and os.path.exists(args.model_path):
        net.load_state_dict(torch.load(args.model_path, map_location=device))
        print('model state has loaded.')

    net = net.to(device)

    optimizer = torch.optim.Adam(net.parameters(), lr=args.lr)
    criterion = torch.nn.CrossEntropyLoss()

    net.train()
    for epoch in range(args.epochs):
        start_time = timeit.default_timer()
        y_true = torch.FloatTensor()
        y_pred = torch.FloatTensor()
        train_loss = 0
        for index, (images, labels) in enumerate(train_loader, 1):
            images = images.to(device)
            labels = labels.to(device)

            outputs = net(images)

            loss = criterion(outputs, labels)
            train_loss += loss.item()

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            y_true = torch.cat((y_true, labels.cpu()))
            y_pred = torch.cat((y_pred, outputs.detach().cpu()))

            print('\repoch %3d/%3d batch %3d/%3d' % (epoch+1, args.epochs, index, len(train_loader)), end='')
            print(' loss %6.4f' % (train_loss / index), end='')
            print(' %5.1fsec' % (timeit.default_timer() - start_time), end='')

        aucs = [metrics.roc_auc_score(y_true[:, i], y_pred[:, i]) for i in range(args.classes)]
        auc_classes = ' '.join(['%5.3f' % (aucs[i]) for i in range(args.classes)])
        print(' mean AUC %5.3f (%s)' % (np.mean(aucs), auc_classes))
        torch.save(net.state_dict(), 'model/checkpoint.pth')

    net.eval()
    start_time = timeit.default_timer()
    y_true = torch.FloatTensor()
    y_pred = torch.FloatTensor()
    test_loss = 0
    for index, (images, labels) in enumerate(test_loader, 1):
        images = images.to(device)
        labels = labels.to(device)

        with torch.no_grad():
            outputs = net(images)

        loss = criterion(outputs, labels)
        test_loss += loss.item()

        y_true = torch.cat((y_true, labels.cpu()))
        y_pred = torch.cat((y_pred, outputs.detach().cpu()))

        print('\rtest batch %3d/%3d' % (index, len(test_loader)), end='')
        print(' loss %6.4f' % (test_loss / index), end='')
        print(' %5.1fsec' % (timeit.default_timer() - start_time), end='')

    aucs = [metrics.roc_auc_score(y_true[:, i], y_pred[:, i]) for i in range(args.classes)]
    auc_classes = ' '.join(['%5.3f' % (aucs[i]) for i in range(args.classes)])
    print(' mean AUC %5.3f (%s)' % (np.mean(aucs), auc_classes))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--model_path', default=None, type=str)
    parser.add_argument('--epochs', default=150, type=int)
    parser.add_argument('--batch_size', default=32, type=int)
    parser.add_argument('--classes', default=8, type=int)
    parser.add_argument('--lr', default=1e-4, type=float) # 5e-5
    parser.add_argument('--momentum', default=0.9, type=float)
    args = parser.parse_args()
    print(vars(args))

    main(args)


text:
from flask import Blueprint, request, render_template, session, redirect, url_for
from flask.wrappers import Response
from loguru import logger
from src.utils.common.data_helper import load_data
from src.utils.common.plotly_helper import PlotlyHelper
from src.utils.common.project_report_helper import ProjectReports
import numpy as np
from src.eda.eda_helper import EDA
from pandas_profiling import ProfileReport
from src.constants.constants import TWO_D_GRAPH_TYPES, TWO_D_GRAPH_TYPES_2
import plotly.figure_factory as ff
import json
import plotly
from src.utils.common.common_helper import immutable_multi_dict_to_str, get_numeric_categorical_columns
import os
from from_root import from_root
import pandas as pd

app_eda = Blueprint('eda', __name__)

@app_eda.route('/eda/<action>')
def eda(action):
    try:
        if 'pid' in session:
            df = load_data()
            if df is not None:
                if action == "data-summary":
                    ProjectReports.insert_record_eda('Redirect To Data Summary')
                    summary = EDA.five_point_summary(df)
                    data = summary.to_html()
                    dtypes = EDA.data_dtype_info(df)
                    return render_template('eda/5point.html', data=data, dtypes=dtypes.to_html(), count=len(df),
                                           column_count=df.shape[1])
                elif action == "profiler":
                    ProjectReports.insert_record_eda('Redirect To Profile Report')
                    return render_template('eda/profiler.html', action=action)

                elif action == "show":
                    ProjectReports.insert_record_eda('Redirect To Show Dataset')
                    data = EDA.get_no_records(df, 100)
                    data = data.to_html()
                    topselected = True
                    bottomSelected = False
                    selectedCount = 100
                    return render_template('eda/showdataset.html', data=data, length=len(df),
                                           bottomSelected=bottomSelected, topselected=topselected, action=action,
                                           selectedCount=selectedCount, columns=df.columns)
                elif action == "missing":
                    ProjectReports.insert_record_eda('Redirect To Missing Value')
                    df = EDA.missing_cells_table(df)

                    if df is not None:

                        graphJSON = PlotlyHelper.barplot(df, x='Column', y='Missing values')
                        pie_graphJSON = PlotlyHelper.pieplot(df, names='Column', values='Missing values',
                                                             title='Missing Values')

                        data = df.drop('Column', axis=1)
                        data = data.to_html()
                        return render_template('eda/missing_values.html', action=action, data=data, barplot=graphJSON,
                                               pieplot=pie_graphJSON, contain_missing=True)
                    else:
                        return render_template('eda/missing_values.html', action=action, contain_missing=False)

                elif action == "outlier":
                    ProjectReports.insert_record_eda('Redirect To Outlier')
                    df = EDA.z_score_outlier_detection(df)
                    graphJSON = PlotlyHelper.barplot(df, x='Features', y='Total outliers')
                    pie_graphJSON = PlotlyHelper.pieplot(
                        df.sort_values(by='Total outliers', ascending=False).loc[: 10 if len(df) > 10 else len(df)-1, :],
                        names='Features', values='Total outliers', title='Top 10 Outliers')
                    data = df.to_html()
                    return render_template('eda/outliers.html', data=data, method='zscore', action=action,
                                           barplot=graphJSON, pieplot=pie_graphJSON)

                elif action == "correlation":
                    ProjectReports.insert_record_eda('Redirect To Correlation')
                    pearson_corr = EDA.correlation_report(df, 'pearson')
                    persion_data = list(np.around(np.array(pearson_corr.values), 2))
                    fig = ff.create_annotated_heatmap(persion_data, x=list(pearson_corr.columns),
                                                      y=list(pearson_corr.columns), colorscale='Viridis')
                    graphJSON = json.dumps(fig, cls=plotly.utils.PlotlyJSONEncoder)
                    return render_template('eda/correlation.html', data=graphJSON, columns=list(pearson_corr.columns),
                                           action=action, method='pearson')

                elif action == "plots":
                    ProjectReports.insert_record_eda('Plots')
                    num_cols, cat_cols = get_numeric_categorical_columns(df)
                    if len(cat_cols) == 0:
                        graph_type_list = TWO_D_GRAPH_TYPES_2
                    else:
                        graph_type_list = TWO_D_GRAPH_TYPES

                    return render_template('eda/plots.html', columns=list(df.columns), x_list=list(df.columns),
                                           y_list=num_cols,
                                           graphs_2d=graph_type_list, action=action, x_column="", y_column="")
                else:
                    return render_template('eda/help.html')
            else:
                return redirect('/')

        else:
            return redirect(url_for('/'))
    except Exception as e:
        ProjectReports.insert_record_eda(e)
        logger.error(e)
        return render_template('500.html', exception=e)

@app_eda.route('/eda/<action>', methods=['POST'])
def eda_post(action):
    try:
        if 'pid' in session:
            df = load_data()
            if df is not None:
                graphJSON = None
                if action == "show":
                    range = request.form['range']
                    optradio = request.form['optradio']
                    columns_for_list = df.columns
                    columns = request.form.getlist('columns')
                    input_str = immutable_multi_dict_to_str(request.form)
                    ProjectReports.insert_record_eda('Show', input=input_str)

                    if len(columns) > 0:
                        df = df.loc[:, columns]

                    data = EDA.get_no_records(df, int(range), optradio)
                    data = data.to_html()
                    topselected = True if optradio == 'top' else False
                    bottomSelected = True if optradio == 'bottom' else False
                    return render_template('eda/showdataset.html', data=data, length=len(df),
                                           bottomSelected=bottomSelected, topselected=topselected, action=action,
                                           selectedCount=range, columns=columns_for_list)
                elif action == "profiler":
                    ProjectReports.insert_record_eda('Download  Profile Report')

                    pr = ProfileReport(df, explorative=True, minimal=True,
                                       correlations={"cramers": {"calculate": False}})

                    report_path = os.path.join(from_root(), "artifacts", f"{session.get('id')}_report.html")
                    pr.to_file(report_path)
                    with open(report_path) as fp:
                        content = fp.read()

                    return Response(
                        content,
                        mimetype="text/csv",
                        headers={"Content-disposition": "attachment; filename=report.html"})

                elif action == "correlation":
                    method = request.form['method']
                    columns = request.form.getlist('columns')

                    input_str = immutable_multi_dict_to_str(request.form, True)
                    ProjectReports.insert_record_eda('Redirect To Correlation', input=input_str)

                    if method is not None:
                        # df=df.loc[:,columns]
                        _corr = EDA.correlation_report(df, method)
                        if len(columns) == 0:
                            columns = _corr.columns

                        _corr = _corr.loc[:, columns]
                        _data = list(np.around(np.array(_corr.values), 2))
                        fig = ff.create_annotated_heatmap(_data, x=list(_corr.columns),
                                                          y=list(_corr.index), colorscale='Viridis')
                        # fig = ff.create_annotated_heatmap(_data, colorscale='Viridis')
                        graphJSON = json.dumps(fig, cls=plotly.utils.PlotlyJSONEncoder)
                        return render_template('eda/correlation.html', data=graphJSON,
                                               columns=list(df.select_dtypes(exclude='object').columns), action=action,
                                               method=method)
                    else:
                        return render_template('eda/help.html')

                elif action == "outlier":
                    method = request.form['method']
                    print(method)
                    lower = 25
                    upper = 75
                    if method == "iqr":
                        lower = request.form['lower']
                        upper = request.form['upper']
                        df = EDA.outlier_detection_iqr(df, int(lower), int(upper))
                        print(df)
                    else:
                        df = EDA.z_score_outlier_detection(df)
                        print('missed')

                    input_str = immutable_multi_dict_to_str(request.form, True)
                    ProjectReports.insert_record_eda('Redirect To Outlier', input=input_str)

                    graphJSON = PlotlyHelper.barplot(df, x='Features', y='Total outliers')

                    pie_graphJSON = PlotlyHelper.pieplot(
                        df.sort_values(by='Total outliers', ascending=False).loc[: 10 if len(df) > 10 else len(df)-1,:],
                        names='Features', values='Total outliers', title='Top 10 Outliers')

                    data = df.to_html()
                    return render_template('eda/outliers.html', data=data, method=method, action=action, lower=lower,
                                           upper=upper, barplot=graphJSON, pieplot=pie_graphJSON)

                elif action == "plots":
                    """All Polots for all kind of features????"""
                    selected_graph_type = request.form['graph']

                    input_str = immutable_multi_dict_to_str(request.form)
                    ProjectReports.insert_record_eda('Plot', input=input_str)
                    num_cols, cat_cols = get_numeric_categorical_columns(df)
                    if len(cat_cols) == 0:
                        graph_type_list = TWO_D_GRAPH_TYPES_2
                    else:
                        graph_type_list = TWO_D_GRAPH_TYPES

                    if selected_graph_type == "Scatter Plot":
                        x_column = request.form['xcolumn']
                        y_column = request.form['ycolumn']
                        graphJSON = PlotlyHelper.scatterplot(df, x=x_column, y=y_column, title='Scatter Plot')

                    elif selected_graph_type == "Pie Chart":

                        x_column = request.form['xcolumn']
                        new_df = df.groupby(x_column).count()
                        temp_df = pd.DataFrame()

                        temp_df[x_column] = list(new_df.index)
                        temp_df['Count'] = list(new_df.iloc[:, 0])

                        graphJSON = PlotlyHelper.pieplot(temp_df, names=x_column, values='Count', title='Pie Chart')

                    elif selected_graph_type == "Bar Graph":
                        x_column = request.form['xcolumn']
                        new_df = df.groupby(x_column).count()
                        temp_df = pd.DataFrame()

                        temp_df[x_column] = list(new_df.index)
                        temp_df['Count'] = list(new_df.iloc[:, 0])

                        graphJSON = PlotlyHelper.barplot(temp_df, x=x_column, y='Count')

                    elif selected_graph_type == "Histogram":
                        x_column = request.form['xcolumn']
                        graphJSON = PlotlyHelper.histogram(df, x=x_column)

                    elif selected_graph_type == "Line Chart":
                        x_column = request.form['xcolumn']
                        y_column = request.form['ycolumn']
                        graphJSON = PlotlyHelper.line(df, x=x_column, y=y_column)

                    elif selected_graph_type == "Box Plot":
                        x_column = request.form['xcolumn']
                        y_column = request.form['ycolumn']
                        graphJSON = PlotlyHelper.boxplot(df, x=x_column, y=y_column)

                    elif selected_graph_type == "Dist Plot":
                        x_column = request.form['xcolumn']
                        y_column = request.form['ycolumn']
                        hist_data = []
                        category_list = list(df[y_column].unique())
                        for category in category_list:
                            hist_data.append(list(df[df[y_column] == category][x_column]))

                        graphJSON = PlotlyHelper.create_distplot(hist_data, category_list)

                    elif selected_graph_type == "Heat Map":
                        graphJSON = PlotlyHelper.heatmap(df)

                    return render_template('eda/plots.html', selected_graph_type=selected_graph_type,
                                           columns=list(df.columns), graphs_2d=graph_type_list,
                                           action=action, graphJSON=graphJSON)
                else:
                    return render_template('eda/help.html')
            else:
                """Manage This"""
                pass

        else:
            return redirect(url_for('/'))
    except Exception as e:
        ProjectReports.insert_record_eda(e)
        return render_template('500.html', exception=e)

@app_eda.route('/x_y_columns', methods=['GET', 'POST'])
def x_y_columns():
    try:
        if 'pid' in session:
            graph_selected = request.args.get('graph_selected')
            df = load_data()
            if df is not None:
                num_cols, cat_cols = get_numeric_categorical_columns(df)
                if graph_selected == "Bar Graph":
                    return render_template('eda/x_y_columns.html', x_list=list(cat_cols),
                                           graph_selected=graph_selected)
                elif graph_selected == "Histogram":
                    return render_template('eda/x_y_columns.html', x_list=list(df.columns), y_list=[],
                                           graph_selected=graph_selected)
                elif graph_selected == "Scatter Plot":
                    return render_template('eda/x_y_columns.html', x_list=list(num_cols), y_list=list(num_cols),
                                           graph_selected=graph_selected)
                elif graph_selected == "Pie Chart":
                    return render_template('eda/x_y_columns.html', x_list=list(cat_cols),
                                           graph_selected=graph_selected)
                elif graph_selected == "Line Chart":
                    return render_template('eda/x_y_columns.html', x_list=list(num_cols), y_list=list(num_cols),
                                           graph_selected=graph_selected)
                elif graph_selected == "Box Plot":
                    return render_template('eda/x_y_columns.html', x_list=list(cat_cols), y_list=list(num_cols),
                                           graph_selected=graph_selected)
                elif graph_selected == "Dist Plot":
                    return render_template('eda/x_y_columns.html', x_list=list(num_cols), y_list=list(cat_cols),
                                           graph_selected=graph_selected)
                elif graph_selected == "Heat Map":
                    return render_template('eda/x_y_columns.html', graph_selected=graph_selected)
                else:
                    return redirect(url_for('/eda/help'))
            else:
                """Manage This"""
                pass
        else:
            return redirect(url_for('/'))
    except Exception as e:
        ProjectReports.insert_record_eda(e)
        return render_template('500.html', exception=e)

max_stars_repo_name: Pankajmalhan/simplified-ml

max_issues_repo_name: Pankajmalhan/simplified-ml

max_forks_repo_name: Pankajmalhan/simplified-ml


text:
"""
This file is used to get the velocity at a fixed point in vtu file. It is provided by Dr. Claire Heaney.
"""

import vtk, vtktools
import sys, os
import numpy as np
import matplotlib.pyplot as plt

coordinates = np.array([[0.4,0.15,0],[0.4,0.2,0],[0.4,0.25,0]])
N = 1000 # time levels
Npred = 0 # prediction starts at 100
print(coordinates.shape[1]+1)
values_at_sensors = np.zeros((N,coordinates.shape[1]+1))
values_at_sensors[:,0] = np.linspace(0,N-1,N)

prediction_at_CAE = np.zeros((N,coordinates.shape[1]+1))
prediction_at_CAE[:,0] = np.linspace(0,N-1,N)

prediction_at_HAE = np.zeros((N,coordinates.shape[1]+1))
prediction_at_HAE[:,0] = np.linspace(0,N-1,N)

prediction_at_SAE = np.zeros((N,coordinates.shape[1]+1))
prediction_at_SAE[:,0] = np.linspace(0,N-1,N)

prediction_at_SVD = np.zeros((N,coordinates.shape[1]+1))
prediction_at_SVD[:,0] = np.linspace(0,N-1,N)

prediction_at_SVDAE = np.zeros((N,coordinates.shape[1]+1))
prediction_at_SVDAE[:,0] = np.linspace(0,N-1,N)

for k in range(N):

    filename = "./FPC_Re3900_DG_new_sum/FPC_Re3900_DG_new/fpc_" + str(k) + ".vtu"
    vtu_data = vtktools.vtu(filename)
    v = vtu_data.ProbeData(coordinates, "Velocity")
    norm_v = np.linalg.norm(v,axis=1)
    values_at_sensors[k,1:] = norm_v

    # filename = "./CAE_64/CAE_reconstructed_" + str(k) + ".vtu"
    # vtu_data = vtktools.vtu(filename)
    # v = vtu_data.ProbeData(coordinates, "Velocity_CAE")
    # norm_v = np.linalg.norm(v,axis=1)
    # prediction_at_CAE[k,1:] = norm_v

    # filename = "./HAE_64/reconstructed_" + str(k) + ".vtu"
    # vtu_data = vtktools.vtu(filename)
    # v = vtu_data.ProbeData(coordinates, "Velocity_CAE")
    # norm_v = np.linalg.norm(v,axis=1)
    # prediction_at_HAE[k,1:] = norm_v

    filename = "./SAE_64/SAE_reconstructed_" + str(k) + ".vtu"
    vtu_data = vtktools.vtu(filename)
    v = vtu_data.ProbeData(coordinates, "Velocity_CAE")
    norm_v = np.linalg.norm(v,axis=1)
    prediction_at_SAE[k,1:] = norm_v

    # filename = "./SVD_LV64/SVD_reconstructed_" + str(k) + ".vtu"
    # vtu_data = vtktools.vtu(filename)
    # v = vtu_data.ProbeData(coordinates, "Velocity_CAE")
    # norm_v = np.linalg.norm(v,axis=1)
    # prediction_at_SVD[k,1:] = norm_v

    # filename = "./SVDAE_64/SVDAE_reconstructed_" + str(k) + ".vtu"
    # vtu_data = vtktools.vtu(filename)
    # v = vtu_data.ProbeData(coordinates, "Velocity_CAE")
    # norm_v = np.linalg.norm(v,axis=1)
    # prediction_at_SVDAE[k,1:] = norm_v

#norm_v = np.atleast_2d( np.linalg.norm(v,axis=1)).T
#print norm_v

plt.rcParams['figure.figsize'] = (8.0, 4.0) # Set the figure size
plt.rcParams['savefig.dpi'] = 300 # Set the figure pixel
plt.rcParams['figure.dpi'] = 300

fig = plt.figure(figsize=(8, 5))
ax1 = fig.add_subplot(111)
ax1.set_xlim(0,1000)
ax1.set_ylim(0,0.08)

plt.tick_params(labelsize=15)
labels = ax1.get_xticklabels() + ax1.get_yticklabels()
[label.set_fontname('Times New Roman') for label in labels]

ax1.plot(values_at_sensors[:,1], linewidth=2.0,label='Original')
# ax1.plot(prediction_at_CAE[Npred:,0],prediction_at_CAE[Npred:,1], linewidth=2.0,label='SFC-CAE')
# ax1.plot(prediction_at_HAE[Npred:,0],prediction_at_HAE[Npred:,1], linewidth=2.0,label='SFC-HAE')
ax1.plot(prediction_at_SAE[Npred:,0],prediction_at_SAE[Npred:,1], linewidth=2.0,label='SFC-SAE')
# ax1.plot(prediction_at_SVD[Npred:,0],prediction_at_SVD[Npred:,1], linewidth=2.0,label='POD')
# ax1.plot(prediction_at_SVDAE[Npred:,0],prediction_at_SVDAE[Npred:,1], linewidth=2.0,label='SVD-AE')
ax1.legend(loc = "best",fontsize=16)
ax1.set_xlabel('$time step$', fontsize=20)
ax1.set_ylabel('$value$', fontsize=20)
plt.savefig('SAE_velocity.png',bbox_inches='tight')