text:
# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
# Copyright 2019 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
This file contains code artifacts adapted from the original implementation:
https://github.com/google-research/google-research/blob/master/schema_guided_dst/baseline/data_utils.py
"""

import collections
import json
import os
import pickle
import re
from typing import Dict, List

import numpy as np

from nemo.collections.nlp.data.dialogue_state_tracking.sgd.input_example import InputExample
from nemo.utils import logging

__all__ = ['SGDDataProcessor']

FILE_RANGES = {
    "sgd_single_domain": {"train": range(1, 44), "dev": range(1, 8), "test": range(1, 12)},
    "sgd_multi_domain": {"train": range(44, 128), "dev": range(8, 21), "test": range(12, 35)},
    "sgd_all": {"train": range(1, 128), "dev": range(1, 21), "test": range(1, 35)},
    "sgd_all_single": {"train": range(1, 128), "dev": range(1, 8), "test": range(1, 12)},
    "multiwoz": {"train": range(1, 18), "dev": range(1, 3), "test": range(1, 3)},
    "debug_sample": {"train": range(1, 2), "dev": range(1, 2), "test": range(1, 2)},
}

class SGDDataProcessor(object):
    """Data generator for SGD dialogues."""

    def __init__(
        self,
        task_name: str,
        data_dir: str,
        dialogues_example_dir: str,
        tokenizer: object,
        schemas: object,
        schema_config: Dict[str, int],
        subsample: bool = False,
    ):
        """
        Constructs SGDDataProcessor
        Args:
            task_name: task name, e.g. "sgd_single_domain"
            data_dir: path to data directory
            dialogues_example_dir: path to store processed dialogue examples
            tokenizer: tokenizer object
            schemas: schema object
            schema_config: schema configuration
            subsample: whether to balance positive and negative samples in dataset
        """
        self.data_dir = data_dir

        self._task_name = task_name
        self.schemas = schemas
        self.schema_config = schema_config

        train_file_range = FILE_RANGES[task_name]["train"]
        dev_file_range = FILE_RANGES[task_name]["dev"]
        test_file_range = FILE_RANGES[task_name]["test"]

        self._file_ranges = {
            "train": train_file_range,
            "dev": dev_file_range,
            "test": test_file_range,
        }

        self._seen_services = {
            "train": set(),
            "dev": set(),
            "test": set(),
        }

        self._tokenizer = tokenizer
        self._subsample = subsample
        self._dialogues_example_dir = dialogues_example_dir

        self.dial_files = {}

        # slots_relation_list.np would contain the candidate list of slots for each (service, slot) which would be
        # looked into when a switch between two services happens in the dialogue and we can not find any value for a slot in the current user utterance.
        # This file would get generated from the dialogues in the training set.
        self.slots_relation_file = os.path.join(dialogues_example_dir, f"{task_name}_train_slots_relation_list.np")
        for dataset in ["train", "dev", "test"]:
            # Process dialogue files
            dial_file = f"{task_name}_{dataset}_examples.processed"
            dial_file = os.path.join(dialogues_example_dir, dial_file)
            self.dial_files[(task_name, dataset)] = dial_file

            dialog_paths = SGDDataProcessor.get_dialogue_files(data_dir, dataset, task_name)
            dialogs = SGDDataProcessor.load_dialogues(dialog_paths)
            for dialog in dialogs:
                self._seen_services[dataset].update(set(dialog['services']))

    def save_dialog_examples(self, overwrite_dial_files: bool):
        """
        Preprocesses dialogues and saves to disk.
        Args:
            overwrite_dial_files: whether or not to overwrite saved file if already exists
        """
        for dataset in ["train", "dev", "test"]:
            dial_file = self.dial_files[(self._task_name, dataset)]
            if not os.path.exists(dial_file) or overwrite_dial_files:
                logging.info(f"Start generating the dialogue examples for {dataset} dataset.")
                if not os.path.exists(self._dialogues_example_dir):
                    os.makedirs(self._dialogues_example_dir)
                dial_examples, slots_relation_list = self._generate_dialog_examples(
                    dataset, self.schemas, self._subsample
                )
                for ex in dial_examples:
                    del ex.service_schema
                    del ex._tokenizer
                    del ex.schema_config
                    del ex.user_utterance
                    del ex.categorical_slot_id
                    del ex.system_utterance
                    del ex.noncategorical_slot_id
                    del ex.categorical_slot_value_id
                    del ex.requested_slot_id
                    del ex.intent_id
                with open(dial_file, "wb") as f:
                    np.save(f, dial_examples)

                if dataset == "train":
                    with open(self.slots_relation_file, "wb") as f:
                        pickle.dump(slots_relation_list, f)
                    logging.info(f"The slot carry-over list for train set is stored at {self.slots_relation_file}")

                logging.info(f"The dialogue examples for {dataset} dataset saved at {dial_file}")
                logging.info(f"Finish generating the dialogue examples for {dataset} dataset.")

    def get_dialog_examples(self, dataset_split: str) -> List[object]:
        """
        Loads preprocessed dialogue examples from disk.
        Args:
            dataset_split: dataset split
        Returns:
            dial_examples:  list of InputExample's.
        """
        if (self._task_name, dataset_split) not in self.dial_files or not os.path.exists(
            self.dial_files[(self._task_name, dataset_split)]
        ):
            raise ValueError(
                f"{dataset_split} dialogue examples were not processed for {self._task_name} task. Re-initialize SGDDataProcessor and add {dataset_split} dataset split to datasets arg."
            )
        dial_file = self.dial_files[(self._task_name, dataset_split)]
        logging.info(f"Loading dialogue examples from {dial_file}.")

        with open(dial_file, "rb") as f:
            dial_examples = np.load(f, allow_pickle=True)
            f.close()

        if not os.path.exists(self.slots_relation_file):
            raise ValueError(
                f"Slots relation file {self.slots_relation_file} does not exist. It is needed for the carry-over mechanism of state tracker for switches between services."
            )
        if os.path.getsize(self.slots_relation_file) > 0:
            with open(self.slots_relation_file, "rb") as f:
                self.schemas._slots_relation_list = pickle.load(f)
            logging.info(
                f"Loaded the slot relation list for value carry-over between services from {self.slots_relation_file}."
            )

        return dial_examples

    def get_seen_services(self, dataset_split: str):
        """
        Returns list of seen services, i.e. both in given and training split
        Args:
            dataset_split: data split
        Returns:
            seen_services: list of seen services
        """
        seen_services = self._seen_services[dataset_split]
        return seen_services

    def _generate_dialog_examples(self, dataset_split: str, schemas: object, subsample: bool):
        """
        Returns a list of `InputExample`s of the data splits' dialogues.
        Args:
            dataset_split: data split, can be "train", "dev", or "test".
            schemas: schema for all services of all datasets
            subsample: whether to balance postive and negative samples in the dataset
        Returns:
            examples: a list of `InputExample`s.
        """
        logging.info(f'Creating examples and slot relation list from the dialogues started...')
        dialog_paths = [
            os.path.join(self.data_dir, dataset_split, "dialogues_{:03d}.json".format(i))
            for i in self._file_ranges[dataset_split]
        ]
        dialogs = SGDDataProcessor.load_dialogues(dialog_paths)

        examples = []
        slot_carryover_candlist = collections.defaultdict(int)
        for dialog_idx, dialog in enumerate(dialogs):
            if dialog_idx % 1000 == 0:
                logging.info(f'Processed {dialog_idx} dialogues.')
            examples.extend(
                self._create_examples_from_dialog(dialog, schemas, dataset_split, slot_carryover_candlist, subsample)
            )

        slots_relation_list = collections.defaultdict(list)
        for slots_relation, relation_size in slot_carryover_candlist.items():
            if relation_size > 0:
                slots_relation_list[(slots_relation[0], slots_relation[1])].append(
                    (slots_relation[2], slots_relation[3], relation_size)
                )
                slots_relation_list[(slots_relation[2], slots_relation[3])].append(
                    (slots_relation[0], slots_relation[1], relation_size)
                )

        return examples, slots_relation_list

    def _create_examples_from_dialog(
        self, dialog: dict, schemas: object, dataset_split: str, slot_carryover_candlist: dict, subsample: bool
    ):
        """
        Create examples for every turn in the dialogue.
        Args:
            dialog: dialogue example
            schemas: schema for all services of all datasets
            dataset_split: data split
            slot_carryover_candlist: a dictionary to keep and count the number of carry-over cases between two slots from two different services
            subsample: whether to balance postive and negative samples in the dataset
        Returns:
            examples: a list of `InputExample`s.
        """
        dialog_id = dialog["dialogue_id"]
        prev_states = {}
        examples = []
        for turn_idx, turn in enumerate(dialog["turns"]):
            # Generate an example for every frame in every user turn.
            if turn["speaker"] == "USER":
                user_utterance = turn["utterance"]
                user_frames = {f["service"]: f for f in turn["frames"]}
                if turn_idx > 0:
                    system_turn = dialog["turns"][turn_idx - 1]
                    system_utterance = system_turn["utterance"]
                    system_frames = {f["service"]: f for f in system_turn["frames"]}
                else:
                    system_utterance = ""
                    system_frames = {}

                turn_id = "{}-{}-{:02d}".format(dataset_split, dialog_id, turn_idx)
                turn_examples, prev_states, slot_carryover_values = self._create_examples_from_turn(
                    turn_id,
                    system_utterance,
                    user_utterance,
                    system_frames,
                    user_frames,
                    prev_states,
                    schemas,
                    subsample,
                )
                examples.extend(turn_examples)

                for value, slots_list in slot_carryover_values.items():
                    if value in ["True", "False"]:
                        continue
                    if len(slots_list) > 1:
                        for service1, slot1 in slots_list:
                            for service2, slot2 in slots_list:
                                if service1 == service2:
                                    continue
                                if service1 > service2:
                                    service1, service2 = service2, service1
                                    slot1, slot2 = slot2, slot1
                                slot_carryover_candlist[(service1, slot1, service2, slot2)] += 1
        return examples

    def _get_state_update(self, current_state: dict, prev_state: dict) -> dict:
        """
        Updates dialogue state
        Args:
            current_state: slot values pairs for the current dialogue turn
            prev_state: slot values pairs for the previous dialogue turns
        Returns:
            state_update: slot values pairs that are added/updated during the current dialogue turn
        """
        state_update = dict(current_state)
        for slot, values in current_state.items():
            if slot in prev_state and prev_state[slot][0] in values:
                # Remove the slot from state if its value didn't change.
                state_update.pop(slot)
        return state_update

    def _create_examples_from_turn(
        self,
        turn_id: int,
        system_utterance: str,
        user_utterance: str,
        system_frames: dict,
        user_frames: dict,
        prev_states: dict,
        schemas: object,
        subsample: bool,
    ):
        """
        Creates an example for each frame in the user turn.
        Args:
            turn_id: turn number
            system_utterance: last system utterance
            user_utterance: lst user utterance
            system_frames: all system utterances and slot - slot value pairs
            user_frames: all user utterances and slot - slot value pairs
            prev_states: slot - slot value pairs from the previous turns
            schemas: schema for all services of all datasets
            subsample: whether to balance postive and negative samples in the dataset
        Returns:
            examples: a list of `InputExample`s.
            prev_states: updated dialogue state e.g. {'Restaurants_1': {'city': ['San Jose'], 'cuisine': ['American']}}
        """
        user_tokens, user_alignments, user_inv_alignments = self._tokenize(user_utterance)
        system_tokens, system_alignments, system_inv_alignments = self._tokenize(system_utterance)
        system_user_utterance = system_utterance + ' ' + user_utterance
        system_user_tokens, system_user_alignments, system_user_inv_alignments = self._tokenize(system_user_utterance)
        states = {}

        examples = []
        slot_carryover_values = collections.defaultdict(list)
        for service, user_frame in user_frames.items():

            base_example = InputExample(schema_config=self.schema_config, tokenizer=self._tokenizer,)
            base_example.service_schema = schemas.get_service_schema(service)
            base_example.service_id = schemas.get_service_schema(service).service_id
            system_frame = system_frames.get(service, None)
            state = user_frame["state"]["slot_values"]
            state_update = self._get_state_update(state, prev_states.get(service, {}))
            states[service] = state
            dataset_split, dialog_id, turn_id_ = turn_id.split('-')
            dialog_id_1, dialog_id_2 = dialog_id.split('_')

            base_example.example_id = f"{turn_id}-{service}"
            base_example.example_id_num = [
                int(dialog_id_1),
                int(dialog_id_2),
                int(turn_id_),
                schemas.get_service_id(service),
            ]

            for model_task in range(self.schema_config["NUM_TASKS"]):
                if model_task == 0:
                    for intent_id, intent in enumerate(schemas.get_service_schema(service).intents):
                        task_example = base_example.make_copy()
                        task_example.task_mask[model_task] = 1
                        task_example.intent_id = intent_id
                        task_example.example_id += f"-{model_task}-{intent_id}-0"
                        task_example.example_id_num.extend([model_task, intent_id, 0])
                        intent_description = (
                            intent + " " + schemas.get_service_schema(service).intent_descriptions[intent]
                        )
                        intent_tokens, intent_alignments, intent_inv_alignments = self._tokenize(intent_description)
                        task_example.add_utterance_features(
                            intent_tokens,
                            intent_inv_alignments,
                            system_user_tokens,
                            system_user_inv_alignments,
                            intent_description,
                            system_user_utterance,
                        )
                        task_example.add_intents(user_frame)
                        examples.append(task_example)

                if model_task == 1:
                    for slot_id, slot in enumerate(schemas.get_service_schema(service).slots):
                        task_example = base_example.make_copy()
                        task_example.task_mask[model_task] = 1
                        task_example.requested_slot_id = slot_id
                        task_example.example_id += f"-{model_task}-{slot_id}-0"
                        task_example.example_id_num.extend([model_task, slot_id, 0])
                        slot_description = slot + " " + schemas.get_service_schema(service).slot_descriptions[slot]
                        slot_tokens, slot_alignments, slot_inv_alignments = self._tokenize(slot_description)
                        task_example.add_utterance_features(
                            slot_tokens,
                            slot_inv_alignments,
                            user_tokens,
                            user_inv_alignments,
                            slot_description,
                            user_utterance,
                        )
                        task_example.add_requested_slots(user_frame)
                        examples.append(task_example)
                if model_task == 2:
                    off_slots = []
                    on_slots = []
                    for slot_id, slot in enumerate(schemas.get_service_schema(service).categorical_slots):
                        task_example = base_example.make_copy()
                        task_example.task_mask[model_task] = 1

                        # assert task_example.task_mask == [0, 0, 1, 0, 0, 0]
                        task_example.categorical_slot_id = slot_id
                        task_example.example_id += f"-{model_task}-{slot_id}-0"
                        task_example.example_id_num.extend([model_task, slot_id, 0])
                        slot_description = slot + " " + schemas.get_service_schema(service).slot_descriptions[slot]
                        slot_tokens, slot_alignments, slot_inv_alignments = self._tokenize(slot_description)
                        task_example.add_utterance_features(
                            slot_tokens,
                            slot_inv_alignments,
                            system_user_tokens,
                            system_user_inv_alignments,
                            slot_description,
                            system_user_utterance,
                        )
                        task_example.add_categorical_slots(state_update)
                        if task_example.categorical_slot_status == 0:
                            off_slots.append(task_example)
                        else:
                            on_slots.append(task_example)
                            examples.append(task_example)
                        old_example = task_example

                        for value_id, value in enumerate(
                            schemas.get_service_schema(service).get_categorical_slot_values(slot)
                        ):
                            if dataset_split != 'train' or task_example.categorical_slot_status == 1:
                                task_example = old_example.make_copy_of_categorical_features()
                                task_example.task_mask[3] = 1
                                # assert task_example.task_mask == [0, 0, 0, 1, 0, 0]
                                task_example.categorical_slot_id = slot_id
                                task_example.categorical_slot_value_id = value_id
                                task_example.example_id = base_example.example_id + f"-3-{slot_id}-{value_id}"
                                task_example.example_id_num = base_example.example_id_num + [3, slot_id, value_id]
                                slot_description = slot + " " + value  # add slot description
                                slot_tokens, slot_alignments, slot_inv_alignments = self._tokenize(slot_description)
                                task_example.add_utterance_features(
                                    slot_tokens,
                                    slot_inv_alignments,
                                    system_user_tokens,
                                    system_user_inv_alignments,
                                    slot_description,
                                    system_user_utterance,
                                )
                                task_example.add_categorical_slots(state_update)
                                assert task_example.categorical_slot_status == old_example.categorical_slot_status
                                examples.append(task_example)

                    if dataset_split == 'train' and subsample:
                        num_on_slots = len(on_slots)
                        examples.extend(
                            np.random.choice(off_slots, replace=False, size=min(max(num_on_slots, 1), len(off_slots)))
                        )
                    else:
                        examples.extend(off_slots)

                if model_task == 4:  # noncat slot status
                    off_slots = []
                    on_slots = []
                    for slot_id, slot in enumerate(schemas.get_service_schema(service).non_categorical_slots):
                        task_example = base_example.make_copy()
                        task_example.task_mask[model_task] = 1
                        # assert task_example.task_mask == [0, 0, 0, 0, 1, 0]
                        task_example.noncategorical_slot_id = slot_id
                        task_example.example_id += f"-{model_task}-{slot_id}-0"
                        task_example.example_id_num.extend([model_task, slot_id, 0])
                        slot_description = slot + " " + schemas.get_service_schema(service).slot_descriptions[slot]
                        slot_tokens, slot_alignments, slot_inv_alignments = self._tokenize(slot_description)
                        task_example.add_utterance_features(
                            slot_tokens,
                            slot_inv_alignments,
                            system_user_tokens,
                            system_user_inv_alignments,
                            slot_description,
                            system_user_utterance,
                        )

                        user_span_boundaries = self._find_subword_indices(
                            state_update,
                            user_utterance,
                            user_frame["slots"],
                            user_alignments,
                            user_tokens,
                            2 + len(slot_tokens) + len(system_tokens),
                        )
                        if system_frame is not None:
                            system_span_boundaries = self._find_subword_indices(
                                state_update,
                                system_utterance,
                                system_frame["slots"],
                                system_alignments,
                                system_tokens,
                                2 + len(slot_tokens),
                            )
                        else:
                            system_span_boundaries = {}
                        task_example.add_noncategorical_slots(
                            state_update, user_span_boundaries, system_span_boundaries
                        )
                        if task_example.noncategorical_slot_status == 0:
                            off_slots.append(task_example)
                        else:
                            on_slots.append(task_example)
                            examples.append(task_example)

                        if dataset_split != 'train' or task_example.noncategorical_slot_status == 1:
                            task_example = task_example.make_copy_of_non_categorical_features()
                            task_example.task_mask[5] = 1
                            # assert task_example.task_mask == [0, 0, 0, 0, 0, 1]
                            task_example.example_id = base_example.example_id + f"-5-{slot_id}-0"
                            task_example.example_id_num = base_example.example_id_num + [5, slot_id, 0]
                            examples.append(task_example)
                    if dataset_split == 'train' and subsample:
                        num_on_slots = len(on_slots)
                        examples.extend(
                            np.random.choice(off_slots, replace=False, size=min(max(num_on_slots, 1), len(off_slots)))
                        )
                    else:
                        examples.extend(off_slots)

            if service not in prev_states and int(turn_id_) > 0:
                for slot_name, values in state_update.items():
                    for value in values:
                        slot_carryover_values[value].append((service, slot_name))
                for prev_service, prev_slot_value_list in prev_states.items():
                    if prev_service == service:
                        continue
                    if prev_service in state:
                        prev_slot_value_list = state[prev_service]
                    for prev_slot_name, prev_values in prev_slot_value_list.items():
                        for prev_value in prev_values:
                            slot_carryover_values[prev_value].append((prev_service, prev_slot_name))

        return examples, states, slot_carryover_values

    def _find_subword_indices(
        self,
        slot_values: dict,
        utterance: str,
        char_slot_spans: dict,
        alignments: List[int],
        subwords: List[str],
        bias: int,
    ) -> dict:
        """
        Find indices for subwords corresponding to slot values.
        Args:
            slot_values: slot - slot value pairs
            utterance: utterance
            char_slot_spans: char - slot spans
            alignments: alignments
            subwords: subtokens mapping
            bias: offset
        Returns:
            span_boundaries: span boundaries
        """
        span_boundaries = {}
        for slot, values in slot_values.items():
            # Get all values present in the utterance for the specified slot.
            value_char_spans = {}
            for slot_span in char_slot_spans:
                if slot_span["slot"] == slot:
                    value = utterance[slot_span["start"] : slot_span["exclusive_end"]]
                    start_tok_idx = alignments[slot_span["start"]]
                    end_tok_idx = alignments[slot_span["exclusive_end"] - 1]
                    if 0 <= start_tok_idx < len(subwords):
                        end_tok_idx = min(end_tok_idx, len(subwords) - 1)
                        value_char_spans[value] = (start_tok_idx + bias, end_tok_idx + bias)
            for v in values:
                if v in value_char_spans:
                    span_boundaries[slot] = value_char_spans[v]
                    break
        return span_boundaries

    def _tokenize(self, utterance: str):
        """
        Tokenize the utterance

        Args:
            utterance: A string containing the utterance to be tokenized.

        Returns:
            bert_tokens: A list of tokens obtained by word-piece tokenization of the
                utterance.
            alignments: A dict mapping indices of characters corresponding to start
                and end positions of words (not subwords) to corresponding indices in
                bert_tokens list.
            inverse_alignments: A list of size equal to bert_tokens. Each element is a
                tuple containing the index of the starting and inclusive ending
                character of the word corresponding to the subword. This list is used
                during inference to map word-piece indices to spans in the original
                utterance.
        """
        # utterance = tokenization.convert_to_unicode(utterance)

        # After _naive_tokenize, spaces and punctuation marks are all retained, i.e.
        # direct concatenation of all the tokens in the sequence will be the
        # original string.
        tokens = SGDDataProcessor._naive_tokenize(utterance)
        # ['I', ' ', 'am', ' ', 'feeling', ' ', 'hungry', ' ', 'so', ' ', 'I', ' ', 'would', ' ', 'like', ' ', 'to', ' ', 'find', ' ', 'a', ' ', 'place', ' ', 'to', ' ', 'eat', '.']
        # Filter out empty tokens and obtain aligned character index for each token.
        alignments = {}
        char_index = 0
        bert_tokens = (
            []
        )  # ['I', 'am', 'feeling', 'hungry', 'so', 'I', 'would', 'like', 'to', 'find', 'a', 'place', 'to', 'eat', '.']
        # These lists store inverse alignments to be used during inference.
        bert_tokens_start_chars = []
        bert_tokens_end_chars = []
        for token in tokens:
            if token.strip():
                subwords = self._tokenizer.text_to_tokens(token)
                # Store the alignment for the index of starting character and the
                # inclusive ending character of the token.
                alignments[char_index] = len(bert_tokens)
                bert_tokens_start_chars.extend([char_index] * len(subwords))
                bert_tokens.extend(subwords)
                # The inclusive ending character index corresponding to the word.
                inclusive_char_end = char_index + len(token) - 1
                alignments[inclusive_char_end] = len(bert_tokens) - 1
                bert_tokens_end_chars.extend([inclusive_char_end] * len(subwords))
            char_index += len(token)
        inverse_alignments = list(zip(bert_tokens_start_chars, bert_tokens_end_chars))
        return bert_tokens, alignments, inverse_alignments

    @classmethod
    def _naive_tokenize(cls, s: str):
        """
        Tokenizes a string, separating words, spaces and punctuations.
        Args:
            s: a string
        Returns:
            seq_tok: list of words, spaces and punctuations from the string
        """
        # Spaces and punctuation marks are all retained, i.e. direct concatenation
        # of all the tokens in the sequence will be the original string.
        seq_tok = [tok for tok in re.split(r"([^a-zA-Z0-9])", s) if tok]
        return seq_tok

    @classmethod
    def load_dialogues(cls, dialog_json_filepaths: List[str]) -> List[dict]:
        """
        Obtain the list of all dialogues from specified json files.
        Args:
            dialog_json_filepaths: list of json files
        Returns:
            dialogs: the list of all dialogues
        """
        dialogs = []
        for dialog_json_filepath in sorted(dialog_json_filepaths):
            with open(dialog_json_filepath, 'r', encoding='utf-8') as f:
                dialogs.extend(json.load(f))
                f.close()
        return dialogs

    @classmethod
    def get_dialogue_files(cls, data_dir: str, dataset_split: str, task_name: str):
        """
        Obtain the list of all dialogue json files
        Args:
            data_dir: path to the data folder
            dataset_split: data split
            task_name: SGD task name, see keys of the FILE_RANGES
        Returns:
            dialog: the list of all dialogue json files paths
        """
        return [
            os.path.join(data_dir, dataset_split, 'dialogues_{:03d}.json'.format(fid))
            for fid in FILE_RANGES[task_name][dataset_split]
        ]

max_stars_repo_name: btarjan/NeMo

max_issues_repo_name: btarjan/NeMo

max_forks_repo_name: btarjan/NeMo


text:
#!/usr/bin/env python
# coding: utf8

"""
    Python oneliner script usage.

    USAGE: python -m spleeter {train,evaluate,separate} ...

    Notes:
        All critical import involving TF, numpy or Pandas are deported to
        command function scope to avoid heavy import on CLI evaluation,
        leading to large bootstraping time.
"""
import json
from functools import partial
from glob import glob
from itertools import product
from os.path import join
from pathlib import Path
from typing import Container, Dict, List, Optional

# pyright: reportMissingImports=false
# pylint: disable=import-error
from typer import Exit, Typer

from . import SpleeterError
from .options import *
from .utils.logging import configure_logger, logger

# pylint: enable=import-error

spleeter: Typer = Typer(add_completion=False, no_args_is_help=True, short_help="-h")
""" CLI application. """

@spleeter.callback()
def default(
    version: bool = VersionOption,
) -> None:
    pass

@spleeter.command(no_args_is_help=True)
def train(
    adapter: str = AudioAdapterOption,
    data: Path = TrainingDataDirectoryOption,
    params_filename: str = ModelParametersOption,
    verbose: bool = VerboseOption,
) -> None:
    """
    Train a source separation model
    """
    import tensorflow as tf

    from .audio.adapter import AudioAdapter
    from .dataset import get_training_dataset, get_validation_dataset
    from .model import model_fn
    from .model.provider import ModelProvider
    from .utils.configuration import load_configuration

    configure_logger(verbose)
    audio_adapter = AudioAdapter.get(adapter)
    audio_path = str(data)
    params = load_configuration(params_filename)
    session_config = tf.compat.v1.ConfigProto()
    session_config.gpu_options.per_process_gpu_memory_fraction = 0.45
    estimator = tf.estimator.Estimator(
        model_fn=model_fn,
        model_dir=params["model_dir"],
        params=params,
        config=tf.estimator.RunConfig(
            save_checkpoints_steps=params["save_checkpoints_steps"],
            tf_random_seed=params["random_seed"],
            save_summary_steps=params["save_summary_steps"],
            session_config=session_config,
            log_step_count_steps=10,
            keep_checkpoint_max=2,
        ),
    )
    input_fn = partial(get_training_dataset, params, audio_adapter, audio_path)
    train_spec = tf.estimator.TrainSpec(
        input_fn=input_fn, max_steps=params["train_max_steps"]
    )
    input_fn = partial(get_validation_dataset, params, audio_adapter, audio_path)
    evaluation_spec = tf.estimator.EvalSpec(
        input_fn=input_fn, steps=None, throttle_secs=params["throttle_secs"]
    )
    logger.info("Start model training")
    tf.estimator.train_and_evaluate(estimator, train_spec, evaluation_spec)
    ModelProvider.writeProbe(params["model_dir"])
    logger.info("Model training done")

@spleeter.command(no_args_is_help=True)
def separate(
    deprecated_files: Optional[str] = AudioInputOption,
    files: List[Path] = AudioInputArgument,
    adapter: str = AudioAdapterOption,
    bitrate: str = AudioBitrateOption,
    codec: Codec = AudioCodecOption,
    duration: float = AudioDurationOption,
    offset: float = AudioOffsetOption,
    output_path: Path = AudioOutputOption,
    stft_backend: STFTBackend = AudioSTFTBackendOption,
    filename_format: str = FilenameFormatOption,
    params_filename: str = ModelParametersOption,
    mwf: bool = MWFOption,
    verbose: bool = VerboseOption,
) -> None:
    """
    Separate audio file(s)
    """
    from .audio.adapter import AudioAdapter
    from .separator import Separator

    configure_logger(verbose)
    if deprecated_files is not None:
        logger.error(
            "⚠ -i option is not supported anymore, audio files must be supplied "
            "using input argument instead (see spleeter separate --help)"
        )
        raise Exit(20)
    audio_adapter: AudioAdapter = AudioAdapter.get(adapter)
    separator: Separator = Separator(
        params_filename, MWF=mwf, stft_backend=stft_backend
    )
    for filename in files:
        separator.separate_to_file(
            str(filename),
            str(output_path),
            audio_adapter=audio_adapter,
            offset=offset,
            duration=duration,
            codec=codec,
            bitrate=bitrate,
            filename_format=filename_format,
            synchronous=False,
        )
    separator.join()

EVALUATION_SPLIT: str = "test"
EVALUATION_METRICS_DIRECTORY: str = "metrics"
EVALUATION_INSTRUMENTS: Container[str] = ("vocals", "drums", "bass", "other")
EVALUATION_METRICS: Container[str] = ("SDR", "SAR", "SIR", "ISR")
EVALUATION_MIXTURE: str = "mixture.wav"
EVALUATION_AUDIO_DIRECTORY: str = "audio"

def _compile_metrics(metrics_output_directory) -> Dict:
    """
    Compiles metrics from given directory and returns results as dict.

    Parameters:
        metrics_output_directory (str):
            Directory to get metrics from.

    Returns:
        Dict:
            Compiled metrics as dict.
    """
    import numpy as np
    import pandas as pd

    songs = glob(join(metrics_output_directory, "test/*.json"))
    index = pd.MultiIndex.from_tuples(
        product(EVALUATION_INSTRUMENTS, EVALUATION_METRICS),
        names=["instrument", "metric"],
    )
    pd.DataFrame([], index=["config1", "config2"], columns=index)
    metrics = {
        instrument: {k: [] for k in EVALUATION_METRICS}
        for instrument in EVALUATION_INSTRUMENTS
    }
    for song in songs:
        with open(song, "r") as stream:
            data = json.load(stream)
        for target in data["targets"]:
            instrument = target["name"]
            for metric in EVALUATION_METRICS:
                sdr_med = np.median(
                    [
                        frame["metrics"][metric]
                        for frame in target["frames"]
                        if not np.isnan(frame["metrics"][metric])
                    ]
                )
                metrics[instrument][metric].append(sdr_med)
    return metrics

@spleeter.command(no_args_is_help=True)
def evaluate(
    adapter: str = AudioAdapterOption,
    output_path: Path = AudioOutputOption,
    stft_backend: STFTBackend = AudioSTFTBackendOption,
    params_filename: str = ModelParametersOption,
    mus_dir: Path = MUSDBDirectoryOption,
    mwf: bool = MWFOption,
    verbose: bool = VerboseOption,
) -> Dict:
    """
    Evaluate a model on the musDB test dataset
    """
    import numpy as np

    configure_logger(verbose)
    try:
        import musdb
        import museval
    except ImportError:
        logger.error("Extra dependencies musdb and museval not found")
        logger.error("Please install musdb and museval first, abort")
        raise Exit(10)
    # Separate musdb sources.
    songs = glob(join(mus_dir, EVALUATION_SPLIT, "*/"))
    mixtures = [join(song, EVALUATION_MIXTURE) for song in songs]
    audio_output_directory = join(output_path, EVALUATION_AUDIO_DIRECTORY)
    separate(
        deprecated_files=None,
        files=mixtures,
        adapter=adapter,
        bitrate="128k",
        codec=Codec.WAV,
        duration=600.0,
        offset=0,
        output_path=join(audio_output_directory, EVALUATION_SPLIT),
        stft_backend=stft_backend,
        filename_format="{foldername}/{instrument}.{codec}",
        params_filename=params_filename,
        mwf=mwf,
        verbose=verbose,
    )
    # Compute metrics with musdb.
    metrics_output_directory = join(output_path, EVALUATION_METRICS_DIRECTORY)
    logger.info("Starting musdb evaluation (this could be long) ...")
    dataset = musdb.DB(root=mus_dir, is_wav=True, subsets=[EVALUATION_SPLIT])
    museval.eval_mus_dir(
        dataset=dataset,
        estimates_dir=audio_output_directory,
        output_dir=metrics_output_directory,
    )
    logger.info("musdb evaluation done")
    # Compute and pretty print median metrics.
    metrics = _compile_metrics(metrics_output_directory)
    for instrument, metric in metrics.items():
        logger.info(f"{instrument}:")
        for metric, value in metric.items():
            logger.info(f"{metric}: {np.median(value):.3f}")
    return metrics

def entrypoint():
    """Application entrypoint."""
    try:
        spleeter()
    except SpleeterError as e:
        logger.error(e)

if __name__ == "__main__":
    entrypoint()

ext: py

max_stars_repo_name: ming-hai/spleeter

max_issues_repo_name: ming-hai/spleeter

max_forks_repo_name: ming-hai/spleeter

reason: import numpy


text:
__author__ = "saeedamen"  # Saeed Amen

#
# Copyright 2016 Cuemacro
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on a "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#

import re

import numpy as np
import pandas as pd

import datetime
from datetime import timedelta

from pandas.tseries.offsets import BDay, CustomBusinessDay, Day, \
    CustomBusinessMonthEnd, DateOffset
from findatapy.timeseries.timezone import Timezone

from findatapy.util.dataconstants import DataConstants
from findatapy.util.loggermanager import LoggerManager

constants = DataConstants()

# To speed up CustomBusinessDay
# https://stackoverflow.com/questions/31523302/performance-of-pandas-custom-business-day-offset

class Calendar(object):
    """Provides calendar based functions for working out options expiries,
    holidays etc. Note, that at present, the expiry _calculations are
    approximate.

    """

    # Approximate mapping from tenor to number of business days
    _tenor_bus_day_dict = {'ON': 1,
                           'TN': 2,
                           '1W': 5,
                           '2W': 10,
                           '3W': 15,
                           '1M': 20,
                           '2M': 40,
                           '3M': 60,
                           '4M': 80,
                           '6M': 120,
                           '9M': 180,
                           '1Y': 252,
                           '2Y': 252 * 2,
                           '3Y': 252 * 3,
                           '5Y': 252 * 5
                           }

    def __init__(self):
        self._holiday_df = pd.read_parquet(constants.holidays_parquet_table)

    def flatten_list_of_lists(self, list_of_lists):
        """Flattens lists of obj, into a single list of strings (rather than
        characters, which is default behavior).

        Parameters
        ----------
        list_of_lists : obj (list)
            List to be flattened

        Returns
        -------
        str (list)
        """

        if isinstance(list_of_lists, list):
            rt = []
            for i in list_of_lists:
                if isinstance(i, list):
                    rt.extend(self.flatten_list_of_lists(i))
                else:
                    rt.append(i)

            return rt

        return list_of_lists

    def _get_full_cal(self, cal):
        holidays_list = []

        # Calendars which have been hardcoded in the parquet file (which users
        # may also edit)
        if len(cal) == 6:
            # Eg. EURUSD (load EUR and USD calendars and combine the holidays)
            holidays_list.append(
                [self._get_full_cal(cal[0:3]), self._get_full_cal(cal[3:6])])
        elif len(cal) == 9:
            holidays_list.append(
                [self._get_full_cal(cal[0:3]), self._get_full_cal(cal[3:6]),
                 self._get_full_cal(cal[6:9])])
        else:
            if cal == 'FX' or cal == 'NYX':
                # Filter for Christmas & New Year's Day
                for i in range(1999, 2025):
                    holidays_list.append(pd.Timestamp(str(i) + "-12-25"))
                    holidays_list.append(pd.Timestamp(str(i) + "-01-01"))

            elif cal == 'NYD' or cal == 'NEWYEARSDAY':
                # Filter for New Year's Day
                for i in range(1999, 2025):
                    holidays_list.append(pd.Timestamp(str(i) + "-01-01"))

            elif cal == 'WDY' or cal == 'WEEKDAY':
                bday = CustomBusinessDay(weekmask='Sat Sun')

                holidays_list.append([x for x in pd.date_range('01 Jan 1999',
                                                               '31 Dec 2025',
                                                               freq=bday)])
            elif cal == 'WKD':  #
                pass
                # holidays_list.append()

            else:
                label = cal + ".holiday-dates"

                try:
                    holidays_list = self._holiday_df[label].dropna().tolist()
                except:
                    logger = LoggerManager().getLogger(__name__)
                    logger.warning(cal + " holiday calendar not found.")

        return holidays_list

    def create_calendar_bus_days(self, start_date, end_date, cal='FX'):
        """Creates a calendar of business days

        Parameters
        ----------
        start_date : DateTime
            start date of calendar
        end_date : DataFrame
            finish date of calendar
        cal : str
            business calendar to use

        Returns
        -------
        list
        """
        hols = self.get_holidays(start_date=start_date, end_date=end_date,
                                 cal=cal)

        return pd.bdate_range(start=start_date, end=end_date, freq='D',
                              holidays=hols)

    def get_holidays(self, start_date=None, end_date=None, cal='FX',
                     holidays_list=[]):
        """Gets the holidays for a given calendar

        Parameters
        ----------
        start_date : DateTime
            start date of calendar
        end_date : DataFrame
            finish date of calendar
        cal : str
            business calendar to use

        Returns
        -------
        list
        """
        # holidays_list ,  = []

        # TODO use Pandas CustomBusinessDays to get more calendars
        holidays_list = self._get_full_cal(cal)
        # .append(lst)

        # Use 'set' so we don't have duplicate dates if we are incorporating
        # multiple calendars
        holidays_list = np.array(
            list(set(self.flatten_list_of_lists(holidays_list))))
        holidays_list = pd.to_datetime(
            holidays_list).sort_values().tz_localize('UTC')

        # Floor start date
        if start_date is not None:
            start_date = pd.Timestamp(start_date).floor('D')

            try:
                start_date = start_date.tz_localize('UTC')
            except:
                pass

            holidays_list = holidays_list[(holidays_list >= start_date)]

        if end_date is not None:
            # Ceiling end date
            end_date = pd.Timestamp(end_date).ceil('D')

            try:
                end_date = end_date.tz_localize('UTC')
            except:
                pass

            holidays_list = holidays_list[(holidays_list <= end_date)]

        # Remove all weekends unless it is WEEKDAY calendar
        if cal != 'WEEKDAY' or cal != 'WKY':
            holidays_list = holidays_list[holidays_list.dayofweek <= 4]

        return holidays_list

    def get_business_days_tenor(self, tenor):
        if tenor in self._tenor_bus_day_dict.keys():
            return self._tenor_bus_day_dict[tenor]

        return None

    def get_dates_from_tenors(self, start, end, tenor, cal=None):
        freq = str(self.get_business_days_tenor(tenor)) + "B"
        return pd.DataFrame(index=pd.bdate_range(start, end, freq=freq))

    def get_delta_between_dates(self, date1, date2, unit='days'):
        if unit == 'days':
            return (date2 - date1).days

    def get_delivery_date_from_horizon_date(self, horizon_date, tenor,
                                            cal=None, asset_class='fx'):
        if 'fx' in asset_class:
            tenor_unit = ''.join(re.compile(r'\D+').findall(tenor))
            asset_holidays = self.get_holidays(cal=cal)

            if tenor_unit == 'ON':
                return horizon_date + CustomBusinessDay(n=1,
                                                        holidays=asset_holidays)
            elif tenor_unit == 'TN':
                return horizon_date + CustomBusinessDay(n=2,
                                                        holidays=asset_holidays)
            elif tenor_unit == 'SP':
                pass
            elif tenor_unit == 'SN':
                tenor_unit = 'D'
                tenor_digit = 1
            else:
                tenor_digit = int(''.join(re.compile(r'\d+').findall(tenor)))

            horizon_date = self.get_spot_date_from_horizon_date(
                horizon_date, cal, asset_holidays=asset_holidays)

            if 'SP' in tenor_unit:
                return horizon_date
            elif tenor_unit == 'D':
                return horizon_date + CustomBusinessDay(
                    n=tenor_digit, holidays=asset_holidays)
            elif tenor_unit == 'W':
                return horizon_date + Day(
                    n=tenor_digit * 7) + CustomBusinessDay(
                    n=0, holidays=asset_holidays)
            else:
                if tenor_unit == 'Y':
                    tenor_digit = tenor_digit * 12

                horizon_period_end = horizon_date + CustomBusinessMonthEnd(
                    tenor_digit + 1)
                horizon_floating = horizon_date + DateOffset(
                    months=tenor_digit)

                cbd = CustomBusinessDay(n=1, holidays=asset_holidays)

                delivery_date = []

                if isinstance(horizon_period_end, pd.Timestamp):
                    horizon_period_end = [horizon_period_end]

                if isinstance(horizon_floating, pd.Timestamp):
                    horizon_floating = [horizon_floating]

                for period_end, floating in zip(horizon_period_end,
                                                horizon_floating):
                    if floating < period_end:
                        delivery_date.append(floating - cbd + cbd)
                    else:
                        delivery_date.append(period_end)

                return pd.DatetimeIndex(delivery_date)

    def get_expiry_date_from_horizon_date(self, horizon_date, tenor, cal=None,
                                          asset_class='fx-vol'):
        """Calculates the expiry date of FX options, based on the horizon date,
        the tenor and the holiday calendar associated with the asset.

        Uses expiry rules from Iain Clark's FX option pricing book

        Parameters
        ----------
        horizon_date : pd.Timestamp (collection)
            Horizon date of contract

        tenor : str
            Tenor of the contract

        cal : str
            Holiday calendar (usually related to the asset)

        asset_class : str
            'fx-vol' - FX options (default)

        Returns
        -------
        pd.Timestamp (collection)
        """
        if asset_class == 'fx-vol':

            tenor_unit = ''.join(re.compile(r'\D+').findall(tenor))

            asset_holidays = self.get_holidays(cal=cal)

            if tenor_unit == 'ON':
                tenor_digit = 1;
                tenor_unit = 'D'
            else:
                tenor_digit = int(''.join(re.compile(r'\d+').findall(tenor)))

            if tenor_unit == 'D':
                return horizon_date + CustomBusinessDay(
                    n=tenor_digit, holidays=asset_holidays)
            elif tenor_unit == 'W':
                return horizon_date + Day(
                    n=tenor_digit * 7) + CustomBusinessDay(
                    n=0, holidays=asset_holidays)
            else:
                horizon_date = self.get_spot_date_from_horizon_date(
                    horizon_date, cal, asset_holidays=asset_holidays)

                if tenor_unit == 'M':
                    pass
                elif tenor_unit == 'Y':
                    tenor_digit = tenor_digit * 12

                cbd = CustomBusinessDay(n=1, holidays=asset_holidays)

                horizon_period_end = horizon_date + CustomBusinessMonthEnd(
                    tenor_digit + 1)
                horizon_floating = horizon_date + DateOffset(
                    months=tenor_digit)

                delivery_date = []

                if isinstance(horizon_period_end, pd.Timestamp):
                    horizon_period_end = [horizon_period_end]

                if isinstance(horizon_floating, pd.Timestamp):
                    horizon_floating = [horizon_floating]

                # TODO: double check this!
                for period_end, floating in zip(horizon_period_end,
                                                horizon_floating):
                    if floating < period_end:
                        delivery_date.append(floating - cbd + cbd)
                    else:
                        delivery_date.append(period_end)

                delivery_date = pd.DatetimeIndex(delivery_date)

                return self.get_expiry_date_from_delivery_date(delivery_date,
                                                               cal)

    def _get_settlement_T(self, asset):
        base = asset[0:3]
        terms = asset[3:6]

        if base in ['CAD', 'TRY', 'RUB'] or terms in ['CAD', 'TRY', 'RUB']:
            return 1

        return 2

    def get_spot_date_from_horizon_date(self, horizon_date, asset,
                                        asset_holidays=None):
        base = asset[0:3]
        terms = asset[3:6]

        settlement_T = self._get_settlement_T(asset)

        if asset_holidays is None:
            asset_holidays = self.get_holidays(cal=asset)

        # First adjustment step
        if settlement_T == 2:
            if base in ['MXN', 'ARS', 'CLP'] or terms in ['MXN', 'ARS', 'CLP']:
                horizon_date = horizon_date + BDay(1)
            else:
                if base == 'USD':
                    horizon_date = horizon_date + CustomBusinessDay(
                        holidays=self.get_holidays(cal=terms))
                elif terms == 'USD':
                    horizon_date = horizon_date + CustomBusinessDay(
                        holidays=self.get_holidays(cal=base))
                else:
                    horizon_date = horizon_date + CustomBusinessDay(
                        holidays=asset_holidays)

        if 'USD' not in asset:
            asset_holidays = self.get_holidays(cal='USD' + asset)

        # Second adjustment step - move forward if horizon_date isn't a good
        # business day in base, terms or USD
        if settlement_T <= 2:
            horizon_date = horizon_date + CustomBusinessDay(
                holidays=asset_holidays)

        return horizon_date

    def get_delivery_date_from_spot_date(self, spot_date, cal):
        pass

    def get_expiry_date_from_delivery_date(self, delivery_date, cal):
        base = cal[0:3]
        terms = cal[3:6]

        if base == 'USD':
            cal = terms
        elif terms == 'USD':
            cal = base

        hols = self.get_holidays(cal=cal + 'NYD')

        # cbd = CustomBusinessDay(1, holidays=self.get_holidays(cal=cal))

        return delivery_date - CustomBusinessDay(self._get_settlement_T(cal),
                                                 holidays=hols)  # - cbd + cbd

    def align_to_NY_cut_in_UTC(self, date_time, hour_of_day=10):

        tstz = Timezone()
        date_time = tstz.localize_index_as_new_york_time(date_time)
        date_time.index = date_time.index + timedelta(hours=hour_of_day)

        return tstz.convert_index_aware_to_UTC_time(date_time)

    def floor_date(self, data_frame):
        data_frame.index = data_frame.index.normalize()

        return data_frame

    def create_bus_day(self, start, end, cal=None):

        if cal is None:
            return pd.bdate_range(start, end)

        return pd.date_range(start, end,
                             hols=self.get_holidays(start_date=start,
                                                    end_date=end, cal=cal))

    def get_bus_day_of_month(self, date, cal='FX', tz=None):
        """ Returns the business day of the month (ie. 3rd Jan, on a Monday,
        would be the 1st business day of the month)
        """

        try:
            # Strip times off the dates - for business dates just want dates!
            date = date.normalize()
        except:
            pass

        start = pd.to_datetime(
            datetime.datetime(date.year[0], date.month[0], 1))
        end = pd.Timestamp(
            datetime.datetime.today())
        # pd.to_datetime(datetime.datetime(date.year[-1], date.month[-1], date.day[-1]))

        holidays = self.get_holidays(start_date=start, end_date=end, cal=cal)

        # bday = CustomBusinessDay(holidays=holidays, weekmask='Mon Tue Wed Thu Fri')

        holidays = holidays.tz_localize(None).date

        bus_dates = pd.bdate_range(start, end)

        # Not most efficient way...
        bus_dates = pd.to_datetime([x for x in bus_dates if x not in holidays])

        month = bus_dates.month

        work_day_index = np.zeros(len(bus_dates))
        work_day_index[0] = 1

        for i in range(1, len(bus_dates)):
            if month[i] == month[i - 1]:
                work_day_index[i] = work_day_index[i - 1] + 1
            else:
                work_day_index[i] = 1

        bus_day_of_month = work_day_index[bus_dates.searchsorted(date)]

        return bus_day_of_month

    def set_market_holidays(self, holiday_df):
        self._holiday_df = holiday_df

max_stars_repo_name: mrderdelo/findatapy

max_issues_repo_name: mrderdelo/findatapy

max_forks_repo_name: mrderdelo/findatapy


text:
#! /usr/bin/python
"""
Compute the number density of each type of stationary point by Monte Carlo integration
"""

import numpy as np
import time
from mcintegrate import MCIntegrate
from math import pi, sin, cos, exp, sqrt
from common import scale, signed_exact

def integrand(values, parameters) :
    """Computes the integrand"""
    l1, l2, l3, a, b, c, d, e, f = values
    N, gamma, nu, sigma0, sigma1, gammanu, kappa, nufact = parameters

    # Construct the determinant
    term1 = a*a*b*b*c*c
    term2 = l1*l2*l3
    term3 = (c*c + e*e + f*f)*l1*l2 + (b*b + d*d)*l1*l3 + a*a*l2*l3
    term4 = -2.0*b*d*e*f*l1 + (c*c + e*e)*(d*d*l1 + a*a*l2) + b*b*(c*c*l1 + f*f*l1 + a*a*l3)
    # Scale each level of term by the number of factors of 3 nu / gamma it needs
    term2 *= nufact**3
    term3 *= nufact**2
    term4 *= nufact
    # Sum all of the terms
    det = term1 + term2 + term3 + term4

    # Compute the other two
    minor1 = nufact * l1 + a*a
    minor2 = minor1 * (nufact * l2 + b*b + d*d) - a*a * d*d

    # Find the type of stationary point
    # Contributions to minima, ++-, +--, maxima, signed number density
    result = np.array([0.0, 0.0, 0.0, 0.0, det])

    if minor1 < 0 and minor2 > 0 and det < 0 :
        # Three negative eigenvalues (maxima)
        result[3] = abs(det)
    elif minor1 > 0 and minor2 > 0 and det > 0 :
        # Three positive eigenvalues (minima)
        result[0] = abs(det)
    elif det > 0 :
        # Two negative, one positive eigenvalues
        result[2] = abs(det)
    else :
        # One negative, two positive eigenvalues
        result[1] = abs(det)

    # Return the result
    return result

def probability(values, parameters) :
    """Returns the (relative) probability density at the given values"""
    l1, l2, l3, a, b, c, d, e, f = values
    N, gamma, nu, sigma0, sigma1, gammanu, kappa, nufact = parameters

    # Compute Q
    trZ = l1 + l2 + l3
    trZ2 = l1*l1 + l2*l2 + l3*l3
    Q = a*a + b*b + c*c + d*d + e*e + f*f + kappa * (gammanu + trZ)**2 + 2.5*(3*trZ2 - trZ*trZ)

    # Construct the probability
    prob = (a * b * c)**(N-4) * a*a * b
    prob *= abs((l1 - l2) * (l2 - l3) * (l3 - l1))
    prob *= exp(-Q/2)

    # Return the result
    return prob

def number_density(parameters, numsamples) :
    """Computes the number density of stationary points"""

    # Construct the full set of parameters we'll need
    N, gamma, nu, sigma0, sigma1 = parameters
    gammanu = gamma * nu
    kappa = 1.0 / (1 - gamma * gamma)
    nufact = 3.0 * nu / gamma
    fullparams = [N, gamma, nu, sigma0, sigma1, gammanu, kappa, nufact]

    # Define the variables of integration and their ranges
    # "n" is standard normal, good enough for initialization for everything
    # "pn" is positive normal
    jump = 0.5
    jump2 = 0.5
    jump3 = 0.5
    variables = [
        ["l1", "n", jump], ["l2", "n", jump], ["l3", "n", jump],
        ["a", "pn", jump2], ["b", "pn", jump2], ["c", "pn", jump2],
        ["d", "n", jump3], ["e", "n", jump3], ["f", "n", jump3]
    ]

    # Perform the Monte Carlo integration
    integrals, errors, acceptance = MCIntegrate(variables, integrand, probability, numsamples, parameters=fullparams)

    # Scale everything by the appropriate coefficient
    scaleval = scale(*parameters)
    integrals *= scaleval
    errors *= scaleval

    # Return the results
    return integrals, errors, signed_exact(*parameters), acceptance

if __name__ == "__main__":
    # parameters = [N, gamma, nu, sigma0, sigma1]
    parameters = [4, 0.8, 1.0, 1.0, 1.0]
    numsamples = 1e5

    print("*" * 60)
    print("Performing test run")
    print("Computing number densities with", int(numsamples), "samples...")
    print("N       = ", parameters[0])
    print("nu      = ", parameters[2])
    print("gamma   = ", parameters[1])
    print("sigma_0 = ", parameters[3])
    print("sigma_1 = ", parameters[4])

    start = time.time()
    integrals, errors, exact, acceptance = number_density(parameters, numsamples)
    end = time.time()

    print("Finished in", round(end - start, 4), "s")
    print("Acceptance rate:", acceptance * 100 , "%")

    print("Minima:      ", integrals[0], "+-", errors[0])
    print("Saddle (++-):", integrals[1], "+-", errors[1])
    print("Saddle (+--):", integrals[2], "+-", errors[2])
    print("Maxima:      ", integrals[3], "+-", errors[3])
    print("Signed:      ", integrals[4], "+-", errors[4])
    print("Signed exact:", exact)
    print("Signed error:", abs(integrals[4] - exact))

max_stars_repo_name: jolyonb/chisquaredfields

max_issues_repo_name: jolyonb/chisquaredfields

max_forks_repo_name: jolyonb/chisquaredfields


text:
"""
===========
Plot Vector
===========

Does this need more documentation?
"""
import numpy as np
import matplotlib.pyplot as plt
from pytransform3d.plot_utils import plot_vector

plot_vector(
    # A vector is defined by start, direction, and s (scaling)
    start=np.array([-0.3, -0.2, -0.3]),
    direction=np.array([1.0, 1.0, 1.0]),
    s=0.5,
    ax_s=0.5,  # Scaling of 3D axes
    lw=0,  # Remove line around arrow
    color="orange"
)
plt.show()


text:
import cv2
import numpy as np
import socket
import threading
import struct
import time

CHAR_q = 113
CHAR_Q = 81

HEADER_SIZE = 32

client = None
def decode_jpg(jpgbuf):
    '''@jpgbuf is bytearray, jpeg data'''

    #jpgdata = np.frombuffer(frame)
    #rgb = cv2.imdecode(np.fromstring(bytes(jpgbuf), dtype=np.uint8), cv2.IMREAD_COLOR)
    rgb = cv2.imdecode(np.asarray(jpgbuf, dtype=np.uint8), cv2.IMREAD_COLOR)
    return rgb

class Camera():
    '''camera sending jpg by tcp'''
    def __init__(self, sock):
        self._sock = sock
        self._rbuf = bytearray()
        pass

    def recv(self):
        chunk = self._sock.recv(64 * 1024)
        if chunk == b'':
            return 0
#            raise RuntimeError("socket peer closed")

        self._rbuf += chunk
        return len(chunk)

    def get_image(self):
        if len(self._rbuf) < HEADER_SIZE:
            return False, 0, 0, "   ", None

        print("_rbuf %d" % (len(self._rbuf)))

        w, h, size = struct.unpack('<III', self._rbuf[0:12])
        print('get_image [%d, %d, %d]' % (w, h, size))

        if len(self._rbuf) < (size + HEADER_SIZE): # not complete packet
            return False, 0, 0, "   ", None

        frame = self._rbuf[HEADER_SIZE: HEADER_SIZE + size]

        # remove this packet
        del self._rbuf[0:HEADER_SIZE + size]

        return True, w, h, '   ', frame

class CamdbgServer():
    def __init__(self, host, port):
        self._host = host
        self._port = port
        self._sock = None

    def start(self):
        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self._sock.bind((self._host, self._port))
        self._sock.listen(1)
        return True

    def accept(self):
        return self._sock.accept()

    def close(self):
        self._sock.close()

def clients_thread():
    sleep_time = 0

    while True:
        time.sleep(sleep_time)
        print('clients_thread loop')
        if not client:
            sleep_time = 0.5
            continue

        ret = client.recv()
        if ret < 0:
            client.close()
            client = None
            sleep_time = 0.5
            continue

        sleep_time = 0.1

        if ret == 0:
            continue

        ret, w, h, title, frame = client.get_image()
        if not ret:
            continue

def main():
    # t = threading.Thread(target=clients_thread)
    # t.start()

    serv = CamdbgServer('192.168.10.152', 5000)
    serv.start()

    print('main start...')

    while True:
        sock, addr = serv.accept()
        if not sock:
            continue

        print('new client')

        c = Camera(sock)
        while True:
            ret = c.recv()
            if not ret:
                cv2.destroyAllWindows()
                print('Client closed')
                break

            r, w, h, title, frame = c.get_image()
            if not r:
                continue
            print('[%dx%d, %d]' % (w, h, len(frame)))
            rgb = decode_jpg(frame)
            cv2.imshow('Frame', rgb)
            key = cv2.waitKey(33)
            if key == CHAR_q or key == CHAR_Q:
                print('User interruptted')
                break

if __name__ == '__main__':
    main()

'''
im1 = cv2.imread("d:/test.jpg")
im2 = cv2.imread("d:/test1.jpg")

while True:
    cv2.imshow('Frame', im1)
    key = cv2.waitKey(33)
    if key == 113:
        break

    cv2.imshow('Frame', im2)
    key = cv2.waitKey(33)
    print(key)
    if key == 113: # key 'q' == 113, 'Q' == 81
        break

cv2.destroyAllWindows()

print('Quit')
'''


text:
from scipy.io import loadmat
import numpy as np
import os

def calculate_sample_percentage(file_names):
    '''
    calculate ramdom selection probability of positive and negative samples
    '''
    num_pos = np.zeros(len(file_names), dtype='float32')
    num_neg = np.zeros(len(file_names), dtype='float32')

    for i in range(len(file_names)):
        assert os.path.exists(file_names[i]), \
            print("File {} does not exist!".format(file_names[i]))

        data = loadmat(file_names[i], struct_as_record=False, squeeze_me=True)
        deepdata = data['deepdata']
        for j in range(len(deepdata)):
            frame = deepdata[j].frameNUM
            num_frame = len(frame)
            if deepdata[j].clipID[0] == -1:
                num_neg[i] = num_frame
            else:
                num_pos[i] = num_frame

    pos_prob = num_pos / num_pos.sum()
    neg_prob = num_neg / num_neg.sum()
    return pos_prob, neg_prob

def gen_vgg3d_batch(file_names, train_pct=0.9, img_sz=(38,38), n_frames=8, batch_sz=32, pos_prob=None, neg_prob=None):
    '''
    a generator that yields training batches for 3D vgg network
    Args:
    file_names: list of file names. Each file has a "deepdata"
    "deepdata", structure variable containing 2 cell arrays, one for each fly from a single experiment.
    each cell is a structure including:
    ".video_cropSTACK", 38x38 by m stack of image frames.
    ".frameNUM", the actual frame number from original video recorded at 30 hz.
    ".clipID" , the labels (-1 is not clipped and 0.1, 0.5, and 1 are ranked clips with 1 being the most).
    ".EuclidDist", measure for each frame of how far the fly moved from the previous frame.
    ".filePATH", directory path where the .avi is saved.
    ".filter_config", fields that contain the the various parameter settings
    train_pct: percentage of video data used in training
    img_sz: image size of each frame
    n_frames: number of continuous frames in a 3D training data (38x38xn_frames)
    batch_sz: batch size
    pos_prob: random selection probability of positive samples
    neg_prob: random selection probability of negative samples
    yield training images in 5D as (num_data,:,:,:,channel=1), and training scores in a 1D array
    '''

    for i in range(len(file_names)):
        assert os.path.exists(file_names[i]), \
            print("File {} does not exist!".format(file_names[i]))
    if pos_prob is not None:
        assert len(file_names) == len(pos_prob), \
            print("Probability of positive samples doesn't match sample size")
    if neg_prob is not None:
        assert len(file_names) == len(neg_prob), \
            print("Probability of negative samples doesn't match sample size")

    while True:
        batch_img = np.zeros((batch_sz, img_sz[0], img_sz[1], n_frames, 1), dtype='float32')
        batch_label = [1]*int(batch_sz/2) + [0]*int(batch_sz/2) # 1-clipped, 0-nonclipped

        for num in range(int(batch_sz/2)):
            # Get positive (clipped) sample
            pos_file = np.random.choice(file_names, p=pos_prob)
            data = loadmat(pos_file, struct_as_record=False, squeeze_me=True)
            deepdata = data['deepdata']
            if deepdata[0].clipID[0] != -1:
                video = deepdata[0].video_cropSTACK
                frame = deepdata[0].frameNUM
            else:
                video = deepdata[1].video_cropSTACK
                frame = deepdata[1].frameNUM
            not_include = True  # if include current data stack in the batch
            while not_include:
                frame_idx = np.random.randint(int(len(frame)*train_pct)-n_frames)
                if frame[frame_idx]+n_frames-1 == frame[frame_idx+n_frames-1]:
                    not_include = False
            batch_img[num,:,:,:,0] = video[:,:,frame_idx:frame_idx+n_frames]

            # Get negative (nonclipped) sample
            neg_file = np.random.choice(file_names, p=neg_prob)
            data = loadmat(neg_file, struct_as_record=False, squeeze_me=True)
            deepdata = data['deepdata']
            if deepdata[0].clipID[0] == -1:
                video = deepdata[0].video_cropSTACK
                frame = deepdata[0].frameNUM
            else:
                video = deepdata[1].video_cropSTACK
                frame = deepdata[1].frameNUM
            not_include = True
            while not_include:
                frame_idx = np.random.randint(int(len(frame)*0.8)-n_frames)
                if frame[frame_idx]+n_frames-1 == frame[frame_idx+n_frames-1]:
                    not_include = False
            batch_img[int(batch_sz/2)+num,:,:,:,0] = video[:,:,frame_idx:frame_idx+n_frames]

        # Data augmentation
        x_flip = np.random.randint(2, size=batch_sz)
        rot_angle = np.random.randint(4, size=batch_sz)
        for i in range(batch_sz):
            if x_flip[i]:
                batch_img[i,:,:,:,0] = np.flip(batch_img[i,:,:,:,0], axis=0)
            if rot_angle[i]:
                batch_img[i,:,:,:,0] = np.rot90(batch_img[i,:,:,:,0], rot_angle[i], axes=(0,1))

        yield batch_img, batch_label

def prepare_validation_data_3d(file_names, test_pct=0.1, img_sz=(38,38), n_frames=6):
    '''
    prepare validation data for 3D vgg network
    Args:
    file_names: list of file names. Each file has a "deepdata"
    "deepdata", structure variable containing 2 cell arrays, one for each fly from a single experiment.
    each cell is a structure including:
    ".video_cropSTACK", 38x38 by m stack of image frames.
    ".frameNUM", the actual frame number from original video recorded at 30 hz.
    ".clipID" , the labels (-1 is not clipped and 0.1, 0.5, and 1 are ranked clips with 1 being the most).
    ".EuclidDist", measure for each frame of how far the fly moved from the previous frame.
    ".filePATH", directory path where the .avi is saved.
    ".filter_config", fields that contain the the various parameter settings
    test_pct: percentage of video data used in testing
    img_sz: image size of each frame
    n_frames: number of continuous frames in a 3D training data (38x38xn_frames)
    '''

    img_pos = []
    img_neg = []

    for i in range(len(file_names)):
        assert os.path.exists(file_names[i]), \
            print("File {} does not exist!".format(file_names[i]))

        data = loadmat(file_names[i], struct_as_record=False, squeeze_me=True)
        deepdata = data['deepdata']
        for j in range(len(deepdata)):
            img = []
            video = deepdata[j].video_cropSTACK
            video = video[:, :, -int(video.shape[2]*test_pct):]
            frame = deepdata[j].frameNUM
            frame = frame[-int(len(frame)*test_pct):]
            for k in range(len(frame)-n_frames):
                if frame[k]+n_frames-1 == frame[k+n_frames-1]:
                    curr_img = video[:,:,k:k+n_frames]
                    img.append(curr_img)
            if deepdata[j].clipID[0] == -1:
                img_neg.extend(img)
            else:
                img_pos.extend(img)

    test_label = [1]*len(img_pos) + [0]*len(img_neg)
    test_img = np.zeros((len(img_pos)+len(img_neg), img_sz[0], img_sz[1], n_frames, 1), dtype='float32')
    for i in range(len(img_pos)):
        test_img[i,:,:,:,0] = img_pos[i]
    for j in range(len(img_neg)):
        test_img[len(img_pos)+j,:,:,:,0] = img_neg[j]

    return test_img, test_label

max_stars_repo_name: JaneliaSciComp/FlySocialPipeline

max_issues_repo_name: JaneliaSciComp/FlySocialPipeline

max_forks_repo_name: JaneliaSciComp/FlySocialPipeline

reason: import numpy,from scipy


text:
#! /usr/bin/env python3

"""
WalArt.wiDev
=======================
wicknec@2016

Customized plot involving wiChart
for processing nanodevice test data
functions related to specific device processing

"""
# NOTE: the actual command documentation is collected from docstrings of the
# commands and is appended to __doc__ after the class has been defined.
"""
Revisions
=================
160202 created
160204 *Plot*,*Prettify*
"""
import numpy as np
from WalArt import wiChart
def SeparateSteps(wo):
    ''' Separate acquired Output curves by gate voltage steps into multiple wiCharts
'''
    Vgs=np.unique(wo.data['Vg'])
    ws=[]
    for V in Vgs:
        w=wiChart.New()
        w.info=wo.info.copy()
        w.info['name']='$V_{gs}=\ %s\ \mathrm{V}$'%V
        w.info['Vgs']=V
        w.info['allvar']=2
        cond=lambda x: x==V
        indices=wiChart.Find(wo.data['Vg'],cond)
        w.info['names']=['Vd','Id']
        w.data['Vd']=[wo.data['Vd'][i] for i in indices]
        w.data['Id']=[wo.data['Id'][i] for i in indices]
        ws.append(w)
    return ws

    #demo mode
if __name__ == '__main__':
    import time
    from WalArt import wiChart
    from wiPlot import *
    w=wiChart.FromTxt('tests/test-Output.txt')
    print(w)
    ws=SeparateSteps(w)
    Plot(0,1,ws)
    Prettify()
    time.sleep(3)

max_stars_repo_name: wicknec/WalArt

max_issues_repo_name: wicknec/WalArt

max_forks_repo_name: wicknec/WalArt


text:
import thulac
import joblib
import gzip
import socket
import threading
import json
import numpy as np
import ipdb
import os

tfidf = joblib.load('./model/tfidf.model')
law = joblib.load('./model/law.model')
accu = joblib.load('./model/accu.model')
time = joblib.load('./model/time.model')

batch_size = 1
cut = thulac.thulac(seg_only = True)

data_path = "law"
for file_name in os.listdir(data_path):
    inf = open(os.path.join(data_path, file_name), "r", encoding='utf-8')
    id2lawid = {}
    for line in inf:
        id2lawid[len(id2lawid)+1] = int(line.strip('\n'))

def predict_law(vec):
    y = [law.predict(vec)[0]]
    y = [id2lawid[temp+1] for temp in y]
    return y

def predict_accu(vec):
    y = accu.predict(vec)
    return [y[0]+1]

def predict_time(vec):
    y = time.predict(vec)[0]

    #返回每一个罪名区间的中位数
    if y == 0:
        return -2
    if y == 1:
        return -1
    if y == 2:
        return 120
    if y == 3:
        return 102
    if y == 4:
        return 72
    if y == 5:
        return 48
    if y == 6:
        return 30
    if y == 7:
        return 18
    else:
        return 6

def predict(content):
    fact = cut.cut(content, text = True)

    vec = tfidf.transform([fact])
    ans = {}

    ans['accusation'] = np.array(predict_accu(vec)).tolist()
    ans['articles'] = np.array(predict_law(vec)).tolist()
    ans['imprisonment'] = predict_time(vec)

    return ans

def main():
    # 创建服务器套接字
    serversocket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
    # 获取本地主机名称
    host = socket.gethostname()
    # 设置一个端口
    port = 12345
    # 将套接字与本地主机和端口绑定
    serversocket.bind((host,port))
    # 设置监听最大连接数
    serversocket.listen(5)
    # 获取本地服务器的连接信息
    myaddr = serversocket.getsockname()
    print("服务器地址:%s"%str(myaddr))
    # 循环等待接受客户端信息
    while True:
        # 获取一个客户端连接
        clientsocket,addr = serversocket.accept()
        print("连接地址:%s" % str(addr))
        try:
            t = ServerThreading(clientsocket)#为每一个请求开启一个处理线程
            t.start()
            pass
        except Exception as identifier:
            print(identifier)
            pass
        pass
    serversocket.close()
    pass

class ServerThreading(threading.Thread):
    def __init__(self,clientsocket,recvsize=1024*1024,encoding="gbk"):
        threading.Thread.__init__(self)
        self._socket = clientsocket
        self._recvsize = recvsize
        self._encoding = encoding
        pass

    def run(self):
        print("开启线程.....")
        try:
            #接受数据
            msg = ''
            while True:
                # 读取recvsize个字节
                rec = self._socket.recv(self._recvsize)
                # 解码
                msg += rec.decode(self._encoding)
                # 文本接受是否完毕，因为python socket不能自己判断接收数据是否完毕，
                # 所以需要自定义协议标志数据接受完毕
                if msg.strip().endswith('over'):
                    msg=msg[:-4]
                    break
            # 解析json格式的数据
            re = json.loads(msg)
            # 调用神经网络模型处理请求
            res = predict(re['content'])
            sendmsg = json.dumps(res)
            # 发送数据
            self._socket.send(("%s"%sendmsg).encode(self._encoding))
            pass
        except Exception as identifier:
            self._socket.send("500".encode(self._encoding))
            print(identifier)
            pass
        finally:
            self._socket.close()
        print("任务结束.....")

        pass

    def __del__(self):

        pass
if __name__ == "__main__":
    main()


text:
import sympy as sp
import numpy as np
from IPython.display import display

## Felipe Gomes

#display(yourobject)

sp.init_printing()

t = sp.Symbol("t", positive=True)

# Propriedades da cabeça
m_c, J_c = sp.symbols("m_c J_c", positive=True)
x_c = sp.Function("x_c", real=True)(t)
y_c = sp.Function("y_c", real=True)(t)
theta_c = sp.Function("theta_c", real=True)(t)

# Propriedades do torço
m_t, J_t, L_t = sp.symbols("m_t J_t L_t", positive=True)
x_t = sp.Function("x_t", real=True)(t)
y_t = sp.Function("y_t", real=True)(t)
theta_t = sp.Function("theta_t", real=True)(t)
L_t = sp.Symbol("L_t", real=True)
L_p = sp.Symbol("L_p",real=True)

# Propriedades dos membros inferiores
x_i = sp.Function("x_i", real=True)(t)
m_i = sp.Symbol("m_i", positive=True)

# Interação air bag-cabeça
k_ab, c_ab = sp.symbols("k_ab c_ab", positive=True)

# Propriedades do pescoço
## Deformação do pescoço
x_p = sp.Function("x_p", positive=True)(t)
## Mola linear
k_p, c_p = sp.symbols("k_p c_p", positive=True)
## Mola de torção
k_rp, c_rp = sp.symbols("k_rp c_rp", positive=True)

# Propriedades do cinto de segurança
k_s, c_s = sp.symbols("k_s c_s", positive=True)

# Propriedades do braço
k_b, c_b = sp.symbols("k_b c_b", positive=True)

# Interação air_bag-torso
k_ab, c_ab = sp.symbols("k_ab c_ab", positive=True)

# Ligamento torço-membros inferiores
k_ri, c_ri = sp.symbols("k_ri c_ri", positive=True)

# Interações de amortecimento dos membros inferiores
k_i, c_i = sp.symbols("k_i c_i", positive=True)

# Desaceleração do carro
a = sp.Symbol("a", real=True)

g = sp.Symbol("g", positive=True)

class Corpo():
    def __init__(self, massa, mom_inercia, x, y, theta):
        self.massa        = massa
        self.mom_inercia  = mom_inercia

        self.x = x
        self.y = y
        self.theta = theta

        try:
            self.y_ponto = y.diff(t)
        except:
            self.y_ponto = 0

        try:
            self.x_ponto = x.diff(t)
        except:
            self.x_ponto = 0

        try:
            self.theta_ponto = theta.diff(t)
        except:
            self.theta_ponto = 0

        self.quad_vel_lin = self.x_ponto**2+self.y_ponto**2
        self.quad_vel_ang = self.theta_ponto**2

    def cinetica(self):
        return (self.massa*self.quad_vel_lin)/2 \
                + (self.mom_inercia*self.quad_vel_ang)/2

    def potencial(self, a_x, a_y):
        return self.massa*(a_x*self.x + a_y*self.y)

    def show(self):
        return (self.massa, self.quad_vel_lin,
                self.mom_inercia, self.quad_vel_ang)

class Mola:
    def __init__(self, k, c, q):
        self.k = k
        self.c = c
        self.q = q

    def potencial(self):
        return (self.k*self.q**2)/2

    def dissipador(self):
        return (self.c*self.q.diff(t)**2)/2

## Diego Kurashima

class Lagrange():
    def __init__(self, T, V, D, var):
        self.L = T - V
        self.D = D
        self.var = var

        try:
            self.dvar = self.var.diff(t)
        except:
            self.dvar = 0

        try:
            self.ddvar = self.dvar.diff(t)
        except:
            self.ddvar = 0

    def equacao(self):
        dL1 = (self.L.diff(self.dvar)).diff(t)
        dL2 = self.L.diff(self.var)
        dD = self.D.diff(self.dvar)
        return dL1 - dL2 + dD

a = Lagrange(0, 0, 0, sp.cos(theta_t)*L_p + x_p*sp.cos(theta_c))
print(a.dvar)

reason: import numpy,import sympy


text:
import point_cloud_based_motion_estimate as pcbme
import numpy as np

### change paths to your localization files (i.e. x,y,z localizations)
x = 'x_results.npy'
y = 'y_results.npy'
z = 'z_results.npy'
times = 'spike_times.npy'
amps = 'max_ptp.npy'

### change path to geometry array
geomarray = np.load('geom.npy') # (num of channels, 2)

### set registration params
reg_win_num = 5 # set to 1 for rigid registration
iteration_num = 1 # supports 1 iteration for now

### length of the recording in seconds
T = 1000
subsampling_rate = np.log(T)/T

### get motion estimate
total_shift = pcbme.estimate_motion(x, y, z, times, amps, geomarray, subsampling_rate=subsampling_rate)

### save results
np.save('total_shift.npy', total_shift) # motion estimate

max_stars_repo_name: int-brain-lab/spikes

max_issues_repo_name: int-brain-lab/spikes


text:
"""Train the cnn model as  described in Lili Mou et al. (2015)
https://arxiv.org/pdf/1409.5718.pdf"""

import os
import logging
import pickle
import tensorflow as tf
import numpy as np
import classifier.tbcnn.network as network
import classifier.tbcnn.sampling as sampling
from classifier.tbcnn.parameters import LEARN_RATE, EPOCHS, \
    CHECKPOINT_EVERY, BATCH_SIZE
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score

def train_model(logdir, infile, embedfile,
                epochs=EPOCHS,
                learning_rate=LEARN_RATE,
                batch_size=BATCH_SIZE,
                checkpoint_every=CHECKPOINT_EVERY):
    """Train a classifier to label ASTs"""

    with open(infile, 'rb') as fh:
        trees, _, labels = pickle.load(fh)

    with open(embedfile, 'rb') as fh:
        embeddings, embed_lookup = pickle.load(fh)
        num_feats = len(embeddings[0])

    # build the inputs and outputs of the network
    nodes_node, children_node, hidden_node = network.init_net(
        num_feats,
        len(labels)
    )

    out_node = network.out_layer(hidden_node)
    labels_node, loss_node = network.loss_layer(hidden_node, len(labels))

    optimizer = tf.train.AdamOptimizer(learning_rate)
    train_step = optimizer.minimize(loss_node)

    tf.summary.scalar('loss', loss_node)

    ### init the graph
    sess = tf.Session()#config=tf.ConfigProto(device_count={'GPU':0}))
    sess.run(tf.global_variables_initializer())

    with tf.name_scope('saver'):
        saver = tf.train.Saver()
        summaries = tf.summary.merge_all()
        writer = tf.summary.FileWriter(logdir, sess.graph)

    checkfile = os.path.join(logdir, 'cnn_tree.ckpt')

    num_batches = len(trees) // batch_size + (1 if len(trees) % batch_size != 0 else 0)
    for epoch in range(1, epochs+1):
        for i, batch in enumerate(sampling.batch_samples(
            sampling.gen_samples(trees, labels, embeddings, embed_lookup), batch_size
        )):
            nodes, children, batch_labels = batch
            step = (epoch - 1) * num_batches + i * batch_size

            if not nodes:
                continue # don't try to train on an empty batch

            _, summary, err, out = sess.run(
                [train_step, summaries, loss_node, out_node],
                feed_dict={
                    nodes_node: nodes,
                    children_node: children,
                    labels_node: batch_labels
                }
            )

            print('Epoch:', epoch,
                  'Step:', step,
                  'Loss:', err,
                  'Max nodes:', len(nodes[0])
            )

            writer.add_summary(summary, step)
            if step % checkpoint_every == 0:
                # save state so we can resume later
                saver.save(sess, os.path.join(checkfile), step)
                print('Checkpoint saved.')

    saver.save(sess, os.path.join(checkfile), step)

    # compute the training accuracy
    correct_labels = []
    predictions = []
    print('Computing training accuracy...')
    for batch in sampling.batch_samples(
        sampling.gen_samples(trees, labels, embeddings, embed_lookup), 1
    ):
        nodes, children, batch_labels = batch
        output = sess.run([out_node],
            feed_dict={
                nodes_node: nodes,
                children_node: children,
            }
        )
        correct_labels.append(np.argmax(batch_labels))
        predictions.append(np.argmax(output))

    target_names = list(labels)
    print('Accuracy:', accuracy_score(correct_labels, predictions))
    print(classification_report(correct_labels, predictions, target_names=target_names))
    print(confusion_matrix(correct_labels, predictions))


text:
from numpy.core.defchararray import array
from tensorflow.keras.models import Model
from tensorflow.keras.models import load_model
from tensorflow.keras.datasets import mnist
from PIL import Image
from imutils import build_montages
import numpy as np
import pickle
import cv2
import sys

# We use the euclidean_distance as our default metric
def euclidean_distance(a, b):
    return np.linalg.norm(a - b)

def perform_search(query_image, index, maxResults=10):
	results = []
	for i in range(0, len(index["features"])):
		# Compute the euclidean distance between our query features
		# and the features for the current image in our index, then
		# update our results list with a 2-tuple consisting of the
		# computed distance and the index of the image
		distance = euclidean_distance(query_image, index["features"][i])
		results.append((distance, i))
	# Sort the results and grab the top ones
	results = sorted(results)[:maxResults]
	return results

# Again we use the MNIST dataset as default
print("[INFO] loading MNIST dataset...")
((trainX, _), (testX, _)) = mnist.load_data()

# Add a channel dimension to every image in the dataset, then scale
# the pixel intensities to the range [0, 1]
trainX = np.expand_dims(trainX, axis=-1)
testX = np.expand_dims(testX, axis=-1)
trainX = trainX.astype("float32") / 255.0
testX = testX.astype("float32") / 255.0

# Load the autoencoder model and index from disk
print("[INFO] loading autoencoder and index...")
autoencoder = load_model("autoencoder.h5")
index = pickle.loads(open("feature_vectors.pickle", "rb").read())

# Create the encoder model which consists of *just* the encoder
# portion of the autoencoder
encoder = Model(inputs=autoencoder.input,
	outputs=autoencoder.get_layer("encoded").output)

# Compute the feature vector of our input image
features = encoder.predict(testX)

# Randomly sample a set of testing query image indexes
queryIdxs = list(range(0, testX.shape[0]))
queryIdxs = np.random.choice(queryIdxs, size=10,
	replace=False)

# Iterate over the testing indexes
for i in queryIdxs:
	# Take the features for the current image, find all similar
	# images in our dataset, and then initialize our list of result
	# images
	queryFeatures = features[i]
	results = perform_search(queryFeatures, index, maxResults=225)
	images = []

	for (d, j) in results:
		# Grab the result image, convert it back to the previous range
		# and then update the images list
		image = (trainX[j] * 255).astype("uint8")
		image = np.dstack([image] * 3)
		images.append(image)
	# Display the query image
	query = (testX[i] * 255).astype("uint8")
	cv2.imshow("Query", query)
	# Build a montage from the results and display it
	montage = build_montages(images, (28, 28), (15, 15))[0]
	cv2.imshow("Results", montage)
	cv2.waitKey(0)

max_stars_repo_name: framtale/image-retrieval

max_issues_repo_name: framtale/image-retrieval

max_forks_repo_name: framtale/image-retrieval

reason: import numpy,from numpy


text:
# USAGE
# python arithmetic.py --image mai-ngoc.jpg

# import the necessary packages
import numpy as np
import argparse
import cv2

# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required=True, help="Path to the image")
args = vars(ap.parse_args())

# load the image and show it
image = cv2.imread(args["image"])
cv2.imshow("Original", image)

# images are NumPy arrays, stored as unsigned 8 bit integers -- this
# that the values of our pixels will be in the range [0, 255]; when
# using functions like cv2.add and cv2.subtract, values will be clipped
# to this range, even if the added or subtracted values fall outside the
# range of [0, 255]. Check out an example:
print("max of 255: {}".format(str(cv2.add(np.uint8([200]), np.uint8([100])))))
print("min of 0: {}".format(str(cv2.subtract(np.uint8([50]), np.uint8([100])))))

# NOTE: if you use NumPy arithmetic operations on these arrays, the value
# will be modulos (wrap around) instead of being  clipped to the [0, 255]
# range. This is important to keep in mind when working with images.
print("wrap around: {}".format(str(np.uint8([200]) + np.uint8([100]))))
print("wrap around: {}".format(str(np.uint8([50]) - np.uint8([100]))))

# let's increase the intensity of all pixels in our image by 100 -- we
# accomplish this by constructing a NumPy array that is the same size of
# our matrix (filled with ones) and the multiplying it by 100 to create an
# array filled with 100's, then we simply add the images together; notice
# how the image is "brighter"
M = np.ones(image.shape, dtype = "uint8") * 100
added = cv2.add(image, M)
cv2.imshow("Added", added)

# similarly, we can subtract 50 from all pixels in our image and make it
# darker
M = np.ones(image.shape, dtype = "uint8") * 50
subtracted = cv2.subtract(image, M)
cv2.imshow("Subtracted", subtracted)
cv2.waitKey(0)


text:
""" A script package helping you make plots for paper.
"""
import os, sys, json
import numpy as np
from numpy.lib.arraysetops import isin
import pandas as pd
import itertools
import matplotlib.pyplot as plt
import matplotlib.lines as mlines
from contextlib import redirect_stderr
from os import devnull

from exptools.launching.variant import flatten_variant4hparams

class ExpDatabase:
	""" A comparison tool that help you get all data paths that match your report needs
	"""
	def __init__(self,
			exp_paths,
			config_filename= "variant_config.json",
			required_files = [],
		):
		"""
		@ Args:
			exp_paths: list[str] absolute paths
			config_filename: the config file that used to check if the experiment is needed
			required_files: a list of strings that are the required files to be in the experiment directory
				NOTE: You cannot specify the files that are in the subdirectory of your experiment directory
		"""
		self.exp_paths = exp_paths
		self.config_filename = config_filename
		self.required_files = required_files
		self.color_map = dict()
		self.df = pd.DataFrame()

		self.interpret_exp_paths()

	def interpret_exp_paths(self):
		""" Go through all paths, and log them into a pandas data frame arranged by params
		"""
		# extract all valid log directory
		for exp_path in self.exp_paths:
			for path, subdirs, subfiles in os.walk(exp_path):
				if self.config_filename in subfiles and all([f in subfiles for f in self.required_files]):
					# get a valid directory
					with open(os.path.join(path, self.config_filename)) as f:
						param = json.load(f)
					# to make all dictionary in the same level, which is easier to process
					param = flatten_variant4hparams(param)
					abspath = os.path.abspath(path)
					param["experiment_log_path"] = abspath
					self.df = self.df.append(param, ignore_index= True)
		if self.df.empty:
			Warning("You got an empty database, please check your exp_paths")

	def get_datapaths(self, configs: dict):
		""" This method will return all experiments paths that match the configs will be collected and returned.
		NOTE: This method will return a list of paths, not a pandas dataframe
		Args
			configs: a dict whose key is a string, value is a list of config values
		Return
			a list of tuple
				(0): each combination of configs (specified by configs)
				(1): a list of all experiment's directory (who satisfies the configs specification)
					(n): the directory of the experiment you want to collect
		"""

		# fill all configs which is assigned as None, and seperate them
		for key, options in configs.items():
			assert isinstance(options, list) or options is None, "You must provide a list of options you want, or simply None"
			if options is None:
				configs[key] = [*(self.df[key].unique())]
		keys = list(configs.keys())
		all_options = list(configs.values()) # a list of options (list)

		return_ = []
		for i, all_option in enumerate(itertools.product(*all_options)):
			# all_option is a list of experiment value
			df = self.df
			for key, option in zip(keys, all_option):
				df = df.loc[df[key] == option]
			data_paths = df["experiment_log_path"]

			if len(data_paths) == 0:
				Warning("No experiment found for {}".format(all_option))
				continue
			else:
				return_.append((
					dict(zip(keys, all_option)), # dict
					data_paths, # list
				))
		return return_

max_stars_repo_name: Shiduo-zh/pybulletSim

max_issues_repo_name: Shiduo-zh/pybulletSim

max_forks_repo_name: Shiduo-zh/pybulletSim