text:
"""
a collection of data structures used throughout this project
"""
import json
import logging
from collections import OrderedDict
from typing import List, Tuple, Union, Dict, Any, Iterable, Callable
import warnings
warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')

import numpy
from gensim.corpora import IndexedCorpus
from gensim.utils import SlicedCorpus

import util

logger = logging.getLogger('index-builder')

# type declarations
CorpusDoc = Union[Dict[str,Any], Iterable[Tuple[int,float]]]
TopicTokens = List[Tuple[str, float]]

class JsonLinesCorpus(IndexedCorpus):
    """
    All corpus formats in gensim are vastly over-engineered, yet at the same time they
    fail to fulfil the most basic purposes:
    - how do we identify documents, if we can't store document IDs?
    - what about additional metadata we'd like to store?
    - why use parsers written with the python-skills of a 5-year-old?

    The sane solution to all these issues is to use python dicts to store data,
    and have a proper convention how to name stuff. Python's json parser has
    proper C bindings that will outperform the custom crap all the other parsers
    use anyways...

    Here's the convention:
    - one document per line
    - each line consists of a single json dictionary (no line breaks within the json)
    - the dictionary has at least these fields:
      * "id": the identifier of the document. defaults to null
      * "tokens": a list of (token_id, count) tuples
    - arbitrary data can be stored under any other key

    Other features
    - can read and write compressed corpora (restriction: no random access, only iteration is available)
    """

    def __init__(self, fname: str):
        """
        Initialize the corpus from an existing file.
        """
        IndexedCorpus.__init__(self, fname)
        logger.info("loading corpus from %s" % fname)
        self.fname = fname
        self.length = None

    def __iter__(self) -> Iterable[List[Tuple[int, float]]]:
        """
        Iterate over the corpus, returning one sparse vector at a time.
        This iterator is basically there fore compatibility with gensim models.
        To iterate over the documents with metadata, use `iter_all()` instead
        :return: a generator that yields lists of (token id, count) tuples
        """
        i = 0
        with util.open_by_ext(self.fname, 'rt', encoding='utf-8') as fp:
            for i, line in enumerate(fp):
                yield self.line2doc(line)
        self.length = i + 1

    def iter_all(self) -> Iterable[Dict[str, Any]]:
        """
        Iterate over the corpus, returning a document with all additional metadata
        as dictionary. Tokens are stored under 'tokens'. The document ID is
        expected under 'id', but not required.
        Alternatively, use `__iter__` to get sparse token vectors.
        :return: a generator that yields each document as dictionary.
        """
        with util.open_by_ext(self.fname, 'rt', encoding='utf-8') as fp:
            for line in fp:
                yield json.loads(line)

    @staticmethod
    def save_corpus(fname: str, corpus: Iterable[CorpusDoc], id2word: Dict[int,str] = None,
                    labels: List[str] = None, progress_cnt: int = 10000, metadata=False):
        """
        Save a corpus in the JsonLinesCorpus format.
        This function is automatically called by `JsonLinesCorpus.serialize`;
        don't call it directly, call `serialize` instead.
        :param fname: the file name of the corpus to store
        :param corpus: the actual corpus. An iterable of documents. A document can be either a
               list of (token, count) tuples, or a dictionary that must have at least a 'tokens' field
        :param id2word: a mapping from token id to token string.
                if specified, all string tokens are converted to integers using this mapping
                and all tokens that are not found in this mapping are dropped!
        :param labels: a mapping from document index to document label. can be used to store
               document IDs, although the canonical way would be to store the document ID under
               the key 'id' in the dictionary.
        :param progress_cnt: log progress information every n documents
        :param metadata: an unused parameter that was reserved by the base class...
        :return: the offsets of each document
        """
        logger.info("converting corpus to JsonLinesCorpus format: %s" % fname)
        token2id = {v: k for k, v in id2word.items()} if id2word else None
        offsets = []
        with util.open_by_ext(fname, 'wt', encoding='utf-8') as fp:
            use_offsets = util.is_stream_seekable(fp)
            for i, doc in enumerate(corpus):
                if use_offsets:
                    offsets.append(fp.tell())
                label = labels[i] if labels else None
                fp.write(JsonLinesCorpus.doc2line(doc, label=label, token2id=token2id))
                if progress_cnt and ((i+1) % progress_cnt) == 0:
                    logger.info("%i documents have been processed so far" % (i+1))
        return offsets

    @classmethod
    def serialize(cls, fname: str, corpus: Iterable[CorpusDoc], id2word: Dict[int,str] = None,
                  index_fname: str = None, progress_cnt: int = 10000, labels: List[str] = None,
                  metadata=False):
        """
        Iterate through the document stream `corpus`, saving the documents to `fname`
        and recording byte offset of each document. Save the resulting index
        structure to file `index_fname` (or `fname`.index is not set).

        The index can be stored in compressed form, if the '.bz2' extension (for bzip2)
        or '.gz' extension (for gzip) are used. In this case, no index will be written
        and random access will be slower.

        >>> JsonLinesCorpus.serialize('test.json', corpus)
        >>> corpus = JsonLinesCorpus('test.json') # document stream now has random access
        >>> print(corpus[42]) # retrieve document no. 42, etc.
        """
        if getattr(corpus, 'fname', None) == fname:
            raise ValueError("identical input vs. output corpus filename, refusing to serialize: %s" % fname)
        if fname.endswith('.bz2') or fname.endswith('.gz'):
            # bypass default serialization routine, just save the corpus
            cls.save_corpus(fname, corpus, id2word=id2word, progress_cnt=progress_cnt, labels=labels)
        else:
            # use the default serialization routine, which also generates a corpus index
            super().serialize(fname, corpus, id2word=id2word, index_fname=index_fname,
                              progress_cnt=progress_cnt, labels=labels, metadata=metadata)

    def convert_tokens_to_ids(self, fname: str, id2word: Dict[int,str], progress_cnt: int = 10000):
        """
        convert the plaintext tokens in this corpus to token IDs, based on the specified dictionary.
        warning: all tokens that are not part of this dictionary will be lost!
        do not apply this function, if this is already a reduced corpus.
        use has_plain_tokens() to check if it contains plaintext tokens.
        """
        self.serialize(fname, self.iter_all(), id2word=id2word, progress_cnt=progress_cnt)

    def has_plain_tokens(self) -> bool:
        """
        returns True, iff this corpus has plaintext tokens (strings) instead of token ids.
        this decision is made based on a small sample (first document)
        """
        sample_tokens = self.__iter__().__next__()
        return all(isinstance(token, str) for token,count in sample_tokens)

    def subset(self, fname: str, filter_function: Callable[[Dict[str, Any]], bool],
               progress_cnt: int = 10000) -> 'JsonLinesCorpus':
        """
        Generate a subset of this corpus. Provide a filter function that
        decides, which documents to keep. The filter has access to the entire
        document dictionary.
        Example: `lambda doc: doc['id'].startswith("foo")`
        :param fname: the name of the file where the new corpus will be written to.
        :param filter_function: this function should return True for every document to keep.
        :param progress_cnt: log progress information every n documents
        :return: a subset of this corpus whose documents match the specified filter condition
        """
        filtered_corpus = (doc for doc in self.iter_all() if filter_function(doc))
        JsonLinesCorpus.serialize(fname, filtered_corpus, progress_cnt=progress_cnt)
        return JsonLinesCorpus(fname)

    def docbyoffset(self, offset):
        """
        Return the document stored at file position `offset`.
        """
        with util.open_by_ext(self.fname, 'rt', encoding='utf-8') as fp:
            fp.seek(offset)
            return self.line2doc(fp.readline())

    def docbyindex(self, index):
        with util.open_by_ext(self.fname, 'rt', encoding='utf-8') as fp:
            for i, line in enumerate(fp):
                if i == index:
                    return self.line2doc(line)

    def _count_lines(self):
        """
        counts the lines of the file, without parsing it's contents
        """
        with util.open_by_ext(self.fname, 'rt', encoding='utf-8') as fp:
            return sum(1 for _ in fp)

    @staticmethod
    def line2doc(line: str) -> List[Tuple[int, float]]:
        """
        Create a document from a single line (string)
        :param line: a line of the corpus file
        :return: the document as a list of (token id, count) tuples
        """
        tokens = json.loads(line).get('tokens')
        return list(tokens.items()) if isinstance(tokens, dict) else tokens

    @staticmethod
    def doc2line(doc: CorpusDoc, label: Any = None, token2id: Dict[str,int] = None):
        """
        Output the document in SVMlight format, as a string. Inverse function to `line2doc`.
        """
        if isinstance(doc, dict):
            if 'tokens' not in doc:
                raise ValueError("the specified dictionary has no 'tokens' field: %s" % str(doc))
            result = doc
        else:
            # assume (token, count) tuples, take ID from label parameter
            result = {'id': label, 'tokens': doc}
        if token2id:
            tokens = list(result['tokens'].items()) if isinstance(result['tokens'], dict) else result['tokens']
            result['tokens'] = [(token2id[token], count) for token, count in tokens if token in token2id]
        return json.dumps(result, separators=None, indent=None) + '\n'

    def __len__(self):
        """
        Return the index length if the corpus is indexed. Otherwise, make a pass
        over self to calculate the corpus length and cache this number.
        """
        if self.index is not None:
            return len(self.index)
        if self.length is None:
            self.length = self._count_lines()
            return self.length
        return self.length

    def __getitem__(self, docno):
        if isinstance(docno, (slice, list, numpy.ndarray)):
            return SlicedCorpus(self, docno)
        elif isinstance(docno, (int, numpy.integer)):
            if self.index is not None:
                return self.docbyoffset(self.index[docno])
            else:
                return self.docbyindex(docno)
        else:
            raise ValueError('Unrecognised value for docno, use either a single integer, a slice or a numpy.ndarray')

    def __str__(self) -> str:
        return "JsonLinesCorpus(fname={}, length={})".format(self.fname, len(self))

    def __repr__(self) -> str:
        return self.__str__()

class Topic:

    def __init__(self, topic_id: str, tokens: TopicTokens = None, layer: int = 1,
                 parent: 'Topic' = None, children: List['Topic'] = None, **kwargs):
        """
        initializes a new topic
        :param topic_id: the ID of this topic
        :param tokens: a list of (token, score) tuples, in descending score order
        :param layer: this topic's layer (depth in the tree, 0 for root node, 1 for
               root's children, and so on)
        """
        super().__init__()
        self.topic_id = topic_id
        self.tokens = tokens
        self.layer = layer
        self.parent = parent
        self.children = children

    def add_child(self, topic_id: str, tokens: TopicTokens = None):
        """
        adds a new child to this topic. automatically infers the topic layer
        and adds references between parent and child
        :param topic_id: the ID of the child topic
        :param tokens: a list of (token, score) tuples, in descending score order
        :return: the new child topic
        """
        layer = 1 if self.layer is None else self.layer+1
        child = Topic(topic_id, tokens, layer=layer, parent=self)
        if self.children is None:
            self.children = []
        self.children.append(child)
        return child

    def store_recursively(self) -> List[Dict[str, Any]]:
        """
        store this topic and all of it's descendants in an easily serializable
        dictionary structure without object references
        """
        topics = self._collect_topics()
        return self.store_topics(topics)

    def _collect_topics(self) -> List['Topic']:
        """
        :return: a list containing this topic and all of it's descendants
        """
        topics = [self]
        if self.children:
            for child in self.children:
                topics.extend(child._collect_topics())
        return topics

    @staticmethod
    def store_topics(topics: Iterable['Topic']) -> List[Dict[str, Any]]:
        """
        transform topics into an easily serializable dictionary structure without object references
        :param topics: a list of topics
        :return: a list of the dictionary representation of each topic (see `to_dict`)
        """
        return [topic.to_dict() for topic in topics]

    @staticmethod
    def restore_topics(topic_dicts: List[Dict[str, Any]]) -> Dict[str, 'Topic']:
        """
        restore a bunch of topics that have been transformed by `store_topics`.
        Also restores object references between topic objects.
        :param topic_dicts: a list of topic dictionaries
        :return: a dictionary mapping from topic id to topic object
        """
        topic_map = {td['topic_id']: Topic._from_dict(td) for td in topic_dicts}
        for topic in topic_map.values():
            topic._restore_links(topic_map)
        return topic_map

    def to_dict(self):
        """
        transforms this topic into an easily serializable dictionary structure without object references
        :return: this topic's fields as dictionary, without object references
        """
        return OrderedDict([
            ('topic_id', self.topic_id),
            ('layer', self.layer),
            ('tokens', self.tokens),
            ('parent', self.parent.topic_id if self.parent else None),
            ('children', [child.topic_id for child in self.children] if self.children else None),
        ])

    @staticmethod
    def _from_dict(topic_dict: Dict[str, Any]) -> 'Topic':
        """
        restores a Topic object from it's dictionary representation.
        Does NOT restore links to other Topic objects. To restore a
        serialized topic model, please refer to `restore_topics`
        :param topic_dict:
        :return:
        """
        return Topic(**topic_dict)

    def _restore_links(self, topic_map: Dict[str, 'Topic']):
        if self.parent:
            self.parent = topic_map[self.parent]
        if self.children:
            self.children = [topic_map[child_id] for child_id in self.children]

    def __hash__(self):
        return hash(self.topic_id) * hash(self.layer)

    def __eq__(self, other):
        if type(other) is type(self):
            return self.topic_id == other.topic_id and self.layer == other.layer
        return False

    def __str__(self) -> str:
        return util.class_str_from_dict("Topic", self.to_dict())

    def __repr__(self) -> str:
        return self.__str__()

class Document:

    def __init__(self, doc_id: str, topics: Dict[Topic, float] = None, categories: List[str] = None, **kwargs):
        """
        initializes a new document
        :param doc_id: the unique identifier of this document
        :param topics: a mapping from topic id to score
        :param categories: the (optional) category of this document
        """
        super().__init__()
        self.doc_id = doc_id
        self.topics = topics
        self.categories = categories

    def add_topic(self, topic: Topic, score: float):
        if self.topics is None:
            self.topics = {}
        self.topics[topic] = score

    def top_topics(self, min_layer: int = None) -> List[Tuple[Topic, float]]:
        """
        get the highest scoring topics for this document
        :param min_layer: only include topics of this layer or higher
        :return: a list of (topic, score) tuples, in descending order of score
        """
        topics = [(topic, score) for topic, score in self.topics.items() if
                  (min_layer is None or topic.layer >= min_layer)]
        return sorted(topics, key=lambda x: x[1], reverse=True)

    @staticmethod
    def store_documents(documents: Iterable['Document']) -> List[Dict[str, Any]]:
        """
        transform documents into an easily serializable dictionary structure without object references
        :param documents: a list of topics
        :return: a list of the dictionary representation of each document (see `to_dict`)
        """
        return [doc.to_dict() for doc in documents]

    @staticmethod
    def restore_documents(doc_dicts: List[Dict[str, Any]], known_topics: Dict[str, 'Topic'] = None) -> Dict[str, 'Document']:
        """
        restore a bunch of documents that have been transformed by `store_documents`.
        Also restores object references from documents to topics.
        :param doc_dicts: a list of document dictionaries
        :return: a dictionary mapping from document id to document object
        """
        return {doc_dict['doc_id']: Document.from_dict(doc_dict, known_topics) for doc_dict in doc_dicts}

    def to_dict(self):
        return OrderedDict([
            ('doc_id', self.doc_id),
            ('categories', self.categories),
            ('topics', [(topic.topic_id, score) for topic, score in self.topics.items()] if self.topics else None),
        ])

    @staticmethod
    def from_dict(doc_dict: Dict[str, Any], known_topics: Dict[str, Topic] = None) -> 'Document':
        topics = None
        if known_topics and doc_dict['topics']:
            topics = {known_topics[topic_id]: score for topic_id, score in doc_dict['topics']}
        return Document(doc_dict['doc_id'], topics=topics, categories=doc_dict['categories'])

    def __str__(self) -> str:
        return util.class_str_from_dict("Document", self.to_dict())

    def __repr__(self) -> str:
        return self.__str__()

class DocumentCollection:
    """
    a document collection consists of
    - a topic definition, with the most relevant tokens for each topic
    - a list of documents, each containing a reference to one or more topics
    """

    def __init__(self, topics: Dict[str, Topic] = None, documents: Dict[str, Document] = None):
        """
        initializes a new document collection
        :param topics: mapping from topic id to Topic (object)
        :param documents: mapping from document id to Document (object)
        """
        super().__init__()
        self.topics = topics
        self.documents = documents

    def to_dict(self):
        return OrderedDict([
            ('topics', Topic.store_topics(self.topics.values())),
            ('documents', [doc.to_dict() for id, doc in self.documents.items()]),
        ])

    @staticmethod
    def from_dict(doc_collection: Dict[str, Any]) -> 'DocumentCollection':
        raw_topics = doc_collection['topics']
        raw_documents = doc_collection['documents']
        topics = Topic.restore_topics(raw_topics)
        documents = {doc_dict['doc_id']: Document.from_dict(doc_dict, topics) for doc_dict in raw_documents}
        return DocumentCollection(topics, documents)

    def __str__(self) -> str:
        return "DocumentCollection(topics: {}, documents: {})".format(
            len(self.topics) if self.topics else 0, len(self.documents) if self.documents else 0)

    def __repr__(self) -> str:
        return self.__str__()

max_stars_repo_name: Klamann/search-index-builder

max_issues_repo_name: Klamann/search-index-builder

max_forks_repo_name: Klamann/search-index-builder


text:
"""
Point collocation method, or regression based polynomial chaos expansion builds
open the idea of fitting a polynomial chaos expansion to a set of generated
samples and evaluations. The experiment can be done as follows:

- Select a :ref:`distributions`::

    >>> distribution = chaospy.Iid(chaospy.Normal(0, 1), 2)

- Generate :ref:`orthogonality`::

    >>> orthogonal_expansion = chaospy.orth_ttr(2, distribution)
    >>> print(orthogonal_expansion)
    [1.0, q1, q0, q1^2-1.0, q0q1, q0^2-1.0]

- Generate samples using :ref:`sampling` (or alternative abscissas from
  :ref:`quadrature`)::

    >>> samples = distribution.sample(
    ...     2*len(orthogonal_expansion), rule="M")
    >>> print(samples[:, :4])
    [[ 0.67448975 -1.15034938  0.31863936 -0.31863936]
     [-1.42607687 -1.02007623 -0.73631592 -0.50240222]]

- A function evaluated using the nodes generated in the second step::

    >>> def model_solver(param):
    ...     return [param[0]*param[1], param[0]*numpy.e**-param[1]+1]
    >>> solves = [model_solver(sample) for sample in samples.T]
    >>> print(numpy.around(solves[:4], 8))
    [[-0.96187423  3.80745414]
     [ 1.17344406 -2.19038608]
     [-0.23461924  1.66539168]
     [ 0.16008512  0.47338898]]

- Bring it all together using `~chaospy.regression.fit_regression`::

    >>> approx_model = chaospy.fit_regression(
    ...      orthogonal_expansion, samples, solves)
    >>> print(chaospy.around(approx_model, 5))
    [q0q1, 0.0478q0^2-1.4354q0q1+0.1108q1^2+1.22377q0-0.0907q1+0.93973]

In this example, the number of collocation points is selected to be twice the
number of unknown coefficients :math:`N+1`. Changing this is obviously
possible. When the number of parameter is equal the number of unknown, the, the
polynomial approximation becomes an interpolation method and overlap with
Lagrange polynomials. If the number of samples are fewer than the number of
unknown, classical least squares can not be used. Instead it possible to use
methods for doing estimation with too few samples.

The function :func:`~chaospy.regression.fit_regression` also takes an optional
``rule`` keyword argument. It allows for the selection of regression method
used when fitting the samples to the polynomials:

+------------+----------------------------------------------+
| Key        | Description                                  |
| Parameters |                                              |
+============+==============================================+
| "LS"       | Ordenary Least Squares                       |
+------------+----------------------------------------------+
| "T"        | Ridge Regression/Tikhonov Regularization     |
| order      | Order of regularization (or custom matrix)   |
| alpha      | Dampning parameter (else estimated from gcv) |
+------------+----------------------------------------------+
| "TC"       | Tikhonov with Cross Validation               |
| order      | Order of regularization (or custom matrix)   |
| alpha      | Dampning parameter (else estimated from gcv) |
+------------+----------------------------------------------+

In addition, the rule can be any initialized regression model from
``scipit-learn``. For example, if one wants to implement orthogonal matching
pursuit : It forces the result to have at most one non-zero coefficient. To
implement it use the keyword ``rule="OMP"``, and to force the number of
coefficients to be for example 1: ``n_nonzero_coefs=1``. In practice::

    >>> from sklearn.linear_model import OrthogonalMatchingPursuit  # doctest: +SKIP
    >>> omp = OrthogonalMatchingPursuit(fit_intercept=False, n_nonzero_coefs=1)  # doctest: +SKIP
    >>> approx_model = chaospy.fit_regression(  # doctest: +SKIP
    ...     orthogonal_expansion, samples, solves, rule=omp)
    >>> print(chaospy.around(approx_model, 8))  # doctest: +SKIP
    [3.46375077q0q1, 11.63750715]

Note that the option ``fit_intercept=False``. This is a prerequisite for
``scikit-learn`` to be compatible with ``chaospy``.
"""
import numpy
from scipy import linalg

import chaospy

def fit_regression(
        polynomials,
        abscissas,
        evals,
        rule="LS",
        retall=False,
        order=0,
        alpha=-1,
):
    """
    Fit a polynomial chaos expansion using linear regression.

    Args:
        polynomials (chaospy.poly.base.Poly):
            Polynomial expansion with ``polynomials.shape=(M,)`` and
            `polynomials.dim=D`.
        abscissas (numpy.ndarray):
            Collocation nodes with ``abscissas.shape == (D, K)``.
        evals (numpy.ndarray):
            Model evaluations with ``len(evals)=K``.
        retall (bool):
            If True return Fourier coefficients in addition to R.
        order (int):
            Tikhonov regularization order.
        alpha (float):
            Dampning parameter for the Tikhonov regularization. Calculated
            automatically if negative.

    Returns:
        (Poly, numpy.ndarray):
            Fitted polynomial with ``R.shape=evals.shape[1:]`` and ``R.dim=D``.
            The Fourier coefficients in the estimation.

    Examples:
        >>> x, y = chaospy.variable(2)
        >>> polynomials = chaospy.Poly([1, x, y])
        >>> abscissas = [[-1,-1,1,1], [-1,1,-1,1]]
        >>> evals = [0,1,1,2]
        >>> print(chaospy.around(chaospy.fit_regression(
        ...     polynomials, abscissas, evals), 14))
        0.5q0+0.5q1+1.0
    """
    abscissas = numpy.asarray(abscissas)
    if len(abscissas.shape) == 1:
        abscissas = abscissas.reshape(1, *abscissas.shape)
    evals = numpy.array(evals)

    poly_evals = polynomials(*abscissas).T
    shape = evals.shape[1:]
    evals = evals.reshape(evals.shape[0], int(numpy.prod(evals.shape[1:])))

    if isinstance(rule, str):
        rule = rule.upper()

    if rule == "LS":
        uhat = linalg.lstsq(poly_evals, evals)[0]

    elif rule == "T":
        uhat = rlstsq(poly_evals, evals, order=order, alpha=alpha, cross=False)

    elif rule == "TC":
        uhat = rlstsq(poly_evals, evals, order=order, alpha=alpha, cross=True)

    else:

        from sklearn.linear_model.base import LinearModel
        assert isinstance(rule, LinearModel)
        uhat = rule.fit(poly_evals, evals).coef_.T

    evals = evals.reshape(evals.shape[0], *shape)

    approx_model = chaospy.poly.sum((polynomials*uhat.T), -1)
    approx_model = chaospy.poly.reshape(approx_model, shape)

    if retall == 1:
        return approx_model, uhat
    elif retall == 2:
        return approx_model, uhat, poly_evals
    return approx_model

def rlstsq(coef_mat, ordinate, order=1, alpha=-1, cross=False):
    """
    Least Squares Minimization using Tikhonov regularization.

    Includes method for robust generalized cross-validation.

    Args:
        coef_mat (numpy.ndarray):
            Coefficient matrix with shape ``(M, N)``.
        ordinate (numpy.ndarray):
            Ordinate or "dependent variable" values with shape ``(M,)`` or
            ``(M, K)``. If ``ordinate`` is two-dimensional, the least-squares
            solution is calculated for each of the ``K`` columns of
            ``ordinate``.
        order (int, numpy.ndarray):
            If int, it is the order of Tikhonov regularization. If
            `numpy.ndarray`, it will be used as regularization matrix.
        alpha (float):
            Lower threshold for the dampening parameter. The real value is
            calculated using generalised cross validation.
        cross (bool):
            Use cross validation to estimate alpha value.
    """
    coef_mat = numpy.array(coef_mat)
    ordinate = numpy.array(ordinate)
    dim1, dim2 = coef_mat.shape

    if cross:
        out = numpy.empty((dim1, dim2) + ordinate.shape[1:])
        coef_mat_ = numpy.empty((dim1-1, dim2))
        ordinate_ = numpy.empty((dim1-1,) + ordinate.shape[1:])
        for i in range(dim1):
            coef_mat_[:i] = coef_mat[:i]
            coef_mat_[i:] = coef_mat[i+1:]
            ordinate_[:i] = ordinate[:i]
            ordinate_[i:] = ordinate[i+1:]
            out[i] = rlstsq(coef_mat_, ordinate_, order, alpha, False)

        return numpy.median(out, 0)

    if order == 0:
        tikhmat = numpy.eye(dim2)

    elif order == 1:
        tikhmat = numpy.zeros((dim2-1, dim2))
        tikhmat[:, :-1] -= numpy.eye(dim2-1)
        tikhmat[:, 1:] += numpy.eye(dim2-1)

    elif order == 2:
        tikhmat = numpy.zeros((dim2-2, dim2))
        tikhmat[:, :-2] += numpy.eye(dim2-2)
        tikhmat[:, 1:-1] -= 2*numpy.eye(dim2-2)
        tikhmat[:, 2:] += numpy.eye(dim2-2)

    elif order is None:
        tikhmat = numpy.zeros(1)

    else:
        tikhmat = numpy.array(order)
        assert tikhmat.shape[-1] == dim2 or tikhmat.shape in ((), (1,))

    if alpha < 0 and order is not None:

        gamma = 0.1

        def rgcv_error(alpha):
            """Calculate Tikhonov dampening parameter."""
            if alpha <= 0:
                return numpy.inf
            coef_mat_ = numpy.dot(
                coef_mat.T, coef_mat)+alpha*(numpy.dot(tikhmat.T, tikhmat))
            try:
                coef_mat_ = numpy.dot(linalg.inv(coef_mat_), coef_mat.T)
            except linalg.LinAlgError:
                return numpy.inf

            abscissas = numpy.dot(coef_mat_, ordinate)
            res2 = numpy.sum((numpy.dot(coef_mat, abscissas)-ordinate)**2)
            coef_mat_2 = numpy.dot(coef_mat, coef_mat_)
            skew = dim1*res2/numpy.trace(numpy.eye(dim1)-coef_mat_2)**2
            mu2 = numpy.sum(coef_mat_2*coef_mat_2.T)/dim1

            return (gamma + (1-gamma)*mu2)*skew

        alphas = 10.**-numpy.arange(0, 16)
        evals = numpy.array([rgcv_error(alpha) for alpha in alphas])
        alpha = alphas[numpy.argmin(evals)]

    out = linalg.inv(
        numpy.dot(coef_mat.T, coef_mat) + alpha*numpy.dot(tikhmat.T, tikhmat))
    out = numpy.dot(out, numpy.dot(coef_mat.T, ordinate))
    return out

ext: py

max_stars_repo_name: krystophny/chaospy

max_issues_repo_name: QianWanghhu/chaospy

max_forks_repo_name: QianWanghhu/chaospy

reason: import numpy,from scipy


text:
from fastai import *
from fastai.vision import *
from fastai.callbacks import *
from fastai.vision import learner
import numpy as np
import torch.nn as nn
import torch.nn.functional as F

from RecurrentModel.ImageSequence import ImageSequence
from RecurrentModel.ImageSequenceList import ImageSequenceList

class VideoSequenceModel(nn.Module):
    def __init__(self, pretrained_classifier,
                 hidden_dim = 1024,#2048,
                 emb_sz=2,
                 ps=0.6,
                 bidirectional=True):
        super().__init__()
        self.hidden_dim, self.emb_sz, self.ps = hidden_dim, emb_sz, ps
        self.bidirectional=bidirectional

        hidden_dim_size = self.hidden_dim * (1+self.bidirectional)

        self.body = pretrained_classifier[:-1]
        self.maxpool = nn.MaxPool2d((10,10))
        self.lstm1 = nn.LSTM(num_features_model(self.body),
                             self.hidden_dim,
                             batch_first=True,
                             num_layers=1,
                             #dropout=self.ps,
                             bidirectional=self.bidirectional) # True returns two times the outputs
        #self.batchnorm1 = nn.BatchNorm1d(hidden_dim_size)
        self.dropout1 = nn.Dropout(self.ps)

        self.linear1 = nn.Linear(hidden_dim_size, self.emb_sz)# hidden_dim_size//2)
        #self.batchnorm2 = nn.BatchNorm1d(hidden_dim_size//2)
        #self.act2 = nn.ReLU()
        #self.dropout2 = nn.Dropout(self.ps)

        #self.linear2 = nn.Linear(hidden_dim_size//2, self.emb_sz)

    def forward(self, x):
        self.lstm1.flatten_parameters() # GPU memory allocation
        # x.shape (bs, sequence_length, ch, w, h)
        bs,sequence_length,ch,w,h = x.shape
        # body requires (bs, ch, w, h) * sequence_length
        out = x.view(bs*sequence_length,ch,w,h)
        out = self.body(out)
        out = self.maxpool(out)
        #c_outs = torch.cat([self.maxpool(self.body(batch)).unsqueeze(dim=0) for batch in x.permute(1,0,2,3,4)],
        #                    dim=0)
        out = out.view(bs, sequence_length,-1)
        out,(h_n, h_c) = self.lstm1(out, None)
        """ h_n shape (n_layers, batch, hidden_size), h_c shape (n_layers, batch, hidden_size) """
        """ None represents zero initial hidden state. RNN_out has shape=(batch, time_step, output_size) """
        out = out[:,-1,:] # take the last time step
        #out = self.batchnorm1(out)
        out = F.relu(out)
        out = self.dropout1(out)
        out = self.linear1(out)
        #out = self.batchnorm2(out)
        #out = self.act2(out)
        #out = self.dropout2(out)
        #out = self.linear2(out)
        return out

class GRUVideoSequenceModel(nn.Module):
    def __init__(self, pretrained_classifier,
                 hidden_dim = 128,
                 emb_sz=2,
                 ps=0.2,
                 bidirectional=True):
        super().__init__()
        self.hidden_dim, self.emb_sz, self.ps = hidden_dim, emb_sz, ps
        self.bidirectional=bidirectional

        hidden_dim_size = self.hidden_dim * (1+self.bidirectional)

        self.body = pretrained_classifier[:-1]
        self.pool = nn.AdaptiveAvgPool2d((1,1))
        self.recurrent = nn.GRU(num_features_model(self.body),
                                self.hidden_dim,
                                batch_first=True,
                                num_layers=1,
                                #dropout=self.ps/2,
                                bidirectional=self.bidirectional) # True returns two times the outputs
        #self.batchnorm1 = nn.BatchNorm1d(hidden_dim_size)
        self.dropout1 = nn.Dropout(self.ps/2)

        self.linear1 = nn.Linear(hidden_dim_size, hidden_dim_size//2)
        #self.batchnorm2 = nn.BatchNorm1d(hidden_dim_size//2)
        self.dropout2 = nn.Dropout(self.ps)
        self.linear2 = nn.Linear(hidden_dim_size//2, self.emb_sz)

    def forward(self, x):
        self.recurrent.flatten_parameters() # GPU memory allocation
        # x.shape (bs, sequence_length, ch, w, h)
        bs,sequence_length,ch,w,h = x.shape
        # body requires (bs, ch, w, h) * sequence_length
        out = x.view(bs*sequence_length,ch,w,h)
        out = self.body(out)
        out = self.pool(out)
        out = out.view(bs, sequence_length,-1)
        out, _ = self.recurrent(out, None)
        out = out[:,-1,:] # take the last time step
        out = F.elu(out)
        out = self.dropout1(out)
        out = self.linear1(out)
        out = F.elu(out)
        out = self.dropout2(out)
        out = self.linear2(out)
        return out

class ConvVideoSequenceModel(nn.Module):
    def __init__(self, pretrained_classifier,
                 hidden_dim = 128,
                 emb_sz=2,
                 ps=0.2):
        super().__init__()
        self.hidden_dim, self.emb_sz, self.ps = hidden_dim, emb_sz, ps

        hidden_dim_size = self.hidden_dim

        self.body = pretrained_classifier[:-1]
        self.pool = nn.AvgPool2d((10,10))
        self.conv1d = nn.Conv1d(in_channels=num_features_model(self.body),
                                   out_channels=hidden_dim,
                                   kernel_size=5)
        #self.batchnorm1 = nn.BatchNorm1d(hidden_dim_size)
        self.dropout1 = nn.Dropout(self.ps/2)

        self.linear1 = nn.Linear(hidden_dim_size, hidden_dim_size//2)
        #self.batchnorm2 = nn.BatchNorm1d(hidden_dim_size//2)
        self.dropout2 = nn.Dropout(self.ps)
        self.linear2 = nn.Linear(hidden_dim_size//2, self.emb_sz)

    def forward(self, x):
        # x.shape (bs, sequence_length, ch, w, h)
        bs,sequence_length,ch,w,h = x.shape
        # body requires (bs, ch, w, h) * sequence_length
        out = x.view(bs*sequence_length,ch,w,h)
        out = self.body(out)
        out = self.pool(out)
        out = out.view(bs, sequence_length,-1)
        out = out.permute(0,2,1)
        out = self.conv1d(out)
        out = out.view(bs, -1)
        out = F.elu(out)
        out = self.dropout1(out)
        out = self.linear1(out)
        out = F.elu(out)
        out = self.dropout2(out)
        out = self.linear2(out)
        return out

class MixedVideoSequenceModel(nn.Module):
    def __init__(self, pretrained_classifier,
                 hidden_dim = 64,
                 emb_sz=2,
                 ps=0.2,
                 bidirectional=True):
        super().__init__()
        self.hidden_dim, self.emb_sz, self.ps = hidden_dim, emb_sz, ps
        self.bidirectional=bidirectional

        hidden_dim_size = self.hidden_dim * (1+self.bidirectional)

        self.body = pretrained_classifier[:-1]
        self.pool = nn.AdaptiveAvgPool2d((1,1))
        self.recurrent = nn.GRU(num_features_model(self.body),
                                self.hidden_dim,
                                batch_first=True,
                                num_layers=1,
                                #dropout=self.ps/2,
                                bidirectional=self.bidirectional) # True returns two times the outputs
        self.conv1d = nn.Conv1d(in_channels=num_features_model(self.body),
                                   out_channels=hidden_dim,
                                   kernel_size=5)
        #self.batchnorm1 = nn.BatchNorm1d(hidden_dim_size)
        self.dropout1 = nn.Dropout(self.ps/2)

        self.linear1 = nn.Linear(hidden_dim_size + hidden_dim, hidden_dim_size//2)
        #self.batchnorm2 = nn.BatchNorm1d(hidden_dim_size//2)
        self.dropout2 = nn.Dropout(self.ps)
        self.linear2 = nn.Linear(hidden_dim_size//2, self.emb_sz)

    def forward(self, x):
        self.recurrent.flatten_parameters() # GPU memory allocation
        # x.shape (bs, sequence_length, ch, w, h)
        bs,sequence_length,ch,w,h = x.shape
        # body requires (bs, ch, w, h) * sequence_length
        out = x.view(bs*sequence_length,ch,w,h)
        out = self.body(out)
        out = self.pool(out)
        out = out.view(bs, sequence_length,-1)

        gru_out, _ = self.recurrent(out, None)
        gru_out = gru_out[:,-1,:] # take the last time step

        conv_out = out.permute(0,2,1)
        conv_out = self.conv1d(conv_out)
        conv_out = conv_out.view(bs, -1)

        out = torch.cat([gru_out,conv_out],dim=1)
        out = F.elu(out)
        out = self.dropout1(out)
        out = self.linear1(out)
        out = F.elu(out)
        out = self.dropout2(out)
        out = self.linear2(out)
        return out

max_stars_repo_name: jpjuvo/deepfake-video-detector

max_issues_repo_name: jpjuvo/deepfake-video-detector

max_forks_repo_name: jpjuvo/deepfake-video-detector

reason: import numpy


text:
# Copyright 2016 Autodesk Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
This module is contains various functions used to create a DNA structure from a caDNAno DNA origami design file.
"""
import numpy as np
from math import sqrt, cos, acos, sin, pi
from .common import CadnanoLatticeType

def generate_coordinates(
    dna_parameters, lattice_type, row, col, helix_num, scaffold_bases, staple_bases
):
    """Generate the axis coordinates, axis reference frames, and nucleotide coordinate for the
    virtual helix.

    Arguments:
        dna_parameters (DnaParameters): The DNA parameters to use when creating the 3D geometry for the design.
        lattice_type (CadnanoLatticeType): The lattice type for this design.
        row (int): The caDNAno row number.
        col (int): The caDNAno column number.
        helix_num (int): The caDNAno virtual helix number.
        scaffold_bases (List[DnaBase]): The list of scaffold bases for this helix.
        staple_bases (List[DnaBase]): The list of staple bases for this helix.

    Returns:
        axis_coords (NumPy Nx3 ndarray[float]): The coordinates of base nodes along the helix axis.
        axis_frames (NumPy 3x3xN ndarray[float]): The coordinate frames of base nodes alonge the helix axis.
        scaffold_coords ((NumPy Nx3 ndarray[float]): The coordinates of nucleotides along the DNA helix.
        staple_coords ((NumPy Nx3 ndarray[float]): The coordinate of nucleotides along the DNA helix.

    The helix coordinates and reference frames are generated for all of the virtural helix
    positions that contain a base. The coordinates and refereance frames are also set for
    scaffold and staple bases.
    """
    # r_strand = dna_parameters.helix_distance / 2.0  # half the distance between the axes of adjacent DNA helices.
    r_helix = dna_parameters.helix_radius  # radius of DNA helices (nm)
    # rise between two neighboring base-pairs (nm)
    dist_bp = dna_parameters.base_pair_rise
    # twisting angle between two neighboring base-pairs (degrees)
    ang_bp = dna_parameters.base_pair_twist_angle
    # angle of the minor groove (degrees)
    ang_minor = dna_parameters.minor_groove_angle

    # Positions of the scaffold nucleotide and staple nucleotide
    # in the local reference frame.
    scaf_local = (
        r_helix
        * np.array(
            [cos(deg2rad(180 - ang_minor / 2)),
             sin(deg2rad(180 - ang_minor / 2)), 0.0]
        ).transpose()
    )
    stap_local = (
        r_helix
        * np.array(
            [cos(deg2rad(180 + ang_minor / 2)),
             sin(deg2rad(180 + ang_minor / 2)), 0.0]
        ).transpose()
    )

    # Set the helix start coordinates.
    init_coord, init_ang = get_start_coordinates_angle(
        dna_parameters, lattice_type, row, col, helix_num
    )

    # Set the direction of the helix axis.
    if helix_num % 2 == 0:
        e3 = np.array([0, 1, 0], dtype=float)
    else:
        e3 = np.array([0, -1, 0], dtype=float)

    # Create a list of sorted base positions.
    base_positions = set()
    for base in scaffold_bases:
        base_positions.add(base.p)
    for base in staple_bases:
        base_positions.add(base.p)

    # Compute helix axis coordinates and frames.
    num_base_positions = len(base_positions)
    axis_coords = np.zeros((num_base_positions, 3), dtype=float)
    axis_frames = np.zeros((3, 3, num_base_positions), dtype=float)
    pos_map = {}
    for i, p in enumerate(sorted(base_positions)):
        pos_map[p] = i
        ref_coord = init_coord + np.array([0, dist_bp * p, 0])
        angle = init_ang + ang_bp * p
        # Base coordinate.
        axis_coords[i, :] = ref_coord
        # Base orientation.
        e2 = np.array([cos(-deg2rad(angle)), 0, sin(-deg2rad(angle))])
        e1 = np.cross(e2, e3)
        axis_frames[:, :, i] = np.array([e1, e2, e3]).transpose()

    # Compute scaffold nucleotide positions and set base coordinates and frame.
    num_scaffold_bases = len(scaffold_bases)
    scaffold_coords = np.zeros((num_scaffold_bases, 3), dtype=float)
    for i, base in enumerate(scaffold_bases):
        j = pos_map[base.p]
        base.coordinates = axis_coords[j]
        base.ref_frame = axis_frames[:, :, j]
        scaffold_coords[i, :] = axis_coords[j, :] + np.dot(
            axis_frames[:, :, j], scaf_local
        )
        base.nt_coords = scaffold_coords[i]

    # Compute staple nucleotide positions and set base coordinates and frame.
    num_staple_bases = len(staple_bases)
    staple_coords = np.zeros((num_staple_bases, 3), dtype=float)
    for i, base in enumerate(staple_bases):
        j = pos_map[base.p]
        base.coordinates = axis_coords[j]
        base.ref_frame = axis_frames[:, :, j]
        staple_coords[i, :] = axis_coords[j, :] + np.dot(
            axis_frames[:, :, j], stap_local
        )
        base.nt_coords = staple_coords[i]

    return axis_coords, axis_frames, scaffold_coords, staple_coords

def get_start_coordinates_angle(dna_parameters, lattice_type, row, col, helix_num):
    """Get the start axis coordinates and angle for a virtual helix.

    Arguments:
        dna_parameters (DnaParameters): The DNA parameters to use when creating the 3D geometry for the design.
        lattice_type (CadnanoLatticeType): The lattice type for this design.
        row (int): The caDNAno row number.
        col (int): The caDNAno column number.
        helix_num (int): The caDNAno virtual helix number.
    """
    # half the distance between the axes of adjacent DNA helices.
    r_strand = dna_parameters.helix_distance / 2.0
    # rise between two neighboring base-pairs (nm)
    # dist_bp = dna_parameters.base_pair_rise
    # twisting angle between two neighboring base-pairs (degrees)
    ang_bp = dna_parameters.base_pair_twist_angle

    # Set the helix start coordinates, based on helix (row,col), and frame orientation,
    # based on helix number (polarity).
    if lattice_type == CadnanoLatticeType.honeycomb:
        xpos = sqrt(3.0) * col * r_strand
        zpos = -3.0 * row * r_strand
        if ((row % 2 == 0) and (col % 2 == 0)) or ((row % 2 != 0) and (col % 2 != 0)):
            zpos = zpos + r_strand
        if helix_num % 2 == 0:
            init_ang = -30 + ang_bp / 2
        else:
            init_ang = 150 + ang_bp / 2
    elif lattice_type == CadnanoLatticeType.square:
        xpos = 2.0 * col * r_strand
        zpos = -2.0 * row * r_strand
        if helix_num % 2 == 0:
            init_ang = 180 + ang_bp / 2
        else:
            init_ang = 0 + ang_bp / 2

    return np.array([xpos, 0.0, zpos]), init_ang

def deg2rad(deg):
    """Convert degrees into radians."""
    rad = (pi / 180) * deg
    return rad

def find_row(neigh, curr_bases):
    tmp = curr_bases - neigh
    s = np.sum(np.abs(tmp), 1)
    ind = np.where(s == 0)[0]
    return ind

def vrrotmat2vec(R):
    """Extract the equivalent rotation about an axis from a rotation matrix."""
    m00 = R[0, 0]
    m01 = R[0, 1]
    m02 = R[0, 2]
    m10 = R[1, 0]
    m11 = R[1, 1]
    m12 = R[1, 2]
    m20 = R[2, 0]
    m21 = R[2, 1]
    m22 = R[2, 2]
    angle = acos((m00 + m11 + m22 - 1) / 2.0)
    x = (m21 - m12) / sqrt(pow(m21 - m12, 2) +
                           pow(m02 - m20, 2) + pow(m10 - m01, 2))
    y = (m02 - m20) / sqrt(pow(m21 - m12, 2) +
                           pow(m02 - m20, 2) + pow(m10 - m01, 2))
    z = (m10 - m01) / sqrt(pow(m21 - m12, 2) +
                           pow(m02 - m20, 2) + pow(m10 - m01, 2))
    return np.array([x, y, z], dtype=float), angle

def vrrotvec2mat(axis, theta):
    """Create a rotation matrix to rotate theta degrees about the axis defined by vec."""
    s = np.sin(theta)
    c = np.cos(theta)
    t = 1 - c
    # print("[vrrotvec2mat] theta=%s" % str(theta))

    # normalize the vector
    x, y, z = axis / np.linalg.norm(axis)
    # print("[vrrotvec2mat] x=%s" % str(x))
    # print("[vrrotvec2mat] y=%s" % str(y))
    # print("[vrrotvec2mat] z=%s" % str(z))

    return np.array(
        [
            [t * x * x + c, t * x * y - s * z, t * x * z + s * y],
            [t * x * y + s * z, t * y * y + c, t * y * z - s * x],
            [t * x * z - s * y, t * y * z + s * x, t * z * z + c],
        ]
    )

def bp_interp(dnode_1, triad_1, dnode_2, triad_2, n: int):
    """Interpolate the position (dnode) and orientation (triad) between two base pairs.

    Solve for rotation matrix R defined as the solution of:
        R * triad_1 = triad_2 (multiply both sides by transpose(triad_1).
    """
    dnode_interp = np.zeros((int(n), 3), dtype=float)
    triad_interp = np.zeros((3, 3, int(n)), dtype=float)

    # Solve for rotation matrix R.
    R = np.dot(triad_2, triad_1.T)

    # Get the equivalent rotation about an axis for R.
    a, theta = vrrotmat2vec(R)

    # Calculate for dNode_interp and triad_interp.
    for i in range(0, n):
        dnode_interp[i, :] = (
            dnode_1 * (n + 1 - (i + 1)) + dnode_2 * (i + 1)) / (n + 1)
        angle = theta * (i + 1) / (n + 1)
        rot_mat = vrrotvec2mat(a, angle)
        triad_interp[:, :, i] = np.dot(rot_mat, triad_1)

    return dnode_interp, triad_interp

max_stars_repo_name: elija-feigl/nanodesign

max_issues_repo_name: elija-feigl/nanodesign

max_forks_repo_name: elija-feigl/nanodesign


text:
# AUTOGENERATED! DO NOT EDIT! File to edit: notebooks/04_metrics.ipynb (unless otherwise specified).

__all__ = ['kde_entropy', 'kde_likelihood', 'ppf', 'cdf', 'quantile', 'kde_quantile', 'quantile_sklearn',
           'theoretical_entropy', 'marginal_variance', 'covariance_matrix', 'bimodal_split', 'agg_smallest_distance',
           'cov_smallest_distance', 'mad', 'bimodal_variance', 'gaussian_distance_entropy',
           'expected_distance_gaussian_likelihood', 'distance_gaussian_likelihood', 'make_outlier_filter',
           'filter_borders']

# Cell
from tqdm.notebook import tqdm

import numpy as np
from sklearn.metrics import pairwise
from scipy import stats
from sklearn.preprocessing import QuantileTransformer

from .core.random_variable import KDE, RandomVariable
from .utils import (
    draw_from,
    _fix_one_sample_2d, _fix_one_dist_2d, _fix_dist_1d,
    _fix_X_1d, _assert_dim_3d, _assert_dim_1d, _assert_dim_2d, _fix_one_dist_1d,
    _add_n_samples_axis
)

# Cell
def _check_kde_metrics_input(y_true, y_dists, frac):
    '''
    preprocesses inputs for kde metrics calculation
    '''
    y_dists = _assert_dim_3d(y_dists)

    if len(y_true.shape) <= 3:
        assert y_true.shape[0] == y_dists.shape[0], f'y_true dim 0 should be equal y_dists dim 0, got {y_true.shape[0]} and {y_dists.shape[0]}'
    else:
        raise Exception(f'y_true dims should less or equal to 3, got {len(y_true.shape)}')

    y_true = _fix_one_sample_2d(y_true)

    assert y_true.shape[-1] == y_dists.shape[-1], f'y_true and y_dists last axis (n_dims) should be equal. got {y_true.shape[-1]} and {y_dists.shape[-1]}'

    idxs = np.arange(y_true.shape[0])
    idxs = draw_from(idxs, frac)

    y_dists = y_dists[idxs]
    y_true = y_true[idxs]
    return y_true, y_dists

# Cell
def kde_entropy(data, sample_size = 200, frac = 1.0, progress_bar = False, **kde_kwargs):
    '''
    Calculates the entropy of multiple continuous distributions. entropy equals np.mean(-np.log(p(x)))
    input should be of shape (n_distributions, n_sample_per_distribution, n_dims_in_distribtuion)
    '''
    data = _fix_one_dist_2d(data)
    data = _assert_dim_3d(data)
    kde = KDE(**kde_kwargs)
    data = draw_from(data, frac)
    if progress_bar:
        return np.array([kde.fit(d).entropy(sample_size = sample_size) for d in tqdm(data)])
    else:
        return np.array([kde.fit(d).entropy(sample_size = sample_size) for d in data])

# Cell
def kde_likelihood(y_true,y_dists, frac = 1.0, progress_bar = False,**kde_kwargs):
    '''
    Calculates the likelihood of y_true in kde estimation of samples
    input should be of shape (n_distributions, n_sample_per_distribution, n_dims_in_distribtuion)
    '''
    y_true, y_dists = _check_kde_metrics_input(y_true, y_dists, frac)

    kde = KDE(**kde_kwargs)
    if progress_bar:
        likelihoods = np.array([kde.fit(y_dists[i]).evaluate(y_true[i]) for i in tqdm([*range(y_dists.shape[0])])])
    else:
        likelihoods =  np.array([kde.fit(y_dists[i]).evaluate(y_true[i]) for i in range(y_dists.shape[0])])

    return _fix_dist_1d(likelihoods)

# Cell
def ppf(percentiles, y_dists):
    '''
    returns the percent point function for given y_dists and percentiles
    expected dims:
    percentiles: (n_dists, n_points)
    y_dists: (n_dists, n_samples, dims)
    '''
    assert len(percentiles.shape) == 2, f'percentiles should have 2 dimensions: (n_dists, n_percentiles), got {len(percentiles.shape)}'
    assert len(y_dists.shape) == 3, f'y_dists should have 3 dimensions: (n_dists, n_samples, n_dims), got {len(y_dists.shape)}'
    assert percentiles.shape[0] == y_dists.shape[0], f'percentiles n_dists should be equal y_dists n_dists. got {percentiles.shape[0]} and {y_dists.shape[0]}'
    values = np.array([np.quantile(y_dists[i], percentiles[i], axis = 0) for i in range(y_dists.shape[0])])
    return _fix_one_sample_2d(values)

# Cell
def _cdf(dist):
    '''
    returns the cdf of each element in a 1d array (distribution)
    '''
    _assert_dim_1d(dist)
    temp = dist.argsort()
    return np.arange(len(dist))[temp.argsort()]/(len(dist)-1)

def cdf(y_dists):
    '''
    returns the cdf of each element in each 1d dist
    '''
    y_dists = _assert_dim_3d(y_dists)
    arr = np.zeros(y_dists.shape)
    for dist in np.arange(y_dists.shape[0]):
        for dim in np.arange(y_dists.shape[-1]):
            arr[dist,:,dim] = _cdf(y_dists[dist,:,dim])

    return arr

# Cell
def quantile(y_true, y_dists):
    '''
    checks in which quantile lies y_true, given the predicted distribution
    y_true shape should be of shape (n_dists, n_samples ,n_dims)
    y_dists_should be of shape (n_dists, n_samples, n_dims)
    '''

    y_true = _fix_one_sample_2d(y_true)
    y_dists = _assert_dim_3d(y_dists)
    assert y_true.shape[0] == y_dists.shape[0], 'number of dists should be the same as number of points'

    values = []
    for i in range(y_true.shape[0]):
        values.append([])
        for j in range(y_true.shape[1]):
            values[i].append((y_true[i,j].T >= y_dists[i]).mean(axis = 0))

    return _fix_one_sample_2d(np.array(values))

# Cell
def kde_quantile(y_true, y_dists, frac = 1.0, progress_bar = False, **kde_kwargs):
    '''
    fits a kde in a distribution and returns the quantile that a point in y_true belongs to in that distribution
    '''
    y_true, y_dists = _check_kde_metrics_input(y_true, y_dists, frac = 1)
    kde = KDE(**kde_kwargs)
    if progress_bar:
        return np.array([kde.fit(y_dists[i]).cdf(y_true[i]) for i in tqdm([*range(len(y_dists))])])
    else:
        return np.array([kde.fit(y_dists[i]).cdf(y_true[i]) for i in range(len(y_dists))])

# Cell
def quantile_sklearn(y_true, y_dists):
    '''checks in which quantile lies y_true, given the predicted distribution, using skleaerns QuantileTransformer'''
    y_true, y_dists = _check_kde_metrics_input(y_true, y_dists, frac = 1)
    return np.array([QuantileTransformer().fit(y_dists[i]).transform(y_true[i]) for i in range(y_dists.shape[0])])

# Cell
def theoretical_entropy(data, dist = 'norm'):
    '''
    returns the entropy of the maximum likelihood estiation of `dist` given `data`
    dist should be one dimensional for cases when `dist` != 'kde'
    '''
    data = _assert_dim_3d(data)
    return np.array([RandomVariable().fit(data, dist).entropy(dist) for d in data])

# Cell
def marginal_variance(data):
    '''
    Calculates the variance for each dimension (marginal) of multiple distributions
    input should be of shape (n_distributions, n_sample_per_distribution, n_dims_in_distribtuion)
    '''
    data = _assert_dim_3d(data)
    return data.var(axis = -2)

# Cell
def covariance_matrix(data):
    '''
    Calculates the variance for each dimension (marginal) of multiple distributions
    input should be of shape (n_distributions, n_sample_per_distribution, n_dims_in_distribtuion)
    '''
    raise NotImplementedError

# Cell
def bimodal_split(data, filter_size = 3, lb = 0.1,ub = 0.9):
    '''
    reutrns siplitting point of single distribution in two according to the highest value of the derivative of cpdf
    '''

    _assert_dim_1d(data)
    filter_size = int(np.ceil(filter_size)) if filter_size >= 1 else int(max(1,np.ceil(data.shape[0]*filter_size)))
    data.sort()
    data = filter_borders(data,lb,ub)
    diff = np.diff(data)
    v = np.ones(filter_size)/filter_size
    diff = np.convolve(a = diff,v = v, mode = 'same')
    argmax = np.argmax(diff)
    split_point = (data[min(data.shape[0] - 1,argmax + filter_size)] + data[argmax])/2
    return split_point

# Cell
def agg_smallest_distance(data, agg_func = np.mean):
    '''
    returns the agregate (defined by agg_func) distance of each point and their closest neighbor
    recieves array of shape (n_dists,n_samples, n_dims) and reutrns array of shape (n_dists, n_dims)
    weights should be of shape (n_dists, n_samples, n_dims)
    '''
    _assert_dim_3d(data)
    data = np.sort(data, axis = 1)
    data = np.diff(data, axis = 1)
    results = agg_func(data, axis = 1)
    return results

def cov_smallest_distance(data, agg_func = np.mean):
    '''
    returns the covariance matrix of the distances of each point and their closest neighbor
    recieves array of shape (n_dists,n_samples, n_dims) and reutrns array of shape (n_dists, n_dims)
    '''
    _assert_dim_3d(data)
    data = np.sort(data, axis = 1)
    data = np.diff(data, axis = 1)
    results = np.array([np.cov(i, rowvar = False) for i in data])
    return results

# Cell
def mad(data):
    '''
    calculates mean average deviance (a robust version of standard deviation)
    '''
    np.median(np.absolute(data - np.median(data)))
    pass

# Cell
def bimodal_variance(data, filter_size = 0.05, lb = 0.1,ub = 0.9):
    '''
    returns weighted marginal variance of splitted data in two according to the highest value of the derivative of cpdf
    returns a variance array of shape (n_dists, n_dims)
    '''
    data = _assert_dim_3d(data)
    #make split point
    #GENERALIZE FOR MULTIDIM
    mses = []
    for dist in range(data.shape[0]):
        mses.append([])
        for dim in range(data.shape[2]):
            d = data[dist,:,dim]
            split_point = bimodal_split(d, filter_size, lb ,ub)
            arr1, arr2 = d[d >= split_point], d[d < split_point]
            mses[dist].append((arr1.shape[0]*(arr1.var()) + arr2.shape[0]*(arr2.var()))/d.shape[0])
    #average variance
    return np.array(mses)

# Cell
def gaussian_distance_entropy(data):
    '''
    calculates the entropy of the distribution of distances from centroid of points in dist, assuming normal distribution
    '''
    return -np.log(expected_distance_gaussian_likelihood(data))

# Cell
def expected_distance_gaussian_likelihood(data):
    '''
    calculates the expected likelihood of the distances from centroid of samples in distributions in dist
    input should be of shape (n_distributions, n_sample_per_distribution, n_dims_in_distribtuion)
    '''
    dist = _assert_dim_3d(data)
    return np.array([distance_gaussian_likelihood(d).mean() for d in data])

# Cell
def distance_gaussian_likelihood(data):
    '''
    calculates the expected likelihood of the distances from the centroid of samples in dist
    '''
    centroid = data.mean(axis = 0).reshape(1,-1)
    distances =  pairwise.euclidean_distances(data, centroid).flatten()
    distance_std = distances.std()
    if distance_std == 0:
        return 1
    z = (distances - distances.mean())/distance_std
    return 1/(distance_std*np.pi**(1/2))*np.exp(-1/2*z**2)

# Cell
def make_outlier_filter(data, lb = 25, ub = 75, c = 1):
    a = np.array(data)
    upper_quartile = np.percentile(a, ub)
    lower_quartile = np.percentile(a, lb)
    iqr = (upper_quartile - lower_quartile) * c
    lb = lower_quartile - iqr
    ub = upper_quartile + iqr
    filter_ = np.zeros(a.shape)
    filter_[(a >= lb) & (a <= ub)] = 1
    return filter_

# Cell
def filter_borders(data, lb = 0.05, ub = 0.95):
    lb = int(len(data)*lb)
    ub = int(len(data)*ub)
    return data[lb:ub]

#def distance_log_variance(dist):
#    '''variance of the distances of points to centroid of distribution'''
#    centroid = dist.mean(axis = 0).reshape(1,-1)
#    distances =  pairwise.euclidean_distances(dist, centroid).flatten()
#    return distances.var()

max_stars_repo_name: AlanGanem/scikit-density

max_issues_repo_name: AlanGanem/scikit-density

max_forks_repo_name: AlanGanem/scikit-density


text:
"""
FLOW: Richards: 1D: Inversion
=============================

The example shows an inversion of Richards equation in 1D with a
heterogeneous hydraulic conductivity function.

The haverkamp model is used with the same parameters as Celia1990_
the boundary and initial conditions are also the same. The simulation
domain is 40cm deep and is run for an hour with an exponentially
increasing time step that has a maximum of one minute. The general
setup of the experiment is an infiltration front that advances
downward through the model over time.

The model chosen is the saturated hydraulic conductivity inside
the hydraulic conductivity function (using haverkamp). The initial
model is chosen to be the background (1e-3 cm/s). The saturation data
has 2% random Gaussian noise added.

The figure shows the recovered saturated hydraulic conductivity
next to the true model. The other two figures show the saturation
field for the entire simulation for the true and recovered models.

Rowan Cockett - 21/12/2016

.. _Celia1990: http://www.webpages.uidaho.edu/ch/papers/Celia.pdf
"""
import matplotlib
import matplotlib.pyplot as plt
import numpy as np

from SimPEG import Mesh
from SimPEG import Maps
from SimPEG import Regularization
from SimPEG import DataMisfit
from SimPEG import Optimization
from SimPEG import InvProblem
from SimPEG import Directives
from SimPEG import Inversion

from SimPEG.FLOW import Richards

def run(plotIt=True):

    M = Mesh.TensorMesh([np.ones(40)], x0='N')
    M.setCellGradBC('dirichlet')
    # We will use the haverkamp empirical model with parameters from Celia1990
    k_fun, theta_fun = Richards.Empirical.haverkamp(
        M, A=1.1750e+06, gamma=4.74, alpha=1.6110e+06,
        theta_s=0.287, theta_r=0.075, beta=3.96
    )

    # Here we are making saturated hydraulic conductivity
    # an exponential mapping to the model (defined below)
    k_fun.KsMap = Maps.ExpMap(nP=M.nC)

    # Setup the boundary and initial conditions
    bc = np.array([-61.5, -20.7])
    h = np.zeros(M.nC) + bc[0]
    prob = Richards.RichardsProblem(
        M,
        hydraulic_conductivity=k_fun,
        water_retention=theta_fun,
        boundary_conditions=bc, initial_conditions=h,
        do_newton=False, method='mixed', debug=False
    )
    prob.timeSteps = [(5, 25, 1.1), (60, 40)]

    # Create the survey
    locs = -np.arange(2, 38, 4.)
    times = np.arange(30, prob.timeMesh.vectorCCx[-1], 60)
    rxSat = Richards.SaturationRx(locs, times)
    survey = Richards.RichardsSurvey([rxSat])
    survey.pair(prob)

    # Create a simple model for Ks
    Ks = 1e-3
    mtrue = np.ones(M.nC)*np.log(Ks)
    mtrue[15:20] = np.log(5e-2)
    mtrue[20:35] = np.log(3e-3)
    mtrue[35:40] = np.log(1e-2)
    m0 = np.ones(M.nC)*np.log(Ks)

    # Create some synthetic data and fields
    stdev = 0.02  # The standard deviation for the noise
    Hs = prob.fields(mtrue)
    survey.makeSyntheticData(mtrue, std=stdev, f=Hs, force=True)

    # Setup a pretty standard inversion
    reg = Regularization.Tikhonov(M, alpha_s=1e-1)
    dmis = DataMisfit.l2_DataMisfit(survey)
    opt = Optimization.InexactGaussNewton(maxIter=20, maxIterCG=10)
    invProb = InvProblem.BaseInvProblem(dmis, reg, opt)
    beta = Directives.BetaSchedule(coolingFactor=4)
    betaest = Directives.BetaEstimate_ByEig(beta0_ratio=1e2)
    target = Directives.TargetMisfit()
    dir_list = [beta, betaest, target]
    inv = Inversion.BaseInversion(invProb, directiveList=dir_list)

    mopt = inv.run(m0)

    Hs_opt = prob.fields(mopt)

    if plotIt:
        plt.figure(figsize=(14, 9))

        ax = plt.subplot(121)
        plt.semilogx(np.exp(np.c_[mopt, mtrue]), M.gridCC)
        plt.xlabel('Saturated Hydraulic Conductivity, $K_s$')
        plt.ylabel('Depth, cm')
        plt.semilogx([10**-3.9]*len(locs), locs, 'ro')
        plt.legend(('$m_{rec}$', '$m_{true}$', 'Data locations'), loc=4)

        ax = plt.subplot(222)
        mesh2d = Mesh.TensorMesh([prob.timeMesh.hx/60, prob.mesh.hx], '0N')
        sats = [theta_fun(_) for _ in Hs]
        clr = mesh2d.plotImage(np.c_[sats][1:, :], ax=ax)
        cmap0 = matplotlib.cm.RdYlBu_r
        clr[0].set_cmap(cmap0)
        c = plt.colorbar(clr[0])
        c.set_label('Saturation $\\theta$')
        plt.xlabel('Time, minutes')
        plt.ylabel('Depth, cm')
        plt.title('True saturation over time')

        ax = plt.subplot(224)
        mesh2d = Mesh.TensorMesh([prob.timeMesh.hx/60, prob.mesh.hx], '0N')
        sats = [theta_fun(_) for _ in Hs_opt]
        clr = mesh2d.plotImage(np.c_[sats][1:, :], ax=ax)
        cmap0 = matplotlib.cm.RdYlBu_r
        clr[0].set_cmap(cmap0)
        c = plt.colorbar(clr[0])
        c.set_label('Saturation $\\theta$')
        plt.xlabel('Time, minutes')
        plt.ylabel('Depth, cm')
        plt.title('Recovered saturation over time')

        plt.tight_layout()

if __name__ == '__main__':
    run()
    plt.show()

max_stars_repo_name: kimjaed/simpeg

max_issues_repo_name: kimjaed/simpeg

max_forks_repo_name: kimjaed/simpeg


text:
from qcor import *
import numpy as np

@qjit
def ansatz(q : qreg, x : float):
    X(q[0])
    Ry(q[1], x)
    CX(q[1], q[0])

# Can convert qjit kernels to a unitary matrix form
# I know the ground state is at x = .59
u_mat = ansatz.as_unitary_matrix(qalloc(2), .59)

# Create a Hamiltonian, map it to a numpy matrix
H = -2.1433 * X(0) * X(1) - 2.1433 * \
    Y(0) * Y(1) + .21829 * Z(0) - 6.125 * Z(1) + 5.907
Hmat = H.to_numpy()

# Compute |psi> = U |0>
zero_state = np.array([1., 0., 0., 0.])
final_state = np.dot(u_mat, np.transpose(zero_state))

# Compute E = <psi| H |psi>
energy = np.dot(final_state, np.dot(Hmat,final_state))
print('Energy: ', energy)

max_stars_repo_name: ausbin/qcor

max_issues_repo_name: ausbin/qcor

max_forks_repo_name: ausbin/qcor


text:
import numpy as np

from source.points import binarize_image
from .filters import *
from .built_in_filters import sobel_horizontal, sobel_vertical
# The process of Canny edge detection algorithm can be broken down to five different steps:

#     Apply Gaussian filter to smooth the image in order to remove the noise
#     Find the intensity gradients of the image
#     Apply gradient magnitude thresholding or lower bound cut-off suppression to get rid of spurious response to edge detection
#     Apply double threshold to determine potential edges
#     Track edge by hysteresis: Finalize the detection of edges by suppressing all the other edges that are weak and not connected to strong edges.

# Edges correspond to a change of pixels’ intensity. To detect it, the easiest way is to apply filters that highlight this intensity change in both directions: horizontal (x) and vertical (y)

def get_neighbors_in_direction(image,i,j,direction):
                #angle 0
    try:
        if (0 <= direction  < 22.5) or (157.5 <= direction <= 180):
            n1= image[i, j+1]
            n2= image[i, j-1]
        #angle 45
        elif (22.5 <= direction < 67.5):
            n1 = image[i+1, j-1]
            n2 = image[i-1, j+1]
        #angle 90
        elif (67.5 <= direction < 112.5):
            n1 = image[i+1, j]
            n2 = image[i-1, j]
        #angle 135
        elif (112.5 <= direction < 157.5):
            n1 = image[i-1, j-1]
            n2 = image[i+1, j+1]

        return (n1,n2)

    except IndexError:
        raise IndexError

def non_max_supression(image,directions_radians):
    n_row,n_col=image.shape
    directions= np.degrees(directions_radians)
    directions[directions<0]+=180
    output= np.zeros(image.shape)

    for i in range(1,n_row-1):
        for j in range(1, n_col-1):
            try:
                angle=directions[i,j]
                n1,n2=get_neighbors_in_direction(image,i,j,angle)
                value=image[i,j]

                if value>=n1 and value>= n2:
                    output[i][j]=value

            except IndexError:
                print("error")

    return output

def hysteresis(image,weak_value,strong_value):

    n_row,n_col=image.shape
    for i in range(n_row):
        for j in range(n_col):

            if image[i][j] == weak_value:
                strong_found=False
                for il in range(-1,2):
                    if not strong_found:

                        for jl in range(-1,2):
                            if il ==0 and jl ==0:
                                pass
                            try:
                                value = image[i+il][j+jl]
                                if value ==strong_value:
                                    strong_found=True
                                    break

                            except IndexError:
                                continue
                if strong_found:
                    image[i][j]=strong_value
                else:
                    image[i][j]=0

def edge(image,method='canny'):

    sobel_v=filter_image(image,sobel_vertical)
    sobel_h=filter_image(image, sobel_horizontal)

    G=np.hypot(sobel_h,sobel_v)
    G=G/ G.max() * 255 # normalization
    angles=np.arctan2(sobel_v,sobel_h)

    edges=non_max_supression(G,angles)

    high_thresh=0.1 * edges.max()
    low_thresh=0.05 *high_thresh

    binarized=binarize_image(edges,low_thresh,btype='both',threshold2= high_thresh, middle_value=30)
    hysteresis(binarized, 30,255)

    return binarized

def canny():
    pass

max_stars_repo_name: jmachalica/PyImage

max_issues_repo_name: jmachalica/PyImage

max_forks_repo_name: jmachalica/PyImage


text:
from neuronmi.simulators.solver.embedding import EmbeddedMesh
from neuronmi.simulators.solver.probing import *
import dolfin as df
import numpy as np
import unittest

class TestCases(unittest.TestCase):

    def test_centers(self):
        mesh = df.UnitCubeMesh(4, 4, 4)
        facet_f = df.MeshFunction('size_t', mesh, 2, 0)
        df.CompiledSubDomain('near(x[0], 0)').mark(facet_f, 1)
        df.CompiledSubDomain('near(x[0], 1)').mark(facet_f, 2)

        x = np.array(get_geom_centers(facet_f, (1, 2)))
        y = np.array([[0, 0.5, 0.5],
                      [1, 0.5, 0.5]])

        self.assertTrue(np.linalg.norm(x - y) < 1E-13)

    def test_probe(self):
        mesh = df.UnitCubeMesh(4, 4, 4)
        f = df.Expression('t*t*x[0] + t*x[1] + x[2]', t=0, degree=1)

        V = df.FunctionSpace(mesh, 'CG', 1)
        fh = df.Function(V)

        locations = np.array([[0.2, 0.3, 0.4], [0.1, 0.5, 0.2]])

        probe = Probe(fh, locations)
        for t in range(1, 5):
            f.t = t
            fh.vector()[:] = df.interpolate(f, V).vector()

            probe.probe(t)
            values = probe.data[-1][1:]
            self.assertTrue(all(abs(f(loc) - v) < 1E-13 for loc, v in zip(locations, values)))

# ------------------------------------------------------------------------------

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: MiroK/nEuronMI

max_issues_repo_name: MiroK/nEuronMI

max_forks_repo_name: MiroK/nEuronMI


text:
import glob
import os
import pickle
import re

import gensim
from gensim.models.callbacks import CallbackAny2Vec
from gensim.models import Word2Vec

import numpy as np

from mat2vec.processing.process import MaterialsTextProcessor

text_processing = MaterialsTextProcessor()

COMMON_TERMS = ["-", "-", b"\xe2\x80\x93", b"'s", b"\xe2\x80\x99s", "from",
                "as", "at", "by", "of", "on", "into", "to", "than", "over",
                "in", "the", "a", "an", "/", "under", ":"]
EXCLUDE_PUNCT = [
    "=",
    ".",
    ",",
    "(",
    ")",
    "<",
    ">",
    "\"",
    "“",
    "”",
    "≥",
    "≤",
    "<nUm>"]
EXCLUDE_TERMS = ["=", ".", ",", "(", ")", "<", ">", "\"", "“", "”", "≥", "≤",
                 "<nUm>", "been", "be", "are",
                 "which", "were", "where", "have", "important", "has", "can",
                 "or", "we", "our",
                 "article", "paper", "show", "there", "if", "these", "could",
                 "publication",
                 "while", "measured", "measure", "demonstrate", "investigate",
                 "investigated",
                 "demonstrated", "when", "prepare", "prepared", "use", "used",
                 "determine",
                 "determined", "find", "successfully", "newly", "present",
                 "reported", "report", "new", "characterize", "characterized",
                 "experimental",
                 "result", "results", "showed", "shown", "such", "after",
                 "but", "this", "that", "via", "is", "was", "and", "using"]
INCLUDE_PHRASES = ["oxygen_reduction_reaction"]

def save_obj(obj, name):
    with open(name + ".pkl", "wb") as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)

def load_obj(name):
    with open(name + ".pkl", "rb") as f:
        return pickle.load(f)

def keep_simple_formula(word, count, min_count):
    if text_processing.is_simple_formula(word):
        return gensim.utils.RULE_KEEP
    else:
        return gensim.utils.RULE_DEFAULT

def compute_epoch_accuracies(root, prefix, analogy_file):
    filenames = glob.glob(os.path.join(root, prefix + "_epoch*.model"))
    nr_epochs = len(filenames)
    accuracies = dict()
    losses = [0] * nr_epochs
    for filename in filenames:
        epoch = int(re.search(r"\d+\.model", filename).group()[:-6])
        m = Word2Vec.load(filename)
        losses[epoch] = m.get_latest_training_loss()
        sections = m.wv.accuracy(analogy_file)
        for sec in sections:
            if sec["section"] not in accuracies:
                accuracies[sec["section"]] = [0] * nr_epochs
            correct, incorrect = len(sec["correct"]), len(sec["incorrect"])
            if incorrect > 0:
                accuracy = correct / (correct + incorrect)
            else:
                accuracy = 0
            accuracies[sec["section"]][epoch] = (correct, incorrect, accuracy)
        save_obj(accuracies, os.path.join("models", prefix + "_accuracies"))
        save_obj(np.concatenate([np.array([losses[0]]), np.diff(
            losses)]), os.path.join("models", prefix + "_loss"))

class EpochSaver(CallbackAny2Vec):
    """Callback to save model after every epoch."""

    def __init__(self, path_prefix):
        self.path_prefix = path_prefix
        self.epoch = 0

    def on_epoch_end(self, m):
        output_path = "{}_epoch{}.model".format(self.path_prefix, self.epoch)
        print("Save model to {}.".format(output_path))
        m.save(output_path)
        self.epoch += 1


text:
# Python Standard Library
import json

# Public libraries
from keras.models import model_from_json
from keras.optimizers import Adam
from keras.utils import np_utils

# Project
import config
import helper
from numpy import genfromtxt

import csv
class_labels = {}
with open('signnames.csv', mode='r') as infile:
    reader = csv.reader(infile)
    class_labels = {key: val for key, val in reader}

DATA_SET = 'test'  # {'test', 'train', 'valid'}

# Load images and labels
#x, y = helper.load_data(DATA_SET)
x = helper.load_images(DATA_SET)
# Load model
model_file = config.MODEL_DEFINITION
with open(model_file, 'r') as jfile:
    model = model_from_json(json.loads(jfile.read()))

# Compile model and load weights
model.compile(optimizer=Adam(), loss='categorical_crossentropy',
              metrics=['accuracy'])
model.load_weights(config.MODEL_WEIGHTS)

# Evaluate model performace
print('Evaluating performance on %d samples' % x.shape[0])
#y_cat = np_utils.to_categorical(y, config.NUM_CLASSES)
scores = model.predict(x, verbose=0)
#names = model.metrics_names
for class_label, score in (class_labels, scores):
    print('%s: \t%.4f' % (class_label, score)

reason: from numpy


text:
# Importing required packages
import pandas
import numpy

#Importing required files
import hd_DataAnalysis as sd
import hd_Preprocessing as prep

# Extracting the features of the three main attributes
def feature_extraction(main_data):
	main_data['search_term'] = main_data['search_term'].map(lambda x: sd.preprocess_word(x))
	main_data['product_title'] = main_data['product_title'].map(lambda x: sd.preprocess_word(x))
	main_data['product_description'] = main_data['product_description'].map(lambda x: sd.preprocess_word(x))

	return main_data

# Create new features in the main data file
def add_attributes(main_data):

	# Integrating all the above three extracted features into a single column.
	main_data['prod_combined_info'] = main_data['search_term'] + "\t" + main_data['product_title'] + "\t" + main_data['product_description']

	main_data['length_of_search_query'] = main_data['search_term'].map(lambda x: len(x.split())).astype(numpy.int64)

	main_data['length_of_title'] = main_data['product_title'].map(lambda x: len(x.split())).astype(numpy.int64)

	main_data['length_of_description'] = main_data['product_description'].map(lambda x: len(x.split())).astype(numpy.int64)

	main_data['query_in_title'] = main_data['prod_combined_info'].map(lambda x: sd.freq_of_words(x.split('\t')[0], x.split('\t')[1], 0))

	main_data['query_in_description'] = main_data['prod_combined_info'].map(lambda x: sd.freq_of_words(x.split('\t')[0], x.split('\t')[2], 0))

	main_data['title_commonWord'] = main_data['prod_combined_info'].map(lambda x: sd.find_common_words(x.split('\t')[0], x.split('\t')[1]))

	main_data['description_commonWord'] = main_data['prod_combined_info'].map(lambda x: sd.find_common_words(x.split('\t')[0], x.split('\t')[2]))

	print("Step2: Feature Extraction & Integrated attributes")
	return main_data


text:
# Copyright: (c) 2021, Edwin G. W. Peters

import zmq
import numpy as np
import time

def send_to_sdr(NUM_TESTS = 1000, TIME_BETWEEN_PACKETS_MS = 1000, PACKET_LEN = 21,ADDR='tcp://127.0.0.1:5501'):

    ctx = zmq.Context()
    sock = ctx.socket(zmq.PUSH)
    sock.connect(ADDR)

    t_tot = 0
    packets = []
    for i in range(NUM_TESTS):
        packets.append(np.mod(np.arange(PACKET_LEN),256).astype(np.uint8))

    for i in range(NUM_TESTS):

        print(f'sending packet {i}')
        p = np.concatenate(( np.array([0x40,0x00,0x00,0x01,0x17],dtype=np.uint8),packets[i],np.array([0x11,0x11],dtype=np.uint8)))
        # p[20] = 0
        sock.send(p)

        time.sleep(TIME_BETWEEN_PACKETS_MS/1000)

    sock.close()

if __name__ == "__main__":

    send_to_sdr()

max_stars_repo_name: mugpahug/pycu-sdr

max_issues_repo_name: mugpahug/pycu-sdr

max_forks_repo_name: mugpahug/pycu-sdr


text:
import os
import xml.etree.ElementTree as et
from transformers import BertTokenizer, T5Tokenizer
import numpy as np
import jsonlines

# Preprocessing functions for the different data sets

def save(file_path, data):
    os.makedirs(file_path.rsplit('/', 1)[0], exist_ok=True)
    np.save(file_path, np.array(data, dtype=object), allow_pickle=True)

def right_tokenizer(model):
    tok = BertTokenizer.from_pretrained('bert-base-uncased')
    if model == 'T5':
        tok = T5Tokenizer.from_pretrained('t5-base')
    return tok

def preprocess_seb(file_path, where_to_save, model):
    def label_to_int(label):
        x = 0
        if label == 'correct':
            x = 2
        if label == 'contradictory':
            x = 1
        return x

    tokenizer = right_tokenizer(model)
    array = []
    files = os.listdir(file_path)
    for file in files:
        root = et.parse(file_path + '/' + file).getroot()
        for ref_answer in root[1]:
            for stud_answer in root[2]:
                text_ref = ref_answer.text[:-1]
                text_stud = stud_answer.text[:-1]
                if model == 'bert':
                    label = label_to_int(stud_answer.get('accuracy'))
                    tokenized = tokenizer(text_ref, text_stud, max_length=128, padding='max_length')
                    array.append([tokenized.input_ids[:128],
                                  tokenized.token_type_ids[:128],
                                  tokenized.attention_mask[:128],
                                  label])
                if model == 'T5':
                    label = tokenizer(stud_answer.get('accuracy'), max_length=4, padding='max_length')

                    tokenized = tokenizer("asag: reference:" + text_ref + tokenizer.eos_token + " student: " +
                                          text_stud, max_length=128, padding='max_length')
                    array.append([tokenized.input_ids[:128], tokenized.attention_mask[:128], label.input_ids,
                                  label.attention_mask])

    save(where_to_save, array)

def preprocess_mnli(file_path, where_to_save, model):
    def label_to_int(lab):
        if lab == 'neutral':
            return 0
        if lab == 'contradiction':
            return 1
        if lab == 'entailment':
            return 2
        else:
            raise ValueError

    tokenizer = right_tokenizer(model)
    file = jsonlines.open(file_path)
    array = []
    for line in file:
        if line['gold_label'] not in ['entailment', 'contradiction', 'neutral']:
            continue
        if model == 'bert':
            label = label_to_int(line['gold_label'])
            tokenized = tokenizer(line['sentence1'], line['sentence2'], max_length=128, padding='max_length')
            array.append([tokenized.input_ids[:128], tokenized.token_type_ids[:128], tokenized.attention_mask[:128],
                          label])
        if model == 'T5':
            label = tokenizer(line['gold_label'], max_length=5, padding='max_length')
            tokenized = tokenizer("mnli: premise: " + line['sentence1'] + tokenizer.eos_token + " hypothesis: " +
                                  line['sentence2'], max_length=128, padding='max_length')
            array.append([tokenized.input_ids[:128], tokenized.attention_mask[:128], label.input_ids,
                          label.attention_mask])

    save(where_to_save, array)

def preprocess_MSpara(file_path, where_to_save, model):
    tokenizer = right_tokenizer(model)
    array = []
    table = np.genfromtxt(file_path, delimiter='\t', dtype=str, encoding='utf-8', comments='###')
    for i in table[1:]:
        if model == 'bert':
            label = int(i[0])
            tokenized = tokenizer(i[3], i[4], max_length=128, padding='max_length')
            array.append([tokenized.input_ids[:128], tokenized.token_type_ids[:128], tokenized.attention_mask[:128],
                          label])
        if model == 'T5':
            label = tokenizer(str(bool(int(i[0]))), max_length=4, padding='max_length')
            tokenized = tokenizer("msrpc: sentence: " + i[3] + tokenizer.eos_token + " paraphrase: " + i[4], max_length=128,
                                  padding='max_length')
            array.append([tokenized.input_ids[:128], tokenized.attention_mask[:128], label.input_ids,
                          label.attention_mask])

    save(where_to_save, array)

def preprocess_QQP(file_path, where_to_save, model):
    tokenizer = right_tokenizer(model)
    array = []
    table = np.genfromtxt(file_path, delimiter='\t', dtype=str, encoding='utf-8', comments='#1#1')
    for i in table[1:]:
        if model == 'bert':
            label = int(i[5])
            tokenized = tokenizer(i[3], i[4], max_length=128, padding='max_length')
            array.append([tokenized.input_ids[:128], tokenized.token_type_ids[:128], tokenized.attention_mask[:128],
                          label])
        if model == 'T5':
            label = tokenizer(str(bool(int(i[5]))), max_length=4, padding='max_length')
            tokenized = tokenizer("qqp: question: " + i[3] + tokenizer.eos_token + " duplicate: " + i[4],
                                  max_length=128, padding='max_length')
            array.append([tokenized.input_ids[:128], tokenized.attention_mask[:128], label.input_ids,
                          label.attention_mask])

    save(where_to_save, array)

def preprocess_RTE(file_path, where_to_save, model):
    def label_to_int(lab):
        if lab == 'entailment':
            return int(1)
        if lab == 'not_entailment':
            return int(0)
        else:
            raise ValueError

    tokenizer = right_tokenizer(model)
    array = []
    table = np.genfromtxt(file_path, delimiter='\t', dtype=str, encoding='utf-8', comments='#1#1')
    for i in table[1:]:
        if model == 'bert':
            label = label_to_int(i[3])
            tokenized = tokenizer(i[1], i[2], max_length=128, padding='max_length')
            array.append([tokenized.input_ids[:128], tokenized.token_type_ids[:128], tokenized.attention_mask[:128],
                          label])
        if model == 'T5':
            label = tokenizer(str(bool(label_to_int(i[3]))), max_length=128, padding='max_length')
            tokenized = tokenizer("rte: reference: " + i[1] + tokenizer.eos_token + " answer: " + i[2], max_length=128,
                                  padding='max_length')
            array.append([tokenized.input_ids[:128], tokenized.attention_mask[:128], label.input_ids,
                          label.attention_mask])

    save(where_to_save, array)

def preprocess_wic(file_path, where_to_save, model):
    tokenizer = right_tokenizer(model)
    array = []
    table = jsonlines.open(file_path)
    for line in table:
        if model == 'bert':
            label = int(line['label'])
            tokenized = tokenizer(line['sentence1'], line['sentence2'], max_length=128, padding='max_length')
            array.append([tokenized.input_ids[:128], tokenized.token_type_ids[:128], tokenized.attention_mask[:128],
                          label])
        if model == 'T5':
            label = tokenizer(str(line['label']), max_length=4, padding='max_length')
            tokenized = tokenizer('wic: word: ' + line['word'] + tokenizer.eos_token + " first: " + line['sentence1'] +
                                  tokenizer.eos_token + "second: " + line['sentence2'], max_length=128,
                                  padding='max_length')
            array.append([tokenized.input_ids[:128], tokenized.attention_mask[:128], label.input_ids,
                          label.attention_mask])

    save(where_to_save, array)

"""
# BERT
# preprocess seb for bert
preprocess_seb('datasets/raw/sciEntsBank_training', 'datasets/preprocessed/bert/seb/train', 'bert')
preprocess_seb('datasets/raw/sciEntsBank_testing/test-unseen-answers', 'datasets/preprocessed/bert/seb/test_ua', 'bert')
preprocess_seb('datasets/raw/sciEntsBank_testing/test-unseen-domains', 'datasets/preprocessed/bert/seb/test_ud', 'bert')
preprocess_seb('datasets/raw/sciEntsBank_testing/test-unseen-questions', 'datasets/preprocessed/bert/seb/test_uq',
                'bert')

# preprocess mnli for bert
preprocess_mnli('datasets/raw/MNLI_matched/original/multinli_1.0_train.jsonl', 'datasets/preprocessed/bert/MNLI/train',
                'bert')
preprocess_mnli('datasets/raw/MNLI_matched/original/multinli_1.0_dev_matched.jsonl',
                'datasets/preprocessed/bert/MNLI/dev_m', 'bert')
preprocess_mnli('datasets/raw/MNLI_matched/original/multinli_1.0_dev_mismatched.jsonl',
                'datasets/preprocessed/bert/MNLI/dev_mm', 'bert')

# preprocess msrpc for bert
preprocess_MSpara('datasets/raw/MSpara/msr_paraphrase_train.txt', 'datasets/preprocessed/bert/MSpara/train', 'bert')
preprocess_MSpara('datasets/raw/MSpara/msr_paraphrase_test.txt', 'datasets/preprocessed/bert/MSpara/test', 'bert')

# preprocess qqp for bert
preprocess_QQP('datasets/raw/QQP/train.tsv', 'datasets/preprocessed/bert/qqp/train', 'bert')
preprocess_QQP('datasets/raw/QQP/dev.tsv', 'datasets/preprocessed/bert/qqp/dev', 'bert')

# preprocess RTE for bert
preprocess_RTE('datasets/raw/RTE/train.tsv', 'datasets/preprocessed/bert/RTE/train', 'bert')
preprocess_RTE('datasets/raw/RTE/dev.tsv', 'datasets/preprocessed/bert/RTE/dev', 'bert')

# preprocess WiC for bert
preprocess_wic('datasets/raw/WiC/train.jsonl', 'datasets/preprocessed/bert/wic/train', 'bert')
preprocess_wic('datasets/raw/WiC/val.jsonl', 'datasets/preprocessed/bert/wic/dev', 'bert')

# T5

# preprocess seb for T5
preprocess_seb('datasets/raw/sciEntsBank_training', 'datasets/preprocessed/T5/seb/train', 'T5')
preprocess_seb('datasets/raw/sciEntsBank_testing/test-unseen-answers', 'datasets/preprocessed/T5/seb/test_ua', 'T5')
preprocess_seb('datasets/raw/sciEntsBank_testing/test-unseen-domains', 'datasets/preprocessed/T5/seb/test_ud', 'T5')
preprocess_seb('datasets/raw/sciEntsBank_testing/test-unseen-questions', 'datasets/preprocessed/T5/seb/test_uq', 'T5')

# preprocess mnli for T5
preprocess_mnli('datasets/raw/MNLI_matched/original/multinli_1.0_train.jsonl', 'datasets/preprocessed/T5/MNLI/train.npy',
                'T5')
preprocess_mnli('datasets/raw/MNLI_matched/original/multinli_1.0_dev_matched.jsonl',
                'datasets/preprocessed/T5/MNLI/dev_m.npy', 'T5')
preprocess_mnli('datasets/raw/MNLI_matched/original/multinli_1.0_dev_mismatched.jsonl',
                'datasets/preprocessed/T5/MNLI/dev_mm.npy', 'T5')

# preprocess msrpc for T5
preprocess_MSpara('datasets/raw/MSpara/msr_paraphrase_train.txt', 'datasets/preprocessed/T5/MSpara/train', 'T5')
preprocess_MSpara('datasets/raw/MSpara/msr_paraphrase_test.txt', 'datasets/preprocessed/T5/MSpara/test', 'T5')

# preprocess qqp for T5
preprocess_QQP('datasets/raw/QQP/train.tsv', 'datasets/preprocessed/T5/qqp/train', 'T5')
preprocess_QQP('datasets/raw/QQP/dev.tsv', 'datasets/preprocessed/T5/qqp/dev', 'T5')

"""
# preprocess WiC for T5
preprocess_wic('datasets/raw/WiC/train.jsonl', 'datasets/preprocessed/T5/wic/train', 'T5')
preprocess_wic('datasets/raw/WiC/val.jsonl', 'datasets/preprocessed/T5/wic/dev', 'T5')

# preprocess RTE for T5
preprocess_RTE('datasets/raw/RTE/train.tsv', 'datasets/preprocessed/T5/RTE/train', 'T5')
preprocess_RTE('datasets/raw/RTE/dev.tsv', 'datasets/preprocessed/T5/RTE/dev', 'T5')