text:
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
import tvm
import numpy as np
from tvm.contrib import cublas
from tvm.contrib import cublaslt

def verify_matmul_add(in_dtype, out_dtype, rtol=1e-5):
    n = 1024
    l = 128
    m = 236
    A = tvm.placeholder((n, l), name='A', dtype=in_dtype)
    B = tvm.placeholder((l, m), name='B', dtype=in_dtype)
    C = cublas.matmul(A, B, dtype=out_dtype)
    s = tvm.create_schedule(C.op)

    def verify(target="cuda"):
        if not tvm.runtime.enabled(target):
            print("skip because %s is not enabled..." % target)
            return
        if not tvm.get_global_func("tvm.contrib.cublas.matmul", True):
            print("skip because extern function is not available")
            return
        ctx = tvm.gpu(0)
        f = tvm.build(s, [A, B, C], target)
        a = tvm.nd.array(np.random.uniform(0, 128, size=(n, l)).astype(A.dtype), ctx)
        b = tvm.nd.array(np.random.uniform(0, 128, size=(l, m)).astype(B.dtype), ctx)
        c = tvm.nd.array(np.zeros((n, m), dtype=C.dtype), ctx)
        f(a, b, c)
        tvm.testing.assert_allclose(
            c.asnumpy(), np.dot(a.asnumpy().astype(C.dtype), b.asnumpy().astype(C.dtype)), rtol=rtol)
    verify()

def roundoff(v, d):
    return int(np.floor((v + d - 1) / d) * d)

def verify_matmul_add_igemm(in_dtype, out_dtype, rtol=1e-5):
    n = 1024
    l = 1024
    m = 1024
    L = roundoff(l, 32)
    N = roundoff(n, 8)
    N_out = roundoff(n, 32)

    A = tvm.placeholder((N, L), name='A', dtype=in_dtype)
    B = tvm.placeholder((m, L), name='B', dtype=in_dtype)
    # C has CUBLASLT_ORDER_COL32 layout, thus a different shape
    C = cublaslt.matmul(A, B, False, True, m, N_out, dtype=out_dtype)
    s = tvm.create_schedule(C.op)

    def verify(target="cuda"):
        if not tvm.runtime.enabled(target):
            print("skip because %s is not enabled..." % target)
            return
        if not tvm.get_global_func("tvm.contrib.cublaslt.matmul", True):
            print("skip because extern function is not available")
            return
        ctx = tvm.gpu(0)
        f = tvm.build(s, [A, B, C], target)
        a_old = np.random.uniform(0, 128, size=(n, l))
        b_old = np.random.uniform(0, 128, size=(l, m))

        # Transform a to become CUBLASLT_ORDER_COL4_4R2_8C layout
        a_new = np.hstack((a_old.astype(A.dtype), np.zeros([n, L-l])))
        a_new = np.vstack((a_new.astype(A.dtype), np.zeros([N-n, L])))
        a_even = np.vsplit(a_new[::2], N / 8)
        a_odd = np.vsplit(a_new[1::2], N / 8)
        a_new = [None]*(len(a_even) + len(a_odd))
        a_new[::2] = a_even
        a_new[1::2] = a_odd
        a_new = np.vstack(a_new)
        a_new = np.vstack(np.vstack(np.vstack(np.hsplit(i, 8)).reshape([4, 32]) for i in np.vsplit(j, N/4)) for j in np.hsplit(a_new, L/32))
        a_new = a_new.reshape([N, L])
        # Transform b to become CUBLASLT_ORDER_COL32 layout
        b_new = np.vstack(np.hsplit(np.hstack((b_old.T.astype(B.dtype), np.zeros([m, L - l]))), L / 32))
        b_new = b_new.reshape([m, L])

        a = tvm.nd.array(a_new.astype(A.dtype), ctx)
        b = tvm.nd.array(b_new.astype(B.dtype), ctx)
        c = tvm.nd.array(np.zeros((m, N_out), dtype=C.dtype), ctx)
        f(a, b, c)
        # Transform output c from layout CUBLASLT_ORDER_COL32 to row major layout
        c_out = c.asnumpy()
        c_out = c_out.reshape([int(m * N_out / 32), 32])
        c_out = np.hstack(np.vsplit(c_out, int(N_out / 32)))
        c_out = c_out[:, :n]
        c_out = c_out.T
        tvm.testing.assert_allclose(
            c_out, np.dot(a_old.astype(C.dtype), b_old.astype(C.dtype)), rtol=rtol)
    verify()

def verify_batch_matmul(in_dtype, out_dtype, rtol=1e-5):
    j = 16
    n = 1024
    l = 128
    m = 236
    A = tvm.placeholder((j, n, l), name='A', dtype=in_dtype)
    B = tvm.placeholder((j, l, m), name='B', dtype=in_dtype)
    C = cublas.batch_matmul(A, B, dtype=out_dtype)
    s = tvm.create_schedule(C.op)

    def verify(target="cuda"):
        if not tvm.runtime.enabled(target):
            print("skip because %s is not enabled..." % target)
            return
        if not tvm.get_global_func("tvm.contrib.cublas.matmul", True):
            print("skip because extern function is not available")
            return
        ctx = tvm.gpu(0)
        f = tvm.build(s, [A, B, C], target)
        a = tvm.nd.array(np.random.uniform(size=(j, n, l)).astype(A.dtype), ctx)
        b = tvm.nd.array(np.random.uniform(size=(j, l, m)).astype(B.dtype), ctx)
        c = tvm.nd.array(np.zeros((j, n, m), dtype=C.dtype), ctx)
        f(a, b, c)
        tvm.testing.assert_allclose(
            c.asnumpy(), np.matmul(a.asnumpy().astype(C.dtype),
                                   b.asnumpy().astype(C.dtype)).astype(C.dtype), rtol=rtol)
    verify()

def test_matmul_add():
    verify_matmul_add('float', 'float', rtol=1e-3)
    verify_matmul_add('float16', 'float')
    verify_matmul_add('float16', 'float16', rtol=1e-2)
    verify_matmul_add('int8', 'int32')

def test_matmul_add_igemm():
    verify_matmul_add_igemm('int8', 'int32')

def test_batch_matmul():
    verify_batch_matmul('float', 'float')
    verify_batch_matmul('float16', 'float')
    verify_batch_matmul('float16', 'float16', rtol=1e-2)

if __name__ == "__main__":
    test_matmul_add()
    test_batch_matmul()
    test_matmul_add_igemm()

max_stars_repo_name: robo-corg/incubator-tvm

max_issues_repo_name: robo-corg/incubator-tvm

max_forks_repo_name: robo-corg/incubator-tvm


text:
from django.db import models
from datasets.models import Dataset, Term, Document, Modality
from django.contrib.auth.models import User
from datetime import datetime
import os
from django.conf import settings
import json
import numpy as np
import pandas as pd
import re
from shutil import rmtree
from django.db import transaction
import traceback
import struct
import time
import algo.arranging.metrics as metrics
from models.bigartm_config import BANNED_WORDS
from django.db.models.signals import pre_delete
from django.dispatch import receiver
from django.contrib import admin

class ArtmModel(models.Model):
    dataset = models.ForeignKey(Dataset, null=False)
    creation_time = models.DateTimeField(null=False, default=datetime.now)
    text_id = models.TextField(null=False, default="")
    # main_modality = models.ForeignKey(Modality, null = True)
    name = models.TextField(null=True)
    author = models.ForeignKey(User, null=True)
    layers_count = models.IntegerField(default=1)
    topics_count = models.TextField(null=False, default="")
    # 0-ready,  1-running, 2-error, 3-empty, 11-running, but not critical
    status = models.IntegerField(null=False, default=0)
    error_message = models.TextField(null=True, blank=True)
    arrangement_mode = models.TextField(default="none", null=False)
    metric = models.TextField(default="jaccard", null=False)
    threshold_hier = models.IntegerField(null=False, default=100)
    threshold_docs = models.IntegerField(null=False, default=100)
    max_parents_hier = models.IntegerField(null=False, default=1)
    topic_naming_top_words = models.IntegerField(null=False, default=3)

    is_public = models.BooleanField(null=False, default=True)

    def __str__(self):
        if not self.name or len(self.name) == 0:
            return self.text_id
        else:
            return self.name

    def get_model(request, modify=False):
        if "model_id" in request.GET:
            model = ArtmModel.objects.get(id=request.GET['model_id'])
        else:
            return None

        if model.is_public:
            return model

        if not model.dataset.is_public and model.dataset.owner != request.user:
            return None

        if not modify and not model.author == request.user:
            return None

        return model

    def create_generic(self, POST):
        mode = POST['mode']
        self.text_id = "model_" + str(self.id)

        try:
            if mode == 'flat':
                self.prepare_log("Creating flat model (auto)")
                iter_count = int(POST.getlist('iter_count')[0])
                self.layers_count = 1
                self.topics_count = "1 " + POST.getlist('num_topics')[0]
                self.save()

                regularizers = {}
                if "regularizers" in POST:
                    regularizers = json.loads(POST["regularizers"])
                artm_object = self.create_simple(
                    iter_count=iter_count, regularizers=regularizers)
                self.save_matrices(artm_object)
            elif mode == "hier":
                self.prepare_log("Creating hierarchical model (auto)")
                self.layers_count = int(POST['num_layers'])
                iter_count = int(POST.getlist('iter_count')[1])
                self.topics_count = "1 " + ' '.join([
                    POST.getlist('num_topics')[i + 1]
                    for i in range(self.layers_count)])
                self.save()
                artm_object = self.create_simple(iter_count=iter_count)
                self.save_matrices(artm_object)
            elif mode == "script":
                script_file_name = os.path.join(
                    settings.DATA_DIR, "scripts", POST['script_name'])
                with open(script_file_name) as f:
                    code = compile(f.read(), script_file_name, "exec")
                batch_vectorizer, dictionary = self.dataset.get_batches()
                local_vars = {
                    "batch_vectorizer": batch_vectorizer,
                    "dictionary": dictionary}
                self.prepare_log("Running custom sript...")
                exec(code, local_vars)
                self.log("Custom script finished.")
                artm_object = local_vars["model"]
                self.save_matrices(artm_object)
            elif mode == "custom":
                raise Exception("You cannot upload scripts.")
            elif mode == "matrices":
                self.text_id = POST["matrices_folder"]
                self.prepare_log("Model will be loaded from matrices")
            elif mode == "empty":
                sample_script_file = os.path.join(
                    self.get_folder(), "sample.py")
                batches_folder = os.path.join(
                    self.dataset.get_folder(), "batches")
                dictionary_file = os.path.join(
                    batches_folder, "dictionary.txt")

                # Creating sample script, which will build simple model and
                # save it to files.
                text = ""
                template_file = os.path.join(settings.BASE_DIR, "templates",
                                             "sample_scripts",
                                             "create_model.py")
                with open(template_file, "r") as f:
                    text = f.read()
                    text = text.replace("{{batches_folder}}", batches_folder)
                    text = text.replace("{{dictionary_file}}", dictionary_file)
                    text = text.replace("{{phi_file}}",
                                        os.path.join(self.get_folder(), "phi"))
                    text = text.replace(
                        "{{theta_file}}", os.path.join(
                            self.get_folder(), "theta"))
                    text = text.replace("\\", "\\\\")

                with open(sample_script_file, "w") as f:
                    f.write(text)
                self.status = 3
                self.save()
                return
            else:
                raise Exception('Unknown mode: ' + mode)

            self.reload()
        except BaseException:
            self.error_message = traceback.format_exc()
            self.status = 2
            self.save()

    def create_simple(self, iter_count, regularizers={}):
        import artm

        self.log("Creating simple model...")
        layers_count = self.layers_count
        num_topics = [int(x) for x in self.topics_count.split()]

        batch_vectorizer, dictionary = self.dataset.get_batches()

        model = artm.hARTM(
            num_document_passes=iter_count,
            theta_columns_naming="id")
        model.cache_theta = True
        layers = [0 for i in range(layers_count)]

        layers[0] = model.add_level(
            num_topics=num_topics[1],
            topic_names=[
                "topic" +
                str(t) for t in range(
                    num_topics[1])])
        layers[0].initialize(dictionary=dictionary)
        self.log("Layer 0 initialized.")

        if (regularizers):
            reg_code = ""
            for name, params in regularizers.items():
                params_init = []
                for pname, value in params.items():
                    if len(value) > 10:
                        raise RuntimeError(
                            "Too long value for parameter %s.%s" %
                            (name, pname))
                    params_init.append(pname + "=" + value)
                reg_code += "layers[0].regularizers.add(artm.%s(%s))\n" % (
                    name, ", ".join(params_init))
            self.log(
                "Regularizers to be applied:
" +
                reg_code.replace(
                    "\n",
                    "
"))
            exec(reg_code)

        layers[0].fit_offline(
            batch_vectorizer=batch_vectorizer,
            num_collection_passes=iter_count)
        self.log("Layer 0 fitted.")

        for layer_id in range(1, layers_count):
            layers[layer_id] = model.add_level(
                parent_level_weight=0.1,
                num_topics=num_topics[layer_id + 1],
                topic_names=[
                    "topic" + str(t)
                    for t in range(num_topics[layer_id + 1])]
            )
            layers[layer_id].initialize(dictionary=dictionary)
            self.log("Layer " + str(layer_id) + " initialized.")
            layers[layer_id].fit_offline(
                batch_vectorizer=batch_vectorizer,
                num_collection_passes=iter_count)
            self.log("Layer " + str(layer_id) + " fitted.")

        self.log("Model built.")
        return model

    def save_matrices(self, artm_object):
        self.log("Saving matrices for model " + str(self.id) + "...")

        self.log("Saving matrix theta...")
        pickle_file = os.path.join(self.get_folder(), "theta")
        artm_object.get_theta().to_pickle(pickle_file)

        self.log("Saving matrix phi...")
        artm_object.get_phi().to_pickle(os.path.join(self.get_folder(), "phi"))

        if self.dataset.modalities_count > 1:
            for modality in Modality.objects.filter(dataset=self.dataset):
                artm_object.get_phi(
                    class_ids=[
                        modality.name]).to_pickle(
                    os.path.join(
                        self.get_folder(),
                        "phi_" +
                        modality.name))

        is_hierarchial = True
        try:
            layers = artm_object._levels
        except BaseException:
            is_hierarchial = False

        if is_hierarchial and len(layers) == 1:
            is_hierarchial = False

        if is_hierarchial:
            self.log("Saving matrices psi...")
            for layer_id in range(1, self.layers_count):
                psi = layers[layer_id].get_psi().to_pickle(
                    os.path.join(self.get_folder(), "psi" + str(layer_id)))

    def gather_phi(self):
        self.log("Loading matrix phi...")
        phi_path = os.path.join(self.get_folder(), "phi")
        phi = None
        topics_count = 0
        if os.path.exists(phi_path):
            phi_raw = pd.read_pickle(os.path.join(self.get_folder(), "phi"))
            if self.dataset.check_terms_order(phi_raw.index):
                phi = phi_raw.values
                self.log("Matrix phi has correct index")
            else:
                self.log(("WARNING! Matrix phi has wrong index. Will restore. "
                          "In case of equal terms in different "
                          "modalitites there will be errors."))
                topics_count = phi_raw.shape[1]
                phi = np.zeros((self.dataset.terms_count, topics_count))
                self.log("Building terms index...")
                terms_index = self.dataset.get_terms_index()
                self.log("Terms index built.")

                ctr = 0
                for row in phi_raw.iterrows():
                    term_text = row[0]
                    try:
                        term_matrix_id = terms_index[term_text]
                    except BaseException:
                        try:
                            self.log(
                                "WARNING! Word " + term_text +
                                " don't belong to dataset dictionary.")
                        except BaseException:
                            self.log(("WARNING! Word (coludn't show) don't "
                                      "belong to dataset dictionary."))
                        continue
                    for j in range(topics_count):
                        phi[term_matrix_id][j] = row[1][j]

                    if ctr % 10000 == 0:
                        self.log(str(ctr))
                    ctr += 1
        else:
            self.log(("WARNING! Phi wasn't detected. "
                      "Will try load from matrices for modalities."))
            for modality in Modality.objects.filter(dataset=self.dataset):
                phi_path = os.path.join(
                    self.get_folder(), "phi_" + modality.name)
                if os.path.exists(phi_path):
                    self.log(
                        "Found matrix phi for modality " +
                        modality.name +
                        ". Will load.")
                    phi_raw = pd.read_pickle(phi_path)
                    terms_index = self.dataset.get_terms_index(
                        modality=modality)
                    if phi is None:
                        topics_count = phi_raw.shape[1]
                        phi = np.zeros(
                            (self.dataset.terms_count, topics_count))
                    if phi_raw.shape[1] != topics_count:
                        raise ValueError((
                            "Fatal error. "
                            "Matrices phi are of different width."))
                    for row in phi_raw.iterrows():
                        term_text = row[0]
                        if term_text in terms_index:
                            term_matrix_id = terms_index[term_text]
                        else:
                            self.log(
                                "WARNING! Word " +
                                term_text +
                                " don't belong to dataset dictionary.")
                            continue
                        for j in range(topics_count):
                            phi[term_matrix_id][j] = row[1][j]
                else:
                    self.log(
                        "Fatal error. Matrix for modality " +
                        modality.name +
                        " wasn't found.")

        if phi is None:
            raise ValueError("No phi matrix")

        self.log("Matrix phi loaded.")

        np.save(os.path.join(self.get_folder(), "phi.npy"), phi)
        self.log("Matrix phi saved in numpy format.")

    def gather_theta(self):
        self.log("Loading matrix theta...")
        theta_raw = pd.read_pickle(os.path.join(self.get_folder(), "theta"))
        self.theta_index = theta_raw.index

        if (1 in theta_raw
                and theta_raw.shape[1] == self.dataset.documents_count):
            self.log("Theta has correct index.")
            theta = theta_raw.sort_index(axis=1).values
        else:
            self.log("Will load theta column by column.")
            if theta_raw.shape[1] != self.dataset.documents_count:
                self.log("WARNING! Not all documents are present in matrix.")
            topics_count = theta_raw.shape[0]
            theta = np.zeros((topics_count, self.dataset.documents_count))
            for document in Document.objects.filter(dataset=self.dataset):
                if document.index_id in theta_raw:
                    column = theta_raw[document.index_id]
                elif document.text_id in theta_raw:
                    column = theta_raw[document.text_id]
                else:
                    self.log(
                        "WARNING! Document " +
                        document.text_id +
                        " wasn't found in matrix theta.")
                    continue

                doc_matrix_id = document.index_id
                for i in range(topics_count):
                    theta[i][doc_matrix_id] = column[i]

        self.log("Checking matrix theta...")
        ones = np.sum(theta, axis=0)
        if (np.max(ones) > 1 + 1e3 or np.min(ones) < 1 - 1e3):
            self.log(str(ones))
            raise ValueError("Fuck! Not stochastic!")

        self.log("Saving matrix theta...")
        np.save(os.path.join(self.get_folder(), "theta.npy"), theta)
        self.log("Matrix theta saved...")

        self.log("Counting topics probabilities using matrix theta")
        pt = np.sum(theta, axis=1) / self.dataset.documents_count
        np.save(os.path.join(self.get_folder(), "pt.npy"), pt)

    # Get probabilities of topics on certain layer
    def get_pt(self, layer=1):
        if layer == 0:
            return np.array([1])
        path = os.path.join(self.get_folder(), "pt.npy")
        if not os.path.exists(path):
            self.gather_theta()
        pt = np.load(path)
        ret = pt[self.get_layer_range(layer)]
        self.log("p(t): " + str(ret))
        return ret

    @transaction.atomic
    def build_hier(self):
        self.log("Building topics hierarchy")
        threshold_hier = self.threshold_hier / 100.0
        TopicInTopic.objects.filter(model=self).delete()
        topics_count = [int(x) for x in self.topics_count.split()]
        self.build_topics_index()

        # Adding topics of top layer as children of root
        for topic_id in self.topics_index[1]:
            relation = TopicInTopic()
            relation.model = self
            relation.parent_id = self.topics_index[0][0]
            relation.child_id = topic_id
            relation.save()

        pt = [self.get_pt(layer=layer)
              for layer in range(self.layers_count + 1)]

        for btm_layer in range(2, self.layers_count + 1):
            top_layer = btm_layer - 1
            psi = self.get_psi(top_layer)
            self.log(
                "Building topics hierarchy between layers %d and %d" %
                (top_layer, btm_layer))
            for bottom_topic_id in range(topics_count[btm_layer]):
                # p = (psi[bottom_topic_id] *
                # pt[top_layer])/pt[btm_layer][bottom_topic_id] #
                # Conditional probabilities(parent_topic | child_topic)
                p = psi[bottom_topic_id]
                best_top_topic_id = np.argmax(p)
                relation = TopicInTopic()
                relation.model = self
                relation.parent_id = \
                    self.topics_index[top_layer][best_top_topic_id]
                relation.child_id = \
                    self.topics_index[btm_layer][bottom_topic_id]
                relation.weight = p[best_top_topic_id]
                relation.is_main = True
                relation.save()

                if threshold_hier <= 0.5 and self.max_parents_hier > 1:
                    pot_parents = np.argsort(p)[-self.max_parents_hier:]
                    for top_topic_id in np.argsort(p)[-self.max_parents_hier:]:
                        if (p[top_topic_id] > threshold_hier and
                                top_topic_id != best_top_topic_id):
                            relation = TopicInTopic()
                            relation.model = self
                            relation.parent_id = \
                                self.topics_index[top_layer][top_topic_id]
                            relation.child_id = \
                                self.topics_index[btm_layer][bottom_topic_id]
                            relation.weight = p[top_topic_id]
                            relation.is_main = False
                            relation.save()

    def extract_docs(self, layer=-1):
        if layer == -1:
            for i in range(1, self.layers_count + 1):
                self.extract_docs(layer=i)
            return

        self.log("Extracting documents in topics for layer %d..." % layer)
        threshold_docs = self.threshold_docs / 100.0
        topics_count = [int(x) for x in self.topics_count.split()]
        total_topics_count = sum(topics_count) - 1
        theta_t = self.get_theta().transpose()[:, self.get_layer_range(layer)]
        document_bags = [[] for i in range(topics_count[self.layers_count])]
        for doc_index_id in range(0, self.dataset.documents_count):
            # doc_id = documents_index[doc_index_id].id
            distr = theta_t[doc_index_id]
            best_topic_id = distr.argmax()

            document_bags[best_topic_id].append(
                (distr[best_topic_id], doc_index_id))
            if threshold_docs <= 0.5:
                for topic_id in range(topics_count[self.layers_count]):
                    if (distr[topic_id] > threshold_docs and
                            topic_id != best_topic_id):
                        document_bags[topic_id].append(
                            distr[topic_id], doc_index_id)

            if doc_index_id % 1000 == 0:
                self.log(str(doc_index_id))

        self.log("Building topics index...")

        self.log("Saving topics...")
        for topic in Topic.objects.filter(
                model=self, layer=layer).order_by("index_id"):
            topic.documents = bytes()
            document_bags[topic.index_id].sort(reverse=True)
            for weight, doc_index_id in document_bags[topic.index_id]:
                topic.documents += \
                    struct.pack('I', doc_index_id) + \
                    struct.pack('f', weight)
            topic.documents_count = len(topic.documents) // 8
            topic.save()

    def build_topics_index(self):
        self.topics_index = [0 for i in range(self.layers_count + 1)]

        for layer_id in range(0, self.layers_count + 1):
            self.topics_index[layer_id] = [
                topic.id for topic in Topic.objects.filter(
                    model=self, layer=layer_id).order_by("index_id")]

    @transaction.atomic
    def reload(self):
        vocab_file = os.path.join(
            settings.DATA_DIR,
            "datasets",
            self.dataset.text_id,
            "UCI",
            "vocab." +
            self.dataset.text_id +
            ".txt")
        self.prepare_log()
        self.log("Reloading model " + str(self.id) + "...")

        rmtree(self.get_visual_folder())
        rmtree(self.get_dist_folder())

        # Loading matrices
        self.gather_phi()
        phi = self.get_phi()
        phi_t = phi.transpose()

        self.gather_theta()
        theta = self.get_theta()

        self.layers_count = 1
        psi = [0]
        for i in range(1, 100):
            path = os.path.join(self.get_folder(), "psi" + str(i))
            if os.path.exists(path):
                self.log("Loading matrix psi" + str(i))
                psi.append(pd.read_pickle(path).values)
                np.save(os.path.join(path + ".npy"), psi[i])
                self.layers_count = i + 1
            else:
                break

        self.log("Counting topics...")
        if self.layers_count == 1:
            self.topics_count = "1 " + str(theta.shape[0])
        else:
            self.topics_count = "1 " + str(psi[1].shape[1])
            for layer_id in range(1, self.layers_count):
                self.topics_count += " " + str(psi[layer_id].shape[0])
        self.log("Topics number: " + self.topics_count)

        terms_count = self.dataset.terms_count
        documents_count = self.dataset.documents_count
        topics_count = [int(x) for x in self.topics_count.split()]
        total_topics_count = sum(topics_count) - 1

        # Extracting topic names from theta index
        topic_names = [[] for i in range(self.layers_count + 1)]
        offset = 0
        for layer_id in range(1, self.layers_count + 1):
            topic_names[layer_id] = self.theta_index[offset: offset +
                                                     topics_count[layer_id]]
            offset += topics_count[layer_id]

        # Building temporary index for terms
        self.log("Building temporary index for words...")
        terms_index = Term.objects.filter(
            dataset=self.dataset).order_by("index_id")
        terms_id_index = [term.id for term in terms_index]

        # Removing existing topics and related objects
        from visual.models import GlobalVisualization
        Topic.objects.filter(model=self).delete()

        GlobalVisualization.objects.filter(model=self).delete()

        # Creating topics, loading top terms, topic labeling
        self.log("Creating topics...")

        # Creating root topic
        root_topic = Topic()
        root_topic.model = self
        root_topic.index_id = 0
        root_topic.title = "root"
        root_topic.layer = 0
        root_topic.save()

        top_terms_size = 100

        banned_words = set(BANNED_WORDS)
        title_size = self.topic_naming_top_words
        terms_weights = self.dataset.get_terms_weights("naming")

        # Number of unique terms in each document
        # TODO: cache this value
        nd = np.array([doc.terms_count for doc in Document.objects.filter(
            dataset=self.dataset).order_by("index_id")])

        # Total number of unique words in collection
        n = np.sum(nd)

        row_counter = 0
        for layer_id in range(1, self.layers_count + 1):
            for topic_id in range(topics_count[layer_id]):
                topic = Topic()
                topic.model = self
                topic.index_id = topic_id
                topic.layer = layer_id
                topic.matrix_id = row_counter
                topic.probability = np.dot(theta[row_counter], nd) / n
                topic.save()

                # Naming and top words extracting
                distr = phi_t[row_counter]
                idx = np.argsort(distr)
                idx = idx[::-1]
                mc = self.dataset.modalities_count
                top_terms_counter = dict()

                for i in idx:
                    weight = distr[i]
                    if weight == 0:
                        break
                    term = terms_index[int(i)]
                    mid = term.modality_id
                    if mid not in top_terms_counter:
                        top_terms_counter[mid] = 0
                    if top_terms_counter[mid] < top_terms_size:
                        relation = TopTerm()
                        relation.topic = topic
                        relation.term = term
                        relation.weight = weight
                        relation.weight_normed = weight * terms_weights[i]
                        relation.save()

                        top_terms_counter[mid] += 1
                        if top_terms_counter[mid] == top_terms_size:
                            mc -= 1
                        if mc == 0:
                            break

                if 'topic' in topic_names[layer_id][topic_id]:
                    terms_to_title = []
                    title_counter = 0
                    idx = np.argsort(phi_t[row_counter] * terms_weights)
                    idx = idx[::-1]
                    for i in idx:
                        term = terms_index[int(i)]
                        if title_counter < title_size:
                            if term.text not in banned_words:
                                title_counter += 1
                                terms_to_title.append(term.text)
                        else:
                            break
                    topic.title = ', '.join(terms_to_title)
                    topic.title_multiline = '\n'.join(terms_to_title)
                else:
                    topic.title = topic_names[layer_id][topic_id]
                    topic.title_multiline = '\n'.join(topic.title.split())

                topic.title_short = topic.title[0:20]
                topic.save()

                row_counter += 1
                if row_counter % 10 == 0:
                    self.log(
                        "Created topic %d/%d." %
                        (row_counter, total_topics_count))

        self.build_hier()
        self.extract_docs()

        self.creation_time = datetime.now()
        self.arrange_topics()
        self.status = 0
        self.save()
        self.log("Model " + str(self.id) + " reloaded.")

    def reload_untrusted(self):
        try:
            self.reload()
        except BaseException:
            self.error_message = traceback.format_exc()
            self.status = 2
            self.save()

    def prepare_log(self, string=""):
        if len(self.text_id) == 0:
            self.text_id = "model_" + str(self.id)
        self.log_file_name = os.path.join(self.get_folder(), "log.txt")
        with open(self.log_file_name, "w") as f:
            f.write("%s
\n" % string)
        self.save()

    def log(self, string):
        if settings.DEBUG:
            print(string)
        if settings.THREADING:
            try:
                with open(self.log_file_name, "a") as f:
                    f.write(string + "
\n")
            except BaseException:
                pass

    def read_log(self):
        try:
            with open(os.path.join(self.get_folder(), "log.txt"), "r") as f:
                return f.read()
        except BaseException:
            return "Model is being processed..."

    def get_topics(self, layer=-1):
        if layer == -1:
            layer = self.layers_count
        return Topic.objects.filter(
            model=self, layer=layer).order_by("index_id")

    def get_related_topics(self, topic, metric="default"):
        metric = metrics.get_metric_by_name(metric)
        topics_index = Topic.objects.filter(
            model=self, layer=topic.layer).order_by("index_id")

        phi_t = self.get_phi_t_norm(topic.layer)
        target_row = phi_t[topic.index_id]
        distances = [metric(target_row, row) for row in phi_t]
        idx = np.argsort(distances)
        return [
            {"distance": distances[int(i)],
             "topic": topics_index[int(i)]}
            for i in idx]

    def get_layer_range(self, layer):
        topics_count = [int(x) for x in self.topics_count.split()]
        shift = 0
        for i in range(1, layer):
            shift += topics_count[i]
        return range(shift, shift + topics_count[layer])

    def get_topics_distances(self, metric="default", layer=-1):
        if layer == -1:
            layer = self.layers_count
        if layer == 0:
            return np.zeros((1, 1))

        matrix_name = os.path.join(
            self.get_dist_folder(),
            metric + "_" + str(layer) + ".npy")
        try:
            ret = np.load(matrix_name)
        except BaseException:
            topics_count = [int(x) for x in self.topics_count.split()]
            ret = np.zeros((topics_count[layer], topics_count[layer]))

            phi_t = self.get_phi_t_norm(layer)

            metric = metrics.get_metric_by_name(metric)

            for i in range(topics_count[layer]):
                for j in range(topics_count[layer]):
                    ret[i][j] = metric(phi_t[i], phi_t[j])
            np.save(matrix_name, ret)
        return ret

    def delete_cached_distances(self):
        rmtree(self.get_dist_folder())
        if hasattr(self, "phi_t_norm"):
            del self.phi_t_norm

    # Only horizontal arranging
    @transaction.atomic
    def arrange_topics(self, mode="default", metric="default", beta=0.8):
        # Counting horizontal relations topic-topic
        self.log("Counting horizontal relations topic-topic...")
        self.topics_index = [
            [topic for topic
             in Topic.objects.filter(model=self, layer=i).order_by("index_id")]
            for i in range(self.layers_count + 1)]

        if metric == "default":
            metric = metrics.default_metric

        TopicRelated.objects.filter(model=self).delete()
        self.topic_distances = [
            self.get_topics_distances(
                metric=metric,
                layer=i) for i in range(
                self.layers_count +
                1)]

        for layer_id in range(1, self.layers_count + 1):
            layer_size = self.get_layer_size(layer_id)
            for i in range(layer_size):
                idx = np.argsort(self.topic_distances[layer_id][i])
                for j in idx[1: 1 + min(10, layer_size - 1)]:
                    relation = TopicRelated()
                    relation.model = self
                    relation.topic1 = self.topics_index[layer_id][i]
                    relation.topic2 = self.topics_index[layer_id][j]
                    relation.weight = self.topic_distances[layer_id][i][j]
                    relation.save()

        topic_hier_relations = TopicInTopic.objects.filter(model=self)

        if mode == "default":
            if self.layers_count == 1:
                mode = "index"
            else:
                mode = "hierarchical"

        if mode == "hierarchical":
            self.arrange_topics_hierarchical(beta=beta)
        else:
            # Building topics spectrum
            for layer_id in range(1, self.layers_count + 1):
                layer_size = self.get_layer_size(layer_id)

                self.log(
                    ("Building topics spectrum for layer %d, "
                     "mode=%s, metric=%s...") % (layer_id, mode, metric))

                # Default - no arranging.
                idx = np.array(range(layer_size))

                if mode == "alphabet":
                    # Topics are sorted by alphabet.
                    titles = [
                        self.topics_index[layer_id][topic_id].title
                        for topic_id in range(0, layer_size)]
                    idx = np.argsort(titles)
                elif mode == "hamilton":
                    # Topics are arranged so semantically close topics are
                    # nearby (by solving Travelling Salesman Problem).
                    from algo.arranging.base import get_arrangement_permutation
                    idx = get_arrangement_permutation(
                        self.topic_distances[layer_id], mode, model=self)

                for i in range(self.get_layer_size(layer_id)):
                    topic = self.topics_index[layer_id][idx[i]]
                    topic.spectrum_index = i
                    topic.save()

        self.status = 0
        self.arrangement_mode = mode
        self.metric = metric
        self.save()

        self.log("Resetting visualizations...")
        self.reset_visuals()

    def arrange_topics_hierarchical(self, beta=0.8, cross_min_mode="auto"):
        if self.layers_count == 1:
            raise ValueError("Model is flat!")

        # Arrange lower level, minimizing NDS
        lower_layer = self.layers_count
        dist = self.topic_distances[lower_layer]

        # Correcting matrix

        high_topics = Topic.objects.filter(model=self, layer=lower_layer - 1)
        pairs = set()
        for topic in high_topics:
            close_topics = [
                t.child.index_id for t in TopicInTopic.objects.filter(
                    parent=topic)]
            for i in close_topics:
                for j in close_topics:
                    pairs.add((i, j))

        for i, j in pairs:
            dist[i][j] *= beta

        # Arranging lower level
        from algo.arranging.base import get_arrangement_permutation
        idx = get_arrangement_permutation(dist, mode="hamilton", model=self)

        for i in range(self.get_layer_size(lower_layer)):
            topic = self.topics_index[lower_layer][idx[i]]
            topic.spectrum_index = i
            topic.save()

        # Arranging higher layers, using mass center principle
        from algo.arranging.crossmin import CrossMinimizer
        for top_layer_id in range(self.layers_count - 1, 0, -1):
            N1 = self.get_layer_size(top_layer_id)
            N2 = self.get_layer_size(top_layer_id + 1)

            A = np.zeros((N1, N2))
            for i in range(N1):
                topic = self.topics_index[top_layer_id][i]
                for rel in TopicInTopic.objects.filter(parent=topic):
                    A[i][rel.child.spectrum_index] = 1

            cm = CrossMinimizer(A)
            idx = cm.solve(mode=cross_min_mode, model=self)

            for i in range(self.get_layer_size(top_layer_id)):
                topic = self.topics_index[top_layer_id][idx[i]]
                topic.spectrum_index = i
                topic.save()

    # layer must be lower level of opair of interest
    def spectrum_crosses_count(self, layer=-1):
        if layer == -1:
            layer = self.layers_count

        links = []
        ans = 0

        for topic in self.get_topics(layer=layer - 1):
            links += [(topic.spectrum_index, rel.child.spectrum_index)
                      for rel in TopicInTopic.objects.filter(parent=topic)]

        for link1 in links:
            for link2 in links:
                if link2[0] > link1[0] and link2[1] < link1[1]:
                    ans += 1
        return ans

    def neighbor_distance_sum(self, metric="default", layer=-1):
        if layer == -1:
            layer = self.layers_count

        perm = [t.index_id for t in Topic.objects.filter(
            model=self, layer=layer).order_by('spectrum_index')]
        dist = self.get_topics_distances(layer=layer, metric=metric)

        return sum([dist[perm[i]][perm[i + 1]] for i in range(len(perm) - 1)])

    def get_folder(self):
        path = os.path.join(
            settings.DATA_DIR,
            "datasets",
            self.dataset.text_id,
            "models",
            self.text_id)
        if not os.path.exists(path):
            os.makedirs(path)
        return path

    def get_visual_folder(self):
        path = os.path.join(
            settings.DATA_DIR,
            "datasets",
            self.dataset.text_id,
            "models",
            self.text_id,
            "visual")
        if not os.path.exists(path):
            os.makedirs(path)
        return path

    def get_dist_folder(self):
        path = os.path.join(
            settings.DATA_DIR,
            "datasets",
            self.dataset.text_id,
            "models",
            self.text_id,
            "dist")
        if not os.path.exists(path):
            os.makedirs(path)
        return path

    def get_phi(self):
        return np.load(os.path.join(self.get_folder(), "phi.npy"))

    def get_theta(self):
        try:
            return self.theta
        except BaseException:
            self.theta = np.load(os.path.join(self.get_folder(), "theta.npy"))
            return self.theta

    # Return phi transposed and normalized by modalities weights, for distance
    # counting.
    def get_phi_t_norm(self, layer):
        if not hasattr(self, "phi_t_norm"):
            self.phi_t_norm = dict()
        if layer not in self.phi_t_norm:
            phi_t_norm = self.get_phi().transpose()[
                self.get_layer_range(layer)]

            if self.dataset.modalities_count > 1:
                self.log("Normalizing phi according to modalitites weights...")
                phi_t_norm = phi_t_norm * \
                    self.dataset.get_terms_weights("spectrum")
                self.log("Normalized.")
                sums = np.sum(phi_t_norm, axis=1)
                if np.min(sums) < 0.95 or np.max(sums) > 1.05:
                    print(sums)
                    raise RuntimeError("Phi is not stochastic!")
                # print(sums)
            self.phi_t_norm[layer] = phi_t_norm
        return self.phi_t_norm[layer]

    def get_theta_t(self):
        try:
            return self.theta_t
        except BaseException:
            self.theta_t = self.get_theta().transpose()
            return self.theta_t

    def get_psi(self, i):
        return np.load(
            os.path.join(
                self.get_folder(),
                "psi" +
                str(i) +
                ".npy"))

    def lower_topics_count(self):
        return int(self.topics_count.split()[-1])

    def get_layer_size(self, layer):
        return int(self.topics_count.split()[layer])

    # Groups documents into matrix Dates-Topics.
    # Elements of matrix are absolute ids of documents.
    # Topics are ordered in order of spectrum_index.
    # Return tuple - matrix, dates.
    # For hierarchical models - topics are taken from the lowest level.
    def group_matrix(self, group_by="day", named_groups=False):
        from algo.utils.date_namer import DateNamer
        # from models.models import Topic
        documents = Document.objects.filter(dataset=self.dataset)
        topics = Topic.objects.filter(
            model=self, layer=self.layers_count).order_by("spectrum_index")
        topics_count = len(topics)
        dn = DateNamer(group_by=group_by, lang=self.dataset.language)

        dates_hashes = set()
        for document in documents:
            dates_hashes.add(dn.date_hash(document.time))
        dates_hashes = sorted(dates_hashes)
        dates = []
        dates_reverse_index = dict()

        dates_count = 0
        for date_h in dates_hashes:
            dates_reverse_index[date_h] = dates_count
            if named_groups:
                dates.append(dn.date_name(date_h))
            else:
                dates.append(dn.hash_date(date_h))
            dates_count += 1

        cells = [[[] for j in range(topics_count)] for i in range(dates_count)]

        for topic in topics:
            y = topic.spectrum_index
            for document in topic.get_documents():
                x = dates_reverse_index[dn.date_hash(document.time)]
                cells[x][y].append(document.id)

        return cells, dates

    # Returns list of related documents to the document with given index_id
    # (sorted from most related)
    def get_related_documents(
            self,
            document_index_id,
            count=20,
            metric="euclidean"):
        documents_count = self.dataset.documents_count
        documents_index = Document.objects.filter(
            dataset=self.dataset).order_by("index_id")
        dist = np.zeros(documents_count)
        theta_t = self.get_theta_t()
        self_distr = theta_t[document_index_id]
        metric = metrics.get_metric_by_name(metric)
        for other_document_id in range(0, documents_count):
            dist[other_document_id] = metric(
                self_distr, theta_t[other_document_id])

        idx = np.argsort(dist)[1: count + 1]
        return [documents_index[int(i)] for i in idx]

    def segmentation_available(self, document):
        return os.path.exists(
            os.path.join(
                self.get_folder(),
                "segmentation",
                document.text_id))

    def get_segmentation(self, document):
        try:
            file_name = os.path.join(self.get_folder(),
                                     "segmentation", document.text_id)
            with open(file_name) as f:
                return json.loads(f.read())["selections"]
        except BaseException:
            return None

    def reset_visuals(self):
        from visual.models import GlobalVisualization
        GlobalVisualization.objects.filter(model=self).delete()

def on_start():
    for model in ArtmModel.objects.filter(status=1):
        model.status = 2
        model.error_message = "Model processing was interrupted."
        model.save()

    for model in ArtmModel.objects.filter(status=11):
        model.status = 0
        model.save()

@receiver(pre_delete, sender=ArtmModel, dispatch_uid='artmmodel_delete_signal')
def remove_model_files(sender, instance, using, **kwargs):
    if settings.DEBUG:
        return
    try:
        rmtree(folder)
    except BaseException:
        pass

class Topic(models.Model):
    model = models.ForeignKey(ArtmModel, null=False)
    index_id = models.IntegerField(null=True)  # in layer
    matrix_id = models.IntegerField(null=True)  # In matrix
    title = models.TextField(null=False)
    title_short = models.TextField(null=True)
    spectrum_index = models.IntegerField(null=True, default=0)
    layer = models.IntegerField(default=1)
    # [4 bytes - document.index_id][4 bytes - weight]
    documents = models.BinaryField(null=True)
    documents_count = models.IntegerField(null=False, default=0)
    # 'a prior' probability of topic, p(t)
    probability = models.FloatField(null=False, default=1)

    def __str__(self):
        return self.title

    def rename(self, new_title):
        self.title = new_title
        self.title_short = new_title[0:30]
        self.save()

    def get_documents(self):
        all_documents = Document.objects.filter(dataset=self.model.dataset)
        for i in range(self.documents_count):
            doc_index_id = struct.unpack(
                'I', self.documents[8 * i: 8 * i + 4])[0]
            yield all_documents.get(index_id=doc_index_id)

    def get_documents_index_ids(self):
        return [struct.unpack('I', self.documents[8 * i: 8 * i + 4])[0]
                for i in range(self.documents_count)]

    def top_words_html(self, count=10):
        ret = ""
        tw = self.top_words(count=count)
        i = 0
        for w in tw:
            ret += w + " "
            i += 1
            if i % 3 == 0:
                ret += "
"
        return ret

    def top_words_list(self, count=10):
        return ', '.join(self.top_words(count=count))

    def top_words(self, count=10):
        # banned_words = set(BANNED_WORDS)
        ret = [x.term.text for x in TopTerm.objects.filter(
            topic=self).order_by('-weight_normed')[0:count]]
        # print(ret)
        return ret
        # return [x for x in ret if not x in banned_words][0:count]

class TopicInDocument(models.Model):
    model = models.ForeignKey(ArtmModel, null=False)
    document = models.ForeignKey(Document, null=False)
    topic = models.ForeignKey(Topic, null=False)
    probability = models.FloatField(default=0)

    def __str__(self):
        return str(self.topic) + " " + \
            "{0:.1f}%".format(100 * self.probability)

class TopicInTopic(models.Model):
    model = models.ForeignKey(ArtmModel, null=False)
    parent = models.ForeignKey(Topic, null=False, related_name='parent')
    child = models.ForeignKey(Topic, null=False, related_name='child')
    weight = models.FloatField(default=0)
    is_main = models.BooleanField(default=True)

class TopicRelated(models.Model):
    model = models.ForeignKey(ArtmModel, null=False)
    topic1 = models.ForeignKey(Topic, null=False, related_name='fk1')
    topic2 = models.ForeignKey(Topic, null=False, related_name='fk2')
    weight = models.FloatField()

    def __str__(self):
        return str(self.topic2) + "{0:.1f}%".format(100 * self.weight)

class TopTerm(models.Model):
    topic = models.ForeignKey(Topic)
    term = models.ForeignKey(Term)
    weight = models.FloatField(default=0)
    weight_normed = models.FloatField(default=0)

class TopicInTerm(models.Model):
    model = models.ForeignKey(ArtmModel, null=False)
    topic = models.ForeignKey(Topic, null=False)
    term = models.ForeignKey(Term, null=False)
    weight = models.FloatField()

    def __str__(self):
        return str(self.term) + " " + str(self.topic) + \
            "{0:.1f}%".format(100 * self.weight)

admin.site.register(ArtmModel)
admin.site.register(Topic)

ext: py

max_stars_repo_name: alexeyev/visartm

max_issues_repo_name: alexeyev/visartm

max_forks_repo_name: alexeyev/visartm

reason: import numpy


text:
# Copyright (c) OpenMMLab. All rights reserved.
import functools
import operator

import cv2
import numpy as np
import pyclipper
from numpy.fft import ifft
from numpy.linalg import norm
from shapely.geometry import Polygon

def points2boundary(points, text_repr_type, text_score=None, min_width=-1):
    """Convert a text mask represented by point coordinates sequence into a
    text boundary.
    Args:
        points (ndarray): Mask index of size (n, 2).
        text_repr_type (str): Text instance encoding type
            ('quad' for quadrangle or 'poly' for polygon).
        text_score (float): Text score.
    Returns:
        boundary (list[float]): The text boundary point coordinates (x, y)
            list. Return None if no text boundary found.
    """
    assert isinstance(points, np.ndarray)
    assert points.shape[1] == 2
    assert text_repr_type in ['quad', 'poly']
    assert text_score is None or 0 <= text_score <= 1

    if text_repr_type == 'quad':
        rect = cv2.minAreaRect(points)
        vertices = cv2.boxPoints(rect)
        boundary = []
        if min(rect[1]) > min_width:
            boundary = [p for p in vertices.flatten().tolist()]

    elif text_repr_type == 'poly':

        height = np.max(points[:, 1]) + 10
        width = np.max(points[:, 0]) + 10

        mask = np.zeros((height, width), np.uint8)
        mask[points[:, 1], points[:, 0]] = 255

        contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL,
                                       cv2.CHAIN_APPROX_SIMPLE)
        boundary = list(contours[0].flatten().tolist())

    if text_score is not None:
        boundary = boundary + [text_score]
    if len(boundary) < 8:
        return None

    return boundary

def box_score_fast(bitmap, _box):
    h, w = bitmap.shape[:2]
    box = _box.copy()
    xmin = np.clip(np.floor(box[:, 0].min()).astype(np.int32), 0, w - 1)
    xmax = np.clip(np.ceil(box[:, 0].max()).astype(np.int32), 0, w - 1)
    ymin = np.clip(np.floor(box[:, 1].min()).astype(np.int32), 0, h - 1)
    ymax = np.clip(np.ceil(box[:, 1].max()).astype(np.int32), 0, h - 1)

    mask = np.zeros((ymax - ymin + 1, xmax - xmin + 1), dtype=np.uint8)
    box[:, 0] = box[:, 0] - xmin
    box[:, 1] = box[:, 1] - ymin
    cv2.fillPoly(mask, box.reshape(1, -1, 2).astype(np.int32), 1)
    return cv2.mean(bitmap[ymin:ymax + 1, xmin:xmax + 1], mask)[0]

def unclip(box, unclip_ratio=1.5):
    poly = Polygon(box)
    distance = poly.area * unclip_ratio / poly.length
    offset = pyclipper.PyclipperOffset()
    offset.AddPath(box, pyclipper.JT_ROUND, pyclipper.ET_CLOSEDPOLYGON)
    expanded = np.array(offset.Execute(distance))
    return expanded

max_stars_repo_name: tabsun/DBSegFormer

max_issues_repo_name: tabsun/DBSegFormer

max_forks_repo_name: tabsun/DBSegFormer

reason: import numpy,from numpy


text:
#!/usr/bin/env python

# Copyright (c) 2013, Carnegie Mellon University
# All rights reserved.
# Authors: Michael Koval <mkoval@cs.cmu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# - Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
# - Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# - Neither the name of Carnegie Mellon University nor the names of its
#   contributors may be used to endorse or promote products derived from this
#   software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import numpy
import openravepy
from openravepy import Robot
from prpy.util import SetTrajectoryTags
from prpy.planning.base import (
    Planner,
    PlanningError,
    LockedPlanningMethod,
    Tags
)
from ..collision import DefaultRobotCollisionCheckerFactory

class SnapPlanner(Planner):
    """Planner that checks the straight-line trajectory to the goal.

    SnapPlanner is a utility planner class that collision checks the
    straight-line trajectory to the goal. If that trajectory is invalid,
    e.g. due to an environment or self collision, the planner
    immediately returns failure by raising a PlanningError.

    SnapPlanner is intended to be used only as a "short circuit" to
    speed-up planning between nearby configurations. This planner is
    most commonly used as the first item in a Sequence meta-planner to
    avoid calling a motion planner when the trivial solution is valid.
    """
    def __init__(self, robot_checker_factory=None):
        super(SnapPlanner, self).__init__()

        if robot_checker_factory is None:
            robot_checker_factory = DefaultRobotCollisionCheckerFactory

        self.robot_checker_factory = robot_checker_factory

    def __str__(self):
        return 'SnapPlanner'

    @LockedPlanningMethod
    def PlanToConfiguration(self, robot, goal, **kw_args):
        """
        Attempt to plan a straight line trajectory from the robot's
        current configuration to the goal configuration. This will
        fail if the straight-line path is not collision free.

        @param robot
        @param goal desired configuration
        @return traj
        """
        return self._Snap(robot, goal, **kw_args)

    def _Snap(self, robot, goal, **kw_args):
        from prpy.util import CheckJointLimits
        from prpy.util import GetLinearCollisionCheckPts
        from prpy.planning.exceptions import CollisionPlanningError
        from prpy.planning.exceptions import SelfCollisionPlanningError

        # Create a two-point trajectory between the
        # current configuration and the goal.
        # (a straight line in joint space)
        env = robot.GetEnv()
        traj = openravepy.RaveCreateTrajectory(env, '')
        cspec = robot.GetActiveConfigurationSpecification('linear')
        active_indices = robot.GetActiveDOFIndices()

        # Check the start position is within joint limits,
        # this can throw a JointLimitError
        start = robot.GetActiveDOFValues()
        CheckJointLimits(robot, start, deterministic=True)

        # Add the start waypoint
        start_waypoint = numpy.zeros(cspec.GetDOF())
        cspec.InsertJointValues(start_waypoint, start, robot,
                                active_indices, False)
        traj.Init(cspec)
        traj.Insert(0, start_waypoint.ravel())

        # Make the trajectory end at the goal configuration, as
        # long as it is not in collision and is not identical to
        # the start configuration.
        CheckJointLimits(robot, goal, deterministic=True)
        if not numpy.allclose(start, goal):
            goal_waypoint = numpy.zeros(cspec.GetDOF())
            cspec.InsertJointValues(goal_waypoint, goal, robot,
                                    active_indices, False)
            traj.Insert(1, goal_waypoint.ravel())

        # Get joint configurations to check
        # Note: this returns a python generator, and if the
        # trajectory only has one waypoint then only the
        # start configuration will be collisioned checked.
        #
        # Sampling function:
        # 'linear'
        # from prpy.util import SampleTimeGenerator
        # linear = SampleTimeGenerator
        # 'Van der Corput'
        from prpy.util import VanDerCorputSampleGenerator
        vdc = VanDerCorputSampleGenerator

        checks = GetLinearCollisionCheckPts(robot, traj,
                                            norm_order=2,
                                            sampling_func=vdc)

        with self.robot_checker_factory(robot) as robot_checker, \
            robot.CreateRobotStateSaver(Robot.SaveParameters.LinkTransformation):
            # Run constraint checks at DOF resolution:
            for t, q in checks:
                # Set the joint positions
                # Note: the planner is using a cloned 'robot' object
                robot.SetActiveDOFValues(q)

                # Check collision (throws an exception on collision)
                robot_checker.VerifyCollisionFree()

        SetTrajectoryTags(traj, {
            Tags.SMOOTH: True,
            Tags.DETERMINISTIC_TRAJECTORY: True,
            Tags.DETERMINISTIC_ENDPOINT: True,
        }, append=True)

        return traj

max_stars_repo_name: MisoRobotics/prpy

max_issues_repo_name: MisoRobotics/prpy

max_forks_repo_name: MisoRobotics/prpy


text:
import cv2
import matplotlib.pyplot as plt
import numpy as np

# All possible color flags
flags = [i for i in dir(cv2) if i.startswith('COLOR_')]

image = cv2.imread('../figures/coloredB-ST1paper.png')
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
plt.imshow(image)
plt.show()


text:
#!/usr/bin/python3
'''
Abstract:
    This is a program for rearrange labels.
Usage:
    arrange_label.py [data] [selector]
Editor:
    Jacob975

##################################
#   Python3                      #
#   This code is made in python3 #
##################################

20181030
####################################
update log
'''
import numpy as np
import time
from sys import argv

#--------------------------------------------
# Main code
if __name__ == "__main__":
    VERBOSE = 0
    # Measure time
    start_time = time.time()
    #-----------------------------------
    # Load argv
    if len(argv) != 3:
        print ('Error')
        print ('The number of arguments is wrong.')
        print ('Usage: arrange_label.py [data] [selector]')
        exit()
    data_name = argv[1]
    selector_name = argv[2]
    #-----------------------------------
    # Load data
    data = np.loadtxt(data_name)
    selector = np.loadtxt(selector_name, dtype = int)
    sub_names = ['star', 'gala', 'ysos']
    for i in range(3):
        sub_selector = np.where(selector == i)
        sub_data = data[sub_selector]
        np.savetxt('{0}_{1}.txt'.format(data_name[:-4], sub_names[i]), sub_data)
    #-----------------------------------
    # Measure time
    elapsed_time = time.time() - start_time
    print("Exiting Main Program, spending ", elapsed_time, "seconds.")


text:
import pygame
from math import sin, cos, pi, radians, degrees, copysign
from pygame.math import Vector2
from pygame.locals import *
import math
import numpy as np
import time
import random

from sensors import Sensors
from action import Action

class HorizontalPad(pygame.sprite.Sprite):
    normal = pygame.image.load('images/race_pads.png')
    hit = pygame.image.load('images/collision.png')

    def __init__(self, position):
        super(HorizontalPad, self).__init__()
        self.rect = pygame.Rect(self.normal.get_rect())
        self.rect.center = position
        self.image = self.normal

    def update(self, hit_list):
        if self in hit_list:
            self.image = self.hit
        else:
            self.image = self.normal

class VerticalPad(pygame.sprite.Sprite):
    normal = pygame.image.load('images/vertical_pads.png')

    def __init__(self, position):
        super(VerticalPad, self).__init__()
        self.rect = pygame.Rect(self.normal.get_rect())
        self.rect.center = position
        self.image = self.normal

class SmallHorizontalPad(pygame.sprite.Sprite):
    normal = pygame.image.load('images/small_horizontal.png')

    def __init__(self, position):
        super(SmallHorizontalPad, self).__init__()
        self.rect = pygame.Rect(self.normal.get_rect())
        self.rect.center = position
        self.image = self.normal

class SmallVerticalPad(pygame.sprite.Sprite):
    normal = pygame.image.load('images/small_vertical.png')

    def __init__(self, position):
        super(SmallVerticalPad, self).__init__()
        self.rect = pygame.Rect(self.normal.get_rect())
        self.rect.center = position
        self.image = self.normal

class CarSprite(pygame.sprite.Sprite):
    TURN_SPEED = 10

    def __init__(self, image, position):
        pygame.sprite.Sprite.__init__(self)
        self.src_image = pygame.image.load(image)
        self.position = position
        self.speed = 5
        self.direction = 0
        self.k_left = self.k_right = 0
        self.crashed = False

    def restart_position(self, position):
        self.position = position
        self.speed = 5
        self.direction = 0
        self.k_left = self.k_right = 0
        self.crashed = False

    def update(self, deltat):
        # SIMULATION
        self.direction += (self.k_right + self.k_left)
        x, y = self.position
        rad = self.direction * pi / 180
        x += -self.speed * sin(rad)
        y += -self.speed * cos(rad)
        self.position = (x, y)
        self.image = pygame.transform.rotate(self.src_image, self.direction)
        self.rect = self.image.get_rect()
        self.rect.center = self.position

pads = [
    VerticalPad((10, 610)),
    HorizontalPad((30, 350)),
    SmallVerticalPad((200, 626)),
    SmallHorizontalPad((314, 500)),
    SmallVerticalPad((270, 220)),
    SmallVerticalPad((425, 380)),
    HorizontalPad((532, 105)),
    SmallHorizontalPad((540, 265)),
    SmallVerticalPad((790, 220)),
    SmallVerticalPad((650, 400)),
    SmallHorizontalPad((788, 512)),
    SmallHorizontalPad((928, 332)),
    SmallHorizontalPad((1012, 332)),
    SmallVerticalPad((925, 625)),
    VerticalPad((1123, 584)),
    VerticalPad((1123, 614)),
    HorizontalPad((680, 740)),
    HorizontalPad((440, 740)),
    HorizontalPad((240, 870)),
    SmallHorizontalPad((550, 870)),
    HorizontalPad((885, 870))
]

pads2 = [
    # outer layer
    # SmallHorizontalPad((1400, 2000)),
    VerticalPad((10, 610)),
    VerticalPad((10, 280)),
    HorizontalPad((250, 40)),
    HorizontalPad((595, 40)),
    HorizontalPad((920, 40)),
    VerticalPad((1170, 610)),
    VerticalPad((1170, 280)),
    HorizontalPad((240, 860)),
    HorizontalPad((590, 860)),
    HorizontalPad((920, 860)),

    # inner layer
    VerticalPad((190, 445)),
    SmallHorizontalPad((306, 205)),
    SmallVerticalPad((426, 318)),
    SmallHorizontalPad((540, 445)),
    SmallHorizontalPad((640, 445)),
    SmallVerticalPad((756, 333)),
    SmallHorizontalPad((869, 205)),
    SmallVerticalPad((590, 153)),
    VerticalPad((980, 445)),
    SmallHorizontalPad((869, 695)),
    SmallVerticalPad((756, 573)),
    SmallHorizontalPad((306, 685)),
    SmallVerticalPad((426, 570)),
    SmallVerticalPad((590, 743)),
]

class Game:
    def __init__(self):
        pygame.init()
        pygame.display.set_caption("AI Car game")
        width = 1200
        height = 900
        self.screen = pygame.display.set_mode((width, height))
        self.clock = pygame.time.Clock()
        self.pad_group = pygame.sprite.RenderPlain(*pads2)
        self.car = CarSprite('images/car.png', (100, 730))
        self.sensors = Sensors(self.car.position, self.car.direction, self.pad_group)
        self.ticks = 60
        self.font = pygame.font.Font(None, 75)
        self.exit = False

    def run(self, action):
        car_group = pygame.sprite.RenderPlain(self.car)
        self.sensors = Sensors(self.car.position, self.car.direction, self.pad_group)

        # zvečujemo hitrost
        #self.car.speed += 0.005

        dt = self.clock.get_time() / 1000

        if action == 0:
            self.car.k_right = -3.5
            self.car.k_left = 0
        elif action == 1:
            self.car.k_left = 3.5
            self.car.k_right = 0

        self.screen.fill((0, 0, 0))
        car_group.update(dt)

        collisions = pygame.sprite.groupcollide(car_group, self.pad_group, False, False, collided=None)
        if collisions != {}:
            self.car.speed = 0
            self.car.k_right = 0
            self.car.k_left = 0
            self.car.crashed = True

        self.sensors.update_sensors(self.car.position, self.car.direction)
        for sens in self.sensors.sens_objs:
            sens.draw(self.screen)

        self.pad_group.update(collisions)
        self.pad_group.draw(self.screen)
        car_group.draw(self.screen)
        # Counter Render
        pygame.display.flip()

        self.clock.tick(self.ticks)

    def is_crashed(self):
        return self.car.crashed

    def get_sensor_values(self):
        return self.sensors.sens_objs

    def step(self, action):

        self.run(action)

        state = [-(200 / (sen.length - 15)) for sen in self.get_sensor_values()]
        state = np.array([state])

        if self.is_crashed():
            reward = -500
            self.car.restart_position((100, 730))
        else:
            reward = 10 + sum(state[0])
        return reward, state

if __name__ == '__main__':
    game = Game()
    while True:
        game.step(random.randint(0, 3))


text:
from io import StringIO
from textwrap import dedent
from datetime import date
from pkg_resources import resource_filename

import nose.tools as nt
import numpy as np
import numpy.testing as nptest
import pandas
import pandas.util.testing as pdtest

from  pycvc import summary

@nt.nottest
def csv_string_as_df(csv_string, **opts):
    return pandas.read_csv(StringIO(dedent(csv_string)), **opts)

@nt.nottest
def load_test_data(filename, **opts):
    return pandas.read_csv(resource_filename('pycvc.tests.testdata', filename), **opts)

def test_classify_storms():
    input_df = pandas.DataFrame({
        'A': np.arange(2, 29, 6),
        'B': np.arange(0, 41, 10),
    })

    expected_df = pandas.DataFrame({
        'A': np.arange(2, 29, 6),
        'B': np.arange(0, 41, 10),
        'C': ['<5 mm', '5 - 10 mm', '10 - 15 mm', '15 - 20 mm', '>25 mm'],
    })
    expected_df['C'] = expected_df['C'].astype('category')

    result_df = summary.classify_storms(input_df, 'A', newcol='C')

    pdtest.assert_frame_equal(result_df, expected_df)

def test_prevalence_table():
    input_df = csv_string_as_df("""\
    site,parameter,season,rescol,sampletype
    A,pA,Fall,1,composite
    A,pA,Fall,1,composite
    A,pA,Summer,1,composite
    A,pA,Summer,1,composite
    A,pA,Spring,1,composite
    A,pA,Winter,1,composite
    B,pA,Summer,1,composite
    B,pA,Fall,1,composite
    B,pA,Fall,1,composite
    B,pB,Spring,1,composite
    B,pB,Winter,1,grab
    B,pB,Winter,1,composite
    """)

    expected_df = pandas.DataFrame({
        'site': ['A', 'A', 'A', 'A', 'B', 'B', 'B', 'B'],
        'season': ['Fall', 'Spring', 'Summer', 'Winter'] * 2,
        'pA': [2, 1, 2, 1, 2, np.nan, 1, np.nan],
        'pB': [np.nan, np.nan, np.nan, np.nan, np.nan, 1, np.nan, 1],
    })[['site', 'season', 'pA', 'pB']]
    expected_df.columns.names = ['parameter']
    result_df = summary.prevalence_table(input_df, rescol='rescol', groupby_col='season')
    pdtest.assert_frame_equal(result_df, expected_df)

def test_remove_load_data_from_storms():
    input_df = pandas.DataFrame({
        'date': [
            pandas.Timestamp('2016-01-02 12:30'),
            pandas.Timestamp('2016-01-02 14:30'),
            pandas.Timestamp('2016-01-03 12:30'),
        ],
        'conc': [1., 2., 3.],
        'load': [1., 2., 4.],
        'load_a': [1., 2., 5.],
        'load_b': [1., 2., 6.]
    })

    expected_df = pandas.DataFrame({
        'date': [
            pandas.Timestamp('2016-01-02 12:30'),
            pandas.Timestamp('2016-01-02 14:30'),
            pandas.Timestamp('2016-01-03 12:30'),
        ],
        'conc': [1., 2., 3.],
        'load': [1., 2., 4.],
        'load_a': [np.nan, np.nan, 5],
        'load_b': [np.nan, np.nan, 6]
    })

    result_df = summary.remove_load_data_from_storms(input_df, [date(2016, 1, 2)], 'date')
    pdtest.assert_frame_equal(result_df, expected_df)

def test_pct_reduction():
    input_df = pandas.DataFrame({
        'A': [75., 50., 25., 10.],
        'B': [50., 50., 10., 20.],
    })

    expected_ser = pandas.Series([33.3333333333, 0., 60., -100.])

    result_ser = summary.pct_reduction(input_df, 'A', 'B')

    pdtest.assert_series_equal(result_ser, expected_ser)

def test_load_reduction_pct():
    input_df = pandas.DataFrame({
        'A':    np.array([75., 50., 25., 10.]),
        'A_lo': np.array([70., 45., 20.,  5.]),
        'A_hi': np.array([80., 55., 30., 15.]),
        'B':    np.array([50., 50., 10., 20.]),
        'B_lo': np.array([45., 45.,  5., 15.]),
        'B_hi': np.array([55., 55., 15., 25.]),
        'site': ['A', 'B'] * 2,
        'parameter': ['A'] * 4,
        'load_units': ['g'] * 4,
        'season': ['summer'] * 4,
    })

    expected_df = pandas.DataFrame({
        'A':    np.array([100., 60.]),
        'A_lo': np.array([ 90., 50.]),
        'A_hi': np.array([110., 70.]),
        'B':    np.array([ 60., 70.]),
        'B_lo': np.array([ 50., 60.]),
        'B_hi': np.array([ 70., 80.]),
        'site': ['A', 'B'],
        'parameter': ['A'] * 2,
        'load_units': ['g'] * 2,
        'season': ['summer'] * 2,
        'load_red': [40., -10/60.],
        'load_red_lower': [20./90., -30./50.],
        'load_red_upper': [60./110., 10./70.],
    })

    result_df = summary.load_reduction_pct(input_df, groupby_col='season',
                                            load_inflow='A',
                                            load_outflow='B',
                                            load_inflow_lower='A_lo',
                                            load_inflow_upper='A_hi',
                                            load_outflow_lower='B_lo',
                                            load_outflow_upper='B_hi')

def test_storm_stats():
    input_df = csv_string_as_df("""\
    site,storm_number,year,season,grouped_season,antecedent_days,start_date,end_date,duration_hours,peak_precip_intensity,total_precip_depth,runoff_m3
    ED-1,2,2011,summer,summer/autumn,3.1041,2011-07-28 08:30,2011-07-28 10:30,2.000,6.0,5.0000,16.4213
    ED-1,3,2011,summer,summer/autumn,0.6458,2011-07-29 02:00,2011-07-29 12:20,10.333,3.6,2.2,7.22535
    ED-1,5,2011,summer,summer/autumn,2.6736,2011-08-03 03:20,2011-08-03 05:20,2.000,12.0,12.8,42.0384
    ED-1,8,2011,summer,summer/autumn,2.7361,2011-08-07 04:10,2011-08-08 00:00,19.833,38.4,26.6,87.36105
    ED-1,9,2011,summer,summer/autumn,1.2638,2011-08-09 06:20,2011-08-09 14:30,8.166,20.4,34.6,113.63505
    ED-1,10,2011,summer,summer/autumn,0.3472,2011-08-09 22:50,2011-08-09 23:40,0.833,14.4,5.800,19.04865
    ED-1,14,2011,summer,summer/autumn,0.4027,2011-08-21 13:50,2011-08-21 16:40,2.833,28.8,6.0,19.7055
    ED-1,17,2011,summer,summer/autumn,0.4166,2011-08-24 20:20,2011-08-24 22:30,2.166,27.6,11.2,36.7836
    ED-1,19,2011,summer,summer/autumn,6.7430,2011-09-01 01:10,2011-09-01 02:30,1.333,4.8,5.0,16.42125
    ED-1,21,2011,summer,summer/autumn,1.8402,2011-09-03 09:20,2011-09-03 14:50,5.500,14.4,4.4,14.4507
    ED-1,24,2011,summer,summer/autumn,0.2638,2011-09-04 19:00,2011-09-04 19:50,0.833,10.8,4.0,13.137
    """, parse_dates=['start_date', 'end_date'])

    expected_df = csv_string_as_df("""\
    site,season,quantity,count,mean,std,min,25%,50%,75%,max
    ED-1,summer,antecedent_days,4.0,1.7726,1.1310,0.4167,1.0521,1.9687,2.689225,2.7361
    ED-1,summer,duration_hours,4.0,8.0417,8.3681,2.0,2.125,5.16667,11.08275,19.8333
    ED-1,summer,peak_precip_intensity,4.0,24.6,11.19285,12.0,18.3,24.0,30.3,38.4
    ED-1,summer,runoff_m3,4.0,69.9545,36.9260,36.7836,40.7247,64.6997,93.92955,113.635
    ED-1,summer,total_precip_depth,4.0,21.3,11.24334,11.2,12.4,19.7,28.6,34.6
    ED-1,summer,year,4.0,2011.0,0.0,2011.0,2011.0,2011.0,2011.0,2011.0
    """)
    expected_df.columns.names = ['stat']

    result_df = summary.storm_stats(input_df, minprecip=8,
                                    excluded_dates=[date(2011, 8, 21)],
                                    groupby_col='season')

    pdtest.assert_frame_equal(result_df, expected_df, check_less_precise=True)

def test_wq_summary():
    datecols = ['samplestart', 'samplestop', 'start_date', 'end_date']
    input_df = load_test_data('test_wq.csv', parse_dates=datecols)

    expected_none = load_test_data('wq_summary_by_nothing.csv')
    expected_season = load_test_data('wq_summary_by_season.csv')

    pdtest.assert_frame_equal(
        summary.wq_summary(input_df),
        expected_none
    )

    pdtest.assert_frame_equal(
        summary.wq_summary(input_df, groupby_col='season'),
        expected_season
    )

def test_load_totals():
    datecols = ['samplestart', 'samplestop', 'start_date', 'end_date']
    input_df = load_test_data('test_wq.csv', parse_dates=datecols)

    expected_none = load_test_data('load_totals_by_nothing.csv')
    expected_season = load_test_data('load_totals_by_season.csv')

    pdtest.assert_frame_equal(
        summary.load_totals(input_df),
        expected_none
    )

    pdtest.assert_frame_equal(
        summary.load_totals(input_df, groupby_col='season'),
        expected_season
    )

max_stars_repo_name: Geosyntec/python-cvc

max_issues_repo_name: Geosyntec/python-cvc

max_forks_repo_name: Geosyntec/python-cvc


text:
from typing import Tuple
import numpy as np
import math

class DataSetWrapper(object):
    def __init__(self, dataset, class_group: Tuple[int], negative_samples=False):
        # The original dataset has been shuffled. Skip shuffling this dataset
        # for consistency.
        self.dataset = dataset
        self.class_group = class_group
        self.negative_samples = negative_samples
        self.targets = np.asarray(self.dataset.targets)
        # This is the bool mask for all classes in the given group.
        positive_mask = np.zeros_like(self.targets, dtype=bool)
        for class_index in class_group:
            positive_mask |= (self.targets == class_index)
        positive_class_indices = np.where(positive_mask)[0]
        if negative_samples:
            # For N negative samples, P positive samples, we need to append
            # (k * N - P) positive samples.
            k = len(class_group)
            P = len(positive_class_indices)
            N = len(self.targets) - P
            assert N >= P, "there are already more positive classes"
            ext_P = k * N - P
            repeat_n = math.ceil(ext_P / P)
            extented_indices = np.repeat(
                positive_class_indices, repeat_n)[:ext_P]
            # fuse and shuffle
            all_indices = np.arange(len(self.targets))
            fullset = np.concatenate([all_indices, extented_indices])
            np.random.shuffle(fullset)
            self.mapping = fullset
        else:
            self.mapping = positive_class_indices

    def __getitem__(self, i):
        index = self.mapping[i]
        data, label = self.dataset[index]
        if label in self.class_group:
            label = list(self.class_group).index(label) + 1
        else:
            label = 0
        return data, label

    def __len__(self):
        return len(self.mapping)

    @property
    def num_classes(self):
        return len(self.class_group) + 1


text:
import numpy as np
import numba
@numba.njit(cache=True)
def sum_square(x):
 return np.sum(np.square(x))

@numba.njit(cache=True)
def project_onto_simplex(v,B):
 u=v[np.argsort(-v)]
 sv=np.cumsum(u)
 for i in range(v.shape[0]):
  cond=u[i]>(sv[i]-B)/(i+1)
  if cond:
   rho=i+1
 theta=(sv[rho-1]-B)/rho
 x=v-theta
 for i in range(x.shape[0]):
  if x[i]<0:
   x[i]=0
 return x

@numba.njit(cache=True)
def linear_chi_square(v,u,rho,epsilon=1e-8):
 lam_l=0
 lam_u=np.inf
 B=1.
 x=project_onto_simplex(u,1.)
 assert(sum_square(x-u)<=rho),'Infeasible'
 lam_s=1.
 while lam_u==np.inf:
  x=project_onto_simplex(u-v/lam_s,B)
  grad=0.5*sum_square(x-u)-rho/2.
  if grad<0:
   lam_u=lam_s
  else:
   lam_s*=2
 while(lam_u-lam_l)>epsilon*lam_s:
  lam=(lam_u+lam_l)/2.
  x=project_onto_simplex(u-v/lam,B)
  grad=0.5*sum_square(x-u)-rho/2.
  if grad<0:
   lam_u=lam
  else:
   lam_l=lam
 lam=(lam_u+lam_l)/2.
 return project_onto_simplex(u-v/lam,B)

if __name__=='__main__':
 B=1.
 temp=np.array([0.3,0.7,0.1,0.2,0.6])
 rho=0.1
 v=np.array([0.1,0.1,0.1,0.1,0.1])
 print(project_onto_simplex(v,B))
 print(linear_chi_square(temp,v/np.sum(v),rho))
 print('')
 v=np.array([0.1,0.1,0.1,0.1,10])
 print(project_onto_simplex(v,B))
 print(linear_chi_square(temp,v/np.sum(v),rho))
 print('')
 v=np.array([0.1,0.1,0.1,0.1,0.2])
 print(project_onto_simplex(v,B))
 print(linear_chi_square(temp,v/np.sum(v),rho))
 print('')
 v=np.array([0.6,0.1,0.1,0.1,0.1])
 print(project_onto_simplex(v,B))
 print(linear_chi_square(temp,v/np.sum(v),rho))
 print('')
 v=np.array([0.3,0.7,0.1,0.2,0.6])
 print(project_onto_simplex(v,B))
 print(linear_chi_square(temp,v/np.sum(v),rho))
 print('')
 v=np.array([0.06,0.1,0.1,0.1,0.06])
 print(project_onto_simplex(v,B))
 print(linear_chi_square(temp,v/np.sum(v),rho))
 print('')
 temp=np.random.rand(100,)
 v=np.random.rand(100,)
 v/=np.sum(v)
 import time
 start=time.time()
 NUM=100
 for _ in range(NUM):
  out=linear_chi_square(temp,v,rho)
 print((time.time()-start)/NUM)
# Created by pyminifier (https://github.com/liftoff/pyminifier)

reason: import numpy,import numba


text:
import numpy as np
from time import time
from pprint import pprint
import re

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Find the smallest prime which, by replacing part of the
# number (not necessarily adjacent digits) with the same
# digit, is part of an eight prime value family.
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

def primesfrom2to(n):
    """ Input n>=6, Returns a array of primes, 2 <= p < n """
    sieve = np.ones(n//3 + (n%6==2), dtype=np.bool)
    for i in range(1,int(n**0.5)//3+1):
        if sieve[i]:
            k=3*i+1|1
            sieve[       k*k//3     ::2*k] = False
            sieve[k*(k-2*(i&1)+4)//3::2*k] = False
    return np.r_[2,3,((3*np.nonzero(sieve)[0][1:]+1)|1)]

start_time = time()
prime_list = primesfrom2to(1000000)[4:]
ending = [1, 3, 7, 9]
regulars = [1, 3, 5, 7, 9]
stringPrimes = [str(x) for x in prime_list]

possibles = []
for numberCount in stringPrimes:
    for numberCheck in range(10):
        if numberCount.count(str(numberCheck)) == 3:
            # print(numberCount)
            indices = [i for i, a in enumerate(numberCount) if a == str(numberCheck)]
            possibles = []
            for value in range(10):
                numberList = list(numberCount)
                numberList[indices[0]] = str(value)
                numberList[indices[1]] = str(value)
                numberList[indices[2]] = str(value)
                ListToInteger = "".join(numberList)
                if len(str(int(ListToInteger))) == len(ListToInteger) and int(ListToInteger) in prime_list:
                    possibles.append(int(ListToInteger))
                    if len(possibles) == 8:
                        answer = possibles
print("The answer is: {}".format(answer[0]))
elapsed_time = time() - start_time
print('The answer was found in {0:.4f} s.'.format(elapsed_time))

max_stars_repo_name: ChristensenCode/energy-viking

max_issues_repo_name: ChristensenCode/energy-viking

max_forks_repo_name: ChristensenCode/energy-viking


text:
# type: ignore[attr-defined]
# This package is mainly used for numpy/scipy... compatible replacement in the future

import numpy as numpy
import scipy as scipy
from scipy import linalg as linalg

# backup important python build-in functions
py_all = all
py_any = any
py_eval = eval
py_slice = slice
py_zip = zip

try:
    # for line_profile
    from builtins import profile as py_profile
except ImportError:
    py_profile = lambda f: f

reason: import numpy,import scipy,from scipy


text:
import os, sys
import numpy as np
import argparse
import time
from scipy import stats

from simple_projections import project_onto_chi_square_ball
from create_datasets import data_loader

def add_intercept(X):
    intercept = np.ones((X.shape[0], 1))
    return np.concatenate((X, intercept), axis=1)

def sigmoid(z):
    return 1.0 / (1 + np.exp(-z))

def loss(h, y):
    return -y * np.log(h) - (1 - y) * np.log(1 - h)

def compute_gradients_vanilla(theta, X, y):  # the vanilla
    h = predict_prob(theta, X)
    gradient = np.dot(X.T, h-y) / y.size
    return gradient

def compute_gradients_individual(theta, X, y):
    h = predict_prob(theta, X)
    gradient = np.multiply(X, np.transpose([h-y]))

    return gradient

def predict_prob(theta, X):
    return sigmoid(np.dot(X, theta))

def predict(theta, X):
    return predict_prob(theta, X).round()

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument('--num_trials',
                        help='run how many times',
                        type=int,
                        default=5)
    parser.add_argument('--obj',
                        help='objective: original, minimax, tilting',
                        type=str,
                        default='original')
    parser.add_argument('--t_in',
                        help='value of t for across sample tilting',
                        type=float,
                        default=1.0)
    parser.add_argument('--t_out',
                        help='value of t for across class tilting',
                        type=float,
                        default=1.0)
    parser.add_argument('--rho',
                        help='value of rho for the distributionally robust opt work',
                        type=float,
                        default=1)
    parser.add_argument('--lr',
                        help='learning rate',
                        type=float,
                        default=0.1)
    parser.add_argument('--num_iters',
                        help='how many iterations of gradient descent',
                        type=int,
                        default=10000)
    parser.add_argument('--corrupt',
                        help='whether crete outliers',
                        type=int,
                        default=0)
    parser.add_argument('--num_corrupted',
                        help='how many data points are outliers/noisy',
                        type=int,
                        default=1500)
    parser.add_argument('--eval_interval',
                        help='eval every how many iterations (of SGD or GD)',
                        type=int,
                        default=1000)
    parser.add_argument('--imbalance',
                        help='whether to make it imbalanced',
                        type=int,
                        default=0)
    parser.add_argument('--gamma',
                        help='gamma for focal loss',
                        type=float,
                        default=0.2)
    parser.add_argument('--dataset',
                        help='dataset (hiv1 or adult)',
                        type=str,
                        default='adult')

    parsed = vars(parser.parse_args())
    num_trials = parsed['num_trials']
    obj = parsed['obj']
    rho = parsed['rho']
    t_in = parsed['t_in']
    t_out = parsed['t_out']
    num_iters = parsed['num_iters']
    lr = parsed['lr']
    corrupt = parsed['corrupt']
    num_corrupted = parsed['num_corrupted']
    interval = parsed['eval_interval']
    imbalance = parsed['imbalance']
    gamma = parsed['gamma']
    dataset = parsed['dataset']
    maxLen = max([len(ii) for ii in parsed.keys()])
    fmtString = '\t%' + str(maxLen) + 's : %s'
    print('Arguments:')
    for keyPair in sorted(parsed.items()):
        print(fmtString % keyPair)

    train_accuracies = []
    test_accuracies = []
    val_accuracies = []
    train_rare = []
    test_rare = []
    train_common = []
    test_common = []

    begin = time.time()

    for i in range(num_trials):
        all_data, all_labels = data_loader("data/hiv1/raw", i)  # have been randomly shuffled using seed i
        print("{} total data points, {} training samples, {}% rare".format(len(all_labels), int(0.8*(len(all_labels))),
                                                                          len(np.where(all_labels==1)[0])/len(all_labels)))

        train_X, train_y = all_data[:int(len(all_labels)*0.8)], all_labels[:int(len(all_labels)*0.8)]
        val_X, val_y = all_data[int(len(all_labels) * 0.8):int(len(all_labels) * 0.9)], all_labels[
                                                                                int(len(all_labels) * 0.8):int(
                                                                                len(all_labels) * 0.9)]
        # 10 samples for each class
        rare_keep_ind = np.where(val_y == 1)[0][:10]
        common_keep_ind = np.where(val_y == 0)[0][:10]
        val_X, val_y = np.concatenate((val_X[common_keep_ind], val_X[rare_keep_ind]), axis=0), \
                   np.concatenate((val_y[common_keep_ind], val_y[rare_keep_ind]), axis=0)

        test_X, test_y = all_data[int(len(all_labels) * 0.9):], all_labels[int(len(all_labels) * 0.9):]

        if imbalance == 1:
            len_common = len(np.where(train_y == 0)[0])
            len_rare = int(len_common * 0.05)
            rare_keep_ind = np.where(train_y == 1)[0][:len_rare]
            common_keep_ind = np.where(train_y == 0)[0]
            train_X, train_y = np.concatenate((train_X[common_keep_ind], train_X[rare_keep_ind]), axis=0), \
                   np.concatenate((train_y[common_keep_ind], train_y[rare_keep_ind]), axis=0)

            np.random.seed(123456 + i)
            perm = np.random.permutation(len(train_y))
            train_X, train_y = train_X[perm], train_y[perm]

        if corrupt == 1:
            np.random.seed(i)
            print(num_corrupted, "corrupted")
            num_common = len(np.where(train_y==0)[0])
            num_rare = len(np.where(train_y==1)[0])
            train_y[:num_corrupted] = np.random.choice([0, 1], num_corrupted, p=[num_common/len(train_y), num_rare/len(train_y)])
            # noisy validation data with the same noise as that in the training set
            perm = np.random.permutation(len(val_y))
            val_X, val_y = val_X[perm], val_y[perm]
            val_y[:6] = np.random.choice([0, 1], 6, p=[num_common / len(train_y), num_rare / len(train_y)])

        theta = np.zeros(len(train_X[0]))

        for j in range(num_iters):
            grads_train_individual = compute_gradients_individual(theta, train_X, train_y)
            grads_train = np.mean(grads_train_individual, axis=0)
            grads_val = compute_gradients_vanilla(theta - lr * grads_train, val_X, val_y)
            #if j % 100 == 0:
            #    print(np.dot(grads_train_individual, grads_val))
            gradients_w = -1 * np.dot(grads_train_individual, grads_val)
            w = np.maximum(np.ones(len(train_X))/len(train_X)-0.1*gradients_w, 0)
            w = w / np.sum(w)
            #w = np.ones(len(train_X)) / len(train_X)
            #print(w)
            theta = theta - lr * np.average(grads_train_individual, axis=0, weights=w)
            #theta = theta - lr * grads_val

            if j % interval == 0:
                if corrupt == 0:
                    rare_sample_train = np.where(train_y == 1)[0]
                    preds_rare = predict(theta, train_X)[rare_sample_train]
                    train_accuracy = (preds_rare == train_y[rare_sample_train]).mean()
                    print("rare training accuracy: ", train_accuracy)
                else:
                    preds_rare = predict(theta, train_X)
                    train_accuracy = (preds_rare == train_y).mean()
                    print("training accuracy: ", train_accuracy)

        preds_train = predict(theta, train_X)
        preds_test = predict(theta, test_X)
        preds_val = predict(theta, val_X)

        train_accuracy = (preds_train == train_y).mean()
        train_accuracies.append(train_accuracy)

        test_accuracy = (preds_test == test_y).mean()
        test_accuracies.append(test_accuracy)

        val_accuracy = (preds_val == val_y).mean()
        val_accuracies.append(val_accuracy)

        rare_sample_train = np.where(train_y == 1)[0]
        preds_rare = preds_train[rare_sample_train]
        rare_train_accuracy = (preds_rare == train_y[rare_sample_train]).mean()
        train_rare.append(rare_train_accuracy)

        rare_sample_test = np.where(test_y == 1)[0]
        preds_rare = preds_test[rare_sample_test]
        rare_test_accuracy = (preds_rare == test_y[rare_sample_test]).mean()
        test_rare.append(rare_test_accuracy)

        common_sample_train = np.where(train_y == 0)[0]
        preds_common = preds_train[common_sample_train]
        common_train_accuracy = (preds_common == 0).mean()
        train_common.append(common_train_accuracy)

        common_sample_test = np.where(test_y == 0)[0]
        preds_common = preds_test[common_sample_test]
        common_test_accuracy = (preds_common == 0).mean()
        test_common.append(common_test_accuracy)

        print("trial {}, training accuracy on the rare class {}".format(i, rare_train_accuracy))
        print("trial {}, testing accuracy on the rare class {}".format(i, rare_test_accuracy))

        print("trial {}, training accuracy on the common class {}".format(i, common_train_accuracy))
        print("trial {}, testing accuracy on the common class {}".format(i, common_test_accuracy))

        print("trial {}, overall training accuracy {}".format(i, train_accuracy))
        print("trial {}, overall testing accuracy {}".format(i, test_accuracy))

    end = time.time()

    print("avg training accuracy on the rare class {}".format(np.mean(np.array(train_rare))))
    print("se training accuracy on the rare class {}".format(stats.sem(np.array(train_rare))))
    print("avg testing accuracy on the rare class {}".format(np.mean(np.array(test_rare))))
    print("se testing accuracy on the rare class {}".format(stats.sem(np.array(test_rare))))

    print("avg training accuracy on the common class {}".format(np.mean(np.array(train_common))))
    print("se training accuracy on the common class {}".format(stats.sem(np.array(train_common))))
    print("avg testing accuracy on the common class {}".format(np.mean(np.array(test_common))))
    print("se testing accuracy on the common class {}".format(stats.sem(np.array(test_common))))

    print("avg overall training accuracy {}".format(np.mean(np.array(train_accuracies))))
    print("se overall training accuracy {}".format(stats.sem(train_accuracies)))
    print("avg overall testing accuracy {}".format(np.mean(np.array(test_accuracies))))
    print("se overall testing accuracy {}".format(stats.sem(test_accuracies)))

    print("avg overall validation accuracy {}".format(np.mean(np.array(val_accuracies))))

if __name__ == '__main__':
    main()

reason: import numpy,from scipy