text:
# Parameters for script.
# These values reflect an actual work load I had that was slow.
num_gigabytes = 10  # Approximate number of giga bytes of the csv
str_columns, str_lens = 200, 13  # How many string columns to include, and what is the uniform length of all strings
float_columns = 200  # How many float columns to include
null_columns = 20  # How many all-null columns to include
num_rounds = 5  # How many calls to read_csv to make
backend = "loky"  # Joblib backend for parallelising creation of csv
n_jobs = -1  # Number of jobs for csv creation parallelisation

#####################################################################

from io import BytesIO
import pyarrow.csv
import time
import pyarrow as pa
import os
import pandas as pd
import numpy as np
import joblib
from multiprocessing import cpu_count

@joblib.delayed
def create_csv(
    num_rows,
    str_columns, str_lens,
    float_columns,
    null_columns,
    random_seed, columns_permutation,
    header
    ):
    """Creates random csv with given parameters"""
    # Create random string columns
    rand = np.random.RandomState(random_seed)
    A, Z = np.array(["A","Z"]).view("int32")
    strs = rand.randint(
        low=A,high=Z,size=(num_rows, str_columns, str_lens),dtype="int32"
    ).view(f"U{str_lens}")[..., 0]

    # Create random float columns
    floats = rand.randn(num_rows, float_columns)

    # Create null columns
    nulls = [[pd.NA] * null_columns] * num_rows

    # Put columns into a pandas DataFrame in same random order
    df = pd.concat([
        pd.DataFrame(
            values,
            columns=[f"{name}_{i}" for i in range(len(values[0]))]
        )
        for name, values in [
            ("strs", strs), ("floats", floats), ("nulls", nulls)
        ]
    ], axis=1)
    df = df.iloc[:, columns_permutation]

    # Create the csv for the DataFrame
    data_raw = df.to_csv(sep="|", header=header).encode()
    return data_raw

if __name__ == "__main__":
    print("pid:", os.getpid())  # Print pid so I can run py-spy on it

    # Compute a random perutation for the columns order.
    columns_permutation = np.random.RandomState(2).permutation(
        str_columns + float_columns + null_columns
    )

    # Approximate number of rows needed to fulfill approximate number of gigabytes
    row_gigabyte = (
        (
            str_columns * (str_lens + 1)
            + float_columns * (19 + 1)  # The floats have aound 19 characters on average
            + null_columns + 2
        ) * cpu_count() / 1024**3
    )
    num_rows = int(num_gigabytes / row_gigabyte)

    print("1/3 Creating csv chunks")

    # Parallelise creation of csv
    with joblib.Parallel(n_jobs=n_jobs, backend=backend) as pool:
        data_raw = pool(
            create_csv(
                num_rows, str_columns, str_lens,
                float_columns, null_columns, i, columns_permutation, i == 0
            )
            for i in range(cpu_count())
        )

    print("2/3 Creating bytes")

    total_size = sum(map(len, data_raw)) / 1024**3
    data_bytes = BytesIO()
    for b in data_raw:
        data_bytes.write(b)

    times = []
    parse_options = pyarrow.csv.ParseOptions(delimiter="|")

    for i in range(num_rounds):
        data_bytes.seek(0)  # Go back to beginning of buffer
        print(f"starting {i}/{num_rounds}")

        # Actual call to read_csv
        t1 = time.time()
        a = pyarrow.csv.read_csv(data_bytes, parse_options=parse_options)
        t2 = time.time()

        times.append(t2 - t1)
        print(f"done in {times[-1]:.2f} secs, {total_size / times[-1]:.2f} GiB/s")
        del a

    print(
        "processed", total_size,
        "in max speed of", total_size / min(times), "GiB/s"
    )
    # Print all times
    print(", ".join("%.2f" % v for v in (total_size / np.array(times))))

max_stars_repo_name: drorspei/arrow-csv-benchmark

max_issues_repo_name: drorspei/arrow-csv-benchmark

max_forks_repo_name: drorspei/arrow-csv-benchmark


text:
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""Test models that are not in API yet."""

import os

import mxnet as mx
import numpy as np
import numpy.testing as npt
import pytest
from mxnet.gluon.utils import _get_repo_url, download

from gluonnlp.data.transforms import GPT2BPEDetokenizer, GPT2BPETokenizer

from ..text_generation.model import get_model

@pytest.mark.remote_required
@pytest.mark.parametrize('model_name', ['gpt2_117m', 'gpt2_345m'])
def test_pretrained_gpt2(model_name, tmp_path):
    sentence = ' natural language processing tools such as gluonnlp and torchtext'
    model, vocab = get_model(model_name, dataset_name='openai_webtext')
    tokenizer = GPT2BPETokenizer()
    detokenizer = GPT2BPEDetokenizer()
    true_data_hash = {'gpt2_117m': '29526682508d03a7c54c598e889f77f7b4608df0',
                      'gpt2_345m': '6680fd2a3d7b737855536f480bc19d166f15a3ad'}
    file_name = '{model_name}_gt_logits-{short_hash}.npy'.format(
            model_name=model_name,
            short_hash=true_data_hash[model_name][:8])
    url_format = '{repo_url}gluon/dataset/test/{file_name}'
    repo_url = _get_repo_url()
    path = os.path.join(str(tmp_path), file_name)
    download(url_format.format(repo_url=repo_url, file_name=file_name),
             path=path,
             sha1_hash=true_data_hash[model_name])
    gt_logits = np.load(path)
    model.hybridize()
    indices = vocab[tokenizer(sentence)]
    nd_indices = mx.nd.expand_dims(mx.nd.array(indices), axis=0)
    logits, new_states = model(nd_indices, None)
    npt.assert_allclose(logits.asnumpy(), gt_logits, 1E-5, 1E-5)

ext: py

max_stars_repo_name: xiaotinghe/gluon-nlp

max_issues_repo_name: xiaotinghe/gluon-nlp

max_forks_repo_name: xiaotinghe/gluon-nlp

reason: import numpy


text:
import cv2
import numpy as np
import face_recognition
from skimage import transform

from constants import landmarks_2D

def get_align_mat(face):
    mat = transform.SimilarityTransform()
    mat.estimate(np.array(face.landmarksAsXY()[17:]), landmarks_2D)
    return mat.params[0:2]

# Extraction class to identify and crop out the face from an image
class Extract(object):
    def extract(self, image, face, size):
        if face.landmarks is None:
            print("Warning! landmarks not found. Switching to crop!")
            return cv2.resize(face.image, (size, size))

        alignment = get_align_mat(face)
        return self.transform(image, alignment, size, padding=48)

    def transform(self, image, mat, size, padding=0):
        mat = mat * (size - 2 * padding)
        mat[:, 2] += padding
        return cv2.warpAffine(image, mat, (size, size))

# Filter class to check if a given image has a specific identity or not
class FaceFilter():
    def __init__(self, reference_file_path, threshold=0.65):
        """
        Works only for single face images
        """
        image = face_recognition.load_image_file(reference_file_path)
        self.encoding = face_recognition.face_encodings(image)[0]
        self.threshold = threshold

    def check(self, detected_face):
        encodings = face_recognition.face_encodings(detected_face.image)
        if len(encodings) > 0:
            encodings = encodings[0]
            score = face_recognition.face_distance([self.encoding], encodings)
        else:
            print("No faces found in the image!")
            score = 0.8
        return score <= self.threshold

class Convert():
    def __init__(self, encoder,
                 blur_size=2,
                 seamless_clone=False,
                 mask_type="facehullandrect",
                 erosion_kernel_size=None,
                 **kwargs):
        self.encoder = encoder

        self.erosion_kernel = None
        if erosion_kernel_size is not None:
            self.erosion_kernel = cv2.getStructuringElement(
                cv2.MORPH_ELLIPSE, (erosion_kernel_size, erosion_kernel_size))

        self.blur_size = blur_size
        self.seamless_clone = seamless_clone
        self.mask_type = mask_type.lower()

    def patch_image(self, image, face_detected):
        size = 64
        image_size = image.shape[1], image.shape[0]

        mat = np.array(get_align_mat(face_detected)).reshape(2, 3) * size

        new_face = self.get_new_face(image, mat, size)

        image_mask = self.get_image_mask(
            image, new_face, face_detected, mat, image_size)

        return self.apply_new_face(image,
                                   new_face,
                                   image_mask,
                                   mat,
                                   image_size,
                                   size)

    def apply_new_face(self,
                       image,
                       new_face,
                       image_mask,
                       mat,
                       image_size,
                       size):
        base_image = np.copy(image)
        new_image = np.copy(image)

        cv2.warpAffine(new_face, mat, image_size, new_image,
                       cv2.WARP_INVERSE_MAP, cv2.BORDER_TRANSPARENT)

        outimage = None
        if self.seamless_clone:
            masky, maskx = cv2.transform(np.array([size / 2, size / 2]).reshape(
                1, 1, 2), cv2.invertAffineTransform(mat)).reshape(2).astype(int)
            outimage = cv2.seamlessClone(new_image.astype(np.uint8), base_image.astype(
                np.uint8), (image_mask * 255).astype(np.uint8), (masky, maskx), cv2.NORMAL_CLONE)
        else:
            foreground = cv2.multiply(image_mask, new_image.astype(float))
            background = cv2.multiply(
                1.0 - image_mask, base_image.astype(float))
            outimage = cv2.add(foreground, background)

        return outimage

    def get_new_face(self, image, mat, size):
        face = cv2.warpAffine(image, mat, (size, size))
        face = np.expand_dims(face, 0)
        new_face = self.encoder(face / 255.0)[0]

        return np.clip(new_face * 255, 0, 255).astype(image.dtype)

    def get_image_mask(self, image, new_face, face_detected, mat, image_size):

        face_mask = np.zeros(image.shape, dtype=float)
        if 'rect' in self.mask_type:
            face_src = np.ones(new_face.shape, dtype=float)
            cv2.warpAffine(face_src, mat, image_size, face_mask,
                           cv2.WARP_INVERSE_MAP, cv2.BORDER_TRANSPARENT)

        hull_mask = np.zeros(image.shape, dtype=float)
        if 'hull' in self.mask_type:
            hull = cv2.convexHull(np.array(face_detected.landmarksAsXY()).reshape(
                (-1, 2)).astype(int)).flatten().reshape((-1, 2))
            cv2.fillConvexPoly(hull_mask, hull, (1, 1, 1))

        if self.mask_type == 'rect':
            image_mask = face_mask
        elif self.mask_type == 'faceHull':
            image_mask = hull_mask
        else:
            image_mask = ((face_mask * hull_mask))

        if self.erosion_kernel is not None:
            image_mask = cv2.erode(
                image_mask, self.erosion_kernel, iterations=1)

        if self.blur_size != 0:
            image_mask = cv2.blur(image_mask, (self.blur_size, self.blur_size))

        return image_mask

# Entity class
class DetectedFace(object):
    def __init__(self, image, x, w, y, h, landmarks):
        self.image = image
        self.x = x
        self.w = w
        self.y = y
        self.h = h
        self.landmarks = landmarks

    def landmarksAsXY(self):
        return [(p.x, p.y) for p in self.landmarks.parts()]


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Sep 14 11:24:48 2018

@author: Lionel Massoulard
"""

import graphviz
import networkx as nx

from aikit.tools.graph_helper import (
    graph_from_edges_string,
    graph_from_edges,
    edges_from_graph,
    is_it_a_partition,
    insert_node_above,
    add_node_after,
    iter_graph,
    get_all_predecessors,
    get_all_successors,
    has_cycle,
    remove_node_keep_connections,
    get_terminal_nodes,
    get_starting_nodes,
    get_connected_graphs,
    subbranch_search,
    merge_nodes,
    get_two_by_two_edges,
)

def test_edges_from_graph():
    G = nx.DiGraph()
    G.add_node("A")
    G.add_node("B")

    assert edges_from_graph(G) == [("A",), ("B",)]

    G = nx.DiGraph()
    G.add_node("B")
    G.add_node("A")
    assert edges_from_graph(G) == [("A",), ("B",)]

    G = graph_from_edges(("A", "B"), ("C", "D"))
    assert set(G.nodes) == {"A", "B", "C", "D"}
    assert set(G.edges) == {("A", "B"), ("C", "D")}

    assert edges_from_graph(G) == [("A", "B"), ("C", "D")]

def test_graph_from_edges_string():

    edges_strings = [
        "A -  B - C  ; D - C",
        "A - B ; B - C ; D - C",
        "A -> B -> C ; D -> C",
        "A-B;B-C;D-C",
        "A->B;B->C;D-C;",
    ]

    for edges_string in edges_strings:
        G = graph_from_edges_string(edges_string)
        assert set(G.nodes) == {"A", "B", "C", "D"}
        assert set(G.edges) == {("A", "B"), ("B", "C"), ("D", "C")}

        edges2 = edges_from_graph(G)
        G2 = graph_from_edges(*edges2)

        assert set(G2.nodes) == set(G.nodes)
        assert set(G2.edges) == set(G.edges)

    edges_strings = ["1 - 2 - 3 ; 4 - 3", "1 - 2 ; 2 - 3 ; 4 - 3"]

    for edges_string in edges_strings:
        G = graph_from_edges_string(edges_string)
        assert set(G.nodes) == {1, 2, 3, 4}
        assert set(G.edges) == {(1, 2), (2, 3), (4, 3)}

        edges2 = edges_from_graph(G)
        G2 = graph_from_edges(*edges2)

        assert set(G2.nodes) == set(G.nodes)
        assert set(G2.edges) == set(G.edges)

def test_is_it_a_partition():
    all_nodes = [1, 2, 3, 4, 5, 6]
    assert is_it_a_partition(all_nodes, [[1], [2], [3, 4, 5, 6]])
    assert is_it_a_partition(all_nodes, [[1, 2], [3, 4, 5, 6]])
    assert is_it_a_partition(all_nodes, [[1, 2, 3, 4, 5, 6], []])

    assert not is_it_a_partition(all_nodes, [[1, 2, 3, 4, 5]])
    assert not is_it_a_partition(all_nodes, [[1, 2, 3, 4], [5]])

    assert not is_it_a_partition(all_nodes, [[1, 2, 3, 4], [5, 6, 7]])
    assert not is_it_a_partition(all_nodes, [[1, 2, 3, 4, 5], [5, 6]])

def test_graph_from_edges():

    edges = [(1, 2, 3), (4, 3)]
    G = graph_from_edges(*edges)
    assert set(G.nodes) == {1, 2, 3, 4}
    assert set(G.edges) == {(1, 2), (2, 3), (4, 3)}

    edges = [("1", "2", "3"), ("4", "3")]
    G = graph_from_edges(*edges)
    assert set(G.nodes) == {"1", "2", "3", "4"}
    assert set(G.edges) == {("1", "2"), ("2", "3"), ("4", "3")}

    edges = [("A", "B", "C")]
    G = graph_from_edges(*edges)
    assert set(G.nodes) == {"A", "B", "C"}
    assert set(G.edges) == {("A", "B"), ("B", "C")}

    edges = [("A", "B", "D"), ("A", "C", "D"), ("E", "C")]
    G = graph_from_edges(*edges)
    assert set(G.nodes) == {"A", "B", "C", "D", "E"}
    assert set(G.edges) == {("A", "B"), ("A", "C"), ("B", "D"), ("C", "D"), ("E", "C")}

    edges = [("A", "B", "D"), ("A", "C", "D"), ("E", "C", "D")]
    G = graph_from_edges(*edges)
    assert set(G.nodes) == {"A", "B", "C", "D", "E"}
    assert set(G.edges) == {("A", "B"), ("A", "C"), ("B", "D"), ("C", "D"), ("E", "C")}

def test_get_two_by_two_edges():
    assert get_two_by_two_edges(("a", "c"), ("b", "c")) == [("a", "c"), ("b", "c")]
    assert get_two_by_two_edges(("b", "c"), ("a", "c")) == [("b", "c"), ("a", "c")]

    assert get_two_by_two_edges(("a", "c", "d"), ("b", "c")) == [("a", "c"), ("c", "d"), ("b", "c")]
    assert get_two_by_two_edges(("a", "c", "d"), ("b", "c")) == [("a", "c"), ("c", "d"), ("b", "c")]

    edges = [("A",), ("B", "C", "D")]
    G = graph_from_edges(*edges)
    assert set(G.nodes) == {"A", "B", "C", "D"}
    assert set(G.edges) == {("B", "C"), ("C", "D")}

def test_insert_node_above():
    G = graph_from_edges_string("1 - 2 - 3")
    G2 = insert_node_above(G, 1, 0)

    assert G is G2
    assert set(G2.nodes) == {0, 1, 2, 3}
    assert set(G2.edges) == {(0, 1), (1, 2), (2, 3)}

    G = insert_node_above(G, 2, 22)
    assert set(G.nodes) == {0, 1, 2, 3, 22}
    assert set(G.edges) == {(0, 1), (1, 22), (22, 2), (2, 3)}

    G = graph_from_edges_string("1 - 2 - 3; 4 - 3")
    G = insert_node_above(G, 3, 33)
    assert set(G.edges) == {(1, 2), (2, 33), (4, 33), (33, 3)}
    assert set(G.nodes) == {1, 2, 3, 4, 33}

def test_remove_node_keep_connections():
    G = graph_from_edges_string("A - B - C; D - C")
    G = remove_node_keep_connections(G, "B")

    assert set(G.edges) == {("A", "C"), ("D", "C")}
    assert set(G.nodes) == {"A", "C", "D"}

    G = graph_from_edges_string("A - B - C; D - C")
    G = remove_node_keep_connections(G, "A")
    assert set(G.edges) == {("B", "C"), ("D", "C")}
    assert set(G.nodes) == {"B", "C", "D"}

    G = graph_from_edges_string("A - B - C - E; D - C - E")
    G = remove_node_keep_connections(G, "C")
    assert set(G.edges) == {("A", "B"), ("B", "E"), ("D", "E")}
    assert set(G.nodes) == {"A", "B", "D", "E"}

    G = graph_from_edges_string("A - B - C - D; C - E")
    G = remove_node_keep_connections(G, "C")
    assert set(G.edges) == {("A", "B"), ("B", "D"), ("B", "E")}
    assert set(G.nodes) == {"A", "B", "D", "E"}

    G = graph_from_edges_string("A - C - D; B - C - E")
    G = remove_node_keep_connections(G, "C")

    assert set(G.edges) == {("A", "D"), ("A", "E"), ("B", "D"), ("B", "E")}
    assert set(G.nodes) == {"A", "B", "D", "E"}

def test_all_graphs_functions():
    G = nx.DiGraph()
    G = add_node_after(G, 1)
    G = add_node_after(G, 2, 1)
    G = add_node_after(G, 3, 2)
    G = add_node_after(G, 4)
    G = add_node_after(G, 5, 4)
    G = add_node_after(G, 6, 5, 3)

    assert set(get_terminal_nodes(G)) == {6}
    assert set(get_starting_nodes(G)) == {1, 4}

    assert set(get_all_successors(G, 1)) == {2, 3, 6}
    assert set(get_all_successors(G, 2)) == {3, 6}
    assert set(get_all_successors(G, 3)) == {6}
    assert set(get_all_successors(G, 4)) == {5, 6}
    assert set(get_all_successors(G, 5)) == {6}
    assert set(get_all_successors(G, 6)) == set()

    assert set(get_all_predecessors(G, 1)) == set()
    assert set(get_all_predecessors(G, 2)) == {1}
    assert set(get_all_predecessors(G, 3)) == {1, 2}
    assert set(get_all_predecessors(G, 4)) == set()
    assert set(get_all_predecessors(G, 5)) == {4}
    assert set(get_all_predecessors(G, 6)) == {1, 2, 3, 4, 5}

def test_iter_graph():
    G1 = nx.DiGraph()
    G1 = add_node_after(G1, 1)
    G1 = add_node_after(G1, 2, 1)
    G1 = add_node_after(G1, 3, 2)
    G1 = add_node_after(G1, 4)
    G1 = add_node_after(G1, 5, 4)
    G1 = add_node_after(G1, 6, 5, 3)

    G2 = graph_from_edges((1, 2), (3, 4))

    for G in (G1, G2):
        done = set()
        for n in iter_graph(G):
            for p in G.predecessors(n):
                assert p in done

            assert n not in done
            done.add(n)

        assert done == set(G.nodes)

    G3 = nx.DiGraph()
    G3.add_node(1)
    G3.add_node(2)  # 2 un-connexted nodes
    assert list(iter_graph(G3)) in ([1, 2], [2, 1])

def test_graphviz():
    G2 = graphviz.Digraph()
    G2.edge("a", "b")
    G2.edge("b", "c")
    G2.edge("c", "f")
    G2.edge("d", "e")
    G2.edge("e", "f")
    G2.node_attr.update(color="lightblue", style="filled")

    G2

def test_has_cycle():
    G = graph_from_edges_string("A - B - C; D - C")
    assert not has_cycle(G)

    G = graph_from_edges_string("A - B - C - A")
    assert has_cycle(G)

def test_get_connected_graph():
    G = graph_from_edges_string("A - B ; B - C")
    Gs = get_connected_graphs(G)
    assert len(Gs) == 1
    assert Gs[0] == G

    def gequal(g1, g2):
        if set(g1.nodes) != set(g2.nodes):
            return False

        if set(g1.edges) != set(g2.edges):
            return False

        return True

    G = graph_from_edges_string("A - B; C - D")
    Gs = get_connected_graphs(G)
    assert len(Gs) == 2
    g1, g2 = graph_from_edges_string("A - B"), graph_from_edges_string("C - D")
    G1, G2 = Gs
    assert (gequal(g1, G1) and gequal(g2, G2)) or (gequal(g1, G2) and gequal(g2, G1))

def test_subbranch_search():

    Graph = graph_from_edges(("A", "B", "C"), ("B", "D"))

    nodes = list(subbranch_search("A", Graph))
    assert nodes == ["A", "B", "C", "D"] or nodes == ["A", "B", "D", "C"]

    nodes = list(subbranch_search("B", Graph))
    assert nodes == []

    nodes = list(subbranch_search("B", Graph, visited={"A"}))
    assert nodes == ["B", "C", "D"] or nodes == ["B", "D", "C"]

    nodes = list(subbranch_search("C", Graph, visited={"B"}))
    assert nodes == ["C"]

    nodes = list(subbranch_search("D", Graph, visited={"B"}))
    assert nodes == ["D"]

    # Graph2
    Graph = graph_from_edges(("A", "B", "C", "E"), ("B", "D", "E"))
    nodes = list(subbranch_search("A", Graph))
    assert nodes == ["A", "B", "C", "D", "E"] or nodes == ["A", "B", "D", "C", "E"]

    Graph = graph_from_edges(("A", "B", "C", "E"), ("B", "D", "E"))
    nodes = list(subbranch_search("B", Graph, visited={"A"}))
    assert nodes == ["B", "C", "D", "E"] or nodes == ["B", "D", "C", "E"]

    Graph = graph_from_edges(("A", "B", "C", "E"), ("B", "D", "E"))
    nodes = list(subbranch_search("C", Graph, visited={"B"}))
    assert nodes == ["C"]

    Graph = graph_from_edges(("A", "B", "C"), ("A", "C"))
    nodes = list(subbranch_search("A", Graph))
    assert nodes == ["A", "B", "C"]

    nodes = list(subbranch_search("B", Graph, visited={"A"}))
    assert nodes == ["B", "C"]

    nodes = list(subbranch_search("C", Graph, visited={"B"}))
    assert nodes == []

def test_merge_nodes():
    Graph = graph_from_edges(("A", "B", "C"), ("B", "D"))

    mGraph = merge_nodes(Graph, {"B": "A"})
    assert set(mGraph.nodes) == {"A", "C", "D"}
    assert set(mGraph.edges) == {("A", "C"), ("A", "D")}
    # check Graph wasn't modified
    assert set(Graph.nodes) == {"A", "B", "C", "D"}

    mGraph = merge_nodes(Graph, {"C": "B", "D": "B"})
    assert set(mGraph.nodes) == {"A", "B"}
    assert set(mGraph.edges) == {("A", "B")}

max_stars_repo_name: gfournier/aikit

max_issues_repo_name: gfournier/aikit

max_forks_repo_name: gfournier/aikit

reason: import networkx


text:
import numpy as np
import PointSet
# from MainProgram.Utils import SENSE_BOUND
# from MainProgram import Utils
import time
import copy

class Coreset(PointSet.PointSet):
    def __init__(self, P=None, W=None, _sense_bound_lambda=None,
                 max_ellipsoid_iters=10, use_svd=True, problem_type=None):
        PointSet.PointSet.__init__(self,P=P, W=W, ellipsoid_max_iters=max_ellipsoid_iters, problem_type=problem_type,
                                   use_svd=use_svd)
        self.rank = 0
        self.sensitivities = None
        self._sense_bound_lambda = _sense_bound_lambda
        self.probability = None
        self.S = None
        self.rank = 0
        self.use_svd = use_svd
        self.time_taken = 0
        self.problem_type = problem_type

    def sampleCoreset(self, P, sensitivity, sample_size, random_state=0):
        startTime = time.time()
        weights = P.W
        # Compute the sum of sensitivities.
        t = np.sum(sensitivity)

        # The probability of a point prob(p_i) = s(p_i) / t
        self.probability = sensitivity.flatten() / t

        # The number of points is equivalent to the number of rows in P.
        n = P.n

        # initialize new seed
        np.random.seed(random_state)

        # Multinomial distribution.
        indxs = np.random.choice(n, sample_size, p=self.probability.flatten())

        # Compute the frequencies of each sampled item.
        hist = np.histogram(indxs, bins=range(n))[0].flatten()
        indxs = copy.deepcopy(np.nonzero(hist)[0])

        # Select the indices.
        S = P.P[indxs, :]

        # Compute the weights of each point: w_i = (number of times i is sampled) / (sampleSize * prob(p_i))
        weights = np.asarray(np.multiply(weights[indxs], hist[indxs]), dtype=float).flatten()

        # Compute the weights of the coreset
        weights = np.multiply(weights, 1.0 / (self.probability[indxs] * sample_size))
        self.time_taken = time.time() - startTime

        self.S = PointSet.PointSet(P=S, W=weights, ellipsoid_max_iters=self.ellipsoid_max_iters,
                                   problem_type=self.problem_type, use_svd=self.use_svd, compute_U=False)
        return self.S, self.time_taken

    def computeSensitivity(self, P, sum_old_weights=None):
        sensitivitiy = np.empty(P.n, )

        if sum_old_weights is None:
            sum_old_weights = np.sum(P.W)

        sensitivity = np.empty((P.n, ))
        if 'lz' in self.problem_type:
            sensitivity = self._sense_bound_lambda(P.P, P.W, P.d)
        else:
            sensitivity[P.pos_idxs] = self._sense_bound_lambda(x=P.U[P.pos_idxs, :], w=P.W[P.pos_idxs],
                                                                args=(P.sum_weights_pos, P.sum_weights_neg,
                                                                      P.sum_weights)
                                                                )
            sensitivity[P.neg_idxs] = self._sense_bound_lambda(x=P.U[P.neg_idxs, :], w=P.W[P.neg_idxs],
                                                                args=(P.sum_weights_neg, P.sum_weights_pos,
                                                                      P.sum_weights)
                                                                )
        return sensitivity

    def mergeCoresets(self, coreset, sample_size):
        self.S.mergePointSet(coreset.S)
        sens = self.computeSensitivity(P=self.S)
        self.sampleCoreset(P=self.S, sensitivity=sens, sample_size=sample_size)
        self.rank += 1

# SENSE_BOUNDS = {
#     'logisitic': (lambda x, w, args=None: 32 / LAMBDA * (2 * w / args[0] + w * np.linalg.norm(x, 2) ** 2) * args[0]),
#     'nclz': (lambda x, w, args=None: w * np.linalg.norm(x, ord=Z, axis=1) ** Z),
#     'svm': (lambda x, w, args=None: max(9 * w / args[0], 2 * w / args[1]) + 13 * w / (4 * args[0]) +
#                                     125 * (args[2]) / (4 * LAMBDA) * (w * np.linalg.norm(x, 2)**2 +
#                                                                           w/(args[0] + args[1]))),
#     'restricted_lz': (lambda x, w, args=None: w * min(np.linalg.norm(x, ord=2),
#                                                     args[1] ** np.abs(0.5 - 1/Z) * np.linalg.norm(args[0]))),
#     'lz': (lambda x, w, args=None: w * np.linalg.norm(x, Z) ** Z if 1 <= Z <= 2
#             else args[0]**(Z/2) * w * np.linalg.norm(x, ord=Z, axis=1)**Z),
#     'lse': (lambda x, w, args=None: w * np.linalg.norm(x, 1, axis=1))
# }

max_stars_repo_name: muradtuk/UnifiedFramework

max_issues_repo_name: muradtuk/UnifiedFramework

max_forks_repo_name: muradtuk/UnifiedFramework


text:
"""
Created by Hamid Eghbal-zadeh at 05.02.21
Johannes Kepler University of Linz
"""

import torch
import torch.nn as nn

import numpy as np

from carla.agents.utils.weight_init import weights_init_he, weights_init_xavier, weights_init_kaiming_normal
from carla.architectures.sebastian.InvRes import Encoder, InvertedResidualVAEModel
from carla.architectures.vae import BasicBetaVAE

def mlp(sizes, activation, output_activation=nn.Identity, normalize_input=False):
    # remove zero in sizes
    sizes = [e for e in sizes if e > 0]
    layers = []
    if normalize_input:
        layers += [nn.BatchNorm1d(sizes[0])]
    for j in range(len(sizes) - 1):
        act = activation if j < len(sizes) - 2 else output_activation
        layers += [nn.Linear(sizes[j], sizes[j + 1]), act()]
    return nn.Sequential(*layers)

def cnn(sizes, obs_shape, out_hidden, activation, kernel_size, stride, output_activation=nn.Identity, bottleneck=False):
    # remove zero in sizes
    sizes = [e for e in sizes if e > 0]
    layers = []
    for j in range(len(sizes) - 1):
        layers += [nn.Conv2d(sizes[j], sizes[j + 1], kernel_size=kernel_size, stride=stride, padding=1), activation()]

    if bottleneck:
        # infer shape
        feat_final_dim = np.prod(nn.Sequential(*layers)(torch.zeros(1, *obs_shape).permute(0, 3, 1, 2)).shape)
        layers += [nn.Flatten(), nn.Linear(feat_final_dim, out_hidden), output_activation()]

    return nn.Sequential(*layers)

# class Permute(nn.Module):
#     def __init__(self, dims):
#         super().__init__()
#         self.dims = dims,  # extra comma

#     def forward(self, x):
#         return x.permute(*self.dims)

def invres(obs_shape, out_hidden, output_activation=nn.Identity):

    layers = [  # Permute([0,3,1,2]),
        Encoder(out_hidden, obs_shape[-1], obs_shape[0]), output_activation()
    ]
    return nn.Sequential(*layers)

def invres_nobn(obs_shape, out_hidden, output_activation=nn.Identity):

    layers = [  # Permute([0,3,1,2]),
        Encoder(out_hidden, obs_shape[-1], obs_shape[0], norm_layer=nn.Identity), output_activation()
    ]
    return nn.Sequential(*layers)

def get_clf_model(init, latent_dim, type, n_context=90):
    from carla.architectures.clf import LinearContextClf, MLPContextClf
    if type == 'linear':
        model = LinearContextClf(n_latent=latent_dim, n_context=n_context)
    elif type == 'mlp':
        model = MLPContextClf(n_latent=latent_dim, n_context=n_context)
    else:
        raise NotImplementedError('{} is not implemented!'.format(type))

    if init == 'he':
        model.init_weights(weights_init_he)
        model.apply(weights_init_he)
    elif init == 'xavier':
        model.init_weights(weights_init_xavier)
        model.apply(weights_init_xavier)
    elif init == 'kaiming':
        model.init_weights(weights_init_kaiming_normal)
        model.apply(weights_init_kaiming_normal)
    return model

def get_model(kernel_sizes, output_padding_lst, latent_dim, hidden_dims, img_size, loss_type, vae_gamma, vae_beta,
              nonlin, enc_bn, dec_bn, dec_out_nonlin, prior, soft_clip, init, strides, paddings, batch_size,
              vae_c_max=25, vae_c_stop_iter=100, vae_geco_goal=0.5, vae_reduction='mean', **kwargs):
    model = BasicBetaVAE(in_channels=3, kernel_sizes=kernel_sizes, latent_dim=latent_dim, hidden_dims=hidden_dims,
                         loss_type=loss_type, img_size=img_size, nonlin=nonlin, enc_bn=enc_bn, dec_bn=dec_bn,
                         output_padding_lst=output_padding_lst, gamma=vae_gamma, beta=vae_beta,
                         dec_out_nonlin=dec_out_nonlin, prior=prior, soft_clip=soft_clip, strides=strides,
                         paddings=paddings, batch_size=batch_size, max_capacity=vae_c_max,
                         vae_c_stop_iter=vae_c_stop_iter, geco_goal=vae_geco_goal, reduction=vae_reduction)

    def count_parameters(model):
        return sum(p.numel() for p in model.parameters() if p.requires_grad)

    print(count_parameters(model))

    d = {}
    for module in model.named_modules():
        n = module[1].__class__.__name__
        print(n, count_parameters(module[1]))
        if not(n in d):
            d[n] = 0

        d[n] += 1

    print(d)
    raise "nope"

    if init == 'he':
        model.init_weights(weights_init_he)
        model.apply(weights_init_he)
    elif init == 'xavier':
        model.init_weights(weights_init_xavier)
        model.apply(weights_init_xavier)
    elif init == 'kaiming':
        model.init_weights(weights_init_kaiming_normal)
        model.apply(weights_init_kaiming_normal)
    return model

# def get_model(latent_dim, img_size, **kwargs):

#     model = InvertedResidualVAEModel(num_latents=latent_dim, num_channels=3, image_size=img_size)

#     # init not important if we use pretrained
#     # if init == 'he':
#     #     model.init_weights(weights_init_he)
#     #     model.apply(weights_init_he)
#     # elif init == 'xavier':
#     #     model.init_weights(weights_init_xavier)
#     #     model.apply(weights_init_xavier)
#     # elif init == 'kaiming':
#     #     model.init_weights(weights_init_kaiming_normal)
#     #     model.apply(weights_init_kaiming_normal)
#     return model

max_stars_repo_name: sebzap/CarlaRL

max_issues_repo_name: sebzap/CarlaRL

max_forks_repo_name: sebzap/CarlaRL


text:
# METHODS USED IN GRAPH EXPANSION

from rtree import index
from tqdm.notebook import tqdm
import networkx as nx
import pandas as pd
import random
from scipy.stats import entropy
import math
import numpy as np
from sklearn.cluster import KMeans
import time
import heapq
import geopandas as gpd
from random import sample
from statistics import mean, median

def load_data(input_file, min_lon=None, max_lon=None, min_lat=None, max_lat=None):

    col_sep = ';'
    header = None
    col_names = ['id', 'name', 'lon', 'lat', 'kwds']
    df = pd.read_csv(input_file, delimiter=col_sep, header=header, names=col_names, error_bad_lines=False)

    df = df[pd.to_numeric(df['lon'], errors='coerce').notnull()]
    df = df[pd.to_numeric(df['lat'], errors='coerce').notnull()]
    df['lon'] = df['lon'].apply(lambda x: float(x))
    df['lat'] = df['lat'].apply(lambda x: float(x))
    df['cat1'] = df['kwds'].apply(lambda x: x.split(',')[0])
    df['cat2'] = df['kwds'].apply(lambda x: x.split(',')[1])
    df = df.drop('kwds', 1)

    if min_lon != None:
        df = df.loc[df['lon'] >= min_lon]
        df = df.loc[df['lon'] <= max_lon]
        df = df.loc[df['lat'] >= min_lat]
        df = df.loc[df['lat'] <= max_lat]

    df.reset_index(drop=True, inplace=True)

    types1 = set(df['cat1'].tolist())
    type1_colors = {t: "#"+''.join([random.choice('0123456789ABCDEF') for j in range(6)]) for t in types1}
    types2 = set(df['cat2'].tolist())
    type2_colors = {t: "#"+''.join([random.choice('0123456789ABCDEF') for j in range(6)]) for t in types2}

    # select category level
    cat_level = 1
    if cat_level == 1:
        cat = 'cat1'
        types = types1
        colors = type1_colors
    else:
        cat = 'cat2'
        types = types2
        colors = type2_colors

    print('#types: ' + str(len(types)))

    type_count_global = {t: len(df.loc[df[cat] == t].index) for t in types}
    n_total_global = len(df.index)
    print('#points: ' + str(n_total_global))

    global_distr = [type_count_global[t] / n_total_global for t in type_count_global]
    print('global entropy: ' + str(entropy(global_distr)))

    max_se = math.log(len(types))
    print('max entropy: ' + str(max_se))

    return df, types, colors

def compute_score(init, params):

    size = sum(init)
    distr = [x / size for x in init]

    rel_se = entropy(distr) / params['settings']['max_se']
    rel_size = size / params['variables']['max_size']['current']

    if params['entropy_mode']['current'] == 'high':
        score = rel_se * (rel_size ** params['variables']['size_weight']['current'])
    else:
        rel_se = 1 - rel_se
        score = rel_se * (rel_size ** params['variables']['size_weight']['current'])

    return score, rel_se

def create_graph(df, eps):

    # create R-tree index
    rtree = index.Index()

    for idx, row in tqdm(df.iterrows()):
        left, bottom, right, top = row['lon'], row['lat'], row['lon'], row['lat']
        rtree.insert(idx, (left, bottom, right, top))

    # construct the graph
    G = nx.Graph()

    for idx, row in tqdm(df.iterrows()):

        # create vertex
        G.add_nodes_from([(idx, {'cat': df.loc[idx]['cat1']})])

        # retrieve neighbors and create edges
        neighbors = list()
        left, bottom, right, top = row['lon'] - eps, row['lat'] - eps, row['lon'] + eps, row['lat'] + eps
        neighbors = [n for n in rtree.intersection((left, bottom, right, top))]
        a = np.array([df.loc[idx]['lon'], df.loc[idx]['lat']])
        for n in neighbors:
            if idx < n:
                b = np.array([df.loc[n]['lon'], df.loc[n]['lat']])
                dist = np.linalg.norm(a - b)
                if dist <= eps:
                    G.add_edge(idx, n)

    # check max node degree
    cc = [d for n, d in G.degree()]
    max_degree = sorted(cc)[-1] + 1
    mean_degree = mean(cc)
    median_degree = median(cc)
    print('Max degree: ' + str(max_degree) + ' Mean degree: ' + str(mean_degree) + ' Median degree: ' + str(median_degree))

    # check connected components
    print('Max connected component: ' + str([len(c) for c in sorted(nx.connected_components(G), key=len, reverse=True)][0]))

    return G, rtree

def create_seeds(df, rtree, seeds_ratio):

    # Compute clusters with k-means
    num_clusters = int(seeds_ratio * len(df.index))
    X = df[['lon', 'lat']].to_numpy()
    kmeans = KMeans(n_clusters=num_clusters).fit(X)
    cluster_labels = kmeans.labels_
    cluster_centers = kmeans.cluster_centers_

    # Pick seeds from cluster centroids
    seeds = []
    for c in cluster_centers:
        seeds.append(list(rtree.nearest([c[0], c[1]], 1))[0])

    return seeds

def init_scores(G, types, params):

    for v in G.nodes:

        # create region
        region_core = {v}
        region_border = set(G[v])
        region = region_core.union(region_border)

        # compute 'init'
        categories = [G.nodes[n]['cat'] for n in region]
        init = [categories.count(t) for t in types]

        # compute score
        score, rel_se = compute_score(init, params)

        # store the information in the node
        G.nodes[v]['init_score'] = score

def init_queue(G, seeds, types, params, topk_regions, start_time, updates):

    queue = []

    # for v in tqdm(seeds):
    for v in seeds:

        # create region
        region_core = {v}
        region_border = set(G[v])
        region = region_core.union(region_border)

        # check if border node is actually core
        border_to_core = set()
        for n in region_border:
            has_new_neighbors = False
            for nn in set(G[n]):
                if nn not in region:
                    has_new_neighbors = True
                    break
            if not has_new_neighbors:
                border_to_core.add(n)
        for n in border_to_core:
            region_border.remove(n)
            region_core.add(n)

        # check max size
        if len(region) > params['variables']['max_size']['current']:
            continue

        # compute 'init'
        categories = [G.nodes[n]['cat'] for n in region]
        init = [categories.count(t) for t in types]

        # compute score
        score, rel_se = compute_score(init, params)

        # update top-k regions
        if score > topk_regions[params['settings']['top_k'] - 1][0]:
            topk_regions.append([score, rel_se,
                                 [region_core.copy(), region_border.copy()],
                                 init.copy(), len(region)])
            topk_regions = sorted(topk_regions, key=lambda topk_regions: topk_regions[0], reverse=True)
            topk_regions = topk_regions[:-1]
            updates[time.time() - start_time] = topk_regions[params['settings']['top_k'] - 1][0]
            # print('init ' + str(score))

        # add to queue if border is not empty
        if len(region_border) > 0:
            heapq.heappush(queue, (-score, (region_core.copy(), region_border.copy())))
            # print(score, (region_core.copy(), region_border.copy()))

    # print('Total queue entries: ' + str(len(queue)))

    return queue, topk_regions

def expand_region(G, region_core, region_border, nodes_to_expand, params, types):

    new_region_core = region_core.copy()
    new_region_border = region_border.copy()

    for n in nodes_to_expand:

        # move selected border node to core
        new_region_border.remove(n)
        new_region_core.add(n)

        # find new neighbors and add them to border
        new_neighbors = set(G[n])
        for nn in new_neighbors:
            if nn not in new_region_core:
                new_region_border.add(nn)

    # get the newly formed region
    new_region = new_region_core.union(new_region_border)

    # check if border node is actually core
    border_to_core = set()
    for n in new_region_border:
        has_extra_neighbors = False
        for nn in set(G[n]):
            if nn not in new_region:
                has_extra_neighbors = True
                break
        if not has_extra_neighbors:
            border_to_core.add(n)
    for n in border_to_core:
        new_region_border.remove(n)
        new_region_core.add(n)

    # compute 'init'
    categories = [G.nodes[n]['cat'] for n in new_region]
    init = [categories.count(t) for t in types]

    # compute score
    score, rel_se = compute_score(init, params)

    return new_region, new_region_core, new_region_border, init, score, rel_se

def process_queue(G, queue, topk_regions, params, types, start_time, updates):

    # POP THE NEXT REGION TO EXPAND
    t = heapq.heappop(queue)
    score, region_core, region_border = -t[0], t[1][0], t[1][1]

    if params['methods']['current'] == 'ExpSingle':   # FIND THE BEST BORDER NODE TO EXPAND

        best_region_core = set()
        best_region_border = set()
        best_region_score = -1
        best_region_rel_se = -1

        for n in region_border:

            # expand region with this border point
            new_region, new_region_core, new_region_border, init, new_score, new_rel_se = expand_region(
                G, region_core, region_border, [n], params, types
            )

            # check max size
            if len(new_region) > params['variables']['max_size']['current']:
                continue

            # update top-k regions
            if new_score > topk_regions[params['settings']['top_k'] - 1][0]:
                topk_regions.append([new_score,
                                     new_rel_se,
                                     [new_region_core.copy(), new_region_border.copy()],
                                     init.copy(),
                                     len(new_region)])
                topk_regions = sorted(topk_regions, key=lambda topk_regions: topk_regions[0], reverse=True)
                topk_regions = topk_regions[:-1]
                updates[time.time() - start_time] = topk_regions[params['settings']['top_k'] - 1][0]

            # update current best score
            if new_score > best_region_score and len(new_region_border) > 0:
                best_region_core = new_region_core.copy()
                best_region_border = new_region_border.copy()
                best_region_score = new_score
                best_region_rel_se = new_rel_se

        # ADD THE BEST FOUND NEW REGION TO QUEUE
        if best_region_score > -1:
            heapq.heappush(queue, (-best_region_score, (best_region_core.copy(), best_region_border.copy())))

        return best_region_score, topk_regions

    elif params['methods']['current'] == 'ExpAll':   # EXPAND THE ENTIRE BORDER

        # expand region with all border points
        new_region, new_region_core, new_region_border, init, new_score, new_rel_se = expand_region(
            G, region_core, region_border, region_border, params, types
        )

        # check max size
        if len(new_region) > params['variables']['max_size']['current']:
            return -1, topk_regions

        # update top-k regions
        if new_score > topk_regions[params['settings']['top_k'] - 1][0]:
            topk_regions.append([new_score,
                                 new_rel_se,
                                 [new_region_core.copy(), new_region_border.copy()],
                                 init.copy(),
                                 len(new_region)])
            topk_regions = sorted(topk_regions, key=lambda topk_regions: topk_regions[0], reverse=True)
            topk_regions = topk_regions[:-1]
            updates[time.time() - start_time] = topk_regions[params['settings']['top_k'] - 1][0]

        # ADD THE NEW REGION TO QUEUE
        if len(new_region_border) > 0:
            heapq.heappush(queue, (-new_score, (new_region_core.copy(), new_region_border.copy())))

        return new_score, topk_regions

def get_region_shapes(df, topk_regions):
    region_shapes = []
    for r in topk_regions:
        region = df.loc[df.index.isin(r[2][0].union(r[2][1]))]
        region = gpd.GeoDataFrame(region, geometry=gpd.points_from_xy(region.lon, region.lat))
        region = region.unary_union.convex_hull
        region_shapes.append(region)
    region_shapes = pd.DataFrame(region_shapes, columns=['geom'])
    region_shapes = gpd.GeoDataFrame(region_shapes, geometry='geom')
    region_shapes.crs = {'init':'epsg:4326'}
    return region_shapes

def run_exp_hybrid(G, seeds, params, types, topk_regions, start_time, updates):

    # create priority queue for regions
    queue = []

    # PART I: For each seed, perform ExpAll
    for v in seeds:

        # initialize best local region
        best_region_score = 0
        best_region = set()

        # initialize region
        region_core = {v}
        region_border = set(G[v])
        region = region_core.union(region_border)

        # expand region until max size
        while len(region) <= params['variables']['max_size']['current'] and len(region_border) > 0 and (time.time() - start_time) < params['variables']['time_budget']['current']:

            # compute 'init'
            categories = [G.nodes[n]['cat'] for n in region]
            init = [categories.count(t) for t in types]

            # compute score
            score, rel_se = compute_score(init, params)

            # update top-k regions
            if score > topk_regions[params['settings']['top_k'] - 1][0]:
                topk_regions.append([score, rel_se,
                                     [region_core.copy(), region_border.copy()],
                                     init.copy(), len(region)])
                topk_regions = sorted(topk_regions, key=lambda topk_regions: topk_regions[0], reverse=True)
                topk_regions = topk_regions[:-1]
                updates[time.time() - start_time] = topk_regions[params['settings']['top_k'] - 1][0]

            # check if local best
            if score > best_region_score:
                best_region_score = score
                best_region = region.copy()

            # check if border node is actually core
            border_to_core = set()
            for n in region_border:
                has_new_neighbors = False
                for nn in set(G[n]):
                    if nn not in region:
                        has_new_neighbors = True
                        break
                if not has_new_neighbors:
                    border_to_core.add(n)
            for n in border_to_core:
                region_border.remove(n)
                region_core.add(n)

            # expand region with all border points
            region, region_core, region_border, init, score, rel_se = expand_region(
                G, region_core, region_border, region_border, params, types
            )

        # add best found region to queue
        if len(best_region) > 0:
            heapq.heappush(queue, (-best_region_score, best_region))

    # PART II: For each seed region, perform ExpSingle
    while len(queue) > 0 and (time.time() - start_time) < params['variables']['time_budget']['current']:

        # get the next seed region
        t = heapq.heappop(queue)
        score, seed_region = -t[0], t[1]

        # pick a seed
        v = seed_region.pop()

        # initialize region
        region_core = {v}
        region_border = set(G[v])
        region = region_core.union(region_border)

        # initialize best local region
        best_local_region_score = 0

        # expand region until max size
        while len(region) <= params['variables']['max_size']['current'] and len(region_border) > 0 and (time.time() - start_time) < params['variables']['time_budget']['current']:

            # compute 'init'
            categories = [G.nodes[n]['cat'] for n in region]
            init = [categories.count(t) for t in types]

            # compute score
            score, rel_se = compute_score(init, params)

            # update top-k regions
            if score > topk_regions[params['settings']['top_k'] - 1][0]:
                topk_regions.append([score, rel_se,
                                     [region_core.copy(), region_border.copy()],
                                     init.copy(), len(region)])
                topk_regions = sorted(topk_regions, key=lambda topk_regions: topk_regions[0], reverse=True)
                topk_regions = topk_regions[:-1]
                updates[time.time() - start_time] = topk_regions[params['settings']['top_k'] - 1][0]

            # check if border node is actually core
            border_to_core = set()
            for n in region_border:
                has_new_neighbors = False
                for nn in set(G[n]):
                    if nn not in region:
                        has_new_neighbors = True
                        break
                if not has_new_neighbors:
                    border_to_core.add(n)
            for n in border_to_core:
                region_border.remove(n)
                region_core.add(n)

            # find best border point to expand

            best_region_core = set()
            best_region_border = set()
            best_region_score = -1
            best_region_rel_se = -1

            for n in region_border:

                # expand region with this border point
                new_region, new_region_core, new_region_border, init, new_score, new_rel_se = expand_region(
                    G, region_core, region_border, [n], params, types
                )

                # check max size
                if len(new_region) > params['variables']['max_size']['current']:
                    continue

                # update top-k regions
                if new_score > topk_regions[params['settings']['top_k'] - 1][0]:
                    topk_regions.append([new_score,
                                         new_rel_se,
                                         [new_region_core.copy(), new_region_border.copy()],
                                         init.copy(),
                                         len(new_region)])
                    topk_regions = sorted(topk_regions, key=lambda topk_regions: topk_regions[0], reverse=True)
                    topk_regions = topk_regions[:-1]
                    updates[time.time() - start_time] = topk_regions[params['settings']['top_k'] - 1][0]

                # update current best score
                if new_score > best_region_score and len(new_region_border) > 0:
                    best_region_core = new_region_core.copy()
                    best_region_border = new_region_border.copy()
                    best_region_score = new_score
                    best_region_rel_se = new_rel_se

            # set current region to best
            region_core = best_region_core
            region_border = best_region_border
            region = region_core.union(region_border)

            # update best local score
            if best_region_score > best_local_region_score:
                best_local_region_score = best_region_score

        # push back to queue with new score
        if len(seed_region) > 0:
            heapq.heappush(queue, (-best_local_region_score, seed_region))

    return topk_regions

def run_method(df, G, rtree, types, params, start_time, seeds):

    # Initialize top-k list
    topk_regions = []
    while len(topk_regions) < params['settings']['top_k']:
        topk_regions.append([0, 0, [set(), set()], [], 0])   # [score, rel_se, [region_core, region_border], init, length]

    iterations = 0
    updates = dict()

    if params['methods']['current'] == 'ExpHybrid':
        topk_regions = run_exp_hybrid(G, seeds, params, types, topk_regions, start_time, updates)
    else:
        # Initialize queue
        queue, topk_regions = init_queue(G, seeds, types, params, topk_regions, start_time, updates)

        # Process queue
        while (time.time() - start_time) < params['variables']['time_budget']['current'] and len(queue) > 0:
            iterations += 1
            score, topk_regions = process_queue(G, queue, topk_regions, params, types, start_time, updates)

    return topk_regions[0][0], topk_regions[0][2][0].union(topk_regions[0][2][1]), updates # return score, region and updates

max_stars_repo_name: smartdatalake/mbrs

max_issues_repo_name: smartdatalake/mbrs

max_forks_repo_name: smartdatalake/mbrs

reason: import numpy,from scipy,import networkx


text:
import numpy as np
import torch
from pytorch_toolbelt.inference.tiles import ImageSlicer, CudaTileMerger
from pytorch_toolbelt.utils.torch_utils import (
    tensor_from_rgb_image,
    rgb_image_from_tensor,
    to_numpy,
)
from torch import nn
from torch.utils.data import DataLoader
import pytest

skip_if_no_cuda = pytest.mark.skipif(
    not torch.cuda.is_available(), reason="Cuda is not available"
)

def test_tiles_split_merge():
    image = np.random.random((500, 500, 3)).astype(np.uint8)
    tiler = ImageSlicer(image.shape, tile_size=51, tile_step=26, weight="mean")
    tiles = tiler.split(image)
    merged = tiler.merge(tiles, dtype=np.uint8)
    np.testing.assert_equal(merged, image)

def test_tiles_split_merge_non_dividable():
    image = np.random.random((563, 512, 3)).astype(np.uint8)
    tiler = ImageSlicer(
        image.shape, tile_size=(128, 128), tile_step=(128, 128), weight="mean"
    )
    tiles = tiler.split(image)
    merged = tiler.merge(tiles, dtype=np.uint8)
    np.testing.assert_equal(merged, image)

@skip_if_no_cuda
def test_tiles_split_merge_non_dividable_cuda():

    image = np.random.random((5632, 5120, 3)).astype(np.uint8)
    tiler = ImageSlicer(
        image.shape, tile_size=(1280, 1280), tile_step=(1280, 1280), weight="mean"
    )
    tiles = tiler.split(image)

    merger = CudaTileMerger(
        tiler.target_shape, channels=image.shape[2], weight=tiler.weight
    )
    for tile, coordinates in zip(tiles, tiler.crops):
        # Integrate as batch of size 1
        merger.integrate_batch(
            tensor_from_rgb_image(tile).unsqueeze(0).float().cuda(), [coordinates]
        )

    merged = merger.merge()
    merged = rgb_image_from_tensor(merged, mean=0, std=1, max_pixel_value=1)
    merged = tiler.crop_to_orignal_size(merged)

    np.testing.assert_equal(merged, image)

def test_tiles_split_merge_2():
    image = np.random.random((5000, 5000, 3)).astype(np.uint8)
    tiler = ImageSlicer(
        image.shape, tile_size=(512, 512), tile_step=(256, 256), weight="pyramid"
    )

    np.testing.assert_allclose(tiler.weight, tiler.weight.T)

    tiles = tiler.split(image)
    merged = tiler.merge(tiles, dtype=np.uint8)
    np.testing.assert_equal(merged, image)

@skip_if_no_cuda
def test_tiles_split_merge_cuda():
    class MaxChannelIntensity(nn.Module):
        def __init__(self):
            super().__init__()

        def forward(self, input):
            max_channel, _ = torch.max(input, dim=1, keepdim=True)
            return max_channel

    image = np.random.random((5000, 5000, 3)).astype(np.uint8)
    tiler = ImageSlicer(
        image.shape, tile_size=(512, 512), tile_step=(256, 256), weight="pyramid"
    )
    tiles = [tensor_from_rgb_image(tile) for tile in tiler.split(image)]

    model = MaxChannelIntensity().eval().cuda()

    merger = CudaTileMerger(tiler.target_shape, 1, tiler.weight)
    for tiles_batch, coords_batch in DataLoader(
        list(zip(tiles, tiler.crops)), batch_size=8, pin_memory=True
    ):
        tiles_batch = tiles_batch.float().cuda()
        pred_batch = model(tiles_batch)

        merger.integrate_batch(pred_batch, coords_batch)

    merged = np.moveaxis(to_numpy(merger.merge()), 0, -1).astype(np.uint8)
    merged = tiler.crop_to_orignal_size(merged)

    np.testing.assert_equal(merged, image.max(axis=2, keepdims=True))

max_stars_repo_name: valyukov/pytorch-toolbelt

max_issues_repo_name: valyukov/pytorch-toolbelt

max_forks_repo_name: valyukov/pytorch-toolbelt


text:
#!/usr/bin/env python

import numpy as np
from abc import abstractmethod
from olympus import Logger
from olympus.campaigns import Observations, Campaign
from olympus.objects import Object, abstract_attribute, ABCMeta, Config

class AbstractPlanner(Object, metaclass=ABCMeta):
    """ This class is intended to contain methods shared by all wrappers, as well as define abstract methods and
    attributes that all planner wrappers need to implement. It is not meant to be exposed to the standard user,
    although more advanced users can use this class to create custom planner classes.
    """

    def __init__(self, *args, **kwargs):
        Object.__init__(self, *args, **kwargs)
        self.num_generated = 0
        self.param_space = None
        self._params = None
        self._values = None
        self.SUBMITTED_PARAMS = []
        self.RECEIVED_VALUES  = []

        # rm all those vars in config that are not needed/used by ScipyWrapper
        for var in ['goal', 'init_guess', 'random_seed']:
            if var in kwargs:
                del kwargs[var]
        self.config = Config(from_dict=kwargs)

        # self.goal is an abstract attribute that needs to be defined by the subclasses of AbstractPlanner
        # Since all planner wrappers are implemented in minimization mode, we flip the measurements if we want to
        # perform a maximization
        if self.goal == 'minimize':
            self.flip_measurements = False
        elif self.goal == 'maximize':
            self.flip_measurements = True
        else:
            message = f'attribute `goal` can only be "minimize" or "maximize". "{self.goal}" is not a valid value'
            Logger.log(message, 'ERROR')

    def __repr__(self):
        if not hasattr(self, 'param_space') or self.param_space is None:
            param_space = 'undefined'
        else:
            param_space = 'defined'
        return f"<Planner (kind={self.kind}, param_space={param_space})>"

    @abstract_attribute
    def goal(self):
        pass

    @property
    def kind(self):
        return type(self).__name__

    @abstractmethod
    def _set_param_space(self, *args, **kwargs):
        """Method that returns the parameter space over which to optimize in the format used by a specific optimization
        library/wrapper.
        """
        pass

    @abstractmethod
    def _tell(self, *args, **kwargs):
        """Method that returns all previous observations as required by a specific optimization library/wrapper.
        """
        pass

    @abstractmethod
    def _ask(self, *args, **kwargs):
        """Method that returns the next query point based on all previous observations.
        """
        pass

    def set_param_space(self, param_space):
        """Defines the parameter space over which the planner will search.

        Args:
            param_space (ParameterSpace): a ParameterSpace object defining the space over which to search.
        """
        self.param_space = param_space
        self._set_param_space(param_space)

    def ask(self, return_as=None):
        """ suggest new set of parameters

        Args:
            return_as (string): choose data type for returned parameters
                allowed options (dict, array)

        Returns:
            ParameterVector: newly generated parameters
        """

        self.num_generated += 1
        param_vector = self._ask()

        # check that the parameters suggested are within the bounds of our param_space
        self._validate_paramvector(param_vector)

        if return_as is not None:
            try:
                param_vector = getattr(param_vector, 'to_{}'.format(return_as))()
            except AttributeError:
                Logger.log('could not return param_vector as "{}"'.format(return_as), 'ERROR')
        return param_vector

    def tell(self, observations=Observations()):
        """Provide the planner with all previous observations.

        Args:
            observations (Observations): an Observation object containing all previous observations. This defines the
                history of the campaign seen by the planner. The default is None, i.e. there are no previous
                observations.
        """
        self._tell(observations)

    def recommend(self, observations=None, return_as=None):
        """Consecutively executes tell and ask: tell the planner about all previous observations, and ask about the
        next query point.

        Args:
            observations (list of ???)
            return_as (string): choose data type for returned parameters
                allowed options (dict, array)

        Returns:
            list: newly generated parameters
        """
        self.tell(observations)
        return self.ask(return_as=return_as)

    def optimize(self, emulator, num_iter=1, verbose=False):
        """Optimizes a surface for a fixed number of iterations.

        Args:
            emulator (object): Emulator or a Surface instance to optimize over.
            num_iter (int): Maximum number of iterations allowed.
            verbose (bool): Whether to print information to screen.

        Returns:
            campaign (Campaign): Campaign object with information about the optimization, including all parameters
                tested and measurements obtained.
        """

        # update num_iter if needed by the specific wrapper
        if hasattr(self, 'num_iter') and self.num_iter != num_iter:
            Logger.log(f'Updating the number of sampling points of planner {type(self).__name__} to {num_iter}', 'INFO')
            self.num_iter = num_iter

        # same for budget
        if hasattr(self, 'budget') and self.budget != num_iter:
            Logger.log(f'Updating the number of sampling points of planner {type(self).__name__} to {num_iter}', 'INFO')
            self.budget = num_iter

        # reset planner if it has a 'reset' method. Assuming that if there is a 'reset' method it is needed here
        # This is used by Deap for example, to clear the latest population before doing another optimization
        if callable(getattr(self, "reset", None)):
            self.reset()

        # use campaign to store info, and then to be returned
        campaign = Campaign()
        campaign.set_planner_specs(self)
        campaign.set_emulator_specs(emulator)

        # provide the planner with the parameter space.
        # param space in emulator as it originates from dataset
        self.set_param_space(emulator.param_space)

        # Optimize: i.e. call the planner recommend method for max_iter times
        for i in range(num_iter):
            if verbose is True:
                Logger.log(f'Optimize iteration {i+1}', 'INFO')
                Logger.log(f'Obtaining parameters from planner...', 'INFO')
            # get new params from planner
            # NOTE: now we get 1 param at a time, a possible future expansion is to return batches
            params = self.recommend(observations=campaign.observations)

            # get measurement from emulator/surface
            if verbose is True:
                Logger.log(f'Obtaining measurement from emulator...', 'INFO')
            values = emulator.run(params.to_array(), return_paramvector=True)

            # store parameter and measurement pair in campaign
            campaign.add_observation(params, values)

        return campaign

    def _validate_paramvector(self, param_vector):
        for key, value in param_vector.to_dict().items():
            param = self.param_space.get_param(name=key)
            if param['type'] == 'continuous':
                if not param['low'] <= value <= param['high']:
                    message = 'Proposed parameter {0} not within defined bounds ({1},{2})'.format(value, param['low'], param['high'])
                    Logger.log(message, 'WARNING')

    def _project_into_domain(self, params):
        for _, bound in enumerate(self.param_space):
            params[_] = np.maximum(bound['low'], params[_])
            params[_] = np.minimum(bound['high'], params[_])
        return params

max_issues_repo_name: kiminh/olympus

max_forks_repo_name: kiminh/olympus


text:
import numpy as np
import imageio as img

class SpectrumPainter(object):
    def __init__(self, Fs=1000000, T_line=0.005):
        self.NFFT = 4096
        self.Fs = Fs
        self.T_line = T_line

    @property
    def repetitions(self):
        return int(np.ceil(self.T_line * self.Fs / self.NFFT))

    def convert_image(self, filename):
        pic = img.imread(filename)
        # Set FFT size to be double the image size so that the edge of the spectrum stays clear
        # preventing some bandfilter artifacts
        self.NFFT = 2*pic.shape[1]

        # Repeat image lines until each one comes often enough to reach the desired line time
        ffts = (np.flipud(np.repeat(pic[:, :, 0], self.repetitions, axis=0) / 16.)**2.) / 256.

        # Embed image in center bins of the FFT
        fftall = np.zeros((ffts.shape[0], self.NFFT))
        startbin = int(self.NFFT/4)
        fftall[:, startbin:(startbin+pic.shape[1])] = ffts

        # Generate random phase vectors for the FFT bins, this is important to prevent high peaks in the output
        # The phases won't be visible in the spectrum
        phases = 2*np.pi*np.random.rand(*fftall.shape)
        rffts = fftall * np.exp(1j*phases)

        # Perform the FFT per image line, then concatenate them to form the final signal
        timedata = np.fft.ifft(np.fft.ifftshift(rffts, axes=1), axis=1) / np.sqrt(float(self.NFFT))
        linear = timedata.flatten()
        linear = linear / np.max(np.abs(linear))
        return linear


text:
import cv2 as cv
import numpy as np
from config import MIN_NUMBER_OF_TESTS, MAX_NUMBER_OF_TESTS, MIN_IMAGE_DIM_SIZE, MAX_IMAGE_DIM_SIZE, MIN_NUMBER_OF_GROUND_TRUTH_DATA, MAX_NUMBER_OF_GROUND_TRUTH_DATA

def test_image(transformations):
    numberOfImages = np.random.randint(MIN_NUMBER_OF_TESTS, MAX_NUMBER_OF_TESTS)

    for i in range(numberOfImages):
        widthOfImage = np.random.randint(MIN_IMAGE_DIM_SIZE, MAX_IMAGE_DIM_SIZE)
        heightOfImage = np.random.randint(MIN_IMAGE_DIM_SIZE, MAX_IMAGE_DIM_SIZE)

        img = np.random.rand(heightOfImage, widthOfImage)
        for t in transformations:
            t.init(img)
            t.image(img)

def test_points(transformations):
    numberOfImages = np.random.randint(MIN_NUMBER_OF_TESTS, MAX_NUMBER_OF_TESTS)

    for i in range(numberOfImages):
        widthOfImage = np.random.randint(MIN_IMAGE_DIM_SIZE, MAX_IMAGE_DIM_SIZE)
        heightOfImage = np.random.randint(MIN_IMAGE_DIM_SIZE, MAX_IMAGE_DIM_SIZE)
        numberOfPoints = np.random.randint(MIN_NUMBER_OF_GROUND_TRUTH_DATA, MAX_NUMBER_OF_GROUND_TRUTH_DATA)

        img = np.random.rand(heightOfImage, widthOfImage)
        points = np.random.rand(numberOfPoints, 2)

        for t in transformations:
            t.init(img)
            t.points(points)

def test_rectangles(transformations):
    numberOfImages = np.random.randint(10, 100)

    for i in range(numberOfImages):
        widthOfImage = np.random.randint(MIN_IMAGE_DIM_SIZE, MAX_IMAGE_DIM_SIZE)
        heightOfImage = np.random.randint(MIN_IMAGE_DIM_SIZE, MAX_IMAGE_DIM_SIZE)
        numberOfRects = np.random.randint(MIN_NUMBER_OF_GROUND_TRUTH_DATA, MAX_NUMBER_OF_GROUND_TRUTH_DATA)

        img = np.random.rand(heightOfImage, widthOfImage)
        rects = np.random.rand(numberOfRects, 4)

        for t in transformations:
            t.init(img)
            t.rectangles(rects)

def test_polygons(transformations):
    numberOfImages = np.random.randint(10, 100)

    for i in range(numberOfImages):
        widthOfImage = np.random.randint(MIN_IMAGE_DIM_SIZE, MAX_IMAGE_DIM_SIZE)
        heightOfImage = np.random.randint(MIN_IMAGE_DIM_SIZE, MAX_IMAGE_DIM_SIZE)
        numberOfPolys = np.random.randint(MIN_NUMBER_OF_GROUND_TRUTH_DATA, MAX_NUMBER_OF_GROUND_TRUTH_DATA)

        polys = []
        for i in range(numberOfPolys):
            numberOfPoints = np.random.randint(MIN_NUMBER_OF_GROUND_TRUTH_DATA, MAX_NUMBER_OF_GROUND_TRUTH_DATA)
            poly = np.random.rand(numberOfPoints, 2)
            polys.append(poly)

        img = np.random.rand(heightOfImage, widthOfImage)

        for t in transformations:
            t.init(img)
            t.polygons(polys)


text:
import os
import pygame
import random
from pygame import *

import numpy as np

from .sprites.sprite_path import sprite_path

import gym
from gym import error, spaces, utils
from gym.utils import seeding

pygame.init()

scr_size = (width,height) = (600,150)
gravity = 0.6

black = (0,0,0)
white = (255,255,255)
background_col = (235,235,235)

screen = pygame.display.set_mode(scr_size)
clock = pygame.time.Clock()
pygame.display.set_caption("T-Rex Rush")

def load_image(
	name,
	sizex=-1,
	sizey=-1,
	colorkey=None,
	):

	#fullname = os.path.join('sprites', name)
	fullname = sprite_path(name)
	image = pygame.image.load(fullname)
	image = image.convert()
	if colorkey is not None:
		if colorkey is -1:
			colorkey = image.get_at((0, 0))
		image.set_colorkey(colorkey, RLEACCEL)

	if sizex != -1 or sizey != -1:
		image = pygame.transform.scale(image, (sizex, sizey))

	return (image, image.get_rect())

def load_sprite_sheet(
		sheetname,
		nx,
		ny,
		scalex = -1,
		scaley = -1,
		colorkey = None,
		):
	#fullname = os.path.join(sheetname)
	fullname = sprite_path(sheetname)
	sheet = pygame.image.load(fullname)
	sheet = sheet.convert()

	sheet_rect = sheet.get_rect()

	sprites = []

	sizex = sheet_rect.width/nx
	sizey = sheet_rect.height/ny

	for i in range(0,ny):
		for j in range(0,nx):
			rect = pygame.Rect((j*sizex,i*sizey,sizex,sizey))
			image = pygame.Surface(rect.size)
			image = image.convert()
			image.blit(sheet,(0,0),rect)

			if colorkey is not None:
				if colorkey is -1:
					colorkey = image.get_at((0,0))
				image.set_colorkey(colorkey,RLEACCEL)

			if scalex != -1 or scaley != -1:
				image = pygame.transform.scale(image,(scalex,scaley))

			sprites.append(image)

	sprite_rect = sprites[0].get_rect()

	return sprites,sprite_rect

def disp_gameOver_msg(retbutton_image,gameover_image):
	retbutton_rect = retbutton_image.get_rect()
	retbutton_rect.centerx = width / 2
	retbutton_rect.top = height*0.52

	gameover_rect = gameover_image.get_rect()
	gameover_rect.centerx = width / 2
	gameover_rect.centery = height*0.35

	screen.blit(retbutton_image, retbutton_rect)
	screen.blit(gameover_image, gameover_rect)

def extractDigits(number):
	if number > -1:
		digits = []
		i = 0
		while(number/10 != 0):
			digits.append(number%10)
			number = int(number/10)

		digits.append(number%10)
		for i in range(len(digits),5):
			digits.append(0)
		digits.reverse()
		return digits

class Dino():
	def __init__(self,sizex=-1,sizey=-1):
		self.images,self.rect = load_sprite_sheet('dino.png',5,1,sizex,sizey,-1)
		self.images1,self.rect1 = load_sprite_sheet('dino_ducking.png',2,1,59,sizey,-1)
		self.rect.bottom = int(0.98*height)
		self.rect.left = width/15
		self.image = self.images[0]
		self.index = 0
		self.counter = 0
		self.score = 0
		self.isJumping = False
		self.isDead = False
		self.isDucking = False
		self.isBlinking = False
		self.movement = [0,0]
		self.jumpSpeed = 11.5

		self.stand_pos_width = self.rect.width
		self.duck_pos_width = self.rect1.width

	def draw(self):
		screen.blit(self.image,self.rect)

	def checkbounds(self):
		if self.rect.bottom > int(0.98*height):
			self.rect.bottom = int(0.98*height)
			self.isJumping = False

	def update(self):
		if self.isJumping:
			self.movement[1] = self.movement[1] + gravity

		if self.isJumping:
			self.index = 0
		elif self.isBlinking:
			if self.index == 0:
				if self.counter % 400 == 399:
					self.index = (self.index + 1)%2
			else:
				if self.counter % 20 == 19:
					self.index = (self.index + 1)%2

		elif self.isDucking:
			if self.counter % 5 == 0:
				self.index = (self.index + 1)%2
		else:
			if self.counter % 5 == 0:
				self.index = (self.index + 1)%2 + 2

		if self.isDead:
			self.index = 4

		if not self.isDucking:
			self.image = self.images[self.index]
			self.rect.width = self.stand_pos_width
		else:
			self.image = self.images1[(self.index)%2]
			self.rect.width = self.duck_pos_width

		self.rect = self.rect.move(self.movement)
		self.checkbounds()

		if not self.isDead and self.counter % 7 == 6 and self.isBlinking == False:
			self.score += 1

		self.counter = (self.counter + 1)

class Cactus(pygame.sprite.Sprite):
	def __init__(self,speed=5,sizex=-1,sizey=-1):
		pygame.sprite.Sprite.__init__(self,self.containers)
		self.images,self.rect = load_sprite_sheet('cacti-small.png',3,1,sizex,sizey,-1)
		self.rect.bottom = int(0.98*height)
		self.rect.left = width + self.rect.width
		self.image = self.images[random.randrange(0,3)]
		self.movement = [-1*speed,0]

	def draw(self):
		screen.blit(self.image,self.rect)

	def update(self):
		self.rect = self.rect.move(self.movement)

		if self.rect.right < 0:
			self.kill()

class Ptera(pygame.sprite.Sprite):
	def __init__(self,speed=5,sizex=-1,sizey=-1):
		pygame.sprite.Sprite.__init__(self,self.containers)
		self.images,self.rect = load_sprite_sheet('ptera.png',2,1,sizex,sizey,-1)
		self.ptera_height = [height*0.82,height*0.75,height*0.60]
		self.rect.centery = self.ptera_height[random.randrange(0,3)]
		self.rect.left = width + self.rect.width
		self.image = self.images[0]
		self.movement = [-1*speed,0]
		self.index = 0
		self.counter = 0

	def draw(self):
		screen.blit(self.image,self.rect)

	def update(self):
		if self.counter % 10 == 0:
			self.index = (self.index+1)%2
		self.image = self.images[self.index]
		self.rect = self.rect.move(self.movement)
		self.counter = (self.counter + 1)
		if self.rect.right < 0:
			self.kill()

class Ground():
	def __init__(self,speed=-5):
		self.image,self.rect = load_image('ground.png',-1,-1,-1)
		self.image1,self.rect1 = load_image('ground.png',-1,-1,-1)
		self.rect.bottom = height
		self.rect1.bottom = height
		self.rect1.left = self.rect.right
		self.speed = speed

	def draw(self):
		screen.blit(self.image,self.rect)
		screen.blit(self.image1,self.rect1)

	def update(self):
		self.rect.left += self.speed
		self.rect1.left += self.speed

		if self.rect.right < 0:
			self.rect.left = self.rect1.right

		if self.rect1.right < 0:
			self.rect1.left = self.rect.right

class Cloud(pygame.sprite.Sprite):
	def __init__(self,x,y):
		pygame.sprite.Sprite.__init__(self,self.containers)
		self.image,self.rect = load_image('cloud.png',int(90*30/42),30,-1)
		self.speed = 1
		self.rect.left = x
		self.rect.top = y
		self.movement = [-1*self.speed,0]

	def draw(self):
		screen.blit(self.image,self.rect)

	def update(self):
		self.rect = self.rect.move(self.movement)
		if self.rect.right < 0:
			self.kill()

class Scoreboard():
	def __init__(self,x=-1,y=-1):
		self.score = 0
		self.tempimages,self.temprect = load_sprite_sheet('numbers.png',12,1,11,int(11*6/5),-1)
		self.image = pygame.Surface((55,int(11*6/5)))
		self.rect = self.image.get_rect()
		if x == -1:
			self.rect.left = width*0.89
		else:
			self.rect.left = x
		if y == -1:
			self.rect.top = height*0.1
		else:
			self.rect.top = y

	def draw(self):
		screen.blit(self.image,self.rect)

	def update(self,score):
		score_digits = extractDigits(score)
		self.image.fill(background_col)
		for s in score_digits:
			self.image.blit(self.tempimages[s],self.temprect)
			self.temprect.left += self.temprect.width
		self.temprect.left = 0

class DinoEnv(gym.Env):
	metadata = {'render.modes': ['human']}

	def __init__(self):
		self.gamespeed = 4
		self.gameOver = False
		self.playerDino = Dino(44,47)
		self.new_ground = Ground(-1*self.gamespeed)
		self.scb = Scoreboard()
		self.counter = 0
		self.done = False

		self.action_space.n = 2
		self.observation_space.shape = (150, 600)
		self.FPS = 60

		self.cacti = pygame.sprite.Group()
		self.pteras = pygame.sprite.Group()
		self.clouds = pygame.sprite.Group()
		self.last_obstacle = pygame.sprite.Group()

		Cactus.containers = self.cacti
		Ptera.containers = self.pteras
		Cloud.containers = self.clouds

		self.temp_images, self.temp_rect = load_sprite_sheet('numbers.png',12,1,11,int(11*6/5),-1)
		HI_image = pygame.Surface((22,int(11*6/5)))
		HI_rect = HI_image.get_rect()
		HI_image.fill(background_col)
		HI_rect.top = height*0.1
		HI_rect.left = width*0.73

	def step(self, action):
		if not self.gameOver:
			if pygame.display.get_surface() == None:
				print("Couldn't load display surface")
				self.gameOver = True
			else:
				if action in [0, 1, 2]:
					if action == 1:
						if not (self.playerDino.isJumping and self.playerDino.isDead):
							self.playerDino.isDucking = True

					if action != 1:
						self.playerDino.isDucking = False
						if action == 2:
							if self.playerDino.rect.bottom == int(0.98*height):
								self.playerDino.isJumping = True
								self.playerDino.movement[1] = -1*self.playerDino.jumpSpeed
				else:
					print("Invalid Action")

			for c in self.cacti:
				c.movement[0] = -1*self.gamespeed
				if pygame.sprite.collide_mask(self.playerDino,c):
					self.playerDino.isDead = True

			for p in self.pteras:
				p.movement[0] = -1*self.gamespeed
				if pygame.sprite.collide_mask(self.playerDino,p):
					self.playerDino.isDead = True

			if len(self.cacti) < 2:
				if len(self.cacti) == 0:
					self.last_obstacle.empty()
					self.last_obstacle.add(Cactus(self.gamespeed,40,40))
				else:
					for l in self.last_obstacle:
						if l.rect.right < width*0.7 and random.randrange(0,50) == 10:
							self.last_obstacle.empty()
							self.last_obstacle.add(Cactus(self.gamespeed, 40, 40))

			if len(self.pteras) == 0 and random.randrange(0,200) == 10 and self.counter > 500:
				for l in self.last_obstacle:
					if l.rect.right < width*0.8:
						self.last_obstacle.empty()
						self.last_obstacle.add(Ptera(self.gamespeed, 46, 40))

			if len(self.clouds) < 5 and random.randrange(0,300) == 10:
				Cloud(width,random.randrange(height/5,height/2))

			self.playerDino.update()
			self.cacti.update()
			self.pteras.update()
			self.clouds.update()
			self.new_ground.update()
			self.scb.update(self.playerDino.score)

			if pygame.display.get_surface() != None:
				screen.fill(background_col)
				self.new_ground.draw()
				self.clouds.draw(screen)
				self.scb.draw()
				self.cacti.draw(screen)
				self.pteras.draw(screen)
				self.playerDino.draw()

			clock.tick(self.FPS)

			self.obs = pygame.surfarray.array3d(pygame.display.get_surface())

			if self.playerDino.isDead:
				self.gameOver = True

			if self.counter%700 == 699:
				self.new_ground.speed -= 1
				self.gamespeed += 1

			self.counter = (self.counter + 1)

		if self.gameOver:
			self.done = True
		return self.obs, self.playerDino.score, self.done, None

	def reset(self):
		self.gamespeed = 4
		self.gameOver = False
		self.playerDino = Dino(44,47)
		self.new_ground = Ground(-1*self.gamespeed)
		self.scb = Scoreboard()
		self.counter = 0
		self.done = False

		self.cacti = pygame.sprite.Group()
		self.pteras = pygame.sprite.Group()
		self.clouds = pygame.sprite.Group()
		self.last_obstacle = pygame.sprite.Group()

		Cactus.containers = self.cacti
		Ptera.containers = self.pteras
		Cloud.containers = self.clouds

		self.temp_images, self.temp_rect = load_sprite_sheet('numbers.png',12,1,11,int(11*6/5),-1)

	def render(self, mode='human'):
		pygame.display.update()

	def close(self):
		pygame.quit()

max_stars_repo_name: Rochan-A/gym-dino

max_issues_repo_name: Rochan-A/gym-dino

max_forks_repo_name: Rochan-A/gym-dino


text:
import numpy as np
from layers import layer, ABC, abstractmethod

eps = 1e-8
class optimiser(ABC):
    '''
    Base class of optimisers for trainable parameters
    '''

    @abstractmethod
    def update(self, id, param, grad):
        """
        Update a single trainable parameter
        :param ids: Unique parameter identifier, for preserving information
        :param params: Trainable parameter tensor
        :param grad: Update gradients tensor
        """

        pass

    def __call__(self, ids, params, grads):
        """
        Update a set of trainable parameters
        :param ids: Tuple of unique parameter identifiers, for preserving information
        :param params: Tuple of trainable parameter tensors, in order of ids
        :param grads: Tuple of update gradient tensors with respect to the corresponding parameters, in order of ids
        """

        # map(lambda t: self.update(*t), zip(ids, params, grads))
        for id_, param, grad in zip(ids, params, grads):
            self.update(id_, param, grad)

class sgd(optimiser):
    def __init__(self, eta):
        """
        :param eta: Learning rate
        """
        self.eta = eta

    def update(self, id_, param, grad):
        param -= grad * self.eta

class momentum(optimiser):
    def __init__(self, eta, gamma):
        """
        :param eta: Learning rate
        :param gamma: Controls acceleration of gradient in a direction
        """

        self.eta = eta
        self.gamma = gamma
        self.v = {}

    def update(self, id_, param, grad):
        if id_ not in self.v:
            self.v[id_] = 0.
        self.v[id_] = self.gamma * self.v[id_] + self.eta * grad
        param -= self.v[id_]

class adagrad(optimiser):
    def __init__(self, eta):
        """
        :param eta: Learning rate
        """

        self.eta = eta
        self.d = {}

    def update(self, id_, param, grad):
        if id_ not in self.d:
            self.d[id_] = 0
        self.d[id_] += grad * grad
        param -= self.eta / np.sqrt(eps + self.d[id_]) * grad

class rmsprop(optimiser):
    def __init__(self, eta, gamma):
        """
        :param eta: Learning rate
        :param gamma: Controls acceleration of gradient in a direction
        """

        self.eta = eta
        self.gamma = gamma
        self.d = {}

    def update(self, id_, param, grad):
        if id_ not in self.d:
            self.d[id_] = 0.
        self.d[id_] = (1 - self.gamma) * grad * grad + self.gamma * self.d[id_]
        param -= self.eta / np.sqrt(eps + self.d[id_]) * grad

class adam(optimiser):
    def __init__(self, eta, gamma1=0.9, gamma2=0.999):
        """
        :param eta: Learning rate
        :param gamma1: Exponential decay for the first momentum estimate
        :param gamma2: Exponential decay for the second momentum estimate

        """

        self.eta = eta
        self.b = gamma1, gamma2
        self.d = {}

    def update(self, id_, param, grad):
        # Array indices for [first order momentum estimate, second order momentum estimate, epochs]
        m_, v_, t_ = 0, 1, 2

        if id_ not in self.d:
            self.d[id_] = [0., 0., 0.]
        self.d[id_][m_] = (1 - self.b[m_]) * grad + self.b[m_] * self.d[id_][m_]
        self.d[id_][v_] = (1 - self.b[v_]) * grad * grad + self.b[v_] * self.d[id_][v_]
        self.d[id_][t_] += 1

        mp = self.d[id_][m_] / (1 - self.b[m_] ** self.d[id_][t_])
        vp = self.d[id_][v_] / (1 - self.b[v_] ** self.d[id_][t_])

        param -= self.eta * mp / (eps + np.sqrt(vp))

max_stars_repo_name: nkarve/twistml

max_issues_repo_name: nkarve/twistml

max_forks_repo_name: nkarve/twistml


text:
import pandas as pd
import numpy as np

def build_stim_channel(trigger_df, num_samples):
    """Builds an indivdiaul stimulus channel to be placed into the .nirs file.

    Args:
        trigger_df: a dataframe with trigger information.
        num_samples(int): the number of samples in each channel of data (n timepoints).

    Returns:
        m_channel(np.array): new stim channel dims = 1 X num_samples.
    """

    m_channel = np.zeros(num_samples) # make an array of len num_samples - set all values to zero.
    for row in trigger_df.iterrows(): # iterate through each onset.
        m_channel[int(row[1]["onset"])] = 1 # mark the onset sample to 1.
        for x in range(int(row[1]["duration"])): # for number of samples in duration.
            m_channel[(int(row[1]["onset"])) + x] = 1 # mark each sample (x [current interation] + onset) to 1.

    return m_channel

num_samples = 50 # hardcoding, will be accessed through .nirs file (nirs_f["t"].shape[0])

trigger_file = pd.read_csv("../stim_info.csv")
by_trigger_groups = trigger_file.groupby("stimType") # will group into two sub dataframes where 1 is trigger, and 2 is trigger.

stim_matrix = [] # create an empty list to 'catch' stim channels through each loop.
for g in by_trigger_groups: # iterate through the groups
    stim_channel = build_stim_channel(g[1], num_samples) #g[1] is the subdataframe from the iteration.
    stim_matrix.append(stim_channel)

print(np.array(stim_matrix).transpose()) # turn list of np.arrays in an np.array of arrays and flip the x,y dims of the array.


text:
#!/usr/bin/env python

"""
Camera Calibration script taken from the Tinker-Twins, Camera-Calibration repo
https://github.com/Tinker-Twins/Camera-Calibration/blob/main/Camera%20Calibration.ipynb

Orignal Authors: Tanmay Samak and Chinmay Samak
"""

######################################################################

import numpy as np
import cv2
import glob
import pickle
import collections
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

######################################################################
ROW = 7 - 1
COL = 10 - 1
PLOT_CALIBRATIONS = False

cell_size = 0.021 #21cm, Size of the square cell (should not affect the result)
objp = np.zeros((ROW*COL,3), np.float32) # Using a 6x9 grid image for calibration
objp[:,:2] = np.mgrid[0:COL,0:ROW].T.reshape(-1,2)*cell_size # Grid points

# Arrays to store object points and image points from all the images
objpoints = [] # 3D points in real world space
imgpoints = [] # 2D points in image plane

# Make a list of calibration images
images = glob.glob('/root/AprilTag/media/calib_img/*.jpg')
print("calibrating according to", len(images), "images")
img1 = mpimg.imread(images[0])
img_size = (img1.shape[1], img1.shape[0]) # Get the image dimensions

# Step through the list and search for grid corners
for image in images:
    img = mpimg.imread(image)
    gray_img = cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)

    # Find the grid corners
    ret, corners = cv2.findChessboardCorners(gray_img, (COL,ROW),None)

    # If found, append object points and image points to the respective arrays
    if ret == True:
        objpoints.append(objp)
        imgpoints.append(corners)

        # Draw and display the corners
        cv2.namedWindow('Corner Detection', cv2.WINDOW_NORMAL)
        img = cv2.drawChessboardCorners(img, (COL,ROW), corners, ret)
        # Plot in a new window
        if PLOT_CALIBRATIONS:
            cv2.imshow('Corner Detection',img)
            cv2.waitKey(500) # Viewing delay in milliseconds

cv2.destroyAllWindows()

ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, img_size, None, None) # Calibrate the camera

# Print the camera calibration result
print('Distortion Coefficients:')
print('dist = \n' + str(dist))
print('')

print('Camera Intrinsic Parameters:')
print('mtx = \n' + str(mtx))
print('')

print('Camera Extrinsic Parameters:')
print('rvecs = \n' + str(rvecs))
print('')
print('tvecs = \n' + str(tvecs))

# Save the camera calibration result for later use
camera_pickle = {}
camera_pickle["mtx"] = mtx
camera_pickle["dist"] = dist
camera_pickle["rvecs"] = rvecs
camera_pickle["tvecs"] = tvecs
pickle.dump(camera_pickle, open("camera_parameters.p", "wb"))

undistorted_image = cv2.undistort(img1, mtx, dist, None, mtx) # Generate the undistorted image

# Plot inline
# %matplotlib inline
# View the results side-by-side
# f, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5))
# f.tight_layout()
# ax1.imshow(img1)
# ax1.set_title('Original Image', fontsize=20)
# ax2.imshow(undistorted_image)
# ax2.set_title('Undistorted Image', fontsize=20)
# plt.subplots_adjust(left=0.0, right=1.0, top=0.9, bottom=0.0)
# plt.show()
print("original image")
cv2.imshow('Corner Detection',img1)
cv2.waitKey(500) # Viewing delay in milliseconds
cv2.imwrite('original.png',img1)
print("undistorted image")
cv2.imshow('Corner Detection',undistorted_image)
cv2.waitKey(500) # Viewing delay in milliseconds
cv2.imwrite('undistorted.png',undistorted_image)

max_stars_repo_name: furrrow/AprilTag

max_issues_repo_name: furrrow/AprilTag

max_forks_repo_name: furrrow/AprilTag


text:
from ..eyesmetriccalculator import EyesMetricCalculator
from ..transition_matrix.aoi import *

import numpy as np
import pytest

TEST_ARRAY = np.array([[0, 0], [100, 50], [78, 31]])
TEST_SCREENDIM = (100, 50)
TEST_SCREENDIM_2 = (1920, 1200)
TEST_CELLX = 50
TEST_CELLY = 10
TEST_RADIUS = 100
TEST_VERTICES_1 = [[ 20,20] , [400,250], [40,400]]
TEST_VERTICES_2 = [[800,800], [1000,800], [700,650]]

TEST_TRAN_PROB = np.array([[0.78, 0.06, 0.16],
                           [0.18, 0.46, 0.36],
                           [0.15, 0.09, 0.76]
                           ])

TEST_STAT_PROB = np.array([0.42, 0.14, 0.44
                           ])

TEST_AOI_DICT = {
                'aoi_poly1': PolyAOI(TEST_SCREENDIM_2,TEST_VERTICES_1),
                'aoi_circle2': CircleAOI(TEST_SCREENDIM_2, TEST_RADIUS, center=[500,500]),
                'aoi_poly2': PolyAOI(TEST_SCREENDIM_2,TEST_VERTICES_2)
                }

GAZE_ARRAY = np.array([
    [41, 41, 0],
    [80, 80, 0],
    [501, 501, 0],
    [100, 200, 0],
    [1002, 1002, 0],
    [400, 400, 0],
    [200, 200, 0],
    [250, 250, 0],
    [540, 540, 0]])

@pytest.fixture
def valid_calc(request):
    return EyesMetricCalculator(TEST_ARRAY,GAZE_ARRAY, TEST_SCREENDIM)

def test_spatialDensity_result(valid_calc):
    assert (valid_calc.spatialDensity(TEST_CELLX, TEST_CELLY).compute()
            == 3 / 10), 'failed spatial density result'

def test_spatialDensity_grid(valid_calc):
    a = np.array([[0., 1.], [0., 1.], [0., 0.], [0., 0.], [1., 0.]])
    assert (np.array_equal(valid_calc.spatialDensity(
        TEST_CELLX, TEST_CELLY).get_grid(), a)), 'Wrong grid structure'

def test_nni(valid_calc):
    assert(valid_calc.NNI().compute() == 2.3200303835449625), 'wrong NNI Value'

@pytest.fixture
def valid_calc2(request):
    return EyesMetricCalculator(TEST_ARRAY,GAZE_ARRAY, TEST_SCREENDIM_2)

def test_entropy(valid_calc2):
    g = valid_calc2.GEntropy(TEST_AOI_DICT,entropy='transition')
    val = np.around(g._calc_stationary(TEST_STAT_PROB),
                      decimals=2)
    assert (val == 1.0), 'Wrong stationary entropy calculation, result: {}'.fomrat(val)
    assert (np.around(g._calc_transition(TEST_STAT_PROB,TEST_TRAN_PROB),
                      decimals=2) == 0.73), 'Wrong transition entropy calculation val'

max_stars_repo_name: nimRobotics/EyeTrackingMetrics

max_issues_repo_name: nimRobotics/EyeTrackingMetrics

max_forks_repo_name: nimRobotics/EyeTrackingMetrics


text:
#  MIT License
#
#  Copyright (c) 2018 Michael J Simms. All rights reserved.
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#  SOFTWARE.

import numpy
import random

MAX_MODS = 'max modifications'
CONFIG_MODIFIERS = 'modifiers'
MOD_CHANGE = 'change'
MOD_INSERT = 'insert'
MOD_DELETE = 'delete'

class Fuzzer(object):
    """Randomly modifies data."""

    def __init__(self, config):
        self.config = config

    def max_modifications(self):
        """Accessor method for retrieving the max modifications setting, or the default if it was not provided."""
        if MAX_MODS in self.config:
            return self.config[MAX_MODS]
        return 1

    def allowed_modifications(self):
        """Accessor method for retrieving the allowed modifications setting, or the default if it was not provided."""
        if CONFIG_MODIFIERS in self.config:
            return self.config[CONFIG_MODIFIERS]
        return [MOD_CHANGE, MOD_INSERT, MOD_DELETE]

    def insert_random_data(self, data, data_len):
        """Inserts random data into the buffer, increasing it's length."""
        index = random.randint(0, data_len - 1)
        num_bytes = int(numpy.random.lognormal() * 10)
        for _ in range(0,num_bytes-1):
            data[index:0] = bytearray([random.randint(0, 255)])
        return data, data_len + 1

    def delete_random_data(self, data, data_len):
        """Removes a random amount of data from the buffer."""
        index = random.randint(0, data_len - 1)
        amount_to_delete = random.randint(0, data_len - index - 1)
        data = data[:index] + data[index + amount_to_delete:]
        data_len = data_len - amount_to_delete
        return data, data_len

    def modify_random_byte(self, data, data_len):
        """Randomly selects and modifies a single byte in the buffer."""
        index = random.randint(0, data_len)
        new_byte = random.randint(0, 255)
        data[index] = new_byte
        return data

    def fuzz_once(self, data, data_len):
        """Performs a single modification to the buffer."""
        modifications = self.allowed_modifications()
        mod_to_perform = modifications[random.randint(0, len(modifications) - 1)]
        if mod_to_perform == MOD_CHANGE:
            data = self.modify_random_byte(data, data_len)
        elif mod_to_perform == MOD_INSERT:
            data, data_len = self.insert_random_data(data, data_len)
        elif mod_to_perform == MOD_DELETE:
            data, data_len = self.delete_random_data(data, data_len)
        return data, data_len

    def fuzz(self, data, data_len):
        """Entry point for fuzzing. Will apply the rules specified by the config that was passed to the constructor."""
        if isinstance(data, basestring):
            data = bytearray(data)
        max_mods = self.max_modifications()
        for _ in range(0,max_mods):
            data, data_len = self.fuzz_once(data, data_len)
        return data, data_len

def main():
    # Test data.
    data = "the quick brown fox jumps over the lazy dog."

    # Fuzz.
    fuzzer = Fuzzer({})
    data, _ = fuzzer.fuzz(data, len(data))
    print(data)

if __name__ == "__main__":
    main()

max_stars_repo_name: msimms/Scripts

max_issues_repo_name: msimms/Scripts

max_forks_repo_name: msimms/Scripts