text:
from typing import Callable, Dict, List, Set, Tuple

import py
import pytest

from dgtest.core.parse import (
    parse_definition_nodes_from_codebase,
    parse_definition_nodes_from_file,
    parse_import_nodes_from_codebase,
    parse_import_nodes_from_file,
    parse_pytest_fixtures_from_codebase,
    parse_pytest_fixtures_from_file,
    parse_pytest_tests_from_codebase,
    parse_pytest_tests_from_file,
    update_dict,
)

@pytest.fixture(scope="function")
def create_mock_codebase(tmpdir: py.path.local) -> Callable:
    """Utility to easily create temporary filesystems"""

    def _create_mock_codebase(
        directory: str, *files: Tuple[str, str]
    ) -> Tuple[str, List[str]]:
        # Create source and tests directories
        my_dir = tmpdir.mkdir(directory)
        my_dir.mkdir("tests")

        my_files = []
        for name, contents in files:
            temp_file = my_dir.join(name)
            temp_file.write(contents)
            my_files.append(temp_file.strpath)
        return my_dir.strpath, my_files

    return _create_mock_codebase

@pytest.fixture(scope="function")
def definition_map() -> Dict[str, Set[str]]:
    """Used to represent the result of `parse_definition_nodes_from_codebase`"""
    return {
        "reverse_string": {"my_package/strings/strings.py"},
        "concatenate_strings": {"my_package/strings/strings.py"},
        "add_two_numbers": {"my_package/math/addition.py"},
        "Square": {"my_package/math/geometry.py"},
        "Triangle": {"my_package/math/geometry.py"},
    }

@pytest.fixture(scope="function")
def fixture_map() -> Dict[str, Set[str]]:
    """Used to represent the result of `parse_pytest_fixtures_from_codebase`"""
    return {"create_shapes": {"my_package/math/geometry.py"}}

def test_parse_definition_nodes_from_file_collects_function_definitions(
    create_mock_codebase: Callable,
) -> None:

    file_contents = """
def add_two_numbers(x, y):
    return x + y

def reverse_string(string):
    return string.reverse()
    """

    _, my_files = create_mock_codebase("my_dir", ("my_file1.py", file_contents))
    file_definition_map = parse_definition_nodes_from_file(my_files[0])

    assert len(file_definition_map) == 2
    assert (
        isinstance(file_definition_map["add_two_numbers"], set)
        and len(file_definition_map["add_two_numbers"]) == 1
    )
    assert (
        isinstance(file_definition_map["reverse_string"], set)
        and len(file_definition_map["reverse_string"]) == 1
    )

def test_parse_definition_nodes_from_file_collects_class_definitions(
    create_mock_codebase: Callable,
) -> None:
    file_contents = """
class Square(Shape):
    pass

class Triangle(Shape):
    def __init__(self, x, y, z):
        self.x = x
        self.y = y
        self.z = z
    """
    _, my_files = create_mock_codebase("my_dir", ("my_file1.py", file_contents))
    definition_map = parse_definition_nodes_from_file(my_files[0])

    assert len(definition_map) == 2
    assert (
        isinstance(definition_map["Square"], set)
        and my_files[0] in definition_map["Square"]
    )
    assert (
        isinstance(definition_map["Triangle"], set)
        and my_files[0] in definition_map["Triangle"]
    )

def test_parse_definition_nodes_from_codebase_accounts_for_namespace_collision(
    create_mock_codebase: Callable,
) -> None:

    file_contents1 = """
class Circle(Shape):
    pass
    """
    file_contents2 = """
class Circle(Shape):
    pass
    """

    _, my_files = create_mock_codebase(
        "my_dir", ("my_file1.py", file_contents1), ("my_file2.py", file_contents2)
    )
    definition_map = parse_definition_nodes_from_codebase(my_files)

    assert len(definition_map) == 1
    assert (
        isinstance(definition_map["Circle"], set) and len(definition_map["Circle"]) == 2
    )

def test_parse_import_nodes_from_file_with_no_relative_imports(
    create_mock_codebase: Callable,
) -> None:
    file_contents = """
import copy
import datetime
import json
import logging
import os
from typing import Dict, List, Optional, Union
from uuid import UUID
    """
    _, my_files = create_mock_codebase("my_dir", ("my_file1.py", file_contents))

    imports = parse_import_nodes_from_file(my_files[0], "my_package", {})
    assert imports == set()

def test_parse_import_nodes_from_file_with_relative_imports(
    create_mock_codebase: Callable,
    definition_map: Dict[str, Set[str]],
) -> None:
    file_contents = """
from my_package.math.addition import add_two_numbers
from my_package.math.shapes import Square
from my_package.strings import reverse_string, concatenate_strings
    """

    _, my_files = create_mock_codebase("my_dir", ("my_file.py", file_contents))
    imports = parse_import_nodes_from_file(my_files[0], "my_package", definition_map)

    assert len(imports) == 3
    assert all(
        path in imports
        for path in (
            "my_package/math/addition.py",
            "my_package/math/geometry.py",
            "my_package/strings/strings.py",
        )
    )

def test_parse_import_nodes_from_file_with_ambiguous_imports(
    create_mock_codebase: Callable,
) -> None:
    file_contents = """
from my_package.math import add_two_numbers
    """
    _, my_files = create_mock_codebase("my_dir", ("my_file.py", file_contents))

    # Namespace collision means we need to pick the closest import
    definition_map = {
        "add_two_numbers": {"my_package/math/addition.py", "my_package/core/utils.py"},
    }

    imports = parse_import_nodes_from_file(my_files[0], "my_package", definition_map)
    assert len(imports) == 1

    # The import statement included "math" so we pick the first option
    assert "my_package/math/addition.py" in imports
    assert "my_package/core/utils.py" not in imports

def test_parse_import_nodes_from_codebase(
    create_mock_codebase: Callable, definition_map: Dict[str, Set[str]]
) -> None:
    file_contents1 = """
import copy
import datetime
import json
import logging

from my_package.core import Square
from my_package.util import add_two_numbers
    """
    file_contents2 = """
import pandas as pd
import numpy as np
import os

from my_package.math import Triangle
from my_package.strings.strings import reverse_string, concatenate_strings
    """

    _, my_files = create_mock_codebase(
        "my_dir", ("my_file1.py", file_contents1), ("my_file2.py", file_contents2)
    )
    imports = parse_import_nodes_from_codebase(my_files, "my_package", definition_map)
    print(imports)

    assert len(imports) == 3
    keys = [
        "my_package/math/addition.py",
        "my_package/strings/strings.py",
        "my_package/math/geometry.py",
    ]
    assert all(file in imports for file in keys)
    assert len(imports["my_package/math/addition.py"]) == 1
    assert len(imports["my_package/strings/strings.py"]) == 1
    assert len(imports["my_package/math/geometry.py"]) == 2  # Imported in both files

def test_parse_pytest_fixtures_from_file_with_simple_fixtures(
    create_mock_codebase: Callable,
) -> None:
    file_contents = """
@pytest.fixture()
def my_empty_fixture():
    pass

@pytest.fixture(scope="session")
def my_other_empty_fixture():
    pass
    """
    _, my_files = create_mock_codebase("my_dir", ("conftest.py", file_contents))

    imports = parse_pytest_fixtures_from_file(my_files[0], {})
    assert imports == {}

def test_parse_pytest_fixtures_from_file_with_fixtures_referencing_symbols(
    create_mock_codebase: Callable,
    definition_map: Dict[str, Set[str]],
) -> None:
    file_contents = """
@pytest.fixture()
def create_shapes():
    # my_package/math/geometry.py
    square = Square()

    # my_package/math/addition.py
    square.x = add_two_numbers(1, 2)
    square.y = add_two_numbers(3, 4)

    return my_class
    """

    _, my_files = create_mock_codebase("my_dir", ("conftest.py", file_contents))
    fixtures = parse_pytest_fixtures_from_file(my_files[0], definition_map)

    assert len(fixtures) == 1
    assert fixtures["create_shapes"] == {
        "my_package/math/geometry.py",
        "my_package/math/addition.py",
    }

# FIXME(cdkini): This functionality needs to be enabled!

# def test_parse_pytest_fixtures_from_file_with_fixtures_referencing_each_other(
#     create_mock_codebase: Callable,
#     definition_map: Dict[str, Set[str]],
# ) -> None:
#     file_contents = """
# @pytest.fixture()
# def my_fixture1():
#     my_class = Square()
#     add_two_numbers(my_class)
#     return my_class

# @pytest.fixture()
# def my_fixture2(my_fixture1): # This should share 'my_fixture1's dependencies
#     return my_fixture1
#     """

#     _, my_files = create_mock_codebase("my_dir", ("conftest.py", file_contents))
#     imports = parse_pytest_fixtures_from_file(my_files[0], definition_map)

#     assert len(imports) == 2
#     assert imports["my_fixture1"] == {
#         "my_package/math/geometry.py",
#         "my_package/math/addition.py",
#     }
#     assert imports["my_fixture2"] == {
#         "my_package/math/geometry.py",
#         "my_package/math/addition.py",
#     }

def test_parse_pytest_fixtures_from_codebase(
    create_mock_codebase: Callable, definition_map: Dict[str, Set[str]]
) -> None:
    file_contents1 = """
@pytest.fixture
def create_circle():
    return Circle()

@pytest.fixture(scope="function")
def create_triangle():
    return Triangle()
    """
    file_contents2 = """
@pytest.fixture
def set_up_strings():
    return concatenate_strings("hello", "world")
    """

    _, my_files = create_mock_codebase(
        "my_dir", ("conftest.py", file_contents1), ("tests/conftest.py", file_contents2)
    )
    fixtures = parse_pytest_fixtures_from_codebase(my_files, definition_map)

    assert len(fixtures) == 2
    print(fixtures.keys())
    assert all(key in fixtures for key in ("create_triangle", "set_up_strings"))
    assert fixtures["create_triangle"] == {"my_package/math/geometry.py"}
    assert fixtures["set_up_strings"] == {"my_package/strings/strings.py"}

def test_parse_pytest_tests_from_file_with_imports(
    create_mock_codebase: Callable, definition_map: Dict[str, Set[str]]
) -> None:
    test_file_name = "test_shapes.py"
    file_contents = """
from my_package.math.shapes import Square, Triangle

def test_Square():
    assert Square()

def test_Triangle():
    assert Triangle()
    """

    _, my_files = create_mock_codebase("my_dir", (test_file_name, file_contents))
    file_graph = parse_pytest_tests_from_file(
        my_files[0], "my_package", definition_map, fixture_map={}
    )

    assert len(file_graph) == 1
    assert len(file_graph["my_package/math/geometry.py"]) == 1
    assert list(file_graph["my_package/math/geometry.py"])[0].endswith("test_shapes.py")

def test_parse_pytest_tests_from_file_with_fixtures(
    create_mock_codebase: Callable,
    definition_map: Dict[str, Set[str]],
    fixture_map: Dict[str, Set[str]],
) -> None:
    test_file_name = "test_my_class.py"
    file_contents = """
def test_my_classes(create_shapes):
    pass
"""

    _, my_files = create_mock_codebase("my_dir", (test_file_name, file_contents))

    file_graph = parse_pytest_tests_from_file(
        my_files[0], "my_package", definition_map, fixture_map
    )

    assert len(file_graph) == 1
    assert len(file_graph["my_package/math/geometry.py"]) == 1
    assert list(file_graph["my_package/math/geometry.py"])[0].endswith(
        "test_my_class.py"
    )

def test_parse_pytest_tests_from_file_with_imports_and_fixtures(
    create_mock_codebase: Callable,
    definition_map: Dict[str, Set[str]],
    fixture_map: Dict[str, Set[str]],
) -> None:
    test_file_name = "test_my_class.py"
    file_contents = """
from my_package.core import Square
from my_package.util import add_two_numbers

def test_shapes(create_shapes):
    pass
"""

    _, my_files = create_mock_codebase("my_dir", (test_file_name, file_contents))
    file_graph = parse_pytest_tests_from_file(
        my_files[0], "my_package", definition_map, fixture_map
    )

    assert len(file_graph) == 2
    for file in ("my_package/math/geometry.py", "my_package/math/addition.py"):
        assert len(file_graph[file]) == 1
        assert list(file_graph[file])[0].endswith("test_my_class.py")

def test_parse_pytest_tests_from_codebase(
    create_mock_codebase: Callable,
    definition_map: Dict[str, Set[str]],
    fixture_map: Dict[str, Set[str]],
) -> None:
    file_contents1 = """
from my_package.math.shapes import Square
from my_package.math import add_two_numbers

def test_shapes(create_shapes):
    pass
    """
    file_contents2 = """
from my_package.math.addition import add_two_numbers

@pytest.fixture()
def add_nums():
    return add_two_numbers()

def test_math(fake_fixture, add_nums):
    pass
    """

    _, my_files = create_mock_codebase(
        "my_dir", ("test_file1.py", file_contents1), ("test_file2.py", file_contents2)
    )

    tests_dependency_graph = parse_pytest_tests_from_codebase(
        my_files, "my_package", definition_map, fixture_map
    )

    assert len(tests_dependency_graph) == 2
    assert len(tests_dependency_graph["my_package/math/geometry.py"]) == 1
    assert len(tests_dependency_graph["my_package/math/addition.py"]) == 2

def test_update_dict_updates_existing_keys() -> None:
    A = {"my_file1": {"a", "b", "c"}}
    B = {"my_file1": {"c", "d", "e"}}

    update_dict(A, B)

    assert len(A) == 1
    assert all(num in A["my_file1"] for num in ("a", "b", "c", "d", "e"))

def test_update_dict_adds_new_keys() -> None:
    A = {"my_file1": {"a", "b", "c"}}
    B = {"bar": {"c", "d", "e"}}

    update_dict(A, B)

    assert len(A) == 2
    assert all(num in A["my_file1"] for num in ("a", "b", "c"))
    assert all(num in A["bar"] for num in ("c", "d", "e"))

max_stars_repo_name: superconductive/dgtest

max_issues_repo_name: superconductive/dgtest

max_forks_repo_name: superconductive/dgtest


text:
# Licensed under a 3-clause BSD style license - see LICENSE.rst
import os
import numpy as np
import pytest
import mica.common

from ..centroid_resid import CentroidResiduals

HAS_L1_ARCHIVE = os.path.exists(os.path.join(mica.common.MICA_ARCHIVE, 'asp1'))
HAS_STARCHECK_ARCHIVE = os.path.exists(os.path.join(mica.common.MICA_ARCHIVE, 'starcheck'))
try:
    from Ska.engarchive import fetch
    fetch.Msidset(['aoattqt*'], '2018:001:00:00:00', '2018:001:00:01:00')
    HAS_QUAT_TELEM = True
except Exception:
    HAS_QUAT_TELEM = False

@pytest.mark.skipif('not HAS_L1_ARCHIVE',
                    reason='No ground solutions without an aspl1 mica archive')
@pytest.mark.skipif('not HAS_STARCHECK_ARCHIVE',
                    reason='No for_slot without a starcheck mica archive')
def test_multi_ai():
    # obsid 15175 has two aspect intervals
    cr = CentroidResiduals.for_slot(obsid=15175, slot=4)
    assert np.all(np.abs(cr.dyags) < 3)
    assert np.all(np.abs(cr.dzags) < 6)
    # check that the right offset was applied from the table in set_offsets
    assert abs(cr.centroid_dt - 0.0553306628318) < 1e-8

@pytest.mark.skipif('not HAS_L1_ARCHIVE',
                    reason='No ground solutions without an aspl1 mica archive')
@pytest.mark.skipif('not HAS_STARCHECK_ARCHIVE',
                    reason='No for_slot without a starcheck mica archive')
def test_obc_centroids():
    cr = CentroidResiduals.for_slot(obsid=15175, slot=5, centroid_source='obc')
    assert np.all(np.abs(cr.dyags) < 7)
    assert np.all(np.abs(cr.dzags) < 4)
    # check that the right offset was applied from the table in set_offsets
    assert abs(cr.centroid_dt - -2.46900481785) < 1e-8

@pytest.mark.skipif('not HAS_STARCHECK_ARCHIVE',
                    reason='No for_slot without a starcheck mica archive')
@pytest.mark.skipif('not HAS_QUAT_TELEM', reason='No AOATTQT* telemetry in Ska.engarchive')
def test_obc():
    cr = CentroidResiduals.for_slot(obsid=15175, slot=6, att_source='obc', centroid_source='obc')
    assert np.all(np.abs(cr.dyags) < 4.5)
    assert np.all(np.abs(cr.dzags) < 5.5)
    # check that the right offset was applied from the table in set_offsets
    assert abs(cr.centroid_dt - -2.45523126997) < 1e-8

@pytest.mark.skipif('not HAS_STARCHECK_ARCHIVE',
                    reason='No for_slot without a starcheck mica archive')
@pytest.mark.skipif('not HAS_QUAT_TELEM', reason='No AOATTQT* telemetry in Ska.engarchive')
def test_er():
    cr = CentroidResiduals.for_slot(obsid=57635, slot=6, att_source='obc', centroid_source='obc')
    assert np.all(np.abs(cr.dyags) < 4)
    assert np.all(np.abs(cr.dzags) < 3)
    # check that the right offset was applied from the table in set_offsets
    assert abs(cr.centroid_dt - -2.53954270068) < 1e-8

def test_or_manual():
    # This test should run with either maude or cxc as the fetch source and without
    # access to mica archive of L1 products or mica starcheck archive
    with fetch.data_source('cxc' if HAS_QUAT_TELEM else 'maude'):
        # Smaller time interval for MAUDE
        stop = '2017:170:05:13:58.190' if HAS_QUAT_TELEM else '2017:169:19:30:00.00'
        cr = CentroidResiduals(start='2017:169:18:54:50.138', stop=stop)
        cr.set_atts('obc')
        cr.set_centroids('obc', slot=5)
        cr.set_star(agasc_id=649201816)
        cr.calc_residuals()

ext: py

reason: import numpy


text:
#   Copyright 2020 Google LLC

#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Infrastructure for ADAPT VQE algorithm"""
from typing import List, Tuple, Union, Dict
import copy

from itertools import product
import numpy as np
import scipy as sp

import openfermion as of
from openfermion import (
    make_reduced_hamiltonian,
    InteractionOperator,
)
from openfermion.chem.molecular_data import spinorb_from_spatial

import fqe
from fqe.wavefunction import Wavefunction
from fqe.hamiltonians.restricted_hamiltonian import RestrictedHamiltonian
from fqe.hamiltonians.hamiltonian import Hamiltonian as ABCHamiltonian
from fqe.fqe_decorators import build_hamiltonian
from fqe.algorithm.brillouin_calculator import (
    get_fermion_op,
    two_rdo_commutator_symm,
    one_rdo_commutator_symm,
)
from fqe.algorithm.generalized_doubles_factorization import \
    doubles_factorization_svd, doubles_factorization_takagi

from fqe.algorithm.low_rank import evolve_fqe_charge_charge_unrestricted, \
    evolve_fqe_givens_unrestricted

from fqe.algorithm.algorithm_util import valdemaro_reconstruction

class OperatorPool:

    def __init__(self, norbs: int, occ: List[int], virt: List[int]):
        """
        Routines for defining operator pools

        Args:
            norbs: number of spatial orbitals
            occ: list of indices of the occupied orbitals
            virt: list of indices of the virtual orbitals
        """
        self.norbs = norbs
        self.occ = occ
        self.virt = virt
        self.op_pool: List[of.FermionOperator] = []

    def singlet_t2(self):
        """
        Generate singlet rotations
        T_{ij}^{ab} = T^(v1a, v2b)_{o1a, o2b} + T^(v1b, v2a)_{o1b, o2a} +
                      T^(v1a, v2a)_{o1a, o2a} + T^(v1b, v2b)_{o1b, o2b}

        where v1,v2 are indices of the virtual obritals and o1, o2 are
        indices of the occupied orbitals with respect to the Hartree-Fock
        reference.
        """
        for oo_i in self.occ:
            for oo_j in self.occ:
                for vv_a in self.virt:
                    for vv_b in self.virt:
                        term = of.FermionOperator()
                        for sigma, tau in product(range(2), repeat=2):
                            op = ((2 * vv_a + sigma, 1), (2 * vv_b + tau, 1),
                                  (2 * oo_j + tau, 0), (2 * oo_i + sigma, 0))
                            if (2 * vv_a + sigma == 2 * vv_b + tau or
                                    2 * oo_j + tau == 2 * oo_i + sigma):
                                continue
                            fop = of.FermionOperator(op, coefficient=0.5)
                            fop = fop - of.hermitian_conjugated(fop)
                            fop = of.normal_ordered(fop)
                            term += fop
                        self.op_pool.append(term)

    def generalized_two_body(self):
        """
        Doubles generators each with distinct Sz expectation value.

        """
        for i, j, k, l in product(range(2 * self.norbs), repeat=4):
            if i != j and k != l:
                op = ((i, 1), (j, 1), (k, 0), (l, 0))
                fop_aa = of.FermionOperator(op)
                fop_aa = fop_aa - of.hermitian_conjugated(fop_aa)
                self.op_pool.append(fop_aa)

    def generalized_two_body_minimal(self):
        """
        Doubles generators each with distinct Sz expectation value.

        """
        for i, j, k, l in product(range(2 * self.norbs), repeat=4):
            if i < j and k < l:
                op = ((i, 1), (j, 1), (k, 0), (l, 0))
                fop_aa = of.FermionOperator(op)
                fop_aa = fop_aa - of.hermitian_conjugated(fop_aa)
                self.op_pool.append(fop_aa)

    def two_body_sz_adapted(self):
        """
        Doubles generators each with distinct Sz expectation value.

        G^{isigma, jtau, ktau, lsigma) for sigma, tau in 0, 1
        """
        for i, j, k, l in product(range(self.norbs), repeat=4):
            if i < j and k < l:
                op_aa = ((2 * i, 1), (2 * j, 1), (2 * k, 0), (2 * l, 0))
                op_bb = ((2 * i + 1, 1), (2 * j + 1, 1), (2 * k + 1, 0),
                         (2 * l + 1, 0))
                fop_aa = of.FermionOperator(op_aa)
                fop_aa = fop_aa - of.hermitian_conjugated(fop_aa)
                fop_bb = of.FermionOperator(op_bb)
                fop_bb = fop_bb - of.hermitian_conjugated(fop_bb)
                fop_aa = of.normal_ordered(fop_aa)
                fop_bb = of.normal_ordered(fop_bb)
                self.op_pool.append(fop_aa)
                self.op_pool.append(fop_bb)

            op_ab = ((2 * i, 1), (2 * j + 1, 1), (2 * k + 1, 0), (2 * l, 0))
            fop_ab = of.FermionOperator(op_ab)
            fop_ab = fop_ab - of.hermitian_conjugated(fop_ab)
            fop_ab = of.normal_ordered(fop_ab)
            if not np.isclose(fop_ab.induced_norm(), 0):
                self.op_pool.append(fop_ab)

    def one_body_sz_adapted(self):
        # alpha-alpha rotation
        # beta-beta rotation
        for i, j in product(range(self.norbs), repeat=2):
            if i > j:
                op_aa = ((2 * i, 1), (2 * j, 0))
                op_bb = ((2 * i + 1, 1), (2 * j + 1, 0))
                fop_aa = of.FermionOperator(op_aa)
                fop_aa = fop_aa - of.hermitian_conjugated(fop_aa)
                fop_bb = of.FermionOperator(op_bb)
                fop_bb = fop_bb - of.hermitian_conjugated(fop_bb)
                fop_aa = of.normal_ordered(fop_aa)
                fop_bb = of.normal_ordered(fop_bb)
                self.op_pool.append(fop_aa)
                self.op_pool.append(fop_bb)

class ADAPT:

    def __init__(self,
                 oei: np.ndarray,
                 tei: np.ndarray,
                 operator_pool,
                 n_alpha: int,
                 n_beta: int,
                 iter_max=30,
                 verbose=True,
                 stopping_epsilon=1.0E-3,
                 delta_e_eps=1.0E-6):
        """
        ADAPT-VQE object.

        Args:
            oei: one electron integrals in the spatial basis
            tei: two-electron integrals in the spatial basis
            operator_pool: Object with .op_pool that is a list of antihermitian
                           FermionOperators
            n_alpha: Number of alpha-electrons
            n_beta: Number of beta-electrons
            iter_max: Maximum ADAPT-VQE steps to take
            verbose: Print the iteration information
            stopping_epsilon: define the <[G, H]> value that triggers stopping

        """
        elec_hamil = RestrictedHamiltonian((oei, np.einsum("ijlk", -0.5 * tei)))
        soei, stei = spinorb_from_spatial(oei, tei)
        astei = np.einsum('ijkl', stei) - np.einsum('ijlk', stei)
        molecular_hamiltonian = InteractionOperator(0, soei, 0.25 * astei)

        reduced_ham = make_reduced_hamiltonian(molecular_hamiltonian,
                                               n_alpha + n_beta)
        self.reduced_ham = reduced_ham
        self.k2_ham = of.get_fermion_operator(reduced_ham)
        self.k2_fop = build_hamiltonian(self.k2_ham,
                                        elec_hamil.dim(),
                                        conserve_number=True)
        self.elec_hamil = elec_hamil
        self.iter_max = iter_max
        self.sdim = elec_hamil.dim()
        # change to use multiplicity to derive this for open shell
        self.nalpha = n_alpha
        self.nbeta = n_beta
        self.sz = self.nalpha - self.nbeta
        self.nele = self.nalpha + self.nbeta
        self.verbose = verbose
        self.operator_pool = operator_pool
        self.stopping_eps = stopping_epsilon
        self.delta_e_eps = delta_e_eps

    def adapt_vqe(self,
                  initial_wf: Wavefunction,
                  opt_method: str = 'L-BFGS-B',
                  opt_options=None,
                  num_opt_var=None,
                  v_reconstruct: bool = True,
                  num_ops_add: int = 1):
        """
        Run ADAPT-VQE using

        Args:
            initial_wf: Initial wavefunction at the start of the calculation
            opt_method: scipy optimizer to use
            opt_options: options  for scipy optimizer
            v_reconstruct: use valdemoro reconstruction
            num_ops_add: add this many operators from the pool to the
                         wavefunction
        """
        if opt_options is None:
            opt_options = {}
        self.num_opt_var = num_opt_var
        operator_pool = []
        operator_pool_fqe: List[ABCHamiltonian] = []
        existing_parameters: List[float] = []
        self.gradients = []
        self.energies = [initial_wf.expectationValue(self.k2_fop)]
        iteration = 0
        while iteration < self.iter_max:
            # get current wavefunction
            wf = copy.deepcopy(initial_wf)
            for fqe_op, coeff in zip(operator_pool_fqe, existing_parameters):
                wf = wf.time_evolve(coeff, fqe_op)

            # calculate rdms for grad
            _, tpdm = wf.sector((self.nele, self.sz)).get_openfermion_rdms()
            if v_reconstruct:
                d3 = 6 * valdemaro_reconstruction(tpdm / 2, self.nele)
            else:
                d3 = wf.sector((self.nele, self.sz)).get_three_pdm()

            # get ACSE Residual and 2-RDM gradient
            acse_residual = two_rdo_commutator_symm(
                self.reduced_ham.two_body_tensor, tpdm, d3)
            one_body_residual = one_rdo_commutator_symm(
                self.reduced_ham.two_body_tensor, tpdm)

            # calculate grad of each operator in the pool
            pool_grad = []
            for operator in self.operator_pool.op_pool:
                grad_val = 0
                for op_term, coeff in operator.terms.items():
                    idx = [xx[0] for xx in op_term]
                    if len(idx) == 4:
                        grad_val += acse_residual[tuple(idx)] * coeff
                    elif len(idx) == 2:
                        grad_val += one_body_residual[tuple(idx)] * coeff
                pool_grad.append(grad_val)

            max_grad_terms_idx = \
                np.argsort(np.abs(pool_grad))[::-1][:num_ops_add]

            pool_terms = [
                self.operator_pool.op_pool[i] for i in max_grad_terms_idx
            ]
            operator_pool.extend(pool_terms)
            fqe_ops: List[ABCHamiltonian] = []
            for f_op in pool_terms:
                fqe_ops.append(
                    build_hamiltonian(1j * f_op,
                                      self.sdim,
                                      conserve_number=True))
            operator_pool_fqe.extend(fqe_ops)
            existing_parameters.extend([0] * len(fqe_ops))

            if self.num_opt_var is not None:
                if len(operator_pool_fqe) < self.num_opt_var:
                    pool_to_op = operator_pool_fqe
                    params_to_op = existing_parameters
                    current_wf = copy.deepcopy(initial_wf)
                else:
                    pool_to_op = operator_pool_fqe[-self.num_opt_var:]
                    params_to_op = existing_parameters[-self.num_opt_var:]
                    current_wf = copy.deepcopy(initial_wf)
                    for fqe_op, coeff in zip(
                            operator_pool_fqe[:-self.num_opt_var],
                            existing_parameters[:-self.num_opt_var]):
                        current_wf = current_wf.time_evolve(coeff, fqe_op)

                new_parameters, current_e = self.optimize_param(
                    pool_to_op,
                    params_to_op,
                    current_wf,
                    opt_method,
                    opt_options=opt_options)

                if len(operator_pool_fqe) < self.num_opt_var:
                    existing_parameters = new_parameters.tolist()
                else:
                    existing_parameters[-self.num_opt_var:] = \
                        new_parameters.tolist()
            else:
                new_parameters, current_e = self.optimize_param(
                    operator_pool_fqe,
                    existing_parameters,
                    initial_wf,
                    opt_method,
                    opt_options=opt_options)
                existing_parameters = new_parameters.tolist()

            if self.verbose:
                print("{: 5d}\t{: 5.15f}\t{: 5.15f}".format(
                    iteration, current_e, max(np.abs(pool_grad))))
            self.energies.append(current_e)
            self.gradients.append(pool_grad)
            if max(np.abs(pool_grad)) < self.stopping_eps or np.abs(
                    self.energies[-2] - self.energies[-1]) < self.delta_e_eps:
                break
            iteration += 1

    def optimize_param(
            self,
            pool: Union[List[of.FermionOperator], List[ABCHamiltonian]],
            existing_params: Union[List, np.ndarray],
            initial_wf: Wavefunction,
            opt_method: str,
            opt_options=None) -> Tuple[np.ndarray, float]:
        """Optimize a wavefunction given a list of generators

        Args:
            pool: generators of rotation
            existing_params: parameters for the generators
            initial_wf: initial wavefunction
            opt_method: Scpy.optimize method
        """
        if opt_options is None:
            opt_options = {}

        def cost_func(params):
            assert len(params) == len(pool)
            # compute wf for function call
            wf = copy.deepcopy(initial_wf)
            for op, coeff in zip(pool, params):
                if np.isclose(coeff, 0):
                    continue
                if isinstance(op, ABCHamiltonian):
                    fqe_op = op
                else:
                    print("Found a OF Hamiltonian")
                    fqe_op = build_hamiltonian(1j * op,
                                               self.sdim,
                                               conserve_number=True)
                if isinstance(fqe_op, ABCHamiltonian):
                    wf = wf.time_evolve(coeff, fqe_op)
                else:
                    raise ValueError("Can't evolve operator type {}".format(
                        type(fqe_op)))

            # compute gradients
            grad_vec = np.zeros(len(params), dtype=np.complex128)
            # avoid extra gradient computation if we can
            if opt_method not in ['Nelder-Mead', 'COBYLA']:
                for pidx, _ in enumerate(params):
                    # evolve e^{iG_{n-1}g_{n-1}}e^{iG_{n-2}g_{n-2}}x
                    # G_{n-3}e^{-G_{n-3}g_{n-3}...|0>
                    grad_wf = copy.deepcopy(initial_wf)
                    for gidx, (op, coeff) in enumerate(zip(pool, params)):
                        if isinstance(op, ABCHamiltonian):
                            fqe_op = op
                        else:
                            fqe_op = build_hamiltonian(1j * op,
                                                       self.sdim,
                                                       conserve_number=True)
                        if not np.isclose(coeff, 0):
                            grad_wf = grad_wf.time_evolve(coeff, fqe_op)
                            # if looking at the pth parameter then apply the
                            # operator to the state
                        if gidx == pidx:
                            grad_wf = grad_wf.apply(fqe_op)

                    # grad_val = grad_wf.expectationValue(self.elec_hamil,
                    # brawfn=wf)
                    grad_val = grad_wf.expectationValue(self.k2_fop, brawfn=wf)

                    grad_vec[pidx] = -1j * grad_val + 1j * grad_val.conj()
                    assert np.isclose(grad_vec[pidx].imag, 0)
            return (wf.expectationValue(self.k2_fop).real,
                    np.array(grad_vec.real, order='F'))

        res = sp.optimize.minimize(cost_func,
                                   existing_params,
                                   method=opt_method,
                                   jac=True,
                                   options=opt_options)
        return res.x, res.fun

max_stars_repo_name: MichaelBroughton/OpenFermion-FQE

max_issues_repo_name: MichaelBroughton/OpenFermion-FQE

max_forks_repo_name: MichaelBroughton/OpenFermion-FQE

reason: import numpy,import scipy


text:
import torch
import numpy as np
import pyredner
import torch
import enum
import math

class Texture:
    def __init__(self,
                 texels,
                 uv_scale = torch.tensor([1.0, 1.0])):
        self.texels = texels
        if len(texels.shape) >= 2:
            # Build a mipmap for texels
            width = max(texels.shape[0], texels.shape[1])
            num_levels = math.ceil(math.log(width, 2) + 1)
            mipmap = texels.unsqueeze(0).expand(num_levels, *texels.shape)
            if len(mipmap.shape) == 3:
                mipmap.unsqueeze_(-1)
            num_channels = mipmap.shape[-1]
            box_filter = torch.ones(num_channels, 1, 2, 2,
                device = texels.device) / 4.0

            # Convert from HWC to NCHW
            base_level = texels.unsqueeze(0).permute(0, 3, 1, 2)
            mipmap = [base_level]
            prev_lvl = base_level
            for l in range(1, num_levels):
                dilation_size = 2 ** (l - 1)
                # Pad for circular boundary condition
                # This is slow. The hope is at some point PyTorch will support
                # circular boundary condition for conv2d
                prev_lvl = torch.cat([prev_lvl, prev_lvl[:,:,0:dilation_size]], dim=2)
                prev_lvl = torch.cat([prev_lvl, prev_lvl[:,:,:,0:dilation_size]], dim=3)
                current_lvl = torch.nn.functional.conv2d(\
                    prev_lvl, box_filter,
                    dilation = dilation_size,
                    groups = num_channels)
                mipmap.append(current_lvl)
                prev_lvl = current_lvl

            mipmap = torch.cat(mipmap, 0)
            # Convert from NCHW to NHWC
            mipmap = mipmap.permute(0, 2, 3, 1)
            texels = mipmap.contiguous()

        self.mipmap = texels
        self.uv_scale = uv_scale

max_stars_repo_name: SuperShinyEyes/redner

max_issues_repo_name: SuperShinyEyes/redner

max_forks_repo_name: SuperShinyEyes/redner


text:
import tensorflow as tf
import matplotlib.pylab as plt

import tensorflow_hub as hub
import tensorflow_datasets as tfds

from tensorflow.keras import layers

import logging
logger = tf.get_logger()
logger.setLevel(logging.ERROR)

CLASSSIFIER_URL = "https://tfhub.dev/google/tf2-preview/mobilenet_v2/classification/2"
IMAGES_RES = 224

model = tf.keras.Sequential([
    hub.KerasLayer(CLASSSIFIER_URL,input_shape=(IMAGES_RES,IMAGES_RES,3))
])

import numpy as np
import PIL.Image as Image

labels_path = tf.keras.utils.get_file('ImageNetLabels.txt','https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt')
imagenet_labels = np.array(open(labels_path).read().splitlines())

(train_examples, validation_examples), info = tfds.load(
    "cats_vs_dogs",
    with_info = True,
    as_supervised = True,
    split = ["train[:80%]","train[80%:]"]
)

num_examples = info.splits["train"].num_examples
num_examples = info.features["label"].num_classes

for i, example_image in enumerate(train_examples.take(3)):
    print("Image {} Shape : {}".format(i+1,example_image[0].shape))

def format_image(image, label):
  image = tf.image.resize(image, (IMAGE_RES, IMAGE_RES))/255.0
  return image, label

BATCH_SIZE = 32

train_batches      = train_examples.shuffle(num_examples//4).map(format_image).batch(BATCH_SIZE).prefetch(1)
validation_batches = validation_examples.map(format_image).batch(BATCH_SIZE).prefetch(1)

image_batch, label_batch = next(iter(train_batches.take(1)))
image_batch = image_batch.numpy()
label_batch = label_batch.numpy()

result_batch = model.predict(image_batch)

predicted_class_names = imagenet_labels[np.argmax(result_batch,axis= -1)]

plt.figure(figsize=(10,9))
for n in range(30):
  plt.subplot(6,5,n+1)
  plt.subplots_adjust(hspace = 0.3)
  plt.imshow(image_batch[n])
  plt.title(predicted_class_names[n])
  plt.axis('off')
_ = plt.suptitle("ImageNet predictions")

max_stars_repo_name: heymadscientist/Tensorflow-practices

max_issues_repo_name: heymadscientist/Tensorflow-practices

max_forks_repo_name: heymadscientist/Tensorflow-practices


text:
'''
Base agents that observe partial environments and take actions based on preferences
'''
import numpy as np
from matplotlib import pyplot as plt
import gym
from heapq import *
import gym_minigrid
from scipy.ndimage import zoom
from gym_minigrid import wrappers
from gym_minigrid.minigrid import OBJECT_TO_IDX, COLOR_TO_IDX, STATE_TO_IDX

DIR_TO_NUM = {
    (1, 0): 0,
    (0, -1): 3,
    (-1, 0): 2,
    (0, 1): 1,
}
NUM_TO_DIR = dict([(v, k) for k, v in DIR_TO_NUM.items()])

ACTION_TO_NUM_MAPPING = {
    'left': 0,
    'right': 1,
    'forward': 2,
    'toggle': 5,
    'done': 6,
}
IDX_TO_OBJECT = dict([(v, k) for k, v in OBJECT_TO_IDX.items()])
VICTIMCOLORS = ['red', 'yellow']
VICTIMCOLORS = list(map(lambda x: COLOR_TO_IDX[x], VICTIMCOLORS))

class PlanAgent:
    # Agent that has a plan! :D
    def __init__(self, env):
        # Given the env, get all the parameters like size and agent details
        # Env would be wrapped in a wrapper that gives agent location and direction
        self.env = env.env
        self.agent_view_size = self.env.agent_view_size
        self.width = self.env.width
        self.height = self.env.height

        self.epsilon = 0
        self.numobjects = len(OBJECT_TO_IDX) - 1
        self.numcolors = len(COLOR_TO_IDX)
        self.numstates = len(STATE_TO_IDX)

        # This is to capture belief
        self.belief = np.ones((self.width + 2*self.agent_view_size, self.height + 2*self.agent_view_size, self.numobjects)) / self.numobjects
        # Keep track of last visited
        self.lastvisited = np.zeros((self.width + 2*self.agent_view_size, self.height + 2*self.agent_view_size))
        # Capture color and states
        self.colors = np.ones((self.width + 2*self.agent_view_size, self.height + 2*self.agent_view_size, self.numcolors)) / self.numcolors
        self.objstates = np.ones((self.width + 2*self.agent_view_size, self.height + 2*self.agent_view_size, self.numstates)) / self.numstates

        self.plan = []
        self.preferences = ['goal', 'box', 'door', 'explore']
        self.agent_pos = None
        self.agent_dir = None
        self._subgoal = None
        self._current_plan = None

    def reset(self):
        self.belief = np.ones((self.width + 2*self.agent_view_size, self.height + 2*self.agent_view_size, self.numobjects)) / self.numobjects
        # Keep track of last visited
        self.lastvisited = np.zeros((self.width + 2*self.agent_view_size, self.height + 2*self.agent_view_size))
        # Capture color and states
        self.colors = np.ones((self.width + 2*self.agent_view_size, self.height + 2*self.agent_view_size, self.numcolors)) / self.numcolors
        self.objstates = np.ones((self.width + 2*self.agent_view_size, self.height + 2*self.agent_view_size, self.numstates)) / self.numstates

        self.plan = []
        self.agent_pos = None
        self.agent_dir = None
        self._subgoal = None
        self._current_plan = None

    def get_max_belief(self):
        A = self.agent_view_size
        belief = self.belief[A:-A, A:-A]
        return np.argmax(belief, 2)

    def get_prob_map(self, classes, zoom_factor=4):
        A = self.agent_view_size
        belief = self.belief[A:-A, A:-A] + 0
        classidx = list(map(lambda x: OBJECT_TO_IDX[x]-1, classes))
        prob = belief[:, :, classidx].sum(2)
        if zoom_factor > 1:
            prob = zoom(prob, zoom_factor, order=1)
            prob[0, 0] = 1
        return prob

    @property
    def subgoal(self):
        return self._subgoal

    def get_entropy(self):
        A = self.agent_view_size
        belief = self.belief[A:-A, A:-A]
        ent = -belief * np.log(1e-100 + belief)
        ent = ent.sum(2)
        ent = -zoom(ent, 4)
        return ent

    def rotate_right(self, grid):
        newgrid = grid * 0
        # Swap elements
        H, W = grid.shape[:2]
        # Swap elements
        for i in range(H):
            for j in range(W):
                newgrid[i, j] = grid[j, H-1-i] + 0
        return newgrid

    def get_bounds(self, agent_pos, agent_dir):
        if agent_dir == 0:
            topX = agent_pos[0]
            topY = agent_pos[1] - self.agent_view_size // 2
        # Facing down
        elif agent_dir == 1:
            topX = agent_pos[0] - self.agent_view_size // 2
            topY = agent_pos[1]
        # Facing left
        elif agent_dir == 2:
            topX = agent_pos[0] - self.agent_view_size + 1
            topY = agent_pos[1] - self.agent_view_size // 2
        # Facing up
        elif agent_dir == 3:
            topX = agent_pos[0] - self.agent_view_size // 2
            topY = agent_pos[1] - self.agent_view_size + 1
        else:
            assert False, "invalid agent direction"

        botX = topX + self.agent_view_size
        botY = topY + self.agent_view_size
        return (topX, topY, botX, botY)

    def update(self, obs):
        # Given the observation, update your belief
        img = obs['image']
        pos = obs['pos']
        agdir = obs['dir']

        # Save the values for planning ahead
        self.agent_pos = pos
        self.agent_dir = agdir

        topX, topY, botX, botY = self.get_bounds(pos, agdir)
        h, w = img.shape[:2]
        img[w//2, h-1, 0] = 0
        #print(topX, topY, botX, botY)
        #print(self.agent_view_size)
        for i in range(agdir + 1):
            img = self.rotate_right(img)

        # Forget some information
        self.belief = (self.belief + self.epsilon)/(1 + self.numstates*self.epsilon)
        self.colors = (self.colors + self.epsilon)/(1 + self.numcolors*self.epsilon)
        self.objstates = (self.objstates + self.epsilon)/(1 + self.numobjects*self.epsilon)

        # Get seen portions only
        objs = img[:, :, 0]
        cols = img[:, :, 1]
        stas = img[:, :, 2]

        # Get locations
        x, y = np.where(objs > 0)
        vals = np.zeros((len(x), self.numobjects))
        vals[np.arange(len(x)), objs[x, y]-1] = 1
        # update colors and states
        colorvals = np.zeros((len(x), self.numcolors))
        colorvals[np.arange(len(x)), cols[x, y]] = 1

        statevals = np.zeros((len(x), self.numstates))
        statevals[np.arange(len(x)), stas[x, y]] = 1

        # update belief accordingly
        self.belief[topX + x + self.agent_view_size, topY + y + self.agent_view_size] = vals
        self.objstates[topX + x + self.agent_view_size, topY + y + self.agent_view_size] = statevals
        self.colors[topX + x + self.agent_view_size, topY + y + self.agent_view_size] = colorvals

    def check_safe_plan(self):
        # TODO: Check if the next step is safe, if not then delete the existing plan
        if len(self.plan) > 0:
            if self.plan[-1] == 'forward':
                dirvec = NUM_TO_DIR[self.agent_dir]
                fwd_cell = self.agent_pos + dirvec
                # Get object, color, state
                fwd_obj, fwd_color, fwd_state = self.query_cell(fwd_cell)
                if fwd_obj in ['wall', 'lava']:
                    self.plan = []
                elif fwd_obj == 'door':
                    self.plan = []
                    if fwd_state == STATE_TO_IDX['closed']:
                        self.plan.append('toggle')
                elif fwd_obj == 'box':
                    self.plan = []
                    self.plan.insert(0, 'toggle')
                elif fwd_obj == 'goal' and fwd_color in VICTIMCOLORS:
                    self.plan = []
                    self.plan.append('toggle')

        elif len(self.plan) == 0:
            # Toggle door as long as it takes
            dirvec = NUM_TO_DIR[self.agent_dir]
            fwd_cell = self.agent_pos + dirvec
            # Get object, color, state
            fwd_obj, fwd_color, fwd_state = self.query_cell(fwd_cell)
            if fwd_obj == 'box':
                self.plan = []
                self.plan.insert(0, 'toggle')
            if fwd_obj == 'goal' and fwd_color in VICTIMCOLORS:
                self.plan = []
                self.plan.append('toggle')

    def query_cell(self, pos):
        A = self.agent_view_size
        belief = self.belief[A + pos[0], A + pos[1]]
        color = self.colors[A + pos[0], A + pos[1]]
        state = self.objstates[A + pos[0], A + pos[1]]
        if max(belief) < 1.5/self.numobjects:
            return 'unknown', None, None
        idx = np.argmax(belief)
        cidx = np.argmax(color)
        sidx = np.argmax(state)
        return IDX_TO_OBJECT[idx + 1], cidx, sidx

    def predict(self, obs):
        self.update(obs)
        self.check_safe_plan()
        if self.plan == []:
            self.update_plan()
        try:
            return ACTION_TO_NUM_MAPPING[self.plan.pop()]
        except:
            print("Returning done")
            return ACTION_TO_NUM_MAPPING['done']

    def get_location_from_preference(self, pref):
        A = self.agent_view_size
        victimidx = [COLOR_TO_IDX['red'], COLOR_TO_IDX['yellow']]
        if pref in ['goal', 'box', 'door']:
            # Search for a high confidence goal according to belief
            goalcode = OBJECT_TO_IDX[pref] - 1
            probgoal = self.belief[A:-A, A:-A, goalcode] + 0
            if pref == 'goal':
                colors = self.colors[A:-A, A:-A, victimidx].sum(2) + 0
                probgoal = probgoal * colors

            x, y = np.where(probgoal > 0.7)
            if len(x) == 0:
                return None
            # Check for unopened doors only
            if pref == 'door':
                doorstate = np.argmax(self.objstates[A+x, A+y], 1)
                doorstate = np.where(doorstate != STATE_TO_IDX['open'])
                x, y = x[doorstate], y[doorstate]
                if len(x) == 0:
                    return None

            # Get the one with min index
            minidx = np.argmin(np.abs(x - self.agent_pos[0]) + np.abs(y - self.agent_pos[1]))
            return [x[minidx], y[minidx]]

        elif pref == 'explore':
            entropy = -self.belief * np.log(self.belief + 1e-100)
            entropy = entropy[A:-A, A:-A].mean(2)
            # Get shape
            H, W = entropy.shape
            # Have a map based on spatial distance
            xx, yy = np.arange(H), np.arange(W)
            xx, yy = np.meshgrid(xx, yy)
            dist = np.sqrt((xx - self.agent_pos[0])**2 + (yy - self.agent_pos[1])**2)
            dist /= H
            entropydist = entropy / (1 + dist)
            # Sample from it
            N = entropy.reshape(-1).shape[0]
            try:
                p = entropydist/entropydist.sum()
                goal = np.random.choice(np.arange(N), p=p.reshape(-1))
                x, y = goal//H, goal%H
                return [x, y]
            except:
                return None

    def update_plan(self):
        # Update a new plan
        assert self.plan == []
        # Check for preferences, and return a location to sample from
        loc = None
        for pref in self.preferences:
            loc = self.get_location_from_preference(pref)
            if loc is not None:
                self._current_plan = pref
                break

        # The location should not be empty
        assert loc is not None
        self._subgoal = loc
        # Make a plan to that location
        self.plan = self.astar(loc)
        return None

    def dist_func(self, a, b, mode='l1'):
        dist = 0
        if mode == 'l2':
            for t1, t2 in zip(a, b):
                dist += (t1 - t2)**2
            return np.sqrt(dist)
        elif mode == 'l1':
            for t1, t2 in zip(a, b):
                dist += np.abs(t1 - t2)
            return np.sqrt(dist)
        else:
            raise NotImplementedError

    def astar(self, dst):
        '''
        Run an A star algorithm to take a sequence of paths to take
        '''
        A = self.agent_view_size
        # Get source and dst
        src = list(self.agent_pos)
        agdir = self.agent_dir
        #print(src, dst)

        locations = []
        # Get belief and visited matrix
        belief = self.belief[A:-A, A:-A, :]
        visited = belief[:, :, 0] * 0
        visited[src[0], src[1]] = 1
        # Get parents
        parents = belief[:, :, :2] * 0 - 1
        # Make a heap for lowest to highest in terms of distance
        # heap elements will be of the form (f, g, pos)
        frontier = []
        heappush(frontier, (self.dist_func(src, dst), 0, src))
        # Expand the next node
        while len(frontier) != 0:
            node = heappop(frontier)
            loc = node[2]
            if loc == dst:
                break
            # Expand its neighbours
            #print('Expanding {}'.format(node))
            nbrs = self.get_neighbours(loc)
            for nbr in nbrs:
                if visited[nbr[0], nbr[1]]:
                    continue
                visited[nbr[0], nbr[1]] = 1
                # Not visited, time to visit this
                prob = belief[nbr[0], nbr[1], [OBJECT_TO_IDX['lava']-1, OBJECT_TO_IDX['wall']-1]].sum()
                gc = node[1] + 1 + int(prob > 0.7)*1e50
                # Get f
                fc = gc + self.dist_func(nbr, dst)
                heappush(frontier, (fc, gc, nbr))
                # update parent of nbr
                parents[nbr[0], nbr[1]] = loc
        # We got the goal location, get the sequence of paths
        seq = [dst]
        while True:
            last = seq[-1]
            parent = list(parents[last[0], last[1]])
            parent = list(map(lambda x: int(x), parent))
            if parent == [-1, -1]:
                assert last == src
                break
            seq.append(parent)

        # Convert this sequence of locations into sequence of actions
        seq = seq[::-1]
        cur_dir = agdir
        action_seq = []
        for i in range(len(seq)-1):
            # Check for directions and take a step forward
            dst_dir = (seq[i+1][0] - seq[i][0], seq[i+1][1] - seq[i][1])
            dst_dir = DIR_TO_NUM[dst_dir]
            action_seq.extend(self.turn(cur_dir, dst_dir))
            action_seq.append('forward')
            # Update current direction
            cur_dir = dst_dir

        # Check for actions and map
        #print(seq)
        #print(action_seq, self.agent_dir)
        #self.finalmap = belief[:, :, 0]* 0
        #for i, s in enumerate(seq):
            #self.finalmap[s[0], s[1]] = i
        #plt.figure()
        #plt.imshow(self.finalmap.T)
        #plt.show()

        # Return sequences of actions
        return action_seq[::-1]

    def turn(self, cur_dir, dst_dir):
        steps = []
        while cur_dir != dst_dir:
            steps.append('right')
            cur_dir = (cur_dir+1)%4
        if len(steps) == 3:
            steps = ['left']
        return steps

    def get_neighbours(self, loc):
        nbrs = []
        x, y = loc[0], loc[1]
        for dx in [-1, 1]:
            if x+dx < 0 or x+dx >= self.height:
                continue
            nbrs.append([x+dx, y])
        for dy in [-1, 1]:
            if y+dy < 0 or y+dy >= self.width:
                continue
            nbrs.append([x, y+dy])
        return nbrs

class PreEmptiveAgent(PlanAgent):
    '''
    This agent stops whereever its going to when it sees a door or goal
    Two parts:
        If going to a door or goal, keep going
        If exploring and see a door/goal, then go to door/goal
    '''
    def check_safe_plan(self):
        # TODO: Check if the next step is safe, if not then delete the existing plan
        if len(self.plan) > 0:
            if self.plan[-1] == 'forward':
                dirvec = NUM_TO_DIR[self.agent_dir]
                fwd_cell = self.agent_pos + dirvec
                # Get object, color, state
                fwd_obj, fwd_color, fwd_state = self.query_cell(fwd_cell)
                if fwd_obj in ['wall', 'lava']:
                    self.plan = []
                elif fwd_obj == 'door':
                    self.plan = []
                    if fwd_state == STATE_TO_IDX['closed']:
                        self.plan.append('toggle')
                elif fwd_obj == 'box':
                    self.plan = []
                    self.plan.insert(0, 'toggle')
                elif fwd_obj == 'goal' and fwd_color in VICTIMCOLORS:
                    self.plan = []
                    self.plan.append('toggle')

        elif len(self.plan) == 0:
            # Toggle door as long as it takes
            dirvec = NUM_TO_DIR[self.agent_dir]
            fwd_cell = self.agent_pos + dirvec
            # Get object, color, state
            fwd_obj, fwd_color, fwd_state = self.query_cell(fwd_cell)
            if fwd_obj == 'box':
                self.plan = []
                self.plan.insert(0, 'toggle')
            elif fwd_obj == 'goal' and fwd_color in VICTIMCOLORS:
                self.plan = []
                self.plan.append('toggle')

        if self._current_plan == 'explore':
            # If you're exploring and see something
            A = self.agent_view_size
            maxprob = self.get_prob_map(['box', 'goal'], zoom_factor=1)
            maxprob = np.max(maxprob)
            # Check for unopened doors
            doorprob = self.get_prob_map(['door'], zoom_factor=1)
            doorprob *= (self.objstates[A:-A, A:-A, STATE_TO_IDX['closed']])
            if maxprob >= 0.7:
                #print("Preemptively ditching plan for goal")
                self.plan = []
            elif np.max(doorprob) >= 0.7:
                #print("Preemptively ditching plan for door")
                self.plan = []

class ScouringAgent(PlanAgent):
    '''
    This agent minimizes overall entropy first and then goes to plan
    Two parts:
        If going to a door or goal, keep going
        If exploring and see a door/goal, then go to door/goal
    '''
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.preferences = ['door', 'explore', 'goal', 'box']
        self.opendoors = 0

    def check_safe_plan(self):
        if len(self.plan) > 0:
            if self.plan[-1] == 'forward':
                dirvec = NUM_TO_DIR[self.agent_dir]
                fwd_cell = self.agent_pos + dirvec
                # Get object, color, state
                fwd_obj, fwd_color, fwd_state = self.query_cell(fwd_cell)
                if fwd_obj in ['wall', 'lava']:
                    self.plan = []
                elif fwd_obj == 'door':
                    self.plan = []
                    if fwd_state == STATE_TO_IDX['closed']:
                        self.plan.append('toggle')
                        self.opendoors += 1
                        # Check if all doors are open, no need to explore anymore now
                        if self.opendoors >= 6:
                            self.preferences.remove('explore')
                            self.preferences.append('explore')

                elif fwd_obj == 'box':
                    self.plan = []
                    self.plan.insert(0, 'toggle')
                elif fwd_obj == 'goal' and fwd_color in VICTIMCOLORS:
                    self.plan = []
                    self.plan.append('toggle')

        elif len(self.plan) == 0:
            # Toggle door as long as it takes
            dirvec = NUM_TO_DIR[self.agent_dir]
            fwd_cell = self.agent_pos + dirvec
            # Get object, color, state
            fwd_obj, fwd_color, fwd_state = self.query_cell(fwd_cell)
            if fwd_obj == 'box':
                self.plan = []
                self.plan.insert(0, 'toggle')
            elif fwd_obj == 'goal' and fwd_color in VICTIMCOLORS:
                self.plan = []
                self.plan.append('toggle')

        if self._current_plan == 'explore':
            # If you're exploring and see a door
            A = self.agent_view_size
            # Check for unopened doors
            doorprob = self.get_prob_map(['door'], zoom_factor=1)
            doorprob *= (self.objstates[A:-A, A:-A, STATE_TO_IDX['closed']])
            if np.max(doorprob) >= 0.7:
                #print("Preemptively ditching plan for door")
                self.plan = []

#########################
## Main code
#########################
env = gym.make('MiniGrid-FourRooms-v0')
env = gym.make('MiniGrid-HallwayWithVictims-v0')
#env = gym.make('MiniGrid-HallwayWithVictimsAndFire-v0')
#env = gym.make('MiniGrid-Empty-Random-10x10-v0')
env = wrappers.AgentExtraInfoWrapper(env)

#agent = PlanAgent(env)
agent = PreEmptiveAgent(env)
#agent = ScouringAgent(env)
obs = env.reset()
act = agent.predict(obs)
#agent.update(obs)
#print(obs['pos'], obs['dir'])
print(OBJECT_TO_IDX)

## Save trajectories here
all_episode_actions = []
current_episode_actions = []
episodes = 0
num_steps = 0

while episodes < 1000:
    #act = int(input("Enter action "))
    #agent.update(obs)
    #act = agent.predict(obs)
    obs, rew, done, info = env.step(act)
    num_steps += 1
    current_episode_actions.append(act)
    if done:
        print("Episode {} done in {} steps".format(episodes + 1, num_steps))
        episodes += 1
        num_steps = 0
        obs = env.reset()
        agent.reset()
        # Add this episode data to all episodes
        all_episode_actions.append(current_episode_actions)
        current_episode_actions = []

    act = agent.predict(obs)
    #print(obs['pos'], obs['dir'])

    if 1:
        plt.clf()
        plt.subplot(221)
        img = env.render('rgb_array')
        plt.imshow(img)

        plt.subplot(222)
        plt.imshow(agent.get_prob_map(['box', 'goal']).T, 'jet')
        plt.title('Victim')

        plt.subplot(223)
        plt.imshow(agent.get_prob_map(['wall']).T, 'jet')
        plt.title('Map')

        plt.subplot(224)
        plt.imshow(agent.get_entropy().T, 'jet')
        plt.title('Certainty')
        plt.draw()
        plt.pause(0.001)

# Get filename
print(agent)
filename = input('Enter filename: ')
filename += '.csv'

with open(filename, 'wt') as fi:
    lines = []
    for act in all_episode_actions:
        stract = ','.join(list(map(lambda x: str(x), act))) + "\n"
        lines.append(stract)
    fi.writelines(lines)

max_stars_repo_name: rohitrango/gym-minigrid

max_issues_repo_name: rohitrango/gym-minigrid

max_forks_repo_name: rohitrango/gym-minigrid

reason: import numpy,from scipy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import warnings
warnings.filterwarnings('ignore')

__tool_name__='STREAM'
print('''
   _____ _______ _____  ______          __  __
  / ____|__   __|  __ \|  ____|   /\   |  \/  |
 | (___    | |  | |__) | |__     /  \  | \  / |
  \___ \   | |  |  _  /|  __|   / /\ \ | |\/| |
  ____) |  | |  | | \ \| |____ / ____ \| |  | |
 |_____/   |_|  |_|  \_\______/_/    \_\_|  |_|
FeatureSelection...
''',flush=True)

import stream as st
import argparse
import multiprocessing
import os
from slugify import slugify
import networkx as nx
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib as mpl
import sys
mpl.use('Agg')
mpl.rc('pdf', fonttype=42)

os.environ['KMP_DUPLICATE_LIB_OK']='True'

print('- STREAM Single-cell Trajectory Reconstruction And Mapping -',flush=True)
print('Version %s\n' % st.__version__,flush=True)
print('sys.argv is', sys.argv)

def main():
    sns.set_style('white')
    sns.set_context('poster')
    parser = argparse.ArgumentParser(description='%s Parameters' % __tool_name__ ,formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("-m", "--data-file", dest="input_filename",default = None, help="input file name, pkl format from Stream preprocessing module", metavar="FILE")
    parser.add_argument("--flag_useprecomputed",dest="flag_useprecomputed", action="store_true", help="use precomputed features for PCA")
    parser.add_argument("--flag_firstpc",dest="flag_firstpc", action="store_true", help="Use the first principal component")
    parser.add_argument("--flag_pca",dest="flag_pca", action="store_true",  help="perform PCA")
    parser.add_argument("--flag_variable",dest="flag_variable", action="store_true",  help="find variable genes")
    parser.add_argument("-of","--of",dest="output_filename_prefix", default="StreamiFSOutput",  help="output file name prefix")

    parser.add_argument("-lf","--loess_fraction",dest="loess_fraction", type=float, default=None, help="loess fraction")
    parser.add_argument("-per",dest="percentile", type = int, default=None,  help="percent of variable genes to find")
    parser.add_argument("-n_g",dest="num_genes", type=int, default=None,  help="num genes")
    parser.add_argument("-n_j",dest="num_jobs", type=int, default=None,  help="num jobs")
    parser.add_argument("-feat",dest="feature", default=None,   help="feature")
    parser.add_argument("-n_pc",dest="num_principal_components", type=int, default=None,  help="num principal components")
    parser.add_argument("-max_pc",dest="max_principal_components", type=int, default=None, help="max principal components")
    parser.add_argument("-fig_width",dest="fig_width", type=int, default=8, help="")
    parser.add_argument("-fig_height",dest="fig_height", type=int, default=8, help="")

    parser.add_argument("--flag",dest="flag", action="store_true",  help="debugging flag")

    args = parser.parse_args()

    print('Starting feature selection procedure...')
    print(args)
    workdir = "./"

    adata = st.read(file_name=args.input_filename, file_format='pkl', experiment='rna-seq', workdir=workdir)

    print('Input: '+ str(adata.obs.shape[0]) + ' cells, ' + str(adata.var.shape[0]) + ' genes')
    #print('N_genes is ' + str(args.num_genes))

    if (args.flag_variable):
        st.select_variable_genes(adata,loess_frac=args.loess_fraction,percentile=args.percentile,n_genes=args.num_genes,n_jobs=args.num_jobs, save_fig=True, fig_name=(args.output_filename_prefix + '_variable_genes.png'), fig_size=(args.fig_width,args.fig_height ), fig_path="./")

    if (args.flag_pca):
        st.select_top_principal_components(adata, feature=args.feature,n_pc=args.num_principal_components,max_pc=args.max_principal_components,first_pc=args.flag_firstpc,use_precomputed=args.flag_useprecomputed, save_fig=True, fig_name=(args.output_filename_prefix + '_pca.png'), fig_size=(args.fig_width,args.fig_height ), fig_path='./')

    st.write(adata,file_name=(args.output_filename_prefix + '_stream_result.pkl'),file_path='./',file_format='pkl')
    print('Output: '+ str(adata.obs.shape[0]) + ' cells, ' + str(adata.var.shape[0]) + ' genes')

    print('Finished computation.')

if __name__ == "__main__":
    main()

reason: import numpy,import networkx


text:
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All"
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

df1=pd.read_csv('../input/forest-cover-type-prediction/train.csv')
df_test1=pd.read_csv('../input/forest-cover-type-prediction/test.csv')
df_test2=pd.read_csv('../input/forest-cover-type-prediction/test3.csv')
df=df1.copy()
df_test=df_test1.copy()
df
pd.set_option('display.max_columns',None)
df.drop(columns=['Id','Cover_Type'],inplace=True)
df_test.drop(columns=['Id'],inplace=True)
df_test
X_train=df
Y_train=df1.iloc[:,-1]
X_train
df_test
from collections import Counter

from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.svm import SVC
from xgboost import XGBClassifier
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve
from lightgbm import LGBMClassifier

sns.set(style='white', context='notebook', palette='deep')
kfold = StratifiedKFold(n_splits=10)

random_state = 2
classifiers = []
classifiers.append(SVC(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state),random_state=random_state,learning_rate=0.1))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
classifiers.append(MLPClassifier(random_state=random_state))
classifiers.append(KNeighborsClassifier())
classifiers.append(LogisticRegression(random_state = random_state))
classifiers.append(LinearDiscriminantAnalysis())
classifiers.append(XGBClassifier(random_state = random_state))
classifiers.append(LGBMClassifier(random_state = random_state))

cv_results = []
for classifier in classifiers :
    score=cross_val_score(classifier, X_train, y = Y_train, scoring = "accuracy", cv = kfold, n_jobs=-1)
    cv_results.append(score)
    print('{} crossvalidation score:{}\n'.format(classifier,score.mean()))
cv_means = []
cv_std = []
for cv_result in cv_results:
    cv_means.append(cv_result.mean())
    cv_std.append(cv_result.std())

cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm":["SVC","DecisionTree","AdaBoost",
"RandomForest","ExtraTrees","GradientBoosting","MultipleLayerPerceptron","KNeighboors","LogisticRegression","LinearDiscriminantAnalysis",'XGboost','LGboost']})

g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
g.set_xlabel("Mean Accuracy")
g = g.set_title("Cross validation scores")

from sklearn.model_selection import train_test_split
xtrain,xtest,ytrain,ytest=train_test_split(X_train.values,Y_train.values,test_size=0.2)
from sklearn.metrics import accuracy_score
RFC = RandomForestClassifier(random_state=random_state)
RFC.fit(xtrain,ytrain)
ypred=RFC.predict(xtest)
score=cross_val_score(RFC,X_train,Y_train,scoring='accuracy',cv=kfold,n_jobs=-1)

# Best score
print('Crossval score for random forest: {}'.format(score.mean()))
print('Accuracy score for random forest: {}'.format(accuracy_score(ytest,ypred)))
RFC.get_params()
from sklearn.metrics import accuracy_score
RFC2 = RandomForestClassifier(random_state=random_state,
                             n_estimators=500,
                             max_depth=32,
                             min_samples_leaf=1,
                             criterion='entropy')
RFC2.fit(xtrain,ytrain)
ypred=RFC2.predict(xtest)
score=cross_val_score(RFC2,X_train,Y_train,scoring='accuracy',cv=kfold,n_jobs=-1)

# Best score
print('Crossval score for random forest: {}'.format(score.mean()))
print('Accuracy score for random forest: {}'.format(accuracy_score(ytest,ypred)))
et=ExtraTreesClassifier(random_state=random_state)
et.fit(xtrain,ytrain)
ypred=et.predict(xtest)
score=cross_val_score(et,X_train,Y_train,scoring='accuracy',cv=kfold,n_jobs=-1)

# Best score
print('Crossval score for extra trees classifier: {}'.format(score.mean()))
print('Accuracy score for extra trees classifier: {}'.format(accuracy_score(ytest,ypred)))

et2=ExtraTreesClassifier()
et2.get_params()
et2=ExtraTreesClassifier(bootstrap=False, ccp_alpha=0.0, class_weight=None,
                     criterion='entropy', max_depth=38, max_features='auto',
                     max_leaf_nodes=None, max_samples=None,
                     min_impurity_decrease=0.0, min_impurity_split=None,
                     min_samples_leaf=1, min_samples_split=2,
                     min_weight_fraction_leaf=0.0, n_estimators=500,
                     n_jobs=None, oob_score=False, random_state=0, verbose=0,
                     warm_start=False)
et2.fit(xtrain,ytrain)
ypred=et2.predict(xtest)
score=cross_val_score(et2,X_train,Y_train,scoring='accuracy',cv=kfold,n_jobs=-1)

# Best score
print('Crossval score for extra trees classifier: {}'.format(score.mean()))
print('Accuracy score for extra trees classifier: {}'.format(accuracy_score(ytest,ypred)))
lgb2=LGBMClassifier(random_state=random_state)
lgb2.fit(xtrain,ytrain)
ypred=lgb2.predict(xtest)
score=cross_val_score(lgb2,X_train,Y_train,scoring='accuracy',cv=kfold,n_jobs=-1)

# Best score
print('Crossval score for Lightgb classifier: {}'.format(score.mean()))
print('Accuracy score for Lightgb classifier: {}'.format(accuracy_score(ytest,ypred)))
lgb=LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,
        importance_type='split', learning_rate=0.2, max_depth=-1,
        min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
        n_estimators=200, n_jobs=4, num_leaves=63, objective=None,
        random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=True,
        subsample=1.0, subsample_for_bin=200000, subsample_freq=0)
lgb.fit(xtrain,ytrain)
ypred=lgb.predict(xtest)
score=cross_val_score(lgb,X_train,Y_train,scoring='accuracy',cv=kfold,n_jobs=-1)

# Best score
print('Crossval score for Lightgb classifier: {}'.format(score.mean()))
print('Accuracy score for Lightgb classifier: {}'.format(accuracy_score(ytest,ypred)))
vc= VotingClassifier(estimators=[('rfc', RFC2), ('extc', et2),
('lgb',lgb)], voting='soft', n_jobs=-1)
vc.fit(xtrain,ytrain)
ypred=vc.predict(xtest)
score=cross_val_score(vc,X_train,Y_train,scoring='accuracy',cv=kfold,n_jobs=-1)

# Best score
print('Crossval score for Lightgb classifier: {}'.format(score.mean()))
print('Accuracy score for Lightgb classifier: {}'.format(accuracy_score(ytest,ypred)))

"""
from sklearn.ensemble import StackingClassifier
estimators = [ ('rf', RFC2),
     ('et', et2)]

sc= StackingClassifier(estimators=estimators, final_estimator=lgb)
sc.fit(xtrain,ytrain)
ypred=sc.predict(xtest)
score=cross_val_score(sc,X_train,Y_train,scoring='accuracy',cv=kfold,n_jobs=-1)

# Best score
print('Crossval score for Lightgb classifier: {}'.format(score.mean()))
print('Accuracy score for Lightgb classifier: {}'.format(accuracy_score(ytest,ypred)))"""
vc.fit(X_train,Y_train)
ypred=vc.predict(df_test.values)

id=df_test1['Id']
dict={'ID':id,'Cover_Type':ypred}
dfsub=pd.DataFrame(dict)
dfsub.to_csv('./submission_ensemblevoting.csv', index=False)
"""
#ExtraTrees
et2= ExtraTreesClassifier()

## Search grid for optimal parameters
ex_param_grid = {

 'criterion': ['gini','entropy'],
 'max_depth':[5,10,25],
 'max_features':[1,3,7],
 'max_samples': [0.2],
 'min_samples_leaf': [1,2,5],
 'min_samples_split': [2,5,7],
 'n_estimators': [100,200,300],
 }

gset = GridSearchCV(et2,param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs=-1, verbose = 1)

gset.fit(X_train,Y_train)
gset_best = gset.best_estimator_

# Best score
print(gset.best_score_)
print(gset.best_estimator_)"""

"""
# RFC Parameters tunning
RFC = RandomForestClassifier()

## Search grid for optimal parameters
rf_param_grid = {"max_depth": [None],
              "max_features": [1, 3, 10],
              "min_samples_split": [2, 3, 10],
              "min_samples_leaf": [1, 3, 10],
              "bootstrap": [False],
              "n_estimators" :[100,300],
              "criterion": ["gini"]}

rf_param_grid = {
    'bootstrap': [True],
    'max_depth': [80, 90, 100, 110],
    'max_features': [2, 3],
    'min_samples_leaf': [3, 4, 5],
    'min_samples_split': [8, 10, 12],
    'n_estimators': [100, 200, 300, 1000]
}

gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)

gsRFC.fit(X_train,Y_train)

RFC_best = gsRFC.best_estimator_

# Best score
gsRFC.best_score_"""
"""
RFC2 = RandomForestClassifier()
rf_param_grid = {
    'bootstrap': [True],
    'max_depth': [32],
    'max_features': [2],
    'min_samples_leaf': [1],
    'min_samples_split': [6],
    'n_estimators': [300]
}

gsRFC2 = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)

gsRFC2.fit(X_train,Y_train)
gsRFC2.best_score_"""
pd.DataFrame(RFC.feature_importances_,
             index=X_train.columns, columns=['Importance']).sort_values(
    by='Importance', ascending=False)[:10]
pd.DataFrame(et.feature_importances_,
             index=X_train.columns, columns=['Importance']).sort_values(
    by='Importance', ascending=False)[:10]
def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
                        n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5)):
    """Generate a simple plot of the test and training learning curve"""
    plt.figure()
    plt.title(title)
    if ylim is not None:
        plt.ylim(*ylim)
    plt.xlabel("Training examples")
    plt.ylabel("Score")
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    plt.grid()

    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std, alpha=0.1,
                     color="r")
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                     test_scores_mean + test_scores_std, alpha=0.1, color="g")
    plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
             label="Training score")
    plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
             label="Cross-validation score")

    plt.legend(loc="best")
    return plt

g = plot_learning_curve(RFC,"Random Forest learning curves",X_train,Y_train,cv=kfold)
g = plot_learning_curve(et,"Extra trees learning curves",X_train,Y_train,cv=kfold)
g = plot_learning_curve(gsRFC2,"Random Forest tuned learning curves",X_train,Y_train,cv=kfold)
#g = plot_learning_curve(gsExtC.best_estimator_,"ExtraTrees learning curves",X_train,Y_train,cv=kfold)

def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
                        n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5)):
    """Generate a simple plot of the test and training learning curve"""
    plt.figure()
    plt.title(title)
    if ylim is not None:
        plt.ylim(*ylim)
    plt.xlabel("Training examples")
    plt.ylabel("Score")
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    plt.grid()

    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std, alpha=0.1,
                     color="r")
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                     test_scores_mean + test_scores_std, alpha=0.1, color="g")
    plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
             label="Training score")
    plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
             label="Cross-validation score")

    plt.legend(loc="best")
    return plt

g = plot_learning_curve(lgb,"lgb tuned learning curves",X_train,Y_train,cv=kfold)
g = plot_learning_curve(lgb2,"Normal lgb learning curves",X_train,Y_train,cv=kfold)

#g = plot_learning_curve(gsExtC.best_estimator_,"ExtraTrees learning curves",X_train,Y_train,cv=kfold)
#g = plot_learning_curve(vc,"voting classifier learning curves",X_train,Y_train,cv=kfold)
gset_best.fit(X_train,Y_train)
ypred=gset_best.predict(df_test.values)

id=df_test1['Id']
dict={'ID':id,'Cover_Type':ypred}
dfsub=pd.DataFrame(dict)
dfsub.to_csv('./submission_gset.csv', index=False)

max_stars_repo_name: josepablocam/janus-public

max_issues_repo_name: josepablocam/janus-public

max_forks_repo_name: josepablocam/janus-public


text:
from setuptools import setup
import numpy

setup(name='gdalutils',
      version='0.1',
      description='GDAL utils',
      url='http://github.com/fathom-global/gdalutils',
      author='Jeison Sosa',
      author_email='sosa.jeison@gmail.com',
      license='MIT',
      packages=['gdalutils'],
      zip_safe=False,
      )

max_stars_repo_name: fathom-global/gdalutils

max_issues_repo_name: fathom-global/gdalutils

max_forks_repo_name: fathom-global/gdalutils


text:
# Copyright (c) Facebook, Inc. and its affiliates.

#%%

import numpy as np
import math
import glob
import pandas as pd
import matplotlib.pyplot as plt
plt.rcParams.update({'font.size': 28})
plt.ion()

from mpl_toolkits.mplot3d import Axes3D
from matplotlib.patches import FancyArrowPatch
from mpl_toolkits.mplot3d import proj3d
from matplotlib import animation

plt.rcParams.update({'font.size': 28})
plt.ion()

BASE_PATH = '/home/paloma/code/fair_ws/tactile-in-hand/inhandpy'
logdir = f'{BASE_PATH}/local/datalogs/'

#%% Helper functions

def pandas_string_to_numpy(arr_str):
    arr_npy = np.fromstring(arr_str.replace('\n', '').replace(
        '[', '').replace(']', '').replace('  ', ' '), sep=', ')
    return arr_npy

def transform_axes_tips(R, t=None, scale=1):
    """
    R: Rotation matrix of shape (N,3,3)
    t: Translation matrix of shape (N,3)
    scale: Length of axes
    Returns (N, 4, 3) coordinate of 4 axes tips (origin, X, Y, Z)
    """
    if R.ndim is 2:
        R = np.expand_dims(R, axis=0)

    if t is None:
        t = np.zeros((R.shape[0],3))
    elif t.ndim is 1:
        t = np.expand_dims(t, axis=0)

    T = np.concatenate((R,  np.expand_dims(t, axis=-1)), axis=-1)
    origin = np.matmul(T, np.expand_dims(np.repeat([[0,0,0,1]], T.shape[0], axis=0), axis=-1)).squeeze(axis=-1)
    Xtip = np.matmul(T, np.expand_dims(np.repeat([[scale,0,0,1]], T.shape[0], axis=0), axis=-1)).squeeze(axis=-1)
    Ytip = np.matmul(T, np.expand_dims(np.repeat([[0,scale,0,1]], T.shape[0], axis=0), axis=-1)).squeeze(axis=-1)
    Ztip = np.matmul(T, np.expand_dims(np.repeat([[0,0,scale,1]], T.shape[0], axis=0), axis=-1)).squeeze(axis=-1)

    return np.stack([origin, Xtip, Ytip, Ztip], axis=1)

class Arrow3D(FancyArrowPatch):
    def __init__(self, verts3d, *args, **kwargs):
        FancyArrowPatch.__init__(self, (0, 0), (0, 0), *args, **kwargs)
        self._verts3d = verts3d[:,0], verts3d[:,1], verts3d[:,2]

    def draw(self, renderer):
        xs3d, ys3d, zs3d = self._verts3d
        xs, ys, zs = proj3d.proj_transform(xs3d, ys3d, zs3d, renderer.M)
        self.set_positions((xs[0], ys[0]), (xs[1], ys[1]))
        FancyArrowPatch.draw(self, renderer)

def plot_axes(ax, axes_tips, alpha=1.0, linewidth=1.0):
    arrow_prop_dict = dict(mutation_scale=20, arrowstyle='-|>', shrinkA=0, shrinkB=0, linewidth=linewidth, alpha=alpha)

    for axes_tip in axes_tips:
        a = Arrow3D(axes_tip[[0,1],:], **arrow_prop_dict, color='#fc8d62')
        ax.add_artist(a)
        a = Arrow3D(axes_tip[[0,2],:], **arrow_prop_dict, color='#66c2a5')
        ax.add_artist(a)
        a = Arrow3D(axes_tip[[0,3],:], **arrow_prop_dict, color='#8da0cb')
        ax.add_artist(a)

def load_dataset(dataset, label, seq_idx):
    csvfiles = sorted(glob.glob(f'{logdir}/{label}/{dataset}/**/*.csv', recursive=True))
    print(f"Loading sequences: {csvfiles}")
    num_seqs = len(csvfiles)

    print(f"Reading sequence: {csvfiles[seq_idx]}")

    dataframe = pd.read_csv(csvfiles[seq_idx])
    num_steps = dataframe.shape[0]

    step = num_steps-1
    graph_obj_trans = pandas_string_to_numpy(dataframe['graph/obj/trans'][step]).reshape((step+1,3))
    graph_obj_rot = pandas_string_to_numpy(dataframe['graph/obj/rot'][step]).reshape((step+1,3,3))

    gt_obj_trans = pandas_string_to_numpy(dataframe['gt/obj/trans'][step]).reshape((step+1,3))
    gt_obj_rot = pandas_string_to_numpy(dataframe['gt/obj/rot'][step]).reshape(step+1,3,3)

    return graph_obj_trans, graph_obj_rot, gt_obj_trans, gt_obj_rot

#%% CUBE

dataset = 'tacto/cube/'
label = 'proposed'
seq_idx = 5

graph_obj_trans, graph_obj_rot, gt_obj_trans, gt_obj_rot = load_dataset(dataset, label, seq_idx)

num_frames = 10
FPS = 15.0
fig = plt.figure(figsize=(num_frames*1.2, 2))
idx = np.round(np.linspace(0, gt_obj_rot.shape[0] - 1, num_frames)).astype(int)
for i in range(num_frames):
    ax = fig.add_subplot(1, num_frames, i+1, projection='3d')
    ax.plot3D(gt_obj_trans[0:idx[i],0], gt_obj_trans[0:idx[i],1], gt_obj_trans[0:idx[i],2], 'gray',linewidth=4,alpha=1.0)
    ax.plot3D(graph_obj_trans[0:idx[i],0], graph_obj_trans[0:idx[i],1], graph_obj_trans[0:idx[i],2], 'gray',linewidth=2,alpha=0.3)
    plot_axes(ax, transform_axes_tips(graph_obj_rot[idx[i],:,:], t=graph_obj_trans[idx[i],:], scale=1), alpha=0.3, linewidth=4.0)
    plot_axes(ax, transform_axes_tips(gt_obj_rot[idx[i],:,:], t=gt_obj_trans[idx[i],:], scale=1), alpha=1.0, linewidth=2.0)
    ax.set_title('t: {:.2f}s'.format(float(idx[i])/FPS), y = 0.0) # increase or decrease y as needed
    ax.title.set_size(8)
    ax.view_init(azim=37.5, elev=30)
    ax.w_xaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
    ax.set_xlim(-0.7, 0.7)
    ax.set_ylim(-0.7, 0.7)
    ax.set_zlim(-0.7, 0.7)
    ax._axis3don = False
    ax.set_xticklabels([])
    ax.set_yticklabels([])
    ax.set_zticklabels([])

plt.subplots_adjust(wspace=-0.1, hspace=0)
plt.savefig(f'{logdir}/{label}/{dataset}/qual_traj_episode_{seq_idx:03d}.png')
plt.show()

#%% SPHERE

# Sphere has mainly translation and no rotation tracking.
# To show translation, one has to plot the traced trajectory and set scale accordingly.

dataset = 'tacto/sphere/'
label = 'proposed'
seq_idx = 0

graph_obj_trans, graph_obj_rot, gt_obj_trans, gt_obj_rot = load_dataset(dataset, label, seq_idx)

num_frames = 10
FPS = 15.0
fig = plt.figure(figsize=(num_frames*1.2, 2))
idx = np.round(np.linspace(0, gt_obj_rot.shape[0] - 1, num_frames)).astype(int)
for i in range(num_frames):
    ax = fig.add_subplot(1, num_frames, i+1, projection='3d')
    ax.plot3D(gt_obj_trans[0:idx[i],0], gt_obj_trans[0:idx[i],1], gt_obj_trans[0:idx[i],2], 'gray',linewidth=2,alpha=1.0)
    ax.plot3D(graph_obj_trans[0:idx[i],0], graph_obj_trans[0:idx[i],1], graph_obj_trans[0:idx[i],2], 'gray',linewidth=2,alpha=0.3)
    plot_axes(ax, transform_axes_tips(graph_obj_rot[idx[i],:,:], t=graph_obj_trans[idx[i],:], scale=0.01), alpha=0.3, linewidth=1.5)
    plot_axes(ax, transform_axes_tips(gt_obj_rot[idx[i],:,:], t=gt_obj_trans[idx[i],:], scale=0.01), alpha=1.0, linewidth=1.5)
    ax.set_title('t: {:.2f}s'.format(float(idx[i])/FPS), y = 0.0) # increase or decrease y as needed
    ax.title.set_size(8)
    ax.view_init(azim=37.5, elev=30)
    ax.w_xaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
    ax.set_xlim(-0.025, -0.007)
    ax.set_ylim(-0.009, 0.009)
    ax.set_zlim(0.026, 0.044)
    ax._axis3don = False
    ax.set_xticklabels([])
    ax.set_yticklabels([])
    ax.set_zticklabels([])

plt.subplots_adjust(wspace=0.1, hspace=0)
plt.savefig(f'{logdir}/{label}/{dataset}/qual_traj_episode_{seq_idx:03d}.png')
plt.show()

#%% TOY BRICK

dataset = 'tacto/toy_brick/'
label = 'proposed'
seq_idx = 0

graph_obj_trans, graph_obj_rot, gt_obj_trans, gt_obj_rot = load_dataset(dataset, label, seq_idx)

num_frames = 10
FPS = 15.0
fig = plt.figure(figsize=(num_frames*1.2, 2))
idx = np.round(np.linspace(0, gt_obj_rot.shape[0] - 1, num_frames)).astype(int)
for i in range(num_frames):
    ax = fig.add_subplot(1, num_frames, i+1, projection='3d')
    ax.plot3D(gt_obj_trans[0:idx[i],0], gt_obj_trans[0:idx[i],1], gt_obj_trans[0:idx[i],2], 'gray',linewidth=2,alpha=1.0)
    ax.plot3D(graph_obj_trans[0:idx[i],0], graph_obj_trans[0:idx[i],1], graph_obj_trans[0:idx[i],2], 'gray',linewidth=2,alpha=0.3)
    plot_axes(ax, transform_axes_tips(graph_obj_rot[idx[i],:,:], t=graph_obj_trans[idx[i],:], scale=1), alpha=0.3, linewidth=4.0)
    plot_axes(ax, transform_axes_tips(gt_obj_rot[idx[i],:,:], t=gt_obj_trans[idx[i],:], scale=1), alpha=1.0, linewidth=2.0)
    ax.set_title('t: {:.2f}s'.format(float(idx[i])/FPS), y = 0.0) # increase or decrease y as needed
    ax.title.set_size(8)
    ax.view_init(azim=37.5, elev=30)
    ax.w_xaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
    ax.set_xlim(-0.7, 0.7)
    ax.set_ylim(-0.7, 0.7)
    ax.set_zlim(-0.7, 0.7)
    ax._axis3don = False
    ax.set_xticklabels([])
    ax.set_yticklabels([])
    ax.set_zticklabels([])

plt.subplots_adjust(wspace=-0.1, hspace=0)
plt.savefig(f'{logdir}/{label}/{dataset}/qual_traj_episode_{seq_idx:03d}.png')
plt.show()

#%% TOY HUMAN

dataset = 'tacto/toy_human/'
label = 'proposed'
seq_idx = 0

graph_obj_trans, graph_obj_rot, gt_obj_trans, gt_obj_rot = load_dataset(dataset, label, seq_idx)

num_frames = 9
FPS = 15.0
fig = plt.figure(figsize=(num_frames*1.2, 2))
idx = np.round(np.linspace(0, gt_obj_rot.shape[0] - 1, num_frames)).astype(int)
for i in range(num_frames):
    ax = fig.add_subplot(1, num_frames, i+1, projection='3d')
    ax.plot3D(gt_obj_trans[0:idx[i],0], gt_obj_trans[0:idx[i],1], gt_obj_trans[0:idx[i],2], 'gray',linewidth=2,alpha=1.0)
    ax.plot3D(graph_obj_trans[0:idx[i],0], graph_obj_trans[0:idx[i],1], graph_obj_trans[0:idx[i],2], 'gray',linewidth=2,alpha=0.3)
    plot_axes(ax, transform_axes_tips(graph_obj_rot[idx[i],:,:], t=graph_obj_trans[idx[i],:], scale=1), alpha=0.3, linewidth=4.0)
    plot_axes(ax, transform_axes_tips(gt_obj_rot[idx[i],:,:], t=gt_obj_trans[idx[i],:], scale=1), alpha=1.0, linewidth=2.0)
    ax.set_title('t: {:.2f}s'.format(float(idx[i])/FPS), y = 0.0) # increase or decrease y as needed
    ax.title.set_size(8)
    ax.view_init(azim=37.5, elev=30)
    ax.w_xaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
    ax.set_xlim(-0.7, 0.7)
    ax.set_ylim(-0.7, 0.7)
    ax.set_zlim(-0.7, 0.7)
    ax._axis3don = False
    ax.set_xticklabels([])
    ax.set_yticklabels([])
    ax.set_zticklabels([])

plt.subplots_adjust(wspace=-0.1, hspace=0)
plt.savefig(f'{logdir}/{label}/{dataset}/qual_traj_episode_{seq_idx:03d}.png')
plt.show()

#%% REAL SPHERE

# Sphere has mainly translation and no rotation tracking.
# To show translation, one has to plot the traced trajectory and set scale accordingly.

dataset = 'real/sphere/'
label = 'proposed'
seq_idx=5

graph_obj_trans, graph_obj_rot, gt_obj_trans, gt_obj_rot = load_dataset(dataset, label, seq_idx)

num_frames = 10
FPS = 15.0
fig = plt.figure(figsize=(num_frames*1.2, 2))
max_frame = 100 # gt_obj_rot.shape[0] - 1
idx = np.round(np.linspace(0, max_frame, num_frames)).astype(int)
for i in range(num_frames):
    ax = fig.add_subplot(1, num_frames, i+1, projection='3d')
    ax.plot3D(gt_obj_trans[0:idx[i],0], gt_obj_trans[0:idx[i],1], gt_obj_trans[0:idx[i],2], 'gray',linewidth=2,alpha=1.0)
    ax.plot3D(graph_obj_trans[0:idx[i],0], graph_obj_trans[0:idx[i],1], graph_obj_trans[0:idx[i],2], 'gray',linewidth=2,alpha=0.4)

    # ARROW_SCALE = 0.1
    # plot_axes(ax, transform_axes_tips(graph_obj_rot[idx[i],:,:], t=graph_obj_trans[idx[i],:], scale=ARROW_SCALE), alpha=0.5, linewidth=1.5)
    # plot_axes(ax, transform_axes_tips(gt_obj_rot[idx[i],:,:], t=gt_obj_trans[idx[i],:], scale=ARROW_SCALE), alpha=1.0, linewidth=1.5)

    ax.set_title('t: {:.2f}s'.format(float(idx[i])/FPS), y = 0.0) # increase or decrease y as needed
    ax.title.set_size(8)
    ax.view_init(azim=37.5, elev=30)
    ax.w_xaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))

    # ax.set_xlim(-0.07, 0.07)
    # ax.set_ylim(-0.07, 0.07)
    # ax.set_zlim(-0.07, 0.07)

    # print(ax.get_xlim())
    # print(ax.get_ylim())
    # print(ax.get_zlim())

    ax._axis3don = False
    ax.set_xticklabels([])
    ax.set_yticklabels([])
    ax.set_zticklabels([])

plt.subplots_adjust(wspace=0.1, hspace=0)
filename = f'{logdir}/{label}/{dataset}/qual_traj_episode_{seq_idx:03d}.png'
plt.savefig(filename)
print(f"Saved figure to: {filename}")
plt.show()

#%% REAL CUBE

dataset = 'real/cube/'
label = 'proposed'
seq_idx = 8

graph_obj_trans, graph_obj_rot, gt_obj_trans, gt_obj_rot = load_dataset(dataset, label, seq_idx)

num_frames = 10
FPS = 15.0
fig = plt.figure(figsize=(num_frames*1.2, 2))
max_frame = 100 # gt_obj_rot.shape[0] - 1
idx = np.round(np.linspace(0, max_frame, num_frames)).astype(int)
for i in range(num_frames):
    ax = fig.add_subplot(1, num_frames, i+1, projection='3d')
    ax.plot3D(gt_obj_trans[0:idx[i],0], gt_obj_trans[0:idx[i],1], gt_obj_trans[0:idx[i],2], 'gray',linewidth=2,alpha=1.0)
    ax.plot3D(graph_obj_trans[0:idx[i],0], graph_obj_trans[0:idx[i],1], graph_obj_trans[0:idx[i],2], 'gray',linewidth=2,alpha=0.3)
    # plot_axes(ax, transform_axes_tips(graph_obj_rot[idx[i],:,:], t=graph_obj_trans[idx[i],:], scale=1), alpha=0.4, linewidth=4.0)
    # plot_axes(ax, transform_axes_tips(gt_obj_rot[idx[i],:,:], t=gt_obj_trans[idx[i],:], scale=1), alpha=1.0, linewidth=2.0)
    ax.set_title('t: {:.2f}s'.format(float(idx[i])/FPS), y = 0.0) # increase or decrease y as needed
    ax.title.set_size(8)
    ax.view_init(azim=37.5, elev=30)
    ax.w_xaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
    # ax.set_xlim(-0.5, 0.5)
    # ax.set_ylim(-0.5, 0.5)
    # ax.set_zlim(-0.5, 0.5)
    ax._axis3don = False
    ax.set_xticklabels([])
    ax.set_yticklabels([])
    ax.set_zticklabels([])

plt.subplots_adjust(wspace=-0.1, hspace=0)
plt.savefig(f'{logdir}/{label}/{dataset}/qual_traj_episode_{seq_idx:03d}.png')
plt.show()

# %%