text:
"""
v1.0
Classe principal, controla jogo, captura de tela e solicita previsao e treinamento
"""

from Chrome_T_Rex_Rush_Like_Gym import main as dinoGame
from captura_tela import CapturaTela
from network import Network
from collections import deque
import numpy as np

class PlayDino:
    """Controla jogo, captura tela e se comunica com a rede (model)"""
    def __init__(self, path_model='model.h5',
                 time_steps=3,
                 prever=True,
                 treinar=True,
                 substituir_model=False,
                 epsilon=0.0,
                 rodada=0,
                 mostra_tela=False,
                 frames_por_previsao=5,
                 frames_iniciais_ignorados=20,
                 shape_frame=(90, 590, 1)):
        """
        Inicializa e executa sistema inteiro

        :param path_model: Caminho em disco para o modelo, caso nao existir sera criado model novo
        :param time_steps: Tamanho da janela de frames observados pela rede LSTM
        :param prever: Se sera feito previsoes pela rede, caso negativo, somente humano ira jogar
        :param treinar: Se a rede sera treinada. So sera treinada se "prever" estiver ativado
        :param substituir_model: Se deseja substituir modelo caso existir no caminho informado
        :param epsilon: Chance de realizar acao aleatoria (sera utilizado somente em rodadas pares). Entre 0.0 e 1.0
        :param rodada: Rodada do treinamento, caso seja dados sejam salvos, ira usar este valor como nome de pasta
        :param mostra_tela: Se deseja visualizar a tela do jogo capturada (frame). Se estiver jogando manual esta tela
        fica em primeiro plano, precisa clicar na tela do jogo
        :param frames_por_previsao: Quantidade de frames que serao executados a cada previsao realizada (quanto menor,
         mais frames serao coletados). Para evitar overfitting devido a muitos dados extremamente parecidos
        :param frames_iniciais_ignorados: Quantidade de frames iniciais de cada rodada que nao serao armazenados para
         serem utilizados no treinamento (nao apresentam relevancia)
        """
        self.cap_tela = CapturaTela()

        if prever:
            # 0 = nao faz nada, 1 = pula e 2 = abaixa
            acoes_game = [0, 1, 2]

            # Armazena a ultima janela para fazer previsao
            deque_steps = deque(maxlen=time_steps)

            # Inicializa rede
            rede = Network(path_model=path_model, time_steps=time_steps, substituir_model=substituir_model,
                           shape_frame=shape_frame)
            # Prepara pasta para salvar frames da rodada
            self.cap_tela.preparar_salvar_frame(rodada)

            # Local onde os frames serao armazenado antes de serem salvos em disco
            armazena_frames = []
            # Usado para ver evolucao do modelo ao longo das rodadas
            historico_armazena_frames = []

            # Apenas para declaram a variavel pois sera utilizada para iniciar o jogo
            previsao = 0

            print("Tudo pronto, pressione espaco para iniciar! ESC para fechar")

        else:
            # Apenas para declaram a variavel pois sera utilizada para iniciar o jogo
            tecla_manual = 0
            print('Mode de jogo manual, pressione espaco para comecar a jogar! ESC para fechar')

        # Inicializa game
        self.dino = dinoGame.GameDino()
        dinoGame.introscreen()
        self.dino.reset_game()

        self.isGameQuit = False
        isGameOver = False

        while not self.isGameQuit:
            #  Computador jogando
            if prever:
                # Quantos frames ira pular a cada acao realizada
                for x in range(frames_por_previsao):
                    self.isGameQuit, isGameOver = self.dino.play(previsao)
                    if self.isGameQuit or isGameOver:
                        break

                # Captura parte da tela
                frame, tecla_manual = self.cap_tela.capturar_frame_tela(mostra_tela)

                # Se for pressionado ESC no teclado o jogo eh encerrado
                if tecla_manual == 3 or self.isGameQuit:
                    self.encerrar()
                    continue

                deque_steps.append(frame)

                if isGameOver:
                    # Se for pressionado "s" no teclado, sera salvo o modelo atual em disco
                    if self.cap_tela.is_salvar_model():
                        if path_model.rfind('.h5') != 1:
                            rede.salvar_model('{}{}.h5'.format(path_model[:path_model.rfind('.h5')], rodada))
                        else:
                            rede.salvar_model('{}{}.h5'.format(path_model, rodada))

                    deque_steps = deque(maxlen=time_steps)

                    # Armazena frames e treina se atingir quantidade minima necessaria para fazer treinamento
                    # CUIDADO: Se alterar valores de coleta de frames, time steps ou frames iniciais ignorados,
                    # pode nao coletar frames suficientes quando morrer no primeiro objeto
                    if treinar and (len(armazena_frames) > frames_iniciais_ignorados + time_steps):
                        # Tira os primeiros frames que nao possuem relevancia
                        armazena_frames_temp = armazena_frames[frames_iniciais_ignorados:]

                        # Salva os frames, menos:
                        # Ultimo = ultima acao antes do game over (muito encima para conseguir alterar resultado)
                        for x in armazena_frames_temp[:-2]:
                            self.cap_tela.salvar_frame(x[0], x[1], 0)
                        # Penultimo frame representa game_over
                        self.cap_tela.salvar_frame(armazena_frames_temp[-2][0], armazena_frames_temp[-2][1], 1)

                        historico_armazena_frames.append(len(armazena_frames))

                        print('Historico jogadas:')
                        print(historico_armazena_frames)
                        print('Rodada salva na pasta: ' + str(rodada) + '. Iniciando treinamento')

                        rede.treinar(rodada)

                        rodada += 1
                        self.cap_tela.preparar_salvar_frame(rodada)

                        armazena_frames = []

                    self.dino.reset_game()
                    isGameOver = False

                # Possui o minimo de steps exigido pela rede LSTM para fazer previsoes
                elif len(deque_steps) >= time_steps:
                    # Faz previsao se nao foi pressionado tecla manualmente no teclado (humano)
                    if tecla_manual == 0:
                        # Sorteia tecla aleatoria
                        if rodada % 2 == 0 and np.random.rand() <= epsilon:
                            previsao = np.random.choice(acoes_game)
                            print("Previsao randomica: " + str(previsao))
                        # Usa conhecimento
                        else:
                            prev = rede.prever_pelo_frame(np.array(deque_steps))
                            previsao = np.argmax(prev)
                            # Se desejar mostrar pontuacao de cada acao prevista
                            # print('{} - {}'.format(prev, previsao))
                    else:
                        previsao = tecla_manual

                    armazena_frames.append([frame, previsao])
            # Humano jogando
            else:
                self.isGameQuit, isGameOver = self.dino.play(tecla_manual)
                if not isGameOver:
                    # Aqui este comando esta sendo usado apenas para pegar tecla pressionada (e mostrar a tela quando
                    # desejado)
                    _, tecla_manual = self.cap_tela.capturar_frame_tela(mostra_tela)

                # Se for pressionado ESC no teclado o jogo eh encerrado
                if tecla_manual == 3 or self.isGameQuit:
                    self.encerrar()
                    continue

    def encerrar(self):
        """Encerra todos servicos e finaliza execucao"""
        self.dino.quit()
        self.cap_tela.encerrar()
        self.isGameQuit = True


text:
from distutils.core import setup
from Cython.Build import cythonize
import numpy as np

setup(
  name = "features",
  ext_modules = cythonize("features.pyx"),
  include_dirs = [np.get_include()]
)

ext: py

reason: import numpy


text:
import numpy as np
from six import iteritems

class PropertyMap(object):
    """Map points in the simulation domain to properties of parts containing the points.

    :param PartMap part_map: Mapper from spatial location to part identifier.
    :param callable prop_map: Map from part identifier to a property value.
    """

    def __init__(self, part_map, prop_map):
        self.partMap = part_map
        self.propMap = prop_map

    def get_part(self, x):
        """Find the part(s) containing one or more points.

        :param x: Coordinate vector or array of coordinate vectors.
        :return: Part identifier (if x is a single coordinate vector); or array of part identifiers,
            of the same shape as x except for the last axis corresponding to coordinate vector
            extent.
        """
        return self.partMap(x)

    def __call__(self, x):
        """Do the mapping.

        :param x: Coordinate vector or array of coordinate vectors.
        :return: Property of the part(s) containing `x`, of the same shape as `x` except for the
            last axis corresponding to coordinate vector extent.
        """
        parts = self.get_part(x)
        if np.isscalar(parts):
            return self.propMap(parts)

        unique_parts = np.unique(parts)
        unique_props = [self.propMap(p) for p in unique_parts]
        obj_type = type(unique_props[0])
        if obj_type is str:
            obj_type = object
        result = np.empty(np.shape(parts), dtype=obj_type)
        for part, prop in zip(unique_parts, unique_props):
            result[parts == part] = prop
        return result

class MaterialPropertyMap(PropertyMap):
    """Map points in the simulation domain to material properties of parts containing the points.

    :param PartMap part_map: Function that takes a spatial location and maps it to a part
    identifier.
    :param dict part_materials: Dict mapping from part identifier to a material name.
    :param qmt.Materials mat_lib: Materials library used to look up the material properties.
    :param str prop_name: Name of the material property to be retrieved for each part.
    :param eunit: Energy unit, passed to `mat_lib.find()`.
    :param fill_value: Value to be filled in places where there is no part or the part does not have
        a material or the material does not have the property `prop_name`. The default behavior
        `fill_value='raise'` is to raise a KeyError in these cases.
    """

    def __init__(self, part_map, part_materials, mat_lib, prop_name, eunit=None,
                 fill_value='raise'):
        self.fillValue = fill_value
        self.materialsDict = dict((p, mat_lib.find(m, eunit)) for p, m in iteritems(part_materials))

        self.partProps = {}
        for p, mat in iteritems(self.materialsDict):
            try:
                if prop_name == 'conductionBandMinimum':
                    self.partProps[p] = mat_lib.conduction_band_minimum(mat)
                elif prop_name == 'valenceBandMaximum':
                    self.partProps[p] = mat_lib.valence_band_maximum(mat)
                elif prop_name == 'lightHoleMass':
                    self.partProps[p] = mat.hole_mass('light', 'dos')
                elif prop_name == 'heavyHoleMass':
                    self.partProps[p] = mat.hole_mass('heavy', 'dos')
                elif prop_name == 'dosHoleMass':
                    self.partProps[p] = mat.hole_mass('dos','dos')
                else:
                    self.partProps[p] = mat[prop_name]
            except KeyError:
                pass

        def prop_map(part):
            try:
                return self.partProps[part]
            except KeyError:
                if self.fillValue == 'raise':
                    raise
                return self.fillValue

        super(MaterialPropertyMap, self).__init__(part_map, prop_map)

max_stars_repo_name: agarwalrounak/qmt

max_issues_repo_name: DalavanCloud/qmt

max_forks_repo_name: DalavanCloud/qmt


text:
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import pandas as pd

import matplotlib

font = {"size": 30}
matplotlib.rc("font", **font)
matplotlib.rcParams["text.usetex"] = True

NUM_SETS_TO_PLOT = 8

# Load BFs and gene set names
treatment_bfs = np.load("./out/geneset_bfs_nutlin.npy")
gene_sets_for_plot = np.load("./out/geneset_names_nutlin.npy")

plt.figure(figsize=(10, 7))

sorted_idx = np.argsort(-np.array(treatment_bfs))[:NUM_SETS_TO_PLOT]
treatment_bfs_to_plot = np.array(treatment_bfs)[sorted_idx]
plt.bar(np.arange(len(treatment_bfs_to_plot)), treatment_bfs_to_plot)
plt.title("Nutlin gene set EBFs")
plt.ylabel("log(EBF)")

plt.xticks(
    np.arange(len(treatment_bfs_to_plot)),
    labels=gene_sets_for_plot[: len(treatment_bfs)][sorted_idx],
)
plt.xticks(rotation=-45, size=20, ha="left")
plt.tight_layout()
plt.savefig("./out/nutlin_gene_sets.png")
# plt.close()
# plt.show()
# import ipdb; ipdb.set_trace()

max_stars_repo_name: andrewcharlesjones/cplvm

max_issues_repo_name: andrewcharlesjones/cplvm

max_forks_repo_name: andrewcharlesjones/cplvm


text:
"""Collection of tools for probability distributions"""
import numpy as np

def random_unitary_basis(kappa):
    Ax1 = random_2d_rotation_in_3d('x', kappa)
    Ay1 = random_2d_rotation_in_3d('y', kappa)
    Az1 = random_2d_rotation_in_3d('z', kappa)
    Ax2 = random_2d_rotation_in_3d('x', kappa)
    Ay1 = random_2d_rotation_in_3d('y', kappa)
    Az1 = random_2d_rotation_in_3d('z', kappa)
    A = np.dot(np.dot(Ax1,Ay1),Az1)
    B = np.dot(np.dot(Az1,Ay1),Ax1)
    return np.dot(A,B)

def random_2d_rotation_in_3d(axis, kappa):
    theta = np.random.vonmises(0, kappa, 1)
    A = np.eye(3)
    if axis is 'z':
        A[0,0] = np.cos(theta)
        A[1,0] = np.sin(theta)
        A[0,1] = - np.sin(theta)
        A[1,1] = np.cos(theta)
        return A
    if axis is 'y':
        A[0,0] = np.cos(theta)
        A[2,0] = np.sin(theta)
        A[0,2] = - np.sin(theta)
        A[2,2] = np.cos(theta)
        return A
    if axis is 'x':
        A[1,1] = np.cos(theta)
        A[2,1] = np.sin(theta)
        A[1,2] = - np.sin(theta)
        A[2,2] = np.cos(theta)
        return A

max_stars_repo_name: McmcNeuron/Generator

max_issues_repo_name: McmcNeuron/Generator

max_forks_repo_name: McmcNeuron/Generator


text:
'''
This code is a rework of
https://github.com/regro/cf-scripts/blob/master/conda_forge_tick/auto_tick.py
This version was not completely importable so the functions had to be copied
here and reimplemented.
We still import some functionality from conda_forge_tick
'''
import logging
import time
import os
import glob
from urllib.error import URLError
import traceback
import json
from uuid import uuid4
from subprocess import SubprocessError, CalledProcessError
import shutil

import github3
import networkx as nx

from conda_forge_tick.utils import (
    frozen_to_json_friendly,
    setup_logger,
    eval_cmd,
    dump_graph,
    load_graph,
    LazyJson
)
from conda_forge_tick.contexts import (
    MigratorContext,
    FeedstockContext,
    MigratorSessionContext
)
from conda_forge_tick.migrators import (
    Version,
    PipMigrator,
    MigrationYaml,
    LicenseMigrator,
    CondaForgeYAMLCleanup,
    ExtraJinja2KeysCleanup,
    Jinja2VarsCleanup,
)
from conda_forge_tick.auto_tick import (
    _compute_time_per_migrator,
)
from conda_forge_tick.status_report import write_version_migrator_status
from conda_forge_tick.git_utils import is_github_api_limit_reached
from conda_forge_tick.xonsh_utils import indir, env
from conda_forge_tick.mamba_solver import is_recipe_solvable

from .git_utils import (
    get_repo,
    push_repo
)
from .dashboard import create_dashboard_from_list

logger = logging.getLogger(__name__)

# change to increase PRs/run of the bot
# NOTE: this number gets doubled for Version migrations
PR_LIMIT = 5
MAX_PR_LIMIT = 50

# Set up types of migrators here
MIGRATORS = [
    Version(
        pr_limit=PR_LIMIT * 2,
        piggy_back_migrations=[
            Jinja2VarsCleanup(),
            PipMigrator(),
            LicenseMigrator(),
            CondaForgeYAMLCleanup(),
            ExtraJinja2KeysCleanup(),
        ],
    ),
]

BOT_RERUN_LABEL = {
    "name": "bot-rerun",
    "color": "#191970",
    "description": (
        "Apply this label if you want the bot to retry "
        "issuing a particular pull-request"
    ),
}

def bot_pr_body(self, feedstock_ctx):
    '''
    Creates the body text of a version pull request
    Used to overwrite conda_forge_tick.migrators.Version.pr_body function

    Parameters
    ----------
    feedstock_ctx: FeedstockContext
        The node attributes of the current feedstock

    Returns
    -------
    str
        Body text of pull request on GitHub for new version releases
    '''
    pred = [
        name
        for name in list(
            self.ctx.effective_graph.predecessors(feedstock_ctx.package_name),
        )
    ]
    # TODO: note that the closing logic needs to be modified when we
    #  issue PRs into other branches for backports
    open_version_prs = [
        muid["PR"]
        for muid in feedstock_ctx.attrs.get("PRed", [])
        if muid["data"].get("migrator_name") == "Version"
        # The PR is the actual PR itself
        and muid.get("PR", {}).get("state", None) == "open"
    ]

    # Display the url so that the maintainer can quickly click on it
    # in the PR body.
    about = feedstock_ctx.attrs.get("meta_yaml", {}).get("about", {})
    upstream_url = about.get("dev_url", "") or about.get("home", "")
    if upstream_url:
        upstream_url_link = ": see [upstream]({upstream_url})".format(
            upstream_url=upstream_url,
        )
    else:
        upstream_url_link = ""

    body = (
        "It is very likely that the current package version for this "
        "feedstock is out of date.\n"
        "Notes for merging this PR:\n"
        "1. Feel free to push to the bot's branch to update this PR if needed.\n"
        "2. The bot will almost always only open one PR per version.\n"
        "Checklist before merging this PR:\n"
        "- [ ] Dependencies have been updated if changed{upstream_url_link}\n"
        "- [ ] Tests have passed \n"
        "- [ ] Updated license if changed and `license_file` is packaged \n"
        "\n"
        "Note that the bot will stop issuing PRs if more than {max_num_prs} "
        "Version bump PRs "
        "generated by the bot are open. If you don't want to package a particular "
        "version please close the PR.\n\n"
        "{closes}".format(
            upstream_url_link=upstream_url_link,
            max_num_prs=self.max_num_prs,
            closes="\n".join(
                [f"Closes: #{muid['number']}" for muid in open_version_prs],
            ),
        )
    )
    # Statement here
    curr_pkg_name = feedstock_ctx.package_name
    pred.insert(0, curr_pkg_name)
    body += (
        "\n\nHere is a list of all the pending dependencies (and their "
        "versions) for this repo. The first package is the current feedstock package. "
        "Please double check all dependencies before merging.\n\n"
    )
    body += create_dashboard_from_list(pred)
    return body

def run(feedstock_ctx, migrator, protocol='ssh', pull_request=True,
        rerender=True, fork=False, organization='nsls-ii-forge', **kwargs):
    """
    For a given feedstock and migration run the migration and possibly submit
    pull request

    Parameters
    ----------
    feedstock_ctx: FeedstockContext
        The node attributes of the feedstock
    migrator: Migrator
        The migrator to run on the feedstock
    protocol: str, optional
        The git protocol to use, defaults to ``ssh``
    pull_request: bool, optional
        If true issue pull request, defaults to true
    rerender: bool
        Whether to rerender, defaults to true
    fork: bool
        If true create a fork, defaults to false
    organization: str, optional
        GitHub organization to get repo from
    gh: github3.GitHub, optional
        Object for communicating with GitHub, if None, build from $GITHUB_USERNAME
        and $GITHUB_PASSWORD, defaults to None
    kwargs: dict
        The key word arguments to pass to the migrator

    Returns
    -------
    migrate_return: MigrationUidTypedDict
        The migration return dict used for tracking finished migrations
    pr_json: dict
        The PR json object for recreating the PR as needed
    """
    # get the repo
    migrator.attrs = feedstock_ctx.attrs

    branch_name = migrator.remote_branch(feedstock_ctx) + "_h" + uuid4().hex[:6]

    # TODO: run this in parallel
    feedstock_dir, repo = get_repo(
        ctx=migrator.ctx.session,
        fctx=feedstock_ctx,
        branch=branch_name,
        organization=organization,
        feedstock=feedstock_ctx.feedstock_name,
        protocol=protocol,
        pull_request=pull_request,
        fork=fork,

    )

    recipe_dir = os.path.join(feedstock_dir, "recipe")

    # migrate the feedstock
    migrator.run_pre_piggyback_migrations(recipe_dir, feedstock_ctx.attrs, **kwargs)

    # TODO - make a commit here if the repo changed

    migrate_return = migrator.migrate(recipe_dir, feedstock_ctx.attrs, **kwargs)

    if not migrate_return:
        logger.critical(
            "Failed to migrate %s, %s",
            feedstock_ctx.package_name,
            feedstock_ctx.attrs.get("bad"),
        )
        eval_cmd(f"rm -rf {feedstock_dir}")
        return False, False

    # TODO - commit main migration here

    migrator.run_post_piggyback_migrations(recipe_dir, feedstock_ctx.attrs, **kwargs)

    # TODO commit post migration here

    # rerender, maybe
    diffed_files = []
    with indir(feedstock_dir), env.swap(RAISE_SUBPROC_ERROR=False):
        msg = migrator.commit_message(feedstock_ctx)  # noqa
        try:
            eval_cmd("git add --all .")
            eval_cmd(f"git commit -am '{msg}'")
        except CalledProcessError as e:
            logger.info(
                "could not commit to feedstock - "
                "likely no changes - error is '%s'" % (repr(e)),
            )
        if rerender:
            head_ref = eval_cmd("git rev-parse HEAD").strip()
            logger.info("Rerendering the feedstock")

            # In the event we can't rerender, try to update the pinnings,
            # then bail if it does not work again
            try:
                eval_cmd(
                    "conda smithy rerender -c auto --no-check-uptodate", timeout=300,
                )
            except SubprocessError:
                return False, False

            # If we tried to run the MigrationYaml and rerender did nothing (we only
            # bumped the build number and dropped a yaml file in migrations) bail
            # for instance platform specific migrations
            gdiff = eval_cmd(f"git diff --name-only {head_ref.strip()}...HEAD")

            diffed_files = [
                _
                for _ in gdiff.split()
                if not (
                    _.startswith("recipe")
                    or _.startswith("migrators")
                    or _.startswith("README")
                )
            ]

    if (
        (
            migrator.check_solvable
            and feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get("automerge")
        )
        or feedstock_ctx.attrs["conda-forge.yml"]
        .get("bot", {})
        .get("check_solvable", False)
    ) and not is_recipe_solvable(feedstock_dir):
        eval_cmd(f"rm -rf {feedstock_dir}")
        return False, False

    if (
        isinstance(migrator, MigrationYaml)
        and not diffed_files
        and feedstock_ctx.attrs["name"] != "conda-forge-pinning"
    ):
        # spoof this so it looks like the package is done
        pr_json = {
            "state": "closed",
            "merged_at": "never issued",
            "id": str(uuid4()),
        }
    else:
        # push up
        try:
            if fork:
                head = f"{migrator.ctx.github_username}:{branch_name}"
            else:
                head = f"{organization}:{branch_name}"
            pr_json = push_repo(
                session_ctx=migrator.ctx.session,
                fctx=feedstock_ctx,
                feedstock_dir=feedstock_dir,
                body=migrator.pr_body(feedstock_ctx),
                repo=repo,
                title=migrator.pr_title(feedstock_ctx),
                head=head,
                branch=branch_name,
                fork=fork,
                organization=organization
            )

        # This shouldn't happen too often any more since we won't double PR
        except github3.GitHubError as e:
            if e.msg != "Validation Failed":
                raise
            else:
                print(f"Error during push {e}")
                print(f'Errors: {e.errors}')
                # If we just push to the existing PR then do nothing to the json
                pr_json = None
                ljpr = None
    if pr_json is not None:
        ljpr = LazyJson(
            os.path.join(migrator.ctx.session.prjson_dir, str(pr_json["id"]) + ".json"),
        )
        ljpr.update(**pr_json)
    else:
        ljpr = None
    # If we've gotten this far then the node is good
    feedstock_ctx.attrs["bad"] = False
    logger.info("Removing feedstock dir")
    eval_cmd(f"rm -rf {feedstock_dir}")
    return migrate_return, ljpr

def initialize_migrators(github_username="", github_password="", github_token=None,
                         dry_run=False):
    '''
    Setup graph, required contexts, and migrators

    Parameters
    ----------
    github_username: str, optional
        Username for bot on GitHub
    github_password: str, optional
        Password for bot on GitHub
    github_token: str, optional
        Token for bot on GitHub
    dry_run: bool, optional
        If true, does not submit pull requests on GitHub

    Returns
    -------
    tuple
        Migrator session to interact with GitHub and list of migrators.
        Currently only returns pre-defined migrators.
    '''
    gx = load_graph()
    smithy_version = eval_cmd("conda smithy --version").strip()
    pinning_version = json.loads(eval_cmd("conda list conda-forge-pinning --json"))[0][
        "version"
    ]
    for m in MIGRATORS:
        print(f'{getattr(m, "name", m)} graph size: {len(getattr(m, "graph", []))}')

    ctx = MigratorSessionContext(
        circle_build_url=os.getenv("CIRCLE_BUILD_URL", ""),
        graph=gx,
        smithy_version=smithy_version,
        pinning_version=pinning_version,
        github_username=github_username,
        github_password=github_password,
        github_token=github_token,
        dry_run=dry_run,
    )

    return ctx, MIGRATORS

def auto_tick(dry_run=False, debug=False, fork=False, organization='nsls-ii-forge'):
    '''
    Automatically update package versions and submit pull requests to
    associated feedstocks

    Parameters
    ----------
    dry_run: bool, optional
        Generate version migration yamls but do not run them
    debug: bool, optional
        Setup logging to be in debug mode
    fork: bool, optional
        Create a fork of the repo from the organization to $GITHUB_USERNAME
    organization: str, optional
        GitHub organization that manages feedstock repositories
    '''
    from conda_forge_tick.xonsh_utils import env

    if debug:
        setup_logger(logger, level="debug")
    else:
        setup_logger(logger)

    # set Version.pr_body to custom pr_body function
    Version.pr_body = bot_pr_body

    # TODO: use ~/.netrc instead
    github_username = env.get("GITHUB_USERNAME", "")
    github_password = env.get("GITHUB_TOKEN", "")
    github_token = env.get("GITHUB_TOKEN")
    global MIGRATORS

    print('Initializing migrators...')
    mctx, MIGRATORS = initialize_migrators(
        github_username=github_username,
        github_password=github_password,
        dry_run=dry_run,
        github_token=github_token,
    )

    # compute the time per migrator
    print('Computing time per migrator')
    (num_nodes, time_per_migrator, tot_time_per_migrator) = _compute_time_per_migrator(
        mctx,
    )
    for i, migrator in enumerate(MIGRATORS):
        if hasattr(migrator, "name"):
            extra_name = "-%s" % migrator.name
        else:
            extra_name = ""

        logger.info(
            "Total migrations for %s%s: %d - gets %f seconds (%f percent)",
            migrator.__class__.__name__,
            extra_name,
            num_nodes[i],
            time_per_migrator[i],
            time_per_migrator[i] / tot_time_per_migrator * 100,
        )

    print('Performing migrations...')
    for mg_ind, migrator in enumerate(MIGRATORS):

        mmctx = MigratorContext(session=mctx, migrator=migrator)
        migrator.bind_to_ctx(mmctx)

        good_prs = 0
        _mg_start = time.time()
        effective_graph = mmctx.effective_graph
        time_per = time_per_migrator[mg_ind]

        if hasattr(migrator, "name"):
            extra_name = "-%s" % migrator.name
        else:
            extra_name = ""

        logger.info(
            "Running migrations for %s%s: %d",
            migrator.__class__.__name__,
            extra_name,
            len(effective_graph.nodes),
        )

        possible_nodes = list(migrator.order(effective_graph, mctx.graph))

        # version debugging info
        if isinstance(migrator, Version):
            logger.info("possible version migrations:")
            for node_name in possible_nodes:
                with effective_graph.nodes[node_name]["payload"] as attrs:
                    logger.info(
                        "    node|curr|new|attempts: %s|%s|%s|%d",
                        node_name,
                        attrs.get("version"),
                        attrs.get("new_version"),
                        (
                            attrs.get("new_version_attempts", {}).get(
                                attrs.get("new_version", ""), 0,
                            )
                        ),
                    )

        for node_name in possible_nodes:
            with mctx.graph.nodes[node_name]["payload"] as attrs:
                # Don't let CI timeout, break ahead of the timeout so we make certain
                # to write to the repo
                # TODO: convert these env vars
                _now = time.time()
                if (
                    (
                        _now - int(env.get("START_TIME", time.time()))
                        > int(env.get("TIMEOUT", 600))
                    )
                    or good_prs >= migrator.pr_limit
                    or (_now - _mg_start) > time_per
                ):
                    break

                fctx = FeedstockContext(
                    package_name=node_name,
                    feedstock_name=attrs["feedstock_name"],
                    attrs=attrs,
                )

                print("\n", flush=True, end="")
                logger.info(
                    "%s%s IS MIGRATING %s",
                    migrator.__class__.__name__.upper(),
                    extra_name,
                    fctx.package_name,
                )
                try:
                    # Don't bother running if we are at zero
                    if (
                        dry_run
                        or mctx.gh.rate_limit()["resources"]["core"]["remaining"] == 0
                    ):
                        break
                    migrator_uid, pr_json = run(
                        feedstock_ctx=fctx,
                        migrator=migrator,
                        rerender=migrator.rerender,
                        protocol="https",
                        hash_type=attrs.get("hash_type", "sha256"),
                        fork=fork,
                        organization=organization
                    )
                    # if migration successful
                    if migrator_uid:
                        d = frozen_to_json_friendly(migrator_uid)
                        # if we have the PR already do nothing
                        if d["data"] in [
                            existing_pr["data"] for existing_pr in attrs.get("PRed", [])
                        ]:
                            pass
                        else:
                            if pr_json is None:
                                pr_json = {
                                    "state": "closed",
                                    "head": {"ref": "<this_is_not_a_branch>"},
                                }
                            d["PR"] = pr_json
                            attrs.setdefault("PRed", []).append(d)
                        attrs.update(
                            {
                                "smithy_version": mctx.smithy_version,
                                "pinning_version": mctx.pinning_version,
                            },
                        )

                except github3.GitHubError as e:
                    if e.msg == "Repository was archived so is read-only.":
                        attrs["archived"] = True
                    else:
                        logger.critical(
                            "GITHUB ERROR ON FEEDSTOCK: %s", fctx.feedstock_name,
                        )
                        if is_github_api_limit_reached(e, mctx.gh):
                            break
                except URLError as e:
                    logger.exception("URLError ERROR")
                    attrs["bad"] = {
                        "exception": str(e),
                        "traceback": str(traceback.format_exc()).split("\n"),
                        "code": getattr(e, "code"),
                        "url": getattr(e, "url"),
                    }
                except Exception as e:
                    logger.exception("NON GITHUB ERROR")
                    attrs["bad"] = {
                        "exception": str(e),
                        "traceback": str(traceback.format_exc()).split("\n"),
                    }
                else:
                    if migrator_uid:
                        # On successful PR add to our counter
                        good_prs += 1
                finally:
                    # Write graph partially through
                    if not dry_run:
                        dump_graph(mctx.graph)

                    eval_cmd(f"rm -rf {mctx.rever_dir}/*")
                    logger.info(os.getcwd())

    if not dry_run:
        logger.info(
            "API Calls Remaining: %d",
            mctx.gh.rate_limit()["resources"]["core"]["remaining"],
        )
    logger.info("Done")

def status_report():
    '''
    Write out the status of current/recent migrations and their
    pull requests on GitHub.

    Only works for Version migrations at the moment.
    '''
    print('Determining current status of migrations...')
    mctx, *_, migrators = initialize_migrators()
    if not os.path.exists("./status"):
        os.mkdir("./status")

    for migrator in migrators:
        if isinstance(migrator, Version):
            write_version_migrator_status(migrator, mctx)

    lst = [
        k
        for k, v in mctx.graph.nodes.items()
        if len(
            [
                z
                for z in v.get("payload", {}).get("PRed", [])
                if z.get("PR", {}).get("state", "closed") == "open"
                and z.get("data", {}).get("migrator_name", "") == "Version"
            ],
        )
        >= Version.max_num_prs
    ]
    with open("./status/could_use_help.json", "w") as f:
        json.dump(
            sorted(
                lst,
                key=lambda z: (len(nx.descendants(mctx.graph, z)), lst),
                reverse=True,
            ),
            f,
            indent=2,
        )
    print('Statuses have been placed in ./status')

def clean(include=None, exclude=None, yes=False):
    '''
    Cleans the current directory for a fresh run of the
    graph building and auto-tick process.

    Parameters
    ----------
    include: list, optional
        Files to be removed (default is all files required
        by the bot except names.txt)
    exclude: list, optional
        Files to keep if already included either by default
        or by user
    yes: bool, optional
        Automatically proceed with deletion if True
    '''
    to_be_removed = [
        './dask-worker-space/*',
        './node_attrs/*',
        './feedstocks/*',
        './pr_json/*',
        './versions/*',
        './status/*',
        'graph.json'
    ]
    to_be_removed = set(to_be_removed)
    if include is not None:
        include = set(include)
        to_be_removed = to_be_removed.union(include)
    if exclude is not None:
        exclude = set(exclude)
        to_be_removed = to_be_removed.difference(exclude)
    to_be_removed = list(to_be_removed)
    rm_string = '\n'.join(to_be_removed)
    warn_msg = (
        'WARNING: This will delete all files associated with creating the graph.\n'
        'This includes:\n\n'
        f'{rm_string}\n\n'
        'If you would like to exclude some file that is included, please run:\n'
        'auto-tick clean --exclude FILE1 FILE2\n\n'
        'If you would like to include some file that is excluded, please run:\n'
        'auto-tick clean --include FILE1 FILE2\n\n'
        'NOTE: If the same file exists in both include and exclude it will NOT be removed.'
    )
    print(warn_msg)
    if not yes:
        while not yes:
            yes_str = input('Do you want to continue? (y/n)\n')
            yes_str = yes_str.lower()[0]
            if yes_str == 'n':
                print('Aborting...')
                return
            elif yes_str == 'y':
                yes = True
    print('Removing files...')
    for path in to_be_removed:
        files = glob.glob(path, recursive=True)
        for file in files:
            try:
                os.remove(file)
            except IsADirectoryError:
                shutil.rmtree(file)
    print('Included files have been removed.')

def _run_handle_args(args):
    auto_tick(dry_run=args.dry_run, debug=args.debug, fork=args.fork,
              organization=args.organization)

def _status_handle_args(args):
    status_report()

def _clean_handle_args(args):
    clean(include=args.include, exclude=args.exclude, yes=args.yes)

if __name__ == '__main__':
    auto_tick(dry_run=True, debug=False)

reason: import networkx


text:
import os
import torch
import numpy as np
import argparse

from itertools import product
from models import LCALSTM as Agent
from task import SequenceLearning
from exp_ms import run_ms
from utils.params import P
from utils.io import build_log_path, load_ckpt, pickle_save_dict, \
    get_test_data_dir, get_test_data_fname, load_env_metadata, pickle_load_dict

#plt.switch_backend('agg')
#sns.set(style='white', palette='colorblind', context='talk')

''' remove parser capability for now
parser = argparse.ArgumentParser()
parser.add_argument('--exp_name', default='test', type=str)
parser.add_argument('--subj_id', default=99, type=int)
parser.add_argument('--n_param', default=16, type=int)
parser.add_argument('--n_branch', default=4, type=int)
parser.add_argument('--pad_len', default=-1, type=int)
parser.add_argument('--def_prob', default=None, type=float)
parser.add_argument('--n_def_tps', default=0, type=int)
parser.add_argument('--enc_size', default=16, type=int)
parser.add_argument('--penalty', default=4, type=int)
parser.add_argument('--penalty_random', default=1, type=int)
parser.add_argument('--penalty_discrete', default=1, type=int)
parser.add_argument('--penalty_onehot', default=0, type=int)
parser.add_argument('--normalize_return', default=1, type=int)
parser.add_argument('--attach_cond', default=0, type=int)
parser.add_argument('--p_rm_ob_enc', default=0.3, type=float)
parser.add_argument('--p_rm_ob_rcl', default=0, type=float)
parser.add_argument('--similarity_max', default=.9, type=float)
parser.add_argument('--similarity_min', default=0, type=float)
parser.add_argument('--n_hidden', default=194, type=int)
parser.add_argument('--n_hidden_dec', default=128, type=int)
parser.add_argument('--lr', default=7e-4, type=float)
parser.add_argument('--eta', default=0.1, type=float)
parser.add_argument('--cmpt', default=0.8, type=float)
parser.add_argument('--n_event_remember', default=2, type=int)
parser.add_argument('--sup_epoch', default=1, type=int)
parser.add_argument('--n_epoch', default=2, type=int)
parser.add_argument('--n_examples', default=256, type=int)
parser.add_argument('--log_root', default='../log/', type=str)
args = parser.parse_args()
print(args)

# process args
exp_name = args.exp_name
subj_id = args.subj_id
n_param = args.n_param
n_branch = args.n_branch
pad_len = args.pad_len
def_prob = args.def_prob
n_def_tps = args.n_def_tps
enc_size = args.enc_size
penalty = args.penalty
penalty_random = args.penalty_random
penalty_discrete = args.penalty_discrete
penalty_onehot = args.penalty_onehot
normalize_return = args.normalize_return
attach_cond = args.attach_cond
p_rm_ob_enc = args.p_rm_ob_enc
p_rm_ob_rcl = args.p_rm_ob_rcl
similarity_max = args.similarity_max
similarity_min = args.similarity_min
n_hidden = args.n_hidden
n_hidden_dec = args.n_hidden_dec
learning_rate = args.lr
cmpt = args.cmpt
eta = args.eta
n_event_remember = args.n_event_remember
n_examples = args.n_examples
n_epoch = args.n_epoch
supervised_epoch = args.sup_epoch
log_root = args.log_root
'''

log_root = '/Users/carsonwardell/Desktop/Thesis/log/'

exp_name = 'vary-test-penalty(trained)'
def_prob = None
n_def_tps = 0

seed = 0
supervised_epoch = 600
epoch_load = 1000

n_branch = 4
n_param = 16
enc_size = 16
# enc_size_test = 8
enc_size_test = enc_size
n_event_remember = 2

penalty_random = 1
# testing param, ortho to the training directory
attach_cond = 0
# loading params
pad_len_load = -1
p_rm_ob_enc_load = .3
p_rm_ob_rcl_load = .3

# testing params
pad_len_test = 0
p_test = 0
p_rm_ob_enc_test = p_test
p_rm_ob_rcl_test = p_test
n_examples_test = 256

similarity_max_test = .9
similarity_min_test = 0

'''loop over conditions for testing'''

slience_recall_times = [range(n_param), None]

subj_ids = np.arange(1)

penaltys_train = [4]
penaltys_test = np.array([2])

all_conds = ['DM']
scramble = False

for slience_recall_time in slience_recall_times:
    for subj_id, penalty_train, fix_cond in product(subj_ids, penaltys_train, all_conds):
        print(
            f'\nsubj : {subj_id}, penalty : {penalty_train}, cond : {fix_cond}')
        print(f'slience_recall_time : {slience_recall_time}')

        penaltys_test_ = penaltys_test[penaltys_test <= penalty_train]
        for fix_penalty in penaltys_test_:
            print(f'penalty_test : {fix_penalty}')

            p = P(
                exp_name=exp_name, sup_epoch=supervised_epoch,
                n_param=n_param, n_branch=n_branch, pad_len=pad_len_load,
                def_prob=def_prob, n_def_tps=n_def_tps,
                enc_size=enc_size, n_event_remember=n_event_remember,
                penalty=penalty_train, penalty_random=penalty_random,
                attach_cond=attach_cond,
                p_rm_ob_enc=p_rm_ob_enc_load, p_rm_ob_rcl=p_rm_ob_rcl_load,
            )
            # create logging dirs
            log_path, log_subpath = build_log_path(
                subj_id, p, log_root=log_root, mkdir=False, verbose=False)

            # create fpath
            test_params = [fix_penalty, pad_len_test, slience_recall_time]
            test_data_dir, _ = get_test_data_dir(
                log_subpath, epoch_load, test_params)
            test_data_fname = get_test_data_fname(
                n_examples_test, fix_cond, scramble)

            if enc_size_test != enc_size:
                test_data_dir = os.path.join(
                    test_data_dir, f'enc_size_test-{enc_size_test}'
                )
                if not os.path.exists(test_data_dir):
                    os.makedirs(test_data_dir)
            fpath = os.path.join(test_data_dir, test_data_fname)
            print(fpath)
            print(test_data_fname)
            print(test_data_dir)
            print(log_subpath)
            print(fpath.split("sup_epoch-600",1)[1])

            # init env
            env_data = load_env_metadata(log_subpath)
            def_path = env_data['def_path']
            p.env.def_path = def_path
            p.update_enc_size(enc_size_test)

            task = SequenceLearning(
                n_param=p.env.n_param, n_branch=p.env.n_branch, pad_len=pad_len_test,
                p_rm_ob_enc=p_rm_ob_enc_test, p_rm_ob_rcl=p_rm_ob_rcl_test,
                similarity_max=similarity_max_test, similarity_min=similarity_min_test,
                similarity_cap_lag=p.n_event_remember,
            )
            x_dim = task.x_dim
            if attach_cond != 0:
                x_dim += 1
            # load the agent back
            agent = Agent(
                input_dim=x_dim, output_dim=p.a_dim,
                rnn_hidden_dim=p.net.n_hidden, dec_hidden_dim=p.net.n_hidden_dec,
                dict_len=p.net.dict_len
            )

            agent, optimizer = load_ckpt(
                epoch_load, log_subpath['ckpts'], agent)

            # if data dir does not exsits ... skip
            if agent is None:
                print('Agent DNE')
                continue

            # agent.parameters

            # freeze memory controlling layer
            for param in agent.parameters():
                param.requires_grad = False
            agent.hpc.requires_grad_ = True

            # training objective
            np.random.seed(seed)
            torch.manual_seed(seed)
            [results, metrics, XY] = run_ms(
                agent, optimizer,
                task, p, n_examples_test, fpath,
                fix_penalty=fix_penalty, slience_recall_time=slience_recall_time,
                learning=False, get_data=True,
            )

            # save the data
            test_params = [fix_penalty, pad_len_test, slience_recall_time]
            test_data_dir, _ = get_test_data_dir(
                log_subpath, epoch_load, test_params)
            test_data_fname = get_test_data_fname(
                n_examples_test, fix_cond, scramble)
            test_data_dict = {
                'results': results, 'metrics': metrics, 'XY': XY
            }
            if enc_size_test != enc_size:
                test_data_dir = os.path.join(
                    test_data_dir, f'enc_size_test-{enc_size_test}'
                )
                if not os.path.exists(test_data_dir):
                    os.makedirs(test_data_dir)
            fpath = os.path.join(test_data_dir, test_data_fname)
            pickle_save_dict(test_data_dict, fpath)

#playing with data:
print(fpath)
import pickle
data = pickle.load(open('/Users/carsonwardell/Desktop/Thesis/n256.pkl', "rb"))

sims_data = data.pop('sims_data')

sims_data_1 = sims_data[0]
sims_data_2 = sims_data[1]
sims_data_1[0]
np.shape(sims_data_2)
plt.plot(sims_data_2)

A= 3
T= 16
t= 14
a_t = 1

from utils.utils import to_sqnp

time_oh_vec = np.identity(T)[t%T]
time_q_vec = np.identity(T)[t%T]
# special case if dk
if a_t == A:
    obs_val_oh_vec =  np.zeros(A)
else:
    obs_val_oh_vec = np.identity(A)[a_t]

retrn1 = np.concatenate([time_oh_vec,obs_val_oh_vec, time_oh_vec])
retrn1.shape
np.shape(to_sqnp(retrn1))

boop = np.reshape(retrn1, (np.shape(retrn1)[0],1))
boop = np.reshape(obs_val_oh_vec, (np.shape(obs_val_oh_vec)[0],1))
boop = np.reshape(time_oh_vec, (np.shape(time_oh_vec)[0],1))
plt.imshow(boop.T)
boop.shape
print(np.shape(time_oh_vec))
print(np.shape(obs_val_oh_vec))

np.shape(np.concatenate([time_oh_vec,obs_val_oh_vec]))

t=2
boop = []
for i in range(t,5):
    boop.append(0)

boop

np.shape(retrn1)
trans_retrn = np.atleast_2d(retrn).T.shape
np.shape(trans_retrn)
log_rets = np.empty(trans_retrn.shape)
np.atleast_2d(retrn).T.shape
log_rets.shape
log_rets = np.vstack([log_rets, retrn])
log_rets = np.vstack([log_rets, retrn1])
log_rets.shape[1]
plt.imshow(log_rets)

training_data = pickle_load_dict(fpath).pop('XY')
X_train = np.array(training_data[0])
Y_train = np.array(training_data[1])

X = np.array(training_data[0])
Y = np.array(training_data[1])

X.shape
Y.shape

import random as rd

i = rd.randint(0,(n_examples_test-1))
print("i:",i)
X_i = X_train[i,:,:]
Y_i = Y_train[i,:,:]

Y_i.shape#[1]
X_i.shape#[0]
import matplotlib.pyplot as plt
plt.imshow(X_i)
plt.imshow(Y_i)

Y_i[13].shape[0]

torch.from_numpy(Y_i)
torch.argmax(torch.from_numpy(Y_i))

j = rd.randint(0,(X_i.shape[0]-1))
X_i_t0 = X_i[j,:]

probs, rewards, values, ents = [], [], [], []
a = torch.empty(1)
a.add_(1)
print(a)

rewards.add(a)

print(rewards)

deep = np.random.choice(15,2, replace=False)

k = 2
n_examples = 256

data_sample_ind=np.random.choice(n_examples,k,replace = False)
X_dict = {}
Y_dict = {}
i=0
for k in data_sample_ind:
    X_dict["X_{0}".format(i)] = "Hello{0}".format(k)
    Y_dict["Y_{0}".format(i)] = "Hello{0}".format(k)
    i+=1
    print(i)
X_dict

X_dict["X_{0}".format(0)]

k=2
for i in range(k):
    print(i)

seed_num = 2
range(1,seed_num)
j = np.random.choice(seed_num)
print(j)

for t in range(16):
    if t<(seed_num):
        print(t)

for sn in range(seed_num):
    print(sn)

def io_convert(a_t, t, T, A):
    '''converts model output to one-hot vector input format
    '''
    time_oh_vec = np.identity(T)[t%T]
    time_q_vec = np.identity(T)[t%T]
    # special case if dk
    if a_t == A:
        obs_val_oh_vec =  np.zeros(A)
    else:
        obs_val_oh_vec = np.identity(A)[a_t]
    return np.concatenate([time_oh_vec,obs_val_oh_vec, time_oh_vec])

A = np.array([0., 0., 0., 1.]).shape[0]
print(A)
T = 16
t = 1
a_t = 3

io_convert(a_t, t, T, A)
# play with results
dpath = '/Users/carsonwardell/Desktop/Thesis/log/subj-1(working-2:22)'

mod_data = pickle_load_dict(dpath)

seed_num = 2
X_dict = [0,1]
print(len(X_dict))
j = np.random.choice(len(X_dict))
print("j", j)

T_part = 16
for t in range(T_part):
    print(t

deep = np.zeros(40)

deep_short = deep[:20,]

np.shape(deep_short)

# compare two X_array_list

outp = np.array([1,1,0,0])
fakemem = np.array([1,0,0,0])
fakemem2 = np.array([1,0,0,0])

if not np.all(outp == fakemem) and not np.all(outp == fakemem2):
    print("other")

mem_1_comp

print(np.sum([[3],[1],[0]]))

rewards = [torch.tensor(-5.)]
R = 0
gamma=0
returns = []
for r in rewards[::-1]:
    R = r + gamma * R
    returns.insert(0, R)
returns = torch.tensor(returns)
# normalize w.r.t to the statistics of this trajectory

print(returns)

deep = np.zeros((5,16))
print(np.shape(np.amax(deep, axis = 1)))

mem_num = 2
for mn in range(mem_num):
    print(mn)

fake_list = []

for i in range(10):
    fake_list.append(i)

fake_list


text:
import numpy as np
import matplotlib.pyplot as plt

def fac(N):
    fact = 1
    for i in range(1, N+1):
        fact *= i
    return fact

def binomial(n, k):
    try:
        int(n)
        int(k)
        if k == 0:
            return 1
        elif k < 0:
            return print('ERRO: k deve ser > 0')
        else:
            bin = fac(n)/(fac(k)*fac(n-k))
            return bin
    except TypeError:
        return print('Os valores devem ser inteiros')

y = np.array([binomial(100, k) for k in range(20, 80)])
x = np.arange(60)
plt.plot(x, y, 'k')
plt.fill_between(x, y)
plt.show()

max_stars_repo_name: Artur-UF/MetCompA

max_issues_repo_name: Artur-UF/MetCompA

max_forks_repo_name: Artur-UF/MetCompA


text:
import os
from datetime import datetime, timedelta
from typing import Union
from urllib.parse import urlparse, urlunparse

import gcsfs
import numpy as np
import pandas as pd
from google.protobuf.duration_pb2 import Duration
from pandas._testing import assert_frame_equal
from pyarrow import parquet

from feast import Client, Entity, Feature, FeatureTable, ValueType
from feast.constants import ConfigOptions as opt
from feast.data_source import BigQuerySource, FileSource
from feast.pyspark.abc import SparkJobStatus

np.random.seed(0)

def read_parquet(uri, azure_account_name=None, azure_account_key=None):
    parsed_uri = urlparse(uri)
    if parsed_uri.scheme == "file":
        return pd.read_parquet(parsed_uri.path)
    elif parsed_uri.scheme == "gs":
        fs = gcsfs.GCSFileSystem()
        files = ["gs://" + path for path in fs.glob(uri + "/part-*")]
        ds = parquet.ParquetDataset(files, filesystem=fs)
        return ds.read().to_pandas()
    elif parsed_uri.scheme == "s3" or parsed_uri.scheme == "s3a":

        s3uri = urlunparse(parsed_uri._replace(scheme="s3"))

        import s3fs

        # AWS_S3_ENDPOINT_URL needs to be set when using minio
        if "AWS_S3_ENDPOINT_URL" in os.environ:
            fs = s3fs.S3FileSystem(
                client_kwargs={"endpoint_url": os.getenv("AWS_S3_ENDPOINT_URL")}
            )
        else:
            fs = s3fs.S3FileSystem()
        files = ["s3://" + path for path in fs.glob(s3uri + "/part-*")]
        ds = parquet.ParquetDataset(files, filesystem=fs)
        return ds.read().to_pandas()
    elif parsed_uri.scheme == "wasbs":
        import adlfs

        fs = adlfs.AzureBlobFileSystem(
            account_name=azure_account_name, account_key=azure_account_key
        )
        uripath = parsed_uri.username + parsed_uri.path
        files = fs.glob(uripath + "/part-*")
        ds = parquet.ParquetDataset(files, filesystem=fs)
        return ds.read().to_pandas()
    else:
        raise ValueError(f"Unsupported URL scheme {uri}")

def generate_data():
    retrieval_date = datetime.utcnow().replace(tzinfo=None)
    retrieval_outside_max_age_date = retrieval_date + timedelta(1)
    event_date = retrieval_date - timedelta(2)
    creation_date = retrieval_date - timedelta(1)

    customers = [1001, 1002, 1003, 1004, 1005]
    daily_transactions = [np.random.rand() * 10 for _ in customers]
    total_transactions = [np.random.rand() * 100 for _ in customers]

    transactions_df = pd.DataFrame(
        {
            "event_timestamp": [event_date for _ in customers],
            "created_timestamp": [creation_date for _ in customers],
            "user_id": customers,
            "daily_transactions": daily_transactions,
            "total_transactions": total_transactions,
        }
    )
    customer_df = pd.DataFrame(
        {
            "event_timestamp": [retrieval_date for _ in customers]
            + [retrieval_outside_max_age_date for _ in customers],
            "user_id": customers + customers,
        }
    )
    return transactions_df, customer_df

def _get_azure_creds(feast_client: Client):
    return (
        feast_client._config.get(opt.AZURE_BLOB_ACCOUNT_NAME, None),
        feast_client._config.get(opt.AZURE_BLOB_ACCOUNT_ACCESS_KEY, None),
    )

def test_historical_features(
    feast_client: Client,
    tfrecord_feast_client: Client,
    batch_source: Union[BigQuerySource, FileSource],
):
    customer_entity = Entity(
        name="user_id", description="Customer", value_type=ValueType.INT64
    )
    feast_client.apply(customer_entity)

    max_age = Duration()
    max_age.FromSeconds(2 * 86400)

    transactions_feature_table = FeatureTable(
        name="transactions",
        entities=["user_id"],
        features=[
            Feature("daily_transactions", ValueType.DOUBLE),
            Feature("total_transactions", ValueType.DOUBLE),
        ],
        batch_source=batch_source,
        max_age=max_age,
    )

    feast_client.apply(transactions_feature_table)

    transactions_df, customers_df = generate_data()
    feast_client.ingest(transactions_feature_table, transactions_df)

    feature_refs = ["transactions:daily_transactions"]

    job_submission_time = datetime.utcnow()
    job = feast_client.get_historical_features(feature_refs, customers_df)
    assert job.get_start_time() >= job_submission_time
    assert job.get_start_time() <= job_submission_time + timedelta(hours=1)

    output_dir = job.get_output_file_uri()

    # will both be None if not using Azure blob storage
    account_name, account_key = _get_azure_creds(feast_client)

    joined_df = read_parquet(
        output_dir, azure_account_name=account_name, azure_account_key=account_key
    )

    expected_joined_df = pd.DataFrame(
        {
            "event_timestamp": customers_df.event_timestamp.tolist(),
            "user_id": customers_df.user_id.tolist(),
            "transactions__daily_transactions": transactions_df.daily_transactions.tolist()
            + [None] * transactions_df.shape[0],
        }
    )

    assert_frame_equal(
        joined_df.sort_values(by=["user_id", "event_timestamp"]).reset_index(drop=True),
        expected_joined_df.sort_values(by=["user_id", "event_timestamp"]).reset_index(
            drop=True
        ),
    )

    job = tfrecord_feast_client.get_historical_features(feature_refs, customers_df)
    job.get_output_file_uri()
    assert job.get_status() == SparkJobStatus.COMPLETED

max_stars_repo_name: chengcheng-pei/feast

max_issues_repo_name: frankgmail/feast

max_forks_repo_name: frankgmail/feast


text:
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.

from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any, cast, Dict, List, Optional, Set, Union
import numpy as np
import numpy.typing as npt
import pandas as pd
from sklearn.base import TransformerMixin
from sklearn.feature_extraction import DictVectorizer
from sklearn.impute import SimpleImputer
from sklearn.pipeline import make_pipeline, make_union
from sklearn.preprocessing import (
    MinMaxScaler,
    PowerTransformer,
    StandardScaler,
)
from tsbench.config import (
    Config,
    EnsembleConfig,
    MODEL_REGISTRY,
    ModelConfig,
    TrainConfig,
)
from tsbench.config.model.models import SeasonalNaiveModelConfig
from tsbench.evaluations.tracking import ModelTracker

class ConfigTransformer(TransformerMixin):
    """
    The config transformer transforms a configuration (model + dataset) into a real-valued vector.
    """

    def __init__(
        self,
        add_model_features: bool = True,
        add_dataset_statistics: bool = True,
        add_seasonal_naive_performance: bool = False,
        add_catch22_features: bool = False,
        tracker: Optional[ModelTracker] = None,
    ):
        """
        Args:
            add_model_type: Whether a one-hot encoding of the model type as well as model
                hyperparameters should be added.
            add_dataset_statistics: Whether simple dataset statistics ought to be added.
            add_seasonal_naive_performance: Whether to add the nCRPS performance of Seasonal Naïve.
                Requires the cacher to be set.
            add_catch_22_features: Whether a dataset's catch22 features ought to be added.
            tracker: An optional tracker to obtain the performance of Seasonal Naïve.
        """
        assert any(
            [
                add_model_features,
                add_dataset_statistics,
                add_seasonal_naive_performance,
                add_catch22_features,
            ]
        ), "ConfigTransformer must be given at least some group of features."
        assert (
            not add_seasonal_naive_performance or tracker is not None
        ), "Tracker must be set if seasonal naive performance is used."

        self.encoders = []
        if add_model_features:
            self.encoders.append(ModelEncoder(transform_full_config=True))
        if add_dataset_statistics:
            self.encoders.append(DatasetStatisticsEncoder())
        if add_seasonal_naive_performance and tracker is not None:
            self.encoders.append(SeasonalNaivePerformanceEncoder(tracker))
        if add_catch22_features:
            self.encoders.append(DatasetCatch22Encoder())

        self.pipeline = make_union(*self.encoders)

    @property
    def feature_names_(self) -> List[str]:
        """
        Returns the feature names for the columns of transformed configurations.
        """
        return [
            f
            for e in self.encoders
            for f in e.feature_names_
            if isinstance(e, Encoder)
        ]

    def fit(self, X: List[Config[ModelConfig]]) -> ConfigTransformer:
        """
        Uses the provided configurations to fit the transformer pipeline.

        Args:
            X: The input configurations.
        """
        self.pipeline.fit(X)
        return self

    def transform(
        self, X: List[Config[ModelConfig]]
    ) -> npt.NDArray[np.float32]:
        """
        Transforms the given configurations according to the fitted transformer pipeline.

        Args:
            X: The input configurations.

        Returns:
            A NumPy array of shape [N, D]. N is the number of input configurations, D the dimension
                of the vectorized representation.
        """
        return cast(npt.NDArray[np.float32], self.pipeline.transform(X))

class EnsembleConfigTransformer(TransformerMixin):
    """
    The config transformer transforms a set of model configs into a set of NumPy arrays.
    """

    model_type_map: Dict[str, int]
    attribute_map: Dict[str, int]

    def __init__(self):
        self.scaler = StandardScaler()
        self.model_type_map = {}
        self.attribute_map = {}

    @property
    def feature_names_(self) -> List[str]:
        """
        Returns the feature names for the columns of transformed configurations.
        """
        return [f"model={m}" for m in self.model_type_map] + list(
            self.attribute_map.keys()
        )

    def fit(
        self, X: List[Config[EnsembleConfig]]
    ) -> EnsembleConfigTransformer:
        """
        Uses the provided configurations to fit the transformer pipeline.

        Args:
            X: The input configurations.
        """
        configs = [model for ensemble in X for model in ensemble.model]

        # Extract model types and attributes
        model_types = {c.name() for c in configs}
        self.model_type_map = {k: i for i, k in enumerate(sorted(model_types))}

        # pylint: disable=no-member
        shared_attributes = set(TrainConfig.__dataclass_fields__.keys())
        attributes = {
            k if k in shared_attributes else f"{c.name()}_{k}"
            for c in configs
            for k in c.__dataclass_fields__.keys()
        }
        self.attribute_map = {
            k: i + len(self.model_type_map)
            for i, k in enumerate(sorted(attributes))
        }

        # Transform the configs to fit the scaler for the attributes
        out = self._transform(configs)
        rhs = out[:, len(MODEL_REGISTRY) :]
        self.scaler.fit(rhs)
        return self

    def transform(
        self, X: List[Config[EnsembleConfig]]
    ) -> List[npt.NDArray[np.float32]]:
        """
        Transforms the given configurations according to the fitted transformer pipeline.

        Args:
            X: The input configurations.

        Returns:
            A NumPy array of shape [N, D] for every ensemble configuration where N is the number of
                ensemble members and D the dimensionality. N might differ for list members.
        """
        configs = [model for ensemble in X for model in ensemble.model]
        out = self._transform(configs)
        lhs = out[:, : len(MODEL_REGISTRY)]
        rhs = out[:, len(MODEL_REGISTRY) :]
        out = np.concatenate([lhs, self.scaler.transform(rhs)], axis=1)
        transformed = np.nan_to_num(out, nan=0)
        chunks = np.cumsum([0] + [len(ensemble.model) for ensemble in X])
        return [
            transformed[lower:upper]
            for lower, upper in zip(chunks, chunks[1:])
        ]

    def _transform(
        self, configs: List[ModelConfig]
    ) -> npt.NDArray[np.float32]:
        result = np.empty((len(configs), len(self.feature_names_)))
        result.fill(np.nan)

        # pylint: disable=no-member
        shared_attributes = set(TrainConfig.__dataclass_fields__.keys())
        for i, config in enumerate(configs):
            result[i, self.model_type_map[config.name()]] = 1
            for field in config.__dataclass_fields__:
                if field in shared_attributes:
                    result[i, self.attribute_map[field]] = getattr(
                        config, field
                    )
                else:
                    result[
                        i, self.attribute_map[f"{config.name()}_{field}"]
                    ] = getattr(config, field)

        return result

# -------------------------------------------------------------------------------------------------
# pylint: disable=missing-class-docstring,missing-function-docstring

class Encoder(ABC):
    @property
    @abstractmethod
    def feature_names_(self) -> List[str]:
        pass

class NanImputer:
    def fit(self, _X: List[Dict[str, Any]], _y: Any = None) -> NanImputer:
        return self

    def transform(
        self, X: List[Dict[str, Any]], _y: Any = None
    ) -> List[Dict[str, Any]]:
        df = pd.DataFrame(X)
        return df.to_dict("records")

class Selector:
    def __init__(
        self, use: Optional[Set[str]] = None, ignore: Optional[Set[str]] = None
    ):
        assert bool(use) != bool(
            ignore
        ), "One of `use` or `ignore` must be set."

        self.use = use or set()
        self.ignore = ignore or set()

    def fit(self, _X: List[Dict[str, Any]], _y: Any = None) -> Selector:
        return self

    def transform(
        self, X: List[Dict[str, Any]], _y: Any = None
    ) -> List[Dict[str, Any]]:
        return [
            {
                k: v
                for k, v in item.items()
                if (not self.use or k in self.use)
                and (not self.ignore or k not in self.ignore)
            }
            for item in X
        ]

# -------------------------------------------------------------------------------------------------

class ModelEncoder(Encoder):
    def __init__(self, transform_full_config: bool = False):
        self.transform_full_config = transform_full_config
        self.model_vectorizer = DictVectorizer(
            dtype=np.float32, sparse=False, sort=True
        )
        self.hp_vectorizer = DictVectorizer(
            dtype=np.float32, sparse=False, sort=True
        )
        self.pipeline = make_pipeline(
            NanImputer(),
            make_union(
                make_pipeline(
                    Selector(use={"model"}),
                    self.model_vectorizer,
                ),
                make_pipeline(
                    Selector(ignore={"model"}),
                    self.hp_vectorizer,
                    StandardScaler(),
                    SimpleImputer(strategy="constant", fill_value=0.0),
                ),
            ),
        )

    @property
    def feature_names_(self) -> List[str]:
        return (
            self.model_vectorizer.feature_names_
            + self.hp_vectorizer.feature_names_
        )

    def fit(
        self,
        X: Union[List[Config[ModelConfig]], List[ModelConfig]],
        _y: Any = None,
    ) -> ModelEncoder:
        if self.transform_full_config:
            self.pipeline.fit(
                [x.model.asdict() for x in cast(List[Config[ModelConfig]], X)]
            )
        else:
            self.pipeline.fit([x.asdict() for x in cast(List[ModelConfig], X)])
        return self

    def transform(
        self,
        X: List[Union[List[Config[ModelConfig]], List[ModelConfig]]],
        _y: Any = None,
    ) -> npt.NDArray[np.float32]:
        if self.transform_full_config:
            return cast(
                npt.NDArray[np.float32],
                self.pipeline.transform(
                    [
                        x.model.asdict()
                        for x in cast(List[Config[ModelConfig]], X)
                    ]
                ),
            )
        return cast(
            npt.NDArray[np.float32],
            self.pipeline.transform(
                [x.asdict() for x in cast(List[ModelConfig], X)]
            ),
        )

class DatasetStatisticsEncoder(Encoder):
    def __init__(self):
        self.unscaled_vectorizer = DictVectorizer(
            dtype=np.float32, sparse=False, sort=True
        )
        self.scaled_vectorizer = DictVectorizer(
            dtype=np.float32, sparse=False, sort=True
        )
        self.pipeline = make_union(
            make_pipeline(
                Selector(use={"integer_dataset", "frequency"}),
                self.unscaled_vectorizer,
            ),
            make_pipeline(
                Selector(ignore={"integer_dataset", "frequency"}),
                self.scaled_vectorizer,
                MinMaxScaler(),  # required for numerical stability
                PowerTransformer(),
            ),
        )

    @property
    def feature_names_(self) -> List[str]:
        return (
            self.unscaled_vectorizer.feature_names_
            + self.scaled_vectorizer.feature_names_
        )

    def fit(
        self, X: List[Config[ModelConfig]], _y: Any = None
    ) -> DatasetStatisticsEncoder:
        self.pipeline.fit(
            [
                {
                    **x.dataset.stats(),
                    "frequency": x.dataset.meta.freq,
                    "prediction_length": x.dataset.meta.prediction_length,
                }
                for x in X
            ]
        )
        return self

    def transform(
        self, X: List[Config[ModelConfig]], _y: Any = None
    ) -> npt.NDArray[np.float32]:
        return cast(
            npt.NDArray[np.float32],
            self.pipeline.transform([x.dataset.stats() for x in X]),
        )

class SeasonalNaivePerformanceEncoder(Encoder):
    def __init__(self, tracker: ModelTracker):
        self.tracker = tracker
        self.scaler = StandardScaler()

    @property
    def feature_names_(self) -> List[str]:
        return ["seasonal_naive_ncrps"]

    def fit(
        self, X: List[Config[ModelConfig]], _y: Any = None
    ) -> SeasonalNaivePerformanceEncoder:
        self.scaler.fit(self._get_performance_array(X))
        return self

    def transform(
        self, X: List[Config[ModelConfig]], _y: Any = None
    ) -> npt.NDArray[np.float32]:
        return cast(
            npt.NDArray[np.float32],
            self.scaler.transform(self._get_performance_array(X)),
        )

    def _get_performance_array(
        self, X: List[Config[ModelConfig]]
    ) -> npt.NDArray[np.float32]:
        return np.array(
            [
                self.tracker.get_performance(
                    Config(SeasonalNaiveModelConfig(), x.dataset)
                ).ncrps.mean
                for x in X
            ]
        )[:, None]

class DatasetCatch22Encoder(Encoder):
    def __init__(self):
        self.vectorizer = DictVectorizer(
            dtype=np.float32, sparse=False, sort=True
        )
        self.pipeline = make_pipeline(
            self.vectorizer,
            PowerTransformer(),
        )

    @property
    def feature_names_(self) -> List[str]:
        return self.vectorizer.feature_names_

    def fit(
        self, X: List[Config[ModelConfig]], _y: Any = None
    ) -> DatasetCatch22Encoder:
        datasets = {x.dataset for x in X}
        features = {d: d.catch22().mean().to_dict() for d in datasets}
        self.pipeline.fit([features[x.dataset] for x in X])
        return self

    def transform(
        self, X: List[Config[ModelConfig]], _y: Any = None
    ) -> npt.NDArray[np.float32]:
        datasets = {x.dataset for x in X}
        features = {d: d.catch22().mean().to_dict() for d in datasets}
        return cast(
            npt.NDArray[np.float32],
            self.pipeline.transform([features[x.dataset] for x in X]),
        )

max_stars_repo_name: RingoIngo/gluon-ts

max_issues_repo_name: RingoIngo/gluon-ts

max_forks_repo_name: RingoIngo/gluon-ts


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# File              : ampel/contrib/hu/t0/DecentFilter.py
# License           : BSD-3-Clause
# Author            : m. giomi <matteo.giomi@desy.de>
# Date              : 06.06.2018
# Last Modified Date: 27.06.2018
# Last Modified By  : m. giomi <matteo.giomi@desy.de>

from numpy import exp, array
import logging
from urllib.parse import urlparse
from astropy.coordinates import SkyCoord
from astropy.table import Table
import sys
try:
	from catsHTM import cone_search
	doCat = True
except ImportError:
	doCat = False
from ampel.base.abstract.AbsAlertFilter import AbsAlertFilter

class MPALensingFilter(AbsAlertFilter):
	"""
                Modification of the Decent filter to only accept candidates with PS match (potential lens).

		General-purpose filter with ~ 0.6% acceptance. It selects alerts based on:
			* numper of previous detections
			* positive subtraction flag
			* loose cuts on image quality (fwhm, elongation, number of bad pixels, and the
			difference between PSF and aperture magnitude)
			* distance to known SS objects
			* real-bogus
			* detection of proper-motion and paralax for coincidence sources in GAIA DR2

		The filter has a very weak dependence on the real-bogus score and it is independent
		on the provided PS1 star-galaxy classification.
	"""

	# Static version info
	version = 1.0
	resources = ('catsHTM.default',)

	def __init__(self, on_match_t2_units, base_config=None, run_config=None, logger=None):
		"""
		"""
		if run_config is None or len(run_config) == 0:
			raise ValueError("Please check you run configuration")

		self.on_match_t2_units = on_match_t2_units
		self.logger = logger if logger is not None else logging.getLogger()

		config_params = (
			'MIN_NDET',					# number of previous detections
			'MIN_TSPAN', 				# minimum duration of alert detection history [days]
			'MAX_TSPAN', 				# maximum duration of alert detection history [days]
			'MIN_RB',					# real bogus score
			'MAX_FWHM',					# sexctrator FWHM (assume Gaussian) [pix]
			'MAX_ELONG',				# Axis ratio of image: aimage / bimage
			'MAX_MAGDIFF',				# Difference: magap - magpsf [mag]
			'MAX_NBAD',					# number of bad pixels in a 5 x 5 pixel stamp
			'MIN_DIST_TO_SSO',			# distance to nearest solar system object [arcsec]
			'MIN_GAL_LAT', 				# minium distance from galactic plane. Set to negative to disable cut.
			'GAIA_RS',					# search radius for GAIA DR2 matching [arcsec]
			'GAIA_PM_SIGNIF',			# significance of proper motion detection of GAIA counterpart [sigma]
			'GAIA_PLX_SIGNIF',			# significance of parallax detection of GAIA counterpart [sigma]
			'GAIA_VETO_GMAG_MIN', 		        # min gmag for normalized distance cut of GAIA counterparts [mag]
			'GAIA_VETO_GMAG_MAX', 		        # max gmag for normalized distance cut of GAIA counterparts [mag]
			'PS1_SGVETO_RAD',			# maximum distance to closest PS1 source for SG score veto [arcsec]
			'PS1_SGVETO_SGTH',			# maximum allowed SG score for PS1 source within PS1_SGVETO_RAD
                        'PS1_MAXIMUM_DIST'                      # Maximum distance of closest PS1 match
			)
		for el in config_params:
			if el not in run_config:
				raise ValueError("Parameter %s missing, please check your channel config" % el)
			if run_config[el] is None:
				raise ValueError("Parameter %s is None, please check your channel config" % el)
			self.logger.info("Using %s=%s" % (el, run_config[el]))

		# ----- set filter proerties ----- #

		# history
		self.min_ndet 					= run_config['MIN_NDET']
		self.min_tspan					= run_config['MIN_TSPAN']
		self.max_tspan					= run_config['MAX_TSPAN']

		# Image quality
		self.max_fwhm					= run_config['MAX_FWHM']
		self.max_elong					= run_config['MAX_ELONG']
		self.max_magdiff				= run_config['MAX_MAGDIFF']
		self.max_nbad					= run_config['MAX_NBAD']
		self.min_rb						= run_config['MIN_RB']

		# astro
		self.min_ssdistnr	 			= run_config['MIN_DIST_TO_SSO']
		self.min_gal_lat				= run_config['MIN_GAL_LAT']
		self.gaia_rs					= run_config['GAIA_RS']
		self.gaia_pm_signif				= run_config['GAIA_PM_SIGNIF']
		self.gaia_plx_signif			= run_config['GAIA_PLX_SIGNIF']
		self.gaia_veto_gmag_min			= run_config['GAIA_VETO_GMAG_MIN']
		self.gaia_veto_gmag_max			= run_config['GAIA_VETO_GMAG_MAX']
		self.ps1_sgveto_rad				= run_config['PS1_SGVETO_RAD']
		self.ps1_sgveto_th				= run_config['PS1_SGVETO_SGTH']
		self.ps1_max_dist			= run_config['PS1_MAXIMUM_DIST']

		# technical
		if doCat:
			self.catshtm_path 			= urlparse(base_config['catsHTM.default']).path
			self.logger.info("using catsHTM files in %s"%self.catshtm_path)
		self.keys_to_check = (
			'fwhm', 'elong', 'magdiff', 'nbad', 'distpsnr1', 'sgscore1', 'distpsnr2',
			'sgscore2', 'distpsnr3', 'sgscore3', 'isdiffpos', 'ra', 'dec', 'rb', 'ssdistnr')

	def _alert_has_keys(self, photop):
		"""
			check that given photopoint contains all the keys needed to filter
		"""
		for el in self.keys_to_check:
			if el not in photop:
				self.logger.debug("rejected: '%s' missing" % el)
				return False
			if photop[el] is None:
				self.logger.debug("rejected: '%s' is None" % el)
				return False
		return True

	def get_galactic_latitude(self, transient):
		"""
			compute galactic latitude of the transient
		"""
		coordinates = SkyCoord(transient['ra'], transient['dec'], unit='deg')
		b = coordinates.galactic.b.deg
		return b

	def is_star_in_PS1(self, transient):
		"""
			apply combined cut on sgscore1 and distpsnr1 to reject the transient if
			there is a PS1 star-like object in it's immediate vicinity
		"""
		if (transient['distpsnr1'] < self.ps1_sgveto_rad and
				transient['sgscore1'] > self.ps1_sgveto_th):
			return True
		else:
			return False

	def is_star_in_gaia(self, transient):
		"""
			match tranient position with GAIA DR2 and uses parallax
			and proper motion to evaluate star-likeliness

			returns: True (is a star) or False otehrwise.
		"""

		transient_coords = SkyCoord(transient['ra'], transient['dec'], unit='deg')
		srcs, colnames, colunits = cone_search(
											'GAIADR2',
											transient_coords.ra.rad, transient_coords.dec.rad,
											self.gaia_rs,
											catalogs_dir=self.catshtm_path)
		my_keys = ['RA', 'Dec', 'Mag_G', 'PMRA', 'ErrPMRA', 'PMDec', 'ErrPMDec', 'Plx', 'ErrPlx']
		if len(srcs) > 0:
			gaia_tab					= Table(srcs, names=colnames)
			gaia_tab					= gaia_tab[my_keys]
			gaia_coords					= SkyCoord(gaia_tab['RA'], gaia_tab['Dec'], unit='rad')

			# compute distance
			gaia_tab['DISTANCE']		= transient_coords.separation(gaia_coords).arcsec
			gaia_tab['DISTANCE_NORM']	= (
				1.8 + 0.6 * exp( (20 - gaia_tab['Mag_G']) / 2.05) > gaia_tab['DISTANCE'])	#TODO: vary mag_G exclusion
			gaia_tab['FLAG_PROX']		= [
											True if x['DISTANCE_NORM'] == True and
											(self.gaia_veto_gmag_min <= x['Mag_G'] <= self.gaia_veto_gmag_max) else
											False for x in gaia_tab
											]

			# check for proper motion and parallax conditioned to distance
			gaia_tab['FLAG_PMRA']		= abs(gaia_tab['PMRA']  / gaia_tab['ErrPMRA']) > self.gaia_pm_signif
			gaia_tab['FLAG_PMDec']		= abs(gaia_tab['PMDec'] / gaia_tab['ErrPMDec']) > self.gaia_pm_signif
			gaia_tab['FLAG_Plx']		= abs(gaia_tab['Plx']   / gaia_tab['ErrPlx']) > self.gaia_plx_signif
			if (any(gaia_tab['FLAG_PMRA'] == True) or
				any(gaia_tab['FLAG_PMDec'] == True) or
				any(gaia_tab['FLAG_Plx'] == True)) and any(gaia_tab['FLAG_PROX'] == True):
				return True
		return False

	def apply(self, alert):
		"""
		Mandatory implementation.
		To exclude the alert, return *None*
		To accept it, either return
			* self.on_match_t2_units
			* or a custom combination of T2 unit names
		"""

		# --------------------------------------------------------------------- #
		#					CUT ON THE HISTORY OF THE ALERT						#
		# --------------------------------------------------------------------- #

		npp = len(alert.pps)
		if npp < self.min_ndet:
			self.logger.debug("rejected: %d photopoints in alert (minimum required %d)"%
				(npp, self.min_ndet))
			return None

		# cut on length of detection history
		detections_jds = alert.get_values('jd', upper_limits=False)
		det_tspan = max(detections_jds) - min(detections_jds)
		if not (self.min_tspan < det_tspan < self.max_tspan):
			self.logger.debug("rejected: detection history is %.3f d long, requested between %.3f and %.3f d"%
				(det_tspan, self.min_tspan, self.max_tspan))
			return None

		# --------------------------------------------------------------------- #
		#							IMAGE QUALITY CUTS							#
		# --------------------------------------------------------------------- #

		latest = alert.pps[0]
		if not self._alert_has_keys(latest):
			return None

		if (latest['isdiffpos'] == 'f' or latest['isdiffpos'] == '0'):
			self.logger.debug("rejected: 'isdiffpos' is %s", latest['isdiffpos'])
			return None

		if latest['rb'] < self.min_rb:
			self.logger.debug("rejected: RB score %.2f below threshod (%.2f)"%
				(latest['rb'], self.min_rb))
			return None

		if latest['fwhm'] > self.max_fwhm:
			self.logger.debug("rejected: fwhm %.2f above threshod (%.2f)"%
				(latest['fwhm'], self.max_fwhm))
			return None

		if latest['elong'] > self.max_elong:
			self.logger.debug("rejected: elongation %.2f above threshod (%.2f)"%
				(latest['elong'], self.max_elong))
			return None

		if abs(latest['magdiff']) > self.max_magdiff:
			self.logger.debug("rejected: magdiff (AP-PSF) %.2f above threshod (%.2f)"%
				(latest['magdiff'], self.max_magdiff))
			return None

		# --------------------------------------------------------------------- #
		#								ASTRONOMY								#
		# --------------------------------------------------------------------- #

		# check for closeby ss objects
		if (0 < latest['ssdistnr'] < self.min_ssdistnr):
			self.logger.debug("rejected: solar-system object close to transient (max allowed: %d)."%
				(self.min_ssdistnr))
			return None

                # check that a ps1 counterpart exists
#		if latest['distpsnr1'] < self.ps1_max_dist and latest['distpsnr1']>0:
                        # Nearby counterpart exists
#                        pass
#                else:
#                        self.logger.debug("rejected: no nearby PS object detected (min distance: %f)."%
#                                (self.p1_max_dist))
#                        return None

		if latest.get('distpsnr1',666) > self.ps1_max_dist:
			self.logger.debug("rejected: no PS1 object within  %.2f arcsec from alert."%
				(self.ps1_max_dist))
			return None

		# cut on galactic latitude
		b = self.get_galactic_latitude(latest)
		if abs(b) < self.min_gal_lat:
			self.logger.debug("rejected: b=%.4f, too close to Galactic plane (max allowed: %f)."%
				(b, self.min_gal_lat))
			return None

		# check ps1 star-galaxy score
		if self.is_star_in_PS1(latest):
			self.logger.debug("rejected: closest PS1 source %.2f arcsec away with sgscore of %.2f"%
				(latest['distpsnr1'], latest['sgscore1']))
			return None

		# check with gaia
		if self.gaia_rs>0:
			if not doCat:
				sys.exit("Cannot match to Gaia without catsHTM!")
			if self.is_star_in_gaia(latest):
				self.logger.debug("rejected: within %.2f arcsec from a GAIA start (PM of PLX)" %
					(self.gaia_rs))
				return None

		# congratulation alert! you made it!
		self.logger.debug("Alert %s accepted. Latest pp ID: %d"%(alert.tran_id, latest['candid']))
		for key in self.keys_to_check:
			self.logger.debug("{}: {}".format(key, latest[key]))
		return self.on_match_t2_units

max_stars_repo_name: wombaugh/Ampel-contrib-MPA

max_issues_repo_name: wombaugh/Ampel-contrib-MPA

max_forks_repo_name: wombaugh/Ampel-contrib-MPA

reason: from numpy,from astropy


text:
"""
observer
    - Beard & McLain, PUP, 2012
    - Last Update:
        3/2/2019 - RWB
"""
import sys
import numpy as np
sys.path.append('..')
import parameters.control_parameters as CTRL
import parameters.simulation_parameters as SIM
import parameters.sensor_parameters as SENSOR
from tools.rotations import Euler2Rotation
from tools.wrap import wrap

from message_types.msg_state import msg_state

class observer:
    def __init__(self, ts_control):
        # initialized estimated state message
        self.estimated_state = msg_state()
        # use alpha filters to low pass filter gyros and accels
        self.lpf_gyro_x = alpha_filter(alpha=0.5)
        self.lpf_gyro_y = alpha_filter(alpha=0.5)
        self.lpf_gyro_z = alpha_filter(alpha=0.5)
        self.lpf_accel_x = alpha_filter(alpha=0.5)
        self.lpf_accel_y = alpha_filter(alpha=0.5)
        self.lpf_accel_z = alpha_filter(alpha=0.5)
        # use alpha filters to low pass filter static and differential pressure
        self.lpf_static = alpha_filter(alpha=0.9)
        self.lpf_diff = alpha_filter(alpha=0.5)
        # ekf for phi and theta
        self.attitude_ekf = ekf_attitude()
        # ekf for pn, pe, Vg, chi, wn, we, psi
        self.position_ekf = ekf_position()

    def update(self, measurements):

        # estimates for p, q, r are low pass filter of gyro minus bias estimate
        self.estimated_state.p =
        self.estimated_state.q =
        self.estimated_state.r =

        # invert sensor model to get altitude and airspeed
        self.estimated_state.h =
        self.estimated_state.Va =

        # estimate phi and theta with simple ekf
        self.attitude_ekf.update(self.estimated_state, measurements)

        # estimate pn, pe, Vg, chi, wn, we, psi
        self.position_ekf.update(self.estimated_state, measurements)

        # not estimating these
        self.estimated_state.alpha = self.estimated_state.theta
        self.estimated_state.beta = 0.0
        self.estimated_state.bx = 0.0
        self.estimated_state.by = 0.0
        self.estimated_state.bz = 0.0
        return self.estimated_state

class alpha_filter:
    # alpha filter implements a simple low pass filter
    # y[k] = alpha * y[k-1] + (1-alpha) * u[k]
    def __init__(self, alpha=0.5, y0=0.0):
        self.alpha = alpha  # filter parameter
        self.y = y0  # initial condition

    def update(self, u):
        self.y =
        return self.y

class ekf_attitude:
    # implement continous-discrete EKF to estimate roll and pitch angles
    def __init__(self):
        self.Q =
        self.Q_gyro =
        self.R_accel =
        self.N =   # number of prediction step per sample
        self.xhat =  # initial state: phi, theta
        self.P =
        self.Ts = SIM.ts_control/self.N

    def update(self, state, measurement):
        self.propagate_model(state)
        self.measurement_update(state, measurement)
        state.phi = self.xhat.item(0)
        state.theta = self.xhat.item(1)

    def f(self, x, state):
        # system dynamics for propagation model: xdot = f(x, u)
        _f =
        return _f

    def h(self, x, state):
        # measurement model y
        _h =
        return _h

    def propagate_model(self, state):
        # model propagation
        for i in range(0, self.N):
             # propagate model
            self.xhat =
            # compute Jacobian
            A = jacobian(self.f, self.xhat, state)
            # compute G matrix for gyro noise
            G =
            # update P with continuous time model
            # self.P = self.P + self.Ts * (A @ self.P + self.P @ A.T + self.Q + G @ self.Q_gyro @ G.T)
            # convert to discrete time models
            A_d =
            G_d =
            # update P with discrete time model
            self.P =

    def measurement_update(self, state, measurement):
        # measurement updates
        threshold = 2.0
        h = self.h(self.xhat, state)
        C = jacobian(self.h, self.xhat, state)
        y = np.array([measurement.accel_x, measurement.accel_y, measurement.accel_z])
        for i in range(0, 3):
            if np.abs(y[i]-h[i,0]) < threshold:
                Ci =
                L =
                self.P =
                self.xhat =

class ekf_position:
    # implement continous-discrete EKF to estimate pn, pe, chi, Vg
    def __init__(self):
        self.Q =
        self.R =
        self.N =   # number of prediction step per sample
        self.Ts = (SIM.ts_control / self.N)
        self.xhat =
        self.P =
        self.gps_n_old = 9999
        self.gps_e_old = 9999
        self.gps_Vg_old = 9999
        self.gps_course_old = 9999

    def update(self, state, measurement):
        self.propagate_model(state)
        self.measurement_update(state, measurement)
        state.pn = self.xhat.item(0)
        state.pe = self.xhat.item(1)
        state.Vg = self.xhat.item(2)
        state.chi = self.xhat.item(3)
        state.wn = self.xhat.item(4)
        state.we = self.xhat.item(5)
        state.psi = self.xhat.item(6)

    def f(self, x, state):
        # system dynamics for propagation model: xdot = f(x, u)
        _f =
        return _f

    def h_gps(self, x, state):
        # measurement model for gps measurements
        _h =
        return _h

    def h_pseudo(self, x, state):
        # measurement model for wind triangale pseudo measurement
        _h =
        return _h

    def propagate_model(self, state):
        # model propagation
        for i in range(0, self.N):
            # propagate model
            self.xhat =
            # compute Jacobian
            A = jacobian(self.f, self.xhat, state)
            # update P with continuous time model
            # self.P = self.P + self.Ts * (A @ self.P + self.P @ A.T + self.Q + G @ self.Q_gyro @ G.T)
            # convert to discrete time models
            A_d =
            # update P with discrete time model
            self.P =

    def measurement_update(self, state, measurement):
        # always update based on wind triangle pseudu measurement
        h = self.h_pseudo(self.xhat, state)
        C = jacobian(self.h_pseudo, self.xhat, state)
        y = np.array([0, 0])
        for i in range(0, 2):
            Ci =
            L =
            self.P =
            self.xhat =

        # only update GPS when one of the signals changes
        if (measurement.gps_n != self.gps_n_old) \
            or (measurement.gps_e != self.gps_e_old) \
            or (measurement.gps_Vg != self.gps_Vg_old) \
            or (measurement.gps_course != self.gps_course_old):

            h = self.h_gps(self.xhat, state)
            C = jacobian(self.h_gps, self.xhat, state)
            y = np.array([measurement.gps_n, measurement.gps_e, measurement.gps_Vg, measurement.gps_course])
            for i in range(0, 4):
                Ci =
                L =
                self.P =
                self.xhat =
            # update stored GPS signals
            self.gps_n_old = measurement.gps_n
            self.gps_e_old = measurement.gps_e
            self.gps_Vg_old = measurement.gps_Vg
            self.gps_course_old = measurement.gps_course

def jacobian(fun, x, state):
    # compute jacobian of fun with respect to x
    f = fun(x, state)
    m = f.shape[0]
    n = x.shape[0]
    eps = 0.01  # deviation
    J = np.zeros((m, n))
    for i in range(0, n):
        x_eps = np.copy(x)
        x_eps[i][0] += eps
        f_eps = fun(x_eps, state)
        df = (f_eps - f) / eps
        J[:, i] = df[:, 0]
    return J