text:
import datetime
import numpy as np

import os
import sys
sys.path.insert(0, os.getcwd())

from hyperopt import hp, fmin, tpe

import qml_workdir.classes.config
from qml.cv import QCV
from qml.models import QXgb, QAvg
from qml_workdir.classes.models import qm

cv = QCV(qm)

model_id = qm.add_by_params(
    QXgb(
** {"alpha": 0.008, "booster": "gbtree", "colsample_bylevel": 0.9, "colsample_bytree": 0.9, "eta": 0.0024, "eval_metric": "logloss",
    "gamma": 0.04, "max_depth": 4, "num_boost_round": 2619, "objective": "binary:logistic", "subsample": 0.7, "tree_method": "hist"}
    ),
    'hyperopt xgb'
)

cv.features_sel_add(model_id, 40, ['age', 'height', 'weight', 'ap_hi', 'ap_lo', 'smoke', 'alco', 'active',
       'gender_male', 'height_low', 'weight_low', 'cholesterol_all',
       'gluc_all', 'cholesterol_1', 'cholesterol_2', 'cholesterol_3', 'gluc_1',
       'gluc_2', 'gluc_3', 'ap_error', 'ap_error_swap', 'imt', 'imt_class_all',
       'imt_class_0', 'imt_class_1', 'imt_class_2', 'imt_class_3',
       'imt_class_4', 'imt_class_5', 'imt_class_6', 'x__age__gluc_all',
       'x__ap_hi__cholesterol_all', 'div6__height__gluc_all__imt',
       'plus__age_norm__ap_hi_norm__gluc_all_norm', 'x__age__weight',
       'div1__age__weight__cholesterol_all',
       'div6__age__weight__cholesterol_all',
       'plus__height_norm__weight_norm__gluc_all_norm',
       'div1__ap_hi__ap_lo__cholesterol_all',
       'div6__ap_hi__ap_lo__cholesterol_all',
       'plus__age_norm__gluc_all_norm__imt_norm',
       'minus6__ap_hi_norm__ap_lo_norm__cholesterol_all_norm',
       'minus1__ap_hi_norm__ap_lo_norm__cholesterol_all_norm',
       'minus6__age_norm__ap_lo_norm__cholesterol_all_norm',
       'minus1__age_norm__ap_lo_norm__cholesterol_all_norm',
       'div6__height__weight__ap_lo', 'div2__ap_lo__cholesterol_all__gluc_all',
       'x__age__ap_hi__gluc_all', 'div5__ap_lo__cholesterol_all__gluc_all',
       'score_scale_val'],
[
'plus__age_norm__cholesterol_all_norm__gluc_all_norm',
'x__age__cholesterol_all__gluc_all',
'minus2__age_norm__cholesterol_all_norm__gluc_all_norm',
'minus5__age_norm__cholesterol_all_norm__gluc_all_norm',
'div5__age__cholesterol_all__gluc_all',
'div1__age__ap_hi__cholesterol_all',
'div2__age__cholesterol_all__gluc_all',
'div6__age__ap_hi__cholesterol_all',
'minus2__ap_lo_norm__cholesterol_all_norm__gluc_all_norm',
'div__cholesterol_all__gluc_all',
'minus1__height_norm__cholesterol_all_norm__imt_norm',
'div6__age__ap_lo__cholesterol_all',
'div1__age__ap_lo__cholesterol_all',
'div1__height__gluc_all__imt',
'div1__age__weight__gluc_all',
'minus2__height_norm__weight_norm__gluc_all_norm',
'div6__age__weight__gluc_all',
'minus6__height_norm__gluc_all_norm__imt_norm',
'div5__height__cholesterol_all__gluc_all',
'div4__age__cholesterol_all__gluc_all',
'div6__age__cholesterol_all__gluc_all',
'minus5__height_norm__weight_norm__gluc_all_norm',
'minus5__height_norm__cholesterol_all_norm__gluc_all_norm',
'div__age__cholesterol_all',
'minus1__height_norm__gluc_all_norm__imt_norm',
'div3__height__weight__gluc_all',
'minus4__age_norm__ap_hi_norm__cholesterol_all_norm',
'x__ap_hi__ap_lo',
'div2__height__cholesterol_all__gluc_all',
'div1__age__cholesterol_all__gluc_all',
'minus5__age_norm__cholesterol_all_norm__imt_norm',
'minus2__height_norm__cholesterol_all_norm__gluc_all_norm',
'minus__cholesterol_all_norm__imt_norm',
'plus__ap_hi_norm__ap_lo_norm',
'minus6__age_norm__cholesterol_all_norm__gluc_all_norm',
'minus1__age_norm__ap_lo_norm__gluc_all_norm',
'minus1__age_norm__cholesterol_all_norm__gluc_all_norm',
'plus__height_norm__weight_norm__ap_lo_norm',
'div4__height__ap_lo__imt',
'minus6__age_norm__ap_lo_norm__gluc_all_norm',
'div2__weight__ap_hi__gluc_all',
'div1__age__ap_lo__imt',
'minus5__age_norm__ap_hi_norm__ap_lo_norm',
'minus3__age_norm__weight_norm__gluc_all_norm',
'minus2__height_norm__weight_norm__cholesterol_all_norm',
'minus5__weight_norm__ap_hi_norm__gluc_all_norm',
'div__age__ap_hi',
'plus__weight_norm__ap_hi_norm__gluc_all_norm',
'div3__weight__ap_hi__gluc_all',
'minus4__weight_norm__cholesterol_all_norm__imt_norm',
'minus5__cholesterol_all_norm__gluc_all_norm__imt_norm',
'minus1__ap_hi_norm__ap_lo_norm__imt_norm',
'div6__height__weight__cholesterol_all',
'minus2__cholesterol_all_norm__gluc_all_norm__imt_norm',
'x__height__ap_hi__gluc_all',
'plus__ap_hi_norm__ap_lo_norm__gluc_all_norm',
'div5__age__cholesterol_all__imt',
'div__weight__gluc_all',
'minus2__age_norm__height_norm__ap_lo_norm',
'plus__age_norm__weight_norm__ap_lo_norm',
'x__height__cholesterol_all__imt',
'x__age__height__imt',
'minus1__weight_norm__cholesterol_all_norm__imt_norm',
'minus5__age_norm__gluc_all_norm__imt_norm',
'minus2__ap_hi_norm__ap_lo_norm__gluc_all_norm',
'div4__age__ap_hi__imt',
'div4__age__height__gluc_all',
'div1__weight__ap_hi__gluc_all',
'minus6__age_norm__height_norm__weight_norm',
'div3__ap_hi__ap_lo__cholesterol_all',
'div3__age__ap_hi__gluc_all',
'div2__ap_hi__cholesterol_all__imt',
'div6__ap_lo__gluc_all__imt',
'div6__weight__ap_hi__gluc_all',
'minus3__weight_norm__ap_lo_norm__cholesterol_all_norm',
'div__ap_hi__ap_lo',
'div2__height__weight__ap_lo',
'div5__age__weight__ap_lo',
'div6__cholesterol_all__gluc_all__imt',
'div1__weight__cholesterol_all__imt',
'div1__cholesterol_all__gluc_all__imt',
'plus__cholesterol_all_norm__imt_norm',
'minus__weight_norm__ap_lo_norm',
'x__ap_hi__imt',
'div5__age__weight__imt',
'plus__weight_norm__cholesterol_all_norm__imt_norm',
'div__age__ap_lo',
'div2__age__ap_hi__imt',
'div5__age__height__ap_hi',
'minus6__age_norm__ap_hi_norm__ap_lo_norm',
'minus1__age_norm__weight_norm__ap_lo_norm',
'div3__height__ap_lo__cholesterol_all',
'minus5__age_norm__height_norm__weight_norm',

], early_stop_cv=lambda x: x > 0.5414, log_file='qml_workdir/logs/feat09.txt')


text:
from __future__ import print_function
from __future__ import division
import torch
import torch.nn as nn
from torch.optim import *
import torch.nn.functional as F
from torch.nn.utils import clip_grad_norm_
import torch.utils.model_zoo as model_zoo
from dataset import get_dataloaders
import os
import numpy as np
import pickle as pkl
import PIL.Image as Image
from tensorboardX import SummaryWriter
import itertools
import matplotlib as mpl
# mpl.use('Agg')
import matplotlib.pyplot as plt
plt.switch_backend('agg')
from sklearn.metrics import confusion_matrix
from torchnet.meter import ConfusionMeter as CM
import seaborn as sn
import pandas as pd

def train_net(model, images, labels, pre_model, save_dir, sum_dir,
              batch_size, lr, log_after, cuda, device):
    print(model)
    if cuda:
        print('GPU')
        model.cuda(device=device)
    # define loss and optimizer
    optimizer = RMSprop(model.parameters(), lr=lr)
    criterion = nn.CrossEntropyLoss()
    train_loader, val_dataloader, test_loader = get_dataloaders(images_path=images,
                                                                labels_path=labels,
                                                                batch_size=batch_size)
    if not os.path.exists(save_dir):
        os.mkdir(save_dir)
    if not os.path.exists(sum_dir):
        os.mkdir(sum_dir)
    writer = SummaryWriter()
    if True:
        i = 0
        m_loss, m_accuracy = [], []
        num_classes = 7
        if pre_model:
            # self.load_state_dict(torch.load(pre_model)['model'])
            model.load_state_dict(torch.load(pre_model))
            print('log: resumed model {} successfully!'.format(pre_model))
            model_number = int(pre_model.split('/')[1].split('-')[1].split('.')[0])
        else:
            print('log: starting anew...')
        while True:
            i += 1
            net_loss = []
            net_accuracy = []
            if not pre_model:
                save_path = os.path.join(save_dir, 'model-{}.pt'.format(i))
            else:
                save_path = os.path.join(save_dir, 'model-{}.pt'.format(i+model_number-1))
            if i > 1 and not os.path.exists(save_path):
                torch.save(model.state_dict(), save_path)
                # remember to save only five previous models, so
                del_this = os.path.join(save_dir, 'model-{}.pt'.format(i+model_number-6))
                if os.path.exists(del_this):
                    os.remove(del_this)
                    print('log: removed {}'.format(del_this))
                print('log: saved {}'.format(save_path))
            list_of_pred = []
            list_of_labels = []
            for idx, data in enumerate(train_loader):
                ##########################
                model.train()
                ##########################
                test_x, label = data['input'], data['label']
                image0 = test_x[0]
                test_x = test_x.cuda(device=device) if cuda else test_x
                size = test_x.size(-1)
                # forward
                out_x, pred = model.forward(test_x)
                pred = pred.cpu(); out_x = out_x.cpu()
                image1 = pred[0]
                image2 = label[0]
                if idx % (len(train_loader)/2) == 0:
                    writer.add_image('input', image0, i)
                    writer.add_image('pred', image1, i)
                    writer.add_image('label', image2, i)

                loss = criterion(out_x, label)
                # get accuracy metric
                accuracy = (pred == label).sum()
                # also convert into np arrays to be used for confusion matrix
                pred_np = pred.numpy(); list_of_pred.append(pred_np)
                label_np = label.numpy(); list_of_labels.append(label_np)

                writer.add_scalar(tag='loss', scalar_value=loss.item(), global_step=i)
                writer.add_scalar(tag='over_all accuracy',
                                  scalar_value=accuracy*100/(test_x.size(0)*size**2),
                                  global_step=i)

                # per class accuracies
                avg = []
                for j in range(num_classes):
                    class_pred = (pred == j)
                    class_label = (label == j)
                    class_accuracy = (class_pred == class_label).sum()
                    class_accuracy = class_accuracy * 100 / (test_x.size(0) * size ** 2)
                    avg.append(class_accuracy)
                    writer.add_scalar(tag='class_{} accuracy'.format(j), scalar_value=class_accuracy, global_step=i)
                classes_avg_acc = np.asarray(avg).mean()
                writer.add_scalar(tag='classes avg. accuracy', scalar_value=classes_avg_acc, global_step=i)

                if idx % log_after == 0 and idx > 0:
                    print('{}. ({}/{}) image size = {}, loss = {}: accuracy = {}/{}'.format(i,
                                                                                idx,
                                                                                len(train_loader),
                                                                                out_x.size(),
                                                                                loss.item(),
                                                                                accuracy,
                                                                                test_x.size(0)*size**2))
                #################################
                # three steps for backprop
                model.zero_grad()
                loss.backward()
                # perform gradient clipping between loss backward and optimizer step
                clip_grad_norm_(model.parameters(), 0.05)
                optimizer.step()
                accuracy = accuracy * 100 / (test_x.size(0)*size**2)
                net_accuracy.append(accuracy)
                net_loss.append(loss.item())
                #################################
            mean_accuracy = np.asarray(net_accuracy).mean()
            mean_loss = np.asarray(net_loss).mean()
            m_loss.append((i, mean_loss))
            m_accuracy.append((i, mean_accuracy))
            print('####################################')
            print('epoch {} -> total loss = {:.5f}, total accuracy = {:.5f}%'.format(i, mean_loss, mean_accuracy))
            print('####################################')

            # # one epoch complete, get new confusion matrix!
            # cm_preds = np.vstack(list_of_pred)
            # cm_preds = cm_preds.reshape(-1)
            # cm_labels = np.vstack(list_of_labels)
            # cm_labels = cm_labels.reshape(-1)
            # cnf_matrix = confusion_matrix(cm_labels, cm_preds)
            # fig1 = plt.figure()
            # plot_confusion_matrix(cnf_matrix, classes=class_names,
            #                       title='Confusion matrix, without normalization')
            # # Plot normalized confusion matrix
            # fig2 = plt.figure()
            # plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True,
            #                       title='Normalized confusion matrix')
            # get1 = np.asarray(fig2img(fig1))
            # get2 = np.asarray(fig2img(fig2))
            # # print(get1.size)

            # validate model
            if i % 10 == 0:
                eval_net(model=model, criterion=criterion, val_loader=val_dataloader,
                         denominator=batch_size * size**2, cuda=cuda, device=device,
                         writer=writer, num_classes=num_classes, batch_size=batch_size, step=i)
    writer.export_scalars_to_json("./all_scalars.json")
    writer.close()

    pass

def eval_net(**kwargs):
    cuda = kwargs['cuda']
    device = kwargs['device']
    model = kwargs['model']
    model.eval()

    if cuda:
        model.cuda(device=device)
    if 'writer' in kwargs.keys():
        num_classes = kwargs['num_classes']
        batch_size = kwargs['batch_size']
        writer = kwargs['writer']
        step = kwargs['step']
        denominator = kwargs['denominator']
        val_loader = kwargs['val_loader']
        model = kwargs['model']
        criterion = kwargs['criterion']
        net_accuracy, net_loss = [], []
        for idx, data in enumerate(val_loader):
            test_x, label = data['input'], data['label']
            test_x = test_x.cuda() if cuda else test_x
            # forward
            out_x, pred = model.forward(test_x)
            pred = pred.cpu()
            loss = criterion(out_x.cpu(), label)
            # get accuracy metric
            accuracy = (pred == label).sum()
            accuracy = accuracy * 100 / (test_x.size(0)*64**2)
            net_accuracy.append(accuracy)
            net_loss.append(loss.item())

            # per class accuracies
            # avg = []
            # for j in range(num_classes):
            #     class_pred = (pred == j)
            #     class_label = (label == j)
            #     class_accuracy = (class_pred == class_label).sum()
            #     class_accuracy = class_accuracy * 100 / (batch_size * 32 ** 2)
            #     avg.append(class_accuracy)
            #     writer.add_scalar(tag='class_{} accuracy'.format(j), scalar_value=class_accuracy, global_step=step)
            # classes_avg_acc = np.asarray(avg).mean()
            # writer.add_scalar(tag='classes avg. accuracy', scalar_value=classes_avg_acc, global_step=step)

            #################################
        mean_accuracy = np.asarray(net_accuracy).mean()
        mean_loss = np.asarray(net_loss).mean()
        writer.add_scalar(tag='eval accuracy', scalar_value=mean_accuracy, global_step=step)
        writer.add_scalar(tag='eval loss', scalar_value=mean_loss, global_step=step)
        print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
        print('log: validation:: total loss = {:.5f}, total accuracy = {:.5f}%'.format(mean_loss, mean_accuracy))
        print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')

    else:
        # model, images, labels, pre_model, save_dir, sum_dir, batch_size, lr, log_after, cuda
        pre_model = kwargs['pre_model']
        images = kwargs['images']
        labels = kwargs['labels']
        batch_size = kwargs['batch_size']
        criterion = nn.CrossEntropyLoss()
        cm = CM(k=7, normalized=True)
        preds, labs = torch.Tensor().long(), torch.Tensor().long()

        model.load_state_dict(torch.load(pre_model))
        print('log: resumed model {} successfully!'.format(pre_model))
        _, _, test_loader = get_dataloaders(images_path=images, labels_path=labels, batch_size=batch_size)
        net_accuracy, net_loss = [], []
        net_class_accuracy_0, net_class_accuracy_1, net_class_accuracy_2, \
        net_class_accuracy_3, net_class_accuracy_4, net_class_accuracy_5,\
        net_class_accuracy_6  = [], [], [], [], [], [], []
        for idx, data in enumerate(test_loader):
            if idx == 400:
                break
            test_x, label = data['input'], data['label']
            # print(test_x.shape)
            if cuda:
                test_x = test_x.cuda()
            # forward
            out_x, pred = model.forward(test_x)
            pred = pred.cpu()
            loss = criterion(out_x.cpu(), label)

            # get accuracy metric
            accuracy = (pred == label).sum()
            accuracy = accuracy * 100 / (pred.view(-1).size(0))
            net_accuracy.append(accuracy)
            net_loss.append(loss.item())
            if idx % 10 == 0:
                print('log: on {}'.format(idx))

            # print(pred.view(-1).size(0))
            # get per-class metrics
            class_pred_0 = (pred == 0)
            class_label_0 = (label == 0)
            class_accuracy_0 = (class_pred_0 == class_label_0).sum()
            class_accuracy_0 = class_accuracy_0 * 100 / (pred.view(-1).size(0))
            net_class_accuracy_0.append(class_accuracy_0)

            class_pred_1 = (pred == 1)
            class_label_1 = (label == 1)
            class_accuracy_1 = (class_pred_1 == class_label_1).sum()
            class_accuracy_1 = class_accuracy_1 * 100 / (pred.view(-1).size(0))
            net_class_accuracy_1.append(class_accuracy_1)

            class_pred_2 = (pred == 2)
            class_label_2 = (label == 2)
            class_accuracy_2 = (class_pred_2 == class_label_2).sum()
            class_accuracy_2 = class_accuracy_2 * 100 / (pred.view(-1).size(0))
            net_class_accuracy_2.append(class_accuracy_2)

            class_pred_3 = (pred == 3)
            class_label_3 = (label == 3)
            class_accuracy_3 = (class_pred_3 == class_label_3).sum()
            class_accuracy_3 = class_accuracy_3 * 100 / (pred.view(-1).size(0))
            net_class_accuracy_3.append(class_accuracy_3)

            class_pred_4 = (pred == 4)
            class_label_4 = (label == 4)
            class_accuracy_4 = (class_pred_4 == class_label_4).sum()
            class_accuracy_4 = class_accuracy_4 * 100 / (pred.view(-1).size(0))
            net_class_accuracy_4.append(class_accuracy_4)

            class_pred_5 = (pred == 5)
            class_label_5 = (label == 5)
            class_accuracy_5 = (class_pred_5 == class_label_5).sum()
            class_accuracy_5 = class_accuracy_5 * 100 / (pred.view(-1).size(0))
            net_class_accuracy_5.append(class_accuracy_5)

            class_pred_6 = (pred == 6)
            class_label_6 = (label == 6)
            class_accuracy_6 = (class_pred_6 == class_label_6).sum()
            class_accuracy_6 = class_accuracy_6 * 100 / (pred.view(-1).size(0))
            net_class_accuracy_6.append(class_accuracy_6)

            preds = torch.cat((preds, pred.long().view(-1)))
            labs = torch.cat((labs, label.long().view(-1)))
            #################################
        mean_accuracy = np.asarray(net_accuracy).mean()
        mean_loss = np.asarray(net_loss).mean()

        class_0_mean_accuracy = np.asarray(net_class_accuracy_0).mean()
        class_1_mean_accuracy = np.asarray(net_class_accuracy_1).mean()
        class_2_mean_accuracy = np.asarray(net_class_accuracy_2).mean()
        class_3_mean_accuracy = np.asarray(net_class_accuracy_3).mean()
        class_4_mean_accuracy = np.asarray(net_class_accuracy_4).mean()
        class_5_mean_accuracy = np.asarray(net_class_accuracy_5).mean()
        class_6_mean_accuracy = np.asarray(net_class_accuracy_6).mean()

        print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
        print('log: test:: total loss = {:.5f}, total accuracy = {:.5f}%'.format(mean_loss, mean_accuracy))
        print('log: class 0:: total accuracy = {:.5f}%'.format(class_0_mean_accuracy))
        print('log: class 1:: total accuracy = {:.5f}%'.format(class_1_mean_accuracy))
        print('log: class 2:: total accuracy = {:.5f}%'.format(class_2_mean_accuracy))
        print('log: class 3:: total accuracy = {:.5f}%'.format(class_3_mean_accuracy))
        print('log: class 4:: total accuracy = {:.5f}%'.format(class_4_mean_accuracy))
        print('log: class 5:: total accuracy = {:.5f}%'.format(class_5_mean_accuracy))
        print('log: class 6:: total accuracy = {:.5f}%'.format(class_6_mean_accuracy))
        print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')

        # class_names = ['background/clutter', 'buildings', 'trees', 'cars',
        #                'low_vegetation', 'impervious_surfaces', 'noise']
        # cm_preds = pred.view(-1).cpu().numpy()
        # cm_labels = label.view(-1).cpu().numpy()
        # cnf_matrix = confusion_matrix(cm_labels, cm_preds)
        #
        # fig1 = plt.figure()
        # plot_confusion_matrix(cnf_matrix, classes=class_names,
        #                       title='Confusion matrix, without normalization')
        # # Plot normalized confusion matrix
        # fig2 = plt.figure()
        # plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True,
        #                       title='Normalized confusion matrix')
        # fig2img(fig1).save('unnormalized.png')
        # fig2img(fig2).save('normalized.png')

        #
        # cm.add(preds.view(-1), labs.view(-1).type(torch.LongTensor))
        # this = cm.value()
        # print(this)
        # df_cm = pd.DataFrame(this, index=[f for f in class_names],
        #                      columns=[f for f in class_names])
        # fig = plt.figure(figsize=(10, 7))
        # sn.heatmap(df_cm, annot=True)
        # fig2img(fig).save('sea.png')
    pass

ext: py

max_stars_repo_name: annusgit/forestcoverUnet

max_issues_repo_name: annusgit/forestcoverUnet

max_forks_repo_name: annusgit/forestcoverUnet

reason: import numpy


text:
"""Define the base data containing classes for precisionguide system

These are designed to be portable across any platform, since they do
not depend on the specifics of the telescsope control system.  Thus,
data can be recorded using this code on, e.g., a Windows system and
uploaded to any other platform for detailed analysis using this same
code.

"""

from copy import deepcopy

import numpy as np

from scipy.signal import convolve2d
from scipy.ndimage.measurements import center_of_mass

from astropy import log
from astropy.io import fits
from astropy import units as u
from astropy.time import Time

from ccdmultipipe.utils.ccddata import FbuCCDData

_NotFound = object()

###########################
# Decorators              #
###########################
class pgproperty(property):
    """Caching property decorator with auto setting and None reset

    . myprop = None resets system and forces the getter to run

    . when getter runs and returns a non-None value, that value is
    cached and returned instead of running the getter

    . myprop = 'some_value' overrides the getter -- 'some_value' is
    cached and returned on subsequent queries of myprop

    . No setter is required to set the property, but if one is
    provided, its return value is used as the cached value that
    overrides the getter (i.e., no knowledge of the internal workings
    of the system is requried to make it work)

    . Deleters can be specified to do something before the cache
    reference is deleted.  Deletion is not permanent -- the next time
    myprop is queried, the getter is run

    . shape_check class variable: None, 0, or tuple

        This class variable affects the treatment of the value
        returned by the setter or if no setter is
        present, value provided by user.
        . None: the value is cached without modification
        . 0: value is converted to a numpy array with np.asarray()
        . tuple: like 0 but shape of value must match shape_check or
        a ValueError is raised

    Inspired by `astropy.utils.lazyproperty` and `property_manager
    <https://github.com/xolox/python-property-manager>`

    .NOTE: This cache operates on the top-level property.  In a
    complicated multi-inheritance system with inter-connected property
    like `pgdata`, some time savings in calculation of quantities
    could be achieved by caching property at each inheritance level
    individually, say by making the `key` a MRO level-unique string,
    e.g. property name, `self.__class__.__name__` and module name.
    Then a None reset would only reset the levels from the top down to
    the MRO at which the property was set to None.  This guarantees
    the children get a freshly calculated value of the affected
    properties and acknowledges that the parents don't care that the
    change was made, since they never depended on the relationship
    between the property in the first place.  Or if they did, they
    should be super()ed into the calculation and the most senior
    setter concerned about the interrelationship would also be calling
    for the reset of the affected other property.  The side-affect of
    caching all of the intermediate results would be more memory use,
    since all of the intermediate property would have long-lived
    references.  For the pgdata coordiate-oriented stuff, this is not
    a big deal, but it is not generally appropriate.

    """

    shape_check = None
    def __init__(self, fget,
                 fset=None,
                 fdel=None,
                 doc=None):
        super().__init__(fget, fset, fdel, doc)
        self._key = self.fget.__name__
        self._none_proxy = 'None proxy'

    def npval(self, val):
        """Turn val into np.array if shape_check is non-None.  Checks
        shape of resulting array if shape_check is a tuple"""
        if self.shape_check is None or val is None:
            return val
        val = np.asarray(val)
        if (self.shape_check != 0
            and val.shape != self.shape_check):
            raise ValueError(f'value "{val}" is not a sensible {self.shape_check}-dimensional coordinate')
        return val

    def __get__(self, obj, owner=None):
        try:
            obj_dict = obj.__dict__
            val = obj_dict.get(self._key, _NotFound)
            if val is self._none_proxy:
                return None
            if val is _NotFound or val is None:
                val = self.fget(obj)
                if val is None:
                    obj_dict[self._key] = self._none_proxy
                    return None
                val = self.npval(val)
                obj_dict[self._key] = val
            return val
        except AttributeError:
            if obj is None:
                return self
            raise

    def __set__(self, obj, val):
        obj_dict = obj.__dict__
        if self.fset:
            val = self.fset(obj, val)
        val = self.npval(val)
        obj_dict[self._key] = val

    def __delete__(self, obj):
        if self.fdel:
            self.fdel(obj)
        obj.__dict__.pop(self._key, None)    # Delete if present

class pgcoordproperty(pgproperty):
    shape_check=(2,)

#######################
# Primary Objects     #
#######################

def center_quality_checker(value):
    if value is None:
        return
    if not isinstance(value, int) or value < 0 or value > 10:
        raise ValueError('quality must be an integer value from 0 to 10')
    return value

class PGCenter():
    """Base class for containing object center and desired center

    Parameters
    ----------
    obj_center : tuple or array
        Center of object, (y, x) in *unbinned* coordinates referenced to the
        *origin* of the CCD.  Note Python C array indexing into 2D array

    desired_center : tuple-like
        Desired center of object, (y, x) in *unbinned* coordinates
        referenced to the *origin* of the CCD.  Note Python C array
        indexing into 2D array

    center_quality : int
        0 -- 10 ranking of quality of ``obj_center``
        determination, with 1 = very bad, 10 = very good, 0 = invalid

    tavg : `~astropy.time.Time`
        Average time of observation.  For simple exposures, this is
        the midpoint.  For exposures integrated from several segments
        one might imagine this would be the average of the exposure
        efficiency as a function of time.  Hoever Rots et al (A&A 574,
        A36 2015) note no standard is defined.  This turns into the
        DATE-AVG FITS keyword

    """
    def __init__(self,
                 obj_center=None,
                 desired_center=None,
                 center_quality=None,
                 tavg=None):
        self.obj_center = obj_center
        self.desired_center = desired_center
        self.center_quality = center_quality
        self.tavg = tavg

    # Our decorator does everything we need :-)
    @pgcoordproperty
    def obj_center(self):
        pass

    @pgcoordproperty
    def desired_center(self):
        pass

    @pgproperty
    def center_quality(self):
        pass

    @center_quality.setter
    def center_quality(self, value):
        """Unset center_quality translates to 0 center_quality"""
        return center_quality_checker(value)

    @pgproperty
    def tavg(self):
        pass

# We want to use astropy's CCDData to store our

# Pretty much all CCD-like detectors present their raw data in adu.
# In the event this needed to change for a particular system, insert a
# subclass redefining raw_unit anywhere on the PGData inheritance
# chain.  That raw_unit will override this one.  Note, doing this as a
# class variable is necessary because of the CCDData read classmethod
class ADU():
    """Simple class to define the class variable `fallback_unit` =
    `~astropy.units.adu` for classes using the
    :class:`ccdmultipipe.FbuCCDData` system

    """
    fallback_unit = u.adu

class PGData(ADU, FbuCCDData):
    """Base class for image data in the `precisionguide` system

    This class stores CCD data and defines methods and property to
    calculate/store four primary quantities: :prop:`obj_center`,
    :prop:`desired_center`, :prop:`center_quality`, and :prop:`tavg`, as
    described in the documentation.

    CCD data are stored using `astropy.nddata.CCDData` as its base
    class with the addition of a class,
    :class:`ccdmultipipe.FbuCCDData`, that ensures the
    `~astropy.nddata.CCDData` will always be defined with a
    :class:`astropy.units.Unit`.  Since all or nearly all CCD-like
    detectors present their raw data in `~astropy.units.adu`, this is
    the default :class:`~astropy.units.Unit` assumed by
    :class:`PGData`.  In the event this needs to change for a
    particular system, a class similar to :class:`ADU` (e.g. "Count")
    could be defined and inserted into the inheritance chain

    superclass and adds properties and methods that calculate/store
    four quantities: :prop:`obj_center`, :prop:`desired_center`,
    :prop:`center_quality`, and :prop:`tavg`.  These four quantities are
    intended to be returned in a :class:`PGCenter` object for
    subsequent lightweight storage and use in the precisionguide
    system.  Because precisionguide controls the absolute position of
    an object (or FOV center) on the CCD, :prop:`desired_center` and
    :prop:`obj_center` always read in *unbinned* pixel values
    referenced to the origin of the CCD itself.  Thus, the image input
    to :class:`PGData` must include both the image FITS header and
    image array(s)

    Parameters
    ----------

    """
    def __init__(self,
                 data,
                 obj_center=None,
                 desired_center=None,
                 center_quality=None,
                 recalculate=False,
                 subframe_origin=None,
                 binning=None,
                 date_obs_key='DATE-OBS',
                 exptime_key='EXPTIME',
                 darktime_key='DARKTIME',
                 copy=False,
                 **kwargs):
        # Pattern after NDData init but skip all the tests
        if isinstance(data, PGData):
            # Sigh.  We have to undo the convenience of our pgproperty
            # lest we trigger the calculation of property, which leads
            # to recursion problems
            obj_dict = data.__dict__
            obj_center = obj_dict.get('obj_center')
            desired_center = obj_dict.get('desired_center')
            center_quality = data._center_quality
            subframe_origin = data.subframe_origin
            binning = data.binning
            date_obs_key = data.date_obs_key
            exptime_key = data.exptime_key
            darktime_key = data.darktime_key
        if copy:
            obj_center = deepcopy(obj_center)
            desired_center = deepcopy(desired_center)
            center_quality = deepcopy(center_quality)
            subframe_origin = deepcopy(subframe_origin)
            binning = deepcopy(binning)
            date_obs_key = deepcopy(date_obs_key)
            exptime_key = deepcopy(exptime_key)
            darktime_key = deepcopy(darktime_key)

        super().__init__(data, copy=copy, **kwargs)
        self.recalculate = recalculate # May be not appropriate at this level
        self.obj_center = obj_center
        self.desired_center = desired_center
        self._center_quality = center_quality
        self.subframe_origin = subframe_origin
        self.binning = binning
        self.date_obs_key = date_obs_key
        self.exptime_key = exptime_key
        self.darktime_key = darktime_key
        self._invalid_obj_center = (-99, -99)
        self._invalid_desired_center = (-99, -99)
        self._invalid_center_quality = 0

    def _init_args_copy(self, kwargs):
        """The NDData _slice and _arithmetic methods create new class instances by running the __init__ method with the appropriate kwarg to copy over relevant property.  Thus we need to copy all our property into a keyword dictionary too"""
        # Grab calculated property directly from our pgproperty
        # decorator system so it doesn't trigger calculations!
        obj_dict = self.__dict__
        obj_center = obj_dict.get('obj_center')
        desired_center = obj_dict.get('desired_center')
        kwargs['obj_center'] = obj_center
        kwargs['desired_center'] = desired_center
        kwargs['center_quality'] = self._center_quality
        kwargs['subframe_origin'] = self.subframe_origin
        kwargs['binning'] = self.binning
        kwargs['date_obs_key'] = self.date_obs_key
        kwargs['exptime_key'] = self.exptime_key
        kwargs['darktime_key'] = self.darktime_key
        return kwargs

    def _arithmetic(self, *args, **kwds):
        """Insert our property when new class instances are created using arithmetic"""
        result, kwargs = super()._arithmetic(*args, **kwds)
        kwargs = self._init_args_copy(kwargs)
        return result, kwargs

    def _slice(self, item):
        """Override NDSlicingMixin definition to move subframe origin"""
        kwargs = super()._slice(item)
        kwargs = self._init_args_copy(kwargs)
        yslice = item[0]
        xslice = item[1]
        yorg = yslice.start or 0
        xorg = xslice.start or 0
        kwargs['subframe_origin'] = (yorg, xorg)
        return kwargs

    @pgcoordproperty
    def obj_center(self):
        """Center of object, (Y, X) in pixel

        Coordinates are referenced to the image stored in the
    :prop:`data` property.  This image may be a binned subframe of
    full detector.  Use :meth:`unbinned(self.obj_center)` to obtain
    the coordinates in raw detector pixels.  Quality of center
    determination must be set as well.  Base class uses out-of-bounds
    value (-99, -99) for center and 0 for center_quality

        Results are stored using the :class:`pgcoordproperty`
        decorator system.  See documentation of that class for
        explanation of features.

        """
        obj_center = self._invalid_obj_center
        self.center_quality = self._invalid_center_quality
        return obj_center

    @pgcoordproperty
    def desired_center(self):
        """Desired center of object (Y, X).  Base class uses center of
    data array.  As with :prop:`obj_center`, :prop:`desired_center` is
    referenced to the data stored in :prop:`data`

        Results are stored using the :class:`pgcoordproperty`
        decorator system.  See documentation of that class for
        explanation of features.

        """
        # Here is where the complicated desired_center calculation is
        # done:
        npshape = np.asarray(self.data.shape)
        desired_center = npshape/2
        return desired_center

    @property
    def center_quality(self):
        """Quality of center determination on a 0 to 10 integer scale.

  Quality should always be set in the :prop:`obj_center` setter

        """
        if self._center_quality is None:
            self.obj_center
        return self._center_quality

    @center_quality.setter
    def center_quality(self, value):
        self._center_quality = center_quality_checker(value)

    @pgproperty
    def tavg(self):
        """`~astropy.time.Time` Average time of observation,  See
        :param:`tavg` of :class:`PGCenter.`"""

        tavg_str = self.meta.get('date-avg')
        if tavg_str is not None:
            return Time(tavg_str, format='fits')
        try:
            exptime = self.meta[self.exptime_key.lower()]
            exptime *= u.s
            dateobs_str = self.meta[self.date_obs_key.lower()]
            return Time(dateobs_str, format='fits') + exptime/2
        except:
            log.error(f'Cannot read sufficient combination of '
                      '{self.date_obs_key}, {self.darktime_key} '
                      'and/or {self.exptime_key} keywords from FITS '
                      'header to establish tavg')
            return None

    @tavg.setter
    def tavg(self, val):
        if not isinstance(val, Time):
            raise ValueError('tavg must be of type `~astropy.time.Time`')

    @pgcoordproperty
    def binning(self):
        """Image binning in Y,X order.

         NOTE: this needs to be overridden in a subclass with the
         actual FITS binning keywords used (which are unfortunately
         not in any FITS standard).  E.g.:
         >>> binning = (self.meta['YBINNING'],
         >>>            self.meta['XBINNING'])
         >>> return binning
        """
        pass

        #binning = (1,1)
        #return binning

    @pgcoordproperty
    def subframe_origin(self):
        """Subframe origin in *unbinned* pixels with full CCD origin =
    (0,0).  Y,X order
         NOTE: this needs to be overridden in a subclass with the
         actual FITS binning keywords used (which are unfortunately
         not in any FITS standard).  E.g.:
         >>> subframe_origin = (self.meta['YORGSUBF'],
         >>>                    self.meta['XORGSUBF'])
         >>> subframe_origin = np.asarray(subframe_origin)
         >>> subframe_origin *= self.binning
         >>> return subframe_origin
        """
        pass
        #subframe_origin = (0,0)
        #return subframe_origin

    def coord_unbinned(self, coords):
        """Returns pixel coords referenced to full CCD given internally stored binning/subim info"""
        coords = np.asarray(coords)
        unbinned = self.binning * coords + self.subframe_origin
        return unbinned.astype(int)

    def x_unbinned(self, coords):
        """Returns x coord referenced to full CCD given internally stored binning/subim info"""
        coords = np.asarray(coords)
        unbinned = self.binning[1] * coords + self.subframe_origin[1]
        return unbinned.astype(int)

    def y_unbinned(self, coords):
        """Returns y coord referenced to full CCD given internally stored binning/subim info"""
        coords = np.asarray(coords)
        unbinned = self.binning[0] * coords + self.subframe_origin[0]
        return unbinned.astype(int)

    def coord_binned(self, coords, limit_edges=False):
        """Assuming coords are referenced to full CCD, return location in binned coordinates relative to the subframe origin.  Limit_edges=True sets output value(s) that would otherwise fall outside the binned image to the appropriate boundary value """
        coords = np.asarray(coords)
        binned_coords = (coords - self.subframe_origin) / self.binning
        if not limit_edges:
            return binned_coords
        s = np.asarray(self.shape)
        if len(coords.shape) == 1:
            binned_coords[0] = np.min((s[0], binned_coords[0]))
            binned_coords[1] = np.min((s[1], binned_coords[1]))
            return binned_coords.astype(int)
        lidx = np.flatnonzero(binned_coords[0, :] > s[0])
        binned_coords[0, lidx] = s[0]
        lidx = np.flatnonzero(binned_coords[1, :] > s[1])
        binned_coords[1, lidx] = s[1]
        return binned_coords.astype(int)

    def x_binned(self, xs, **kwargs):
        """Assuming x coords are referenced to full CCD, return location in binned coordinates relative to the subframe origin.  Limit_edges=True sets output value(s) that would otherwise fall outside the binned image to the appropriate boundary value """
        if np.isscalar(xs):
            unbinned_coords = np.asarray((-99, xs))
            binned_coords = self.coord_binned(unbinned_coords, **kwargs)
            xs = binned_coords[1]
        else:
            xs = np.asarray(xs)
            ys = len(xs)*(-99,)
            unbinned_coords = list(zip(ys, xs))
            binned_coords = self.coord_binned(unbinned_coords, **kwargs)
            xs = binned_coords[:, 1]
        return xs

    def y_binned(self, ys, **kwargs):
        """Assuming x coords are referenced to full CCD, return location in binned coordinates relative to the subframe origin.  Limit_edges=True sets output value(s) that would otherwise fall outside the binned image to the appropriate boundary value """
        if np.isscalar(ys):
            unbinned_coords = np.asarray((ys, -99))
            binned_coords = self.coord_binned(unbinned_coords, **kwargs)
            ys = binned_coords[0]
        else:
            ys = np.asarray(ys)
            xs = len(ys)*(-99,)
            unbinned_coords = list(zip(ys, xs))
            binned_coords = self.coord_binned(unbinned_coords, **kwargs)
            ys = binned_coords[:, 0]
        return ys

    def im_unbinned(self, a):
        """Returns an unbinned version of a.  a must be same shape as self.data
        NOTE: to preserve flux, divide by (np.prod(self.binning)
        """
        if a is None:
            return None
        assert a.shape == self.data.shape
        # Don't bother if we are already unbinned
        if np.sum(self.binning) == 2:
            return a
        newshape = self.binning * a.shape
        # From http://scipy-cookbook.readthedocs.io/items/Rebinning.html
        assert len(a.shape) == len(newshape)

        slices = [ slice(0,old, float(old)/new)
                   for old,new in zip(a.shape,newshape) ]
        coordinates = np.mgrid[slices]
        indices = coordinates.astype('i')   #choose the biggest smaller integer index
        unbinned = a[tuple(indices)]
        # Check to see if we need to make a larger array into which to
        # plop unbinned array
        if np.sum(self.subframe_origin) > 0:
            # Note subframe origin reads in binned pixels
            origin = self.unbinned(self.subframe_origin)
            full_unbinned = np.zeros(origin + np.asarray(unbinned.shape))
            full_unbinned[origin[0]:, origin[1]:] = unbinned
            unbinned = full_unbinned
        return unbinned

    @pgproperty
    def self_unbinned(self):
        """Returns an unbinned version of this object.  NOTE: just as with the
    primary object, if this object needs to be modified, it should be
    copied first, particularly since, in the case the image is not
    binned, it might just point to self

        """
        if (self.binning.sum() == 2
            and self.subframe_origin.sum() == 0):
            # We are already unbinned, don't bother to copy
            return self
        # If we made it here, we need to unbin
        self_unbinned = deepcopy(self)
        self_unbinned.data = self.im_unbinned(self.data)
        self_unbinned.mask = self.im_unbinned(self.mask)
        self_unbinned.uncertainty = self.im_unbinned(self.uncertainty)
        self_unbinned.binning = np.asarray((1,1))
        self_unbinned.subframe_origin = np.asarray((0,0))
        # --> NOTE we are not messing with any other metadata
        return self_unbinned

    @property
    def pgcenter(self):
        """Returns a :class:`PGCenter` object with the *unbinned* obj and
    desired center coordinates.  Working in *unbinned* coordinates is
    essential for the internal workings of the precisionguide
    telescope control system, but rather confusing and abstract for
    the user in any other context.

        """
        # I may want to also export the binning information for
        # display purposes in precisionguide
        return PGCenter(self.coord_unbinned(self.obj_center),
                        self.coord_unbinned(self.desired_center),
                        self.center_quality,
                        self.tavg)

    def _card_write(self):
        """Writes FITS header cards for obj and desired center coordinates in
        *binned* coordinates (i.e., as they would likely presesnt in
        image analysis software showing the image)

        """
        if not np.all(np.isfinite(self.obj_center)):
            self.obj_center = self._invalid_obj_center
            log.warning(f'non-finite obj_center found, converting to '
                        f'{self.obj_center}')
        # Note pythonic y, x coordinate ordering
        self.meta['OBJ_CR0'] = (self.obj_center[1], 'Calculated object '
                                'center X')
        self.meta['OBJ_CR1'] = (self.obj_center[0], 'Calculated object '
                                'center Y')
        self.meta['DES_CR0'] = (self.desired_center[1], 'Desired center X')
        self.meta['DES_CR1'] = (self.desired_center[0], 'Desired center Y')
        self.meta['HIERARCH CENTER_QUALITY'] = (
            self.center_quality or self._invalid_center_quality,
            'Quality on 0-10 scale '
            'of center determination')
        # As per https://www.aanda.org/articles/aa/pdf/2015/02/aa24653-14.pdf
        if not self.meta.get('DATE-AVG'):
            self.meta.insert('DATE-OBS',
                             ('DATE-AVG', self.tavg.value,
                              'midpoint of exposure, UT'),
                             after=True)

    def write(self, *args, **kwargs):
        self._card_write()
        super().write(*args, **kwargs)

class NoCenterPGD(PGData):
    """Sets :prop:`obj_center` to invalid value and :prop:`center_quality` to 0`"""
    pass

class FITSReaderPGD(PGData):
    """Read FITS keys to set defaults"""

    ## Inefficient way -- FITS header is read each time object is
    ## initialized even if it never uses the property
    #def __init__(self, *args,
    #             obj_center=None,
    #             desired_center=None,
    #             center_quality=0,
    #             recalculate=False,
    #             **kwargs):
    #    super().__init__(*args, **kwargs)
    #    try:
    #        assert not recalculate, ('Recalculation requested')
    #        log.debug('Trying to read center info from FITS header,')
    #        cx = self.meta['OBJ_CR0']
    #        cy = self.meta['OBJ_CR1']
    #        dx = self.meta['DES_CR0']
    #        dy = self.meta['DES_CR1']
    #        q = self.meta['CENTER_QUALITY']
    #        tavg_str = self.meta['DATE-AVG']
    #        self.obj_center = (cy, cx)
    #        self.desired_center = (dy, dx)
    #        self.center_quality = q
    #        self.tavg = Time(tavg_str, format='fits')
    #        log.info('Center info set from FITS header, use '
    #                 'recalculate=True to avoid')
    #    except Exception as e:
    #        log.debug(str(e))
    #        log.debug('Setting obj_center, desired_center, and center_quality '
    #                  'from object instantiation keywords')
    #        self.obj_center = obj_center
    #        self.desired_center = desired_center
    #        self.center_quality = center_quality

    @pgproperty
    def obj_center(self):
        """Center of object, (Y, X).  Quality of center determination must be
    set as well.  Base class uses out-of-bounds value (-99, -99) for
    center and 0 for center_quality

        Results are stored using the :class:`pgcoordproperty`
        decorator system.  See documentation of that class for
        explanation of features.

        """
        try:
            assert not self.recalculate, ('Recalculation requested')
            log.debug('Trying to read center info from FITS header,')
            cx = self.meta['OBJ_CR0']
            cy = self.meta['OBJ_CR1']
            obj_center = np.asarray((cy, cx))
        except Exception as e:
            log.debug(str(e))
            log.debug('Not setting center from FITS header,')
            obj_center = None
            self.center_quality = None
        return obj_center

    @pgproperty
    def desired_center(self):
        """Desired center of object (Y, X).  Base class uses center of
    data array.

        Results are stored using the :class:`pgcoordproperty`
        decorator system.  See documentation of that class for
        explanation of features.

        """
        try:
            assert not self.recalculate, ('Recalculation requested')
            log.debug('Trying to read desired center info from FITS header,')
            dx = self.meta['DES_CR0']
            dy = self.meta['DES_CR1']
            desired_center = np.asarray((dy, dx))
        except Exception as e:
            log.debug(str(e))
            log.debug('Not setting desired center from FITS header,')
            desired_center = None
        return desired_center

    @pgproperty
    def center_quality(self):
        """Quality of center determination.  center_quality should always be
        set in the obj_center setter

        """
        try:
            assert not self.recalculate, ('Recalculation requested')
            log.debug('Trying to read center_quality info from FITS header,')
            q = self.meta['CENTER_QUALITY']
            return int(q)
        except Exception as e:
            log.debug(str(e))
            log.debug('Not setting center_quality from FITS header,')
            # obj_center should set center_quality
            self.obj_center

    @center_quality.setter
    def center_quality(self, value):
        """Unset center_quality translates to 0 center_quality"""
        if value is not None:
            value = center_quality_checker(value)
        return value

#### In the end, I think CameraData is a bad idea, since this can all
#### go into and out of metadata.  Unit in principle could go there
#### too (e.g. BUNIT), however it is implemented as separate property
#### in the underlying CCDData/NDData.  No need to add that level of
#### complexity to these.  FITS card metadata property with comments
#### is good enough.
###
#### Although CameraData is intended to be for containing just camera
#### information, make it a subclass of FbuCCDData so that the raw_unit
#### can be properly inserted when CCD images are read in.
###
#### I envision either the camera file being use with a camera name and
#### method to initialize all of the propriety, or just a new class being
#### prepared that has the property hand-coded as class variables
###class CameraData(FbuCCDData):
###    raw_unit = u.adu
###    camera_data_file = None # some kind of file that would contain this info
###    camera_name = None
###    camera_description = None
###    full_naxis1 = None
###    full_naxis2 = None
###    satlevel = None
###    nonlinlevel = None
###    gain = None
###    readnoise = None
###
###    def __init__(self,
###                 *args,
###                 camera_data_file=None,
###                 camera_name=None,
###                 camera_description=None,
###                 raw_unit=None, # This will not affect the read classmethod
###                 full_naxis1=None,
###                 full_naxis2=None,
###                 satlevel=None,
###                 nonlinlevel=None,
###                 gain=None,
###                 readnoise=None,
###                 **kwargs):
###        self.camera_data_file = camera_data_file or self.camera_data_file
###        self.camera_name = camera_name or self.camera_name
###        self.camera_description = camera_description or self.camera_description
###        self.raw_unit = raw_unit or self.raw_unit
###        self.full_naxis1 = full_naxis1 or self.full_naxis1
###        self.full_naxis2 = full_naxis2 or self.full_naxis2
###        self.satlevel = satlevel or self.satlevel
###        self.nonlinlevel = nonlinlevel or self.nonlinlevel
###        self.gain = gain or self.gain
###        self.readnoise = readnoise or self.readnoise
###        super().__init__(*args, fallback_unit=self.raw_unit, **kwargs)
###
###    @classmethod
###    def read(cls, filename,
###             raw_unit=None,
###             **kwargs):
###        """Use ``raw_unit`` instead of ``fallback_unit``."""
###        raw_unit = raw_unit or cls.raw_unit
###        return super(CameraData, cls).read(filename,
###                                       fallback_unit=raw_unit,
###                                       **kwargs)
###
###    def _card_write(self):
###        """Write FITS header cards for camera
###
###        """
###        self.meta['GAIN'] = (self.gain, f'CCD charge gain '
###                             '{self.gain.unit.to_str()}')
###        self.meta['SATLEVEL'] = (self.satlevel, f'CCD saturation level '
###                                 '{self.satlevel.unit.to_str()}')
###        self.meta['NONLIN'] = (self.nonlinlevel, f'CCD nonlinearity '
###                               'level {self.nonlinlevel.unit.to_str()}')
###        self.meta['RDNOISE'] = (self.readnoise, f'CCD readnoise '
###                               'level {self.readnoise.unit.to_str()}')
###
###class SX694(CameraData):
###    camera_name = 'SX694'
###    camera_description = 'Starlight Xpress Trius SX694 mono, 2017 model version'
###    raw_unit = u.adu
###    # naxis1 = fastest changing axis in FITS primary image = X in
###    # Cartesian thought
###    # naxis1 = next to fastest changing axis in FITS primary image = Y in
###    # Cartesian thought
###    full_naxis1 = 2750*u.pix
###    full_naxis2 = 2200*u.pix
###    # 16-bit A/D converter
###    satlevel = (2**16-1)*raw_unit
###    nonlinlevel = (42000 - 1811)*raw_unit
###    # Gain measured in /data/io/IoIO/observing/Exposure_Time_Calcs.xlsx.
###    # Value agrees well with Trius SX-694 advertised value (note, newer
###    # "PRO" model has a different gain value).  Stored in GAIN keyword
###    gain = 0.3 * u.electron/raw_unit
###    # Sample readnoise measured as per ioio.notebk
###    # Tue Jul 10 12:13:33 2018 MCT jpmorgen@byted
###    # Readnoies is measured regularly as part of master bias creation and
###    # stored in the RDNOISE keyword.  This is used as a sanity check.
###    readnoise = 15.475665*raw_unit

class Ex(PGData):
    @pgcoordproperty
    def obj_center(self):
        # Pattern for all that want to read the FITS
        print('in obj_center')
        t = super().obj_center
        return t

class ExampleFITSReaderPGD(FITSReaderPGD):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        print('in init')

    @pgcoordproperty
    def obj_center(self):
        # Pattern for all that want to read the FITS
        t = FITSReaderPGD(self).obj_center
        if  t is not None:
            return t
        # Real calculation starts here
        obj_center = self.desired_center
        self.center_quality = 10
        return obj_center

    @pgcoordproperty
    def desired_center(self):
        # Pattern for all that want to read the FITS
        t = FITSReaderPGD(self).desired_center
        if t is not None:
            return t
        # Real calculation starts here
        npshape = np.asarray(self.shape)
        desired_center = npshape/2
        return desired_center

    @pgproperty
    def center_quality(self):
        # Pattern for all that want to read the FITS
        t = FITSReaderPGD(self).center_quality
        if t is not None:
            return t
        self.obj_center

class CenteredPGD(PGData):
    """Sets :prop:`obj_center` to :prop:`desired_center`"""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._old_desired_center = None

    @pgcoordproperty
    def obj_center(self):
        self.center_quality = 10
        return self.desired_center

    @pgcoordproperty
    def desired_center(self):
        return super().desired_center

    @desired_center.setter
    def desired_center(self, value):
        # Reset obj_center for proper recalculation
        del self.obj_center
        return value

class OffsetPGD(PGData):
    """Offsets :prop:`obj_center` by :prop:`center_offset`  Note:
    proper object that determins center needs to be inherited before
    this for sensible center: e.g.:
    class MyPGD(PGDOffset, PGDCentered): pass"""

    def __init__(self, *args,
                 center_offset=None,
                 **kwargs):
        super().__init__(*args, **kwargs)
        # Center offset is used in the calculation of a
        # pgcoordproperty such that we want to delete that
        # property for recalculation in the event center_offset
        # changes.  Particularly considering we end up in np.arrays
        # and the += operator increments the object numpy object
        # itself (e.g. pgd.center_offset += (10,10)), this is just a
        # little too complex for the pgcoordproperty system to
        # handle, so do it the olde fashionede waye

        if center_offset is None:
            center_offset = (0,0)
        self._old_center_offset = center_offset
        self.center_offset = center_offset

    @pgcoordproperty
    def _old_center_offset(self):
        pass

    @pgcoordproperty
    def center_offset(self):
        pass

    @center_offset.setter
    def center_offset(self, value):
        if value is None:
            value = (0,0)
        value = np.asarray(value)
        if not np.all(self._old_center_offset == value):
            del self.obj_center
            # This copy is key for avoiding subtle assign by reference bugs
            self._old_center_offset = value.copy()
            self.obj_center = None
        return value

    @pgcoordproperty
    def obj_center(self):
        return super().obj_center + self.center_offset

    def _card_write(self):
        """Writes FITS header cards in *binned* coordinates (i.e., as they
would likely presesnt in image analysis software showing the image)

        """
        # Note pythonic y, x coordinate ordering
        self.meta['OFF_CR0'] = (self.center_offset[1], 'Offset from '
                                'orig obj center X')
        self.meta['OFF_CR1'] = (self.center_offset[0], 'Offset from '
                                'orig obj center Y')
        super()._card_write()

class MaxPGD(PGData):
    @pgcoordproperty
    def obj_center(self):
        self.center_quality = 6
        obj_center = np.unravel_index(np.argmax(self), self.shape)
        return obj_center

# --> Still working on this.  Looks like I am going to need a
# readnoise property in the object if I really want to do this.  Maybe
# we need a readnoise_keyword or something like that....  Very camera
# specific

# --> this could be property in pgdata and use biweight_local
class BackgroundPGD(PGData):
    """Hold background.  Might need this to expand to hold other
    things.  Util is a bit too general a name, but that is sort of
    what I would envision"""

    @pgproperty
    def background(self):
        return np.median(self)

class CenterOfMassPGD(BackgroundPGD):
    """Use simple center-of-mass calculation to find center of
    brightest object.  Works best if pixels not likely to be part of
    the source are set to zero."""

    @pgcoordproperty
    def obj_center(self):
        #bsub = self.subtract(self.background, handle_meta='first_found')
        #return center_of_mass(bsub.data)
        bsub = self.data - self.background
        return center_of_mass(bsub)

class BrightestPGD(PGData):
    def __init__(self, *args,
                 seeing=2,
                 **kwargs):
        super().__init__(*args, **kwargs)
        self.seeing = seeing

    @pgcoordproperty
    def obj_center(self):
        # https://keflavich-astropy.readthedocs.io/en/convolve_fft_profiling/convolution/performance.html
        # suggests that astropy convolv is only needed when their are NaNs

        # Still working on this
        return super().obj_center + self.center_offset

class MaxImPGD(PGData):
    """Handles MaxIM DL :prop:`binning` and :prop:`subframe_origin`"""

    # It is important to just read these once, since we may modify
    # them in the object before write
    @pgcoordproperty
    def binning(self):
        """Image binning in Y,X order"""
        binning = (self.meta['YBINNING'],
                   self.meta['XBINNING'])
        binning = np.asarray(binning)
        return binning

    @pgcoordproperty
    def subframe_origin(self):
        """Subframe origin in *unbinned* pixels with full CCD origin = (0,0).  Y,X order"""
        subframe_origin = (self.meta['YORGSUBF'],
                           self.meta['XORGSUBF'])
        subframe_origin = np.asarray(subframe_origin)
        subframe_origin *= self.binning
        return subframe_origin

    def _card_write(self):
        """Write FITS card unique to MaxIMPGD"""
        # Note pythonic y, x coordinate ordering
        self.meta['XBINNING'] = self.binning[1]
        self.meta['YBINNING'] = self.binning[0]
        self.meta['XORGSUBF'] = self.subframe_origin[1]
        self.meta['YORGSUBF'] = self.subframe_origin[0]
        super()._card_write()

    def write(self, *args, **kwargs):
        self._card_write()
        super().write(*args, **kwargs)

if __name__ == "__main__":
    log.setLevel('DEBUG')
    #pgc = PGCenter()
    #pgc.obj_center = (1,1)
    #print(pgc.obj_center)
    ##pgc.obj_center = 1
    ###pgc = PGCenter((1,2), (3,4))
    #pgc = PGCenter([1,2], (3,4))
    #print(pgc.obj_center, pgc.desired_center)

    #pgd = PGData()
    fname = '/data/io/IoIO/raw/2020-07-15/HD87696-0016_Na_off.fit'

    class MyPGD(OffsetPGD, CenteredPGD, PGData):
        pass

    #pgd = PGData.read(fname)
    #print(pgd.obj_center, pgd.desired_center)
    #pgd.obj_center = (1,1)
    #print(pgd.obj_center, pgd.desired_center)
    #pgd = PGData.read(fname, obj_center=(2,2))
    #print(pgd.obj_center, pgd.desired_center)
    #pgd.desired_center = (1,1)
    #print(pgd.obj_center, pgd.desired_center)
    #del pgd.desired_center
    #print(pgd.obj_center, pgd.desired_center)
    #pgd.obj_center = None
    #print(pgd.obj_center, pgd.desired_center)
    #
    #pgd = CenteredPGD.read(fname)
    #print(pgd.obj_center, pgd.desired_center)

    #pgd = MyPGD.read(fname)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_offset)
    #pgd.center_offset += np.asarray((10, -10))
    #pgd.center_offset += (10, -10)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_offset)
    #pgd.center_offset = None
    #pgd.desired_center = (3,3)
    #pgd.center_offset += (10, -30)
    #pgd.center_offset = (10, -10)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_offset)
    #pgd.obj_center = (0,0)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_offset)
    #pgd.obj_center = None
    ##del pgd.obj_center
    ###pgd.calculated_center = (0,0)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_offset)
    #pgd.center_offset = pgd.center_offset + (10, -10)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_offset)
    #pgd.center_offset += (10, -30)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_offset)
    #pgd.write('/tmp/testfits.fits', overwrite=True)

    #class MyPGD(OffsetPGD, CenteredPGD, PGData):
    class MyPGD(OffsetPGD, CenteredPGD, MaxImPGD):#, SX694):
        pass
    #pgd = MyPGD.read(fname)
    #pgd.center_offset += (10, -30)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_offset)
    #pgd.write('/tmp/testfits.fits', overwrite=True)

    #pgd = MyPGD(pgd)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_offset)
    #pgd = MyPGD(pgd.data)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_offset)

    #pgd = Ex.read(fname)
    #print(pgd.obj_center, pgd.desired_center)

    #pgd = FITSReaderPGD.read('/tmp/testfits.fits')
    #print(pgd.obj_center, pgd.desired_center)
    #
    #pgd = ExampleFITSReaderPGD.read('/tmp/testfits.fits')
    #print(pgd.obj_center, pgd.desired_center, pgd.center_quality)
    #
    #pgd = ExampleFITSReaderPGD.read(fname)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_quality)

    #pgd = CenterOfMassPGD.read(fname)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_quality)
    #
    #off_filt = '/data/io/IoIO/raw/2018-01-28/R-band_off_ND_filter.fit'
    #pgd = CenterOfMassPGD.read(off_filt)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_quality)

    #pgd.obj_center = (1,1)
    #pgd.desired_center = (1,1)
    #print(pgd.obj_center, pgd.desired_center)
    #del pgd.desired_center
    #print(pgd.obj_center, pgd.desired_center)
    #pgd.center_offset = (20. -10)
    #print(pgd.obj_center, pgd.desired_center, pgd.center_offset)

    #log.setLevel('DEBUG')
    #pgd = MaxPGD.read(fname)
    #print(pgd.obj_center)
    #wname = '/tmp/test_kwd_write.fits'
    #pgd.write(wname, overwrite=True)
    #pgd = MaxPGD.read(wname)
    #print(pgd.obj_center, pgd.desired_center)
    #pgd = PGData.read(wname, recalculate=True)
    #print(pgd.obj_center, pgd.desired_center)
    #class MyPGD(OffsetPGD, MaxPGD):
    #    pass
    ##pgd = MyPGD.read(fname, center_offset=(-10,20))
    ##print(pgd.obj_center, pgd.desired_center, pgd.center_offset)
    #
    #bname = '/data/io/IoIO/reduced/Calibration/2020-07-07_ccdT_-10.3_bias_combined.fits'
    #bpgd = MyPGD.read(bname)
    #epgd = MyPGD.read(bname, unit='adu')
    #ccd = FbuCCDData.read(bname)

    #pgd = MyPGD.read(fname)

    #print(pgd.desired_center)

    #
    #ccd = CCDData.read(fname, unit='adu')
    #
    #rpgd = PGData(ccd.data, meta=ccd.meta)
    #print(pgd.desired_center)
    #
    #pgd = PGData.read(fname)
    #print(pgd.desired_center)
    #
    #pgd = PGData.read(fname, desired_center=(2,2))
    #print(pgd.desired_center)
    #
    #dspgd = PGData.read(fname, desired_center=(100,100))
    #print(dspgd.obj_center, dspgd.desired_center)
    #
    #dspgd.write('/tmp/test.fits', overwrite=True)
    #
    #pgdc = PGDCentered.read(fname)
    #print(pgdc.obj_center, pgdc.desired_center)
    #
    #pgdo = PGDOffset.read(fname)
    #print(pgdo.obj_center, pgdo.desired_center)
    #
    #pgdo = PGDOffset(pgd.data, meta=pgd.meta, center_offset=(20,10))
    #print(pgdo.obj_center, pgdo.desired_center)
    #
    #pgdo = PGDOffset.read(fname, center_offset=(20,10))
    #print(pgdo.obj_center, pgdo.desired_center)
    #
    #class MyPGD(PGDOffset, PGDCentered):
    #    pass
    #
    #mpgd = MyPGD.read(fname, center_offset=(20,10))
    #print(mpgd.obj_center, mpgd.desired_center)
    #
    #print('done')

    #bname = '/data/io/IoIO/reduced/Calibration/2020-07-07_ccdT_-10.3_bias_combined.fits'
    ##ccd = CCDData.read(bname)
    #
    #fname1 = '/data/Mercury/raw/2020-05-27/Mercury-0005_Na-on.fit'
    ##ccd = CCDData.read(fname1)
    #ccd = FbuCCDData.read(fname1, fallback_unit='adu')
    ##ccd = FbuCCDData.read(fname1, fallback_unit='aduu')
    #
    #ccd = FbuCCDData.read(fname1, unit='electron')

    fname1 = '/data/Mercury/raw/2020-05-27/Mercury-0005_Na-on.fit'
    pgd = CenterOfMassPGD.read(fname1)
    print(pgd.obj_center, pgd.desired_center)

fname1 = '/data/IoIO/raw/20210310/HD 132052-S001-R001-C002-R.fts'
pgd = MaxImPGD.read(fname1)
##print(pgd.meta)
#sub = pgd[0:100, 20:100]
#print(sub.subframe_origin)

#fname = '/data/IoIO/raw/2021-04_Astrometry/Main_Astrometry_East_of_Pier.fit'
#c = MaxImPGD.read(fname)
#print(c.binning)
#print(c.x_binned(6))
#print(c.x_binned(np.asarray((6, 12, 24, 48))))
#print(c.y_binned(6))
#print(c.y_binned(np.asarray((6, 12, 24, 48))))

max_stars_repo_name: jpmorgen/precisionguide

max_issues_repo_name: jpmorgen/precisionguide

max_forks_repo_name: jpmorgen/precisionguide

reason: import numpy,from scipy,from astropy


text:
# -*- coding: utf-8 -*-
from pathfinding.core.diagonal_movement import DiagonalMovement
from pathfinding.core.grid import Grid
from pathfinding.finder.a_star import AStarFinder
import numpy as np

BORDERLESS_GRID = """
xxx
xxx
"""

BORDER_GRID = """
+---+
|   |
|   |
+---+
"""

WALKED_GRID = """
+---+
|s# |
|xe |
+---+
"""

SIMPLE_MATRIX = [
  [1, 1, 1],
  [1, 0, 1],
  [1, 1, 1]
]

SIMPLE_WALKED = """
+---+
|sx |
| #x|
|  e|
+---+
"""

def test_str():
    """
    test printing the grid
    """
    grid = Grid(height=2, width=3)
    assert grid.grid_str(border=False, empty_chr='x') == BORDERLESS_GRID[1:-1]
    assert grid.grid_str(border=True) == BORDER_GRID[1:-1]
    grid.nodes[0][1].walkable = False
    start = grid.nodes[0][0]
    end = grid.nodes[1][1]
    path = [(0, 1)]
    assert grid.grid_str(path, start, end) == WALKED_GRID[1:-1]

def test_empty():
    """
    special test for empty values
    """
    matrix = ()
    grid = Grid(matrix=matrix)
    assert grid.grid_str() == '++\n||\n++'

    matrix = np.array(matrix)
    grid = Grid(matrix=matrix)
    assert grid.grid_str() == '++\n||\n++'

def test_numpy():
    """
    test grid from numpy array
    """
    matrix = np.array(SIMPLE_MATRIX)
    grid = Grid(matrix=matrix)

    start = grid.node(0, 0)
    end = grid.node(2, 2)

    finder = AStarFinder(diagonal_movement=DiagonalMovement.always)
    path, runs = finder.find_path(start, end, grid)

    assert grid.grid_str(path, start, end) == SIMPLE_WALKED[1:-1]

if __name__ == '__main__':
    test_str()

max_stars_repo_name: HSTEHSTEHSTE/python-pathfinding

max_issues_repo_name: HSTEHSTEHSTE/python-pathfinding

max_forks_repo_name: HSTEHSTEHSTE/python-pathfinding

reason: import numpy,from numpy


text:
""" Module for base classes for AND/OR trees and some tree utility code """
from __future__ import annotations
import random
import abc
from typing import TYPE_CHECKING

import networkx as nx

from aizynthfinder.reactiontree import ReactionTree, ReactionTreeLoader
from aizynthfinder.chem import UniqueMolecule

if TYPE_CHECKING:
    from aizynthfinder.context.config import Configuration
    from aizynthfinder.context.stock import Stock
    from aizynthfinder.chem import FixedRetroReaction
    from aizynthfinder.utils.type_utils import StrDict, Any, List, Union, Optional

class TreeNodeMixin:
    """ A mixin class for node in a tree """

    @property
    def prop(self) -> StrDict:
        """ Dictionary with publicly exposed properties """
        return {}

    @property
    def children(self) -> List["TreeNodeMixin"]:
        """ List of children nodes """
        return []

class AndOrSearchTreeBase(abc.ABC):
    """ A base class for a search tree based on an AND/OR structure """

    def __init__(self, config: Configuration, root_smiles: str = None) -> None:
        self.config = config
        self._root_smiles = root_smiles

    @property
    def mol_nodes(self) -> List[TreeNodeMixin]:
        """ Return the molecule nodes of the tree """
        return []

    @abc.abstractmethod
    def one_iteration(self) -> bool:  # pylint: disable=no-self-use
        """ Perform one iteration of the search """
        return False

    @abc.abstractmethod
    def routes(self) -> List[ReactionTree]:  # pylint: disable=no-self-use
        """ Return the routes of the tree """
        return []

class SplitAndOrTree:
    """
    Encapsulation of an algorithm to split an AND/OR tree into separate routes

    This is a modified algorithm of the one detailed in the CompRet paper:
    Shibukawa et al. (2020) J. Cheminf. 12, 52

    This implementation sets an upper-bound on the number of extracted routes
    to avoid combinatorial explosion.

    The routes are extracted on instantiation and the routes can be access from
    the `routes` attribute.

    :param root_node: the root of the AND/OR tree
    :param stock: the stock of the search
    :param max_routes: the maximum number of routes to extract
    """

    def __init__(
        self, root_node: TreeNodeMixin, stock: Stock, max_routes: int = 25000
    ) -> None:
        self._traces: List[_AndOrTrace] = []
        self._black_list: List[TreeNodeMixin] = []
        graph = _AndOrTrace(root_node)
        self._samples = {child: 0 for child in root_node.children}
        if root_node.children:
            self._sampling_cutoff = max_routes / len(root_node.children)
        else:
            self._sampling_cutoff = max_routes
        self._partition_search_tree(graph, root_node)
        self.routes = [
            ReactionTreeFromAndOrTrace(trace, stock).tree for trace in self._traces
        ]

    def _partition_search_tree(self, graph: _AndOrTrace, node: TreeNodeMixin) -> None:
        # fmt: off
        if self._sampling_cutoff and len(graph) > 1 and self._samples[graph.first_reaction] > self._sampling_cutoff:
            return
        # fmt: on

        if not node.children:
            self._traces.append(graph)

        children_to_search = [
            child for child in node.children if child not in self._black_list
        ]
        if not children_to_search:
            for child in node.children:
                self._black_list.remove(child)
            return

        graph_copy = graph.copy()

        child_node = self._select_child_node(children_to_search)
        graph.add_edge(node, child_node)
        for grandchild in child_node.children:
            graph.add_edge(child_node, grandchild)
        self._black_list.append(child_node)

        leaves = [node for node in graph if not graph[node] and node.children]
        if not leaves:
            self._traces.append(graph)
            self._samples[graph.first_reaction] += 1
        else:
            self._partition_search_tree(graph, leaves[0])

        self._partition_search_tree(graph_copy, node)

    def _select_child_node(self, children: List[TreeNodeMixin]) -> TreeNodeMixin:
        # This is what makes this algorithm different from what was
        # detailed in the CompRet paper
        if not self._sampling_cutoff:
            return children[0]

        solved_children = [
            child for child in children if child.prop.get("solved", False)
        ]
        if solved_children:
            return random.choice(solved_children)
        return random.choice(children)

class _AndOrTrace(nx.DiGraph):
    """Helper class for the SplitAndOrTree class."""

    def __init__(self, root: TreeNodeMixin = None) -> None:
        super().__init__()
        self.root = root
        self._first_reaction: Optional[TreeNodeMixin] = None
        self._reaction_tree: Optional[Any] = None
        if root:
            self.add_node(root)

    @property
    def first_reaction(self) -> TreeNodeMixin:
        """ Return the first reaction or raise an exception """
        assert self._first_reaction is not None
        return self._first_reaction

    def add_edge(
        self, u_of_edge: TreeNodeMixin, v_of_edge: TreeNodeMixin, **attr: Any
    ) -> None:
        if u_of_edge is self.root:
            if self._first_reaction is not None:
                raise ValueError(
                    "Re-defining trace. Trying to set first reaction twice."
                )
            self._first_reaction = v_of_edge
        super().add_edge(u_of_edge, v_of_edge, **attr)

    def copy(self, as_view: bool = False) -> "_AndOrTrace":
        other = super().copy(as_view)
        assert isinstance(other, _AndOrTrace)
        other.root = self.root
        other._first_reaction = self._first_reaction  # pylint: disable=protected-access
        return other

class ReactionTreeFromAndOrTrace(ReactionTreeLoader):
    """Creates a reaction tree object from an AND/OR Trace"""

    def _load(self, andor_trace: nx.DiGraph, stock: Stock) -> None:  # type: ignore
        """
        :param andor_trace: the trace graph
        :param stock: stock object
        """
        self._stock = stock
        self._trace_graph = andor_trace
        self._trace_root = self._find_root()

        self._add_node(
            self._unique_mol(self._trace_root.prop["mol"]),
            depth=0,
            transform=0,
            in_stock=self._trace_root.prop["mol"] in self._stock,
        )
        for node1, node2 in andor_trace.edges():
            rt_node1 = self._make_rt_node(node1)
            rt_node2 = self._make_rt_node(node2)
            self.tree.graph.add_edge(rt_node1, rt_node2)

    def _add_node_with_depth(
        self, node: Union[UniqueMolecule, FixedRetroReaction], base_node: TreeNodeMixin
    ) -> None:
        if node in self.tree.graph:
            return

        depth = nx.shortest_path_length(self._trace_graph, self._trace_root, base_node)
        if isinstance(node, UniqueMolecule):
            self._add_node(
                node, depth=depth, transform=depth // 2, in_stock=node in self._stock
            )
        else:
            self._add_node(node, depth=depth)

    def _find_root(self) -> TreeNodeMixin:
        for node, degree in self._trace_graph.in_degree():  # type: ignore
            if degree == 0:
                return node
        raise ValueError("Could not find root!")

    def _make_rt_node(
        self, node: TreeNodeMixin
    ) -> Union[UniqueMolecule, FixedRetroReaction]:
        if "mol" in node.prop:
            unique_obj = self._unique_mol(node.prop["mol"])
            self._add_node_with_depth(unique_obj, node)
            return unique_obj

        reaction_obj = self._unique_reaction(node.prop["reaction"])
        reaction_obj.reactants = (
            tuple(self._unique_mol(child.prop["mol"]) for child in node.children),
        )
        self._add_node_with_depth(reaction_obj, node)
        return reaction_obj

max_stars_repo_name: cambiegroup/aizynthfinder

max_issues_repo_name: cambiegroup/aizynthfinder

max_forks_repo_name: cambiegroup/aizynthfinder

reason: import networkx


text:
import numpy as np
import torch
from torch.autograd import Variable
import torch.nn.functional as F
import torch.optim as optim
from tqdm import tqdm

from relnet.agent.baseline.baseline_agent import BaselineAgent
from relnet.agent.supervised.sl_net import SLNet
from relnet.agent.pytorch_agent import PyTorchAgent
from relnet.utils.config_utils import get_device_placement

class SLAgent(PyTorchAgent, BaselineAgent):
    algorithm_name = "sl"
    is_deterministic = False
    is_trainable = True

    def __init__(self, environment):
        super().__init__(environment)
        self.obj_fun_scale_multiplier = 1

    def setup(self, options, hyperparams):
        super().setup(options, hyperparams)
        # overriding provided options
        self.validation_check_interval = 500
        self.max_validation_consecutive_steps = 10000

        self.setup_net()

    def train(self, train_g_list, validation_g_list, max_steps, **kwargs):
        self.setup_graphs(train_g_list, validation_g_list)
        self.setup_sample_idxes(len(train_g_list))

        self.setup_histories_file()
        self.setup_training_parameters(max_steps)

        pbar = tqdm(range(max_steps + 1), unit='steps', disable=None)
        optimizer = optim.Adam(self.net.parameters(), lr=self.learning_rate)
        for self.step in pbar:
            self.logger.warn("starting validation loss check...")
            self.check_validation_loss(self.step, max_steps)
            self.logger.warn("completed validation loss check...")

            true_values, pred_values = self.run_training_batch()
            true_values = Variable(true_values.view(-1, 1))

            loss = F.mse_loss(Variable(true_values), pred_values)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            pbar.set_description('loss: %0.5f' % ( loss))
            should_stop = self.check_stopping_condition(self.step, max_steps)
            if should_stop:
                break

    def setup_net(self):
        self.net = SLNet(self.hyperparams, s2v_module=None)
        if get_device_placement() == 'GPU':
            self.net = self.net.cuda()
        if self.restore_model:
            self.restore_model_from_checkpoint()

    def setup_training_parameters(self, max_steps):
        self.learning_rate = self.hyperparams['learning_rate']

    def finalize(self):
        pass

    def pick_actions_using_strategy(self, t, i):
        g = self.environment.g_list[i]

        non_edges = list(self.environment.get_graph_non_edges(i))

        if len(non_edges) == 0:
            return (-1, -1)

        next_g_batch = []
        for first, second in non_edges:
            g_copy = g.copy()
            next_g, _ = g_copy.add_edge(first, second)
            next_g_batch.append(next_g)

        argmax_val_index, vals = self.net(next_g_batch, return_argmax=True)
        return non_edges[argmax_val_index]

    def run_training_batch(self):
        selected_idx = self.advance_pos_and_sample_indices()
        batch_graph = [self.train_g_list[idx] for idx in selected_idx]
        batch_true_obj_values = torch.Tensor([self.train_initial_obj_values[idx] for idx in selected_idx])
        batch_true_obj_values = torch.mul(batch_true_obj_values, self.obj_fun_scale_multiplier)
        if get_device_placement() == 'GPU':
            batch_true_obj_values = batch_true_obj_values.cuda()

        _, batch_pred_values = self.net(batch_graph)
        return batch_true_obj_values, batch_pred_values

    def post_env_setup(self):
        pass

    def get_default_hyperparameters(self):
        hyperparams = {'learning_rate': 0.0001,
                       'latent_dim': 64,
                       'hidden': 128,
                       'embedding_method': 'mean_field',
                       'max_lv': 3}
        return hyperparams

max_stars_repo_name: VictorDarvariu/graph-construction-rl

max_issues_repo_name: VictorDarvariu/graph-construction-rl

max_forks_repo_name: VictorDarvariu/graph-construction-rl


text:
'''
Copyright (c) 2018-2019 Daniel Fedai Larsen
LICENSE: See end of file for license information.
'''

from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np

REGULAR_SAMPLE = False
VARYING_Z = False
COSINE_WEIGHTED = False
COSINE_WEIGHTED_BLUE_NOISE = False
COSINE_WEIGHTED_GOLDEN_RATIO = False
FIBONACCI_SPIRAL = True
numOcclusionSteps = 7;
totalOcclusionSamples = numOcclusionSteps * numOcclusionSteps;

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
u = np.linspace(0, 2 * np.pi, 100)
v = np.linspace(0, 0.5 * np.pi, 100)
x = np.outer(np.cos(u), np.sin(v))
y = np.outer(np.sin(u), np.sin(v))
z = np.outer(np.ones(np.size(u)), np.cos(v))
ax.plot_surface(x, y, z, color='w', alpha=0.4)

if REGULAR_SAMPLE:
	points_x = []
	points_y = []
	points_z = []
	for i in range(0, totalOcclusionSamples + 1):
		u0 = float(i) / float(numOcclusionSteps)
		u1 = float(i) / float(totalOcclusionSamples)
		phi = 2.0 * np.pi * u0;
		theta = np.arccos(u1);
		points_x.append(np.sin(theta) * np.cos(phi))
		points_y.append(np.sin(theta) * np.sin(phi))
		points_z.append(u1)
	ax.scatter(points_x, points_y, points_z, marker=".")

if VARYING_Z:
	points_x = []
	points_y = []
	points_z = []
	for i in range(0, totalOcclusionSamples + 1):
		u0 = float(i) / float(numOcclusionSteps)
		u1 = float(i) / float(totalOcclusionSamples)
		phi = 2 * np.pi * u0;
		theta = np.arccos(u1);
		x = np.sin(theta) * np.cos(phi)
		y = np.sin(theta) * np.sin(phi)
		z = u1 + (np.sin(phi * 2 * np.pi) * 0.1)
		length = x**2 + y**2 + z**2
		x = x / length
		y = y / length
		z = z / length
		points_x.append(x)
		points_y.append(y)
		points_z.append(z)
	ax.scatter(points_x, points_y, points_z, marker=".")

if COSINE_WEIGHTED:
	points_x = []
	points_y = []
	points_z = []
	for y in range(0,numOcclusionSteps+1):
		for x in range(0,numOcclusionSteps+1):
			idx = y * numOcclusionSteps + x;
			u0 = float(idx) / float(numOcclusionSteps)
			u1 = float(idx) / float(totalOcclusionSamples)
			phi = 2.0 * np.pi * u0
			theta = np.sqrt(u1)
			points_x.append(theta * np.cos(phi))
			points_y.append(theta * np.sin(phi))
			points_z.append(np.sqrt(1.0 - u1))
	ax.scatter(points_x, points_y, points_z, marker=".", color='b')

if COSINE_WEIGHTED_BLUE_NOISE:
	us = [[0.281250, 0.893555],
		[0.125000, 0.616211],
		[0.726562, 0.806641],
		[0.378906, 0.380859],
		[0.367188, 0.957031],
		[0.572266, 0.280273],
		[0.772461, 0.512695],
		[0.080078, 0.168945],
		[0.105469, 0.861328],
		[0.375977, 0.178711],
		[0.668945, 0.005859],
		[0.587891, 0.540039],
		[0.397461, 0.661133],
		[0.756836, 0.230469],
		[0.533203, 0.814453],
		[0.949219, 0.063477],
		[0.078125, 0.399414],
		[0.244141, 0.454102],
		[0.848633, 0.140625],
		[0.895508, 0.789062],
		[0.228516, 0.147461],
		[0.963867, 0.529297],
		[0.115234, 0.025391],
		[0.238281, 0.778320],
		[0.739258, 0.673828],
		[0.808594, 0.928711],
		[0.748047, 0.377930],
		[0.977539, 0.923828],
		[0.368164, 0.805664],
		[0.501953, 0.122070],
		[0.548828, 0.964844],
		[0.597656, 0.666016],
		[0.473633, 0.458984],
		[0.130859, 0.734375],
		[0.179688, 0.312500],
		[0.366211, 0.500977],
		[0.945312, 0.653320],
		[0.229492, 0.945312],
		[0.248047, 0.626953],
		[0.979492, 0.253906],
		[0.329102, 0.072266],
		[0.637695, 0.124023],
		[0.474609, 0.735352],
		[0.623047, 0.404297],
		[0.464844, 0.302734],
		[0.287109, 0.240234],
		[0.835938, 0.635742],
		[0.006836, 0.731445],
		[0.085938, 0.518555],
		[0.766602, 0.057617],
		[0.973633, 0.833984],
		[0.481445, 0.889648],
		[0.468750, 0.028320],
		[0.025391, 0.006836],
		[0.300781, 0.884766],
		[0.083984, 0.275391],
		[0.488281, 0.611328],
		[0.198242, 0.555664],
		[0.567383, 0.188477],
		[0.670898, 0.566406],
		[0.955078, 0.424805],
		[0.876953, 0.234375],
		[0.228516, 0.041992],
		[0.735352, 0.141602]]
	points_x = []
	points_y = []
	points_z = []
	for y in range(0,numOcclusionSteps+1):
		for x in range(0,numOcclusionSteps+1):
			idx = y * (numOcclusionSteps+1) + x;
			u0 = us[idx][0]
			u1 = us[idx][1]
			phi = 2.0 * np.pi * u0
			theta = np.sqrt(u1)
			points_x.append(theta * np.cos(phi))
			points_y.append(theta * np.sin(phi))
			points_z.append(np.sqrt(1.0 - u1))
	ax.scatter(points_x, points_y, points_z, marker=".", color='g')

if COSINE_WEIGHTED_GOLDEN_RATIO:
	us = [[0.253906, 0.069336],
		[0.008789, 0.639648],
		[0.763672, 0.208984],
		[0.518555, 0.779297],
		[0.273438, 0.348633],
		[0.028320, 0.918945],
		[0.783203, 0.488281],
		[0.538086, 0.058594],
		[0.292969, 0.627930],
		[0.047852, 0.198242],
		[0.802734, 0.767578],
		[0.557617, 0.337891],
		[0.312500, 0.907227],
		[0.067383, 0.477539],
		[0.822266, 0.046875],
		[0.577148, 0.617188],
		[0.332031, 0.186523],
		[0.086914, 0.756836],
		[0.841797, 0.326172],
		[0.596680, 0.896484],
		[0.351562, 0.465820],
		[0.106445, 0.036133],
		[0.861328, 0.605469],
		[0.616211, 0.175781],
		[0.371094, 0.745117],
		[0.125977, 0.315430],
		[0.880859, 0.884766],
		[0.635742, 0.455078],
		[0.390625, 0.024414],
		[0.145508, 0.594727],
		[0.900391, 0.165039],
		[0.655273, 0.734375],
		[0.410156, 0.304688],
		[0.165039, 0.874023],
		[0.919922, 0.444336],
		[0.674805, 0.013672],
		[0.429688, 0.583984],
		[0.184570, 0.153320],
		[0.939453, 0.723633],
		[0.694336, 0.292969],
		[0.449219, 0.863281],
		[0.204102, 0.432617],
		[0.958984, 0.002930],
		[0.713867, 0.572266],
		[0.468750, 0.142578],
		[0.223633, 0.711914],
		[0.978516, 0.282227],
		[0.733398, 0.851562],
		[0.488281, 0.421875],
		[0.243164, 0.991211],
		[0.998047, 0.561523],
		[0.752930, 0.130859],
		[0.507812, 0.701172],
		[0.262695, 0.270508],
		[0.017578, 0.840820],
		[0.772461, 0.410156],
		[0.527344, 0.980469],
		[0.282227, 0.549805],
		[0.037109, 0.120117],
		[0.791992, 0.689453],
		[0.546875, 0.259766],
		[0.301758, 0.830078],
		[0.056641, 0.399414],
		[0.811523, 0.969727]]
	points_x = []
	points_y = []
	points_z = []
	for y in range(0,numOcclusionSteps+1):
		for x in range(0,numOcclusionSteps+1):
			idx = y * (numOcclusionSteps+1) + x;
			u0 = us[idx][0]
			u1 = us[idx][1]
			phi = 2.0 * np.pi * u0
			theta = np.sqrt(u1)
			points_x.append(theta * np.cos(phi))
			points_y.append(theta * np.sin(phi))
			points_z.append(np.sqrt(1.0 - u1))
	ax.scatter(points_x, points_y, points_z, marker=".", color='g')

if FIBONACCI_SPIRAL:
	us = [[0.088853, 2.399963],
		[0.154306, 4.799926],
		[0.199744, 7.199889],
		[0.236984, 9.599853],
		[0.269456, 11.999816],
		[0.298726, 14.399778],
		[0.325667, 16.799742],
		[0.350824, 19.199705],
		[0.374564, 21.599669],
		[0.397148, 23.999632],
		[0.418771, 26.399595],
		[0.439582, 28.799557],
		[0.459699, 31.199520],
		[0.479216, 33.599483],
		[0.498213, 35.999447],
		[0.516752, 38.399410],
		[0.534891, 40.799374],
		[0.552676, 43.199337],
		[0.570149, 45.599300],
		[0.587346, 47.999264],
		[0.604299, 50.399227],
		[0.621037, 52.799191],
		[0.637585, 55.199154],
		[0.653969, 57.599113],
		[0.670209, 59.999077],
		[0.686326, 62.399040],
		[0.702339, 64.799004],
		[0.718268, 67.198967],
		[0.734127, 69.598930],
		[0.749935, 71.998894],
		[0.765708, 74.398857],
		[0.781461, 76.798820],
		[0.797210, 79.198784],
		[0.812971, 81.598747],
		[0.828760, 83.998711],
		[0.844591, 86.398674],
		[0.860483, 88.798637],
		[0.876452, 91.198601],
		[0.892515, 93.598564],
		[0.908691, 95.998528],
		[0.925000, 98.398491],
		[0.941463, 100.798454],
		[0.958103, 103.198418],
		[0.974945, 105.598381],
		[0.992016, 107.998344],
		[1.009347, 110.398308],
		[1.026971, 112.798271],
		[1.044927, 115.198227],
		[1.063261, 117.598190],
		[1.082021, 119.998154],
		[1.101269, 122.398117],
		[1.121075, 124.798080],
		[1.141526, 127.198044],
		[1.162727, 129.598007],
		[1.184810, 131.997971],
		[1.207944, 134.397934],
		[1.232354, 136.797897],
		[1.258348, 139.197861],
		[1.286370, 141.597824],
		[1.317101, 143.997787],
		[1.351690, 146.397751],
		[1.392380, 148.797714],
		[1.444973, 151.197678],
		[1.570796, 153.597641]]
	points_x = []
	points_y = []
	points_z = []
	golden_angle = np.pi * (3.0 - np.sqrt(5.0))
	for y in range(0,numOcclusionSteps+1):
		for x in range(0,numOcclusionSteps+1):
			idx = y * (numOcclusionSteps+1) + x
			sin_theta = np.sin(us[idx][0])
			cos_theta = np.cos(us[idx][0])
			phi = golden_angle * (idx + 1.0)
			cos_phi = np.cos(us[idx][1])
			sin_phi = np.sin(us[idx][1])
			points_x.append(cos_phi * sin_theta)
			points_y.append(sin_phi * sin_theta)
			points_z.append(cos_theta)
	ax.scatter(points_x, points_y, points_z, marker=".", color='g')

plt.show()

'''
MIT License

Copyright (c) 2018-2019 Daniel Fedai Larsen

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''


text:
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import emoji

##### EDA STEPS #####
# Column names and definition

# Column datatypes and possible values

# Sample (head) of the data

# Statistical summaries
## Number of categorical and continuous features
## Number of dtypes: DONE

# Categorical Data
## Missing values: DONE
## Relative frequencies
## Correlations

# Continuous Data
## Missing values
## Histograms for distribution
## Means, standard deviation, medians, etc
## Correlations

# Text Data
## Pre-processing
### Convert emojis into text
### Remove tagged users "@"
### Remove URLs
## wordclouds
## text length
## word counts
## top unigrams (individual words)
### before and after removing words
## top bigrams
### before and after removing words
## Topic Modeling
### TF-IDF
    '''
    pass
    '''
    def __init__(self, data, categorical=None, continuous=None, text=None):
        '''
        data: a pandas dataframes with the data types already set to be the correct ones
        categorical: a list columns that are categorical
        continuous: a list of columns that are continuous
        '''
        self.data = data
        self.dtypes = ['O', 'float', 'int', 'datetime64[ns]', 'bool', 'other']
        self.categorical = categorical
        self.continuous = continuous
        self.text = text
        self._util()

    def _util(self):
        self.num_categorical = len(self.categorical) if self.categorical != None else 0
        self.num_continuous = len(self.continuous) if self.continuous != None else 0
        self.num_text = len(self.text) if self.text != None else 0

    def _remove_url(self):
        pattern=r'(?i)\b((?:[a-z][\w-]+:(?:/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’]))'
        for col_i in self.text:
            self.data[col_i] = self.data[col_i].str.replace(pattern,'')

    def _replace_emojis(self):
        for col_i in self.text:
            self.data[col_i] = self.data[col_i].apply(emoji.demojize, delimiters=("", " "))

    def _remove_tags(self):
        pattern = r"\@\w+[,]|\@\w+|[,]\@\w+"
        for col_i in self.text:
            self.data[col_i] = self.data[col_i].str.replace(pattern,'')

    def len_of_text(self, column):
        if column not in self.text:
            raise Exception("No Text columns to analyze, please add them to text")

        return self.data[column].apply(lambda x: len(x))

    def number_of_words(self,column):
        if column not in self.text:
            raise Exception("No Text columns to analyze, please add them to text")

        return self.data[column].str.count('\w+')

    def _calc_number_of_dtypes(self):
        unique_dtypes = set(self.data.dtypes.to_dict().values())
        dtypes_lookup = {}
        for i in unique_dtypes:
            dtype_bool = False
            for j in self.dtypes:
                if i==j:
                    dtypes_lookup[i] = j
                    dtype_bool=True
                    break
            if not dtype_bool:
                dtypes_lookup[i] = 'other'

        self.num_dtypes = {key:0 for key in self.dtypes}
        for key, value in self.data.dtypes.to_dict().items():
            # Look up value and add it
            self.num_dtypes[dtypes_lookup[value]] += 1

    def head(self, *args, **kwargs):
        # Show head of data, pass arguements through
        return self.data.head(*args, **kwargs)

    def _calculate_number_of_missing_values(self, percentage=False, plot=False, **kwargs):
        if percentage:
            results = self.data.isna().sum()/self.data.shape[0]
        else:
            results = self.data.isna().sum()

        if plot:
            results = results.plot.bar(**kwargs)

        return results

    def _plot_categories_pct_of_total(self, column_to_groupby:str, column_to_agg:str, column_to_agg_value_order: list, figsize= (10,8)):
        '''
        TODO: write docstrings
        '''
        figure = (self.data.groupby(column_to_groupby)[column_to_agg]
        .value_counts(normalize = True)
        .mul(100)
        .rename('percent')
        .reset_index())

        fig, ax = plt.subplots(figsize = figsize)

        sns.barplot(x=column_to_groupby,
                    y="percent",
                    hue=column_to_agg,
                    data= figure,
                    ax = ax,
                    order = column_to_agg_value_order,
                    hue_order = column_to_agg_value_order)

        ax.xaxis.set_tick_params(rotation=90)
        ax.legend(title = column_to_agg, loc = 'best')
        plt.show()

max_stars_repo_name: FeardaMeow/Data-Science-Templates

max_issues_repo_name: FeardaMeow/Data-Science-Templates

max_forks_repo_name: FeardaMeow/Data-Science-Templates