text:
# Copyright 2020 Makani Technologies LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for the overrides_util library."""

import unittest

from makani.config import overrides_util
from makani.lib.python import c_helpers
from makani.sim import sim_types
import numpy

class OverridesUtilTest(unittest.TestCase):

  def testSimOpt(self):
    overrides_struct = {
        'sim': {
            'sim_opt': ['kSimOptImperfectSensors', 1]
        }
    }
    overrides = overrides_util.PreprocessOverrides(overrides_struct)
    self.assertIn('sim', overrides)
    self.assertIn('sim_opt', overrides['sim'])
    self.assertEqual(1 | sim_types.kSimOptImperfectSensors,
                     overrides['sim']['sim_opt'])

  def testFaultsSim(self):
    faults_in = [{
        't_start': 100.0,
        't_end': 200.0,
        'component': 'Pitot/actual_P_dyn',
        'type': 'kSimFaultMeasurementRescale',
        'parameters': [1.0, 2.0, 3.0],
    }, {
        't_start': 100.0,
        'component': 'Pitot/actual_P_dyn',
        'type': 'kSimFaultMeasurementRescale',
    }]

    overrides_struct = {'sim': {'faults_sim': faults_in}}
    overrides = overrides_util.PreprocessOverrides(overrides_struct)

    # Check the overall structure of the created fields.
    self.assertTrue('sim' in overrides and 'faults_sim' in overrides['sim']
                    and 'num_fault_events' in overrides['sim']['faults_sim']
                    and 'fault_events' in overrides['sim']['faults_sim'])
    self.assertEqual(2, overrides['sim']['faults_sim']['num_fault_events'])

    faults_out = overrides['sim']['faults_sim']['fault_events']
    self.assertEqual(sim_types.MAX_FAULT_EVENTS, len(faults_out))

    # Check the fault events that we specified.
    for (fault_in, fault_out) in zip(faults_in, faults_out[:len(faults_in)]):
      self.assertEqual(fault_in['component'], fault_out['component'])
      self.assertEqual(sim_types.kSimFaultMeasurementRescale,
                       fault_out['type'])

      if 'parameters' in fault_in:
        params_in = fault_in['parameters']
        params_out = fault_out['parameters']
        self.assertEqual(sim_types.MAX_FAULT_EVENT_PARAMETERS,
                         len(params_out))
        self.assertEqual(params_in, params_out[:len(params_in)])
      else:
        self.assertEqual([0.0] * sim_types.MAX_FAULT_EVENT_PARAMETERS,
                         fault_out['parameters'])

    # Check basic reasonableness of the unused faults.
    for fault in faults_out[len(faults_in):]:
      self.assertEqual(sim_types.kSimFaultNoFault, fault['type'])
      self.assertEqual(sim_types.MAX_FAULT_EVENT_PARAMETERS,
                       len(fault['parameters']))

  def testWingSimXg0(self):
    # Test Cartesian coordinates.
    with self.assertRaises(overrides_util.InvalidArgument):
      overrides_util.PreprocessOverrides({
          'sim': {'wing_sim': {'Xg_0': [22.0, 22.0]}}
      })

    with self.assertRaises(overrides_util.InvalidArgument):
      overrides_util.PreprocessOverrides({
          'sim': {'wing_sim': {'Xg_0': [22.0, 22.0, 22.0, 22.0]}}
      })

    overrides = overrides_util.PreprocessOverrides({
        'sim': {'wing_sim': {'Xg_0': [22.0, 22.0, 22.0]}}
    })
    self.assertTrue('sim' in overrides
                    and 'wing_sim' in overrides['sim']
                    and 'Xg_0' in overrides['sim']['wing_sim'])
    self.assertEqual([22.0, 22.0, 22.0],
                     overrides['sim']['wing_sim']['Xg_0'])

    # Test spherical coordinates.
    with self.assertRaises(overrides_util.InvalidArgument):
      overrides_util.PreprocessOverrides({
          'sim': {'wing_sim': {'Xg_0': {
              'ele': 22.0, 'azi': 0.0, 'r': 0.0, 'bad': 0.0
          }}}})

    with self.assertRaises(overrides_util.InvalidArgument):
      overrides_util.PreprocessOverrides({
          'sim': {'wing_sim': {'Xg_0': {
              'ele': 22.0, 'azi': 0.0
          }}}})

    overrides = overrides_util.PreprocessOverrides({
        'sim': {'wing_sim': {'Xg_0': {
            'ele': numpy.pi/3, 'azi': 0.0, 'r': 1.0
        }}}})
    self.assertTrue('sim' in overrides
                    and 'wing_sim' in overrides['sim']
                    and 'Xg_0' in overrides['sim']['wing_sim'])
    expected = [-0.5, 0.0, -numpy.sqrt(3.0)/2.0]
    for i in range(3):
      self.assertAlmostEqual(expected[i],
                             overrides['sim']['wing_sim']['Xg_0'][i])

  def testBadEnumValues(self):
    # Test overriding the flight_plan.
    with self.assertRaises(c_helpers.EnumError):
      overrides_util.PreprocessOverrides({
          'system': {'flight_plan': 'kFlightPlanEit'}
      })

    # Test a name that doesn't start with FLIGHT_PLAN.
    with self.assertRaises(c_helpers.EnumError):
      overrides_util.PreprocessOverrides({
          'system': {'flight_plan': 'kSimOptPerch'}
      })

    # Test a name that doesn't exist.
    with self.assertRaises(c_helpers.EnumError):
      overrides_util.PreprocessOverrides({'sim': {'sim_opt': ['kSimOptEit']}})

    # Test a name that doesn't start with SIM_OPT.
    with self.assertRaises(c_helpers.EnumError):
      overrides_util.PreprocessOverrides({
          'sim': {'sim_opt': ['kFlightPlanTurnKey']}
      })

    # Test a name that doesn't exist.
    with self.assertRaises(c_helpers.EnumError):
      overrides_util.PreprocessOverrides({
          'sim': {'faults_sim': [{'type': 'kSimFaultUpdateEit'}]}
      })

    # Test a name that doesn't start with SIM_FAULT.
    with self.assertRaises(c_helpers.EnumError):
      overrides_util.PreprocessOverrides({
          'sim': {'faults_sim': [{'type': 'kSimOptPerch'}]}
      })

    # Test a name that doesn't exist.
    with self.assertRaises(c_helpers.EnumError):
      overrides_util.PreprocessOverrides({
          'sim': {'joystick_sim': {
              'updates': [{'type': 'kSimJoystickUpdateEit'}]
          }}
      })

    # Test a name that doesn't start with SIM_JOYSTICK_UPDATE.
    with self.assertRaises(c_helpers.EnumError):
      overrides_util.PreprocessOverrides({
          'sim': {'joystick_sim': {
              'updates': [{'type': 'kSimJoystickTypeProgrammed'}]
          }}
      })

  def testFaultsSimError(self):
    overrides_in = {
        'sim': {
            'faults_sim': [{
                't_start': 100.0,
                'component': 'Pitot/actual_P_dyn',
                'type': 'kSimFaultMeasurementRescale',
            }] * (1 + sim_types.MAX_FAULT_EVENTS)
        }
    }
    with self.assertRaises(overrides_util.InvalidArrayLength):
      overrides_util.PreprocessOverrides(overrides_in)

    overrides_in = {
        'sim': {
            'faults_sim': [{
                't_start': 100.0,
                'component': 'Pitot/actual_P_dyn',
                'type': 'kSimFaultMeasurementRescale',
                'parameters': [0.0] * (1 + sim_types.MAX_FAULT_EVENT_PARAMETERS)
            }]
        }
    }
    with self.assertRaises(overrides_util.InvalidArrayLength):
      overrides_util.PreprocessOverrides(overrides_in)

  def testJoystickSimJoystickType(self):
    overrides_struct = {'sim': {'joystick_sim': {'joystick_type': 'Hardware'}}}
    overrides = overrides_util.PreprocessOverrides(overrides_struct)

    self.assertTrue('sim' in overrides and 'joystick_sim' in overrides['sim']
                    and 'joystick_type' in overrides['sim']['joystick_sim'])
    self.assertEqual(sim_types.kSimJoystickTypeHardware,
                     overrides['sim']['joystick_sim']['joystick_type'])

  def testJoystickSimUpdates(self):
    updates_in = [{
        't_update': 100.0,
        'type': 'kSimJoystickUpdateSwitchMiddle'
    }, {
        't_update': 100.0,
        'type': 'kSimJoystickUpdateThrottle',
        'enum_value': 'kSimJoystickThrottleManual',
        'value': 22.0
    }, {
        't_update': 200.0,
        'type': 'kSimJoystickUpdateThrottle',
        'enum_value': 'kSimJoystickThrottleCrosswindNormal'
    }, {
        't_update': 200.0,
        'type': 'kSimJoystickUpdateThrottle',
        'enum_value': 'kSimJoystickThrottleCrosswindNormal',
        'value': -1.0
    }, {
        't_update': 200.0,
        'type': 'kSimJoystickUpdateSwitchMiddle',
        'enum_value': 0
    }]

    overrides_struct = {'sim': {'joystick_sim': {'updates': updates_in}}}
    overrides = overrides_util.PreprocessOverrides(overrides_struct)

    self.assertTrue('sim' in overrides and 'joystick_sim' in overrides['sim']
                    and 'num_updates' in overrides['sim']['joystick_sim']
                    and 'updates' in overrides['sim']['joystick_sim'])
    self.assertEqual(5, overrides['sim']['joystick_sim']['num_updates'])
    self.assertEqual(sim_types.MAX_JOYSTICK_UPDATES,
                     len(overrides['sim']['joystick_sim']['updates']))

    self.assertEqual(0.0,
                     overrides['sim']['joystick_sim']['updates'][0]['value'])

    for update in overrides['sim']['joystick_sim']['updates'][len(updates_in):]:
      self.assertEqual(sim_types.kSimJoystickUpdateNone, update['type'])

  def testJoystickSimError(self):
    overrides_in = {
        'sim': {
            'joystick_sim': {
                'updates': [{
                    't_update': 100.0,
                    'type': 'kSimJoystickUpdateSwitchMiddle'
                }] * (1 + sim_types.MAX_JOYSTICK_UPDATES)
            }
        }
    }
    with self.assertRaises(overrides_util.InvalidArrayLength):
      overrides_util.PreprocessOverrides(overrides_in)

  def testWindModel(self):
    overrides_in = {
        'sim': {
            'phys_sim': {
                'wind_model': sim_types.kWindModelDrydenTurbulence,
            },
        },
    }
    overrides_out = overrides_util.PreprocessOverrides(overrides_in)
    self.assertEqual(sim_types.kWindModelDrydenTurbulence,
                     overrides_out['sim']['phys_sim']['wind_model'])

    overrides_in = {
        'sim': {
            'phys_sim': {
                'wind_model': 'DrydenTurbulence',
            },
        },
    }
    overrides_out = overrides_util.PreprocessOverrides(overrides_in)
    self.assertEqual(sim_types.kWindModelDrydenTurbulence,
                     overrides_out['sim']['phys_sim']['wind_model'])

    overrides_in = {
        'sim': {
            'phys_sim': {
                'wind_model': 'NotAWindModel',
            },
        },
    }
    with self.assertRaises(c_helpers.EnumError):
      overrides_util.PreprocessOverrides(overrides_in)

  def testWindSpeedUpdates(self):
    updates_in = [{
        't_update': 20.0,
        'offset': -5.0
    }, {
        't_update': 40.0,
        'offset': 10.0
    }]

    overrides_struct = {'sim': {'phys_sim': {'wind_speed_update': updates_in}}}
    overrides = overrides_util.PreprocessOverrides(overrides_struct)

    self.assertTrue(
        'num_updates' in overrides['sim']['phys_sim']['wind_speed_update'] and
        'offsets' in overrides['sim']['phys_sim']['wind_speed_update'])
    self.assertEqual(
        2, overrides['sim']['phys_sim']['wind_speed_update']['num_updates'])
    self.assertEqual(
        sim_types.MAX_WIND_SPEED_UPDATES,
        len(overrides['sim']['phys_sim']['wind_speed_update']['offsets']))

if __name__ == '__main__':
  unittest.main()


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import sys
import tensorflow.contrib.slim as slim
from functools import partial
from glob import glob

import cv2
import data
import image_utils as im
import models
import numpy as np
import tensorflow as tf
import utils
from utils import gradient_penalty
import importer_tar

def train():
    epoch = 200
    batch_size = 1
    lr = 0.0002
    crop_size = 128
    load_size = 128
    tar_db_a = "cameron_images.tgz"
    tar_db_b = "teresa_images.tgz"
    db_a_i = importer_tar.Importer(tar_db_a)
    db_b_i = importer_tar.Importer(tar_db_b)
    image_a_names = db_a_i.get_sorted_image_name()
    image_b_names = db_b_i.get_sorted_image_name()
    train_a_size = int(len(image_a_names) * 0.8)
    train_b_size = int(len(image_b_names) * 0.8)

    image_a_train_names = image_a_names[0:train_a_size]
    image_b_train_names = image_b_names[0:train_b_size]

    image_a_test_names = image_a_names[train_a_size:]
    image_b_test_names = image_b_names[train_b_size:]

    print("A train size:{},test size:{}".format(len(image_a_train_names),len(image_a_test_names)))
    print("B train size:{},test size:{}".format(len(image_b_train_names),len(image_b_test_names)))

    """ graph """
    # models
    generator_a2b = partial(models.generator, scope='a2b')
    generator_b2a = partial(models.generator, scope='b2a')
    discriminator_a = partial(models.discriminator, scope='a')
    discriminator_b = partial(models.discriminator, scope='b')

    # operations
    a_real_in = tf.placeholder(tf.float32, shape=[None, load_size, load_size, 3], name="a_real")
    b_real_in = tf.placeholder(tf.float32, shape=[None, load_size, load_size, 3], name="b_real")
    a_real = utils.preprocess_image(a_real_in, crop_size=crop_size)
    b_real = utils.preprocess_image(b_real_in, crop_size=crop_size)

    a2b = generator_a2b(a_real)
    b2a = generator_b2a(b_real)
    b2a2b = generator_a2b(b2a)
    a2b2a = generator_b2a(a2b)

    a_logit = discriminator_a(a_real)
    b2a_logit = discriminator_a(b2a)
    b_logit = discriminator_b(b_real)
    a2b_logit = discriminator_b(a2b)

    # losses
    g_loss_a2b = -tf.reduce_mean(a2b_logit)
    g_loss_b2a = -tf.reduce_mean(b2a_logit)

    cyc_loss_a = tf.losses.absolute_difference(a_real, a2b2a)
    cyc_loss_b = tf.losses.absolute_difference(b_real, b2a2b)
    g_loss = g_loss_a2b + g_loss_b2a + cyc_loss_a * 10.0 + cyc_loss_b * 10.0

    wd_a = tf.reduce_mean(a_logit) - tf.reduce_mean(b2a_logit)
    wd_b = tf.reduce_mean(b_logit) - tf.reduce_mean(a2b_logit)

    gp_a = gradient_penalty(a_real, b2a, discriminator_a)
    gp_b = gradient_penalty(b_real, a2b, discriminator_b)

    d_loss_a = -wd_a + 10.0 * gp_a
    d_loss_b = -wd_b + 10.0 * gp_b

    # summaries
    utils.summary({g_loss_a2b: 'g_loss_a2b',
                               g_loss_b2a: 'g_loss_b2a',
                               cyc_loss_a: 'cyc_loss_a',
                               cyc_loss_b: 'cyc_loss_b'})
    utils.summary({d_loss_a: 'd_loss_a'})
    utils.summary({d_loss_b: 'd_loss_b'})
    for variable in slim.get_model_variables():
        tf.summary.histogram(variable.op.name, variable)
    merged = tf.summary.merge_all()

    im1_op = tf.summary.image("real_a",a_real_in)
    im2_op = tf.summary.image("a2b",a2b)
    im3_op = tf.summary.image("b2a2b",b2a2b)
    im4_op = tf.summary.image("real_b",b_real_in)
    im5_op = tf.summary.image("b2a",b2a)
    im6_op = tf.summary.image("b2a2b",b2a2b)

    # optim
    t_var = tf.trainable_variables()
    d_a_var = [var for var in t_var if 'a_discriminator' in var.name]
    d_b_var = [var for var in t_var if 'b_discriminator' in var.name]
    g_var = [var for var in t_var if 'a2b_generator' in var.name or 'b2a_generator' in var.name]

    d_a_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(d_loss_a, var_list=d_a_var)
    d_b_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(d_loss_b, var_list=d_b_var)
    g_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(g_loss, var_list=g_var)

    """ train """
    ''' init '''
    # session
    config = tf.ConfigProto(allow_soft_placement=True)
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)

    # counter
    it_cnt, update_cnt = utils.counter()

    ''' summary '''
    summary_writer = tf.summary.FileWriter('./outputs/summaries/', sess.graph)

    ''' saver '''
    saver = tf.train.Saver(max_to_keep=5)

    ''' restore '''
    ckpt_dir = './outputs/checkpoints/'
    utils.mkdir(ckpt_dir)
    try:
        utils.load_checkpoint(ckpt_dir, sess)
    except:
        sess.run(tf.global_variables_initializer())

    '''train'''
    try:
        batch_epoch = min(train_a_size,train_b_size) // batch_size
        max_it = epoch * batch_epoch
        for it in range(sess.run(it_cnt), max_it):
            sess.run(update_cnt)
            epoch = it // batch_epoch
            it_epoch = it % batch_epoch + 1

            # read data
            a_real_np = cv2.resize(db_a_i.get_image(
                    image_a_train_names[it_epoch % batch_epoch]),(load_size,load_size))
            b_real_np = cv2.resize(db_b_i.get_image(
                    image_b_train_names[it_epoch % batch_epoch]),(load_size,load_size))

            # train G
            sess.run(g_train_op,
                     feed_dict={a_real_in: [a_real_np], b_real_in: [b_real_np]})

            # train discriminator
            sess.run([d_a_train_op, d_b_train_op],
                         feed_dict={a_real_in: [a_real_np],
                                    b_real_in: [b_real_np]})

            # display
            if it % 100 == 0:
                # make summary
                summary  = sess.run(merged,
                        feed_dict={a_real_in: [a_real_np],
                        b_real_in: [b_real_np]})
                summary_writer.add_summary(summary, it)
                print("Epoch: (%3d) (%5d/%5d)" % (epoch, it_epoch, batch_epoch))

            # save
            if (it + 1) % 1000 == 0:
                save_path = saver.save(sess, '{}/epoch_{}_{}.ckpt'.format(ckpt_dir, epoch, it_epoch))
                print('###Model saved in file: {}'.format(save_path))

            # sample
            if (it + 1) % 1000 == 0:
                a_test_index = int(np.random.uniform(high=len(image_a_test_names)))
                b_test_index = int(np.random.uniform(high=len(image_b_test_names)))
                a_real_np = cv2.resize(db_a_i.get_image(image_a_test_names[a_test_index]),(load_size,load_size))
                b_real_np = cv2.resize(db_b_i.get_image(image_b_test_names[b_test_index]),(load_size,load_size))

                [a_opt, a2b_opt, a2b2a_opt, b_opt, b2a_opt, b2a2b_opt] = sess.run([a_real, a2b, a2b2a, b_real, b2a, b2a2b],
                    feed_dict={a_real_in: [a_real_np], b_real_in: [b_real_np]})

                sample_opt = np.concatenate((a_opt, a2b_opt, a2b2a_opt, b_opt, b2a_opt, b2a2b_opt), axis=0)
                [im1_sum,im2_sum,im3_sum,im4_sum,im5_sum,im6_sum] = \
                    sess.run([im1_op,im2_op,im3_op,im4_op,im5_op,im6_op],
                             feed_dict={a_real_in: [a_real_np], b_real_in: [b_real_np]})

                summary_writer.add_summary(im1_sum, it)
                summary_writer.add_summary(im2_sum, it)
                summary_writer.add_summary(im3_sum, it)
                summary_writer.add_summary(im4_sum, it)
                summary_writer.add_summary(im5_sum, it)
                summary_writer.add_summary(im6_sum, it)

                save_dir = './outputs/sample_images_while_training/'
                utils.mkdir(save_dir)
                im.imwrite(im.immerge(sample_opt, 2, 3),
                           '{}/epoch_{}_it_{}.jpg'.
                           format(save_dir, epoch, it_epoch))
    except:
        raise

    finally:
        save_path = saver.save(sess, '{}/epoch_{}_{}.ckpt'.format(ckpt_dir, epoch, it_epoch))
        print('###Model saved in file: {}'.format(save_path))
        sess.close()

if __name__ == "__main__":
    train()

ext: py

reason: import numpy


text:
"""
Mcts implementation modified from
https://github.com/brilee/python_uct/blob/master/numpy_impl.py
"""
import collections
import math

import numpy as np

class Node:
    def __init__(self, action, obs, done, reward, state, mcts, parent=None):
        self.env = parent.env
        self.action = action  # Action used to go to this state

        self.is_expanded = False
        self.parent = parent
        self.children = {}

        self.action_space_size = self.env.action_space.n
        self.child_total_value = np.zeros(
            [self.action_space_size], dtype=np.float32)  # Q
        self.child_priors = np.zeros(
            [self.action_space_size], dtype=np.float32)  # P
        self.child_number_visits = np.zeros(
            [self.action_space_size], dtype=np.float32)  # N
        self.valid_actions = obs["action_mask"].astype(np.bool)

        self.reward = reward
        self.done = done
        self.state = state
        self.obs = obs

        self.mcts = mcts

    @property
    def number_visits(self):
        return self.parent.child_number_visits[self.action]

    @number_visits.setter
    def number_visits(self, value):
        self.parent.child_number_visits[self.action] = value

    @property
    def total_value(self):
        return self.parent.child_total_value[self.action]

    @total_value.setter
    def total_value(self, value):
        self.parent.child_total_value[self.action] = value

    def child_Q(self):
        # TODO (weak todo) add "softmax" version of the Q-value
        return self.child_total_value / (1 + self.child_number_visits)

    def child_U(self):
        return math.sqrt(self.number_visits) * self.child_priors / (
            1 + self.child_number_visits)

    def best_action(self):
        """
        :return: action
        """
        child_score = self.child_Q() + self.mcts.c_puct * self.child_U()
        masked_child_score = child_score
        masked_child_score[~self.valid_actions] = -np.inf
        return np.argmax(masked_child_score)

    def select(self):
        current_node = self
        while current_node.is_expanded:
            best_action = current_node.best_action()
            current_node = current_node.get_child(best_action)
        return current_node

    def expand(self, child_priors):
        self.is_expanded = True
        self.child_priors = child_priors

    def get_child(self, action):
        if action not in self.children:
            self.env.set_state(self.state)
            obs, reward, done, _ = self.env.step(action)
            next_state = self.env.get_state()
            self.children[action] = Node(
                state=next_state,
                action=action,
                parent=self,
                reward=reward,
                done=done,
                obs=obs,
                mcts=self.mcts)
        return self.children[action]

    def backup(self, value):
        current = self
        while current.parent is not None:
            current.number_visits += 1
            current.total_value += value
            current = current.parent

class RootParentNode(object):
    def __init__(self, env):
        self.parent = None
        self.child_total_value = collections.defaultdict(float)
        self.child_number_visits = collections.defaultdict(float)
        self.env = env

class MCTS:
    def __init__(self, model, mcts_param):
        self.model = model
        self.temperature = mcts_param["temperature"]
        self.dir_epsilon = mcts_param["dirichlet_epsilon"]
        self.dir_noise = mcts_param["dirichlet_noise"]
        self.num_sims = mcts_param["num_simulations"]
        self.exploit = mcts_param["argmax_tree_policy"]
        self.add_dirichlet_noise = mcts_param["add_dirichlet_noise"]
        self.c_puct = mcts_param["puct_coefficient"]

    def compute_action(self, node):
        for _ in range(self.num_sims):
            leaf = node.select()
            if leaf.done:
                value = leaf.reward
            else:
                child_priors, value = self.model.compute_priors_and_value(
                    leaf.obs)
                if self.add_dirichlet_noise:
                    child_priors = (1 - self.dir_epsilon) * child_priors
                    child_priors += self.dir_epsilon * np.random.dirichlet(
                        [self.dir_noise] * child_priors.size)

                leaf.expand(child_priors)
            leaf.backup(value)

        # Tree policy target (TPT)
        tree_policy = node.child_number_visits / node.number_visits
        tree_policy = tree_policy / np.max(
            tree_policy)  # to avoid overflows when computing softmax
        tree_policy = np.power(tree_policy, self.temperature)
        tree_policy = tree_policy / np.sum(tree_policy)
        if self.exploit:
            # if exploit then choose action that has the maximum
            # tree policy probability
            action = np.argmax(tree_policy)
        else:
            # otherwise sample an action according to tree policy probabilities
            action = np.random.choice(
                np.arange(node.action_space_size), p=tree_policy)
        return tree_policy, action, node.children[action]


text:
"""
Scattering efficiency of a sphere
=================================

PyMieSim makes it easy to create a source and a scatterer. With these objects
defined, it is possible to use PyMieSim to find the scattering efficiency of the
scatterer. This feature can be used to plot a graph of the scattering efficiency
of a sphere as a function of the permittivity and the size parameter.
"""

# sphinx_gallery_thumbnail_path = '../images/ScatteringEfficiency.png'

def run(Plot, Save):
    # import PyMieSim
    from PyMieSim.Source import PlaneWave
    from PyMieSim.Scatterer import Sphere

    # import other librairies
    import math
    import numpy as np
    import matplotlib.pyplot as plt
    from mpl_toolkits.axes_grid1 import make_axes_locatable
    import matplotlib

    # set the source using PyMieSim
    Source = PlaneWave(Wavelength   = 450e-9,
                       Polarization = 0,
                       Amplitude    = 1)
    # create an empty list that will contain the heatmap values
    heatmap = []

    # loop through all the values to generate the heatmap
    # first, loop through the diameter values
    for i in range(1,517):
        x = []
        # second, loop through the index values
        for j in range(-100,501):
            if j == 0:
                continue

            Scat = Sphere(Diameter    = i/3 * 1e-9,
                          Source      = Source,
                          Index       = (j/10)**0.5) # square root since permittivity
                                                     # is the index squared
            # Get the scattering efficiency using GetProperties()
            prop = Scat.GetProperties()
            Qsca = prop[1]
            x.append(Qsca)
        # append list (row) to the heatmap (list of lists)
        heatmap.append(x)

    # convert the heatmap to a numpy array
    data = np.array(heatmap)

    # create the plot
    fig, ax = plt.subplots()
    # show the data and adjust the color scale
    im = ax.imshow(data, norm=matplotlib.colors.LogNorm(vmin = 0.1, vmax = 10), cmap='viridis')

    # graph title
    ax.set_title("Scattering efficiency of a sphere")

    # x axis settings
    ax.set_xlabel("Permittivity")
    ax.set_xticks(np.linspace(0, len(heatmap[0]), 7))
    ax.set_xticklabels([-10, 0, 10, 20, 30, 40, 50])

    # y axis settings
    ax.set_ylabel("Size parameter")
    ax.invert_yaxis()
    ax.set_yticks(np.linspace(0, len(heatmap), 7))
    ax.set_yticklabels([0, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2])

    # colorbar settings
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.05)
    plt.colorbar(im, cax=cax)

    # display the plot in a tight layout
    fig.tight_layout()
    plt.show()

if __name__ == '__main__':
    run(Plot=True, Save=False)

max_stars_repo_name: MartinPdS/PyMieSim

max_issues_repo_name: MartinPdS/PyMieSim

max_forks_repo_name: MartinPdS/PyMieSim


text:
from __future__ import unicode_literals

from IPython.display import HTML
from pandas.core.indexing import _non_reducing_slice
import pandas as pd

if pd.__version__ >= '0.18.1':
    from pandas.formats.style import Styler
else:
    from pandas.core.style import Styler

import numpy as np

from collections import namedtuple
from itertools import product
from functools import partial
import warnings

from .formatters import as_percent, as_money, as_unit, as_currency, LOCALE_OBJ

def apply_pretty_globals():
    """Apply global CSS to make dataframes pretty.

    This function injects HTML and CSS code into the notebook in order to make
    tables look pretty. Third party hosts of notebooks advise against using
    this and some don't support it. As long as you are okay with HTML injection
    in your notebook, go ahead and use this. Otherwise use the ``PrettyPandas``
    class.
    """

    return HTML("""
        <style type='text/css'>
            /* Pretty Pandas Dataframes */
            .dataframe * {border-color: #c0c0c0 !important;}
            .dataframe th{background: #eee;}
            .dataframe td{
                background: #fff;
                text-align: right;
                min-width:5em;
            }

            /* Format summary rows */
            .dataframe-summary-row tr:last-child,
            .dataframe-summary-col td:last-child{
                background: #eee;
                font-weight: 500;
            }
        </style>
        """)

Formatter = namedtuple("Formatter", "subset, function")

class PrettyPandas(Styler):
    """Pretty pandas dataframe Styles.

    Parameters
    ----------
    :param data: Series or DataFrame
    :param precision: int
        precision to round floats to, defaults to pd.options.display.precision
    :param table_styles: list-like, default None
        list of {selector: (attr, value)} dicts. These values overwrite the
        default style.
    :param uuid: str, default None
        a unique identifier to avoid CSS collisons; generated automatically
    :param caption: str, default None
        caption to attach to the table
    :param summary_rows:
        list of single-row dataframes to be appended as a summary
    :param summary_cols:
        list of single-row dataframes to be appended as a summary
    """

    #: Default colour for header backgrounds
    DEFAULT_BACKGROUND = "#eee"

    #: Default color for table borders
    DEFAULT_BORDER_COLOUR = '#c0c0c0'

    #: CSS style for header rows and column.
    HEADER_PROPERTIES = [('background', DEFAULT_BACKGROUND),
                         ('font-weight', '500')]

    #: CSS style for summary content cells.
    SUMMARY_PROPERTIES = HEADER_PROPERTIES

    #: Base styles
    STYLES = [
        {'selector': 'th', 'props': HEADER_PROPERTIES},
        {'selector': 'td', 'props': [('text-align', 'right'),
                                     ('min-width', '3em')]},
        {'selector': '*', 'props': [('border-color', DEFAULT_BORDER_COLOUR)]},
    ]

    #: Default local for formatting functions
    DEFAULT_LOCALE = LOCALE_OBJ

    def __init__(self,
                 data,
                 summary_rows=None,
                 summary_cols=None,
                 formatters=None,
                 *args,
                 **kwargs):

        kwargs['table_styles'] = self.STYLES + kwargs.get('table_styles', [])

        self.summary_rows = summary_rows or []
        self.summary_cols = summary_cols or []
        self.formatters = formatters or []

        return super(self.__class__, self).__init__(data, *args, **kwargs)

    @classmethod
    def set_locale(cls, locale):
        """Set the PrettyPandas default locale."""
        cls.DEFAULT_LOCALE = locale

    def _append_selector(self, selector, *props):
        """Add a CSS selector and style to this Styler."""
        self.table_styles.append({'selector': selector, 'props': props})

    def summary(self, func=np.sum, title='Total', axis=0, **kwargs):
        """Add multiple summary rows or columns to the dataframe.

        Parameters
        ----------
        :param func: Iterable of functions to be used for a summary.
        :param titles: Iterable of titles in the same order as the functions.
        :param axis:
            Same as numpy and pandas axis argument. A value of None will cause
            the summary to be applied to both rows and columns.
        :param kwargs: Keyword arguments passed to all the functions.

        The results of summary can be chained together.
        """
        return self.multi_summary([func], [title], axis, **kwargs)

    def multi_summary(self, funcs, titles, axis=0, **kwargs):
        """Add multiple summary rows or columns to the dataframe.

        Parameters
        ----------
        :param funcs: Iterable of functions to be used for a summary.
        :param titles: Iterable of titles in the same order as the functions.
        :param axis:
            Same as numpy and pandas axis argument. A value of None will cause
            the summary to be applied to both rows and columns.
        :param kwargs: Keyword arguments passed to all the functions.
        """
        if axis is None:
            return self.multi_summary(funcs, titles, axis=0, **kwargs)\
                       .multi_summary(funcs, titles, axis=1, **kwargs)

        output = [self.data.apply(f, axis=axis, **kwargs).to_frame(t)
                  for f, t in zip(funcs, titles)]

        if axis == 0:
            self.summary_rows += [row.T for row in output]
        elif axis == 1:
            self.summary_cols += output
        else:
            ValueError("Invalid axis selected. Can only use 0, 1, or None.")

        return self

    def total(self, title="Total", **kwargs):
        """Add a total summary to this table.

        :param title: Title to be displayed.
        :param kwargs: Keyword arguments passed to ``numpy.sum``.
        """
        return self.summary(np.sum, title, **kwargs)

    def average(self, title="Average", **kwargs):
        """Add a mean summary to this table.

        :param title: Title to be displayed.
        :param kwargs: Keyword arguments passed to ``numpy.mean``.
        """
        return self.summary(np.mean, title, **kwargs)

    def median(self, title="Median", **kwargs):
        """Add a median summary to this table.

        :param title: Title to be displayed.
        :param kwargs: Keyword arguments passed to ``numpy.median``.
        """
        return self.summary(np.median, title, **kwargs)

    def max(self, title="Maximum", **kwargs):
        """Add a maximum summary to this table.

        :param title: Title to be displayed.
        :param kwargs: Keyword arguments passed to ``numpy.max``.
        """
        return self.summary(np.max, title, **kwargs)

    def min(self, title="Minimum", **kwargs):
        """Add a minimum summary to this table.

        :param title: Title to be displayed.
        :param kwargs: Keyword arguments passed to ``numpy.min``.
        """
        return self.summary(np.min, title, **kwargs)

    def as_percent(self, subset=None, precision=None, locale=None):
        """Represent subset of dataframe as percentages.

        Parameters:
        -----------
        :param subset: Pandas slice to convert to percentages
        :param precision: int
            Number of decimal places to round to
        :param locale: Locale to be used (e.g. 'en_US')
        """
        # TODO: Find good way to implement precision

        add_formatter = partial(self._format_cells,
                                as_percent,
                                subset=subset)
        if locale is not None:
            return add_formatter(locale=locale)
        else:
            return add_formatter(locale=self.DEFAULT_LOCALE)

    def as_currency(self, subset=None, currency='USD', locale=None):
        """Represent subset of dataframe as currency.

        Parameters:
        -----------
        :param subset: Pandas slice to convert to percentages
        :param currency: Currency or currency symbol to be used
        :param locale: Locale to be used (e.g. 'en_US')
        """
        add_formatter = partial(self._format_cells,
                                as_currency,
                                subset=subset,
                                currency=currency)

        if locale is not None:
            return add_formatter(locale=locale)
        else:
            return add_formatter(locale=self.DEFAULT_LOCALE)

    def as_unit(self, unit, subset=None, precision=None, location='prefix'):
        """Represent subset of dataframe as a special unit.

        Parameters:
        -----------
        :param unit: string representing unit to be used.
        :param subset: Pandas slice to convert to percentages
        :param precision: int
            Number of decimal places to round to
        :param location: 'prefix' or 'suffix' indicating where the currency symbol
            should be.
        """
        precision = self.precision if precision is None else precision

        return self._format_cells(as_unit,
                                  subset=subset,
                                  precision=precision,
                                  unit=unit,
                                  location=location)

    def as_money(self,
                 subset=None,
                 precision=None,
                 currency='$',
                 location='prefix'):
        """[DEPRECATED] Represent subset of dataframe as currency.

        Parameters:
        -----------
        :param precision: int
            Number of decimal places to round to
        :param subset: Pandas slice to convert to percentages
        :param currency: Currency string
        :param location: 'prefix' or 'suffix' indicating where the currency
            symbol should be.
        """
        with warnings.catch_warnings():
            warnings.simplefilter("always")
            warnings.warn("`as_money` is depricated in favour of "
                          "`as_currency`.",
                          DeprecationWarning)

        precision = self.precision if precision is None else precision

        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            return self._format_cells(as_money,
                                      currency=currency,
                                      precision=precision,
                                      subset=subset,
                                      location=location)

    def _format_cells(self, func, subset=None, **kwargs):
        """Add formatting function to cells."""

        # Create function closure for formatting operation
        def fn(*args):
            return func(*args, **kwargs)

        self.formatters.append(Formatter(subset=subset, function=fn))
        return self

    def _apply_formatters(self):
        """Apply all added formatting."""
        for subset, function in self.formatters:
            if subset is None:
                subset = self.data.index
            else:
                subset = _non_reducing_slice(subset)
            self.data.loc[subset] = self.data.loc[subset].applymap(function)
        return self

    def _apply_summaries(self):
        """Add all summary rows and columns."""
        colnames = list(self.data.columns)
        summary_colnames = [series.columns[0] for series in self.summary_cols]
        summary_rownames = [series.index[0] for series in self.summary_rows]

        rows, cols = self.data.shape
        ix_rows = self.data.index.size
        ix_cols = len(self.data.index.names)

        # Add summary rows and columns
        self.data = pd.concat([self.data] + self.summary_cols,
                              axis=1,
                              ignore_index=False)
        self.data = pd.concat([self.data] + self.summary_rows,
                              axis=0,
                              ignore_index=False)

        # Update CSS styles
        for i, _ in enumerate(self.summary_rows):
            index = rows + i + 1
            self._append_selector('tr:nth-child({})'.format(index),
                                  *self.SUMMARY_PROPERTIES)

        for i, _ in enumerate(self.summary_cols):
            index = cols + ix_cols + i + 1
            self._append_selector('td:nth-child({})'.format(index),
                                  *self.SUMMARY_PROPERTIES)

        # Sort column names
        self.data = self.data[colnames + summary_colnames]

        # Fix shared summary cells to be empty
        for row, col in product(summary_rownames, summary_colnames):
            self.data.loc[row, col] = ''

        return self

    def _translate(self):
        """Apply styles and formats before rendering."""
        data = self.data.copy()

        self._apply_summaries()
        self._apply_formatters()
        result = super(self.__class__, self)._translate()

        # Revert changes to inner data
        self.data = data
        return result

max_stars_repo_name: nnamon/PrettyPandas

max_issues_repo_name: nnamon/PrettyPandas

max_forks_repo_name: nnamon/PrettyPandas


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from enum import Enum
import matplotlib.pyplot as plt
import numpy as np
from gurobipy import *

# The implementation of the solver using the dynamic programming method.
# In fact, it does not use the bottom-up method normally required  by the dynamic
# programming methods, but rather a backward recursion using memoization to avoid
# recomputing the same values again and again.
class DynamicSolver:
    # Used to determine which trace we're computing
    class For(Enum):
        LINE = 0,
        COLUMN = 1

    # A simple enumeration to determine the color of the cells
    class Color(Enum):
        UNDEFINED = 0,
        WHITE = 1,
        BLACK = 2

    def __init__(self):
        self.line_validity_trace = []
        self.column_validity_trace = []
        self.unload_instance()

    def unload_instance(self):
        self.instance = [[]]
        self.lSeq = []
        self.cSeq = []

    # Load an instance from a text file and do the necessary setup
    def load_instance(self, filename):
        f = open(filename, "r")

        isLineSeq = True
        for line in f:
            if line[:-1] == "#":
                isLineSeq = False
                continue
            if line[-1] == '\n':
                seq = [int(x) for x in line[:-1].split(" ") if x != ""]
            else:
                seq = [int(x) for x in line.split(" ") if x != ""]
            if isLineSeq:
                self.lSeq.append(seq)
            else:
                self.cSeq.append(seq)

        self.current_line = -1
        self.current_column = -1

        self.instance = [[self.Color.UNDEFINED] * len(self.cSeq) for i in range(len(self.lSeq))]

        f.close()

    # A function used during the developpement to check the validity of the solution
    # using different online solver, most of which use this particular format.
    def get_instance_as_JSON(self):
        return "{\"hor\":" + str(self.cSeq) + ",\"ver\":" + str(self.lSeq) + "}"

    # Compute the validity trace, the T(j, l).
    def compute_validity_trace(self, i, j, l, t=For.LINE):
        if t == self.For.LINE:
            if self.current_line != i:
                self.current_line = i
                self.line_validity_trace = [[None] * len(self.cSeq) for i in range(len(self.lSeq[i]) + 1)]
            if self.line_validity_trace[l][j] is None:
                self.line_validity_trace[l][j] = self.compute_validity_trace_line(i, j, l)

            return self.line_validity_trace[l][j]
        else:
            if self.current_column != j:
                self.current_column = j
                self.column_validity_trace = [[None] * len(self.lSeq) for i in range(len(self.cSeq[j]) + 1)]
            if self.column_validity_trace[l][i] is None:
                self.column_validity_trace[l][i] = self.compute_validity_trace_column(i, j, l)

            return self.column_validity_trace[l][i]

    # Auxilliary function usd to compute the validity trace for a line
    def compute_validity_trace_line(self, i, j, l):
        if l == 0:
            if j == 0:
                return self.instance[i][j] != self.Color.BLACK
            return self.instance[i][j] != self.Color.BLACK and self.compute_validity_trace(i, j - 1, 0)

        sl = self.lSeq[i][l - 1]
        if j < sl - 1:
            return False
        if self.instance[i][j] == self.Color.WHITE:
            if j - 1 >= 0:
                return self.compute_validity_trace(i, j - 1, l, self.For.LINE)
            else:
                return False
        else:
            if self.instance[i][j] == self.Color.UNDEFINED and self.compute_validity_trace(i, j - 1, l, self.For.LINE):
                return True
            found_white_cell = False
            for j_prime in range(j - sl + 1, j + 1):
                if self.instance[i][j_prime] == self.Color.WHITE:
                    found_white_cell = True
                    break
            if found_white_cell:
                return False

        if j - sl - 1 >= 0:
            if self.instance[i][j - sl] == self.Color.BLACK:
                return False
            return self.compute_validity_trace(i, j - sl - 1, l - 1, self.For.LINE)
        elif l - 1 == 0:
            return True
        else:
            return False

    # Auxilliary function usd to compute the validity trace for a line
    def compute_validity_trace_column(self, i, j, l):
        if l == 0:
            if i == 0:
               return self.instance[i][j] != self.Color.BLACK
            return self.instance[i][j] != self.Color.BLACK and self.compute_validity_trace(i - 1, j, 0, self.For.COLUMN)

        sl = self.cSeq[j][l - 1]
        if i < sl - 1:
            return False
        if self.instance[i][j] == self.Color.WHITE:
            if i - 1 >= 0:
                return self.compute_validity_trace(i - 1, j, l, self.For.COLUMN)
            else:
                return False
        else:
            if self.instance[i][j] == self.Color.UNDEFINED and self.compute_validity_trace(i - 1, j, l, self.For.COLUMN):
                return True
            found_white_cell = False
            for i_prime in range(i - sl + 1, i + 1):
                if self.instance[i_prime][j] == self.Color.WHITE:
                    found_white_cell = True
                    break
            if found_white_cell:
                return False

        if i - sl >= 0:
            if self.instance[i - sl][j] == self.Color.BLACK:
                return False
        if i - sl - 1 >= 0:
            return self.compute_validity_trace(i - sl - 1, j, l - 1, self.For.COLUMN)
        elif l - 1 == 0:
            return True
        else:
            return False

    def set_cell_color(self, i, j, c):
        # TODO  :test if C is of type enum Color
        self.instance[i][j] = c

    # Show the instance grid
    def show_grid(self):
        grid = np.array([[0x000000 if j == DynamicSolver.Color.BLACK else 0xffffff if j != DynamicSolver.Color.UNDEFINED else 0xaaaaaa for j in self.instance[i]] for i in range(len(self.instance))])

        plt.imshow(grid, cmap="gray")
        #plt.gca().tick_params(axis="x", direction="out", top=1, bottom=0, labelbottom=0, labeltop=1)
        #plt.xticks(range(grid.shape[1]), [ str(x)[1:-1].replace(',','\n') for x in self.cSeq], rotation='horizontal')
        #plt.yticks(range(grid.shape[0]), [ str(x)[1:-1].replace(',',' ') for x in self.lSeq])

    # Solve the nonogram using the propagation algorithm given in the subject.
    def solve(self):
        instance_height = len(self.instance)
        instance_width = len(self.instance[0])
        instance_size = instance_height * instance_width
        fixed_cells_count = 0

        # At first, we need to look at every line and column.
        line_to_explore = [i for i in range(instance_height)]
        column_to_explore = [j for j in range(instance_width)]

        # If we still have cells to color, we need to continue.
        while fixed_cells_count != instance_size:

            # If there is no remaining line or column to explore, but we did not color
            # the full grid, then we failed.
            if len(line_to_explore) == 0 and len(column_to_explore) == 0:
                return "ERROR"

            for i in line_to_explore:
                for j in range(instance_width):

                    if self.instance[i][j] == self.Color.UNDEFINED:

                        self.set_cell_color(i, j, self.Color.BLACK)
                        # Reinit the trace
                        self.line_validity_trace = [[None] * len(self.cSeq) for i in range(len(self.lSeq[i]) + 1)]

                        success_black_coloring = self.compute_validity_trace(i, instance_width - 1, len(self.lSeq[i]), self.For.LINE)

                        #success_black_coloring = (line_test and column_test)
                        self.set_cell_color(i, j, self.Color.WHITE)

                        # Reinit the trace
                        self.line_validity_trace = [[None] * len(self.cSeq) for i in range(len(self.lSeq[i]) + 1)]

                        success_white_coloring = self.compute_validity_trace(i, instance_width - 1, len(self.lSeq[i]), self.For.LINE)

                        #success_white_coloring = (line_test and column_test)
                        had_change = False
                        if not success_black_coloring and success_white_coloring:
                            self.set_cell_color(i, j, self.Color.WHITE)
                            fixed_cells_count += 1
                            had_change = True
                        elif success_black_coloring and not success_white_coloring:
                            self.set_cell_color(i, j, self.Color.BLACK)
                            fixed_cells_count += 1
                            had_change = True
                        elif not success_black_coloring and not success_white_coloring:
                            return "ERROR"
                        else:
                            self.set_cell_color(i, j, self.Color.UNDEFINED)
                            #print("no coloring")

                        if had_change and not j in column_to_explore:
                            column_to_explore.append(j)

            line_to_explore = []

            for j in column_to_explore:
                for i in range(instance_height):
                    #print(self.instance)

                    if self.instance[i][j] == self.Color.UNDEFINED:
                        self.set_cell_color(i, j, self.Color.BLACK)
                        # Reinit the trace
                        self.column_validity_trace = [[None] * len(self.lSeq) for i in range(len(self.cSeq[j]) + 1)]

                        success_black_coloring = self.compute_validity_trace(instance_height - 1, j, len(self.cSeq[j]), self.For.COLUMN)

                        #success_black_coloring = (line_test and column_test)
                        self.set_cell_color(i, j, self.Color.WHITE)

                        # Reinir the trace
                        self.column_validity_trace = [[None] * len(self.lSeq) for i in range(len(self.cSeq[j]) + 1)]

                        success_white_coloring = self.compute_validity_trace(instance_height - 1, j, len(self.cSeq[j]), self.For.COLUMN)

                        #success_white_coloring = (line_test and column_test)
                        had_change = False

                        if not success_black_coloring and success_white_coloring:
                            self.set_cell_color(i, j, self.Color.WHITE)
                            fixed_cells_count += 1
                            had_change = True
                        elif success_black_coloring and not success_white_coloring:
                            self.set_cell_color(i, j, self.Color.BLACK)
                            fixed_cells_count += 1
                            had_change = True
                        elif not success_black_coloring and not success_white_coloring:
                            return "ERROR"
                        else:
                            self.set_cell_color(i, j, self.Color.UNDEFINED)

                        if had_change and not i in line_to_explore:
                            line_to_explore.append(i)

            column_to_explore = []

# A solver using MIP (PLNE in french) to solve the grid.
class LinearSolver:
    # A simple enumeration to determine the color of the cells
    class Color(Enum):
        UNDEFINED = 0,
        WHITE = 1,
        BLACK = 2

    def __init__(self):
        self.variable_count = 0
        self.instance_width = 0
        self.instance_height = 0
        self.lSeq = []
        self.cSeq = []

    # Load an instance from a text file and perform the necessary setup.
    def load_instance(self, filename):
        f = open(filename, "r")
        self.variable_count = 0
        self.instance_file = filename

        isLineSeq = True
        for line in f:
            if line[:-1] == "#":
                isLineSeq = False
                continue
            if line[-1] == '\n':
                seq = [int(x) for x in line[:-1].split(" ") if x != ""]
            else:
                seq = [int(x) for x in line.split(" ") if x != ""]
            if isLineSeq:
                self.lSeq.append(seq)
            else:
                self.cSeq.append(seq)

        self.instance = [[self.Color.UNDEFINED] * len(self.cSeq) for i in range(len(self.lSeq))]
        self.instance_width = len(self.instance[0])
        self.instance_height = len(self.instance)
        self.variable_count += self.instance_width * self.instance_height

        for seq in self.lSeq:
            self.variable_count += len(seq) * self.instance_width
        for seq in self.cSeq:
            self.variable_count += len(seq) * self.instance_height

        self.model = Model(filename)

        f.close()

    # Build the model variables from the instance parameters and sequences.
    def build_model_variables(self, partial_solution):
        x = []
        y = []
        z = []

        for i in range(self.instance_height):
            for j in range(self.instance_width):
                if not partial_solution is None and partial_solution[i][j] == DynamicSolver.Color.BLACK:
                    x.append(self.model.addVar(vtype=GRB.BINARY, lb=1, ub=1, name="x%d,%d" % (((i), (j)))))
                elif not partial_solution is None and partial_solution[i][j] == DynamicSolver.Color.WHITE:
                    x.append(self.model.addVar(vtype=GRB.BINARY, lb=0, ub=0, name="x%d,%d" % (((i), (j)))))
                else:
                    x.append(self.model.addVar(vtype=GRB.BINARY, name="x%d,%d" % (((i), (j)))))

        for i in range(self.instance_height):
            l = i * self.instance_width
            for j in range(self.instance_width):
                y.append([])
                acc = 0
                seq_sum = sum(self.lSeq[i])
                num_seq = len(self.lSeq[i])

                for t in range(len(self.lSeq[i])):
                    st = self.lSeq[i][t]
                    if (j < acc + t) or (j > self.instance_width - (seq_sum - acc +  num_seq - t - 1)):
                        y[l + j].append(None)
                        acc += st
                        continue
                    else:
                        y[l + j].append(self.model.addVar(vtype=GRB.BINARY, name="y%d,%d,%d" % ((i), (j), (t + 1))))
                    acc += st

        for j in range(self.instance_width):
            c = j * self.instance_height
            for i in range(self.instance_height):
                z.append([])
                acc = 0
                seq_sum = sum(self.cSeq[j])
                num_seq = len(self.cSeq[j])

                for t in range(len(self.cSeq[j])):
                    st = self.cSeq[j][t]
                    if (i < acc + t) or i > (self.instance_height - (seq_sum - acc  + num_seq - t - 1)):
                        z[c + i].append(None)
                        acc += st
                        continue
                    else:
                        z[c + i].append(self.model.addVar(vtype=GRB.BINARY, name="z%d,%d,%d" % ((i), (j), (t + 1))))
                    acc += st
        return x, y, z

    def show_grid(self):
        grid = np.array([[0x000000 if j == 1 else 0xffffff for j in self.instance[i]] for i in range(len(self.instance))])

        plt.imshow(grid, cmap="gray")
        #plt.gca().tick_params(axis="x", direction="out", top=1, bottom=0, labelbottom=0, labeltop=1)
        #plt.xticks(range(grid.shape[1]), [ str(x)[1:-1].replace(',','\n') for x in self.cSeq], rotation='horizontal')
        #plt.yticks(range(grid.shape[0]), [ str(x)[1:-1].replace(',',' ') for x in self.lSeq])

    # Solve the nonogram using MIP
    def solve(self, use_presolving = False, seed = 35594):
        partial_solution = None

        # If we're using presolving, we try to solve what we can with the dynamic programming
        # approach, and then fix the variable accordingly.
        if use_presolving:
            presolver = DynamicSolver()
            presolver.load_instance(self.instance_file)
            presolver.solve()
            partial_solution = presolver.instance

        x, y, z = self.build_model_variables(partial_solution)

        self.model.update()

        # Builds all the constraints for the lines.
        for i in range(self.instance_height):
            l = i * self.instance_width
            self.model.addConstr(quicksum(x[l + k] for k in range(self.instance_width)) == sum(self.lSeq[i]))

            max_t = 0
            for j in range(self.instance_width):
                for t in range(len(y[l + j])):
                    max_t = max(t, max_t)
                    seq_var = y[l + j][t]
                    if seq_var is None:
                        continue
                    st = self.lSeq[i][t]
                    self.model.addConstr(quicksum(x[l + k] for k in range(j, j + st)) >= st * seq_var)
                    if t < len(y[l + j]) - 1:
                        self.model.addConstr(quicksum(y[l + k][t + 1] for k in range(j + st + 1, self.instance_width) if not y[l + k][t + 1] is None) >= seq_var)
            if not len(y[l]) == 0:
                for t in range(max_t + 1):
                    self.model.addConstr(quicksum(y[l + k][t] for k in range(0, self.instance_width) if not y[l + k][t] is None) == 1)

        # Builds all the contraints for the columns.
        for j in range(self.instance_width):
            c = j * self.instance_height
            self.model.addConstr(quicksum(x[j + k * self.instance_width] for k in range(self.instance_height)) == sum(self.cSeq[j]))

            max_t = 0
            for i in range(self.instance_height):
                for t in range(len(z[c + i])):
                    max_t = max(t, max_t)
                    seq_var = z[c + i][t]
                    if seq_var is None:
                        continue
                    st = self.cSeq[j][t]
                    self.model.addConstr(quicksum(x[j + k * self.instance_width] for k in range(i, i + st)) >= st * seq_var)
                    if t < len(z[c + i]) - 1:
                        self.model.addConstr(quicksum(z[c + k][t + 1] for k in range(i + st + 1, self.instance_height) if not z[c + k][t + 1] is None) >= seq_var)
            if not len(z[c]) == 0:
                for t in range(max_t + 1):
                    self.model.addConstr(quicksum(z[c + k][t] for k in range(0, self.instance_height) if not z[c + k][t] is None) == 1)

        # Objective : maximizing the x_ij.
        self.model.setObjective(quicksum(x[idx] for idx in range(len(x))) ,GRB.MAXIMIZE)
        #self.model.update()
        self.model.setParam(GRB.Param.Seed, seed)
        self.model.write("debug.lp")

        # Optimizes the model
        self.model.optimize()

        # Exracts the variables' values.
        self.instance = [[x[idx].x for idx in range(l * self.instance_width, (l + 1) * self.instance_width)] for l in range(self.instance_height)]

# An example using the linear programming solver to solve the instance 15.
# The dynamic solver works in almost exactly the same way, without any
# arguments inside the "solve()" method.
solver = LinearSolver()
solver.load_instance("instances/14.txt")
#solver.solve()
solver.solve(True, 21488320)

solver.show_grid()
#21488320
#

max_stars_repo_name: Redchards/NonogramSolver

max_issues_repo_name: Redchards/NonogramSolver

max_forks_repo_name: Redchards/NonogramSolver


text:
# coding:utf-8
import numpy as np
from .classifier import Classifier
from .regressor import Regressor

class Node(object):
    def __init__(self, min_list, max_list, tau, is_leaf, stat, parent=None, delta=None, xi=None):
        self.parent = parent
        self.tau = tau
        self.is_leaf = is_leaf
        self.min_list = min_list
        self.max_list = max_list
        self.delta = delta
        self.xi = xi
        self.left = None
        self.right = None
        self.stat = stat

    def update_leaf(self, x, label):
        self.stat.add(x, label)

    def update_internal(self):
        self.stat = self.left.stat.merge(self.right.stat)

    def get_parent_tau(self):
        if self.parent is None:
            return 0.0
        return self.parent.tau

    def __repr__(self):
        return "<mondrianforest.Node tau={} min_list={} max_list={} is_leaf={}>".format(
            self.tau,
            self.min_list,
            self.max_list,
            self.is_leaf,
        )

class ClassifierFactory(object):
    def create(self):
        return Classifier()

class RegressorFactory(object):
    def create(self):
        return Regressor()

class MondrianTree(object):
    def __init__(self):
        self.root = None
        self.classes = set()

    def create_leaf(self, x, label, parent):
        leaf = Node(
            min_list=x.copy(),
            max_list=x.copy(),
            is_leaf=True,
            stat=self.stat_factory.create(),
            tau=1e9,
            parent=parent,
        )
        leaf.update_leaf(x, label)
        return leaf

    def extend_mondrian_block(self, node, x, label):
        '''
            return root of sub-tree
        '''
        e_min = np.maximum(node.min_list - x, 0)
        e_max = np.maximum(x - node.max_list, 0)
        e_sum = e_min + e_max
        rate = np.sum(e_sum) + 1e-9
        E = np.random.exponential(1.0/rate)
        if node.get_parent_tau() + E < node.tau:
            e_sample = np.random.rand() * np.sum(e_sum)
            delta = (e_sum.cumsum() > e_sample).argmax()
            if x[delta] > node.min_list[delta]:
                xi = np.random.uniform(node.min_list[delta], x[delta])
            else:
                xi = np.random.uniform(x[delta], node.max_list[delta])
            parent = Node(
                min_list=np.minimum(node.min_list, x),
                max_list=np.maximum(node.max_list, x),
                is_leaf=False,
                stat=self.stat_factory.create(),
                tau=node.get_parent_tau() + E,
                parent=node.parent,
                delta=delta,
                xi=xi,
            )
            sibling = self.create_leaf(x, label, parent=parent)
            if x[parent.delta] <= parent.xi:
                parent.left = sibling
                parent.right = node
            else:
                parent.left = node
                parent.right = sibling
            node.parent = parent
            parent.update_internal()
            return parent
        else:
            node.min_list = np.minimum(x, node.min_list)
            node.max_list = np.maximum(x, node.max_list)
            if not node.is_leaf:
                if x[node.delta] <= node.xi:
                    node.left = self.extend_mondrian_block(node.left, x, label)
                else:
                    node.right = self.extend_mondrian_block(node.right, x, label)
                node.update_internal()
            else:
                node.update_leaf(x, label)
            return node

    def partial_fit(self, X, y):
        for x, label in zip(X, y):
            self.classes |= {label}
            if self.root is None:
                self.root = self.create_leaf(x, label, parent=None)
            else:
                self.root = self.extend_mondrian_block(self.root, x, label)

    def fit(self, X, y):
        self.root = None
        self.partial_fit(X, y)

    def _predict(self, x, node, p_not_separeted_yet):
        d = node.tau - node.get_parent_tau()
        eta = np.sum(np.maximum(x-node.max_list, 0) + np.maximum(node.min_list - x, 0))
        p = 1.0 - np.exp(-d*eta)
        result = node.stat.create_result(x, p_not_separeted_yet * p)
        if node.is_leaf:
            w = p_not_separeted_yet * (1.0 - p)
            return result.merge(node.stat.create_result(x, w))
        if x[node.delta] <= node.xi:
            child_result = self._predict(x, node.left, p_not_separeted_yet*(1.0-p))
        else:
            child_result = self._predict(x, node.right, p_not_separeted_yet*(1.0-p))
        return result.merge(child_result)

    def get_params(self, deep):
        return {}

# TODO: extends BaseClassifier
class MondrianTreeClassifier(MondrianTree):
    def __init__(self):
        MondrianTree.__init__(self)
        self.stat_factory = ClassifierFactory()

    def predict_proba(self, X):
        res = []
        for x in X:
            prob = self._predict(x, self.root, 1.0).get()
            res.append(np.array([prob[l] for l in self.classes]))
        return res

    def score(self, X, y):
        probs = self.predict_proba(X)
        classes = np.array([c for c in self.classes])
        correct = 0.0
        for prob, label in zip(probs, y):
            correct += prob.argmax() == (classes == label).argmax()
        return correct / len(X)

class MondrianTreeRegressor(MondrianTree):
    def __init__(self):
        MondrianTree.__init__(self)
        self.stat_factory = RegressorFactory()

    def predict(self, X):
        res = []
        for x in X:
            predicted = self._predict(x, self.root, 1.0).get()
            res.append(predicted)
        return np.array(res)


text:
"""
Code for cheating in the inversion programming.

Includes functions for:

* forward modelling with prisms
* generating a synthetic model
* plotting the models and solutions

"""
import numpy as np
import matplotlib.pyplot as plt

def plot_prisms(
    depths,
    basin_boundaries,
    ax=None,
    color="#00000000",
    edgecolor="black",
    linewidth=1,
    label=None,
    figsize=(9, 3),
):
    """
    Plot the prism model using matplotlib.
    """
    # Create lines with the outline of the prisms
    boundaries = np.linspace(*basin_boundaries, depths.size + 1)
    x = [boundaries[0]]
    y = [0]
    for i in range(depths.size):
        x.extend([boundaries[i], boundaries[i + 1]])
        y.extend([depths[i], depths[i]])
    x.append(boundaries[-1])
    y.append(0)
    x = np.array(x) / 1000
    y = np.array(y) / 1000
    # Plot the outline with optional filling
    if ax is None:
        fig = plt.figure(figsize=figsize)
        ax = plt.subplot(111)
        ax.set_xlabel("x [km]")
        ax.set_ylabel("depth [km]")
    ax.fill_between(
        x, y, color=color, edgecolor=edgecolor, linewidth=linewidth, label=label
    )
    ax.set_ylim(max(y) * 1.05, 0)
    return ax

def gaussian(x, shift, std, amplitude):
    """
    A simple Gaussian function we'll use to make a model
    """
    return amplitude * np.exp(-(((x - shift) / std) ** 2))

def synthetic_model():
    """
    Generate a synthetic model using Gaussian functions
    """
    size = 100
    basin_boundaries = (0, 100e3)
    boundaries = np.linspace(*basin_boundaries, size + 1)
    x = boundaries[:-1] + 0.5 * (boundaries[1] - boundaries[0])
    depths = gaussian(x, shift=45e3, std=20e3, amplitude=5e3) + gaussian(
        x, shift=80e3, std=10e3, amplitude=1e3
    )
    # Make sure the boundaries are at zero to avoid steps
    depths -= depths.min()
    return depths, basin_boundaries

def prism_boundaries(depths, basin_boundaries):
    """
    Calculate the x coordinate of the boundaries of all prisms.
    Will have depths.size + 1 elements.
    """
    boundaries = np.linspace(*basin_boundaries, depths.size + 1)
    return boundaries

def forward_model(depths, basin_boundaries, density, x):
    """
    Calculate the predicted gravity for a given basin at x locations
    """
    easting = x
    boundaries = prism_boundaries(depths, basin_boundaries)
    result = np.zeros_like(x)
    for m in range(depths.size):
        prism_gravity(
            x, boundaries[m], boundaries[m + 1], 0, depths[m], density, output=result
        )
    return result

def prism_gravity(x, east, west, top, bottom, density, output=None):
    """
    Calculate the gravity of a single prism.
    Append the result to output if it's given.
    """
    prism = [east, west, -200e3, 200e3, -bottom, -top]
    si2mgal = 1e5
    # The gravitational constant in SI units
    GRAVITATIONAL_CONST = 0.00000000006673
    scale = GRAVITATIONAL_CONST * density * si2mgal
    northing = 0
    upward = 10
    if output is None:
        output = np.zeros_like(x)
    # Iterate over the prism boundaries to compute the result of the
    # integration (see Nagy et al., 2000)
    for i in range(2):
        for j in range(2):
            for k in range(2):
                shift_east = prism[1 - i]
                shift_north = prism[3 - j]
                shift_upward = prism[5 - k]
                output += (
                    (-1) ** (i + j + k)
                    * scale
                    * kernel(
                        shift_east - x,
                        shift_north - northing,
                        shift_upward - upward,
                    )
                )
    return output

def kernel(easting, northing, upward):
    """
    The kernel function for calculating the vertical component of gravity
    """
    radius = np.sqrt(easting ** 2 + northing ** 2 + upward ** 2)
    result = (
        easting * np.log(northing + radius)
        + northing * np.log(easting + radius)
        + upward * np.arctan2(easting * northing, -upward * radius)
    )
    return result


text:
"""Validation class for ChemKED schema.
"""
from warnings import warn
import re

from pkg_resources import resource_filename
import yaml

import numpy as np
import pint
from requests.exceptions import HTTPError, ConnectionError
from cerberus import Validator, SchemaError
import habanero
from .orcid import search_orcid

units = pint.UnitRegistry()
"""Unit registry to contain the units used in PyKED"""

units.define('cm3 = centimeter**3')
Q_ = units.Quantity

crossref_api = habanero.Crossref(mailto='prometheus@pr.omethe.us')

# Load the ChemKED schema definition file
schema_file = resource_filename(__name__, 'schemas/chemked_schema.yaml')
with open(schema_file, 'r') as f:
    schema_list = f.readlines()

inc_start = None
inc_end = None
inc_list = []
no_includes = False
for l_num, l in enumerate(schema_list):
    if l.startswith('!include'):
        if no_includes:  # pragma: no cover
            raise SchemaError('All included files must be first in the main schema')

        if inc_start is None:
            inc_start = l_num

        if inc_end is not None:  # pragma: no cover
            raise SchemaError('All included files must be first in the main schema')

        inc_fname = l.split('!include')[1].strip()
        inc_fname = resource_filename(__name__, 'schemas/' + inc_fname)
        with open(inc_fname, 'r') as f:
            inc_list.extend(f.readlines())
    else:
        if not l.strip() or l.startswith('#') or l.startswith('---'):
            continue

        if inc_start is None:  # pragma: no cover
            no_includes = True

        if inc_start is not None and inc_end is None:
            inc_end = l_num

schema_list[inc_start:inc_end] = inc_list
schema = yaml.safe_load(''.join(schema_list))

# These top-level keys in the schema serve as references for lower-level keys.
# They are removed to prevent conflicts due to required variables, etc.
for key in ['author', 'value-unit-required', 'value-unit-optional',
            'composition', 'ignition-type', 'value-with-uncertainty',
            'value-without-uncertainty',
            ]:
    del schema[key]

# SI units for available value-type properties
property_units = {
    'temperature': 'kelvin',
    'compressed-temperature': 'kelvin',
    'pressure': 'pascal',
    'compressed-pressure': 'pascal',
    'ignition-delay': 'second',
    'first-stage-ignition-delay': 'second',
    'pressure-rise': '1.0 / second',
    'compression-time': 'second',
    'volume': 'meter**3',
    'time': 'second',
    'piston position': 'meter',
    'emission': 'dimensionless',
    'absorption': 'dimensionless',
    'concentration': 'mole/meter**3',
    'stroke': 'meter',
    'clearance': 'meter',
    'compression-ratio': 'dimensionless',
}

def compare_name(given_name, family_name, question_name):
    """Compares a name in question to a specified name separated into given and family.

    The name in question ``question_name`` can be of varying format, including
    "Kyle E. Niemeyer", "Kyle Niemeyer", "K. E. Niemeyer", "KE Niemeyer", and
    "K Niemeyer". Other possibilities include names with hyphens such as
    "Chih-Jen Sung", "C. J. Sung", "C-J Sung".

    Examples:
        >>> compare_name('Kyle', 'Niemeyer', 'Kyle E Niemeyer')
        True
        >>> compare_name('Chih-Jen', 'Sung', 'C-J Sung')
        True

    Args:
        given_name (`str`): Given (or first) name to be checked against.
        family_name (`str`): Family (or last) name to be checked against.
        question_name (`str`): The whole name in question.

    Returns:
        `bool`: The return value. True for successful comparison, False otherwise.
    """
    # lowercase everything
    given_name = given_name.lower()
    family_name = family_name.lower()
    question_name = question_name.lower()

    # rearrange names given as "last, first middle"
    if ',' in question_name:
        name_split = question_name.split(',')
        name_split.reverse()
        question_name = ' '.join(name_split).strip()

    # remove periods
    question_name = question_name.replace('.', '')
    given_name = given_name.replace('.', '')
    family_name = family_name.replace('.', '')

    # split names by , <space> - .
    given_name = list(filter(None, re.split(r"[, \-.]+", given_name)))
    num_family_names = len(list(filter(None, re.split("[, .]+", family_name))))

    # split name in question by , <space> - .
    name_split = list(filter(None, re.split(r"[, \-.]+", question_name)))
    first_name = [name_split[0]]
    if len(name_split) > 2:
        first_name += [n for n in name_split[1:-num_family_names]]

    if len(first_name) > 1 and len(given_name) == len(first_name):
        # both have same number of first and middle names/initials
        for i in range(1, len(first_name)):
            first_name[i] = first_name[i][0]
            given_name[i] = given_name[i][0]
    elif len(given_name) != len(first_name):
        min_names = min(len(given_name), len(first_name))
        first_name = first_name[:min_names]
        given_name = given_name[:min_names]

    # first initial
    if len(first_name[0]) == 1 or len(given_name[0]) == 1:
        given_name[0] = given_name[0][0]
        first_name[0] = first_name[0][0]

    # first and middle initials combined
    if len(first_name[0]) > 1 or len(given_name[0]) > 1:
        given_name[0] = given_name[0][0]
        first_name[0] = name_split[0][0]

    # Hyphenated last name may need to be reconnected
    if num_family_names == 1 and '-' in family_name:
        num_hyphen = family_name.count('-')
        family_name_compare = '-'.join(name_split[-(num_hyphen + 1):])
    else:
        family_name_compare = ' '.join(name_split[-num_family_names:])

    return given_name == first_name and family_name == family_name_compare

class OurValidator(Validator):
    """Custom validator with rules for Quantities and references.
    """
    def _validate_isvalid_t_range(self, isvalid_t_range, field, values):
        """Checks that the temperature ranges given for thermo data are valid
        Args:
            isvalid_t_range (`bool`): flag from schema indicating T range is to be checked
            field (`str`): T_range
            values (`list`): List of temperature values indicating low, middle, and high ranges

        The rule's arguments are validated against this schema:
            {'isvalid_t_range': {'type': 'bool'}, 'field': {'type': 'str'},
             'value': {'type': 'list'}}
        """
        if all([isinstance(v, (float, int)) for v in values]):
            # If no units given, assume Kelvin
            T_low = Q_(values[0], 'K')
            T_mid = Q_(values[1], 'K')
            T_hi = Q_(values[2], 'K')
        elif all([isinstance(v, str) for v in values]):
            T_low = Q_(values[0])
            T_mid = Q_(values[1])
            T_hi = Q_(values[2])
        else:
            self._error(field, 'The temperatures in the range must all be either with units or '
                               'without units, they cannot be mixed')
            return False

        if min([T_low, T_mid, T_hi]) != T_low:
            self._error(field, 'The first element of the T_range must be the lower limit')
        if max([T_low, T_mid, T_hi]) != T_hi:
            self._error(field, 'The last element of the T_range must be the upper limit')

    def _validate_isvalid_unit(self, isvalid_unit, field, value):
        """Checks for appropriate units using Pint unit registry.
        Args:
            isvalid_unit (`bool`): flag from schema indicating units to be checked.
            field (`str`): property associated with units in question.
            value (`dict`): dictionary of values from file associated with this property.

        The rule's arguments are validated against this schema:
            {'isvalid_unit': {'type': 'bool'}, 'field': {'type': 'str'},
             'value': {'type': 'dict'}}
        """
        quantity = 1.0 * units(value['units'])
        try:
            quantity.to(property_units[field])
        except pint.DimensionalityError:
            self._error(field, 'incompatible units; should be consistent '
                        'with ' + property_units[field]
                        )

    def _validate_isvalid_history(self, isvalid_history, field, value):
        """Checks that the given time history is properly formatted.

        Args:
            isvalid_history (`bool`): flag from schema indicating units to be checked.
            field (`str`): property associated with history in question.
            value (`dict`): dictionary of values from file associated with this property.

        The rule's arguments are validated against this schema:
            {'isvalid_history': {'type': 'bool'}, 'field': {'type': 'str'},
             'value': {'type': 'dict'}}
        """
        # Check the type has appropriate units
        history_type = value['type']
        if history_type.endswith('emission'):
            history_type = 'emission'
        elif history_type.endswith('absorption'):
            history_type = 'absorption'
        quantity = 1.0*(units(value['quantity']['units']))
        try:
            quantity.to(property_units[history_type])
        except pint.DimensionalityError:
            self._error(field, 'incompatible units; should be consistent '
                        'with ' + property_units[history_type])

        # Check that time has appropriate units
        time = 1.0*(units(value['time']['units']))
        try:
            time.to(property_units['time'])
        except pint.DimensionalityError:
            self._error(field, 'incompatible units; should be consistent '
                        'with ' + property_units['time'])

        # Check that the values have the right number of columns
        n_cols = len(value['values'][0])
        max_cols = max(value['time']['column'],
                       value['quantity']['column'],
                       value.get('uncertainty', {}).get('column', 0)) + 1
        if n_cols > max_cols:
            self._error(field, 'too many columns in the values')
        elif n_cols < max_cols:
            self._error(field, 'not enough columns in the values')

    def _validate_isvalid_quantity(self, isvalid_quantity, field, value):
        """Checks for valid given value and appropriate units.

        Args:
            isvalid_quantity (`bool`): flag from schema indicating quantity to be checked.
            field (`str`): property associated with quantity in question.
            value (`list`): list whose first element is a string representing a value with units

        The rule's arguments are validated against this schema:
            {'isvalid_quantity': {'type': 'bool'}, 'field': {'type': 'str'},
             'value': {'type': 'list'}}
        """
        quantity = Q_(value[0])
        low_lim = 0.0 * units(property_units[field])

        try:
            if quantity <= low_lim:
                self._error(
                    field, 'value must be greater than 0.0 {}'.format(property_units[field]),
                )
        except pint.DimensionalityError:
            self._error(field, 'incompatible units; should be consistent '
                        'with ' + property_units[field]
                        )

    def _validate_isvalid_uncertainty(self, isvalid_uncertainty, field, value):
        """Checks for valid given value and appropriate units with uncertainty.

        Args:
            isvalid_uncertainty (`bool`): flag from schema indicating uncertainty to be checked
            field (`str`): property associated with the quantity in question.
            value (`list`): list with the string of the value of the quantity and a dictionary of
                the uncertainty

        The rule's arguments are validated against this schema:
            {'isvalid_uncertainty': {'type': 'bool'}, 'field': {'type': 'str'},
             'value': {'type': 'list'}}
        """
        self._validate_isvalid_quantity(True, field, value)

        # This len check is necessary for reasons that aren't quite clear to me
        # Cerberus calls this validation method even when lists have only one element
        # and should therefore be validated only by isvalid_quantity
        if len(value) > 1 and value[1]['uncertainty-type'] != 'relative':
            if value[1].get('uncertainty') is not None:
                self._validate_isvalid_quantity(True, field, [value[1]['uncertainty']])

            if value[1].get('upper-uncertainty') is not None:
                self._validate_isvalid_quantity(True, field, [value[1]['upper-uncertainty']])

            if value[1].get('lower-uncertainty') is not None:
                self._validate_isvalid_quantity(True, field, [value[1]['lower-uncertainty']])

    def _validate_isvalid_reference(self, isvalid_reference, field, value):
        """Checks valid reference metadata using DOI (if present).

        Args:
            isvalid_reference (`bool`): flag from schema indicating reference to be checked.
            field (`str`): 'reference'
            value (`dict`): dictionary of reference metadata.

        The rule's arguments are validated against this schema:
            {'isvalid_reference': {'type': 'bool'}, 'field': {'type': 'str'},
             'value': {'type': 'dict'}}

        """
        if 'doi' in value:
            try:
                ref = crossref_api.works(ids=value['doi'])['message']
            except (HTTPError, habanero.RequestError):
                self._error(field, 'DOI not found')
                return
            except ConnectionError:
                warn('network not available, DOI not validated.')
                return

            # Assume that the reference returned by the DOI lookup always has a container-title
            ref_container = ref.get('container-title')[0]
            # TODO: Add other container types: value.get('journal') or value.get('report') or ...
            # note that there's a type field in the ref that is journal-article, proceedings-article
            container = value.get('journal')
            if container is None or container != ref_container:
                self._error(field, 'journal should be {}'.format(ref_container))

            # Assume that the reference returned by DOI lookup always has a year
            ref_year = ref.get('published-print') or ref.get('published-online')
            ref_year = ref_year['date-parts'][0][0]
            year = value.get('year')
            if year is None or year != ref_year:
                self._error(field, 'year should be {}'.format(ref_year))

            # Volume number might not be in the reference
            ref_volume = ref.get('volume')
            volume = value.get('volume')
            if ref_volume is None:
                if volume is not None:
                    self._error(field, 'Volume was specified in the YAML but is not present in the '
                                'DOI reference.')
            else:
                if volume is None or int(volume) != int(ref_volume):
                    self._error(field, 'volume should be {}'.format(ref_volume))

            # Pages might not be in the reference
            ref_pages = ref.get('page')
            pages = value.get('pages')
            if ref_pages is None:
                if pages is not None:
                    self._error(field, 'Pages were specified in the YAML but are not present in '
                                'the DOI reference.')
            else:
                if pages is None or pages != ref_pages:
                    self._error(field, 'pages should be {}'.format(ref_pages))

            # check that all authors present
            authors = value['authors'][:]
            author_names = [a['name'] for a in authors]
            for author in ref['author']:
                # find using family name
                author_match = next(
                    (a for a in authors if
                     compare_name(author['given'], author['family'], a['name'])
                     ),
                    None
                    )
                # error if missing author in given reference information
                if author_match is None:
                    self._error(field, 'Missing author: ' +
                                ' '.join([author['given'], author['family']])
                                )
                else:
                    author_names.remove(author_match['name'])

                    # validate ORCID if given
                    orcid = author.get('ORCID')
                    if orcid:
                        # Crossref may give ORCID as http://orcid.org/####-####-####-####
                        # so need to strip the leading URL
                        orcid = orcid[orcid.rfind('/') + 1:]

                        if 'ORCID' in author_match:
                            if author_match['ORCID'] != orcid:
                                self._error(
                                    field, author_match['name'] + ' ORCID does ' +
                                    'not match that in reference. Reference: ' +
                                    orcid + '. Given: ' + author_match['ORCID']
                                    )
                        else:
                            # ORCID not given, suggest adding it
                            warn('ORCID ' + orcid + ' missing for ' + author_match['name'])

            # check for extra names given
            if len(author_names) > 0:
                self._error(field, 'Extra author(s) given: ' +
                            ', '.join(author_names)
                            )

    def _validate_isvalid_orcid(self, isvalid_orcid, field, value):
        """Checks for valid ORCID if given.

        Args:
            isvalid_orcid (`bool`): flag from schema indicating ORCID to be checked.
            field (`str`): 'author'
            value (`dict`): dictionary of author metadata.

        The rule's arguments are validated against this schema:
            {'isvalid_orcid': {'type': 'bool'}, 'field': {'type': 'str'},
             'value': {'type': 'dict'}}

        """
        if isvalid_orcid and 'ORCID' in value:
            try:
                res = search_orcid(value['ORCID'])
            except ConnectionError:
                warn('network not available, ORCID not validated.')
                return
            except HTTPError:
                self._error(field, 'ORCID incorrect or invalid for ' +
                            value['name']
                            )
                return

            family_name = res['name']['family-name']['value']
            given_name = res['name']['given-names']['value']
            if not compare_name(given_name, family_name, value['name']):
                self._error(field, 'Name and ORCID do not match. Name supplied: ' +
                            value['name'] + '. Name associated with ORCID: ' +
                            ' '.join([given_name, family_name])
                            )

    def _validate_isvalid_composition(self, isvalid_composition, field, value):
        """Checks for valid specification of composition.

        Args:
            isvalid_composition (bool): flag from schema indicating
                composition to be checked.
            field (str): 'composition'
            value (dict): dictionary of composition

        The rule's arguments are validated against this schema:
            {'isvalid_composition': {'type': 'bool'}, 'field': {'type': 'str'},
             'value': {'type': 'dict'}}
        """
        sum_amount = 0.0
        if value['kind'] in ['mass fraction', 'mole fraction']:
            low_lim = 0.0
            up_lim = 1.0
            total_amount = 1.0
        elif value['kind'] in ['mole percent']:
            low_lim = 0.0
            up_lim = 100.0
            total_amount = 100.0
        else:
            self._error(field, 'composition kind must be "mole percent", "mass fraction", or '
                        '"mole fraction"')
            return False

        for sp in value['species']:
            amount = sp['amount'][0]
            sum_amount += amount

            # Check that amount within bounds, based on kind specified
            if amount < low_lim:
                self._error(field, 'Species ' + sp['species-name'] + ' ' +
                            value['kind'] + ' must be greater than {:.1f}'.format(low_lim)
                            )
            elif amount > up_lim:
                self._error(field, 'Species ' + sp['species-name'] + ' ' +
                            value['kind'] + ' must be less than {:.1f}'.format(up_lim)
                            )

        # Make sure mole/mass fraction sum to 1
        if not np.isclose(total_amount, sum_amount):
            self._error(field, 'Species ' + value['kind'] +
                        's do not sum to {:.1f}: '.format(total_amount) +
                        '{:f}'.format(sum_amount)
                        )
        # TODO: validate InChI, SMILES, or atomic-composition

max_stars_repo_name: pr-omethe-us/PyKED

max_issues_repo_name: mefuller/PyKED

max_forks_repo_name: mefuller/PyKED