text:
import os
from contextlib import closing

import numpy as np
from numpy import testing
import tables

from ..file import H5File
from ..dict_node import H5DictNode
from ..table_node import H5TableNode
from .utils import open_h5file, temp_h5_file

H5_TEST_FILE = '_temp_test_filt.h5'

def teardown():
    try:
        os.remove(H5_TEST_FILE)
    except OSError:
        pass

def test_reopen():
    h5 = H5File(H5_TEST_FILE, mode='w')
    assert h5.is_open
    h5.close()
    assert not h5.is_open
    h5.open()
    assert h5.is_open
    h5.close()

def test_open_from_pytables_object():
    with closing(tables.File(H5_TEST_FILE, 'w')) as pyt_file:
        pyt_file.create_group('/', 'my_group')
        with open_h5file(pyt_file) as h5:
            assert '/my_group' in h5

def test_open_from_closed_pytables_object():
    with closing(tables.File(H5_TEST_FILE, 'w')) as pyt_file:
        pyt_file.create_group('/', 'my_group')
        pyt_file.close()
        with open_h5file(pyt_file) as h5:
            assert '/my_group' in h5

def test_create_array_with_H5File():
    array = np.arange(3)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5array = h5.create_array('/array', array)
        # Test returned array
        testing.assert_allclose(h5array, array)
        # Test stored array
        testing.assert_allclose(h5['/array'], array)

def test_create_array_with_H5Group():
    array = np.arange(3)
    node_path = '/tardigrade/array'
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        group = h5.create_group('/tardigrade')
        h5array = group.create_array('array', array)
        # Test returned array
        testing.assert_allclose(h5array, array)
        # Test stored array
        testing.assert_allclose(h5[node_path], array)

def test_getitem_failure():
    array = np.arange(3)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5.create_array('/array', array)
        testing.assert_raises(NameError, h5.__getitem__, '/not_there')

def test_iteritems():
    node_paths = ['/foo', '/bar', '/baz']
    array = np.arange(3)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        for path in node_paths:
            h5.create_array(path, array)

        # We expect to see the root node when calling iteritems...
        node_paths.append('/')
        iter_paths = []

        # 2to3 converts the iteritems blindly to items which is incorrect,
        # so we resort to this ugliness.
        items = getattr(h5, 'iteritems')()
        for path, node in items:
            iter_paths.append(path)

    assert set(node_paths) == set(iter_paths)

def test_create_plain_array_with_H5File():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5array = h5.create_array('/array', np.arange(3), chunked=False)
        assert isinstance(h5array, tables.Array)
        assert not isinstance(h5array, tables.CArray)

def test_create_plain_array_with_H5Group():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5array = h5.root.create_array('/array', np.arange(3), chunked=False)
        assert isinstance(h5array, tables.Array)
        assert not isinstance(h5array, tables.CArray)

def test_create_chunked_array_with_H5File():
    array = np.arange(3, dtype=np.uint8)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5array = h5.create_array('/array', array, chunked=True)
        testing.assert_allclose(h5array, array)
        assert isinstance(h5array, tables.CArray)

def test_create_chunked_array_with_H5Group():
    array = np.arange(3, dtype=np.uint8)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5array = h5.root.create_array('/array', array, chunked=True)
        testing.assert_allclose(h5array, array)
        assert isinstance(h5array, tables.CArray)

def test_create_extendable_array_with_H5File():
    array = np.arange(3, dtype=np.uint8)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5array = h5.create_array('/array', array, extendable=True)
        testing.assert_allclose(h5array, array)
        assert isinstance(h5array, tables.EArray)

def test_create_extendable_array_with_H5Group():
    array = np.arange(3, dtype=np.uint8)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5array = h5.root.create_array('/array', array, extendable=True)
        testing.assert_allclose(h5array, array)
        assert isinstance(h5array, tables.EArray)

def test_str_and_repr():
    array = np.arange(3)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5.create_array('/array', array)

        assert repr(h5) == repr(h5._h5)
        assert str(h5) == str(h5._h5)

def test_shape_and_dtype():
    array = np.ones((3, 4), dtype=np.uint8)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        for node, chunked in (('/array', False), ('/carray', True)):
            h5array = h5.create_array(node, array.shape, dtype=array.dtype,
                                      chunked=chunked)
            assert h5array.dtype == array.dtype
            assert h5array.shape == array.shape

def test_shape_only_raises():
    shape = (3, 4)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        testing.assert_raises(ValueError, h5.create_array, '/array', shape)

def test_create_duplicate_array_raises():
    array = np.arange(3)
    with open_h5file(H5_TEST_FILE, mode='w', delete_existing=False) as h5:
        h5.create_array('/array', array)
        testing.assert_raises(ValueError, h5.create_array, '/array', array)

def test_delete_existing_array_with_H5File():
    old_array = np.arange(3)
    new_array = np.ones(5)
    with open_h5file(H5_TEST_FILE, mode='w', delete_existing=True) as h5:
        h5.create_array('/array', old_array)
        # New array with the same node name should delete old array
        h5.create_array('/array', new_array)
        testing.assert_allclose(h5['/array'], new_array)

def test_delete_existing_array_with_H5Group():
    old_array = np.arange(3)
    new_array = np.ones(5)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5.create_array('/array', old_array)
        # New array with the same node name should delete old array
        h5.root.create_array('/array', new_array, delete_existing=True)
        testing.assert_allclose(h5['/array'], new_array)

def test_delete_existing_dict_with_H5File():
    old_dict = {'a': 'Goose'}
    new_dict = {'b': 'Quail'}
    with open_h5file(H5_TEST_FILE, mode='w', delete_existing=True) as h5:
        h5.create_dict('/dict', old_dict)
        # New dict with the same node name should delete old dict
        h5.create_dict('/dict', new_dict)
        assert h5['/dict'].data == new_dict

def test_delete_existing_dict_with_H5Group():
    old_dict = {'a': 'Goose'}
    new_dict = {'b': 'Quail'}
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5.create_dict('/dict', old_dict)
        # New dict with the same node name should delete old dict
        h5.root.create_dict('/dict', new_dict, delete_existing=True)
        assert h5['/dict'].data == new_dict

def test_delete_existing_table_with_H5File():
    old_description = [('Honk', 'int'), ('Wink', 'float')]
    new_description = [('Toot', 'float'), ('Pop', 'int')]
    with open_h5file(H5_TEST_FILE, mode='w', delete_existing=True) as h5:
        h5.create_table('/table', old_description)
        # New table with the same node name should delete old table
        h5.create_table('/table', new_description)
        tab = h5['/table']
        tab.append({'Pop': (1,), 'Toot': (np.pi,)})
        assert tab.ix[0][0] == np.pi

def test_delete_existing_table_with_H5Group():
    old_description = [('Honk', 'int'), ('Wink', 'float')]
    new_description = [('Toot', 'float'), ('Pop', 'int')]
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5.create_table('/table', old_description)
        # New table with the same node name should delete old table
        h5.root.create_table('/table', new_description, delete_existing=True)
        tab = h5['/table']
        tab.append({'Pop': (1,), 'Toot': (np.pi,)})
        assert tab.ix[0][0] == np.pi

def test_delete_existing_group_with_H5File():
    with open_h5file(H5_TEST_FILE, mode='w', delete_existing=True) as h5:
        h5.create_group('/group')
        grp = h5['/group']
        grp.attrs['test'] = 4

        assert grp.attrs['test'] == 4

        h5.create_group('/group')
        grp = h5['/group']
        grp.attrs['test'] = 6

        assert grp.attrs['test'] == 6

def test_delete_existing_group_with_H5Group():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5.create_group('/group')
        grp = h5['/group']
        grp.attrs['test'] = 4

        assert grp.attrs['test'] == 4

        h5.root.create_group('/group', delete_existing=True)
        grp = h5['/group']
        grp.attrs['test'] = 6

        assert grp.attrs['test'] == 6

def test_remove_group_with_H5File():
    with open_h5file(H5_TEST_FILE, mode='w', delete_existing=True) as h5:
        h5.create_group('/group')
        assert '/group' in h5
        h5.remove_group('/group')
        assert '/group' not in h5

def test_remove_group_with_H5Group():
    node_path = '/waterbear/group'
    with open_h5file(H5_TEST_FILE, mode='w', delete_existing=True) as h5:
        group = h5.create_group('/waterbear')
        group.create_group('group')
        assert node_path in h5
        group.remove_group('group')
        assert node_path not in h5

@testing.raises(ValueError)
def test_remove_group_with_remove_node():
    node_path = '/group'
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5.create_group(node_path)
        h5.remove_node(node_path)  # Groups should be removed w/ `remove_group`

def test_remove_node_with_H5File():
    with open_h5file(H5_TEST_FILE, mode='w', delete_existing=True) as h5:
        h5.create_array('/array', np.arange(3))
        assert '/array' in h5
        h5.remove_node('/array')
        assert '/array' not in h5

def test_remove_node_with_H5Group():
    node_path = '/waterbear/array'
    with open_h5file(H5_TEST_FILE, mode='w', delete_existing=True) as h5:
        group = h5.create_group('/waterbear')
        h5.create_array(node_path, np.arange(3))
        assert node_path in h5
        group.remove_node('array')
        assert node_path not in h5

def test_read_mode_raises_on_nonexistent_file():
    cm = open_h5file('_nonexistent_.h5', mode='r')
    testing.assert_raises(IOError, cm.__enter__)
    cm = open_h5file('_nonexistent_.h5', mode='r+')
    testing.assert_raises(IOError, cm.__enter__)

def test_cleanup():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5_pytables = h5._h5  # This reference gets deleted on close
    assert not h5_pytables.isopen

def test_create_group_with_H5File():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5.create_group('/group')
        assert '/group' in h5

def test_create_group_with_H5Group():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        group = h5['/']
        group.create_group('group')
        assert '/group' in h5

def test_split_path():
    path, node = H5File.split_path('/')
    assert path == '/'
    assert node == ''

    path, node = H5File.split_path('/node')
    assert path == '/'
    assert node == 'node'

    path, node = H5File.split_path('/group/node')
    assert path == '/group'
    assert node == 'node'

def test_join_path():
    path = H5File.join_path('/', 'a', 'b', 'c')
    assert path == '/a/b/c'

    path = H5File.join_path('a', 'b/c')
    assert path == '/a/b/c'

    path = H5File.join_path('a', '/b', '/c')
    assert path == '/a/b/c'

def test_auto_groups():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5.auto_groups = True
        h5.create_array('/group/array', np.arange(3))
        assert '/group/array' in h5

def test_auto_groups_deep():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5.auto_groups = True
        h5.create_array('/group1/group2/array', np.arange(3))
        assert '/group1/group2/array' in h5

def test_groups():
    array = np.arange(3)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5.create_array('/a/b/array', array)
        group_a = h5['/a']
        # Check that __contains__ works for groups
        assert 'b' in group_a
        assert 'b/array' in group_a
        group_b = group_a['b']
        # Check that __contains__ works for arrays
        assert 'array' in group_b
        testing.assert_allclose(group_b['array'], array)
        testing.assert_allclose(group_a['b/array'], array)

def test_group_attributes():
    value_1 = 'foo'
    value_2 = 'bar'
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5['/'].attrs['name'] = value_1
        assert h5['/'].attrs['name'] == value_1
        group = h5['/']

        # Make sure changes to the attribute consistent
        group._h5_group._v_attrs['name'] = value_2
        assert group.attrs['name'] == value_2

def test_group_properties():
    with open_h5file(H5_TEST_FILE, mode='w', auto_groups=True) as h5:
        h5.create_array('/group1/group2/array', np.arange(3))
        h5.create_array('/group1/array1', np.arange(3))

        assert h5['/group1'].name == 'group1'

        child_names = h5['/group1'].children_names
        assert sorted(child_names) == sorted(['group2', 'array1'])

        sub_names = h5['/group1'].subgroup_names
        assert sub_names == ['group2']

        assert h5['/group1'].root.name == '/'
        assert h5['/group1/group2'].root.name == '/'

def test_iter_groups():
    with open_h5file(H5_TEST_FILE, mode='w', auto_groups=True) as h5:
        h5.create_array('/group1/array', np.arange(3))
        h5.create_array('/group1/subgroup/deep_array', np.arange(3))
        group = h5['/group1']
        assert set(n.name for n in group.iter_groups()) == set(['subgroup'])

def test_mapping_interface_for_file():
    with open_h5file(H5_TEST_FILE, mode='w', auto_groups=True) as h5:
        array = h5.create_array('/array', np.arange(3))
        h5.create_array('/group/deep_array', np.arange(3))
        # `deep_array` isn't a direct descendent and isn't counted.
        assert len(h5) == 2
        assert '/group' in h5
        assert '/array' in h5
        testing.assert_allclose(h5['/array'], array)
        assert set(n.name for n in h5) == set(['array', 'group'])

def test_mapping_interface_for_group():
    with open_h5file(H5_TEST_FILE, mode='w', auto_groups=True) as h5:
        array = h5.create_array('/group1/array', np.arange(3))
        h5.create_array('/group1/subgroup/deep_array', np.arange(3))
        group = h5['/group1']
        # `deep_array` isn't a direct descendent and isn't counted.
        assert len(group) == 2
        assert 'subgroup' in group
        assert 'array' in group
        testing.assert_allclose(group['array'], array)
        assert set(n.name for n in group) == set(['array', 'subgroup'])

def test_group_str_and_repr():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        group = h5['/']

        assert str(group) == str(group._h5_group)
        assert repr(group) == repr(group._h5_group)

def test_attribute_translation():
    value_1 = (1, 2, 3)
    value_1_array = np.array(value_1)
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        h5['/'].attrs['name'] = value_1
        assert isinstance(h5['/'].attrs['name'], np.ndarray)
        testing.assert_allclose(h5['/'].attrs['name'], value_1_array)

def test_get_attribute():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        attrs = h5['/'].attrs
        attrs['name'] = 'hello'
        assert attrs.get('name') == attrs['name']

def test_del_attribute():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        attrs = h5['/'].attrs
        attrs['name'] = 'hello'
        del attrs['name']
        assert 'name' not in attrs

def test_get_attribute_default():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        assert h5['/'].attrs.get('missing', 'null') == 'null'

def test_attribute_update():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        attrs = h5['/'].attrs
        attrs.update({'a': 1, 'b': 2})
        assert attrs['a'] == 1
        assert attrs['b'] == 2
        attrs.update({'b': 20, 'c': 30})
        assert attrs['b'] == 20
        assert attrs['c'] == 30

def test_attribute_iteration_methods():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        attrs = h5['/'].attrs
        attrs['organ'] = 'gallbladder'
        attrs['count'] = 42
        attrs['alpha'] = 0xff

        items = list(attrs.items())

        assert all(isinstance(x, tuple) for x in items)

        # unfold the pairs
        keys, vals = [list(item) for item in zip(*items)]

        assert keys == list(attrs.keys())
        assert vals == list(attrs.values())

        # Check that __iter__ is consistent
        assert keys == list(iter(attrs))
        assert len(attrs) == len(keys)

def test_bad_node_name():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        testing.assert_raises(ValueError, h5.create_array,
                              '/attrs', np.zeros(3))

def test_bad_group_name():
    with open_h5file(H5_TEST_FILE, mode='w') as h5:
        testing.assert_raises(ValueError, h5.create_array,
                              '/attrs/array', np.zeros(3))

def test_create_dict_with_H5File():
    data = {'a': 1}
    with temp_h5_file() as h5:
        h5.create_dict('/dict', data)
        assert isinstance(h5['/dict'], H5DictNode)
        assert h5['/dict']['a'] == 1

def test_create_dict_with_H5Group():
    node_path = '/bananas/dict'
    data = {'a': 1}
    with temp_h5_file() as h5:
        group = h5.create_group('/bananas')
        group.create_dict('dict', data)
        assert isinstance(h5[node_path], H5DictNode)
        assert h5[node_path]['a'] == 1

def test_create_table_with_H5File():
    description = [('foo', 'int'), ('bar', 'float')]
    with temp_h5_file() as h5:
        h5.create_table('/table', description)
        tab = h5['/table']
        assert isinstance(tab, H5TableNode)
        tab.append({'foo': (1,), 'bar': (np.pi,)})
        assert tab.ix[0][0] == 1
        assert tab.ix[0][1] == np.pi

        h5.remove_node('/table')
        assert '/table' not in h5

def test_create_table_with_H5Group():
    node_path = '/rhinocerous/table'
    description = [('foo', 'int'), ('bar', 'float')]
    with temp_h5_file() as h5:
        group = h5.create_group('/rhinocerous')
        group.create_table('table', description)
        tab = h5[node_path]
        assert isinstance(tab, H5TableNode)
        tab.append({'foo': (1,), 'bar': (np.pi,)})
        assert tab.ix[0][0] == 1
        assert tab.ix[0][1] == np.pi

        group.remove_node('table')
        assert node_path not in h5

if __name__ == '__main__':
    testing.run_module_suite()

max_stars_repo_name: adamcvj/SatelliteTracker

max_issues_repo_name: adamcvj/SatelliteTracker

max_forks_repo_name: adamcvj/SatelliteTracker


text:
from __future__ import print_function
import argparse
import torch
from train_loop import TrainLoop
import torch.optim as optim
import torch.utils.data
import model as model_
import numpy as np
from data_load import Loader
from torch.utils.tensorboard import SummaryWriter
from utils import *

# Training settings
parser = argparse.ArgumentParser(description='Speaker embbedings with contrastive loss')
parser.add_argument('--model', choices=['lstm', 'resnet', 'resnet_pca', 'wideresnet', 'lcnn_9', 'lcnn_29', 'lcnn_9_pca', 'lcnn_29_pca', 'lcnn_9_prodspec', 'lcnn_9_icqspec', 'lcnn_9_CC', 'lcnn_29_CC', 'resnet_CC', 'TDNN', 'TDNN_multipool', 'TDNN_ablation', 'TDNN_LSTM', 'FTDNN', 'mobilenet', 'densenet', 'VGG'], default='lcnn_9', help='Model arch')
parser.add_argument('--resnet-type', choices=['18', '28', '34', '50', '101', 'se_18', 'se_28', 'se_34', 'se_50', 'se_101', '2net_18', '2net_se_18'], default='18', help='Resnet arch')
parser.add_argument('--vgg-type', choices=['VGG11', 'VGG13', 'VGG16', 'VGG19'], default='VGG16', help='VGG arch')
parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)')
parser.add_argument('--valid-batch-size', type=int, default=64, metavar='N', help='input batch size for validation (default: 64)')
parser.add_argument('--epochs', type=int, default=500, metavar='N', help='number of epochs to train (default: 500)')
parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.001)')
parser.add_argument('--momentum', type=float, default=0.9, metavar='alpha', help='Alpha (default: 0.9)')
parser.add_argument('--l2', type=float, default=1e-5, metavar='L2', help='Weight decay coefficient (default: 0.00001)')
parser.add_argument('--max-gnorm', type=float, default=100., metavar='clip', help='Max gradient norm (default: 100.0)')
parser.add_argument('--patience', type=int, default=5, metavar='N', help='number of epochs without improvement to wait before reducing lr (default: 5)')
parser.add_argument('--smoothing', type=float, default=0.1, metavar='l', help='Label smoothing (default: 0.1) - Disable by setting it to 0')
parser.add_argument('--checkpoint-epoch', type=int, default=None, metavar='N', help='epoch to load for checkpointing. If None, training starts from scratch')
parser.add_argument('--checkpoint-path', type=str, default=None, metavar='Path', help='Path for checkpointing')
parser.add_argument('--pretrained-path', type=str, default=None, metavar='Path', help='Path for pre trained model')
parser.add_argument('--logdir', type=str, default=None, metavar='Path', help='Path for checkpointing')
parser.add_argument('--train-hdf-path', type=str, default='./data/train.hdf', metavar='Path', help='Path to hdf data')
parser.add_argument('--valid-hdf-path', type=str, default=None, metavar='Path', help='Path to hdf data')
parser.add_argument('--workers', type=int, help='number of data loading workers', default=4)
parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)')
parser.add_argument('--save-every', type=int, default=1, metavar='N', help='how many epochs to wait before logging training status. Default is 1')
parser.add_argument('--n-frames', type=int, default=1000, metavar='N', help='maximum number of frames per utterance (default: 1000)')
parser.add_argument('--n-cycles', type=int, default=30000, metavar='N', help='number of examples to complete 1 epoch')
parser.add_argument('--valid-n-cycles', type=int, default=1000, metavar='N', help='number of examples to complete 1 epoch')
parser.add_argument('--ncoef', type=int, default=90, metavar='N', help='Number of cepstral coefs for the LA case (default: 90)')
parser.add_argument('--init-coef', type=int, default=0, metavar='N', help='First cepstral coefs (default: 0)')
parser.add_argument('--lists-path', type=str, default=None, metavar='Path', help='Path to list files per attack')
parser.add_argument('--no-cuda', action='store_true', default=False, help='Disables GPU use')
args = parser.parse_args()
args.cuda = True if not args.no_cuda and torch.cuda.is_available() else False

print(args)

if args.cuda:
	device = get_freer_gpu()
else:
	device = torch.device('cpu')

torch.manual_seed(args.seed)
if args.cuda:
	torch.cuda.manual_seed(args.seed)

train_dataset = Loader(hdf5_clean = args.train_hdf_path+'train_clean.hdf', hdf5_attack = args.train_hdf_path+'train_attack.hdf', max_nb_frames = args.n_frames, n_cycles=args.n_cycles, augment=True, label_smoothing=args.smoothing)
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers)

if args.valid_hdf_path is not None:
	valid_dataset = Loader(hdf5_clean = args.valid_hdf_path+'valid_clean.hdf', hdf5_attack = args.valid_hdf_path+'valid_attack.hdf', max_nb_frames = args.n_frames, n_cycles=args.valid_n_cycles, augment=False, label_smoothing=0.0)
	valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=args.valid_batch_size, shuffle=False, worker_init_fn=set_np_randomseed)
else:
	valid_loader=None

if args.model == 'lstm':
	model = model_.cnn_lstm()
elif args.model == 'resnet':
	model = model_.ResNet(resnet_type=args.resnet_type)
elif args.model == 'VGG':
	model = model_.VGG(vgg_name=args.vgg_type)
elif args.model == 'resnet_pca':
	model = model_.ResNet_pca(resnet_type=args.resnet_type)
elif args.model == 'wideresnet':
	model = model_.WideResNet()
elif args.model == 'lcnn_9':
	model = model_.lcnn_9layers()
elif args.model == 'lcnn_29':
	model = model_.lcnn_29layers_v2()
elif args.model == 'lcnn_9_pca':
	model = model_.lcnn_9layers_pca()
elif args.model == 'lcnn_29_pca':
	model = model_.lcnn_29layers_v2_pca()
elif args.model == 'lcnn_9_icqspec':
	model = model_.lcnn_9layers_icqspec()
elif args.model == 'lcnn_9_prodspec':
	model = model_.lcnn_9layers_prodspec()
elif args.model == 'lcnn_9_CC':
	model = model_.lcnn_9layers_CC(ncoef=args.ncoef, init_coef=args.init_coef)
elif args.model == 'lcnn_29_CC':
	model = model_.lcnn_29layers_CC(ncoef=args.ncoef, init_coef=args.init_coef)
elif args.model == 'resnet_CC':
	model = model_.ResNet_CC(ncoef=args.ncoef, init_coef=args.init_coef, resnet_type=args.resnet_type)
elif args.model == 'TDNN':
	model = model_.TDNN(ncoef=args.ncoef, init_coef=args.init_coef)
elif args.model == 'TDNN_multipool':
	model = model_.TDNN_multipool(ncoef=args.ncoef, init_coef=args.init_coef)
elif args.model == 'TDNN_ablation':
	model = model_.TDNN_ablation(ncoef=args.ncoef, init_coef=args.init_coef)
elif args.model == 'TDNN_LSTM':
	model = model_.TDNN_LSTM(ncoef=args.ncoef)
elif args.model == 'FTDNN':
	model = model_.FTDNN(ncoef=args.ncoef, init_coef=args.init_coef)
elif args.model == 'mobilenet':
	model = model_.MobileNetV3_Small()
elif args.model == 'densenet':
	model = model_.DenseNet()

if args.pretrained_path is not None:
	ckpt = torch.load(args.pretrained_path, map_location = lambda storage, loc: storage)

	try:
		model.load_state_dict(ckpt['model_state'], strict=True)
	except RuntimeError as err:
		print("Runtime Error: {0}".format(err))
		model.load_state_dict(ckpt['model_state'], strict=False)
	except:
		print("Unexpected error:", sys.exc_info()[0])
		raise

model = model.to(device)

optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.l2)

if args.logdir:
	writer = SummaryWriter(log_dir=args.logdir, comment=args.model, purge_step=0 if args.checkpoint_epoch is None else int(args.checkpoint_epoch*len(train_loader)))
else:
	writer = None

trainer = TrainLoop(model, optimizer, train_loader, valid_loader, patience=args.patience, max_gnorm=args.max_gnorm, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda, logger=writer)

print('\n')
print(model)
print('\n')

args_dict = dict(vars(args))
for key in args_dict:
	print('{}: {}'.format(key, args_dict[key]))
print('\n')

trainer.train(n_epochs=args.epochs, save_every=args.save_every)

ext: py

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Dec  5 09:25:46 2017

@author: ben
"""
import numpy as np
import scipy.sparse as sp
from LSsurf.fd_grid import fd_grid

class lin_op:
    def __init__(self, grid=None, row_0=0, col_N=None, col_0=None, name=None):
        # a lin_op is an operator that represents a set of linear equations applied
        # to the nodes of a grid (defined in fd_grid.py)
        if col_0 is not None:
            self.col_0=col_0
        elif grid is not None:
            self.col_0=grid.col_0
        self.col_N=None
        if col_N is not None:
            self.col_N=col_N
        elif grid is not None:
            self.col_N=grid.col_N
        self.row_0=row_0
        self.N_eq=0
        self.name=name
        self.id=None
        self.r=np.array([], dtype=int)
        self.c=np.array([], dtype=int)
        self.v=np.array([], dtype=float)
        self.ind0=np.zeros([0], dtype=int)
        self.TOC={'rows':dict(),'cols':dict()}
        self.grid=grid
        self.dst_grid=None
        self.dst_ind0=None
        self.expected=None
        self.shape=None
        self.size=None

    def __update_size_and_shape__(self):
        self.shape = (self.N_eq, self.col_N)

    def diff_op(self, delta_subs, vals,  which_nodes=None, valid_equations_only=True):
        # build an operator that calculates linear combination of the surrounding
        # values at each node of a grid.
        # A template, given by delta_subs and vals contains a list of offsets
        # in each direction of the grid, and a list of values corresponding
        # to each offset.  Only those nodes for which the template falls
        # entirely inside the grid are included in the operator

        if valid_equations_only:
            # compute the maximum and minimum offset in each dimension.  These
            # will be used to eliminate equations that extend outside the model
            # domain
            max_deltas=[np.max(delta_sub) for delta_sub in delta_subs]
            min_deltas=[np.min(delta_sub) for delta_sub in delta_subs]
        else:
            # treat the maximum and minimum offset in each dimension as zero,
            # so no equations are truncated
            max_deltas=[0 for delta_sub in delta_subs]
            min_deltas=[0 for delta_sub in delta_subs]
        #generate the center-node indices for each calculation
        # if in dimension k, min_delta=-a and max_delta = +b, the number of indices is N,
        # then the first valid center is a and the last is N-b
        sub0s=np.meshgrid(*[np.arange(np.maximum(0, -min_delta), np.minimum(Ni, Ni-max_delta)) for Ni, min_delta, max_delta in zip(self.grid.shape, min_deltas, max_deltas)], indexing='ij')
        sub0s=[sub.ravel() for sub in sub0s]
        if which_nodes is not None:
            temp_mask=np.in1d(self.grid.global_ind(sub0s), which_nodes)
            sub0s=[temp[temp_mask] for temp in sub0s]
        self.r, self.c=[np.zeros((len(sub0s[0]), len(delta_subs[0])), dtype=int) for _ in range(2)]
        self.v=np.zeros_like(self.r, dtype=float)
        self.N_eq=len(sub0s[0])
        # loop over offsets
        for ii in range(len(delta_subs[0])):
            # build a list of subscripts over dimensions
            this_sub=[sub0+delta[ii] for sub0, delta in zip(sub0s, delta_subs)]
            self.r[:,ii]=self.row_0+np.arange(0, self.N_eq, dtype=int)
            if valid_equations_only:
                self.c[:,ii]=self.grid.global_ind(this_sub)
                self.v[:,ii]=vals[ii].ravel()
            else:
                # need to remove out-of-bound subscripts
                self.c[:,ii], valid_ind=self.grid.global_ind(this_sub, return_valid=True)
                self.v[:,ii]=vals[ii].ravel()*valid_ind.ravel()
        #if not valid_equations_only: [Leave this commented until it causes a problem]
        #    # remove the elements that have v=0
        #    nonzero_v = self.v.ravel() != 0
        #    self.r = self.r.ravel()[nonzero_v]
        #    self.c = self.c.ravel()[nonzero_v]
        #    self.v = self.v.ravel()[nonzero_v]
        self.ind0 = self.grid.global_ind(sub0s).ravel()
        self.TOC['rows'] = {self.name:range(self.N_eq)}
        self.TOC['cols'] = {self.grid.name:np.arange(self.grid.col_0, self.grid.col_0+self.grid.N_nodes)}
        self.__update_size_and_shape__()
        return self

    def add(self, op):
        # combine a set of operators into a composite operator by adding them.
        # the same thing could be accomplished by converting the operators to
        # sparse arrays and adding the arrays, but this method keeps track of the
        # table of contents for the operators.
        # if a list of operators is provided, all are added together, or a single
        # operator can be added to an existing operator.
        if isinstance(op, list) or isinstance(op, tuple):
            for this_op in op:
                op.add(self, this_op)
            return self
        if self.r is not None:
            self.r=np.append(self.r, op.r)
            self.c=np.append(self.c, op.c)
            self.v=np.append(self.v, op.v)
            self.ind0=np.append(self.ind0, op.ind0)
        else:
            self.r=op.r
            self.c=op.c
            self.v=op.v
            self.ind0=op.ind0
        # assume that the new op may have columns that aren't in self.cols, and
        # add any new columns to the table of contents
        for key in op.TOC['cols'].keys():
            self.TOC['cols'][key]=op.TOC['cols'][key]
        self.col_N=np.maximum(self.col_N, op.col_N)
        self.__update_size_and_shape__()
        return self

    def interp_mtx(self, pts):
        # create a matrix that, when it multiplies a set of nodal values,
        # gives the bilinear interpolation between those nodes at a set of
        # data points
        pts=[pp.ravel() for pp in pts]
        # Identify the nodes surrounding each data point
        # The floating-point subscript expresses the point locations in terms
        # of their grid positions
        ii=self.grid.float_sub(pts)
        cell_sub=self.grid.cell_sub_for_pts(pts)
        # calculate the fractional part of each cell_sub
        i_local=[a-b for a, b in zip(ii,cell_sub)]
        # find the index of the node below each data point
        global_ind=self.grid.global_ind(cell_sub)
        # make a list of dimensions based on the dimensions of the grid
        if self.grid.N_dims==1:
            list_of_dims=np.mgrid[0:2]
        elif self.grid.N_dims==2:
            list_of_dims=np.mgrid[0:2, 0:2]
        elif self.grid.N_dims==3:
            list_of_dims=np.mgrid[0:2, 0:2, 0:2]
        delta_ind=np.c_[[kk.ravel() for kk in list_of_dims]]
        n_neighbors=delta_ind.shape[1]
        Npts=len(pts[0])
        rr=np.zeros([Npts, n_neighbors], dtype=int)
        cc=np.zeros([Npts, n_neighbors], dtype=int)
        vv= np.ones([Npts, n_neighbors], dtype=float)
        # make lists of row and column indices and weights for the nodes
        for ii in range(n_neighbors):
            rr[:,ii]=np.arange(len(pts[0]), dtype=int)
            cc[:,ii]=global_ind+np.sum(self.grid.stride*delta_ind[:,ii])
            for dd in range(self.grid.N_dims):
                if delta_ind[dd, ii]==0:
                    vv[:,ii]*=(1.-i_local[dd])
                else:
                    vv[:,ii]*=i_local[dd]
        self.r=rr
        self.c=cc
        self.v=vv
        self.N_eq=Npts
        # in this case, sub0s is the index of the data points
        self.ind0=np.arange(0, Npts, dtype='int')
        # report the table of contents
        self.TOC['rows']={self.name:np.arange(self.N_eq, dtype='int')}
        self.TOC['cols']={self.grid.name:np.arange(self.grid.col_0, self.grid.col_0+self.grid.N_nodes)}
        self.__update_size_and_shape__()
        return self

    def grad(self, DOF='z'):
        coeffs=np.array([-1., 1.])/(self.grid.delta[0])
        dzdx=lin_op(self.grid, name='d'+DOF+'_dx').diff_op(([0, 0],[-1, 0]), coeffs)
        dzdy=lin_op(self.grid, name='d'+DOF+'_dy').diff_op(([-1, 0],[0, 0]), coeffs)
        self.vstack((dzdx, dzdy))
        self.__update_size_and_shape__()
        return self

    def grad_dzdt(self, DOF='z', t_lag=1):
        coeffs=np.array([-1., 1., 1., -1.])/(t_lag*self.grid.delta[0]*self.grid.delta[2])
        d2zdxdt=lin_op(self.grid, name='d2'+DOF+'_dxdt').diff_op(([ 0, 0,  0, 0], [-1, 0, -1, 0], [-t_lag, -t_lag, 0, 0]), coeffs)
        d2zdydt=lin_op(self.grid, name='d2'+DOF+'_dydt').diff_op(([-1, 0, -1, 0], [ 0, 0,  0, 0], [-t_lag, -t_lag, 0, 0]), coeffs)
        self.vstack((d2zdxdt, d2zdydt))
        self.__update_size_and_shape__()
        return self

    def diff(self, lag=1, dim=0):
        coeffs=np.array([-1., 1.])/(lag*self.grid.delta[dim])
        deltas=[[0, 0] for this_dim in range(self.grid.N_dims)]
        deltas[dim]=[0, lag]
        self.diff_op((deltas), coeffs)
        self.__update_size_and_shape__()
        return self

    def dzdt(self, lag=1, DOF='dz'):
        coeffs=np.array([-1., 1.])/(lag*self.grid.delta[2])
        self.diff_op(([0, 0], [0, 0], [0, lag]), coeffs)
        self.__update_size_and_shape__()
        self.update_dst_grid([0, 0, 0.5*lag*self.grid.delta[2]], np.array([1, 1, 1]))
        return self

    def d2z_dt2(self, DOF='dz', t_lag=1):
        coeffs=np.array([-1, 2, -1])/((t_lag*self.grid.delta[2])**2)
        self=lin_op(self.grid, name='d2'+DOF+'_dt2').diff_op(([0,0,0], [0,0,0], [-t_lag, 0, t_lag]), coeffs)
        self.__update_size_and_shape__()
        return self

    def grad2(self, DOF='z'):
        coeffs=np.array([-1., 2., -1.])/(self.grid.delta[0]**2)
        d2zdx2=lin_op(self.grid, name='d2'+DOF+'_dx2').diff_op(([0, 0, 0],[-1, 0, 1]), coeffs)
        d2zdy2=lin_op(self.grid, name='d2'+DOF+'_dy2').diff_op(([-1, 0, 1],[0, 0, 0]), coeffs)
        d2zdxdy=lin_op(self.grid, name='d2'+DOF+'_dxdy').diff_op(([-1, -1, 1,1],[-1, 1, -1, 1]), 0.5*np.array([-1., 1., 1., -1])/(self.grid.delta[0]**2))
        self.vstack((d2zdx2, d2zdy2, d2zdxdy))
        self.__update_size_and_shape__()
        return self

    def grad2_dzdt(self, DOF='z', t_lag=1):
        coeffs=np.array([-1., 2., -1., 1., -2., 1.])/(t_lag*self.grid.delta[0]**2.*self.grid.delta[2])
        d3zdx2dt=lin_op(self.grid, name='d3'+DOF+'_dx2dt').diff_op(([0, 0, 0, 0, 0, 0],[-1, 0, 1, -1, 0, 1], [-t_lag,-t_lag,-t_lag, 0, 0, 0]), coeffs)
        d3zdy2dt=lin_op(self.grid, name='d3'+DOF+'_dy2dt').diff_op(([-1, 0, 1, -1, 0, 1], [0, 0, 0, 0, 0, 0], [-t_lag, -t_lag, -t_lag, 0, 0, 0]), coeffs)
        coeffs=np.array([-1., 1., 1., -1., 1., -1., -1., 1.])/(self.grid.delta[0]**2*self.grid.delta[2])
        d3zdxdydt=lin_op(self.grid, name='d3'+DOF+'_dxdydt').diff_op(([-1, 0, -1, 0, -1, 0, -1, 0], [-1, -1, 0, 0, -1, -1, 0, 0], [-t_lag, -t_lag, -t_lag, -t_lag, 0, 0, 0, 0]),  coeffs)
        self.vstack((d3zdx2dt, d3zdy2dt, d3zdxdydt))
        self.__update_size_and_shape__()
        return self

    def normalize_by_unit_product(self, wt=1):
        # normalize an operator by its magnitude's product with a vector of ones.
        # optionally rescale the result by a factor of wt
        unit_op=lin_op(col_N=self.col_N)
        unit_op.N_eq=self.N_eq
        unit_op.r, unit_op.c, unit_op.v = [self.r, self.c, np.abs(self.v)]
        unit_op.__update_size_and_shape__()
        norm = unit_op.toCSR(row_N=unit_op.N_eq).dot(np.ones(self.shape[1]))
        scale = np.zeros_like(norm)
        scale[norm>0] = 1./norm[norm>0]
        self.v *= scale[self.r]*wt

    def mean_of_bounds(self, bds, mask=None):
        # make a linear operator that calculates the mean of all points
        # in its grid that fall within bounds specified by 'bnds',  If an
        # empty matrix is specified for a dimension, the entire dimension is
        # included.
        # optionally, a 'mask' variable can be used to select from within the
        # bounds.

        coords=np.meshgrid(*self.grid.ctrs, indexing='ij')
        in_bds=np.ones_like(coords[0], dtype=bool)
        for dim, bnd in enumerate(bds):
            if bds[dim] is not None:
                in_bds=np.logical_and(in_bds, np.logical_and(coords[dim]>=bnd[0], coords[dim] <= bnd[1]));
        if mask is not None:
            in_bds=np.logical_and(in_bds, mask)
        self.c=self.grid.global_ind(np.where(in_bds))
        self.r=np.zeros(in_bds.ravel().sum(), dtype=int)
        self.v=np.ones(in_bds.ravel().sum(), dtype=float)/np.sum(in_bds.ravel())
        self.TOC['rows']={self.name:self.r}
        self.TOC['cols']={self.name:self.c}
        self.N_eq=1.
        self.__update_size_and_shape__()
        return self

    def mean_of_mask(self, mask, dzdt_lag=None):
        # make a linear operator that takes the mean of points multiplied by
        # a 2-D mask.  If the grid has a time dimension, the operator takes the
        # mean for each time slice.  If dzdt_lags are provided, it takes the
        # mean dzdt as a function of time
        coords=np.meshgrid(*self.grid.ctrs[0:2], indexing='ij')
        mask_g=mask.interp(coords[1], coords[0])
        mask_g[~np.isfinite(mask_g)]=0
        i0, j0 = np.nonzero(mask_g)
        if self.grid.cell_area is None:
            v0=mask_g.ravel()[np.flatnonzero(mask_g)]
        else:
            v0=(mask_g*self.grid.cell_area).ravel()[np.flatnonzero(mask_g)]
        v0 /= v0.sum()
        y0=np.sum((self.grid.bds[0][0]+i0.ravel()*self.grid.delta[0])*v0)
        x0=np.sum((self.grid.bds[1][0]+j0.ravel()*self.grid.delta[1])*v0)

        if len(self.grid.shape) < 3:
            # no time dimension: Just average the grid
            self.r = np.zeros_like(i0)
            self.c = self.grid.global_ind([i0,j0])
            self.v = v0
            self.N_eq=1
            self.col_N=np.max(self.c)+1
            self.__update_size_and_shape__()
            self.dst_grid = fd_grid( [[y0, y0], [x0, x0]], \
                                self.grid.delta, 0,  col_N=0,
                                srs_proj4=self.grdi.srs_proj4)
            self.dst_ind0 = np.array([0]).astype(int)
            return self
        rr, cc, vv = [[],[], []]
        if dzdt_lag is None:
            # average each time slice
            for ii in range(self.grid.shape[2]):
                rr += [np.zeros_like(i0)+ii]
                cc += [self.grid.global_ind([i0, j0, np.zeros_like(i0)+ii])]
                vv += [v0]
            t_vals=self.grid.ctrs[2]
        else:
            for ii in range(self.grid.shape[2]-dzdt_lag):
                for dLag in [0, dzdt_lag]:
                    rr += [np.zeros_like(i0)+ii]
                    cc += [self.grid.global_ind([i0, j0, np.zeros_like(i0) + ii+ dLag])]
                    if dLag==0:
                        vv += [-v0/dzdt_lag/self.grid.delta[2]]
                    else:
                        vv += [v0/dLag/self.grid.delta[2]]
            t_vals=self.grid.ctrs[-1][:-dzdt_lag] + self.grid.delta[-1]*dzdt_lag/2
        self.r, self.c, self.v = [ np.concatenate(ii) for ii in [rr, cc, vv]]
        self.dst_grid = fd_grid( [[y0, y0], [x0, x0], [t_vals[0], t_vals[-1]]], \
                                self.grid.delta, 0,  col_N=self.r.max()+1,
                                srs_proj4=self.grid.srs_proj4)
        self.N_eq = self.r.max()+1
        self.__update_size_and_shape__()
        self.dst_ind0=np.arange(self.N_eq, dtype=int)
        return self

    def sum_to_grid3(self, kernel_size,  sub0s=None, lag=None, taper=True, valid_equations_only=True, dims=None):
        # make an operator that adds values with a kernel of size kernel_size pixels
        # centered on the grid cells identified in sub0s
        # optional: specify 'lag' to compute a dz/dt
        # specify taper=True to include half-weights on edge points and
        #          quarter-weights on corner points
        if taper:
            half_kernel=np.floor((kernel_size-1)/2).astype(int)
        else:
            half_kernel=np.floor(kernel_size/2).astype(int)

        if dims is None:
            dims=range(len(self.grid.shape))
            n_dims=len(dims)
        else:
            n_dims=len(dims)

        if sub0s is None:
            if taper:
                if valid_equations_only:
                    sub0s = np.meshgrid( *[np.arange(half_kernel+1, self.grid.shape[ii], kernel_size[ii]-1, dtype=int) for ii in dims], indexing='ij')
                else:
                    sub0s = np.meshgrid( *[np.arange(0, self.grid.shape[ii]+1, kernel_size[ii]-1, dtype=int) for ii in dims], indexing='ij')
            else:
                sub0s = np.meshgrid(*[np.arange(half_kernel, self.grid.shape[ii], kernel_size[ii], dtype=int) for ii in dims], indexing='ij')
        ind0 = self.grid.global_ind(sub0s[0:n_dims])

        if np.mod(kernel_size[0]/2,1)==0:
            # even case
            di, dj = np.meshgrid(np.arange(-half_kernel[0], half_kernel[0]),\
                             np.arange(-half_kernel[1], half_kernel[1]), indexing='ij')
            grid_shift=[-self.grid.delta[0]/2, -self.grid.delta[1]/2, 0][0:n_dims]
        else:
            # odd_case
            di, dj = np.meshgrid(np.arange(-half_kernel[0], half_kernel[0]+1),\
                             np.arange(-half_kernel[1], half_kernel[1]+1), indexing='ij')
            grid_shift=[0, 0, 0][0:len(dims)]

        # make the weighting matrix:
        wt0=np.ones(kernel_size[0:2], dtype=float)
        if taper:
            for ii in [0, -1]:
                wt0[ii, :] /= 2
                wt0[:, ii] /= 2
        wt0 = wt0.ravel()

        if lag is None:
            delta_subs=[di.ravel(), dj.ravel(), np.zeros_like(di.ravel())]
            wt=wt0
            grid_shift=[0, 0, 0]
        else:
            delta_subs=[
                np.concatenate([di.ravel(), di.ravel()]),
                np.concatenate([dj.ravel(), dj.ravel()]),
                np.concatenate([np.zeros_like(di.ravel(), dtype=int), np.zeros_like(di.ravel(), dtype=int)+lag])]
            wt = np.concatenate([-wt0, wt0])/(lag*self.grid.delta[2])
            grid_shift[2] = 0.5*lag*self.grid.delta[2]

        self.diff_op( delta_subs, wt.astype(float), which_nodes = ind0,\
                     valid_equations_only=valid_equations_only )
        if taper:
            self.update_dst_grid(grid_shift, kernel_size-1)
        else:
            self.update_dst_grid(grid_shift, kernel_size)

        return self

    def update_dst_grid(self, grid_shift, kernel_size):
        rcv0 = np.unravel_index(self.ind0-self.grid.col_0, self.grid.shape)
        # make a destination grid that spans the output data
        # the grid centers are shifted by grid_shift in each dimension
        dims=range(len(self.grid.shape))
        self.dst_grid = fd_grid(\
            [ [ self.grid.ctrs[dim][rcv0[dim][jj]] + grid_shift[dim] for jj in [0, -1]] for dim in dims],\
            kernel_size*self.grid.delta, name=self.name)
        # map the dst_ind0 value in the output grid
        out_subs = [ ((rcv0[dim]-rcv0[dim][0])/kernel_size[dim]).astype(int) for dim in dims ]
        self.dst_ind0 = np.ravel_multi_index( out_subs, self.dst_grid.shape)

        return self

    def data_bias(self, ind, val=None, col=None):
        # make a linear operator that returns a particular model parameter.
        # can be used to add one model parameter to a set of other parameters,
        # when added to another matrix, or to force a model parameter towards
        # a particular value, when appended to another matrix
        if col is None:
            col=self.col_N
            self.col_N +=1
        self.r=ind
        self.c=np.zeros_like(ind, dtype='int')+col
        if val is None:
            self.v=np.ones_like(ind, dtype='float')
        else:
            self.v=val.ravel()
        self.TOC['rows']={self.name:np.unique(self.r)}
        self.TOC['cols']={self.name:np.unique(self.c)}
        self.N_eq=np.max(ind)+1
        self.__update_size_and_shape__()
        return self

    def grid_prod(self, m, grid=None):
        # dot an operator with a vector, map the result to a grid
        if grid is None:
            if self.dst_grid is None:
                grid=self.grid
            else:
                grid=self.dst_grid
        if self.dst_ind0 is None:
            ind0=self.ind0
        else:
            ind0=self.dst_ind0
        P=np.zeros(grid.col_N+1)+np.NaN
        P[ind0]=self.toCSR(row_N=ind0.size).dot(m).ravel()
        return P[grid.col_0:grid.col_N].reshape(grid.shape)

    def grid_error(self, Rinv, grid=None):
        # calculate the error estimate for an operator and map the result to a grid
        if grid is None:
            if self.dst_grid is None:
                grid=self.grid
            else:
                grid=self.dst_grid
        if self.dst_ind0 is None:
            ind0=self.ind0
        else:
            ind0=self.dst_ind0
        E=np.zeros(self.col_N)+np.NaN
        E[ind0]=np.sqrt((self.toCSR(row_N=ind0.size).dot(Rinv)).power(2).sum(axis=1)).ravel()
        return E[grid.col_0:grid.col_N].reshape(grid.shape)

    def vstack(self, ops, order=None, name=None, TOC_cols=None):
        # combine a set of operators by stacking them vertically to form
        # a composite operator.  This could also be done by converting
        # the operators to sparse matrices and stacking them using the
        # vstack function, but using the lin_op.vstack() keeps track of
        # the different equations in the TOC
        if isinstance(ops, lin_op):
            ops=(self, ops)
        if order is None:
            order=range(len(ops))
        if name is not None:
            self.name=name
        if TOC_cols is None:
            TOC_cols=dict()
            col_array=np.array([], dtype=int)
            for op in ops:
                for key in op.TOC['cols'].keys():
                    TOC_cols[key]=op.TOC['cols'][key]
                    col_array=np.append(col_array, op.TOC['cols'][key])
            # add an entry for this entire operator
            if self.name is not None:
                TOC_cols[self.name]=np.unique(col_array)
        if self.col_N is None:
            self.col_N=np.max(np.array([op.col_N for op in ops]))

        self.TOC['cols']=TOC_cols
        # rr, cc, and vv are lists that will be populated with the nonzero
        # entries for each matrix being combined.
        rr=list()
        cc=list()
        vv=list()
        last_row=0
        for ind in order:
            # append the nonzero entries to the list of entries
            rr.append(ops[ind].r.ravel()+last_row)
            cc.append(ops[ind].c.ravel())
            vv.append(ops[ind].v.ravel())
            # label these equations in the TOC
            this_name=ops[ind].name
            if this_name is None:
                this_name='eq'
            # if TOC already contains the name we've specified, add a number to
            # distinguish this
            temp_name=this_name
            name_suffix=0
            while temp_name in self.TOC['rows'].keys():
                name_suffix+=1
                temp_name="%s_%d" %(this_name, name_suffix)
            if name_suffix>0:
                this_name="%s_%d" %(this_name, name_suffix)
            # shift the TOC entries and keep track of what sub-operators make up the current operator
            this_row_list=list()
            for key in ops[ind].TOC['rows'].keys():
                these_rows=np.array(ops[ind].TOC['rows'][key], dtype='int')+last_row
                self.TOC['rows'][key]=these_rows
                this_row_list.append(these_rows.ravel())
            # add a TOC entry for all of the sub operators together, if it's
            # not there already (which happens if we're concatenating composite operators)
            if this_name not in self.TOC['rows']:
                self.TOC['rows'][this_name]=np.concatenate(this_row_list)
            last_row+=ops[ind].N_eq
        # Combine the nonzero entries
        self.N_eq=last_row
        self.r=np.concatenate(rr)
        self.c=np.concatenate(cc)
        self.v=np.concatenate(vv)

        self.ind0=np.concatenate([op.ind0 for op in ops])
        if self.name is not None and len(self.name) >0:
            self.TOC['rows'][self.name]=np.arange(0, last_row)
        self.__update_size_and_shape__()
        return self

    def mask_for_ind0(self, mask_scale=None):
        """
        Sample the mask at the central indices for a linear operator

        This function samples the linear operator's mask field at the indices
        corresponding to the 'ind0' for each row of the operator.  The only
        input is:
             mask_scale (dict, or none):   gives the mapping between key values
                  and output values:  if mask_scale={1:0, 2:1}, then all mask
                  values equal to 1 will be returned as zero, and all mask values
                  equal to 2 will be returned as 1.
        """

        if self.grid.mask is None:
            return np.ones_like(self.ind0, dtype=float)
        # if the operator's grid has more dimensions than the mask does,
        # need to use the first two indices to pick the grid cells
        if len(self.grid.shape) > len(self.grid.mask.shape):
            temp=np.unravel_index(self.ind0-self.grid.col_0, self.grid.shape)
            subs=tuple([temp[ii] for ii in range(len(self.grid.mask.shape))])
        else:
            inds=self.ind0-self.grid.col_0
            subs=np.unravel_index(inds, self.grid.mask.shape)
        temp=self.grid.mask[subs]
        if mask_scale is not None:
            temp2=np.zeros_like(temp, dtype=np.float)
            for key in mask_scale.keys():
                temp2[temp==key]=mask_scale[key]
            return temp2
        else:
            return temp

    def apply_2d_mask(self, mask=None):
        # multiply array elements by the values in a mask
        # The mask must have dimensions equal to the first two dimensions of
        # self.grid
        # if no mask is specified, use self.grid.mask
        if mask is None:
            mask=self.grid.mask
        csr=self.toCSR()

        for row in range(csr.shape[0]):
            # get the indices of the nonzero entries for the row
            inds=csr[row,:].nonzero()[1]
            # get the mask subscripts for the indices
            subs=np.unravel_index(inds-self.grid.col_0, self.grid.shape)
            # query the mask at those points
            mask_ind=np.ravel_multi_index([subs[0], subs[1]], mask.shape)
            # want to do: csr[row,inds] *= mask.ravel()[mask_ind]
            # but need to add a toarray() step to avoid broadcasting rules
            temp = csr[row, inds].toarray()
            csr[row,inds] = temp.ravel()*mask.ravel()[mask_ind]
        temp=csr.tocoo()
        self.r, self.c, self.v=[temp.row, temp.col, temp.data]
        return self

    def print_TOC(self):
        for rc in ('cols','rows'):
            print(rc)
            rc_min={k:np.min(self.TOC[rc][k]) for k in self.TOC[rc].keys()}
            for key in sorted(rc_min, key=rc_min.get):
                print("\t%s\t%d : %d" % (key, np.min(self.TOC[rc][key]), np.max(self.TOC[rc][key])))

    def fix_dtypes(self):
        self.r=self.r.astype(int)
        self.c=self.c.astype(int)

    def toCSR(self, col_N=None, row_N=None):
        # transform a linear operator to a sparse CSR matrix
        if col_N is None:
            col_N=self.col_N
        self.fix_dtypes()
        good=self.v.ravel() != 0
        if row_N is None:
            row_N=np.max(self.r.ravel()[good])+1
        return sp.csr_matrix((self.v.ravel()[good],(self.r.ravel()[good], self.c.ravel()[good])), shape=(row_N, col_N))

max_stars_repo_name: SmithB/LSsurf

max_issues_repo_name: tsutterley/LSsurf

max_forks_repo_name: tsutterley/LSsurf

reason: import numpy,import scipy


text:
import base64
import io
import json
import os
import urllib
from collections import defaultdict
from typing import Dict, Tuple

import click
import numpy
import pandas
import seaborn
from matplotlib import pyplot
from tqdm import tqdm

seaborn.set_palette("colorblind")

def _extract_molecule_profiles(
    energy_frame: pandas.DataFrame, smiles: str
) -> Tuple[numpy.ndarray, numpy.ndarray, Dict[str, numpy.ndarray]]:
    """Extracts the QM and MM energies profiles from a master pandas data frame
    for a particular molecule defined by its SMILES pattern.

    Returns
        The grid angles, QM profile and MM profiles for each force field.
    """

    smiles_frame = energy_frame[energy_frame["SMILES"] == smiles]

    qm_energies = {}
    mm_energies = defaultdict(dict)

    for _, row in smiles_frame.iterrows():
        qm_energies[int(row["Grid ID"])] = row["QM Energy"]
        mm_energies[row["Force Field"]][int(row["Grid ID"])] = row["MM Energy"]

    grid_ids = sorted(grid_id for grid_id in qm_energies)

    qm_energies = numpy.array([qm_energies[grid_id] for grid_id in grid_ids])

    mm_energies = {
        mm_label: numpy.array([mm_energies[mm_label][grid_id] for grid_id in grid_ids])
        for mm_label in mm_energies
    }

    return numpy.array(grid_ids), qm_energies, mm_energies

def _compute_profile_rmse(
    qm_energies: numpy.ndarray, mm_energies: numpy.ndarray, normalize: bool, shift: bool
) -> float:
    """Compute the RMSE between a QM and an MM torsion profile after optimally
    superimposing the two."""

    if shift:
        mm_energies = mm_energies + (qm_energies - mm_energies).mean()

    if normalize:
        qm_energies = qm_energies / (qm_energies.max() - qm_energies.min())
        mm_energies = mm_energies / (mm_energies.max() - mm_energies.min())

    delta = numpy.sqrt(
        (qm_energies - mm_energies).dot(qm_energies - mm_energies) / len(qm_energies)
    )

    return float(delta)

def _smiles_to_img(smiles: str) -> str:
    """A helper function for creating an HTML image from a SMILES pattern."""

    from rdkit import Chem
    from rdkit.Chem.Draw import rdMolDraw2D

    rdkit_molecule = Chem.MolFromSmiles(smiles)

    tagged_bonds = [
        bond.GetIdx()
        for bond in rdkit_molecule.GetBonds()
        if bond.GetBeginAtom().GetAtomMapNum() != 0
        and bond.GetEndAtom().GetAtomMapNum() != 0
    ]

    # Generate a set of 2D coordinates.
    Chem.rdDepictor.Compute2DCoords(rdkit_molecule)

    drawer = rdMolDraw2D.MolDraw2DSVG(400, 300)
    rdMolDraw2D.PrepareAndDrawMolecule(
        drawer,
        rdkit_molecule,
        highlightBonds=tagged_bonds,
    )
    drawer.FinishDrawing()

    svg_content = drawer.GetDrawingText()

    return (
        f''
    )

def plot_torsion_profile(
    angles: numpy.ndarray,
    qm_energies: numpy.ndarray,
    mm_energies: Dict[str, numpy.ndarray],
) -> str:
    """Plots a set of QM and MM torsion profiles and returns the result as a HTML
    encoded image.
    """

    figure = pyplot.figure(figsize=(6.0, 4.0))

    pyplot.plot(angles, qm_energies, label="QM")

    for mm_label in mm_energies:
        pyplot.plot(angles, mm_energies[mm_label], label=mm_label)

    pyplot.xlabel("Angle")
    pyplot.ylabel("Energy (kcal / mol)")

    pyplot.legend()

    pyplot.tight_layout()

    with io.BytesIO() as buffer:

        pyplot.gcf().savefig(buffer, format="png")
        buffer.seek(0)
        plot_string = base64.b64encode(buffer.read())

    pyplot.close(figure)

    img_html = f''
    return img_html

def plot_per_force_field_rmse(
    energy_frame: pandas.DataFrame,
    output_path: str,
    normalize: bool,
    shift: bool,
    bootstrap_iterations: int = 2000,
    percentile=0.95,
):
    """Plots the mean RMSE between the QM torsion profile and the MM torsion profile
    computed for a set of force fields, showing bootstrapped error bars."""

    rmse_values = defaultdict(list)

    for smiles in tqdm(energy_frame["SMILES"].unique()):

        angles, qm_energies, mm_energies = _extract_molecule_profiles(
            energy_frame, smiles
        )

        for mm_label in mm_energies:

            rmse_values[mm_label].append(
                _compute_profile_rmse(
                    numpy.array(qm_energies),
                    numpy.array(mm_energies[mm_label]),
                    normalize=normalize,
                    shift=shift,
                )
            )

    method_labels = [*rmse_values]
    plot_data = [numpy.array(rmse_values[mm_label]) for mm_label in method_labels]

    # Bootstrap the data to compute the error bars.
    sample_count = len(plot_data[0])

    sample_values = {
        method: numpy.zeros((bootstrap_iterations, 1)) for method in method_labels
    }

    for sample_index in range(bootstrap_iterations):

        samples_indices = numpy.random.randint(
            low=0, high=sample_count, size=sample_count
        )

        for method, method_data in zip(method_labels, plot_data):
            sample_values[method][sample_index] = method_data[samples_indices].mean()

    lower_percentile_index = int(bootstrap_iterations * (1 - percentile) / 2)
    upper_percentile_index = int(bootstrap_iterations * (1 + percentile) / 2)

    # Produce the figure.
    figure = pyplot.figure(figsize=(4.0, 4.0))

    for i, (method, method_data) in enumerate(zip(method_labels, plot_data)):

        sorted_samples = numpy.sort(sample_values[method], axis=0)

        mean_value = method_data.mean()

        confidence_intervals = numpy.array(
            [
                [float(numpy.abs(mean_value - sorted_samples[lower_percentile_index]))],
                [float(numpy.abs(mean_value - sorted_samples[upper_percentile_index]))],
            ]
        )

        pyplot.errorbar([i], [mean_value], yerr=confidence_intervals, marker="x")

    pyplot.xticks(
        ticks=[*range(len(method_labels))], labels=method_labels, rotation=45.0
    )

    pyplot.tight_layout()
    pyplot.savefig(output_path)
    pyplot.close(figure)

def plot_per_molecule_profile(energy_frame: pandas.DataFrame, output_path: str):

    plot_rows = []

    for smiles in tqdm(energy_frame["SMILES"].unique()):

        angles, qm_energies, mm_energies = _extract_molecule_profiles(
            energy_frame, smiles
        )

        plot_rows.append(
            {
                "Molecule": _smiles_to_img(smiles),
                "Profile": plot_torsion_profile(
                    angles=angles, qm_energies=qm_energies, mm_energies=mm_energies
                ),
                **{
                    f"{mm_label} {normalize_label}": _compute_profile_rmse(
                        numpy.array(qm_energies),
                        numpy.array(mm_energies[mm_label]),
                        normalize=normalize,
                        shift=True,
                    )
                    for mm_label in mm_energies
                    for normalize, normalize_label in [
                        (False, "RMSE"),
                        (False, "Norm RMSE"),
                    ]
                },
            }
        )

    plot_frame = pandas.DataFrame(plot_rows)

    with open(output_path, "w") as file:
        file.write(plot_frame.to_html(escape=False))

@click.command()
@click.option(
    "-i",
    "--input",
    "input_path",
    default="01-residuals.json",
    show_default=True,
    type=click.STRING,
    help="The path to the JSON file containing the energies and residuals produced by "
    "step 01.",
)
@click.option(
    "-o",
    "--output",
    "output_directory",
    default="02-plots",
    show_default=True,
    type=click.STRING,
    help="The directory to save the plots into.",
)
def main(input_path, output_directory):

    os.makedirs(output_directory, exist_ok=True)

    # Store all of the data in a pandas frame for easier manipulation.
    with open(input_path) as file:
        results = json.load(file)

    record_smiles = results["record_smiles"]
    energies = results["energies"]

    energy_frame = pandas.DataFrame(
        [
            {
                "QC Record": record_id,
                "SMILES": record_smiles[record_id],
                "Grid ID": grid_id,
                "Force Field": force_field_label,
                "QM Energy": energies[record_id][grid_id][force_field_label][0],
                "MM Energy": energies[record_id][grid_id][force_field_label][1],
                "MM Residual": energies[record_id][grid_id][force_field_label][2],
            }
            for record_id in energies
            for grid_id in energies[record_id]
            for force_field_label in energies[record_id][grid_id]
        ]
    )

    # Plot the per molecule profiles and global RMSE values.
    print("1) Plotting RMSE values")

    plot_per_force_field_rmse(
        energy_frame,
        os.path.join(output_directory, "torsion-rmse.png"),
        normalize=False,
        shift=True,
    )

    plot_per_force_field_rmse(
        energy_frame,
        os.path.join(output_directory, "torsion-norm-rmse.png"),
        normalize=True,
        shift=True,
    )

    print("2) Plotting per molecule profiles")

    plot_per_molecule_profile(
        energy_frame, os.path.join(output_directory, "torsion-profiles.html")
    )

if __name__ == "__main__":
    main()

max_stars_repo_name: ocmadin/openff-sage

max_issues_repo_name: ocmadin/openff-sage

max_forks_repo_name: ocmadin/openff-sage


text:
import streamlit as st
import numpy as np
import pandas as pd
import time
import altair as alt
import json

@st.cache
def Your_Covid_Data():
  with open('repeated_data.json') as json_f:
    repeated_json = json.load(json_f)
    return alt.Data(values=repeated_json["features"])

@st.cache
def Your_World_Data():
  with open('world_data.json') as json_f:
    world_json = json.load(json_f)
    return alt.Data(values=world_json["features"])

st.title("How Data-Driven are our Covid fears?")

expander = st.beta_expander("Instructions")
expander.write("This visualization app allows you to view how the public sentiment of fear has changed in 15 countries as a result of the novel Coronavirus. By clicking a country on either of the two graphs below, you can view how the Fear Index varied in comparison to new Covid cases and deaths. \n\n By holding shift and pressing multiple countries you can view how Covid fear and Covid statistics vary among different countries. To learn more about Interesting Trends, the Fear Index and Variable Selection, please view the descriptions below. The graphs start at week 11 of the year 2020, which corresponds to the data March 9, 2020.")

repeated_data = Your_Covid_Data()
world_data = Your_World_Data()

my_selector = alt.selection_multi(fields = ["properties.location"],init = [{"properties.location": "United States"},{"properties.location": "India"}])

chart = alt.Chart(repeated_data,title = "Percentage of Elderly and GDP for each Country").mark_circle().encode(
    x = alt.X("properties.gdp_per_capita:Q", title='GDP per Capita'),
    y=alt.Y("properties.aged_65_older:Q", title='Percentage of Elderly'),
    # color=alt.condition(my_selector, alt.Color("properties.Continent:N",title = "Continent"), alt.value('lightgray')),
    color=alt.condition(my_selector, alt.Color("properties.location:N",title = "Countries"), alt.value('lightgray')),
    opacity = alt.condition(my_selector, alt.value(1.0), alt.value(0.15)),
    size=alt.Size("properties.population:Q",scale=alt.Scale(domain=(-400000000, 1850000000)), legend=None),
    tooltip=alt.Tooltip('properties.location:N',title = "Country")

).add_selection(
    my_selector
).properties(
    width=300,
    height=200
)

one_country = alt.Chart(repeated_data).mark_geoshape().encode(
    color= alt.condition(
        # my_selector, alt.Color("properties.Continent:N"), alt.value('gray')
        my_selector, alt.Color("properties.location:N",title = "Countries",scale=alt.Scale(scheme='tableau20'),legend = alt.Legend(orient="top",columns=5, labelOpacity=0.8, columnPadding=67,clipHeight=12)), alt.value('gray')
    ),
    opacity = alt.condition(my_selector, alt.value(1.0), alt.value(0.15)),
    tooltip=alt.Tooltip('properties.location:N',title = "Country")
).properties(
    width=375,
    height=200
).add_selection(
    my_selector
)

world = alt.Chart(world_data).mark_geoshape().encode(
    color=alt.value('lightgray'),
    tooltip=alt.Tooltip('properties.location:N',title = "Country")
).properties(
    width=375,
    height=200
)

boundaries = alt.Chart(world_data, title='World Map').mark_geoshape(
    stroke='white',
    strokeWidth=1,
    fill=None
)

fear = alt.Chart(repeated_data, title='Twitter Fear over Time').mark_line().encode(
    x=alt.X('properties.week:O',title= "Week"),
    y=alt.Y('properties.fear_percentage:Q',title = "Fear Index"),
    color= alt.Color("properties.location:N", legend = alt.Legend())
    # y=alt.Y('mean(properties.fear_percentage):Q',title = "Fear Index"),
    # color= alt.value("gray")
).transform_filter(
    my_selector
).properties(
    width = 700,
    height = 150
)

cases = alt.Chart(repeated_data, title='Covid Infections over Time').mark_line().encode(
    x=alt.X('properties.week:O', title = "Week"),
    y=alt.Y('properties.new_cases_per_million:Q',title="New Cases per Million"),
    color= alt.Color("properties.location:N")
).transform_filter(
    my_selector
).properties(
    width = 700,
    height = 150
)

deaths = alt.Chart(repeated_data, title='Covid Deaths over Time').mark_line().encode(
    x=alt.X('properties.week:O', title = "Week"),
    y=alt.Y('properties.new_deaths_per_million:Q',title="New Deaths per Million"),
    color= alt.Color("properties.location:N")
).transform_filter(
    my_selector
).properties(
    width = 700,
    height = 150
)

st.write((chart | world + one_country  + boundaries) & fear & cases  & deaths)

expander = st.beta_expander("Interesting Trends")
expander.write("Have a look at how the Fear Index precedes the number of infections in first world countries. In most of these countries, the fear index halves by the time Covid cases hit their peak. Do we just become bored when the pandemic is most deadly? \n\nNotice how the death rate is very low in UAE compared to in Ireland; perhaps the lower percentage of elderly could explain this. \n\n Have a look at Covid infections and deaths in third world countries. Could a lack of testing explain the low cases and deaths even when fear is high? \n\n Maybe look at how similar the fear and Covid trends are within countries of the same continent, maybe shared cultural and geographical aspects could be the reason?")

expander = st.beta_expander("What is the Fear Index")
expander.write("The Fear Index is the percentage of Covid-related tweets that contain keywords related to fear. These keywords include conjugations of synonyms of 'fear' such as 'scared', 'afraid' and 'nervous'. \n\n To collect the Tweets, we utilize the publically available Harvard 'Coronavirus Tweet Ids' dataset.")

expander = st.beta_expander("Variable Selection")
expander.write("The variables GDP per Capita and percentage of population above 65 were chosen because our intuition would tell us that our Covid fears should increase as these variables increase. This is because Covid is more dangerous when hospital beds cannot be afforded or when it affects a member of the elderly.")


text:
import pygame
import numpy as np

class Control:

    def __init__(self):

        self.key = np.array([-1.5, 0.0, -0.5, 0.0, 0.0, 1.0])
        '''key[0] - W S
           key[1] - A D
           key[2] - W S
           key[3] - LEFT RIGHT
           key[4] - UP DOWN
           key[5] - Z X
        '''

    def pressed_keys(self):
        keys = pygame.key.get_pressed()

        if keys[pygame.K_w]:
            self.key[0] -= 0.06
        if keys[pygame.K_s]:
            self.key[0] += 0.06
        if keys[pygame.K_a]:
            self.key[1] -= 0.04
        if keys[pygame.K_d]:
            self.key[1] += 0.04
        if keys[pygame.K_q]:
            self.key[2] += 0.04
        if keys[pygame.K_e]:
            self.key[2] -= 0.04
        if keys[pygame.K_LEFT]:
            self.key[3] += 0.02
        if keys[pygame.K_RIGHT]:
            self.key[3] -= 0.02
        if keys[pygame.K_UP]:
            self.key[4] += 0.02
        if keys[pygame.K_DOWN]:
            self.key[4] -= 0.02
        if keys[pygame.K_z]:
            self.key[5] -= 1.0
            self.key[5] = max(1, self.key[5])
        if keys[pygame.K_x]:
            self.key[5] += 1.0
            self.key[5] = min(6.0, self.key[5])


text:
from config import *

from selenium import webdriver
from selenium.webdriver.common.keys import Keys

from time import sleep
from getpass import getpass

from os import remove

import zipfile
import pandas as pd
import numpy as np
from lxml import etree as et

def _parseBgeXml(f):
    timestamp = []
    consumed = []
    cost = []

    timezone = config.get('global','timezone')
    for e,elem in et.iterparse(f, tag='{http://naesb.org/espi}IntervalReading'):
        timestamp.append(elem.findall('.//{http://naesb.org/espi}start')[0].text)
        consumed.append( elem.findall('{http://naesb.org/espi}value')[0].text)
        cost.append(     elem.findall('{http://naesb.org/espi}cost')[0].text)
        nt = np.array(timestamp,dtype=int).astype('datetime64[s]')
        nc = np.array(consumed,dtype=float)
        no = np.array(cost,dtype=float)

    nc /= 1e5
    no /= 1e5

    consumed = pd.Series(nc,index=nt).tz_localize('UTC').tz_convert(timezone)
    cost     = pd.Series(no,index=nt).tz_localize('UTC').tz_convert(timezone)

    return (consumed,cost)

def getData(daterange):

    downloadDir = cache_directory

    customer_id = config.get('bge','customer_id')

    if config.has_option('bge','username') and config.has_option('bge','password'):
        username    = config.get('bge','username')
        password    = config.get('bge','password')
    else:
        username = raw_input('Username for bge.com:')
        password = getpass()

    begin = daterange[0].strftime('%Y-%m-%d')
    end   = daterange[-1].strftime('%Y-%m-%d')

    profile = webdriver.FirefoxProfile()
    profile.set_preference('browser.download.folderList',2)
    profile.set_preference('browser.download.manager.showWhenStarting', False)
    profile.set_preference('browser.download.dir',downloadDir)
    profile.set_preference('browser.helperApps.neverAsk.saveToDisk', 'application/octet-stream')

    browser = webdriver.Firefox(profile)
    browser.implicitly_wait(60)

    browser.get('https://www.bge.com/')

    username_element = browser.find_element_by_id('USER')
    password_element = browser.find_element_by_id('PASSWORD')

    username_element.send_keys(username)
    password_element.send_keys(password)
    password_element.send_keys(Keys.RETURN)

    browser.find_element_by_link_text('My Energy Use').click()
    browser.find_element_by_link_text('My Usage Details').click()

    download_url = 'https://bgesmartenergymanager.com/ei/app/modules/customer/%s/energy/download?exportFormat=ESPI_AMI&xmlFrom=%s&xmlTo=%s'%(customer_id,begin,end)

    browser.get(download_url)

    sleep(5)

    browser.quit()

    zf = downloadDir + '/bgec_interval_data_%s_to_%s.zip'%(begin,end)

    with zipfile.ZipFile(zf,'r') as z:
        files = [ data_directory + '/' + fn for fn in z.namelist() ]
        z.extractall(data_directory)

    df = pd.DataFrame()

    for f in files:
        if 'gas' in f:
            consumed_name = 'gas_consumed'
            cost_name = 'gas_cost'
        if 'electric' in f:
            consumed_name = 'electric_consumed'
            cost_name = 'electric_cost'

        consumed,cost = _parseBgeXml(f)

        df.loc[:,consumed_name] = consumed
        df.loc[:,cost_name]     = cost

    remove(zf)
    for f in files:
        remove(f)

    return df


text:
"""
 Copyright (c) 2018 Intel Corporation

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import networkx as nx
import numpy as np

from mo.graph.graph import Node
from mo.ops.op import Op
from mo.utils.error import Error

def duplicate_shared_weights(graph: nx.MultiDiGraph):
    """
    This function finds all const data nodes that have more that one consumer and then duplicate them
    """
    data_nodes = [Node(graph, id) for id in graph.nodes() if Node(graph, id).soft_get('kind') == 'data']
    for node in data_nodes:
        # Check that node has const values and more than one consumer
        if len(node.out_nodes()) > 1 and node.value is not None:
            # Here we delete all edges between base node and it's consumers (except first), and then duplicate this
            # node to connect with other consumers
            while len(node.out_nodes()) > 1:
                out_node = node.out_node(1)

                if len(graph.get_edge_data(node.id, out_node.id)) != 1:
                    raise Error('There is more than one edge from {} node to {} node.'.format(node.id, out_node.id))
                e_attrs = graph.get_edge_data(node.id, out_node.id)[0]

                graph.remove_edge(node.id, out_node.id)
                data = Op.create_input_data_node(graph, "Copy_{}".format(node.id), np.array(node.value), graph.node[node.id])

                graph.add_edges_from([(data.id, out_node.id, e_attrs)])

max_stars_repo_name: apexxs/dldt

max_issues_repo_name: apexxs/dldt

max_forks_repo_name: apexxs/dldt

reason: import numpy,import networkx


text:
import numpy as np
from ..mtd import _ChainBase
from typing import Tuple, Optional, List

class ChainGenerator(_ChainBase):
    """
    Class for generating discrete space datasets

    Parameters
    ----------
    values: list or tuple of strings
        States to be used for generation.

    order: int
        Number of lags of the model.

    sep: string, optional (default='>')
        Separator between states.

    min_len: int, optional (default=None)
        Minimal length of the sequence. If None, then equal to order.

    max_len: int, optional (default=None)
        Maximal length of the sequence. If None, then equal to order.

    transition_matrix: NumPy array, optional (default=None)
        Markov model array, to be used for data generation. If None, then the matrix will be generated.

    lambdas: NumPy array, optional (default=None)
        MTD model weights vector, to be used for data generation. If None, then the vector will be generated.

    transition_matrices: NumPy array, optional (default=None)
        MTD model transition matrices, to be used for data generation. If None, then the matrices will be generated.

    random_state: int, optional (default=None)
        Random state, to be used for NumPy seed.

    Example
    ----------
    from mtdlearn.datasets import ChainGenerator

    cg = ChainGenerator(['A', 'B', 'C'], 4)

    x, y = cg.generate_data(100)

"""

    def __init__(self, values: List[str], order: int, sep: str = '>', min_len: Optional[int] = None,
                 max_len: Optional[int] = None, transition_matrix: Optional[np.ndarray] = None,
                 lambdas: Optional[np.ndarray] = None, transition_matrices: Optional[np.ndarray] = None,
                 random_state: Optional[int] = None) -> None:
        super().__init__(order=order, values=values)
        self._n_dimensions = len(values)
        self.order = order
        self.sep = sep
        if not ((min_len is None and max_len is None) or (min_len is not None and max_len is not None)):
            raise ValueError('if min_len is passed max_len has to be specified and vice versa')
        if min_len is None:
            self.min_len = order
        elif min_len < order:
            raise ValueError('min_len cannot be smaller that order')
        else:
            self.min_len = min_len
        if max_len is None:
            self.max_len = order
        elif max_len < order:
            raise ValueError('max_len cannot be smaller that order')
        else:
            self.max_len = max_len
        if max_len is not None and min_len is not None:
            if max_len < min_len:
                raise ValueError('max_len cannot be smaller that min_len')
        self.transition_matrix = transition_matrix
        self.lambdas = lambdas
        self.transition_matrices = transition_matrices
        self.random_state = random_state
        if self.transition_matrix is None:
            self._generate_mtd_model()
            self._create_markov()
        self._label_dict = {i: j for i, j in enumerate(values)}

    def generate_data(self, samples: int, random_state: Optional[int] = None) -> Tuple[np.ndarray, np.ndarray]:
        """
        Generate data

        :param samples: int
                        Number of samples to be generated.
        :param random_state: int
                             Random state for generation
        :return: NumPy array of shape (samples, 1)
        :return: NumPy array of shape (samples,)
        """

        if random_state is not None:
            np.random.seed(random_state)

        x = np.random.choice(list(self._label_dict.keys()), (samples, self.order))
        y = self.predict_random(x)

        if self.max_len > self.order:
            x_to_add = np.random.randint(self.min_len, self.max_len + 1, samples)
            x = [np.hstack([np.random.choice(list(self._label_dict.keys()), i[1] - self.order), i[0]])
                 for i
                 in zip(x.tolist(), list(x_to_add))]

        x = [self.sep.join(list(map(self._label_dict.get, i))) for i in x]

        return np.array(x).reshape(-1, 1), np.array(y)

    def _generate_mtd_model(self) -> None:
        if self.random_state is not None:
            np.random.seed(self.random_state)
        if self.lambdas is None:
            lambdas = np.random.rand(self.order)
            self.lambdas = lambdas / lambdas.sum()
        if self.transition_matrices is None:
            transition_matrices = np.random.rand(self.order, self._n_dimensions, self._n_dimensions)
            self.transition_matrices = transition_matrices / transition_matrices.sum(2).reshape(self.order,
                                                                                self._n_dimensions, 1)

data_values3_order2_full = dict()
data_values3_order2_full['x'] = np.array([['A>A>A'],
                                          ['A>A>B'],
                                          ['A>A>C'],
                                          ['A>B>A'],
                                          ['A>B>A'],
                                          ['A>B>A'],
                                          ['A>B>B'],
                                          ['A>B>B'],
                                          ['A>B>B'],
                                          ['A>B>C'],
                                          ['A>B>C'],
                                          ['A>B>C'],
                                          ['A>C>A'],
                                          ['A>C>A'],
                                          ['A>C>A'],
                                          ['A>C>B'],
                                          ['A>C>B'],
                                          ['A>C>B'],
                                          ['A>C>C'],
                                          ['A>C>C'],
                                          ['A>C>C'],
                                          ['B>A>A'],
                                          ['B>A>A'],
                                          ['B>A>A'],
                                          ['B>A>B'],
                                          ['B>A>B'],
                                          ['B>A>B'],
                                          ['B>A>C'],
                                          ['B>A>C'],
                                          ['B>A>C'],
                                          ['B>B>A'],
                                          ['B>B>A'],
                                          ['B>B>A'],
                                          ['B>B>B'],
                                          ['B>B>B'],
                                          ['B>B>B'],
                                          ['B>B>C'],
                                          ['B>B>C'],
                                          ['B>B>C'],
                                          ['B>C>A'],
                                          ['B>C>A'],
                                          ['B>C>A'],
                                          ['B>C>B'],
                                          ['B>C>B'],
                                          ['B>C>B'],
                                          ['B>C>C'],
                                          ['B>C>C'],
                                          ['B>C>C'],
                                          ['C>A>A'],
                                          ['C>A>A'],
                                          ['C>A>A'],
                                          ['C>A>B'],
                                          ['C>A>B'],
                                          ['C>A>B'],
                                          ['C>A>C'],
                                          ['C>A>C'],
                                          ['C>A>C'],
                                          ['C>B>A'],
                                          ['C>B>A'],
                                          ['C>B>A'],
                                          ['C>B>B'],
                                          ['C>B>B'],
                                          ['C>B>B'],
                                          ['C>B>C'],
                                          ['C>B>C'],
                                          ['C>B>C'],
                                          ['C>C>A'],
                                          ['C>C>A'],
                                          ['C>C>A'],
                                          ['C>C>B'],
                                          ['C>C>B'],
                                          ['C>C>B'],
                                          ['C>C>C'],
                                          ['C>C>C'],
                                          ['C>C>C']])

data_values3_order2_full['y'] = np.array(['A',
                                          'A',
                                          'A',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C',
                                          'A',
                                          'B',
                                          'C'])

data_values3_order2_full['sample_weight'] = np.array([1000,
                                                      1000,
                                                      1000,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      1000,
                                                      0,
                                                      0,
                                                      1000,
                                                      0,
                                                      0,
                                                      1000,
                                                      0,
                                                      0,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      1000,
                                                      0,
                                                      0,
                                                      1000,
                                                      0,
                                                      0,
                                                      1000,
                                                      0,
                                                      0,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100,
                                                      100])

max_stars_repo_name: PiotrekGa/mtd-learn

max_issues_repo_name: PiotrekGa/mtd-learn

max_forks_repo_name: PiotrekGa/mtd-learn


text:
# --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
# --------------------------------------------------------------------------------------------
"""
general utility functions
"""

import os

import numpy as np
import pandas as pd
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.metrics import roc_auc_score
from sklearn.utils.testing import assert_greater

# select columns from DataFrame insize a pipeline

class ColumnSelector(BaseEstimator, TransformerMixin):
    def __init__(self, columns, ravel=False):
        self.columns = columns
        self.ravel = ravel

    def fit(self, x, y=None):
        return self

    def transform(self, df, drop=False):
        if drop:
            df = df.drop[self.columns]
        else:
            df = df[self.columns]
        if self.ravel:
            return np.ravel(df[self.columns].values)
        else:
            return df[self.columns]

def load_img(filename):
    from imageio import imread
    from scipy.misc import imresize
    with open(filename, "r") as f:
        N = sum(1 for line in f)

    H = 32
    W = 32
    C = 3
    X = np.zeros((N, H, W, C))
    y = np.zeros(N, dtype=np.uint8)
    base_path = os.path.dirname(filename)
    with open(filename, "r") as f:
        line = f.readline().strip()
        i = 0
        while line:
            #            if (i % 10 == 0):
            #                print(i)

            relative_path, label = line.split('\t')
            full_path = os.path.join(base_path, relative_path)
            #            print("{0} {1}".format(full_path, label))

            img = imread(full_path)
            img_resize = imresize(img, (H, W))

            X[i] = img_resize.astype(np.float64)
            y[i] = int(label)

            line = f.readline().strip()
            i += 1

    return X, y

# extract label and features from train/test fiels
def get_X_y(filename, label_column=None, label_index=None, header=0, sep='\t',
            nrows=None, features=None, names=None,
            encoding="iso-8859-1"):  # default encoding = "ISO-8859-1"
    df = pd.read_csv(
        filename,
        header=header,
        sep=sep,
        nrows=nrows,
        error_bad_lines=False,
        encoding=encoding,
        names=names)
    if header is not None:
        df.rename(columns=lambda x: x.strip(), inplace=True)
    else:
        df.columns = ['V' + str(x) for x in range(len(df.columns))]
    X = None
    y = None
    if label_column is not None:
        X = df.loc[:, df.columns != label_column]
        if features is not None:
            X = X[features]
        y = df.loc[:, df.columns == label_column]
    elif label_index is not None:
        X = df.ix[:, [i for i in range(len(df.columns)) if i != label_index]]
        y = df.ix[:, [label_index]]
    return X, y

def evaluate_binary_classifier(target, predicted, probabilities=None):
    accuracy = np.mean(target == predicted)
    auc_score = None
    if probabilities is not None:
        auc_score = roc_auc_score(target, probabilities)
    return (accuracy, auc_score)

def check_accuracy(test_file, label_column, predictions, threshold, sep=','):
    (test, label) = get_X_y(test_file, label_column, sep=sep)
    accuracy = np.mean(label[label_column].values ==
                       predictions.ix[:, 'PredictedLabel'].values)
    assert_greater(
        accuracy,
        threshold,
        "accuracy should be greater than %s" %
        threshold)

def check_accuracy_scikit(
        test_file,
        label_column,
        predictions,
        threshold,
        sep=','):
    (test, label) = get_X_y(test_file, label_column, sep=sep)
    accuracy = np.mean(label[label_column].values == predictions.values)
    assert_greater(
        accuracy,
        threshold,
        "accuracy should be greater than %s" %
        threshold)

max_stars_repo_name: GalOshri/NimbusML

max_issues_repo_name: GalOshri/NimbusML

max_forks_repo_name: GalOshri/NimbusML

reason: import numpy,from scipy


text:
import numpy as np
from numpy.testing import assert_array_equal, assert_raises

def assert_array_not_equal(x, y):
    assert_raises(AssertionError, assert_array_equal, x, y)

def assert_list_of_tensors_equal(unittest_obj, input, other):
    import torch
    tensor_pairs = zip([item for item in input], [item for item in other])
    is_equal = [torch.equal(first, second)
                for first, second in tensor_pairs]
    unittest_obj.assertTrue(np.alltrue(is_equal))

max_stars_repo_name: chschroeder/small-text

max_issues_repo_name: chschroeder/small-text

max_forks_repo_name: chschroeder/small-text

reason: import numpy,from numpy