text:
import enum
import re
import sqlite3
import time
from pathlib import Path

import numpy as np
import pandas as pd
import pytest
from sqlalchemy import Column, Integer, String, Float, Enum
from sqlalchemy.ext.declarative import declarative_base

from cascade.dismod.constants import DensityEnum
from cascade.dismod.db import DismodFileError
from cascade.dismod.db.metadata import Base as DismodFileBase
from cascade.dismod.db.wrapper import DismodFile, get_engine, _validate_data, _ordered_by_foreign_key_dependency

@pytest.fixture
def engine():
    return get_engine(None)

@pytest.fixture
def base_file(engine):
    dm_file = DismodFile(engine)
    density = pd.DataFrame({"density_name": [x.name for x in DensityEnum]})
    dm_file.density = density.assign(density_id=density.index)

    ages = pd.DataFrame({"age": np.array([6.0, 22.0, 48.0])})
    ages["age_id"] = ages.index
    dm_file.age = ages
    dm_file.time = pd.DataFrame({"time": [1997.0, 2005.0, 2017.0]})
    dm_file.integrand = pd.DataFrame({"integrand_name": ["prevalence"], "minimum_meas_cv": [0.0]})

    return dm_file

@pytest.fixture
def dummy_data_row():
    return pd.DataFrame(
        {
            "data_name": "foo",
            "integrand_id": 1,
            "density_id": 1,
            "node_id": 1,
            "weight_id": 1,
            "hold_out": 0,
            "meas_value": 0.0,
            "meas_std": 0.0,
            "eta": np.nan,
            "nu": np.nan,
            "age_lower": 0,
            "age_upper": 10,
            "time_lower": 1990,
            "time_upper": 2000,
            "x_s_source": 0,  # Dismod-AT allows arbitrary columns starting with x_.
            "x_sex": 2.0,
        },
        index=[0],
    )

@pytest.fixture
def new_data():
    return pd.DataFrame(dict(
        data_name=pd.Series(["foo", "bar"]),
        integrand_id=pd.Series([1, 2], dtype="Int64"),
        density_id=pd.Series([1, 1], dtype="Int64"),
        node_id=pd.Series([37, 102], dtype="Int64"),
        weight_id=pd.Series([1, 1], dtype="Int64"),
        hold_out=pd.Series([0, 1], dtype="Int64"),
        meas_value=pd.Series([0.0, 0.5]),
        meas_std=pd.Series([0.01, 0.01], dtype=np.float),
        eta=pd.Series([np.nan, 0.0005], dtype=np.float),
        nu=pd.Series([np.nan, 5], dtype=np.float),
        age_lower=pd.Series([0, 45], dtype=np.float),
        age_upper=pd.Series([10, 70], dtype=np.float),
        time_lower=pd.Series([1990, 2000], dtype=np.float),
        time_upper=pd.Series([2000, 2010], dtype=np.float),
        x_s_source=pd.Series([0, 2], dtype="Int64"),
        x_sex=pd.Series([2.0, 4.0], dtype=np.float),
    ))

@pytest.mark.parametrize(
    "in_tables,expected",
    [
        ("age time integrand density", "age density integrand time"),
        ("density prior age", "age density prior"),
        ("node", "node"),
        ("weight_grid age time weight", "age time weight weight_grid"),
        ("avgint node weight", "node weight avgint"),
    ],
)
def test_ordering_of_tables(in_tables, expected):
    out = list(_ordered_by_foreign_key_dependency(DismodFileBase.metadata, in_tables.split()))
    assert out == expected.split()

def test_ordering_unhappy():
    with pytest.raises(DismodFileError):
        next(_ordered_by_foreign_key_dependency(DismodFileBase.metadata, "nonexistent age".split()))

def test_wrong_type(base_file):
    ages = pd.DataFrame({"age": np.array(["strings", "for", "ages"])})
    base_file.age = ages
    with pytest.raises(DismodFileError):
        base_file.flush()

def test_non_existent_table(base_file):
    with pytest.raises(KeyError):
        base_file.integrands_with_an_s = pd.DataFrame()

def test_empty_table__regular_tables(base_file):
    table = base_file.empty_table("node")
    assert set(table.columns) == {"node_id", "node_name", "parent"}
    assert table.dtypes.node_id == np.int64
    assert table.dtypes.node_name == np.object
    assert table.dtypes.parent == np.int64

def test_empty_table__table_with_mutable_columns(base_file, dummy_data_row):
    table = base_file.empty_table("data")
    assert "x_sex" not in table.columns
    assert "x_s_source" not in table.columns

    base_file.data = dummy_data_row
    base_file.flush()

    table = base_file.empty_table("data")
    assert "x_sex" in table.columns
    assert "x_s_source" in table.columns

def test_attribute_access_of_empty_table(base_file):
    table = base_file.node
    assert set(table.columns) == {"node_id", "node_name", "parent"}

def test_is_dirty__initially(base_file):
    assert base_file._is_dirty("age")

def test_is_dirty__after_flush(base_file):
    base_file.flush()
    assert not base_file._is_dirty("age")

def test_is_dirty__after_modification(base_file):
    base_file.flush()
    base_file.age.loc[0, "age"] *= 2
    assert base_file._is_dirty("age")

def test_is_dirty__on_read(base_file, engine):
    base_file.flush()

    dm_file2 = DismodFile(engine)

    dm_file2.age

    assert not dm_file2._is_dirty("age")

def test_is_dirty__not_yet_read(base_file):
    assert not base_file._is_dirty("foo_bar")

def test_add_log_has_primary_key(base_file):
    base_file.flush()
    base_file.log = pd.DataFrame(
        {
            "message_type": ["command"],
            "table_name": np.array([None], dtype=np.object),
            "row_id": np.NaN,
            "unix_time": int(round(time.time())),
            "message": ["fit_no_covariates.py"],
        }
    )
    base_file.flush()

def test_dmfile_read(base_file, engine):
    ages = base_file.age
    times = base_file.time
    base_file.flush()

    dm_file2 = DismodFile(engine)
    assert ages.sort_index("columns").equals(dm_file2.age.sort_index("columns"))
    assert times.sort_index("columns").equals(dm_file2.time.sort_index("columns"))

def test_reading_modified_columns(base_file, engine):
    base_file.flush()
    base_file.age.loc[0, "age"] *= 2
    ages = base_file.age.copy()
    base_file.flush()

    dm_file2 = DismodFile(engine)
    assert ages.sort_index("columns").equals(dm_file2.age.sort_index("columns"))

DummyBase = declarative_base()

class DummyTable(DummyBase):
    __tablename__ = "test_table"
    primary_key_column = Column(Integer(), primary_key=True)
    integer_column = Column(Integer())
    float_column = Column(Float())
    nonnullable_column = Column(Integer(), nullable=False)
    string_column = Column(String())
    enum_column = Column(Enum(enum.Enum("Bee", "bumble honey carpenter wool_carder")))

def test_validate_data__happy_path():
    data = pd.DataFrame(
        {
            "integer_column": [1, 2, 3],
            "float_column": [1.0, 2.0, 3.0],
            "string_column": ["a", "b", "c"],
            "nonnullable_column": [1, 2, 3],
        }
    )
    _validate_data(DummyTable.__table__, data)

def test_validate_data__bad_integer():
    data = pd.DataFrame({"integer_column": np.array(["1", "2", "3"], dtype=np.str), "nonnullable_column": [1, 2, 3]})
    with pytest.raises(DismodFileError) as excinfo:
        _validate_data(DummyTable.__table__, data)

    assert "integer_column" in str(excinfo.value)

def test_validate_data__bad_float():
    data = pd.DataFrame({"float_column": ["1.0", "2.0", "3.0"], "nonnullable_column": [1, 2, 3]})
    with pytest.raises(DismodFileError) as excinfo:
        _validate_data(DummyTable.__table__, data)

    assert "float_column" in str(excinfo.value)

def test_validate_data__bad_string():
    data = pd.DataFrame({"string_column": [1, 2, 3], "nonnullable_column": [1, 2, 3]})
    with pytest.raises(DismodFileError) as excinfo:
        _validate_data(DummyTable.__table__, data)
    assert "string_column" in str(excinfo.value)

def test_validate_data__extra_column():
    data = pd.DataFrame({"nonnullable_column": [1, 2, 3], "other_column": [1, 2, 3]})
    with pytest.raises(DismodFileError) as excinfo:
        _validate_data(DummyTable.__table__, data)
    assert "other_column" in str(excinfo.value)

def test_validate_data__missing_column():
    data = pd.DataFrame({"integer_column": [1, 2, 3]})
    with pytest.raises(DismodFileError) as excinfo:
        _validate_data(DummyTable.__table__, data)
    assert "nonnullable_column" in str(excinfo.value)

def test_write_covariate_column__success(base_file):
    new_data = pd.DataFrame(
        {
            "data_name": "foo",
            "integrand_id": 1,
            "density_id": 1,
            "node_id": 1,
            "weight_id": 1,
            "hold_out": 0,
            "meas_value": 0.0,
            "meas_std": 0.0,
            "eta": np.nan,
            "nu": np.nan,
            "age_lower": 0,
            "age_upper": 10,
            "time_lower": 1990,
            "time_upper": 2000,
            "x_s_source": 0,
            "x_sex": 2.0,
        },
        index=[0],
    )
    base_file.data = new_data
    base_file.flush()

def test_write_integer_column__success(base_file, new_data):
    # Constructs a DataFrame with explicitly-typed nullable integer types.
    assert new_data.hold_out.dtype == pd.Int64Dtype()
    base_file.data = new_data
    base_file.flush()

def test_written_schema(new_data, tmp_path):
    """Constructs a DataFrame with explicitly-typed nullable integer types."""
    # tests whether it really gets written to the db with the expected type.
    db_file = Path(tmp_path) / "file.db"
    engine = get_engine(db_file)
    dm_file = DismodFile(engine)
    dm_file.data = new_data
    dm_file.flush()

    found_column = dict()
    conn = sqlite3.connect(str(db_file))
    c = conn.cursor()
    # This schema description is a CREATE TABLE statement.
    schema = c.execute('''SELECT sql FROM sqlite_master WHERE type = 'table' and tbl_name = 'data';''').fetchone()
    for line in schema[0].split("\n"):
        print(line)
        m = re.search(r"(\w+)\s+(\w+)", line)
        if m:
            col, kind = (m.group(1), m.group(2))
            print(f"col {col} kind {kind}")
            if col in new_data.dtypes:
                expected = new_data.dtypes[col]
                if expected == np.float:
                    assert kind == "real"
                elif expected == np.dtype("O"):
                    assert kind == "text"
                elif expected == pd.Int64Dtype():
                    assert kind == "integer"
                else:
                    assert False, f"expected {expected} found {line}"
                found_column[col] = kind
            elif col == "data_id":
                assert kind == "integer"
            elif col == "CREATE":
                continue
            else:
                assert False, f"Unknown column {line}"
    assert len(found_column) == len(new_data.dtypes)

def test_read_covariate_column__success(base_file, dummy_data_row):
    base_file.data = dummy_data_row
    base_file.flush()

    new_dm = DismodFile(base_file.engine)
    assert list(new_dm.data.x_s_source) == [0]
    assert list(new_dm.data.x_sex) == [2.0]

def test_write_covariate_column__bad_name(base_file, dummy_data_row):
    dummy_data_row = dummy_data_row.rename(columns={"x_sex": "not_x_sex"})
    base_file.data = dummy_data_row
    with pytest.raises(ValueError) as excinfo:
        base_file.flush()
    assert "not_x_sex" in str(excinfo.value)

def test_write_covariate_column__schema_changes_are_isolated(dummy_data_row):
    dm_file = DismodFile(get_engine(None))
    dm_file.data = dummy_data_row
    dm_file.flush()

    dm_file2 = DismodFile(get_engine(None))
    table = dm_file2.data
    assert "x_sex" not in table

def test_write_empty_table(tmp_path):
    """Ensure an empty table gets written."""
    db_file = Path(tmp_path) / "file.db"
    dm_file = DismodFile(get_engine(db_file))
    dm_file.age = dm_file.age.append(dict(age_id=0, age=48.0), ignore_index=True)
    covariate = dm_file.covariate
    assert covariate.empty
    dm_file.flush()
    dm_file.engine.dispose()

    # Use sqlite3 module to search for entries in this table.
    assert db_file.exists()
    conn = sqlite3.connect(str(db_file))
    c = conn.cursor()
    c.execute('''SELECT * FROM age''')
    res_age = c.fetchall()
    print(res_age)
    assert res_age is not None
    c.execute('''SELECT * FROM covariate''')
    res = c.fetchall()
    print(res)
    assert not res

max_stars_repo_name: adolgert/cascade

max_issues_repo_name: adolgert/cascade

max_forks_repo_name: adolgert/cascade


text:
from __future__ import division
from __future__ import print_function

import numpy as np
import scipy as sp

from sklearn.metrics import mean_squared_error

from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import AdaBoostClassifier
from sklearn.linear_model import Perceptron as PerceptronClf

from sklearn.linear_model import Lasso

from sklearn.model_selection import KFold

RANDOM_STATE = 0

# Error calculator for class average for each fold
def error_calc(test_labels, predictions):
    error = []
    eps = 1e-15

    #  Calculate the custom metric 1- 0.5(Specificity + Sensitivity)
    for i in np.unique(test_labels):
        # true positives
        tp = ((test_labels == i) & (predictions == i)).sum()

        # true negatives
        tn = ((test_labels != i) & (predictions != i)).sum()

        # false positives
        fp = ((test_labels != i) & (predictions == i)).sum()

        # false negatives
        fn = ((test_labels == i) & (predictions != i)).sum()

        tp_new = sp.maximum(eps, tp)
        pos_new = sp.maximum(eps, tp+fn)
        tn_new = sp.maximum(eps, tn)
        neg_new = sp.maximum(eps, tn+fp)

        error.append(1 - 0.5*(tp_new/pos_new) - 0.5*(tn_new/neg_new))

    # convert the error list into numpy array
    error_np = np.array(error)
    return np.mean(error_np)

# error calculation for regression tasks (mean-squared error)
def error_calc_reg(test_values, predictions):
    return mean_squared_error(test_values, predictions)

# returns the kfold CV error given data and classifier
def kfolderror(data_numeric, data_labels, clf, num_splits, classification=True):
    error_avg = 0
    predictions=np.zeros(shape=data_labels.shape)
    kf = KFold(n_splits=num_splits, shuffle=True, random_state=RANDOM_STATE)
    for train, test in kf.split(data_numeric):
        x_train = data_numeric[train, :]
        y_train = data_labels[train]
        x_test = data_numeric[test, :]
        y_test = data_labels[test]

        clf.fit(x_train, y_train)
        predictions[test] = clf.predict(x_test)
        if classification:
            error_avg += error_calc(y_test, predictions[test])
        else:
            error_avg += error_calc_reg(y_test, predictions[test])

    error_avg = error_avg/num_splits
    return error_avg, predictions

# classification algorithms

def kNN(data_numeric, data_labels, num_splits=10, k=5, p=2, verbose=True):
    clf = KNeighborsClassifier(n_neighbors=k, p=p)
    error_avg, predictions = kfolderror(data_numeric, data_labels, clf, num_splits)
    if verbose:
        print("kNN finished (k={}, shape={})".format(k, data_numeric.shape))
    return error_avg, predictions, clf

def CART(data_numeric, data_labels, min_samples_split=2, num_splits=10, verbose=True):
    clf = DecisionTreeClassifier(min_samples_split=min_samples_split, random_state=RANDOM_STATE)
    error_avg, predictions = kfolderror(data_numeric, data_labels, clf, num_splits)
    if verbose:
        print("CART finished (min_samples_split={}, shape={})".format(min_samples_split, data_numeric.shape))
    return error_avg, predictions, clf

def RF(data_numeric, data_labels, min_samples_split=2, num_splits=10, num_trees=100, num_jobs=1,
       type_num_features='sqrt', verbose=True):

    clf = RandomForestClassifier(n_estimators=num_trees, min_samples_split=min_samples_split,
                                 max_features=type_num_features, n_jobs=num_jobs, random_state=RANDOM_STATE)
    error_avg, predictions = kfolderror(data_numeric, data_labels, clf, num_splits)
    if verbose:
        print("RF finished (min_samples_split={}, shape={})".format(min_samples_split, data_numeric.shape))
    return error_avg, predictions, clf

def Perceptron(data_numeric, data_labels, num_splits=10, num_iter=5, penalty=None, verbose=True):
    clf = PerceptronClf(penalty=penalty, n_iter=num_iter, random_state=RANDOM_STATE)
    error_avg, predictions = kfolderror(data_numeric, data_labels, clf, num_splits)
    if verbose:
        print("Perceptron finished (shape={})".format(data_numeric.shape))
    return error_avg, predictions, clf

def Adaboost(data_numeric, data_labels, n_estimators=50, learning_rate=1.0, num_splits=10, verbose=True):
    clf = AdaBoostClassifier(n_estimators=n_estimators, learning_rate=learning_rate, random_state=RANDOM_STATE)
    error_avg, predictions = kfolderror(data_numeric, data_labels, clf, num_splits)
    if verbose:
        print("Adaboost finished (n_estimators={}, learning_rate={}, shape={})"
              .format(n_estimators, learning_rate, data_numeric.shape))
    return error_avg, predictions, clf

def GNB(data_numeric, data_labels, num_splits=10, verbose=True):

    clf = GaussianNB()
    error_avg, predictions = kfolderror(data_numeric, data_labels, clf, num_splits)
    if verbose:
        print("Gaussian Naive Bayes finished (shape={})".format(data_numeric.shape))
    return error_avg, predictions, clf

def GB(data_numeric, data_labels, learning_rate=0.1, num_splits=10, verbose=True):

    clf = GradientBoostingClassifier(n_estimators=100, learning_rate=learning_rate, random_state=RANDOM_STATE)
    error_avg, predictions = kfolderror(data_numeric, data_labels, clf, num_splits)
    if verbose:
        print("GradientBoosting finished (learning_rate={}".format(data_numeric.shape))
    return error_avg, predictions, clf

def lSVM(data_numeric, data_labels, C=1.0, num_splits=10, verbose=True):
    clf = LinearSVC(C=C, multi_class='ovr', random_state=RANDOM_STATE)
    error_avg, predictions = kfolderror(data_numeric, data_labels, clf, num_splits)
    if verbose:
        print("LinearSVM finished (C={}, shape={})".format(C, data_numeric.shape))
    return error_avg, predictions, clf

def kSVM(data_numeric, data_labels, C=1.0, num_splits=10, kernel='rbf', verbose=True):
    clf = SVC(C=C, decision_function_shape='ovo',kernel=kernel, random_state=RANDOM_STATE)
    error_avg, predictions = kfolderror(data_numeric, data_labels, clf, num_splits)
    if verbose:
        print("KernelSVM finished (C={}, shape={})".format(C, data_numeric.shape))
    return error_avg, predictions, clf

def Logit(data_numeric, data_labels, C=1.0, num_splits=10, penalty='l2', verbose=True):
    clf = LogisticRegression(C=C, penalty=penalty, random_state=RANDOM_STATE)
    error_avg, predictions = kfolderror(data_numeric, data_labels, clf, num_splits)
    if verbose:
        print("Logistic Regression finished (C={}, shape={})".format(C, data_numeric.shape))
    return error_avg, predictions, clf

def NeuralNet(data_numeric, data_labels, num_splits=10, solver='sgd', alpha=1e-5, hidden_layer_sizes=(5, 2),
              verbose=True):
    clf = MLPClassifier(solver=solver, alpha=alpha,hidden_layer_sizes=hidden_layer_sizes, random_state=RANDOM_STATE)
    error_avg, predictions = kfolderror(data_numeric, data_labels, clf, num_splits)
    if verbose:
        print("Neural Network finished (shape={})".format(data_numeric.shape))
    return error_avg, predictions, clf

# regression algorithms
# def Lasso(data_numeric, data_labels, num_splits=10, alpha=1.0, verbose=True):
#     reg = Lasso(alpha=alpha, random_state=RANDOM_STATE)
#     error_avg, predictions = kfolderror(data_numeric, data_labels, reg, num_splits, classification=False)
#     if verbose:
#         print("Lasso finished (shape={})".format(data_numeric.shape))
#     return error_avg, predictions, clf

ext: py

max_stars_repo_name: yujiakimoto/lowrankautoml

max_issues_repo_name: yujiakimoto/lowrankautoml

max_forks_repo_name: yujiakimoto/lowrankautoml

reason: import numpy,import scipy


text:
#
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import sys

# This sample uses an MNIST PyTorch model to create a TensorRT Inference Engine
import model
import numpy as np
import pycuda.autoinit
import tensorrt as trt

trt_version = trt.__version__
trt_v7 = True if trt_version.startswith("7") else False

sys.path.insert(1, os.path.join(sys.path[0], ".."))
import common

# You can set the logger severity higher to suppress messages (or lower to display more messages).
TRT_LOGGER = trt.Logger(trt.Logger.WARNING)

class ModelData(object):
    INPUT_NAME = "data"
    INPUT_SHAPE = (1, 1, 28, 28)
    OUTPUT_NAME = "prob"
    OUTPUT_SIZE = 10
    DTYPE = trt.float32

def populate_network(network, weights):
    # Configure the network layers based on the weights provided.
    input_tensor = network.add_input(name=ModelData.INPUT_NAME, dtype=ModelData.DTYPE, shape=ModelData.INPUT_SHAPE)

    conv1_w = weights['conv1.weight'].cpu().numpy()
    conv1_b = weights['conv1.bias'].cpu().numpy()
    conv1 = network.add_convolution(input=input_tensor, num_output_maps=20, kernel_shape=(5, 5), kernel=conv1_w, bias=conv1_b)
    conv1.stride = (1, 1)

    pool1 = network.add_pooling(input=conv1.get_output(0), type=trt.PoolingType.MAX, window_size=(2, 2))
    pool1.stride = (2, 2)

    conv2_w = weights['conv2.weight'].cpu().numpy()
    conv2_b = weights['conv2.bias'].cpu().numpy()
    conv2 = network.add_convolution(pool1.get_output(0), 50, (5, 5), conv2_w, conv2_b)
    conv2.stride = (1, 1)

    pool2 = network.add_pooling(conv2.get_output(0), trt.PoolingType.MAX, (2, 2))
    pool2.stride = (2, 2)

    fc1_w = weights['fc1.weight'].cpu().numpy()
    fc1_b = weights['fc1.bias'].cpu().numpy()
    fc1 = network.add_fully_connected(input=pool2.get_output(0), num_outputs=500, kernel=fc1_w, bias=fc1_b)

    relu1 = network.add_activation(input=fc1.get_output(0), type=trt.ActivationType.RELU)

    fc2_w = weights['fc2.weight'].cpu().numpy()
    fc2_b = weights['fc2.bias'].cpu().numpy()
    fc2 = network.add_fully_connected(relu1.get_output(0), ModelData.OUTPUT_SIZE, fc2_w, fc2_b)

    fc2.get_output(0).name = ModelData.OUTPUT_NAME
    network.mark_output(tensor=fc2.get_output(0))

def build_engine(weights):
    # For more information on TRT basics, refer to the introductory samples.
    builder = trt.Builder(TRT_LOGGER)
    network = builder.create_network(common.EXPLICIT_BATCH)
    config = builder.create_builder_config()
    runtime = trt.Runtime(TRT_LOGGER)

    config.max_workspace_size = common.GiB(1)
    # Populate the network using weights from the PyTorch model.
    populate_network(network, weights)
    # Build and return an engine.

    if trt_v7:
        #plan = builder.build_cuda_engine(network)
        plan = builder.build_engine(network,config)
        return plan
    plan = builder.build_serialized_network(network, config)
    return runtime.deserialize_cuda_engine(plan)

# Loads a random test case from pytorch's DataLoader
def load_random_test_case(model, pagelocked_buffer):
    # Select an image at random to be the test case.
    img, expected_output = model.get_random_testcase()
    # Copy to the pagelocked input buffer
    np.copyto(pagelocked_buffer, img)
    return expected_output

def main():
    common.add_help(description="Runs an MNIST network using a PyTorch model")
    # Train the PyTorch model
    mnist_model = model.MnistModel()
    mnist_model.learn()
    weights = mnist_model.get_weights()
    # Do inference with TensorRT.
    engine = build_engine(weights)

    #save trt engine
    f = open("mnist.trt","wb")
    f.write(engine.serialize())
    f.close()

    #load trt engine
    f = open("mnist.trt","rb")
    engine = trt.Runtime(TRT_LOGGER)
    engine = engine.deserialize_cuda_engine(f.read())
    f.close()

    # Build an engine, allocate buffers and create a stream.
    # For more information on buffer allocation, refer to the introductory samples.
    inputs, outputs, bindings, stream = common.allocate_buffers(engine)
    context = engine.create_execution_context()

    case_num = load_random_test_case(mnist_model, pagelocked_buffer=inputs[0].host)
    # For more information on performing inference, refer to the introductory samples.
    # The common.do_inference function will return a list of outputs - we only have one in this case.
    [output] = common.do_inference_v2(context, bindings=bindings, inputs=inputs, outputs=outputs, stream=stream)
    pred = np.argmax(output)
    print("Test Case: " + str(case_num))
    print("Prediction: " + str(pred))

if __name__ == '__main__':
    main()

max_stars_repo_name: ChenShisen/TensorRT

max_issues_repo_name: ChenShisen/TensorRT

max_forks_repo_name: ChenShisen/TensorRT

reason: import numpy,import pycuda


text:
##############################################################################################
##
##  Date: July 2nd 2015
##  Details: This is a Python  example for digital channel data being queried on the MSO2000/B scopes
##
##############################################################################################

import visa # http://pyvisa.sourceforge.net/  version 1.6
import numpy # http://www.numpy.org/      version 1.6.1
import pylab #plotting tool

print 'Starting program'

rm = visa.ResourceManager()

#myscope = rm.open_resource('TCPIP::192.168.1.50::INSTR')  #insert IP address of device
myscope = rm.open_resource('USB::0x0699::0x03A4::C000001::INSTR') #use this for USB devices

#identify connected scope
IDN = myscope.ask('*IDN?')
print 'Successfully connected to: ', IDN

#check error status register
ESR = myscope.ask('*ESR?')
print 'Error status register: ', ESR

ALLEV = myscope.ask('ALLEV?')
print 'All Event details: ', ALLEV

myscope.write('data:encdg SRP')
print 'Encoding type: ', myscope.ask('data:encdg?')

myscope.write('DATA:WIDTH 1')
print 'Data width: ', myscope.ask('DATA:WIDTH?')

myscope.write('wfmoutpre:byt_nr 1')
print 'wfm preamble btye size: ', myscope.ask('wfmoutpre:byt_nr?')

myscope.write('data:start 1')
print 'Start of data: ', myscope.ask('data:start?')

myscope.write('data:stop 200000')
print 'Stop of data: ', myscope.ask('data:stop?')

myscope.write('DATa:RESOlution FULL')
print 'Data resolution setup: ', myscope.ask('DATa:RESOlution?')

myscope.write('DATa:COMP SINGULAR_YT')
print 'Data composition type: ', myscope.ask('DATa:COMP?')

xincr = float(myscope.ask('WFMPRE:XINCR?'))

digital_channel = range(16)
d0 = []
d1 = []
d2 = []
d3 = []
d4 = []
d5 = []
d6 = []
d7 = []
d8 = []
d9 = []
d10 = []
d11 = []
d12 = []
d13 = []
d14 = []
d15 = []

#loop to grab all digital channel data
for i in digital_channel:

    #selected digital channel
    myscope.write('data:source D' + str(digital_channel[i]))
    print 'Transmitting Data From Source: ', myscope.ask('data:source?')

    #query the digital channel
    myscope.write('curve?')
    data = myscope.read_raw()

    headerlen = 2 + int(data[1])
    header = data[:headerlen]
    digital_wave = data[headerlen:-1]

    temp8bits = numpy.fromstring(digital_wave, dtype = 'u1')

    extracted8bit = numpy.unpackbits(temp8bits.reshape((temp8bits.shape[0],1)), axis=1)

    exec 'd%s = extracted8bit[:,7]' %digital_channel[i]

Time = numpy.arange(0, xincr * len(d0), xincr)

pylab.plot(Time, d0+15, label='D0')
pylab.plot(Time, d1+14, label='D1')
pylab.plot(Time, d2+13, label='D2')
pylab.plot(Time, d3+12, label='D3')
pylab.plot(Time, d4+11, label='D4')
pylab.plot(Time, d5+10, label='D5')
pylab.plot(Time, d6+9, label='D6')
pylab.plot(Time, d7+8, label='D7')
pylab.plot(Time, d8+7, label='D8')
pylab.plot(Time, d9+6, label='D9')
pylab.plot(Time, d10+5, label='D10')
pylab.plot(Time, d11+4, label='D11')
pylab.plot(Time, d12+3, label='D12')
pylab.plot(Time, d13+2, label='D13')
pylab.plot(Time, d14+1, label='D14')
pylab.plot(Time, d15+0, label='D15')

pylab.ylim(-1,17)
pylab.legend()
pylab.show()

reason: import numpy


text:
# -*- coding: utf-8 -*-

import math
import re
import warnings

import numpy as np

hour_re = re.compile(r"(?P<sign>[+-]*)"
                     r"(?P<hours>\d+):"
                     r"(?P<mins>\d+):"
                     r"(?P<secs>\d+\.?\d*)")

deg_re = re.compile(r"(?P<sign>[+-])*"
                    r"(?P<degs>\d+)\."
                    r"(?P<mins>\d+)\."
                    r"(?P<secs>\d+\.?\d*)")

def _hour_converter(hour_str):
    m = hour_re.match(hour_str)

    if not m:
        raise ValueError("Error parsing '%s'" % hour_str)

    value = float(m.group("hours")) / 24.0
    value += float(m.group("mins")) / (24.0*60.0)
    value += float(m.group("secs")) / (24.0*60.0*60.0)

    if m.group("sign") == '-':
        value = -value

    return 2.0 * math.pi * value

def _deg_converter(deg_str):
    m = deg_re.match(deg_str)

    if not m:
        raise ValueError(f"Error parsing '{deg_str}'")

    value = float(m.group("degs")) / 360.0
    value += float(m.group("mins")) / (360.0*60.0)
    value += float(m.group("secs")) / (360.0*60.0*60.0)

    if m.group("sign") == '-':
        value = -value

    return 2.0 * math.pi * value

def arcsec2rad(arcseconds=0.0):
    return np.deg2rad(float(arcseconds) / 3600.)

def spi_converter(spi):
    spi = spi.strip("[] ")

    if not spi:
        return []

    return [float(c) for c in spi.split(",")]

_COLUMN_CONVERTERS = {
    'Name': str,
    'Type': str,
    'Ra': _hour_converter,
    'Dec': _deg_converter,
    'I': float,
    'SpectralIndex': spi_converter,
    'LogarithmicSI': lambda x: bool(x == "true"),
    'ReferenceFrequency': float,
    'MajorAxis': arcsec2rad,
    'MinorAxis': arcsec2rad,
    'Orientation': lambda x=0.0: np.deg2rad(float(x)),
}

# Split on commas, ignoring within [] brackets
_COMMA_SPLIT_RE = re.compile(r',\s*(?=[^\]]*(?:\[|$))')

# Parse columm headers, handling possible defaults
_COL_HEADER_RE = re.compile(r"^\s*?(?P<name>.*?)"
                            r"(\s*?=\s*?'(?P<default>.*?)'\s*?){0,1}$")

def _parse_col_descriptor(column_descriptor):
    components = [c.strip() for c in column_descriptor.split(",")]

    columns = []
    defaults = []

    for column in components:
        m = _COL_HEADER_RE.search(column)

        if m is None:
            raise ValueError(f"'{column}' is not a valid column header")

        name, default = m.group('name', 'default')

        columns.append(name)
        defaults.append(default)

    return columns, defaults

def _parse_header(header):
    format_str, col_desc = (c.strip() for c in header.split("=", 1))

    if format_str != "Format":
        raise ValueError(f"'{format_str}' does not "
                         f"appear to be a wsclean header")

    return _parse_col_descriptor(col_desc)

def _parse_lines(fh, line_nr, column_names, defaults, converters):
    source_data = [[] for _ in range(len(column_names))]

    for line_nr, line in enumerate(fh, line_nr):
        components = [c.strip() for c in re.split(_COMMA_SPLIT_RE, line)]

        if len(components) != len(column_names):
            raise ValueError(f"line {line_nr} '{line}' should "
                             f"have {len(column_names)} components")

        # Iterate through each column's data
        it = zip(column_names, components, converters, source_data, defaults)

        for name, comp, conv, data_list, default in it:
            if not comp:
                if default is None:
                    try:
                        default = conv()
                    except Exception as e:
                        raise ValueError(
                            f"No value supplied for column '{name}' "
                            f"on line {line_nr} and no default was "
                            f"supplied either. Attempting to "
                            f"generate a default produced the "
                            f"following exception {e}")

                value = default
            else:
                value = comp

            data_list.append(conv(value))

    columns = dict(zip(*(column_names, source_data)))

    # Zero any spectral model's with nan/inf values
    try:
        name_column = columns["Name"]
        flux_column = columns["I"]
        spi_column = columns["SpectralIndex"]
        log_spi_column = columns["LogarithmicSI"]
    except KeyError as e:
        raise ValueError(f"WSClean Model File missing "
                         f"required column {str(e)}")

    it = zip(name_column, flux_column, spi_column, log_spi_column)

    # Zero flux and spi's in-place
    for i, (name, flux, spi, log_spi) in enumerate(it):
        good = True

        if not math.isfinite(flux):
            warnings.warn(f"Non-finite I {flux} encountered "
                          f"for source {name}. This source model will "
                          f"be zeroed.")
            good = False

        if not all(map(math.isfinite, spi)):
            warnings.warn(f"Non-finite SpectralIndex {spi} encountered "
                          f"for source {name}. This source model will "
                          f"be zeroed.")
            good = False

        if good:
            continue

        # np.log(1.0) = 0.0
        flux_column[i] = 1.0 if log_spi else 0.0

        for j in range(len(spi)):
            spi[j] = 0.0

    return list(columns.items())

def load(filename):
    """
    Loads wsclean component model.

    .. code-block:: python

        sources = load("components.txt")
        sources = dict(sources)  # Convert to dictionary

        I = sources["I"]
        ref_freq = sources["ReferenceFrequency"]

    See the `WSClean Component List
    <https://sourceforge.net/p/wsclean/wiki/ComponentList/>`_
    for further details.

    Parameters
    ----------
    filename : str or iterable
        Filename of wsclean model file or iterable
        producing the lines of the file.

    See Also
    --------
    africanus.model.wsclean.spectra

    Returns
    -------
    list of (name, list of values) tuples
        list of column (name, value) tuples
    """

    if isinstance(filename, str):
        fh = open(filename, "r")
        fh = iter(fh)
        close_filename = True
    else:
        fh = iter(filename)
        close_filename = False

    try:
        # Search for a header until we find a non-empty string
        header = ''
        line_nr = 1

        for headers in fh:
            header = headers.split("#", 1)[0].strip()

            if header:
                break

            line_nr += 1

        if not header:
            raise ValueError(f"'{filename}' does not contain "
                             f"a valid wsclean header")

        column_names, defaults = _parse_header(header)

        try:
            converters = [_COLUMN_CONVERTERS[n] for n in column_names]
        except KeyError as e:
            raise ValueError(f"No converter registered for column {str(e)}")

        return _parse_lines(fh, line_nr, column_names, defaults, converters)

    finally:
        if close_filename:
            fh.close()

max_stars_repo_name: ratt-ru/codex-africanus

max_issues_repo_name: ratt-ru/codex-africanus

max_forks_repo_name: ska-sa/codex-africanus


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 30 20:51:57 2019

@author: PRJ
"""

import os
import numpy as np
import pandas as pd
from PyPLS import PLSR, PLSR_numba

import CV
import time
from sklearn.cross_decomposition import PLSRegression
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import KFold

import sys
import warnings
if not sys.warnoptions:
    warnings.simplefilter("ignore")

def data_generation(dataset = 'house', datatype = 'csv', K = 10):
    cwd =  os.getcwd()
    X = np.loadtxt(open(cwd+'/data/'+dataset+'_X.'+datatype, "rb"), delimiter=",")
    Y = np.loadtxt(open(cwd+'/data/'+dataset+'_Y.'+datatype, "rb"), delimiter=",")
    if not os.path.exists(cwd+'/data/'+dataset+'/'):
        os.mkdir(cwd+'/data/'+dataset+'/')
    kf = KFold(n_splits = K)
    kf.get_n_splits(X)
    it = 0
    for train_index, test_index in kf.split(X):
        X_train, X_test = X[train_index], X[test_index]
        Y_train, Y_test = Y[train_index], Y[test_index]

        np.savetxt(cwd+'/data/'+dataset+'/train'+'_'+str(it)+'_X.'+datatype, X_train, delimiter=",")
        np.savetxt(cwd+'/data/'+dataset+'/train'+'_'+str(it)+'_Y.'+datatype, Y_train, delimiter=",")
        np.savetxt(cwd+'/data/'+dataset+'/test'+'_'+str(it)+'_X.'+datatype, X_test, delimiter=",")
        np.savetxt(cwd+'/data/'+dataset+'/test'+'_'+str(it)+'_Y.'+datatype, Y_test, delimiter=",")

        it += 1

def main():
    cwd =  os.getcwd()
    model_list = [PLSRegression, PLSR.PLSR, LinearRegression, 'PLSR_fix']
    N_model = len(model_list)
    dataset = 'sim'
    datatype = 'csv'
    N_dataset = 5
    K = 5
    d_mat = np.zeros((N_dataset, N_model), dtype = np.int)
    err_mat = np.zeros((N_dataset, N_model), dtype = np.float)
    Err_mix = np.zeros(N_model, dtype = np.float) #Frobenius norm
    Time_mix = np.zeros(N_model, dtype = np.float) #Frobenius norm

    data_generation(dataset, datatype, N_dataset)

    for it in range(N_dataset):
        print (it)
        X = np.loadtxt(open(cwd+'/data/'+dataset+'/train'+'_'+str(it)+'_X.'+datatype, "rb"), delimiter=",")
        Y = np.loadtxt(open(cwd+'/data/'+dataset+'/train'+'_'+str(it)+'_Y.'+datatype, "rb"), delimiter=",")
        X_test = np.loadtxt(open(cwd+'/data/'+dataset+'/test'+'_'+str(it)+'_X.'+datatype, "rb"), delimiter=",")
        Y_test = np.loadtxt(open(cwd+'/data/'+dataset+'/test'+'_'+str(it)+'_Y.'+datatype, "rb"), delimiter=",")

        Mx = X.shape[1]
        Y_center = Y.mean(0)
        Y_std = Y.std(0, ddof = 1)
        Y_scale = 1.0 / Y_std

        for jt in range(N_model):
            T_start = time.time()
            model = model_list[jt]
            if model == LinearRegression:
                lr = model()
                lr.fit(X,Y)
                Y_pred = lr.predict(X_test)
            elif type(model) == str:
                plsr = PLSRegression(n_components = int(0.3*Mx))
                plsr.fit(X,Y)
                Y_pred = plsr.predict(X_test)
            else:
                d_cv = CV.CV_Regression(X, Y, K, None, model)
                plsr = model(n_components = d_cv)
                plsr.fit(X,Y)
                Y_pred = plsr.predict(X_test)
            T_end = time.time()
            err_mat[it, jt] = np.sum(((Y_pred-Y_test)*Y_scale)**2)
            d_mat[it, jt] = d_cv
            Time_mix[jt] += T_end-T_start
    Err_mix = np.sqrt(np.sum(err_mat, axis = 0))
    print (Err_mix)
    print (Time_mix)
    print (d_mat)

if __name__ == "__main__":
    main()

max_stars_repo_name: showhei/show-time

max_issues_repo_name: showhei/show-time

max_forks_repo_name: showhei/show-time


text:
import numpy as np
import time
import sys
if sys.version_info.major == 2:
    import Tkinter as tk
else:
    import tkinter as tk

UNIT = 30
MAZE_H = 12
MAZE_W = 12

class Mz(tk.Tk, object):

    def __init__(self):
        super(Mz, self).__init__()
        self.action_space = ['u', 'd', 'l', 'r']
        self.n_actions = len(self.action_space)
        self.n_features = 9
        self.escapeOrigin = np.array([6, 6])
        self.angentOriginList = [
            np.array([5, 8]), np.array([8, 4]), np.array([5, 3])]
        self.angentList = []
        self.title('Pursuit-Escape')
        self.geometry('{0}x{1}'.format(MAZE_W * UNIT, (MAZE_H+3) * UNIT))
        self._build_maze()

    def _build_maze(self):
        self.canvas = tk.Canvas(self, bg='white',
                                height=MAZE_H * UNIT,
                                width=MAZE_W * UNIT)

        # draw grids
        for c in range(0, MAZE_W * UNIT, UNIT):
            x0, y0, x1, y1 = c, 0, c, MAZE_W * UNIT
            self.canvas.create_line(x0, y0, x1, y1)
        for r in range(0, MAZE_H * UNIT, UNIT):
            x0, y0, x1, y1 = 0, r, MAZE_H * UNIT, r
            self.canvas.create_line(x0, y0, x1, y1)

        for angentOrigin in self.angentOriginList:
            self.angentList.append(self.canvas.create_rectangle(
                angentOrigin[0]*UNIT - 10, angentOrigin[1]*UNIT - 10,
                angentOrigin[0]*UNIT + 10, angentOrigin[1]*UNIT + 10,
                fill='red'
            ))

        self.escape = self.canvas.create_oval(
            self.escapeOrigin[0]*UNIT-10, self.escapeOrigin[1]*UNIT-10,
            self.escapeOrigin[0]*UNIT+10, self.escapeOrigin[1]*UNIT+10,
            fill='green'
        )

        self.canvas.pack()

    def reset(self):
        self.update()
        time.sleep(1)
        for angent in self.angentList:
            self.canvas.delete(angent)

        self.angentList.clear()

        for angentOrigin in self.angentOriginList:
            self.angentList.append(self.canvas.create_rectangle(
                angentOrigin[0]*UNIT - 10, angentOrigin[1]*UNIT - 10,
                angentOrigin[0]*UNIT + 10, angentOrigin[1]*UNIT + 10,
                fill='red'
            ))

        self.canvas.delete(self.escape)
        self.escape = self.canvas.create_oval(
            self.escapeOrigin[0]*UNIT-10, self.escapeOrigin[1]*UNIT-10,
            self.escapeOrigin[0]*UNIT+10, self.escapeOrigin[1]*UNIT+10,
            fill='green'
        )

        observation = np.array([])
        observation = np.hstack(
            (observation, (np.array(self.canvas.coords(self.angentList[0])))[:2]))
        for angent in self.angentList:
            observation = np.hstack((observation, np.array(self.canvas.coords(angent))[
                                    :2]-np.array(self.canvas.coords(self.escape))[:2]))

        observation /= UNIT
        observation = np.hstack((0, observation))
        return observation

    def step(self, num, action):
        s = self.canvas.coords(self.angentList[num])
        e = self.canvas.coords(self.escape)
        distance = np.sqrt(((np.array(s) - np.array(e))**2).sum()) / UNIT
        base_action = np.array([0, 0])
        if action == 0:   # up
            if s[1] > UNIT:
                base_action[1] -= 2*UNIT
        elif action == 1:   # down
            if s[1] < (MAZE_H - 1) * UNIT:
                base_action[1] += 2*UNIT
        elif action == 2:   # right
            if s[0] < (MAZE_W - 1) * UNIT:
                base_action[0] += 2*UNIT
        elif action == 3:   # left
            if s[0] > UNIT:
                base_action[0] -= 2*UNIT

        self.canvas.move(self.angentList[num], base_action[0], base_action[1])
        next_coords = self.canvas.coords(self.angentList[num])
        _distance = np.sqrt(
            ((np.array(next_coords) - np.array(e))**2).sum()) / UNIT

        if next_coords == e:
            reward = 2
            done = True
        elif _distance < distance:
            reward = 1
            done = False
        elif _distance >= distance:
            reward = -1
            done = False
        else:
            reward = 0
            done = False

        _observation = np.array([])
        _observation = np.hstack((_observation, (np.array(s))[:2]))
        for angent in self.angentList:
            _observation = np.hstack(
                (_observation, np.array(self.canvas.coords(angent))[:2] - np.array(self.canvas.coords(self.escape))[:2]))
        _observation /= UNIT

        if num == 2:
            _observation = np.hstack((0, _observation))
        else:
            _observation = np.hstack((num+1, _observation))

        if (action == 0) & (s[1] > UNIT):
            reward = -1
        elif (action == 1) & (s[1] < (MAZE_H - 1) * UNIT):
            reward = -1
        elif (action == 2) & (s[0] < (MAZE_W - 1) * UNIT):
            reward = -1
        elif (action == 3) & (s[0] > UNIT):
            reward = -1

        return _observation, reward, done

    def doEscape(self):
        n = int(np.random.random()*4)
        e = self.canvas.coords(self.escape)
        base_action = np.array([0, 0])
        if n == 0:   # up
            if e[1] > UNIT:
                base_action[1] -= 2*UNIT
        elif n == 1:   # down
            if e[1] < (MAZE_H - 1) * UNIT:
                base_action[1] += 2*UNIT
        elif n == 2:   # right
            if e[0] < (MAZE_W - 1) * UNIT:
                base_action[0] += 2*UNIT
        elif n == 3:   # left
            if e[0] > UNIT:
                base_action[0] -= 2*UNIT
        self.canvas.move(self.escape, base_action[0], base_action[1])

    def render(self):
        time.sleep(0.01)
        self.update()


text:
#! /usr/bin/env python

import sudoku
import sys
import numpy as np
from sudoku import Board

problems = np.genfromtxt('p096_sudoku.txt', comments='G', dtype=str)
problems = np.split(problems, 50)

token = 0
for i, problem in enumerate(problems):
  nums = map(lambda x: map(int, list(x)), problem)
  board = Board(np.array(nums))
  boards = [board]

  print "Solving board", i
  try:
    sudoku.solve(boards)
  except IndexError:
    "Board", i, "has no solution!"
    sys.exit(1)

  solved_board = boards[-1]
  token += int(''.join(map(str, solved_board[0][:3])))

print
print "Solution:", token

max_stars_repo_name: joe-antognini/euler

max_issues_repo_name: joe-antognini/euler

max_forks_repo_name: joe-antognini/euler


text:
"""

Python BioSemi ActiveTwo: main class
Copyright 2015, Ilya Kuzovkin
Licensed under MIT

Builds on example code by Jack Keegan
https://batchloaf.wordpress.com/2014/01/17/real-time-analysis-of-data-from-biosemi-activetwo-via-tcpip-using-python/

"""

import socket
import numpy as np

class ActiveTwo():
    """
    Main class which implements major functions needed for communication with BioSemi ActiveTwo device
    """

    #: Host where ActiView acquisition software is running
    host = None

    #: This is the port ActiView listens on
    port = None

    #: Number of channles
    nchannels = None

    #: Data packet size (default: 32 channels @ 512Hz)
    buffer_size = None

    def __init__(self, host='127.0.0.1', sfreq=512, port=778, nchannels=32, tcpsamples=4):
        """
        Initialize connection and parameters of the signal
        :param host: IP address where ActiView is running
        :param port: Port ActiView is listening on
        :param nchannels: Number of EEG channels
        """

        # store parameters
        self.host = host
        self.port = port
        self.nchannels = nchannels
        self.sfreq = sfreq
        self.tcpsamples = tcpsamples
        self.buffer_size = self.nchannels * self.tcpsamples * 3

        # open connection
        self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.s.connect((self.host, self.port))

    def read(self, duration):
        """
        Read signal from the EEG device
        :param duration: How long to read in seconds
        :return: Signal in the matrix form: samples x channels
        """

        # initialize final data array
        rawdata = np.empty((0,32))

        # The reader process will run until requested amount of data is collected
        samples = 0
        while samples < duration * self.sfreq:

            # Create a 16-sample signal_buffer
            signal_buffer = np.zeros((self.nchannels, self.tcpsamples))

            # Read the next packet from the network
            # sometimes there is an error and packet is smaller than needed, read until get a good one
            data = []
            while len(data) != self.buffer_size:
                data = self.s.recv(self.buffer_size)

            # Extract 16 samples from the packet (ActiView sends them in 16-sample chunks)
            for m in range(self.tcpsamples):

                # extract samples for each channel
                for ch in range(self.nchannels):
                    offset = m * 3 * self.nchannels + (ch * 3)

                    # The 3 bytes of each sample arrive in reverse order
                    sample = (ord(data[offset+2]) << 16)
                    sample += (ord(data[offset+1]) << 8)
                    sample += ord(data[offset])

                    # Store sample to signal buffer
                    signal_buffer[ch, m] = sample

            # update sample counter
            samples += self.tcpsamples

            # transpose matrix so that rows are samples
            signal_buffer = np.transpose(signal_buffer)

            # add to the final dataset
            rawdata = np.concatenate((rawdata, signal_buffer), axis=0)

        return rawdata

max_stars_repo_name: kuz/pyactivetwo

max_issues_repo_name: kuz/pyactivetwo

max_forks_repo_name: kuz/pyactivetwo


text:
"""
Script to extract the SONG CCFs

The intial steps are shown in the jupyter notebook
in this folder
"""
import sys
import os
import argparse as ap
from astropy.io import fits
import matplotlib.pyplot as plt
import numpy as np
import glob as g
iSpec_location = '/Users/jmcc/iSpec_v20161118'
sys.path.insert(0, os.path.abspath(iSpec_location))
import ispec

strongLines = iSpec_location + '/input/regions/strong_lines/absorption_lines.txt'
line_lists_parent = iSpec_location + '/input/linelists/CCF/'
telluricLines = line_lists_parent + "Synth.Tellurics.500_1100nm/mask.lst"
atomicMaskLines = {'A0': line_lists_parent + 'HARPS_SOPHIE.A0.350_1095nm/mask.lst',
                   'F0': line_lists_parent + 'HARPS_SOPHIE.F0.360_698nm/mask.lst',
                   'G2': line_lists_parent + 'HARPS_SOPHIE.G2.375_679nm/mask.lst',
                   'K0': line_lists_parent + 'HARPS_SOPHIE.K0.378_679nm/mask.lst',
                   'K5': line_lists_parent + 'HARPS_SOPHIE.K5.378_680nm/mask.lst',
                   'M5': line_lists_parent + 'HARPS_SOPHIE.M5.400_687nm/mask.lst'}
INSTRUMENT = {'RESOLUTION': 90000}

def arg_parse():
    """
    """
    p = ap.ArgumentParser()
    p.add_argument('action', choices=['orders', 'ccfs'])
    return p.parse_args()

def normaliseContinuum(spec):
    """
    Based on example.py
    normalize_whole_spectrum_strategy1_ignoring_prefixed_strong_lines function
    """
    model = 'Splines'
    degree = 2
    nknots = None
    from_resolution = INSTRUMENT['RESOLUTION']

    # continuum fit
    order = 'median+max'
    median_wave_range = 0.01
    max_wave_range = 1.0

    strong_lines = ispec.read_line_regions(strongLines)
    continuum_model = ispec.fit_continuum(spec, \
                                          from_resolution=from_resolution, \
                                          nknots=nknots, \
                                          degree=degree, \
                                          median_wave_range=median_wave_range, \
                                          max_wave_range=max_wave_range, \
                                          model=model, \
                                          order=order, \
                                          automatic_strong_line_detection=True, \
                                          strong_line_probability=0.5, \
                                          use_errors_for_fitting=True)
    # continuum normalisation
    spec_norm = ispec.normalize_spectrum(spec, \
                                         continuum_model, \
                                         consider_continuum_errors=False)
    return spec_norm

def measureRadialVelocityWithMask(spec, ccf_mask):
    """
    Radial velocity measurement using atomic line list

    Based on example.py determine_radial_velocity_with_mask() function
    """
    models, ccf = ispec.cross_correlate_with_mask(spec, \
                                                  ccf_mask, \
                                                  lower_velocity_limit=-200, \
                                                  upper_velocity_limit=200, \
                                                  velocity_step=0.50, \
                                                  mask_depth=0.01, \
                                                  fourier=False)

    # Number of models represent the number of components
    components = len(models)
    # First component:
    try:
        rv = np.round(models[0].mu(), 2) # km/s
        rv_err = np.round(models[0].emu(), 2) # km/s
    except IndexError:
        print '\n\n\nPROBLEM RV WITH MASK, SKIPPING...\n\n\n'
        return 0.0, 0.0, components, models, ccf
    return rv, rv_err, components, models, ccf

if __name__ == "__main__":
    args = arg_parse()
    data_dir = "/Users/jmcc/Dropbox/data/SONG/20180907"
    os.chdir(data_dir)
    fitsfiles = sorted(g.glob("*.fits"))

    if args.action == 'orders':
        # loop over all fits files and extract the orders and 1D spectrum
        for fitsfile in fitsfiles:
            fits_dir = fitsfile.split('.')[0]
            if not os.path.exists(fits_dir):
                os.mkdir(fits_dir)
            with fits.open(fitsfile) as ff:
                fluxes = ff[0].data[0]
                blazes = ff[0].data[2]
                waves = ff[0].data[3]

            wave_out, flux_out, error_out = [], [], []

            fig, ax = plt.subplots(4, figsize=(20, 20), sharex=True)
            o = 1
            for wave, flux, blaze in zip(waves, fluxes, blazes):
                # normalise the blaze
                blazen = blaze / np.average(blaze)
                error = np.zeros(len(wave))
                flux_corr = flux/blazen
                # keep this for the final spectrum
                wave_out.append(wave)
                flux_out.append(flux_corr)
                error_out.append(error)

                # save per order files for doing the CCF
                order_file = "{}/{}_o{:02d}.txt".format(fits_dir, fitsfile.split('.')[0], o)
                # do nan filtering on the output
                wave_filt, flux_corr_filt, error_filt = [], [], []
                for nw, nf, ne in zip(wave, flux_corr, error):
                    if nw == nw and nf == nf and ne == ne and nf != 0:
                        # also divide wave by 10 to get nm
                        wave_filt.append(nw/10.0)
                        flux_corr_filt.append(nf)
                        error_filt.append(ne)
                wave_filt = np.array(wave_filt)
                flux_corr_filt = np.array(flux_corr_filt)
                error_filt = np.array(error_filt)
                np.savetxt(order_file,
                           np.c_[wave_filt, flux_corr_filt, error_filt],
                           fmt='%.5f\t%.4f\t%.4f',
                           header='Wave_nm  Flux  Error')

                # do the plots
                _ = ax[0].plot(wave, flux, 'k-')
                _ = ax[0].set_ylabel('Flux', fontsize=18)
                _ = ax[1].plot(wave, blaze, 'r-')
                _ = ax[1].set_ylabel('Blaze Flat', fontsize=18)
                _ = ax[2].plot(wave, blazen, 'r-')
                _ = ax[2].set_ylabel('Blaze Flat (norm_avg)', fontsize=18)
                _ = ax[3].plot(wave, flux_corr, 'g-')
                _ = ax[3].set_ylabel('Flux / Blaze Flat (norm_avg)', fontsize=18)
                _ = ax[3].set_xlabel('Wavelength (Angstroms)', fontsize=18)
                o += 1

            fig.subplots_adjust(hspace=0.0)
            fig.tight_layout()
            fig.savefig('{}/{}.png'.format(fits_dir, fitsfile.split('.')[0]), dpi=400)

            # stack and sort the orders
            wave_out = np.hstack(wave_out)
            flux_out = np.hstack(flux_out)
            error_out = np.hstack(error_out)

            n = np.where(wave_out > 4700)[0]
            wave_out = wave_out[n]
            flux_out = flux_out[n]
            error_out = error_out[n]

            temp = zip(wave_out, flux_out, error_out)
            temp = sorted(temp)
            wave_out_s, flux_out_s, error_out_s = zip(*temp)

            # save the full 1D spectrum
            np.savetxt("{}/{}_1D.txt".format(fits_dir, fitsfile.split('.')[0]),
                       np.c_[wave_out_s, flux_out_s, error_out_s],
                       fmt='%.5f\t%.4f\t%.4f',
                       header='Wave_Ang  Flux  Error')
    else:
        mask_type = 'G2'
        ccf_mask = ispec.read_cross_correlation_mask(atomicMaskLines[mask_type])

        for fitsfile in fitsfiles:
            fits_dir = fitsfile.split('.')[0]
            os.chdir(fits_dir)
            orders = sorted(g.glob('*.txt'))[10:-4]
            fig, ax = plt.subplots(len(orders), figsize=(5, 10), sharex=True)
            fig_total, ax_total = plt.subplots(2, figsize=(5, 5), sharex=True)
            for i, order in enumerate(orders):
                # read in the order
                spec = ispec.read_spectrum(order)
                print('{} Loaded...'.format(order))
                spec = normaliseContinuum(spec)
                print('{} Continuum normalised...'.format(order))
                # measure the radial velocity using a atomic mask line list
                mask_rv, mask_rv_err, mask_components, mask_models, \
                mask_ccf = measureRadialVelocityWithMask(spec, ccf_mask)
                ax[i].plot(mask_ccf['x'], mask_ccf['y'], 'k-', lw=1)
                if i==0:
                    total_ccf = np.zeros(len(mask_ccf['x']))
                if mask_rv != 0.0 and mask_rv_err != 0.0:
                    total_ccf += mask_ccf['y']
            ax[len(orders)-1].set_xlabel('RV (km/s)')
            fig.tight_layout()
            fig.subplots_adjust(hspace=0)
            fig.savefig('{}_order_ccfs.png'.format(fits_dir), dpi=300)

            # fit the CCF to normalise the baseline
            n = np.where(((mask_ccf['x'] < -50) | (mask_ccf['x'] > 50)))
            coeffs = np.polyfit(mask_ccf['x'][n], total_ccf[n], 1)
            besty = np.polyval(coeffs, mask_ccf['x'])
            total_ccf_norm = total_ccf / besty

            # plot the combined CCF and the final normalised version
            ax_total[0].plot(mask_ccf['x'], total_ccf, 'k-', lw=1)
            ax_total[0].plot(mask_ccf['x'], besty, 'r--', lw=1)
            ax_total[0].set_ylabel('CCF contrast')
            ax_total[1].plot(mask_ccf['x'], total_ccf_norm, 'k-', lw=1)
            ax_total[1].set_ylabel('CCF contrast norm')
            ax_total[1].set_xlabel('RV (km/s)')
            fig_total.tight_layout()
            fig_total.subplots_adjust(hspace=0.0)
            fig_total.savefig('{}_total_ccf.png'.format(fits_dir), dpi=300)

            # output the final CCF
            np.savetxt('{}.ccf'.format(fits_dir),
                       np.c_[mask_ccf['x'], total_ccf_norm],
                       fmt='%.3f  %.5f',
                       header='RV_kms  Contrast')

            os.chdir('../')

reason: import numpy,from astropy


text:
import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr

@testing.parameterize(
    {'out_shape': (2, 2, 6)},
    {'out_shape': (2, -1, 6)},
)
class TestReshape(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (4, 3, 2)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (2, 2, 6)).astype(numpy.float32)

    def check_forward(self, x_data):
        shape = self.out_shape
        x = chainer.Variable(x_data)
        y = functions.reshape(x, shape)
        self.assertEqual(y.data.dtype, numpy.float32)
        self.assertTrue((self.x.reshape(shape) == cuda.to_cpu(y.data)).all())

    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(
            functions.Reshape(self.gy.shape), x_data, y_grad)

    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

testing.run_module(__name__, __file__)


text:
import torch
import torch.nn as nn
import numpy as np
from torch import optim
from torch.autograd import Variable
import torch.nn.functional as F

bandit_arms = [0.2, 0, -0.2, -2]
num_arms = len(bandit_arms)

total_episodes = 1000 #Set total number of episodes to train agent on.
total_reward = np.zeros(num_arms) #Set scoreboard for bandit arms to 0.

def pullBandit(bandit):
    #Get a random number.
    result = np.random.randn(1)
    if result > bandit:
        #return a positive reward.
        return 1
    else:
        #return a negative reward.
        return -1

weights = torch.ones(num_arms, requires_grad= True)
soft_m = nn.Softmax(dim=-1)
output = Variable(soft_m(weights))

lr = 1e-3
i = 0
while i < total_episodes:
    # Choose action according to Boltzmann distribution.
    actions = soft_m(weights)
    dummyActs = actions.detach().numpy()
    a = np.random.choice(dummyActs, p=dummyActs)
    action = np.argmax(dummyActs == a)

    # Get our reward from picking one of the bandit arms.
    reward = pullBandit(bandit_arms[action])
    responsible_output = actions[action]
    loss = (-torch.log(responsible_output) * reward)

    # Update the network
    loss.backward()
    with torch.no_grad():
        weights -= weights.grad * lr
        weights.grad.zero_()

    total_reward[action] += reward

    if i % 50 == 0:
        print("Running reward for the " + str(num_arms) + " arms of the bandit: " + str(total_reward))
        # print("Weight : " + str(weights))
    i += 1
ww = weights.detach().numpy()
print("\nThe agent thinks arm " + str(np.argmax(ww).item() + 1) + " is the most promising....")
if np.argmax(ww).item() == np.argmax(-np.array(bandit_arms)):
    print("...and it was right!")
else:
    print("...and it was wrong!")


text:
# python imports
import json
import xlsxwriter
import pandas as pd
from numpy import nan
import openpyxl as op
from time import sleep
from bs4 import BeautifulSoup
from selenium import webdriver

MAIN_PAGE = 'https://sellercentral.amazon.in/gspn/?ref=spn_rc&sellFrom=IN&sellIn=IN&ref_=sc_gspn_na_hp'

CATEGORY_LIST = {
    'Taxes' : 'https://sellercentral.amazon.in/gspn/searchpage/Taxes?ref_=sc_gspn_clst_talst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Account Management' : 'https://sellercentral.amazon.in/gspn/searchpage/Account%20Management?ref_=sc_gspn_hp_blst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Imaging' : 'https://sellercentral.amazon.in/gspn/searchpage/Imaging?ref_=sc_gspn_hp_ilst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Cataloging' : 'https://sellercentral.amazon.in/gspn/searchpage/Cataloguing?ref_=sc_gspn_hp_clst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Advertising Optimization' : 'https://sellercentral.amazon.in/gspn/searchpage/Advertising%20Optimization?ref_=sc_gspn_hp_alst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Domestic Shipping' : 'https://sellercentral.amazon.in/gspn/searchpage/Domestic%20Shipping?ref_=sc_gspn_alst_dslst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Training' : 'https://sellercentral.amazon.in/gspn/searchpage/Training?ref_=sc_gspn_dslst_trlst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Seller Reinstatement' : 'https://sellercentral.amazon.in/gspn/searchpage/Seller%20Reinstatement?ref_=sc_gspn_trlst_srelst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Accounting' : 'https://sellercentral.amazon.in/gspn/searchpage/Accounting?ref_=sc_gspn_srelst_acclst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Enhanced Brand Content' : 'https://sellercentral.amazon.in/gspn/searchpage/Enhanced%20Brand%20Content?ref_=sc_gspn_acclst_ebclst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'IP Accelerator' : 'https://sellercentral.amazon.in/gspn/searchpage/IP%20Accelerator?ref_=sc_gspn_ebclst_iplst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Excess Inventory' : 'https://sellercentral.amazon.in/gspn/searchpage/Excess%20Inventory?ref_=sc_gspn_iplst_eilst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Compliance' : 'https://sellercentral.amazon.in/gspn/searchpage/Compliance?ref_=sc_gspn_eilst_colst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'FBA Preparation' : 'https://sellercentral.amazon.in/gspn/searchpage/FBA%20Preparation?ref_=sc_gspn_colst_fplst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'International Returns' : 'https://sellercentral.amazon.in/gspn/searchpage/International%20Returns?ref_=sc_gspn_fplst_irlst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'International Shipping' : 'https://sellercentral.amazon.in/gspn/searchpage/International%20Shipping?ref_=sc_gspn_irlst_islst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Storage' : 'https://sellercentral.amazon.in/gspn/searchpage/Storage?ref_=sc_gspn_islst_slst&sellFrom=IN&sellIn=IN&localeSelection=en_US',
    'Translation' : 'https://sellercentral.amazon.in/gspn/searchpage/Translation?ref_=sc_gspn_slst_tralst&sellFrom=IN&sellIn=IN&localeSelection=en_US'
}

class SPN:
    data = {}
    count = 1
    limit = 0
    clicked = 0
    source = None
    driver = None
    browser = None
    sheetname = ''
    filename = 'companies_data.xlsx'
    headers = ['Company Name', 'Web-Link', 'Provider Description', 'Requests', 'Reviews', 'Product Category', 'Lab Location', 'Provider Location', 'Shipping Mode', 'Ship from location',
                'Ship to location', 'Return from location', 'Return to location', 'Provider Language', 'Source language', 'Target language', 'Value Added Services', 'Service Type']

    def __init__(self):
        self.browser = webdriver.Chrome('chromedriver.exe')
        self.browser.set_window_position(680, 30)
        self.browser.set_window_size(700, 700)
        self.create_excel_file()

    def start_data_collection(self):
        for cat in CATEGORY_LIST:
            self.browser.get(CATEGORY_LIST[cat])
            self.browser.execute_script("window.scrollTo(0,document.body.scrollHeight)")
            self.add_excel_sheet(cat)
            self.reset_variables()
            print(f'----------------------------')
            print(f'{cat}  [{self.check_limit()}]')
            print('----------------------------')
            self.get_companies()
            self.write_to_excel()
        self.close_browser()

    def reset_variables(self):
        self.data = {}
        self.count = 1
        self.clicked = 0

    def check_limit(self):
        lis = self.browser.find_elements_by_class_name("pagination-pageNumber")
        self.limit = len(lis)
        return self.limit

    def dump_json(self):
        print('\n ----- Dumping to JSON ----- ')
        with open ('data.json', 'w+') as my_data:
            json.dump(self.data, my_data, indent=4)

    def get_companies(self):
        if self.clicked >= self.limit:
            # print(f'\nClicked = {self.clicked}\n----------------------------\n')
            return
        try:
            li = self.browser.find_element_by_class_name("a-last")
        except:
            # print('\n --- END OF PAGES --- \n')
            pass

        soup = BeautifulSoup(self.browser.page_source, features='lxml')
        main_div = soup.find('div', attrs={'class':'a-column a-span12 a-ws-span9 searchResultsRightDiv a-span-last'})
        div = main_div.find('div')
        a_tags = div.find_all('a')
        for a in a_tags:
            self.get_company_data(a['href'])
            self.count += 1
        # print(self.count, end=" ")
        try:
            li.click()
            self.browser.execute_script("window.scrollTo(0,document.body.scrollHeight)")
            self.clicked += 1
            self.get_companies()
        except:
            if self.clicked < self.limit:
                self.get_companies()
            else:
                print('\n\n -------- Completed ------- \n\n')
                sleep(10)

    def get_driver(self):
        if self.driver: return self.driver
        options = webdriver.ChromeOptions()
        options.add_argument("--incognito")
        self.driver = webdriver.Chrome(options=options)
        # self.driver = webdriver.Chrome('chromedriver.exe')
        self.driver.set_window_position(680, 30)
        self.driver.set_window_size(700, 700)
        return self.driver

    def get_company_data(self, link):
        company = {
            'Web-Link':'',
            'Reviews':'N/A',
            'Requests':'N/A',
            'Company Name': '',
            'Service Type': ['N/A'],
            'Lab Location': ['N/A'],
            'Source language' : ['N/A'],
            'Target language' : ['N/A'],
            'Product Category': ['N/A'],
            'Provider Location': ['N/A'],
            'Provider Language': ['N/A'],
            'Provider Description': 'N/A',
            'Return to location' : ['N/A'],
            'Value Added Services': ['N/A'],
            'Return from location' : ['N/A'],
        }

        browser = self.get_driver()
        company_name = ''
        browser.get(link)
        soup = BeautifulSoup(browser.page_source, features='lxml')
        try:
            first_div = soup.find('div', attrs={'class':'a-column a-span12 a-spacing-base a-ws-span12 a-ws-spacing-none a-span-last a-ws-span-last'})
            span = first_div.find('span', attrs={'class':'a-size-medium a-color-base providerNamePadding'})
            print('\n-------------------------------------------------------------------------------------')
            company_name = str(span.get_text())
            company['Company Name'] = company_name
            print(self.count, ' Company = ', company_name)
        except: pass
        company['Web-Link'] = link
        print('Web-Link: ', link)
        try:
            span = first_div.find('span', attrs={'id':'service-request-completed-count'})
            requests = str(span.get_text()).replace('More than','')
            requests = requests.replace('Less than','')
            company['Requests'] = requests
            print('Requests = ', requests)
        except: pass
        try:
            div = first_div.find('div', attrs={'class':'view-reviews center-align-metrics'})
            span = div.find('span', attrs={'class':'a-color-tertiary'})
            reviews = str(span.get_text()).replace('Reviews','')
            company['Reviews'] = reviews
            print('Reviews = ', reviews)
        except: pass
        try:
            second_div = soup.find('div', attrs={'class':'a-row category-specific-data'})
            div = second_div.find('div', attrs={'class':'a-column a-span12 a-text-left a-spacing-large'})
            divs = div.find_all('div', attrs={'class':'a-row'})
        except: pass
        try:
            desc = divs[2].find('div', attrs={'class':'a-column a-span12 a-spacing-base sectionDescription'})
            spans = desc.find_all('span')
            company['Provider Description'] = str( spans[1].get_text() )
            print('Provider Description: ', company['Provider Description'])
        except: pass
        # Service Type
        try:
            services = []
            span = divs[3].find('span', attrs={'class':'a-size-medium'})
            first = str(span.get_text())
            first = first.lower()
            print(first, end=" = ")
            div = divs[3].find('div', attrs={'class':'a-column a-span9 a-text-left'})
            spans = div.find_all('span', attrs={'class':'sectionDescription'})
            for span in spans:
                service = str(span.get_text())
                services.append(service)
                print(service, end="  ")
            if first.__contains__('service type'):              company['Service Type']             = services
            elif first.__contains__('value added services'):    company['Value Added Services']     = services
            elif first.__contains__('ship to location'):       company['Ship to location']          = services
            elif first.__contains__('return to location'):     company['Return to location']        = services
            elif first.__contains__('ship from location'):     company['Ship from location']        = services
            elif first.__contains__('return from location'):   company['Return from location']      = services
        except:
            # print('\nno service type')
            pass
        # value added services
        try:
            added_services = []
            span = divs[4].find('span', attrs={'class':'a-size-medium'})
            second = str(span.get_text())
            second = second.lower()
            print()
            print(second, end=" = ")
            div = divs[4].find('div', attrs={'class':'a-column a-span9 a-text-left'})
            spans = div.find_all('span', attrs={'class':'sectionDescription'})
            for span in spans:
                added_service = str(span.get_text())
                added_services.append(added_service)
                print(added_service, end="  ")
            if second.__contains__('value added services'):     company['Value Added Services']     = added_services
            elif second.__contains__('provider location'):      company['Provider Location']        = added_services
            elif second.__contains__('product category'):       company['Product Category']         = added_services
            elif second.__contains__('source language'):        company['Source Category']          = added_services
            elif second.__contains__('target language'):        company['Target Category']          = added_services
            elif second.__contains__('ship to location'):       company['Ship to location']         = added_services
            elif second.__contains__('return to location'):     company['Return to location']       = added_services
            elif second.__contains__('ship from location'):     company['Ship from location']       = added_services
            elif second.__contains__('return from location'):   company['Return from location']     = added_services
        except:
            # print('\nno value added service')
            pass
        # Provider Location
        try:
            locations = []
            span = divs[5].find('span', attrs={'class':'a-size-medium'})
            third = str(span.get_text())
            third = third.lower()
            print()
            print(third, end=" = ")
            div = divs[5].find('div', attrs={'class':'a-column a-span9 a-text-left'})
            spans = div.find_all('span', attrs={'class':'sectionDescription'})
            for span in spans:
                location = str(span.get_text())
                locations.append(location)
                print(location, end="  ")
            if third.__contains__('provider location'):         company['Provider Location']        = locations
            elif third.__contains__('lab location'):            company['Lab Location']             = locations
            elif third.__contains__('provider language'):       company['Provider Language']        = locations
            elif third.__contains__('source language'):         company['Source Language']          = locations
            elif third.__contains__('target language'):         company['Target Language']          = locations
            elif third.__contains__('ship to location'):        company['Ship to location']         = locations
            elif third.__contains__('return to location'):      company['Return to location']       = locations
            elif third.__contains__('ship from location'):      company['Ship from location']       = locations
            elif third.__contains__('return from location'):    company['Return from location']     = locations
        except:
            # print('\nno location')
            pass
        # Provider Language
        try:
            languages = []
            span = divs[6].find('span', attrs={'class':'a-size-medium'})
            fourth = str(span.get_text())
            fourth = fourth.lower()
            print()
            print(fourth, end=" = ")
            div = divs[6].find('div', attrs={'class':'a-column a-span9 a-text-left'})
            spans = div.find_all('span', attrs={'class':'sectionDescription'})
            for span in spans:
                language = str(span.get_text())
                languages.append(language)
                print(language, end="  ")
            if fourth.__contains__('provider location'):        company['Provider Location']        = languages
            elif fourth.__contains__('provider language'):      company['Provider Language']        = languages
            elif fourth.__contains__('source language'):        company['Source language']          = languages
            elif fourth.__contains__('target language'):        company['Target language']          = languages
            elif fourth.__contains__('ship to location'):       company['Ship to location']         = languages
            elif fourth.__contains__('return to location'):     company['Return to location']       = languages
            elif fourth.__contains__('ship from location'):     company['Ship from location']       = languages
            elif fourth.__contains__('return from location'):   company['Return from location']     = languages
            elif fourth.__contains__('shipping mode'):           company['Shipping Mode']            = languages
        except:
            # print('\nNo language')
            pass
        for index in range(7,13):
            try:
                values = []
                span = divs[index].find('span', attrs={'class':'a-size-medium'})
                text = str(span.get_text())
                text = text.lower()
                print()
                print(text, end=" = ")
                div = divs[index].find('div', attrs={'class':'a-column a-span9 a-text-left'})
                spans = div.find_all('span', attrs={'class':'sectionDescription'})
                for span in spans:
                    value = str(span.get_text())
                    values.append(value)
                    print(value, end="  ")
                # print('\n\n Values = ', values)
                if text.__contains__('provider location'):     company['Provider Location']    = values
                elif text.__contains__('provider language'):   company['Provider Language']    = values
                elif text.__contains__('shipping mode'):       company['Shipping Mode']        = values
            except: pass

        print('\n')
        self.data[company_name] = company

    def create_excel_file(self):
        # creating new excle file
        workbook = xlsxwriter.Workbook(self.filename)
        workbook.add_worksheet('Categories')
        workbook.close()
        workbook = op.load_workbook(self.filename, False)
        worksheet = workbook['Categories']
        worksheet.append(['Categories', 'Web-Links'])
        for cat in CATEGORY_LIST:
            cats = []
            cats.append(cat)
            cats.append(CATEGORY_LIST[cat])
            worksheet.append(cats)
        workbook.save(self.filename)
        workbook.close()

    def add_excel_sheet(self, sheetname):
        self.sheetname = sheetname
        workbook = op.load_workbook(self.filename, False)
        workbook.create_sheet(sheetname)
        worksheet = workbook[sheetname]
        worksheet.append(self.headers)
        workbook.save(self.filename)
        workbook.close()

    def write_to_excel(self):
        workbook = op.load_workbook(self.filename, False)
        worksheet = workbook[self.sheetname]
        for company in self.data:
            saving_data = []
            company = self.data[company]
            for data_field in self.headers:
                try:
                    if data_field == 'Service Type':
                        for v in company[data_field]: saving_data.append(v)
                    else:
                        if isinstance(company[data_field], list):
                            new_list = ''
                            for v in company[data_field]:
                                if len(company[data_field]) > 1:    new_list += v + ' , '
                                else: new_list += v
                            saving_data.append(new_list)
                        else: saving_data.append(company[data_field])
                except: saving_data.append('N/A')
            worksheet.append(saving_data)
        workbook.save(self.filename)
        workbook.close()

    def close_browser(self):
        self.browser.quit()
        self.driver.quit()

if __name__ == '__main__':
    spn = SPN()
    spn.start_data_collection()
    sleep(1)

""" TODO

1. separate headers for each category
2. headers = {}
3. headers['category']

"""

reason: from numpy


text:
import numpy as np
import base64

VERTICES = np.array([0.,0.,0.,    0.,1.,0.,    1.,0.,0.], dtype=np.float32)
INDICES = np.array([0, 1, 2], dtype=np.ushort)

HOWMANY = 3
MAX_X = 1
MAX_Y = 1
MAX_Z = 0
MIN_X = 0
MIN_Y = 0
MIN_Z = 0
MAX = 2
MIN = 0

HOWMANYBYTES_V = VERTICES.nbytes
HOWMANYBYTES_I = INDICES.nbytes

B64_VERTICES = base64.b64encode(VERTICES)
B64_INDICES = base64.b64encode(INDICES)

gltf = {
    "asset": {
        "version": "2.0",
        "generator": "CS460 Magic Fingers"
    },

  "accessors": [
        {
            "bufferView": 0,
            "byteOffset": 0,
            "componentType": 5126,
            "count": HOWMANY,
            "type": "VEC3",
            "max": [MAX_X, MAX_Y, MAX_Z],
            "min": [MIN_X, MIN_Y, MIN_Z]
        },
        {
            "bufferView": 1,
            "byteOffset": 0,
            "componentType": 5123,
            "count": HOWMANY,
            "type": "SCALAR",
            "max": [MAX],
            "min": [MIN]
        }
    ],

    "bufferViews": [
        {
            "buffer": 0,
            "byteOffset": 0,
            "byteLength": HOWMANYBYTES_V,
            "target": 34962
        },
        {
            "buffer": 1,
            "byteOffset": 0,
            "byteLength": HOWMANYBYTES_I,
            "target": 34963
        }
    ],

    "buffers": [
        {
            "uri": "data:application/octet-stream;base64,"+str(B64_VERTICES, 'utf-8'),
            "byteLength": HOWMANYBYTES_V
        },
        {
            "uri": "data:application/octet-stream;base64,"+str(B64_INDICES, 'utf-8'),
            "byteLength": HOWMANYBYTES_I
        }
    ],

    "meshes": [
        {
            "primitives": [{
                 "mode": 4,
                 "attributes": {
                     "POSITION": 0
                 },
                 "indices": 1
            }]
        }
    ],

    "nodes": [
        {
            "mesh": 0
        }
    ],

    "scenes": [
        {
            "nodes": [
                0
            ]
        }
    ],

    "scene": 0
}

print ( str(gltf).replace("'", '"') ) # we need double quotes instead of single quotes