text:
from typing import Dict, List

import numpy
import random
import os
import matplotlib.pyplot as plt
from dotenv import load_dotenv

load_dotenv()

HOST = os.getenv("BYTIE_HOST") or 'http://localhost/'
PATH = os.getenv("BYTIE_PATH") or './.tmp'

sum = numpy.sum
mean = numpy.mean
median = numpy.median

def quantile(args: List) -> str:
    nums = args[0]
    q = args[1]
    return str(numpy.quantile(nums, q))

def draw_random_numbers(args: List) -> str:
    n = args[0]
    return str([random.random() for i in range(n)])

def plot(nums: List) -> str:
    randstr = str(random.randrange(10000000))
    filename = f"plot_{randstr}.png"
    filepath = f"{PATH}/{filename}"
    url = f"{HOST}/{filename}"
    plt.plot(nums)
    plt.title(f"Grifik çizdircik bişki prigrim yik mi ki birdisin? (:")
    plt.savefig(filepath)
    plt.close()
    return url

ext: py

max_stars_repo_name: KubilayGazioglu/bytie

max_issues_repo_name: KubilayGazioglu/bytie

max_forks_repo_name: KubilayGazioglu/bytie

reason: import numpy


text:
'''
   usage: python famus-launch.py batch.setup
   see scripts/famus-launch/ directory for sample input files

   The program reads a table and sample files
   to setup a batch of .input and .sh

   The table lives in batch_name.setup.
   Read sample.input and b-sample.sh

   Updated: 19 Jan 2021
'''

import sys
import numpy as np

f_sample = 'sample.input'
b_sample = 'b-sample.sh'
N_shell  = 1

### load (optional) user input
def load_option(args):
    #print('additonal options:', args)

    if (args[0] == '-f'):
        print(' detected user input -f')
        global f_sample
        f_sample = args[1]

    if (args[0] == '-b'):
        print(' detected user input -b')
        global b_sample
        b_sample = args[1]

    if (args[0] == '-s'):
        print(' detected user input -s')
        global N_shell
        N_shell = int(args[1])

    try:
        load_option(args[2:])
    except:
        pass

def load(fname):
    try:
        with open(fname) as f:
            sample_input = f.readlines()
        print(' reading sample: ', fname)
    except:
        print(' sample input \'%s\' not found' % fname)
        exit()

    return sample_input

def load_setup(fin):
    # reads a .setup file and splits lines, removing white spaces
    with open(fin) as f:
        input_data = f.readlines()

    indata = []
    for line in input_data:
        if (line.find('#') > -1):
            continue
        data   =  line.strip().split(',')
        values = [x.strip() for x in data]
        indata.append(values)

    return np.array(indata)

def write_input(new_input, value):

    with open(new_input, 'w') as f:
        for line in sample_input:

            for j in np.arange(N_edits):
                knob = input_knobs[j]
                if (line.find( knob ) > -1):
                    line = ' {:15} =   {},\n'.format(knob, value[j])
                    #print('  %s'%line)
            f.write(line)
        print('  writing {:40}: {}'.format(new_input,value) )

def write_batch(new_batch, tag):

    #tag = '.'.join(new_input.split('.')[:-1])
    with open(new_batch, 'w') as f:
        for line in sample_batch:
           f.write(line)

        cmd = 'srun ~/CODE/FOCUS/bin/xfocus {}.input > {}.log'.format(tag,tag)
        f.write('\n{}\n'.format(cmd) )

    print('  writing {}'.format(new_batch) )

def append_batch(open_batch, tag):

    with open(open_batch, 'a') as f:

        cmd = 'srun ~/CODE/FOCUS/bin/xfocus {}.input > {}.log'.format(tag,tag)
        f.write('{}\n'.format(cmd) )

    print('    updating {}'.format(open_batch) )

# load options
if (len(sys.argv) > 2):
    try:
        load_option( sys.argv[2:] )
    except:
        print('usage: python famus-launch.py batch.setup -f alt-opt.input -b alt-opt.sh')

sample_input = load(f_sample)

fin = sys.argv[1]
batch_name = '.'.join(fin.split('.')[:-1])
#print(' batch name:', batch_name)

indata = load_setup(fin)
target_name = indata[0][0]
input_knobs = indata[0][1:]

# write inputs
N_edits = len(input_knobs)
print(' batch name: {:37}: {}'.format(batch_name, input_knobs) )
for line in indata[1:]:
    tag   = line[0]
    edits = line[1:]

    new_input = '{}-{}-{}.input'.format(target_name, batch_name, tag)
    write_input(new_input, edits)

# write batch
sample_batch = load(b_sample)
f_script = 'run-{}.sh'.format(batch_name)
fsh = open(f_script,'w')

tags = indata[1:,0]
for j in np.arange( len(tags) ):
    tag = tags[j]
    handle = '{}-{}-{}'.format(target_name, batch_name, tag)

    recycle = j % (N_shell)
    if (recycle):
        append_batch(new_batch, handle)
        continue

    new_batch = 'b-{}-{}.sh'.format(batch_name, tag)
    write_batch(new_batch, handle)
    fsh.write('sbatch %s\n' % new_batch)

print(' exporting script: {}'.format(f_script) )

max_stars_repo_name: tmqian/MUSE

max_issues_repo_name: tmqian/MUSE

max_forks_repo_name: tmqian/MUSE


text:
# -*- coding: utf-8 -*-
# *****************************************************************************
# Copyright (c) 2020, Intel Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     Redistributions of source code must retain the above copyright notice,
#     this list of conditions and the following disclaimer.
#
#     Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions and the following disclaimer in the documentation
#     and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# *****************************************************************************

import numpy as np
import pandas as pd
import unittest

from itertools import (combinations_with_replacement, product, filterfalse, chain)

from sdc.tests.test_base import TestCase
from sdc.utilities.sdc_typing_utils import kwsparams2list
from sdc.tests.test_series import _make_func_from_text
from numba.core.errors import TypingError

test_global_index_names = [None, 'abc', 'index']
test_global_range_member_values = [1, 2, 10, -5, 0, None]

def _generate_valid_range_params():

    def valid_params_predicate(range_params):
        # if step is zero or all start/stop/step are None range is invalid
        return (range_params[-1] == 0
                or all(map(lambda x: x is None, range_params)))

    return filterfalse(
        valid_params_predicate,
        combinations_with_replacement(test_global_range_member_values, 3)
    )

def _generate_range_indexes_fixed(size, start=1, step=3):
    yield pd.RangeIndex(size)
    yield pd.RangeIndex(size, name='abc')
    yield pd.RangeIndex(stop=step * size, step=step)
    yield pd.RangeIndex(stop=2*step*size, step=2*step)
    yield pd.RangeIndex(start=start, stop=start + size*step - step//2, step=step)
    yield pd.RangeIndex(start=start + step, stop=start + (size + 1)*step, step=step)

def _generate_index_param_values(n):
    return chain([None], _generate_range_indexes_fixed(n))

class TestRangeIndex(TestCase):

    def test_range_index_create_and_box(self):
        def test_impl(start, stop, step, name):
            return pd.RangeIndex(start, stop, step, name=name)
        sdc_func = self.jit(test_impl)

        for params in _generate_valid_range_params():
            start, stop, step = params
            for name in test_global_index_names:
                with self.subTest(start=start, stop=stop, step=step, name=name):
                    result = sdc_func(start, stop, step, name)
                    result_ref = test_impl(start, stop, step, name)
                    pd.testing.assert_index_equal(result, result_ref)

    def test_range_index_unbox_and_box(self):
        def test_impl(index):
            return index
        sdc_func = self.jit(test_impl)

        for params in _generate_valid_range_params():
            start, stop, step = params
            for name in test_global_index_names:
                index = pd.RangeIndex(start, stop, step, name=name)
                with self.subTest(index=index):
                    result = sdc_func(index)
                    result_ref = test_impl(index)
                    pd.testing.assert_index_equal(result, result_ref)

    @unittest.skip("TODO: support boxing/unboxing and parent ref for Python ranges in Numba")
    def test_range_index_unbox_data_id_check(self):
        def test_impl(index):
            return index
        sdc_func = self.jit(test_impl)

        index = pd.RangeIndex(11, name='abc')
        result = sdc_func(index)
        result_ref = test_impl(index)
        self.assertIs(index._range, result_ref._range)
        self.assertIs(result._range, result_ref._range)

    @unittest.skip("TODO: add support for integers as floats in ctor")
    def test_range_index_create_from_floats(self):
        def test_impl(*args):
            return pd.RangeIndex(*args)
        sdc_func = self.jit(test_impl)

        start, stop, step = 3.0, 15.0, 2.0
        result = sdc_func(start, stop, step)
        result_ref = test_impl(start, stop, step)
        pd.testing.assert_index_equal(result, result_ref)

    def test_range_index_create_invalid1(self):
        def test_impl(start, stop, step):
            return pd.RangeIndex(start, stop, step)
        sdc_func = self.jit(test_impl)

        # zero step is not allowed by pandas
        start, stop, step = 3, 5, 0
        with self.assertRaises(Exception) as context:
            test_impl(start, stop, step)
        pandas_exception = context.exception

        with self.assertRaises(type(pandas_exception)) as context:
            sdc_func(start, stop, step)
        sdc_exception = context.exception
        self.assertIn(str(sdc_exception), str(pandas_exception))

    def test_range_index_create_invalid2(self):
        def test_impl():
            return pd.RangeIndex(name='index')
        sdc_func = self.jit(test_impl)

        # all start, stop and step cannot be None at the same time
        with self.assertRaises(Exception) as context:
            test_impl()
        pandas_exception = context.exception

        with self.assertRaises(type(pandas_exception)) as context:
            sdc_func()
        sdc_exception = context.exception
        self.assertIn(str(sdc_exception), str(pandas_exception))

    def test_range_index_create_defaults(self):
        func_lines = [
            'def test_impl():',
            '  return pd.RangeIndex({})'
        ]
        test_impl_text = '\n'.join(func_lines)

        # use non default values for all parameters except one (tested)
        non_default_params = {'start': 2, 'stop': 7, 'step': 2, 'name': "'index'"}
        for arg in non_default_params.keys():
            with self.subTest(omitted=arg):
                kwargs = {key: val for key, val in non_default_params.items() if key != arg}
                func_text = test_impl_text.format(', '.join(kwsparams2list(kwargs)))
                test_impl = _make_func_from_text(func_text, global_vars={'pd': pd})
                sdc_func = self.jit(test_impl)
                result = sdc_func()
                result_ref = test_impl()
                pd.testing.assert_index_equal(result, result_ref)

    def test_range_index_create_param_copy(self):
        def test_impl(stop, value):
            return pd.RangeIndex(stop, copy=value)
        sdc_func = self.jit(test_impl)

        with self.assertRaises(TypingError) as raises:
            sdc_func(11, False)
        self.assertIn("SDCLimitation: pd.RangeIndex(). Unsupported parameter",
                      str(raises.exception))

    def test_range_index_create_param_name_literal_str(self):
        def test_impl(stop):
            return pd.RangeIndex(stop, name='index')
        sdc_func = self.jit(test_impl)

        n = 11
        result = sdc_func(n)
        result_ref = test_impl(n)
        pd.testing.assert_index_equal(result, result_ref)

    def test_range_index_create_param_dtype(self):
        def test_impl(stop, dtype):
            return pd.RangeIndex(stop, dtype=dtype)
        sdc_func = self.jit(test_impl)

        n = 11
        supported_dtypes = [None, np.int64, 'int64']
        for dtype in supported_dtypes:
            with self.subTest(dtype=dtype):
                result = sdc_func(n, dtype)
                result_ref = test_impl(n, dtype)
                pd.testing.assert_index_equal(result, result_ref)

    def test_range_index_create_param_dtype_invalid(self):
        def test_impl(stop, dtype):
            return pd.RangeIndex(stop, dtype=dtype)
        sdc_func = self.jit(test_impl)

        n = 11
        invalid_dtypes = ['float', np.int32, 'int32']
        for dtype in invalid_dtypes:
            with self.subTest(dtype=dtype):
                with self.assertRaises(Exception) as context:
                    test_impl(n, dtype)
                pandas_exception = context.exception

                with self.assertRaises(type(pandas_exception)) as context:
                    sdc_func(n, dtype)
                sdc_exception = context.exception
                self.assertIn(str(sdc_exception), str(pandas_exception))

    def test_range_index_attribute_start(self):
        def test_impl(*args):
            index = pd.RangeIndex(*args)
            return index.start
        sdc_func = self.jit(test_impl)

        for params in _generate_valid_range_params():
            start, stop, step = params
            with self.subTest(start=start, stop=stop, step=step):
                result = sdc_func(*params)
                result_ref = test_impl(*params)
                self.assertEqual(result, result_ref)

    def test_range_index_attribute_stop(self):
        def test_impl(*args):
            index = pd.RangeIndex(*args)
            return index.stop
        sdc_func = self.jit(test_impl)

        for params in _generate_valid_range_params():
            start, stop, step = params
            with self.subTest(start=start, stop=stop, step=step):
                result = sdc_func(*params)
                result_ref = test_impl(*params)
                self.assertEqual(result, result_ref)

    def test_range_index_attribute_step(self):
        def test_impl(*args):
            index = pd.RangeIndex(*args)
            return index.step
        sdc_func = self.jit(test_impl)

        for params in _generate_valid_range_params():
            start, stop, step = params
            with self.subTest(start=start, stop=stop, step=step):
                result = sdc_func(*params)
                result_ref = test_impl(*params)
                self.assertEqual(result, result_ref)

    def test_range_index_attribute_dtype(self):
        def test_impl(index):
            return index.dtype
        sdc_func = self.jit(test_impl)

        index = pd.RangeIndex(11)
        result = sdc_func(index)
        result_ref = test_impl(index)
        self.assertEqual(result, result_ref)

    def test_range_index_attribute_name(self):
        def test_impl(index):
            return index.name
        sdc_func = self.jit(test_impl)

        n = 11
        for name in test_global_index_names:
            with self.subTest(name=name):
                index = pd.RangeIndex(n, name=name)
                result = sdc_func(index)
                result_ref = test_impl(index)
                self.assertEqual(result, result_ref)

    def test_range_index_len(self):
        def test_impl(*args):
            index = pd.RangeIndex(*args)
            return len(index)
        sdc_func = self.jit(test_impl)

        for params in _generate_valid_range_params():
            start, stop, step = params
            with self.subTest(start=start, stop=stop, step=step):
                result = sdc_func(*params)
                result_ref = test_impl(*params)
                self.assertEqual(result, result_ref)

    def test_range_index_attribute_values(self):
        def test_impl(index):
            return index.values
        sdc_func = self.jit(test_impl)

        for params in _generate_valid_range_params():
            index = pd.RangeIndex(*params)
            with self.subTest(index=index):
                result = sdc_func(index)
                result_ref = test_impl(index)
                np.testing.assert_array_equal(result, result_ref)

    def test_range_index_contains(self):
        def test_impl(index, value):
            return value in index
        sdc_func = self.jit(test_impl)

        index = pd.RangeIndex(1, 11, 2)
        values_to_test = [-5, 15, 1, 11, 5, 6]
        for value in values_to_test:
            with self.subTest(value=value):
                result = sdc_func(index, value)
                result_ref = test_impl(index, value)
                np.testing.assert_array_equal(result, result_ref)

    def test_range_index_copy(self):
        def test_impl(index, new_name):
            return index.copy(name=new_name)
        sdc_func = self.jit(test_impl)

        for params in _generate_valid_range_params():
            start, stop, step = params
            for name, new_name in product(test_global_index_names, repeat=2):
                index = pd.RangeIndex(start, stop, step, name=name)
                with self.subTest(index=index, new_name=new_name):
                    result = sdc_func(index, new_name)
                    result_ref = test_impl(index, new_name)
                    pd.testing.assert_index_equal(result, result_ref)

    def test_range_index_getitem_scalar(self):
        def test_impl(index, idx):
            return index[idx]
        sdc_func = self.jit(test_impl)

        for params in _generate_valid_range_params():
            index = pd.RangeIndex(*params)
            n = len(index)
            if not n:  # test only non-empty ranges
                continue
            values_to_test = [-n, n // 2, n - 1]
            for idx in values_to_test:
                with self.subTest(index=index, idx=idx):
                    result = sdc_func(index, idx)
                    result_ref = test_impl(index, idx)
                    self.assertEqual(result, result_ref)

    def test_range_index_getitem_scalar_idx_bounds(self):
        def test_impl(index, idx):
            return index[idx]
        sdc_func = self.jit(test_impl)

        n = 11
        index = pd.RangeIndex(n, name='abc')
        values_to_test = [-(n + 1), n]
        for idx in values_to_test:
            with self.subTest(idx=idx):
                with self.assertRaises(Exception) as context:
                    test_impl(index, idx)
                pandas_exception = context.exception

                with self.assertRaises(type(pandas_exception)) as context:
                    sdc_func(index, idx)
                sdc_exception = context.exception
                self.assertIsInstance(sdc_exception, type(pandas_exception))
                self.assertIn("out of bounds", str(sdc_exception))

    def test_range_index_getitem_slice(self):
        def test_impl(index, idx):
            return index[idx]
        sdc_func = self.jit(test_impl)

        index_len = 17
        start_values, step_values = [0, 5, -5], [1, 2, 7]
        slices_params = combinations_with_replacement(
            [None, 0, -1, index_len // 2, index_len, index_len - 3, index_len + 3, -(index_len + 3)],
            2
        )

        for start, step, slice_step in product(start_values, step_values, step_values):
            stop = start + index_len
            for slice_start, slice_stop in slices_params:
                idx = slice(slice_start, slice_stop, slice_step)
                index = pd.RangeIndex(start, stop, step, name='abc')
                with self.subTest(index=index, idx=idx):
                    result = sdc_func(index, idx)
                    result_ref = test_impl(index, idx)
                    pd.testing.assert_index_equal(result, result_ref)

    @unittest.skip("Needs writable native struct type members in Numba")
    def test_range_index_named_set_name(self):
        def test_impl(index):
            index.name = 'def'
            return index
        sdc_func = self.jit(test_impl)

        n = 11
        index1 = pd.RangeIndex(n, name='abc')
        index2 = index1.copy(deep=True)
        result = sdc_func(index1)
        result_ref = test_impl(index2)
        pd.testing.assert_index_equal(result, result_ref)

    @unittest.skip("Needs writable native struct type members and single common type for name")
    def test_range_index_unnamed_set_name(self):
        def test_impl(index):
            index.name = 'def'
            return index
        sdc_func = self.jit(test_impl)

        n = 11
        index1 = pd.RangeIndex(n, name='abc')
        index2 = index1.copy(deep=True)
        result = sdc_func(index1)
        result_ref = test_impl(index2)
        pd.testing.assert_index_equal(result, result_ref)

    def _test_range_indexes(self, test_impl, indexes, size, apply_func):
        for index in indexes:
            expected_res = pd.RangeIndex(size) if index is None else index
            with self.subTest(series_index=index):
                args = apply_func(size, index)
                result = test_impl(args)
                pd.testing.assert_index_equal(result, expected_res)

    def test_range_index_unbox_series_with_index(self):
        @self.jit
        def test_impl(S):
            # TO-DO: this actually includes calling 'index' attribute overload, should really be S._index,
            # but this requires separate type (e.g. DefaultIndexType) instead of types.none as native index
            return S.index

        n = 11
        for index in _generate_index_param_values(n):
            expected_res = pd.RangeIndex(n) if index is None else index
            with self.subTest(series_index=index):
                S = pd.Series(np.ones(n), index=index)
                result = test_impl(S)
                pd.testing.assert_index_equal(result, expected_res)

    def test_range_index_create_series_with_index(self):
        @self.jit
        def test_impl(data, index):
            S = pd.Series(data=data, index=index)
            return S.index

        n = 11
        series_data = np.ones(n)
        for index in _generate_index_param_values(n):
            expected_res = pd.RangeIndex(n) if index is None else index
            with self.subTest(series_index=index):
                result = test_impl(series_data, index)
                pd.testing.assert_index_equal(result, expected_res)

    def test_range_index_box_series_with_index(self):
        def test_impl(data, index):
            return pd.Series(data=data, index=index)
        sdc_func = self.jit(test_impl)

        n = 11
        series_data = np.ones(n)
        for index in _generate_index_param_values(n):
            with self.subTest(series_index=index):
                result = sdc_func(series_data, index)
                result_ref = test_impl(series_data, index)
                pd.testing.assert_series_equal(result, result_ref)

    def test_range_index_get_series_index(self):
        def test_impl(S):
            return S.index
        sdc_func = self.jit(test_impl)

        n = 11
        for index in _generate_index_param_values(n):
            with self.subTest(series_index=index):
                S = pd.Series(np.ones(n), index=index)
                result = sdc_func(S)
                result_ref = test_impl(S)
                pd.testing.assert_index_equal(result, result_ref)

    def test_range_index_unbox_df_with_index(self):
        @self.jit
        def test_impl(df):
            return df.index

        n = 11
        for index in _generate_index_param_values(n):
            expected_res = pd.RangeIndex(n) if index is None else index
            with self.subTest(df_index=index):
                df = pd.DataFrame({'A': np.ones(n), 'B': np.arange(n)}, index=index)
                result = test_impl(df)
                pd.testing.assert_index_equal(result, expected_res)

    def test_range_index_create_df_with_index(self):
        @self.jit
        def test_impl(A, B, index):
            df = pd.DataFrame({'A': A, 'B': B}, index=index)
            return df.index

        n = 11
        A, B = np.ones(n), np.arange(n)
        for index in _generate_index_param_values(n):
            expected_res = pd.RangeIndex(n) if index is None else index
            with self.subTest(df_index=index):
                result = test_impl(A, B, index)
                pd.testing.assert_index_equal(result, expected_res)

    def test_range_index_box_df_with_index(self):
        def test_impl(A, B, index):
            return pd.DataFrame({'A': A, 'B': B}, index=index)
        sdc_func = self.jit(test_impl)

        n = 11
        A, B = np.ones(n), np.arange(n, dtype=np.intp)
        for index in _generate_index_param_values(n):
            with self.subTest(series_index=index):
                result = sdc_func(A, B, index)
                result_ref = test_impl(A, B, index)
                pd.testing.assert_frame_equal(result, result_ref)

    def test_range_index_get_df_index(self):
        def test_impl(df):
            return df.index
        sdc_func = self.jit(test_impl)

        n = 11
        for index in _generate_index_param_values(n):
            with self.subTest(series_index=index):
                df = pd.DataFrame({'A': np.ones(n)}, index=index)
                result = sdc_func(df)
                result_ref = test_impl(df)
                pd.testing.assert_index_equal(result, result_ref)

    def test_range_index_iterator_1(self):
        def test_impl(index):
            res = []
            for i, label in enumerate(index):
                res.append((i, label))
            return res
        sdc_func = self.jit(test_impl)

        index = pd.RangeIndex(1, 21, 3)
        result = sdc_func(index)
        result_ref = test_impl(index)
        self.assertEqual(result, result_ref)

    def test_range_index_iterator_2(self):
        def test_impl(index):
            res = []
            for label in index:
                if not label % 2:
                    res.append(label)
            return res
        sdc_func = self.jit(test_impl)

        index = pd.RangeIndex(1, 21, 3)
        result = sdc_func(index)
        result_ref = test_impl(index)
        self.assertEqual(result, result_ref)

    def test_range_index_nparray(self):
        def test_impl(index):
            return np.array(index)
        sdc_func = self.jit(test_impl)

        index = pd.RangeIndex(1, 21, 3)
        result = sdc_func(index)
        result_ref = test_impl(index)
        np.testing.assert_array_equal(result, result_ref)

    def test_range_index_operator_eq_index_1(self):
        """ Verifies operator.eq implementation for pandas RangeIndex in a case of equal range sizes """
        def test_impl(index1, index2):
            return index1 == index2
        sdc_func = self.jit(test_impl)

        n = 11
        for index1, index2 in product(_generate_range_indexes_fixed(n), repeat=2):
            with self.subTest(index1=index1, index2=index2):
                result = np.asarray(sdc_func(index1, index2))   # FIXME_Numba#5157: remove np.asarray
                result_ref = test_impl(index1, index2)
                np.testing.assert_array_equal(result, result_ref)

    def test_range_index_operator_eq_index_2(self):
        """ Verifies operator.eq implementation for pandas RangeIndex in a case of non equal range sizes """
        def test_impl(index1, index2):
            return index1 == index2
        sdc_func = self.jit(test_impl)

        index1 = pd.RangeIndex(1, 22, 5)
        index2 = pd.RangeIndex(1, 22, 10)
        with self.assertRaises(Exception) as context:
            test_impl(index1, index2)
        pandas_exception = context.exception

        with self.assertRaises(type(pandas_exception)) as context:
            sdc_func(index1, index2)
        sdc_exception = context.exception
        self.assertIn(str(sdc_exception), str(pandas_exception))

    def test_range_index_operator_eq_scalar(self):
        """ Verifies operator.eq implementation for pandas RangeIndex and a scalar value """
        def test_impl(A, B):
            return A == B
        sdc_func = self.jit(test_impl)

        n = 11
        A = pd.RangeIndex(n)
        scalars_to_test = [
            A.start,
            float(A.start),
            A.start + 1,
            (A.start + A.stop) / 2,
            A.stop,
        ]
        for B in scalars_to_test:
            for swap_operands in (False, True):
                if swap_operands:
                    A, B = B, A
                with self.subTest(left=A, right=B):
                    result = np.asarray(sdc_func(A, B))  # FIXME_Numba#5157: remove np.asarray
                    result_ref = test_impl(A, B)
                    np.testing.assert_array_equal(result, result_ref)

    def test_range_index_operator_eq_nparray(self):
        """ Verifies operator.eq implementation for pandas RangeIndex and a numpy array """
        def test_impl(A, B):
            return A == B
        sdc_func = self.jit(test_impl)

        n = 11
        for A, B in product(
            _generate_range_indexes_fixed(n),
            map(lambda x: np.array(x), _generate_range_indexes_fixed(n))
        ):
            for swap_operands in (False, True):
                if swap_operands:
                    A, B = B, A
                with self.subTest(left=A, right=B):
                    result = np.asarray(sdc_func(A, B))  # FIXME_Numba#5157: remove np.asarray
                    result_ref = test_impl(A, B)
                    np.testing.assert_array_equal(result, result_ref)

    def test_range_index_operator_ne_index(self):
        """ Verifies operator.ne implementation for pandas RangeIndex in a case of non equal range sizes """
        def test_impl(index1, index2):
            return index1 != index2
        sdc_func = self.jit(test_impl)

        n = 11
        for index1, index2 in product(_generate_range_indexes_fixed(n), repeat=2):
            with self.subTest(index1=index1, index2=index2):
                result = np.asarray(sdc_func(index1, index2))   # FIXME_Numba#5157: remove np.asarray
                result_ref = test_impl(index1, index2)
                np.testing.assert_array_equal(result, result_ref)

    @unittest.skip("Need support unboxing Python range in Numba with parent ref")
    def test_range_index_operator_is_1(self):
        def test_impl(index1, index2):
            return index1 is index2
        sdc_func = self.jit(test_impl)

        # positive testcase
        with self.subTest(subtest="same indexes"):
            index1 = pd.RangeIndex(1, 21, 3)
            index2 = index1
            result = sdc_func(index1, index2)
            result_ref = test_impl(index1, index2)
            self.assertEqual(result, result_ref)

        # negative testcase
        with self.subTest(subtest="not same indexes"):
            index1 = pd.RangeIndex(1, 21, 3)
            index2 = pd.RangeIndex(1, 21, 3)
            result = sdc_func(index1, index2)
            result_ref = test_impl(index1, index2)
            self.assertEqual(result, result_ref)

    def test_range_index_operator_is_2(self):
        def test_impl_1(*args):
            index1 = pd.RangeIndex(*args)
            index2 = index1
            return index1 is index2
        sdc_func_1 = self.jit(test_impl_1)

        def test_impl_2(*args):
            index1 = pd.RangeIndex(*args)
            index2 = pd.RangeIndex(*args)
            return index1 is index2
        sdc_func_2 = self.jit(test_impl_2)

        # positive testcase
        params = 1, 21, 3
        with self.subTest(subtest="same indexes"):
            result = sdc_func_1(*params)
            result_ref = test_impl_1(*params)
            self.assertEqual(result, result_ref)

        # negative testcase
        with self.subTest(subtest="not same indexes"):
            result = sdc_func_2(*params)
            result_ref = test_impl_2(*params)
            self.assertEqual(result, result_ref)

    def test_range_index_getitem_by_mask(self):
        def test_impl(index, mask):
            return index[mask]
        sdc_func = self.jit(test_impl)

        n = 11
        np.random.seed(0)
        mask = np.random.choice([True, False], n)
        for index in _generate_range_indexes_fixed(n):
            result = sdc_func(index, mask)
            result_ref = test_impl(index, mask)
            # FIXME: replace with pd.testing.assert_index_equal when Int64Index is supported
            np.testing.assert_array_equal(result, result_ref.values)

    def test_range_index_support_reindexing(self):
        from sdc.datatypes.common_functions import sdc_reindex_series

        def pyfunc(data, index, name, by_index):
            S = pd.Series(data, index, name=name)
            return S.reindex(by_index)

        @self.jit
        def sdc_func(data, index, name, by_index):
            return sdc_reindex_series(data, index, name, by_index)

        n = 100
        np.random.seed(0)
        mask = np.random.choice([True, False], n)
        name = 'asdf'
        index1 = pd.RangeIndex(n)
        index2 = index1[::-1]
        result = sdc_func(mask, index1, name, index2)
        result_ref = pyfunc(mask, index1, name, index2)
        pd.testing.assert_series_equal(result, result_ref)

    def test_range_index_support_join(self):
        from sdc.datatypes.common_functions import sdc_join_series_indexes

        def pyfunc(index1, index2):
            return index1.join(index2, how='outer', return_indexers=True)

        @self.jit
        def sdc_func(index1, index2):
            return sdc_join_series_indexes(index1, index2)

        index1 = pd.RangeIndex(1, 21, 3, name='asv')
        index2 = pd.RangeIndex(19, -1, -3, name='df')
        result = sdc_func(index1, index2)
        result_ref = pyfunc(index1, index2)
        results_names = ['result index', 'left indexer', 'right indexer']
        for i, name in enumerate(results_names):
            result_elem = result[i]
            result_ref_elem = result_ref[i].values if not i else result_ref[i]
            np.testing.assert_array_equal(result_elem, result_ref_elem, f"Mismatch in {name}")

    def test_range_index_support_take(self):
        from sdc.datatypes.common_functions import _sdc_take

        def pyfunc(index1, indexes):
            return index1.values.take(indexes)

        @self.jit
        def sdc_func(index1, indexes):
            return _sdc_take(index1, indexes)

        n, k = 1000, 200
        np.random.seed(0)
        index = pd.RangeIndex(stop=3 * n, step=3, name='asd')
        indexes = np.random.choice(np.arange(n), n)[:k]
        result = sdc_func(index, indexes)
        result_ref = pyfunc(index, indexes)
        np.testing.assert_array_equal(result, result_ref)

    def test_range_index_support_astype(self):
        from sdc.functions.numpy_like import astype

        def pyfunc(index):
            return index.values.astype(np.int64)

        @self.jit
        def sdc_func(index):
            return astype(index, np.int64)

        index = pd.RangeIndex(stop=11, name='asd')
        np.testing.assert_array_equal(sdc_func(index), pyfunc(index))

    def test_range_index_support_array_equal(self):
        from sdc.functions.numpy_like import array_equal

        def pyfunc(index1, index2):
            return np.array_equal(index1.values, index2.values)

        @self.jit
        def sdc_func(index1, index2):
            return array_equal(index1, index2)

        for params1, params2 in product(_generate_valid_range_params(), repeat=2):
            for name1, name2 in product(test_global_index_names, repeat=2):
                index1 = pd.RangeIndex(*params1, name=name1)
                index2 = pd.RangeIndex(*params2, name=name2)
                with self.subTest(index1=index1, index2=index2):
                    result = sdc_func(index1, index2)
                    result_ref = pyfunc(index1, index2)
                    self.assertEqual(result, result_ref)

    def test_range_index_support_copy(self):
        from sdc.functions.numpy_like import copy

        @self.jit
        def sdc_func(index):
            return copy(index)

        for params in _generate_valid_range_params():
            for name in test_global_index_names:
                index = pd.RangeIndex(*params, name=name)
                with self.subTest(index=index):
                    result = sdc_func(index)
                    pd.testing.assert_index_equal(result, index)

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: densmirn/sdc

max_issues_repo_name: densmirn/sdc

max_forks_repo_name: densmirn/sdc

reason: import numpy,from numba


text:
''' Runs experiments for our alife20 paper, exploring the effect of extinction events on the
complexity of maps generated by a trained RL agent.'''
import os
import shutil
import re
import time

import matplotlib.pyplot as plt
import gym
import numpy as np
import torch
from PIL import Image

import game_of_life
import gym_city
from arguments import get_parser
from envs import VecPyTorch, make_vec_envs
from evaluate import Evaluator
from model import Policy
from utils import get_vec_normalize

#plt.switch_backend('agg')

def parse_cl_args():
    #TODO: away with this function, in theory.
    '''
    Takes arguments from the command line and ignores as many of them as possible.
    '''

    # assume the user passes no args, and these are defaults/dummy
    #TODO: trash all of this
    parser = get_parser()

    parser.add_argument('--non-det', action='store_true', default=False,
                        help='whether to use a non-deterministic policy')
    parser.add_argument('--active-column', default=None, type=int,
                        help='Run only one vertical column of a fractal model to see what it\
                        has learnt independently')
    parser.add_argument('--evaluate', action='store_true', default=False,
                        help='record trained network\'s performance')
    # add any experiment-specific args here
    args = parser.parse_args()
    args.im_render = True
   #args.render = True
    args.random_terrain = False
    args.random_builds = False
    return args

class ExtinctionEvaluator():
    '''Run a series of experiments to evaluate the effect of extinction events on the complexity
    of the behaviour of a trained agent.'''
    def __init__(self, args, im_log_dir):
        self.im_log_dir = im_log_dir
        self.log_dir = args.load_dir
        env_name = args.env_name
        if torch.cuda.is_available() and not args.no_cuda:
            args.cuda = True
            device = torch.device('cuda')
            map_location = torch.device('cuda')
        else:
            args.cuda = False
            device = torch.device('cpu')
            map_location = torch.device('cpu')
        try:
            checkpoint = torch.load(os.path.join(args.load_dir, env_name + '.tar'),
                                    map_location=map_location)
        except FileNotFoundError:
            print('load-dir does not start with valid gym environment id, using command line args')
            env_name = args.env_name
            checkpoint = torch.load(os.path.join(args.load_dir, env_name + '.tar'),
                                map_location=map_location)
        saved_args = checkpoint['args']
        past_frames = checkpoint['n_frames']
        args.past_frames = past_frames
        env_name = saved_args.env_name

        if 'Micropolis' in env_name:
            args.power_puzzle = saved_args.power_puzzle

        if not args.evaluate and not 'GoLMulti' in env_name:
            # assume we just want to observe/interact w/ a single env.
            args.num_proc = 1
        dummy_args = args
        envs = make_vec_envs(env_name, args.seed + 1000, args.num_processes,
                            None, args.load_dir, args.add_timestep, device=device,
                            allow_early_resets=False,
                            args=dummy_args)
        print(args.load_dir)

        if isinstance(envs.observation_space, gym.spaces.Discrete):
            in_width = 1
            num_inputs = envs.observation_space.n
        elif isinstance(envs.observation_space, gym.spaces.Box):
            if len(envs.observation_space.shape) == 3:
                in_w = envs.observation_space.shape[1]
                in_h = envs.observation_space.shape[2]
            else:
                in_w = 1
                in_h = 1
            num_inputs = envs.observation_space.shape[0]

        if isinstance(envs.action_space, gym.spaces.Discrete):
            out_w = 1
            out_h = 1
            num_actions = int(envs.action_space.n // (in_w * in_h))
           #if 'Micropolis' in env_name:
           #    num_actions = env.venv.venv.envs[0].num_tools
           #elif 'GameOfLife' in env_name:
           #    num_actions = 1
           #else:
           #    num_actions = env.action_space.n
        elif isinstance(envs.action_space, gym.spaces.Box):
            out_w = envs.action_space.shape[0]
            out_h = envs.action_space.shape[1]
            num_actions = envs.action_space.shape[-1]
        # We need to use the same statistics for normalization as used in training
        #actor_critic, ob_rms = \
        #            torch.load(os.path.join(args.load_dir, args.env_name + ".pt"))

        if saved_args.model == 'fractal':
            saved_args.model = 'FractalNet'
        actor_critic = Policy(envs.observation_space.shape, envs.action_space,
                base_kwargs={'map_width': args.map_width,
                             'recurrent': args.recurrent_policy,
                            'in_w': in_w, 'in_h': in_h, 'num_inputs': num_inputs,
                    'out_w': out_w, 'out_h': out_h },
                             curiosity=args.curiosity, algo=saved_args.algo,
                             model=saved_args.model, args=saved_args)
        actor_critic.to(device)
        torch.nn.Module.dump_patches = True
        actor_critic.load_state_dict(checkpoint['model_state_dict'])
        ob_rms = checkpoint['ob_rms']

        if 'fractal' in args.model.lower():
            new_recs = args.n_recs - saved_args.n_recs

            for nr in range(new_recs):
                actor_critic.base.auto_expand()
            print('expanded network:\n', actor_critic.base)

            if args.active_column is not None \
                    and hasattr(actor_critic.base, 'set_active_column'):
                actor_critic.base.set_active_column(args.active_column)
        vec_norm = get_vec_normalize(envs)

        if vec_norm is not None:
            vec_norm.eval()
            vec_norm.ob_rms = ob_rms
        self.actor_critic = actor_critic
        self.envs = envs
        self.args = args

    def run_experiment(self, n_epis, max_step, map_width, extinction_type, extinction_prob,
            extinction_dels):
        '''Evaluate the effect of a single type of extinction event (or none).'''
        args = self.args
        actor_critic = self.actor_critic
        envs = self.envs
        im_log_dir = '{}/xttyp:{}_stp:{}'.format(
                self.im_log_dir,
                extinction_type,
               #extinction_prob,
               #map_width,
                max_step
                )
        envs.venv.venv.set_log_dir(im_log_dir)
        # adjust envs in general
        envs.venv.venv.setMapSize(map_width, max_step=max_step, render=args.render)
        if extinction_type is not None:
            # adjust extinguisher wrapper
            envs.venv.venv.set_extinction_type(extinction_type, extinction_prob, extinction_dels)
        # adjust image render wrapper
        envs.venv.venv.reset_episodes(im_log_dir)
        envs.venv.venv.init_storage()
        recurrent_hidden_states = torch.zeros(1, actor_critic.recurrent_hidden_state_size)
        masks = torch.zeros(1, 1)
        obs = envs.reset()
        #obs = torch.Tensor(obs)
        player_act = None
        n_episode = 0

        while n_episode < n_epis:
            with torch.no_grad():
                value, action, _, recurrent_hidden_states = actor_critic.act(
                    obs, recurrent_hidden_states, masks, deterministic=not args.non_det,
                    player_act=player_act)
            # Observe reward and next obs
            obs, reward, done, infos = envs.step(action)
            if args.render:
                envs.venv.venv.render()

            if done.any():
                n_episode += np.sum(done.astype(int))
            player_act = None

            if infos[0]:
                if 'player_move' in infos[0].keys():
                    player_act = infos[0]['player_move']
           #masks.fill_(0.0 if done else 1.0)
        envs.reset()

#def run_experiment():
#    '''Measure True under various conditions.'''
#    map_sizes = self.map_sizes
#    extinction_types = self.extinction_types
#    extinction_intervals = self.extinction_intervals
#    evaluator = ExtinctionEvaluator()
#
#    for map_size in map_sizes:
#        for extinction_type in extinction_types:
#            for extinction_interval in extinction_intervals:
#                evaluator.run_experiment(map_size, extinction_type, extinction_interval)

def get_xy(exp_dir):
    '''Plot the mean episode by mean size of the functional jpeg in terms of timestep.
    - exp_dir: location of images
    Return xy coordinates of mean episode
    '''
    ims = os.listdir(exp_dir)
    # map timesteps to a tuple (mean_size, num_ims)
    step2size = {}

    for im in ims:
        step_search = re.search(r'([\d]+)\.jpg', im)
        step = step_search.group(1)
        im_path = os.path.join(exp_dir, im)
        size = os.stat(im_path).st_size

        if step in step2size:
            mean_size, num_ims = step2size[step]
            mean_size = (mean_size * num_ims + size) / (num_ims + 1)
            num_ims += 1
            step2size[step] = (mean_size, num_ims)
        else:
            step2size[step] = (size, 1)
    xs = []
    ys = []
    for x, (y, _) in step2size.items():
        xs += [x]
        ys += [y]
    xy = zip(xs, ys)
    xy = sorted(xy, key = lambda x: int(x[0]))
    xs, ys = zip(*xy)
    return xs, ys

def visualize_experiments(log_dir):
    '''Visualize results from extinction-compressibility experiments.
     - load-dir: stores folder of experiments, within which are compressed images named by rank and
       episode
    '''
    log_dir = log_dir
    xtinct_dirs = os.listdir(log_dir)
    xt_dir_paths = [os.path.join(log_dir, xt_dir) for xt_dir in xtinct_dirs]
    # make sure the order of local and global paths correspond
    dirs_types = zip(xtinct_dirs, xt_dir_paths)
    dirs_types = sorted(dirs_types, key = lambda x: str(x[0]))
    xtinct_dirs, xt_dir_paths = zip(*dirs_types)
    xt_ims = [os.listdir(xt_dir) for xt_dir in xt_dir_paths if not os.path.isfile(xt_dir) ]

    for i, trial_name in enumerate(xtinct_dirs):
        print(trial_name)
        srch_xttyp = re.search(r'xttyp\:([a-zA-Z]+)', trial_name)
        if srch_xttyp is None:
            continue
        xt_type = srch_xttyp.group(1)
        xt_dir = xt_dir_paths[i]
        exp_title = ' '.join(xt_dir.split('/')[-2:])
        if os.path.isfile(xt_dir):
            continue
        x, y = get_xy(xt_dir)
        exp_plot, = plt.plot(x, y)
        print(xt_type)
        exp_plot.set_label(xt_type)

    srch_xtprob = re.search(r'xtprob\:(\d.[\d]+)', log_dir)
    xt_prob = srch_xtprob.group(1)
    xt_interval = int(1 / float(xt_prob))
    graph_title = 'extinction interval = {}'.format(xt_interval)
    plt.title(graph_title)
    plt.xlabel('timesteps')
    plt.ylabel('bytes per jpg')
    plt.xticks([25 * i for i in range(5)])
    plt.legend()
    plt.savefig(os.path.join(log_dir, '{}.png'.format(graph_title)), format='png')
       #plt.show()

class ExtinctionExperimenter():
    '''
    Coordinate between experimentation and visualization.
    '''
    def __init__(self, log_dir):
        args = parse_cl_args()
        env_name = log_dir.split('/')[-1].split('_')[0]
        args.env_name = env_name
        # Experiment global parameters
        self.n_epis = 20
        self.max_step = [1000]
        self.map_sizes = [
                16,
               #32,
               #64
                ]
        self.xt_types = [
                None,
                'age',
                'spatial',
                'random'
                ]
        # TODO: automate xt_probs
        self.xt_probs = [args.extinction_prob]
        self.xt_dels = [15]
        self.map_sizes = [args.map_width]
       #self.xt_probs = [
       #        0.005,
       #       #0.01,
       #       #0.05,
       #       #0.1
       #        ]
        exp_name = 'test_map:{}_xtprob:{}_xtdels:{}'.format(
                self.map_sizes[0],
                float(self.xt_probs[0]),
                self.xt_dels[0])
        self.log_dir = log_dir
        args.load_dir = log_dir
        im_log_dir = os.path.join(log_dir, exp_name)
        try:
            os.mkdir(im_log_dir)
        except FileExistsError:
           #shutil.rmtree(im_log_dir)
           #os.mkdir(im_log_dir)
            pass
        self.im_log_dir = im_log_dir
        self.evaluator = ExtinctionEvaluator(args, im_log_dir)

    def run_experiments(self):
        '''Run experiments and produce data.'''
        evaluator = self.evaluator
        for mst in self.max_step:
            for msz in self.map_sizes:
                for xtd in self.xt_dels:
                    for xtt in self.xt_types:
                        if xtt is None:
                            xtp = 0
                            evaluator.run_experiment(self.n_epis, mst, msz, xtt, xtp, xtd)
                        else:
                            for xtp in self.xt_probs:
                                evaluator.run_experiment(self.n_epis, mst, msz, xtt, xtp, xtd)

    def visualize_experiments(self):
        '''
        Visualize compressibility data stored in subfolders of the current directory.
        '''
        return visualize_experiments(self.im_log_dir)

if __name__ == "__main__":
    VIS_ONLY = False
   #VIS_ONLY = True
    LOG_DIR = os.path.abspath(os.path.join(
        'trained_models',
        'a2c_FractalNet_drop',
        'MicropolisEnv-v0_w16_1000s_noExtinct.bkp',
        ))
    EXPERIMENTER = ExtinctionExperimenter(LOG_DIR)

    #TODO: hacky; detect incomplete folders automatically,
    #     should save numpy object w/ stats in folder
    if not VIS_ONLY:
        try:
            EXPERIMENTER.run_experiments()
        # broadcast problem when sizing up map #TODO: adjust vec_envs to prevent this
        except ValueError as ve:
            print(ve)
    EXPERIMENTER.visualize_experiments()

max_stars_repo_name: smearle/gym-micropolis

max_issues_repo_name: smearle/gym-micropolis

max_forks_repo_name: smearle/gym-micropolis


text:
# Copyright 2020 The TensorStore Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests of tensorstore.IndexDomain"""

import pickle
import re

import pytest
import tensorstore as ts
import numpy as np

def test_init_rank():
  x = ts.IndexDomain(rank=2)
  assert x.rank == 2
  np.testing.assert_equal(x.inclusive_min, [-ts.inf] * 2)
  np.testing.assert_equal(x.inclusive_max, [+ts.inf] * 2)
  np.testing.assert_equal(x.exclusive_max, [+ts.inf + 1] * 2)
  np.testing.assert_equal(x.shape, [2 * ts.inf + 1] * 2)
  assert x.labels == ("", "")
  np.testing.assert_equal(x.implicit_lower_bounds, [1, 1])
  np.testing.assert_equal(x.implicit_upper_bounds, [1, 1])

def test_init_inclusive_min():
  x = ts.IndexDomain(inclusive_min=[1, 2])
  assert x.rank == 2
  np.testing.assert_equal(x.inclusive_min, [1, 2])
  np.testing.assert_equal(x.inclusive_max, [+ts.inf] * 2)
  np.testing.assert_equal(x.exclusive_max, [+ts.inf + 1] * 2)
  assert x.labels == ("", "")
  np.testing.assert_equal(x.implicit_lower_bounds, [0, 0])
  np.testing.assert_equal(x.implicit_upper_bounds, [1, 1])

def test_init_exclusive_max():
  x = ts.IndexDomain(exclusive_max=[1, 2])
  assert x.rank == 2
  np.testing.assert_equal(x.inclusive_min, [-ts.inf] * 2)
  np.testing.assert_equal(x.exclusive_max, [1, 2])
  assert x.labels == ("", "")
  np.testing.assert_equal(x.implicit_lower_bounds, [1, 1])
  np.testing.assert_equal(x.implicit_upper_bounds, [0, 0])

def test_init_inclusive_max():
  x = ts.IndexDomain(inclusive_max=[1, 2])
  assert x.rank == 2
  np.testing.assert_equal(x.inclusive_min, [-ts.inf] * 2)
  np.testing.assert_equal(x.inclusive_max, [1, 2])
  assert x.labels == ("", "")
  np.testing.assert_equal(x.implicit_lower_bounds, [1, 1])
  np.testing.assert_equal(x.implicit_upper_bounds, [0, 0])

def test_init_shape():
  x = ts.IndexDomain(shape=[1, 2])
  assert x.rank == 2
  np.testing.assert_equal(x.inclusive_min, [0] * 2)
  np.testing.assert_equal(x.exclusive_max, [1, 2])
  np.testing.assert_equal(x.shape, [1, 2])
  assert x.labels == ("", "")
  np.testing.assert_equal(x.implicit_lower_bounds, [0, 0])
  np.testing.assert_equal(x.implicit_upper_bounds, [0, 0])

def test_init_labels():
  x = ts.IndexDomain(labels=["x", "y"])
  assert x.rank == 2
  np.testing.assert_equal(x.inclusive_min, [-ts.inf] * 2)
  np.testing.assert_equal(x.inclusive_max, [+ts.inf] * 2)
  assert x.labels == ("x", "y")
  np.testing.assert_equal(x.implicit_lower_bounds, [1, 1])
  np.testing.assert_equal(x.implicit_upper_bounds, [1, 1])

def test_init_implicit_lower_bounds():
  x = ts.IndexDomain(implicit_lower_bounds=[0, 1])
  assert x.rank == 2
  np.testing.assert_equal(x.inclusive_min, [-ts.inf] * 2)
  np.testing.assert_equal(x.inclusive_max, [+ts.inf] * 2)
  np.testing.assert_equal(x.exclusive_max, [+ts.inf + 1] * 2)
  assert x.labels == ("", "")
  np.testing.assert_equal(x.implicit_lower_bounds, [0, 1])
  np.testing.assert_equal(x.implicit_upper_bounds, [1, 1])

def test_init_rank_mismatch():
  with pytest.raises(
      ValueError,
      match=r"Rank specified by `inclusive_min` \(1\) does not "
      r"match rank specified by `rank` \(2\)",
  ):
    ts.IndexDomain(rank=2, inclusive_min=[1])

def test_init_duplicate_upper_bound():
  with pytest.raises(
      ValueError,
      match="Cannot specify both `exclusive_max` and `inclusive_max`"):
    ts.IndexDomain(inclusive_max=[1, 2], exclusive_max=[1, 2])

def test_init_missing_rank():
  with pytest.raises(ValueError, match="Must specify `rank`"):
    ts.IndexDomain()

def test_init_implicit_upper_bounds():
  x = ts.IndexDomain(implicit_upper_bounds=[0, 1])
  assert x.rank == 2
  np.testing.assert_equal(x.inclusive_min, [-ts.inf] * 2)
  np.testing.assert_equal(x.inclusive_max, [+ts.inf] * 2)
  np.testing.assert_equal(x.exclusive_max, [+ts.inf + 1] * 2)
  assert x.labels == ("", "")
  np.testing.assert_equal(x.implicit_lower_bounds, [1, 1])
  np.testing.assert_equal(x.implicit_upper_bounds, [0, 1])

def test_getitem_index():
  d = ts.IndexDomain(labels=["x", "y"], shape=[3, 4])
  assert d["x"] == ts.Dim(label="x", size=3)
  assert d["y"] == ts.Dim(label="y", size=4)
  assert d[1] == ts.Dim(label="y", size=4)
  assert d["x", "y"] == d
  assert d["y", "x"] == [d[1], d[0]]
  assert d[::-1] == [d[1], d[0]]
  with pytest.raises(IndexError):
    d["z"]
  with pytest.raises(IndexError):
    d[2]
  with pytest.raises(ValueError):
    d[1:3]

def test_getitem_index_domain():
  d = ts.IndexDomain(labels=["x", "y", "z"], shape=[10, 11, 12])
  assert d[ts.IndexDomain(
      labels=["y", "x"], inclusive_min=[1, 2],
      exclusive_max=[6, 7])] == ts.IndexDomain(
          labels=["x", "y", "z"],
          inclusive_min=[2, 1, 0],
          exclusive_max=[7, 6, 12])

def test_eq():
  a = ts.IndexDomain(labels=["x", "y"])
  b = ts.IndexDomain(labels=["x", "y"])
  c = ts.IndexDomain(labels=["x", "z"])
  assert a == b
  assert a != c

def test_pickle():
  x = ts.IndexDomain(
      labels=["x", "y"],
      inclusive_min=[1, 2],
      exclusive_max=[3, 4],
      implicit_lower_bounds=[1, 0],
      implicit_upper_bounds=[0, 1],
  )
  assert pickle.loads(pickle.dumps(x)) == x

max_stars_repo_name: stuarteberg/tensorstore

max_issues_repo_name: stuarteberg/tensorstore

max_forks_repo_name: stuarteberg/tensorstore


text:
## Plot the overall eccentricity fit for all of the single planets
##

import numpy as np
import matplotlib.pyplot as plt
import random
import scipy
from scipy import stats
import os
from scipy import interpolate
from cksecc_helper import *

cks_data, koi_data, koi_errs = load_data()
koilist, propertymatrix = divide_into_singles_and_multis(cks_data, koi_data, koi_errs, singles=True)

e, inc, like, emaxval = read_in_singles()

print "e"
print e
print "inc"
print inc

propertymatrix.append(like)
koilist, propertymatrix = clean_sample(koilist, propertymatrix, customremove=True)

msun = propertymatrix[0]
rsun = propertymatrix[1]
period = propertymatrix[2]
radius = propertymatrix[3]
teff = propertymatrix[4]
metal = propertymatrix[5]
age = propertymatrix[6]
dilution = propertymatrix[7]
like = propertymatrix[-1]

alltotal = np.sum(np.log(like), axis=0)
allmaxllike = max(alltotal)
allllike = alltotal-allmaxllike

print "overall loglike"
print allmaxllike

from s_g_funs import quadratic_intersections, savitzky_golay

def makeplot(propertylist, cutlist, propertystring):

  nbins = len(cutlist)+1
  first = 0
  last = nbins-1

  koigroups = []

  for i in range(nbins):
    if i==first:
      koigroups.append(np.where(propertylist < cutlist[i]))
    elif i==last:
      koigroups.append(np.where(propertylist > cutlist[i-1]))
    else:
      koigroups.append(np.where(np.logical_and(cutlist[i-1] < propertylist, propertylist < cutlist[i])))

  likegroups = [like[koigroup] for koigroup in koigroups]
  liketotals = [np.sum(np.log(likegroup), axis=0) for likegroup in likegroups]
  normlikes = [liketotal - max(liketotal) for liketotal in liketotals]

  strings = []
  for i in range(nbins):
    if i==first:
      strings.append('%s < %s (N=%i)' % (propertystring, str(cutlist[i]), len(likegroups[i])))
    elif i==last:
      strings.append('%s > %s (N=%i)' % (propertystring, str(cutlist[i-1]), len(likegroups[i])))
    else:
      strings.append('%s < %s < %s (N=%i)' % (str(cutlist[i-1]), propertystring, str(cutlist[i]), len(likegroups[i])))

  colors = ['r', 'b', 'g', 'y', 'c', 'm', 'tan', 'maroon', 'mediumspringgreen', 'cornflowerblue', 'orange', 'gold']

  maxes = []
  maxeerrs1 = []
  maxeerrs2 = []

  #plt1, ax1 = plt.plot(e, allllike, color='k', label='All')
  plt.figure(1)
  plt.plot(e, allllike, color='k', label='All')

  nlall = allllike
  allfit = savitzky_golay(nlall, 7, 3)
  plt.plot(e, allfit, color='k', ls='dashed')

  for i in range(nbins):
    plt.figure(1)
    nl = normlikes[i]
    plt.plot(e, nl, color=colors[i % len(colors)], label=strings[i])
    nl10i = np.where(nl > -10.)
    e10 = e[nl10i]
    nl10 = nl[nl10i]
    poly = np.polyfit(e10, nl10, 2)
    quadfit = np.poly1d(poly)
    xfit = np.linspace(0., 0.4, 50)
    #plt.plot(xfit, quadfit(xfit), color=colors[i % len(colors)], ls='dashed')
    ix = quadratic_intersections(poly, [-0.5])
    #plt.scatter(*ix, marker='x', color=colors[i % len(colors)], s=40, linewidth=2)

    lfit = savitzky_golay(nl, 7, 3)
    plt.plot(e, lfit, color='k', ls='dashed')

    xfit = np.linspace(0.01, 0.39, 1000)
    f = interpolate.interp1d(e,lfit, kind='quadratic')
    fgrid = f(xfit)
    maxi = np.argmax(fgrid)
    maxl = fgrid[maxi]
    maxe = xfit[maxi]
    fgrid1s = fgrid-maxl+0.5
    dotp = fgrid1s[0:-1]*fgrid1s[1:]
    #print maxi, maxl, maxe
    #print dotp
    errsi = np.where(dotp < 0)
    #print errsi
    evalerrs = []
    for erri in errsi[0]:
      evalerr = (xfit[erri]+xfit[erri+1])/2.
      evalerrs.append(evalerr)
    if len(evalerrs) == 1:
      if evalerrs[0] > maxe:
        evalerrs = [0.] + evalerrs
      else:
        evalerrs = evalerrs + [0.]
    if len(evalerrs) == 0:
      evalerrs = [0., 0.4]
    if len(evalerrs) > 2:
      #raise
      pass

    print "this cut"
    print cutlist, i
    print evalerrs[0], maxe, evalerrs[1]

    plt.scatter([evalerrs[0], maxe, evalerrs[1]], [maxl-0.5,maxl-0,maxl-0.5], marker='x', color=colors[i % len(colors)], s=40, linewidth=2)
    plt.plot(xfit, f(xfit), color='gray', ls='dotted')

    maxes.append(maxe)
    maxeerrs1.append(evalerrs[0])
    maxeerrs2.append(evalerrs[1])

  plt.figure(2)
  lc = len(cutlist)
  if lc > 1:
    xbinvals = np.array(np.array(cutlist[0:-1])+np.array(cutlist[1:]))/2.
    print xbinvals
    xbinvals = [xbinvals[0]+(cutlist[0]-xbinvals[0])*2] + [val for val in xbinvals]
    xbinvals = [val for val in xbinvals] + [xbinvals[-1]-(-cutlist[-1]+xbinvals[-1])*2]
    print xbinvals
  else:
    xbinvals = [-1,1]
  xbinvals = np.array(xbinvals)
  plt.errorbar(xbinvals, maxes, yerr=[np.array(maxes)-np.array(maxeerrs1), np.array(maxeerrs2)-np.array(maxes)], fmt='o')
  outputname1 = propertystring+'_trend_R<15.png'
  plt.xlabel(propertystring, fontsize=12)
  plt.ylabel('$\sigma_e$', fontsize=12)
  plt.ylim((0.,0.4))
  plt.savefig('figs4/'+outputname1, bbox_inches='tight')
  plt.close()

  plt.figure(1)
  plt.plot([min(e), max(e)], [-2, -2], color='gray', ls='dotted')
  plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, mode="expand")
  plt.xlabel('$\sigma_e$', fontsize=12)
  plt.ylabel('Natural Log Likelihood Difference', fontsize=12)
  plt.grid()

  plt.xlim((0.,0.4))
  plt.ylim((-20, 5))

  outputname1 = propertystring+'_R<15.png'
  plt.savefig('figs4/'+outputname1, bbox_inches='tight')
  plt.close()

  print "plotted "+propertystring

  return

################
# Make plot for all planets, with uncertainties

plt.figure(1)

nlall = allllike
allfit = savitzky_golay(nlall, 7, 3)
offset = np.max(allfit)
allfit -= offset

plt.plot(e, allllike - offset, 'o', color='k', label='All')

plt.plot(e, allfit, color='k')
plt.plot([min(e), max(e)], [-0.5, -0.5], color='gray', ls='dotted')
plt.plot([min(e), max(e)], [-2, -2], color='gray', ls='-.')
plt.plot([min(e), max(e)], [-4.5, -4.5], color='gray', ls='dashed')

x2 = np.linspace(0., max(e), num=1000, dtype=np.float)
y2 = np.interp(x2, e, allfit)
plt.fill_between(x2, y2*0-40, y2, where=y2 >= -4.5, facecolor='green', interpolate=True)
plt.fill_between(x2, y2*0-40, y2, where=y2 >= -0.5, facecolor='blue', interpolate=True)

print "bestfit = ", x2[np.argmax(y2)]
print "1 sigma = ", x2[np.where(y2 >= -0.5)]
print "2 sigma = ", x2[np.where(y2 >= -2)]
print "3 sigma = ", x2[np.where(y2 >= -4.5)]

#plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, mode="expand")
plt.xlabel('$\sigma_e$', fontsize=12)
plt.ylabel('Natural Log Likelihood Difference', fontsize=12)
plt.grid()

plt.xlim((0.,0.4))
plt.xlim((0.1,0.3))
plt.ylim((-40, 8))

outputname1 = 'All_R<15.png'
plt.savefig('figs4/'+outputname1, bbox_inches='tight')
plt.close()

# Make plot for all planets, with uncertainties
# Also add multiplanet system line

plt.figure(1)

nlall = allllike
allfit = savitzky_golay(nlall, 7, 3)
offset = np.max(allfit)
allfit -= offset

plt.plot(e, allllike - offset, 'o', color='k', label='All')

plt.plot(e, allfit, color='k')
plt.plot([min(e), max(e)], [-0.5, -0.5], color='gray', ls='dotted')
plt.plot([min(e), max(e)], [-2, -2], color='gray', ls='-.')
plt.plot([min(e), max(e)], [-4.5, -4.5], color='gray', ls='dashed')

x2 = np.linspace(0., max(e), num=1000, dtype=np.float)
y2 = np.interp(x2, e, allfit)
plt.fill_between(x2, y2*0-40, y2, where=y2 >= -4.5, facecolor='green', interpolate=True)
plt.fill_between(x2, y2*0-40, y2, where=y2 >= -0.5, facecolor='blue', interpolate=True)

print "bestfit = ", x2[np.argmax(y2)]
print "1 sigma = ", x2[np.where(y2 >= -0.5)]
print "2 sigma = ", x2[np.where(y2 >= -2)]
print "3 sigma = ", x2[np.where(y2 >= -4.5)]

#plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, mode="expand")
plt.xlabel('$\sigma_e$', fontsize=12)
plt.ylabel('Natural Log Likelihood Difference', fontsize=12)
plt.grid()

gridm = np.loadtxt('./fgrid.txt')
gride = np.loadtxt('./xgrid.txt')
gridi = np.loadtxt('./ygrid.txt')

imaxi, emaxi = np.unravel_index(np.argmax(gridm, axis=None), gridm.shape)

evals = gride
llvalsm = gridm[imaxi]
llvalsm -= np.max(llvalsm)

plt.plot(evals, llvalsm, color='c', alpha=0.5)
#x2 = np.linspace(0., max(e), num=1000, dtype=np.float)
#y2 = np.interp(x2, e, allfit)
plt.fill_between(evals, llvalsm*0-35, llvalsm, where=llvalsm >= -4.5, facecolor='green', interpolate=True, alpha=0.5)
plt.fill_between(evals, llvalsm*0-35, llvalsm, where=llvalsm >= -0.5, facecolor='blue', interpolate=True, alpha=0.5)

plt.xlim((0.,0.4))
plt.xlim((0.0,0.3))
plt.ylim((-34.5, 8))

outputname1 = 'All_R<15_plusmultis.png'
plt.savefig('figs4/'+outputname1, bbox_inches='tight')
plt.close()

reason: import numpy,import scipy,from scipy


text:
import pandas as pd
import numpy as np
import math
import operator
from sklearn.model_selection import KFold, cross_val_score, GridSearchCV
from sklearn import metrics, svm

# data_path
DATA_PATH = "dataset/leaf/leaf.csv"

columns = ["Class",
           "Specimen",
           "Eccentricity",
           "Aspect Ratio",
           "Elongation",
           "Solidity",
           "Stochastic Convexity",
           "Isometric Factor",
           "Maximal Indentation Depth",
           "Lobedness",
           "Average Contrast",
           "Smoothness",
           "Third Moment",
           "Uniformity",
           "Entropy"]

def load_data(path=DATA_PATH):
    data = pd.read_csv(path, index_col=False, header=None, names=columns)
    return data

# test_data = load_data()

# group = test_data.groupby("Class")
# list_temp = []

# print(test_data["Class"].max())

def split_training_and_testing_data(data, split_percentage=0.8):
    training_data = []
    testing_data = []
    for classes, classes_region in data.groupby("Class"):
        # print(int(len(classes_region)))
        split_index = int(len(classes_region) * split_percentage)
        # print(classes_region[:split_index])
        training_data.append(classes_region[:split_index])
        # print("----------------")
        testing_data.append(classes_region[split_index:])
        # print(classes_region[split_index+1:])
        # print(split_index)
        # print("----------------")

    return training_data, testing_data

"""training_data, testing_data = split_training_and_testing_data(load_data())

total_train = 0
for x in range(0,len(training_data)):
    total_train += len(training_data[x])

print("Train: ", total_train )
total_train = 0
for x in range(0,len(testing_data)):
    total_train += len(testing_data[x])

print("Test: ", total_train )

print(len(training_data[29]))
print("----------------")
print(len(testing_data[29]))

print(len(training_data[28]))
print("----------------")
print(len(testing_data[28]))

print(testing_data[0].head())"""

def euclidean_distance(data1, data2, length):
    distance = 0
    # ignore first data because it is class label.
    # print("Euclidean")
    # print(data1)
    # print(data2)
    # print(length)
    # print("Euclidean")
    for x in range(2, length):
        distance += np.square(data1[x] - data2[x])

    # print("Euclidean Result: " + str(distance))
    return np.sqrt(distance)

def k_nearest_neighbourhood(training_set, test_instance, k):
    distances = {}
    sort = {}

    length = len(test_instance)

    # print("Test Instance Shape", test_instance.shape, len(test_instance))

    # print("Training Set Length", len(training_set))

    #### Start of STEP 3
    # Calculating euclidean distance between each row of training data and test data
    for x in range(len(training_set)):
        #### Start of STEP 3.1
        dist = euclidean_distance(test_instance, training_set[x], length)

        distances[x] = dist
        #### End of STEP 3.1

    # print(len(distances))

    #### Start of STEP 3.2
    # Sorting them on the basis of distance
    # print("Distance Items:\n", distances.items())
    sorted_d = sorted(distances.items(), key=operator.itemgetter(1))
    #### End of STEP 3.2

    # print("Sorted Distances ")

    # print(sorted_d)
    # print(sorted_d[0][0])
    neighbors = []

    #### Start of STEP 3.3
    # Extracting top k neighbors
    for x in range(k):
        neighbors.append(sorted_d[x][0])
    #### End of STEP 3.3
    classVotes = {}

    # print(neighbors)
    #### Start of STEP 3.4
    # Calculating the most freq class in the neighbors
    for x in range(len(neighbors)):
        response = training_set[neighbors[x]][0]

        if response in classVotes:
            classVotes[response] += 1
        else:
            classVotes[response] = 1
    #### End of STEP 3.4

    #### Start of STEP 3.5

    # print(classVotes)
    sortedVotes = sorted(classVotes.items(), key=operator.itemgetter(1), reverse=True)
    return (sortedVotes[0][0], neighbors)
    #### End of STEP 3.5

"""
training_data, testing_data = split_training_and_testing_data(load_data(DATA_PATH))

total_train = 0
for x in range(0, len(training_data)):
    total_train += len(training_data[x])

print("Train: ", total_train)
total_test = 0
for x in range(0, len(testing_data)):
    total_test += len(testing_data[x])
print("Test: ", total_test)

train_numpy = np.asarray(training_data[0])
test_numpy = np.asarray(testing_data[0])
for x in range(1, len(training_data)):
    train_numpy = np.concatenate((train_numpy, np.asarray(training_data[x])), axis=0)

for x in range(1, len(testing_data)):
    test_numpy = np.concatenate((test_numpy, np.asarray(testing_data[x])), axis=0)

# print(test_numpy)
print(train_numpy.shape)
print(test_numpy.shape)

print(test_numpy[0])
print(train_numpy[0])

test_index = 67;
result, neigbours = k_nearest_neighbourhood(train_numpy, test_numpy[test_index], 5)
print("Expected Class: " + str(int(test_numpy[test_index][0])))
print("Predicted Class: ", int(result))
print("Nearest Neighbours: ", neigbours)"""

# print(train_numpy[neigbours])
# print(test_numpy[0])

# normalize data:

"""positive = 0
negative = 0
print(len(test_numpy[0]))
"""

def find_best_k_value_without_norm(train_numpy):
    train_results = {}
    split_size = 5
    kf = KFold(n_splits=split_size, random_state=False, shuffle=True)

    splits = kf.get_n_splits(train_numpy)

    print("Splits: " + str(splits))

    for k_size in range(1, 15, 2):
        mean_pos = 0
        X_train = []
        X_test = []
        for train_index, test_index in kf.split(train_numpy):
            # print("TRAIN:", train_index, "TEST:", test_index)
            X_train, X_test = train_numpy[train_index], train_numpy[test_index]

            positive = 0
            negative = 0
            for x in range(0, len(X_test)):
                sorted2, neigbours2 = k_nearest_neighbourhood(X_train, X_test[x], k_size)
                if int(X_test[x][0]) == int(sorted2):
                    positive += 1
                else:
                    negative += 1

            temp_acc = (100 * positive) / (positive + negative)
            mean_pos += temp_acc
            #print("TrainSize: " + str(len(X_train)) + " TestSize:" + str(len(X_test)))
        # print("K : " + str(k_size) + " True:" + str(positive) + " False: " + str(negative) + "  Positive Accuracy Rate % " + str(
        # (100 * positive) / (positive + negative)))
        train_results[k_size] = (100 * positive) / (positive + negative)
        # print("K : " + str(k_size) + " True:" + str(positive) + " False: " + str(negative) + "  Negative Accuracy Rate % " + str(
        # (100 * negative) / (positive + negative)))

        print("K SIZE : " + str(k_size) + " MEAN ACC  % " + str(mean_pos / split_size))
    return train_results

"""

for x in range(0, len(test_numpy), 1):
    temp_point = test_numpy[x]
    # print("BEFORE_NORM")
    # print(temp_point)
    total_sum = sum(temp_point) - test_numpy[x][0] - test_numpy[x][1]
    for y in range(2, len(temp_point)):
        temp_point[y] = temp_point[y] / total_sum

    # print("AFTER_NORM")
    # print(temp_point)
    test_numpy[x] = temp_point

for x in range(0, len(train_numpy), 1):
    temp_point = train_numpy[x]
    # print("BEFORE_NORM")
    # print(temp_point)
    total_sum = sum(temp_point) - train_numpy[x][0] - train_numpy[x][1]
    for y in range(2, len(temp_point)):
        temp_point[y] = temp_point[y] / total_sum

    # print("AFTER_NORM")
    # print(temp_point)
    train_numpy[x] = temp_point
"""
# print("K : " + str(k_size) + " True:" + str(positive) + " False: " + str(negative) + "  Negative Accuracy Rate % " + str(
# (100 * negative) / (positive + negative)))
"""
split_size = 5
kf = KFold(n_splits=split_size, random_state=False, shuffle=True)

splits = kf.get_n_splits(train_numpy)

print("Splits: " + str(splits))

for k_size in range(1, 15, 2):
    mean_pos = 0
    for train_index, test_index in kf.split(train_numpy):
        # print("TRAIN:", train_index, "TEST:", test_index)
        X_train, X_test = train_numpy[train_index], train_numpy[test_index]

        positive = 0
        negative = 0
        for x in range(0, len(X_test)):
            sorted2, neigbours2 = k_nearest_neighbourhood(X_train, X_test[x], k_size)
            if int(X_test[x][0]) == int(sorted2):
                positive += 1
            else:
                negative += 1

        temp_acc = (100 * positive) / (positive + negative)
        mean_pos += temp_acc
        print("TrainSize: " + str(len(X_train)) + " TestSize:" + str(len(X_test)))
    # print("K : " + str(k_size) + " True:" + str(positive) + " False: " + str(negative) + "  Positive Accuracy Rate % " + str(
    # (100 * positive) / (positive + negative)))
    # print("K : " + str(k_size) + " True:" + str(positive) + " False: " + str(negative) + "  Negative Accuracy Rate % " + str(
    # (100 * negative) / (positive + negative)))

    print("K SIZE : " + str(k_size) + " MEAN ACC  % " + str(mean_pos / split_size))

for test_k in range(1, 15, 2):
    positive = 0
    negative = 0
    for x in range(0, len(test_numpy)):
        sorted2, neigbours2 = k_nearest_neighbourhood(train_numpy, test_numpy[x], test_k)
        if int(test_numpy[x][0]) == int(sorted2):
            positive += 1
        else:
            negative += 1
    print("TEST DATA K : " + str(test_k) + " True:" + str(positive) + " False: " + str(
        negative) + "  Positive Accuracy Rate % " + str(
        (100 * positive) / (positive + negative)))

scores = cross_val_score(k_nearest_neighbourhood, train_numpy, train_numpy[0], cv=5, scoring='f1_macro')

from sklearn.neighbors import KNeighborsClassifier

# Create KNN classifier
"""

"""

"""

# print(knn_gscv.best_estimator_)
# print(knn_gscv.best_params_)

def manhattan_distance(data1, data2):
    if (len(data1) != len(data2)):
        print("Be sure that both vectors are the same dimension!")
        return

    return sum([abs(data1[i] - data2[i]) for i in range(len(data1))])

def k_nearest_neighbourhood_manhattan(training_set, test_instance, k):
    distances = {}
    sort = {}

    length = len(test_instance)

    # print("Test Instance Shape", test_instance.shape, len(test_instance))

    # print("Training Set Length", len(training_set))

    #### Start of STEP 3
    # Calculating euclidean distance between each row of training data and test data
    for x in range(len(training_set)):
        #### Start of STEP 3.1
        dist = manhattan_distance(test_instance, training_set[x])

        distances[x] = dist
        #### End of STEP 3.1

    # print(len(distances))

    #### Start of STEP 3.2
    # Sorting them on the basis of distance
    # print("Distance Items:\n", distances.items())
    sorted_d = sorted(distances.items(), key=operator.itemgetter(1))
    #### End of STEP 3.2

    # print("Sorted Distances ")

    # print(sorted_d)
    # print(sorted_d[0][0])
    neighbors = []

    #### Start of STEP 3.3
    # Extracting top k neighbors
    for x in range(k):
        neighbors.append(sorted_d[x][0])
    #### End of STEP 3.3
    classVotes = {}

    # print(neighbors)
    #### Start of STEP 3.4
    # Calculating the most freq class in the neighbors
    for x in range(len(neighbors)):
        response = training_set[neighbors[x]][0]

        if response in classVotes:
            classVotes[response] += 1
        else:
            classVotes[response] = 1
    #### End of STEP 3.4

    #### Start of STEP 3.5

    # print(classVotes)
    sortedVotes = sorted(classVotes.items(), key=operator.itemgetter(1), reverse=True)
    return (sortedVotes[0][0], neighbors)
    #### End of STEP 3.5

"""
for test_k in range(1, 13, 2):
    positive = 0
    negative = 0
    for x in range(0, len(test_numpy)):
        sorted2, neigbours2 = k_nearest_neighbourhood_manhattan(train_numpy, test_numpy[x], test_k)
        if int(test_numpy[x][0]) == int(sorted2):
            positive += 1
        else:
            negative += 1

    print("Without Norm Test K : " + str(test_k) + "  Positive:" + str(positive))
    print("Without Norm Test K : " + str(test_k) + "  Negative:" + str(negative))
for k_size in range(1, 15, 2):
    mean_pos = 0
    for train_index, test_index in kf.split(train_numpy):
        # print("TRAIN:", train_index, "TEST:", test_index)
        X_train, X_test = train_numpy[train_index], train_numpy[test_index]

        positive = 0
        negative = 0
        for x in range(0, len(X_test)):
            sorted2, neigbours2 = k_nearest_neighbourhood_manhattan(X_train, X_test[x], k_size)
            if int(X_test[x][0]) == int(sorted2):
                positive += 1
            else:
                negative += 1

        temp_acc = (100 * positive) / (positive + negative)
        mean_pos += temp_acc
        print("TrainSize: " + str(len(X_train)) + " TestSize:" + str(len(X_test)))
    # print("K : " + str(k_size) + " True:" + str(positive) + " False: " + str(negative) + "  Positive Accuracy Rate % " + str(
    # (100 * positive) / (positive + negative)))
    # print("K : " + str(k_size) + " True:" + str(positive) + " False: " + str(negative) + "  Negative Accuracy Rate % " + str(
    # (100 * negative) / (positive + negative)))

    print("K SIZE : " + str(k_size) + " MEAN ACC  % " + str(mean_pos / split_size))

for test_k in range(1, 15, 2):
    positive = 0
    negative = 0
    for x in range(0, len(test_numpy)):
        sorted2, neigbours2 = k_nearest_neighbourhood_manhattan(train_numpy, test_numpy[x], test_k)
        if int(test_numpy[x][0]) == int(sorted2):
            positive += 1
        else:
            negative += 1
    print("TEST DATA K : " + str(test_k) + " True:" + str(positive) + " False: " + str(negative) + "  Positive Accuracy Rate % " + str(
     (100 * positive) / (positive + negative)))

"""

############# SVM STARTS HERE #######################

"""
train_labels = []
test_labels = []

def delete_first_two_column(data_array):
    label_array = np.empty(len(data_array))
    for index_label in range(0, len(data_array), 1):
        print("Doing:" + str(index_label))
        print("DATA: " + str(data_array[index_label][0]))
        label_array[index_label] = data_array[index_label][0]

    data_array = np.delete(data_array, np.s_[0:2], axis=1)

    return data_array, label_array

print(train_numpy[0][0])

print(len(train_numpy))
train_numpy, train_labels = delete_first_two_column(train_numpy)
test_numpy, test_labels = delete_first_two_column(test_numpy)

print(train_numpy[0])
print(train_labels[0])

print(test_numpy[80])
print(test_labels[80])

clf = svm.LinearSVC(penalty='l2', verbose=0, max_iter=10000, class_weight=None)

clf.fit(train_numpy, train_labels, sample_weight=None)

# print(str(train_numpy))
print(str(test_labels))

print("###---- WHOLE TRAIN DATA START ----####")
print(clf.score(train_numpy, train_labels))
print("###---- WHOLE TRAIN DATA END  ----####")

print("###---- WHOLE TEST DATA START ----####")
print(clf.score(test_numpy, test_labels))
print("###---- WHOLE TEST DATA END  ----####")

average_train = 0
average_test = 0
for train_index, test_index in kf.split(train_numpy):
    print("####################333")

    # print("TRAIN:", train_index, "TEST:", test_index)
    X_train, X_test = train_numpy[train_index], train_numpy[test_index]
    X_train_labels, X_test_labels = train_labels[train_index], train_labels[test_index]
    clf.fit(X_train, X_train_labels)
    average_train += clf.score(X_train, X_train_labels)
    average_test += clf.score(X_test, X_test_labels)
    print(clf.score(X_train, X_train_labels))
    print(clf.score(X_test, X_test_labels))

print("AVERAGE RESULTS")
print(str(average_train / 5))
print(str(average_test / 5))

poly_svm = svm.SVC(kernel="poly", degree=3)

poly_svm.fit(train_numpy, train_labels)

print("POLY NOT MIXED RESULTS")

print(poly_svm.score(train_numpy, train_labels))
print(poly_svm.score(test_numpy, test_labels))

for train_index, test_index in kf.split(train_numpy):
    print("####################4444")

    # print("TRAIN:", train_index, "TEST:", test_index)
    X_train, X_test = train_numpy[train_index], train_numpy[test_index]
    X_train_labels, X_test_labels = train_labels[train_index], train_labels[test_index]
    poly_svm.fit(X_train, X_train_labels)
    average_train += poly_svm.score(X_train, X_train_labels)
    average_test += poly_svm.score(X_test, X_test_labels)
    print(poly_svm.score(X_train, X_train_labels))
    print(poly_svm.score(X_test, X_test_labels))

print("AVERAGE KFOLD RESULTS")
print(str(average_train / 5))
print(str(average_test / 5))

########## DECISION TREE START ##################

from sklearn import tree
from sklearn.model_selection import cross_val_score

my_decision_tree = tree.DecisionTreeClassifier(criterion="gini", class_weight="balanced")

my_decision_tree.fit(X_train, X_train_labels)

print("DTREE RESULTS")

print(my_decision_tree.score(X_train, X_train_labels))
print(my_decision_tree.score(X_test, X_test_labels))

print(cross_val_score(my_decision_tree, X_train, X_train_labels, cv=3))
print(cross_val_score(my_decision_tree, X_test, X_test_labels, cv=3))

def get_lineage(tree, feature_names):
    left = tree.tree_.children_left
    right = tree.tree_.children_right
    threshold = tree.tree_.threshold
    features = [feature_names[i] for i in tree.tree_.feature]

    # get ids of child nodes
    idx = np.argwhere(left == -1)[:, 0]

    def recurse(left, right, child, lineage=None):
        if lineage is None:
            lineage = [child]
        if child in left:
            parent = np.where(left == child)[0].item()
            split = 'l'
        else:
            parent = np.where(right == child)[0].item()
            split = 'r'

        lineage.append((parent, split, threshold[parent], features[parent]))

        if parent == 0:
            lineage.reverse()
            return lineage
        else:
            return recurse(left, right, parent, lineage)

    for child in idx:
        for node in recurse(left, right, child):
            print(node)

# get_lineage(my_decision_tree,X_train_labels)

dtree = tree.DecisionTreeClassifier(class_weight="balanced", criterion='gini', max_depth=150,
                                    max_features=13, max_leaf_nodes=50, min_samples_leaf=1,
                                    min_samples_split=2, min_weight_fraction_leaf=0.0,
                                    presort=False, random_state=None, splitter='random')
dtree.fit(X_train, X_train_labels)

print(dtree.score(X_train, X_train_labels))
print(dtree.score(X_test, X_test_labels))

# print(cross_val_score(dtree, X_train_labels, cv=5))
"""


text:
#!/usr/bin/env python
#
# Kevin J. Walchko 3 April 2016
#

from __future__ import print_function
from __future__ import division
# import time
import multiprocessing as mp
import logging
import datetime as dt
# import argparse
import numpy as np
from math import sin, cos, sqrt
from pyrk import RK4
import pygecko.ZmqClass as zmq
import pygecko.Messages as msg
from numpy import dot, cross
from numpy.linalg import inv
# from pyrk import RK4
import numdifftools as nd
# import lib.FileStorage as fs

def ecef(lat, lon, H):
	# phi = lat
	# lambda = lon
	e = 1.0
	re = 6378137.0  # m
	rm = re * (1.0 - e**2) / pow(1.0 - e**2 * sin(lat)**2, 3.0 / 2.0)
	rn = re / sqrt(1.0 - e**2 * sin(lat)**2)
	x = (rn + H) * cos(lat) * cos(lon)
	y = (rn + H) * cos(lat) * sin(lon)
	z = (rm + H) * sin(lat)
	return x, y, z

def normalizeQuaternion(w, x, y, z):
	m = sqrt(w**2 + x**2 + y**2 + z**2)
	return w/m, x/m, y/m, z/m

class EOM(object):
	"""
	EoM for navigation
	"""
	def __init__(self):
		self.t = 0.0
		self.rk = RK4(self.eom)
		self.epoch = dt.datetime.now()

	@staticmethod
	def eom(t, X, u):
		"""
		X = [vx vy vz px py pz qw qx qy qz]
		v - velocity
		p - position
		q - quaternion (orientation)

		u = [fx fy fz wx wy wz]
		f - force
		w - angular velocity
		"""
		q = X[6:]
		f = u[0:3]
		wx, wy, wz = u[3:]
		p = X[3:6]
		v = X[0:3]
		wie = np.array([0, 0, 7.292115E-15])
		Ceb = np.eye(3)
		W = np.array([
			(0, wz, -wy, wx),
			(-wz, 0, wz, -wy),
			(wy, -wx, 0, wz),
			(-wx, -wy, -wz, 0)
		])
		vd = -2.0*np.cross(wie, np.cross(wie, v))-np.cross(wie, np.cross(wie, p)) + Ceb.dot(f)
		pd = v
		qd = 0.5 * W.dot(q)

		# print('vd', vd)
		# print('pd', pd)
		# print('qd', qd)

		XX = np.hstack((vd, pd, qd))
		return XX

	def step(self, X, u):
		delta = (dt.datetime.now() - self.epoch).total_seconds()
		# t = self.epoch.total_seconds()
		t = self.t
		# X = self.X
		# dt = self.dt
		X = self.rk.step(X, u, t, delta)

		# fix q
		w, x, y, z = normalizeQuaternion(*X[6:])
		X[6] = w
		X[7] = x
		X[8] = y
		X[9] = z

		self.t += delta
		self.epoch = dt.datetime.now()
		return X

class EKF(object):
	"""
	Extended Kalman Filter (EKF)

	def func(time, x, u):
		some nonlinear eqns
		return dx

	ekf = EKF(size_x,size_z)
	ekf.init(x, func, R, Q)

	while True:
		ekf.predict(u)
		x_hat = ekf.update(z)
	"""
	def __init__(self, dim_x, dim_z):
		# self.x = x  # init state
		# self.rk = RK4(f)
		self.dt = dt
		self.F = np.eye(dim_x)
		self.H = 0
		self.P = np.eye(dim_x)
		self.R = np.eye(dim_z)
		self.Q = np.eye(dim_x)
		self.I = np.eye(dim_x)

	def init(self, x, f, r, q):
		"""
		f - dx = f(x, u)
		r -
		q -
		"""
		self.x = x
		self.rk = RK4(f)
		self.jacob = nd.Jacobian(f)
		self.R = r
		self.Q = q
		self.time = 0.0

	def something(self, u):
		rk = self.rk
		dt = self.dt
		t = self.time
		x = self.x

		y = rk.step(x, u, t, dt)
		self.F = self.jacob(y)

		self.time = t+dt
		self.x = y

		return y

	def predict(self, u):
		self.x = self.something(u)

		F = self.F
		Q = self.Q
		P = self.P

		self.P = dot(F, dot(P, F.T)) + Q

	def update(self, z):
		H = self.H
		R = self.R
		I = self.I
		P = self.P
		x = self.x

		K = dot(P, dot(H.T, inv(dot(H, dot(P, H.T)) + R)))
		x = x + K.dot(z-H)
		p = (I - K.dot(H)).dot(P)

		self.x = x
		self.P = p

		return x

def skew(a):
	return a

def kf_eqns(t, x, u):
	"""
	Calculates navigation corrections
	input:
		t - time
		x - state
		u - [s_b(accel) w_b(gyros)]
	output:
		dev - dot error velocity or acceleration error
		dep - velocity error
		dea - attitude error
		return [dev, dep, dea]
	"""
	err_v = x[0:3]
	err_p = x[3:6]
	err_a = x[6:]
	w_b = u[3:]   # gyros
	s_b = u[0:3]  # accel
	Reb = np.eye(3)
	wie = np.array([0, 0, 7.292115E-15])  # earth's rotation rate in ECEF
	web = Reb.dot(w_b)  # gyros in ECEF
	se = skew(Reb.dot(s_b))  # accels in ECEF

	dev = -2.0*cross(wie, err_v) + cross(wie, cross(wie, err_p)) + cross(se, err_a) + Reb.dot(err_s)
	dep = err_v
	dea = cross(web, err_a) - web

	return np.hstack((dev, dep, dea))

class NavigationServer(mp.Process):
	"""
	Still needs lots of work!
	"""
	def __init__(self, host="localhost", port='9000'):
		mp.Process.__init__(self)
		# self.epoch = dt.datetime.now()
		self.host = host
		self.port = port
		# self.sub = Sub('/cmd','tcp://%s:%s'%(host,port))
		# logging.basicConfig(level=logging.INFO)
		mp.log_to_stderr()
		self.logger = logging.getLogger(__name__)
		# self.kf = KF()
		self.eom = EOM()

	def run(self):
		self.logger.info(str(self.name) + '[' + str(self.pid) + '] started on' + str(self.host) + ':' + str(self.port) + ', Daemon: ' + str(self.daemon))
		sub_imu = zmq.Sub('imu', (self.host, self.port))
		sub_vo = zmq.Sub('vo', (self.host, self.port))
		pub = zmq.Pub(('localhost', '9100'))
		# self.logger.info('Openned camera: '+str(self.camera_num))
		v = np.array([0., 0., 0.])
		gps = ecef(39.0, 104.7, 1000.0)
		x = np.array(gps)
		q = np.array([1.0, 0., 0., 0.0])
		X = np.hstack((v, x, q))
		w = np.array([0, 0, 0])

		self.epoch = dt.datetime.now()

		# x_init = np.array([1, 1])
		ekf = EKF(1, 1)
		Q = np.diag([1, 2, 3, 4, 5, 6])
		R = np.diag([1, 2, 3, 4, 5, 6])
		ekf.init(X, kf_eqns, R, Q)

		def printX(x, q):
			print('------------------------------------------------')
			print('pos: {:.2f} {:.2f} {:.2f}'.format(*x[0:3]))
			print('vel: {:.2f} {:.2f} {:.2f}'.format(*x[3:6]))
			print('qua: {:.2f} {:.2f} {:.2f} {:.2f}'.format(*x[6:]))
			print('qu2: {:.2f} {:.2f} {:.2f} {:.2f}'.format(*q))

		try:
			# ans = {}
			while True:
				topic, imu = sub_imu.recv()
				if imu:
					# print(imu)
					# get this from imu
					x = imu['linear_acceleration']['x']
					y = imu['linear_acceleration']['y']
					z = imu['linear_acceleration']['z']
					f = np.array([x, y, z])

					x = imu['angular_velocity']['x']
					y = imu['angular_velocity']['y']
					z = imu['angular_velocity']['z']
					w = np.array([x, y, z])

					qw = imu['orientation']['w']
					qx = imu['orientation']['x']
					qy = imu['orientation']['y']
					qz = imu['orientation']['z']

					# integrate navigation EoM
					u = np.hstack((f, w))
					X = self.eom.step(X, u)
					printX(X, [qw, qx, qy, qz])

				topic, vo = sub_vo.recv()
				if vo:
					# self.kf.predict()
					# self.kf.??
					print('wtf ... not implemented yet')
				else:
					vo = None

				if 0:
					u = np.array([0, 0])
					z = np.array([0, 0])
					ekf.predict(u)
					ekf.update(z)
				# xar.append(X[3])
				# yar.append(X[4])
				# plot(fig, li, xx, yy)

				# [odom]-----------------------------------------------------
				# pose(position[vector], orientation[quaternion])
				# twist(linear[vector], angular[vector])
				odom = msg.Odom()
				odom['position']['position']['x'] = X[3]
				odom['position']['position']['y'] = X[4]
				odom['position']['position']['z'] = X[5]

				odom['position']['orientation']['x'] = X[6]
				odom['position']['orientation']['y'] = X[7]
				odom['position']['orientation']['z'] = X[8]
				odom['position']['orientation']['z'] = X[9]

				odom['velocity']['linear']['x'] = X[0]
				odom['velocity']['linear']['y'] = X[1]
				odom['velocity']['linear']['z'] = X[2]

				odom['velocity']['angular']['x'] = w[0]
				odom['velocity']['angular']['y'] = w[1]
				odom['velocity']['angular']['z'] = w[2]

				# print(odom)

				pub.pub('/nav', odom)
				# time.sleep(0.05)

		except KeyboardInterrupt:
			print('Navigation: shutting down ...')
			# pass

def main():
	nav = NavigationServer()
	nav.run()

if __name__ == '__main__':
	main()

max_stars_repo_name: gecko-robotics/pygecko

max_issues_repo_name: walchko/pygecko

max_forks_repo_name: MomsFriendlyRobotCompany/pygecko

reason: import numpy,from numpy


text:
import csv
import os
import sys
import time

import numpy as np
#import matplotlib.pyplot as plt

from modules_scott.path import Path
from modules_scott.vector_math import *
from modules_janto.paths import *
#from find_matches import *

#********************
#**** this function reads a CSV file and returns a header, and a list that has been converted to float
#********************
def read_csv_float_with_header(file1,list1):

   fileopen = open(file1,'rb')
   fileobject = csv.reader(fileopen)

   # get the header
   header = next(fileobject)

   # read each line in the CSV, and convert the values to float before appending to list1
   for row in fileobject:
      float_line = []
      for subrow in row:
         float_line.append( float(subrow))

      list1.append( float_line)

   fileopen.close()   # close the file that we read from

   return  header, list1
#********************
#**** end function reads a CSV file and returns a header, and a list that has been converted to float
#********************

#*************** get the 10, 30, 50, 70, 90 percent numbers
def get_quintiles(float_list):
   float_list.sort()

   quintiles = []
   list_length = len(float_list)

   if ( list_length > 5):

      for x in range( 10, 110, 20):
        index1 = int(list_length * x / 100.)
        value1 = float_list[index1]

        quintiles.append(value1)

   else:
      quintiles = [0, 0, 0, 0, 0]

   return quintiles

#********************
#**** main code
#********************
#@profile
def mainloop(driver_id, final_out_csv, Input_Path):

    start_time = time.time()

    list_of_paths = []
    list_of_lengths = []

    # read all the routes for this driver
    for cnt in range(1,201):

       # initialize this path
       path = Path(1,cnt)   # start with driver 1, route 1

       input_coords=[]
#       file_name = "Test_Set\\driver_1\\" + str(cnt) + ".csv"
       file_name = os.path.join(Input_Path,str(driver_id),str(cnt) + ".csv")
       header, input_coords = read_csv_float_with_header(file_name,input_coords)
       path_array = np.array(input_coords)

       path.route = path_array
       path.time  = len(path.route)   # 1 second per data file

       # only analyze this path if it is not within a 90 meter bound of the starting point
       max_value = np.amax(path.route)
       min_value = np.amin(path.route)
       if ( max_value < 90 and min_value > -90):
           path.is_zero = 1   # this is a zero length route
           path.matched = 0   # the jitter is done differently

       # find the total distance along the route
       path.distance = path.get_route_distance(0, path.time)

       speed_hold = path.speed
       accel_hold = path.acceleration

       path.speed_quintiles = get_quintiles(speed_hold)
       path.acceleration_quintiles = get_quintiles(accel_hold)

       path.energy_per_distance = path.total_energy / path.distance
       path.energy_per_time = path.total_energy / path.time

       list_of_lengths.append(path.distance)

       #if ( path.is_zero == 0) :
       #    # get features on this path
       #    path.generate_features(rdp_tolerance)
       #
       #    # get angles between each of the consective features
       #    path.generate_angles()

       list_of_paths.append(path)

    for cnt1, path1 in enumerate(list_of_paths):
       path1.distance = round(path1.distance,4)
       path1.time = round(path1.time,4)
       path1.is_zero = round(path1.is_zero,4)

       speed1 = round(path1.speed_quintiles[0],4)
       speed2 = round(path1.speed_quintiles[1],4)
       speed3 = round(path1.speed_quintiles[2],4)
       speed4 = round(path1.speed_quintiles[3],4)
       speed5 = round(path1.speed_quintiles[4],4)

       accel1 = round(path1.acceleration_quintiles[0],4)
       accel2 = round(path1.acceleration_quintiles[1],4)
       accel3 = round(path1.acceleration_quintiles[2],4)
       accel4 = round(path1.acceleration_quintiles[3],4)
       accel5 = round(path1.acceleration_quintiles[4],4)

       time_at_speed =[]
       for i in range(0,10):
          time_at_speed.append(  round(path1.time_in_speed[i] / (path1.time-2),5) )

       total_energy = round(path1.total_energy,4)
       energy_per_distance = round(path1.energy_per_distance,4)
       energy_per_time = round(path1.energy_per_time,4)

       final_out_csv.writerow([driver_id, path1.routeid, path1.distance, path1.time, path1.is_zero,speed1,speed2,speed3,speed4,speed5,accel1,accel2,accel4,accel5,total_energy,energy_per_distance,energy_per_time,time_at_speed[0],time_at_speed[1],time_at_speed[2],time_at_speed[3],time_at_speed[4],time_at_speed[5],time_at_speed[6],time_at_speed[7],time_at_speed[8],time_at_speed[9] ])

    end_time = time.time()
    print("\tMinutes elapsed ",(end_time-start_time) / 60.,"\n")

    #********************
    #**** end main code
    #********************

# Path to raw data
Input_Path =   DATA

final_out = open("featurematrix2.csv",'wb')
final_out_csv = csv.writer(final_out)
final_out_csv.writerow(['driver','route','distance','time','is_jitter','speed10pct','speed30pct','speed50pct','speed70pct','speed90pct','accel10pct','accel30pct','accel70pct','accel90pct','total_energy','energy_per_distance','energy_per_time'])

# data_path_file = open("..\\..\\..\\SETTINGS.json",'rb')
# file_paths = data_path_file.readlines()
# for file_path in file_paths:
#    if ('DATA' in file_path):
#       Input_Paths = file_path.split("\"")
#       for cnt1, string1 in enumerate(Input_Paths):
#          if (string1 == 'DATA'):
#             location1 = cnt1
#       Input_Path2 = Input_Paths[location1 +2]
#       Input_Path =''
#       for char1 in Input_Path2:
#          if (char1 == '\\'):
#             Input_Path = Input_Path + '\\' + '\\'
#          else:
#             Input_Path = Input_Path + char1

#       Input_Path = Input_Path +  '\\' + '\\'
#       try:
#          file_name =  Input_Path +str(1)+"\\" + str(1) + ".csv"
#          fileopen = open(file_name,'rb')
#          fileopen.close()   # close the file that we read from
#       except:
#          print("Could not open the raw data from the path read in from Settings.json   ")
#          print("quiting")
#          sys.exit()

print("\n\nExtracting feature set 2...\n\n")

for driver_id in range(1,3613):

   #if (driver_id <= 3613):
   #   print("here 1")
   try:
      file_name =  os.path.join(Input_Path,str(driver_id),str(1) + ".csv")
      fileopen = open(file_name,'rb')
      fileopen.close()   # close the file that we read from

      print("Processing driver ",driver_id)
      mainloop(driver_id, final_out_csv, Input_Path)
   except:
      x=1

final_out.close()

print('\nfeaturematrix 2 created.')

max_stars_repo_name: Keesiu/meta-kaggle

max_issues_repo_name: Keesiu/meta-kaggle

max_forks_repo_name: Keesiu/meta-kaggle