text:
import pandas as pd
from matplotlib import rcParams
import matplotlib.pyplot as plt
import pickle
import util.plot
import util.data
import numpy as np
from pandas.plotting import scatter_matrix

np.random.seed(123)

data = pd.read_csv('data/training_set_VU_DM_clean.csv', sep=';',nrows=10000)

# some feature engineering
data["srch_person_per_room"] = (data["srch_adults_count"]+data["srch_children_count"])/data["srch_room_count"]
data.loc[~(data["srch_person_per_room"]<10000),"srch_person_per_room"] = 0
data["srch_adults_per_room"] = data["srch_adults_count"]/data["srch_room_count"]

columns = list(data.columns)
srch_crit  = [k for k in columns if 'srch' in k]
correlation_grid(data, srch_crit[2:8])

for k in columns[3:]:
    makeBarchart(data,k,len(set(data[k])))

scatter_matrix(data[srch_crit[2:8]], figsize=(6, 6))
plt.show()

def makeBarchart(data, k, n):
    class Encoders:
        discretizers = {}
        encoders = {}
    newData = pd.DataFrame(data[k])
    #util.data.discretize(newData, k, Encoders,n)
    newData["click_bool"] = data["click_bool"]
    newData["booking_bool"]= data["booking_bool"]
    # data to plot
    clicks = np.zeros(n)
    books = np.zeros(n)
    sets = list(set(newData[k]))
    for i in range(0,n):
        clicks[i] = np.mean(newData.loc[newData[k]==sets[i],"click_bool"])
        books[i] = np.mean(newData.loc[newData[k]==sets[i],"booking_bool"])

    # create plot
    fig, ax = plt.subplots()
    index = np.arange(n)
    bar_width = 0.35
    opacity = 0.8

    rects1 = plt.bar(index, clicks, bar_width,
    alpha=opacity,
    color='b',
    label='clicks')

    rects2 = plt.bar(index + bar_width, books, bar_width,
    alpha=opacity,
    color='g',
    label='books')

    plt.xlabel(k)
    plt.ylabel('mean')
    plt.title('mean by type')
    plt.xticks(index + bar_width, (sets))
    plt.legend()

    plt.tight_layout()
    plt.show()

max_stars_repo_name: voschezang/Data-Mining

max_issues_repo_name: voschezang/Data-Mining

max_forks_repo_name: voschezang/Data-Mining


text:
from helpers import *
import numpy as np

# in this code we will use integer based arithmetic code insted of the infinite prescision version
# this version is based upon this youtube tutorial : https://www.youtube.com/watch?v=ouYV3rBtrTI&list=PLE125425EC837021F&index=41

# creating arbitrary stream to test on
stream = [1,2,3,4,5,7,85,4,5,7,-5,516,8,1,68,1,313,-5,18,9,7,351,32,15,6,84,13,20,56,85,4,55,6,7,85,4,-5,516,8,1,68,1,313,-5,18,9,7,351,32,15,6,5,5,7,4,5,6,7,86,7,8,8,7,6,85,7,4,5,6,7]
stream=stream*20 # this is just made to increase the size without writing too much :)

 # Unlike the infinite precision case, here we are confined with a finite amount of bits
 # the number of precision bits that from which we will try to handle the code
precision=32

# call the encoder function
code,dic =Arithmetic_encode(stream,precision)

print (code)
print ('number of bits is : ',len(code))

# call the decoder function
message=Arithmetic_decode (code,dic,precision)

stream.pop()# remove the end of file symbol so that we could compare the stream with the decoded messsage

print(stream)
print (message)
print(stream == message)

ext: py

max_stars_repo_name: Khaled-Abdelhamid/Arithmetic-code

max_issues_repo_name: yifeipet/Arithmetic-code

max_forks_repo_name: yifeipet/Arithmetic-code

reason: import numpy


text:
"""
Theano is an optimizing compiler in Python, built to evaluate
complicated expressions (especially matrix-valued ones) as quickly as
possible.  Theano compiles expression graphs (see :doc:`graph` ) that
are built by Python code. The expressions in these graphs are called
`Apply` nodes and the variables in these graphs are called `Variable`
nodes.

You compile a graph by calling `function`, which takes a graph, and
returns a callable object.  One of theano's most important features is
that `function` can transform your graph before compiling it.  It can
replace simple expressions with faster or more numerically stable
implementations.

To learn more, check out:

- Op List (:doc:`oplist`)

The markup language used in the docstrings is ReStructured Text,
which may be rendered with Sphinx. A rendered version is
maintained at http://www.deeplearning.net/software/theano/library/

"""
from __future__ import absolute_import, print_function, division

__docformat__ = "restructuredtext en"

# Set a default logger. It is important to do this before importing some other
# theano code, since this code may want to log some messages.
import logging

import sys

if sys.platform == 'win32' and sys.version_info[0:2] == (3, 5):
    raise RuntimeError(
        "Theano do not support Python 3.5 on Windows. Use Python 2.7 or 3.4.")

theano_logger = logging.getLogger("theano")
logging_default_handler = logging.StreamHandler()
logging_default_formatter = logging.Formatter(
    fmt='%(levelname)s (%(name)s): %(message)s')
logging_default_handler.setFormatter(logging_default_formatter)
theano_logger.addHandler(logging_default_handler)
theano_logger.setLevel(logging.WARNING)

# Version information.
from theano.version import version as __version__

from theano.configdefaults import config

import os
import re
from subprocess import check_call, check_output, CalledProcessError

try:
    check_call(['icpc', '-v'])
    using_icpc = True
except (CalledProcessError, OSError):
    using_icpc = False

if not using_icpc:
    config.cxx = 'g++'
    config.gcc.cxxflags = '-fopenmp -O3 -opt-prefetch=2 -funroll-loops'

on_xeon_phi = False

cpuinfo = check_output(['cat', '/proc/cpuinfo']).decode('utf-8')
lines = cpuinfo.split('\n')

regex = r'^model\s+:\s+(\d+)'
for line in lines:
    m = re.match(regex, line)
    if m:
        model = int(m.group(1))
        on_xeon_phi = model == 87 or model == 133
        break

if on_xeon_phi:

    core_ids = set()

    core_id_regex = r'^core id\s+:\s+(\d+)'
    for line in lines:
        m = re.search(core_id_regex, line)
        if m:
            core_ids.add(m.group(1))
    num_cores = len(core_ids)

    # Xeon Phi recommended defaults
    env_vars = {
        'KMP_BLOCKTIME': '1',
        'KMP_AFFINITY': 'verbose,granularity=core,noduplicates,compact,0,0',
        'OMP_NUM_THREADS': str(num_cores - 2),
    }

    for key, val in env_vars.iteritems():
        if os.getenv(key, None) is None:
            os.environ[key] = val
            theano_logger.info("Setting environment variable '{}' to '{}' as optimal recommendation".format(key, val))
        else:
            theano_logger.info("Environment variable '{}' is set to '{}'".format(key, os.getenv(key)))
            theano_logger.info("Recommended value is '{}'".format(val))

# This is the api version for ops that generate C code.  External ops
# might need manual changes if this number goes up.  An undefined
# __api_version__ can be understood to mean api version 0.
#
# This number is not tied to the release version and should change
# very rarely.
__api_version__ = 1

from theano.gof import (
    CLinker, OpWiseCLinker, DualLinker, Linker, LocalLinker, PerformLinker,
    Container,
    InconsistencyError, FunctionGraph,
    Apply, Variable, Constant,
    Op, OpenMPOp,
    opt,
    toolbox,
    Type, Generic, generic,
    object2, utils)

from theano.compile import (
    SymbolicInput, In,
    SymbolicOutput, Out,
    Mode,
    predefined_modes, predefined_linkers, predefined_optimizers,
    FunctionMaker, function, function_dump, OpFromGraph,
    ProfileMode, ProfileStats,
    Param, shared, as_op)

from theano.misc.safe_asarray import _asarray

from theano.printing import pprint, pp

from theano.scan_module import scan, map, reduce, foldl, foldr, clone

from theano.updates import OrderedUpdates

# scan_module import above initializes tensor and scalar making these imports
# redundant

# import tensor
# import scalar

# we don't import by default as we don't want to force having scipy installed.

# import sparse

from theano.gradient import Rop, Lop, grad, subgraph_grad

# This need to be before the init of GPU, as it add config variable
# needed during that phase.
import theano.tests
if hasattr(theano.tests, "TheanoNoseTester"):
    test = theano.tests.TheanoNoseTester().test
else:
    def test():
        raise ImportError("The nose module is not installed."
                          " It is needed for Theano tests.")

if config.device.startswith('gpu') or config.init_gpu_device.startswith('gpu'):
    import theano.sandbox.cuda
    # We can't test the driver during import of theano.sandbox.cuda as
    # this cause circular import dependency. So we also test it manually
    # after the import
    if theano.sandbox.cuda.cuda_available:
        import theano.sandbox.cuda.tests.test_driver

        if config.enable_initial_driver_test:
            theano.sandbox.cuda.tests.test_driver.test_nvidia_driver1()

if (config.device.startswith('cuda') or
        config.device.startswith('opencl') or
        config.init_gpu_device.startswith('cuda') or
        config.init_gpu_device.startswith('opencl') or
        config.contexts != ''):
    import theano.gpuarray

# This was added for using Intel MKL dnn primitive,
# which contains optimal implementaion Ops.
if config.device.startswith('cpu'):
    try:
        import theano.sandbox.mkl
    except Exception as e:
        pass

# Use config.numpy to call numpy.seterr
import numpy

if config.numpy.seterr_all == 'None':
    _all = None
else:
    _all = config.numpy.seterr_all
if config.numpy.seterr_divide == 'None':
    _divide = None
else:
    _divide = config.numpy.seterr_divide
if config.numpy.seterr_over == 'None':
    _over = None
else:
    _over = config.numpy.seterr_over
if config.numpy.seterr_under == 'None':
    _under = None
else:
    _under = config.numpy.seterr_under
if config.numpy.seterr_invalid == 'None':
    _invalid = None
else:
    _invalid = config.numpy.seterr_invalid
numpy.seterr(
    all=_all,
    divide=_divide,
    over=_over,
    under=_under,
    invalid=_invalid)
del _all, _divide, _over, _under, _invalid

# This is defined here because it is designed to work across symbolic
#   datatypes (Sparse and Tensor)

def dot(l, r):
    """Return a symbolic matrix/dot product between l and r """
    rval = NotImplemented
    e0, e1 = None, None

    if rval == NotImplemented and hasattr(l, '__dot__'):
        try:
            rval = l.__dot__(r)
        except Exception as e0:
            rval = NotImplemented
    if rval == NotImplemented and hasattr(r, '__rdot__'):
        try:
            rval = r.__rdot__(l)
        except Exception as e1:
            rval = NotImplemented
    if rval == NotImplemented:
        raise NotImplementedError("Dot failed for the following reasons:",
                                  (e0, e1))
    return rval

def get_scalar_constant_value(v):
    """return the constant scalar(0-D) value underlying variable `v`

    If v is the output of dimshuffles, fills, allocs, rebroadcasts, cast
    this function digs through them.

    If theano.sparse is also there, we will look over CSM op.

    If `v` is not some view of constant data, then raise a
    tensor.basic.NotScalarConstantError.
    """
    # Is it necessary to test for presence of theano.sparse at runtime?
    if 'sparse' in globals() and isinstance(v.type, sparse.SparseType):
        if v.owner is not None and isinstance(v.owner.op, sparse.CSM):
            data = v.owner.inputs[0]
            return tensor.get_scalar_constant_value(data)
    return tensor.get_scalar_constant_value(v)

def sparse_grad(var):
    """This function return a new variable whose gradient will be
    stored in a sparse format instead of dense.

    Currently only variable created by AdvancedSubtensor1 is supported.
    i.e. a_tensor_var[an_int_vector].

    .. versionadded:: 0.6rc4
    """
    assert isinstance(var.owner.op, tensor.AdvancedSubtensor1)
    ret = var.owner.op.__class__(sparse_grad=True)(*var.owner.inputs)
    return ret

__import__('theano.tensor.shared_randomstreams')

max_stars_repo_name: TaoLv/Theano

max_issues_repo_name: TaoLv/Theano

max_forks_repo_name: TaoLv/Theano

reason: import numpy,import theano,from theano


text:
# plot_web_api_realtime.py
"""
A live auto-updating plot of random numbers pulled from a web API
"""

import time
import datetime as dt
import requests
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import pandas as pd
import numpy as np

url = "https://opendata-geohive.hub.arcgis.com/datasets/d8eb52d56273413b84b0187a4e9117be_0.csv?outSR=%7B%22latestWkid%22%3A3857%2C%22wkid%22%3A102100%7D"

df = pd.read_csv(url)

print(df)
print(df.info())
print(df.columns)
print(df.sort_values("ConfirmedCovidCases"))
print(df["ConfirmedCovidCases"].mean())

df['month_year'] = pd.to_datetime(df['Date']).dt.to_period('M')
print(df.head())
print(df.pivot_table(values="ConfirmedCovidCases", index="month_year", aggfunc=[np.mean, np.median]))

# create the figure and axis objects
#fig, ax = plt.subplots()

# plot the data and customize
x= df['Date']
y= df['ConfirmedCovidCases']

#ax.plot(x, y)
#ax.set_xlabel('Day Number')
#ax.set_ylabel('Count')
#ax.set_title('Cases')

# save and show the plot
#fig.savefig('static_plot.png')
#plt.show()

# function to pull out a float from the requests response object
def pull_float(response):
    jsonr = response.json()
    strr = jsonr["ConfirmedCovidCases"][0]
    if strr:
        fltr = round(float(strr), 2)
        return fltr
    else:
        return None

# Create figure for plotting
fig, ax = plt.subplots()
xs = []
ys = []

def animate(i, xs:list, ys:list):
    # grab the data from thingspeak.com
    response = requests.get(url)
    flt = pull_float(response)
    # Add x and y to lists
    xs.append(dt.datetime.now().strftime('%H:%M:%S'))
    ys.append(flt)
    # Limit x and y lists to 10 items
    xs = xs[-10:]
    ys = ys[-10:]
    # Draw x and y lists
    ax.clear()
    ax.plot(xs, ys)
    # Format plot
    ax.set_ylim([0,255])
    plt.xticks(rotation=45, ha='right')
    plt.subplots_adjust(bottom=0.20)
    ax.set_title('Plot of random numbers from https://qrng.anu.edu.au')
    ax.set_xlabel('Date Time (hour:minute:second)')
    ax.set_ylabel('Random Number')

# Set up plot to call animate() function every 1000 milliseconds
ani = animation.FuncAnimation(fig, animate, fargs=(xs,ys), interval=1000)

plt.show()


text:
import pandas as pd
import numpy as np

model_data = pd.get_dummies(pd.read_csv('abalone.csv'),
                            drop_first=True)

model_data[['age']] = model_data[['rings']] + 1
model_data = model_data.drop(columns="rings")
train_data, validation_data = np.split(model_data.sample(
    frac=1, random_state=123), [int(0.8 * len(model_data))])

pd.concat([train_data['age'], train_data.drop(['age'], axis=1)],
          axis=1).to_csv('../abalone_train.csv', index=False, header=False)
pd.concat([validation_data['age'], validation_data.drop(['age'], axis=1)], axis=1).to_csv(
    '../abalone_validation.csv', index=False, header=False)

max_stars_repo_name: TomasBeuzen/machine-learning-tutorials

max_issues_repo_name: mgelbart/machine-learning-tutorials

max_forks_repo_name: TomasBeuzen/machine-learning-tutorials


text:
import os, glob
import unittest
import numpy as np
from numpy.testing import assert_allclose, assert_equal, assert_array_equal
import trackpy as tp
from trackpy.utils import cKDTree, pandas_sort, make_pandas_strict
from trackpy.artificial import SimulatedImage
from matplotlib.pyplot import imread

class StrictTestCase(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        # Suppress logging messages
        tp.quiet()
        # Catch attempts to set values on an inadvertent copy of a Pandas object.
        make_pandas_strict()
        # Make numpy strict
        np.seterr('raise')

def sort_positions(actual, expected):
    tree = cKDTree(actual)
    deviations, argsort = tree.query([expected])
    if len(set(range(len(actual))) - set(argsort[0])) > 0:
        raise AssertionError("Position sorting failed. At least one feature is "
                             "very far from where it should be.")
    return deviations, actual[argsort][0]

def assert_coordinates_close(actual, expected, atol):
    assert_equal(len(actual), len(expected))
    _, sorted_actual = sort_positions(actual, expected)
    assert_allclose(sorted_actual, expected, atol=atol)

def assert_traj_equal(actual, expected, pos_atol=1):
    assert_equal(len(actual), len(expected))
    actual = pandas_sort(actual, 'frame').reset_index(drop=True)
    expected = pandas_sort(expected, 'frame').reset_index(drop=True)
    actual_order = []
    for frame_no in expected['frame'].unique():
        actual_f = actual[actual['frame'] == frame_no]
        expected_f = expected[expected['frame'] == frame_no]
        assert_equal(len(actual_f), len(expected_f),
                     err_msg='Actual and expected numbers of features '
                             'differ in frame %i' % frame_no)
        tree = cKDTree(actual_f[['y', 'x']].values)
        devs, argsort = tree.query(expected_f[['y', 'x']].values)
        assert_allclose(devs, 0., atol=pos_atol)
        actual_order.extend(actual_f.index[argsort].tolist())

    actual = actual.loc[actual_order].reset_index(drop=True, inplace=False)
    for p_actual in actual.particle.unique():
        actual_ind = actual.index[actual['particle'] == p_actual]
        p_expected = expected.loc[actual_ind[0], 'particle']
        expected_ind = expected.index[expected['particle'] == p_expected]
        assert_array_equal(actual_ind, expected_ind,
                           err_msg='Actual and expected linking results '
                           'differ for actual particle %i/expected particle %i'
                           '' % (p_actual, p_expected))

class TrackpyFrame(np.ndarray):
    """Simplified version of pims.Frame, for testing purposes only."""
    def __new__(cls, input_array, frame_no=None):
        # get a view of the input data as a Frame object
        obj = np.asarray(input_array).view(cls)
        obj.frame_no = frame_no
        return obj

    def __array_finalize__(self, obj):
        if obj is None:
            return
        self.frame_no = getattr(obj, 'frame_no', None)

    def __array_wrap__(self, out_arr, context=None):
        # Handle scalars so as not to break ndimage.
        # See http://stackoverflow.com/a/794812/1221924
        if out_arr.ndim == 0:
            return out_arr[()]

        return np.ndarray.__array_wrap__(self, out_arr, context)

class TrackpyImageSequence:
    """Simplified version of pims.ImageSequence, for testing purposes only."""
    def __init__(self, path_spec):
        self._get_files(path_spec)

        tmp = imread(self._filepaths[0])
        self._first_frame_shape = tmp.shape

    def _get_files(self, path_spec):
        # deal with if input is _not_ a string
        if not isinstance(path_spec, str):
            # assume it is iterable and off we go!
            self._filepaths = list(path_spec)
            self._count = len(self._filepaths)
            return

        self.pathname = os.path.abspath(path_spec)  # used by __repr__
        if os.path.isdir(path_spec):
            directory = path_spec
            filenames = os.listdir(directory)
            make_full_path = lambda filename: (
                os.path.abspath(os.path.join(directory, filename)))
            filepaths = list(map(make_full_path, filenames))
        else:
            filepaths = glob.glob(path_spec)
        self._filepaths = list(sorted(filepaths))
        self._count = len(self._filepaths)

        # If there were no matches, this was probably a user typo.
        if self._count == 0:
            raise OSError("No files were found matching that path.")

    def __getitem__(self, j):
        image = imread(self._filepaths[j])
        image = (image * 255).astype(np.uint8)
        return TrackpyFrame(image, frame_no=j)

    def __len__(self):
        return self._count

    @property
    def frame_shape(self):
        return self._first_frame_shape

    @property
    def dtype(self):
        return np.uint8

class CoordinateReader:
    """Generate a pims.FramesSquence-like object that draws features at
    given coordinates"""
    def __init__(self, f, shape, size, t=None, **kwargs):
        self._f = f.copy()
        self.pos_columns = ['z', 'y', 'x'][-len(shape):]
        self.shape = shape
        self.size = size
        self.kwargs = kwargs
        self.im = SimulatedImage(shape, size, **self.kwargs)
        if t is None:
            self._len = int(f['frame'].max() + 1)
            self._inds = range(self._len)
        else:
            self._len = len(t)
            self._inds = t

    def __len__(self):
        return self._len

    def __iter__(self):
        # this is actually a hack to get find_link working with float-typed indices
        return (self.get_frame(i) for i in self._inds)

    def __getitem__(self, key):
        return self.get_frame(key)

    def get_frame(self, ind):
        self.im.clear()
        pos = self._f.loc[self._f['frame'] == ind, self.pos_columns].values
        for _pos in pos:
            self.im.draw_feature(_pos)
        return TrackpyFrame(self.im(), frame_no=ind)

    @property
    def pixel_type(self):
        return self.im.dtype

    @property
    def frame_shape(self):
        return self.im.shape

max_stars_repo_name: rbnvrw/trackpy

max_issues_repo_name: rbnvrw/trackpy

max_forks_repo_name: rbnvrw/trackpy

reason: import numpy,from numpy


text:
"""Joint maximum a posteriori state-path and parameter estimator for SDEs."""

import numpy as np

from . import oem

class Problem(oem.Problem):
    """Joint MAP state-path and parameter estimation problem."""

    def __init__(self, model, t, y, u):
        super().__init__(model, t, y, u)

        # Add fictitious log-density of tubes to objective function
        self.add_objective(model.tube_L, self.npieces)

        # Add collocation defect penalty
        if getattr(model, 'use_penalty', False):
            self.add_objective(model.penalty, self.npieces)

    def _init_collocation_variables(self):
        """Register problem collocation variables."""
        super()._init_collocation_variables()
        col = self.collocation
        model = self.model

        # Coordinates over the basis for the process noise intensity w
        self.add_decision('wc', (self.npieces, col.ninterv, model.nw))

    def variables(self, dvec):
        """Get all variables needed to evaluate problem functions."""
        return {'penweight': getattr(self.model, 'penweight', None),
                'G': self.model.G, **super().variables(dvec)}

max_stars_repo_name: dimasad/ceacoest

max_issues_repo_name: dimasad/ceacoest

max_forks_repo_name: dimasad/ceacoest


text:
from sklearn.cross_validation import train_test_split
from sklearn.svm import LinearSVC
import numpy as np
from sklearn.externals import joblib
import random

def duplicate_classes(trainX, trainY, num, class_0, class_1):

    counts = [len(list(np.where(trainY == i)[0])) for i in range(0,5)]
    print "0s : {0}\n1s : {1}\n2s : {2}\n3s : {3}\n4s : {4}\n".format(counts[0] , counts[1], counts[2], counts[3], counts[4])

    indexes = list(np.where(trainY == class_1)[0])

    print len(indexes)
    newX = np.zeros((num, trainX.shape[1]))
    newY = np.zeros((num, 1))
    zeros = 0
    ones = 0
    i = 0

    count = len(indexes)
    print count
    while(zeros < num - count  or ones < count):

        if len(indexes) > 0:
            random_index = indexes.pop()
        else:
            random_index = random.randint(0, trainX.shape[0] - 1)

        label = int(trainY[random_index, 0])
        if label == class_1 and ones < count:

            newX[i,:] = trainX[random_index,:]
            newY[i,0] = 1
            ones += 1
            i += 1

        elif label == class_0 and zeros < num - count:
            newX[i,:] = trainX[random_index,:]
            newY[i,0] = 0
            zeros += 1
            i += 1

    from sklearn.utils import shuffle
    newX, newY  = shuffle(newX, newY)
    from sklearn.feature_selection import SelectKBest, chi2
    X_new = SelectKBest(chi2, k=500).fit_transform(newX, newY)
    return X_new, newY

if __name__ == '__main__':
    import os
    x_1 = np.load('features/test_features_windowsize_6.npy')
    x_2 = np.load('features/test_features_windowsize_8.npy')
    x_3 = np.load('features/test_features_windowsize_16.npy')
    x_4 = np.load('features/train_features_windowsize_16.npy')
    trainY = np.load('features/trainY_corrected.npy')

#
    print "Feature vector dimensions: {0}".format(np.hstack((x_1, x_2, x_3, x_4)).shape[1])

    trainXC, y_train = duplicate_classes(np.hstack((x_1,x_2,x_3,x_4)), trainY, 2000, 0, 4)

    trainXC_mean = np.mean(trainXC, axis=0, dtype=np.float32, keepdims=False)
    trainXC_sd = np.sqrt(np.var(trainXC, axis=0, dtype=np.float32, ddof=1, keepdims=False)+0.01)
    trainXCs = (trainXC - trainXC_mean) / trainXC_sd
    x_train, x_test, y_train, y_test = train_test_split(trainXCs, y_train, test_size=0.25)

    classifier = LinearSVC(penalty='l2', loss='l2', dual=True, tol=0.0001, C=100.0, multi_class='ovr', fit_intercept=True, intercept_scaling=1, class_weight='auto', verbose=1)
    classifier.fit(x_train, y_train)
#    joblib.dump(classifier, 'svm_incomplete_classifier.pkl')
    y_train_predicted = classifier.predict(x_train)
    y_test_predicted = classifier.predict(x_test)

    from sklearn.metrics import accuracy_score, classification_report, confusion_matrix

    train_accuracy = accuracy_score(y_train, y_train_predicted)
    test_accuracy = accuracy_score(y_test, y_test_predicted)

    print 'Train accuracy = ' + str(train_accuracy)
    print 'Test accuracy = ' + str(test_accuracy)

    print 'TRAIN: Classification report'
    print classification_report(y_train, y_train_predicted)
    print confusion_matrix(y_train, y_train_predicted)

    print 'TEST: Classification report'
    print classification_report(y_test, y_test_predicted)
    print confusion_matrix(y_test, y_test_predicted)

    #print confusion_matrix(old_samples, prediction)
   # from time import sleep
   # for i in range(0, len(old_samples)):
   #     print old_samples[i], decision[i], prediction[i]
   #     sleep(1)
    #from lmdbWriter import open_csv
    #data_dict = open_csv('../data/trainLabels.csv')
    #files = os.listdir('../data/resized/train512')

    #full_classifier = LinearSVC(penalty='l2', loss='l2', dual=True, tol=0.0001, C=100.0, multi_class='ovr', fit_intercept=True, intercept_scaling=1, class_weight=None, verbose=1,    random_state=None)
    #full_classifier.fit(trainXCs, trainY)
#    joblib.dump(full_classifier, 'svm_classifier.pkl')

    #y_full_predicted = full_classifier.predict(trainXCs)
    #full_accuracy = accuracy_score(trainY, y_full_predicted)
    #print 'FULL TRAIN = ' + str(full_accuracy)

    #print 'FULL Classification report'
    #print classification_report(trainY, y_full_predicted)

    #print confusion_matrix(trainY, y_full_predicted)

max_stars_repo_name: Keesiu/meta-kaggle

max_issues_repo_name: Keesiu/meta-kaggle

max_forks_repo_name: Keesiu/meta-kaggle


text:
import numpy as np
from scipy.ndimage.filters import convolve
from colour.utilities import tstack

def mhc_demosaic(cfa, raw):

    Gr_Gb = np.asarray([[0, 0, -1, 0, 0], [0, 0, 2, 0 , 0], [-1, 2, 4, 2, -1], [0, 0, 2, 0, 0], [0, 0, -1, 0, 0]], dtype=np.float64) / 8

    Rg_r_Bg_r = np.asarray([[0, 0, 0.5, 0, 0], [0, -1, 0, -1, 0], [-1, 4, 5, 4, -1], [0, -1, 0, -1, 0], [0, 0, 0.5, 0, 0]], dtype=np.float64) / 8

    Rg_b_Bg_b = np.transpose(Rg_r_Bg_r)

    Rb_Br = np.asarray([[0, 0, -1.5, 0, 0], [0, 2, 0, 2, 0], [-1.5, 0, 6, 0, -1.5], [0, 2, 0, 2, 0], [0, 0, -1.5, 0, 0]], dtype=np.float64) / 8

    R = np.zeros(cfa.shape, dtype=np.float64)
    G = np.zeros(cfa.shape, dtype=np.float64)
    B = np.zeros(cfa.shape, dtype=np.float64)

    # R pixels
    R[raw.raw_colors_visible==0] = cfa[raw.raw_colors_visible==0]
    G[raw.raw_colors_visible==0] = convolve(cfa, Gr_Gb)[raw.raw_colors_visible==0]
    B[raw.raw_colors_visible==0] = convolve(cfa, Rb_Br)[raw.raw_colors_visible==0]

    # G pixels at R rows
    R[raw.raw_colors_visible==1] = convolve(cfa, Rg_r_Bg_r)[raw.raw_colors_visible==1]
    G[raw.raw_colors_visible==1] = cfa[raw.raw_colors_visible==1]
    B[raw.raw_colors_visible==1] = convolve(cfa, Rg_r_Bg_r)[raw.raw_colors_visible==1]

    # B pixels
    R[raw.raw_colors_visible==2] = convolve(cfa, Rb_Br)[raw.raw_colors_visible==2]
    B[raw.raw_colors_visible==2] = cfa[raw.raw_colors_visible==2]
    G[raw.raw_colors_visible==2] = convolve(cfa, Gr_Gb)[raw.raw_colors_visible==2]

    # G pixels at B rows
    R[raw.raw_colors_visible==3] = convolve(cfa, Rg_b_Bg_b)[raw.raw_colors_visible==3]
    G[raw.raw_colors_visible==3] = cfa[raw.raw_colors_visible==3]
    B[raw.raw_colors_visible==3] = convolve(cfa, Rg_b_Bg_b)[raw.raw_colors_visible==3]

    # Combine 3 channels
    return tstack([R, G, B])

if __name__ == '__main__':
    pass

reason: import numpy,from scipy


text:
# --------------
# Importing header files
import numpy as np
import warnings
#Reading file
warnings.filterwarnings('ignore')
# path for the file
data = np.genfromtxt(path, delimiter=",", skip_header=1)

#New record
new_record=[[50,  9,  4,  1,  0,  0, 40,  0]]
census=np.concatenate((data,new_record),axis=0)
age=np.array(census[:,[0]])

max_age=np.max(age)
min_age=np.min(age)
age_mean=np.mean(age)
age_std=np.std(age)
race_0=[]
race_1=[]
race_2=[]
race_3=[]
race_4=[]
for i in range(0,1001):
    item=census[i][2]
    if item == 0:
        race_0.append(census[i])
    elif item == 1:
        race_1.append(census[i])
    elif item == 2:
        race_2.append(census[i])
    elif item == 3:
        race_3.append(census[i])
    elif item == 4:
        race_4.append(census[i])
race_0=np.array(race_0)
race_1=np.array(race_1)
race_2=np.array(race_2)
race_3=np.array(race_3)
race_4=np.array(race_4)
len_0=len(race_0)
len_1=len(race_1)
len_2=len(race_2)
len_3=len(race_3)
len_4=len(race_4)
a=min(len_0,len_1,len_2,len_3,len_4)
if a==len_0:
    minority_race=0
elif a==len_1:
    minority_race=1
elif a==len_2:
    minority_race=2
elif a==len_3:
    minority_race=3
elif a==len_4:
    minority_race=4
print(minority_race)

senior_citizen=[]
for i in range(0,1001):
    item=census[i][0]
    if item > 60:
        senior_citizen.append(census[i])
senior_citizen=np.array(senior_citizen)

temp=np.array(senior_citizen[:,[6]])
working_hours_sum=np.sum(temp)
senior_citizens_len=len(senior_citizen)
avg_working_hours= working_hours_sum/senior_citizens_len
print(avg_working_hours)

high=[]
low=[]
for i in range(0,1001):
    item=census[i][1]
    if item >10:
        high.append(census[i])
    elif item <=10:
        low.append(census[i])
high=np.array(high)
low=np.array(low)
temp1=np.array(high[:,[7]])
avg_pay_high=np.mean(temp1)
temp2=np.array(low[:,[7]])
avg_pay_low=np.mean(temp2)


text:
import numpy as np

import sys
import numpy as np
from numpy import float32, int32, uint8, dtype, genfromtxt

N = len( sys.argv )

direction = sys.argv[ 1 ]
cx = float(sys.argv[ 2 ])
cx = float(sys.argv[ 2 ])
cy = float(sys.argv[ 3 ])
cz = float(sys.argv[ 4 ])

t=np.array([[1.0,0.0,0.0,cx],[0.0,1.0,0.0,cy],[0.0,0.0,1.0,cz],[0.0,0.0,0.0,1.0]])

sqrt2=0.70710678118

def translation_inv( t ):
    ti = np.copy( t )
    ti[0,3] = -ti[0,3]
    ti[1,3] = -ti[1,3]
    ti[2,3] = -ti[2,3]
    return ti

ti=translation_inv(t)

rotcw  = np.array([[sqrt2,sqrt2,0.0,0.0],[-sqrt2,sqrt2,0.0,0.0],[0.0,0.0,0.0,0.0],[0.0,0.0,0.0,1.0]])
rotccw = np.array([[sqrt2,-sqrt2,0.0,0.0],[sqrt2,sqrt2,0.0,0.0],[0.0,0.0,0.0,0.0],[0.0,0.0,0.0,1.0]])

if( direction == 'CW' or direction == 'cw' ):
    rot = rotcw
else:
    rot = rotccw

# Concatenate
xfm = np.matmul( t, np.matmul( rot, ti ))

print( xfm )

max_stars_repo_name: mzouink/template-building

max_issues_repo_name: mzouink/template-building

max_forks_repo_name: mzouink/template-building


text:
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import os
import sys
import json
from ast import literal_eval
import warnings
from sklearn.feature_extraction.text import CountVectorizer
import argparse
import random

warnings.filterwarnings(action='ignore')
fashions=pd.read_csv("/workspace/mj_nodejs/csv/coordiList.csv",encoding='utf-8')

fashions_df=fashions[['idnum','styles','coordi_literal','dress','top','bottom','outwear','temp','weight','count']]

from sklearn.metrics.pairwise import euclidean_distances
from sklearn.feature_extraction.text import TfidfVectorizer

cnt_vect=CountVectorizer(min_df=0) #ngram_range=(1,2))
coordi_vect=cnt_vect.fit_transform(fashions_df['coordi_literal']).toarray()

coordi_sim=1/(euclidean_distances(coordi_vect,coordi_vect)+1e-5)

st=sys.argv[1].split(',')
temp=int(sys.argv[2])
style=sys.argv[3]

coordi_sim_idx=coordi_sim.argsort()[:,::-1]
def find_sim_coordi2(df, sorted_idx,coordi_idnum,top_n=30):
            coordi_id=df[df['idnum']==coordi_idnum]
            coordi_idx=coordi_id.index.values

            top_sim_idx=sorted_idx[coordi_idx,:top_n]
            #print(top_sim_idx)
            top_sim_idx=top_sim_idx.reshape(-1,)
            similar_co=df.iloc[top_sim_idx]

            return similar_co

similar_coordi=pd.DataFrame(columns=['idnum','styles','coordi_literal','dress','top','bottom','outwear','temp','weight','count'])

for v in st:
    s=pd.DataFrame(columns=['idnum','styles','coordi_literal','dress','top','bottom','outwear','temp','weight','count'])
    v=int(v)
    s=find_sim_coordi2(fashions_df,coordi_sim_idx,v)
    similar_coordi=pd.concat([similar_coordi,s],ignore_index=True)

#print(similar_coordi[['idnum','coordi_literal','styles','temp']])
similar_style=similar_coordi.loc[(similar_coordi['styles']==style)&(similar_coordi['temp']==temp)]
#randomNum = random.sample([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], 2)
similar_style=similar_style.sample(2)
#similar_json=similar_style.to_json(orient ='records')
#similar_style=similar_style[randomNum[0]]

#similar_json=similar_style.to_json(orient ='split')
#print(similar_json)

similar_list=list(similar_style['idnum'])
print(similar_list)

#print(type(similar_style))
#print(similar_style[['idnum','coordi_literal','styles','temp']])


text:
import numpy as np
import cv2
import matplotlib.pyplot as plt
from skimage.transform import PiecewiseAffineTransform, warp
from skimage import data

im = cv2.imread("assets\\testgrid.png")
im = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)

# src_cols = np.linspace(0, cols, 5)
# src_rows = np.linspace(0, rows, 5)
# src_rows, src_cols = np.meshgrid(src_rows, src_cols)
# src = np.dstack([src_cols.flat, src_rows.flat])[0]

# print( src)

# # add sinusoidal oscillation to row coordinates
# dst_rows = src[:, 1] + np.cos(np.linspace(0, 2 * np.pi, src.shape[0])) * 64
# dst_cols = src[:, 0]
# dst = np.vstack([dst_cols, dst_rows]).T

# tform = cv2.getPerspectiveTransform(src, dst)
# out = cv2.warpPerspective(image,tform,(rows,cols))

# fig, ax = plt.subplots()
# ax.imshow(out)
# ax.plot(tform.inverse(src)[:, 0], tform.inverse(src)[:, 1], '.b')
# ax.axis((0, out_cols, out_rows, 0))
# plt.show()

loop = 0
while True:
  loop = loop + 1

  # generate a test image
  h, w = im.shape[0], im.shape[1]

  # Number of grid cells
  nx, ny = 8, 8
  p0 = np.meshgrid(np.linspace(0, w-1, nx+1, dtype='f'), np.linspace(0, h-1, ny+1, dtype='f'))
  p1 = [v.copy() for v in p0]

  for i in range(nx+1):
    for j in range(ny+1):
      p1[0][i,j] += np.sin(i/4*np.pi + (loop/6 % (2*np.pi))) * 40
      p1[1][i,j] += np.cos(j/4*np.pi + (loop/6 % (2*np.pi))) * 40

  im1 = np.zeros_like(im)
  for i in range(nx):
    for j in range(ny):
      x0, y0 = p0[0][j,i], p0[1][j,i]

      sourceSqr = np.stack((p0[0][j:(j+2),i:(i+2)].ravel() - x0 , p0[1][j:(j+2),i:(i+2)].ravel() - y0 ))
      destSqr = np.stack((p1[0][j:(j+2),i:(i+2)].ravel() , p1[1][j:(j+2),i:(i+2)].ravel() ))
      iSourceSqr = np.round(np.stack((p0[0][j:(j+2),i:(i+2)].ravel(), p0[1][j:(j+2),i:(i+2)].ravel()))).astype('i')
      iDestSqr = np.round(np.stack((p1[0][j:(j+2),i:(i+2)].ravel(), p1[1][j:(j+2),i:(i+2)].ravel()))).astype('i')

      #pad the squares to try to eliminate seam
      # if i > 0:
      #   iSourceSqr[0] += [-1, 0, -1, 0]
      #   iDestSqr[0] += [-1, 0, -1, 0]
      # if i < nx-1:
      #   iSourceSqr[0] += [0, 1, 0, 1]
      #   iDestSqr[0] += [0, 1, 0, 1]
      # if j > 0:
      #   iSourceSqr[1] += [-1, -1, 0, 0]
      #   iDestSqr[1] += [-1, -1, 0, 0]
      # if j < ny-1:
      #   iSourceSqr[1] += [0, 0, 1, 1]
      #   iDestSqr[1] += [0, 0, 1, 1]

      for indexSet in ([0,1,2],[1,3,2]):
        srcTri = sourceSqr.T[indexSet]
        destTri = destSqr.T[indexSet]
        iDestTri = iDestSqr.T[indexSet]
        M = cv2.getAffineTransform(srcTri, destTri)
        imw = cv2.warpAffine(im[iSourceSqr[1,0]:iSourceSqr[1,3], iSourceSqr[0,0]:iSourceSqr[0,3]], M, (w, h))
        # cv2.imshow("Test_" + str(i) + "_" + str(j) + "_" + str(indexSet[0]), imw)
        shard = cv2.fillConvexPoly(np.zeros_like(im), iDestTri, 255) & imw
        # im1 = np.where(shard == 0, im1, shard) // poor performance
        im1 |= shard

  cv2.imshow("Test", im1)

  key = cv2.waitKey(1)

  if key == ord('q'):
      break
  if key == 27: #esc
      break

max_stars_repo_name: eugenexlin/nice-face

max_issues_repo_name: eugenexlin/nice-face

max_forks_repo_name: eugenexlin/nice-face


text:
import cv2
import numpy as np
import io as lib

from imageio import imread
from matplotlib import pyplot
from Comparacao_Pascal import *
import PIL
from skimage import data, io, segmentation, color, morphology, img_as_ubyte

#Pega os nomes dos arquivos e manda para uma lista
from os import listdir
from os.path import isfile, join

from scipy.stats import norm, kurtosis,skew

mypath_all = ["exemplos_granito/28-05-2021_10-34-24_B68715", \
         "exemplos_granito/28-05-2021_11-42-34_B68703", \
         "exemplos_granito/28-05-2021_14-43-44_B69480", \
         "exemplos_granito/28-05-2021_14-51-12_B58905", \
         "exemplos_granito/28-05-2021_14-56-01_B58940", \
         "exemplos_granito/28-05-2021_15-05-12_B58609" ]
indice_all = 0
numero_fotos = 0
media_anterior = 0
constante_log = 110
numero_fotos = 0
media_all = 0
indice = 0
for i in range(0,6,1):
    mypath = mypath_all[i]
    onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]
    for index in range(0,len(onlyfiles),1):
        """
        mypath="exemplos_granito/28-05-2021_10-13-28_B69428_only_left"
        im = imread(mypath+"/1_40_left.png")
        mypath="exemplos_granito/28-05-2021_10-13-28_B69428_only_left"
        im = imread(mypath+"/1_390_left.png")
        """
        """
        #kernal sensitive to horizontal lines
        kernel = np.array([[-1.5, -1.5, 3, -1.5,-1.5],
                            [-1.5, 3.0, -18.0,3.0, -1.5],
                            [-1.5, -1.5, 3, -1.5,-1.5]])
        kernel = kernel
        kernel = kernel/(np.sum(kernel) if np.sum(kernel)!=0 else 1)

        #filter the source image
        im = cv2.filter2D(im,-1,kernel)
        """

        im = imread(mypath + "/1_" + str(index) + "_left.png")
        im = cv2.resize(im, (500,500), interpolation = cv2.INTER_CUBIC)
        # assign blue channel to zeros
        im = cv2.GaussianBlur(im,(5,5),5)

        """
        sobelx_filter = cv2.Sobel(im, ddepth=-1, dx=1, dy=0, ksize=11, scale=1, borderType=cv2.BORDER_CONSTANT)
        sobely_filter = cv2.Sobel(im, ddepth=-1, dx=0, dy=1, ksize=11, scale=1, borderType=cv2.BORDER_CONSTANT)
        sobel_filter = sobelx_filter + sobely_filter
        im = im - sobel_filter
        """

        print(onlyfiles)
        print("size: ",len(onlyfiles))

        im_cinza = color.rgb2gray(im)
        #im_cinza = 1*(im_cinza)**(5)
        im_cinza[im_cinza >= 255] = 255
        #im_cinza = cv2.Laplacian(im_cinza,cv2.CV_64F,3,borderType=cv2.BORDER_CONSTANT )
        im_cinza = cv2.Laplacian(im_cinza,cv2.CV_64F,4,borderType=cv2.BORDER_CONSTANT )
        im_cinza[im_cinza < 0] = 0
        #constante de ativação log about
        # incio constante_log = 100 -> 200
        im_cinza = 0 + 110*np.log(im_cinza + 1)

        im_cinza = im_cinza.astype(np.uint8)
        im_cinza = cv2.equalizeHist(im_cinza)

        windows = np.array([17,17])
        print(windows)
        half_win = np.floor(windows/2) # função escada, joga um valor quebrado para um inteiro
        half_win = half_win.astype(int) # transforma essa data em tipo inteiro
        width = im_cinza.shape[0] # Largura
        length = im_cinza.shape[1] # Altura
        print("half_win[0]: ",half_win[0])
        print("half_win[1]: ",half_win[1])
        segment = np.zeros((width-2*half_win[0]+1,length-2*half_win[1]+1), dtype=np.uint8)
        media_array = np.zeros((600,1),dtype=np.uint8)
        desvio_array = np.zeros((600,1),dtype=np.uint8)
        kur_array = np.zeros((600,1),dtype=np.uint8)
        print(segment.shape) # (511,509)
        """
        for i in range(half_win[0],width-half_win[0]+1):
            for j in range(half_win[1],length-half_win[1]+1):
                media = np.mean(im_cinza[i-half_win[0]:i+half_win[0],j-half_win[1]:j+half_win[1]])
                desvio = np.std(im_cinza[i-half_win[0]:i+half_win[0],j-half_win[1]:j+half_win[1]])
                #print("i: ",i)
                media_array[i] = media
                desvio_array[i] = desvio
        """
        for i in range( (int(length/2)-1) - half_win[0],(int(length/2)-1) + half_win[0] + 1):
            for j in range( (int(width/2)-1) - half_win[1],(int(width/2)-1) + half_win[1] +1):
                media = np.mean(im_cinza[i-half_win[0]:i+half_win[0],j-half_win[1]:j+half_win[1]])
                desvio = np.std(im_cinza[i-half_win[0]:i+half_win[0],j-half_win[1]:j+half_win[1]])
                kur = kurtosis(im_cinza[i-half_win[0]:i+half_win[0],j-half_win[1]:j+half_win[1]],fisher=False)
                kur_mean = np.mean(kur)
                print("kur: ",kur)
                kur_array[i] = kur_mean
                media_array[i] = media
                desvio_array[i] = desvio
        # extracao de valores para limiar
        #pyplot.plot(media_array)
        media_array_modf = media_array[media_array != 0]
        media_total= np.mean(media_array_modf)
        media_limiar_down= int(media_total*0.7)
        media_limiar_up= int(media_total*1.5)
        #media_limiar_down= int(media_total*0.8)
        #media_limiar_up= int(media_total*1.5)
        #print("Media total: ", media_total)
        #pyplot.plot(desvio_array)
        desvio_array_modf = desvio_array[desvio_array != 0]
        desvio_total = np.mean(desvio_array_modf)
        desvio_limiar_down = int(desvio_total*0.7)
        desvio_limiar_up = int(desvio_total*1.5)
        #desvio_limiar_down = int(desvio_total*0.8)
        #desvio_limiar_up = int(desvio_total*1.3)
        #print("Desvio total: ", desvio_total)
        kur_array_modf = kur_array[kur_array != 0]
        kur_total = np.mean(kur_array_modf)
        kur_limiar_down = kur_total*0.85
        kur_limiar_up = kur_total*1.3
        print("kur total: ", kur_total)
        print("Image number pack:",numero_fotos)

        for i in range(half_win[0],width-half_win[0]+1):
            for j in range(half_win[1],length-half_win[1]+1):
                media = np.mean(im_cinza[i-half_win[0]:i+half_win[0],j-half_win[1]:j+half_win[1]])
                desvio = np.std(im_cinza[i-half_win[0]:i+half_win[0],j-half_win[1]:j+half_win[1]])
                kur = kurtosis(im_cinza[i-half_win[0]:i+half_win[0],j-half_win[1]:j+half_win[1]],fisher=False)
                kur_mean = np.mean(kur)
                #print("i: ",i)
                if ( media_limiar_down < media and  media < media_limiar_up) \
                and ( desvio_limiar_down < desvio and desvio < desvio_limiar_up) \
                and (  kur_mean > kur_limiar_down ) :
                    segment[i-half_win[0],j-half_win[1]] = 255

        values_l = np.zeros(segment.shape[1], float)
        for j in range(segment.shape[1]):
            values_l[j] = np.sum(segment[:,j])
            values_l[j] = values_l[j]/(255)
        media = np.mean(values_l)
        print("media_values_l:",np.mean(values_l))
        for j in range(segment.shape[1]):
            if (values_l[j] < int(media)):
                #print("--->")
                segment[:,j] = np.zeros(segment.shape[0],dtype = np.uint8)

        values_w = np.zeros(segment.shape[0], float)
        for i in range(segment.shape[0]):
            values_w[i] = np.sum(segment[i,:])
            values_w[j] = values_w[j]/255
        media = np.mean(values_w)
        print("media_values_w:",np.mean(values_w))
        for i in range(segment.shape[0]):
            if (values_w[i] < int(media)):
                #print("-------->")
                segment[i,:] = np.zeros(segment.shape[1],dtype = np.uint8)

        #ordem de cores
        #azul,laranja,verde,vermelho
        #pyplot.plot(values_l) #como slider vertical
        values_l = cv2.GaussianBlur(values_l,(21,21),10)
        div = np.zeros((values_l.shape[0],1))
        h=1/5
        for i in range(0,values_l.shape[0],1):
            if i!=values_l.shape[0]-1:
                div[i] = int((values_l[i+1]-values_l[i])/h)
                print(div[i])

        min_value = np.min(div)
        result = np.where(div == min_value)
        index_minimo = result[0][0]
        print("min index: ", index_minimo)

        for j in range(0,segment.shape[0],1):
            segment[j][index_minimo]=255

        max_value = np.max(div)
        result = np.where(div == max_value)
        index_maximo = result[0][0]
        print("max index: ",index_maximo)

        for j in range(0,segment.shape[0],1):
            segment[j][index_maximo]=255

        div = cv2.GaussianBlur(div,(21,21),15)
        pyplot.plot(values_l/1000) #como slider vertical
        pyplot.plot(div) #como slider vertical

        values_w = cv2.GaussianBlur(values_w,(21,21),10)
        div = np.zeros((values_w.shape[0],1))

        h=1/5
        for i in range(0,values_w.shape[0],1):
            if i!=values_w.shape[0]-1:
                div[i] = int((values_w[i+1]-values_w[i])/h)
                print(div[i])

        min_value = np.min(div)
        result = np.where(div == min_value)
        index_minimo = result[0][0]
        print("min index: ", index_minimo)

        for i in range(0,segment.shape[1],1):
            segment[index_minimo][i]=255

        max_value = np.max(div)
        result = np.where(div == max_value)
        index_maximo = result[0][0]
        print("max index: ",index_maximo)

        for i in range(0,segment.shape[1],1):
            segment[index_maximo][i]=255

        # slider horizontal
        faixa = 0
        anterior = 0
        largura_minima = 5
        for j in range(0,segment.shape[0],1):
            for i in range(0,segment.shape[1],1):
                if segment[j][i]==255:
                    faixa = faixa + 1
                    anterior = 1
                elif segment[j][i]==0 and anterior==1:
                    # faixa fora da largura minima
                    if(faixa <= largura_minima):
                        for x in range(1,faixa+1,1):
                            segment[j][i-x]=0
                if segment[j][i]==0:
                    faixa = 0
                    anterior = 0

        # slider vertical
        faixa = 0
        anterior = 0
        altura_minima = 5
        for i in range(0,segment.shape[1],1):
            for j in range(0,segment.shape[0],1):
                if segment[j][i]==255:
                    faixa = faixa + 1
                    anterior = 1
                elif segment[j][i]==0 and anterior==1:
                    # faixa fora da largura minima
                    if(faixa <= altura_minima):
                        for x in range(1,faixa+1,1):
                            segment[j-x][i]=0
                if segment[j][i]==0:
                    faixa = 0
                    anterior = 0

        div = cv2.GaussianBlur(div,(21,21),15)

        pyplot.plot(values_w) #como slider vertical
        pyplot.plot(div) #como slider vertical

        # Configs to plot in only one page
        fig = pyplot.figure(figsize = (30,30))
        rows = 2
        columns = 4

        fig.add_subplot(rows, columns, 1)
        pyplot.imshow(im_cinza,cmap='gray')
        pyplot.title("Imagem tratada")

        fig.add_subplot(rows, columns, 2)
        pyplot.imshow(segment, cmap='gray')
        pyplot.title("Imagem segmentada")

        print("finish")

        footprint = morphology.rectangle(11,11)
        #segment2 = cv2.erode(segment,(15,15),iterations=5)
        segment2 = morphology.binary_closing(segment, footprint)

        fig.add_subplot(rows, columns, 3)
        pyplot.imshow(segment2,cmap='gray')
        pyplot.title("Imagem segmentada técnica de fechamento")

        segment3 = morphology.convex_hull_image(segment2 == 1)

        print(segment3)
        segment3 = segment3.astype(np.uint8)  #convert to an unsigned byte
        segment3*=255
        #PIL_image = PIL.Image.fromarray(np.uint8(segment3))
        #segment3 = img_as_ubyte(segment3)
        #PIL_image = PIL.Image.fromarray(numpy_image.astype('uint8'), 'RGB')
        #segment3 = PIL.Image.fromarray(segment3, "RGB")
        print(segment3)
        segment3 = cv2.resize(segment3,(500,500),cv2.INTER_CUBIC)
        #x,y,w,h = cv2.boundingRect(segment3)
        #cv2.rectangle(segment3,(x,y),(x+w,y+h),255,-1)

        fig.add_subplot(rows, columns, 4)
        pyplot.imshow(segment3,cmap='gray')
        pyplot.title("Imagem segmentada pós-processada")

        indice, bitwiseAND, bitwiseOR, img = indice_segmentacao(segment3,mypath,index)

        fig.add_subplot(rows, columns, 5)
        pyplot.imshow(img,cmap='gray')
        pyplot.title("Imagem marcada manualmente")

        fig.add_subplot(rows, columns, 6)
        pyplot.imshow(bitwiseAND,cmap='gray')
        pyplot.title("Interseção de áreas")

        fig.add_subplot(rows, columns, 7)
        pyplot.imshow(bitwiseOR,cmap='gray')
        pyplot.title("União de áreas")

        fig.add_subplot(rows, columns, 8)
        pyplot.axis([0, 10, 0, 10])
        per=85
        pyplot.text(2, 5, "Indice: " + str(indice) + "\n" \
                            "Constante log: " + "110" + "\n" \
                            "Gaussian Blur kernel: " + "(5,5)" + "\n" \
                            "Média limiar: " + "70% da média entre 150% da média " + "\n" \
                            "Desvio padrão limiar: " + "70% da média entre 150% da média" + "\n" \
                            "Kur limiar: " + "maior que " + str(per) + "%" + " da média" + "\n" \
                            "Windows size: " + "(17,17)" + "\n" \
                            "Filtro tamanho de faixa: " + "Sim" + "\n" \
                            "Tamanho de faixa vert. e hor.: " + "5" + "\n" \
                            "Binary closing footprint: " + "(11,11)" + "\n" \
            , style='italic',
            bbox={'facecolor': 'red', 'alpha': 0.5, 'pad': 15})

        pyplot.savefig( "Analise_" + mypath + "/" + "Analise_1_" + str(index) + "_left.png")
        #pyplot.show()
        pyplot.close()

        indice_all = indice_all + indice
    numero_fotos = numero_fotos + len(onlyfiles)
    print("Numero: ",numero_fotos)
    print("Indice_all: ",indice_all)
        #print("Constante Log:", constante_log)

indice_media = indice_all/numero_fotos
print("media_all: ",indice_media)
f = open("Analise_exemplos_granito.txt","a")
#reset the txt
f.writelines("@" + "D_M_fix(70 entre 150) e kur_per:" +  str(per)  + "@" + str(indice_media) + "@" + "\n")
f.close()

max_stars_repo_name: lorenzoppx/Classifica-o-de-imagens

max_issues_repo_name: lorenzoppx/Classifica-o-de-imagens

max_forks_repo_name: lorenzoppx/Classifica-o-de-imagens


text:
#  * Copyright (c) 2020-2022. Authors: see NOTICE file.
#  *
#  * Licensed under the Apache License, Version 2.0 (the "License");
#  * you may not use this file except in compliance with the License.
#  * You may obtain a copy of the License at
#  *
#  *      http://www.apache.org/licenses/LICENSE-2.0
#  *
#  * Unless required by applicable law or agreed to in writing, software
#  * distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.
import logging
from functools import cached_property
from typing import List, Optional, Union
import pathlib
import numpy as np
import orjson
from pyvips import Image as VIPSImage

from pims.api.exceptions import MetadataParsingProblem
from pims.config import get_settings
from pims.formats import AbstractFormat
from pims.formats.utils.abstract import CachedDataPath
from pims.formats.utils.checker import SignatureChecker
from pims.formats.utils.engines.vips import (
    VipsSpatialConvertor
)
from pims.formats.utils.histogram import DefaultHistogramReader
from pims.formats.utils.parser import AbstractParser
from pims.formats.utils.reader import AbstractReader
from pims.formats.utils.structures.metadata import ImageChannel, ImageMetadata, MetadataStore
from pims.formats.utils.structures.planes import PlanesInfo
from pims.formats.utils.structures.pyramid import Pyramid
from pims.processing.adapters import RawImagePixels, convert_to
from pims.processing.region import Region, Tile
from pims.utils.dtypes import dtype_to_bits
from pims.utils.iterables import ensure_list
from pims.utils.types import parse_datetime
from pims.utils.vips import bandjoin, fix_rgb_interpretation

log = logging.getLogger("pims.formats")

VIRTUAL_STACK_SLUG_SCHEMA = "virtual/stack"

def _json_load(path):
    if pathlib.Path(path).is_dir():
        return {}
    else:
        with open(path, "rb") as f:
            return orjson.loads(f.read())

def cached_json(format: Union[AbstractFormat, CachedDataPath]) -> dict:
    return format.get_cached(
        '_json', _json_load, str(format.path.resolve())
    )

class VirtualStackChecker(SignatureChecker):
    @classmethod
    def match(cls, pathlike: CachedDataPath) -> bool:
        try:
            json_data = cached_json(pathlike)
            return json_data.get("schema") == VIRTUAL_STACK_SLUG_SCHEMA
        except ValueError:
            return False

class VirtualStackParser(AbstractParser):
    def parse_main_metadata(self) -> ImageMetadata:
        image = cached_json(self.format)
        metadata = image.get("image")
        if metadata is None:
            raise MetadataParsingProblem(self.format.path)

        imd = ImageMetadata()
        imd.width = metadata.get("width")
        imd.height = metadata.get("height")
        imd.depth = metadata.get("depth")
        imd.duration = metadata.get("duration")
        imd.n_concrete_channels = metadata.get(
            "n_concrete_channels",
            metadata.get("n_intrinsic_channels", metadata.get("n_channels"))
        )
        imd.n_samples = metadata.get(
            "n_samples",
            metadata.get("n_channels_per_read", 1)
        )

        imd.pixel_type = np.dtype(metadata.get("pixel_type"))
        imd.significant_bits = dtype_to_bits(imd.pixel_type)

        for i, channel in enumerate(image.get("channels")):
            imd.set_channel(
                ImageChannel(index=i, color=channel.get("color"))
            )

        return imd

    def parse_known_metadata(self) -> ImageMetadata:
        imd = super().parse_known_metadata()

        image = cached_json(self.format)
        metadata = image.get("image")
        if metadata is None:
            raise MetadataParsingProblem(self.format.path)

        imd.description = metadata.get("description")
        imd.acquisition_datetime = parse_datetime(
            metadata.get("acquired_at")
        )

        imd.physical_size_x = metadata.get("physical_size_x")
        imd.physical_size_y = metadata.get("physical_size_y")
        imd.physical_size_z = metadata.get("physical_size_z")
        imd.frame_rate = metadata.get("frame_rate")

        for parsed_channel, channel in zip(imd.channels, image.get("channels")):
            parsed_channel.suggested_name = channel.get("suggested_name")
            parsed_channel.emission_wavelength = channel.get("emission_wavelength")
            parsed_channel.excitation_wavelength = channel.get("excitation_wavelength")

        imd.is_complete = True
        return imd

    def parse_raw_metadata(self) -> MetadataStore:
        store = super().parse_raw_metadata()
        return store

    def parse_pyramid(self) -> Pyramid:
        return super().parse_pyramid()  # TODO (?)

    def parse_planes(self) -> PlanesInfo:
        imd = self.format.main_imd
        pi = PlanesInfo(
            imd.n_concrete_channels, imd.depth, imd.duration,
            ['location'], ['U255']
        )

        image = cached_json(self.format)
        planes = image.get("planes")
        for c in range(imd.n_concrete_channels):
            for z in range(imd.depth):
                for t in range(imd.duration):
                    key = f"C{c}_Z{z}_T{t}"
                    pi.set(c, z, t, location=planes[key]["location"])

        return pi

class VirtualStackReader(AbstractReader):
    @staticmethod
    def _get_underlying_format(filepath):
        from pims.formats.utils.factories import SpatialReadableFormatFactory
        FILE_ROOT_PATH = get_settings().root
        return SpatialReadableFormatFactory(match_on_ext=True).match(
            Path(FILE_ROOT_PATH, filepath).get_spatial()
        )

    def read_thumb(self, out_width: int, out_height: int, precomputed: bool = None,
                   c: Optional[Union[int, List[int]]] = None, z: Optional[int] = None,
                   t: Optional[int] = None) -> RawImagePixels:
        bands = list()
        if c is None:
            channels = list(range(self.format.main_imd.n_channels))
        else:
            channels = ensure_list(c)

        for c in channels:
            bands.append(
                convert_to(
                    self._get_underlying_format(
                        self.format.planes_info.get(c, z, t, "location")
                    ).reader.read_thumb(out_width, out_height, precomputed),
                    VIPSImage
                )
            )

        im = bandjoin(bands)
        if c == [0, 1, 2]:
            im = fix_rgb_interpretation(im)
        return im

    def read_window(self, region: Region, out_width: int, out_height: int,
                    c: Optional[Union[int, List[int]]] = None, z: Optional[int] = None,
                    t: Optional[int] = None) -> RawImagePixels:
        bands = list()
        if c is None:
            channels = list(range(self.format.main_imd.n_channels))
        else:
            channels = ensure_list(c)

        for c in channels:
            bands.append(
                convert_to(
                    self._get_underlying_format(
                        self.format.planes_info.get(c, z, t, "location")
                    ).reader.read_window(region, out_width, out_height),
                    VIPSImage
                )
            )

        im = bandjoin(bands)
        if c == [0, 1, 2]:
            im = fix_rgb_interpretation(im)
        return im

    def read_tile(self, tile: Tile, c: Optional[Union[int, List[int]]] = None,
                  z: Optional[int] = None, t: Optional[int] = None) -> RawImagePixels:
        return self.read_window(
            tile, int(tile.width), int(tile.height), c, z, t
        )

class VirtualStackFormat(AbstractFormat):
    """
    VirtualStack Format.

    A stack of 2D images, organized to mimic up to a 5D (XYCZT) image.
    The stack is described by a JSON file with the following structure:

    ```
    {
        "schema": "virtual/stack",
        "image": {
            "width": (int),
            "height": (int),
            "depth": (int),
            "duration": (int),
            "physical_size_x": (float-nullable),
            "physical_size_y": (float-nullable),
            "physical_size_z": (float-nullable),
            "frame_rate": (float-nullable),
            "n_concrete_channels": (int),
            "n_distinct_channels": (int),
            "acquired_at": (str-datetime-nullable),
            "description": (str-nullable),
            "pixel_type": (str-pixel-type)("uint8", "uint16"),
            "significant_bits": (int),
            "n_samples": (int)
        },
        "channels": [
            {
                "index": (int),
                "suggested_name": (str-nullable),
                "emission_wavelength": (float-nullable),
                "excitation_wavelength": (float-nullable),
                "color": (str-nullable)
            }
        ],
        "instrument": {
            "microscope": {
              "model": (str-nullable)
            },
            "objective": {
              "nominal_magnification": (float-nullable),
              "calibrated_magnification": (float-nullable)
            }
        },
        "planes": {
            "C0_Z0_T0": {
              "location": (str-pims-filepath-without-root)
            },
             "C0_Z0_T1": {
              "location": (str-pims-filepath-without-root)
            },
            ...
        }
    ```
    """
    checker_class = VirtualStackChecker
    parser_class = VirtualStackParser
    reader_class = VirtualStackReader
    histogram_reader_class = DefaultHistogramReader
    convertor_class = VipsSpatialConvertor

    def __init__(self, *args, **kwargs):
        super(VirtualStackFormat, self).__init__(*args, **kwargs)
        self._enabled = True

    @classmethod
    def is_spatial(cls):
        return True

    @cached_property
    def need_conversion(self):
        return False

    @classmethod
    def is_importable(cls) -> bool:
        # We do not allow to import virtual stacks as we do for regular images !
        # As virtual stacks make references to other uploads, virtual stacks could
        # be used to get privileges on other uploads !!
        # The virtual stack file (JSON) is expected to be created by other means
        # than the regular import, and by a service who has enough rights and is
        # security-aware of access rights on images.
        return False

max_stars_repo_name: hurondp/pims

max_issues_repo_name: hurondp/pims

max_forks_repo_name: hurondp/pims


text:
import sys
import struct

import numpy as np
import scipy.ndimage as im

# reading data (up to "limit" samples) from the
# (unpacked) original  MNIST data files. two separate
# path to image and label data must be specified
def get_data(image_path, label_path, limit):
    with open(image_path, "rb") as f:
        data = f.read()
        magic, n, rows, cols = struct.unpack_from(">4i", data)
        bytes = struct.unpack_from(">{0}B".format(min(n, limit) * rows * cols), data, offset=16)
        images = np.asarray(bytes).reshape((min(n, limit), rows * cols))

    with open(label_path, "rb") as f:
        data = f.read()
        magic, n = struct.unpack_from(">2i", data)
        bytes = struct.unpack_from(">{0}B".format(min(n, limit)), data, offset=8)
        labels = np.asarray(bytes)

    return images, labels

# reading and pre-processing training data ("limit", if defined, specifies the number of samples to read)
def get_training_data(limit=sys.maxint,
                      image_path="train-images-idx3-ubyte", label_path="train-labels-idx1-ubyte",
                      normalize=False, add_ones=False, deskew=False):
    return preprocess_data(get_data(image_path, label_path, limit), normalize, add_ones, deskew)

# reading and pre-processing testing data ("limit", if defined, specifies the number of samples to read)
def get_testing_data(limit=sys.maxint,
                     image_path="t10k-images-idx3-ubyte", label_path="t10k-labels-idx1-ubyte",
                     normalize=False, add_ones=False, deskew=False):
    return preprocess_data(get_data(image_path, label_path, limit), normalize, add_ones, deskew)

# deskewing the images to achieve better predictive stability.
# scipy.ndimage.interpolation.affine_transform is used for
# the interpolation of the deskewed images' pixel values
def deskew_images(data):
    images = np.array(data) / 255.0
    count = len(images)
    summed = np.sum(images, axis=1)
    dim = int(np.sqrt(images.shape[1]))
    images = images.reshape(count, dim, dim)

    grid_x, grid_y = np.mgrid[:dim, :dim]

    mean_x = (np.sum(np.sum(images * grid_x, axis=1), axis=1) / summed).reshape(count, 1, 1)
    mean_y = (np.sum(np.sum(images * grid_y, axis=1), axis=1) / summed).reshape(count, 1, 1)

    mean_free_grid_x = np.tile(grid_x, (count, 1, 1)) - mean_x
    mean_free_grid_y = np.tile(grid_y, (count, 1, 1)) - mean_y

    var_x = (np.sum(np.sum(mean_free_grid_x ** 2 * images, axis=1), axis=1) / summed).reshape(count, 1, 1)
    cov_x_y = (np.sum(np.sum(mean_free_grid_x * mean_free_grid_y * images, axis=1), axis=1) / summed)\
        .reshape(count, 1, 1)

    alpha = cov_x_y / var_x

    center = np.array([dim, dim]) / 2.0
    result = np.zeros_like(data, dtype=np.float64)

    for i in range(count):
        affine = np.array([[1.0, 0.0], [alpha[i, 0, 0], 1.0]])
        offset = np.array([mean_x[i, 0, 0], mean_y[i, 0, 0]]) - np.dot(affine, center)
        transformed = im.interpolation.affine_transform(images[i], affine, offset=offset).ravel()
        t_min, t_max = transformed.min(), transformed.max()
        result[i, :] = (transformed - t_min) / (t_max - t_min)

    return result * 255.0

# three input pre-processing options: normalization,
# adding a column of ones (bias input), and deskewing
def preprocess_data(data, normalize, add_ones, deskew):
    X, T = data[0], data[1]

    if deskew:
        X = deskew_images(X)
    if normalize:
        X = (X - X.mean(axis=0)) / 255.0
    if add_ones:
        X = np.hstack((np.ones((len(X), 1)), X))

    return X, T

max_stars_repo_name: aakhundov/mnist-challenge

max_issues_repo_name: aakhundov/mnist-challenge

max_forks_repo_name: aakhundov/mnist-challenge

reason: import numpy,import scipy


text:
import pandas as pd
from pprint import pprint
from glob import glob
import numpy as np
import re
import csv
import sys

followUp = {}
ack = {}
nonIntimate = {}
intimate = {}
featureList = {}
questionType_DND = {}
questionType_PN = {}

discriminativeVectors = []
nonDiscriminativeVectors = []

def readHelperData():
    global followUp, ack, nonIntimate, intimate, questionType_PN, questionType_DND
    utterrances = pd.read_csv('data/misc/IdentifyingFollowUps.csv')
    disc_nondisc = pd.read_csv('data/misc/DND_Annotations.csv')
    pos_neg = pd.read_csv('data/misc/PN_Annotations.csv')

    # Discriminative/Non-discriminative annotations
    for i in xrange(len(disc_nondisc)):
        question = disc_nondisc.iloc[i]['Questions']
        qType = disc_nondisc.iloc[i]['Annotations']
        questionType_DND[question] = qType

    # Positive/Negative annotations
    for i in xrange(len(pos_neg)):
        question = pos_neg.iloc[i]['Questions']
        qType = pos_neg.iloc[i]['Annotations']
        questionType_PN[question] = qType

    for item in utterrances.itertuples():
        if item[3] == "#follow_up" and item[1] not in followUp:
            followUp[item[1]] = item[2]
        elif item[3] == "#ack" and item[1] not in ack:
            ack[item[1]] = item[2]
        elif item[3] == "#non_int" and item[1] not in nonIntimate:
            nonIntimate[item[1]] = item[2]
        elif item[3] == "#int" and item[1] not in intimate:
            intimate[item[1]] = item[2]

def readTranscript():
    global featureList
    transcriptFiles = glob(sys.argv[1] + '[0-9][0-9][0-9]_P/[0-9][0-9][0-9]_TRANSCRIPT.csv')
    for i in range(0, len(transcriptFiles)):
        t = pd.read_csv(transcriptFiles[i], delimiter='\t')
        t = t.fillna("")
        captureStarted = False
        startTime = 0.0
        endTime = 0.0
        prevQuestion = ""
        participantNo = transcriptFiles[i][-18:-15]
        for j in xrange(len(t)):
            question = re.search(".*\((.*)\)$", t.iloc[j]['value'])
            if question is not None:
                question = question.group(1)
            else:
                question = t.iloc[j]['value']
            question = question.strip()

            if t.iloc[j]['speaker'] == 'Ellie':
                if question in nonIntimate and captureStarted:
                    if (participantNo, prevQuestion) not in featureList:
                        featureList[(participantNo, prevQuestion)] = [startTime, endTime]
                    else:
                        featureList[(participantNo, prevQuestion)][1] = endTime
                    captureStarted = False

                elif question in intimate and question in questionType_DND and captureStarted:
                    if (participantNo, prevQuestion) not in featureList:
                        featureList[(participantNo, prevQuestion)] = [startTime, endTime]
                    else:
                        featureList[(participantNo, prevQuestion)][1] = endTime
                    startTime = t.iloc[j]['start_time']
                    endTime = t.iloc[j]['stop_time']
                    prevQuestion = question

                elif question in intimate and question in questionType_DND and not captureStarted:
                    startTime = t.iloc[j]['start_time']
                    endTime = t.iloc[j]['stop_time']
                    prevQuestion = question
                    captureStarted = True

                elif question in intimate and question not in questionType_DND and captureStarted:
                    if (participantNo, prevQuestion) not in featureList:
                        featureList[(participantNo, prevQuestion)] = [startTime, endTime]
                    else:
                        featureList[(participantNo, prevQuestion)][1] = endTime
                    captureStarted = False

                elif question in followUp or question in ack and captureStarted:
                    endTime = t.iloc[j]['stop_time']

            elif t.iloc[j]['speaker'] == 'Participant' and captureStarted:
                endTime = t.iloc[j]['stop_time']

def readCLM_DND():
    groupByQuestion = {}
    dFile = open('data/disc_nondisc/discriminative_CLM.csv', 'w')
    ndFile = open('data/disc_nondisc/nondiscriminative_CLM.csv', 'w')
    dWriter = csv.writer(dFile)
    ndWriter = csv.writer(ndFile)
    header = ["video", "question", "starttime", "endtime","frame", "timestamp", "confidence", "success", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9",
              "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", "x24",
              "x25", "x26", "x27", "x28", "x29", "x30", "x31", "x32", "x33", "x34", "x35", "x36", "x37", "x38", "x39",
              "x40", "x41", "x42", "x43", "x44", "x45", "x46", "x47", "x48", "x49", "x50", "x51", "x52", "x53", "x54",
              "x55", "x56", "x57", "x58", "x59", "x60", "x61", "x62", "x63", "x64", "x65", "x66", "x67", "y0", "y1", "y2",
              "y3", "y4", "y5", "y6", "y7", "y8", "y9", "y10", "y11", "y12", "y13", "y14", "y15", "y16", "y17", "y18",
              "y19", "y20", "y21", "y22", "y23", "y24", "y25", "y26", "y27", "y28", "y29", "y30", "y31", "y32", "y33",
              "y34", "y35", "y36", "y37", "y38", "y39", "y40", "y41", "y42", "y43", "y44", "y45", "y46", "y47", "y48",
              "y49", "y50", "y51", "y52", "y53", "y54", "y55", "y56", "y57", "y58", "y59", "y60", "y61", "y62", "y63",
              "y64", "y65", "y66", "y67"]
    dWriter.writerow(header)
    ndWriter.writerow(header)
    for item in featureList:
        if item[0] not in groupByQuestion:
            groupByQuestion[item[0]] = [(item[1], featureList[item])]
        else:
            groupByQuestion[item[0]].append((item[1], featureList[item]))

    for item in groupByQuestion:
        fileName = sys.argv[1] + item + '_P/' + item + '_CLM_features.txt'
        f = pd.read_csv(fileName, delimiter=', ')

        for instance in groupByQuestion[item]:
            startTime = instance[1][0]
            endTime = instance[1][1]

            startFrame = f.ix[(f['timestamp'] - startTime).abs().argsort()[:1]].index.tolist()[0]
            endFrame = f.ix[(f['timestamp'] - endTime).abs().argsort()[:1]].index.tolist()[0]
            features = f.ix[startFrame:endFrame].mean(0).tolist()
            vector = instance[1][:]
            vector += features
            vector.insert(0, instance[0])
            vector.insert(0, item)
            vector = np.asarray(vector)
            # print item, instance[0], startTime, endTime

            if questionType_DND[instance[0]] == 'D':
                dWriter.writerow(vector)
            else:
                ndWriter.writerow(vector)
    dFile.close()
    ndFile.close()

def readCLM_PN():
    groupByQuestion = {}
    pFile = open('data/pos_neg/positive_CLM.csv', 'w')
    nFile = open('data/pos_neg/negative_CLM.csv', 'w')
    pWriter = csv.writer(pFile)
    nWriter = csv.writer(nFile)
    header = ["video", "question", "starttime", "endtime","frame", "timestamp", "confidence", "success", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9",
              "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", "x24",
              "x25", "x26", "x27", "x28", "x29", "x30", "x31", "x32", "x33", "x34", "x35", "x36", "x37", "x38", "x39",
              "x40", "x41", "x42", "x43", "x44", "x45", "x46", "x47", "x48", "x49", "x50", "x51", "x52", "x53", "x54",
              "x55", "x56", "x57", "x58", "x59", "x60", "x61", "x62", "x63", "x64", "x65", "x66", "x67", "y0", "y1", "y2",
              "y3", "y4", "y5", "y6", "y7", "y8", "y9", "y10", "y11", "y12", "y13", "y14", "y15", "y16", "y17", "y18",
              "y19", "y20", "y21", "y22", "y23", "y24", "y25", "y26", "y27", "y28", "y29", "y30", "y31", "y32", "y33",
              "y34", "y35", "y36", "y37", "y38", "y39", "y40", "y41", "y42", "y43", "y44", "y45", "y46", "y47", "y48",
              "y49", "y50", "y51", "y52", "y53", "y54", "y55", "y56", "y57", "y58", "y59", "y60", "y61", "y62", "y63",
              "y64", "y65", "y66", "y67"]

    pWriter.writerow(header)
    nWriter.writerow(header)
    for item in featureList:
        if item[0] not in groupByQuestion:
            groupByQuestion[item[0]] = [(item[1], featureList[item])]
        else:
            groupByQuestion[item[0]].append((item[1], featureList[item]))

    for item in groupByQuestion:
        fileName = sys.argv[1] + item + '_P/' + item + '_CLM_features.txt'
        f = pd.read_csv(fileName, delimiter=', ')

        for instance in groupByQuestion[item]:
            startTime = instance[1][0]
            endTime = instance[1][1]

            startFrame = f.ix[(f['timestamp'] - startTime).abs().argsort()[:1]].index.tolist()[0]
            endFrame = f.ix[(f['timestamp'] - endTime).abs().argsort()[:1]].index.tolist()[0]

            features = f.ix[startFrame:endFrame].mean(0).tolist()
            vector = instance[1][:]
            vector += features
            vector.insert(0, instance[0])
            vector.insert(0, item)
            vector = np.asarray(vector)
            # print item, instance[0], startTime, endTime

            if questionType_PN[instance[0]] == 'P':
                pWriter.writerow(vector)
            else:
                nWriter.writerow(vector)
    pFile.close()
    nFile.close()

if __name__ == "__main__":
    readHelperData()
    readTranscript()
    readCLM_DND()
    readCLM_PN()


text:
"""
Functions for fine-tuning GPT2 models on Twitter conversations or any foreign data.
Includes visualization functions as well.
"""
import random
from collections import defaultdict
import os
import torch
import torch.nn.functional as F
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sn
from transformers import GPT2Tokenizer, GPT2LMHeadModel, AdamW, get_linear_schedule_with_warmup
from tqdm import tqdm
from PIL import Image
from IPython import display
#from IPython.display import display as dsp
from .inference import predict
from .label_dictionaries import classes_dict, label_dict

def train(
    dataset,
    question,
    validation_set=None,
    validation_labels=None,
    labels_dict=label_dict, #defaults to global label_dict but better to use customized
    model=GPT2LMHeadModel,
    tokenizer=GPT2Tokenizer,
    batch_size=5,
    epochs=4,
    lr=2e-5,
    max_seq_len=1024,
    warmup_steps=5000,
    gpt2_type="gpt2",
    device="cuda",
    output_dir=".",
    output_prefix="gpt2_fintune",
    save_model_on_epoch=True,
    eval_between_epochs=True,
    validation_file="validation",
    download=True,
    foreign_data=False,
    plot_loss=True,
    prompt=None,
):
    """
    Please see README for more detailed parameter comments
    Training loop for `Conversation` objects or foreign data. Hugging face's GPT2 transformer
    models
    """
    tokenizer = tokenizer.from_pretrained(gpt2_type,pad_token="<pad>", padding_side="left") if download else tokenizer
    model = model.from_pretrained(gpt2_type,pad_token_id=tokenizer.pad_token_id) if download else model
    #acc_steps = 100

    model = model.to(device)
    model.train()

    optimizer = AdamW(model.parameters(), lr=lr,weight_decay=0.0)
    scheduler = get_linear_schedule_with_warmup(
        optimizer, num_warmup_steps=warmup_steps, num_training_steps=-1
    )

    dataset = [conv for conv in dataset if len(tokenizer.encode(conv.template)) < max_seq_len] if not foreign_data else dataset
    #accumulating_batch_count = 0
    plt.ion()
    fig, (ax_accuracy, ax_loss, ax_labels) = plt.subplots(3, 1, figsize=(20,25), tight_layout=True)
    accuracy = []
    soft_accuracy = []
    validation_loss = []
    acc_loss = []
    running_loss = 0
    #label_softmax = []
    #all_model_responses = defaultdict()
    running_label_loss = {
                  " No": [],
                  " Remote": [],
                  " Unsure": [],
                  " Probably": [],
                  " Yes": [],
                  " Neutral": [],
                  " None": [],
                  " Positive": [],
                  " Defensive": [],
                  " Negative": [],
                  " Opposition": [],
                  " Discussion": [],
                  " Agreement": []
    }
    for epoch in range(epochs):

        model.train()

        random.shuffle(dataset)

        print(f"Training epoch {epoch+1}")

        for index, batch in tqdm(enumerate(dataset),ascii=True, desc="current batch"):

            batched_tensors, labels = tokenize_data(batch, tokenizer, device, foreign_data, prompt)
            outputs = model(batched_tensors, labels=labels)
            loss = outputs[0]
            loss.backward()
            #loss_val = loss.item()
            #running_loss += loss_val
            logits_last_token = outputs[1][...,-2,:][:]
            current_loss_last_token = F.cross_entropy(logits_last_token.view(-1, logits_last_token.size(-1)), labels[-1].view(-1))
            running_loss += current_loss_last_token.item()

            if (index+1)%batch_size==0 or index == len(dataset)-1:
                optimizer.step()
                scheduler.step()
                optimizer.zero_grad()
                model.zero_grad()

            if index == len(dataset)-1:
                acc_loss.append(running_loss/(index+1))
                running_loss = 0
                #print(f"loss on {index} iteration of the {epoch} epoch: {loss_val}")

        if eval_between_epochs:
            model.eval()
            model_stats = predict(validation_set,
                                    tokenizer,
                                    model,
                                    json_file_out=validation_file+f"_{epoch}",
                                    labels=validation_labels,
                                    labels_dict=labels_dict,
                                    foreign_data=foreign_data
                          )
            #model_responses = defaultdict()
            #model_responses["correct answer"] = [labels_dict["all_values"][model_stats[str(index)][2]] for index in range(len(validation_set))]
            #model_responses["1st choice"] = [model_stats[str(index)][3][0][0] for index in range(len(validation_set))]
            #model_responses["2nd choice"] = [model_stats[str(index)][3][1][0] for index in range(len(validation_set))]
            #model_responses["3rd choice"] = [model_stats[str(index)][3][2][0] for index in range(len(validation_set))]
            #all_model_responses[epoch] = model_responses
            accuracy.append(float(model_stats["accuracy"]))
            soft_accuracy.append(float(model_stats["soft_accuracy"]))
            validation_loss.append(float(model_stats["validation_loss"]))
            label_softmaxes = model_stats["label_softmaxes"]
            for label in label_softmaxes.keys():
                running_label_loss[label].append(label_softmaxes[label])
            #top_model_predict = [int(model_stats[str(index)][3][0][1]) for index in range(len(validation_set))]
            #labels_top_model_predict = [model_stats[str(index)][3][0][0] for index in range(len(validation_set))]
            #correct_answers = [labels_dict["all_values"][model_stats[str(index)][2]] for index in range(len(validation_set))]
            #hist_data = model_stats["hist_data"]
            ground_truth = [labels_dict["all_values"][model_stats[str(index)][2]] for index in range(len(validation_set))]
            model_predictions = [model_stats[str(index)][3][0][0] for index in range(len(validation_set))]
            if plot_loss:
                plot_loss_accuracy(accuracy,
                                    soft_accuracy,
                                    labels_dict["baseline_accuracy"],
                                    acc_loss,
                                    validation_loss,
                                    ax_accuracy,
                                    ax_loss,
                                    ax_labels,
                                    epoch+2,
                                    fig,
                                    running_label_loss,
                                    labels_dict=labels_dict
                )
            create_confusion_matrix(ground_truth, model_predictions, question, epoch+1, lr, output_prefix)

        if save_model_on_epoch:
            torch.save(
                model.state_dict(),
                os.path.join(output_dir, f"{output_prefix}-{epoch}.pt"),
            )
    #for stat in all_model_responses:
        #top_predicts = pd.DataFrame.from_dict(all_model_responses[stat])
        #print(f"epoch {stat+1}: ")
        #dsp(top_predicts)
    if plot_loss:
        fig.savefig(f"loss_accuracy_graph_{output_prefix}.png")
    return model

def create_batches(data, batch_size, max_seq_len):
    """batches dataset"""
    data_ = sorted(data, key=lambda x: len(x.template))
    data_ = [record for record in data_ if len(record.template) <= max_seq_len]
    data_ = [data_[index:index+batch_size] for index in range(0,len(data_), batch_size)]
    random.shuffle(data_)
    return data_

def tokenize_data(conversation, tokenizer, device, foreign_data, prompt):
    """returns two lists of tokenized torch tensors: inputs and labels"""
    tokens = tokenizer.encode(conversation.template) if not foreign_data else tokenizer.encode(conversation)
    label_check = " " + conversation.template.split(" ")[-1] if not foreign_data else " " + conversation.split(" ")[-1]
    assert len(tokenizer.encode(label_check)) == 1
    assert label_check == tokenizer.decode(tokens[-1])
    inputs = torch.LongTensor(tokens).to(device)
    #for the labels, pad conversation part of template with -100
    conv_thread = ""
    if not foreign_data:
        for tweet in conversation.thread:
            conv_thread+= f"{tweet.user_handle}: {tweet.content}\n"
        label_mask = len(tokenizer.encode(conv_thread))
        token_labels = [-100. if index < label_mask else tokens[index] for index in range(len(tokens))]
        labels = torch.LongTensor(token_labels).to(device)
    else:
        len_input = len(tokens) - len(tokenizer.encode(prompt)) -1
        token_labels = [-100. if index < len_input else tokens[index] for index in range(len(tokens))]
        labels = torch.LongTensor(token_labels).to(device)
    return inputs, labels

def plot_loss_accuracy(updated_accuracy,
                        updated_soft_accuracy,
                        base_accuracy,
                        updated_loss,
                        validation_loss,
                        ax_accuracy,
                        ax_loss,
                        ax_labels,
                        epochs,
                        fig,
                        labels_softmaxes,
                        labels_dict=None
):
    """
    plots loss and accuracy graphs, updated each epoch.
    saves plot when finetuning is done
    """
    x_axis = [count for count in range(1,epochs)]
    ax_accuracy.clear()
    ax_loss.clear()
    ax_labels.clear()
    ax_accuracy.set_ylim([0,1])
    ax_labels.set_xlabel("epoch", fontsize=15)
    ax_labels.set_ylabel("avg. softmax", fontsize=15)
    ax_loss.set_xlabel("epoch", fontsize=15)
    ax_loss.set_ylabel("loss", fontsize=15)
    ax_accuracy.set_xlabel("epoch", fontsize=15)
    ax_accuracy.set_ylabel("accuracy", fontsize=15)
    ax_labels.set_title("avg. softmax for each label")
    ax_accuracy.set_title("validation acccuracy")
    ax_loss.set_title("avg loss of final token per epoch")
    ax_accuracy.plot(x_axis, updated_accuracy, '--ro', label='hard accuracy')
    ax_accuracy.plot(x_axis, updated_soft_accuracy, '--yo', label='soft accuracy')
    base_plot = [base_accuracy for _ in range(1, epochs)]
    ax_accuracy.plot(x_axis, base_plot, '--go', label='arg. max accuracy')
    color=iter(plt.cm.rainbow(np.linspace(0,1,len(labels_softmaxes.keys()))))
    for index, key in enumerate(labels_softmaxes.keys()):
        next_color=next(color)
        curr_ax = ax_labels.plot(x_axis, labels_softmaxes[key], c=next_color, label=f"{key}")
    ax_loss.plot(x_axis, updated_loss,'--ro', label='training')
    ax_loss.plot(x_axis, validation_loss, '--bo', label='validation')
    ax_loss.legend()
    ax_accuracy.legend()
    ax_labels.legend()
    #plt.close()
    #display.clear_output(wait=True)
    #display.display(plt.gcf())

def create_confusion_matrix(
                            y_true,
                            y_pred,
                            question,
                            epoch=None,
                            lr=None,
                            output_prefix=None,
                            classes = classes_dict,
                            out_file="confusion.png"
):
    """
    creates and plots a confusion matrix given two list (ground_truths and model predictions)
    :param list y_true: list of all ground truths
    :param list y_pred: list of all model predictions
    :param dict classes: dict of classes, converts naturl language response to indexing integer
    """
    y_pred = [prediction if prediction in classes[question].keys() else "invalid" for prediction in y_pred]
    amount_classes = len(list(classes[question].keys()))

    confusion_matrix = pd.DataFrame(np.zeros(shape=[amount_classes, amount_classes]),
                                    columns=list(classes[question].keys()))
    for index in range(len(y_true)):
        target = y_true[index]

        output = y_pred[index]

        confusion_matrix[target][classes[question][output]] += 1
    sn.set(rc={'figure.figsize':(11.7,8.27)})
    sn.set(font_scale=2.0) # for label size
    plt.figure()
    confusion_matrix = confusion_matrix/confusion_matrix.sum()
    confusion_matrix.round(3)
    sn.heatmap(confusion_matrix,
                xticklabels=classes[question].keys(),
                yticklabels=classes[question].keys(),
                annot=True,
                annot_kws={"size": 16},
                fmt=".3"
    ) # font size
    #plot.figure.show()
    display.clear_output(wait=True)
    display.display(plt.gcf())
    if epoch is not None:
        plt.savefig(f"confusion_matrix_{output_prefix}_{lr}_{epoch}.png")
        #display.display(plt.gcf())
        #plt.close()
    else:
        plt.savefig(out_file)

def autolabel(rects, ax):
    """Attach a text label above each bar in *rects*, displaying its height."""
    for rect in rects:
        height = rect.get_height()
        ax.annotate('{}'.format(height),
                    xy=(rect.get_x() + rect.get_width() / 2, height),
                    xytext=(0, 3),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom')

def pil_grid(images, max_horiz=np.iinfo(int).max, file_name='image.png'):
    """returns images as a single grided image"""
    n_images = len(images)
    n_horiz = min(n_images, max_horiz)
    h_sizes = [0] * n_horiz
    v_sizes = [0] * (n_images // n_horiz) if not n_images // n_horiz else  [0] * ((n_images // n_horiz)+1)
    for index, img in enumerate(images):
        height, vert = index % n_horiz, index // n_horiz
        h_sizes[height] = max(h_sizes[height], img.size[0])
        v_sizes[vert] = max(v_sizes[vert], img.size[1])
    h_sizes, v_sizes = np.cumsum([0] + h_sizes), np.cumsum([0] + v_sizes)
    im_grid = Image.new('RGB', (h_sizes[-1], v_sizes[-1]), color='white')
    for index, img in enumerate(images):
        im_grid.paste(img, (h_sizes[index % n_horiz], v_sizes[index // n_horiz]))
    im_grid.save(file_name)
    return im_grid

max_stars_repo_name: credwood/sportsbot

max_issues_repo_name: credwood/sportsbot

max_forks_repo_name: credwood/sportsbot


text:
"""Implementation of :class:`RealField` class. """

from __future__ import print_function, division

import math

from sympy.polys.domains.field import Field
from sympy.polys.domains.simpledomain import SimpleDomain
from sympy.polys.domains.characteristiczero import CharacteristicZero
from sympy.polys.domains.mpelements import MPContext

from sympy.polys.polyerrors import DomainError, CoercionFailed
from sympy.core.numbers import Float
from sympy.utilities import public

@public
class RealField(Field, CharacteristicZero, SimpleDomain):
    """Real numbers up to the given precision. """

    rep = 'RR'

    is_RealField = is_RR = True

    is_Exact = False
    is_Numerical = True

    has_assoc_Ring = False
    has_assoc_Field = True

    _default_precision = 53

    @property
    def has_default_precision(self):
        return self.precision == self._default_precision

    @property
    def precision(self):
        return self._context.prec

    @property
    def dps(self):
        return self._context.dps

    @property
    def tolerance(self):
        return self._context.tolerance

    def __init__(self, prec=_default_precision, dps=None, tol=False):
        context = MPContext(prec, dps, tol)
        context._parent = self
        self._context = context

        self.dtype = context.mpf
        self.zero = self.dtype(0)
        self.one = self.dtype(1)

    def __eq__(self, other):
        return (isinstance(other, RealField)
           and self.precision == other.precision
           and self.tolerance == other.tolerance)

    def __hash__(self):
        return hash((self.__class__.__name__, self.dtype, self.precision, self.tolerance))

    def to_sympy(self, element):
        """Convert ``element`` to SymPy number. """
        return Float(element, self.dps)

    def from_sympy(self, expr):
        """Convert SymPy's number to ``dtype``. """
        number = expr.evalf(n=self.dps)

        if number.is_Number:
            return self.dtype(number)
        else:
            raise CoercionFailed("expected real number, got %s" % expr)

    def from_ZZ_python(self, element, base):
        return self.dtype(element)

    def from_QQ_python(self, element, base):
        return self.dtype(element.numerator) / element.denominator

    def from_ZZ_gmpy(self, element, base):
        return self.dtype(int(element))

    def from_QQ_gmpy(self, element, base):
        return self.dtype(int(element.numerator)) / int(element.denominator)

    def from_RealField(self, element, base):
        if self == base:
            return element
        else:
            return self.dtype(element)

    def from_ComplexField(self, element, base):
        if not element.imag:
            return self.dtype(element.real)

    def to_rational(self, element, limit=True):
        """Convert a real number to rational number. """
        return self._context.to_rational(element, limit)

    def get_ring(self):
        """Returns a ring associated with ``self``. """
        raise DomainError('there is no ring associated with %s' % self)

    def get_exact(self):
        """Returns an exact domain associated with ``self``. """
        from sympy.polys.domains import QQ
        return QQ

    def gcd(self, a, b):
        """Returns GCD of ``a`` and ``b``. """
        return self.one

    def lcm(self, a, b):
        """Returns LCM of ``a`` and ``b``. """
        return a*b

    def almosteq(self, a, b, tolerance=None):
        """Check if ``a`` and ``b`` are almost equal. """
        return self._context.almosteq(a, b, tolerance)

max_stars_repo_name: lidavidm/sympy

max_issues_repo_name: shashank-agg/sympy

max_forks_repo_name: shashank-agg/sympy

reason: from sympy


text:
import argparse
import os
import rosbag
import tf
import numpy as np

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("bag_file", help="input bag file")
    args = parser.parse_args()

    output_file = os.path.splitext(os.path.basename(args.bag_file))[0] + ".csv"

    bag = rosbag.Bag(args.bag_file)

    # matrix = np.empty([0, 10])
    last_est = None
    # last_pos = None
    start_time = None
    print("t, statex, goalx, viconx, statez, goalz, viconz")
    for topic, msg, t in bag.read_messages(topics=['/cf06/log1']):
        time = msg.header.stamp.secs + msg.header.stamp.nsecs / 1e9
        vals = [str(v) for v in msg.values];
        print("{}, {}".format(time, ", ".join(vals)))
        # print(time)
        # print(msg.values)
        # if start_time is None:
        #     start_time = t.to_sec()
        # print(msg)
        # if topic == "/cf06/log1":
        #     for m in msg.transforms:
        #         if m.child_frame_id == "/vicon/cf_config1/cf_config1":
        #             if last_est:
        #                 # row = np.array([])
        #                 row = np.append(row, t.to_sec() - start_time)
        #                 row = np.append(row, [m.transform.translation.x, m.transform.translation.y, m.transform.translation.z])
        #                 row = np.append(row, [last_est.translation.x, last_est.translation.y, last_est.translation.z])
        #                 quaternion = (
        #                     m.transform.rotation.x,
        #                     m.transform.rotation.y,
        #                     m.transform.rotation.z,
        #                     m.transform.rotation.w)
        #                 euler = tf.transformations.euler_from_quaternion(quaternion)
        #                 row = np.append(row, [euler[0], euler[1], euler[2]])

        #                 matrix = np.append(matrix, [row], axis=0)
        #         if m.child_frame_id == "tracker_test1":
        #             last_est = m.transform
    bag.close()

    # np.savetxt(output_file, matrix, delimiter=",", header="t,x_vicon,y_vicon,z_vicon,x_ext,y_est,z_est,roll_vicon,pitch_vicon,yaw_vicon")

max_stars_repo_name: hugokohli/SwarmInterface

max_issues_repo_name: TaoChenOSU/crazyswarm-nn

max_forks_repo_name: TaoChenOSU/crazyswarm-nn


text:
import math

import numpy as np
import pandas as pd
# import sys
# sys.path.append('C:\\Users\\Moritz Eck\\code\\fs18\\intro_ml\\shared')
import reader
from sklearn.linear_model import Ridge
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold, RepeatedKFold, train_test_split

# FILE PATH: TRAINING FILE
FILE_PATH_TRAIN = "train1a.csv"
TEST_SIZE = 0.2

# alpha parameter values
alphas = [0.1, 1.0, 10.0, 100.0, 1000.0]

# training data
data = reader.read_csv(FILE_PATH_TRAIN, show_info=False)

# drop the first column
data = pd.DataFrame.drop(data, columns='Id')

# x-values
X = pd.DataFrame(data, columns=['x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7', 'x8', 'x9', 'x10'], copy=True)
X = X.values

# y-values
Y = pd.DataFrame(data, columns=['y'], copy=True)
Y = Y['y'].values

# split into intermediate (training + validation) and test set
x_intermediate, x_test, y_intermediate, y_test = train_test_split(X, Y, test_size=TEST_SIZE, shuffle=False)

# storing the evaluation scores
eval = []

for entry in alphas:
    # creating the 10-fold split
    ten_fold = KFold(n_splits=10, shuffle=False)
    repeat_ten_fold = RepeatedKFold(n_splits=10, n_repeats=10)

    # fold counter
    i = 1
    # create model
    ridge = Ridge(alpha=entry, fit_intercept=True, normalize=False,
                  copy_X=True, max_iter=None, tol=1e-5, solver='auto')
    # errors
    errors = []

    for train_ind, val_ind in repeat_ten_fold.split(X, Y):
    # for train_ind, val_ind in repeat_ten_fold.split(x_intermediate, y_intermediate):
        # x & y training set (no separated test set)
        x_train = X[train_ind, : ]
        y_train = Y[train_ind]

        # # x & y training set with separated test set
        # x_train = x_intermediate[train_ind, : ]
        # y_train = y_intermediate[train_ind]

        # x & y validation set (no separated test set)
        x_val = X[val_ind, : ]
        y_val = Y[val_ind]

        # # x & y validation set with separated test set
        # x_val = x_intermediate[val_ind, : ]
        # y_val = y_intermediate[val_ind]

        # train the model
        ridge.fit(x_train, y_train)

        # evaluate with validation set
        y_val_predictions = ridge.predict(x_val)

        # compute root mean square error
        rmse = math.sqrt(mean_squared_error(y_val, y_val_predictions))
        errors.append(rmse)

        # printing alpha, fold number and root mean squared error
        print("Alpha:\t", str(entry)+"\t", "Fold:", str(i)+"\t", "RMSE:\t", rmse)

        # increment fold counter
        i += 1

    # compute mean errors for rmse of each alpha
    eval.append(np.mean(errors))
    print()

# print out the rmse averages for all 5 lambdas
for ind in range(0,5):
    print("Alpha:\t", alphas[ind], "\t\t", "Average RMSE:\t", eval[ind])

# writing the result to file
# data = {"Result":eval}
# alpha_rmnse_output = pd.DataFrame(data=eval)
# print(alpha_rmnse_output.head(5))
# alpha_rmnse_output.to_csv("alpha_mean_rmse_ecm_3.csv", sep=',', index=False, header=False)