text:
import numpy as np
import cv2
import glob
import os
import pickle
import matplotlib.pyplot as plt

# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
nx = 9
ny = 6
objp = np.zeros((ny*nx, 3), np.float32)
objp[:,:2] = np.mgrid[0:nx, 0:ny].T.reshape(-1, 2)

# Arrays to store object points and image points from all the images.
objpoints = []  # 3d points in real world space
imgpoints = []  # 2d points in image plane.

# Make a list of calibration images
images = glob.glob('camera_cal/calibration*.jpg')

# Step through the list and search for chessboard corners
for fname in images:
    img = cv2.imread(fname)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # Find the chessboard corners
    ret, corners = cv2.findChessboardCorners(gray, (nx, ny), None)

    # If found, add object points, image points
    if ret is True:
        objpoints.append(objp)
        imgpoints.append(corners)

        # Draw and display the corners
        img = cv2.drawChessboardCorners(img, (nx, ny), corners, ret)
        # cv2.imshow('img', img)
        # cv2.waitKey(500)

cv2.destroyAllWindows()

ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)

camera_params = {"mtx": mtx, "dist": dist}
print(mtx)
print(dist)

params_path = 'output/camera_params.pickle'

with open(params_path, 'wb') as f:
    # Pickle the 'data' dictionary using the highest protocol available.
    pickle.dump(camera_params, f, pickle.HIGHEST_PROTOCOL)

with open(params_path, 'rb') as f:
    # The protocol version used is detected automatically, so we do not
    # have to specify it.
    data = pickle.load(f)
    print(data["mtx"])
    print(data["dist"])

for fname in images:
    img = cv2.imread(fname)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    dst = cv2.undistort(img, mtx, dist, None, mtx)
    # cv2.imshow('img', dst)
    # cv2.waitKey(500)
    basefname = os.path.basename(fname)
    new_name = os.path.join('output_images', basefname)
    cv2.imwrite(new_name, dst)

cv2.destroyAllWindows()


text:
import zlib
from typing import List, Iterator, Dict, Iterable
from operator import itemgetter

import pandas as pd
from scipy.optimize import curve_fit

from utils.misc import depreciated

def calculate_repetition(text: str, method: str = 'DEFLATE') -> float:
    b_str = bytes(text, encoding='utf-8')
    if method == 'DEFLATE':
        # default level = 6
        res = zlib.compress(b_str)
    else:
        raise NotImplementedError("Other methods not supported yet")
    ratio = len(res) / len(b_str)
    return 1 - ratio

def get_words(s: str) -> Iterator[str]:
    it = iter(s)
    lst = []
    for c in it:
        if c in ' -,\n()"':
            yield ''.join(lst)
            lst = []
        else:
            lst.append(c)

def get_sorted_bag_of_words(words: Iterable[str]) -> Dict[str, str]:
    bow = {}
    for word in words:
        word = word.lower()
        bow[word] = bow.get(word, 0) + 1
    sorted_bow = dict(sorted(bow.items(), key=itemgetter(1), reverse=True))
    return sorted_bow

@depreciated(reason="Use get_bow_dataframe() for better perf")
def convert_bow_to_dataframe(sorted_bow: Dict[str, str]) -> pd.DataFrame:
    lst = []
    for r, (k, v) in enumerate(sorted_bow.items()):
        lst.append({
            'freq': v,
            'rank': r + 1,
            'word': k,
        })
    df = pd.DataFrame(lst)
    return df

def get_bow_dataframe(words: Iterable[str]) -> pd.DataFrame:
    bow = {}
    for word in words:
        word = word.lower()
        bow[word] = bow.get(word, 0) + 1
    sorted_bow = sorted(bow.items(), key=itemgetter(1), reverse=True)
    lst = []
    for r, (k, v) in zip(range(1, len(bow) + 1), sorted_bow):
        lst.append({
            'freq': v,
            'rank': r,
            'word': k,
        })
    df = pd.DataFrame(lst)
    return df

def zipf_getter(N: int):
    def zipf(k, s):
        # TODO: optimize using a stdlib function
        return (1 / k ** s) / sum(1 / n ** s for n in range(1, N + 1)) * N
    return zipf

def fit_to_zipf(df: pd.DataFrame) -> (float, float):
    """

    Args:
        df: A bag of words in a DataFrame format.

    Returns:
        s: Fitted value of the parameter (?) s of the zipf function.
        pcov: Covariant of the fitting (?) curve_fit.pcov
    """
    N = sum(df.get('freq'))
    zipf = zipf_getter(N)
    popt, pcov = curve_fit(zipf, df.get('rank'), df.get('freq'))
    return popt[0], pcov

def plot_line_fit_to_zipf(df: pd.DataFrame, s: float) -> None:
    N = sum(df.get('freq'))
    zipf = zipf_getter(N)
    plt.plot(df.get('rank'), df.get('freq'), 'b-', label='data')
    plt.plot(df.get('rank'), zipf(df.get('rank'), s), 'r-',
             label='fit: s=%5.3f' % s)
    # plt.plot(df.get('rank'), zipf(df.get('rank'), 1.07), 'g--',
    #          label='ctrl: s=1.07')
    plt.legend()
    plt.show()
    return

def plot_bar_comp_to_zipf(df: pd.DataFrame, s: float) -> None:
    N = sum(df.get('freq'))
    zipf = zipf_getter(N)
    L = 10
    plt.bar(df.get('word')[:L], df.get('freq')[:L])
    plt.bar(df.get('word')[:L], zipf(df.get('rank')[:L], s))
    plt.show()

if __name__ == '__main__':
    import re
    with open('tests/data/G.U.Y.txt', 'r', encoding='utf-8') as f:
        lines = f.readlines()
    cleaned_lines = [line for line in lines if re.match(f'^\[.*]\\n?$', line) is None]
    cleaned_text = ''.join(cleaned_lines)
    # print(cleaned_text)
    print(calculate_repetition(cleaned_text))

ext: py

max_stars_repo_name: FuckBrains/slap

max_issues_repo_name: FuckBrains/slap

max_forks_repo_name: FuckBrains/slap

reason: from scipy


text:
import numpy as np

from .frames import Frame
from .local import to_qsw
from .orient import Orientation
from .center import Center
from ..propagators.base import AnalyticalPropagator

class LagrangePropagator(AnalyticalPropagator):
    """Propagate the Lagrange point of body1/body2 in a frame centred on body2 in QSW"""

    def __init__(self, frame1, body2, number):
        self.frame1 = frame1
        self.body2 = body2
        self.number = number

        if not (1 <= number <= 5):  # pragma: no cover
            raise ValueError(f"Unknown Lagrange point : {number}")

    def copy(self):  # pragma: no cover
        return self.__class__(self.frame1, self.body2)

    def propagate(self, date):
        orb = self.body2.propagate(date).copy(frame=self.frame1, form="cartesian")
        offset = np.zeros(6)
        r = np.linalg.norm(orb[:3])

        if self.number in (1, 2):
            offset[0] = r * (self.body2.m / (3 * self.frame1.center.body.m)) ** (1 / 3)
            if self.number == 1:
                offset *= -1
        elif self.number == 3:
            offset[0] = r * (5 * self.body2.m / (12 * self.frame1.center.body.m) - 2)
        elif self.number in (4, 5):
            offset[0] = -r / 2
            offset[1] = r * np.sqrt(3) / 2
            if self.number == 5:
                offset[1] *= -1
        else:  # pragma: no cover
            raise ValueError(f"Unknown Lagrange point : {number}")

        return offset

class LagrangeOrient(Orientation):
    def __init__(self, frame1, body2):

        name = f"{frame1.center.body.name}{body2.name}Lagrange"
        super().__init__(name)

        self.frame1 = frame1
        self.body2 = body2

        mtd = f"{name}_to_{frame1.orientation.name}"
        setattr(Orientation, mtd, self._to_parent)
        frame1.orientation + self

    def _to_parent(self, date):
        orb = self.body2.propagate(date).copy(frame=self.frame1, form="cartesian")
        return to_qsw(orb).T, None

def lagrange(frame1, frame2, number, name=None, orientation=None):
    """Create a reference frame centred on Lagrange points

    Args:
        frame1 (Frame) : Most massive object frame
        frame2 (Frame) : Less massive object frame
        number (int) : Lagrange point number (1 for L1, 2 for L2, ...)
        name (str) : Name of the frame created.
        orientation (Orientation): If ``None``, the created frame will be sinodic (i.e. oriented in QSW relative to frame2)

    Return:
        Frame : Centred on the specified Lagrange point.
    """

    c_name = f"{frame1.center.body.name}{frame2.center.body.name}L{number}"

    if name is None:  # pragma: no cover
        name = c_name

    # The offset of the Lagrange point is provided by the propagator, in a frame
    # centred on body2, and oriented in QSW
    l_orient = LagrangeOrient(frame1, frame2.center.body)
    l_prop = LagrangePropagator(frame1, frame2.center.body, number)
    l_center = Center(c_name)
    l_center.add_link(frame2.center, l_orient, l_prop)

    if orientation is None:
        orientation = l_orient

    return Frame(name, orientation, l_center)

max_forks_repo_name: galactics/space-api

reason: import numpy


text:
from collections import namedtuple
import cv2 as cv
from matplotlib import pyplot as plt
import numpy as np
from numba import jit
import time

_MATCHING_THRESHOLD = 0.9

Pattern = namedtuple('Pattern', ['piece', 'mask', 'name', 'threshold'])

def build_pattern(file_path, name=None, shape=None, circle_mask=False,
                  threshold=_MATCHING_THRESHOLD):
    name = name or file_path
    img = cv.imread(file_path, cv.IMREAD_UNCHANGED)
    if shape:
        img = cv.resize(img, shape)
    (_, mask) = create_mask(img)
    if circle_mask:
        # circle mask removes confusion around cursor corners
        # [el][el]
        (_, cmask) = create_circle_mask(img)
        mask = cv.bitwise_and(cmask, mask)
    return Pattern(piece=img[:, :, :3], mask=mask, name=name,
                   threshold=threshold)

def create_circle_mask(img):
    h, w, _ = img.shape
    mask = np.zeros((h, w), dtype=np.uint8)
    cx, cy = h//2, w//2
    r = int(h/2 * 0.5)
    return None, cv.circle(mask, (cx, cy), r, 255, thickness=cv.FILLED)

def create_mask(transparent_img, thresh=cv.THRESH_BINARY):
    return cv.threshold(transparent_img[:, :, 3], 0, 255, thresh)

@jit
def find_piece(img, pattern, coeff=cv.TM_CCOEFF_NORMED):
    # matchTempalte for full image is a bit slow because piece pattern is tried
    # outside of the regular grid boundaries
    res = cv.matchTemplate(img, pattern.piece, coeff, mask=pattern.mask)
    loc = np.where(res >= pattern.threshold)
    return [pt for pt in zip(*loc[::-1])]

def find_board_rectangle(img, head_img, width, height, width_adjustment,
                         height_adjustment):
    img = img.copy()
    # Apply template Matching
    res = cv.matchTemplate(img, head_img, cv.TM_CCOEFF)
    min_val, max_val, min_loc, max_loc = cv.minMaxLoc(res)
    # If the method is TM_SQDIFF or TM_SQDIFF_NORMED, take minimum
    top_left = (max_loc[0] - width_adjustment, max_loc[1] + height_adjustment)
    bottom_right = (top_left[0] + width, top_left[1] + height)
    return (top_left, bottom_right, res)

def plot_result(img, top_left, bottom_right, res):
    cv.rectangle(img, top_left, bottom_right, 255, 2)
    plt.subplot(121), plt.imshow(res, cmap='gray')
    plt.title('Matching Result'), plt.xticks([]), plt.yticks([])
    plt.subplot(122), plt.imshow(img, cmap='gray')
    plt.title('Detected Point'), plt.xticks([]), plt.yticks([])
    plt.suptitle('cv.TM_CCOEFF')
    plt.show()

def plot_pieces(img, piece_img):
    w, h = piece_img.shape[0], piece_img.shape[1]
    pieces = find_piece(img, piece_img)
    plot_rectangles(img.copy(), pieces, w, h)

def draw_rectangles(img, top_lefts, w, h, color=255):
    img = img.copy()
    for tl in top_lefts:
        cv.rectangle(img, tl, (tl[0] + w, tl[1] + h), color, 2)
    return img

def plot_rectangles(img, top_lefts, w, h, color=255):
    img = draw_rectangles(img, top_lefts, w, h, color)
    plt.imshow(img)
    plt.show()

max_stars_repo_name: vjeranc/puzpirobot

max_issues_repo_name: vjeranc/puzpirobot

max_forks_repo_name: vjeranc/puzpirobot

reason: import numpy,from numba


text:
from typing import Tuple

import numpy as np
import pyqtgraph.opengl as gl
from PyQt5 import QtGui

from pedrec.models.constants.skeleton_pedrec import SKELETON_PEDREC_LIMB_COLORS, SKELETON_PEDREC, SKELETON_PEDREC_JOINT_COLORS, \
    SKELETON_PEDREC_JOINTS

def add_grid(view: gl.GLViewWidget, size: QtGui.QVector3D = QtGui.QVector3D(3, 3, 1)):
    gx = gl.GLGridItem(size=size)
    gx.rotate(90, 0, 1, 0)
    gx.translate(-1.5, 0, 0)
    view.addItem(gx)
    gy = gl.GLGridItem(size=size)
    gy.rotate(90, 1, 0, 0)
    gy.translate(0, -1.5, 0)
    view.addItem(gy)
    gz = gl.GLGridItem(size=size)
    gz.translate(0, 0, -1.5)
    view.addItem(gz)

def get_limb_positions(skeleton: np.array, min_score: float = 0.3) -> Tuple[np.ndarray, np.ndarray]:
    limbs = []
    colors = []

    # create the coordinate list for the lines
    for limb_idx, limb in enumerate(SKELETON_PEDREC):

        if skeleton[limb[0], 3] < min_score or skeleton[limb[1], 3] < min_score:
            continue
        for i in range(2):
            limbs.append(skeleton[limb[i], :3])
        limbs.append([None, None, None])
        color = SKELETON_PEDREC_LIMB_COLORS[limb_idx].rgba_float_list
        colors.append(color)
        colors.append(color)
        colors.append(color)
    return np.array(limbs), np.array(colors)

def get_joint_positions(skeleton: np.array, min_score: float = 0.3) -> Tuple[np.ndarray, np.ndarray]:
    joints = []
    colors = []
    for joint_idx, joint in enumerate(SKELETON_PEDREC_JOINTS):
        if skeleton[joint_idx, 3] < min_score:
            continue
        joints.append(skeleton[joint_idx, :3])
        colors.append(SKELETON_PEDREC_JOINT_COLORS[joint_idx].rgba_float_list)
    return np.array(joints), np.array(colors)

max_stars_repo_name: noboevbo/PedRec

max_issues_repo_name: noboevbo/PedRec

max_forks_repo_name: noboevbo/PedRec


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm
import warnings
warnings.filterwarnings("ignore")

# load train and test features
train_df = pd.read_csv("./train_features.csv")
test_df = pd.read_csv("./test_features.csv")

data_df = pd.concat([train_df,test_df],axis=0,ignore_index=True)

group_feats = []

# top-k features
features = ['median','xr','mean','Quartile_75','std','form_factor','Quartile_25','Quartile','Skewness','par']

# adaptive feature aggregation and crosses
for f in tqdm(features):
    data_df['cavitation_{}_median'.format(f)] = data_df.groupby(['valve_opening','upstream_pressure'])[f].transform('median')
    data_df['cavitation_{}_mean'.format(f)] = data_df.groupby(['valve_opening','upstream_pressure'])[f].transform('mean')
    data_df['cavitation_{}_max'.format(f)] = data_df.groupby(['valve_opening','upstream_pressure'])[f].transform('max')
    data_df['cavitation_{}_min'.format(f)] = data_df.groupby(['valve_opening','upstream_pressure'])[f].transform('min')

    group_feats.append('cavitation_{}_median'.format(f))
    group_feats.append('cavitation_{}_mean'.format(f))
    group_feats.append('cavitation_{}_max'.format(f))
    group_feats.append('cavitation_{}_min'.format(f))

for f1 in tqdm(features + group_feats):
    for f2 in features + group_feats:
        if f1 != f2:
            colname1 = '{}_{}_ratio'.format(f1,f2)
            data_df[colname1] = data_df[f1].values/data_df[f2].values
            colname2 = '{}_{}_subtraction'.format(f1,f2)
            data_df[colname2] = data_df[f1].values-data_df[f2].values
data_df = data_df.fillna(method='bfill')

del data_df['valve_opening']
del data_df['upstream_pressure']
train_count = train_df.shape[0]
train_df = data_df[:train_count].copy().reset_index(drop=True)
test_df = data_df[train_count:].copy().reset_index(drop=True)

train_df.to_csv("./train_feature_transform.csv",index=False)
test_df.to_csv("./test_feature_transform.csv",index=False)


text:
### Importing the necessary modules
import numpy as np
import math
from xclib.data import data_utils
from pathlib import Path
import os
import csv
from scipy.sparse import csr_matrix
import matplotlib.pyplot as plt
import time
import copy

### Getting all the directories required
# To get the current directory
current_directory = os.path.dirname(os.path.abspath(__file__))

# To get the directory of the files -> Assuming that the folder containing data will be in the current directory only!
train_x_path = os.path.join(current_directory, 'ass3_parta_data/ass3_parta_data/train_x.txt')
train_y_path = os.path.join(current_directory, 'ass3_parta_data/ass3_parta_data/train_y.txt')

test_x_path = os.path.join(current_directory, 'ass3_parta_data/ass3_parta_data/test_x.txt')
test_y_path = os.path.join(current_directory, 'ass3_parta_data/ass3_parta_data/test_y.txt')

valid_x_path = os.path.join(current_directory, 'ass3_parta_data/ass3_parta_data/valid_x.txt')
valid_y_path = os.path.join(current_directory, 'ass3_parta_data/ass3_parta_data/valid_y.txt')

### Reading the sparse features and storing them correspondingly
features_train = data_utils.read_sparse_file(train_x_path).toarray().astype(np.int)
features_test = data_utils.read_sparse_file(test_x_path).toarray().astype(np.int)
features_valid = data_utils.read_sparse_file(valid_x_path).toarray().astype(np.int)

### Reading the class values
class_train = []
class_test = []
class_valid = []

with open(train_y_path,'r') as f:
	reader = csv.reader(f)
	for row in reader:
		class_train.append(row[0])

with open(test_y_path,'r') as f:
	reader = csv.reader(f)
	for row in reader:
		class_test.append(row[0])

with open(valid_y_path,'r') as f:
	reader = csv.reader(f)
	for row in reader:
		class_valid.append(row[0])

class_train = np.array(class_train).astype(np.int)
class_test = np.array(class_test).astype(np.int)
class_valid = np.array(class_valid).astype(np.int)

### Different useful functions for the decision tree formation

## Function for calculating the Entropy
def entropy(V):

	sum_1 = np.count_nonzero(V)
	prob_1 = (1.0 * sum_1)/len(V)
	prob_0 = 1. - prob_1

	# If any of the probabilities are at the extremes then the Entropy is 0
	if prob_0==1 or prob_1==1:
		sumVal = 0.0
	else:
		sumVal = - ((prob_0 * math.log2(prob_0)) + (prob_1 * math.log2(prob_1)))
	return sumVal, prob_0, prob_1

## Function to get the mutual information
def mutual(Y,X,ind):

	## Splitting as per the median
	X_ind = X[:,ind]

	# To ignore the cases of a variable which is just hidden everywhere
	sumTemp = np.count_nonzero(X_ind)
	if sumTemp==0:
		return 0.0,0

	# Getting the Median
	med_ind = np.median(X_ind)

	## We consider the elements <median as 0 and >=median as 1
	X_new = []
	Y_1 = []
	Y_0 = []

	for i in range(X_ind.shape[0]):
		if (X_ind[i] <= med_ind):
			X_new.append(0)
			Y_0.append(Y[i])
		else:
			X_new.append(1)
			Y_1.append(Y[i])

	# To take into consideration the case where all the elements get classified into one sub-division. In that case, the mutual information will be 0 (Kinda trivial)
	if len(Y_0)==0 or len(Y_1)==0:
		return 0.,med_ind

	## Calculating the probabilities
	H_b, prob_b_0, prob_b_1 = entropy(X_new)

	# Getting the Entropy of A for=> I(A,B) = H(A) - H(A|B)
	H_a = entropy(Y_1 + Y_0)[0]

	return (H_a - (prob_b_0 *entropy(Y_0)[0] + prob_b_1 * entropy(Y_1)[0])), med_ind

## Function to compute the best mutual information at a node
def best_expansion(Y,X):

	## Vectorizing the code
	vfunc = np.vectorize(mutual, excluded=['Y','X'])
	indices = list(range(0,X.shape[1]))
	maxOut = vfunc(Y = Y,X = X, ind= indices)[0]

	maxVal = maxOut.max()
	maxInd = maxOut.argmax()
	maxMed = mutual(Y,X,maxInd)[1]

	return maxVal, maxMed, maxInd

### Getting the class for the Trees
class Node:

	## Use isGrown to indicate that the node can have children now
	def __init__(self, Y, X, ind, med, isGrown, isLeaf, parent, Y_test, X_test, Y_valid, X_valid):

		self.left = None
		self.right = None
		self.Y = Y
		self.X = X
		self.ind = ind
		self.med = med
		self.isGrown = isGrown
		self.isLeaf = isLeaf
		self.parent = parent

		## Adding the test and valiation data
		self.Y_test = Y_test
		self.X_test = X_test
		self.Y_valid = Y_valid
		self.X_valid = X_valid

	## Printing the Tree for debugging purposes
	def printTree(self):
		if self.left!=None and self.right!=None:
			return("[" + str(self.ind) + "-" + self.left.printTree() + self.right.printTree() +"]")
		elif self.left!=None:
			return("[" + str(self.ind) + "-" + self.left.printTree() + "[]" +"]")
		elif self.right!=None:
			return("[" + str(self.ind) + "-" + self.right.printTree() + "[]" +"]")
		else:
			return("[" + str(self.ind) + "-" + "[]" + "[]" +"]")

	## Getting the Height of the tree:
	def height(self):
		if self.left!=None and self.right!=None:
			return 1 + max(self.left.height(), self.right.height())
		elif self.left!=None:
			return 1 + self.left.height()
		elif self.right!=None:
			return 1 + self.right.height()
		else:
			return 0

	## Getting the Number of Nodes of the tree:
	def total_nodes(self):
		if self.left!=None and self.right!=None:
			return 1 + (self.left.total_nodes()) + (self.right.total_nodes())
		elif self.left!=None:
			return 1 + (self.left.total_nodes())
		elif self.right!=None:
			return 1 + (self.right.total_nodes())
		else:
			return 1

	## Defining a function to get one-step accuracy. Returns 1 for true and 0 for false
	def step_accuracy(self, y, x):

		## The current Node is Grown
		if self.isGrown:

			# The node is a leaf now!
			if self.isLeaf!=-1:
				if y==self.isLeaf:
					return 1
				else:
					return 0

			# The node is not a leaf now!
			else:

				# Move to the left!
				if (x[self.ind] <= self.med):
					return self.left.step_accuracy(y, x)
				# Move to the Right!
				else:
					return self.right.step_accuracy(y, x)

		## The current Node is not grown and so it is a leaf in the given limited scenario
		else:

			# The majority value will be the median. And in the case of equal 0 and 1 -> we take the majority value as 0 (rather than the 0.5)
			y_pred = math.floor(np.median(self.Y))
			if (y==y_pred):
				return 1
			else:
				return 0

	## Defining an overall accuracy function
	def overall_accuracy(self, Y, X):
		m = Y.shape[0]
		sumVal = 0.
		for i in range(m):
			sumVal += self.step_accuracy(Y[i], X[i,:])
		return sumVal/m

	## Getting the accuracies at a single node
	def accuracy_node(self, data_status):

		# Training data
		if data_status==0:
			if self.Y.shape[0]!=0:
				nz_count = np.count_nonzero(self.Y)
				z_count = self.Y.shape[0] - nz_count
				return max(((1. * nz_count)/(self.Y.shape[0])),((1. * z_count)/(self.Y.shape[0])))
			else:
				return 0.

		# Test Data
		elif data_status==1:
			if self.Y_test.shape[0]!=0:
				nz_count = np.count_nonzero(self.Y_test)
				z_count = self.Y_test.shape[0] - nz_count
				return max(((1. * nz_count)/(self.Y_test.shape[0])),((1. * z_count)/(self.Y_test.shape[0])))
			else:
				return 0.

		# Validation data:
		else:
			if self.Y_valid.shape[0]!=0:
				nz_count = np.count_nonzero(self.Y_valid)
				z_count = self.Y_valid.shape[0] - nz_count
				return max(((1. * nz_count)/(self.Y_valid.shape[0])),((1. * z_count)/(self.Y_valid.shape[0])))
			else:
				return 0.

	## Defining a one step insert function, increasing the Height of the tree by one
	def insert(self):

		## The current Node is not grown
		if not self.isGrown:

			# Number of 1's in the Class list
			nonzeroCount = np.count_nonzero(np.array(self.Y))

			# All the elements are Zero
			if nonzeroCount==0:

				# Making it a zero leaf
				self.isLeaf=0
				self.isGrown= True

			# All the elements are Ones
			elif nonzeroCount==len(self.Y):

				# Making it a One leaf
				self.isLeaf=1
				self.isGrown=True

			else:
				# Obtaining the best choice
				maxMI, maxMed, maxInd = best_expansion(self.Y, self.X)

				# Creating the lists for the left and right children respectively
				Y_left = []
				Y_right = []
				Y_left_test = []
				Y_right_test = []
				Y_left_valid = []
				Y_right_valid = []

				X_left = []
				X_right = []
				X_left_test = []
				X_right_test = []
				X_left_valid = []
				X_right_valid = []

				# Getting the X array for the given index
				X_ind = self.X[:,maxInd]

				if self.Y_test.shape[0]!=0:
					X_ind_test = self.X_test[:,maxInd]
				if self.Y_valid.shape[0]!=0:
					X_ind_valid = self.X_valid[:,maxInd]

				# Storing the corresponding Train matrices
				for i in range(X_ind.shape[0]):
					if (X_ind[i]<= maxMed):
						Y_left.append(self.Y[i])
						X_left.append(self.X[i,:])
					else:
						Y_right.append(self.Y[i])
						X_right.append(self.X[i,:])

				if self.Y_test.shape[0]!=0:
					# Storing the corresponding Test matrices
					for i in range(X_ind_test.shape[0]):
						if (X_ind_test[i]<= maxMed):
							Y_left_test.append(self.Y_test[i])
							X_left_test.append(self.X_test[i,:])
						else:
							Y_right_test.append(self.Y_test[i])
							X_right_test.append(self.X_test[i,:])

				if self.Y_valid.shape[0]!=0:
					# Storing the corresponding Train matrices
					for i in range(X_ind_valid.shape[0]):
						if (X_ind_valid[i]<= maxMed):
							Y_left_valid.append(self.Y_valid[i])
							X_left_valid.append(self.X_valid[i,:])
						else:
							Y_right_valid.append(self.Y_valid[i])
							X_right_valid.append(self.X_valid[i,:])

				# Marking the current node as grown and marking it's indices
				self.isGrown=True

				## Checking for the case where none of the Two Yi's come to be empty
				if len(Y_left)!=0 and len(Y_right)!=0:

					self.ind = maxInd
					self.med = maxMed

					# Creating the new children
					self.left = Node(np.array(Y_left), np.array(X_left), -1, -1, False, -1, copy.copy(self), np.array(Y_left_test), np.array(X_left_test), np.array(Y_left_valid), np.array(X_left_valid))
					self.right = Node(np.array(Y_right), np.array(X_right), -1, -1, False, -1, copy.copy(self), np.array(Y_right_test), np.array(X_right_test), np.array(Y_right_valid), np.array(X_right_valid))
					# Bug fix!
					self.left.parent = self
					self.right.parent = self

				## If no partition is happening such that the two y's get non-empty, then we take the majority value to the given node
				else:
					self.isLeaf = math.floor(np.median(self.Y))

		## If the current Node is grown. So we will need to traverse further!
		else:

			## Checking if it is an actual leaf in the overall tree
			if (self.isLeaf==-1):
				# Inserting at the left and right child, thus increasing the Height
				self.left.insert()
				self.right.insert()

	### A function to get to the root of the tree given any node
	def get_root(self):

		## Checking if it is a root
		if self.parent==None:
			return self
		else:
			return self.parent.get_root()

	### A function to update the root as we go upwards
	def get_updated_root(self):

		if self.parent==None:
			return self
		else:

			## Getting the right or left child status
			if (self.ind == self.parent.left.ind):
				self.parent.left = self
				return self.parent.get_updated_root()
			else:
				self.parent.right = self
				return self.parent.get_updated_root()

	### A function to prune a node
	def prune_step(self):

		selfCopy = copy.copy(self)

		self.left.parent = selfCopy
		self.right.parent = selfCopy

		self.left=None
		self.right=None
		self.isLeaf=-1
		self.isGrown=False

	###

	## Function to get the best prune
	def get_best_prune(self, orig_accA, Y,X):

		### Making shallow copies so that the changes are not reflected
		self_old = copy.copy(self)
		self_new = copy.copy(self)
		self_temp = copy.copy(self)

		### Checking if it is a leaf
		if (self_old.left==None and self_old.right==None) or self_old.isGrown==False or self_old.isLeaf!=-1:
			return self.accuracy_node(2), self, 0
		else:
			left_acc, subTree0, status_left = self.left.get_best_prune(orig_accA, Y,X)
			right_acc, subTree1, status_right = self.right.get_best_prune(orig_accA, Y,X)

			self_acc = self.accuracy_node(2)

			## Checking if the best tree has already been obtained
			if status_left==-1:
				return left_acc, subTree0, -1
			elif status_right==-1:
				return right_acc, subTree1, -1

			# If the given node is no better than it's chldren. We move upawrds
			elif self_acc < max(left_acc, right_acc):
				return self.accuracy_node(2), self, 0

			# If the given node is better than it's chldren, we get out from there
			else:
				## Taking a copy of the parent since it was getting changed!
				self_parent = copy.copy(self_old.parent)

				# Knowing if it is a left child or a right child
				if (self_old.ind==self_parent.left.ind):
					isLeft=True
				else:
					isLeft=False

				self_temp.prune_step()
				if isLeft:
					self_parent.left=self_temp
				else:
					self_parent.right=self_temp

				self_new = copy.copy(self_parent.get_updated_root())

				return self_acc, self_new, -1

	'''
	## Function to prune all the nodes and get the corresponding accuracies
	def prune_all(self, Y, X):

		### Making shallow copies so that the changes are not reflected
		self_old = copy.copy(self)
		self_new = copy.copy(self)
		self_temp = copy.copy(self)

		## Checking if it is a root
		if self_old.parent==None:

			self_temp.prune_step()
			self_new = copy.copy(self_temp)
		else:

			## Taking a copy of the parent since it was getting changed!
			self_parent = copy.copy(self_old.parent)

			# Knowing if it is a left child or a right child
			if (self_old.ind==self_parent.left.ind):
				isLeft=True
			else:
				isLeft=False

			self_temp.prune_step()
			if isLeft:
				self_parent.left=self_temp
			else:
				self_parent.right=self_temp

			self_new = copy.copy(self_parent.get_updated_root())

		## If both the left and right children are leaves
		if (self_old.left.isLeaf!=-1 or (self_old.left.left==None and self_old.left.right==None)) and (self_old.left.isLeaf!=-1 or (self_old.right.left==None and self_old.right.right==None)):
			return [self_new]

		## If only left child is a leaf
		elif self_old.left.isLeaf!=-1 or (self_old.left.left==None and self_old.left.right==None):

			## Bug fix
			self_old.right.parent = self_old

			return  (self_old.right.prune_all()) + [self_new]

		## If only right child is leaf
		elif self_old.left.isLeaf!=-1 or (self_old.right.left==None and self_old.right.right==None):

			## Bug fix
			self_old.left.parent = self_old

			return (self_old.left.prune_all()) + [self_new]

		## If none are leaves
		else:
			### Bug fix
			self_old.left.parent = self_old
			self_old.right.parent = self_old

			return (self_old.left.prune_all()) + (self_old.right.prune_all()) + [self_new]
	'''
'''
### An independant total accuracy function
def total_accuracy(root, Y, X):
	return root.overall_accuracy(Y,X)

### An independant function for Getting the best pruned tree!
def best_prune(tList, Y, X):
	vfunc = np.vectorize(total_accuracy, excluded = ['Y','X'])
	output = vfunc(root = tList,Y = Y,X = X)
	maxAcc = output.max()
	maxInd = output.argmax()
	print(len(output), maxAcc, output[0])

	return tList[maxInd], maxAcc
'''

#### Initiating our tree:
## (1): Ind, Med is -1 indicating that no variable is conditioned here
## (2): isGrown is False here
## (3): isLeaf is -1 here to indicate that the given node is not a leaf in the true
print("Decision Tree without Pruning started")
start_time = time.time()

root = Node(class_train, features_train, -1, -1, False, -1, None, class_test, features_test, class_valid, features_valid)

## Initiating the plot variables
height_list = []
node_list = []
train_acc_list = []
test_acc_list = []
valid_acc_list = []

### Running the loop depending on the Overall height of the tree
### The maximum height of the tree obtained is 52. Overall Nodes about 20,000! Train accuracy of 90%, Test and Valid accuracies of 77%
for i in range(60):
	print(root.height())

#	node_list.append(root.total_nodes())
#	height_list.append(root.height())
#	train_acc_list.append(root.overall_accuracy(class_train,features_train))
#	test_acc_list.append(root.overall_accuracy(class_test,features_test))
#	valid_acc_list.append(root.overall_accuracy(class_valid,features_valid))

	root.insert()

print("The required tree has been constructed")
print()

'''
## Total time taken
print("Total time taken is ", time.time() - start_time)

### Plotting them

## Adding the few further elements
node_list.append(20008)
train_acc_list.append(train_acc_list[len(train_acc_list)-1])
test_acc_list.append(test_acc_list[len(test_acc_list)-1])
valid_acc_list.append(valid_acc_list[len(valid_acc_list)-1])

plt.plot(node_list, train_acc_list, label='Train Accuracy')
plt.plot(node_list, test_acc_list, label='Test Accuracy')
plt.plot(node_list, valid_acc_list, label='Validation Accuracy')
plt.legend()
plt.xlabel('Total Number of Nodes')
plt.ylabel('Accuracy')
plt.show()
'''
initial_nodes = root.total_nodes()
new_nodes = initial_nodes

orig_train_acc = root.overall_accuracy(class_train, features_train)
orig_test_acc = root.overall_accuracy(class_test, features_test)
orig_valid_acc = root.overall_accuracy(class_valid, features_valid)

node_list.append(0)
train_acc_list.append(orig_train_acc)
test_acc_list.append(orig_test_acc)
valid_acc_list.append(orig_valid_acc)

#### Pruning the maximum tree. Also limiting the size of the nodes
while(new_nodes>=10000):

	'''
	## Storing stuff in the Lists for plotting the graph
	node_list.append(root.total_nodes())
	train_acc_list.append(root.overall_accuracy(class_train,features_train))
	test_acc_list.append(root.overall_accuracy(class_test,features_test))
	valid_acc_list.append(root.overall_accuracy(class_valid,features_valid))
	'''

	## Creating a copy of the older root
	root_old = copy.copy(root)
#	tList = root.prune_all()

	## Storing the original Validation accuracy
	orig_acc = root_old.overall_accuracy(class_valid, features_valid)
	print(orig_acc, root_old.total_nodes(), root_old.height())

	## Best Accuracy
	dump, bestPrune, status = root_old.get_best_prune(orig_acc, class_valid, features_valid)
	bestAcc = bestPrune.overall_accuracy(class_valid, features_valid)
	new_nodes = bestPrune.total_nodes()
	print(new_nodes)

	## Getting the accuracies in the list
	if bestAcc>orig_acc:
		node_list.append(initial_nodes - new_nodes)
		train_acc_list.append(bestPrune.overall_accuracy(class_train, features_train))
		test_acc_list.append(bestPrune.overall_accuracy(class_test, features_test))
		valid_acc_list.append(bestAcc)

	## Getting the maximum pruned root
#	bestPrune, bestAcc = best_prune(tList,class_valid, features_valid)

	## Checking if the pruned trees are better or worse
	if status!=-1:
		break
	else:
		root = copy.copy(bestPrune)

## Total time taken
print("Total time taken is ", time.time() - start_time)

### Plotting them
#plt.plot(node_list, train_acc_list, label='Train Accuracy')
#plt.plot(node_list, test_acc_list, label='Test Accuracy')
plt.plot(node_list, valid_acc_list, label='Validation Accuracy')
plt.legend()
plt.xlabel('Number of Nodes removed')
plt.ylabel('Accuracy')
plt.show()

reason: import numpy,from scipy


text:
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import style
from scipy.optimize import curve_fit

plt.rcParams["image.cmap"] = "Spectral"
style.use("dark_background")

def func(x, a, b):
    return a * x * x + b

x_data = list(range(0, 10))
y_data = (np.random.normal(1, 0.1, 10)) * [func(n, 5, 200) for n in x_data]

popt, pcov = curve_fit(func, x_data, y_data, method="trf")

fit_data = [func(n, popt[0], popt[1]) for n in x_data]

print(popt, "\n", pcov, flush=True)

plt.plot(x_data, y_data)
plt.plot(x_data, fit_data)
plt.show()

max_stars_repo_name: YodaEmbedding/experiments

max_issues_repo_name: YodaEmbedding/experiments

max_forks_repo_name: YodaEmbedding/experiments