from matplotlib import pyplot as plt
import numpy as np
import requests
import json
import os
from dotenv import load_dotenv
load_dotenv() #take envirment variables from .env
import gspread #pip install gspread
from datetime import date, datetime

gc = gspread.service_account_from_dict(json.loads(os.environ.get('credentials')))
sh = gc.open_by_key('119aFQyS4gKwkqjDKhrxLkEuEA5WxFfz5rNm8OiPI4T0')
worksheet = sh.sheet1 #set the worksheet within google sheets
lstRows = worksheet.get_all_values() #list of rows

intDay = datetime.now().timetuple().tm_yday

API_KEY = os.environ.get('API_KEY')

dictGames = {'d':'Duels', 'D':'Duels', 'duels':'Duels', 'Duels':'Duels','bw':'Bedwars', 'BW':'Bedwars', 'Bw':'Bedwars', 'bW':'Bedwars'}
dictGameStats = {"Duels":["bridge_doubles_wins", "bridge_four_wins", "bridge_3v3v3v3_losses", "bridge_3v3v3v3_wins", "bridge_2v2v2v2_losses", "bridge_2v2v2v2_wins", "bridge_doubles_losses", "bridge_four_losses","bridge_duel_wins", "bridge_duel_losses"]}

strPlayer = input("Who should we look up? ")
strPlayer2 = input("Who should we look up? ")
strGame = dictGames[input('What Game? ')]

def getInfo(call):
    r = requests.get(call)
    return r.json()
def graph(x1, y1, x2 = [], y2 = [], title = ''):
    styles = plt.style.available
    plt.style.use(styles[8])

    axes = plt.gca()
    axes.yaxis.grid() #Set horizontal gridlines only
    if x2 and y2:
             plt.plot(x2, y2, color = 'r', marker = 'o') #plot the data
    plt.plot(x1, y1, color = 'k', linestyle='-', marker = 'o') #plot the data
    plt.tick_params(axis = "x", which = "both", bottom = True, top = False)#only want ticks on the bottem not the top
    plt.tick_params(axis = "y", which = "both", left = True, right = False) #only want ticks on the left not the right
    #plt.xticks(np.arange(x1[2], x1[-1]+1, 10.0)) ##How often I want a tick on the x axis

    i = 0
    for spine in plt.gca().spines.values():
        if i == 2:
            spine.set_visible(True)
        else:
            spine.set_visible(False)
        i += 1

    #plt.axis(xmin=, xmax=35, ymin=40000, ymax=80000)
    plt.savefig(f'{title}.png',bbox_inches='tight',transparent=True, pad_inches=0)
    plt.show()
def updateSheet(player, game):
    url = f'https://api.hypixel.net/player?key={API_KEY}&name={player}'
    data = getInfo(url)
    if data == {'success': False, 'cause': 'You have already looked up this name recently'}:
        print('you already entered this username recently')
        stat = ''
    else:
        for stats in dictGameStats[strGame]:
            try:
                stat = data['player']['stats'][game][stats]
                lstSheetRow = [intDay, player, game, stats, stat]
                worksheet.insert_row(lstSheetRow, 2)
            except:
                lstSheetRow = [intDay, player, game, stats, 0]
def purge(Day):
    lstRows = worksheet.get_all_values()
    if int(Day) == 0:
        i = 1
        while i < len(lstRows):
            worksheet.delete_rows(i+1, len(lstRows))
            break
    i = 1
    while i < len(lstRows):
        if int(Day)-14 > int(lstRows[i][0]):
            worksheet.delete_rows(i+1, len(lstRows))
        i += 1
def duelsToXY(player, targetStat):
    x = []
    y = []
    for rows in lstRows:
        if (rows[1] == player) and (rows[3] == targetStat):
            y.append(rows[4])
            x.append(rows[0])
    return x, y
for stats in dictGameStats['Duels']:
    x2 = duelsToXY(strPlayer2, stats)[0]
    y2 = duelsToXY(strPlayer2, stats)[1]
    print(x2, y2)
    graph(duelsToXY(strPlayer, stats)[0], duelsToXY(strPlayer, stats)[1], x2, y2, stats) #duelsToXY(strPlayer2, stats)[0], duelsToXY(strPlayer2, stats)[1],
#client.run(os.environ.get('DiscordToken'))


''' Houses the PyFunctions class for flagging Python files for documentation '''
# Internal
import  config
from    modules.documentation import Docs
from    modules.logging       import Log

log = Log(__name__)

class PyFunctions(Docs):
    ''' Inserts function related parsing flags into the file_lines class variable'''
    @log.performance
    def __init__(self) -> None:
        self.config = config.modules.Documentation
        super().__init__()
        # Ensure that the Doc class variables are filled
        self.readLines()

    @classmethod
    @log.performance
    def from_test_conf(cls, config):
        ''' Loads a test configuration file and returns an instance of the class '''
        test_class = cls()
        test_class.config = config
        log.verbose('Test class instantiated')
        return test_class

    @log.performance
    def processPyFunctionFlags(self):
        ''' Runs all of the flagging in order. When in doubt this is the correct order '''
        log.verbose('Start flagging Python functions')
        self.__flag_functions()
        self.__flag_function_params()
        self.__flag_function_docstring()
        self.__flag_nested_functions()
        log.verbose('Stop flagging Python function')

    @log.performance
    def __flag_functions(self):
        ''' Find and flag functions '''
        for file in self.file_lines:
            in_function = False
            prev_line_w_text  = 0
            # Ignore non python files and continue to next file
            if not self.isFileOfExtension(file.get('file_path'),'py'):
                continue

            for line in file.get('lines'):
                # First make sure that we are in a class and not flagging a nested function
                if 'cls' in line.get('flags'):
                    continue # If in a class it's a method and will get picked up in the classes file
                if in_function == True:
                    # Check if the function is finished
                    if line.get('whitespace') == 0:
                        first_char = line.get('line')[0] if len(line.get('line')) > 0 else ''
                        if first_char == '@':
                            # Technically we are starting a method here but since its not being
                            # declared we set it False
                            in_function = False
                            file['lines'][prev_line_w_text]['flags'].append('func end')
                            # Clear the extra flags from the blank lines
                            clear_flags = 'func'
                            for i in range(prev_line_w_text + 1, line.get('line_no')):
                                file['lines'][i]['flags'] = [ x for x in file['lines'][i]['flags'] if x != clear_flags ]

                        # If def continues over 1 line its possible to have the closing parenthesis inline with the def
                        elif first_char != ')':
                            in_function = False
                            file['lines'][prev_line_w_text]['flags'].append('func end')
                            # clear flags between the end of the method and the line we are on
                            clear_flags = 'func'
                            for i in range(prev_line_w_text + 1, line.get('line_no')):
                                file['lines'][i]['flags'] = [ x for x in file['lines'][i]['flags'] if x != clear_flags ]

                    # If still in a method and tag as such
                    if in_function:
                        line['flags'].append('func')

                if in_function == False:
                    # Look for additional decorators
                    if line.get('whitespace') == 0:
                        first_char = line.get('line')[0] if len(line.get('line')) > 0 else ''
                        if first_char == '@':
                            # Technically we are starting a method here but since its not being
                            # declared we set it False
                            in_function = False
                            line['flags'].append('decorated')
                            # Clear the extra flags from the blank lines
                            clear_flags = 'func'
                            for i in range(prev_line_w_text + 1, line.get('line_no')):
                                file['lines'][i]['flags'] = [ x for x in file['lines'][i]['flags'] if x != clear_flags ]

                    if line.get('line')[0:4] == 'def ':
                        in_function = True
                        line['flags'].append('func start')
                        line['flags'].append('func')

                if line.get('line').strip() != '':
                    prev_line_w_text = line.get('line_no')

            if in_function: # we are past the for loop now close off any open tags
                file['lines'][-1]['flags'].append('func end')

    @log.performance
    def __flag_function_params(self):
        ''' Adds flags for function parameters'''
        for file in self.file_lines:
            if not self.isFileOfExtension(file.get('file_path'),'py'): # Ignore non python files
                continue

            in_function_params = False
            for line in file.get('lines'):
                # These 3 if's should remain as three ifs to ensure the right flags stay on
                if 'func start' in line.get('flags'):
                    in_function_params = True
                    line['flags'].append('func param start')

                if in_function_params:
                    line['flags'].append('func param')

                if line.get('line').find(')') > -1 and in_function_params:
                    in_function_params = False
                    line['flags'].append('func param end')
                    # Look for a return hint
                    if line.get('line').find('->') > -1:
                        line['flags'].append('func return')

    @log.performance
    def __flag_function_docstring(self):
        ''' Adds flags for function docstring'''
        for file in self.file_lines:
            if not self.isFileOfExtension(file.get('file_path'),'py'): # Ignore non python files
                continue

            line_after_function_def = False
            in_docstring = False
            docs_start_pos = 0
            for line in file.get('lines'):
                if 'func param end' in line.get('flags'):
                    # If its the start of the function, start looking for the docstring.
                    line_after_function_def = True
                    continue # to the next line. This line wont have the docstring

                if 'func' in line.get('flags'):
                    if line_after_function_def:
                        # docstrings can start with either ''' or """. Find returns -1 if not in string
                        # So check both and if max > -1 then we found one
                        docs_start_pos = max([ line.get('line').find("'''"), line.get('line').find('"""') ])
                        if docs_start_pos > -1:
                            in_docstring = True
                            line['whitespace'] += 3
                            line['flags'].append('func docs start')
                        else:
                            # There is no docstring stop looking
                            line_after_function_def = False
                            continue

                if in_docstring:
                    line['flags'].append('func docs')
                    # Check for single liner
                    if line_after_function_def:
                        closing_quotes = max([ line.get('line')[docs_start_pos+3:].find("'''"), line.get('line')[docs_start_pos+3:].find('"""') ])
                    else:
                        closing_quotes = max([ line.get('line').find("'''"), line.get('line').find('"""') ])

                    if closing_quotes > -1:
                        in_docstring = False
                        line['flags'] = [ x for x in line['flags'] if x != 'docs']
                        line['flags'].append('func docs end')

                # We need to use the line_after_function_def flag to check for 2 sets of triple quotes only
                if 'func param end' not in line.get('flags'):
                    line_after_function_def = False

    @log.performance
    def __flag_nested_functions(self):
        ''' Adds all flags for nested functions'''
        for file in self.file_lines:
            in_nested_function = False
            in_params = False
            in_docs = False
            nested_whitespace = 0
            prev_line_w_text  = 0
            # Ignore non python files and continue to next file
            if not self.isFileOfExtension(file.get('file_path'),'py'):
                continue

            for line in file.get('lines'):
                # First make sure that we are in a function
                if 'func' in line.get('flags'):
                    if in_nested_function == False:
                        if 'func' not in line.get('flags'):
                            continue # to next line
                        if 'func start' in line.get('flags'):
                            continue # wont start a nested function here

                        # Look for the start of methods
                        start = line.get('whitespace')
                        end = start + 4
                        if line.get('line')[start:end] == 'def ':
                            in_nested_function = True
                            in_params = True
                            nested_whitespace = line.get('whitespace')
                            line['flags'].append('nest func start')
                            line['flags'].append('nest func param start')

                        if line.get('line').find(')') > -1 and in_params:
                            in_params = False
                            line['flags'].append('nest func param')
                            line['flags'].append('nest func param end')
                            if line.get('line').find('->') > -1:
                                line['flags'].append('nest func return')

                    if in_nested_function == True:
                        if in_params:
                            line['flags'].append('nest func param')
                            if line.get('line').find(')') > -1:
                                in_params = False
                                line['flags'].append('nest func param end')
                                if line.get('line').find('->') > -1:
                                    line['flags'].append('nest func return')

                        # Check that another method has not ended
                        if line.get('whitespace') == nested_whitespace and 'nest func start' not in line.get('flags'):
                            in_nested_function = False
                            file['lines'][prev_line_w_text]['flags'].append('nest func end')
                            # Clear flags between the end of the method and the line we are on
                            clear_flags = 'nest func'
                            for i in range(prev_line_w_text + 1, line.get('line_no')):
                                file['lines'][i]['flags'] = [ x for x in file['lines'][i]['flags'] if x != clear_flags ]

                            if line.get('line')[line.get('whitespace'):line.get('whitespace')+4] == 'def ':
                                line['flags'].append('nest func start')
                                line['flags'].append('nest func param start')
                                in_nested_function = True
                                in_params = True
                            # Check that the params didn't just close.
                            if line.get('line').find(')') > -1 and in_params:
                                in_params = False
                                line['flags'].append('nest func param')
                                line['flags'].append('nest func param end')
                                if line.get('line').find('->') > -1:
                                    line['flags'].append('nest func return')

                        # Docstring
                        if in_docs or line.get('line').find("'''") > -1 or line.get('line').find('"""') > -1:
                            line['flags'].append('nest func docs')
                            first_three = line.get('line')[line.get('whitespace'):line.get('whitespace')+3]
                            if first_three == '"""' or first_three == "'''":
                                if in_docs:
                                    in_docs = False
                                    line['flags'].append('nest func docs end')
                                    continue # To next line
                                in_docs = True
                                line['flags'].append('nest func docs start')
                                remaining_line = line.get('line')[line.get('whitespace')+3:]
                                if remaining_line.find('"""') > -1 or remaining_line.find("'''") > -1:
                                    in_docs = False
                                    line['flags'].append('nest func docs end')
                            else: # This line must be an additional line
                                # Check if it closes
                                if line.get('line').find('"""') > -1 or line.get('line').find("'''") > -1:
                                    in_docs = False
                                    line['flags'].append('nest func docs end')

                        # If still in a method and tag as such
                        if in_nested_function:
                            line['flags'].append('nest func')

                    if line.get('line').strip() != '':
                        prev_line_w_text = line.get('line_no')


#!/usr/bin/env python
# coding: utf-8

# In[47]:

#Load the dataset into the tool.
import pandas as pd

data = pd.read_csv('housing.csv')

# In[48]:

# Univariate analysis
import matplotlib.pyplot as plt

# Histogram
plt.hist(data['price'], bins=20)
plt.xlabel('Price')
plt.ylabel('Area')
plt.title('Price VS Area')
plt.show()

# Bar plot
data['bedrooms'].value_counts().plot(kind='bar')
plt.xlabel('Bedrooms')
plt.ylabel('Stories')
plt.title('Bedrooms VS Stories')
plt.show()

# In[11]:

# Bivariate analysis
import seaborn as sns

# Scatter plot
sns.scatterplot(x='area', y='price', data=data)
plt.xlabel('Area')
plt.ylabel('Price')
plt.title('Price vs. Area')
plt.show()

# Box plot
sns.boxplot(x='parking', y='price', data=data)
plt.xlabel('Parking')
plt.ylabel('Price')
plt.title('Parking vs Price')
plt.show()

# In[12]:

# Multivariate analysis
# Correlation matrix
corr_matrix = data.corr()
sns.heatmap(corr_matrix, annot=True, cmap='coolwarm')
plt.title('Correlation Matrix')
plt.show()

# Pair plot
sns.pairplot(data)
plt.show()

# In[13]:

# Descriptive statistics
statistics = data.describe()
print(statistics)

# In[15]:

# Check for missing values
missing_values = data.isnull().sum()
print(missing_values)
data = data.dropna()

# In[17]:

#detecting outliers
import numpy as np
def detect_outliers_zscore(data, threshold=3):
    z_scores = np.abs((data - data.mean()) / data.std())
    outliers = data[z_scores > threshold]
    return outliers
numerical_columns = ['price', 'area', 'bedrooms', 'bathrooms','stories','parking']
for column in numerical_columns:
    outliers = detect_outliers_zscore(data[column])
    print(f"Outliers in {column}:")
    print(outliers)

# In[18]:

#replacing outliers
def replace_outliers(data, column, method='median', threshold=3):
    z_scores = np.abs((data[column] - data[column].mean()) / data[column].std())
    outliers = data[z_scores > threshold]

    if method == 'median':
        replacement_value = data[column].median()
    elif method == 'mean':
        replacement_value = data[column].mean()
    else:
        raise ValueError("Invalid replacement method. Choose 'median' or 'mean'.")

    data.loc[z_scores > threshold, column] = replacement_value

    return data

numerical_columns = ['price', 'area', 'bedrooms', 'bathrooms','stories','parking']
for column in numerical_columns:
    data = replace_outliers(data, column, method='median')
data.to_csv('housing_without_outliers.csv', index=False)

# In[35]:

# Check for categorical columns
categorical_columns = data.select_dtypes(include=['object']).columns

# Perform one-hot encoding
data_encoded = pd.get_dummies(data, columns=categorical_columns)

# In[36]:

# Split into dependent and independent variables
X = data.drop('price', axis=1)  # Independent variables
y = data['price']  # Dependent variable

# In[37]:

from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# In[46]:

# Split the data into training and testing
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)

# In[45]:

# Build the model
from sklearn.linear_model import LinearRegression

model = LinearRegression()

# In[44]:

# Train the model
model.fit(X_train, y_train)

# In[41]:

# Test the model
y_pred = model.predict(X_test)

# In[43]:

# Measure performance
from sklearn.metrics import mean_squared_error, r2_score

rmse = np.sqrt(mean_squared_error(y_test, y_pred))
r2 = r2_score(y_test, y_pred)

print("RMSE:", rmse)
print("R-squared:", r2)


import matplotlib.pyplot as plt
import networkx as nx
import numpy as np
import os
from numpy import linalg as LA
from utils import load_data_with_features

def plot_all_filters(dataset, k, root):
    data, features, labels = load_data_with_features(dataset)
    features = np.array(features.todense())
    labels = np.argmax(labels, axis=-1)

    # Sort the labels according to their counts from small to big
    unique_labels, counts_labels = np.unique(labels, return_counts=True)
    res = sorted(zip(counts_labels, unique_labels), key=lambda x: x[0])
    res = zip(*res)
    res = [list(a) for a in res]
    sorted_counts = res[0]
    sorted_labels = res[1]
    print(dataset)
    print(f"Sorted labels: {sorted_labels}")
    print(f"Labels counts: {sorted_counts}")

    # Eigendecomposition
    graph = nx.from_dict_of_lists(data)
    norm_lap = nx.normalized_laplacian_matrix(graph)
    e, U = LA.eigh(norm_lap.A)
    print('# of nodes: {}'.format(graph.number_of_nodes()))
    print('# of edges: {}'.format(graph.number_of_edges()))

    # To-do: save e and U

    # Create folder
    if not os.path.exists(root):
        os.mkdir(root)

    # Save features
    np.savetxt(f"{root}/features.csv", features, delimiter=",")

    # Plot all filters
    for i in range(k):
        label = sorted_labels[i]
        binary_labels = [1 if x == label else 0 for x in labels]
        savedir = f"{root}/label_{label}/"
        if not os.path.exists(savedir):
            os.mkdir(savedir)
            # Save labels
            np.savetxt(f"{savedir}/labels_{label}.csv", binary_labels, delimiter=",")
        for j in range(features.shape[1]):
            filename = savedir + f"label_{label}_vs_feature_{j+1}.png"
            plot_filter(features[:, j], binary_labels, e, U, j, label, filename)

    # features = np.array(features.todense()[:, feature_id])  # One feature only
    # features = [i[0] for i in features]  # 2D array to 1D
    # np.clip(labels, 0, 1, out=labels)

    # plt.hist(features, bins=2)
    # plt.title('Cora feature {}'.format(feature_id))
    # plt.xlabel('feature value')
    # plt.ylabel('count')
    # plt.show()

def plot_filter(x, y, e, U, feature_id, label, filename):
    # if feature_id not in [30, 108]:
    #     return

    eps = 1e-4
    xh = np.dot(U.T, x)
    yh = np.dot(U.T, y)

    output = np.divide(yh, xh, out=np.zeros(yh.shape, dtype=float), where=np.abs(xh) > eps)

    fig = plt.figure()
    plt.scatter(e, output, s=10)
    plt.title("g(x) of label {} vs feature {}".format(label, feature_id+1))
    plt.savefig(filename)
    plt.close(fig)
    return

plot_all_filters('cora', 3, 'img.nosync/cora')
plot_all_filters('citeseer', 3, 'img.nosync/citeseer')
plot_all_filters('pubmed', 3, 'img.nosync/pubmed')


#!/usr/bin/python3

# references:
# - http://ccg.doc.gold.ac.uk/ccg_old/teaching/ludic_computing/ludic16.pdf
# - https://int8.io/monte-carlo-tree-search-beginners-guide/

# To be verified: "eval" recalculation in backpropagation

balance_constant = 50 # exploration vs exploitation balance
move_timeframe = 20  # number of seconds for evaluation
modeleval = None # model will be loaded in protover call

import Const
from queue import *
import chess.uci
import sys
from math import (sqrt, log)
from FeaturesExtraction import extract_features
import numpy as np
from copy import deepcopy

class MctsNode(object):

    def __init__(self, board, parent=None):
        #h#global evaltable, hitsevaltable
        self.board = board
        self.parent = parent
        self.children = []
        self.movetochild = []
        self.visitcount = 1
        #h#try:
        #h#    self.anneval, self.movestoexpand = evaltable[" ".join(self.board.fen().split()[0:4])]
        #h#    print("# evaltable ", self.board.fen(), " = ", self.anneval)
        #h#    hitsevaltable += 1
        #h#except KeyError:
        if True: #h#
            self.movestoexpand = Queue()
            if self.board.is_checkmate():
                self.anneval = -999999
            else:
                if board.turn == chess.WHITE:
                    b = self.board
                else:
                    b = self.board.mirror()
                y = modeleval.predict(np.array([extract_features(b)]), batch_size=1, verbose=0)
                self.anneval = y[0][0] #* (1 if self.board.turn==chess.WHITE else -1)
                print("# evaluated ", self.board.fen(), " = ", self.anneval)
                l = []
                self.terminalnode = True
                for m in self.board.generate_legal_moves():
                    a = str(m).upper()
                    l.append((m, y[1][0][ord(a[0])-65, ord(a[1])-49, ord(a[2])-65, ord(a[3])-49]))
                    self.terminalnode = False
                print("# sorted moves ", end="")
                for m in sorted(l, key=lambda e: e[1], reverse=True):
                    self.movestoexpand.put(m[0]) # moves ordered via policy values
                    print(str(m[0]), " ", end="")
                print("")
        #h#        evaltable[" ".join(self.board.fen().split()[0:4])] = (self.anneval, deepcopy(self.movestoexpand))
        self.value = self.anneval
        self.isroot = False # root will be moved during game progress

    def boardcopy(self):
        return self.board.copy()

    def setroot(self):
        self.isroot = True

def traverse(parent, mainline=""):

    parent.visitcount += 1
    print("# visit ", parent.visitcount, " of ", mainline if mainline!="" else "root")

    if parent.terminalnode:
        print("# terminal node")
        return parent

    if not parent.movestoexpand.empty():
        # pick_univisted_children and create node for it
        move = parent.movestoexpand.get()
        parent.movetochild.append(move)
        board = parent.boardcopy()
        board.push(move)
        node = MctsNode(board, parent=parent)
        parent.children.append(node)
        print("# picked unvisited ", move)
        return node

    # traverse child with best upper confidence bound
    max = -float('inf')
    bestchild = None
    for i in range(len(parent.children)):
        exploit = -parent.children[i].value
        explore = balance_constant * sqrt( log( parent.visitcount ) / parent.children[i].visitcount )
        e = exploit + explore
        print("# deciding on move ", parent.movetochild[i], \
              " exploit ", exploit, " explore ", explore)
        if e > max:
            bestchild = i
            max = e
    print("# picked move ", parent.movetochild[bestchild], \
          " visited already ", parent.children[bestchild].visitcount, " times")

    return traverse(parent.children[bestchild], mainline+str(parent.movetochild[bestchild])+",")

def backpropagate(node, eval):

    # recalculate max (to be optimized)
    if len(node.children)>1:
        print("# backpropagated value ", eval)
        mx = eval
        for n in node.children:
            mx = max( mx, -n.value) # minus because is adversary
        eval = mx

    if node.movestoexpand.empty() or not node.value:
        node.value = eval
        print("# backpropagated assignment: old node value ", node.value, " new node value ", eval)
        if not node.isroot:
            print("# continue backpropagation")
            backpropagate(node.parent, -eval)
    else:
        print("# backpropagation stops here")

    return

#=========================
root = MctsNode(chess.Board())
root.setroot() # stop backpropagation here
while true:
	leaf = traverse(root) # to pick unvisited/best node
	eval = leaf.value # simulation
	backpropagate(leaf.parent, -eval)


import confluent_kafka as ck
import pytest

from cudf.testing._utils import assert_eq

@pytest.mark.parametrize("commit_offset", [1, 45, 100, 22, 1000, 10])
@pytest.mark.parametrize("topic", ["cudf-kafka-test-topic"])
def test_kafka_offset(kafka_client, topic, commit_offset):
    offsets = [ck.TopicPartition(topic, 0, commit_offset)]
    kafka_client.commit(offsets=offsets)

    # Get the offsets that were just committed to Kafka
    retrieved_offsets = kafka_client.committed(offsets)

    for off in retrieved_offsets:
        assert_eq(off.topic, offsets[0].topic)
        assert_eq(off.partition, offsets[0].partition)
        assert_eq(off.offset, offsets[0].offset)


from functools import reduce

pessoas = [
    {'nome': 'Pedro', 'idade': 11},
    {'nome': 'Mariana', 'idade': 18},
    {'nome': 'Arthur', 'idade': 26},
    {'nome': 'Rebeca', 'idade': 6},
    {'nome': 'Tiago', 'idade': 19},
    {'nome': 'Gabriela', 'idade': 17}
]

print(int(reduce(lambda i,s: i + s, filter(lambda i: i < 18, map(lambda p: p['idade'], pessoas)), 0)))


from flask_app.config.mysqlconnection import connectToMySQL
import re
from flask import flash

EMAIL_REGEX = re.compile(r'^[a-zA-Z0-9.+_-]+@[a-zA-Z0-9._-]+\.[a-zA-Z]+$')

class User:
    def __init__(self, data):
        self.id = data['id']
        self.email= data['email']
        self.created_at=data['created_at']
        self.updated_at=data['updated_at']
    @classmethod
    def get_all(cls):
        query= "SELECT * FROM users;"
        result= connectToMySQL('email_validation_schema').query_db(query)
        users=[]
        for user in result:
            users.append(cls(user))
        return users
    @classmethod
    def save(cls, data ):
        query = "INSERT INTO users (  email  ) VALUES (  %(email)s );"
        return connectToMySQL('email_validation_schema').query_db( query, data )
    @classmethod
    def delete(cls, data):
        query = "DELETE FROM users WHERE id = (%(id)s);"
        return connectToMySQL('email_validation_schema').query_db(query, data)
    @staticmethod
    def validate_user( user ):
        is_valid = True
        # test whether a field matches the pattern
        if not EMAIL_REGEX.match(user['email']):
            flash("Invalid email address!")
            is_valid = False
        for email in User.get_all():
            if user['email']== email.email:
                flash("This email is taken")
                is_valid= False
        return is_valid


import argparse
import json

import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions

class ProcessMessages(beam.DoFn):
    def process(self, element):
        yield element

def run(argv=None):
    # Initialize pipeline options
    parser = argparse.ArgumentParser()
    v, pipeline_args = parser.parse_known_args(argv)
    pipeline_options = PipelineOptions(pipeline_args)

    with beam.Pipeline(options=pipeline_options) as p:
        # Sample JSON data as Python lists
        json_data1 = [
            {'id': '4697948','title': 'How do I get two integer values from the client by the terminal in this code - GO RPC','creation_date': '2023-06-23T13:21:32'},
            {"id": "3", "name": "Bob"}
        ]

        json_data2 = [
            {"id": "4697948", "age": 25},
            {"id": "2", "age": 30},
        ]

        # Convert Python lists to PCollections
        json_pcoll1 = p | "Create JSON 1 PCollection" >> beam.Create(json_data1)
        json_pcoll2 = p | "Create JSON 2 PCollection" >> beam.Create(json_data2)

        # Map key-value pairs for JSON data
        json_mapped1 = json_pcoll1 | 'Map JSON 1' >> beam.Map(lambda x: (x['id'], x))
        json_mapped2 = json_pcoll2 | 'Map JSON 2' >> beam.Map(lambda x: (x['id'], x))

        # Join the JSON data using CoGroupByKey
        joined_data = ({'json1': json_mapped1, 'json2': json_mapped2}
                       | 'CoGroupByKey' >> beam.CoGroupByKey())

        # Process and print the joined data
        joined_data | 'Print joined data' >> beam.Map(print)

if __name__ == '__main__':
    run()


from pathlib import Path

import pytest

from dpipe.config import Locker
from dpipe.layout.base import change_current_dir

def test_locker(tmpdir):
    locker = Locker(tmpdir)
    assert (tmpdir / '.lock').exists()
    with pytest.raises(FileExistsError):
        Locker(tmpdir)

    locker.run()
    assert not (tmpdir / '.lock').exists()

def test_chdir(tmpdir):
    with change_current_dir(tmpdir):
        locker = Locker()
        lock = Path('.lock')
        assert lock.exists()
        with pytest.raises(FileExistsError):
            Locker()

        locker.run()
        assert not lock.exists()

        def nested():
            assert lock.exists()
            with pytest.raises(FileExistsError):
                Locker()

        assert not lock.exists()
        Locker().run(nested())
        assert not lock.exists()

def test_no_instance():
    with pytest.raises(AttributeError):
        Locker.run()


import functools
import itertools
import logging
import time
from http import HTTPStatus
from typing import Any, List, Literal, Mapping, Optional, TypedDict, Union

from datasets import (
    Dataset,
    Features,
    IterableDataset,
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_split_names,
    load_dataset,
)
from datasets.data_files import EmptyDatasetError as _EmptyDatasetError
from libcommon.exceptions import CustomError
from libcommon.simple_cache import SplitFullName as _SplitFullName
from libcommon.utils import orjson_dumps

from datasets_based.config import AppConfig, FirstRowsConfig
from datasets_based.features import get_cell_value
from datasets_based.worker import ConfigNotFoundError, JobInfo, SplitNotFoundError
from datasets_based.workers._datasets_based_worker import DatasetsBasedWorker

FirstRowsWorkerErrorCode = Literal[
    "SplitsNamesError",
    "EmptyDatasetError",
    "InfoError",
    "FeaturesError",
    "StreamingRowsError",
    "NormalRowsError",
    "RowsPostProcessingError",
]

class FirstRowsWorkerError(CustomError):
    """Base class for exceptions in this module."""

    def __init__(
        self,
        message: str,
        status_code: HTTPStatus,
        code: FirstRowsWorkerErrorCode,
        cause: Optional[BaseException] = None,
        disclose_cause: bool = False,
    ):
        super().__init__(message, status_code, str(code), cause, disclose_cause)

class SplitsNamesError(FirstRowsWorkerError):
    """Raised when the split names could not be fetched."""

    def __init__(self, message: str, cause: Optional[BaseException] = None):
        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "SplitsNamesError", cause, True)

class EmptyDatasetError(FirstRowsWorkerError):
    """Raised when the dataset has no data."""

    def __init__(self, message: str, cause: Optional[BaseException] = None):
        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "EmptyDatasetError", cause, True)

class InfoError(FirstRowsWorkerError):
    """Raised when the info could not be fetched."""

    def __init__(self, message: str, cause: Optional[BaseException] = None):
        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "InfoError", cause, True)

class FeaturesError(FirstRowsWorkerError):
    """Raised when the features could not be fetched."""

    def __init__(self, message: str, cause: Optional[BaseException] = None):
        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "FeaturesError", cause, True)

class StreamingRowsError(FirstRowsWorkerError):
    """Raised when the rows could not be fetched in streaming mode."""

    def __init__(self, message: str, cause: Optional[BaseException] = None):
        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "StreamingRowsError", cause, True)

class NormalRowsError(FirstRowsWorkerError):
    """Raised when the rows could not be fetched in normal mode."""

    def __init__(self, message: str, cause: Optional[BaseException] = None):
        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "NormalRowsError", cause, True)

class RowsPostProcessingError(FirstRowsWorkerError):
    """Raised when the rows could not be post-processed successfully."""

    def __init__(self, message: str, cause: Optional[BaseException] = None):
        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "RowsPostProcessingError", cause, False)

def retry():
    def decorator_retry(func):
        """retries with an increasing sleep before every attempt"""
        SLEEPS = [1, 7, 70, 7 * 60, 70 * 60]
        MAX_ATTEMPTS = len(SLEEPS)

        @functools.wraps(func)
        def decorator(*args, **kwargs):
            attempt = 0
            last_err = None
            while attempt < MAX_ATTEMPTS:
                try:
                    """always sleep before calling the function. It will prevent rate limiting in the first place"""
                    duration = SLEEPS[attempt]
                    logging.info(f"Sleep during {duration} seconds to preventively mitigate rate limiting.")
                    time.sleep(duration)
                    return func(*args, **kwargs)
                except ConnectionError as err:
                    logging.info("Got a ConnectionError, possibly due to rate limiting. Let's retry.")
                    last_err = err
                    attempt += 1
            raise RuntimeError(f"Give up after {attempt} attempts with ConnectionError") from last_err

        return decorator

    return decorator_retry

Row = Mapping[str, Any]

class FeatureItem(TypedDict):
    feature_idx: int
    name: str
    type: Mapping[str, Any]

class RowItem(TypedDict):
    row_idx: int
    row: Mapping[str, Any]
    truncated_cells: List[str]

class FirstRowsResponse(TypedDict):
    dataset: str
    config: str
    split: str
    features: List[FeatureItem]
    rows: List[RowItem]

@retry()
def get_rows(
    dataset: str,
    config: str,
    split: str,
    streaming: bool,
    rows_max_number: int,
    use_auth_token: Union[bool, str, None] = False,
) -> List[Row]:
    ds = load_dataset(
        dataset,
        name=config,
        split=split,
        streaming=streaming,
        use_auth_token=use_auth_token,
    )
    if streaming:
        if not isinstance(ds, IterableDataset):
            raise TypeError("load_dataset should return an IterableDataset in streaming mode")
    elif not isinstance(ds, Dataset):
        raise TypeError("load_dataset should return a Dataset in normal mode")
    rows_plus_one = list(itertools.islice(ds, rows_max_number + 1))
    # ^^ to be able to detect if a split has exactly ROWS_MAX_NUMBER rows
    if len(rows_plus_one) <= rows_max_number:
        logging.debug(f"all the rows in the split have been fetched ({len(rows_plus_one)})")
    else:
        logging.debug(f"the rows in the split have been truncated ({rows_max_number} rows)")
    return rows_plus_one[:rows_max_number]

def get_json_size(obj: Any) -> int:
    """Returns the size of an object in bytes once serialized as JSON

    Args:
        obj (Any): the Python object

    Returns:
        int: the size of the serialized object in bytes
    """
    return len(orjson_dumps(obj))

# from https://stackoverflow.com/a/43848928/7351594
def utf8_lead_byte(b: int) -> bool:
    """A UTF-8 intermediate byte starts with the bits 10xxxxxx."""
    return (b & 0xC0) != 0x80

def utf8_byte_truncate(text: str, max_bytes: int) -> str:
    """If text[max_bytes] is not a lead byte, back up until a lead byte is
    found and truncate before that character."""
    utf8 = text.encode("utf8")
    if len(utf8) <= max_bytes:
        return text
    i = max_bytes
    while i > 0 and not utf8_lead_byte(utf8[i]):
        i -= 1
    return utf8[:i].decode("utf8", "ignore")

# Mutates row_item, and returns it anyway
def truncate_row_item(row_item: RowItem, min_cell_bytes: int) -> RowItem:
    row = {}
    for column_name, cell in row_item["row"].items():
        # for now: all the cells above min_cell_bytes are truncated to min_cell_bytes
        # it's done by replacing the cell (which can have any type) by a string with
        # its JSON serialization, and then truncating it to min_cell_bytes
        cell_json = orjson_dumps(cell)
        if len(cell_json) <= min_cell_bytes:
            row[column_name] = cell
        else:
            cell_json_str = cell_json.decode("utf8", "ignore")
            row_item["truncated_cells"].append(column_name)
            row[column_name] = utf8_byte_truncate(text=cell_json_str, max_bytes=min_cell_bytes)
    row_item["row"] = row
    return row_item

COMMA_SIZE = 1  # the comma "," is encoded with one byte in utf-8

# Mutates row_items, and returns them anyway
def truncate_row_items(row_items: List[RowItem], min_cell_bytes: int, rows_max_bytes: int) -> List[RowItem]:
    # compute the current size
    rows_bytes = sum(get_json_size(row_item) for row_item in row_items) + COMMA_SIZE * (len(row_items) - 1)

    # Loop backwards, so that the last rows are truncated first
    for row_item in reversed(row_items):
        if rows_bytes < rows_max_bytes:
            break
        previous_size = get_json_size(row_item) + COMMA_SIZE
        row_item = truncate_row_item(row_item=row_item, min_cell_bytes=min_cell_bytes)
        new_size = get_json_size(row_item) + COMMA_SIZE
        rows_bytes += new_size - previous_size
        row_idx = row_item["row_idx"]
        logging.debug(f"the size of the rows is now ({rows_bytes}) after truncating row idx={row_idx}")
    return row_items

def to_row_item(row_idx: int, row: Row) -> RowItem:
    return {
        "row_idx": row_idx,
        "row": row,
        "truncated_cells": [],
    }

def create_truncated_row_items(
    rows: List[Row],
    min_cell_bytes: int,
    rows_max_bytes: int,
    rows_min_number: int,
) -> List[RowItem]:
    row_items = []
    rows_bytes = 0

    # two restrictions must be enforced:
    # - at least rows_min_number rows
    # - at most rows_max_bytes bytes. Note that it's the limit to the sum of the rows sizes. The JSON response size
    #   will be greater, due to the other fields (row_idx, truncated_cells, features, etc.).
    # To enforce this:
    # 1. first get the first rows_min_number rows
    for row_idx, row in enumerate(rows[:rows_min_number]):
        row_item = to_row_item(row_idx=row_idx, row=row)
        rows_bytes += get_json_size(row_item) + COMMA_SIZE
        row_items.append(row_item)

    # 2. if the total is over the bytes limit, truncate the values, iterating backwards starting
    # from the last rows, until getting under the threshold
    # caveat: the truncation might not be enough to get under the threshold if:
    # - the number of columns is too high
    # - rows_max_bytes is too low (or even negative)
    if rows_bytes >= rows_max_bytes:
        logging.debug(
            f"the size of the first {rows_min_number} rows ({rows_bytes}) is above the max number of bytes"
            f" ({rows_max_bytes}), they will be truncated"
        )
        return truncate_row_items(row_items=row_items, min_cell_bytes=min_cell_bytes, rows_max_bytes=rows_max_bytes)

    # 3. else: add the remaining rows until the end, or until the bytes threshold
    for idx, row in enumerate(rows[rows_min_number:]):
        row_idx = rows_min_number + idx
        row_item = to_row_item(row_idx=row_idx, row=row)
        rows_bytes += get_json_size(row_item) + COMMA_SIZE
        if rows_bytes >= rows_max_bytes:
            logging.debug(
                f"the rows in the split have been truncated to {row_idx} row(s) to keep the size"
                f" ({rows_bytes}) under the limit ({rows_max_bytes})"
            )
            break
        row_items.append(row_item)
    return row_items

def transform_rows(
    dataset: str,
    config: str,
    split: str,
    rows: List[Row],
    features: Features,
    assets_base_url: str,
    assets_directory: str,
) -> List[Row]:
    return [
        {
            featureName: get_cell_value(
                dataset=dataset,
                config=config,
                split=split,
                row_idx=row_idx,
                cell=row[featureName] if featureName in row else None,
                featureName=featureName,
                fieldType=fieldType,
                assets_base_url=assets_base_url,
                assets_directory=assets_directory,
            )
            for (featureName, fieldType) in features.items()
        }
        for row_idx, row in enumerate(rows)
    ]

# in JSON, dicts do not carry any order, so we need to return a list
#
# > An object is an *unordered* collection of zero or more name/value pairs, where a name is a string and a value
#   is a string, number, boolean, null, object, or array.
# > An array is an *ordered* sequence of zero or more values.
# > The terms "object" and "array" come from the conventions of JavaScript.
# from https://stackoverflow.com/a/7214312/7351594 / https://www.rfc-editor.org/rfc/rfc7159.html
def to_features_list(features: Features) -> List[FeatureItem]:
    features_dict = features.to_dict()
    return [
        {
            "feature_idx": idx,
            "name": name,
            "type": features_dict[name],
        }
        for idx, name in enumerate(features)
    ]

class SplitFullName(TypedDict):
    dataset: str
    config: str
    split: str

def get_dataset_split_full_names(dataset: str, use_auth_token: Union[bool, str, None] = False) -> List[SplitFullName]:
    logging.info(f"get dataset '{dataset}' split full names")
    return [
        {"dataset": dataset, "config": config, "split": split}
        for config in get_dataset_config_names(path=dataset, use_auth_token=use_auth_token)
        for split in get_dataset_split_names(path=dataset, config_name=config, use_auth_token=use_auth_token)
    ]

def compute_first_rows_response(
    dataset: str,
    config: str,
    split: str,
    assets_base_url: str,
    hf_token: Optional[str],
    min_cell_bytes: int,
    max_size_fallback: int,
    rows_max_bytes: int,
    rows_max_number: int,
    rows_min_number: int,
    assets_directory: str,
) -> FirstRowsResponse:
    """
    Get the response of /first-rows for one specific split of a dataset from huggingface.co.
    Dataset can be private or gated if you pass an acceptable token.

    It is assumed that the dataset exist and can be accessed using the token.

    Args:
        dataset (`str`):
            A namespace (user or an organization) and a repo name separated
            by a `/`.
        config (`str`):
            A configuration name.
        split (`str`):
            A split name.
        assets_base_url (`str`):
            The base url of the assets.
        hf_endpoint (`str`):
            The Hub endpoint (for example: "https://huggingface.co")
        hf_token (`str` or `None`):
            An authentication token (See https://huggingface.co/settings/token)
        max_size_fallback (`int`):
            The maximum number of bytes of the split to fallback to normal mode if the streaming mode fails.
        rows_max_bytes (`int`):
            The maximum number of bytes of the response (else, the response is truncated).
        rows_max_number (`int`):
            The maximum number of rows of the response.
        rows_min_number (`int`):
            The minimum number of rows of the response.
    Returns:
        [`FirstRowsResponse`]: The list of first rows of the split.
    <Tip>
    Raises the following errors:
        - [`~libcommon.worker.ConfigNotFoundError`]
          If the config does not exist in the dataset.
        - [`~libcommon.worker.SplitNotFoundError`]
          If the split does not exist in the dataset.
        - [`~worker.utils.InfoError`]
          If the config info could not be obtained using the datasets library.
        - [`~worker.utils.FeaturesError`]
          If the split features could not be obtained using the datasets library.
        - [`~worker.utils.StreamingRowsError`]
          If the split rows could not be obtained using the datasets library in streaming mode.
        - [`~worker.utils.NormalRowsError`]
          If the split rows could not be obtained using the datasets library in normal mode.
        - [`~worker.utils.RowsPostProcessingError`]
          If the post-processing of the split rows failed, e.g. while saving the images or audio files to the assets.
    </Tip>
    """
    logging.info(f"get first-rows for dataset={dataset} config={config} split={split}")
    use_auth_token: Union[bool, str, None] = hf_token if hf_token is not None else False
    # first ensure the tuple (dataset, config, split) exists on the Hub
    # get the list of splits
    try:
        split_full_names = get_dataset_split_full_names(dataset=dataset, use_auth_token=use_auth_token)
    except _EmptyDatasetError as err:
        raise EmptyDatasetError("The dataset is empty.", cause=err) from err
    except Exception as err:
        raise SplitsNamesError("Cannot get the split names for the dataset.", cause=err) from err
    # ^ can raise DatasetNotFoundError or SplitsNamesError
    if config not in [split_full_name["config"] for split_full_name in split_full_names]:
        raise ConfigNotFoundError(f"config {config} does not exist for dataset {dataset}")
    if {"dataset": dataset, "config": config, "split": split} not in [
        {
            "dataset": split_full_name["dataset"],
            "config": split_full_name["config"],
            "split": split_full_name["split"],
        }
        for split_full_name in split_full_names
    ]:
        raise SplitNotFoundError("The config or the split does not exist in the dataset")
    # get the features
    try:
        info = get_dataset_config_info(
            path=dataset,
            config_name=config,
            use_auth_token=use_auth_token,
        )
    except Exception as err:
        raise InfoError("The info cannot be fetched for the dataset config.", cause=err) from err
    if not info.features:
        try:
            # https://github.com/huggingface/datasets/blob/f5826eff9b06ab10dba1adfa52543341ef1e6009/src/datasets/iterable_dataset.py#L1255
            iterable_dataset = load_dataset(
                path=dataset,
                name=config,
                split=split,
                streaming=True,
                use_auth_token=use_auth_token,
            )
            if not isinstance(iterable_dataset, IterableDataset):
                raise TypeError("load_dataset should return an IterableDataset")
            iterable_dataset = iterable_dataset._resolve_features()
            if not isinstance(iterable_dataset, IterableDataset):
                raise TypeError("load_dataset should return an IterableDataset")
            features = iterable_dataset.features
        except Exception as err:
            raise FeaturesError("The split features (columns) cannot be extracted.", cause=err) from err
    else:
        features = info.features
    # get the rows
    try:
        rows = get_rows(
            dataset=dataset,
            config=config,
            split=split,
            streaming=True,
            rows_max_number=rows_max_number,
            use_auth_token=use_auth_token,
        )
    except Exception as err:
        if info.size_in_bytes is None or info.size_in_bytes > max_size_fallback:
            raise StreamingRowsError(
                "Cannot load the dataset split (in streaming mode) to extract the first rows.",
                cause=err,
            ) from err
        try:
            rows = get_rows(
                dataset=dataset,
                config=config,
                split=split,
                streaming=False,
                rows_max_number=rows_max_number,
                use_auth_token=use_auth_token,
            )
        except Exception as err:
            raise NormalRowsError(
                "Cannot load the dataset split (in normal download mode) to extract the first rows.",
                cause=err,
            ) from err
    # transform the rows, if needed (e.g. save the images or audio to the assets, and return their URL)
    try:
        transformed_rows = transform_rows(
            dataset=dataset,
            config=config,
            split=split,
            rows=rows,
            features=features,
            assets_base_url=assets_base_url,
            assets_directory=assets_directory,
        )
    except Exception as err:
        raise RowsPostProcessingError(
            "Server error while post-processing the split rows. Please report the issue.",
            cause=err,
        ) from err
    # get the size of the surrounding JSON (without the rows)
    features_list = to_features_list(features=features)
    response: FirstRowsResponse = {
        "dataset": dataset,
        "config": config,
        "split": split,
        "features": features_list,
        "rows": [],
    }
    surrounding_json_size = get_json_size(response)
    # truncate the rows to fit within the restrictions, and prepare them as RowItems
    row_items = create_truncated_row_items(
        rows=transformed_rows,
        min_cell_bytes=min_cell_bytes,
        rows_max_bytes=rows_max_bytes - surrounding_json_size,
        rows_min_number=rows_min_number,
    )
    response["rows"] = row_items
    # return the response
    return response

class FirstRowsWorker(DatasetsBasedWorker):
    first_rows_config: FirstRowsConfig

    @staticmethod
    def get_job_type() -> str:
        return "/first-rows"

    @staticmethod
    def get_version() -> str:
        return "2.0.0"

    def __init__(self, job_info: JobInfo, app_config: AppConfig, first_rows_config: FirstRowsConfig) -> None:
        super().__init__(job_info=job_info, app_config=app_config)
        self.first_rows_config = first_rows_config

    def compute(self) -> Mapping[str, Any]:
        if self.config is None or self.split is None:
            raise ValueError("config and split are required")
        return compute_first_rows_response(
            dataset=self.dataset,
            config=self.config,
            split=self.split,
            assets_base_url=self.first_rows_config.assets.base_url,
            assets_directory=self.first_rows_config.assets.storage_directory,
            hf_token=self.common_config.hf_token,
            min_cell_bytes=self.first_rows_config.min_cell_bytes,
            max_size_fallback=self.first_rows_config.fallback_max_dataset_size,
            rows_max_bytes=self.first_rows_config.max_bytes,
            rows_max_number=self.first_rows_config.max_number,
            rows_min_number=self.first_rows_config.min_number,
        )

    def get_new_splits(self, _: Mapping[str, Any]) -> set[_SplitFullName]:
        """Get the set of new splits, from the content created by the compute."""
        if self.config is None or self.split is None:
            raise ValueError("config and split are required")
        return {_SplitFullName(dataset=self.dataset, config=self.config, split=self.split)}


# type: ignore

import json
import logging; log = logging.getLogger()

class Emotes:
    def __init__(self, bot) -> None:
        self.bot = bot
        with open(f"data/emotes.json", "r", encoding="utf8", errors="ignore") as f:
            self.emotes = json.load(f)
            if bot.config.dev:
                self.emotes.update({
                    "YES": "✅",
                    "NO": "❌"
                })

    def get(self, key: str) -> str:
        try:
            return self.emotes[key]
        except KeyError:
            log.warn("[AutoMod] Failed to obtain an emoji with key {}".format(key), extra={"loc": f"PID {os.getpid()}"})
            return "❓"

    def reload(self) -> None:
        self.__init__(self.bot)


from operator import add
from sqlalchemy import create_engine, MetaData, Table, func as sqlfunc
from sqlalchemy.orm import sessionmaker

DATABASE = 'red30'
engine = create_engine(f'postgresql://postgres:@localhost:5432/{DATABASE}')
Session = sessionmaker()
session = Session()

def get_last_row(session, table):
    row_count = session.query(table).count()
    query = sales_table.select().offset(row_count - 1)
    return session.execute(query).first()

def add_filler():
    print('-'*10)

with engine.connect() as conn:

    meta = MetaData(engine)
    sales_table = Table('sales', meta, autoload=True, autoload_with=engine)

    # Select first 10 rows
    print('Selecting first 10 rows...')
    add_filler()
    select_statement = sales_table.select().limit(10)
    result = conn.execute(select_statement)
    for row in result:
        print(row)

    add_filler()

    # Delete previously inserted data
    delete_statement = sales_table.delete().where(sales_table.c.cust_name == 'Akshay Bhatia')
    conn.execute(delete_statement)

    # Get max of order_num
    max_order_num = session.query(sqlfunc.max(sales_table.c.order_num)).scalar()

    # Create data to insert
    sale_data = dict(
        order_num=max_order_num+1,
        order_type='Retail',
        cust_name='Akshay',
        cust_state='Prague',
        prod_category='Electronics',
        prod_number='EL420',
        prod_name='PS5 Digital Edition',
        quantity=1,
        price=499,
        discount_perc=20
    )

    # Calculate other fields
    initial_total = sale_data['price']*sale_data['quantity']
    sale_data['discount'] = initial_total*sale_data['discount_perc']/100
    sale_data['order_total'] = initial_total - sale_data['discount']
    del sale_data['discount_perc']

    # Insert data
    print('Inserting new sale data...')
    add_filler()
    insert_statement = sales_table.insert().values(**sale_data)
    conn.execute(insert_statement)

    # Show inserted data
    print('Selecting inserted data...')
    add_filler()
    select_statement = sales_table.select().where(sales_table.c.order_num == max_order_num+1)
    result = conn.execute(select_statement)
    for row in result:
        print(row)

    add_filler()

    # Update Akshay's name
    print('Updating cust_name in inserted data...')
    add_filler()
    update_statement = sales_table.update().where(sales_table.c.order_num == max_order_num+1).values(cust_name='Akshay Bhatia')
    conn.execute(update_statement)

    # Reselect inserted data
    print('Reselecting updated data...')
    select_statement = sales_table.select().where(sales_table.c.order_num == max_order_num+1)
    result = conn.execute(select_statement)
    for row in result:
        print(row)

    add_filler()


import unittest, doctest

import util.testcase as testcase
from osaf import pim
from application import schema
import osaf.pim.mail as Mail
import osaf.pim.calendar.Calendar as Calendar
import logging

from datetime import datetime
from chandlerdb.util.Path import Path
from osaf.pim.stamping import has_stamp, StampAlreadyPresentError, StampNotPresentError

def test_stamping_doc():
    return doctest.DocFileSuite(
        'stamping.txt',
        optionflags=doctest.ELLIPSIS, package='osaf.pim',
    )

verbose = False
compareWhos = False
compareDates = False # date shouldn't automatically carry over
testFailureCases = True

class SavedAttrs:
    """
    Saved Attributes stored in one of these
    """
    pass

class StampingTest(testcase.SharedSandboxTestCase):

    """ Test Stamping in the Domain Model """
    def setAttributes(self, item, doWho=True):
        try:
            savedItems = self.savedAttrs
        except AttributeError:
            self.savedAttrs = {}

        savedAttrs = SavedAttrs()
        self.savedAttrs[item.itsName] = savedAttrs

        displayName = 'aTitleOrHeadline'
        item.displayName = displayName
        savedAttrs.displayName = displayName

        item.createdOn = self.savedAttrs[item.itsName].createdOn = datetime.now(item.itsView.tzinfo.default)

        if has_stamp(item, pim.TaskStamp):
            task = pim.TaskStamp(item)
            # Add some attributes here...

        if has_stamp(item, pim.EventStamp):
            event = pim.EventStamp(item)

        if has_stamp(item, Mail.MailStamp):
            mail = Mail.MailStamp(item)

        """
        item.displayName = anAbout
        aDate = datetime.now()
        item.date = aDate
        aWhoList = []
        view = item.itsView
        if doWho:
            aWhoList.append(generate.GenerateCalendarParticipant(view))
            aWhoList.append(generate.GenerateCalendarParticipant(view))
        if compareWhos:
            item.who = aWhoList

        savedAttrs.displayName = anAbout
        savedAttrs.date = aDate
        savedAttrs.who = aWhoList
        """

    def assertAttributes(self, item):
        itemAttrs = self.savedAttrs[item.itsName]
        self.assertEqual(item.displayName, itemAttrs.displayName)
        # compare the dates
        if compareDates:
            self.assertEqual(item.createdOn, itemAttrs.createdOn)
        # compare the whos
        if compareWhos:
            self.assertEqual(len(item.who), len(itemAttrs.who))
            i = 0
            for whom in item.who:
                self.assertEqual(whom, itemAttrs.who[i])
                i += 1

    #def assertKinds(self, item, stampsList):
    #    self.assertAttributes(item)
    #    for stamp in stampsList:
    #        self.failUnless(stamp in Stamp(item).stamps)

    def traverseStampSquence(self, item, sequence):
        for operation, stampClass in sequence:
            if verbose:
                message = "stamping %s: %s %s" % \
                        (item.itsKind.itsName,
                         operation,
                         stampClass)
                logging.info(message)
            stampItem = stampClass(item)
            getattr(stampItem, operation)() # i.e. stampClass(item).add() or
                                            # stampClass(item).remove()
            self.assertAttributes(item)
            if operation == 'add':
                self.failUnless(has_stamp(item, stampClass))

    def testStamping(self):
        view = self.sandbox.itsView

        # Get the stamp kinds
        mailStamp = Mail.MailStamp
        taskStamp = pim.TaskStamp
        eventStamp = Calendar.EventStamp
        noteKind = pim.Note.getKind(view)

        # start out with a Note
        aNote = pim.Note("noteItem1", itsParent=self.sandbox)
        self.setAttributes(aNote, doWho=False)
        self.assertAttributes(aNote)
        add = 'add'
        remove = 'remove'

        # stamp everything on and off the note
        self.traverseStampSquence(aNote, ((add, mailStamp),
                                          (add, taskStamp),
                                          (add, eventStamp),
                                          (remove, eventStamp),
                                          (remove, taskStamp),
                                          (remove, mailStamp)))

        # stamp everything on again, remove in a different order
        self.traverseStampSquence(aNote, ((add, mailStamp),
                                          (add, taskStamp),
                                          (add, eventStamp),
                                          (remove, mailStamp),
                                          (remove, taskStamp),
                                          (remove, eventStamp)))
        self.assertAttributes(aNote)

        # Create a Task, and do all kinds of stamping on it
        aTask = pim.Task("aTask", itsParent=self.sandbox).itsItem
        self.setAttributes(aTask)

        self.traverseStampSquence(aTask, ((add, eventStamp),
                                          (remove, taskStamp)))
        # now it's an Event

        self.traverseStampSquence(aTask, ((add, mailStamp),
                                          (remove, mailStamp)))

        self.traverseStampSquence(aTask, ((add, mailStamp),
                                          (add, taskStamp),
                                          (remove, mailStamp),
                                          (remove, taskStamp)))

        self.traverseStampSquence(aTask, ((add, taskStamp),
                                          (add, mailStamp),
                                          (remove, mailStamp),
                                          (remove, taskStamp)))

        self.traverseStampSquence(aTask, ((add, mailStamp),
                                          (remove, eventStamp)))
        # now it's a Mail

        self.traverseStampSquence(aTask, ((add, taskStamp),
                                          (remove, mailStamp)))
        # it's a Task again

        self.traverseStampSquence(aTask, ((add, mailStamp),
                                          (remove, taskStamp)))

        self.traverseStampSquence(aTask, ((add, taskStamp),
                                          (remove, mailStamp)))
        # it's a Task again

        self.traverseStampSquence(aTask, ((add, eventStamp),
                                          (remove, taskStamp),
                                          (add, mailStamp),
                                          (remove, eventStamp),
                                          (add, taskStamp),
                                          (remove, mailStamp)))
        self.failUnless(has_stamp(aTask, taskStamp))

        # check stamping on an Event
        anEvent = Calendar.CalendarEvent("anEvent", itsParent=self.sandbox).itsItem
        self.setAttributes(anEvent)

        # round-robin its Kind back to event
        self.traverseStampSquence(anEvent, ((add, mailStamp),
                                            (remove, eventStamp),
                                            (add, taskStamp),
                                            (remove, mailStamp),
                                            (add, eventStamp),
                                            (remove, taskStamp)))
        self.failUnless(has_stamp(anEvent, eventStamp))

        # check stamping on a Mail Message
        aMessage = Mail.MailMessage("aMessage", itsParent=self.sandbox).itsItem
        self.setAttributes(aMessage)
        self.traverseStampSquence(aMessage, ((add, eventStamp),
                                             (add, taskStamp),
                                             (remove, eventStamp),
                                             (remove, taskStamp)))
        self.failUnless(has_stamp(aMessage, mailStamp))

        # now mixin some arbitrary Kind
        #anotherKind = view.findPath('//parcels/osaf/framework/blocks/Block')

        # stamp an event, mail, task with another kind
        #aNote.StampKind(add, anotherKind)
        #aTask.StampKind(add, anotherKind)
        #anEvent.StampKind(add, anotherKind)
        #aMessage.StampKind(add, anotherKind)

        #self.assertKinds(aNote, (noteKind, anotherKind))
        #self.assertKinds(aTask, (taskKind, anotherKind))
        #self.assertKinds(anEvent, (eventKind, anotherKind))
        #self.assertKinds(aMessage, (mailKind, anotherKind))

        # unstamp with another kind
        #aNote.StampKind(remove, anotherKind)
        #aTask.StampKind(remove, anotherKind)
        #anEvent.StampKind(remove, anotherKind)
        #aMessage.StampKind(remove, anotherKind)

        # see that they still have their attributes
        #self.assertKinds(aNote, (noteKind, ))
        #self.assertKinds(aTask, (taskKind, ))
        #self.assertKinds(anEvent, (eventKind, ))
        #self.assertKinds(aMessage, (mailKind, ))

        # Test some failure cases
        # These cases should produce suitable warning messages in Chandler.log
        if testFailureCases:
            anotherEvent = Calendar.CalendarEvent("anotherEvent", itsParent=self.sandbox).itsItem
            self.setAttributes(anotherEvent)
            self.failUnless(has_stamp(anotherEvent, eventStamp))
            # Could use assertRaises here, but it's syntax with respect to parameters is
            #   not clear with my complex arguments, so try/except/else is more readable.
            try:
                # double stamping
                self.traverseStampSquence(anotherEvent, ((add, mailStamp),
                                                         (add, mailStamp)))
            except StampAlreadyPresentError:
                pass
            else:
                self.failUnless(False, "Double stamping should raise an exception!")

            try:
                # unstamping something not present
                self.traverseStampSquence(anotherEvent, ((remove, taskStamp), ))
            except StampNotPresentError:
                pass
            else:
                self.failUnless(False, "Unstamping a stamp not present should raise an exception!")
            # Test for Bug:6151: Make sure items don't disappear
            # from the all collection if they're unstamped
            #
            # Make an email ...
            aMessage = Mail.MailMessage("aNewMessage", itsParent=self.sandbox)
            self.setAttributes(aMessage.itsItem)

            # Make sure it's in "Out"
            aMessage.fromMe = True
            outCollection = schema.ns("osaf.pim", view).outCollection

            self.failUnless(aMessage.itsItem in outCollection)

            # unstamp its emailness
            self.traverseStampSquence(aMessage.itsItem,
                                      (('add', taskStamp),
                                      ('remove', mailStamp)))

            allCollection = schema.ns("osaf.pim", view).allCollection
            self.failUnless(aMessage.itsItem in allCollection)

    def testStampWelcomeNote(self):

        # Look up the welcome note ...
        welcome = schema.ns("osaf.app", self.view).WelcomeEvent

        # stamp it as a mail ...
        stampedWelcome = Mail.MailStamp(welcome)
        stampedWelcome.add()

        self.failUnlessEqual(list(stampedWelcome.mimeContent.mimeParts), [])
        self.failUnlessEqual(stampedWelcome.mimeContent.mimeType,
                             'message/rfc822')

class S1(pim.Stamp):
    schema.kindInfo(annotates=pim.ContentItem)

class S2(pim.Stamp):
    schema.kindInfo(annotates=pim.ContentItem)

class S3(pim.Stamp):
    schema.kindInfo(annotates=pim.ContentItem)

class StampMergeTestCase(testcase.SharedSandboxTestCase):

    def setUp(self):
        super(StampMergeTestCase, self).setUp()
        item = pim.ContentItem(itsParent=self.sandbox, displayName=u'Item')
        schema.itemFor(S1, self.sandbox.itsView)
        self.view.commit()
        self.view2 = self.view.repository.createView('Woo')
        self.uuid = item.itsUUID

    def tearDown(self):
        self.view2.cancel()
        super(StampMergeTestCase, self).tearDown()

    def testAdd(self):
        item1 = self.view.find(self.uuid)
        item2 = self.view2.find(self.uuid)

        S1(item1).add()
        item1.itsView.commit()

        S2(item2).add()
        item2.itsView.commit()

        self.failUnlessEqual(
            set(pim.Stamp(item2).stamp_types),
            set([S1, S2])
        )

    def testRemove(self):
        item1 = self.view.find(self.uuid)
        item2 = self.view2.find(self.uuid)

        S1(item1).add()
        S2(item1).add()
        item1.itsView.commit()
        item2.itsView.refresh()

        S1(item1).remove()
        item1.itsView.commit()

        S2(item2).remove()
        item2.itsView.commit()

        self.failUnlessEqual(
            set(pim.Stamp(item2).stamp_types),
            set(),
        )

    def testRemoveAdd(self):
        item1 = self.view.find(self.uuid)
        item2 = self.view2.find(self.uuid)

        S1(item1).add()
        S2(item1).add()
        item1.itsView.commit()
        item2.itsView.refresh()

        S3(item1).add()
        item1.itsView.commit()

        S2(item2).remove()
        item2.itsView.commit()

        self.failUnlessEqual(
            set(pim.Stamp(item2).stamp_types),
            set([S1, S3])
        )

def additional_tests():
    return unittest.TestSuite(
        [ test_stamping_doc(), ]
    )

if __name__ == "__main__":
    # Just using unittest.main() here isn't good enough, since
    # that wouldn't pick up the tests in additional_tests().
    # For that, the magic ScanningLoader() below is needed.
    from util import test_finder
    unittest.main(testLoader=test_finder.ScanningLoader())


# -*- coding: utf-8 -*-

import os
import zipfile

INCOMING_FILE = 'icons.zip'  # относительный путь к файлу
PATH_NORMALIZED = os.path.normpath(INCOMING_FILE)  # нормализация пути
DESTINATION_FOLDER = 'files_by_year'  # папка назначения

# Скрипт раскладывает файлы из zip архива по годам и месяцам в другую папку. Новая папка создается на этом же уровне.
#   результирующая папка
#       files_by_year/2018/05/cat.jpg
#       files_by_year/2018/05/man.jpg
#       files_by_year/2017/12/new_year_01.jpg

class ZipFileOrganizer:

    def __init__(self, incoming_file, output_folder):
        self.incoming_file_name = incoming_file
        self.output_folder = output_folder

    def sorting_files(self):
        with zipfile.ZipFile(self.incoming_file_name, 'r') as zf:
            # Проходим только по файлам
            for file in zf.infolist():
                if file.is_dir():  # не берем в работу каталоги
                    continue
                parent_dir = os.path.dirname(PATH_NORMALIZED)  # родительская директория входящего файла
                year = str(file.date_time[0])
                month = str(file.date_time[1])
                # оставляем у файла только базовое имя и убираем весь его путь в архиве,
                # чтобы при извлечении он не наложился на директорию назначения
                file.filename = os.path.basename(file.filename)
                # новый путь назначения для файла: родительская_директория/папка_назначения/год/месяц
                new_dir_name = os.path.join(parent_dir, self.output_folder, year, month)
                os.makedirs(new_dir_name, exist_ok=True)  # рекурсивное создание директории
                zf.extract(file, new_dir_name)  # извлекаем файл в новую папку
        print('Файлы перемещены')

icon_arrange = ZipFileOrganizer(incoming_file=PATH_NORMALIZED, output_folder=DESTINATION_FOLDER)
icon_arrange.sorting_files()


import math
import time
import random
import os
from typing import List, Tuple
import shapely
from shapely.geometry import Point, Polygon, LineString, GeometryCollection
from shapely import affinity
from shapely.ops import split, nearest_points
import geopandas as gpd
import matplotlib.pyplot as plt
import numpy as np

def longest_edge(poly):
    """
    Finds the longest edge in a polygon

    Parameters
    ----------
    poly: Polygon
        The polygon that we want to find the longest edge of

    Returns
    -------
    start: Point
        First point of the longest edge

    end: Point
        Other point of the longest edge
    """
    poly_coords = list(poly.exterior.coords)
    max_length = 0
    prev_p = Point(poly_coords[-1])
    for p in poly_coords:
        curr_p = Point(p)
        length = curr_p.distance(prev_p)
        if length > max_length:
            max_length = length
            start = prev_p
            end = curr_p

        prev_p = curr_p

    return start, end

def get_farthest_point(arc, base_poly, remaining_empty_space):
    """
    Find the point on a given arc that is farthest away from the base polygon.
    In other words, the point on which the largest circle can be drawn without going outside the base polygon.

    Parameters
    ----------
    arc: Polygon
        The arc in question

    base_poly: Polygon
        The base polygon

    remaining_empty_space: Polygon
        The polygon representing the space left to be filled in the base polygon

    ax: matplotlib Axes
        Used for plotting

    fig: matplotlib Figure
        Used for plotting

    Returns
    -------
    farthest_point: Point
        The point on the arc that is farthest away from the base polygon

    longest_distance: float
        How far away the polygon is from the farthest point

    point_on_poly: Point
        The point on the base polygon that is closest to the arc
    """
    longest_distance = 0
    farthest_point = Point([0, 0])

    # Handle input for polygons and LineString
    # The first arc begins on a LineString rather than a Polygon
    if arc.geom_type == 'Polygon':
        arc_coords = arc.exterior.coords
    elif arc.geom_type == 'LineString':
        arc_coords = np.linspace(list(arc.coords)[0], list(arc.coords)[1])
    else:
        print('get_farthest_distance: Wrong shape type given')

    # For every point in the arc, find out which point is farthest away from the base polygon
    for p in list(arc_coords):
        distance = Point(p).distance(base_poly)
        if (distance > longest_distance) and ((remaining_empty_space.buffer(1e-9).contains(Point(p)))):
            longest_distance = distance
            farthest_point = Point(p)

    point_on_poly = nearest_points(base_poly, farthest_point)[0]
    return farthest_point, longest_distance, point_on_poly

def move_toward_point(start_point, target_point, distance):
    """Moves a point a set distance toward another point"""

    # Calculate the direction in which to move
    dx = target_point.x - start_point.x
    dy = target_point.y - start_point.y

    # Normalize the direction
    magnitude = (dx**2 + dy**2)**0.5
    dx /= magnitude
    dy /= magnitude

    # Move the point in the direction of the target by the set distance
    return Point(start_point.x + dx*distance, start_point.y + dy*distance)

def get_boundary_line(poly, p1):
    """
    Find the geometry that the arcs will build out to approach.
    In this implementation, it is simply the base_poly without the starting line
    Arcs cannot start from, or terminate on this line.
    """
    poly_coords = poly.exterior.coords
    for i, p in enumerate(poly_coords):
        # find the starting point in the polygon
        if p == list(p1.coords)[0]:
            # arrange the boundary line so first coordinate is at one end, and the final coordinate is at the end
            base_poly_1 = poly_coords[:i+1]
            base_poly_2 = poly_coords[i+1:]
            base_poly_2 += base_poly_1
            return base_poly_2

def create_circle(x, y, radius, n):
    """
    Create a circle
    - with center at point (x,y)
    - with specified radius
    - using n segments
    """
    return Polygon([[radius*np.sin(theta)+x, radius*np.cos(theta)+y] for theta in np.linspace(0, 2*np.pi - 2*np.pi/n, int(n))])

def create_rect(x, y, length, width, from_center):
    """
    Create a rectangle either from the center or from the bottom-left corner
    """

    if from_center:
        return Polygon([(x - width/2, y - length/2),
                        (x - width/2, y + length/2),
                        (x + width/2, y + length/2),
                        (x + width/2, y - length/2)])
    else:
        return Polygon([(x, y),
                        (x, y + length),
                        (x + width, y + length),
                        (x + width, y)])

def image_number(list_of_files):
    if list_of_files:
        return str(int(list_of_files[-1][:-4])+1)
    else:
        return "1"

def get_exterior(poly):

    #extract just the first polygon if there's ever a multipolygon or geometry collection
    for geom in getattr(poly, 'geoms', [poly]):
        if geom.geom_type == 'Polygon':
            poly = geom
            break

    return [
        coord
        for geom in getattr(poly, 'geoms', [poly])
        for coord in geom.exterior.coords
    ]

def create_arc(circle, remaining_empty_space, ax, depth):
    """
    Turns a circle into an arc

    Parameters
    ----------
    circle: Polygon
        The circle to convert into an arc

    remaining_empty_space: Polygon
        The polygon representing the space left to be filled in the base polygon

    ax: matplotlib Axes
        Used for plotting

    depth: int
        How deep are we into recursion? Used for rainbow coloring based on depth.

    Returns
    -------
    Polygon
        A properly oriented arc shape. (looks like a "D") This will be the path the nozzle takes.
    """

    # Turn "moon" shaped arcs into D shapes
    crescent = circle.intersection(remaining_empty_space)

    # Remove all the points in the concave section of the crescent shape, turning it into a "D" shape instead
    crescent_exterior = get_exterior(crescent)

    # Plot the arcs. Comment the following 2 lines to make the code run faster
    crescent_geoseries = gpd.GeoSeries(Polygon(crescent_exterior))
    crescent_geoseries.plot(ax=ax[0], color='none', edgecolor='black', linewidth=1) # set color='none' for black and white plotting
    crescent_geoseries.plot(ax=ax[1], color=num_to_rgb(depth), edgecolor='black', linewidth=1) # set color='none' for black and white plotting

    empty_exterior = get_exterior(remaining_empty_space)

    arc = []
    for coord in crescent_exterior:
        if (not coord in empty_exterior) and (not coord in arc):
            arc.append(coord)

    if len(arc) == 0:
        print("CIRCLE COMPLETELY ENGULFED")
        return None

    elif len(arc) <= 2:
        arc = Polygon(crescent_exterior).intersection(remaining_empty_space)

    else:
        arc = Polygon(arc)

    # Make sure first point is on one corner "D", and last point is on the other.
    # This makes sure the arcs start from one end, and go all the away around to the other

    start, _ = longest_edge(arc)
    fixed_arc = get_boundary_line(arc, start)

    return Polygon(fixed_arc)

def arc_overhang(arc, boundary, starting_line_angle, n, prev_poly, prev_circle, threshold, ax, fig, depth,
                 filename_list, r_max, min_arcs, line_width, gcode_file, layer_height, filament_diameter, e_multiplier, feedrate):
    """
    Main recursive function (I'm deeply sorry for the number of arguments here.)

    TODO collapse all the hardcoded parameters into a single parameter

    Parameters
    ----------
    circle: Polygon
        The circle to convert into an arc

    remaining_empty_space: Polygon
        The polygon representing the space left to be filled in the base polygon

    ax: matplotlib Axes
        Used for plotting

    depth: int
        How deep are we into recursion? Used for rainbow coloring based on depth.
    """
    # Find the next center point of arc, the radius of the new arc, and the closest point on the boundary to the center point.
    next_point, r_final, _ = get_farthest_point(arc, boundary, prev_poly)
    # Limit maximum circle size
    r_final = min(r_final - threshold, r_max)

    small_arc_radius = 0.5  #1

    # Overlap arc with the previous one.
    circle_moved = False
    if r_final > small_arc_radius:
        next_point = move_toward_point(next_point, prev_circle.centroid, 0) # change 0 to any distance in mm to move the arcs
        circle_moved = True

    # Update the current boundary polygon to include the previous circle
    remaining_empty_space = prev_poly.difference(prev_circle)

    # Create multiple layers
    r = line_width
    if r > r_final:
        print("WARNING: r", r, "should not be bigger than r_final", r_final)

    while r < r_final:
        # Create a circle based on point location, radius, n
        next_circle = create_circle(next_point.x, next_point.y, r, n)
        next_circle = affinity.rotate(next_circle, starting_line_angle, origin = 'centroid', use_radians=True)

        # Plot arc
        next_arc = create_arc(next_circle, remaining_empty_space, ax, depth)
        if not next_arc:
            r += line_width
            continue
        curr_arc = Polygon(next_arc)
        longest_distance = r

        #Slow down and reduce flow for all small arcs
        if circle_moved and r < small_arc_radius:
            speed_modifier = 0.25
            e_modifier = 0.25
        elif r <= line_width:
            speed_modifier = 0.25
            e_modifier = 0.1
        else:
            speed_modifier = 1
            e_modifier = 1

        # Write gcode
        if r_final > 0:
            write_gcode(gcode_file, next_arc, line_width, layer_height, filament_diameter, e_multiplier*e_modifier, feedrate*speed_modifier, False)

        r += line_width
        # Create image
        #filename = image_number(filename_list)
        #plt.savefig(filename, dpi=200)
        #filename_list.append(filename + ".png")

    remaining_empty_space = remaining_empty_space.difference(next_circle)

    next_point, longest_distance, _ = get_farthest_point(curr_arc, boundary, remaining_empty_space)
    branch = 0
    # Create new arcs on the same base arc until no more points on the base arc are farther than the threshold distance.
    while (longest_distance > threshold + min_arcs*line_width):
        branch += 1

        #Create a new arc on curr_arc
        next_arc, remaining_empty_space, filename_list = arc_overhang(
            curr_arc, boundary, starting_line_angle, n, remaining_empty_space, next_circle, threshold, ax, fig, depth + 1,
            filename_list, r_max, min_arcs, line_width, gcode_file, layer_height, filament_diameter, e_multiplier, feedrate)

        # Get the farthest distance between curr_arc and the boundary
        next_point, longest_distance, closest_point_on_poly = get_farthest_point(
            curr_arc, boundary, remaining_empty_space)

        # Optional: Draw lines showing distance to outer polygon
        #if list(next_point.coords)[0] != (0,0):
        #    nextPointGeoSeries = gpd.GeoSeries(LineString([next_point, closest_point_on_poly]))
        #    nextPointGeoSeries.plot(ax=ax[0], color='green', linewidth=1)

    print("Depth = ", depth, "Arcs this layer", branch)
    return next_arc, remaining_empty_space, filename_list

def write_gcode(file_name, arc, line_width, layer_height, filament_diameter, e_multiplier, feedrate, close_loop):
    ## TODO try using circles instead of D shapes for better surface quality
    ## TODO use a dict or something to reduce # parameters
    #line_width = print_settings["line_width"]
    #layer_height = print_settings["layer_height"]
    #e_multiplier = print_settings["e_multiplier"]
    #filament_diameter = print_settings["filament_diameter"]
    #feedrate = print_settings["feedrate"]
    feedrate_travel = feedrate * 20
    feedrate_printing = feedrate

    #extract just the first polygon if there's ever a multipolygon or geometry collection
    for geom in getattr(arc, 'geoms', [arc]):
        if geom.geom_type == 'Polygon':
            arc = geom
            break

    with open(file_name, 'a') as gcode_file:

        for geom in getattr(arc, 'geoms', [arc]):
            if geom.geom_type == 'LineString':
                coord_list = arc.coords

            elif geom.geom_type == 'Polygon':
                if close_loop == False:
                    coord_list = arc.exterior.coords[:-1]
                else:
                    coord_list = arc.exterior.coords

        first_coord = True
        prev_coordinate = coord_list[0]
        for coordinate in coord_list:
            if not first_coord and coordinate == prev_coordinate:
                continue
            else:
                first_coord = False
            # Calculate extrusion amount
            # Extrusion number = height of cylinder with equal volume to amount of filament required
            distance = Point(coordinate).distance(Point(prev_coordinate))
            volume = line_width * layer_height * distance
            e_distance = e_multiplier * volume / (3.1415 * (filament_diameter / 2)**2)

            if e_distance <= 0.0001:
                feedrate = feedrate_travel
                gcode_file.write("G1 E-1 F1500\n") # retract
            else:
                feedrate = feedrate_printing

            gcode_file.write(f"G0 "
                            f"X{'{0:.3f}'.format(coordinate[0])} "
                            f"Y{'{0:.3f}'.format(coordinate[1])} "
                            f"E{'{0:.8f}'.format(e_distance)} "
                            f"F{feedrate*60}\n")

            if e_distance <= 0.000001:
                feedrate = feedrate_travel
                gcode_file.write("G1 E1 F1500\n") # deretract

            prev_coordinate = coordinate
    return

def num_to_rgb(val, max_val=10):
    i = (val * 255 / max_val)
    r = round(math.sin(0.024 * i + 0) * 127 + 128)
    g = round(math.sin(0.024 * i + 2) * 127 + 128)
    b = round(math.sin(0.024 * i + 4) * 127 + 128)
    r /= 256
    g /= 256
    b /= 256
    return (r, g, b)

def generate_polygon(center: Tuple[float, float], avg_radius: float,
                     irregularity: float, spikiness: float,
                     num_vertices: int) -> List[Tuple[float, float]]:
    """
    Start with the center of the polygon at center, then creates the
    polygon by sampling points on a circle around the center.
    Random noise is added by varying the angular spacing between
    sequential points, and by varying the radial distance of each
    point from the centre.

    Args:
        center (Tuple[float, float]):
            a pair representing the center of the circumference used
            to generate the polygon.
        avg_radius (float):
            the average radius (distance of each generated vertex to
            the center of the circumference) used to generate points
            with a normal distribution.
        irregularity (float):
            variance of the spacing of the angles between consecutive
            vertices.
        spikiness (float):
            variance of the distance of each vertex to the center of
            the circumference.
        num_vertices (int):
            the number of vertices of the polygon.
    Returns:
        List[Tuple[float, float]]: list of vertices, in CCW order.
    """
    # Parameter check
    if irregularity < 0 or irregularity > 1:
        raise ValueError("Irregularity must be between 0 and 1.")
    if spikiness < 0 or spikiness > 1:
        raise ValueError("Spikiness must be between 0 and 1.")

    irregularity *= 2 * math.pi / num_vertices
    spikiness *= avg_radius
    angle_steps = random_angle_steps(num_vertices, irregularity)

    # now generate the points
    points = []
    angle = random.uniform(0, 2 * math.pi)
    for i in range(int(num_vertices)):
        radius = clip(random.gauss(avg_radius, spikiness), 0, 2 * avg_radius)
        point = (center[0] + radius * math.cos(angle),
                 center[1] + radius * math.sin(angle))
        points.append(point)
        angle += angle_steps[i]

    return points

def random_angle_steps(steps: int, irregularity: float) -> List[float]:
    """Generates the division of a circumference in random angles.

    Args:
        steps (int):
            the number of angles to generate.
        irregularity (float):
            variance of the spacing of the angles between consecutive vertices.
    Returns:
        List[float]: the list of the random angles.
    """
    # generate n angle steps
    angles = []
    lower = (2 * math.pi / steps) - irregularity
    upper = (2 * math.pi / steps) + irregularity
    cumsum = 0
    for i in range(int(steps)):
        angle = random.uniform(lower, upper)
        angles.append(angle)
        cumsum += angle

    # normalize the steps so that point 0 and point n+1 are the same
    cumsum /= (2 * math.pi)
    for i in range(int(steps)):
        angles[i] /= cumsum
    return angles

def clip(value, lower, upper):
    """
    Given an interval, values outside the interval are clipped to the interval
    edges.
    """
    return min(upper, max(value, lower))


import operator
import shlex
from functools import reduce

from django.apps import AppConfig
from django.contrib.postgres.indexes import GinIndex
from django.contrib.postgres.search import (CombinedSearchQuery, SearchQuery,
                                            SearchRank, SearchVector,
                                            SearchVectorField)
from django.db import models
from django.db.models.signals import class_prepared, post_init, post_save
from django.dispatch import receiver

from django_searchable.exceptions import SearchableException

class SearchableTextField(models.TextField):

    vector_name = None

    def contribute_to_class(self, cls, name):
        if not cls._meta.abstract:
            self.vector_name = f'{name}_vector'

            # make sure the field doesn't already exist with this name
            if hasattr(cls, self.vector_name):
                raise SearchableException(
                    'Error trying to create vector field. '
                    f'Field with name "{self.vector_name}" already exists'
                )

            vector_field = SearchVectorField(f'SearchVector storage for {name}', null=True)
            vector_field.contribute_to_class(cls, self.vector_name)
            # add index
            cls._meta.indexes.append(GinIndex(fields=[self.vector_name]))

        super(SearchableTextField, self).contribute_to_class(cls, name)

class SearchableQuerySet(models.QuerySet):

    def _generate_search_field_names(self, fields=None):
        for field in self.model.get_searchable_fields():
            if fields is None or field.name in fields:
                yield (field.vector_name, f'{field.name}_rank')

    def _get_query(self, query):
        # allow passing in of complex query objects
        if isinstance(query, (CombinedSearchQuery, SearchQuery)):
            return query

        if isinstance(query, list):
            terms = query
        else:
            try:
                terms = shlex.split(query)
            except ValueError:
                terms = query.replace("'", '').replace('"', '').split()

        return reduce(operator.or_, [SearchQuery(term) for term in terms])

    def search(self, query, fields=None):
        '''
        query is either a string or list of terms or SearchQuery object
        '''
        query = self._get_query(query)
        ranks = {}
        rank = 0
        for vector_name, rank_name in self._generate_search_field_names(fields=fields):
            ranks[rank_name] = SearchRank(models.F(vector_name), query)
            rank += models.F(rank_name)

        return self.annotate(
            **ranks,
        ).annotate(
            rank=rank,  # all ranks added together
        ).filter(
            rank__gt=0.0,
        ).order_by('-rank')

class SearchableModel(models.Model):

    objects = SearchableQuerySet.as_manager()

    class Meta:
        abstract = True

    @classmethod
    def get_searchable_fields(cls):
        for field in cls._meta.get_fields():
            if isinstance(field, SearchableTextField):
                yield field

    def save(self, *args, **kwargs):
        Model = type(self)
        previous = Model.objects.get(id=self.id) if self.id else None
        super(SearchableModel, self).save(*args, **kwargs)

        updates = {'id': self.id}
        for field in self.get_searchable_fields():
            if previous is None or getattr(previous, field.name) != getattr(self, field.name):
                updates[field.vector_name] = SearchVector(field.name)

        if len(updates) > 1:
            Model.objects.update(**updates)