from application_logger import logger
import os, shutil
import pandas as pd
import deepdiff
import re, glob

# builder pattern for rawdatavalidator
class ValidatorBuilder:
    def __init__(self, type_of_data):
        self.validator = RawDataValidator(type_of_data)

    def add_filepath(self, filepath):
        self.validator.file_path = filepath
        return self

    def add_filename(self, filename):
        self.validator.filename = filename
        return self

    def add_no_of_columns(self, number):
        self.validator.number_of_columns = number
        return self

    def add_filename_pattern(self, pattern):
        self.validator.filename_pattern = pattern
        return self

    def add_length_of_datestamp(self, length_of_datestamp):
        self.validator.length_of_datestamp = length_of_datestamp
        return self

    def add_length_of_timestamp(self, length_of_timestamp):
        self.validator.length_of_timestamp = length_of_timestamp
        return self

    def add_column_name(self, name):
        self.validator.columns_name = name
        return self

    def add_logger(self, logger):
        self.validator.logger = logger
        return self

    def build(self):
        return self.validator

class RawDataValidator:

    def __init__(self, type_of_data):
        self.type_of_data = type_of_data

    #phising_08012020_120000.csv
    def validate_file_properties(self):
        try:
            self.logger.log("validation_logs", "validation.log", "info", "Validation of the Training Data Started..")
            if re.match(self.filename_pattern, self.filename):
                splitting_at_dot = self.filename.split(".")
                name, date_stamp, time_stamp = splitting_at_dot[0].split('_')
                _error = False

                client_dic = { }
                client_file = pd.read_csv(self.file_path + self.filename)

                for column in client_file.columns:
                    if client_file[column].dtype == "O":
                        client_dic[column] = "OBJECT"
                    else :
                        client_dic[column] = "INTEGER"

                if splitting_at_dot[1] != "csv":
                    self.logger.log("validation_logs", "validation.log", "error", f"Filename extension is not .csv. Moving the file into bad directory of {self.type_of_data}")
                    _error = True

                if name != "phising" or len(time_stamp) != self.length_of_timestamp or len(date_stamp) != self.length_of_datestamp:
                    self.logger.log("validation_logs", "validation.log", "error", f"Filename schema is not valid. Moving the file into bad directory {self.type_of_data}")
                    _error = True

                if self.number_of_columns != len(pd.read_csv(self.file_path + self.filename).columns):
                    print(self.number_of_columns != len(pd.read_csv(self.file_path + self.filename).columns))
                    self.logger.log("validation_logs", "validation.log", "error", f"Number of Columns in the file are not valid. Moving the file into bad Directory {self.type_of_data}")
                    _error = True

                if len(deepdiff.DeepDiff(self.columns_name, client_dic)) > 0:
                    print(deepdiff.DeepDiff(self.columns_name, client_dic))
                    self.logger.log("validation_logs", "validation.log", "error", f"Number of Columns in the file are not valid. Moving the file into bad Directory {self.type_of_data}")
                    _error = True

                if _error == False:
                    self.logger.log("validation_logs", "validation.log", "info", f"Filename schema is valid. Moving the file into good directory {self.type_of_data}")

                    if self.type_of_data == "training":
                        shutil.copy(glob.glob(self.file_path + self.filename)[0], 'data_segregation/training/good_data/')
                    else:
                        shutil.copy(glob.glob(self.file_path + self.filename)[0], 'data_segregation/testing/good_data/')

                else:
                    if self.type_of_data == "training":
                        shutil.copy(glob.glob(self.file_path + self.filename)[0], 'data_segregation/training/bad_data/')
                    else :
                        shutil.copy(glob.glob(self.file_path + self.filename)[0], 'data_segregation/testing/bad_data/')

            else:
                if self.type_of_data == "training":
                #  print(file.read())
                    shutil.copy(glob.glob(self.file_path + self.filename)[0], 'data_segregation/training/bad_data/')
                else:
                    shutil.copy(glob.glob(self.file_path + self.filename)[0], 'data_segregation/testing/bad_data/')

                self.logger.log("validation_logs", "validation.log", "error", f"Filename schema is not valid. Moving the file into bad directory {self.type_of_data}")

        except Exception as error:
            self.logger.log("validation_logs", "validation.log", "error", f"Error during the validation of the training data {error}")

        else:
            self.logger.log("validation_logs", "validation.log", "info", f"Completed the raw data validation")


# -*- coding: utf-8 -*-
import math
import random
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sopt.SGA import SGA

from utils.MyCount import C

def Compare(a,b):
    length = len(a)
    Mysum = 0
    for i in range(length):
        temp= (a[i]-b[i])*(a[i]-b[i])
        Mysum += temp
    com = math.sqrt(Mysum)
    return com

def algorithm_function(beta):
    anomaly = []
    normal = []
    Beta = beta[0]
    for i in dataset:
        temp = []
        for item in centroids:
            temp.append(Compare(i,item))
        if 10*min(temp) > Beta:
            anomaly.append(i)
        if 10*min(temp) < Beta:
            normal.append(i)

    score = len(normal)/(len(normal)+len(anomaly))
    print(score)
    Main_anomaly = anomaly
    Main_normal = normal
    mycount.vaildCount(anomaly,normal)
    if 0 < score <= 0.96:
        return score
    elif score > 0.97:
        return 0.9
    else:
        return 0.1

if __name__ == "__main__":
    mycount = C()
    # dataset = np.random.rand(500, 2)  # 随机生成500个二维[0,1)平面点
    X_outliers = np.random.uniform(low=-4, high=4, size=(20, 2))
    X = 0.3 * np.random.randn(100, 2)

    Y = 0.5 * np.random.randn(100, 2)

    # Z = 0.8 * np.random.randn(100, 2)
    # Y = np.append(X-2,Y,axis=0)
    # Y = np.append(X+1.2,Z-2.5,axis=0)
    X_train = np.r_[X + 2.3, Y -1]

    dataset = np.append(X_train,X_outliers,axis=0)

    for i in dataset:
        plt.scatter(i[0],i[1], c='g')
    plt.xlim(xmax=5,xmin=-5)
    plt.ylim(ymax=5,ymin=-5)
    plt.show()

    # estimator = KMeans(n_clusters=3)#构造聚类器
    # estimator.fit(dataset)#聚类
    # label_pred = estimator.labels_ #获取聚类标签
    # centroids = estimator.cluster_centers_ #获取聚类中心
    # import matplotlib.pyplot as plt
    # plt.figure(figsize=(8,10))
    # colors = ['b', 'g','r']
    # markers = ['o', 's','d']
    # for i,l in enumerate(label_pred):
    #     plt.plot(dataset[i][0],dataset[i][1],color=colors[l],marker=markers[l],ls='None')
    # print(centroids)
    # plt.plot(centroids[0][0],centroids[0][1],color = 'r',marker = 'o',ls='None')
    # plt.plot(centroids[1][0],centroids[1][1],color = 'r',marker = 'o',ls='None')
    # plt.show()

    # import matplotlib.pyplot as plt
    # from scipy.spatial.distance import cdist
    # K=range(1,10)
    # meandistortions=[]
    # for k in K:
    #     kmeans=KMeans(n_clusters=k)
    #     kmeans.fit(dataset)
    #     meandistortions.append(sum(np.min(cdist(
    #         X,kmeans.cluster_centers_,"euclidean"),axis=1))/X.shape[0])
    # K_list = {}
    # print(meandistortions)
    # for i in range(0,len(meandistortions)):
    #     K_list[i] = meandistortions[i-1]- meandistortions[i]
    #
    # K_list = sorted(K_list.items(), key=lambda item:item[1], reverse=True)
    # a = K_list[0]
    # plt.plot(K,meandistortions,'bx-')
    # plt.xlabel('k')
    # plt.ylabel(u'平均畸变程度')
    # plt.title('best k is '+ str(a[0]+1))
    # plt.show()

    #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    Main_anomaly = []
    Main_normal = []
    sga = SGA.SGA(func = algorithm_function,func_type = 'max',variables_num = 2,
                  lower_bound = [1,2],upper_bound = [10,5],generations = 2,
                  binary_code_length = 10)
    sga.run()
    algorithm_function(sga.global_best_point)

    # normal_df, outlier_df = mycount.getInfo()
    Main_anomaly = mycount.final_anomalys_DF
    Main_normal = mycount.final_nomalys_DF
    # print('anomaly',Main_anomaly)
    # print('normal', Main_normal)

    import matplotlib.pyplot as plt
    plt.figure()
    for i in Main_anomaly:
        print(i)
        plt.scatter(i[0],i[1],c='r')
    for i in Main_normal:
        plt.scatter(i[0],i[1],c='g')
    plt.show()


from django.urls import path
from .views import *

urlpatterns = [
    # root urls
    path('creator_dashboard/', creatorDashboardPageView, name='creator_dashboard'),

    # world urls
    path('create_realm/', createRealmPageView, name='create_realm'),
    path('view_realm/', viewRealmPageView, name='view_realm'),
    path('edit_realm/', editRealmPageView, name='edit_realm'),
    # character urls
    path('view_character/', characterPageView, name='view_character'),
    path('edit_character/', editCharacterPageView, name='edit_character'),
    # plot arc urls
    path('view_plot_arc/', viewPlotArcPageView, name='view_plot_arc'),
    path('edit_plot_arc/', editPlotArcPageView, name='edit_plot_arc'),
    # other page urls
    path('view_other_page/', viewOtherPageView, name='view_other_page'),
    path('edit_other_page/', editOtherPageView, name='edit_other_page'),
]


import pyaudio
import wave
import os
import time
import sys
import board
import paho.mqtt.client as mqtt
from datetime import datetime
import json
from array import array

FORMAT=pyaudio.paInt16
CHANNELS=2
RATE=44100
CHUNK=100024
RECORD_SECONDS=15

THINGSBOARD_HOST = 'iot.ceisufro.cl'
ACCESS_TOKEN = '####'

# Data capture and upload interval in seconds. Less interval will eventually hang the DHT22.
INTERVAL=2

sensor_data = {'estado': False,'ruido':0, 'fecha':'date'}

next_reading = time.time()

client = mqtt.Client()

# Set access token
client.username_pw_set(ACCESS_TOKEN)

# Connect to ThingsBoard using default MQTT port and 60 seconds keepalive interval
#client.connect(THINGSBOARD_HOST, 8080, 60)
client.connect(THINGSBOARD_HOST, 1883, 60)

client.loop_start()

audio=pyaudio.PyAudio() #instantiate the pyaudio

        #recording prerequisites
stream=audio.open(format=FORMAT,channels=CHANNELS,
                  rate=RATE,
                  input=True,
                  frames_per_buffer=CHUNK)
frames=[]

SERVER = 'smtp-mail.outlook.com'
FROM = "proyectomonitordci@outlook.com"
TO = ["proyectomonitordci@yopmail.com"]

SUBJECT = "Alerta!"
TEXT = "Se registra un ruido muy fuerte\n Revisar: http://localhost:8081/"

message = """From: %s\r\nTo: %s\r\nSubject: %s\r\n\

%s
""" % (FROM, ", ".join(TO), SUBJECT, TEXT)

# Send the mail
import smtplib
try:
    server = smtplib.SMTP('smtp-mail.outlook.com', 587)
except Exception as e:
    print(e)
    server = smtplib.SMTP_SSL('smtp-mail.outlook.com', 465)
server.ehlo()
server.starttls()
server.login("proyectomonitordci@outlook.com", "PASSWORD")

while True:
    for i in range(0,int(RATE/CHUNK*RECORD_SECONDS)):

        data=stream.read(CHUNK)
        data_chunk=array('h',data)
        vol=max(data_chunk)
        if(vol>=16000):
            estado = True
            ruido=vol
            print("Se registra ruido")
            sensor_data['estado'] = estado
            sensor_data['ruido'] = vol
            fecha = datetime.now()
            fecha = fecha.strftime("%b %d %Y %H:%M:%S")
            sensor_data['fecha'] = fecha
            print(u"Estado: {:g}, Ruido: {:g}, Fecha: ".format(estado,ruido))
            print(fecha)
            print(message)
            server.sendmail(FROM, TO, message)

            frames.append(data)
            client.publish('v1/devices/me/telemetry', json.dumps(sensor_data), 1)
            next_reading += INTERVAL
            sleep_time = next_reading-time.time()
            if sleep_time > 0:

                    time.sleep(sleep_time)
        else:
            estado=False
            print("nothing")
        print("\n")

client.loop_stop()
client.disconnect()


import json

filename = "themes/qtlike-dark-theme.json"
# filename = "themes/test.json"

settings = 'settings'
foreground = 'foreground'
fontStyle = 'fontStyle'
scope = 'scope'

class MergeResult:

    def __init__(self, success: bool, merged: dict):
        self.success = success
        self.merged = merged

    success: bool = False
    merged: dict

def make_list(input) -> list:
    if (not isinstance(input, list)):
        input.replace(" ", "")
        return input.split(",")

    return input

def merge_scope_tag(lhs: dict, rhs: dict) -> dict:
    lhs[scope] = make_list(lhs[scope])
    rhs[scope] = make_list(rhs[scope])
    lhs[scope] = lhs[scope] + rhs[scope]
    return lhs

def compare_scope_tag(lhs: dict, rhs: dict) -> bool:

    settings1 = lhs[settings]
    settings2 = rhs[settings]
    if (settings1[foreground] == settings2[foreground]):

        if ((fontStyle in settings1) and (fontStyle in settings2)):  # must match
            if (settings1[fontStyle] == settings2[fontStyle]):
                return True

            return False
        elif (not ((fontStyle in settings1) or (fontStyle in settings2))):  # nothing to match
            return True
        else:  # one has, other doesn't
            return False

    return False

def compare_and_merge(lhs: dict, rhs: dict) -> MergeResult:

    if (compare_scope_tag(lhs, rhs)):
        return MergeResult(True, merge_scope_tag(lhs, rhs))

    return MergeResult

def merge_all_scopes(objects: list) -> list:

    i = 0

    while (i < len(objects)):

        j = i + 1

        while (j < len(objects)):

            later_element = objects[j]
            result = compare_and_merge(objects[i], later_element)

            if (result.success):
                objects[i] = result.merged
                objects.pop(j)
            else:
                j = j + 1

        i = i + 1

    return objects

with open(filename, 'r') as file:

    # Read the json file into a dictionary
    objects = json.load(file)

    # Sort all of the members of "tokenColors" based on the "settings" : "foreground" parameter
    tokenColors = objects['tokenColors']
    tokenColors.sort(key=lambda x: x[settings][foreground])

    # Merge all of the scopes together so that everything with the same colour is in the same object
    merged = merge_all_scopes(tokenColors)

with open(f'{filename} totally merged.json', 'x') as out_file:

    # Save all of the sorted information into a file
    text = json.dumps(merged)
    out_file.write(text)


from django.shortcuts import redirect, render
from create.models import Level, Lesson, Entry
from .models import AudioClip
from create.models import Entry
from pydub import AudioSegment, playback
from pydub.silence import split_on_silence
from itertools import chain

def dash(request):
    selected_level = request.POST.get('level')

    entries = Entry.objects.all()

    for entry in entries:
        if entry.info:
            print(entry.eng)
    print('done')
    if request.POST:
        gap = request.POST.get('gap')
        volume = request.POST.get('volume')
        selected_lesson = request.POST.get('lesson')
        mp3 = request.FILES.get('mp3')
        if mp3:
            sound = AudioSegment.from_mp3(mp3)
            audio_chunks = split_on_silence(sound, min_silence_len=int(gap), silence_thresh=-int(volume))
            counter = 1
            for i, chunk in enumerate(audio_chunks):
                output_file = f"static/temp/chunk{counter}.mp3"
                print("Exporting file", output_file)
                chunk.export(output_file, format="mp3")
                counter += 1
            return redirect('audio:match-audio', level=selected_level, lesson=selected_lesson,chunks=counter)
        lessons = Lesson.objects.filter(level=selected_level)
        selected_level = int(selected_level)

    else:
        lessons = []

    levels = Level.objects.all()
    context = {
        'levels': levels,
        'lessons': lessons,
        'selected_level': selected_level
    }
    return render(request, 'audio/dash.html', context)

def match_audio(request, level, lesson, chunks):
    lesson_obj = Lesson.objects.get(pk=lesson)
    if request.POST:
        names = request.POST.getlist('file-name')
        files = request.FILES.getlist('clip')
        ids = []
        i = 0
        for clip in files:
            clip.name = names[i]
            AudioClip.objects.create(file=clip)
            split_name = names[i].split('-')
            entry_id = split_name[0]
            ids.append(int(entry_id))
            i += 1

        entries_with_audio = []
        for id in ids:
            total = ids.count(id)
            if total == 2:
                if id not in entries_with_audio:
                    entries_with_audio.append(id)
        for id in entries_with_audio:
            Entry.objects.filter(pk=id).update(audio='true')
        return redirect('audio:dash')

    urls = []
    num_of_chunks = list(range(1, chunks))

    for i in num_of_chunks:
        url = f'/static/temp/chunk{i}.mp3'
        urls.append(url)
        i += 1
    if lesson_obj.type == 'dialogue':
        entries = Entry.objects.filter(id__in=lesson_obj.entries)
    else:
        entries = Entry.objects.filter(id__in=lesson_obj.entries).filter(audio='false')
        for entry in entries:
            try: alt_ids = entry.alts[str(lesson)]
            except: alt_ids = []
            if alt_ids:
                alts = Entry.objects.filter(id__in=alt_ids).filter(audio='false')
                entries = list(chain(entries, alts))

    context = {
        'level': Level.objects.get(level=level),
        'lesson': lesson_obj,
        'entries': entries,
        'urls': urls,

    }
    return render(request, 'audio/match-audio.html', context)

def get_alts(entries):
    for id in entries:
        entry = Entry.objects.get(pk=id)
        print(entry.alts)


import argparse
import os
from Data import get_available_collections, get_collection

def main():

    parser = argparse.ArgumentParser(description='PairsDataPrep')

    parser.add_argument('--collection', type=str, required=True, help=f'{get_available_collections()}')
    parser.add_argument('--set', type=str, default='test')
    parser.add_argument('--output_dir', type=str, required=True)
    parser.add_argument('--queries_path', type=str, required=False,
                            help='The path to the test queries .tsv file: q_id, query')
    parser.add_argument('--run_path', type=str, required=False,
                            help='The path to the run file .tsv file : q_id, doc_id, score, rank, judgement.')
    parser.add_argument('--collection_path', type=str, required=False,
                            help='The path to the documents .tsv file: doc_id, title, body.')
    parser.add_argument('--set_name', type=str, required=True,
                            help='Name of the experiment.')
    parser.add_argument('--num_eval_docs', default=1000, type=int,
                            help='The number of documents retrieved per query.')

    parser.add_argument('--dataset_path', type=str, required=False)
    parser.add_argument('--do_udel', action='store_true', default=False)

    args, other = parser.parse_known_args()

    if not os.path.exists(args.output_dir):
        os.makedirs(args.output_dir)

    col = get_collection(args.collection)

    prep = col.get_prep()

    if args.set in ('test', 'dev'):
        prep.convert_eval_dataset(args)
    elif args.set == 'train':
        prep.convert_train_dataset(args)
    else :
        raise ValueError("Set must be in ['train', 'dev', 'test] !")

if __name__ == '__main__':
    main()


import logging

import sqlalchemy_dao
from sqlalchemy_dao import Dao

from SmartAnno.utils.ConfigReader import ConfigReader
from SmartAnno.db.ORMs import Filter
from SmartAnno.gui.Workflow import Workflow
from SmartAnno.utils.AnnotationTypeDef import AnnotationTypeDef
from SmartAnno.utils.IntroStep import IntroStep
from SmartAnno.utils.KeywordsFiltering import KeywordsFiltering
from SmartAnno.utils.KeywordsUMLSExtender import KeywordsUMLSExtender
from SmartAnno.utils.KeywordsUMLSExtenderSetup import KeywordsUMLSExtenderSetup

logging.getLogger().setLevel(logging.DEBUG)

ConfigReader('../conf/smartanno_conf.json')

wf = Workflow(config_file=ConfigReader.config_file)
wf.api_key = ConfigReader.getValue("api_key")
wf.dao = Dao('sqlite+pysqlite:///../data/test.sqlite', sqlalchemy_dao.POOL_DISABLED)
wf.task_name = 'language'
wf.append(AnnotationTypeDef(
    '
Annotation types:

List all the types you want to identify below. Each type per line.
If you'
    'have too many types, try set up them separately, so that you won't need to choose from a long list '
    'for each sample.
', name='types'))
wf.append(KeywordsFiltering(
    name='keywords'))
wf.append(KeywordsUMLSExtenderSetup(name='umls_extender_setup'))
wf.append(KeywordsUMLSExtender(name='umls_extender', sources=ConfigReader.getValue("umls/sources"),
                               filter_by_length=ConfigReader.getValue("umls/filter_by_length"),
                               filter_by_contains=ConfigReader.getValue("umls/filter_by_contains"),
                               max_query=ConfigReader.getValue("umls/max_query")))
wf.append(
    IntroStep('
Welcome to SmartAnno!

First, let's import txt data from a directory.
',
              name='intro'))
wf.start()

wf.steps[0].complete()

with wf.dao.create_session() as session:
    records = session.query(Filter).filter(Filter.task_id == wf.task_id) \
        .filter(Filter.type_name == 'Eng')
    record = records.first()
    record.keyword = 'Eng\nEnglish'
    session.commit()

wf.steps[1].complete()
wf.steps[2].to_umls_ext_filters['Eng'].value = ('English',)
wf.steps[2].complete()
wf.steps[3].loop_workflow.steps[0].selections.value = ('Walnut Nut',)
wf.steps[3].loop_workflow.steps[0].complete()

with wf.dao.create_session() as session:
    records = session.query(Filter).filter(Filter.task_id == wf.task_id) \
        .filter(Filter.type_name == 'Eng')
    record = records.first()
    print(record.keyword)
    assert (record.keyword == 'Eng\nEnglish\nWalnut Nut')
    record.keyword = 'Eng\nEnglish'


from flask import Flask,request,render_template,Response
import json
import foxess
app = Flask(__name__)

@app.route('/soc')
def solar_soc():
    push_variables = ['SoC']
    token = foxess.get_token()
    print(token)
    variables = push_variables
    print(variables)
    var_string = '"' + '","'.join(variables) + '"'
    data = foxess.get_raw(token, var_string, days=0, allday=False, timezone=1)['result']

    if not data[0]['data']:
        data = foxess.get_raw(token, var_string, days=0, allday=False, timezone=0)['result']
    try:
        latest_data = {r['variable']:r['data'][-1]['value'] for r in data if r['variable'] in push_variables}
        response = json.dumps(latest_data['SoC'])
    except(TypeError):
        response = '50'

    print(response)
    return Response(response,200)

@app.route('/net')
def solar_net():
    push_variables = ['loadsPower','pvPower','batChargePower']
    token = foxess.get_token()
    print(token)
    variables = push_variables
    print(variables)
    var_string = '"' + '","'.join(variables) + '"'
    data = foxess.get_raw(token, var_string, days=0, allday=False, timezone=1)['result']
    if not data[0]['data']:
        data = foxess.get_raw(token, var_string, days=0, allday=False, timezone=0)['result']

    print(data)
    try:
        latest_data = {r['variable']:r['data'][-1]['value'] for r in data if r['variable'] in push_variables}
        print(latest_data)
        response = float(json.dumps(latest_data['pvPower']))-float(json.dumps(latest_data['loadsPower']))

    except(TypeError):
        response = '1.8'

    print(response)

    response = str(int(max(min((float(response)/1.8),1),-1)*50)+50)
    print(response)
    return Response(response,200)

if __name__ == "__main__":

    print(solar_soc())
    print(solar_net())


from app.models import db, Watchlist, environment, SCHEMA
from sqlalchemy.sql import text

# Adds a watchlist, you can add other users here if you want
def seed_watchlists():
    watchlist1 = Watchlist(
        list_name = "Tech Meme Stocks",
        user_id = 1
    )
    watchlist2 = Watchlist(
        list_name = "Dividend Stocks",
        user_id = 1
    )
    watchlist3 = Watchlist(
        list_name = "Stuff I believe In",
        user_id = 1
    )

    all_watchlists = [watchlist1, watchlist2, watchlist3]

    for watchlist in all_watchlists:
        db.session.add(watchlist)

    db.session.commit()

def undo_watchlists():
    if environment == "production":
        db.session.execute(f"TRUNCATE table {SCHEMA}.watchlists RESTART IDENTITY CASCADE;")
    else:
        db.session.execute(text("DELETE FROM watchlists"))

    db.session.commit()


from typing import Any, Callable, Dict, List, Optional, Union, Iterable

import cloudpickle
import pandas as pd
from fugue import transform
from pycaret.internal.tabular import (
    _append_display_container,
    pull,
    _create_display,
    _get_context_lock,
)
from pycaret.internal.Display import Display
from threading import RLock
from math import ceil
import random
from .parallel_backend import NoDisplay, ParallelBackend

try:
    import fugue_dask
except Exception:
    pass

try:
    import fugue_spark
except Exception:
    pass

class _DisplayUtil:
    def __init__(
        self, display: Optional[Display], progress: int, verbose: bool, sort: str
    ):
        self._lock = RLock()
        self._display = display or _create_display(
            progress, verbose=verbose, monitor_rows=None
        )
        self._sort = sort
        self._df: Optional[pd.DataFrame] = None
        self._display.display_progress()

    def update(self, df: pd.DataFrame) -> None:
        with self._lock:
            if self._df is None:
                self._df = df
            else:
                self._df = pd.concat([self._df, df]).sort_values(
                    self._sort, ascending=False
                )
            self._display.move_progress(df.shape[0])
            self._display.replace_master_display(self._df)
            self._display.display_master_display()

    def finish(self) -> None:
        self._display.display_master_display()

class FugueBackend(ParallelBackend):
    """
    Fugue Backend for PyCaret

    Example
    -------

    >>> from pycaret.datasets import get_data
    >>> from pyspark.sql import SparkSession
    >>> from pycaret.parallel import FugueBackend
    >>> juice = get_data('juice')
    >>> from pycaret.classification import *
    >>> exp_name = setup(data = juice,  target = 'Purchase')
    >>> spark = SparkSessiong.builder.getOrCreate()
    >>> best_model = compare_models(parallel=FugueBackend(spark))

    engine: Any, default = None
        A ``SparkSession`` instance or "spark" to get the current ``SparkSession``.
        "dask" to get the current Dask client. "native" to test locally using single
        thread. None means using Fugue's Native execution engine to run jobs
        sequentially on local machine, it is for testing/development purpose.

    conf: Any, default = None
        Fugue ExecutionEngine configs.

    batch_size: int, default = 1
        Batch size to partition the tasks. For example if there are 16 tasks,
        and with ``batch_size=3`` we will have 6 batches to be processed
        distributedly. Smaller batch sizes will have better load balance but
        worse overhead, and vise versa.

    display_remote: bool, default = False
        Whether show progress bar and metrics table when using a distributed
        backend. By setting it to True, you must also enable the
        `callback settings <https://fugue-project.github.io/tutorials/tutorials/advanced/rpc.html>`_
        to receive realtime updates.

    top_only: bool, default = False
        Whether only return the top ``n_select`` models from each worker. When top only,
        the overall execution time can be faster.
    """

    def __init__(
        self,
        engine: Any = None,
        conf: Any = None,
        batch_size: int = 1,
        display_remote: bool = False,
        top_only: bool = False,
    ):
        super().__init__()
        self._engine = engine
        self._conf: Dict[str, Any] = conf or {}
        self._batch_size = batch_size
        self._display_remote = display_remote
        self._top_only = top_only
        self._func: Optional[Callable] = None
        self._params: Optional[Dict[str, Any]] = None

    def __getstate__(self) -> Dict[str, Any]:
        res = dict(self.__dict__)
        del res["_engine"]
        return res

    def compare_models(
        self, func: Callable, params: Dict[str, Any]
    ) -> Union[Any, List[Any]]:
        self._func = func
        self._params = dict(params)
        assert "include" in self._params
        assert "sort" in self._params

        shuffled_idx = pd.DataFrame(
            dict(
                idx=random.sample(
                    range(len(self._params["include"])), len(self._params["include"])
                )
            )
        )
        du: Optional[Display] = (
            None
            if not self._display_remote
            else _DisplayUtil(
                self._params.get("display", None),
                progress=shuffled_idx.shape[0],
                verbose=self._params.get("verbose", False),
                sort=self._params["sort"],
            )
        )
        outputs = transform(
            shuffled_idx,
            self._remote_compare_models,
            schema="output:binary",
            partition={
                "num": ceil(shuffled_idx.shape[0] / self._batch_size),
                "algo": "even",
            },
            engine=self._engine,
            engine_conf=self._conf,
            callback=None if du is None else du.update,
            force_output_fugue_dataframe=True,
            as_local=True,
        ).as_array()
        res = pd.concat(cloudpickle.loads(x[0]) for x in outputs)
        res = res.sort_values(self._params["sort"], ascending=False)
        top = res.head(self._params.get("n_select", 1))
        _append_display_container(res.iloc[:, :-1])
        top_models = [cloudpickle.loads(x) for x in top._model]
        if du is not None:
            du.finish()
        return top_models[0] if len(top_models) == 1 else top_models

    def _remote_compare_models(
        self, idx: List[List[Any]], report: Optional[Callable]
    ) -> List[List[Any]]:
        include = [self._params["include"][i[0]] for i in idx]
        self.remote_setup()
        params = dict(self._params)
        params.pop("include")
        params["display"] = NoDisplay()
        results: List[List[Any]] = []
        with _get_context_lock():
            top = (
                min(params.get("n_select", 1), len(include))
                if self._top_only
                else len(include)
            )
            params["n_select"] = top
            m = self._func(include=include, **params)
            if not isinstance(m, list):
                m = [m]
            res = pull()[:top]
            if report is not None:
                report(res)
            results.append(
                [
                    cloudpickle.dumps(
                        res.assign(_model=[cloudpickle.dumps(x) for x in m])
                    )
                ]
            )
        return results


from datetime import datetime, timedelta

import numpy as np
import pandas as pd
from pyomo.environ import (
    ConcreteModel,
    ConstraintList,
    Objective,
    Reals,
    Var,
    maximize,
    quicksum,
)
from pyomo.opt import SolverFactory, check_available_solvers

from assume.common.base import BaseStrategy, SupportsMinMaxCharge
from assume.common.market_objects import MarketConfig, Orderbook, Product

def shift(prc, type_: str = "first"):
    """
    Shifts the price curve up or down

    :param prc: price curve
    :type prc: np.array
    :param type_: type of shift. default is 'first'
    :type type_: str
    :return: shifted price curve
    :rtype: np.array
    """
    new_prices, num_prices = [], len(
        prc
    )  # new_prices is the list of prices after the shift

    for p, index in zip(prc, range(num_prices)):
        if type_ == "first":
            new_prices += [
                p * (0.9 + index * 0.2 / num_prices)
            ]  # calculate the new price by multiplying the old price with a factor
        else:
            new_prices += [p * (1.1 - index * 0.2 / num_prices)]

    return np.asarray(new_prices)

def shaping(prc, type_: str = "peak"):
    """
    Shifts the price curve up or down

    :param prc: price curve
    :type prc: np.array
    :param type_: type of shift. default is 'peak'. other options are 'pv' and 'demand'.
    :type type_: str
    :return: shifted price curve
    :rtype: np.array
    """
    if type_ == "peak":
        prc[
            8:20
        ] *= 1.1  # between 8 and 20 hours, the price is multiplied by 1.1 (general production is high)
    elif type_ == "pv":
        prc[
            10:14
        ] *= 0.9  # between 10 and 14 hours, the price is multiplied by 0.9 (Pv production is high)
    elif type_ == "demand":
        prc[
            6:9
        ] *= 1.1  # between 6 and 9 hours, the price is multiplied by 1.1 (demand is high)
        prc[17:20] *= 1.1  # between 17 and 20 hours, the price is multiplied by 1.1
    return prc

PRICE_FUNCS = {
    "left": lambda prc: np.roll(prc, -1),
    "right": lambda prc: np.roll(prc, 1),
    "normal": lambda prc: prc,
    # 'first': lambda prc: shift(prc, type_='first'),
    # 'last': lambda prc: shift(prc, type_='last'),
    # 'peak_off_peak': lambda prc: shaping(prc, type_='peak'),
    "pv_sink:": lambda prc: shaping(prc, type_="pv"),
    "demand": lambda prc: shaping(prc, type_="demand"),
}

def get_solver_factory(solvers_str=["glpk", "cbc", "gurobi", "cplex"]):
    """
    Returns the first available solver from the list of solvers

    :param solvers_str: list of solvers
    :type solvers_str: list
    :return: solver factory
    :rtype: SolverFactory
    """
    solvers = check_available_solvers(*solvers_str)
    if len(solvers) < 1:
        raise Exception(f"None of {solvers_str} are available")
    return SolverFactory(solvers[0])

class DmasStorageStrategy(BaseStrategy):
    """
    Strategy for a storage unit that uses DMAS to optimize its operation
    """

    def __init__(self, *args, **kwargs):
        """
        Initializes the strategy

        :param args: arguments
        :type args: list
        :param kwargs: keyword arguments
        :type kwargs: dict"""
        super().__init__(*args, **kwargs)

        self.model = ConcreteModel("storage")
        self.opt = get_solver_factory()

    def build_model(self, unit: SupportsMinMaxCharge, start: datetime, hour_count: int):
        """
        Builds the optimization model

        :param unit: unit to dispatch
        :type unit: SupportsMinMaxCharge
        :param start: start time
        :type start: datetime
        :param hour_count: number of hours to optimize
        :type hour_count: int
        :return: power
        :rtype: np.array"""
        self.model.clear()
        time_range = range(hour_count)

        self.model.p_plus = Var(
            time_range, within=Reals, bounds=(0, -unit.max_power_charge)
        )
        self.model.p_minus = Var(
            time_range, within=Reals, bounds=(0, unit.max_power_discharge)
        )
        self.model.volume = Var(time_range, within=Reals, bounds=(0, unit.max_volume))

        self.power = [
            -self.model.p_minus[t] / unit.efficiency_discharge
            + self.model.p_plus[t] * unit.efficiency_charge
            for t in time_range
        ]

        self.model.vol_con = ConstraintList()
        soc0 = unit.get_soc_before(start)
        v0 = unit.max_volume * soc0

        for t in time_range:
            if t == 0:
                self.model.vol_con.add(self.model.volume[t] == v0 + self.power[t])
            else:
                self.model.vol_con.add(
                    self.model.volume[t] == self.model.volume[t - 1] + self.power[t]
                )

        # always end with half full SoC
        self.model.vol_con.add(self.model.volume[hour_count - 1] == unit.max_volume / 2)
        return self.power

    def optimize_result(self, unit: SupportsMinMaxCharge, committed_power: np.array):
        """
        Optimizes the result

        :param unit: unit to dispatch
        :type unit: SupportsMinMaxCharge
        :param committed_power: committed power
        :type committed_power: np.array
        :return: optimization result
        :rtype: pyomo.opt.results.SolverResults
        """
        # if day ahead result is known minimize the difference
        bid_count = len(committed_power)
        time_range = range(bid_count)

        self.model.power_difference = Var(time_range, within=Reals)
        self.model.minus = Var(time_range, within=Reals, bounds=(0, None))
        self.model.plus = Var(time_range, within=Reals, bounds=(0, None))

        difference = [committed_power[t] - self.power[t] for t in time_range]

        self.model.difference = ConstraintList()
        for t in time_range:
            self.model.difference.add(
                self.model.plus[t] - self.model.minus[t] == difference[t]
            )
        abs_difference = [self.model.plus[t] + self.model.minus[t] for t in time_range]
        costs = [
            abs_difference[t] * np.abs(unit.forecaster["price_EOM"][t] * 2)
            for t in time_range
        ]

        profit = [
            -self.power[t] * unit.forecaster["price_EOM"][t] - costs[t]
            for t in time_range
        ]
        self.model.obj = Objective(
            quicksum(profit[t] for t in time_range), sense=maximize
        )
        r = self.opt.solve(self.model)
        return r

    def optimize(
        self,
        unit: SupportsMinMaxCharge,
        start: datetime,
        hour_count: int,
    ):
        """
        Optimizes the unit operation

        :param unit: unit to dispatch
        :type unit: SupportsMinMaxCharge
        :param start: start time
        :type start: datetime
        :param hour_count: number of hours to optimize
        :type hour_count: int
        :return: optimization results
        :rtype: dict
        """
        opt_results = {key: np.zeros(hour_count) for key in PRICE_FUNCS.keys()}
        time_range = range(hour_count)

        base_price = unit.forecaster["price_EOM"][
            start : start + timedelta(hours=hour_count)
        ]

        for key, func in PRICE_FUNCS.items():
            prices = func(base_price.values)
            self.power = self.build_model(unit, start, hour_count)
            profit = [-self.power[t] * prices[t] for t in time_range]
            self.model.obj = Objective(
                expr=quicksum(profit[t] for t in time_range), sense=maximize
            )
            r = self.opt.solve(self.model)
            power = np.asarray(
                [
                    -self.model.p_minus[t].value * unit.efficiency_discharge
                    + self.model.p_plus[t].value
                    for t in time_range
                ]
            )
            profit = [-power[t] * prices[t] for t in time_range]
            volume = np.asarray([self.model.volume[t].value for t in time_range])
            opt_results[key] = power
            if key == "normal":
                end = start + unit.index.freq * (hour_count - 1)

                # unit.outputs["total"][self.power < 0] = -self.power[self.power < 0]
                # unit.outputs["demand"][self.power > 0] = self.power[self.power > 0]
                # unit.outputs["storage"] = self.power
                unit.outputs["volume"].loc[start:end] = volume
                unit.outputs["generation"].loc[start:end] = power
                unit.outputs["profit"].loc[start:end] = profit
        return opt_results

    def calculate_bids(
        self,
        unit: SupportsMinMaxCharge,
        market_config: MarketConfig,
        product_tuples: list[Product],
        **kwargs,
    ) -> Orderbook:
        """
        Takes information from a unit that the unit operator manages and
        defines how it is dispatched to the market

        Returns a list of bids that the unit operator will submit to the market
        :param unit: unit to dispatch
        :type unit: SupportsMinMaxCharge
        :param market_config: market configuration
        :type market_config: MarketConfig
        :param product_tuples: list of products to dispatch
        :type product_tuples: list[Product]
        :param kwargs: additional arguments
        :type kwargs: dict
        :return: bids
        :rtype: Orderbook
        """
        assert "exclusive_id" in market_config.additional_fields
        start = product_tuples[0][0]
        end = product_tuples[-1][1]
        hour_count = (end - start) // timedelta(hours=1)
        opt_results = self.optimize(unit, start, hour_count)
        total_orders = {}
        block_id = 0
        power_prices = unit.forecaster["price_EOM"][
            start : start + timedelta(hours=hour_count)
        ]
        for key, power in opt_results.items():
            prc = np.zeros(hour_count)
            bid_hours = np.argwhere(power < 0).flatten()
            ask_hours = np.argwhere(power > 0).flatten()
            if len(bid_hours) > 1:
                max_charging_price = power_prices.values[bid_hours].max()
            else:
                max_charging_price = 0
            min_discharging_price = max_charging_price / (
                unit.efficiency_discharge * unit.efficiency_discharge
            )
            prc[ask_hours] = (power_prices.iloc[ask_hours] + min_discharging_price) / 2
            prc[bid_hours] = power_prices.values[bid_hours]
            add = True
            for orders in total_orders.values():
                if any(prc != orders["price"]) or any(power != orders["volume"]):
                    add = False
            if add:
                total_orders[block_id] = dict(price=prc, volume=power)
                block_id += 1
        dfs = []
        time_range = range(hour_count)
        for block_id, bids in total_orders.items():
            df = pd.DataFrame(data=bids)
            df["bid_id"] = unit.id
            df["exclusive_id"] = block_id
            df["hour"] = time_range
            df = df.set_index(["exclusive_id", "hour", "bid_id"])
            dfs.append(df)

        bids = pd.concat(dfs)

        if not bids.empty:
            bids = bids.reset_index()
            bids["start_time"] = bids.apply(
                lambda o: start + timedelta(hours=o["hour"]), axis=1
            )
            bids["end_time"] = bids.apply(
                lambda o: start + timedelta(hours=o["hour"]) + unit.index.freq, axis=1
            )
            del bids["hour"]
        return bids.to_dict("records")


from binance.client import Client

import random

import keys
import tests

client = Client(keys.api_key, keys.api_secret)

# Проверяем корректность вхожных данных
def validate_input(data):
    if not isinstance(data, dict):
        raise ValueError("Input data should be a dictionary")

    required_keys = ["volume", "number", "amountDif", "side", "priceMin", "priceMax"]
    for key in required_keys:
        if key not in data:
            raise ValueError(f"Missing required key '{key}' in input data.")

    if not isinstance(data["volume"], (int, float)) or data["volume"] <= 0:
        raise ValueError("Volume must be a positive number")

    if not isinstance(data["number"], int) or data["number"] <= 0:
        raise ValueError("Number of orders must be a positive integer")

    if not isinstance(data["amountDif"], (int, float)) or data["amountDif"] <= 0:
        raise ValueError("Amount difference must be a positive number")

    if data["side"] not in ["BUY", "SELL"]:
        raise ValueError("Side must be either 'BUY' or 'SELL'")

    if not isinstance(data["priceMin"], (int, float)) or data["priceMin"] < 0:
        raise ValueError("Minimum price must be a non-negative number")

    if not isinstance(data["priceMax"], (int, float)) or data["priceMax"] <= 0:
        raise ValueError("Maximum price must be a positive number")

    if data["priceMax"] <= data["priceMin"]:
        raise ValueError("Maximum price must be greater than minimum price")

    return True

def create_order(symbol, side, quantity, price):
    order = client.create_order(
        symbol=symbol,
        side=side,
        type='LIMIT',
        timeInForce='GTC',
        quantity=quantity,
        price=price
    )
    return order

def create_orders(data):
    volume = data['volume']
    number = data['number']
    amount_dif = data['amountDif']
    side = data['side']
    price_min = data['priceMin']
    price_max = data['priceMax']

    # Не продолжаем работу с некорректными данными
    validate_input(data)

    # Разбиваем общий объем на равные части для каждого ордера
    volume_per_order = volume / number

    for i in range(number):
        # Генерируем случайные цены и объемы для ордеров
        price = round(random.uniform(price_min, price_max), 2)
        volume_with_dif = round(random.uniform(volume_per_order - amount_dif, volume_per_order + amount_dif), 2)

        # Округляем объем до шага торговли
        step_size = client.get_symbol_info('BTCUSDT')['filters'][1]['stepSize']
        quantity = round(volume_with_dif / price, int(step_size.find('1') - 1))

        order = create_order('BTCUSDT', side, quantity, price)
        print(f'Created order: {order}')

# проверяем тесты
create_orders(tests.test1)
create_orders(tests.test2)
create_orders(tests.test3)
create_orders(tests.test4)
create_orders(tests.test5)
create_orders(tests.test6)
create_orders(tests.test7)
create_orders(tests.test8)
create_orders(tests.test9)
create_orders(tests.test10)


from django.core.management.base import BaseCommand
from faker import Faker
import faker.providers
from resources.models import InstitutionResource
from institutions.models import Institution, Department
import random
import decimal
from django.contrib.auth import authenticate, login

TYPE = [
    "published",
    "draft",
]
DEPARTMENT = Department.objects.all()
INSTITUTIONS = Institution.objects.all()

class Provider(faker.providers.BaseProvider):
    def department(self):
        return self.random_element(DEPARTMENT)

    def institution(self):
        return self.random_element(INSTITUTIONS)

    def workspace_type(self):
        return self.random_element(TYPE)

class Command(BaseCommand):
    help = "Command Information"

    def handle(self, *args, **kwargs):

        fake = Faker(["en_US"])
        fake.add_provider(Provider)

        for _ in range(5):

            name = fake.unique.bs()
            description = fake.unique.sentence()
            status = fake.workspace_type()
            institution = fake.institution()
            department = fake.department()

            InstitutionResource.objects.create(
                name=name, description=description, status=status, institution=institution, department=department
            )
            print(name, status, institution, department)


import sqlite3
from contextlib import contextmanager
from inspect import Parameter
from typing import Any, Callable, Iterator, List, Optional, Tuple, Union

from molten import (
    HTTP_201, HTTP_204, HTTP_403, HTTP_404, App, Component, Field, Header, HTTPError, Include,
    Middleware, Request, ResponseRendererMiddleware, Route, schema
)
from molten.openapi import HTTPSecurityScheme, Metadata, OpenAPIHandler, OpenAPIUIHandler

def AuthorizationMiddleware(handler: Callable[..., Any]) -> Callable[..., Any]:
    def middleware(request: Request, authorization: Optional[Header]) -> Any:
        if authorization != "Bearer secret" and request.path not in ["/_docs", "/_schema"]:
            raise HTTPError(HTTP_403, {"error": "forbidden"})
        return handler()
    return middleware

class DB:
    def __init__(self) -> None:
        self._db = sqlite3.connect(":memory:")
        self._db.row_factory = sqlite3.Row

        with self.get_cursor() as cursor:
            cursor.execute("create table todos(description text, status text)")

    @contextmanager
    def get_cursor(self) -> Iterator[sqlite3.Cursor]:
        cursor = self._db.cursor()

        try:
            yield cursor
            self._db.commit()
        except Exception:
            self._db.rollback()
            raise
        finally:
            cursor.close()

class DBComponent:
    is_cacheable = True
    is_singleton = True

    def can_handle_parameter(self, parameter: Parameter) -> bool:
        return parameter.annotation is DB

    def resolve(self) -> DB:
        return DB()

@schema
class Todo:
    id: Optional[int] = Field(response_only=True)
    description: str
    status: str = Field(choices=["todo", "done"], default="todo")

class TodoManager:
    def __init__(self, db: DB) -> None:
        self.db = db

    def create(self, todo: Todo) -> Todo:
        with self.db.get_cursor() as cursor:
            cursor.execute("insert into todos(description, status) values(?, ?)", [
                todo.description,
                todo.status,
            ])

            todo.id = cursor.lastrowid
            return todo

    def get_all(self) -> List[Todo]:
        with self.db.get_cursor() as cursor:
            cursor.execute("select rowid as id, description, status from todos")
            return [Todo(**data) for data in cursor.fetchall()]

    def get_by_id(self, todo_id: int) -> Optional[Todo]:
        with self.db.get_cursor() as cursor:
            cursor.execute("select rowid as id, description, status from todos where rowid = ? limit 1", [todo_id])
            data = cursor.fetchone()
            if data is None:
                return None

            return Todo(**data)

    def delete_by_id(self, todo_id: int) -> None:
        with self.db.get_cursor() as cursor:
            cursor.execute("delete from todos where rowid = ?", [todo_id])

class TodoManagerComponent:
    is_cacheable = True
    is_singleton = True

    def can_handle_parameter(self, parameter: Parameter) -> bool:
        return parameter.annotation is TodoManager

    def resolve(self, db: DB) -> TodoManager:
        return TodoManager(db)

def list_todos(manager: TodoManager) -> List[Todo]:
    return manager.get_all()

def get_todo(todo_id: str, manager: TodoManager) -> Todo:
    todo = manager.get_by_id(int(todo_id))
    if todo is None:
        raise HTTPError(HTTP_404, {"error": f"todo {todo_id} not found"})
    return todo

def create_todo(todo: Todo, manager: TodoManager) -> Tuple[str, Todo]:
    return HTTP_201, manager.create(todo)

def delete_todo(todo_id: str, manager: TodoManager) -> Tuple[str, None]:
    manager.delete_by_id(int(todo_id))
    return HTTP_204, None

components: List[Component] = [
    DBComponent(),
    TodoManagerComponent(),
]

middleware: List[Middleware] = [
    ResponseRendererMiddleware(),
    AuthorizationMiddleware,
]

get_docs = OpenAPIUIHandler()

get_schema = OpenAPIHandler(
    metadata=Metadata(
        title="Todo API",
        description="An API for managing todos.",
        version="0.0.0",
    ),
    security_schemes=[
        HTTPSecurityScheme("default", "bearer"),
    ],
    default_security_scheme="default",
)

routes: List[Union[Route, Include]] = [
    Include("/v1/todos", [
        Route("/", list_todos),
        Route("/", create_todo, method="POST"),
        Route("/{todo_id}", get_todo),
        Route("/{todo_id}", delete_todo, method="DELETE"),
    ]),

    Route("/_docs", get_docs),
    Route("/_schema", get_schema),
]

app = App(
    components=components,
    middleware=middleware,
    routes=routes,
)


import numpy as np
from PIL import Image
import matplotlib.pyplot as plt
import cv2 as cv

def kmeans(img, k, tol = 0.001, max_iter = 300):
    img = pil_to_np(img)

    shape = img.shape
    img = img.reshape((-1,3))
    img = np.float32(img)

    criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, max_iter, 1.0)
    comp, label, centers = cv.kmeans(img, k, None, criteria, 10, cv.KMEANS_RANDOM_CENTERS)

    centers = np.uint8(centers)
    res = centers[label.flatten()]
    res = res.reshape((shape))
    return Image.fromarray(normalize(res).astype('uint8'))

if __name__ == '__main__':
    import sys
    from toolbox import *

    if len(sys.argv) >= 2:
        img = sys.argv[1]
        k = int(sys.argv[2]) if len(sys.argv) >= 3 else 5
        tol = float(sys.argv[3]) if len(sys.argv) >= 4 else 0.001
        max_iter = int(sys.argv[4]) if len(sys.argv) >= 5 else 300

        res = kmeans(img, k=k, tol=tol, max_iter=max_iter)

        save_image(res, sys.argv[1])
else:
    from lib.toolbox import *


import time
import logging

#The intention of this library file is to provide an API for csv2 processes to register for watchdog coverage
# and allow the ProcessMonitor to use the watchdog to detect threads in a stalled state and restart them
# providing automatic recovery when a request to an external API causes a process to stall or become a zombie

def watchdog_register_process(config, pid, host_id):
    update_dict = {
        "pid": pid,
        "host_id": host_id,
        "last_heartbeat": time.time()
    }
    logging.debug("Registering %s" % pid)
    rc, msg = config.db_merge('csv2_watchdog', update_dict)
    config.db_commit()
    return 0

def watchdog_remove_process(config, pid, host_id):
    where = "pid='%s' and host_id='%s'" % (pid, host_id)
    logging.debug("Removing %s" % pid)
    config.db_delete('csv2_watchdog', where=where)
    config.db_commit()
    return 0

def watchdog_check_process(config, pid, host_id):
    where = "pid='%s' and host_id='%s'" % (pid, host_id)
    rc, msg, rows = config.db_query('csv2_watchdog', where=where)
    if rc !=0:
        # query error
        logging.error("Query error: %s" % msg)
        #returning false will restart the process, which we may or may not want to do if there is database errors
        return False
    try:
        row = rows[0]
    except Exception as exc:
        #no process registered
        logging.info("%s not found to be registered" % pid)
        return False

    current_time = time.time()
    # if last heartbeat is over an hour old return false and restart the process
    hbeat_diff = current_time - row["last_heartbeat"]
    logging.debug("Heartbeat differential: %s" % hbeat_diff)
    if (hbeat_diff > 3600):
        watchdog_send_warning(config, pid, host_id)
        logging.info("PID: %s found to be stuck" % pid)
        return False
    return True

def watchdog_send_warning(config, pid, host_id):
    return 0

def watchdog_send_heartbeat(config, pid, host_id):
    update_dict = {
        "pid": pid,
        "host_id": host_id,
        "last_heartbeat": time.time()
    }
    config.db_update('csv2_watchdog', update_dict)
    config.db_commit()
    logging.debug("watchdog updated for: %s" % pid)
    return 0

def watchdog_cleanup(config):
    #clean out all entries that havent been updated in 24 hrs
    where = "last_heartbeat <= %s" % (time.time() - (3600 * 24))
    rc, msg = config.db_delete('csv2_watchdog', where=where)
    config.db_commit()


import argparse
import sys

from GraphMaker import *
from JsonVisualiser import JsonLoader
from StepParser import StepParser

QUEST_FOLDER_PATH = None
DIALOG_FOLDER_PATH = None
OUTPUT_NAME = "./output/idralwelGraph"  # CAN NOT BE graph

def args_parsing():
    global QUEST_FOLDER_PATH, DIALOG_FOLDER_PATH, OUTPUT_NAME
    parser = argparse.ArgumentParser(
        description="Idralwel Json Visualiser let you create and store dot graph out of special JSON formatted Adventure based off the Idralwel Project")
    parser.add_argument('-f', '--folder', action='store',
                        help="The Adventure folder to search from, needs to have a quest folder and Dialogs folder in it")
    parser.add_argument('-o', '--output', action='store',
                        help="The output file name to create (do not include extension)")
    parser.add_argument('-v', '--vertical', action='store_true',
                        help="The output graph will be vertical, default horizontal")
    parser.add_argument('--pdf', action='store_true', help="Output a pdf file")
    parser.add_argument('--dot', action='store_true', help="Output a dot file")
    parser.add_argument('--png', action='store_true', help="Output a png file")

    result = parser.parse_args()
    print("L. Argument Parsing")
    if result.folder == None:
        print("|\tError, folder must be specified using option -f or --file\n|\tOption -h or --help for help\n'")
        sys.exit()
    if result.output == None:
        print("|\tWarning, output file not defined, using ./output/idralwelGraph by default\n|\tCan be specified using option -o or --output\n|\tOption -h or --help for help\n'")
    elif result.output == "graph":
        print(f"|\t{result.output} is an invalid name, using default name ./output/idralwelGraph\n'")
    else:
        OUTPUT_NAME = result.output
    print("|\tArgs : ", result.__dict__.__str__()[1:-1])
    QUEST_FOLDER_PATH = result.folder+"/quest/"
    DIALOG_FOLDER_PATH = result.folder+"/Dialogs/"

    return result

if __name__ == '__main__':
    print("[Idralwel Json Visualiser]")

    args = args_parsing()
    print("L. Loading Files")
    jsonLoader = JsonLoader(QUEST_FOLDER_PATH, DIALOG_FOLDER_PATH)
    dialogs = jsonLoader.dialogs_list
    quests = jsonLoader.json_list
    print(f"|\tLoaded {len(quests)} quests from {QUEST_FOLDER_PATH}")
    print(f"|\tLoaded {len(dialogs)} dialogs from {DIALOG_FOLDER_PATH}")
    for json in quests.keys():
        print(f"L. Loading {json}...")
        stepParser = StepParser(jsonLoader.json_list[json]["steps"], jsonLoader.dialogs_list)
        print(f"|\tLoaded {len(stepParser.steps)} steps")
        print(f"|\tLoaded {stepParser.conditions_loaded} conditions")
        print(f"|\tLoaded {stepParser.response_loaded} responses")
        print(f"|\tLoaded {stepParser.start_link} start link")
        print(f"|\tLoaded {stepParser.stop_link} stop link")
        print("'\t Done !")

        print(f"+\tStarting graph generation for {json}...")
        print("|\tThat can take a few seconds !")
        create_step_graph(stepParser, OUTPUT_NAME, pdf=args.pdf, png=args.png, dot=args.dot,
                          horizontal=not args.vertical)
        print("'\t Done !")


## Computer DJ

#Libraries
import pandas as pd
import os
import spotipy
from spotipy.oauth2 import SpotifyClientCredentials
from inputimeout import inputimeout, TimeoutOccurred
import vlc
import time
import select
import sys
from mutagen.mp3 import MP3
import random
from sklearn.preprocessing import StandardScaler
from sklearn.metrics.pairwise import euclidean_distances
import numpy as np

def get_song_ids(playlist_id, sp):

    # Get playlist tracks
    tracks = sp.playlist_tracks(playlist_id)

    # Create a list to store the song IDs
    song_ids = []

    # Initialize the offset and initial call flag
    offset = 0
    initial_call = True

    # Retrieve playlist tracks using offset to get all songs
    while True:
        tracks = sp.playlist_tracks(playlist_id, offset=offset)
        items = tracks['items']

        # Iterate over each track in the current set of items and extract the song ID
        for item in items:
            track = item['track']
            song_id = track['id']
            song_ids.append(song_id)

        # Check if it's the initial call
        if initial_call:
            total_tracks = tracks['total']
            initial_call = False

        # Increment the offset by the maximum limit (100) for the next call
        offset += len(items)

        # Break the loop if all tracks have been fetched
        if offset >= total_tracks:
            break

    return song_ids

def build_song_df(mp3_dir, all_song_ids, sp, comp_dj_dir):

    song_df_path = comp_dj_dir + 'songs_df.csv'
    try:
        curr_df = pd.read_csv(song_df_path )
        starting_index = len(curr_df)
    except:
        starting_index = 0

    # Get a list of all files with ".mp3" extension in the directory
    file_list = [file for file in os.listdir(mp3_dir) if file.endswith('.mp3')]

    # Create a list of song names without the ".mp3" extension
    songs = [os.path.splitext(file)[0] for file in file_list]

    # Create an empty dataframe with a column named "file name"

    song_dict_list = []
    counter = 0
    # Iterate over each song and search for matching tracks on Spotify
    print("Creating song dataframe...")

    for song in songs[starting_index:]:
        counter += 1
        print("{} of {}".format(counter, len(songs)))

        if counter%10 ==0 :
            print("Saving intermediate df...")
            # Create a list of dataframes from the dictionaries
            dfs = [pd.DataFrame([dictionary]) for dictionary in song_dict_list]
            # Concatenate the dataframes into a single dataframe
            songs_df = pd.concat(dfs, ignore_index=True)
            songs_df.to_csv(song_df_path, index=True)

        # Search for tracks using the song name
        results = sp.search(q=song, type='track', limit=20)
        items = results['tracks']['items']
        returned_ids = []
        for item in items:
            returned_ids.append( item['id'])

        match_index = -1
        for index, id in enumerate(returned_ids):
            if id in all_song_ids:
                match_index = index

        if match_index != -1:
            correct_song = items[match_index]

            # Retrieve track information using the Spotify API
            track_baseline = sp.track(correct_song['id'])
            track_baseline['album'] = track_baseline['album']['name']
            track_baseline['artists'] = track_baseline['artists'][0]['name']
            vars_to_keep = ['id', 'name', 'artists', 'album', 'popularity']
            track_baseline = {key: track_baseline[key] for key in vars_to_keep if key in track_baseline}

            track_features = sp.audio_features(correct_song['id'])[0]

            vars_to_keep = ['danceability', 'energy', 'key', 'speechiness', 'acousticness', 'instrumentalness', 'liveness', 'valence', 'tempo',  'duration_m']
            track_features['duration_m'] = round( (track_features['duration_ms']/1000)/60, 4)

            track_features =  {key: track_features[key] for key in vars_to_keep if key in track_features}

            track_dict = {**track_baseline, **track_features}
            track_dict['file name'] = song

            # Append the row to the dataframe
            song_dict_list.append(track_dict)

    # Create a list of dataframes from the dictionaries
    dfs = [pd.DataFrame([dictionary]) for dictionary in song_dict_list]

    # Concatenate the dataframes into a single dataframe
    songs_df = pd.concat(dfs, ignore_index=True)

    return songs_df

def load_song( player, song_path):

    media = vlc.Media(song_path)
    player.set_media(media)

    # Get the duration of the media in milliseconds
    mutagen_mp3 = MP3(song_path)
    duration_sec = int( mutagen_mp3.info.length )

    return (player, duration_sec)

def determine_next_song_index(songs_df, curr_index, already_played_indices):

    vars_for_sim = songs_df[ ['popularity', 'danceability',
           'energy', 'key', 'speechiness', 'acousticness', 'instrumentalness',
           'liveness', 'valence', 'tempo' ]  ]

    # Standardize the variables
    scaler = StandardScaler()
    df_standardized = pd.DataFrame( scaler.fit_transform(vars_for_sim), columns = vars_for_sim.columns)

    #weights
    weights = {
    'popularity': 5,
    'danceability': 5,
    'energy': 6,
    'key': 5,
    'speechiness': 3,
    'acousticness': 5,
    'instrumentalness': 3,
    'liveness': 3,
    'valence': 2,
    'tempo': 20
    }

    for key, value in weights.items():
        df_standardized[key] = df_standardized[key] * value

    # Compute distances between observations
    distances = euclidean_distances(df_standardized)
    distances_to_curr = distances[curr_index, :]

    # Aritifically inflate distances for those songs that have been played
    weighter_played = np.array(songs_df['played']) * 100000000000 +1
    distances_to_curr *= weighter_played

    closest_index =  np.argsort(distances_to_curr)[1]

    counter = 2
    while closest_index ==curr_index or closest_index in already_played_indices:
        closest_index =  np.argsort(distances_to_curr)[counter]
        counter+=1

    return closest_index

def song_player(songs_df, mp3_dir):

     songs_df['played'] = False
     already_played_indices = []

     # Create a VLC media player instance
     player = vlc.MediaPlayer()

     #Get a set of random songs to cycle through
     songs_random_indices = random.sample( range(len(songs_df)), len(songs_df))
     random_counter=0

     #Initialize the first song as the first random song
     curr_index = songs_random_indices[random_counter]
     random_counter += 1

     #Allow user to cycle between first song seed
     random_input_usr = '0'
     while random_input_usr !='1':
         random_input_usr = input("First song is: {} by {}\n1.Okay\n2.Next song\n".format(songs_df['name'][curr_index] , songs_df['artists'][curr_index] ))
         if random_input_usr != "1":
             curr_index = songs_random_indices[random_counter]
             random_counter += 1

     #Default start
     user_input='1'
     automatic_mode = False
     automatic_num=4

     while user_input != '4':

         if user_input == '1':

             #Allow user to pick how many next songs to play
             if not automatic_mode:
                 num_songs = input("How many songs?\n")
                 while num_songs not in [str(i) for i in range(100)]:
                     num_songs = input("Enter number between 1 and 99\n")
                 num_songs = int(num_songs)
                 if num_songs ==0:
                     num_songs+=1
             else:
                 num_songs = automatic_num

             song_counter = 0

             #Play through number of "next closest" songs
             while song_counter <num_songs:
                 song_counter += 1

                 # Load the MP3 file
                 fp_song = mp3_dir + songs_df['file name'][curr_index] + '.mp3'
                 player, duration_sec = load_song(player, fp_song )
                 #duration_sec = 5
                 start_time = time.time()
                 elapsed_time = 0

                 print("Now playing: {} by {}...".format(songs_df['name'][curr_index] , songs_df['artists'][curr_index] ) )
                 player.play()

                 #Get next closest song using distance metric
                 next_song_index = determine_next_song_index(songs_df, curr_index, already_played_indices)

                 #Play song
                 while elapsed_time < duration_sec-3:

                     #Print out a loading bar
                     dots = 60
                     pct = int( dots* elapsed_time / duration_sec)
                     pct_mins = dots - pct
                     elapsed_time = time.time() - start_time
                     elapsed_str = "{}:{}".format( int(elapsed_time//60), str(int(elapsed_time%60)).zfill(2)  )
                     duration_str = "{}:{}".format( int(duration_sec//60), str(int(duration_sec%60)).zfill(2)  )
                     print( "[", '.'*pct, " "*pct_mins, "]" ," ", elapsed_str, " / ", duration_str, sep="")
                     time.sleep(3)

                 #Add song to "already Played" list
                 already_played_indices.append(curr_index)

                 #Stop playing
                 player.stop()
                 songs_df.loc[curr_index, 'played'] = True

                 #Set up loop to play next song
                 curr_index = next_song_index

             #In automatic mode, automatically set next seed and keep playing
             if automatic_mode:
                 print("*"*50, "\n\nContinuing in automatic mode - NEW SEED...\n")
                 curr_index = songs_random_indices[random_counter]
                 random_counter += 1

         #New seed
         if user_input == '2':
             random_input_usr = '0'
             while random_input_usr !='1':
                 random_input_usr = input("Next seed song is: {} by {}\n1.Okay\n2.Next song\n".format(songs_df['name'][curr_index] , songs_df['artists'][curr_index] ))
                 if random_input_usr != "1":
                     curr_index = songs_random_indices[random_counter]
                     random_counter += 1

         if user_input == '3':
             automatic_mode = True
             automatic_num = 4
             user_input = '1'

         if user_input == '4':
             # Stop the player
             player.stop()

         if not automatic_mode:
             menu_txt = '1. Play songs\n2. New Seed\n3. Automatic Mode\n4. Quit\n'
             user_input = input(menu_txt)
             while user_input not in ['1', '2', '3']:
                 user_input = input("Try Again")
         else:
             user_input = "1"

     return player

def main():

    print("\n\n", "-"*50, sep="")
    print("\n\n\nComputer DJ\n\n\n")
    print("-"*50)

    print("\n\nloading....")
    time.sleep(1)

    rips_archived_id = 'spotify:playlist:4Ra7m4IDwu9GVMq7FC51Kj'
    mp3_dir = '/Users/coltonlapp/Dropbox/Personal/Music/DJ/MP3s/Spotify/'
    comp_dj_dir = '/Users/coltonlapp/Dropbox/Personal/Music/DJ/SCRIPTS/ComputerDJ/'

    client_id= 'temp'
    client_secret = 'temp'

    with open( comp_dj_dir + "credentials.txt", "r") as file:
        credentials = file.readlines()

    # Extract the values and assign them to variables
    client_id = credentials[0].strip().split('=')[1].strip().strip("'")
    client_secret = credentials[1].strip().split('=')[1].strip().strip("'")

    # Authenticate with Spotify API
    client_credentials_manager = SpotifyClientCredentials(client_id=client_id, client_secret=client_secret)
    sp = spotipy.Spotify(client_credentials_manager=client_credentials_manager)

    all_song_ids = get_song_ids(rips_archived_id, sp)

    # Check if the file exists
    if not os.path.exists(comp_dj_dir + 'songs_df.csv'):
        # File does not exist, so run the build_songs function
        songs_df = build_song_df(mp3_dir, all_song_ids, sp, comp_dj_dir)
        songs_df.to_csv( comp_dj_dir + 'songs_df.csv', index=False)
    else:
        # File exists, read the CSV file into a dataframe
        songs_df = pd.read_csv(comp_dj_dir + 'songs_df.csv')

    print("Loaded {} songs...\n".format(len(songs_df)))
    player = song_player(songs_df, mp3_dir)
    player.stop()

if __name__ == '__main__':
    main()


import os
import sys
import arcade
import numpy as np
from sklearn.linear_model import LinearRegression, Lasso, Ridge

# Graphics constants
SCREEN_WIDTH  = 800
SCREEN_HEIGHT = 800
SCREEN_TITLE  = "Shape Fitter"
FPS = 60

class MyGame(arcade.Window):

    def __init__(self, shape):
        # Call the parent class initializer
        super().__init__(SCREEN_WIDTH, SCREEN_HEIGHT, SCREEN_TITLE)
        arcade.set_background_color(arcade.color.LIGHT_GRAY)

        # List of points so far
        self.points = None

        # List of best parameters so far
        if shape == 'l' or shape == 'line':
            self.params = np.zeros(2)
        else:
            self.params = np.zeros(3)

        # Store shape of interest
        self.shape = shape

        # Store the regularization strengths
        self.alpha_l1 = 0.5
        self.alpha_l2 = 0.05

        max_iter = 1e7

        # Create regression objects for later
        self.regressors = [
            LinearRegression(fit_intercept=False),
            Lasso(alpha=self.alpha_l1, fit_intercept=False, max_iter=max_iter),
            Ridge(alpha=self.alpha_l2, fit_intercept=False, max_iter=max_iter)
        ]

        # Colors for reach type of regression
        self.colors = [arcade.color.RED, arcade.color.ARMY_GREEN, arcade.color.BLUE]

    def draw_stats(self):

        # First show Regularizers
        arcade.draw_text(f'LASSO Regularization = {self.alpha_l1:.4f}', SCREEN_WIDTH // 7, 7 * SCREEN_HEIGHT // 8, self.colors[1])
        arcade.draw_text(f'Ridge Regularization = {self.alpha_l2:.4f}', SCREEN_WIDTH // 7, 7 * SCREEN_HEIGHT // 8 - 20, self.colors[2])

        # Draw legend @ top right
        x_legend = SCREEN_WIDTH - 200
        y_legend = SCREEN_HEIGHT - 100
        for col, reg in zip(self.colors, self.regressors):
            name = str(type(reg)).split(".")[-1][:-2]

            arcade.draw_text(name, x_legend, y_legend, col, 12)
            y_legend -= 0.02 * SCREEN_HEIGHT

    def on_draw(self):
        # This command has to happen before we start drawing
        arcade.start_render()

        # Show the numbers
        self.draw_stats()

        # Go through each type of regression
        if self.points is not None and len(self.points) > 3:
            for regressor, col in zip(self.regressors, self.colors):
                params = regressor.coef_
                if self.shape == 'c' or self.shape == 'circle':
                    # Find circle center and radius
                    xc = -params[0] / 2
                    yc = -params[1] / 2
                    r = np.sqrt(xc ** 2 + yc ** 2 + params[2])

                    # Draw best circle so far
                    arcade.draw_circle_outline(xc, yc, r, col, 1)
                elif self.shape == 'p' or self.shape == 'parabola':
                    a, b, c = params
                    x_axis = np.arange(SCREEN_WIDTH).reshape((-1,1))
                    points = np.hstack((x_axis, a * x_axis ** 2 + b * x_axis + c))

                    # Draw best parabola so far
                    arcade.draw_lines(points, col, 1)
                elif self.shape == 'l' or self.shape == 'line':
                    # Draws line given y = mx + b (m and b)
                    m, b = params
                    x0 = -1
                    x1 = SCREEN_WIDTH + 1
                    y0 = m * x0 + b
                    y1 = m * x1 + b

                    # Draw best line so far
                    arcade.draw_line(x0, y0, x1, y1, col, 1)

        # Draw all points
        if self.points is not None:
            if len(self.points.shape) != 2:
                arcade.draw_point(self.points[0], self.points[1], arcade.color.BLACK, 3)
            else:
                for row in self.points:
                    arcade.draw_point(row[0], row[1], arcade.color.BLACK, 3)

    # Helper for estimating next best curve
    def fit_regressors(self):
        # Best fit circle  when the time is right
        if len(self.points) > 3:
            # Find X and Y depending on what shape we want
            if self.shape == 'c' or self.shape == 'circle':
                X = np.hstack((self.points, -np.ones(len(self.points)).reshape((-1,1))))
                y = -(self.points[:,0] ** 2 + self.points[:,1] ** 2)
            elif self.shape == 'p' or self.shape == 'parabola':
                x_temp = self.points[:,0].reshape((-1,1))
                X = np.hstack((x_temp ** 2, x_temp, np.ones(len(x_temp)).reshape((-1,1))))
                y = self.points[:,1]
            elif self.shape == 'l' or self.shape == 'line':
                X = np.hstack((self.points[:,0].reshape((-1, 1)), np.ones(len(self.points)).reshape((-1,1))))
                y = self.points[:,1]
            # perform regression
            for i, regressor in enumerate(self.regressors):
                self.regressors[i] = regressor.fit(X, y)

    # Called whenever the mouse is pressed
    def on_mouse_press(self, x, y, button, modifiers):
        # New point
        new_point = np.array([x, y])

        # Append to list of points
        if self.points is None:
            self.points = new_point
        else:
            self.points = np.vstack((self.points, new_point))

        # Fit the curves
        self.fit_regressors()

    # Called whenever a key is pressed
    def on_key_press(self, key, modifiers):
        # If the player presses a key, update the speed
        if key == arcade.key.SPACE:
            self.points = None
            arcade.cleanup_texture_cache()
            arcade.set_background_color(arcade.color.LIGHT_GRAY)
            arcade.finish_render()
        elif key == arcade.key.LEFT or key == arcade.key.RIGHT or key == arcade.key.DOWN or key == arcade.key.UP:
            if key == arcade.key.LEFT:
                self.alpha_l1 *= 0.9
            elif key == arcade.key.RIGHT:
                self.alpha_l1 *= 1.1
            elif key == arcade.key.UP:
                self.alpha_l2 *= 1.1
            elif key == arcade.key.DOWN:
                self.alpha_l2 *= 0.9
            self.regressors[1].alpha = self.alpha_l1
            self.regressors[2].alpha = self.alpha_l2
            self.fit_regressors()

def main(shape):
    game = MyGame(shape)
    arcade.run()

if __name__ == "__main__":
    assert len(sys.argv) == 2
    main(sys.argv[1])