import settings
import discord
from discord.ext import commands

logger = settings.logging.getLogger("bot")

def run():
    """ running """
    # print(settings.DISCORD_API_TOKEN)
    intents = discord.Intents.default()
    intents.message_content = True

    bot = commands.Bot(command_prefix="!", intents=intents)

    @bot.event
    async def on_ready():
        logger.info(f"User: {bot.user} (ID: {bot.user.id})")
        logger.info(f"Guild ID: {bot.guilds[0].id}")
        # bot.tree.copy_global_to(guild=settings.GUILDS_ID)
        # await bot.tree.sync(guild=settings.GUILDS_ID)

        for cmd_file in settings.CMDS_DIR.glob("*.py"):
            if cmd_file != "__init.py":
                await bot.load_extension(f"cmds.{cmd_file.name[:-3]}")

        for cmd_file in settings.COGS_DIR.glob("*.py"):
            if cmd_file != "__init.py":
                await bot.load_extension(f"cogs.{cmd_file.name[:-3]}")

    @bot.event
    async def on_command_error(ctx, error):
        if isinstance(error, commands.MissingRequiredArgument):
            await ctx.send("handled error globally")

    bot.run(settings.DISCORD_API_TOKEN, root_logger=True)

if __name__ == "__main__":
    run()


from django.shortcuts import render
from django.shortcuts import render, redirect, get_object_or_404
from django.contrib.auth.decorators import login_required
from django.contrib import messages

from clients.models import Client
from clients.forms import AddClientForm
from team.models import Team

@login_required
def clients_list(request):
    clients = Client.objects.filter(created_by=request.user)
    return render(request, "clients/clients_list.html", {"clients": clients})

@login_required
def clients_details(request, pk):
    client = get_object_or_404(Client, created_by=request.user, pk=pk)
    return render(request, "clients/clients_detail.html", {"client": client})

@login_required
def client_delete(request, pk):
    client = get_object_or_404(Client, created_by=request.user, pk=pk)
    client.delete()

    messages.success(request, "The client was deleted")
    return redirect("clients:list")

@login_required
def edit_client(request, pk):
    client = get_object_or_404(Client, created_by=request.user, pk=pk)

    if request.method == "POST":
        form = AddClientForm(request.POST, instance=client)

        if form.is_valid():
            form.save()

            messages.success(request, "The changes have been saved")

            return redirect("clients:list")
    else:
        form = AddClientForm(instance=client)

    return render(request, "clients/edit_client.html", {"form": form})

@login_required
def add_client(request):
    team = Team.objects.filter(created_by=request.user)
    if request.method == "POST":
        form = AddClientForm(request.POST)
        if form.is_valid():
            client = form.save(commit=False)
            client.created_by = request.user
            team = Team.objects.filter(created_by=request.user)[0]
            client.team = team
            client.save()

            messages.success(request, "The client was created")

            return redirect("clients:list")
    else:
        form = AddClientForm()

    return render(request, "clients/add_client.html", {"form": form, "team": team})


import argparse
import os
import requests
from datetime import date
from bs4 import BeautifulSoup

# Base URL of the website
base_url = "https://newsdig.tbs.co.jp/"

# Function to fetch and parse the content of a URL
def get_parsed_page(url):
    response = requests.get(url)
    if response.status_code == 200:
        return BeautifulSoup(response.content, "html.parser")
    else:
        return None

# Function to scrape and save article text
def scrape_and_save_article(article_url, save_path):
    article_soup = get_parsed_page(article_url)
    if article_soup:
        paragraphs = article_soup.find("main").find_all("p")
        text = "\n".join(p.get_text() for p in paragraphs if p.get_text().strip())
        if text:
            article_number = article_url.split("/")[-1].split("?")[0]
            with open(os.path.join(save_path, f"{article_number}.txt"), 'w', encoding='utf8') as file:
                file.write(text)

# Main scraping function
def scrape(download_path):
    menu_url = base_url
    menu_page = get_parsed_page(menu_url)

    if menu_page:
        menu_bar = menu_page.find("div", {"class": "g-menu-wrap"}).find("ul")
        menu_items = menu_bar.find_all("li", {"class": "g-menu-main"})

        for item in menu_items:
            href = item.find("a")["href"]
            genre = href.split('/')[-1]
            genre_url = base_url + href
            genre_page = get_parsed_page(genre_url)

            if genre_page:
                articles = genre_page.find_all("article")
                for article in articles:
                    article_url = base_url + article.find("a")["href"]
                    save_folder = os.path.join(download_path, genre, date.today().strftime("%Y_%m_%d"))
                    os.makedirs(save_folder, exist_ok=True)
                    scrape_and_save_article(article_url, save_folder)
                    print(f"Saved article from {article_url} to {save_folder}")

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--download_directory", help='Path to save scraped data in.')
    args = parser.parse_args()

    scrape(args.download_directory)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

import optparse
import speex
import sys

from twistar.registry import Registry
from twisted.enterprise import adbapi
from twisted.internet import reactor
from twisted.python import log

from sequoia.media.media import AudioMixer
from sequoia.protocol import ServerClientsFactory, ServerMediaProtocol
from sequoia.security import ServerContextFactory

def parse_args():
    usage = """usage: %prog [--host=HOST] [--cport=CPORT] [--mport=MPORT]"""
    parser = optparse.OptionParser(usage)

    help = "Host to run server on. Default: localhost"
    parser.add_option('--host', default='localhost', help=help)

    help = "Port for clients to connect to. Default: 0"
    parser.add_option('--cport', type='int', default=0, help=help)

    help = "Port for receiving media data. Default: 0"
    parser.add_option('--mport', type='int', default=0, help=help)

    options, args = parser.parse_args()
    return options.host, options.cport, options.mport

def show_connector_info(connector, description):
    info = connector.getHost()
    log.msg("{0} on {1}:{2}.".format(description, info.host, info.port))

def main():
    host, cport, mport = parse_args()
    use_codec = False

    Registry.DBPOOL = adbapi.ConnectionPool(
        'sqlite3', "sequoia/tests/auth/server_1/database.db",
        check_same_thread=False)

    mixer = AudioMixer()
    reactor.callWhenRunning(mixer.start)
    codec = speex.new() if use_codec else None
    media_tx = ServerMediaProtocol(codec, mixer)
    clients_factory = ServerClientsFactory(media_tx, use_codec)
    ctx_factory = ServerContextFactory(
        "sequoia/tests/auth/server_1/private_key.pem",
        "sequoia/tests/auth/server_1/certificate.pem",
        "sequoia/tests/auth/root/root_ca.crt")

    clients_listener = reactor.listenSSL(cport, clients_factory, ctx_factory,
        interface=host)
    show_connector_info(clients_listener, "Listening clients")
    media_listener = reactor.listenUDP(mport, media_tx,
        interface=host)
    show_connector_info(media_listener, "Serving media")

    reactor.run()

if __name__ == "__main__":
    log.startLogging(sys.stdout)
    main()


import numpy as np
import csv
import matplotlib.pyplot as plt
import skfuzzy as fuzz
from sklearn.metrics import mean_squared_error, r2_score

# Custom Gaussian bell-shaped membership function
def gbellmf(x, params):
    a, b, c = params
    return 1 / (1 + ((x - c) / a) ** (2 * b))

def calculate_output1(mynet):
    mparams = mynet['mparams']

    for i in range(mynet['ni']):
        for j in range(mynet['mf']):
            ind = mynet['ni'] + i * mynet['mf'] + j

            x = mynet['nodes'][i]
            a = mparams[i * mynet['mf'] + j, 0]
            b = mparams[i * mynet['mf'] + j, 1]
            c = mparams[i * mynet['mf'] + j, 2]

            tmp1 = (x - c) / a
            if tmp1 == 0:
                tmp2 = 0
            else:
                tmp2 = (tmp1 ** 2) ** b
            mynet['nodes'][ind] = 1 / (1 + tmp2)
    mynet['nodes'] = mynet['nodes']
    return mynet

def calculate_output2(mynet):
    st = mynet['ni'] + mynet['ni'] * mynet['mf']
    for i in range(st, st + mynet['nc']):
        I = [idx for idx, val in enumerate(mynet['config'][:, i]) if val == 1]
        tmp = 1
        for idx in I:
            tmp *= mynet['nodes'][idx]
        mynet['nodes'][i] = tmp
    return mynet

def calculate_output3(mynet):
    st = mynet['ni'] + mynet['ni'] * mynet['mf'] + mynet['nc']
    for i in range(st , st + mynet['nc'] ):
        I = [idx for idx, val in enumerate(mynet['config'][:, i]) if val == 1]
        denom = sum([mynet['nodes'][idx] for idx in I])
        mynet['nodes'][i] = mynet['nodes'][i - mynet['nc']] / denom
    return mynet

def calculate_output4(mynet):
    st = mynet['ni'] + mynet['ni'] * mynet['mf'] + 2 * mynet['nc']
    inp = mynet['nodes'][:mynet['ni']]
    kparam = mynet['kparams']

    for i in range(mynet['nc']):
        wn = mynet['nodes'][i + st - mynet['nc']]
        mynet['nodes'][i + st] = wn * (np.sum(kparam[i, :-1] * inp) + kparam[i, -1])

    # print(mynet['nodes'])
    # print(np.shape(mynet['nodes']))
    # exit()
    return mynet

def calculate_output5(mynet):
    mynet['nodes'][-1] = sum(mynet['nodes'][-mynet['nc']-1:-1])
    # print(np.shape(mynet['nodes']))
    # print(mynet['nodes'])
    # exit()
    return mynet

def get_kalman_data(mynet, target):

    kalman_data = np.zeros(((mynet['ni'] + 1) * mynet['nc'] + 1, 1))

    st = mynet['ni'] + mynet['ni'] * mynet['mf'] + mynet['nc']
    j = 0
    # print(st + mynet['nc'] + 1)
    for i in range(st, st + mynet['nc']):
        for k in range(mynet['ni']):
            kalman_data[j] = mynet['nodes'][i] * mynet['nodes'][k]
            j += 1
        kalman_data[j] = mynet['nodes'][i]
        j += 1

    kalman_data[j] = target
    return kalman_data

def mykalman(mynet, kalman_data, k):
    k_p_n = (mynet['ni'] + 1) * mynet['nc']

    alpha = 1000000
    # print(k)
    if k == 0:
        mynet['P'] = np.zeros((k_p_n, 1))
        mynet['S'] = alpha * np.eye(k_p_n)

    # print(mynet['P'])
    # print(mynet['S'])

    # exit()
    x = kalman_data[:-1]
    y = kalman_data[-1]

    x_temp = x.flatten()
    tmp1 = np.dot(x_temp, mynet['S'])
    tmp1 = tmp1[:, np.newaxis]

    denom = 1 + np.sum(tmp1 * x)
    tmp1 = np.dot(mynet['S'], x_temp)
    tmp1 = tmp1[:, np.newaxis]

    tmp2 = np.dot(x_temp, mynet['S'])
    tmp2 = tmp2[:, np.newaxis]
    tmp_m = np.outer(tmp1, tmp2)
    tmp_m = -1 / denom * tmp_m
    mynet['S'] = mynet['S'] + tmp_m

    diff = y - np.sum(x * mynet['P'])
    tmp1 = diff * np.dot(mynet['S'], x_temp)
    tmp1 = tmp1[:, np.newaxis]

    mynet['P'] = mynet['P'] + tmp1
    mynet['kparams'] = mynet['P'].reshape(( mynet['nc'],mynet['ni'] + 1))
    # mynet['kparams'] = mynet['P'].reshape(8,4)

    return mynet

def clear_de_dp(mynet):
    mynet['mparam_de_do'] = np.zeros((mynet['ni'] * mynet['mf'], 3))
    mynet['kparam_de_do'] = np.zeros((mynet['nc'], mynet['ni'] + 1))

    return mynet

def calculate_de_do(mynet, de_dout):
    mynet['de_do'] = np.zeros_like(mynet['nodes'])
    mynet['de_do'][-1] = de_dout
    tmp2 = []

    for i in range(len(mynet['nodes']) - 2, mynet['ni']-1, -1):
        # print("i = ", i)
        # print(len(mynet['nodes']) - 2)
        # print(mynet['ni']+1)

        de_do = 0
        II = np.where(mynet['config'][i, :] == 1)[0]
        # print(mynet['config'][i, :])
        # print("II =", II)
        I = np.where(II > i)[0]
        # print("I =",I)
        for j in I:
            jj = II[j]
            tmp1 = mynet['de_do'][jj]
            tmp2 = derivative_o_o(mynet, i, jj)
            # print("tmp2 =",tmp2)
            # ui = input()
            de_do = de_do + tmp1 * tmp2
        mynet['de_do'][i] = de_do
        # print(tmp2)
        # ui = input()
        # print(I, mynet['de_do'][i], tmp1, tmp2)
        # exit()
    # print(np.shape(mynet['de_do']))
    # print((mynet['de_do']))
    # exit()
    return mynet

def derivative_o_o(mynet, i, j):
    if i > mynet['ni'] + mynet['ni'] * mynet['mf'] +  2 * mynet['nc']-1:
        # print(i, "\t In 1", mynet['ni'] + mynet['ni'] * mynet['mf'] + 2 * mynet['nc']-1)
        return 1
    elif i > mynet['ni'] + mynet['ni'] * mynet['mf'] + mynet['nc']-1:
        # print(i, "\t In 2", mynet['ni'] + mynet['ni'] * mynet['mf'] + mynet['nc'])
        return do4_do3(mynet, i, j)
    elif i > mynet['ni'] + mynet['ni'] * mynet['mf']-1:
        # print(i, "\t In 3", mynet['ni'] + mynet['ni'] * mynet['mf'])
        return do3_do2(mynet, i, j)
    elif i > mynet['ni']-1:
        # print(i, "\t In 4",mynet['ni'])
        # print(mynet['nodes'][j])
        # print(mynet['nodes'][i])
        return mynet['nodes'][j] / mynet['nodes'][i]

def do4_do3(mynet, i, j):
    kparam = mynet['kparams']
    inp = mynet['nodes'][:mynet['ni']]
    # print("inp =", inp)
    jj = j - mynet['ni'] - mynet['ni'] * mynet['mf'] - 2 * mynet['nc']
    # print("jj =", jj)
    #print("inp and jj - ", inp,jj )
    return np.sum(kparam[jj, :-1] * inp) + kparam[jj, -1]

def do3_do2(mynet, i, j):
    II = np.where(mynet['config'][:, j] == 1)[0]
    I = np.where(II < j)[0]
    total = np.sum(mynet['nodes'][II[I]])
    # print("total = ",total)
    # print("j = ",j)
    # print("mynet['nc'] = ",mynet['nc'])
    if j - i == mynet['nc']:
        # print(("r1 = ",total - mynet['nodes'][i]) / (total**2))
        return (total - mynet['nodes'][i]) / (total**2)

    else:
        # print("r2 = ",-mynet['nodes'][j - mynet['nc']] / (total**2))
        return -mynet['nodes'][j - mynet['nc']] / (total**2)

def update_de_do(mynet):
    s = 0
    for i in range(mynet['ni'], mynet['ni'] + mynet['ni'] * mynet['mf']):
        for j in range(3):
            do_dp = dmf_dp(mynet, i, j)
            mynet['mparam_de_do'][s, j] = mynet['mparam_de_do'][s, j] + mynet['de_do'][i] * do_dp
        s += 1

    s = 0
    for i in range(1 + mynet['ni'] + mynet['ni'] * mynet['mf'] + 2 * mynet['nc'], len(mynet['config'])):
        for j in range(mynet['ni'] + 1):
            do_dp = dconsequent_dp(mynet, i, j)
            mynet['kparam_de_do'][s, j] = mynet['kparam_de_do'][s, j] + mynet['de_do'][i] * do_dp
        s += 1

    return mynet

def dmf_dp(mynet, i, j):
    I = np.where(mynet['config'][:, i] == 1)[0]
    x = mynet['nodes'][I]
    a = mynet['mparams'][i - mynet['ni'], 0]
    b = mynet['mparams'][i - mynet['ni'], 1]
    c = mynet['mparams'][i - mynet['ni'], 2]
    tmp1 = (x - c) / a
    if tmp1 == 0:
        tmp2 = 0
    else:
        tmp2 = (tmp1**2)**b
    denom = (1 + tmp2) * (1 + tmp2)
    if j == 0:
        tmp = (2 * b * tmp2) / (a * denom)
    elif j == 1 and tmp1 == 0:
        tmp = 0
    elif j == 1 and tmp1 != 0:
        tmp = (-np.log(tmp1**2) * tmp2) / denom
    elif j == 2 and x == c:
        tmp = 0
    elif j == 2 and x != c:
        tmp = (2 * b * tmp2) / ((x - c) * denom)
    return tmp

def dconsequent_dp(mynet, i, j):
    wn = mynet['nodes'][i - mynet['nc']-1]
    inp = mynet['nodes'][:mynet['ni']]
    inp = np.append(inp, 1)
    return wn * inp[j]

def update_step_size(mynet, error_array, iter, step_size, decrease_rate, increase_rate):
    if check_decrease_ss(error_array, mynet['last_decrease_ss'], iter):
        step_size = step_size * decrease_rate
        mynet['last_decrease_ss'] = iter
    elif check_increase_ss(error_array, mynet['last_increase_ss'], iter):
        step_size = step_size * increase_rate
        mynet['last_increase_ss'] = iter

    return mynet, step_size

def check_decrease_ss(error_array, last_change, current):
    if current - last_change < 4:
        return False
    elif (error_array[current] < error_array[current - 1] and
          error_array[current - 1] > error_array[current - 2] and
          error_array[current - 2] < error_array[current - 3] and
          error_array[current - 3] > error_array[current - 4]):
        return True
    else:
        return False

def check_increase_ss(error_array, last_change, current):
    if current - last_change < 4:
        return False
    elif (error_array[current] < error_array[current - 1] and
          error_array[current - 1] < error_array[current - 2] and
          error_array[current - 2] < error_array[current - 3] and
          error_array[current - 3] < error_array[current - 4]):
        return True
    else:
        return False

def update_parameter(mynet, step_size):
    tmp = mynet['mparam_de_do']
    tmp = tmp * tmp
    norm_factor = np.sqrt(np.sum(tmp))
    mynet['mparams'] = mynet['mparams'] - step_size * mynet['mparam_de_do'] / norm_factor
    return mynet

def evalmyanfis(mynet, inputs):
    anfis_output = np.zeros((inputs.shape[0], 1))
    for i in range(inputs.shape[0]):
        mynet['nodes'][:mynet['ni']] = inputs[i]
        # mynet['nodes'][:mynet['ni']] = inputs[j].T.tolist()
        mynet = calculate_output1(mynet)
        mynet = calculate_output2(mynet)
        mynet = calculate_output3(mynet)
        mynet = calculate_output4(mynet)
        mynet = calculate_output5(mynet)

        anfis_output[i, 0] = mynet['nodes'][-1]

    return anfis_output

def myanfis(data, inputs, epoch_n, mf, step_size, decrease_rate, increase_rate):
    # Divide data into input and output
    output = data[:, -1:]  # The last column is the output

    ndata = data.shape[0]  # Data length

    # Define minimum and maximum of input to determine initial membership functions
    mn = np.min(inputs, axis=0)
    mx = np.max(inputs, axis=0)
    mm = mx - mn
    ni = inputs.shape[1]  # Number of inputs
    nc = mf**ni  # Number of rules
    Node_n = ni + ni * mf + 3 * nc + 1  # Total number of nodes

    min_RMSE = 999999999999  # Define minimum RMSE

    # Define initial membership functions
    mparams = []
    for i in range(ni):
        tmp = np.linspace(mn[i], mx[i], mf)
        mparams.extend(np.column_stack([np.full(mf, mm[i] / 6), np.full(mf, 2), tmp]))

    # Define initial Kalman parameters with all zeros
    kparams = np.zeros((nc, ni + 1))

    # Create connection matrix and node array
    config = np.zeros((Node_n, Node_n))
    nodes = np.zeros(Node_n)

    # Inputs - layer1 connections
    st = ni
    for i in range(ni):
        config[i, st : st + mf] = 1
        st = st + mf

    # Layer1 - Layer2 connections
    st = ni + ni * mf
    if ni == 2:
        for i in range(mf):
            for j in range(mf):
                config[ni + i, st] = 1
                config[ni + mf + j, st] = 1
                st = st + 1
    elif ni == 3:
        for i in range(mf):
            for j in range(mf):
                for k in range(mf):
                    config[ni + i, st] = 1
                    config[ni + mf + j, st] = 1
                    config[ni + 2 * mf + k, st] = 1
                    st = st + 1
    elif ni == 4:
        for i in range(mf):
            for j in range(mf):
                for k in range(mf):
                    for l in range(mf):
                        config[ni + i, st] = 1
                        config[ni + mf + j, st] = 1
                        config[ni + 2 * mf + k, st] = 1
                        config[ni + 3 * mf + l, st] = 1
                        st = st + 1
    elif ni == 5:
        for i in range(mf):
            for j in range(mf):
                for k in range(mf):
                    for l in range(mf):
                        for m in range(mf):
                            config[ni + i, st] = 1
                            config[ni + mf + j, st] = 1
                            config[ni + 2 * mf + k, st] = 1
                            config[ni + 3 * mf + l, st] = 1
                            config[ni + 4 * mf + m, st] = 1
                            st = st + 1

    # Layer2 - Layer3 connections
    for i in range(nc):
        for j in range(nc):
            config[ni + ni * mf + i, ni + ni * mf + nc + j] = 1

    # Layer3 - Layer4 connections
    for i in range(nc):
        config[ni + ni * mf + nc + i, ni + ni * mf + 2 * nc + i] = 1

    # Layer4 - Layer5 connections
    for i in range(nc):
        config[ni + ni * mf + 2 * nc + i, -1] = 1

    # Inputs - Layer4 connections
    for i in range(ni):
        for j in range(nc):
            config[i, ni + ni * mf + 2 * nc + j] = 1

    mynet = {
        "config": config,
        "mparams": np.array(mparams),
        "kparams": kparams,
        "nodes": nodes,
        "ni": ni,
        "mf": mf,
        "nc": nc,
        "last_decrease_ss": 1,
        "last_increase_ss": 1,
    }

    num_nodes = len(mynet['nodes'])  # You need to determine this

    layer_1_to_3_output = np.zeros((num_nodes, ndata))  # Initialize with the correct shape

    for iter in range(0, epoch_n):
        for j in range(ndata):

            mynet['nodes'] = mynet['nodes'].flatten()
            mynet['nodes'][:mynet['ni']] = inputs[j]

            mynet = calculate_output1(mynet)
            mynet = calculate_output2(mynet)
            mynet = calculate_output3(mynet)

            # Save outputs of layer 1 to 3
            layer_1_to_3_output[:, j] = mynet['nodes'].flatten()
            # print((mynet['nodes'].flatten()))
            # print((layer_1_to_3_output))
            # exit()
            kalman_data = get_kalman_data(mynet, output[j])

            # exit()
            mynet = mykalman(mynet, kalman_data, j)

        mynet = clear_de_dp(mynet)

        # print(np.shape(layer_1_to_3_output))
        # print((layer_1_to_3_output))
        # file_path = "layer_1_to_3_output.csv"

        ## Use numpy.savetxt to save the array to a CSV file
        # np.savetxt(file_path, layer_1_to_3_output, delimiter=',')
        # exit()
        anfis_output = np.zeros((ndata, 1))
        RMSE = np.zeros((epoch_n, 1))  # Initialize RMSE as a NumPy array of zeros

        for j in range(ndata):
            mynet['nodes'] = layer_1_to_3_output[:, j]

            mynet = calculate_output4(mynet)
            mynet = calculate_output5(mynet)

            anfis_output[j, 0] = mynet['nodes'][-1]
            target = output[j]

            de_dout = -2 * (target - anfis_output[j, 0])

            mynet = calculate_de_do(mynet, de_dout)

            mynet = update_de_do(mynet)

        diff = anfis_output - output
        total_squared_error = np.sum(diff**2)
        RMSE[iter - 1, 0] = np.sqrt(total_squared_error / ndata)
        print(f'{iter}. RMSE error: {RMSE[iter - 1, 0]}')
        # input()
        if RMSE[iter - 1, 0] < min_RMSE:
            bestnet = mynet
            min_RMSE = RMSE[iter - 1, 0]

        mynet = update_parameter(mynet, step_size)
        mynet, step_size = update_step_size(mynet, RMSE, iter, step_size, decrease_rate, increase_rate)

# At this point, bestnet contains the trained ANFIS model, and RMSE contains the RMSE values for each epoch.
    for j in range(ndata):
        # print(j)
        # mynet['nodes'][:mynet['ni']].flatten()
        # print(np.shape(mynet['nodes'][:mynet['ni']]))
        # print(np.shape(inputs[j]))

        mynet['nodes'][:mynet['ni']] = inputs[j]
        # mynet['nodes'][:mynet['ni']] = inputs[j].T.tolist()
        mynet = calculate_output1(mynet)
        mynet = calculate_output2(mynet)
        mynet = calculate_output3(mynet)
        mynet = calculate_output4(mynet)
        mynet = calculate_output5(mynet)

        anfis_output[j, 0] = mynet['nodes'][-1]

    return bestnet,anfis_output,RMSE

def plot_Nodes(mynet):
    # Plot the Node Connections
    plt.figure()
    plt.imshow(mynet['config'], aspect='auto', cmap='cool')
    plt.title('Node Connections')
    plt.show()

def plot_mf(mynet, data):
    plt.figure(figsize=(12, 4))  # Create a single figure for all plots
    # print(mynet['mparams'])
    k = 0
    for i in range(0,mynet['ni']*mynet['mf'],mynet['mf']):
        # print(k)
        plt.subplot(2, 2, k+1)  # Create subplots for each input variable
        plt.title(f'Input {k+1}')
        plt.xlabel('X')
        plt.ylabel('Degree of Membership')
        min_val = np.min(data[:, k])
        max_val = np.max(data[:, k])
        step = 0.1
        x = np.arange(min_val, max_val, step)
        x = np.arange(-10, 10, step)

        for j in range(mynet['mf']):
            mf_x = fuzz.gbellmf(x, mynet['mparams'][i+j][0], mynet['mparams'][i+j][1], mynet['mparams'][i+j][2])
            plt.plot(x, mf_x, label=f'Membership Function {j+1}')
        k = k+1
        plt.tight_layout()
        plt.legend()
    plt.show()

def plot_predictions(actual_output,anfis_predictions):
    plt.figure()
    plt.plot(actual_output, 'b*', label='Actual Output')
    plt.plot(anfis_predictions, 'r-', linewidth=0.5, label='ANFIS Prediction')
    plt.xlabel('Data Point')
    plt.ylabel('Output Value')
    plt.legend()
    plt.show()

def calc_rmse(x,y):
    # Calculate Root Mean Squared Error (RMSE)
    mse = mean_squared_error(x, y)
    rmse = np.sqrt(mse)

    return rmse

def calc_r2(x,y):
    # Calculate R-squared
    r_squared = r2_score(x, y)

    return r_squared

def plot_r2(x,y):

    r_squared = r2_score(x, y)
     # Create a scatter plot for the data points
    plt.figure(figsize=(8, 6))
    plt.scatter(x, y, label=f'R-squared = {r_squared:.4f}')
    plt.plot([y.min(), y.max()], [y.min(), y.max()], 'k--', lw=2)  # Add the identity line
    plt.xlabel('Actual Output')
    plt.ylabel('Predicted Output')
    plt.title("Correlation_Coefficient (R2)")
    plt.legend(loc='upper left')
    plt.tight_layout()
    # Show the plot
    plt.show


import pytest

from math import sqrt
from pdsa.cardinality.probabilistic_counter import ProbabilisticCounter

def test_init():
    pc = ProbabilisticCounter(10)
    assert pc.sizeof() == 40, "Unexpected size in bytes"

    with pytest.raises(ValueError) as excinfo:
        pc = ProbabilisticCounter(0)
    assert str(excinfo.value) == 'At least one simple counter is required'

def test_repr():
    pc = ProbabilisticCounter(10)

    assert repr(pc) == (
        "<ProbabilisticCounter (length: 320, num_of_counters: 10)>")

def test_add():
    pc = ProbabilisticCounter(10)

    for word in ["test", 1, {"hello": "world"}]:
        pc.add(word)

def test_count():
    num_of_counters = 256
    pc = ProbabilisticCounter(num_of_counters)
    std = 0.78 / sqrt(num_of_counters)

    errors = []

    boundary = 20 * num_of_counters

    cardinality = 0
    for i in range(10000):
        cardinality += 1
        element = "element_{}".format(i)
        pc.add(element)

        if cardinality < boundary:
            # For small cardinalities we need to use correction,
            # that we will test in another case.
            continue

        error = (cardinality - pc.count()) / float(cardinality)
        errors.append(error)

    avg_error = abs(sum(errors)) / float(len(errors))

    assert avg_error >= 0
    assert avg_error <= std

def test_count_small():
    num_of_counters = 256
    pc = ProbabilisticCounter(
        num_of_counters, with_small_cardinality_correction=True)

    # Actually, for small cardinalities we have no estimate. It is
    # just seems that the the errors have to be bigger.
    std = 0.78 / sqrt(num_of_counters)

    boundary = 2 * num_of_counters

    errors = []

    cardinality = 0
    for i in range(boundary):
        cardinality += 1
        element = "element_{}".format(i)
        pc.add(element)

        error = (cardinality - pc.count()) / float(cardinality)
        errors.append(error)

    avg_error = abs(sum(errors)) / float(len(errors))

    assert avg_error >= 0
    assert avg_error <= 3 * std  # There is no known theoretical expectation.

def test_correction():
    pc_with_corr = ProbabilisticCounter(
        256, with_small_cardinality_correction=True)
    pc = ProbabilisticCounter(256)

    errors = []
    errors_with_corr = []

    cardinality = 0
    for i in range(100):
        cardinality += 1
        element = "element_{}".format(i)
        pc_with_corr.add(element)
        pc.add(element)

        error_with_corr = (
            cardinality - pc_with_corr.count()) / float(cardinality)
        errors_with_corr.append(error_with_corr)

        error = abs(cardinality - pc.count()) / float(cardinality)
        errors.append(error)

    avg_error_with_corr = abs(sum(errors_with_corr)) / \
        float(len(errors_with_corr))
    avg_error = abs(sum(errors)) / float(len(errors))

    assert avg_error_with_corr < avg_error

def test_len():
    pc = ProbabilisticCounter(10)
    assert len(pc) == 320


from django.conf.urls import url
from . import views

app_name = 'accounts'

urlpatterns = [

    url(r'login/$', views.LoginView , name="login"),
    url(r'signup/$', views.SignUpView , name="signup"),
    url(r'profile/$', views.ProfileView , name="profile"),
    url(r'logout/$', views.LogoutView , name="logout"),
]


from django import forms
from topics.models import topic

class CreateTopicForm(forms.ModelForm):

    class Meta:
        model = topic
        fields = ['topic_name', 'research_direction', 'type', 'content']

class EditTopicForUserForm(forms.ModelForm):

    class Meta:
        model = topic
        fields = ['topic_name', 'research_direction', 'type', 'content']

    def save(self, commit=True):
        topic_post = self.instance
        topic_post.topic_name = self.cleaned_data['topic_name']
        topic_post.content = self.cleaned_data['content']
        topic_post.research_direction = self.cleaned_data['research_direction']
        topic_post.type = self.cleaned_data['type']

        if commit:
            topic_post.save()
        return topic_post

class EditTopicForSuperUserForm(forms.ModelForm):

    class Meta:
        model = topic
        fields = ['rate', 'review_date', 'process']

    def save(self, commit=True):
        topic_post = self.instance
        topic_post.rate = self.cleaned_data['rate']
        topic_post.review_date = self.cleaned_data['review_date']
        topic_post.process = self.cleaned_data['process']

        if commit:
            topic_post.save()
        return topic_post


from django.urls import path
from . import views
from .views import *

urlpatterns = [
    path('api/todo/', views.TodoView.as_view()),
    path('api/todo/delete/', views.TodoDeleteView),
    path('api/event/', views.EventView.as_view()),
    path('api/event/delete/', views.EventDeleteView),
    path('api/add-members/<event_id>/', views.add_members),
    path('api/invite/<event_id>/', views.invite),
]


import threading
import json
import os
import sys
import signal
import time
import pycdp
import traceback
from config_helper import *

class CDPInterface(threading.Thread):
    def __init__(self):
        super(CDPInterface, self).__init__()
        cdp_config = self.get_cdp_config()
        self._term_event = threading.Event()

        self.daemon = True # so we can kill it with CTRL-C
        # Create the CDP object and a registry entry for a REDIS transport
        self.cdp = pycdp.pycdp(cdp_config, log_level=pycdp.log_debug, log_id="test")
        self.in_channels = []

    def publish(self, out_channel, json_obj):
        json_str = json.dumps(json_obj)
        self.cdp.pub(out_channel, json_str)

    def subscribe(self,channels_list):
        for chan in channels_list:
            self.cdp.sub(chan, self.sub_handler)

    def set(self, key, json_obj):
        json_str = json.dumps(json_obj)
        self.cdp.set(key, json_str)

    def get(self, key):
        (status, m) = self.cdp.get(key)
        if status != 200:
            return None

        try:
            json_obj = json.loads(m)
            return(json_obj)
        except:
            print ("INFO: CDP Error Get() exception:", traceback.format_exc())
        return None

    def exists(self, key):
        (s, m) = self.cdp.get("/exists?key=%s" % (key))
        if s != 200:
            print("INFO: CDP query exists not successful--code: [%d]  key: %s" % (s, key))
            return False
        else:
            #print >>sys.stderr, "CDP: query exists code: [%d],  key: %s, message %s" % (s, key, m)
            if int(m) == 1:
                return True
            return False

    def sets(self, key, strval):
        self.cdp.set(key, strval)

    def gets(self, key):
        (s, m) = self.cdp.get(key)
        return m

    def keys(self, key_pattern):
        ## Returns of matching keys
        (s,m) = self.cdp.get("/keys?pattern=%s" %(key_pattern))
        key_list = json.loads(m)
        return key_list

    def append_in_channel(self, channel):
        self.in_channels.append(channel)
        print("INFO: Subscription to %s was added" % channel)
        s = self.cdp.sub(channel, self.sub_handler)
        print("INFO: Channel subsrciption code ", s)

    def add_subscription(self, channel):
        self.append_in_channel(channel)

    def cancel_cdp(self):
        self._term_event.set()

    def cdp_is_terminated(self):
        return self._term_event.is_set()

    def sub_handler(self, source, channel, raw_message):
        try:
            # Message accidently encodes the b' as a byte string, strip it
            msg = raw_message[2:-1]
            json_obj = json.loads(msg)  # need better error handling
        except:
            print("ERROR: On channel %s json.loads %s" % (channel, traceback.format_exc()))
            return

        try:
            print("DEBUG", channel, json_obj)
            # Notice Calls the derived class version of work
            self.work(channel, json_obj)
        except:
            print("DEBUG: Exception calling work", traceback.format_exc())

    def run(self):
        # This gets called by a call to threads start routine
        while not self.cdp_is_terminated():
            time.sleep(0.01)  # be nice to the system :)
        print("INFO: run thread terminated")

    def external_cdp_config(self):
        # check Environment variable CDP_CONFIG first
        # if not exist, try to load config.json file
        # if not successful, return None for get_cdp_config() processing
        cdp_config = os.getenv('CDP_CONFIG', None)
        if cdp_config == None:
            try:
                cdp_config_path = os.getenv("CDP_CONFIG_PATH", 'future_config.json')
                #TODOLIST: need to be able to handle future_config
                cdp_config_path = get_config_path(cdp_config_path)
                with open (cdp_config_path) as cdp_config_file:
                    cdp_config = json.load(cdp_config_file)
            except:
                print("INFO: could not load CDP_CONFIG from %s, use default cdp_config." % cdp_config_path)
                return None
        return cdp_config

    def get_cdp_config(self):
        cdp_config = self.external_cdp_config()
        if cdp_config == None:
            cdp_config = json.dumps({
                  "info":{
                    "file_version":1.0
                  },
                  "config":{
                    "plane1":{
                    "transport_addr":"redis://edgeservices_cdp_redis"
                    },
                    "plane2":{
                    "transport_addr":"mqtt-tcp://edgeservices_cdp_mqtt"
                    }
                  },
                  "mapping":[
                    ["plane1",12, ".*", ""],
                    ["plane2",3, ".*", ""]
                  ]
            })
        return cdp_config


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import requests
import wget
import os
import time
from requests import ReadTimeout, ConnectTimeout, HTTPError, Timeout, ConnectionError

meme_images = ['https://images.dontbuymeme.com/collections/bitcoin-origins/mp4/wheel-of-fortune.mp4',
               'https://images.dontbuymeme.com/collections/bitcoin-origins/static/inception.jpg',
               'https://images.dontbuymeme.com/collections/bitcoin-origins/static/proliferation.jpg',
               'https://images.dontbuymeme.com/collections/bitcoin-origins/static/propagation.jpg',
               'https://images.dontbuymeme.com/collections/bitcoin-origins/static/the-affirmation-seed.jpg',
               'https://images.dontbuymeme.com/collections/bitcoin-origins/static/pizza-day.jpg',
               'https://images.dontbuymeme.com/collections/bitcoin-origins/static/pride-of-the-lotus.jpg',
              ]

def main():

    # get all images from the meme pool
    if not os.path.exists("meme_pool"):
        os.makedirs("meme_pool")
    for i in meme_images:
        meme_path = "meme_pool"+os.path.sep+i.split('/')[-1]
        if not os.path.isfile(meme_path):
            try:
                wget.download(i, out=meme_path)
            except (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError) as e:
                print("Oops we hit an error - " + e + "\n")
                print("Don't worry we're going to try again in 5 seconds\n")
                time.sleep(5)
                wget.download(i, out=meme_path)
            except Exception as e:
                print("Oops we really hit an error - " + e + "\n")
                print("Don't worry we're going to try again in 10 seconds\n")
                time.sleep(10)
                wget.download(i, out=meme_path)

    # get the number of templates on atomichub
    r = requests.get('https://wax.api.atomicassets.io/atomicassets/v1/collections/bitcoinorign/stats')
    tmp = r.json()
    num_templates = int(tmp['data']['templates'])
    # get all templates
    r = requests.get('https://wax.api.atomicassets.io/atomicassets/v1/templates?limit='+str(num_templates)+'&page=1&collection_name=bitcoinorign')
    templates = r.json()
    for i in range(int(num_templates)):
        img = templates['data'][i]['immutable_data']['img']
        back_img = templates['data'][i]['immutable_data']['backimg'] if 'backimg' in templates['data'][i]['immutable_data'] else None
        name = templates['data'][i]['name']
        moment = templates['data'][i]['immutable_data']['moment'] if 'moment' in templates['data'][i]['immutable_data'] else None
        rarity = templates['data'][i]['immutable_data']['rarity'] if 'rarity' in templates['data'][i]['immutable_data'] else None

        folder = 'moment '+moment if moment is not None else 'other'
        if not os.path.exists(folder):
            os.makedirs(folder)

        filename_1 = name+'_'+rarity if rarity is not None else name
        filename_2 = filename_1+'_'+'backimage' if back_img is not None else None
        filetype_1 = requests.head("https://ipfs.io/ipfs/"+img)
        filetype_1 = filetype_1.headers['Content-Type'].split('/')[-1]

        if filename_2 is not None:
            filetype_2 = requests.head("https://ipfs.io/ipfs/"+back_img)
            filetype_2 = filetype_2.headers['Content-Type'].split('/')[-1]

        path_1 = folder+os.path.sep+filename_1+'.'+filetype_1
        if not os.path.isfile(path_1):
            try:
                wget.download("https://ipfs.io/ipfs/"+img, out=path_1)
            except (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError) as e:
                print("Oops we hit an error - " + e + "\n")
                print("Don't worry we're going to try again in 5 seconds\n")
                time.sleep(5)
                wget.download("https://ipfs.io/ipfs/"+img, out=path_1)
            except Exception as e:
                print("Oops we really hit an error - " + e + "\n")
                print("Don't worry we're going to try again in 10 seconds\n")
                time.sleep(10)
                wget.download("https://ipfs.io/ipfs/"+img, out=path_1)

        if back_img is not None:
            path_2 = folder+os.path.sep+filename_2+'.'+filetype_2
            if not os.path.isfile(path_2):
                try:
                    wget.download("https://ipfs.io/ipfs/"+back_img, out=path_2)
                except (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError) as e:
                    print("Oops we hit an error - " + e + "\n")
                    print("Don't worry we're going to try again in 5 seconds =)")
                    time.sleep(5)
                    wget.download("https://ipfs.io/ipfs/"+back_img, out=path_2)
                except Exception as e:
                    print("Oops we really hit an error - " + e + "\n")
                    print("Don't worry we're going to try again in 10 seconds =)")
                    time.sleep(10)
                    wget.download("https://ipfs.io/ipfs/"+back_img, out=path_2)

    print("\nFinished")

if __name__ == "__main__":
    # execute only if run as a script
    main()


import datetime
import numpy as np
import pandas as pd
import scipy as sp
import scipy.optimize as sco
import statsmodels.api as sm
from scipy.optimize import curve_fit
from scipy.optimize import leastsq as least_squares

import albio.algo_holtwinters as ht
import albio.series_stat as a_s
import albio.forecast as a_f

#-------------------------series-class----------------------------

def standardConf():
    return {"obs_time":0,"decay":[1],"hist_adj": 0.8,"model": "holt"
            ,"res":[14,0.03,0.01,0.02,0.15,0.12,0.13,0.11,0.22]
            ,"holt": [5,0.2,0.1,0.05,0.8]
            ,"poly": [-0.26,-0.043,-1.24,0.0024,-1.6e-06,3.6e-10]
            ,"response": [1.37,1.09,-0.57,0.023,0.110]
            ,"autocor": [0.095,-1.01,-0.40,0.15,8.26,1.0,1.0,1.0,1.0]
            ,"freq": [10.38,5.19]
            ,"arma": [4,0,0]
            ,"sin": [0.,1.,0.03,0.31,2.80,1.0,1.0,1.0,1.0]
    }

class series:
    """statistical properties and forecast of series"""

    def __init__(y, t=[None], conf=None, isPlot=False):
        """analyze time series"""
        ny = len(y)
        if t[0] == None:
            t0 = datetime.datetime.today()-datetime.timedelta(ny)
            t = [t0+datetime.timedelta(x) for x in range(ny)]
        if conf == None:
            conf = standardConf()
        week = [x.isocalendar()[1] for x in t]
        month = [x.month for x in t]
        year = [x.isocalendar()[0] for x in t]
        ser = pd.DataFrame({"y":y,"t":t,"week":week,"month":month,"year":year})
        ser.loc[:,"ts"] = [x.timestamp() for x in t]
        sWeek = ser[['ts','y','week']].groupby(["week"]).agg(np.mean)
        s = ser.iloc[0]
        s1 = pd.DataFrame({'ts':s['ts']-1,'y':s['y']},index=[s['week']])
        s = ser.iloc[-1]
        s2 = pd.DataFrame({'ts':s['ts']+1,'y':s['y']},index=[s['week']])
        sWeek = pd.concat([s1,sWeek,s2])
        sMont = ser[['ts','y','month']].groupby(["month"]).agg(np.mean)
        sYear = ser[['ts','y','year']].groupby(["year"]).agg(np.mean)
        ser = ser.merge(sWeek[['y']],on="week" ,how="left",suffixes=["","_week"])
        ser = ser.merge(sMont[['y']],on="month",how="left",suffixes=["","_month"])
        ser = ser.merge(sYear[['y']],on="year" ,how="left",suffixes=["","_year"])
        ser.loc[:,"roll"] = a_s.serSmooth(ser['y'],4,11)
        ser.loc[:,"e_av"] = ser['y'].ewm(halflife=1).mean()
        s2 = pd.DataFrame({"week":s['week'],'ts':s['ts'],'y':s['y']})
        ser['inte'] = sp.interpolate.interp1d(sWeek['ts'],sWeek['y'],kind="cubic")(ser['ts'])
        ser['deri'] = ser['y'] - ser['y'].shift()
        ser['diff'] = ser['y'] - ser['e_av']
        ser['rema'] = ser['y'] - ser['roll']
        ser['hist'] = ser['y'] - ser['inte']
        ser['adju'] = ( (ser['hist']-ser['hist'].min())/(ser['hist'].max()-ser['hist'].min()) + .5)*conf['hist_adj']
        ser.drop(columns={"week","month","year"},inplace=True)
        ser.replace(float('nan'),0.,inplace=True)
        self.ser = ser
        self.sWeek = sWeek
        if isPlot:
            plt.bar( ser['t'],ser['y_year'],label="yearly",alpha=0.1)
            plt.bar( ser['t'],ser['y_month'],label="monthly",alpha=0.1)
            plt.bar( ser['t'],ser['y_week'],label="weekly",alpha=0.1)
            plt.plot(ser['t'],ser['y'],label="series",linewidth=2)
            plt.plot(ser['t'],ser['roll'],label="rolling average",linewidth=1)
            plt.plot(ser['t'],ser['e_av'],label="average",linewidth=1)
            plt.plot(ser['t'],ser['inte'],label="interpolate week",linewidth=1)
            plt.legend()
            plt.xticks(rotation=15)
            plt.show()
            plt.plot(ser['t'],ser['deri'],label="derivative",linewidth=1)
            plt.plot(ser['t'],ser['diff'],label="remainder exp",linewidth=1)
            plt.plot(ser['t'],ser['rema'],label="remainder roll",linewidth=1)
            plt.plot(ser['t'],ser['hist'],label="historical week",linewidth=1)
            plt.legend()
            plt.xticks(rotation=15)
            plt.show()

    def trendFluct(self, n_ahead=14):
        """predict the signal decomposing the trend from the fluctuations"""
        ser = self.ser
        t1, y1, pol = a_f.trendFluct(ser['t'],ser['t'],ser['y'],n=14,conf=conf)
        return t1, y1, pol

##---------------------------series-preprocessing------------------------

##---------------------------forecast-methods----------------------------

def serLsq(sDay, nAhead, x0, hWeek):
    nFit = sDay.shape[0]  # if int(x0['obs_time']) <= 14 else int(x0['obs_time'])
    predS, x0 = getHistory(sDay, nAhead, x0, hWeek)
    predS = predS.tail(nFit + nAhead)
    freqP = x0['freq']
    res_lsq = least_squares(ser_sin_min, x0['lsq'], args=(sDay.t, sDay.stat, x0['freq']))  # loss='soft_l1',f_scale=0.1,
    x0['lsq'] = [x for x in res_lsq[0]]
    predS['lsq'] = ser_sin(res_lsq[0], predS.t, x0['freq'])  # fun(res_robust.x,t_test)
    sDay['resid'] = sDay['y'] - ser_sin(res_lsq[0], sDay.t, x0['freq']) / sDay['hist'] - ser_poly(x0['poly'], sDay[
        't'])  # lm.predict(sDay)
    rSquare = (sDay['resid'].tail(x0['res'][0]) - sDay['resid'].tail(x0['res'][0]).mean()) ** 2
    x0['res'][1] = rSquare.sum()
    x0['res'][2] = rSquare.sum() / sDay['y'].tail(x0['res'][0]).sum()
    # predS['pred'] = (predS['lsq'] + lm.predict(predS))*predS['hist']*x0['hist_adj']
    predS['pred'] = ser_sin(res_lsq[0], predS.t, x0['freq']) * sDay['hist'] + ser_poly(x0['poly'], predS['t'])
    predS = predS.drop(predS.index[0])
    return predS, x0

def bestArima(sDay, nAhead, x0, hWeek):
    dta = sDay['y']
    dta.index = [pd.datetime.strptime(str(x)[0:10], '%Y-%m-%d') for x in dta.index]
    t_line = [float(calendar.timegm(x.utctimetuple())) / 1000000 for x in dta.index]
    t_line1 = [float(calendar.timegm(x.utctimetuple())) / 1000000 for x in hWeek.index]
    sExog = pd.DataFrame({'y': sp.interpolate.interp1d(t_line1, hWeek.y, kind="cubic")(t_line)})
    grid = (slice(1, 4, 1), slice(1, 2, 1), slice(1, 3, 1))

    def objfunc(order, endog, exog):
        fit = sm.tsa.ARIMA(endog, order).fit(trend="c", method='css-mle', exog=exog)
        return fit.aic

    par = sco.brute(objfunc, grid, args=(dta, sExog), finish=None)
    return par

def serArma(sDay, nAhead, x0, hWeek):
    predS, x0 = getHistory(sDay, nAhead, x0, hWeek)
    dta = sDay['y']
    dta.index = [pd.datetime.strptime(str(x)[0:10], '%Y-%m-%d') for x in dta.index]
    sDay.index = dta.index
    t_line = [float(calendar.timegm(x.utctimetuple())) / 1000000 for x in dta.index]
    t_line1 = [float(calendar.timegm(x.utctimetuple())) / 1000000 for x in hWeek.index]
    sExog = pd.DataFrame({'y': sp.interpolate.interp1d(t_line1, hWeek.y, kind="cubic")(t_line)})
    # par = bestArima(dta,sExog)
    sExog.index = dta.index
    result = sm.tsa.ARIMA(dta, (x0['arma'][0], x0['arma'][1], x0['arma'][2])).fit(trend="c", method='css-mle',
                                                                                exog=sExog)
    predT = [str(dta.index[0])[0:10], str(dta.index[len(dta) - 1])[0:10]]
    histS = pd.DataFrame({'pred': result.predict(start=predT[0], end=predT[1])})
    # predT = [str(dta.index[0])[0:10],str(dta.index[len(dta)-1]+datetime.timedelta(days=nAhead))[0:10]]
    predS = predS[0:(len(dta) - 1)]
    predS['pred'] = result.predict(start=predT[0], end=predT[1])
    # predS = pd.DataFrame({'pred':result.predict(start=predT[0],end=predT[1])})
    # predS.index = [dta.index[0]+datetime.timedelta(days=x) for x in range(0,len(dta)+nAhead)]
    # predS['t'] = [float(calendar.timegm(x.utctimetuple()))/1000000 for x in predS.index]
    # predS['hist'] = sp.interpolate.interp1d(t_line1,hWeek.y,kind="cubic")(predS['t'])
    # predS['hist'] = predS['hist']/predS['hist'].mean()
    # predS['pred'] = predS['pred']*predS['hist']
    predS['trend'] = ser_poly(x0['poly'], predS.t)
    predS['y'] = sDay['y']
    predS['pred'] = (predS['pred'] * predS['hist'] + predS['trend'])
    x0['response'] = [x for x in result.params]
    sDay['resid'] = sDay['y'] - predS['pred'][0:sDay.shape[0]]
    rSquare = (sDay['resid'].tail(x0['res'][0]) - sDay['resid'].tail(x0['res'][0]).mean()) ** 2
    x0['res'][1:2] = [rSquare.sum(), rSquare.sum() / sDay['y'].tail(x0['res'][0]).sum()]
    return predS, x0
    # plt.plot(dta,'-k',label="series")
    # plt.plot(sExog,label="exo")
    # plt.plot(hWeek.y,label="hist")
    # plt.plot(predS,'-b',label="pred")
    # plt.legend()
    # plt.show()
    # steps = 1
    # tsa.arima_model._arma_predict_out_of_sample(res.params,steps,res.resid,res.k_ar,res.k_ma,res.k_trend,res.k_exog,endog=dta, exog=None, start=len(dta))

def SerBayes(sDay, nAhead, x0, hWeek):
    import pydlm
    dta = sDay['y']
    dta.index = [pd.datetime.strptime(str(x)[0:10], '%Y-%m-%d') for x in dta.index]
    t_line = [float(calendar.timegm(x.utctimetuple())) / 1000000 for x in dta.index]
    dta.index = t_line
    model = pydlm.dlm(dta)
    model = model + pydlm.trend(degree=1, discount=0.98, name='a', w=10.0)
    model = model + pydlm.dynamic(features=[[v] for v in t_line], discount=1, name='b', w=10.0)
    model = model + pydlm.autoReg(degree=3, data=dta.values, name='ar3', w=1.0)
    allStates = model.getLatentState(filterType='forwardFilter')
    model.evolveMode('independent')
    model.noisePrior(2.0)
    model.fit()
    model.plot()
    model.turnOff('predict')
    model.plotCoef(name='a')
    model.plotCoef(name='b')
    model.plotCoef(name='ar3')

def serHolt(sDay, nAhead, x0, hWeek):
    predS, x0 = getHistory(sDay, nAhead, x0, hWeek)
    Y = [x for x in sDay.y]
    ##Yht, alpha, beta, gamma, rmse = ht.additive([x for x in sDay.y],int(x0[0]),nAhead,x0[1],x0[2],x0[3])
    nAv = int(x0['holt'][0]) if int(x0['holt'][0]) > 1 else 5
    Yht, alpha, beta, gamma, rmse = ht.additive([x for x in sDay.y], x0['holt'][0], nAhead, x0['holt'][1],
                                                x0['holt'][2], x0['holt'][3])
    sDay['resid'] = sDay['y'] - Yht[0:sDay.shape[0]]
    x0['holt'] = [x0['holt'][0], alpha, beta, gamma, rmse]
    nLin = sDay.shape[0] + nAhead
    t_test = np.linspace(sDay['t'][0], sDay['t'][sDay.shape[0] - 1] + sDay.t[nAhead] - sDay.t[0], nLin)
    # predS = pd.DataFrame({'t':t_test},index=[sDay.index[0]+datetime.timedelta(days=x) for x in range(nLin)])
    predS['pred'] = Yht
    # predS['hist'] = sp.interpolate.interp1d(hWeek.t,hWeek.y,kind="cubic")(predS['t'])
    # predS['hist'] = predS['hist']/predS['hist'].mean()
    # predS['pred'] = predS['pred']*predS['hist']*x0['hist_adj']
    # predS['trend'] = ser_poly(x0['poly'],predS.t)
    # predS['trend'].ix[0:(sDay.shape[0]-nFit)] = predS['trend'][sDay.shape[0]-nFit]
    predS['lsq'] = 0
    predS['y'] = sDay['y']
    sDay['resid'] = sDay['y'] - predS['pred'][0:sDay.shape[0]]
    rSquare = (sDay['resid'].tail(x0['res'][0]) - sDay['resid'].tail(x0['res'][0]).mean()) ** 2
    x0['res'][1] = rSquare.sum()
    x0['res'][2] = rSquare.sum() / sDay['y'].tail(x0['res'][0]).sum()
    return predS, x0

def serAuto(sDay, nAhead, x0, hWeek):
    predS, x0 = getHistory(sDay, nAhead, x0, hWeek)
    todayD = datetime.datetime.today()
    todayD = todayD.replace(hour=0, minute=0, second=0, microsecond=0)
    dta = pd.DataFrame({'y': sDay.y})
    dta['day'] = sDay.index.weekday
    phase = dta.head(int(x0['obs_time'])).groupby(['day']).mean()
    phase['std'] = dta.groupby(['day']).std()['y']
    phase = phase.sort_values(['y'], ascending=False)
    phase['csum'] = phase['y'].cumsum() / phase['y'].sum()
    phaseN = phase.index[0] - todayD.weekday()
    r, q, p = sm.tsa.acf(sDay['y'].tail(phaseN + int(x0['obs_time'])).squeeze(), qstat=True)
    popt, pcov = curve_fit(ser_exp, np.array(range(0, 6)), r[0:6] - min(r), p0=(x0['decay'][0]))
    X = np.array(range(0, r.size, 7))
    popt1, pcov1 = curve_fit(ser_exp, X, r[X], p0=(x0['decay'][0]))
    autD = pd.DataFrame({'r': r, 'exp': ser_exp(range(0, r.size), popt), 'exp1': ser_exp(range(0, r.size), popt1)})
    x0['decay'] = [x for x in popt]
    wN = 0
    sY = np.random.normal(phase['y'].head(1), dta.y.std())
    for i in predS.index:
        wN = 6 - np.abs(phase.index[0] - i.weekday())
        wN = wN + 1 if wN < 6 else 0
        if (wN == 0):
            sY = np.random.normal(phase['y'].head(1), dta.y.std() / 2)
        sY = sY * (1 + predS['hist'][i] * x0['hist_adj'])
        predS.loc[i, 'pred'] = sY * ser_exp(float(wN), popt)

    predS['pred'] = serSmooth(predS['pred'], 16, 5)
    sDay['resid'] = sDay['y'] - predS['pred'][0:sDay.shape[0]]
    freqP = x0['freq']
    res_lsq = least_squares(ser_fun_min, x0['lsq'], args=(sDay['t'], sDay['resid'], x0['freq']))
    predS['lsq'] = ser_sin(res_lsq[0], predS['t'], x0['freq'])  # fun(res_robust.x,t_test)
    x0['lsq'][0:res_lsq[0].size] = res_lsq[0]
    predS['pred2'] = predS['pred']
    predS['pred'] = predS['pred'] + predS['lsq']
    sDay['resid'] = sDay['y'] - predS['pred'][0:sDay.shape[0]]
    rSquare = (sDay['resid'].tail(x0['res'][0]) - sDay['resid'].tail(x0['res'][0]).mean()) ** 2
    x0['res'][1] = rSquare.sum()
    x0['res'][2] = rSquare.sum() / sDay['y'].tail(x0['res'][0]).sum()
    # sDay.to_csv('tmpAuto1.csv')
    # predS.to_csv('tmpAuto2.csv')
    # autD.to_csv('tmpAuto3.csv')
    return predS, x0


import urllib
from unittest.mock import MagicMock, create_autospec, Mock

import pytest

from server_administrator.server_administrator import ServerAdministrator, ServerConfiguration

class Fixtures:

    @pytest.fixture
    def mock_seconds_of_sleep(self):
        return Mock()

    @pytest.fixture
    def mock_urlretrieve(self, mocker):
        mock = MagicMock()
        mocker.patch("urllib.request.urlretrieve", mock)

    @pytest.fixture
    def mock_subprocess_run(self, mocker):
        mock = MagicMock()
        mocker.patch("subprocess.run", mock)
        return mock

    @pytest.fixture
    def mock_directory_builder(self):
        from server_administrator.directory_builder import DirectoryBuilder
        mock_builder = create_autospec(DirectoryBuilder)
        return mock_builder("")

    @pytest.fixture
    def mock_eula_editor(self):
        from server_administrator.editor import EulaEditor
        mock_editor = create_autospec(EulaEditor)
        return mock_editor("")

    @pytest.fixture(autouse=True)
    def administrator(self, mock_directory_builder, server_configuration, mock_eula_editor, mock_seconds_of_sleep):
        return ServerAdministrator(
            config=server_configuration,
            builder=mock_directory_builder,
            editor=mock_eula_editor,
            sleeper=mock_seconds_of_sleep
        )

    @pytest.fixture
    def server_configuration(self):
        return ServerConfiguration(
            directory="/opt/minecraft/server",
            port=25565,
            max_memory=1024,
            min_memory=1024,
        )

class TestRunner(Fixtures):

    # Todo: consider abstracting
    def test_can_download_the_server_jar(self, administrator, mock_urlretrieve, mock_eula_editor, mock_subprocess_run):
        administrator.build_and_launch_server()

        urllib.request.urlretrieve.assert_called_with(
            url="https://piston-data.mojang.com/v1/objects/c9df48efed58511cdd0213c56b9013a7b5c9ac1f/server.jar",
            filename="/opt/minecraft/server/server.jar"
        )

    def test_can_correctly_launch_minecraft_jar(self, mock_directory_builder, mock_eula_editor,
                                                mock_subprocess_run, mock_seconds_of_sleep, mock_urlretrieve):
        config = ServerConfiguration(
            max_memory=4096,
            min_memory=512,
            directory="/test/directory",
            port=5555,
        )
        administrator = ServerAdministrator(config, mock_directory_builder, mock_eula_editor, mock_seconds_of_sleep)

        administrator.build_and_launch_server()

        mock_subprocess_run.assert_called_with(
            [
                "/usr/bin/java",
                f'-Xmx4096M',
                f'-Xms512M',
                "-jar", "server.jar",
                "nogui"
            ],
            cwd="/test/directory"
        )

    # * I'm not actually sure the best way of spelling out this logic in a test other than this.
    # * the basic idea is that with the minecraft server we launch the jar once, the jar creates (amongst other things)
    # * the end user license agreement (the eula we need to modify) and then the jar's process exits asking you to
    # * update the eula. once it's updated we can launch the jar again and the server will run.
    # * It feels like in the spirit of "tests as documentation" we'd do something like assert the first call, then the
    # * eula update, then the second call in that order, but I don't know if that's actually possible.
    def test_can_update_eula(self, administrator, mock_eula_editor, mock_urlretrieve, mock_subprocess_run):
        administrator.build_and_launch_server()

        assert mock_subprocess_run.call_count == 2
        mock_eula_editor.update_state.assert_called_with("/opt/minecraft/server/eula.txt")


from flask import Flask, request

app = Flask(__name__)

visitors = {}

def get_hit_count(remote_ip):
    if remote_ip in visitors:
        visitors[remote_ip] = visitors[remote_ip] + 1
    else:
        visitors[remote_ip] = 1
    hits = visitors[remote_ip]
    return hits

@app.route('/',  methods=["GET"])
def get_ip():
    hits = get_hit_count(str(request.environ.get('HTTP_X_REAL_IP', request.remote_addr)))
    ip_res = "User Address: " + request.environ.get('HTTP_X_REAL_IP', request.remote_addr)
    hit_count = "\nHits: " + str(hits)
    return ip_res+hit_count

if __name__ == '__main__':
    app.run(debug=True,host='0.0.0.0', port=80)


import argparse
import os
import shutil
import time

import pixmix_utils as utils
import numpy as np

import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torch.nn.functional as F
from torchvision import datasets
from torchvision import transforms
from tqdm import tqdm
import torchvision.transforms.functional as TF

from PIL import Image
import cv2

k = 4
beta = 4
all_ops = True
aug_severity = 1

IMAGENET_MEAN = [0.485, 0.456, 0.406]
IMAGENET_STD = [0.229, 0.224, 0.225]

to_tensor = transforms.ToTensor()
normalize = transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)
standard_preprocess = {'tensorize': to_tensor, 'normalize': normalize}

def pixmix(orig, mixing_pic, preprocess):

    mixings = utils.mixings
    tensorize, normalize = preprocess['tensorize'], preprocess['normalize']
    if np.random.random() < 0.5:
        mixed = tensorize(augment_input(orig))
    else:
        mixed = tensorize(orig)

    for _ in range(np.random.randint(k + 1)):

        if np.random.random() < 0.5:
            aug_image_copy = tensorize(augment_input(orig))
        else:
            aug_image_copy = tensorize(mixing_pic)

        mixed_op = np.random.choice(mixings)
        mixed = mixed_op(mixed, aug_image_copy, beta)
        mixed = torch.clip(mixed, 0, 1)

    # return normalize(mixed)
    return mixed

def augment_input(image):
    aug_list = utils.augmentations_all if all_ops else utils.augmentations
    op = np.random.choice(aug_list)
    return op(image.copy(), aug_severity)

# Load the image
# sample_image = Image.open('panda.jpg')
mixing_image = Image.open("fractal.jpg")

# sample_image = sample_image.resize((256,256))
mixing_image = mixing_image.resize((256,256))

# # Apply the pixmix
# pixed = pixmix(sample_image, mixing_image, standard_preprocess)

# image = TF.to_pil_image(pixed)
# image.save('pixed_panda.jpg')

# a = to_tensor(sample_image)
# b = to_tensor(mixing_image)
# res = utils.multiply(a,b, beta)
# res = torch.clip(res, 0, 1)
# res = TF.to_pil_image(res)
# res.save('pixed_panda.jpg')

# apply pixmix to a video
video_path = "v_Archery_g01_c04.avi"
output_path = "output.avi"

cap = cv2.VideoCapture(video_path)
fps = cap.get(cv2.CAP_PROP_FPS)
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

fourcc = cv2.VideoWriter_fourcc(*'MJPG')
out = cv2.VideoWriter(output_path, fourcc, fps, (frame_width, frame_height))

for i in tqdm(range(frame_count)):
    ret, frame = cap.read()
    if ret == True:
        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        frame = Image.fromarray(frame)
        frame = frame.resize((256,256))
        # pixed = pixmix(frame, mixing_image, standard_preprocess)

        a = to_tensor(frame)
        b = to_tensor(mixing_image)
        if np.random.random() < 0.5:
            pixed = utils.add(a,b, beta)
        else:
            pixed = utils.multiply(a,b, beta)
        pixed = torch.clip(pixed, 0, 1)

        pixed = TF.to_pil_image(pixed)
        pixed = cv2.cvtColor(np.array(pixed), cv2.COLOR_RGB2BGR)
        pixed = cv2.resize(pixed, (frame_width, frame_height))
        out.write(pixed)
    else:
        break

cap.release()
out.release()


import pickle
from bioservices.kegg import KEGG
k = KEGG()

import pandas
from statsmodels.stats import proportion

from capblood_seq import common
from capblood_seq import resources
from capblood_seq import config

from . import diurnality as diurnality_data
from . import individuality as individuality_data

GENE_PATHWAY_DICT = None
PATHWAY_CLASS_LABELS = None
PATHWAY_CLASSES = None
PATHWAY_ENRICHMENT_BY_THRESHOLD = None

def load_pathway_data():

    global GENE_PATHWAY_DICT
    global PATHWAY_CLASS_LABELS
    global PATHWAY_CLASSES
    global PATHWAY_ENRICHMENT_BY_THRESHOLD

    if GENE_PATHWAY_DICT is None:
        GENE_PATHWAY_DICT = pickle.load(
            open(common.get_resource_path(
                resources, "gene_pathway_dict.pickle"
            ), "rb")
        )

        with open(common.get_resource_path(
                resources, "pathway_class_labels.tsv"), "r") as \
                pathway_class_labels_file:

            PATHWAY_CLASS_LABELS = {}

            pathway_class_labels_file.readline()
            for line in pathway_class_labels_file.readlines():
                line = line.strip()
                pathway_class_label = line.split("\t")
                PATHWAY_CLASS_LABELS[pathway_class_label[0]] = \
                    pathway_class_label[1]

        pathway_class_df = pandas.read_csv(
            common.get_resource_path(resources, "pathway_classes.csv"),
            index_col=0, header=0)

        PATHWAY_CLASSES = {}

        for row in pathway_class_df.iterrows():
            pathway = row[0]
            PATHWAY_CLASSES[pathway] = row[1][1]

        PATHWAY_ENRICHMENT_BY_THRESHOLD = {}

def get_pathway(gene):

    global GENE_PATHWAY_DICT

    if gene in GENE_PATHWAY_DICT:
        return GENE_PATHWAY_DICT[gene]
    else:
        GENE_PATHWAY_DICT[gene] = k.get_pathway_by_gene(gene, "hsa")

def get_pathway_enrichment(num_top_genes=250):

    load_pathway_data()

    global PATHWAY_CLASS_LABELS
    global PATHWAY_CLASSES
    global PATHWAY_ENRICHMENT_BY_THRESHOLD

    if num_top_genes in PATHWAY_ENRICHMENT_BY_THRESHOLD:
        return PATHWAY_ENRICHMENT_BY_THRESHOLD[num_top_genes]

    diurnality_df = diurnality_data.get_gene_diurnality()
    diurnality_df = diurnality_df.sort_values(by="By Cell Type p-value")
    individuality_df = individuality_data.get_gene_individuality()
    individuality_df = individuality_df.sort_values(by="By Cell Type p-value")

    pathway_set = set()

    diurnal_pathway_counts = {}
    diurnal_label_counts = {}
    nondiurnal_pathway_counts = {}
    nondiurnal_label_counts = {}
    individual_pathway_counts = {}
    individual_label_counts = {}
    nonindividual_pathway_counts = {}
    nonindividual_label_counts = {}

    for gene_index, gene in enumerate(diurnality_df.index.values):

        pathways = get_pathway(gene)

        if pathways is None:
            continue

        pathway_count = 1/len(pathways)

        for pathway_id, pathway in pathways.items():
            pathway_set.add(pathway)
            pathway_class = PATHWAY_CLASSES[pathway]
            pathway_label = PATHWAY_CLASS_LABELS[pathway_class]
            if gene_index < num_top_genes:
                if pathway not in diurnal_pathway_counts:
                    diurnal_pathway_counts[pathway] = 0
                diurnal_pathway_counts[pathway] += pathway_count
                if pathway_label not in diurnal_label_counts:
                    diurnal_label_counts[pathway_label] = 0
                diurnal_label_counts[pathway_label] += pathway_count
            else:
                if pathway not in nondiurnal_pathway_counts:
                    nondiurnal_pathway_counts[pathway] = 0
                nondiurnal_pathway_counts[pathway] += pathway_count
                if pathway_label not in nondiurnal_label_counts:
                    nondiurnal_label_counts[pathway_label] = 0
                nondiurnal_label_counts[pathway_label] += pathway_count

    for gene_index, gene in enumerate(individuality_df.index.values):

        pathways = get_pathway(gene)

        if pathways is None:
            continue

        pathway_count = 1 / len(pathways)

        for pathway_id, pathway in pathways.items():
            pathway_set.add(pathway)
            pathway_class = PATHWAY_CLASSES[pathway]
            pathway_label = PATHWAY_CLASS_LABELS[pathway_class]
            if gene_index < num_top_genes:
                if pathway not in individual_pathway_counts:
                    individual_pathway_counts[pathway] = 0
                individual_pathway_counts[pathway] += pathway_count
                if pathway_label not in individual_label_counts:
                    individual_label_counts[pathway_label] = 0
                individual_label_counts[pathway_label] += pathway_count
            else:
                if pathway not in nonindividual_pathway_counts:
                    nonindividual_pathway_counts[pathway] = 0
                nonindividual_pathway_counts[pathway] += pathway_count
                if pathway_label not in nonindividual_label_counts:
                    nonindividual_label_counts[pathway_label] = 0
                nonindividual_label_counts[pathway_label] += pathway_count

    pathway_list = []
    pathway_diurnal_enrichments = []
    pathway_individual_enrichments = []

    for pathway in pathway_set:

        if pathway not in diurnal_pathway_counts:
            diurnal_pathway_counts[pathway] = 0
        if pathway not in nondiurnal_pathway_counts:
            nondiurnal_pathway_counts[pathway] = 0
        if pathway not in individual_pathway_counts:
            individual_pathway_counts[pathway] = 0
        if pathway not in nonindividual_pathway_counts:
            nonindividual_pathway_counts[pathway] = 0

        pathway_list.append(pathway)

        diurnal_enrichment, diurnal_p = proportion.proportions_ztest(
            [
                diurnal_pathway_counts[pathway],
                nondiurnal_pathway_counts[pathway]
            ],
            [
                sum(diurnal_pathway_counts.values()),
                sum(nondiurnal_pathway_counts.values())
            ]
        )

        individual_enrichment, individual_p = proportion.proportions_ztest(
            [
                individual_pathway_counts[pathway],
                nonindividual_pathway_counts[pathway]
            ],
            [
                sum(individual_pathway_counts.values()),
                sum(nonindividual_pathway_counts.values())
            ]
        )

        pathway_diurnal_enrichments.append(diurnal_enrichment)
        pathway_individual_enrichments.append(individual_enrichment)

    pathway_label_data = {}

    for class_index, class_label in enumerate(config.PATHWAY_LABELS):

        x_values = []
        y_values = []
        text_labels = []

        for pathway_index, pathway in enumerate(pathway_list):
            pathway_class = PATHWAY_CLASSES[pathway]
            pathway_label = PATHWAY_CLASS_LABELS[pathway_class]
            if pathway_label == class_label:
                x_values.append(pathway_individual_enrichments[pathway_index])
                y_values.append(pathway_diurnal_enrichments[pathway_index])
                text_labels.append(pathway)

        if class_label not in diurnal_label_counts:
            diurnal_label_counts[class_label] = 0
        if class_label not in nondiurnal_label_counts:
            nondiurnal_label_counts[class_label] = 0
        if class_label not in individual_label_counts:
            individual_label_counts[class_label] = 0
        if class_label not in nonindividual_label_counts:
            nonindividual_label_counts[class_label] = 0

        individual_enrichment, _ = proportion.proportions_ztest(
            [
                individual_label_counts[class_label],
                nonindividual_label_counts[class_label]
            ],
            [
                sum(individual_label_counts.values()),
                sum(nonindividual_label_counts.values())
            ]
        )

        diurnal_enrichment, _ = proportion.proportions_ztest(
            [
                diurnal_label_counts[class_label],
                nondiurnal_label_counts[class_label]
            ],
            [
                sum(diurnal_label_counts.values()),
                sum(nondiurnal_label_counts.values())
            ]
        )

        x_values.append(individual_enrichment)
        y_values.append(diurnal_enrichment)
        text_labels.append(class_label)

        pathway_label_data[class_label] = {
            "x_values": x_values,
            "y_values": y_values,
            "text_labels": text_labels
        }

    PATHWAY_ENRICHMENT_BY_THRESHOLD[num_top_genes] = pathway_label_data

    return pathway_label_data


"""
This is a Python3 PyQt5 firmware upload GUI for the SparkFun RTK products - based on ESP32 esptool.exe

MIT license

Please see the LICENSE.md for more details

"""

# import action things - the .syntax is used since these are part of the package
from .au_worker import AUxWorker
from .au_action import AxJob
from .au_act_esptool import AUxEsptoolDetectFlash, AUxEsptoolUploadFirmware, AUxEsptoolResetESP32

import darkdetect
import sys
import os
import os.path
import platform

import serial

from time import sleep

from typing import Iterator, Tuple

from PyQt5.QtCore import QSettings, QProcess, QTimer, Qt, QIODevice, pyqtSignal, pyqtSlot, QObject
from PyQt5.QtWidgets import QWidget, QLabel, QComboBox, QGridLayout, \
    QPushButton, QApplication, QLineEdit, QFileDialog, QPlainTextEdit, \
    QAction, QActionGroup, QMenu, QMenuBar, QMainWindow, QMessageBox
from PyQt5.QtGui import QCloseEvent, QTextCursor, QIcon, QFont
from PyQt5.QtSerialPort import QSerialPort, QSerialPortInfo

_APP_NAME = "RTK Firmware Uploader"

# sub folder for our resource files
_RESOURCE_DIRECTORY = "resource"

#https://stackoverflow.com/a/50914550
def resource_path(relative_path):
    """ Get absolute path to resource, works for dev and for PyInstaller """
    base_path = getattr(sys, '_MEIPASS', os.path.dirname(os.path.abspath(__file__)))
    return os.path.join(base_path, _RESOURCE_DIRECTORY, relative_path)

def get_version(rel_path: str) -> str:
    try:
        with open(resource_path(rel_path), encoding='utf-8') as fp:
            for line in fp.read().splitlines():
                if line.startswith("__version__"):
                    delim = '"' if '"' in line else "'"
                    return line.split(delim)[1]
            raise RuntimeError("Unable to find version string.")
    except:
        raise RuntimeError("Unable to find _version.py.")

_APP_VERSION = get_version("_version.py")

# ----------------------------------------------------------------
# hack to know when a combobox menu is being shown. Helpful if contents
# of list are dynamic -- like serial ports.

class AUxComboBox(QComboBox):

    popupAboutToBeShown = pyqtSignal()

    def showPopup(self):
        self.popupAboutToBeShown.emit()
        super().showPopup()

#----------------------------------------------------------------
# ux_is_darkmode()
#
# Helpful function used during setup to determine if the Ux is in
# dark mode
_is_darkmode = None
def ux_is_darkmode() -> bool:
    global _is_darkmode

    if _is_darkmode is not None:
        return _is_darkmode

    osName = platform.system()

    if osName == "Darwin":
        _is_darkmode = darkdetect.isDark()

    elif osName == "Windows":
        # it appears that the Qt interface on Windows doesn't apply DarkMode
        # So, just keep it light
        _is_darkmode = False
    elif osName == "Linux":
        # Need to check this on Linux at some pont
        _is_darkmod = False

    else:
        _is_darkmode = False

    return _is_darkmode

# Setting constants
SETTING_PORT_NAME = 'port_name'
SETTING_FILE_LOCATION = 'file_location'
#SETTING_PARTITION_LOCATION = 'partition_location'
SETTING_BAUD_RATE = 'baud'

def gen_serial_ports() -> Iterator[Tuple[str, str, str]]:
    """Return all available serial ports."""
    ports = QSerialPortInfo.availablePorts()
    return ((p.description(), p.portName(), p.systemLocation()) for p in ports)

# noinspection PyArgumentList

class MainWidget(QWidget):
    """Main Widget."""

    sig_message = pyqtSignal(str)
    sig_finished = pyqtSignal(int, str, int)

    def __init__(self, parent: QWidget = None) -> None:
        super().__init__(parent)

        self.flashSize = 0

        # File location line edit
        self.file_label = QLabel(self.tr('Firmware File:'))
        self.fileLocation_lineedit = QLineEdit()
        self.file_label.setBuddy(self.fileLocation_lineedit)
        self.fileLocation_lineedit.setEnabled(False)
        self.fileLocation_lineedit.returnPressed.connect(self.on_browse_btn_pressed)

        # Browse for new file button
        self.browse_btn = QPushButton(self.tr('Browse'))
        self.browse_btn.setEnabled(True)
        self.browse_btn.pressed.connect(self.on_browse_btn_pressed)

        # # Partition file location line edit
        # self.partition_label = QLabel(self.tr('Partition File:'))
        # self.partitionFileLocation_lineedit = QLineEdit()
        # self.partition_label.setBuddy(self.partitionFileLocation_lineedit)
        # self.partitionFileLocation_lineedit.setEnabled(False)
        # self.partitionFileLocation_lineedit.returnPressed.connect(self.on_partition_browse_btn_pressed)

        # # Browse for new file button
        # self.partition_browse_btn = QPushButton(self.tr('Browse'))
        # self.partition_browse_btn.setEnabled(True)
        # self.partition_browse_btn.pressed.connect(self.on_partition_browse_btn_pressed)

        # Port Combobox
        self.port_label = QLabel(self.tr('COM Port:'))
        self.port_combobox = AUxComboBox()
        self.port_label.setBuddy(self.port_combobox)
        self.update_com_ports()
        self.port_combobox.popupAboutToBeShown.connect(self.on_port_combobox)

        # Reset Button
        self.reset_btn = QPushButton(self.tr('Reset ESP32'))
        self.reset_btn.clicked.connect(self.tera_term_reset)

        # Baudrate Combobox
        self.baud_label = QLabel(self.tr('Baud Rate:'))
        self.baud_combobox = QComboBox()
        self.baud_label.setBuddy(self.baud_combobox)
        self.update_baud_rates()

        # Upload Button
        myFont=QFont()
        myFont.setBold(True)
        self.upload_btn = QPushButton(self.tr('  Upload Firmware  '))
        self.upload_btn.setFont(myFont)
        self.upload_btn.clicked.connect(self.on_upload_btn_pressed)

        # Messages Bar
        self.messages_label = QLabel(self.tr('Status / Warnings:'))

        # Messages Window
        self.messageBox = QPlainTextEdit()
        color =  "C0C0C0" if ux_is_darkmode() else "424242"
        self.messageBox.setStyleSheet("QPlainTextEdit { color: #" + color + ";}")
        self.messageBox.setReadOnly(True)
        self.messageBox.clear()

        # Arrange Layout
        layout = QGridLayout()

        layout.addWidget(self.file_label, 1, 0)
        layout.addWidget(self.fileLocation_lineedit, 1, 1)
        layout.addWidget(self.browse_btn, 1, 2)

        # layout.addWidget(self.partition_label, 2, 0)
        # layout.addWidget(self.partitionFileLocation_lineedit, 2, 1)
        # layout.addWidget(self.partition_browse_btn, 2, 2)

        layout.addWidget(self.port_label, 2, 0)
        layout.addWidget(self.port_combobox, 2, 1)
        layout.addWidget(self.reset_btn, 2, 2)

        layout.addWidget(self.baud_label, 3, 0)
        layout.addWidget(self.baud_combobox, 3, 1)
        layout.addWidget(self.upload_btn, 3, 2)

        layout.addWidget(self.messages_label, 4, 0)
        layout.addWidget(self.messageBox, 5, 0, 5, 3)

        self.setLayout(layout)

        self.settings = QSettings()
        #self._clean_settings() # This will delete all existing settings! Use with caution!
        self._load_settings()

        self.setWindowTitle( _APP_NAME + " - " + _APP_VERSION)

        # setup our background worker thread ...

        # connect the signals from the background processor to callback
        # methods/slots. This makes it thread safe
        self.sig_message.connect(self.appendMessage)
        self.sig_finished.connect(self.on_finished)

        # Create our background worker object, which also will do work in it's
        # own thread.
        self._worker = AUxWorker(self.on_worker_callback)

        # add the actions/commands for this app to the background processing thread.
        # These actions are passed jobs to execute.
        self._worker.add_action(AUxEsptoolDetectFlash(), AUxEsptoolUploadFirmware(), AUxEsptoolResetESP32())

    #--------------------------------------------------------------
    # callback function for the background worker.
    #
    # It is assumed that this method is called by the background thread
    # so signals and used to relay the call to the GUI running on the
    # main thread

    def on_worker_callback(self, *args): #msg_type, arg):

        # need a min of 2 args (id, arg)
        if len(args) < 2:
            self.writeMessage("Invalid parameters from the uploader.")
            return

        msg_type = args[0]
        if msg_type == AUxWorker.TYPE_MESSAGE:
            self.sig_message.emit(args[1])
        elif msg_type == AUxWorker.TYPE_FINISHED:
            # finished takes 3 args - status, job type, and job id
            if len(args) < 4:
                self.writeMessage("Invalid parameters from the uploader.");
                return;

            self.sig_finished.emit(args[1], args[2], args[3])

    @pyqtSlot(str)
    def appendMessage(self, msg: str) -> None:
        if msg.startswith("\r"):
            self.messageBox.moveCursor(QTextCursor.StartOfLine, QTextCursor.KeepAnchor)
            self.messageBox.cut()
            self.messageBox.insertPlainText(msg[1:])
        else:
            self.messageBox.insertPlainText(msg)
        self.messageBox.ensureCursorVisible()
        self.messageBox.repaint()

        if msg.find("Detected flash size: 4MB") >= 0:
            self.flashSize = 4
        elif msg.find("Detected flash size: 8MB") >= 0:
            self.flashSize = 8
        elif msg.find("Detected flash size: 16MB") >= 0:
            self.flashSize = 16
        elif msg.find("Detected flash size: ") >= 0:
            self.flashSize = 0

    @pyqtSlot(str)
    def writeMessage(self, msg: str) -> None:
        self.messageBox.moveCursor(QTextCursor.End)
        #self.messageBox.ensureCursorVisible()
        self.messageBox.appendPlainText(msg)
        self.messageBox.ensureCursorVisible()
        self.messageBox.repaint()
        #QApplication.processEvents()

    #--------------------------------------------------------------
    # on_finished()
    #
    #  Slot for sending the "on finished" signal from the background thread
    #
    #  Called when the backgroudn job is finished and includes a status value
    @pyqtSlot(int, str, int)
    def on_finished(self, status, action_type, job_id) -> None:

        # If the flash detection is finished, trigger the upload
        if action_type == AUxEsptoolDetectFlash.ACTION_ID:
            self.writeMessage("Flash detection complete. Uploading firmware...")
            self.do_upload()

        # If the upload is finished, trigger a reset
        elif action_type == AUxEsptoolUploadFirmware.ACTION_ID:
            self.writeMessage("Firmware upload complete. Resetting ESP32...")
            self.esptool_reset()

        # re-enable the UX
        else:
            self.writeMessage("Reset complete...")
            self.disable_interface(False)

    # --------------------------------------------------------------
    # on_port_combobox()
    #
    # Called when the combobox pop-up menu is about to be shown
    #
    # Use this event to dynamically update the displayed ports
    #
    @pyqtSlot()
    def on_port_combobox(self):
        self.update_com_ports()

    def _load_settings(self) -> None:
        """Load settings on startup."""
        port_name = self.settings.value(SETTING_PORT_NAME)
        if port_name is not None:
            index = self.port_combobox.findData(port_name)
            if index > -1:
                self.port_combobox.setCurrentIndex(index)

        lastFile = self.settings.value(SETTING_FILE_LOCATION)
        if lastFile is not None:
            self.fileLocation_lineedit.setText(lastFile)

        # lastFile = self.settings.value(SETTING_PARTITION_LOCATION)
        # if lastFile is not None:
        #     self.partitionFileLocation_lineedit.setText(lastFile)
        # else:
        #     self.partitionFileLocation_lineedit.setText(resource_path("RTK_Surveyor.ino.partitions.bin"))

        baud = self.settings.value(SETTING_BAUD_RATE)
        if baud is not None:
            index = self.baud_combobox.findData(baud)
            if index > -1:
                self.baud_combobox.setCurrentIndex(index)

    def _save_settings(self) -> None:
        """Save settings on shutdown."""
        self.settings.setValue(SETTING_PORT_NAME, self.port)
        self.settings.setValue(SETTING_FILE_LOCATION, self.theFileName)
        # self.settings.setValue(SETTING_PARTITION_LOCATION, self.thePartitionFileName)
        self.settings.setValue(SETTING_BAUD_RATE, self.baudRate)

    def _clean_settings(self) -> None:
        """Clean (remove) all existing settings."""
        self.settings.clear()

    def show_error_message(self, msg: str) -> None:
        """Show a Message Box with the error message."""
        QMessageBox.critical(self, QApplication.applicationName(), str(msg))

    def update_com_ports(self) -> None:
        """Update COM Port list in GUI."""
        previousPort = self.port # Record the previous port before we clear the combobox

        self.port_combobox.clear()

        index = 0
        indexOfPrevious = -1
        for desc, name, sys in gen_serial_ports():
            longname = desc + " (" + name + ")"
            self.port_combobox.addItem(longname, sys)
            if(sys == previousPort): # Previous port still exists so record it
                indexOfPrevious = index
            index = index + 1

        if indexOfPrevious > -1: # Restore the previous port if it still exists
            self.port_combobox.setCurrentIndex(indexOfPrevious)

    def update_baud_rates(self) -> None:
        """Update baud rate list in GUI."""
        # Highest speed first so code defaults to that
        # if settings.value(SETTING_BAUD_RATE) is None
        self.baud_combobox.clear()
        self.baud_combobox.addItem("921600", 921600)
        self.baud_combobox.addItem("460800", 460800)
        self.baud_combobox.addItem("115200", 115200)

    @property
    def port(self) -> str:
        """Return the current serial port."""
        return str(self.port_combobox.currentData())

    @property
    def baudRate(self) -> str:
        """Return the current baud rate."""
        return str(self.baud_combobox.currentData())

    @property
    def theFileName(self) -> str:
        """Return the file name."""
        return self.fileLocation_lineedit.text()

    # @property
    # def thePartitionFileName(self) -> str:
    #     """Return the partition file name."""
    #     return self.partitionFileLocation_lineedit.text()

    def closeEvent(self, event: QCloseEvent) -> None:
        """Handle Close event of the Widget."""
        try:
            self._save_settings()
        except:
            pass

        # shutdown the background worker/stop it so the app exits correctly
        self._worker.shutdown()

        event.accept()

    def on_browse_btn_pressed(self) -> None:
        """Open dialog to select bin file."""
        options = QFileDialog.Options()
        fileName, _ = QFileDialog.getOpenFileName(
            None,
            "Select Firmware to Upload",
            "",
            "Firmware Files (*.bin);;All Files (*)",
            options=options)
        if fileName:
            self.fileLocation_lineedit.setText(fileName)

    # def on_partition_browse_btn_pressed(self) -> None:
    #     """Open dialog to select partition bin file."""
    #     options = QFileDialog.Options()
    #     fileName, _ = QFileDialog.getOpenFileName(
    #         None,
    #         "Select Partition File",
    #         "",
    #         "Parition Files (*.bin);;All Files (*)",
    #         options=options)
    #     if fileName:
    #         self.partitionFileLocation_lineedit.setText(fileName)

    #--------------------------------------------------------------
    # disable_interface()
    #
    # Enable/Disable portions of the ux - often used when a job is running
    #
    def disable_interface(self, bDisable=False):

        self.upload_btn.setDisabled(bDisable)
        self.reset_btn.setDisabled(bDisable)

    def on_upload_btn_pressed(self) -> None:
        """Get ready to upload the firmware. First, detect the flash size"""
        portAvailable = False
        for desc, name, sys in gen_serial_ports():
            if (sys == self.port):
                portAvailable = True
        if (portAvailable == False):
            self.writeMessage("Port No Longer Available")
            return

        try:
            self._save_settings() # Save the settings in case the command fails
        except:
            pass

        self.flashSize = 0

        self.writeMessage("Detecting flash size\n\n")

        command = []
        command.extend(["--chip","esp32"])
        command.extend(["--port",self.port])
        command.extend(["--baud",self.baudRate])
        command.extend(["--before","default_reset","--after","no_reset"])
        command.extend(["flash_id"])

        # Create a job and add it to the job queue. The worker thread will pick this up and
        # process the job. Can set job values using dictionary syntax, or attribute assignments
        #
        # Note - the job is defined with the ID of the target action
        theJob = AxJob(AUxEsptoolDetectFlash.ACTION_ID, {"command":command})

        # Send the job to the worker to process
        self._worker.add_job(theJob)

        self.disable_interface(True)

    def do_upload(self) -> None:
        """Upload the firmware"""
        portAvailable = False
        for desc, name, sys in gen_serial_ports():
            if (sys == self.port):
                portAvailable = True
        if (portAvailable == False):
            self.writeMessage("Port No Longer Available")
            self.disable_interface(False)
            return

        fileExists = False
        try:
            f = open(self.theFileName)
            fileExists = True
        except IOError:
            fileExists = False
        finally:
            if (fileExists == False):
                self.writeMessage("File Not Found")
                self.disable_interface(False)
                return
            f.close()

        # fileExists = False
        # try:
        #     f = open(self.thePartitionFileName)
        #     fileExists = True
        # except IOError:
        #     fileExists = False
        # finally:
        #     if (fileExists == False):
        #         self.writeMessage("File Not Found")
        #         return
        #     f.close()

        try:
            self._save_settings() # Save the settings in case the command fails
        except:
            pass

        if self.flashSize == 0:
            self.writeMessage("Flash size not detected! Defaulting to 16MB\n")
            self.flashSize = 16
        else:
            self.writeMessage("Flash size is " + str(self.flashSize) + "MB\n")

        thePartitionFileName = ''
        firmwareSizeCorrect = True
        if self.flashSize == 16:
            thePartitionFileName = resource_path("RTK_Surveyor_Partitions_16MB.bin")
            # if self.theFileName.find("16MB") < 0:
            #     firmwareSizeCorrect = False
        else:
            thePartitionFileName = resource_path("RTK_Surveyor_Partitions_4MB.bin")
            # if self.theFileName.find("4MB") < 0:
            #     firmwareSizeCorrect = False

        if firmwareSizeCorrect == False:
            reply = QMessageBox.warning(self, "Firmware size mismatch", "Do you want to continue?", QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
            if reply == QMessageBox.No:
                self.disable_interface(False)
                return

        sleep(1.0);
        self.writeMessage("Uploading firmware\n")

        command = []
        #command.extend(["--trace"]) # Useful for debugging
        command.extend(["--chip","esp32"])
        command.extend(["--port",self.port])
        command.extend(["--baud",self.baudRate])
        command.extend(["--before","default_reset","--after","no_reset","write_flash","-z","--flash_mode","dio","--flash_freq","80m","--flash_size","detect"])
        command.extend(["0x1000",resource_path("RTK_Surveyor.ino.bootloader.bin")])
        command.extend(["0x8000",thePartitionFileName])
        command.extend(["0xe000",resource_path("boot_app0.bin")])
        command.extend(["0x10000",self.theFileName])

        #print("python esptool.py %s\n\n" % " ".join(command)) # Useful for debugging - cut and paste into a command prompt

        # Create a job and add it to the job queue. The worker thread will pick this up and
        # process the job. Can set job values using dictionary syntax, or attribute assignments
        #
        # Note - the job is defined with the ID of the target action
        theJob = AxJob(AUxEsptoolUploadFirmware.ACTION_ID, {"command":command})

        # Send the job to the worker to process
        self._worker.add_job(theJob)

        self.disable_interface(True) # Redundant... Interface is still disabled from flash detect

    def esptool_reset(self) -> None:
        """Tell the ESP32 to reset/restart"""
        portAvailable = False
        for desc, name, sys in gen_serial_ports():
            if (sys == self.port):
                portAvailable = True
        if (portAvailable == False):
            self.writeMessage("Port No Longer Available")
            self.disable_interface(False)
            return

        try:
            self._save_settings() # Save the settings in case the command fails
        except:
            pass

        sleep(1.0);
        self.writeMessage("Resetting ESP32\n")

        # ---- The esptool method -----

        command = []
        command.extend(["--chip","esp32"])
        command.extend(["--port",self.port])
        command.extend(["--before","default_reset","run"])

        # Create a job and add it to the job queue. The worker thread will pick this up and
        # process the job. Can set job values using dictionary syntax, or attribute assignments
        #
        # Note - the job is defined with the ID of the target action
        theJob = AxJob(AUxEsptoolResetESP32.ACTION_ID, {"command":command})

        # Send the job to the worker to process
        self._worker.add_job(theJob)

        self.disable_interface(True)

    def tera_term_reset(self) -> None:
        """Reset the ESP32 the TeraTerm way"""
        portAvailable = False
        for desc, name, sys in gen_serial_ports():
            if (sys == self.port):
                portAvailable = True
        if (portAvailable == False):
            self.writeMessage("Port No Longer Available")
            return

        try:
            self._save_settings() # Save the settings in case the command fails
        except:
            pass

        self.writeMessage("Resetting ESP32\n")

        # ---- The pySerial method -----

        self.disable_interface(True)

        sleep(0.1)

        try:
            ser = serial.Serial()
            ser.port = self.port
            ser.setDTR(False) # DTR High
            ser.setRTS(False) # RTS High
            with ser as s:
                s.setRTS(True) # RTS Low - before DTR
                s.setDTR(True) # DTR Low - after RTS
                sleep(1.0)
                self.writeMessage("Waiting for reset to complete")
                sleep(1.0)
                self.writeMessage("Waiting for reset to complete")
                sleep(1.0)
                self.writeMessage("Waiting for reset to complete")
                sleep(1.0)
                self.writeMessage("Waiting for reset to complete\n")
                sleep(1.0)
        except:
            self.writeMessage("Could not open serial port\n")
            self.disable_interface(False)
            return

        self.writeMessage("Reset complete...")
        self.disable_interface(False)

def startUploaderGUI():
    """Start the GUI"""
    from sys import exit as sysExit
    app = QApplication([])
    app.setOrganizationName('SparkFun Electronics')
    app.setApplicationName(_APP_NAME + ' - ' + _APP_VERSION)
    app.setWindowIcon(QIcon(resource_path("RTK.png")))
    app.setApplicationVersion(_APP_VERSION)
    w = MainWidget()
    w.show()
    sys.exit(app.exec_())

if __name__ == '__main__':
    startUploaderGUI()