import torch
import numpy as np
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import random_split, DataLoader
from sklearn import metrics
from sklearn import tree
from sklearn.tree.export import export_text

from feature_dataset import SafetyFeatureDataset

class SafetyFeatureModel(nn.Module):
    def __init__(self, input_dim, hidden_dim):
        super(SafetyFeatureModel, self).__init__()
        self.layer = nn.Linear(input_dim, hidden_dim)
        self.output = nn.Linear(hidden_dim, 2)

    def forward(self, features):
        output = self.layer(features)
        output = self.output(output)
        return output

def train(model, training, validation, loss_fn, optimizer, num_epochs):
    for epoch in range(num_epochs):
        training_loss, validation_loss = [], []
        training_labels, validation_labels = [], []
        training_predictions, validation_predictions = [], []
        model.train()
        print("Going over training data")
        for i, (feature, label) in enumerate(training):
            if torch.cuda.is_available():
                feature = feature.cuda()
                label = label.cuda()
            model.zero_grad()
            output = model(feature)
            loss = loss_fn(output, label)
            loss.backward()
            optimizer.step()
            training_labels.append(label.cpu().detach().numpy())
            training_predictions.append(np.argmax(output.cpu().detach().numpy(), -1))
            training_loss.append(loss.item())
            training_accuracy = metrics.accuracy_score(training_labels[-1], training_predictions[-1])
            all_same = "ALL SAME" if len(set(training_predictions[-1])) == 1 else ""
            print("Iteration {} loss: {}, accuracy: {} {}".format(i, loss.item(), training_accuracy, all_same))
        model.eval()
        print("Going over validation data")
        for i, (feature, label) in enumerate(validation):
            if torch.cuda.is_available():
                feature = feature.cuda()
                label = label.cuda()
            output = model(feature)
            loss = loss_fn(output, label)
            validation_loss.append(loss.item())
            validation_labels.append(label.cpu().detach().numpy())
            validation_predictions.append(np.argmax(output.cpu().detach().numpy(), -1))
        training_accuracy = metrics.accuracy_score(np.concatenate(training_labels, 0), np.concatenate(training_predictions, 0))
        validation_accuracy = metrics.accuracy_score(np.concatenate(validation_labels, 0), np.concatenate(validation_predictions, 0))
        confusion_matrix = str(metrics.confusion_matrix(np.concatenate(validation_labels, 0), np.concatenate(validation_predictions, 0)))
        print("Epoch {}, Training loss: {}, Training accuracy: {}, Validation loss: {}, Validation accuracy: {}, Validation confusion matrix: {}".format(epoch, np.mean(training_loss), np.mean(validation_loss), training_accuracy, validation_accuracy, confusion_matrix))

if __name__ == '__main__':
    dataset = SafetyFeatureDataset("/Users/arkadyark/Downloads/data/labels/part-00000-e9445087-aa0a-433b-a7f6-7f4c19d78ad6-c000.csv", "/Users/arkadyark/Downloads/data/features/*.csv")
    train_size = int(0.9*len(dataset))
    val_size = len(dataset) - train_size
    training, val = random_split(dataset, [train_size, val_size])
    train_loader = DataLoader(training, batch_size=train_size, shuffle=True)
    val_loader = DataLoader(val, batch_size=val_size, shuffle=False)
    print("Dataset loaded!")
    trees = []
    for i, (feature, label) in enumerate(train_loader):
        for max_depth in (2, 3, 5, 7, 10):
            print("Max depth: {}".format(max_depth))
            clf = tree.DecisionTreeClassifier(max_depth=max_depth)
            clf = clf.fit(np.array(feature), np.array(label))
            training_predictions = clf.predict(np.array(feature))
            training_accuracy = metrics.accuracy_score(np.array(label), training_predictions)
            for j, (val_feature, val_label) in enumerate(val_loader):
                validation_predictions = clf.predict(np.array(val_feature))
                validation_accuracy = metrics.accuracy_score(np.array(val_label), validation_predictions)
                confusion_matrix = str(metrics.confusion_matrix(np.array(val_label), validation_predictions))
            print(training_accuracy)
            print(validation_accuracy)
            print(confusion_matrix)
            import pdb
            pdb.set_trace()
            r = export_text(clf, feature_names=('max_speed', 'max_dec', 'max_acc', 'max_acc_total'))
            print(r)
    # model = SafetyFeatureModel(4, 2)
    # weight = torch.tensor([1, 3], dtype=torch.float)
    # if torch.cuda.is_available():
    #     model.cuda()
    #     weight = weight.cuda()
    # loss_function = nn.CrossEntropyLoss(weight)
    # optimizer = optim.SGD(model.parameters(), lr=0.00001)
    # train(model, train_loader, val_loader, loss_function, optimizer, 500)


from decimal import Decimal
from django.db import models
from .abstract_model import TimestampedModel

class Category(models.Model):
    name = models.CharField(
        max_length=100, verbose_name='Category name'
    )
    slug = models.SlugField(
        max_length=70, unique=True
    )

    class Meta:
        verbose_name_plural = 'Categories'

    def __str__(self):
        return self.name

class Size(models.Model):
    name = models.CharField(max_length=5)

    def __str__(self):
        return self.name

class Brand(models.Model):
    name = models.CharField(max_length=50)
    logo = models.ImageField(upload_to='images/brand', blank=True, null=True)

class Image(models.Model):
    product = models.ForeignKey(
        'shop.Product',
        on_delete=models.CASCADE,
        related_name='images',
    )
    image = models.ImageField(
        upload_to='images/product', max_length=300
    )
    url = models.URLField(
        max_length=512, verbose_name='Image URL'
    )
    size = models.PositiveIntegerField(
        blank=True, null=True
    )

    def __str__(self):
        return self.image.url

class Product(TimestampedModel):
    source_url = models.URLField(
        verbose_name='Source URL', max_length=500
    )
    title = models.CharField(
        max_length=200
    )
    slug = models.SlugField(
        max_length=150, unique=True
    )
    description = models.TextField(
        blank=True, default=''
    )
    characteristics = models.JSONField(
        default=dict, blank=True
    )
    price = models.DecimalField(
        max_digits=10, decimal_places=2,
        blank=True, null=True
    )
    old_price = models.DecimalField(
        max_digits=10, decimal_places=2,
        blank=True, null=True, verbose_name='Old Price'
    )
    discount = models.DecimalField(
        max_digits=10, decimal_places=2,
        blank=True, null=True
    )
    article = models.CharField(
        max_length=30
    )
    categories = models.ManyToManyField(
        Category,
        related_name='products'
    )
    sizes = models.ManyToManyField(
        Size,
        related_name='products'
    )
    brand = models.ForeignKey(
        Brand,
        related_name='products',
        on_delete=models.CASCADE
    )

    def save(self, *args, **kwargs) -> None:
        if self.old_price:
            self.discount = Decimal(
                round((float(self.price) / float(self.old_price))
                      * 100 - 100, 2)
            )
        super().save(*args, **kwargs)

    def __str__(self):
        return self.title


# -*- coding: utf-8 -*-
import config

__author__ = 'magenn'

import requests
import json
import base64
from insurance_crm.dal import dal_django
from insurance_site import settings

def send_sign_document_request_to_client(client_id):
    client = dal_django.get_client_from_id(client_id)
    agent = client.agent
    agent_name = agent.first_name + " " + agent.last_name

    headers = {'Content-type': 'application/json',
               'Accept': 'application/json',
               "Authorization": "Token  {0}".format(settings.CELLOSIGN_AUTHORIZATION_TOKEN)
               }

    inputvalues = {
        "client_name":"%s %s" % (client.first_name, client.last_name),
        "client_id":client.id_number,
        "agent_name":agent_name,
        "agent_name_2":agent_name,
        "agent_license_number":agent.license_number,
        "agent_phone":agent.phone_number,
    }
    data ={
        "input_file": inputvalues,
        "destination": client.phone_number,  # post SMS w/link to client
        "template": "bd343311cd4645078c9fd94b5b1960b3",
        "rep_email": config.MAIL_ADDRESS,
        "client_redirect": "http://www.google.com",
        "invite_message": config.CLIENT_INVITE_MESSAGE,
        "client_id": client.id
    }
    json_file = json.dumps(data)

    request = requests.post(config.CELLOSIGN_REQUEST_ADDRESS,
                            headers=headers, data=json_file)

    #return request
    return "NIR"

'''
data={
    "client_copy": "omer@cellosign.com",
    "rep_email": ["omer@cellosign.com","iris@cellosign.com"],
    "template":"283c6009a4aa4b1e89ca3ac6b880f25b"
}
input_file = {
    "customer_name" : "קובי שגב",
    "customer_id": "028732436"
}
data['input_file'] = input_file
data = json.dumps(data)

request = requests.post('http://test.cellosign.com/ayalon/rep/create_session',headers=headers,data=data)

'''


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
import re

from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
analyser = SentimentIntensityAnalyzer()

vaderData = pd.read_csv("VADER_Pruned_Final.csv")
vaderData.drop('Unnamed: 0', axis=1, inplace=True)
vaderData.drop('Unnamed: 0.1', axis=1, inplace=True)

scores = []
# Declare variables for scores
compound_list = []
positive_list = []
negative_list = []
neutral_list = []
for i in range(vaderData['text'].shape[0]):
#print(analyser.polarity_scores(sentiments_pd['text'][i]))
    compound = analyser.polarity_scores(vaderData['text'][i])["compound"]
    pos = analyser.polarity_scores(vaderData['text'][i])["pos"]
    neu = analyser.polarity_scores(vaderData['text'][i])["neu"]
    neg = analyser.polarity_scores(vaderData['text'][i])["neg"]

    scores.append({"Compound": compound,
                       "Positive": pos,
                       "Negative": neg,
                       "Neutral": neu
                  })

sentiments_score = pd.DataFrame.from_dict(scores)
vaderData = vaderData.join(sentiments_score)
vaderData.to_csv("VADER_Sent_Scores.csv", index=True, encoding='utf8')


from utils import DealData

class SearchElm:

    def __init__(self, ganZhiDict=None):
        if ganZhiDict:
            self.ganZhiDict = ganZhiDict.copy()
            self.values = list(ganZhiDict.values())
            for i in range(len(self.values)):
                self.values[i] = DealData.getZhi(self.values[i])

        self.pat = ""
        self.type = ""
        self.year = 0
        self.month = []
        self.dateStr = []
        self.combine = {}

    def __str__(self):
        s = "查找类型为%s，匹配的组是%s，是%d年，" % (self.type, self.pat, self.year)
        for key in self.combine:
            s = s + "其中" + key + "为" + self.combine[key] + "，"
        return s

    def setPattern(self, pat):
        self.pat = ""
        for ch in pat:
            self.pat += ch

    def markLiuYue(self, liuYue):
        for yue in liuYue:
            month = yue.getIndex() + 1
            yueZhi = DealData.getZhi(yue.getGanZhi())
            if self.type == "三合" or self.type == "反拱":
                mid = self.pat[1]
                if yueZhi == mid:
                    self.month.append(month)
            elif self.type == "对冲" or self.type == "复吟":
                if yueZhi == DealData.getZhi(self.combine["流年"]):
                    self.month.append(month)
        self.setDateStr()

    def setDateStr(self):
        for m in self.month:
            ds = str(self.year) + '-' + str(m)
            self.dateStr.append(ds)

def searchSpecType(searchType, *args):
    if searchType == "复吟":
        return searchFuYin(*args)
    else:
        return searchPattern(searchType, *args)

def searchPattern(searchType, ganZhiDict, year, patList):
    targetPat = None
    for pat in patList:
        if DealData.getZhi(ganZhiDict['流年']) in pat:
            targetPat = pat.copy()
            break
    if targetPat is None:
        return None
    searchElm = SearchElm(ganZhiDict)
    searchElm.setPattern(targetPat)
    targetPat.remove(DealData.getZhi(ganZhiDict['流年']))
    if set(targetPat).issubset(set(searchElm.values)):
        searchElm.combine["流年"] = DealData.getZhi(ganZhiDict["流年"])
        searchElm.year = year
        for key in ganZhiDict:
            if DealData.getZhi(ganZhiDict[key]) in targetPat:
                searchElm.combine[key] = DealData.getZhi(ganZhiDict[key])
        searchElm.type = searchType
        return searchElm
    else:
        return None

#   查询复吟
def searchFuYin(ganZhiDict, year, patList):
    elm = SearchElm()
    elm.year = year
    elm.type = "复吟"
    liuNianGanZhi = ganZhiDict["流年"]
    for key in ganZhiDict:
        if key != "流年" and ganZhiDict[key] == liuNianGanZhi:
            elm.combine[key] = ganZhiDict[key]
    if len(elm.combine) == 0:
        return None
    else:
        elm.combine["流年"] = liuNianGanZhi
        return elm


import os
import subprocess
from flask import Flask, request, jsonify, render_template

app = Flask(__name__)

# Get the absolute path of the directory containing the app.py file
current_directory = os.path.dirname(os.path.abspath(__file__))

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/process', methods=['POST'])
def process():
    user_input = request.form['user_input']

    if user_input == "Run Test Suite":
        options = ["Functional Testing", "Regression Testing", "Security Testing"]
        button_html = ""
        for option in options:
            button_html += f'<button type="button" class="option-button" data-option="{option}">{option}</button>'
        bot_response = f"Please select an option:<br/>{button_html}"
    elif user_input == "Regression Testing":
        try:
            # Run the Maven command
            subprocess.run(['mvn', 'clean', 'test'], cwd=current_directory, shell=True)

            bot_response = "Regression testing started."
        except FileNotFoundError:
            bot_response = "Maven or testng.xml file not found. Make sure you have the required files."
    else:
        bot_response = "Invalid command. Please try again."

    return jsonify({'bot_response': bot_response})

if __name__ == '__main__':
    app.run()


import qdrant_client
import streamlit as st
import tiktoken
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.schema import AIMessage, HumanMessage, SystemMessage
from langchain.vectorstores import Qdrant

def embedding_search(query, k, api_token):
    embeddings = OpenAIEmbeddings(
        openai_api_key=api_token,
    )
    client = qdrant_client.QdrantClient(
        url=st.secrets["qdrant_url"], api_key=st.secrets["qdrant_api_key"]
    )
    db = Qdrant(client=client, embeddings=embeddings, collection_name="documentations")
    return db.similarity_search(query, k=k)

def format_context(docs):
    context = "\n\n".join(
        [
            f'From file {d.metadata["document_id"]}:\n' + str(d.page_content)
            for d in docs
        ]
    )
    return context

def system_message_prompt(query, api_token):
    docs = embedding_search(query, k=5, api_token=api_token)
    context = format_context(docs)

    prompt = """Given the following context and code, answer the following question. Do not use outside context, and do not assume the user can see the provided context. Try to be as detailed as possible and reference the components that you are looking at. Keep in mind that these are only code snippets, and more snippets may be added during the conversation.
    Reference the exact code snippets that you have been provided with. If you are going to write code, make sure to specify the language of the code. For example, if you were writing Python, you would write the following:

    ```python
    <python code goes here>
    ```

    Now, here is the relevant context:

    Context: {context}
    """
    docs = [doc.page_content for doc in docs]
    return prompt.format(context=context), docs

def query_message_prompt(query: str, api_token: str):
    docs = embedding_search(query, k=2, api_token=api_token)
    context = format_context(docs)
    docs = [doc.page_content for doc in docs]
    return (
        f"""Relevant context: {context}

    {query}""",
        docs,
    )

def get_encoding_length(text: str) -> int:
    encoding_name = "cl100k_base"
    encoding = tiktoken.get_encoding(encoding_name)
    ecoding_length = len(encoding.encode(text))
    return ecoding_length


import json, urllib3
from datetime import datetime, timedelta

from django.http import HttpResponse, JsonResponse
from django.shortcuts import render
from django.utils.cache import patch_response_headers
from core.oauth.utils import login_customrequired
from core.views import initRequest
from core.utils import is_json_request
from core.libs.DateEncoder import DateEncoder
from core.libs.cache import setCacheEntry, getCacheEntry
from core.libs.datetimestrings import parse_datetime

from core.oi.utils import round_time
import matplotlib

from core.utils import removeParam

from django.template.defaulttags import register

CACHE_TIMEOUT = 5
OI_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S"

@register.filter(takes_context=True)
def to_float(value):
    return float(value)

def formatError(json):
    outstr = ""
    for errorcat, errormessages in json.items():
        outstr += '
Error cathegory: '+errorcat+'
'
        for message, num in errormessages.items():
            outstr += 'Error message:' + message + ': ' +str(num)+ '
'
    return outstr

@login_customrequired
def jbhome(request):

    if is_json_request(request):
        response = JsonResponse({"error": "decommissioned"}, status=410)
    else:
        valid, response = initRequest(request)
        data = {
            'request': request,
            'requestParams': request.session['requestParams'],
            'viewParams': request.session['viewParams'],
            'title': 'Jobs Buster',
            'subtitle': 'Jobs Buster',
            'built': datetime.now().strftime("%H:%M:%S"),
            'message': 'It was decided to decommission the JobsBuster view.',
        }
        response = render(request, '_decommissioned.html', data, content_type='text/html')
    return response

    valid, response = initRequest(request)
    if not valid:
        return response
    # Here we try to get cached data
    data = getCacheEntry(request, "jobProblem")
    data = None
    if data is not None and len(data) > 10:
        data = json.loads(data)
        if not ('message' in data and 'warning' in data['message'] and len(data['message']['warning']) > 1):
            data['request'] = request
            response = render(request, 'jobsbuster.html', data, content_type='text/html')
            patch_response_headers(response, cache_timeout=request.session['max_age_minutes'] * 60)
            return response

    message = {}

    # process params
    metric = None
    computetype = None
    if 'metric' in request.session['requestParams'] and request.session['requestParams']['metric']:
        metric = request.session['requestParams']['metric']

    if 'hours' in request.session['requestParams'] and request.session['requestParams']['hours']:
        hours = int(request.session['requestParams']['hours'])
        endtime = datetime.now()
        starttime = datetime.now() - timedelta(hours=hours)
    elif 'endtime_from' in request.session['requestParams'] and 'endtime_to' in request.session['requestParams']:
        endtime = parse_datetime(request.session['requestParams']['endtime_to'])
        starttime = parse_datetime(request.session['requestParams']['endtime_from'])
    else:
        default_hours = 12
        endtime = datetime.now()
        starttime = datetime.now() - timedelta(hours=default_hours)

    if 'jobtype' in request.session['requestParams'] and request.session['requestParams']['jobtype']:
        jobtype = request.session['requestParams']['jobtype']
    else:
        jobtype = 'prod'

    if 'computetype' in request.session['requestParams'] and request.session['requestParams']['computetype']:
        computetype = request.session['requestParams']['computetype']

    # getting data from jobbuster API
    base_url = 'http://aipanda030.cern.ch:8010/jobsbuster/api/?'
    url = base_url + 'timewindow={}|{}'.format(
        round_time(starttime, timedelta(minutes=1)).strftime(OI_DATETIME_FORMAT),
        round_time(endtime, timedelta(minutes=1)).strftime(OI_DATETIME_FORMAT))
    if metric:
        url += '&metric=' + metric
    if computetype:
        url += '&computetype=' + computetype

    http = urllib3.PoolManager()
    try:
        resp = http.request('GET', url, timeout=500)
    except:
        resp = None
        message['warning'] = "Can not connect to jobbuster API, please try later"

    resp_data = None
    if resp and len(resp.data) > 10:
        try:
            resp_data = json.loads(resp.data)
        except:
            message['warning'] = "No data was received"
    else:
        message['warning'] = "No data was received"
    http.clear()

    # processing data
    plots = {}
    spots = []
    names = []
    if resp_data:

        for i, problem in enumerate(resp_data['mesuresW']):
            resp_data['mesuresW'][i] = list(map(lambda x: x/3.154e+7 if not type(x) is str else x, problem))

        names = [i[0] for i in resp_data['mesuresW']]

    resp_dict = None
    timeticks = []
    errormessages = {}
    if resp_data and 'ticks' in resp_data:
        timeticks = [parse_datetime(tick) for tick in resp_data['ticks']]
        timeticks = ['x'] + [tick.strftime("%Y-%m-%d %H:%M:%S") for tick in timeticks]
        resp_data['mesuresW'].insert(0,timeticks)
        resp_data['mesuresNF'].insert(0,timeticks)

        colors = {}
        for name, color in zip(names, resp_data['colorsW']):
            colors[name] = matplotlib.colors.to_hex(color)

        for issue in resp_data['issues']:
            card = {}
            card['color'] = colors[issue['name']]
            card['impactloss'] = str(round(issue['walltime_loss'] / 3.154e+7, 2))
            card['impactfails'] = issue['nFailed_jobs']
            card['name'] = issue['name']
            card['params'] = {}
            id = str(len(errormessages.keys()))
            card['errormessagesid'] = id
            errormessages[id] = formatError(json.loads(issue['err_messages']))
            urlstr = "https://bigpanda.cern.ch/jobs/?endtimerange=" + str(issue['observation_started']).replace(" ", "T") + "|" + str(issue['observation_finished']).replace(" ", "T")

            for key,value in issue['features'].items():
                card['params'][key] = value

                # if isinstance(value, tuple):
                #     propname = value[i]
                # else:
                propname = value
                urlstr += "&" + str(key).lower() + "=" + str(propname)

            urlstr += "&mode=nodrop&prodsourcelabel=managed"
            card['url'] = urlstr
            spots.append(card)

        measures = resp_data['mesuresW'] if not metric or metric=='loss' else resp_data['mesuresNF']
        resp_dict = {
            'mesures': measures,
            'ticks': resp_data['ticks'],
            'issnames': names,
            'doGroup': False if len(names) < 2 else True,
            'colors': colors,
            'spots':spots,
        }

    url_no_computetype = removeParam(request.get_full_path(), 'computetype')

    request.session['timerange'] = [starttime.strftime(OI_DATETIME_FORMAT), endtime.strftime(OI_DATETIME_FORMAT)]
    data = {
        'request': request,
        'requestParams': request.session['requestParams'],
        'viewParams': request.session['viewParams'],
        'timerange': request.session['timerange'],
        'message': message,
        'mesures': [],
        'metric': metric,
        'urlBase': url_no_computetype + ('&' if url_no_computetype.find('?') > -1 else '?'),
        'errormessages': json.dumps(errormessages)
        #'plots': plots,
        #'spots': spots,
    }
    if resp_dict:
        data.update(resp_dict)

    if (not (('HTTP_ACCEPT' in request.META) and (request.META.get('HTTP_ACCEPT') in ('application/json',))) and (
                'json' not in request.session['requestParams'])):
        response = render(request, 'jobsbuster.html', data, content_type='text/html')
    else:
        response = HttpResponse(json.dumps(data, cls=DateEncoder), content_type='application/json')
    if resp and len(resp.data) > 10:
        setCacheEntry(request, "jobProblem", json.dumps(data, cls=DateEncoder), 60 * CACHE_TIMEOUT)
    patch_response_headers(response, cache_timeout=request.session['max_age_minutes'] * 60)
    return response


import torch
import numpy as np

device = 'cuda' if torch.cuda.is_available() else 'cpu'

def predict(dataset, model, text, next_words=100):
    model.eval()

    words = text.lower().split(' ')
    state_h, state_c = model.init_state(len(words))

    for i in range(0, next_words):
        try:
            x = torch.tensor([[dataset.word_to_index[w] for w in words[i:]]]).to(device)
        except:
            words_index = []
            for w in words[i:]:
                if w in dataset.word_to_index:
                    words_index.append(dataset.word_to_index[w])
                else:
                    words_index.append(0)
            x = torch.tensor([words_index]).to(device)
        y_pred, (state_h, state_c) = model(x, (state_h, state_c))

        last_word_logits = y_pred[0][-1]
        p = torch.nn.functional.softmax(last_word_logits, dim=0).detach().cpu().numpy()
        word_index = np.random.choice(len(last_word_logits), p=p)
        words.append(dataset.index_to_word[word_index])

    words[0] = words[0][0].upper() + words[0][1:]
    return " ".join(words)


from django.test import TestCase,SimpleTestCase
from .models import Post,Review,Categories
from django.urls import resolve, reverse
from django.contrib.auth.models import User
from . import views
from datetime import datetime
# Create your tests here.

class TestPostUrls(SimpleTestCase):

    def test_homepage_url_is_resolved(self):
        url = reverse("posts:homepage")
        self.assertEqual(resolve(url).func,views.HomeList)

    def test_createPost_url_is_resolved(self):
        url = reverse("posts:post-create")
        self.assertEqual(resolve(url).func.view_class,views.CreatePostView)

    def test_search_url_is_resolved(self):
        url = reverse("posts:search")
        self.assertEqual(resolve(url).func,views.SearchFunction)

    def test_user_profile_url_is_resolved(self):
        url = reverse("posts:profile",args=[1])
        self.assertEqual(resolve(url).func,views.userProfile)

    def test_editPost_url_is_resolved(self):
        url = reverse("posts:post-update",args=[1])
        self.assertEqual(resolve(url).func.view_class,views.UpdatePostView)

    def test_deletePost_url_is_resolved(self):
        url = reverse("posts:post-delete",args=[1])
        self.assertEqual(resolve(url).func.view_class,views.DeletePostView)

    def test_category_url_is_resolved(self):
        url = reverse("posts:category",args=[1])
        self.assertEqual(resolve(url).func,views.show_categories)

    def test_post_detail_url_is_resolved(self):
        url = reverse("posts:post-detail",args=[1])
        self.assertEqual(resolve(url).func,views.PostDetail)

class TestPost(TestCase):

    def setUp(self):
    #creating instances of objects in User
       self.user1 = User.objects.create(username="PmPranaya",email="Pmpranaya@gmail.com")
       self.category = Categories.objects.create(name="Arts")
       self.post = Post.objects.create(
            user=self.user1,
            title="Test Title",
            description="This is Test Description",
            date=datetime.now(),
            category=self.category,
            price = 1400)
       self.review = Review.objects.create(
           user=self.user1,
           date=datetime.now(),
           post=self.post,
           review="This is Test Review",
           rating=5
       )

    def test_post_data(self):
        post1 = Post.objects.get(title="Test Title")
        self.assertEqual(post1.description,"This is Test Description")
        self.assertEqual(post1.price,1400)
        self.assertEqual(post1.user.username,"PmPranaya")
        self.assertEqual(post1.category.name,"Arts")

    def test_review_data(self):
        review1 = Review.objects.get(review="This is Test Review")
        self.assertEqual(review1.user.username,"PmPranaya")
        self.assertEqual(review1.post.title,"Test Title")


from scipy.io.wavfile import read as read_wav
import os
import wave
import contextlib

Sound_paht_dir = './datas'
Sound = os.listdir(Sound_paht_dir)
##Sound.remove('noise')
Sound.remove('test')

for k in Sound:
    print(k)
    Sound = k
    path_dir = './datas/'+k+'/'
    file_list = os.listdir(path_dir)
    print(len(file_list))
    file_string = '\n'.join(file_list)
    ##print(file_string)

    for i in range(0, len(file_list)):
        # check frame_rate
    ##    print( path_dir + str(i) )

        os.chdir('./') # path
        sampling_rate, data = read_wav(path_dir + file_list[i])
        if sampling_rate < 16000:
            print("File name : " + file_list[i] ,"Frame_Rate : " + str(sampling_rate), "Data : " + str(data))

        with contextlib.closing(wave.open(path_dir + file_list[i], 'r')) as f:
            frames = f.getnframes()
            rate = f.getframerate()
            length = frames / float(rate)

            if length > 1:
                print("over one sec : ",file_list[i] , length)

    # check channels

##    f1 = wave.open(path_dir + str(i), 'r')
##    print("Channels : " + str(f1.getnchannels()))


import re
import itertools
import datetime
import numpy as np

from dateparser.search import search_dates
from sklearn.feature_extraction.text import TfidfVectorizer

from autociter.data.storage import Table, Record
from autociter.data.queries import contains
from autociter.utils.debugging import debug
from autociter.utils.decorators import timeit

def standardize(data, datatype):
    """Define methods that standardize fields into a singular format that is logical
    and searchable
    """

    def std_markdown(markdown):
        """Remove the substring 'Image\n\n' from markdown."""
        for substring in {"Image\n\n"}:
            markdown = markdown.replace(substring, "")
        return markdown

    def std_html(html):
        """Remove script and style elements from HTML."""

        def remove_elements(html, tag_name):
            """Remove elements of a given type from HTML."""
            while "<" + tag_name in html:
                open_tag_start = html.find("<" + tag_name)
                open_tag_end = html.find(">", open_tag_start)
                close_tag_start = html.find("</" + tag_name, open_tag_end)
                close_tag_end = html.find(">", close_tag_start)
                html = html[:open_tag_start] + html[close_tag_end + 1:]
            return html

        html = remove_elements(html, "script")
        html = remove_elements(html, "style")
        return html

    def std_table(table):
        """Standardizes a default Table object so that it can be processed by
        pipeline. (i.e Author field is created from first, last, first1, last1, etc.)
        """
        std_fields = [
            "title", "author", "publisher", "date", "url", "archive-url"
        ]
        ret = Table(fields=std_fields)
        for rec in table.records:
            author_fields = [("first", "last"), ("first1", "last1"),
                             ("first2", "last2")]
            authors = []
            for i in author_fields:
                author = (rec[i[0]] + " " + rec[i[1]]).strip()
                if author != "":
                    authors.append(author)
            values = []
            for attr in std_fields:
                if contains(attr)(rec):
                    values.append(rec[attr])
                elif attr == 'author':
                    values.append(list(set(authors)))
                else:
                    values.append("")
            ret.add(Record(std_fields, values))
        return ret

    def std_text(text):
        """Standardizes a string representing article text
        Resources: https://stackoverflow.com/questions/19785458/capitalization-of-sentences-in-python
        """

        def clean_text(text):
            """Method that cleans a string to only include relevant characters and words"""
            text = text.replace('\'', '')
            text = text.replace('\"', '')
            matched_words = re.findall(r'\S+|\n', re.sub(
                "[^\w#\n]", " ", text))
            words_and_pound_newline = [
                i for i, j in itertools.zip_longest(
                    matched_words, matched_words[1:]) if i != j
            ]
            words_and_pound_newline = [('#' if '#' in x else x)
                                       for x in words_and_pound_newline]
            words_and_pound_newline = [(x.replace('_', '') if '_' in x else x)
                                       for x in words_and_pound_newline]
            ret = ''
            for i in range(len(words_and_pound_newline)):
                word = words_and_pound_newline[i]
                if i == 0 or word == '\n' or (
                        i > 0 and words_and_pound_newline[i - 1] == '\n'):
                    ret += word
                else:
                    ret += (" " + word)
            return ret

        def uppercase(matchobj):
            return matchobj.group(0).upper()

        def capitalize(s):
            return re.sub('^([a-z])|[\.|\?|\!]\s*([a-z])|\s+([a-z])(?=\.)',
                          uppercase, s)

        text = clean_text(text)
        return capitalize(text)

    def std_author(authors):
        """Method for standardizing a field if it is a list of authors"""
        return [
            ' '.join([
                name.capitalize() for name in author.replace('.', '').replace(
                    '-', ' ').split(' ')
            ]) for author in authors
        ]

    def std_date(date):
        """Method for standardizing a field if it is a date"""
        base = datetime.datetime(1000, 1, 1, 0, 0)
        matches = search_dates(
            date, settings={
                'STRICT_PARSING': True,
                'RELATIVE_BASE': base
            })
        if matches and matches[0][1] - base > datetime.timedelta(days=2 * 365):
            return matches[0][1].strftime('%m/%d/%y')
        return date.lower().replace(',', ' ').replace('-', ' ')

    def std_title(title):
        """Method for standardizing a field if it is a title"""
        return title.title()

    def std_url(url):
        """Method to standardize a url"""
        return url

    try:
        if datatype.lower() in [
                "markdown", "html", "table", "text", "author", "date", "title",
                "url"
        ]:
            return eval("std_{0}(data)".format(datatype.lower()))
        else:
            debug(
                "*** Error in standardization.standardize: {0} not standardizable"
                .format(datatype))
            return data
    except Exception as e:
        debug("*** Error in standardization.standardize: {0}".format(e))
        return ""

def find(field, text, datatype, start=0, end=None, threshold_value=0.6):
    """Attempts to locate a field as a substring of text based on its datatype.
    Assumes text is cleaned by pipeline's clean_text"""

    def find_generic(field, text, start=0, end=None):
        """Basic method for finding any generic field within a text"""
        text = text[start:end]
        index = text.find(field)
        if index != -1:
            return (start + index, start + index + len(field))
        return (-1, -1)

    def find_fuzzy_fast(field, text, start=0, end=None):
        """Fast fuzzy string matching
        References:
        https://bergvca.github.io/2017/10/14/super-fast-string-matching.html
        http://blog.christianperone.com/2013/09/machine-learning-cosine-similarity-for-vector-space-models-part-iii/
        https://stackoverflow.com/questions/52048562/efficient-way-to-compute-cosine-similarity-between-1d-array-and-all-rows-in-a-2d
        https://stackoverflow.com/questions/36013295/find-best-substring-match
        """

        def generate_chunks(string, length):
            return [
                string[0 + i:length + i]
                for i in range(0, len(string), length) if length + i < len(string)
            ]

        def generate_ngrams(string, n=3):
            """Helper method for generating ngrams"""
            ngrams = zip(*[string[i:] for i in range(n)])
            return [''.join(ngram) for ngram in ngrams]

        try:
            text = text[start:end]
            chunks = [field] + generate_chunks(text, len(field))
            vectorizer = TfidfVectorizer(min_df=1, analyzer=generate_ngrams)
            tf_idf_matrix = vectorizer.fit_transform(chunks)
            vec, arr = tf_idf_matrix[0].toarray().flatten(
            ), tf_idf_matrix[1:].toarray()
            a, b = np.linalg.norm(arr, axis=1), np.linalg.norm(vec)
            out = (arr @ vec) / (a * b)
            index = np.argmax(out) * len(field)

            ngrams = [field]
            for i in range(
                    max(0, index - len(field)), min(index + len(field),
                                                    len(text))):
                if i+len(field) >= len(text):
                    break
                ngrams.append(text[i:i + len(field)])

            vectorizer = TfidfVectorizer(min_df=1, analyzer=generate_ngrams)
            tf_idf_matrix = vectorizer.fit_transform(ngrams)
            vec, arr = tf_idf_matrix[0].toarray().flatten(
            ), tf_idf_matrix[1:].toarray()
            a, b = np.linalg.norm(arr, axis=1), np.linalg.norm(vec)
            out = (arr @ vec) / (a * b)
            ans = np.argmax(out) + max(0, index - len(field))
            if np.max(out) < threshold_value:
                return (-1, -1)
            ret = (ans, ans + len(field))
            return (start + ret[0].item(), start + ret[1].item())
        except Exception as e:
            return (-1, -1)

    def find_singular_author(author, text, start=0, end=None):
        """Helper method for finding a singular author"""
        return find_generic(author.lower(), text.lower(), start, end)

    def find_author(authors, text, start=0, end=None):
        """Method for finding an "author" field (a list of authors) in a text"""
        authors = standardize(authors, 'author')
        ret = []
        for author in authors:
            pos = find_singular_author(author, text, start, end)
            ret.append(pos)
        return ret

    def find_date(date, text, start=0, end=None):
        """Method for finding a date field in a text"""
        text = text[start:end]
        date = datetime.datetime.strptime(date, '%m/%d/%y')
        # Pass an impossible relative base so that relative words like "today" won't be detected
        matches = search_dates(
            text,
            settings={
                'STRICT_PARSING': True,
                'RELATIVE_BASE': datetime.datetime(1000, 1, 1, 0, 0)
            })
        if matches:
            for original_text, match in matches:
                if date.date() == match.date():
                    return find_generic(original_text.lower(), text.lower(), start, end)
        return (-1, -1)

    def find_title(title, text, start=0, end=None):
        """Method for finding a title field in a text
        ?? 'http://www.digitalspy.com/gaming/retro-corner/news/a381156/retro-corner-wolfenstein-3d/'
        """
        # return find_generic(title.lower(), text.lower())
        return find_fuzzy_fast(title.lower(), text.lower(), start, end)

    try:
        if datatype.lower() in ["author", "date", "title"]:
            return eval("find_{0}(field, text)".format(datatype.lower()))
        else:
            debug(
                "*** Warning in standardization.find: {0} not a findable field"
                .format(datatype))
            return find_generic(field, text, start, end)
    except Exception as e:
        debug("*** Error in standardization.find: {0}".format(e))
        return (-1, -1)

def clean_to_ascii(foreign_char):
    """Converts a non-ASCII character into it's ASCII equivalent

        >>> clean_to_ascii('ç')
        'c'
    """
    special_chars = {
        'a': ['à', 'á', 'â', 'ä', 'æ', 'ã', 'å', 'ā'],
        'c': ['ç', 'ć', 'č'],
        'e': ['è', 'é', 'ê', 'ë', 'ē', 'ė', 'ę'],
        'i': ['î', 'ï', 'í', 'ī', 'į', 'ì'],
        'l': ['ł'],
        'n': ['ñ', 'ń'],
        'o': ['ô', 'ö', 'ò', 'ó', 'œ', 'ø', 'ō', 'õ'],
        's': ['ß', 'ś', 'š'],
        'u': ['û', 'ü', 'ù', 'ú', 'ū'],
        'y': ['ÿ'],
        'z': ['ž', 'ź', 'ż'],
        'A': ['À', 'Á', 'Â', 'Ä', 'Æ', 'Ã', 'Å', 'Ā'],
        'C': ['Ç', 'Ć', 'Č'],
        'E': ['È', 'É', 'Ê', 'Ë', 'Ē', 'Ė', 'Ę'],
        'I': ['Î', 'Ï', 'Í', 'Ī', 'Į', 'Ì'],
        'L': ['Ł'],
        'N': ['Ñ', 'Ń'],
        'O': ['Ô', 'Ö', 'Ò', 'Ó', 'Œ', 'Ø', 'Ō', 'Õ'],
        'S': ['Ś', 'Š'],
        'U': ['Û', 'Ü', 'Ù', 'Ú', 'Ū'],
        'Y': ['Ÿ'],
        'Z': ['Ž', 'Ź', 'Ż']
    }
    if foreign_char in sum(special_chars.values(), []):
        for k in special_chars:
            if foreign_char in special_chars[k]:
                return k
    debug("Can't convert: " + str(foreign_char))
    return ' '


"""Correspondence generator that utilizes direct matching of keypoints across an image pair, without descriptors.

Authors: John Lambert
"""
from typing import Any, Dict, List, Optional, Tuple

from dask.distributed import Client, Future
import numpy as np

from gtsfm.common.image import Image
from gtsfm.common.keypoints import Keypoints
from gtsfm.common.pose_prior import PosePrior
from gtsfm.common.types import CALIBRATION_TYPE, CAMERA_TYPE
from gtsfm.frontend.correspondence_generator.correspondence_generator_base import CorrespondenceGeneratorBase
from gtsfm.frontend.correspondence_generator.keypoint_aggregator.keypoint_aggregator_base import KeypointAggregatorBase
from gtsfm.frontend.correspondence_generator.keypoint_aggregator.keypoint_aggregator_dedup import (
    KeypointAggregatorDedup,
)
from gtsfm.frontend.correspondence_generator.keypoint_aggregator.keypoint_aggregator_unique import (
    KeypointAggregatorUnique,
)
from gtsfm.frontend.matcher.image_matcher_base import ImageMatcherBase
from gtsfm.two_view_estimator import TWO_VIEW_OUTPUT, TwoViewEstimator

class ImageCorrespondenceGenerator(CorrespondenceGeneratorBase):
    """Pair-wise direct matching of images (e.g. transformer-based)."""

    def __init__(self, matcher: ImageMatcherBase, deduplicate: bool = True) -> None:
        """
        Args:
            matcher: Matcher to use.
            deduplicate: Whether to de-duplicate with a single image the detections received from each image pair.
        """
        self._matcher = matcher

        self._aggregator: KeypointAggregatorBase = (
            KeypointAggregatorDedup() if deduplicate else KeypointAggregatorUnique()
        )

    def __repr__(self) -> str:
        return f"""
        ImageCorrespondenceGenerator:
           {self._matcher}
           {self._aggregator}
        """

    def generate_correspondences(
        self,
        client: Client,
        images: List[Future],
        image_pairs: List[Tuple[int, int]],
    ) -> Tuple[List[Keypoints], Dict[Tuple[int, int], np.ndarray]]:
        """Apply the correspondence generator to generate putative correspondences.

        Args:
            client: Dask client, used to execute the front-end as futures.
            images: List of all images, as futures.
            image_pairs: Indices of the pairs of images to estimate two-view pose and correspondences.

        Returns:
            List of keypoints, one entry for each input images.
            Putative correspondence as indices of keypoints, for pairs of images.
        """

        def apply_image_matcher(
            image_matcher: ImageMatcherBase, image_i1: Image, image_i2: Image
        ) -> Tuple[Keypoints, Keypoints]:
            return image_matcher.match(image_i1=image_i1, image_i2=image_i2)

        image_matcher_future = client.scatter(self._matcher, broadcast=False)
        pairwise_correspondence_futures = {
            (i1, i2): client.submit(apply_image_matcher, image_matcher_future, images[i1], images[i2])
            for i1, i2 in image_pairs
        }

        pairwise_correspondences: Dict[Tuple[int, int], Tuple[Keypoints, Keypoints]] = client.gather(
            pairwise_correspondence_futures
        )

        keypoints_list, putative_corr_idxs_dict = self._aggregator.aggregate(keypoints_dict=pairwise_correspondences)
        return keypoints_list, putative_corr_idxs_dict

    def generate_correspondences_and_estimate_two_view(
        self,
        client: Client,
        images: List[Image],
        image_pairs: List[Tuple[int, int]],
        camera_intrinsics: List[CALIBRATION_TYPE],
        relative_pose_priors: Dict[Tuple[int, int], PosePrior],
        gt_cameras: List[Optional[CAMERA_TYPE]],
        gt_scene_mesh: Optional[Any],
        two_view_estimator: TwoViewEstimator,
    ) -> Tuple[List[Keypoints], Dict[Tuple[int, int], TWO_VIEW_OUTPUT]]:
        """Apply the correspondence generator to generate putative correspondences and subsequently process them with
        two view estimator to complete the front-end.

        Args:
            client: Dask client, used to execute the front-end as futures.
            images: List of all images.
            image_pairs: Indices of the pairs of images to estimate two-view pose and correspondences.
            camera_intrinsics: List of all camera intrinsics.
            relative_pose_priors: Priors on relative pose between two cameras.
            gt_cameras: GT cameras, used to evaluate metrics.
            gt_scene_mesh: GT mesh of the 3D scene, used to evaluate metrics.
            two_view_estimator: Two view estimator, which is used to verify correspondences and estimate pose.

        Returns:
            List of keypoints, one entry for each input images.
            Two view output for image_pairs.
        """

        def apply_image_matcher(
            image_matcher: ImageMatcherBase, image_i1: Image, image_i2: Image
        ) -> Tuple[Keypoints, Keypoints]:
            return image_matcher.match(image_i1=image_i1, image_i2=image_i2)

        def apply_two_view_estimator(
            two_view_estimator: TwoViewEstimator,
            keypoints_i1: Keypoints,
            keypoints_i2: Keypoints,
            putative_corr_idxs: np.ndarray,
            camera_intrinsics_i1: CALIBRATION_TYPE,
            camera_intrinsics_i2: CALIBRATION_TYPE,
            i2Ti1_prior: Optional[PosePrior],
            gt_camera_i1: Optional[CAMERA_TYPE],
            gt_camera_i2: Optional[CAMERA_TYPE],
            gt_scene_mesh: Optional[Any] = None,
        ) -> TWO_VIEW_OUTPUT:
            return two_view_estimator.run_2view(
                keypoints_i1=keypoints_i1,
                keypoints_i2=keypoints_i2,
                putative_corr_idxs=putative_corr_idxs,
                camera_intrinsics_i1=camera_intrinsics_i1,
                camera_intrinsics_i2=camera_intrinsics_i2,
                i2Ti1_prior=i2Ti1_prior,
                gt_camera_i1=gt_camera_i1,
                gt_camera_i2=gt_camera_i2,
                gt_scene_mesh=gt_scene_mesh,
            )

        image_matcher_future = client.scatter(self._matcher, broadcast=False)
        two_view_estimator_future = client.scatter(two_view_estimator, broadcast=False)
        pairwise_correspondence_futures = {
            (i1, i2): client.submit(apply_image_matcher, image_matcher_future, images[i1], images[i2])
            for i1, i2 in image_pairs
        }

        pairwise_correspondences: Dict[Tuple[int, int], Tuple[Keypoints, Keypoints]] = client.gather(
            pairwise_correspondence_futures
        )

        keypoints_list, putative_corr_idxs_dict = self._aggregator.aggregate(keypoints_dict=pairwise_correspondences)

        two_view_output_futures = {
            (i1, i2): client.submit(
                apply_two_view_estimator,
                two_view_estimator_future,
                keypoints_list[i1],
                keypoints_list[i2],
                putative_corr_idxs_dict[(i1, i2)],
                camera_intrinsics[i1],
                camera_intrinsics[i2],
                relative_pose_priors.get((i1, i2)),
                gt_cameras[i1],
                gt_cameras[i2],
                gt_scene_mesh,
            )
            for (i1, i2) in image_pairs
        }

        two_view_output_dict = client.gather(two_view_output_futures)
        return keypoints_list, two_view_output_dict


"""
First version: 16 May 2022
@author: Wilson W.C. Yiu
"""

import enum
import itertools
import math
import os
import pickle
import time
import traceback
from collections import defaultdict, namedtuple
from datetime import datetime
from functools import cmp_to_key
from math import atan2, degrees, radians, cos
from pathlib import Path
import numpy as np
from imutils import paths
import SimpleITK as sitk

def main():

    ## settings
    folder_path: str = 'D:/viterbi linkage/dataset/'
    raw_3d_data_dir: str = folder_path + '3D raw data_seg data_find center coordinate/4 Segmentation dataset/'
    default_save_dir = folder_path + 'save_directory_enhancement/'

    ## user input
    input_series_name_sub_dir_dict: dict = {}
    # input_series_name_sub_dir_dict["20190621++2_8layers_M3a_Step98"] = "1 8layers mask data/20190621++2_8layers_M3a_Step98"
    # input_series_name_sub_dir_dict["20190701--1_8layers_M3a_Step98"] = "1 8layers mask data/20190701--1_8layers_M3a_Step98"
    # input_series_name_sub_dir_dict["20190701--2_8layers_M3a_Step98"] = "1 8layers mask data/20190701--2_8layers_M3a_Step98"
    # input_series_name_sub_dir_dict["20200716++1_8layers_M3s_Step98"] = "1 8layers mask data/20200716++1_8layers_M3s_Step98"
    # input_series_name_sub_dir_dict["20200716++2_8layers_M3a_Step98"] = "1 8layers mask data/20200716++2_8layers_M3a_Step98"
    # input_series_name_sub_dir_dict["20200802--2_9layers_M3a_Step98"] = "2 9layers mask data/20200802--2_9layers_M3a_Step98"
    # input_series_name_sub_dir_dict["20200829++1_9layers_M3a_Step98"] = "2 9layers mask data/20200829++1_9layers_M3a_Step98"
    # input_series_name_sub_dir_dict["20200829++2_9layers_M3a_Step98"] = "2 9layers mask data/20200829++2_9layers_M3a_Step98"
    # input_series_name_sub_dir_dict["20200829--1_9layers_M3a_Step98"] = "2 9layers mask data/20200829--1_9layers_M3a_Step98"

    input_series_name_sub_dir_dict["20190621++2_inter_29layers_mask_3a"] = "5 29layers inter mask data/model3a/20190621++2_inter_29layers_mask_3a"
    input_series_name_sub_dir_dict["20190701--1_inter_29layers_mask_3a"] = "5 29layers inter mask data/model3a/20190701--1_inter_29layers_mask_3a"
    input_series_name_sub_dir_dict["20190701--2_inter_29layers_mask_3a"] = "5 29layers inter mask data/model3a/20190701--2_inter_29layers_mask_3a"
    input_series_name_sub_dir_dict["20200716++1_inter_29layers_mask_3a"] = "5 29layers inter mask data/model3a/20200716++1_inter_29layers_mask_3a"
    input_series_name_sub_dir_dict["20200716++2_inter_29layers_mask_3a"] = "5 29layers inter mask data/model3a/20200716++2_inter_29layers_mask_3a"
    input_series_name_sub_dir_dict["20200802--2_inter_33layers_mask_3a"] = "6 33layers inter mask data/20200802--2_inter_33layers_mask_3a"
    input_series_name_sub_dir_dict["20200829++1_inter_33layers_mask_3a"] = "6 33layers inter mask data/20200829++1_inter_33layers_mask_3a"
    input_series_name_sub_dir_dict["20200829++2_inter_33layers_mask_3a"] = "6 33layers inter mask data/20200829++2_inter_33layers_mask_3a"
    input_series_name_sub_dir_dict["20200829--1_inter_33layers_mask_3a"] = "6 33layers inter mask data/20200829--1_inter_33layers_mask_3a"

    ## hyper parameter settings
    weight_tuple_list: list = [WeightTuple(0.3, 0.4, 0.3)]
    max_moving_distance_list: list = [40]
    coord_length_for_vector_list: list = [6]
    average_movement_step_length_list: list = [6]
    minimum_track_length_list: list = [1]

    ## end of user input

    discount_rate_per_layer_list: list = [None]      # depricated

    start_datetime: Date = datetime.now()
    date_str: str = start_datetime.strftime("%Y%m%d-%H%M%S")
    py_file_name: str = Path(__file__).name.replace(".py", "")
    individual_result_dir: str = default_save_dir + date_str + "_" + py_file_name + "/"
    os.makedirs(individual_result_dir)

    score_log_dir: str = individual_result_dir + "score_log/"
    os.makedirs(score_log_dir)

    hyper_para_combination_list = list(itertools.product(
                                                         weight_tuple_list,
                                                         max_moving_distance_list,
                                                         coord_length_for_vector_list,
                                                         average_movement_step_length_list,
                                                         minimum_track_length_list,
                                                         discount_rate_per_layer_list
                                                         ))

    hyper_para_list: list = []
    for hyper_para_combination in hyper_para_combination_list:
        weight_tuple = hyper_para_combination[0]
        max_moving_distance = hyper_para_combination[1]
        coord_length_for_vector = hyper_para_combination[2]
        average_movement_step_length = hyper_para_combination[3]
        minimum_track_length: int = hyper_para_combination[4]
        discount_rate_per_layer: [str, float] = hyper_para_combination[5]

        hyper_para: HyperPara = HyperPara(
                                            weight_tuple,
                                            max_moving_distance,
                                            coord_length_for_vector,
                                            average_movement_step_length,
                                            minimum_track_length,
                                            discount_rate_per_layer)

        hyper_para_list.append(hyper_para)

    total_para_size: int = len(hyper_para_list)
    for hyperpara_idx, hyper_para in enumerate(hyper_para_list):
        para_set_num: int = hyperpara_idx + 1

        print(f"start. Parameter set: {para_set_num}/ {total_para_size}; {hyper_para.__str__()}")

        start_time = time.perf_counter()

        feature_based_result_dict: dict = {}
        series_frame_num_score_log_mtx_dict_dict: dict = {}
        try:
            for series_name, series_sub_dir in input_series_name_sub_dir_dict.items():
                print(f"working on series_name: {series_name}")

                frame_num_node_id_coord_dict_dict = read_series_data(raw_3d_data_dir + series_sub_dir, series_name)

                return_series, final_result_list, frame_num_score_log_mtx_dict = cell_tracking_core_flow(series_name,
                                                                                frame_num_node_id_coord_dict_dict,
                                                                                hyper_para
                                                                                )
                feature_based_result_dict[series_name] = final_result_list
                series_frame_num_score_log_mtx_dict_dict[series_name] = frame_num_score_log_mtx_dict

        except Exception as e:
            time.sleep(1)
            print()
            traceback.print_exc()
            print(f"series_name {series_name}. para {para_set_num}.  hyper_para: {hyper_para.__str__()}")
            exit()

        hyper_para_str: str = str(hyper_para.weight_tuple) + \
                                    "MD(" + str(hyper_para.max_moving_distance) + ")_" + \
                                    "CL(" + str(hyper_para.coord_length_for_vector) + ")_" + \
                                    "AML(" + str(hyper_para.average_movement_step_length) + ")_" + \
                                    "MTL(" + str(hyper_para.minimum_track_length) + ")_" + \
                                    "DR(" +  str(hyper_para.discount_rate_per_layer) + ")_"

        result_file_name: str = py_file_name + "_hp" + str(hyperpara_idx+1).zfill(3) + "__" + hyper_para_str
        abs_file_path: str = individual_result_dir + result_file_name

        py_file_name: str = Path(__file__).name.replace(".py", "_")
        save_score_log_to_excel(series_frame_num_score_log_mtx_dict_dict, score_log_dir, py_file_name)

        execution_time = time.perf_counter() - start_time
        generate_txt_file(hyperpara_idx, hyper_para, execution_time, abs_file_path)

        print("save_track_dictionary: ", abs_file_path)

        print(f"Execution time: {np.round(execution_time, 4)} seconds")

def __________object_start_label():
    raise Exception("for labeling only")

CoordTuple = namedtuple("CoordTuple", "x y z")
WeightTuple = namedtuple("WeightTuple", "directional distance average_movement")

class DataStore(object):

    def __init__(self):
        profit_mtx_data_dict = {}

    @staticmethod
    def obtain_data_a(input_str: str):
        # return data_map_a_dict[input_str]
        print("Hi,this tutorial is about static class in python: " + input_str)

class ROUTING_STRATEGY_ENUM(enum.Enum):
    ALL_LAYER = 1
    ONE_LAYER = 2

class CUT_STRATEGY_ENUM(enum.Enum):
    DURING_ROUTING = 1
    AFTER_ROUTING = 2

class BOTH_CELL_BELOW_THRESHOLD_STRATEGY_ENUM(enum.Enum):
    SHARE = 1

class HyperPara():
    def __init__(self,
                 weight_tuple: tuple,
                 max_moving_distance: int,
                 coord_length_for_vector: int,
                 average_movement_step_length: int,
                 minimum_track_length: int,
                 discount_rate_per_layer: float):

        self.weight_tuple                     = weight_tuple
        self.max_moving_distance              = max_moving_distance
        self.coord_length_for_vector          = coord_length_for_vector
        self.average_movement_step_length     = average_movement_step_length
        self.minimum_track_length             = minimum_track_length
        self.discount_rate_per_layer          = discount_rate_per_layer

    def __str__(self, separator=""):
        return  f"weight_tuple: {self.weight_tuple}; {separator}" \
                f"max_moving_distance: {self.max_moving_distance}; {separator}" \
                f"coord_length_for_vector: {self.coord_length_for_vector}; {separator}" \
                f"average_movement_step_length: {self.average_movement_step_length}; {separator}" \
                f"minimum_track_length: {self.minimum_track_length}; {separator}" \
                f"discount_rate_per_layer: {self.discount_rate_per_layer}; {separator}"

class CellId():

    def __init__(self, start_frame_num: int, cell_idx: int):
        self.start_frame_num = start_frame_num
        self.cell_idx = cell_idx

    def str_short(self):
        return f"{self.start_frame_num}-{self.cell_idx};"

    def __str__(self):
        # return f"CellId(start_frame_num: {self.start_frame_num}; cell_idx: {self.cell_idx})"
        return f"CellId({self.start_frame_num}, {self.cell_idx})"

    def __eq__(self, other):
        if self.start_frame_num == other.start_frame_num and self.cell_idx == other.cell_idx:
            return True

        return False

    def __hash__(self):
        return hash((self.start_frame_num, self.cell_idx))

def compare_cell_id(cell_1, cell_2):
    if cell_1.start_frame_num == cell_2.start_frame_num:
        if cell_1.cell_idx < cell_2.cell_idx:           return -1
        elif cell_1.cell_idx > cell_2.cell_idx:         return 1
        else:                                           raise Exception("cell_1.node_idx == cell_2.node_idx")

    if cell_1.start_frame_num < cell_2.start_frame_num:            return -1
    elif cell_1.start_frame_num > cell_2.start_frame_num:          return 1

def __________flow_function_start_label():
    raise Exception("for labeling only")

def read_series_data(raw_3d_data_dir: str, series_name: str):
    image_path_list = sorted(list(paths.list_images(raw_3d_data_dir)))

    # construct image series list
    series_image_path_list_dict: dict = defaultdict(list)
    for idx, img_path in enumerate(image_path_list):
        series_image_path_list_dict[series_name].append(img_path)

    # sort
    for series_name, image_list in series_image_path_list_dict.items():
        series_image_path_list_dict[series_name] = sorted(image_list)

    threshold = 100  # if the volume of object is small than the threshold, remove the object
    total_size = len(series_image_path_list_dict)
    for idx, (series_name, img_path_list) in enumerate(series_image_path_list_dict.items()):

        # frame_num_coord_tuple_list_dict = {}
        frame_num_node_id_coord_dict_dict: dict = defaultdict(dict)
        total_frame = len(img_path_list)
        for frame_idx, img_path in enumerate(img_path_list):
            frame_num = frame_idx + 1
            print(f"\rseries: {idx + 1}/ {total_size}; frame: {frame_num}/{total_frame}", end='')

            img_stack = sitk.ReadImage(os.path.join(img_path))
            img_stack = sitk.GetArrayFromImage(img_stack)

            # get the intensity value and volume of each cell object
            img_stack_label, img_stack_cellvolume_counts = np.unique(img_stack, return_counts=True)

            # remove the small cell which volume is lower than threshold
            for l in range(len(img_stack_label)):
                if img_stack_cellvolume_counts[l] < threshold:
                    img_stack[img_stack == img_stack_label[l]] = 0
            labels = np.unique(img_stack)

            label_coord_tuple_dict = cell_center(img_stack)

            for label, coord_tuple in label_coord_tuple_dict.items():
                # key = str(frame_num) + ":" + str(label)
                # frame_num_coord_tuple_list_dict[key] = coord_tuple
                x, y, z = coord_tuple[0], coord_tuple[1], coord_tuple[2]

                frame_num_node_id_coord_dict_dict[frame_num][label] = CoordTuple(x, y, x)

        return frame_num_node_id_coord_dict_dict

def cell_center(seg_img):
    label_coord_tuple_dict = {}
    for label in np.unique(seg_img):
        if label != 0:
            all_points_z, all_points_x, all_points_y = np.where(seg_img == label)
            avg_z = np.round(np.mean(all_points_z))
            avg_x = np.round(np.mean(all_points_x))
            avg_y = np.round(np.mean(all_points_y))

            label_coord_tuple_dict[label] = (avg_z, avg_x, avg_y)

    return label_coord_tuple_dict

def cell_tracking_core_flow(series: str, frame_num_node_id_coord_dict_dict: dict, hyper_para: HyperPara):

    all_track_dict, score_log_mtx = execute_cell_tracking_task_bon(frame_num_node_id_coord_dict_dict,
                                                                   hyper_para
                                                                   )

    sorted_cell_id_key_list = sorted(list(all_track_dict.keys()), key=cmp_to_key(compare_cell_id))
    sorted_dict: dict = {}
    for sorted_key in sorted_cell_id_key_list:
        sorted_dict[sorted_key] = all_track_dict[sorted_key]
    all_track_dict = sorted_dict

    track_list_list: list = list(all_track_dict.values())

    code_validate_track_list(track_list_list)

    return series, track_list_list, score_log_mtx

def __________component_function_start_label():
    raise Exception("for labeling only")

def execute_cell_tracking_task_bon(frame_num_node_id_coord_dict_dict: dict, hyper_para: HyperPara):
    to_handle_cell_id_list: list = derive_initial_cell_id_list(frame_num_node_id_coord_dict_dict)

    frame_num_node_idx_occupation_list_list_dict = initiate_frame_num_node_idx_cell_id_occupation_list_list_dict(frame_num_node_id_coord_dict_dict)

    total_frame: int = max(frame_num_node_id_coord_dict_dict.keys())

    all_valid_cell_track_idx_dict: dict = {}        # [cell_id][frame_num] = node_idx
    all_valid_cell_track_score_dict: dict = {}       # [cell_id][frame_num] = probability

    score_log_mtx = init_score_log(frame_num_node_id_coord_dict_dict)

    while len(to_handle_cell_id_list) != 0:
        to_handle_cell_id_list.sort(key=cmp_to_key(compare_cell_id))

        to_handle_cell_id: CellId = to_handle_cell_id_list[0]

        print(f"{to_handle_cell_id.str_short()}")

        if to_handle_cell_id in all_valid_cell_track_idx_dict:
            frame_num_node_idx_occupation_list_list_dict = remove_track_from_cell_occupation_list_list_dict(frame_num_node_idx_occupation_list_list_dict, to_handle_cell_id, all_valid_cell_track_idx_dict[to_handle_cell_id])
            del all_valid_cell_track_idx_dict[to_handle_cell_id]
            del all_valid_cell_track_score_dict[to_handle_cell_id]

        is_new_cell: bool = (len(frame_num_node_idx_occupation_list_list_dict[to_handle_cell_id.start_frame_num][to_handle_cell_id.cell_idx]) == 0)
        if not is_new_cell:
            has_exist_data: bool = (to_handle_cell_id in all_valid_cell_track_idx_dict)
            if has_exist_data:
                frame_num_node_idx_occupation_list_list_dict = remove_track_from_cell_occupation_list_list_dict(frame_num_node_idx_occupation_list_list_dict, to_handle_cell_id, all_valid_cell_track_idx_dict[to_handle_cell_id])
                del all_valid_cell_track_idx_dict[to_handle_cell_id]
                del all_valid_cell_track_score_dict[to_handle_cell_id]

            to_handle_cell_id_list.remove(to_handle_cell_id)
            continue

        handling_cell_frame_num_track_idx_dict: dict = { to_handle_cell_id.start_frame_num: to_handle_cell_id.cell_idx}
        handling_cell_frame_num_score_dict: dict = {}

        redo_cell_id_set: set = set()

        second_frame_num: int = to_handle_cell_id.start_frame_num + 1
        for connect_to_frame_num in inclusive_range(second_frame_num, total_frame):
            total_node: int = len(frame_num_node_id_coord_dict_dict[connect_to_frame_num])
            is_connect_to_frame_has_no_cell = (total_node == 0)
            if is_connect_to_frame_has_no_cell:
                break

            node_id_coord_dict: dict = frame_num_node_id_coord_dict_dict[connect_to_frame_num]

            cut_threshold = 1000
            best_idx, best_prob, score_log_mtx, redo_cell_id_set = derive_best_node_idx_to_connect(to_handle_cell_id,
                                                                                handling_cell_frame_num_track_idx_dict,
                                                                                node_id_coord_dict,
                                                                                connect_to_frame_num,
                                                                                frame_num_node_idx_occupation_list_list_dict,
                                                                                all_valid_cell_track_idx_dict,
                                                                                all_valid_cell_track_score_dict,
                                                                                frame_num_node_id_coord_dict_dict,
                                                                                score_log_mtx,
                                                                                cut_threshold,
                                                                                redo_cell_id_set,
                                                                                hyper_para.weight_tuple,
                                                                                hyper_para.max_moving_distance,
                                                                                hyper_para.coord_length_for_vector,
                                                                                hyper_para.average_movement_step_length)

            has_valid_connection: bool = (best_prob != 0)
            if not has_valid_connection:
                break

            if connect_to_frame_num in handling_cell_frame_num_track_idx_dict:   raise Exception("code validation check")
            if connect_to_frame_num in handling_cell_frame_num_score_dict:       raise Exception("code validation check")

            handling_cell_frame_num_track_idx_dict[connect_to_frame_num] = best_idx
            handling_cell_frame_num_score_dict[connect_to_frame_num] = best_prob

        is_track_above_min_length: bool = (len(handling_cell_frame_num_track_idx_dict.keys()) > hyper_para.minimum_track_length)

        if not is_track_above_min_length:
            to_handle_cell_id_list.remove(to_handle_cell_id)
            continue

        # use final track to check if any redo cells occur
        for redo_cell_id in redo_cell_id_set:
            if redo_cell_id not in to_handle_cell_id_list:
                to_handle_cell_id_list.append(redo_cell_id)

        # add to occupation list
        frame_num_node_idx_occupation_list_list_dict = add_track_to_cell_occupation_list_list_dict(frame_num_node_idx_occupation_list_list_dict, to_handle_cell_id, handling_cell_frame_num_track_idx_dict)

        # add track to valid track list
        all_valid_cell_track_idx_dict[to_handle_cell_id] = handling_cell_frame_num_track_idx_dict
        all_valid_cell_track_score_dict[to_handle_cell_id] = handling_cell_frame_num_score_dict

        to_handle_cell_id_list.remove(to_handle_cell_id)

    print("----> finish")

    all_cell_id_track_list_dict: dict = defaultdict(list)
    for cell_id, handling_cell_frame_num_track_idx_dict in all_valid_cell_track_idx_dict.items():
        track_tuple_list: list = []
        previous_node_idx: int = -1
        for connect_to_frame_num, node_idx in handling_cell_frame_num_track_idx_dict.items():
            frame_idx: int = connect_to_frame_num - 1
            track_tuple_list.append((node_idx, frame_idx, previous_node_idx))
            previous_node_idx = node_idx

        all_cell_id_track_list_dict[cell_id] = track_tuple_list

    return all_cell_id_track_list_dict, score_log_mtx

def __________unit_function_start_label():
    raise Exception("for labeling only")

def generate_txt_file(hyperpara_idx: int, hyper_para: HyperPara, execution_time, abs_file_path: str):
    with open(abs_file_path + ".txt", 'w') as f:
        f.write(f"Execution time: {np.round(execution_time, 4)} seconds\n")
        f.write("hyper_para--- ID: " + str(hyperpara_idx + 1) + "; \n" + hyper_para.__str__(separator="\n"))

def derive_average_movement_score(to_handle_cell_id,
                                  current_frame_num,
                                  last_frame_node_coord,
                                  candidate_node_coord,
                                  step_length,
                                  handling_cell_frame_num_track_idx_dict,
                                  frame_num_node_id_coord_dict_dict,
                                  max_moving_distance: float):

    start_frame = current_frame_num - step_length
    start_frame = max(start_frame, 1, to_handle_cell_id.start_frame_num)
    end_frame = current_frame_num

    actual_step_length: int = (end_frame - start_frame) + 1
    if actual_step_length > 1:
        sum_distance: float = 0

        previous_node_idx: int = handling_cell_frame_num_track_idx_dict[start_frame]
        previous_coord_tuple: CoordTuple = frame_num_node_id_coord_dict_dict[start_frame][previous_node_idx]
        for frame_num in inclusive_range(start_frame+1, end_frame):
            current_node_idx: int = handling_cell_frame_num_track_idx_dict[frame_num]
            current_coord_tuple: CoordTuple = frame_num_node_id_coord_dict_dict[frame_num][current_node_idx]

            x_y_distance: float = ((current_coord_tuple.x - previous_coord_tuple.x)**2 + (current_coord_tuple.y - previous_coord_tuple.y)**2)**0.5
            distance: float = (x_y_distance**2 + (current_coord_tuple.z - previous_coord_tuple.z)**2)**0.5
            sum_distance += distance

            previous_coord_tuple = current_coord_tuple
        average_movement: float = sum_distance / actual_step_length

        new_candidate_node_distance: float = ((candidate_node_coord.x - last_frame_node_coord.x)**2 + (candidate_node_coord.y - last_frame_node_coord.y)**2)**0.5

        movement_difference: float = abs(average_movement - new_candidate_node_distance)

        if movement_difference > max_moving_distance:
            normalized_average_movement_score = 0
        else:
            normalized_average_movement_score: float = (max_moving_distance - movement_difference) / max_moving_distance

    else:
        normalized_average_movement_score = 0.5

    return normalized_average_movement_score

def derive_distance_score(current_node_coord, last_frame_node_coord, max_moving_distance):
    x_y_distance: float = ((current_node_coord.x - last_frame_node_coord.x)**2 + (current_node_coord.y - last_frame_node_coord.y)**2)**0.5

    x_y_z_distance: float = (x_y_distance**2 + (current_node_coord.z - last_frame_node_coord.z)**2)**0.5
    x_y_z_distance = np.round(x_y_z_distance, 4)

    if x_y_z_distance > max_moving_distance:
        distance_score = 0
    else:
        distance_score = (max_moving_distance - x_y_z_distance) / max_moving_distance

    return distance_score

def derive_directional_score(to_handle_cell_id, current_frame_num, last_frame_node_coord, candidate_node_coord,
                             coord_length_of_vector, handling_cell_frame_num_track_idx_dict, frame_num_node_id_coord_dict_dict):
    previous_coord_list = []
    start_frame = current_frame_num - coord_length_of_vector
    start_frame = max(start_frame, 1, to_handle_cell_id.start_frame_num)
    end_frame = current_frame_num

    coord_length: int = (end_frame - start_frame) + 1
    if coord_length > 1:
        for frame_num in inclusive_range(start_frame, end_frame):
            node_id: int = handling_cell_frame_num_track_idx_dict[frame_num]
            coord_tuple: CoordTuple = frame_num_node_id_coord_dict_dict[frame_num][node_id]
            previous_coord_list.append(coord_tuple)
        previous_vec: CoordTuple = derive_vector_from_coord_list(previous_coord_list)

        new_candidate_vec: CoordTuple = derive_vector_from_coord_list([last_frame_node_coord, candidate_node_coord])

        degree_diff: float = derive_degree_diff_from_two_vectors(previous_vec, new_candidate_vec)

        directional_score: float = (cos(radians(degree_diff)) + 1) * 0.5  # +1 and *0.5 to shift up and make it stay between 1 and 0

    else:
        directional_score: float = 0.5

    return directional_score

def init_score_log(frame_num_node_id_coord_dict_dict):
    frame_num_score_log_mtx = {}

    start_frame_num = min(frame_num_node_id_coord_dict_dict.keys())
    end_frame_num = max(frame_num_node_id_coord_dict_dict.keys())
    for frame_num in range(start_frame_num, end_frame_num):
        num_node_id_coord_dict = frame_num_node_id_coord_dict_dict[frame_num]
        next_frame_num_node_id_coord_dict = frame_num_node_id_coord_dict_dict[frame_num+1]
        total_row = max(num_node_id_coord_dict.keys()) + 1
        total_col = max(next_frame_num_node_id_coord_dict) + 1
        row_list = []

        for row_idx in range(total_row):
            col_list = []
            for col_idx in range(total_col):
                col_list.append(0)
            row_list.append(col_list)
        frame_num_score_log_mtx[frame_num] = row_list

    return frame_num_score_log_mtx

def save_score_log_to_excel(series_frame_num_score_log_mtx_dict_dict: dict, excel_output_dir_path: str, file_name_prefix: str = ""):
    import pandas as pd

    for series, frame_num_score_log_mtx_dict in series_frame_num_score_log_mtx_dict_dict.items():
        file_name: str = file_name_prefix + f"series_{series}.xlsx"
        filepath = excel_output_dir_path + file_name;
        writer = pd.ExcelWriter(filepath, engine='xlsxwriter') #pip install xlsxwriter

        for frame_num, prof_matrix in frame_num_score_log_mtx_dict.items():
            frame_data_array: np.arrays = frame_num_score_log_mtx_dict[frame_num]

            df = pd.DataFrame (frame_data_array)

            sheet_name: str = "frame_1" if frame_num == 1 else str(frame_num)

            df.to_excel(writer, sheet_name=sheet_name, index=True)

            workbook = writer.book
            cell_format = workbook.add_format({'text_wrap': True})        # to make \n work

            worksheet = writer.sheets[sheet_name]
            worksheet.set_column('A:BO', cell_format=cell_format)

            for col_idx in range(40):
                if col_idx == 0:    col_width = 3
                else:               col_width = 25
                worksheet.set_column(col_idx, col_idx, col_width)

        writer.save()

def derive_vector_from_coord_list(coord_tuple_list: list):
    if len(coord_tuple_list) == 1:
        raise Exception("code validation check")

    start_vec: CoordTuple = coord_tuple_list[0]
    end_vec: CoordTuple = coord_tuple_list[-1]

    reversed_y_coord = -(end_vec.y - start_vec.y) # reverse y because coord raw data of y-axis is reversed
    z_coord = end_vec.z - start_vec.z
    final_coord_tuple = CoordTuple(end_vec.x - start_vec.x, reversed_y_coord, z_coord)

    return final_coord_tuple

def derive_initial_cell_id_list(frame_num_node_id_coord_dict_dict: dict):
    to_handle_cell_id_list: list = []
    for frame_num, node_id_coord_dict in frame_num_node_id_coord_dict_dict.items():
        for node_id in node_id_coord_dict.keys():
            cell_start_frame: int = frame_num
            to_handle_cell_id_list.append(CellId(cell_start_frame, node_id))

    return to_handle_cell_id_list

def derive_best_node_idx_to_connect(to_handle_cell_id,
                                    handling_cell_frame_num_track_idx_dict,
                                    node_id_coord_dict,
                                    connect_to_frame_num,
                                    frame_num_node_idx_occupation_list_list_dict,
                                    all_valid_cell_track_idx_dict,
                                    all_valid_cell_track_prob_dict,
                                    frame_num_node_id_coord_dict_dict,
                                    score_log_mtx,
                                    cut_threshold,
                                    redo_cell_id_set,
                                    weight_tuple: WeightTuple,
                                    max_moving_distance: int,
                                    coord_length_for_vector: int,
                                    average_movement_step_length: int):

    current_frame_num = connect_to_frame_num - 1
    round_to = 2

    current_frame_node_id: int = handling_cell_frame_num_track_idx_dict[current_frame_num]
    current_frame_node_coord: CoordTuple = frame_num_node_id_coord_dict_dict[current_frame_num][current_frame_node_id]

    best_prob: float = 0
    best_node_idx: int = None

    for candidate_node_id, candidate_node_coord in node_id_coord_dict.items():
        final_score: float = 0

        is_use_directional_score: bool = (weight_tuple.directional > 0)
        if is_use_directional_score:
            directional_score: float = derive_directional_score(to_handle_cell_id,
                                                                current_frame_num,
                                                                current_frame_node_coord,
                                                                candidate_node_coord,
                                                                coord_length_for_vector,
                                                                handling_cell_frame_num_track_idx_dict,
                                                                frame_num_node_id_coord_dict_dict)

            weighted_directional_score = np.round(weight_tuple.directional * directional_score, round_to)

            if math.isnan(weighted_directional_score):
                weighted_directional_score = 0

            final_score += weighted_directional_score

        is_use_distance_score: bool = (weight_tuple.distance > 0)
        if is_use_distance_score:
            distance_score = derive_distance_score(candidate_node_coord, current_frame_node_coord, max_moving_distance)

            weighted_distance_score = np.round(weight_tuple.distance * distance_score, round_to)
            final_score += weighted_distance_score

        is_use_avg_movement_score: bool = (weight_tuple.average_movement > 0)
        if is_use_avg_movement_score:
            avg_movement_score = derive_average_movement_score(to_handle_cell_id,
                                                               current_frame_num,
                                                               current_frame_node_coord,
                                                               candidate_node_coord,
                                                               average_movement_step_length,
                                                               handling_cell_frame_num_track_idx_dict,
                                                               frame_num_node_id_coord_dict_dict,
                                                               max_moving_distance)
            weighted_avg_mov_score = np.round(weight_tuple.average_movement * avg_movement_score, round_to)
            final_score += weighted_avg_mov_score

        final_score = np.round(final_score, round_to)

        score_log_mtx[current_frame_num][current_frame_node_id][candidate_node_id] = final_score

    return best_node_idx, best_prob, score_log_mtx, redo_cell_id_set

def initiate_frame_num_node_idx_cell_id_occupation_list_list_dict(frame_num_node_id_coord_dict_dict: dict):
    frame_num_node_idx_occupation_list_list_dict: dict = defaultdict(dict)

    for frame_num, node_id_coord_dict in frame_num_node_id_coord_dict_dict.items():
        for node_id in node_id_coord_dict.keys():
            frame_num_node_idx_occupation_list_list_dict[frame_num][node_id] = []

    return frame_num_node_idx_occupation_list_list_dict

def remove_track_from_cell_occupation_list_list_dict(frame_num_node_idx_occupation_tuple_vec_dict: dict, cell_id, frame_num_cell_track_idx_dict: dict):
    for frame_num, node_idx in frame_num_cell_track_idx_dict.items():
        if cell_id not in frame_num_node_idx_occupation_tuple_vec_dict[frame_num][node_idx]:
            raise Exception("code validation")

        frame_num_node_idx_occupation_tuple_vec_dict[frame_num][node_idx].remove(cell_id)

    return frame_num_node_idx_occupation_tuple_vec_dict

def add_track_to_cell_occupation_list_list_dict(frame_num_node_idx_occupation_tuple_vec_dict: dict, cell_id, frame_num_cell_track_idx_dict: dict):
    for frame_num, node_idx in frame_num_cell_track_idx_dict.items():
        if cell_id in frame_num_node_idx_occupation_tuple_vec_dict[frame_num][node_idx]:
            raise Exception("code validation")

        frame_num_node_idx_occupation_tuple_vec_dict[frame_num][node_idx].append(cell_id)

    return frame_num_node_idx_occupation_tuple_vec_dict

def __________general_function_start_label():
    raise Exception("for labeling only")

def derive_degree_diff_from_two_vectors(vector_coord_tuple_1: CoordTuple, vector_coord_tuple_2: CoordTuple): #These can also be four parameters instead of two arrays
    """ Returns the angle in radians between vectors 'v1' and 'v2'::

        angle_between((1, 0, 0), (0, 1, 0))
        1.5707963267948966
        angle_between((1, 0, 0), (1, 0, 0))
        0.0
        angle_between((1, 0, 0), (-1, 0, 0))
        3.141592653589793
    """

    vec1_unit = unit_vector(vector_coord_tuple_1)
    vec2_unit = unit_vector(vector_coord_tuple_2)

    diff_radius: float = np.arccos(np.clip(np.dot(vec1_unit, vec2_unit), -1.0, 1.0))
    diff_degree: float = degrees(diff_radius)

    if diff_degree < 0:
        diff_degree = abs(diff_degree)

    return diff_degree

def unit_vector(vector):
    """ Returns the unit vector of the vector.  """
    return vector / np.linalg.norm(vector)

def derive_degree_from_vector(vector_coord_tuple_1: CoordTuple): #These can also be four parameters instead of two arrays
    default_zero_degree_vector = CoordTuple(0, 1)

    dot = default_zero_degree_vector.x * vector_coord_tuple_1.x + default_zero_degree_vector.y * vector_coord_tuple_1.y      # dot product
    det = default_zero_degree_vector.y * vector_coord_tuple_1.x - default_zero_degree_vector.x * vector_coord_tuple_1.y      # determinant
    angle = atan2(det, dot)  # atan2(y, x) or atan2(sin, cos)

    angle = degrees(angle)

    if angle < 0:
        angle += 360

    return angle

def inclusive_range(start: int, end: int):
    return range(start, end+1)

def dev_print(*arg):
    print(*arg)

def save_data_dict_to_file(data_dict: dict, save_file_abs_path: str):
    if not os.path.exists(save_file_abs_path):
        with open(save_file_abs_path, 'w'):
            pass
    pickle_out = open(save_file_abs_path, "wb")
    pickle.dump(data_dict, pickle_out)
    pickle_out.close()

def __________code_validation_function_start_label():
    raise Exception("for labeling only")

def code_validate_track_list(track_tuple_list_list: list):

    for track_tuple_list in track_tuple_list_list:
        frame_idx = track_tuple_list[0][1]

        for track_tuple in track_tuple_list[1:]:
            next_frame_idx = track_tuple[1]

            if next_frame_idx != (frame_idx + 1):
                raise Exception(track_tuple_list[0], next_frame_idx, track_tuple_list)

            frame_idx = next_frame_idx

if __name__ == '__main__':
    main()


"""A collection tkinter frame children, customized for the
application's needs.
"""

import ttkbootstrap as tb
from ttkbootstrap.dialogs.dialogs import Messagebox
from ttkbootstrap.scrolled import ScrolledFrame

from src.components.navigation import ButtonPanel, ContextMenu
from src.components.notifications import Notifications

class AutoLayoutFrame(tb.Frame):
    """Provides a frame class which supports automatic grid
    configuration and placement of labels.
    """

    def __init__(self, master, config, labels) -> None:
        super().__init__(master=master)
        for idx, weight in config["rowconfigure"].items():
            self.grid_rowconfigure(idx, weight=weight)
        for idx, weight in config["columnconfigure"].items():
            self.grid_columnconfigure(idx, weight=weight)

        for label in labels:
            tb.Label(self, text=label["text"], font=label["font"]).grid(
                row=label["row"],
                column=label["col"],
                rowspan=label["rowspan"],
                columnspan=label["columnspan"],
                padx=label["padx"],
                pady=label["pady"],
                sticky=label["sticky"],
            )

class RefreshMixin:
    """A mixin that allows a class to register its sub-components
    and to refresh those upon request.
    """

    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
        self.registered_components = []

    def register(self, component):
        """Registers a component to be refreshed, whenever the refresh
        method of the tab frame is called.
        """
        self.registered_components.append(component)

    def refresh(self):
        """Refreshes all registered components inside this frame.

        This method is invoked by whichever parent holds those tab
        frames.
        """
        for component in self.registered_components:
            component.refresh()

class ListFrame(tb.Frame, RefreshMixin):
    """A frame with a default left sidebar to depict a selection of
    objects. The central frame will be filled with the given detail
    view frame and the given form frame.

    Parameters
    ----------
    form_edit : forms.Form
        A form for adding or editing an object of the list.
    detail_view : cards.DetailView
        A frame for displaying the selected item in great detail.
    """

    def __init__(
        self,
        master,
        app,
        db_service,
        db_session,
        form_edit,
        detail_view,
        db_delete_item,
    ) -> None:
        super().__init__(master=master)
        self.app = app
        self.db_service = db_service
        self.db_session = db_session
        self.form_edit = form_edit
        self.detail_view = detail_view
        self.db_delete_item = db_delete_item

        self.scrolled_frame = None
        self.item_detail_view = None

        self.objects = None
        self.item_key_var = tb.IntVar()
        self.item_str_var = tb.StringVar()
        self.item_str_var.trace("w", self.select_handler)

        self.grid_columnconfigure(0, weight=1)
        self.grid_columnconfigure(2, weight=7)
        self.grid_rowconfigure(0, weight=1)

        self.build_gui_components()

    def build_gui_components(self):
        """Create the GUI elements for this component."""
        for separator in self.app.definitions.TAB_FRAME_LIST["separators"]:
            sep_new = tb.Separator(self, orient=separator["orient"])
            sep_new.grid(
                row=separator["row"],
                column=separator["col"],
                rowspan=separator["rowspan"],
                sticky=separator["sticky"],
            )

        btn_new_item = tb.Button(
            self, text="New", bootstyle="success", command=self.open_form
        )
        btn_new_item.grid(row=2, column=0, pady=25)

        self.register(self)
        self.refresh()

    def select_handler(self, *_args):
        if self.item_detail_view:
            self.item_detail_view.grid_forget()
        item = self.objects[self.item_key_var.get()]

        detail_scroll = ScrolledFrame(self)
        detail_scroll.grid(row=0, rowspan=3, column=2, sticky="nsew")
        self.item_detail_view = self.detail_view(detail_scroll, self.app, item)
        self.item_detail_view.pack(expand=True, fill="both", padx=25)

    def refresh(self):
        if self.scrolled_frame:
            self.scrolled_frame.grid_remove()

        list_layout = self.app.definitions.TAB_FRAME_LIST["sidebar"]
        self.scrolled_frame = ScrolledFrame(master=self, autohide=True)
        self.scrolled_frame.grid(
            row=list_layout["row"],
            column=list_layout["col"],
            sticky=list_layout["sticky"],
        )

        # Context menu
        context_menu = ContextMenu(self.app)
        context_menu.add_command(label="Edit", command=self.edit_entry)
        context_menu.add_command(label="Delete", command=self.delete_entry)

        # Item list
        items = self.db_service.get_all(self.db_session).all()
        item_dict = {}
        self.objects = {}
        for item in items:
            item_dict[item.id] = item.name
            self.objects[item.id] = item
        frm_item_list = ButtonPanel(
            parent=self.scrolled_frame,
            ttk_string_var=self.item_str_var,
            labels=item_dict,
            styling=self.app.definitions.LIST_ITEM,
            ttk_key_var=self.item_key_var,
            context_menu=context_menu,
        )
        frm_item_list.pack(expand=True, fill="both")

    def open_form(self):
        """Create a new form instance and put it on the grid layout."""
        form = self.form_edit(self, self.app, self.db_service, self.db_session)
        form.grid(row=0, rowspan=3, column=2, sticky="nsew")

    def edit_entry(self, *_args):
        """Opens a form corresponding to the selected object to allow
        editing the entity.
        """
        item = self.objects[self.item_key_var.get()]
        form = self.form_edit(
            master=self,
            app=self.app,
            db_service=self.db_service,
            db_session=self.db_session,
            selected_item=item,
        )
        form.grid(row=0, rowspan=3, column=2, sticky="nsew")

    def delete_entry(self, *_args):
        """Allows to delete the selected entity. Will warn the user via
        a popup beforehand.
        """
        usr_answ = Messagebox.okcancel(
            message="Are you sure you want to delete that entry?",
            title="Attention!",
        )
        if usr_answ == "OK":
            status = self.db_service.delete(self.db_session, self.item_key_var.get())
            self.refresh()
            if status == 1:
                Notifications.show_info(
                    message="The entry has been deleted from the database."
                )
            elif status == 0:
                Notifications.show_error(message="The entry couldn't be deleted.")
            else:
                Notifications.show_error(message="There has been an unknown error!")


# -*- coding: utf-8 -*-

from itertools import zip_longest
import logging
import os
import re

from gsheets import GSheets

HXYDRA_SLATE_PROJECT_NICKNAME_RANGE = 'D:D'
HXYDRA_INTAKE_ASSIGNMENT_HEADER_RANGE = 'A12:H'

HXYDRA_INTAKE_HEADER_MAP = {
    'name': 'assignment_name',
    'instructions': 'instructions',
    'date': 'release_date',
    'link': 'link',
    'notes': 'extra_info',
    'type': 'type',
}
# unpack dict keys into a list; python>=3.5
HXYDRA_INTAKE_HEADER_KEYS = [*HXYDRA_INTAKE_HEADER_MAP]

HXYDRA_SLATE_HEADER_MAP = {
    'launch date': 'launch_date',
    'end date': 'end_date',
    'project nickname': 'course_nickname',
    'project title': 'course_title',
    'project lead': 'project_lead',
    'other team members': 'other_team_members',
    'duration _wks': 'duration_in_weeks',
    'platform': 'platform',
    'common project name': 'course_group_name',
    'approximate course id for courses on edx.org only': 'context_id',
    'public course url': 'public_edx_url',
}
HXYDRA_USERS_HEADER_MAP = {
    'row': 'row',
}

logger = logging.getLogger(__name__)

if __name__ == '__main__':
    """ main() works as usage example and live test.

    requires environment variables:
        GOOGLE_APPLICATION_CREDENTIALS: path to json key file
        SLATE_SHEET_ID: spreadsheet id
        INTAKE_SHEET_ID: spreadsheet id
    """
    import json

    credentials_path = os.getenv('GOOGLE_APPLICATION_CREDENTIALS', None)
    if credentials_path:
        credentials = GSheets.get_credentials(credentials_path)
        if credentials is None:
            print('no credentials; failing...')
            exit(1)
    else:
        print('missing GOOGLE_APPLICATION_CREDENTIALS')
        exit(1)

    #print(
    #    'project_id: {}; scopes: {}, token: {}'.format(
    #        credentials.project_id,
    #        credentials.scopes,
    #        credentials.token,
    #    )
    #)

    gsheets = GSheets(credentials)

    slate_sheet_id = os.getenv('SLATE_SHEET_ID', None)
    if slate_sheet_id:
        # range for all info in the slate
        slate_rows = gsheets.get_range(
            slate_sheet_id,
            'A:AF',
        )
    else:
        print('missing SLATE_SHEET_ID')
        exit(1)

    #print('slate results have ({}) rows'.format(len(slate_rows)))
    #print('{}'.format(json.dumps(slate_rows, indent=4)))
    #print('{}'.format(json.dumps(slate_rows[0], indent=4)))

    slate_items = GSheets.map_rows_into_dict(
        HXYDRA_SLATE_HEADER_MAP,
        slate_rows,
    )

    # cleanup a little
    for item in slate_items:
        nickname = item['course_nickname'].lower()
        course_group = nickname.split('_')[0]
        item['course_nickname'] = nickname
        item['course_group'] = course_group

    print('{}'.format(json.dumps(slate_items, indent=4)))

    '''
    intake_sheet_id = os.getenv('INTAKE_SHEET_ID', None)
    if intake_sheet_id:
        intake_assignment_rows = gsheets.get_range(
            intake_sheet_id,
            HXYDRA_INTAKE_ASSIGNMENT_HEADER_RANGE,
        )
    else:
        print('missing INTAKE_SHEET_ID')
        exit(1)

    intake = GSheets.map_rows_into_dict(HXYDRA_INTAKE_HEADER_MAP, intake_assignment_rows)

    print('INTAKE: {}'.format(json.dumps(intake, indent=4)))
    '''

    users_sheet_id = os.getenv('USERNAME_SHEET_ID', None)
    if not users_sheet_id:
        print('missing USERNAME_SHEET_ID')
        exit(1)

    users_row = gsheets.get_range(users_sheet_id, 'A:F')
    users_items = GSheets.map_rows_into_dict(
        HXYDRA_USERS_HEADER_MAP, users_row)

    print(json.dumps(users_items, indent=4))

    # range for usernames