"""
Decision Tree Classifier
CSCI 230 Final Project
Grace MacDonald, Warren Seeds, Carson Cooley, Petra Ilic, Sarah Martin
"""
import eda
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import GridSearchCV, validation_curve, StratifiedKFold
from sklearn.tree import DecisionTreeClassifier, export_text, plot_tree, export_graphviz, plot_tree
from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score, confusion_matrix, plot_confusion_matrix, classification_report, roc_auc_score, ConfusionMatrixDisplay
from sklearn.feature_selection import RFE

print('\n\n====================================== Beginning Decision Tree Classifier ============================================ \n\n')
df = eda.df
X_train = eda.X_train
X_test = eda.X_test
y_train = eda.y_train
y_test = eda.y_test
X_val = eda.X_val
y_val = eda.y_val

print('\n\n--------------------------- Addressing over and underfitting with validation curves ------------------------\n\n')

param_range = [1, 5, 10, 50, 100, 250, 500, 1000]
train_scores, test_scores = validation_curve(
                estimator=DecisionTreeClassifier(random_state=0),
                X=X_train,
                y=y_train,
                param_name='max_depth',
                param_range=param_range,
                cv=10)
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.plot(param_range, train_mean,
         color='blue', marker='o',
         markersize=5, label='Training accuracy')
plt.fill_between(param_range, train_mean + train_std,
                 train_mean - train_std, alpha=0.15,
                 color='blue')
plt.plot(param_range, test_mean,
         color='green', linestyle='--',
         marker='s', markersize=5,
         label='Validation accuracy')
plt.fill_between(param_range,
                 test_mean + test_std,
                 test_mean - test_std,
                 alpha=0.15, color='green')
plt.grid()
plt.xscale('log')
plt.legend(loc='lower right')
plt.xlabel('Parameter max_depth')
plt.ylabel('Accuracy')
plt.ylim([0.2, 1.03])
plt.tight_layout()
plt.show()

print('\n\n------------------------------------------- DTC Grid Search ------------------------------------------------ \n\n')

param_grid = [{'criterion': ['gini', 'entropy', 'log_loss'],
               'max_depth': [3,5,10,20,50,100,200],
               'min_samples_split': [2,6,10],
               'splitter': ['best', 'random'],
               'min_samples_leaf': [1,5,10],
               'min_weight_fraction_leaf': [0,0.2,0.35,0.5],
               'max_features': ['sqrt', 'log2']
               }]

gs = GridSearchCV(estimator=DecisionTreeClassifier(random_state=0),
                  param_grid=param_grid,
                  scoring='accuracy',
                  refit=True,
                  cv=10,
                  n_jobs=-1)

gs = gs.fit(X_val, y_val)
print(gs.best_score_)
print(gs.best_params_)
params = gs.best_params_

dtc = DecisionTreeClassifier(random_state=0, criterion=params['criterion'], max_depth=params['max_depth'],
                            max_features=params['max_features'],
                            min_samples_leaf=params['min_samples_leaf'], min_samples_split=params['min_samples_split'],
                            min_weight_fraction_leaf=params['min_weight_fraction_leaf'], splitter=params['splitter'])
dtc.fit(X_train, y_train)
y_pred = dtc.predict(X_test)

'''
print('\n\n------------------------------------------ Importance of Each Feature ----------------------------------------------- \n\n')
feature_importances = pd.DataFrame(dtc.feature_importances_,
                                index = X_train.columns)

print(feature_importances)
'''

print('\n\n------------------------------------------ Evaluating DTC ----------------------------------------------- \n\n')
k = 10
kf = StratifiedKFold(n_splits=k, shuffle=True, random_state=123)

X_kf = np.concatenate((X_train, X_test))
y_kf = np.concatenate((y_train, y_test))

predicted_targets = np.array([])
actual_targets = np.array([])

accuracies = []
precisions = []
recalls = []
f1s = []
roc_aucs = []
for train_index, test_index in kf.split(X_kf, y_kf):
    X_train_kf, X_test_kf = X_kf[train_index], X_kf[test_index]
    y_train_kf, y_test_kf = y_kf[train_index], y_kf[test_index]

    dtc.fit(X_train_kf, y_train_kf)
    y_pred_kf = dtc.predict(X_test_kf)

    predicted_targets = np.append(predicted_targets, y_pred_kf)
    actual_targets = np.append(actual_targets, y_test_kf)

    accuracies.append(dtc.score(X_test_kf, y_test_kf))
    precisions.append(precision_score(y_true=y_test_kf, y_pred=y_pred_kf))
    recalls.append(recall_score(y_true=y_test_kf, y_pred=y_pred_kf))
    f1s.append(f1_score(y_true=y_test_kf, y_pred=y_pred_kf))
    roc_aucs.append(roc_auc_score(y_test_kf, y_pred_kf))

cm_kf = confusion_matrix(y_true=actual_targets, y_pred=predicted_targets)
cm_kf_display = ConfusionMatrixDisplay(cm_kf)
cm_kf_display.from_predictions(y_true=actual_targets, y_pred=predicted_targets, cmap=plt.cm.Blues)
plt.title("10-Fold Cross Validation Confusion Matrix")
plt.show()

print('\n\n------------------------------------ 10-Fold Cross Validation Metrics ----------------------------------------- \n\n')

print("Accuracy: ", round(sum(accuracies)/k, 6))
print("Precision:", round(sum(precisions)/k, 6))
print("Recall:   ", round(sum(recalls)/k, 6))
print("F1:       ", round(sum(f1s)/k, 6))
print("ROC-AUC:  ", round(sum(roc_aucs)/k, 6))

print('\n\n')
print('Classification report:\n', classification_report(actual_targets, predicted_targets))

# print('\n\n----------------------------- Printing Graphical Representation of DTC ------------------------------------- \n\n')

# plot_tree(dtc, fontsize=5, feature_names=eda.X_train.columns)
# plt.tight_layout()
# plt.show()

# print('\n\n------------------------------------------ Feature Importance ----------------------------------------------- \n\n')

# feature_importance = pd.DataFrame(dtc.feature_importances_,
#                                 index = X_train.columns,
#                                 columns=['importance']).sort_values('importance',
#                                                                     ascending=False)

# print(feature_importance)

# feature_importance = feature_importance[feature_importance['importance'] != 0]

# ax = feature_importance.plot.bar(y='importance',rot=0)
# plt.show()

print('\n\n------------------------------------- Recursive Feature Elimination ------------------------------------- \n\n')

estimator = DecisionTreeClassifier(random_state=0, criterion=params['criterion'], max_depth=params['max_depth'],
                            max_features=params['max_features'],
                            min_samples_leaf=params['min_samples_leaf'], min_samples_split=params['min_samples_split'],
                            min_weight_fraction_leaf=params['min_weight_fraction_leaf'], splitter=params['splitter'])

selector = RFE(estimator, n_features_to_select=0.3, step=1)

predicted_targets_RFE = np.array([])
actual_targets_RFE = np.array([])

accuracies_RFE = []
precisions_RFE = []
recalls_RFE = []
f1s_RFE = []
roc_aucs_RFE = []
for train_index, test_index in kf.split(X_kf, y_kf):
    X_train_kf, X_test_kf = X_kf[train_index], X_kf[test_index]
    y_train_kf, y_test_kf = y_kf[train_index], y_kf[test_index]

    selector.fit(X_train_kf, y_train_kf)
    y_pred_kf = selector.predict(X_test_kf)

    predicted_targets_RFE = np.append(predicted_targets_RFE, y_pred_kf)
    actual_targets_RFE = np.append(actual_targets_RFE, y_test_kf)

    accuracies_RFE.append(round(accuracy_score(y_true=y_test_kf, y_pred=y_pred_kf), 6))
    precisions_RFE.append(round(precision_score(y_true=y_test_kf, y_pred=y_pred_kf), 6))
    recalls_RFE.append(round(recall_score(y_true=y_test_kf, y_pred=y_pred_kf), 6))
    f1s_RFE.append(round(f1_score(y_true=y_test_kf, y_pred=y_pred_kf), 6))
    roc_aucs_RFE.append(round(roc_auc_score(y_test_kf, y_pred_kf), 6))

cm_kf_RFE = confusion_matrix(y_true=actual_targets_RFE, y_pred=predicted_targets_RFE)
cm_kf_display_RFE = ConfusionMatrixDisplay(cm_kf_RFE)
cm_kf_display_RFE.from_predictions(y_true=actual_targets_RFE, y_pred=predicted_targets_RFE, cmap=plt.cm.Blues)
plt.title("10-Fold Cross Validation Confusion Matrix")
plt.show()

print('\n\n------------------------------------ 10-Fold Cross Validation Metrics with RFE ----------------------------------------- \n\n')

print("Accuracy: ", round(sum(accuracies_RFE)/len(accuracies_RFE), 6))
print("Precision:", round(sum(precisions_RFE)/len(precisions_RFE), 6))
print("Recall:   ", round(sum(recalls_RFE)/len(recalls_RFE), 6))
print("F1:       ", round(sum(f1s_RFE)/len(f1s_RFE), 6))
print("ROC-AUC:  ", round(sum(roc_aucs_RFE)/len(roc_aucs_RFE), 6))

print("\n\nAccuracy: ", accuracies_RFE)
print("Precision:", precisions_RFE)
print("Recall:   ", recalls_RFE)
print("F1:       ", f1s_RFE)
print("ROC-AUC:  ", roc_aucs_RFE)

print('\n\n')
print('Classification report:\n', classification_report(actual_targets_RFE, predicted_targets_RFE))


from __future__ import print_function
from combo_core.source_maintainer import *
from combo_core.importer import *
from combo_core.compat import connection_error
import requests
import json

MAX_RESPONSE_LENGTH = 4096

class ServerConnectionError(ComboException, connection_error):
    pass

class NackFromServer(ComboException):
    pass

class RemoteSourceLocator(SourceLocator):
    def __init__(self, address):
        self._addr = address
        self._url = 'http://' + ':'.join(str(x) for x in address)

    def _extended_url(self, *args):
        return '/'.join((self._url, ) + args)

    def get_source(self, project_name, version):
        req_url = self._extended_url('get_source')
        params = {'project_name': project_name, 'project_version': str(version)}

        try:
            response = requests.get(req_url, params=params)
        except BaseException as e:
            raise ServerConnectionError(
                'Could not get response for request to "{}" with params "{}"'.format(req_url, params), e)

        try:
            source = json.loads(response.content.decode())
        except BaseException as e:
            raise UndefinedProject('Server could not locate project "{}" version "{}"'.format(project_name, version), e)

        return source

    def all_sources(self):
        req_url = self._extended_url('get_available_versions')

        try:
            response = requests.get(req_url)
        except BaseException as e:
            raise ServerConnectionError('Could not get response for request to "{}"'.format(req_url), e)

        try:
            sources_dict = json.loads(response.content.decode())
        except BaseException as e:
            raise ServerConnectionError('Server did not return a list of the available versions', e)

        return sources_dict

class RemoteSourceMaintainer(RemoteSourceLocator, SourceMaintainer):
    def __init__(self, address):
        super(RemoteSourceMaintainer, self).__init__(address)

    def add_project(self, project_name, source_type=None):
        req_url = self._extended_url('add_project')

        data = {'project_name': project_name}
        if source_type:
            data['source_type'] = source_type

        try:
            response = requests.post(req_url, data=data)
            print('Server response: {}'.format(response.content))
        except BaseException as e:
            raise ServerConnectionError('Could not post new project {}'.format(project_name), e)

    def add_version(self, version_details, **kwargs):
        # kwargs is not relevant here, because it is not sent to the server anyway

        req_url = self._extended_url('add_version')
        data = {'version_details': json.dumps(version_details)}

        try:
            response = requests.post(req_url, data=data)
            print('Server response: {}'.format(response.content))
        except BaseException as e:
            raise ServerConnectionError('Could not post version located at {}'.format(version_details), e)

class RemoteImporter(Importer):
    def __init__(self, sources_locator):
        """
        Construct a dependencies importer which uses the combo server
        :param sources_locator: A RemoteSourceLocator object
        """
        if not isinstance(sources_locator, RemoteSourceLocator):
            raise UnhandledComboException(
                'Invalid sources locator for type for server importer: {}'.format(type(sources_locator)))
        super(RemoteImporter, self).__init__(sources_locator)

    def get_all_sources_map(self):
        return self._source_locator.all_sources()

    # TODO: Return this optimized implementation once the server has a real implementation of the all sources map
    # def get_dep_hash(self, dep):
    #     """
    #     :param dep: A combo dependency
    #     :return: The hash of the given dependency
    #     """
    #     # If already cached, return the cached hash
    #     if self._cached_data.has_dep(dep):
    #         return self._cached_data.get_hash(dep)
    #
    #     # Dependency is not cached, use the all sources json instead
    #     sources_map = self.get_all_sources_map()
    #     assert str(dep) in sources_map, 'Dependency "{}" not found on the remote sources map'.format(dep)
    #
    #     return sources_map[str(dep)]['hash']


"""Create's a harmony at a fixed diatonic interval for any midi file which does
not contain accidental notes.  If an acidental is encountered, the harmony will
be P8 or -P8."""

from __future__ import print_function
import music21 as m21
import os
import argparse

def cmp(a, b):
    return (a > b) - (a < b)

def get_parser():
    """Get parser for command line arguments."""
    parser = argparse.ArgumentParser(description="Harmonize-With-Midi")
    parser.add_argument(
        "input_midi",
        help="Input midi file."
    )
    parser.add_argument(
        'interval',
        type=int,
        help="The interval (use positive integer ascending and negative for "
             "descending).  "
             "Can be ..., -7, -6, -5, -4, -3, -2, 2, 3, 4, 5, 6,  or 7, ..."
    )
    parser.add_argument(
        "-k",
        "--key",
        help=('(Optional) The Key. Use capital for major, lowercase for minor, '
              'e.g. "C#" for C# Major and "c#" for C# Minor.  If no key is '
              'chosen, the code will attempt to key find the key automatically.'
              '\n\n'

              'Note: Midi files containing key changes are not currently '
              'supported.\n\n'

              'Note: non-diatonic notes will always be harmonized using P8.'),
        default=None,
    )
    parser.add_argument(
        "-o",
        "--output_midi",
        default=None,
        help="Name for output midi file.  Defaults to the name of you input "
             "file with '_harmony_<interval>' before the extension.  "
             "E.g. 'test.mid' could output 'test_harmony_3.mid'"
    )
    parser.add_argument(
        "-t",
        "--fix_timing",
        action='store_true',
        default=False,
        help="OK, so using `music21` to make this was maybe a bad choice.  "
             "Turns out there's automatic quantization.  This flag will up the "
             "`ticksPerQuarter` default in `music21` to 2^14, which should "
             "help prevent too much quantization. This is not a perfect fix."
    )
    parser.add_argument(
        "-f",
        "--fixed_octave",
        action='store_true',
        default=False,
        help="Use to force harmony to stay a one octave range."
    )
    return parser

def parentdir(path_, n=1):
    for i in range(n):
        path_ = os.path.dirname(path_)
    return path_

def sgn(x):
    return cmp(x, 0)

def nzint(interval_):
    """converts a diatonic degree/interval to be a unit with a natural 0.
    I.e. 1, -1 notes become 0; 2, -2 become 1, etc."""
    return sgn(interval_)*(abs(interval_) - 1)

def harmonize_pitch_in_place(p, interval, key, fix_octave=False):
    """Create's a harmony with pitch `p` with a distance determined by the
    input diatonic interval, `interval` and key.
    If p is not in the input key, the harmony will
    be P8 or -P8 (match the sign of `interval')."""
    key_notes = [kp.midi % 12 for kp in key.pitches]
    try:
        degree = key_notes.index(p.midi % 12) + 1
    except ValueError:
        p.transpose(sgn(interval)*12, inPlace=True)
        return
    new_degree = degree + nzint(interval)

    if fix_octave:
        octave_adjustment = 0
    else:
        octave_adjustment = nzint(new_degree)//6
    new_pitch = key.pitchFromDegree(new_degree)
    adjustment = (new_pitch.midi - p.midi) % 12 + 12*octave_adjustment
    # from pdb import set_trace; set_trace()
    p.transpose(adjustment, inPlace=True)

if __name__ == '__main__':

    # Parse input and user args
    args = get_parser().parse_args()

    if args.output_midi:
        output_file = args.output_midi
    else:
        name, ext = os.path.splitext(args.input_midi)
        output_file = name + '_' + 'harmony' + '_' + str(args.interval) + '.mid'

    if args.fix_timing:
        m21.defaults.ticksPerQuarter = 1024*16

    # read the midi into music21
    mf = m21.midi.MidiFile()
    mf.open(args.input_midi)
    mf.read()
    mf.close()
    score = m21.midi.translate.midiFileToStream(mf, quantizePost=False)
    #score = m21.converter.parse(args.input_midi)

    # figure out the key (if use didn't suggest it)
    if not args.key:
        try:
            key = score.analyze('key')
        except:
            raise Exception("Having difficulty determining key.  "
                            "Please specify a key.")
    else:
        key = m21.key.Key(args.key)
    print("Using key:", key)

    # Harmonize the score
    for part in score.parts:
        for p in part.pitches:
            print(p, end=' -> ')
            # print ''
            # score.show('t')
            harmonize_pitch_in_place(p, interval=args.interval,
                                        key=key,
                                        fix_octave=args.fixed_octave)
            # print ''
            # score.show('t')
            # print ''
            print(p)
            # raw_input()

    m = m21.midi.translate.streamToMidiFile(score)
    m.open(output_file, 'wb')
    m.write()
    m.close()


from bs4 import BeautifulSoup
import requests, openpyxl

wb = openpyxl.Workbook()
sheet = wb.active
sheet.title = "Top 250 Movies"
sheet.append(["Rank", "Movie Name", "Rating", "Year of release"])

try:
    response = requests.get("https://www.imdb.com/chart/top/")
    soup = BeautifulSoup(response.text, "html.parser")
    movie = soup.find("tbody", class_="lister-list").find_all("tr")

    for movies in movie:
        rank = movies.find("td", class_="titleColumn").get_text(strip=True).split(".")[0]
        movie_name = movies.find("td", class_ ="titleColumn").a.text
        rating = movies.find("td", class_="ratingColumn imdbRating").strong.text
        year = movies.find("td", class_="titleColumn").span.text.replace("("," ")
        year = year.replace(")"," ")
        sheet.append([rank, movie_name, rating, year])
except Exception as e:
    print(e)

wb.save(r"C:\Users\rknav\Downloads\Movie_list.xlsx")


import torch
import torch.nn as nn
import torch.nn.functional as F
import os
import os.path as osp
import sys
path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.append(path)

import clip
from tools.configer import Configer
from lib.get_dataloader import get_data_loader
import cv2
import numpy as np
import matplotlib.pyplot as plt
import clip
from PIL import Image
import lib.transform_cv2 as T
import torch.nn.functional as F
import random
import pickle

def get_img_for_everyclass(configer, dataset_id=None):
    n_datasets = configer.get("n_datasets")

    num_classes = []
    for i in range(1, n_datasets + 1):
        num_classes.append(configer.get("dataset" + str(i), "n_cats"))

    dls = get_data_loader(configer, aux_mode='ret_path', distributed=False)

    img_lists = []
    lb_lists  = []
    lb_info_list = []
    for i in range(0, n_datasets):
        this_img_lists = []
        this_lb_lists = []
        lb_info_list.append(dls[i].dataset.lb_map)

        if dataset_id != None and i != dataset_id:
            img_lists.append(this_img_lists)
            lb_lists.append(this_lb_lists)
            continue
        # print("cur dataset id： ", i)

        for label_id in range(0, num_classes[i]):
            this_img_lists.append([])
            this_lb_lists.append([])

        for im, lb, lbpth in dls[i]:
            im = im[0]
            lb = lb.squeeze()
            for label_id in range(0, num_classes[i]):
                if len(this_img_lists[label_id]) < 100 and (lb == label_id).any():
                    this_img_lists[label_id].append(im)
                    this_lb_lists[label_id].append(lbpth)

        for j, lb in enumerate(this_lb_lists):
            if len(lb) == 0:
                print("the number {} class has no image".format(j))

        img_lists.append(this_img_lists)
        lb_lists.append(this_lb_lists)

    return img_lists, lb_lists, lb_info_list

def get_img_for_everyclass_single(configer, dls):
    n_datasets = configer.get("n_datasets")

    num_classes = []
    for i in range(1, n_datasets + 1):
        num_classes.append(configer.get("dataset" + str(i), "n_cats"))

    # dls = get_data_loader(configer, aux_mode='ret_path', distributed=False)
    dl_iters = [iter(dl) for dl in dls]

    img_lists = []
    lb_lists  = []
    for i in range(0, n_datasets):
        # print("cur dataset id： ", i)
        this_img_lists = []
        this_lb_lists = []
        for label_id in range(0, num_classes[i]):
            this_img_lists.append([])
            this_lb_lists.append([])

        cur_num = 0
        while True:
            try:
                im, lb, lbpth = next(dl_iters[i])
                while torch.min(lb) == 255:
                    im, lb, lbpth = next(dl_iters[i])

                if not len(im) == 1:
                    raise StopIteration
            except StopIteration:
                break

            im = im[0]
            lb = lb.squeeze()
            for label_id in range(0, num_classes[i]):
                if (lb == label_id).any():
                    this_img_lists[label_id].append(im)
                    this_lb_lists[label_id].append(lbpth)

            # print('cur_num: ', cur_num)
            # print('num_classes: ', num_classes[i])
            # print(lbpth)
            # if cur_num == num_classes[i]:
            #     break

        for j, lb in enumerate(this_lb_lists):
            if len(lb) == 0:
                print("the number {} class has no image".format(j))

        img_lists.append(this_img_lists)
        lb_lists.append(this_lb_lists)

    return img_lists, lb_lists

def crop_image_by_label_value(img, label, label_value):
    # 将标签二值化
    binary = np.zeros_like(label, dtype=np.uint8)
    binary[label == label_value] = 255

    binary = cv2.convertScaleAbs(binary)

    # 执行闭运算操作
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
    closed = cv2.morphologyEx(binary, cv2.MORPH_CLOSE, kernel)

    # 计算轮廓
    contours, _ = cv2.findContours(closed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

#     # 找到覆盖所有轮廓的最小矩形
#     max_rect = cv2.minAreaRect(np.concatenate(contours))

#     # 获取最大包围盒的坐标
#     x, y, w, h = cv2.boundingRect(np.int0(cv2.boxPoints(max_rect)))
#     print(x, y, w, h)

    # 计算每个包围盒的面积并找到面积最大的包围盒
    max_area = 0
    max_bbox = None
    for contour in contours:
        bbox = cv2.boundingRect(contour)
        area = bbox[2] * bbox[3]
        if area > max_area:
            max_area = area
            max_bbox = bbox

    # 如果没有找到任何包围盒，返回空图像
    if max_bbox is None:
        return np.zeros_like(img)

    # 裁剪图像
    x, y, w, h = max_bbox

#     # 获取包围盒
#     x, y, w, h = cv2.boundingRect(contours[4])
#     print(x, y, w, h)

    # 裁剪图像
    # print(img.shape)
    cropped = img[y:y+h, x:x+w, :]

    # 将不属于该标签的像素点替换为指定的值
    label_roi = binary[y:y+h, x:x+w]

    k = np.ones((5, 5), np.uint8)
    dilated = cv2.dilate(label_roi, kernel=k)

    mask = np.ones_like(cropped, dtype=bool)
    mask[dilated != 255] = False
    cropped[~mask] = 128

    h, w, _ = cropped.shape
    if h < w:
        top_padding = (w - h) // 2
        bottom_padding = w - h - top_padding
        cropped = cv2.copyMakeBorder(cropped, top_padding, bottom_padding, 0, 0, cv2.BORDER_CONSTANT, value=[128, 128, 128])
    elif h > w:
        left_padding = (h - w) // 2
        right_padding = h - w - left_padding
        cropped = cv2.copyMakeBorder(cropped, 0, 0, left_padding, right_padding, cv2.BORDER_CONSTANT, value=[128, 128, 128])

    # 返回裁剪后的图像
    return cropped

def gen_image_features(configer, dataset_id=None):
    img_lists, lb_lists, lb_info_list = get_img_for_everyclass(configer, dataset_id)

    n_datasets = configer.get('n_datasets')
    to_tensor = T.ToTensor(
                mean=(0.48145466, 0.4578275, 0.40821073), # clip , rgb
                std=(0.26862954, 0.26130258, 0.27577711),
            )
    device = "cuda" if torch.cuda.is_available() else "cpu"
    model, _ = clip.load("ViT-B/32", device=device)
    with torch.no_grad():
        out_features = []
        for idx in range(0, n_datasets):
            this_label_info = lb_info_list[idx]
            if dataset_id != None and idx != dataset_id:
                continue
            print("dataset_id: ", idx)
            for i, im_lb_list in enumerate(zip(img_lists[idx], lb_lists[idx])):
                im_list, lb_list = im_lb_list
                if len(im_list) == 0:
                    print("why dataset_id: ", idx)
                    continue
                image_features_list = []
                for im_path, lb_path in zip(im_list, lb_list):
                    image = cv2.imread(im_path)
                    # print(lb_path[0])
                    lb = cv2.imread(lb_path[0], 0)
                    lb = this_label_info[lb]
                    if image is None:
                        print(im_path)
                        continue
                    # lb = lb.numpy()
                    cropped_img = crop_image_by_label_value(image, lb, i)

                    im_lb = dict(im=cropped_img, lb=lb)
                    im_lb = to_tensor(im_lb)
                    img = im_lb['im'].cuda()
                    img = F.interpolate(img.unsqueeze(0), size=(224, 224))
                    image_features = model.encode_image(img).type(torch.float32)
                    image_features_list.append(image_features)

                # print("im_lb_list: ", im_lb_list)
                img_feat = torch.cat(image_features_list, dim=0)
                mean_feats = torch.mean(img_feat, dim=0, keepdim=True)
                # print(mean_feats.shape)
                out_features.append(mean_feats)

    return out_features

def gen_image_features_single(configer, dls, gen_feature=False):
    if gen_feature is False:
        img_lists, lb_lists = get_img_for_everyclass_single(configer, dls)
    else:
        img_lists, lb_lists = dls

    n_datasets = configer.get('n_datasets')
    to_tensor = T.ToTensor(
                mean=(0.48145466, 0.4578275, 0.40821073), # clip , rgb
                std=(0.26862954, 0.26130258, 0.27577711),
            )
    device = "cuda" if torch.cuda.is_available() else "cpu"
    model, _ = clip.load("ViT-B/32", device=device)
    with torch.no_grad():
        out_features = []
        for dataset_id in range(0, n_datasets):
            print("dataset_id: ", dataset_id)
            for i, im_lb_list in enumerate(zip(img_lists[dataset_id], lb_lists[dataset_id])):
                im_list, lb_list = im_lb_list
                im_lb_zip_list = list(zip(im_list, lb_list))
                if len(im_list) == 0:
                    print("why dataset_id: ", dataset_id)
                    continue
                image_features_list = []
                im_path, lb_path = random.choice(im_lb_zip_list)
                image = cv2.imread(im_path)
                # print(lb_path[0])
                lb = cv2.imread(lb_path[0], 0)
                if image is None:
                    print(im_path)
                    continue
                # lb = lb.numpy()
                cropped_img = crop_image_by_label_value(image, lb, i)

                im_lb = dict(im=cropped_img, lb=lb)
                im_lb = to_tensor(im_lb)
                img = im_lb['im'].cuda()
                img = F.interpolate(img.unsqueeze(0), size=(224, 224))
                image_features = model.encode_image(img).type(torch.float32)
                image_features_list.append(image_features)

                # print("im_lb_list: ", im_lb_list)
                img_feat = torch.cat(image_features_list, dim=0)
                mean_feats = torch.mean(img_feat, dim=0, keepdim=True)
                # print(mean_feats.shape)
                out_features.append(mean_feats)

    return out_features, [img_lists, lb_lists]

def gen_image_features_storage(configer, dataset_id):
    img_lists, lb_lists = get_img_for_everyclass(configer, dataset_id)

    n_datasets = configer.get('n_datasets')
    to_tensor = T.ToTensor(
                mean=(0.48145466, 0.4578275, 0.40821073), # clip , rgb
                std=(0.26862954, 0.26130258, 0.27577711),
            )
    device = "cuda" if torch.cuda.is_available() else "cpu"
    model, _ = clip.load("ViT-B/32", device=device)
    with torch.no_grad():
        this_datasets_feats = []
        print("dataset_id: ", dataset_id)
        for i, im_lb_list in enumerate(zip(img_lists[dataset_id], lb_lists[dataset_id])):
            im_list, lb_list = im_lb_list
            if len(im_list) == 0:
                print("why dataset_id: ", dataset_id)
                continue
            image_features_list = []
            for im_path, lb_path in zip(im_list, lb_list):
                image = cv2.imread(im_path)
                # print(lb_path[0])
                lb = cv2.imread(lb_path[0], 0)
                if image is None:
                    print(im_path)
                    continue
                # lb = lb.numpy()
                cropped_img = crop_image_by_label_value(image, lb, i)

                im_lb = dict(im=cropped_img, lb=lb)
                im_lb = to_tensor(im_lb)
                img = im_lb['im'].cuda()
                img = F.interpolate(img.unsqueeze(0), size=(224, 224))
                image_features = model.encode_image(img).type(torch.float32)
                image_features_list.append(image_features)

            # print("im_lb_list: ", im_lb_list)
            img_feat = torch.cat(image_features_list, dim=0)
            # mean_feats = torch.mean(img_feat, dim=0, keepdim=True)
            # print(mean_feats.shape)
            this_datasets_feats.append(img_feat)
    # out_features = torch.cat(out_features, dim=0)
    return this_datasets_feats

def get_encode_lb_vec(configer, datasets_id=None):
    n_datasets = configer.get('n_datasets')
    text_feature_vecs = []
    with torch.no_grad():
        clip_model, _ = clip.load("ViT-B/32", device="cuda")
        for i in range(0, n_datasets):
            if datasets_id != None and i != datasets_id:
                continue
            lb_name = configer.get("dataset"+str(i+1), "label_names")
            lb_name = ["a photo of " + name + "." for name in lb_name]
            text = clip.tokenize(lb_name).cuda()
            text_features = clip_model.encode_text(text).type(torch.float32)
            text_feature_vecs.append(text_features)

    return text_feature_vecs

def gen_graph_node_feature(configer):
    n_datasets = configer.get("n_datasets")

    if not osp.exists(configer.get('res_save_pth')): os.makedirs(configer.get('res_save_pth'))

    file_name = configer.get('res_save_pth') + 'graph_node_features'
    dataset_names = []
    for i in range(0, configer.get('n_datasets')):
        # file_name += '_'+str(configer.get('dataset'+str(i+1), 'data_reader'))
        dataset_names.append(str(configer.get('dataset'+str(i+1), 'data_reader')))

    # file_name += '.pt'
    out_features = []
    for i in range(0, n_datasets):
        this_file_name = file_name + f'_{dataset_names[i]}.pt'
        if osp.exists(this_file_name):
            this_graph_node_features = torch.load(this_file_name, map_location='cpu')

            out_features.append(this_graph_node_features)
        else:
            print(f'gen_graph_node_featuer: {i}')
            img_feature_vecs = gen_image_features(configer, i)
            img_feat_tensor = torch.cat(img_feature_vecs, dim=0)

            text_feature_vecs = get_encode_lb_vec(configer, i)[0]
            this_graph_node_features = torch.cat([text_feature_vecs, img_feat_tensor], dim=1)

            print("gen finished")
            torch.save(this_graph_node_features.clone(), this_file_name)

            out_features.append(this_graph_node_features)

    out_features = torch.cat(out_features, dim=0)
    print(out_features.shape)
    return out_features

    # if not osp.exists(configer.get('res_save_pth')): os.makedirs(configer.get('res_save_pth'))

    # file_name = configer.get('res_save_pth') + 'graph_node_features'+str(configer.get('n_datasets'))
    # for i in range(0, configer.get('n_datasets')):
    #     file_name += '_'+str(configer.get('dataset'+str(i+1), 'data_reader'))

    # file_name += '.pt'

    # if osp.exists(file_name):
    #     graph_node_features = torch.load(file_name)
    # else:
    #     print("gen_graph_node_feature")
    #     text_feature_vecs = get_encode_lb_vec(configer)
    #     text_feat_tensor = torch.cat(text_feature_vecs, dim=0)
    #     print(text_feat_tensor.shape)
    #     print("gen_text_feature_vecs")
    #     img_feature_vecs = gen_image_features(configer)
    #     img_feat_tensor = torch.cat(img_feature_vecs, dim=0)
    #     print(img_feat_tensor.shape)
    #     print("gen_img_features")
    #     graph_node_features = torch.cat([text_feat_tensor, img_feat_tensor], dim=1)
    #     # graph_node_features = (text_feat_tensor+img_feat_tensor)/2
    #     print(graph_node_features.shape)
    #     torch.save(graph_node_features.clone(), file_name)

    # return graph_node_features

def gen_graph_node_feature_single(configer, img_feature_vecs, text_feat_tensor=None):
    if text_feat_tensor is None:
        text_feature_vecs = get_encode_lb_vec(configer)
        text_feat_tensor = torch.cat(text_feature_vecs, dim=0)

    # img_feature_vecs, lbpth_list = gen_image_features_single(configer, dls, gen_feature)
    n_datasets = configer.get("n_datasets")
    img_feat_tensor = []
    for i in range(0, n_datasets):
        n_cats = configer.get(f'dataset{i+1}', 'n_cats')
        for j in range(0, n_cats):
            num_samples, dim = img_feature_vecs[i][j].shape
            if num_samples == 1:
                img_feat_tensor.append(img_feature_vecs[i][j])
                continue

            choice_samples = int(num_samples/2)

            # print(f'the shape of dataset{i}: ', img_feature_vecs[i][j].shape)
            random_img_sample = random.sample(list(img_feature_vecs[i][j]), choice_samples)
            avg_img_feat = torch.mean(torch.cat(random_img_sample).view(-1, dim), dim=0)
            img_feat_tensor.append(avg_img_feat[None])

    img_feat_tensor = torch.cat(img_feat_tensor, dim=0)

    # graph_node_features = torch.cat([text_feat_tensor, img_feat_tensor], dim=1)
    graph_node_features = (text_feat_tensor+img_feat_tensor)/2

    return graph_node_features, text_feat_tensor

def gen_graph_node_feature_storage(configer):

    # text_feature_vecs = get_encode_lb_vec(configer)
    # text_feat_tensor = torch.cat(text_feature_vecs, dim=0)
    n_datasets = configer.get("n_datasets")

    if not osp.exists(configer.get('res_save_pth')): os.makedirs(configer.get('res_save_pth'))

    file_name = configer.get('res_save_pth') + 'img_feature_vecs'+str(configer.get('n_datasets'))
    dataset_names = []
    for i in range(0, configer.get('n_datasets')):
        # file_name += '_'+str(configer.get('dataset'+str(i+1), 'data_reader'))
        dataset_names.append(str(configer.get('dataset'+str(i+1), 'data_reader')))

    # file_name += '.pt'
    out_features = []
    for i in range(0, n_datasets):
        this_file_name = file_name + f'_{dataset_names[i]}.pkl'
        if osp.exists(this_file_name):
            # img_feature_vecs = torch.load(this_file_name)
            with open(this_file_name, 'rb') as file:
                img_feature_vecs = pickle.load(file)

            out_features.append(img_feature_vecs)
        else:
            print("gen_img_feature_vecs")
            img_feature_vecs = gen_image_features_storage(configer, i)
            print("gen finished")
            with open(this_file_name, 'wb') as file:
                pickle.dump(img_feature_vecs, file)
            # _ = [torch.save(img_feature_vecs.clone(), file_name + f'_dataset{idx}.pt' ) for idx, img_feature_vecs in enumerate(out_features)]
            out_features.append(img_feature_vecs)

    return out_features

if __name__ == "__main__":
    configer = Configer(configs="configs/ltbgnn_5_datasets.json")
    img_feature_vecs = gen_graph_node_feature_storage(configer)
    # print(img_feature_vecs[0][0])
    # print(img_feature_vecs)

    # print(graph_node_features.shape)
    # norm_adj_feat = F.normalize(graph_node_features, p=2, dim=1)
    # similar_matrix = torch.einsum('nc, mc -> nm', norm_adj_feat, norm_adj_feat)
    # print("similar_matrix_max:", torch.max(similar_matrix))
    # print("similar_matrix_min:", torch.min(similar_matrix))
    # torch.set_printoptions(profile="full")
    # print(similar_matrix)


import numpy as np
import pandas as pd
import matplotlib as mpl
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
import seaborn as sns
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings("ignore")
import sys
from pydotplus import graph_from_dot_data
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_auc_score
import os
os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/'
#%%-----------------------------------------------------------------------
#Libraries to display decision tree
from sklearn.tree import export_graphviz
import webbrowser
#%%-----------------------------------------------------------------------
data = pd.read_csv('train_users_final.csv')
label = data.loc[:,'country_destination']
X_data = data.drop(['country_destination'], axis=1)
X = X_data.drop(['id'],axis=1)

test_data = pd.read_csv('test_users_final.csv')
id_test = test_data['id']
test = test_data.drop(['id'], axis=1)

class_le = LabelEncoder()
# fit and transform the class
y = class_le.fit_transform(label)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=100)
#%%-----------------------------------------------------------------------
# perform training with gini Index.
# creating the classifier object
clf_gini = DecisionTreeClassifier(criterion="gini", random_state=100, max_depth=7, min_samples_leaf=50)

# performing training
clf_gini.fit(X_train, y_train)
#%%-----------------------------------------------------------------------

# perform training with entropy.
# Decision tree with entropy
clf_entropy = DecisionTreeClassifier(criterion="entropy", random_state=100, max_depth=7, min_samples_leaf=50)

# Performing training
clf_entropy.fit(X_train, y_train)

#%%-----------------------------------------------------------------------
# make predictions
# predicton on test using gini
y_pred_gini = clf_gini.predict(X_test)
# predicton on test using entropy
y_pred_entropy = clf_entropy.predict(X_test)
#%%-----------------------------------------------------------------------
# calculate metrics gini model
print("\n")
print("Results Using Gini Index: \n")
print("Classification Report: ")
print(classification_report(y_test,y_pred_gini))
print("\n")
print("Accuracy : ", accuracy_score(y_test, y_pred_gini) * 100)
print("\n")
print ('-'*80 + '\n')
# calculate metrics entropy model
print("\n")
print("Results Using Entropy: \n")
print("Classification Report: ")
print(classification_report(y_test,y_pred_entropy))
print("\n")
print("Accuracy : ", accuracy_score(y_test, y_pred_entropy) * 100)
print ('-'*80 + '\n')
#%%-----------------------------------------------------------------------
# confusion matrix for gini model
conf_matrix = confusion_matrix(y_test, y_pred_gini)
class_names = data.country_destination.unique()
df_cm = pd.DataFrame(conf_matrix, index=class_names, columns=class_names )

plt.figure(figsize=(5,5))
hm = sns.heatmap(df_cm, cbar=False, annot=True, square=True, fmt='d', annot_kws={'size': 10}, yticklabels=df_cm.columns, xticklabels=df_cm.columns)
hm.yaxis.set_ticklabels(hm.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=20)
hm.xaxis.set_ticklabels(hm.xaxis.get_ticklabels(), rotation=0, ha='right', fontsize=20)
plt.ylabel('True label',fontsize=20)
plt.xlabel('Predicted label',fontsize=20)
plt.tight_layout()
plt.show()
#%%-----------------------------------------------------------------------
# confusion matrix for entropy model
conf_matrix = confusion_matrix(y_test, y_pred_entropy)
class_names = data.country_destination.unique()
df_cm = pd.DataFrame(conf_matrix, index=class_names, columns=class_names )

plt.figure(figsize=(5,5))
hm = sns.heatmap(df_cm, cbar=False, annot=True, square=True, fmt='d', annot_kws={'size': 10}, yticklabels=df_cm.columns, xticklabels=df_cm.columns)
hm.yaxis.set_ticklabels(hm.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=20)
hm.xaxis.set_ticklabels(hm.xaxis.get_ticklabels(), rotation=0, ha='right', fontsize=20)
plt.ylabel('True label',fontsize=20)
plt.xlabel('Predicted label',fontsize=20)
plt.tight_layout()
plt.show()

#Feature importance for decision tree
feature_importance = clf_gini.feature_importances_
feature_importance = 100*(feature_importance / feature_importance.max())
sorted_idx = np.argsort(feature_importance)
pos = np.arange(sorted_idx.shape[0])+.5
plt.subplot(1,2,2)
plt.barh(pos,feature_importance[sorted_idx],align='center')
plt.yticks(pos,data.columns[sorted_idx])
plt.ylim(150,169)
plt.xlabel('Relative Importance')
plt.title('Feature Importance',fontsize=15)
plt.show()
#%%-----------------------------------------------------------------------
# display decision tree
dot_data = export_graphviz(clf_gini, filled=True, rounded=True, class_names=class_names, feature_names=X.iloc[:, :].columns, out_file=None)
graph = graph_from_dot_data(dot_data)
graph.write_pdf("decision_tree_gini_1.pdf")
webbrowser.open_new(r'decision_tree_gini_1.pdf')

#%%-----------------------------------------------------------------------

dot_data = export_graphviz(clf_entropy, filled=True, rounded=True, class_names=class_names, feature_names=X.iloc[:, :].columns, out_file=None)

graph = graph_from_dot_data(dot_data)
graph.write_pdf("decision_tree_entropy_1.pdf")
webbrowser.open_new(r'decision_tree_entropy_1.pdf')

# make prediction with gini
y_pred_gini = clf_gini.predict(test)
y_pred_score_gini = clf_gini.predict_proba(test)

ids = []
cts = []
for i in range(len(id_test)):
    idx = id_test[i]
    ids += [idx]*5
    cts += class_le.inverse_transform(np.argsort(y_pred_score_gini[i])[::-1])[:5].tolist()

sub_dt = pd.DataFrame(np.column_stack((ids,cts)),columns=['id','country'])
sub_dt.to_csv('sub_dt.csv',index=False)
#make prediction with entropy
y_pred_entropy = clf_entropy.predict(test)
y_pred_score_entropy = clf_entropy.predict_proba(test)
ids = []
cts = []
for i in range(len(id_test)):
    idx = id_test[i]
    ids += [idx]*5
    cts += class_le.inverse_transform(np.argsort(y_pred_score_entropy[i])[::-1])[:5].tolist()

sub_dt_en = pd.DataFrame(np.column_stack((ids,cts)),columns=['id','country'])
sub_dt_en.to_csv('sub_dt_en.csv',index=False)

#%%------------------------------Random Forest-----------------------------------------
X_train=X_train.values[:,:]
X_test =X_test.values[:,:]
clf = RandomForestClassifier(n_estimators=100)

# perform training
clf.fit(X_train, y_train)
# get feature importances
importances = clf.feature_importances_

# convert the importances into one-dimensional 1darray with corresponding df column names as axis labels
f_importances = pd.Series(importances, X.columns)

# sort the array in descending order of the importances
f_importances.sort_values(ascending=False, inplace=True)

# make the bar Plot from f_importances
f_importances.plot(x='Features', y='Importance', kind='bar', figsize=(16, 9), rot= 90, fontsize=15)
plt.title('Feature Importance',fontsize=20,fontname="Times New Roman Bold")
plt.xlim(-0.5,19.5)
# show the plot
plt.tight_layout()
plt.show()

#%%-----------------------------------------------------------------------
#select features to perform training with random forest with k columns
# select the training dataset on k-features
newX_train = X_train[:, clf.feature_importances_.argsort()[::-1][:15]]

# select the testing dataset on k-features
newX_test = X_test[:, clf.feature_importances_.argsort()[::-1][:15]]

#%%-----------------------------------------------------------------------
#perform training with random forest with k columns
# specify random forest classifier
clf_k_features = RandomForestClassifier(n_estimators=100)

# train the model
clf_k_features.fit(newX_train, y_train)
#%%-----------------------------------------------------------------------
#accuracy of random forest
#accuracy of all features
y_pred = clf.predict(X_test)
print("\n")
print("Results Using All Features: \n")

print("Classification Report: ")
print(classification_report(y_test,y_pred))
print("\n")

print("Accuracy : ", accuracy_score(y_test, y_pred) * 100)
print("\n")

#accuracy of K features
y_pred_k_features = clf_k_features.predict(newX_test)
print("\n")
print("Results Using K features: \n")
print("Classification Report: ")
print(classification_report(y_test,y_pred_k_features))
print("\n")
print("Accuracy : ", accuracy_score(y_test, y_pred_k_features) * 100)
# %%-----------------------------------------------------------------------
# confusion matrix for all features
conf_matrix = confusion_matrix(y_test, y_pred)
class_names = data['country_destination'].unique()

df_cm = pd.DataFrame(conf_matrix, index=class_names, columns=class_names )

plt.figure(figsize=(5,5))

hm = sns.heatmap(df_cm, cbar=False, annot=True, square=True, fmt='d', annot_kws={'size': 20}, yticklabels=df_cm.columns, xticklabels=df_cm.columns)

hm.yaxis.set_ticklabels(hm.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=20)
hm.xaxis.set_ticklabels(hm.xaxis.get_ticklabels(), rotation=0, ha='right', fontsize=20)
plt.ylabel('True label',fontsize=20)
plt.xlabel('Predicted label',fontsize=20)
# Show heat map
plt.tight_layout()
# %%-----------------------------------------------------------------------
# predicton on test using all features
y_pred_rf = clf.predict(test)
y_pred_score1_rf = clf.predict_proba(test)
ids = []
cts = []
for i in range(len(id_test)):
    idx = id_test[i]
    ids += [idx]*5
    cts += class_le.inverse_transform(np.argsort(y_pred_score1_rf[i])[::-1])[:5].tolist()

sub_dt = pd.DataFrame(np.column_stack((ids,cts)),columns=['id','country'])
sub_dt.to_csv('sub_rf.csv',index=False)

print ('-'*40 + 'End Console' + '-'*40 + '\n')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Jul  9 04:41:00 2017

@author: Shankar
"""

from stateequation import stateupdateequation, computevoltageforthreshold,getinput
from RULPredictor import calculateRUL
from Parameters import params
import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import norm
from math import sqrt

def convertoriginaltostd(X,Means,StdDev):

    U=[0,0,0,0,0]
    for i in range(0,len(X)):
        U[i]=(X[i]-Means[i])/StdDev[i]

    return U

def convertstdtooriginal(U,Means,StdDev):

    X=[0,0,0,0,0]
    for i in range(0,len(U)):
        X[i]=U[i]*StdDev[i]+Means[i]

    return X

Prm=params()
Means=[Prm.Init0,Prm.Init1,Prm.Init2,Prm.Init3,2]
StdDev=[Prm.Init0*0.1,Prm.Init1*0.1,0.1,0.1,0.2]

# What is the probability that the RUL is less than given number
RUL_Upper_Limit = 3286;
# P(RUL<given number) = ?
# we need to get to the curve RUL=RUL_Upper_Limit
# Every point on this curve has a probability of occurrence, we find that point with maximum probability

derivative=[0,0,0,0,0]
alpha=[0,0,0,0,0] #gradient in the standard normal space

Old_U=[0,0,0,0,0]

continue_counter=1

while(continue_counter==1 or continue_counter=='1' ):
    Old_X=convertstdtooriginal(U=Old_U,Means=Means,StdDev=StdDev)
    Gfunction=calculateRUL(Old_X,Prm)
    perturb=0.01

    for i in range(0,len(Old_X)):

        argument=Old_X
        argument[i]=argument[i]+perturb
        perturbGfunction=calculateRUL(argument,Prm)
        derivative[i]=(perturbGfunction-Gfunction)/perturb
        alpha[i]=derivative[i]*StdDev[i]

    normvalue=np.linalg.norm(np.array(alpha))

    Array_OldU=np.array(Old_U)
    Array_alpha=np.array(alpha)
    New_U=(np.dot(Array_OldU,Array_alpha)-Gfunction+RUL_Upper_Limit)*Array_alpha/(normvalue*normvalue)
    Old_U=New_U
    print(Gfunction)
    print(Old_X)
    continue_counter=input("Would you like to continue (if so, say 1)?")

Probability_Value=norm.cdf(-sqrt(np.dot(Old_U,Old_U)))
print("Final Probability Value")
print(Probability_Value)


# pip install paho-mqtt
# https://test.mosquitto.org/ssl/mosquitto.org.crt

import os, sys, time, ssl
from os.path import join
from datetime import datetime
import paho.mqtt.publish as publish
import paho.mqtt.client as mqtt

DIR  = os.path.dirname(os.path.realpath(__file__))
HOST = "test.mosquitto.org"
PORT = 8883

TLS = {
    'ca_certs' : join(DIR, "mosquitto.org.crt"),
    'tls_version' : ssl.PROTOCOL_TLSv1_2
}

print ("---BEGIN---")
publish.single( 'wizio',
    payload = "Hello world:" + str(datetime.now()),
    hostname = HOST, port = PORT,
    protocol  = mqtt.MQTTv311,
    client_id = "CLIENT_ID_" + str( time.time() ),
    tls = TLS
)
print ("----END----")


#

import pybullet as p
import time

p.connect(p.GUI)

p.setAdditionalSearchPath("../URDF_model")

p.createCollisionShape(p.GEOM_PLANE)
p.createMultiBody(0, 0)

# sphereRadius = 0.05
# colSphereId = p.createCollisionShape(p.GEOM_SPHERE, radius=sphereRadius)
# colBoxId = p.createCollisionShape(p.GEOM_BOX,
#                                   halfExtents=[sphereRadius, sphereRadius, sphereRadius])

# mass = 1
# visualShapeId = -1

cubeStartPos = [0,0,1]
cubeStartOrientation = p.getQuaternionFromEuler([0,0,0])
boxId = p.loadURDF("single_body.urdf",cubeStartPos, cubeStartOrientation)

# link_Masses = [1]
# linkCollisionShapeIndices = [colBoxId]
# linkVisualShapeIndices = [-1]
# linkPositions = [[0, 0, 0.11]]
# linkOrientations = [[0, 0, 0, 1]]
# linkInertialFramePositions = [[0, 0, 0]]
# linkInertialFrameOrientations = [[0, 0, 0, 1]]
# indices = [0]
# jointTypes = [p.JOINT_REVOLUTE]
# axis = [[0, 0, 1]]

# basePosition = [0,0,0.5]
# baseOrientation = [0, 0, 0, 1]

# sphereUid = p.createMultiBody(mass, colSphereId, visualShapeId, basePosition,
#                                       baseOrientation)
# p.changeDynamics(sphereUid,
#                        -1,
#                        spinningFriction=0.001,
#                        rollingFriction=0.001,
#                        linearDamping=0.0)

p.setGravity(0, 0, -10)
p.setRealTimeSimulation(1)

# p.getNumJoints(sphereUid)
# for i in range(p.getNumJoints(sphereUid)):
#   p.getJointInfo(sphereUid, i)

for i in range (10000):
    p.stepSimulation()
    time.sleep(1./240.)
# cubePos, cubeOrn = p.getBasePositionAndOrientation(boxId)
# print(cubePos,cubeOrn)
p.disconnect()


import pytest
import sqlite3

from ..src import initialization
from ..src import db as data

from .test_database.fixtures import TEST_DB_CONN_STRING, drop_it_all_sql

@pytest.fixture
def test_db_teardown():
    """
    A fixture to tear down the test database which must necessarily
    be created in order to test the cold start functionality.
    """
    # No set up; yield immediately to the test
    yield

    # Spin up a connection and execute the drop_it_all
    # script from the test_database package on the
    # designated test database file.
    conn = sqlite3.connect(TEST_DB_CONN_STRING)
    cursor = conn.cursor()
    cursor.executescript(drop_it_all_sql)
    conn.commit()
    conn.close()

class TestInitialization:
    def test_cold_start(self, test_db_teardown):
        # Run the cold start script
        initialization.execute_cold_start(TEST_DB_CONN_STRING)

        # Get a connection to the provided test DB
        conn = sqlite3.connect(TEST_DB_CONN_STRING)

        # Assure each of the tables exists and is non-empty
        assert data.wars.select_latest_war(conn) is not None

        assert len(data.maps.select_latest_maps(conn)) > 0
        maps: initialization.Map = data.maps.select_latest_maps(conn)[0]

        assert len(data.labels.select_latest_labels_for_map(maps[0], conn)) > 0
        assert len(data.icons.get_latest_icons_for_map(maps[0], conn)) > 0


from pyatlan.cache.custom_metadata_cache import CustomMetadataCache
from pyatlan.cache.enum_cache import EnumCache
from pyatlan.client.atlan import AtlanClient
from pyatlan.error import NotFoundError
from pyatlan.model.assets import Badge
from pyatlan.model.enums import (
    AtlanCustomAttributePrimitiveType,
    BadgeComparisonOperator,
    BadgeConditionColor,
)
from pyatlan.model.structs import BadgeCondition
from pyatlan.model.typedef import AttributeDef, CustomMetadataDef, EnumDef
from pyatlan.utils import get_logger

ENUM_NAME = "QDRating"
CUSTOM_METADATA_NAME = "Quality Data"

client = AtlanClient()
logger = get_logger(level="INFO")

def create_custom_metadata_options():
    """
    Create custom metadata options (enumeration), if it does not yet exist.
    """
    if EnumCache.get_by_name(ENUM_NAME):
        logger.info(f"{ENUM_NAME} enumeration has already been created.")
        return

    enum_def = EnumDef.create(name=ENUM_NAME, values=["Poor", "OK", "Great"])
    response = client.create_typedef(enum_def)
    assert response  # noqa: S101
    logger.info(f"{ENUM_NAME} enumeration created.")

def create_custom_metadata_structure():
    """
    Create a logo-branded custom metadata structure, if it does not yet exist.
    """
    try:
        CustomMetadataCache.get_id_for_name(CUSTOM_METADATA_NAME)
        logger.info(
            f"{CUSTOM_METADATA_NAME} custom metadata structure has "
            f"already been created."
        )
    except NotFoundError:
        cm_def = CustomMetadataDef.create(display_name=CUSTOM_METADATA_NAME)
        cm_def.attribute_defs = [
            AttributeDef.create(
                display_name="Rating",
                attribute_type=AtlanCustomAttributePrimitiveType.OPTIONS,
                options_name=ENUM_NAME,
            ),
            AttributeDef.create(
                display_name="Passed count",
                attribute_type=AtlanCustomAttributePrimitiveType.INTEGER,
            ),
            AttributeDef.create(
                display_name="Failed count",
                attribute_type=AtlanCustomAttributePrimitiveType.INTEGER,
            ),
            AttributeDef.create(
                display_name="Detailed reports",
                attribute_type=AtlanCustomAttributePrimitiveType.URL,
                multi_valued=True,
            ),
        ]
        cm_def.options = CustomMetadataDef.Options.with_logo_from_url(
            url="http://assets.atlan.com/assets/atlan-a-logo-blue-background.png"
        )
        result = client.create_typedef(cm_def)
        assert result  # noqa: S101
        logger.info(f"{CUSTOM_METADATA_NAME} custom metadata structure created.")

def create_badge():
    """
    Create a badge for the enumerated values of the "Rating" attribute in the
    custom metadata structure.
    """
    badge = Badge.create(
        name="Rating",
        cm_name=CUSTOM_METADATA_NAME,
        cm_attribute="Rating",
        badge_conditions=[
            BadgeCondition.create(
                badge_condition_operator=BadgeComparisonOperator.EQ,
                badge_condition_value='"Great"',
                badge_condition_colorhex=BadgeConditionColor.GREEN,
            ),
            BadgeCondition.create(
                badge_condition_operator=BadgeComparisonOperator.EQ,
                badge_condition_value='"OK"',
                badge_condition_colorhex=BadgeConditionColor.YELLOW,
            ),
            BadgeCondition.create(
                badge_condition_operator=BadgeComparisonOperator.EQ,
                badge_condition_value='"Poor"',
                badge_condition_colorhex=BadgeConditionColor.RED,
            ),
        ],
    )
    badge.user_description = "Overall quality rating for the asset."
    response = client.upsert(badge)
    assert response.assets_created(asset_type=Badge)  # noqa: S101
    logger.info(f"Badge for {CUSTOM_METADATA_NAME}:Rating created / updated.")

def main():
    create_custom_metadata_options()
    create_custom_metadata_structure()
    create_badge()

if __name__ == "__main__":
    main()


from page_scripts import helpers, food_page
from pages import PAGES_TO_SCRAPE
from wikipedia import wiki_http as http
import models
import pages

# logic to scrape a page with a table of foods
def scrape():
    categories = []
    for category in PAGES_TO_SCRAPE.get('table_categories'):

        page_url = f'{pages.WIKI_BASE}{category[1]}'

        categories.append(models.WikiCategory(
            category[0],
            helpers.scape_description(page_url),
            page_url,
            [],
            scrape_page(page_url, category[2])
        ))

    return categories

def scrape_page(page_url, column):
    ''' scrapes pages with the structure:
        1) h2 = name
        2) p = description
        3) table = list
    '''
    soup = http.request(page_url)

    # each table is a category
    tables = soup.select(':not(.navbox) > table')
    return [models.WikiCategory(
        category_name(table),
        category_description(table),
        page_url,
        category_foods(table, column)
    ) for table in tables]

def category_name(table):
    # scrapes the preceeding H2 text as the category name
    helpers.remove_superscripts(table)

    for sibling in table.previous_siblings:
        if sibling.name == 'h2':
            return helpers.scrub_string(sibling.text)

def category_foods(table, column=1):
    # gets the food names and links to dedicated pages,
    # ultimately scraping the description

    foods = []
    for anchor in [row.select_one(f'td:nth-child({column}) a') for row in table.select('tbody tr')]:
        if anchor and 'redlink=1' not in anchor["href"]:
            foods.append(food_page.create_food_from_anchor(anchor))

    return foods

def category_description(table):
    # scrapes the preceeding paragraph as a description of the category
    helpers.remove_superscripts(table)

    for sibling in table.previous_siblings:
        if sibling.name == 'p':
            return helpers.scrub_string(sibling.text)


import firebase_admin
from firebase_admin import credentials, firestore
import datetime
root_collection = 'active-hosts'
##### IMPORTANT WRITE ALL OF THIS INTO A CLASS!!!!

class firebase_dns:
    def __init__(self):
        cred = credentials.Certificate('./serviceAccountKey.json')
        firebase_admin.initialize_app(cred)
        self.db = firestore.client()

    def write_data_db(self,host_ip , queried_domain  , data : dict):
        self.db.collection(root_collection).document(host_ip).collection(queried_domain).add(data)

    def get_all_dns_by_host_ip(self,host_ip):
        doc_ref = self.db.collection(root_collection).document(host_ip)
        dns_data_snapshot = doc_ref.get()
        if dns_data_snapshot.exists:
            dns_data = dns_data_snapshot.to_dict()
            print(dns_data)

        for key in dns_data:
            for queried_domain in dns_data[key]:
               self.get_queried_domain_by_host( host_ip , queried_domain)
    def get_queried_domain_by_host(self,host_ip,queried_domain):
        doc_ref = self.db.collection(root_collection).document(host_ip).collection(queried_domain)
        dns_data_snapshot = doc_ref.get()  # this is duplicate code , refactor
        # returns documents , and we print them
        for snapshot in dns_data_snapshot:
            dns_data = snapshot.to_dict()
            print(dns_data)

    def get_all_dns(self):
        pass

#get_queried_domain_by_host('127.0.0.1','google.com')
#get_all_dns_by_host_ip('127.0.0.1')


import re
from .emails import send_alert
from datetime import datetime
from django.contrib.postgres.fields import ArrayField
from django.db import models
from django.template import Template, Context
from megedc.billing.measue_calculators import Calculators
from megedc.general.models import Project, Local, UnitCost
from timezone_field import TimeZoneField
from tinymce.models import HTMLField

class Gateway(models.Model):

    gateway_type = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    sn = models.CharField(
        max_length=128
    )

    name = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    ver = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    site = models.JSONField()

    owner = models.JSONField()

    room = models.JSONField()

    extra_data = models.JSONField(default=dict, null=True, blank=True)

    secret_extra_data = models.JSONField(default=dict, null=True, blank=True)

    time_zone = TimeZoneField(
        null=True,
        blank=True
    )

    enabled = models.BooleanField(default=True)

    created_at = models.DateTimeField(
        auto_now_add=True,
    )

    updated_at = models.DateTimeField(
        auto_now=True
    )

    removed_at = models.DateTimeField(
        null=True,
    )

    project = models.ForeignKey(
        Project,
        on_delete=models.PROTECT,
        related_name='gateways',
    )

    class Meta:
        unique_together = [
            ['project', 'sn']
        ]

    @property
    def timezone(self):
        if not self.time_zone:
            return self.project.timezone
        return self.time_zone

    def __str__(self):
        if self.name is None:
            return self.sn
        return self.name

class Device(models.Model):

    dev_id = models.AutoField(primary_key=True)

    channel = models.CharField(max_length=128)

    id = models.CharField(max_length=10)

    name = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    desc = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    ready = models.BooleanField(default=False)

    created_at = models.DateTimeField(
        auto_now_add=True,
    )

    updated_at = models.DateTimeField(
        auto_now=True
    )

    removed_at = models.DateTimeField(
        null=True,
    )

    gateway = models.ForeignKey(
        Gateway,
        on_delete=models.PROTECT,
        related_name='devices',
    )

    def __str__(self):
        if self.name is None:
            return '%s [%s-%s]' % (self.gateway, self.channel, self.id)
        return '%s %s' % (self.gateway, self.name)

    class Meta:
        unique_together = [
            ['gateway', 'channel', 'id']
        ]

class RTData(models.Model):

    logdt = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    date_time = models.DateTimeField(auto_now_add=True)

    discard = models.BooleanField(default=False)

    gateway = models.ForeignKey(
        Gateway,
        on_delete=models.PROTECT,
        related_name='data',
    )

    removed_at = models.DateTimeField(
        null=True,
    )

    class Meta:
        unique_together = [
            ['gateway', 'date_time'],
        ]

class DataNone(models.Model):

    dblink = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    name = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    value = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    unit = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    discard = models.BooleanField(default=False)

    data = models.ForeignKey(
        RTData,
        related_name='nodes',
        on_delete=models.PROTECT
    )

    device = models.ForeignKey(
        Device,
        related_name='nodes',
        on_delete=models.PROTECT
    )

    removed_at = models.DateTimeField(
        null=True,
    )

    class Meta:
        indexes = [
            models.Index(fields=['device', 'data', 'name'])
        ]

class RTAlarm(models.Model):

    logdt = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    date_time = models.DateTimeField(auto_now_add=True)

    name = models.CharField(
        max_length=256,
        null=True,
        blank=True
    )

    value = models.CharField(
        max_length=2048,
        null=True,
        blank=True
    )

    threadhold_value = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    unit = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    type = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    flag = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    removed_at = models.DateTimeField(
        null=True,
    )

    device = models.ForeignKey(
        Device,
        related_name='alarms',
        on_delete=models.PROTECT
    )

    class Meta:
        indexes = [
            models.Index(fields=['device', 'logdt'])
        ]

    re_int_charts = re.compile(r'^\D*(\d*).*$')

    @property
    def int_value(self):
        int_chars = self.re_int_charts.match(self.value)
        if int_chars:
            int_val = int_chars.groups()[0]
            if int_val:
                return int(int_val)

class TrendLogData(models.Model):

    date_time = models.DateTimeField()

    dblink = models.CharField(
        max_length=128,
        null=True,
        blank=True,
    )

    name = models.CharField(
        max_length=128,
        null=True,
        blank=True,
    )

    value = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    unit = models.CharField(
        max_length=128,
        null=True,
        blank=True
    )

    removed_at = models.DateTimeField(
        null=True,
    )

    device = models.ForeignKey(
        Device,
        related_name='trend_log_data',
        on_delete=models.PROTECT
    )

    class Meta:
        unique_together = [
            ['device', 'dblink', 'date_time'],
        ]

        indexes = [
            models.Index(fields=['device', 'name', 'date_time'])
        ]

class Measure(models.Model):

    name = models.CharField(
        max_length=128,
        help_text='Measure name'
    )

    desc = models.TextField(
        null=True,
        blank=True,
        help_text='Measure description'
    )

    var_name = models.CharField(
        max_length=128,
        help_text='Device variable name',
        null=True,
        blank=True,
    )

    calculator = models.CharField(
        max_length=20,
        help_text='Measuer calculator',
        choices=Calculators.choices(),
        default=Calculators.default_calculator
    )

    calculator_args = models.JSONField(
        default=list,
        null=True,
        blank=True
    )

    calculator_kwargs = models.JSONField(
        default=dict,
        null=True,
        blank=True
    )

    created_at = models.DateTimeField(
        auto_now_add=True,
    )

    updated_at = models.DateTimeField(
        auto_now=True
    )

    removed_at = models.DateTimeField(
        null=True
    )

    extra_data = models.JSONField(default=dict, null=True, blank=True)

    local = models.ForeignKey(
        Local,
        on_delete=models.CASCADE,
        related_name='measures',
    )

    unit_cost = models.ForeignKey(
        UnitCost,
        on_delete=models.CASCADE,
        related_name='measures',
    )

    def make_invoice_item_desc(self, result_data):
        if self.unit_cost.invoice_item_format is None:
            return 'Cargo de energía'
        context = dict(result_data)
        context.update({
            'start_date': datetime.fromisoformat(result_data['start_date']),
            'end_date': datetime.fromisoformat(result_data['start_date']),
            'local': self.local,
            'unit_cost': self.unit_cost,
        })
        template = Template(self.unit_cost.invoice_item_format)
        return template.render(Context(context))

    def calculate(self, *args, **kwargs):
        calculator = Calculators.get(self.calculator)
        return calculator(self, *args, **kwargs)

    def __str__(self):
        return self.name

class Alert(models.Model):

    name = models.CharField(max_length=128)

    enabled = models.BooleanField(default=True)

    emails = ArrayField(
        models.EmailField(),
        null=True,
        blank=True
    )

    from_email = models.EmailField(null=True, blank=True)

    from_email_name = models.CharField(max_length=50, null=True, blank=True)

    device_channel = models.CharField(max_length=128)

    device_id = models.CharField(max_length=10)

    subject_template = models.TextField(null=True)

    msg_template = models.TextField(null=True)

    html_msg_template = HTMLField(null=True)

    gateway = models.ForeignKey(
        Gateway,
        on_delete=models.PROTECT,
        related_name='alerts',
    )

    created_at = models.DateTimeField(
        auto_now_add=True,
    )

    updated_at = models.DateTimeField(
        auto_now=True
    )

    removed_at = models.DateTimeField(
        null=True
    )

    indexes = [
        models.Index(fields=['gateway', 'device_channel', 'device_id'])
    ]

    def send(self, rt_alarm):
        return send_alert(self, rt_alarm)


import json
import time

from platformio.cache import ContentCache
from platformio.compat import aio_create_task
from platformio.home.rpc.handlers.base import BaseRPCHandler
from platformio.home.rpc.handlers.os import OSRPC

class MiscRPC(BaseRPCHandler):
    async def load_latest_tweets(self, data_url):
        cache_key = ContentCache.key_from_args(data_url, "tweets")
        cache_valid = "180d"
        with ContentCache() as cc:
            cache_data = cc.get(cache_key)
            if cache_data:
                cache_data = json.loads(cache_data)
                # automatically update cache in background every 12 hours
                if cache_data["time"] < (time.time() - (3600 * 12)):
                    aio_create_task(
                        self._preload_latest_tweets(data_url, cache_key, cache_valid)
                    )
                return cache_data["result"]

        return await self._preload_latest_tweets(data_url, cache_key, cache_valid)

    @staticmethod
    async def _preload_latest_tweets(data_url, cache_key, cache_valid):
        result = json.loads((await OSRPC.fetch_content(data_url)))
        with ContentCache() as cc:
            cc.set(
                cache_key,
                json.dumps({"time": int(time.time()), "result": result}),
                cache_valid,
            )
        return result


#!/usr/bin/python
# -*- coding:utf-8 -*-

import cx_Oracle
from conf import Config as Config

def connect_db():
    # 连接数据库
    db_user = Config().getconf("db").ttw
    # 这里的顺序是用户名 / 密码 @ oracleserver的ip地址 / 数据库名字
    conn = cx_Oracle.connect(db_user)
    # 使用cursor()方法获取操作游标
    get_redis_ares = conn.cursor()
    get_redis_ares.execute("SELECT ra.id,ra.fullname FROM REDIS_AREA ra ")
    redis_ares = get_redis_ares.fetchall()
    get_job_class = conn.cursor()
    get_job_class.execut
    a = len(redis_ares)
    print(a)
    get_redis_ares .close()
    # conn.commit()
    conn.close()

if __name__ == '__main__':
    connect_db()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Building Web Maps using Folium"""

from __future__ import (division, absolute_import, print_function,
                        unicode_literals)

import os
import pandas as pd
import folium
from folium.plugins import MarkerCluster

# Get File Directory
WORK_DIR = os.path.dirname((os.path.realpath(__file__)))

# Loading the json data as python dictionary
DATA = pd.read_csv(WORK_DIR + "/data/Volcanoes_USA.txt")
LAT = DATA['LAT']
LON = DATA['LON']
ELEVATION = DATA['ELEV']

# Function to change colors
def color_change(elev):
    """Change Color depending on Mountain Elevation"""

    if elev < 1000:
        return 'green'
    if 1000 <= elev < 3000:
        return 'orange'
    return 'red'

# Create base map
MAP = folium.Map(
    location=[37.296933, -121.9574983],
    zoom_start=5,
    tiles="CartoDB dark_matter")

# Create Cluster
MARKER_CLUSTER = MarkerCluster().add_to(MAP)

# Plot Markers
for LAT, LON, ELEVATION in zip(LAT, LON, ELEVATION):
    folium.CircleMarker(
        location=[LAT, LON],
        radius=9,
        popup=str(ELEVATION) + " m",
        fill_color=color_change(ELEVATION),
        color="gray",
        fill_opacity=0.9).add_to(MARKER_CLUSTER)

# ave the map
MAP.save("map.html")


import logging
from io import BytesIO

import numpy as np
from scipy.io import wavfile
from voxpopuli import Voice, PhonemeList
from voxpopuli.phonemes import Phoneme

from cock_music_smart_music import get_random_progression_freqs
from tools.audio import to_f32_16k, to_wav_bytes, mix_tracks
from tools.melody import get_trinote_progression_freqs
from tools.player import AudioPlayer

TEMPO = 50
BEATS_PER_MEASURE = 4
CHORDS_PROGRESSION = ["Am2", "Dm2", "Am2", "E2"]
LOOPS_COUNT = 40

BEATS_TRACK_FILE = None
BEATS_TRACK_FILE = "tools/data/beats/tight_beat.wav"
BEATS_TRACK_BEAT_COUNT = 16

if __name__ == "__main__":
    logging.getLogger().setLevel(logging.INFO)

    if BEATS_TRACK_FILE is None:
        beat_duration = 60 / TEMPO
    else:
        rate, beats_track = wavfile.read(BEATS_TRACK_FILE)
        beat_duration = len(beats_track) / (rate * BEATS_TRACK_BEAT_COUNT)
        beats_track_looped = np.tile(beats_track,
                                     LOOPS_COUNT * BEATS_PER_MEASURE * len(CHORDS_PROGRESSION) // BEATS_TRACK_BEAT_COUNT)

    logging.info("Beat time : %dms" % (beat_duration * 1000))
    logging.info("Measure time : %dms" % (beat_duration * 1000 * 4))

    # prog_freqs = get_trinote_progression_freqs(CHORDS_PROGRESSION, BEATS_PER_MEASURE)
    # logging.info("First freq progression: \n %s" % (str(prog_freqs)))
    track_freqs = []
    for _ in range(LOOPS_COUNT):
        track_freqs += get_trinote_progression_freqs(CHORDS_PROGRESSION, BEATS_PER_MEASURE)
        # track_freqs += get_random_progression_freqs(CHORDS_PROGRESSION)

    progression_phonems = PhonemeList([])
    for freq in track_freqs:
        progression_phonems.append(
            Phoneme("o", int(beat_duration * 1000), [(0,freq), (100,freq)])
        )
    logging.info("Rendering audio")

    voice = Voice(lang="fr", voice_id=4)
    wav = voice.to_audio(progression_phonems)
    if BEATS_TRACK_FILE is not None:
        rate, wave_array = to_f32_16k(wav)
        mixed_tracks = mix_tracks(beats_track_looped * 0.5, wave_array * 0.7, align="left")
        wav = to_wav_bytes(mixed_tracks, 16000)
    player = AudioPlayer()
    with open("/tmp/gros_beat.wav", "wb") as wavfile:
        wavfile.write(wav)
    player.set_file(BytesIO(wav))
    player.play()
    player.close()


# Module imports
import requests
import os
import sys
import pandas as pd

# Retrieve ID and TOKEN from environment variables
## (Set with $ export NAME=VALUE)
ID = os.getenv('ID')
TOKEN = os.environ.get('TOKEN')

# Retrieve Twitter data for a given username (30 days by default)
def get_socialblade_data(user):
    twit_user = user
    client_id = ID
    token = TOKEN

    query_url = f'https://matrix.sbapis.com/b/twitter/statistics?query={twit_user}&clientid={client_id}&token={token}'
    x = requests.get(query_url)

    if x.status_code==200:
        print(f"Got data for {user}")
        resp = x.json()
        # call process_response() function on data object
        processed_data = process_response(resp)
        return processed_data
    else:
        print(f"Query for [{user}] returned status {x.status_code}")
        exit()

# Use pandas to format response object into dataframe
## (Subroutine of get_socialblade_data)
def process_response(resp):
    user_data = resp['data']['id']
    df = pd.DataFrame(resp['data']['daily']).sort_values(by='date').copy()

    df['username'] = user_data['username']
    df['display_name'] = user_data['display_name']

    # reorganize columns
    df = df[['username','display_name','date', 'followers', 'following', 'tweets']]

    # calculate follower change
    df['change'] = df.followers.diff()
    df['percent_change'] = df.followers.pct_change()
    print(f"Compiled data for {user_data['username']}")
    return df

# Save dataframe to CSV in a given directory (e.g. political alignment)
def write_to_csv(df, directory, username):
    if not os.path.exists(directory):
        os.makedirs(directory)
        print(f"Created directory {directory}")

    df.to_csv(f'{directory}/{username}.csv', index=False)
    print(f'Saved {username} to csv.')

# Main function - requests data from socialblade, writes to CSV
## Takes second argument
def save_follow_data(user, alignment):
    df = get_socialblade_data(user)
    write_to_csv(df, alignment, user)

# Run function if called from command line
## (Requires 2 arguments: handle and category/directory)
if __name__ == "__main__":
    handle = sys.argv[1]
    category = sys.argv[2]
    save_follow_data(handle, category)


from django.http import Http404, HttpResponse
from django.core import serializers
from django.views.decorators.csrf import csrf_exempt
from news.models import News
from .models import likes, dislikes, comment as commentTable
import json
from users.models import userExtraField

def add_like(request):
    if request.user.is_authenticated:

        news_id = request.GET.get('articale_id')
        news_is = News.objects.get(id = news_id)
        try:
            like_obj = likes.objects.get(for_news = news_is)
        except likes.DoesNotExist:
            like_obj = likes(for_news=news_is)
            like_obj.save()

        like_obj.like_by.add(request.user)
        like_obj.save()
        data = {
            'total' : like_obj.like_by.all().exclude(id = news_is.created_by.id).count()
        }
        return HttpResponse(json.dumps(data))

    else:
        raise Http404()

def remove_like(request):
    if request.user.is_authenticated:

        news_id = request.GET.get('articale_id')
        news_is = News.objects.get(id = news_id)
        like_obj = likes.objects.get(for_news = news_is)
        like_obj.like_by.remove(request.user)
        like_obj.save()
        data = {
            'total' : like_obj.like_by.all().exclude(id = news_is.created_by.id).count()
        }
        return HttpResponse(json.dumps(data))

    else:
        raise Http404()

def add_dislike(request):
    if request.user.is_authenticated:

        news_id = request.GET.get('articale_id')
        news_is = News.objects.get(id = news_id)
        try:
            dislike_obj = dislikes.objects.get(for_news = news_is)
        except dislikes.DoesNotExist:
            dislike_obj = dislikes(for_news=news_is)
            dislike_obj.save()

        dislike_obj.dislike_by.add(request.user)
        dislike_obj.save()
        data = {
            'total' : dislike_obj.dislike_by.all().exclude(id = news_is.created_by.id).count()
        }
        return HttpResponse(json.dumps(data))

    else:
        raise Http404()

def remove_dislike(request):
    if request.user.is_authenticated:

        news_id = request.GET.get('articale_id')
        news_is = News.objects.get(id = news_id)
        like_obj = dislikes.objects.get(for_news = news_is)
        like_obj.dislike_by.remove(request.user)
        like_obj.save()
        data = {
            'total' : like_obj.dislike_by.all().exclude(id = news_is.created_by.id).count()
        }
        return HttpResponse(json.dumps(data))

    else:
        raise Http404()

@csrf_exempt
def add_comment(request):
    if request.user.is_authenticated:
        if request.method == 'POST':
            article = request.POST['articale_id']
            message = request.POST['message']
            news = News.objects.get(id=article)
            extra= userExtraField.objects.get(user=request.user)
            comments = commentTable(comment_for = news, commented_by=request.user, message=message)
            comments.save()
            data = {'data': serializers.serialize('json', [comments, ])}
            if extra.profile_pic:
                data['profile_pic'] = extra.profile_pic.url
            else:
                data['profile_pic'] = '/static/icon/default-avatar.png'
            return HttpResponse(json.dumps(data))

    raise Http404()

def get_comment(request):
    news_id = request.GET.get('article_id')
    news_is = News.objects.get(id=news_id)
    main_comment = commentTable.objects.filter(comment_for=news_is, parent=None).order_by('-created_at')
    child_comment = commentTable.objects.filter(comment_for=news_is).exclude(parent=None).order_by('-created_at')
    main_comments = []
    child_comments = []
    for comment in main_comment:
        extra = userExtraField.objects.get(user=comment.commented_by)
        obj = {
            'username': comment.commented_by.username,
            'message': comment.message,
            'parent': comment.parent,
            'pk': comment.id,
            # 'created_at': comment.created_at
        }
        if extra.profile_pic:
            obj['profile_pic'] = extra.profile_pic.url
        else:
            obj['profile_pic'] = '/static/icon/default-avatar.png'

        main_comments.append(obj)

    for comment in child_comment:
        extra = userExtraField.objects.get(user=comment.commented_by)
        obj = {
            'username': comment.commented_by.username,
            'message': comment.message,
            'parent': comment.parent.id,
            'pk': comment.id,
            # 'created_at': comment.created_at
        }
        if extra.profile_pic:
            obj['profile_pic'] = extra.profile_pic.url
        else:
            obj['profile_pic'] = '/static/icon/default-avatar.png'

        child_comments.append(obj)
    data = {
        'main_comment': main_comments,
        'child_comment': child_comments
    }
    return HttpResponse(json.dumps(data))

@csrf_exempt
def add_reply(request):
    if request.user.is_authenticated:
        if request.method == 'POST':
            news_is = News.objects.get(id = request.POST['articale_id'])
            messages = request.POST['message']
            parent_comment = commentTable.objects.get(id=request.POST['parent'])
            comments = commentTable(comment_for=news_is, commented_by=request.user, message=messages, parent=parent_comment)
            comments.save()
            data = serializers.serialize('json', [comments, ])
            return HttpResponse(json.dumps(data))

    raise Http404()

@csrf_exempt
def edit_reply(request):
    if request.method == 'POST':
        comments = commentTable.objects.get(id = request.POST['comment_id'])
        messages = request.POST['message']
        comments.message = messages
        comments.save()
        data = serializers.serialize('json', [comments, ])
        return HttpResponse(json.dumps(data))

@csrf_exempt
def delete_reply(request, id):
    if request.user.is_authenticated:
        try:
            comments = commentTable.objects.get(id = id)
            comments.delete()
            return HttpResponse('done')
        except commentTable.DoesNotExist:
            raise Http404
    else:
        return HttpResponse('Unauthorized', status=401)


#coding:utf-8

"""
ID:          issue-3130
ISSUE:       3130
TITLE:       isql hangs when trying to show a view based on a procedure
DESCRIPTION:
JIRA:        CORE-2735
FBTEST:      bugs.core_2735
"""

import pytest
from firebird.qa import *

init_script = """set term ^;
create procedure p returns(a int) as begin a = 9; suspend; end^
create view vp1 as select a from p^
set term ;^
COMMIT;"""

db = db_factory(charset='UTF8', init=init_script)

test_script = """show view vp1;
"""

act = isql_act('db', test_script)

expected_stdout = """A                               INTEGER Nullable
View Source:
==== ======
 select a from p
"""

@pytest.mark.version('>=2.5.0')
def test_1(act: Action):
    act.expected_stdout = expected_stdout
    act.execute()
    assert act.clean_stdout == act.clean_expected_stdout


import json
from flask import Flask, render_template, request
app = Flask(__name__)

with open('data/movie5000.json') as f:
    mv = json.load(f)
with open('data/movie50000.json') as f:
    mv2 = json.load(f)

actors_movies = {}
movies_actors = {}
movies = {}
genres = {}
year = {}

def preprocess(data, actors_movies, movies_actors):
    for k, v in data.items():
        if k not in actors_movies.keys():
            actors_movies[k] = []
        for item in v:
            # actors_movies
            if item['title'] not in actors_movies[k]:
                actors_movies[k].append(item['title'])

            # movies_actors
            if item['title'] not in movies_actors.keys():
                movies_actors[item['title']] = [k]
            elif item['title'] in movies_actors[item['title']]:
                continue
            else:
                movies_actors[item['title']].append(k)

            # genres
            for genre in item['genres']:
                if genre not in genres.keys():
                    genres[genre] = [item['title']]
                else:
                    genres[genre].append(item['title'])
            # year
            if item['release_date'][:4] not in year.keys():
                year[item['release_date'][:4]] = [item['title']]
            else:
                year[item['release_date'][:4]].append(item['title'])
            # movies
            if item['title'] not in movies.keys():
                movies[item['title']] = {'vote_count': item['vote_count'], 'vote_avg': item['vote_avg'],
                                         'release_date': item['release_date'], 'poster_path': item['poster_path'],
                                         'genres': item['genres'], 'characters': [k]}
            else:
                if k not in movies[item['title']]['characters']:
                    movies[item['title']]['characters'].append(k)

preprocess(mv, actors_movies, movies_actors)
preprocess(mv2, actors_movies, movies_actors)
del year['2030']
del year['2028']
del year['2026']
del year['2024']
del year['2023']
del year['2022']

class Vertex:
    def __init__(self, key):
        self.id = key
        self.connectedTo = {}  # empty dictionary

    def addNeighbor(self, nbr, weight=0):
        self.connectedTo[nbr] = weight

    def getId(self):
        return self.id

    def getConnection(self):
        return self.connectedTo.keys()

class Graph:
    def __init__(self):
        self.verList = {}
        self.numVertices = 0

    def addVertex(self, key):
        self.numVertices += 1
        newVertex = Vertex(key)
        self.verList[key] = newVertex
        return newVertex

    def addEdge(self, f, t, weight=0):
        if f not in self.verList:
            nv = self.addVertex(f)
        if t not in self.verList:
            nv = self.addVertex(t)
        self.verList[f].addNeighbor(self.verList[t], weight)

def Backtrace(dic, start, end):
    res = [end]
    while 1:
        if start == end:
            break
        for k, v in dic.items():
            for i in v:
                if i == end:
                    res.append(k)
                    end = k
                    break
    return res

graph = Graph()
for actor in actors_movies.keys():
    graph.addVertex("A-"+actor)
for movie in movies_actors.keys():
    graph.addVertex("M-"+movie)

for k, v in movies_actors.items():
    for item in v:
        graph.addEdge("M-"+k, "A-"+item, 1)
        graph.addEdge("A-"+item,"M-"+k,1)

@app.route('/')
def KBGame():
    return render_template('game.jinja')

@app.route('/game/result', methods=['POST'])
def KBGame_result():
    backtrace={}
    def BFS(s, goal):
        s="A-"+s
        goal = "A-"+goal
        start = s
        visited = {}
        for key in graph.verList.keys():
            visited[key] = False
        queue = []
        queue.append(s)
        visited[s] = True

        while queue:
            s = queue.pop(0)
            for i in graph.verList[s].getConnection():
                if visited[i.getId()] == False:
                    if s not in backtrace.keys():
                        backtrace[s] = [i.getId()]
                    else:
                        backtrace[s].append(i.getId())
                    queue.append(i.getId())
                    visited[i.getId()] = True
                if i == graph.verList[goal]:
                    return Backtrace(backtrace, start, goal)
    actor1 = request.form['actor1']
    actor2 = request.form['actor2']
    noactor1 = ""
    noactor2 =""
    if "A-"+actor1 not in graph.verList:
        noactor1 = 'There is no result for ' + actor1
    if "A-"+actor2 not in graph.verList:
        noactor2 = 'There is no result for ' + actor2
    if len(noactor1)!=0 or len(noactor2)!=0:
        return render_template('game-result.jinja', noactor1=noactor1, noactor2=noactor2)

    result = BFS(actor2, actor1)

    return render_template('game-result.jinja', actor1=actor1, actor2=actor2, result=result)

@app.route('/popular')
def popular():
    return render_template('popular.jinja',)

@app.route('/popular/result', methods=['POST'])
def popular_res():
    def BFS_all(s):
        s="A-"+s
        if s not in graph.verList.keys():
            return []
        visited = {}
        for key in graph.verList.keys():
            visited[key] = False
        queue = []

        queue.append(s)
        visited[s] = True

        result={}
        inner = []
        count = 0.5
        while queue:
            s = queue.pop(0)

            for i in graph.verList[s].getConnection():

                if visited[i.getId()] == False:

                    inner.append(i.getId())
                    visited[i.getId()] = True

            if len(queue) == 0:
                result[count]=inner
                count+=0.5
                queue+=inner
                inner = []
            if count == 7.5:
                break
        return result

    actor = request.form['actor']
    all_degree = BFS_all(actor)
    if len(all_degree)==0:
        r = "No Search. Please enter again."
        return render_template('popular-result.jinja', nosearch=r)
    star = len(actors_movies[actor])

    score = []
    for i in range(1, 8):
        try:
            if all_degree[i]:
                score.append(len(all_degree[i]))
        except:
            score.append(0)

    if len(score)==5:
        score.append(0)
    if len(score)==6:
        score.append(0)

    popular_score = (1*score[0]+2*score[1]+3*score[2]+4*score[3]+5*score[4]+6*score[5] +
                     7*score[6])/(score[0]+score[1]+score[2]+score[3]+score[4]+score[5]+score[6])

    return render_template('popular-result.jinja', actor=actor, popular=round(popular_score,2), score=score, star=star)

@app.route('/recommend')
def recommend_page():
    all_genre = list(genres.keys())

    all_year = list(sorted(year.keys()))
    all_year.reverse()

    return render_template('recommend.jinja',all_genre=all_genre, all_year = all_year)

@app.route('/recommend/result', methods=['POST'])
def recommend_res():
    def Diff(li1, li2):
        return list(set(li1) - set(li2)) + list(set(li2) - set(li1))
    def listToString(s):
        str1 = " "
        separator = ', '
        return (separator.join(s))
    actor=request.form['actor']
    genre = request.form['genre']
    from_year = request.form['from_year']
    to_year = request.form['to_year']

    def recommend(genre, actor=None, from_year=None, to_year=None):
        movie = []
        delete = []
        if genre == 'Science':
            genre = 'Science Fiction'
        if genre == 'TV':
            genre=='TV Movie'
        movie = genres[genre]
        for m in movie:
            if actor:
                if actor not in movies_actors[m]:
                    delete.append(m)
            if from_year:
                if movies[m]['release_date'][:4] < from_year:
                    delete.append(m)
            if to_year:
                if movies[m]['release_date'][:4] >to_year:
                    delete.append(m)
            if movies[m]['vote_count']<1000:
                delete.append(m)
        result = Diff(movie, delete)
        sorting = {}
        for res in result:
            sorting[res] = movies[res]['vote_avg']
        sorting = dict(sorted(sorting.items(), key=lambda item: item[1], reverse=True))
        top20 = {}
        count=1
        for item in list(sorting.keys()):
            if item not in top20.keys():
                top20[item]=movies[item]

            count+=1
            if count==20:break

        return top20

    result= recommend(genre, actor=actor,from_year=from_year, to_year=to_year)

    title=[]
    for k in result.keys():
        title.append(k)
    vote_count = []
    vote_avg =[]
    year_list = []
    photo=[]
    genres_list=[]
    for item in result.values():
        vote_count.append(item['vote_count'])
        vote_avg.append(item['vote_avg'])
        year_list.append(item['release_date'][:4])
        photo.append(item['poster_path'])
        genres_list.append(item['genres'])
    r = []
    for i in range(len(title)):
        r.append([title[i],year_list[i],vote_count[i],vote_avg[i],genres_list[i],\
          'https://image.tmdb.org/t/p/w500/'+photo[i]])
    if len(r) == 0:
        r = "No Search. Please enter again."
        return render_template('recommend-result.jinja', nosearch = r)

    return render_template('recommend-result.jinja', result = r)

@app.route('/contact')
def contact():
    return render_template('contact.html')

if __name__ == '__main__':
    app.run(debug=True)