import cv2
from ultralytics import YOLO
from picamera2 import Picamera2

class YOLOCloudDetector:
    def __init__(self, model_path):
        # Initialize the YOLO model
        self.model = YOLO(model_path)

        # Initialize the Raspberry Pi Camera
        self.piCam = Picamera2()
        self.piCam.preview_configuration.main.size = (1280, 720)
        self.piCam.preview_configuration.main.format = "RGB888"
        self.piCam.preview_configuration.align()
        self.piCam.resolution = (1280, 720)
        self.piCam.configure("preview")
        self.piCam.start()

    def detect_objects_and_clouds(self):
        # Capture a frame
        frame = self.piCam.capture_array()

        # Perform object detection using the YOLO model
        results = self.model(frame)
        annotated_frame = results[0].plot()

        # Convert to grayscale for cloud detection
        gray = cv2.cvtColor(annotated_frame, cv2.COLOR_BGR2GRAY)
        ret, binary = cv2.threshold(gray, 175, 255, cv2.THRESH_BINARY)

        # Find contours of clouds
        contours, _ = cv2.findContours(binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        cv2.drawContours(annotated_frame, contours, -1, (0, 0, 255), 3)

        # Display the image with cloud contours
        cv2.imshow("YOLOv8 Detection with Cloud Contours", annotated_frame)

        # Check for exit condition
        if cv2.waitKey(1) & 0xFF == ord("q"):
            return

# Usage
model_path = '/home/pi/Desktop/cloud/best.pt'  # Ensure this is the correct model path
detector = YOLOCloudDetector(model_path)

# Continuously detect objects and clouds
while True:
    detector.detect_objects_and_clouds()


import tweepy
import time
from tweepy import OAuthHandler

import twitter_credentials

auth = OAuthHandler(twitter_credentials.CONSUMER_KEY, twitter_credentials.CONSUMER_SECRET)
auth.set_access_token(twitter_credentials.ACCESS_TOKEN, twitter_credentials.ACCESS_TOKEN_SECRET)
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)

user = api.me()
search = 'Javascript'
numberOfTweets = 500

for tweet in tweepy.Cursor(api.search, search).items(numberOfTweets):
    try:
        print('Tweet liked')
        tweet.favorite()
        time.sleep(10)
    except tweepy.TweepError as e:
        print(e.reason)
    except StopIteration:
        break


from django.urls import path
from . import views

#Template Tagging
app_name="MyApp"

urlpatterns=[
#path('mane/',views.mane_view,name="mane"),
path('other/',views.other_view,name="other"),
path('relative_templates/',views.relative_templates_view,name="relative"),
]


import requests
import argparse
import threading
import queue

results_lock = threading.Lock()
results = queue.Queue()

# ANSI escape codes for text color
RED = "\033[91m"
GREEN = "\033[92m"
ENDC = "\033[0m"

def send_request(url):
    try:
        response = requests.get(url)
        headers = response.headers
        if any("HIT" in value for value in headers.values()):
            status = GREEN + "HIT" + ENDC
            result = url
            print(f"{result} => {status}")  # print progress
            with open("cache-status.txt", "a") as file:
                file.write(result + "\n")  # save "HIT" URLs to file
        elif any("MISS" in value for value in headers.values()):
            status = RED + "MISS" + ENDC
            print(f"{url} => {status}")  # print progress
        else:
            status = "No CDN"
            print(f"{url} => {status}")  # print progress
        return None
    except Exception as e:
        result = f"Error for {url}: {e}"
        print(result)  # print error message
        return result

def worker():
    while True:
        url = urls.get()
        if url is None:
            break
        result = send_request(url)
        if result:
            results.put(result)
        urls.task_done()

def main():
    parser = argparse.ArgumentParser(description="Cache Checker Tool")
    parser.add_argument("-f", "--file", help="File containing URLs to check")
    parser.add_argument("-t", "--threads", type=int, help="Number of threads to use", default=80)
    parser.add_argument("-e", "--encoding", help="File encoding", default="utf-8")
    args = parser.parse_args()

    if args.file:
        with open(args.file, "r", encoding=args.encoding, errors="ignore") as file:
            global urls
            urls = queue.Queue()
            for url in file.read().splitlines():
                urls.put(url)
    else:
        parser.error("Please provide a file containing URLs using the -f/--file argument.")
        return

    threads = []
    for _ in range(args.threads):
        thread = threading.Thread(target=worker)
        threads.append(thread)
        thread.start()

    # Stop worker threads
    for _ in range(args.threads):
        urls.put(None)
    for thread in threads:
        thread.join()

if __name__ == "__main__":
    main()


import numpy as np
from copy import deepcopy
from collections import defaultdict

class KMeans:

    def __init__(self, k, max_iter=100):
        self.k = k
        self.max_iter = max_iter
        self.centroids = dict()

    def _euc_dist(self, pt1, pt2):
        return np.linalg.norm(pt1 - pt2)

    def _find_closest_centroid(self, pt):
        distances = list()
        for cluster_label in self.centroids:
            dist = self._euc_dist(self.centroids[cluster_label], pt)
            distances.append((cluster_label, dist))
        return sorted(distances, key=lambda x: x[1])[0][0]

    def fit(self, X, track_progress=False):
        # Randomly initialize centroids
        for i in range(self.k):
            self.centroids[i] = X[np.random.choice(len(X))]

        if track_progress:
            progress = list()

        # iterations
        for _ in range(self.max_iter):
            # For each point, find closest centroid
            clusters = defaultdict(list)
            for pt in X:
                closest_centroid = self._find_closest_centroid(pt)
                clusters[closest_centroid].append(pt)

            # Update centroids
            for i in self.centroids:
                self.centroids[i] = sum(clusters[i]) / len(clusters[i])

            if track_progress:
                progress.append((deepcopy(self.centroids), deepcopy(clusters)))

        if track_progress:
            return progress

    def predict(self, X):
        predictions = list()
        for pt in X:
            predictions.append(self._find_closest_centroid(pt))
        return np.array(predictions)


from typing import Final, Sequence, Tuple
import datetime
from spideriment_ng.logger.LogSeverity import LogSeverity
from spideriment_ng.logger.LoggingStrategyIface import LoggingStrategyIface

class Logger:
    _LOG_DATETIME_FORMAT: Final[str] = "%Y-%m-%d %H:%M:%S"

    def __init__(self, logging_strategy: LoggingStrategyIface, logged_severities: Sequence[LogSeverity], source_identifier: str):
        self._logging_strategy: Final[LoggingStrategyIface] = logging_strategy
        self._logged_severities: Final[Tuple[LogSeverity, ...]] = tuple(logged_severities)
        self._source_identifier: Final[str] = source_identifier
        self._operational: bool = True

    def log(self, severity: LogSeverity, message: str) -> None:
        assert self._operational

        if severity not in self._logged_severities:
            return

        log_string = "[{datetime} / {source} / {severity}] {message}".format(
            datetime=datetime.datetime.now().strftime(self.__class__._LOG_DATETIME_FORMAT),
            source=self._source_identifier,
            severity=self._get_string_representation_of_log_severity(severity),
            message=message
        )

        self._logging_strategy.process_log_string(log_string)

    def _get_string_representation_of_log_severity(self, severity: LogSeverity) -> str:
        severity_string = ({
            LogSeverity.ERROR: "ERROR",
            LogSeverity.WARNING: "WARN",
            LogSeverity.INFO: "INFO",
            LogSeverity.DEBUG: "DEBUG"
        }.get(severity, None))

        assert (severity_string is not None)

        return severity_string

    def terminate(self) -> None:
        assert self._operational
        self._operational = False

        self._logging_strategy.terminate()


""" Training augmented model """
import os
from math import sqrt

import numpy as np
import torch
import torch.nn as nn

import utils
from bert_fineturn.data_processor.glue import glue_compute_metrics as compute_metrics
from config import AugmentConfig
from dist_util_torch import init_gpu_params, FileLogger
from kdTool import Emd_Evaluator, distillation_loss
from modeling import TinyBertForSequenceClassification
from models.augment_cnn import AugmentCNN

acc_tasks = ["mnli", "mrpc", "sst-2", "qqp", "qnli", "rte", "books"]
corr_tasks = ["sts-b"]
mcc_tasks = ["cola"]
os.environ['CUDA_VISIBLE_DEVICES'] = '7'
device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
from transformers import RobertaForSequenceClassification, GPT2ForSequenceClassification

def convert_to_attn(hidns, mask):
    if type(hidns[0]) is not tuple:
        hdim = hidns[0].shape[-1]
        attns = [torch.matmul(x, x.transpose(2, 1)) / sqrt(hdim) for x in hidns]
        mask = mask.unsqueeze(1)
        mask = (1.0 - mask) * -10000.0
        attns = [torch.nn.functional.softmax(x + mask, dim=-1) for x in attns]
    else:
        hidns = [torch.stack(x, dim=1) for x in hidns]
        hdim = hidns[0][0].shape[-1]
        attns = [torch.matmul(x, x.transpose(-1, -2)) / sqrt(hdim) for x in hidns]
        mask = mask.unsqueeze(1).unsqueeze(2)
        mask = (1.0 - mask) * -10000.0
        attns = [torch.nn.functional.softmax(x + mask, dim=-1) for x in attns]
    return attns

def main():
    config = AugmentConfig()

    init_gpu_params(config)
    logger = FileLogger('./log', config.is_master, config.is_master)

    use_emd = config.use_emd

    # set seed
    np.random.seed(config.seed)
    torch.manual_seed(config.seed)
    torch.cuda.manual_seed_all(config.seed)

    torch.backends.cudnn.benchmark = True

    ############# LOADING DATA /START ###############
    task_name = config.datasets
    train_dataloader, _, eval_dataloader, output_mode, n_classes, config = utils.load_glue_dataset(config)
    logger.info(f"train_loader length {len(train_dataloader)}")

    ############# LOADING DATA /END ###############

    ############### BUILDING MODEL /START ###############

    model = AugmentCNN(config, n_classes, output_mode, auxiliary=False)
    pre_d, new_d = {}, {}
    for k, v in model.named_parameters():
        pre_d[k] = torch.sum(v)
    if config.teacher_type == 'gpt2':
        utils.load_gpt2_embedding_weight(model,
                                    config.teacher_model)
    elif config.teacher_type == 'bert':
        utils.load_bert_embedding_weight(model,
                                    config.teacher_model)
    elif config.teacher_type == 'roberta':
        utils.load_roberta_embedding_weight(model,
                                         config.teacher_model)
    for k, v in model.named_parameters():
        new_d[k] = torch.sum(v)

    logger.info("=" * 10 + "alter" + "=" * 10)
    for k in pre_d.keys():
        if pre_d[k] != new_d[k]:
            logger.info(k)
    del pre_d, new_d

    model = model.to(device)
    emd_tool = None
    if config.use_emd and config.use_kd:
        emd_tool = Emd_Evaluator(config.layers, 12, device)

    if not config.use_kd:
        teacher_model = None
    else:

        if config.teacher_type == 'gpt2':
            teacher_model = GPT2ForSequenceClassification.from_pretrained(config.teacher_model, num_labels=n_classes)
        elif config.teacher_type == 'bert':
            teacher_model = TinyBertForSequenceClassification.from_pretrained(config.teacher_model,num_labels=n_classes)
        elif config.teacher_type == 'roberta':
            teacher_model = RobertaForSequenceClassification.from_pretrained(config.teacher_model)
        teacher_model = teacher_model.to(device)
        teacher_model.eval()

    # model size
    mb_params = utils.param_size(model)
    if config.is_master:
        logger.info("Model size = {:.3f} MB".format(mb_params))

    optimizer = torch.optim.SGD(
        model.parameters(), config.lr, momentum=config.momentum, weight_decay=config.weight_decay)

    lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, config.epochs/2, eta_min=config.lr_min)
    torch.autograd.set_detect_anomaly(True)
    best_top1 = 0

    ############### BUILDING MODEL /END ###############

    ############### TRAIN /START ###############
    # training loop
    for epoch in range(config.epochs):
        drop_prob = config.drop_path_prob * epoch / config.epochs
        model.drop_path_prob(drop_prob)

        # training
        train(logger, config, train_dataloader, model, teacher_model, optimizer, epoch, task_name.lower(), emd_tool=emd_tool)

        lr_scheduler.step()
        # validation
        cur_step = (epoch + 1) * len(train_dataloader)
        top1 = validate(logger, config, eval_dataloader, model, epoch, cur_step, task_name.lower(), "val")
        # top1 = validate(test_loader, model, criterion, epoch, cur_step, "test", len(task_types))
        # save
        if best_top1 < top1:
            best_top1 = top1
            is_best = True
        else:
            is_best = False
        utils.save_checkpoint(model, config.path, is_best)
        logger.info("Present best Prec@1 = {:.4%}".format(best_top1))
        print("")

    logger.info("Final best Prec@1 = {:.4%}".format(best_top1))
    ############### TRAIN /END ###############

def train(logger, config, train_loader, model, teacher_model, optimizer, epoch, task_name, emd_tool=None):
    top1 = utils.AverageMeter()
    losses = utils.AverageMeter()

    total_num_step = len(train_loader)
    cur_step = epoch * len(train_loader)
    cur_lr = optimizer.param_groups[0]['lr']

    logger.info("Epoch {} LR {}".format(epoch, cur_lr))
    # writer.add_scalar('train/lr', cur_lr, cur_step)
    model.train()

    if model.output_mode == "classification":
        criterion = nn.CrossEntropyLoss()
    elif model.output_mode == "regression":
        criterion = nn.MSELoss()

    for step, data in enumerate(train_loader):
        data = [x.to("cuda", non_blocking=True) for x in data]
        input_ids, input_mask, segment_ids, label_ids, seq_lengths = data
#       [32,128]    [32,128]   [32,128]      [32]         [32]
        X = [input_ids, input_mask, segment_ids, seq_lengths]
        y = label_ids

        N = X[0].size(0)

        optimizer.zero_grad()
        logits = model(X)  #[32,2] , [[32,128,768],[32,128,768],[32,128,768],[32,128,768]]
        if config.use_emd:
            logits, s_layer_out = logits
        if config.use_kd:
            # with torch.no_grad():
            #     check_ids = input_ids.cpu()  #[32,128]
            #     check_seg = segment_ids.cpu() #[32,128]
            #     mask_check = input_mask.cpu() #[32,128]

            if config.teacher_type == 'roberta' or config.teacher_type == 'gpt2':
                output_dict = teacher_model(input_ids, attention_mask=input_mask, output_hidden_states=True,return_dict=True)

                teacher_logits, teacher_reps = output_dict.logits,output_dict.hidden_states
            elif config.teacher_type == 'bert':
                teacher_logits, teacher_reps = teacher_model(input_ids, segment_ids, input_mask)
            # print(np.argmax(teacher_logits.detach().cpu().numpy(),axis=1))
            # print(label_ids.cpu().numpy())
            # print("#####################################################")
            kd_loss, _, _ = distillation_loss(logits, y, teacher_logits, model.output_mode, alpha=config.kd_alpha)
            rep_loss = 0
            if config.use_emd:
                if config.hidn2attn:
                    s_layer_out = convert_to_attn(s_layer_out, input_mask)
                    teacher_reps = convert_to_attn(teacher_reps, input_mask)
                rep_loss, flow, distance = emd_tool.loss(s_layer_out, teacher_reps, return_distance=True)
                if config.update_emd:
                    emd_tool.update_weight(flow, distance)
            # loss = kd_loss * config.emd_only + rep_loss * config.emd_rate
            loss = kd_loss
        else:
            loss = criterion(logits, y)
        loss.backward()

        nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip)
        optimizer.step()

        preds = logits.detach().cpu().numpy()
        if model.output_mode == "classification":
            preds = np.argmax(preds, axis=1)
        elif model.output_mode == "regression":
            preds = np.squeeze(preds)        # top5.update(prec5.item(), N)
        result = compute_metrics(task_name, preds, y.detach().cpu().numpy())

        if task_name == "cola":
            train_acc = result['mcc']
        elif task_name in ["sst-2", "mnli", "mnli-mm", "qnli", "rte", 'books']:
            train_acc = result['acc']
        elif task_name in ["mrpc", "qqp"]:
            train_acc = result['f1']
        elif task_name == "sts-b":
            train_acc = result['corr']
        losses.update(loss.item(), N)
        top1.update(train_acc, N)

        if step % config.print_freq == 0 or step == total_num_step - 1:
            logger.info(
                "Train: , [{:2d}/{}] Step {:03d}/{:03d} Loss {:.3f} Prec {top1:.1%}"
                .format(epoch + 1, config.epochs, step, total_num_step - 1, losses.avg, top1=train_acc))
        cur_step += 1

def validate(logger, config, data_loader, model, epoch, cur_step, task_name, mode="dev"):
    top1 = utils.AverageMeter()
    losses = utils.AverageMeter()
    if model.output_mode == "classification":
        criterion = nn.CrossEntropyLoss()
    elif model.output_mode == "regression":
        criterion = nn.MSELoss()
    eval_labels = []
    model.eval()
    preds = []
    with torch.no_grad():
        for step, data in enumerate(data_loader):
            data = [x.to(device, non_blocking=True) for x in data]
            # input_ids, input_mask, segment_ids, label_ids, seq_lengths = data
            input_ids, input_mask, segment_ids, label_ids, seq_lengths = data

            X = [input_ids, input_mask, segment_ids, seq_lengths]
            y = label_ids
            N = X[0].size(0)
            logits = model(X)
            if config.use_emd:
                logits, _ = logits
            loss = criterion(logits, y)
            correct = torch.sum(torch.argmax(logits, axis=1) == y)

            if len(preds) == 0:
                preds.append(logits.detach().cpu().numpy())
            else:
                preds[0] = np.append(preds[0], logits.detach().cpu().numpy(), axis=0)
            eval_labels.extend(y.detach().cpu().numpy())

        preds = preds[0]
        if model.output_mode == "classification":
            preds = np.argmax(preds, axis=1)
        elif model.output_mode == "regression":
            preds = np.squeeze(preds)
        result = compute_metrics(task_name, preds, eval_labels)
        print(np.sum(preds == eval_labels), len(eval_labels), result)
        if task_name == "cola":
            acc = result['mcc']
        elif task_name in ["sst-2", "mnli", "mnli-mm", "qnli", "rte", 'books']:
            acc = result['acc']
        elif task_name in ["mrpc", "qqp"]:
            acc = result['f1']
        elif task_name == "sts-b":
            acc = result['corr']

    logger.info(mode + ": [{:2d}/{}] Final Prec@1 {:.4%}".format(epoch + 1, config.epochs, acc))
    return acc

if __name__ == "__main__":
    main()


import pubchempy as pcp
import csv
import requests

def get_data(cid):
    url = f"https://pubchem.ncbi.nlm.nih.gov/rest/pug_view/data/compound/{cid}/json/"
    response = requests.get(url)
    if response.status_code == 200:
        data = response.json()
        data = str(data)
        dataout ={'cid':cid, 'ld50':'' , 'smiles':''} #create a dictionary to store the data
        #find the index of the ld50 in data string
        try:
            index = data.find("LD50 Rat")
            dataout['ld50'] = data[index+14:index+24]
        except:
            dataout['ld50'] = None
    else:
        #return a request error

        return TimeoutError("Error: API request unsuccessful.")

# an empty list to store the compound ids
compound_ids = []
with open('PubChem_compound_list_aS7My_0hmJ2vsxCqktJZgGjZFLlr-0oYMD1RVCssQ1UrNX8.csv', newline='') as csvfile:
    reader = csv.reader(csvfile)
    for row in reader:
        compound_ids.append(row[0])
#remove the first element of the compound ids
compound_ids.pop(0)
#remove the leading empty spcae of the first element
compound_ids[0] = compound_ids[0].lstrip()

datatable = []

#get and print the first 10 ld50 values
for i in range(10):
    datatable.append(get_data(compound_ids[i]))

print(datatable)


import cv2
import numpy as np
import copy
import math

cap_region_x_begin=0.5
cap_region_y_end=0.8
threshold = 60  # 默认阈值
blurValue = 41  # 高斯模糊参数
bgSubThreshold = 50
learningRate = 0

isBgCaptured = 0   # 布尔值, 是否捕获背景
triggerSwitch = False  # 如果是true, 键盘仿真器工作

def printThreshold(thr):
    print("阈值改变为："+str(thr))

def removeBG(frame):
    fgmask = bgModel.apply(frame,learningRate=learningRate)

    kernel = np.ones((3, 3), np.uint8)
    fgmask = cv2.erode(fgmask, kernel, iterations=1)
    res = cv2.bitwise_and(frame, frame, mask=fgmask)
    return res

def calculateFingers(res,drawing):
    hull = cv2.convexHull(res, returnPoints=False)
    if len(hull) > 3:
        defects = cv2.convexityDefects(res, hull)
        if type(defects) != type(None):

            cnt = 0
            for i in range(defects.shape[0]):  # calculate the angle
                s, e, f, d = defects[i][0]
                start = tuple(res[s][0])
                end = tuple(res[e][0])
                far = tuple(res[f][0])
                a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2)
                b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2)
                c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2)
                angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c))  # cosine theorem
                if angle <= math.pi / 2:  # angle less than 90 degree, treat as fingers
                    cnt += 1
                    cv2.circle(drawing, far, 8, [211, 84, 0], -1)
            return True, cnt
    return False, 0

camera = cv2.VideoCapture(0)
camera.set(10,200)
cv2.namedWindow('trackbar')
cv2.createTrackbar('trh1', 'trackbar', threshold, 100, printThreshold)

while camera.isOpened():
    ret, frame = camera.read()
    threshold = cv2.getTrackbarPos('trh1', 'trackbar')
    frame = cv2.bilateralFilter(frame, 5, 50, 100)
    frame = cv2.flip(frame, 1)
    cv2.rectangle(frame, (int(cap_region_x_begin * frame.shape[1]), 0),
                 (frame.shape[1], int(cap_region_y_end * frame.shape[0])), (255, 0, 0), 2)
    cv2.imshow('original', frame)

    if isBgCaptured == 1:
        img = removeBG(frame)
        img = img[0:int(cap_region_y_end * frame.shape[0]),
                    int(cap_region_x_begin * frame.shape[1]):frame.shape[1]]
        cv2.imshow('mask', img)

        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        blur = cv2.GaussianBlur(gray, (blurValue, blurValue), 0)
        cv2.imshow('blur', blur)
        ret, thresh = cv2.threshold(blur, threshold, 255, cv2.THRESH_BINARY)
        cv2.imshow('ori', thresh)

        thresh1 = copy.deepcopy(thresh)
        contours, hierarchy = cv2.findContours(thresh1, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        length = len(contours)
        maxArea = -1
        if length > 0:
            for i in range(length):
                temp = contours[i]
                area = cv2.contourArea(temp)
                if area > maxArea:
                    maxArea = area
                    ci = i

            res = contours[ci]
            hull = cv2.convexHull(res)
            drawing = np.zeros(img.shape, np.uint8)
            cv2.drawContours(drawing, [res], 0, (0, 255, 0), 2)
            cv2.drawContours(drawing, [hull], 0, (0, 0, 255), 3)

            isFinishCal, cnt = calculateFingers(res,drawing)
            if triggerSwitch is True:
                if isFinishCal is True and cnt <= 2:
                    print (cnt)

        cv2.imshow('output', drawing)

    k = cv2.waitKey(10)
    if k == 27:  # 按ESC退出
        camera.release()
        cv2.destroyAllWindows()
        break
    elif k == ord('b'):  #按b捕捉背景
        bgModel = cv2.createBackgroundSubtractorMOG2(0, bgSubThreshold)
        isBgCaptured = 1
        print( '背景捕捉到啦')
    elif k == ord('r'):  #按r重置背景
        bgModel = None
        triggerSwitch = False
        isBgCaptured = 0
        print ('重置背景')
    elif k == ord('n'):
        triggerSwitch = True
        print ('开始啦')


#! /usr/bin/env python3

import numpy as np
import matplotlib.pyplot as plt
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
from FeatureExtractor_ResNet50 import FeatureExtractor
from math import floor
from sklearn.metrics import confusion_matrix
import tensorflow as tf
'''config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config)'''

#os.environ['CUDA_VISIBLE_DEVICES'] = '0'
#CUDA_VISIBLE_DEVICES=""
#os.environ['CUDA_VISIBLE_DEVICES'] = '-1'

class SVCClassificator:

    def __init__(self, ds_selection = "", kernel= ""):
        self.ds_selection = ds_selection
        self.kernel = kernel

    def execute(self):

        #device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        #device = torch.device('cpu')
        #print('Using device:' , device)
        #gpus = tf.config.experimental.list_physical_devices('CPU')

        '''gpus = tf.config.experimental.list_physical_devices('GPU')
        for device in gpus:
                tf.config.experimental.set_memory_growth(device, True)
        if gpus:
        # Restrict TensorFlow to only allocate 2GB of memory on the first GPU
            try:
                tf.config.experimental.set_virtual_device_configuration(
                    gpus[0],
                    [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=3800)])
                logical_gpus = tf.config.experimental.list_logical_devices('GPU')
                print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
            except RuntimeError as e:
                # Virtual devices must be set before GPUs have been initialized
                print(e)
        else:
            print('no gpus')'''

        # Confusion matrix lists
        Ccm_list = []
        Fcm_list = []
        Mcm_list = []

        for i in range(30):
                gpus = tf.config.list_physical_devices('CPU')
                print('CICLE: ' + str(i))
                '''gc.collect()
                torch.cuda.empty_cache()
                gpus = tf.config.list_physical_devices('GPU')
                if gpus:
                # Restrict TensorFlow to only allocate 1GB of memory on the first GPU
                        try:
                                tf.config.set_logical_device_configuration(
                                gpus[0],
                                [tf.config.LogicalDeviceConfiguration(memory_limit=3000)])
                                logical_gpus = tf.config.list_logical_devices('GPU')
                                print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
                        except RuntimeError as e:
                # Virtual devices must be set before GPUs have been initialized
                                print(e)
                else:
                        print('Using CPU')'''
                ############################################################
                ############### READ DATA ##################################
                itd = FeatureExtractor(self.ds_selection)

                CX = itd.CX
                CXT = itd.CXT
                CY = itd.CY
                CYT = itd.CYT

                FX = itd.FX
                FXT = itd.FXT
                FY = itd.FY
                FYT = itd.FYT

                MX = itd.MX
                MXT = itd.MXT
                MY = itd.MY
                MYT = itd.MYT

                #####################################################################
                ################### MODEL SELECTION (HYPERPARAMETER TUNING)##########
                #device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
                #print('Using device:' , device)
                points = 80
                print('MODEL SELECTION AND TUNING')
                if self.kernel == 'rbf':
                    logspaceC = np.logspace(-2,2.5,points)
                    logspaceGamma = np.logspace(-2,2.5,points)
                if self.kernel == 'linear':
                    logspaceC = np.logspace(-2,2.5,points)
                    logspaceGamma = np.logspace(-2,2.5,points)
                grid = {'C':        logspaceC,
                        'kernel':   [self.kernel],
                        'gamma':    logspaceGamma}
                MS = GridSearchCV(estimator = SVC(),
                                param_grid = grid,
                                scoring = 'balanced_accuracy',
                                cv = 10,
                                verbose = 0)

                if self.ds_selection == "chinese":
                    H = MS.fit(CX,CY)
                if self.ds_selection == "french":
                    H = MS.fit(FX,FY)
                if self.ds_selection == "mix":
                    H = MS.fit(MX,MY)

                print('CLASSIFICATION')
                print('C best param')
                print(H.best_params_['C'])
                print('gamma best param')
                print(H.best_params_['gamma'])

                M = SVC(C = H.best_params_['C'],
                        kernel = H.best_params_['kernel'],
                        gamma = H.best_params_['gamma'])

                if self.ds_selection == "chinese":
                    M = MS.fit(CX,CY)
                if self.ds_selection == "french":
                    M = MS.fit(FX,FY)
                if self.ds_selection == "mix":
                    M = MS.fit(MX,MY)

                ####################################################
                ################## TESTING #########################

                print('PREDICTING CHINESE TEST SET')
                CYF = M.predict(CXT)
                cm = confusion_matrix(CYT,CYF)
                print(cm)
                Ccm_list.append(cm)
                print('Predicting FRENCH TEST SET')
                CFYF = M.predict(FXT)
                cm = confusion_matrix(FYT,CFYF)
                print(cm)
                Fcm_list.append(cm)
                print('PREDICTING MIX TEST SET')
                MYF = M.predict(MXT)
                cm = confusion_matrix(MYT,MYF)
                print(cm)
                Mcm_list.append(cm)
                #device = torch.device('cpu')

        ######################################################
        ###################### RESULTS #######################
        print('RESULTS')
        def calculate_percentage_confusion_matrix(confusion_matrix_list, tot):
                pcms = []
                for i in confusion_matrix_list:
                        true_negative = (i[0,0]/tot)*100
                        false_negative = (i[1,0]/tot)*100
                        true_positive = (i[1,1]/tot)*100
                        false_positive = (i[0,1]/tot)*100
                        pcm = np.array([[true_negative , false_positive],[false_negative, true_positive]])
                        pcms.append(pcm)
                return pcms

        def return_tot_elements(cm):
                tot = cm[0,0] + cm[0,1] + cm[1,0] + cm[1,1]
                return tot

        def return_statistics_pcm(pcms):
                max_true_negative = 0
                max_false_negative = 0
                max_true_positive = 0
                max_false_positive = 0
                min_true_negative = 100
                min_false_negative = 100
                min_true_positive = 100
                min_false_positive = 100
                count_true_negative = 0
                count_false_negative = 0
                count_true_positive = 0
                count_false_positive = 0
                for i in pcms:
                        true_negative = i[0,0]
                        false_negative = i[1,0]
                        true_positive = i[1,1]
                        false_positive = i[0,1]

                        count_true_negative += true_negative
                        count_false_negative += false_negative
                        count_false_positive += false_positive
                        count_true_positive += true_positive

                        if true_negative > max_true_negative:
                                max_true_negative = true_negative
                        if false_negative > max_false_negative:
                                max_false_negative = false_negative
                        if true_positive > max_true_positive:
                                max_true_positive = true_positive
                        if false_positive > max_false_positive:
                                max_false_positive = false_positive

                        if true_negative < min_true_negative:
                                min_true_negative = true_negative
                        if false_negative < min_false_negative:
                                min_false_negative = false_negative
                        if true_positive < min_true_positive:
                                min_true_positive = true_positive
                        if false_positive < min_false_positive:
                                min_false_positive = false_positive

                mean_true_negative = count_true_negative/len(pcms)
                mean_false_negative = count_false_negative/len(pcms)
                mean_true_positive = count_true_positive/len(pcms)
                mean_false_positive = count_false_positive/len(pcms)

                mean_matrix = np.array([[mean_true_negative, mean_false_positive],[mean_false_negative, mean_true_positive]])
                max_matrix = np.array([[max_true_negative, max_false_positive],[max_false_negative, max_true_positive]])
                min_matrix = np.array([[min_true_negative, min_false_positive],[min_false_negative, min_true_positive]])

                matrix = []
                matrix.append(mean_matrix)
                matrix.append(max_matrix)
                matrix.append(min_matrix)
                return matrix

        Ctot = return_tot_elements(Ccm_list[0])
        Ccm_list = calculate_percentage_confusion_matrix(Ccm_list, Ctot)

        Ftot = return_tot_elements(Fcm_list[0])
        Fcm_list = calculate_percentage_confusion_matrix(Fcm_list, Ftot)

        Mtot = return_tot_elements(Mcm_list[0])
        Mcm_list = calculate_percentage_confusion_matrix(Mcm_list, Mtot)

        statistic_C = return_statistics_pcm(Ccm_list)
        statistic_F = return_statistics_pcm(Fcm_list)
        statistic_M = return_statistics_pcm(Mcm_list)

        print('CHINESE')
        for i in statistic_C:
                print(i)
        #print(statistic_C)
        print('FRENCH')
        for i in statistic_F:
                print(i)
        #print(statistic_F)
        print('MIX')
        for i in statistic_M:
                print(i)
        #print(statistic_M)

        ###################################################################
        ################## PRINT RESULTS ##################################
        accuracy_C = statistic_C[0][0][0] + statistic_C[0][1][1]
        accuracy_F = statistic_F[0][0][0] + statistic_F[0][1][1]
        accuracy_M = statistic_M[0][0][0] + statistic_M[0][1][1]
        print('Chinese Accuracy Out 0 ', accuracy_C, '%')
        print('French Accuracy Out 0 ', accuracy_F, '%')
        print('Mixed Accuracy Out 0 ', accuracy_M, '%')

        ####################################################################
        ###################### PLOT IMAGE ##################################
        print('PLOT IMAGE')
        plt.figure()
        plt.imshow(np.reshape(CX[30], (itd.size,itd.size)))
        plt.show()


import argparse
import os
from os import listdir
from os.path import isfile, join
from pathlib import Path
from pypdf import PdfMerger
import sys

def merger_pdfs(pdfs: list[str], output_path: Path) -> None:
    """
    Merge pdfs to one pdf

    Parameters
        pdfs: A list of complet path to the PDFs
        output_path: Path to output directory where the merge PDF will be located
    """
    merger = PdfMerger()

    for pdf in pdfs:
        merger.append(pdf)

    merger.write(output_path)
    merger.close()

def get_files(path: Path) -> list[str]:
    """
    Get the complet path to PDFs from a directory

    Parameters
        path: Path to directory

    Returns
        pdfs: A list of complet path to the PDFs in directory
    """
    all_files = [file for file in listdir(path) if isfile(join(path, file))]
    incomplete_pdfs = [file for file in all_files if file[-4:] == ".pdf"]
    pdfs = [f"{path}\\" + pdf for pdf in incomplete_pdfs]

    if len(pdfs) == 0:
        sys.exit("Den valgte mappe indenholder ingen PDf'er")

    return pdfs

if __name__ == "__main__":
    # -------------------------
    # Argsparse
    # -------------------------
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-i", "--input", help="Path to directory where PDFs are located"
    )
    parser.add_argument(
        "-o", "--output", help="Path where the merge PDF will be located"
    )
    args = parser.parse_args()

    if args.output:
        output_path = args.output + r"\result.pdf"
    else:
        # Default output path
        output_path = os.path.join(sys.path[0], r"result\result.pdf")

    if args.input:
        input_path = args.input
    else:
        # Default output path
        input_path = os.path.join(sys.path[0], r"PDFs")

    # -------------------------
    # Main
    # -------------------------
    pdfs = get_files(input_path)
    merger_pdfs(pdfs, output_path)


import os.path as osp
import torch
from torch import nn
import torch.nn.functional as F
from torch_geometric.nn import GraphUNet
from examples.core.base_model import BaseModel

class GraphUNetModel(BaseModel):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.save_hyperparameters()

        assert kwargs["num_layers"] >= 2

        self.unet = GraphUNet(
            kwargs["num_features"],
            kwargs["hidden_channels"],
            kwargs["num_classes"],
            depth=kwargs["num_layers"],
            pool_ratios=kwargs["pool_ratios"],
        )

    def forward(self, batch):
        x = self.unet(batch.x, batch.edge_index[0])
        return x, 0


from django.conf import settings
from django.db import models

from bleach import clean

from .utils import get_bleach_default_options

class BleachField(models.TextField):
    def __init__(self, allowed_tags=None, allowed_attributes=None,
                 allowed_styles=None, strip_tags=None, strip_comments=None,
                 *args, **kwargs):

        super(BleachField, self).__init__(*args, **kwargs)

        self.bleach_kwargs = get_bleach_default_options()

        if allowed_tags:
            self.bleach_kwargs['tags'] = allowed_tags
        if allowed_attributes:
            self.bleach_kwargs['attributes'] = allowed_attributes
        if allowed_styles:
            self.bleach_kwargs['styles'] = allowed_styles
        if strip_tags:
            self.bleach_kwargs['strip'] = strip_tags
        if strip_comments:
            self.bleach_kwargs['strip_comments'] = strip_comments

    def pre_save(self, model_instance, add):
        return clean(
            getattr(model_instance, self.attname),
            **self.bleach_kwargs
        )

if 'south' in settings.INSTALLED_APPS:
    from south.modelsinspector import add_introspection_rules
    # Bleach attributes don't influence on data representation so we use
    # default introspection rules of TextField
    add_introspection_rules([], ['^django_bleach\.models\.BleachField'])


# coding: utf-8

"""
    ExtremeCloud IQ API

    ExtremeCloud IQ RESTful API for external and internal applications.  # noqa: E501

    The version of the OpenAPI document: 23.6.0.46
    Contact: support@extremenetworks.com
    Generated by: https://openapi-generator.tech
"""

import pprint
import re  # noqa: F401

import six

from extremecloudiq.configuration import Configuration

class XiqValidDuringDateSettings(object):
    """NOTE: This class is auto generated by OpenAPI Generator.
    Ref: https://openapi-generator.tech

    Do not edit the class manually.
    """

    """
    Attributes:
      openapi_types (dict): The key is attribute name
                            and the value is attribute type.
      attribute_map (dict): The key is attribute name
                            and the value is json key in definition.
    """
    openapi_types = {
        'start_date_time': 'XiqDateTimeType',
        'end_date_time': 'XiqDateTimeType',
        'time_zone': 'str'
    }

    attribute_map = {
        'start_date_time': 'start_date_time',
        'end_date_time': 'end_date_time',
        'time_zone': 'time_zone'
    }

    def __init__(self, start_date_time=None, end_date_time=None, time_zone=None, local_vars_configuration=None):  # noqa: E501
        """XiqValidDuringDateSettings - a model defined in OpenAPI"""  # noqa: E501
        if local_vars_configuration is None:
            local_vars_configuration = Configuration()
        self.local_vars_configuration = local_vars_configuration

        self._start_date_time = None
        self._end_date_time = None
        self._time_zone = None
        self.discriminator = None

        self.start_date_time = start_date_time
        self.end_date_time = end_date_time
        self.time_zone = time_zone

    @property
    def start_date_time(self):
        """Gets the start_date_time of this XiqValidDuringDateSettings.  # noqa: E501

        :return: The start_date_time of this XiqValidDuringDateSettings.  # noqa: E501
        :rtype: XiqDateTimeType
        """
        return self._start_date_time

    @start_date_time.setter
    def start_date_time(self, start_date_time):
        """Sets the start_date_time of this XiqValidDuringDateSettings.

        :param start_date_time: The start_date_time of this XiqValidDuringDateSettings.  # noqa: E501
        :type: XiqDateTimeType
        """
        if self.local_vars_configuration.client_side_validation and start_date_time is None:  # noqa: E501
            raise ValueError("Invalid value for `start_date_time`, must not be `None`")  # noqa: E501

        self._start_date_time = start_date_time

    @property
    def end_date_time(self):
        """Gets the end_date_time of this XiqValidDuringDateSettings.  # noqa: E501

        :return: The end_date_time of this XiqValidDuringDateSettings.  # noqa: E501
        :rtype: XiqDateTimeType
        """
        return self._end_date_time

    @end_date_time.setter
    def end_date_time(self, end_date_time):
        """Sets the end_date_time of this XiqValidDuringDateSettings.

        :param end_date_time: The end_date_time of this XiqValidDuringDateSettings.  # noqa: E501
        :type: XiqDateTimeType
        """
        if self.local_vars_configuration.client_side_validation and end_date_time is None:  # noqa: E501
            raise ValueError("Invalid value for `end_date_time`, must not be `None`")  # noqa: E501

        self._end_date_time = end_date_time

    @property
    def time_zone(self):
        """Gets the time_zone of this XiqValidDuringDateSettings.  # noqa: E501

        The date/time timezone  # noqa: E501

        :return: The time_zone of this XiqValidDuringDateSettings.  # noqa: E501
        :rtype: str
        """
        return self._time_zone

    @time_zone.setter
    def time_zone(self, time_zone):
        """Sets the time_zone of this XiqValidDuringDateSettings.

        The date/time timezone  # noqa: E501

        :param time_zone: The time_zone of this XiqValidDuringDateSettings.  # noqa: E501
        :type: str
        """
        if self.local_vars_configuration.client_side_validation and time_zone is None:  # noqa: E501
            raise ValueError("Invalid value for `time_zone`, must not be `None`")  # noqa: E501

        self._time_zone = time_zone

    def to_dict(self):
        """Returns the model properties as a dict"""
        result = {}

        for attr, _ in six.iteritems(self.openapi_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value

        return result

    def to_str(self):
        """Returns the string representation of the model"""
        return pprint.pformat(self.to_dict())

    def __repr__(self):
        """For `print` and `pprint`"""
        return self.to_str()

    def __eq__(self, other):
        """Returns true if both objects are equal"""
        if not isinstance(other, XiqValidDuringDateSettings):
            return False

        return self.to_dict() == other.to_dict()

    def __ne__(self, other):
        """Returns true if both objects are not equal"""
        if not isinstance(other, XiqValidDuringDateSettings):
            return True

        return self.to_dict() != other.to_dict()


from typing import List
class Solution:
    def pancakeSort(self, arr: List[int]) -> List[int]:
        n = len(arr) -1
        output = []

        def flip(start,end):
            while start < end:
                arr[start],arr[end] = arr[end],arr[start]
                start += 1
                end -= 1
        for i in range(n,-1,-1):
            maxVIndex = i
            for j in range(0,i):
                if arr[j] > arr[maxVIndex]:
                    maxVIndex = j

            if maxVIndex != i:
                flip(0,maxVIndex) # bring the maximum element to index 0
                output.append(maxVIndex+1)
                flip(0,i)               #flip it to the last index of the array
                output.append(i+1)

        return output

s = Solution()
print(s.pancakeSort([3,2,4,1]))


import cv2

cap = cv2.VideoCapture(0, cv2.CAP_DSHOW)

genislik = cap.get(3)  # get(3) ile genislik degeri 4 ile de yukseklik degeri float olarak bulundu
yukseklik = cap.get(4)

print(genislik)
print(yukseklik)

cap.set(3, genislik*2)  # set ile istenilen aralik yeniden boyutlandirildi
cap.set(4, yukseklik*2)

genislik = cap.get(3)
yukseklik = cap.get(4)

print(genislik)
print(yukseklik)

while True:
    ret, frame = cap.read()

    if ret == 0:
        break

    frame = cv2.flip(frame, 1)

    cv2.imshow("frame", frame)

    if cv2.waitKey(5) & 0xFF == ord("q"):
        break

cap.release()
cv2.destroyAllWindows()


from torch.utils.data import Dataset
import glob
import numpy as np
import SimpleITK as sitk
import torch
import albumentations as A

transformer = A.Compose([
    A.RandomCrop(height=160, width=160, always_apply=True),
    A.Resize(height=224, width=224)
])

class LGE_TrainSet(Dataset):
    def __init__(self, dir, sample_num, train):
        self.imgdir = dir + '/LGE/'

        if train:
            self.imgsname = glob.glob(self.imgdir + 'LGE_Test/*LGE.nii*')
        else:
            self.imgsname = glob.glob(self.imgdir + 'LGE_Vali/*LGE.nii*')

        imgs = np.zeros((1, 192, 192))
        labs = np.zeros((1, 192, 192))
        for img_num in range(sample_num):
            itkimg = sitk.ReadImage(self.imgsname[img_num])
            npimg = sitk.GetArrayFromImage(itkimg)  # Z,Y,X,220*240*1

            imgs = np.concatenate((imgs, npimg), axis=0)

            labname = self.imgsname[img_num].replace('.nii', '_manual.nii')
            itklab = sitk.ReadImage(labname)
            nplab = sitk.GetArrayFromImage(itklab)
            nplab = (nplab == 200) * 1 + (nplab == 500) * 2 + (nplab == 600) * 3

            labs = np.concatenate((labs, nplab), axis=0)

        self.imgs = imgs[1:, :, :]
        self.labs = labs[1:, :, :]
        self.imgs.astype(np.float32)
        self.labs.astype(np.float32)

    def __getitem__(self, item):
        npimg = self.imgs[item]
        nplab = self.labs[item]

        transformed = transformer(image=npimg, mask=nplab)
        npimg = transformed['image']
        nplab = transformed['mask']

        return torch.from_numpy(npimg).unsqueeze(0).type(dtype=torch.FloatTensor), \
               torch.from_numpy(nplab).type(dtype=torch.LongTensor)

    def __len__(self):
        return self.imgs.shape[0]

class C0_TrainSet(Dataset):
    def __init__(self, dir, sample_num):
        self.imgdir = dir + '/C0/'

        self.imgsname = glob.glob(self.imgdir + '*C0.nii*')

        imgs = np.zeros((1, 192, 192))
        labs = np.zeros((1, 192, 192))
        for img_num in range(sample_num):
            itkimg = sitk.ReadImage(self.imgsname[img_num])
            npimg = sitk.GetArrayFromImage(itkimg)  # Z,Y,X,220*240*1

            imgs = np.concatenate((imgs, npimg), axis=0)

            labname = self.imgsname[img_num].replace('.nii', '_manual.nii')
            itklab = sitk.ReadImage(labname)
            nplab = sitk.GetArrayFromImage(itklab)
            nplab = (nplab == 200) * 1 + (nplab == 500) * 2 + (nplab == 600) * 3

            labs = np.concatenate((labs, nplab), axis=0)

        self.imgs = imgs[1:, :, :]
        self.labs = labs[1:, :, :]
        self.imgs.astype(np.float32)
        self.labs.astype(np.float32)

    def __getitem__(self, item):
        npimg = self.imgs[item]
        nplab = self.labs[item]

        transformed = transformer(image=npimg, mask=nplab)
        npimg = transformed['image']
        nplab = transformed['mask']

        return torch.from_numpy(npimg).unsqueeze(0).type(dtype=torch.FloatTensor), \
               torch.from_numpy(nplab).type(dtype=torch.LongTensor)

    def __len__(self):
        return self.imgs.shape[0]

class Train_Dataset(Dataset):
    def __init__(self, dir, train):
        # dir = '/data02/imucs_data/machine58_img/wangxiaoming/dataset/MMWHS/Patch192'
        self.C0_dataset = C0_TrainSet(dir, 35)
        self.LGE_dataset = LGE_TrainSet(dir, 40, train)

        self.len_C0_dataset = self.C0_dataset.__len__()
        self.len_LGE_dataset = self.LGE_dataset.__len__()

        # print(self.len_C0_dataset)
        # print(self.len_LGE_dataset)

    def __len__(self):
        return self.len_C0_dataset if self.len_C0_dataset > self.len_LGE_dataset else self.len_LGE_dataset

    def __getitem__(self, item):
        source_image, source_label = self.C0_dataset.__getitem__(np.random.randint(low=0, high=self.len_C0_dataset))
        target_image, target_label = self.LGE_dataset.__getitem__(item)

        return source_image, source_label, target_image, target_label

if __name__ == '__main__':
    import os

    os.environ["CUDA_VISIBLE_DEVICES"] = '5'
    from torch.utils.data import Dataset, DataLoader

    path = '/data02/imucs_data/machine58_img/wangxiaoming/dataset/MMWHS/Patch192'
    train = False

    SourceData = LGE_TrainSet(path, 5, train)
    # SourceData = Train_Dataset(path, train)
    SourceData_loader = DataLoader(SourceData, batch_size=12, shuffle=True, num_workers=2, pin_memory=True,
                                   drop_last=True)
    print(SourceData.__len__())
    # for i in SourceData_loader:
    #     img, label = i
    #     print(img.shape)  # 12x1x160x160
    #     print(label.shape)  # 12x160x160

    '''
    数据读取一大缺陷：2D、没有增强

    数量：
    C0：35
    LGE,vali:5
    LGE,test:40

    预训练：C0数据用作预训练
    训练：C0与LGE的image组成对来进行训练，与C0的标签进行比较
    验证：LGE的image与label进行比较
    测试：LGE的test数据集中image与label进行比较
    '''

    # 数据集的分配可能有点问题，训练集少于验证集


import json
import os
import random
import shutil
import time
from abc import abstractmethod
from pathlib import Path

import accelerate
import json5
import numpy as np
import torch
from accelerate.logging import get_logger
from accelerate.utils import ProjectConfiguration
from torch.utils.data import ConcatDataset, DataLoader
from tqdm import tqdm

from models.base.base_sampler import build_samplers
from optimizer.optimizers import NoamLR

class BaseTrainer(object):
    r"""The base trainer for all tasks. Any trainer should inherit from this class."""

    def __init__(self, args=None, cfg=None):
        super().__init__()

        self.args = args
        self.cfg = cfg

        cfg.exp_name = args.exp_name

        # init with accelerate
        self._init_accelerator()
        self.accelerator.wait_for_everyone()

        # Use accelerate logger for distributed training
        with self.accelerator.main_process_first():
            self.logger = get_logger(args.exp_name, log_level=args.log_level)

        # Log some info
        self.logger.info("=" * 56)
        self.logger.info("||\t\t" + "New training process started." + "\t\t||")
        self.logger.info("=" * 56)
        self.logger.info("\n")
        self.logger.debug(f"Using {args.log_level.upper()} logging level.")
        self.logger.info(f"Experiment name: {args.exp_name}")
        self.logger.info(f"Experiment directory: {self.exp_dir}")
        self.checkpoint_dir = os.path.join(self.exp_dir, "checkpoint")
        if self.accelerator.is_main_process:
            os.makedirs(self.checkpoint_dir, exist_ok=True)
        self.logger.debug(f"Checkpoint directory: {self.checkpoint_dir}")

        # init counts
        self.batch_count: int = 0
        self.step: int = 0
        self.epoch: int = 0
        self.max_epoch = (
            self.cfg.train.max_epoch if self.cfg.train.max_epoch > 0 else float("inf")
        )
        self.logger.info(
            "Max epoch: {}".format(
                self.max_epoch if self.max_epoch < float("inf") else "Unlimited"
            )
        )

        # Check values
        if self.accelerator.is_main_process:
            self.__check_basic_configs()
            # Set runtime configs
            self.save_checkpoint_stride = self.cfg.train.save_checkpoint_stride
            self.checkpoints_path = [
                [] for _ in range(len(self.save_checkpoint_stride))
            ]
            self.keep_last = [
                i if i > 0 else float("inf") for i in self.cfg.train.keep_last
            ]
            self.run_eval = self.cfg.train.run_eval

        # set random seed
        with self.accelerator.main_process_first():
            start = time.monotonic_ns()
            self._set_random_seed(self.cfg.train.random_seed)
            end = time.monotonic_ns()
            self.logger.debug(
                f"Setting random seed done in {(end - start) / 1e6:.2f}ms"
            )
            self.logger.debug(f"Random seed: {self.cfg.train.random_seed}")

        # setup data_loader
        with self.accelerator.main_process_first():
            self.logger.info("Building dataset...")
            start = time.monotonic_ns()
            self.train_dataloader, self.valid_dataloader = self._build_dataloader()
            end = time.monotonic_ns()
            self.logger.info(f"Building dataset done in {(end - start) / 1e6:.2f}ms")

        # setup model
        with self.accelerator.main_process_first():
            self.logger.info("Building model...")
            start = time.monotonic_ns()
            self.model = self._build_model()
            end = time.monotonic_ns()
            self.logger.debug(self.model)
            self.logger.info(f"Building model done in {(end - start) / 1e6:.2f}ms")
            self.logger.info(
                f"Model parameters: {self.__count_parameters(self.model)/1e6:.2f}M"
            )
        # optimizer & scheduler
        with self.accelerator.main_process_first():
            self.logger.info("Building optimizer and scheduler...")
            start = time.monotonic_ns()
            self.optimizer = self.__build_optimizer()
            self.scheduler = self.__build_scheduler()
            end = time.monotonic_ns()
            self.logger.info(
                f"Building optimizer and scheduler done in {(end - start) / 1e6:.2f}ms"
            )

        # accelerate prepare
        self.logger.info("Initializing accelerate...")
        start = time.monotonic_ns()
        (
            self.train_dataloader,
            self.valid_dataloader,
            self.model,
            self.optimizer,
            self.scheduler,
        ) = self.accelerator.prepare(
            self.train_dataloader,
            self.valid_dataloader,
            self.model,
            self.optimizer,
            self.scheduler,
        )
        end = time.monotonic_ns()
        self.logger.info(f"Initializing accelerate done in {(end - start) / 1e6:.2f}ms")

        # create criterion
        with self.accelerator.main_process_first():
            self.logger.info("Building criterion...")
            start = time.monotonic_ns()
            self.criterion = self._build_criterion()
            end = time.monotonic_ns()
            self.logger.info(f"Building criterion done in {(end - start) / 1e6:.2f}ms")

        # Resume or Finetune
        with self.accelerator.main_process_first():
            if args.resume:
                ## Automatically resume according to the current exprimental name
                self.logger.info("Resuming from {}...".format(self.checkpoint_dir))
                start = time.monotonic_ns()
                ckpt_path = self.__load_model(
                    checkpoint_dir=self.checkpoint_dir, resume_type=args.resume_type
                )
                end = time.monotonic_ns()
                self.logger.info(
                    f"Resuming from checkpoint done in {(end - start) / 1e6:.2f}ms"
                )
                self.checkpoints_path = json.load(
                    open(os.path.join(ckpt_path, "ckpts.json"), "r")
                )
            elif args.resume_from_ckpt_path and args.resume_from_ckpt_path != "":
                ## Resume from the given checkpoint path
                if not os.path.exists(args.resume_from_ckpt_path):
                    raise ValueError(
                        "[Error] The resumed checkpoint path {} don't exist.".format(
                            args.resume_from_ckpt_path
                        )
                    )

                self.logger.info(
                    "Resuming from {}...".format(args.resume_from_ckpt_path)
                )
                start = time.monotonic_ns()
                ckpt_path = self.__load_model(
                    checkpoint_path=args.resume_from_ckpt_path,
                    resume_type=args.resume_type,
                )
                end = time.monotonic_ns()
                self.logger.info(
                    f"Resuming from checkpoint done in {(end - start) / 1e6:.2f}ms"
                )

        # save config file path
        self.config_save_path = os.path.join(self.exp_dir, "args.json")

    ### Following are abstract methods that should be implemented in child classes ###
    @abstractmethod
    def _build_dataset(self):
        r"""Build dataset for model training/validating/evaluating."""
        pass

    @staticmethod
    @abstractmethod
    def _build_criterion():
        r"""Build criterion function for model loss calculation."""
        pass

    @abstractmethod
    def _build_model(self):
        r"""Build model for training/validating/evaluating."""
        pass

    @abstractmethod
    def _forward_step(self, batch):
        r"""One forward step of the neural network. This abstract method is trying to
        unify ``_train_step`` and ``_valid_step`` and avoid redundant implementation.
        However, for special case that using different forward step pattern for
        training and validating, you could just override this method with ``pass`` and
        implement ``_train_step`` and ``_valid_step`` separately.
        """
        pass

    @abstractmethod
    def _save_auxiliary_states(self):
        r"""To save some auxiliary states when saving model's ckpt"""
        pass

    ### Abstract methods end ###

    ### THIS IS MAIN ENTRY ###
    def train_loop(self):
        r"""Training loop. The public entry of training process."""
        # Wait everyone to prepare before we move on
        self.accelerator.wait_for_everyone()
        # dump config file
        if self.accelerator.is_main_process:
            self.__dump_cfg(self.config_save_path)
        self.model.train()
        self.optimizer.zero_grad()
        # Wait to ensure good to go
        self.accelerator.wait_for_everyone()
        while self.epoch < self.max_epoch:
            self.logger.info("\n")
            self.logger.info("-" * 32)
            self.logger.info("Epoch {}: ".format(self.epoch))

            ### TODO: change the return values of _train_epoch() to a loss dict, or (total_loss, loss_dict)
            ### It's inconvenient for the model with multiple losses
            # Do training & validating epoch
            train_loss = self._train_epoch()
            self.logger.info("  |- Train/Loss: {:.6f}".format(train_loss))
            valid_loss = self._valid_epoch()
            self.logger.info("  |- Valid/Loss: {:.6f}".format(valid_loss))
            self.accelerator.log(
                {"Epoch/Train Loss": train_loss, "Epoch/Valid Loss": valid_loss},
                step=self.epoch,
            )

            self.accelerator.wait_for_everyone()
            # TODO: what is scheduler?
            self.scheduler.step(valid_loss)  # FIXME: use epoch track correct?

            # Check if hit save_checkpoint_stride and run_eval
            run_eval = False
            if self.accelerator.is_main_process:
                save_checkpoint = False
                hit_dix = []
                for i, num in enumerate(self.save_checkpoint_stride):
                    if self.epoch % num == 0:
                        save_checkpoint = True
                        hit_dix.append(i)
                        run_eval |= self.run_eval[i]

            self.accelerator.wait_for_everyone()
            if self.accelerator.is_main_process and save_checkpoint:
                path = os.path.join(
                    self.checkpoint_dir,
                    "epoch-{:04d}_step-{:07d}_loss-{:.6f}".format(
                        self.epoch, self.step, train_loss
                    ),
                )
                self.tmp_checkpoint_save_path = path
                self.accelerator.save_state(path)
                print(f"save checkpoint in {path}")
                json.dump(
                    self.checkpoints_path,
                    open(os.path.join(path, "ckpts.json"), "w"),
                    ensure_ascii=False,
                    indent=4,
                )
                self._save_auxiliary_states()

                # Remove old checkpoints
                to_remove = []
                for idx in hit_dix:
                    self.checkpoints_path[idx].append(path)
                    while len(self.checkpoints_path[idx]) > self.keep_last[idx]:
                        to_remove.append((idx, self.checkpoints_path[idx].pop(0)))

                # Search conflicts
                total = set()
                for i in self.checkpoints_path:
                    total |= set(i)
                do_remove = set()
                for idx, path in to_remove[::-1]:
                    if path in total:
                        self.checkpoints_path[idx].insert(0, path)
                    else:
                        do_remove.add(path)

                # Remove old checkpoints
                for path in do_remove:
                    shutil.rmtree(path, ignore_errors=True)
                    self.logger.debug(f"Remove old checkpoint: {path}")

            self.accelerator.wait_for_everyone()
            if run_eval:
                # TODO: run evaluation
                pass

            # Update info for each epoch
            self.epoch += 1

        # Finish training and save final checkpoint
        self.accelerator.wait_for_everyone()
        if self.accelerator.is_main_process:
            self.accelerator.save_state(
                os.path.join(
                    self.checkpoint_dir,
                    "final_epoch-{:04d}_step-{:07d}_loss-{:.6f}".format(
                        self.epoch, self.step, valid_loss
                    ),
                )
            )
            self._save_auxiliary_states()

        self.accelerator.end_training()

    ### Following are methods that can be used directly in child classes ###
    def _train_epoch(self):
        r"""Training epoch. Should return average loss of a batch (sample) over
        one epoch. See ``train_loop`` for usage.
        """
        self.model.train()
        epoch_sum_loss: float = 0.0
        epoch_step: int = 0
        for batch in tqdm(
            self.train_dataloader,
            desc=f"Training Epoch {self.epoch}",
            unit="batch",
            colour="GREEN",
            leave=False,
            dynamic_ncols=True,
            smoothing=0.04,
            disable=not self.accelerator.is_main_process,
        ):
            # Do training step and BP
            with self.accelerator.accumulate(self.model):
                loss = self._train_step(batch)
                self.accelerator.backward(loss)
                self.optimizer.step()
                self.optimizer.zero_grad()
            self.batch_count += 1

            # Update info for each step
            # TODO: step means BP counts or batch counts?
            if self.batch_count % self.cfg.train.gradient_accumulation_step == 0:
                epoch_sum_loss += loss
                self.accelerator.log(
                    {
                        "Step/Train Loss": loss,
                        "Step/Learning Rate": self.optimizer.param_groups[0]["lr"],
                    },
                    step=self.step,
                )
                self.step += 1
                epoch_step += 1

        self.accelerator.wait_for_everyone()
        return (
            epoch_sum_loss
            / len(self.train_dataloader)
            * self.cfg.train.gradient_accumulation_step
        )

    @torch.inference_mode()
    def _valid_epoch(self):
        r"""Testing epoch. Should return average loss of a batch (sample) over
        one epoch. See ``train_loop`` for usage.
        """
        self.model.eval()
        epoch_sum_loss = 0.0
        for batch in tqdm(
            self.valid_dataloader,
            desc=f"Validating Epoch {self.epoch}",
            unit="batch",
            colour="GREEN",
            leave=False,
            dynamic_ncols=True,
            smoothing=0.04,
            disable=not self.accelerator.is_main_process,
        ):
            batch_loss = self._valid_step(batch)
            epoch_sum_loss += batch_loss.item()

        self.accelerator.wait_for_everyone()
        return epoch_sum_loss / len(self.valid_dataloader)

    def _train_step(self, batch):
        r"""Training forward step. Should return average loss of a sample over
        one batch. Provoke ``_forward_step`` is recommended except for special case.
        See ``_train_epoch`` for usage.
        """
        return self._forward_step(batch)

    @torch.inference_mode()
    def _valid_step(self, batch):
        r"""Testing forward step. Should return average loss of a sample over
        one batch. Provoke ``_forward_step`` is recommended except for special case.
        See ``_test_epoch`` for usage.
        """
        return self._forward_step(batch)

    def __load_model(
        self,
        checkpoint_dir: str = None,
        checkpoint_path: str = None,
        resume_type: str = "",
    ):
        r"""Load model from checkpoint. If checkpoint_path is None, it will
        load the latest checkpoint in checkpoint_dir. If checkpoint_path is not
        None, it will load the checkpoint specified by checkpoint_path. **Only use this
        method after** ``accelerator.prepare()``.
        """
        if checkpoint_path is None:
            ls = [str(i) for i in Path(checkpoint_dir).glob("*")]
            ls.sort(key=lambda x: int(x.split("_")[-3].split("-")[-1]), reverse=True)
            checkpoint_path = ls[0]
            self.logger.info("Resume from {}...".format(checkpoint_path))

        if resume_type in ["resume", ""]:
            # Load all the things, including model weights, optimizer, scheduler, and random states.
            self.accelerator.load_state(input_dir=checkpoint_path)

            # set epoch and step
            self.epoch = int(checkpoint_path.split("_")[-3].split("-")[-1]) + 1
            self.step = int(checkpoint_path.split("_")[-2].split("-")[-1]) + 1

        elif resume_type == "finetune":
            # Load only the model weights
            accelerate.load_checkpoint_and_dispatch(
                self.accelerator.unwrap_model(self.model),
                os.path.join(checkpoint_path, "pytorch_model.bin"),
            )
            self.logger.info("Load model weights for finetune...")

        else:
            raise ValueError("Resume_type must be `resume` or `finetune`.")

        return checkpoint_path

    # TODO: LEGACY CODE
    def _build_dataloader(self):
        Dataset, Collator = self._build_dataset()

        # build dataset instance for each dataset and combine them by ConcatDataset
        datasets_list = []
        for dataset in self.cfg.dataset:
            subdataset = Dataset(self.cfg, dataset, is_valid=False)
            datasets_list.append(subdataset)
        train_dataset = ConcatDataset(datasets_list)
        train_collate = Collator(self.cfg)
        _, batch_sampler = build_samplers(train_dataset, self.cfg, self.logger, "train")
        self.logger.debug(f"train batch_sampler: {list(batch_sampler)}")
        self.logger.debug(f"length: {train_dataset.cumulative_sizes}")
        # TODO: use config instead of (sampler, shuffle, drop_last, batch_size)
        train_loader = DataLoader(
            train_dataset,
            collate_fn=train_collate,
            batch_sampler=batch_sampler,
            num_workers=self.cfg.train.dataloader.num_worker,
            pin_memory=self.cfg.train.dataloader.pin_memory,
        )

        # Build valid dataloader
        datasets_list = []
        for dataset in self.cfg.dataset:
            subdataset = Dataset(self.cfg, dataset, is_valid=True)
            datasets_list.append(subdataset)
        valid_dataset = ConcatDataset(datasets_list)
        valid_collate = Collator(self.cfg)
        _, batch_sampler = build_samplers(valid_dataset, self.cfg, self.logger, "valid")
        self.logger.debug(f"valid batch_sampler: {list(batch_sampler)}")
        self.logger.debug(f"length: {valid_dataset.cumulative_sizes}")
        valid_loader = DataLoader(
            valid_dataset,
            collate_fn=valid_collate,
            batch_sampler=batch_sampler,
            num_workers=self.cfg.train.dataloader.num_worker,
            pin_memory=self.cfg.train.dataloader.pin_memory,
        )
        return train_loader, valid_loader

    @staticmethod
    def _set_random_seed(seed):
        r"""Set random seed for all possible random modules."""
        random.seed(seed)
        np.random.seed(seed)
        torch.random.manual_seed(seed)

    def _check_nan(self, loss, y_pred, y_gt):
        if torch.any(torch.isnan(loss)):
            self.logger.fatal("Fatal Error: Training is down since loss has Nan!")
            self.logger.error("loss = {:.6f}".format(loss.item()), in_order=True)
            if torch.any(torch.isnan(y_pred)):
                self.logger.error(
                    f"y_pred has Nan: {torch.any(torch.isnan(y_pred))}", in_order=True
                )
            else:
                self.logger.debug(
                    f"y_pred has Nan: {torch.any(torch.isnan(y_pred))}", in_order=True
                )
            if torch.any(torch.isnan(y_gt)):
                self.logger.error(
                    f"y_gt has Nan: {torch.any(torch.isnan(y_gt))}", in_order=True
                )
            else:
                self.logger.debug(
                    f"y_gt has nan: {torch.any(torch.isnan(y_gt))}", in_order=True
                )
            if torch.any(torch.isnan(y_pred)):
                self.logger.error(f"y_pred: {y_pred}", in_order=True)
            else:
                self.logger.debug(f"y_pred: {y_pred}", in_order=True)
            if torch.any(torch.isnan(y_gt)):
                self.logger.error(f"y_gt: {y_gt}", in_order=True)
            else:
                self.logger.debug(f"y_gt: {y_gt}", in_order=True)

            # TODO: still OK to save tracking?
            self.accelerator.end_training()
            raise RuntimeError("Loss has Nan! See log for more info.")

    ### Protected methods end ###

    ## Following are private methods ##
    ## !!! These are inconvenient for GAN-based model training. It'd be better to move these to svc_trainer.py if needed.
    def __build_optimizer(self):
        r"""Build optimizer for model."""
        # Make case-insensitive matching
        if self.cfg.train.optimizer.lower() == "adadelta":
            optimizer = torch.optim.Adadelta(
                self.model.parameters(), **self.cfg.train.adadelta
            )
            self.logger.info("Using Adadelta optimizer.")
        elif self.cfg.train.optimizer.lower() == "adagrad":
            optimizer = torch.optim.Adagrad(
                self.model.parameters(), **self.cfg.train.adagrad
            )
            self.logger.info("Using Adagrad optimizer.")
        elif self.cfg.train.optimizer.lower() == "adam":
            optimizer = torch.optim.Adam(self.model.parameters(), **self.cfg.train.adam)
            self.logger.info("Using Adam optimizer.")
        elif self.cfg.train.optimizer.lower() == "adamw":
            optimizer = torch.optim.AdamW(
                self.model.parameters(), **self.cfg.train.adamw
            )
        elif self.cfg.train.optimizer.lower() == "sparseadam":
            optimizer = torch.optim.SparseAdam(
                self.model.parameters(), **self.cfg.train.sparseadam
            )
        elif self.cfg.train.optimizer.lower() == "adamax":
            optimizer = torch.optim.Adamax(
                self.model.parameters(), **self.cfg.train.adamax
            )
        elif self.cfg.train.optimizer.lower() == "asgd":
            optimizer = torch.optim.ASGD(self.model.parameters(), **self.cfg.train.asgd)
        elif self.cfg.train.optimizer.lower() == "lbfgs":
            optimizer = torch.optim.LBFGS(
                self.model.parameters(), **self.cfg.train.lbfgs
            )
        elif self.cfg.train.optimizer.lower() == "nadam":
            optimizer = torch.optim.NAdam(
                self.model.parameters(), **self.cfg.train.nadam
            )
        elif self.cfg.train.optimizer.lower() == "radam":
            optimizer = torch.optim.RAdam(
                self.model.parameters(), **self.cfg.train.radam
            )
        elif self.cfg.train.optimizer.lower() == "rmsprop":
            optimizer = torch.optim.RMSprop(
                self.model.parameters(), **self.cfg.train.rmsprop
            )
        elif self.cfg.train.optimizer.lower() == "rprop":
            optimizer = torch.optim.Rprop(
                self.model.parameters(), **self.cfg.train.rprop
            )
        elif self.cfg.train.optimizer.lower() == "sgd":
            optimizer = torch.optim.SGD(self.model.parameters(), **self.cfg.train.sgd)
        else:
            raise NotImplementedError(
                f"Optimizer {self.cfg.train.optimizer} not supported yet!"
            )
        return optimizer

    def __build_scheduler(self):
        r"""Build scheduler for optimizer."""
        # Make case-insensitive matching
        if self.cfg.train.scheduler.lower() == "lambdalr":
            scheduler = torch.optim.lr_scheduler.LambdaLR(
                self.optimizer, **self.cfg.train.lambdalr
            )
        elif self.cfg.train.scheduler.lower() == "multiplicativelr":
            scheduler = torch.optim.lr_scheduler.MultiplicativeLR(
                self.optimizer, **self.cfg.train.multiplicativelr
            )
        elif self.cfg.train.scheduler.lower() == "steplr":
            scheduler = torch.optim.lr_scheduler.StepLR(
                self.optimizer, **self.cfg.train.steplr
            )
        elif self.cfg.train.scheduler.lower() == "multisteplr":
            scheduler = torch.optim.lr_scheduler.MultiStepLR(
                self.optimizer, **self.cfg.train.multisteplr
            )
        elif self.cfg.train.scheduler.lower() == "constantlr":
            scheduler = torch.optim.lr_scheduler.ConstantLR(
                self.optimizer, **self.cfg.train.constantlr
            )
        elif self.cfg.train.scheduler.lower() == "linearlr":
            scheduler = torch.optim.lr_scheduler.LinearLR(
                self.optimizer, **self.cfg.train.linearlr
            )
        elif self.cfg.train.scheduler.lower() == "exponentiallr":
            scheduler = torch.optim.lr_scheduler.ExponentialLR(
                self.optimizer, **self.cfg.train.exponentiallr
            )
        elif self.cfg.train.scheduler.lower() == "polynomiallr":
            scheduler = torch.optim.lr_scheduler.PolynomialLR(
                self.optimizer, **self.cfg.train.polynomiallr
            )
        elif self.cfg.train.scheduler.lower() == "cosineannealinglr":
            scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
                self.optimizer, **self.cfg.train.cosineannealinglr
            )
        elif self.cfg.train.scheduler.lower() == "sequentiallr":
            scheduler = torch.optim.lr_scheduler.SequentialLR(
                self.optimizer, **self.cfg.train.sequentiallr
            )
        elif self.cfg.train.scheduler.lower() == "reducelronplateau":
            scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
                self.optimizer, **self.cfg.train.reducelronplateau
            )
        elif self.cfg.train.scheduler.lower() == "cycliclr":
            scheduler = torch.optim.lr_scheduler.CyclicLR(
                self.optimizer, **self.cfg.train.cycliclr
            )
        elif self.cfg.train.scheduler.lower() == "onecyclelr":
            scheduler = torch.optim.lr_scheduler.OneCycleLR(
                self.optimizer, **self.cfg.train.onecyclelr
            )
        elif self.cfg.train.scheduler.lower() == "cosineannearingwarmrestarts":
            scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(
                self.optimizer, **self.cfg.train.cosineannearingwarmrestarts
            )
        elif self.cfg.train.scheduler.lower() == "noamlr":
            scheduler = NoamLR(self.optimizer, **self.cfg.train.lr_scheduler)
        else:
            raise NotImplementedError(
                f"Scheduler {self.cfg.train.scheduler} not supported yet!"
            )
        return scheduler

    def _init_accelerator(self):
        self.exp_dir = os.path.join(
            os.path.abspath(self.cfg.log_dir), self.args.exp_name
        )
        project_config = ProjectConfiguration(
            project_dir=self.exp_dir,
            logging_dir=os.path.join(self.exp_dir, "log"),
        )
        self.accelerator = accelerate.Accelerator(
            gradient_accumulation_steps=self.cfg.train.gradient_accumulation_step,
            log_with=self.cfg.train.tracker,
            project_config=project_config,
        )
        if self.accelerator.is_main_process:
            os.makedirs(project_config.project_dir, exist_ok=True)
            os.makedirs(project_config.logging_dir, exist_ok=True)
        with self.accelerator.main_process_first():
            self.accelerator.init_trackers(self.args.exp_name)

    def __check_basic_configs(self):
        if self.cfg.train.gradient_accumulation_step <= 0:
            self.logger.fatal("Invalid gradient_accumulation_step value!")
            self.logger.error(
                f"Invalid gradient_accumulation_step value: {self.cfg.train.gradient_accumulation_step}. It should be positive."
            )
            self.accelerator.end_training()
            raise ValueError(
                f"Invalid gradient_accumulation_step value: {self.cfg.train.gradient_accumulation_step}. It should be positive."
            )
        # TODO: check other values

    @staticmethod
    def __count_parameters(model):
        model_param = 0.0
        if isinstance(model, dict):
            for key, value in model.items():
                model_param += sum(p.numel() for p in model[key].parameters())
        else:
            model_param = sum(p.numel() for p in model.parameters())
        return model_param

    def __dump_cfg(self, path):
        os.makedirs(os.path.dirname(path), exist_ok=True)
        json5.dump(
            self.cfg,
            open(path, "w"),
            indent=4,
            sort_keys=True,
            ensure_ascii=False,
            quote_keys=True,
        )

    ### Private methods end ###


import uuid
from datetime import datetime
from decimal import Decimal

from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models
from pytz import utc

from users.models import Base

ONSITE = 'OnSite'
OFFSITE = 'OffSite'
BOTH = 'Both'

ITEM_GROUP_TYPE_CHOICES = (
    (OFFSITE, OFFSITE),
    (ONSITE, ONSITE),
    (BOTH, BOTH),
)

class EventItem(Base):
    uuid = models.UUIDField(default=uuid.uuid4, db_index=True)
    item_master = models.ForeignKey('items.ItemMaster', on_delete=models.CASCADE)
    group = models.ForeignKey('events.EventItemGroup', on_delete=models.CASCADE)
    event = models.ForeignKey('events.Event', on_delete=models.CASCADE)
    event_registration_type = models.ForeignKey('events.EventRegistrationType', on_delete=models.CASCADE)
    group_type = models.CharField(max_length=10, choices=ITEM_GROUP_TYPE_CHOICES)
    item_capacity = models.PositiveIntegerField(validators=[MinValueValidator(int(1))])
    items_booked = models.PositiveIntegerField(default=0)
    item_sharing_count = models.PositiveIntegerField(validators=[MinValueValidator(int(1))])
    discount_before_early_bird = models.DecimalField(max_digits=5, decimal_places=2,
                                                     validators=[MinValueValidator(Decimal('0.00')),
                                                                 MaxValueValidator(Decimal('100.00'))])  # percentage
    discount_after_early_bird = models.DecimalField(max_digits=5, decimal_places=2,
                                                    validators=[MinValueValidator(Decimal('0.00')),
                                                                MaxValueValidator(Decimal('100.00'))])  # percentage
    is_day_pass_item = models.BooleanField()
    senior_citizen_discount_applicable = models.BooleanField(default=False)
    # start of fields applicable only for transportation items
    ask_for_arrival_datetime = models.BooleanField(default=False)
    ask_for_departure_datetime = models.BooleanField(default=False)
    ask_for_pickup_location = models.BooleanField(default=False)
    transportation_pickup_locations = models.ManyToManyField('events.TransportationPickupLocation', blank=True)
    # this is only for accommodation items
    accommodation_rooms = models.ManyToManyField('events.AccommodationRoom', blank=True)

    #  end of fields appliacable for only transportation items

    def __str__(self):
        return '{}'.format(self.item_master.name)

    def __repr__(self):
        return '{}'.format(self.item_master.name)

    class Meta:
        unique_together = ('item_master', 'event_registration_type', 'event', 'group_type')
        ordering = ('-created_at',)

    def get_early_bird_rate(self):
        return Decimal(self.item_master.item_rate) - (self.discount_before_early_bird / 100 * Decimal(
            self.item_master.item_rate))

    def get_late_bird_rate(self):
        return Decimal(self.item_master.item_rate) - (self.discount_after_early_bird / 100 * Decimal(
            self.item_master.item_rate))

    def get_current_rate(self):
        current_datetime = datetime.utcnow()
        if current_datetime.replace(tzinfo=utc) < self.event.early_bird_date.replace(tzinfo=utc):
            return self.get_early_bird_rate()
        else:
            return self.get_late_bird_rate()

# following statement to get the value of event items in csv from already existing database
# \copy (SELECT item_master_id,group_id,event_id,event_registration_type_id,group_type,item_capacity,
# items_booked,item_sharing_count,discount_before_early_bird,discount_after_early_bird,is_day_pass_item
# FROM events_eventitem) to 'eventitem_backup.csv' with csv;


__all__ = [
    "DropBoxHomeResource",
    "DropBoxCollectionResource",
]

from twext.python.log import Logger
from txweb2 import responsecode
from txdav.xml import element as davxml
from txweb2.dav.http import ErrorResponse
from txweb2.dav.resource import DAVResource, TwistedACLInheritable
from txweb2.dav.util import joinURL

from twisted.internet.defer import inlineCallbacks, returnValue

from twistedcaldav.config import config
from twistedcaldav.customxml import calendarserver_namespace

log = Logger()

class DropBoxHomeResource (DAVResource):
    """
    Drop box collection resource.
    """

    def resourceType(self):
        return davxml.ResourceType.dropboxhome  # @UndefinedVariable

    def isCollection(self):
        return True

    def http_PUT(self, request):
        return responsecode.FORBIDDEN

    @inlineCallbacks
    def accessControlList(self, request, *args, **kwargs):
        """
        Override this to give write proxies DAV:write-acl privilege so they can add attachments too.
        """

        acl = (yield super(DropBoxHomeResource, self).accessControlList(request, *args, **kwargs))

        if config.EnableProxyPrincipals:
            owner = (yield self.ownerPrincipal(request))

            newaces = tuple(acl.children)
            newaces += (
                # DAV:write-acl access for this principal's calendar-proxy-write users.
                davxml.ACE(
                    davxml.Principal(davxml.HRef(joinURL(owner.principalURL(), "calendar-proxy-write/"))),
                    davxml.Grant(
                        davxml.Privilege(davxml.WriteACL()),
                    ),
                    davxml.Protected(),
                    TwistedACLInheritable(),
                ),
            )

            returnValue(davxml.ACL(*newaces))

        else:
            returnValue(acl)

class DropBoxCollectionResource (DAVResource):
    """
    Drop box resource.
    """

    def resourceType(self):
        return davxml.ResourceType.dropbox  # @UndefinedVariable

    def isCollection(self):
        return True

    def writeNewACEs(self, newaces):
        """
        Write a new ACL to the resource's property store. We override this for calendar collections
        and force all the ACEs to be inheritable so that all calendar object resources within the
        calendar collection have the same privileges unless explicitly overridden. The same applies
        to drop box collections as we want all resources (attachments) to have the same privileges as
        the drop box collection.

        @param newaces: C{list} of L{ACE} for ACL being set.
        """
        # Add inheritable option to each ACE in the list
        edited_aces = []
        for ace in newaces:
            if TwistedACLInheritable() not in ace.children:
                children = list(ace.children)
                children.append(TwistedACLInheritable())
                edited_aces.append(davxml.ACE(*children))
            else:
                edited_aces.append(ace)

        # Do inherited with possibly modified set of aces
        return super(DropBoxCollectionResource, self).writeNewACEs(edited_aces)

    def http_PUT(self, request):
        return ErrorResponse(
            responsecode.FORBIDDEN,
            (calendarserver_namespace, "valid-drop-box"),
            "Cannot store resources in dropbox",
        )


import os
import logging
from huggingface_deploy.nlp.token_classification_demo.classifier import TokenClassifer

logger = logging.getLogger(__name__)

class HuaziExtract():
    def __init__(self, file_name=None, **kwargs):
        super(HuaziExtract, self).__init__(file_name, **kwargs)
        tokenizer_dir = file_name
        model_path = os.path.join(file_name, kwargs.get("onnx_name", "best.onnx"))
        model="bert"
        label_list = kwargs.get("label_list", [])
        self.classifier = TokenClassifer(tokenizer_dir, model_path, model, label_list)

    def convert_input(self, data):
        """
        转换输入数据的格式

        Parameters
        ----------
        data : list[dict]
            输入的数据
        data[*].raw_str : str
            原始的字符串

        Returns
        -------
        classifier_input : dict
            输入的数据
        classifier_input.tokens : list[str]
            输入的句子列表
        """
        classifier_input = {"tokens": []}
        for item in data:
            classifier_input["tokens"].append(item["raw_str"])
        return classifier_input

    def predict(self, data, **kwargs):
        """
        通过tokenizer处理输入的字符串，并通过onnx对token进行分类

        Parameters
        ----------
        data : list[dict]
            输入的数据
        data[*].raw_str : str
            原始的字符串

        Returns
        -------
            即self.classifier的输出
        """
        classifier_input = self.convert_input(data)
        results = self.classifier.predict(classifier_input, **kwargs)
        return results

if __name__ == "__main__":
    model_path = ""
    kwargs = {
        "onnx_name": "best.onnx",
        "label_list": ["O", "B-MAIN", "I-MAIN"]
    }

    model = HuaziExtract(file_name=model_path, **kwargs)

    data = [
        {"raw_str": "这瓶雅诗兰黛面霜效果非常好，能让脸润润的，有光泽"}
    ]

    result = model.predict(data)
    logger.fatal("result:{}".format(result))
    print("result:{}".format(result))


# # Importing the necessary libraries:

import re
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.svm import NuSVC
from sklearn.metrics import classification_report
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import pandas as pd
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score

# # Loading the 20 Newsgroups dataset for the specified classes:

categories = ['comp.graphics', 'misc.forsale', 'rec.sport.baseball', 'soc.religion.christian', 'talk.politics.guns']
newsgroups_data = fetch_20newsgroups(categories=categories, subset='all', remove=('headers', 'footers', 'quotes'))

newsgroups_data.filenames.shape

# # Get the category labels

labels = newsgroups_data.target

# #  Prepocessing the texts :
# * Transorm to LowerCase
# * Remove Punctuation
# * Remove dates
# * Tokenize
# * Remove StopWords

preprocessed_texts = []
for text in newsgroups_data.data:
    # Lowercase the text
    text = text.lower()
    # Remove punctuations, quotations, and dates
    text = re.sub(r'[^\w\s]', '', text)
    text = re.sub(r'\d+', '', text)
    # Tokenize the text
    tokens = word_tokenize(text)
    # Remove stopwords
    tokens = [word for word in tokens if word not in stopwords.words('english')]

    # Append the preprocessed text to the list
    preprocessed_texts.append(' '.join(tokens))

# ## Create a DataFrame with labels and preprocessed texts:

data = pd.DataFrame({'labels': labels, 'preprocessed_texts': preprocessed_texts})
data

# # Feature Extraction Based On TF-IDF :

# ## Initialize the TF-IDF vectorizer

vectorizer = TfidfVectorizer()

# ## Fit the vectorizer on the preprocessed texts to learn the vocabulary and IDF values

vectorizer.fit(preprocessed_texts)

# ## Transform the preprocessed texts into TF-IDF feature vectors

features = vectorizer.transform(preprocessed_texts)

print(features.shape)
print(labels.shape)

# # Classification using SVM :

# ## Splitting into Train-Test

X_train, X_test, y_train, y_test = train_test_split(features, labels, test_size=0.2, shuffle=True, random_state=42)

# ## Initialize the Nu-SVC classifier with the specified maximum value of nu (0.5 in this case)

svm = NuSVC(nu=0.5)

# ## Train the SVM classifier

svm.fit(X_train, y_train)

# ## Make predictions on the testing data

y_pred = svm.predict(X_test)

# ## Calculate precision

precision = precision_score(y_test, y_pred, average='weighted')
print(f"Precision: {precision:.4f}")

# ## Calculate recall

recall = recall_score(y_test, y_pred, average='weighted')
print(f"Recall: {recall:.4f}")

# ## Calculate F1-score

f1 = f1_score(y_test, y_pred, average='weighted')
print(f"F1-score: {f1:.4f}")

# ## Calculate the classification report

report = classification_report(y_test, y_pred)

print(report)


from collections import deque
class Node:
    def __init__(self, v, children=None):
        if children is None:
            children = []
        self.children = children
        self.v = v
    def __eq__(self, other):
        return self.__dict__ == other.__dict__

def serialize(t):
    s = str(t.v)
    for c in t.children:
        s += ' ' + serialize(c)
    s += ' |'
    return s

def deserialize(s):
    a = deque(s.split(' '))
    return dehelper(a)

def dehelper(a):
    n = Node(int(a[0]))
    a.popleft()
    while a[0] != '|':
        n.children.append(dehelper(a))
    a.popleft()
    return n

def dfs(t, l):
    print(t.v, l)
    for c in t.children:
        dfs(c, l+1)

a = Node(3, [Node(4), Node(5, [Node(7), Node(8)]), Node(6)])
b = serialize(a)
c = deserialize(b)
print(b)
print(a == c)