"""
This script processes an MRC file, recorded with the Technai TEM, containing multiple images, corrects the pixel sizes, rescales the intensity of the images,
adds scale bars and annotations, and saves the result as a TIFF file.

Author: Christoph Karfusehr
Date: 8.4.2023
Email: c.karfusehr@tum.de

The script imports several libraries including:
re, numpy, skimage, scipy, mrcfile, tifffile, PIL, concurrent, os, tkinter

The script contains several backend functions used for:
- Reading magnifications from an mdoc file
- Mapping magnifications to pixel size
- Rescaling intensity to 16-bit
- Resizing images to target shapes
- Drawing rectangles
- Drawing text
- Calculating scale bar parameters
- Drawing scale bars on images
- Combining images horizontally
- Reading MRC files into arrays
- Multithread processing of single images

The script contains a GUI application for browsing and processing MRC files.
"""

#### Importing packages ####

import os
import tkinter as tk
from tkinter import ttk, filedialog
import concurrent.futures #for multi-threading
import re
import numpy as np
from skimage import exposure
from scipy.ndimage import zoom
import mrcfile
import tifffile
from PIL import Image, ImageDraw, ImageFont
import sys

#### Constants ####

MAG_TO_PIXEL_DICT = {
100: 108.97,
120: 93.2,
145: 74.3,
240: 45.31,
260: 45.31,
290: 37.88,
380: 28.77,
470: 23.05,
560: 19.45,
1100: 9.734,
1650: 6.519,
2100: 5.071,
2700: 3.824,
3200: 3.354,
4400: 2.45,
6500: 1.644,
11000: 0.9945,
15000: 0.7096,
21000: 0.5129,
26000: 0.4215,
30000: 0.3653,
42000: 0.261,
52000: 0.211,
67000: 0.163,
110000: 0.102
}

PIXEL_LENGTH_TO_SCALEBAR_LENGTH_IN_nm_DICT = {
108.97: 10000,
93.2: 10000,
74.3: 10000,
45.31: 10000,
37.88: 100,
28.77: 10000,
23.05: 10000,
19.45: 10000,
9.734: 5000,
6.519: 5000,
5.071: 5000,
3.824: 1000,
3.354: 1000,
2.45: 1000,
1.644: 1000,
0.9945: 1000,
0.7096: 500,
0.5129: 1000,
0.4215: 200,
0.3653: 200,
0.261: 200,
0.211: 100,
0.163: 100,
0.102: 100
}

SPLIT_OUTPUT_TO_SINGLE_FILES = False;

#### Backend functions ####

def read_magnifications_from_mdoc(mrc_file_path):
    """
    Reads the magnification values from a .mdoc file corresponding to the given .mrc file.

    :param mrc_file_path: str, path to the .mrc file
    :return: list of int, magnification values extracted from the .mdoc file
    """

    base_path, extension = os.path.splitext(mrc_file_path)
    mdoc_file_path = base_path + extension + ".mdoc"

    magnifications = []
    with open(mdoc_file_path, "r") as mdoc_file:
        for line in mdoc_file:
            match = re.search(r"Magnification = (\d+)", line)
            if match:
                magnifications.append(int(match.group(1)))
    return magnifications

def map_magnifications_to_pixel_size(magnifications):
    """
    Maps the magnification values to pixel sizes using the predefined MAG_TO_PIXEL_DICT.

    :param magnifications: list of int, magnification values
    :return: list of float, corresponding pixel sizes in nanometers
    """

    return [MAG_TO_PIXEL_DICT[mag] for mag in magnifications]

def rescale_intensity_to_16bit(image):
    """
    Rescales the intensity of the input image to 16-bit.

    :param image: numpy array, input image
    :return: numpy array, rescaled image with intensity values in the range of 0 to 65535 (16-bit)
    """

    return exposure.rescale_intensity(image, out_range=(0, 65535)).astype(np.uint16)

def resize_image(image, target_shape):
    """
    Resizes the input image to the specified target shape using the scipy.ndimage.zoom function.

    :param image: numpy array, input image
    :param target_shape: tuple, target shape in the form (height, width)
    :return: numpy array, resized image with the specified target shape
    """

    factors = [t / s for t, s in zip(target_shape, image.shape)]
    return zoom(image, factors, order=1)

def draw_rectangle(image, x1, y1, x2, y2, fill_value):
    """
    Draws a filled rectangle on the input image with the specified coordinates and fill value.

    :param image: numpy array, input image
    :param x1: int, x-coordinate of the top-left corner of the rectangle
    :param y1: int, y-coordinate of the top-left corner of the rectangle
    :param x2: int, x-coordinate of the bottom-right corner of the rectangle
    :param y2: int, y-coordinate of the bottom-right corner of the rectangle
    :param fill_value: int, fill value for the rectangle
    :return: numpy array, image with the rectangle drawn
    """

    image[y1:y2, x1:x2] = fill_value
    return image

def draw_text(image, text, x, y, font):
    """
    Draws the specified text on the input image at the given (x, y) coordinates using the provided font.

    :param image: numpy array, input image
    :param text: str, text to be drawn on the image
    :param x: int, x-coordinate of the text position
    :param y: int, y-coordinate of the text position
    :param font: PIL.ImageFont instance, font to be used for the text
    :return: numpy array, image with the text drawn
    """

    image_pil = Image.fromarray(image)
    draw = ImageDraw.Draw(image_pil)
    draw.text((x, y), text, font=font, fill=0)
    return np.array(image_pil)

def calculate_scalebar_parameters(img_height, img_width, pixel_length, position=None):
    """
    Calculates the parameters necessary to draw a scale bar on an image.

    :param img_height: int, the height of the input image in pixels.
    :param img_width: int, the width of the input image in pixels.
    :param pixel_length: float, the length of one pixel in nanometers.
    :param position: tuple, optional, a tuple containing the (x, y) coordinates of the top-left corner of the scale bar.
                     If None, the scale bar is placed in the lower-right corner of the image.
    :return: tuple, a tuple containing the following scale bar parameters:
             - scale_thickness (int): the thickness of the scale bar in pixels.
             - scale_length_in_nm (float): the length of the scale bar in nanometers.
             - x1 (int): the x-coordinate of the top-left corner of the scale bar.
             - y1 (int): the y-coordinate of the top-left corner of the scale bar.
             - x2 (int): the x-coordinate of the bottom-right corner of the scale bar.
             - y2 (int): the y-coordinate of the bottom-right corner of the scale bar.
    """

    # Define the scalebar parameters
    scale_thickness = img_height // 125
    scale_length_in_nm = PIXEL_LENGTH_TO_SCALEBAR_LENGTH_IN_nm_DICT[pixel_length]
    scale_length_in_px = int(round(scale_length_in_nm/pixel_length))

    # Determine the scalebar position and dimensions
    if position is None:
        x_buffer = img_width // 30
        y_buffer = img_height // 30
        x1, y1 = img_width - x_buffer - scale_length_in_px, img_height - y_buffer - scale_thickness
    else:
        x1, y1 = position
    x2, y2 = x1 + scale_length_in_px, y1 + scale_thickness

    return scale_thickness, scale_length_in_nm, x1, y1, x2, y2

def draw_scalebar(image, pixel_length, position=None):
    """
    Draws a scale bar and corresponding annotations on the input image based on the provided pixel length.

    :param image: numpy array, input image
    :param pixel_length: float, pixel length in nanometers
    :param position: tuple, optional, x and y coordinates of the scale bar's starting position; if not provided,
                     the scale bar will be placed in the lower left corner of the image
    :return: numpy array, image with the scale bar and annotations drawn
    """

    img_height, img_width = image.shape
    scale_thickness, scale_length_in_nm, x1, y1, x2, y2 = calculate_scalebar_parameters(img_height, img_width, pixel_length, position)

    # Draw the scalebar and text on the image
    image = draw_rectangle(image, x1 - 30, y1 - 60, x2 + 30, y2 + 60, 65535)  # Frame
    image = draw_rectangle(image, x1, y1, x2, y2, 0)
    # new rectangle for white background goes here

    font_size = img_height // 90
    font = ImageFont.truetype("arial.ttf", font_size)

    length_scale = "nm"
    if scale_length_in_nm >= 1000:
        scale_length_in_nm /= 1000
        length_scale = "µm"

    # Get the text width using getbbox
    text = f"{scale_length_in_nm} {length_scale}"
    text_width, text_height = font.getbbox(text)[2:]

    # Add the scale length to the image
    text_x = (x1 + x2) / 2 - text_width / 2
    text_y = y1 - text_height - scale_thickness / 2
    image = draw_text(image, text, text_x, text_y, font)

    # Add the pixel length to the image
    text = f"{pixel_length} {length_scale}/px"
    text_y = y1 + scale_thickness
    image = draw_text(image, text, text_x, text_y, font)

    return image

def combine_images_horizontally(images):
    """
    Combines a list of images horizontally, assuming all input images have the same shape.

    :param images: list of numpy arrays, input images
    :return: numpy array, combined image
    """

    if not all(img.shape == images[0].shape for img in images[1:]):
        raise ValueError("All images should have the same size.")
    return np.concatenate(images, axis=1)

def process_single_image(image, pixel_size):
    """
    Processes a single image for equalization, rescaling, and scalebar drawing.

    :param image: numpy array, input image
    :param pixel_size: float, pixel size of the image in nanometers
    :return: numpy array, processed image with added scalebar and annotations
    """

    original = image
    equalize_adapthist = exposure.equalize_adapthist(image, clip_limit=0.01, nbins=256)

    original_rescaled = rescale_intensity_to_16bit(original)
    equalize_adapthist_rescaled = rescale_intensity_to_16bit(equalize_adapthist)

    original_rescaled = draw_scalebar(original_rescaled, pixel_size)
    equalize_adapthist_rescaled = draw_scalebar(equalize_adapthist_rescaled, pixel_size)

    images = [original_rescaled, equalize_adapthist_rescaled]
    combined_image = combine_images_horizontally(images)

    return combined_image

def read_mrc_file_to_array(mrc_file_path):
    """
    Reads an MRC file and returns its data as a numpy array.

    :param mrc_file_path: str, path to the MRC file
    :return: tuple of (numpy array, str), the MRC data array and the output TIFF file path
    """

    with mrcfile.open(mrc_file_path, mode='r', permissive=True) as mrc:
        mrc_data = np.array(mrc.data)

    return mrc_data

def multithread_process_single_images(mrc_data_array, pixel_sizes):
    """
    Processes multiple images in parallel, each image with its corresponding pixel size.

    :param mrc_data_array: numpy array, input MRC data array
    :param pixel_sizes: list of float, list of pixel sizes in nanometers
    :return: numpy array, array of processed images
    """

    processed_images = []
    with concurrent.futures.ThreadPoolExecutor() as executor:
        results = executor.map(process_single_image, mrc_data_array, pixel_sizes)
        for result in results:
            processed_images.append(result)

    return np.array(processed_images)

def process_mrc_file_main(mrc_file_path):
    """
    Processes an MRC file containing multiple images, adds scale bars and annotations, and saves the result as a TIFF file.

    :param mrc_file_path: str, path to the input MRC file
    """

    mrc_data_array = read_mrc_file_to_array(mrc_file_path)
    magnifications = read_magnifications_from_mdoc(mrc_file_path)
    pixel_sizes = map_magnifications_to_pixel_size(magnifications)
    processed_images_arr = multithread_process_single_images(mrc_data_array, pixel_sizes)

    base_path, extension = os.path.splitext(mrc_file_path)

    if SPLIT_OUTPUT_TO_SINGLE_FILES is False:
        output_tiff_path = base_path + "_processed.tiff"
        tifffile.imwrite(output_tiff_path, processed_images_arr);
    else:
        output_tiff_path = base_path + "_processed"
        for i in range(len(processed_images_arr)):
            tifffile.imwrite(output_tiff_path+'_'+str(i)+'.tiff', processed_images_arr[i]);

def start_gui():
    root = tk.Tk()
    root.geometry("600x200")
    app = Application(master=root)
    app.mainloop()

class Application(tk.Frame):
    def __init__(self, master=None):
        super().__init__(master)
        self.master = master
        self.master.title("Technai File Processor")
        self.style = ttk.Style()
        self.style.theme_use("clam")

        # Define new style for orange button
        self.style.configure("Orange.TButton", foreground="black", background="orange")

        # Define new style for green button
        self.style.configure("Green.TButton", foreground="black", background="green")

        self.pack(fill="both", expand=True)
        self.create_widgets()

    def create_widgets(self):
        self.selected_files_listbox = tk.Listbox(self, height=10)
        self.selected_files_listbox.pack(side="left", padx=10, pady=10, fill="both", expand=True)

        self.browse_button = ttk.Button(self, text="Browse", command=self.browse_files)
        self.browse_button.pack(side="top", padx=10, pady=10)

        # Set initial button style to green
        self.process_button = ttk.Button(self, text="Process .mrc", command=self.process_files, style="Green.TButton")
        self.process_button.pack(side="top", padx=10, pady=10)

    def browse_files(self):
        file_path = filedialog.askopenfilename(filetypes=[("MRC files", "*.mrc")])
        if file_path:
            self.selected_files_listbox.insert(tk.END, file_path)

    def process_files(self):

        for index in range(self.selected_files_listbox.size()):
            file_path = self.selected_files_listbox.get(index)
            process_mrc_file_main(file_path)

        # Change button color back to green
        self.process_button.configure(style="Green.TButton")
        self.process_button.update()

        self.selected_files_listbox.delete(0, tk.END)

    def quit(self):
        self.master.destroy()

def cmd_wrapper():
    # remove the first argument, which is the name of the script itself
    args = sys.argv[1:]

    for options in args:
        if options == '-s':
            print('-s option is activated. Processed MRC stack files will be saved as individual TIFF images.');
            global SPLIT_OUTPUT_TO_SINGLE_FILES;
            SPLIT_OUTPUT_TO_SINGLE_FILES = True;

        else:
            mrc_file_path = options;
            try:
                print(f"Started processing {mrc_file_path}")
                process_mrc_file_main(mrc_file_path)
                print(f"Finished processing {mrc_file_path}")
            except:
                print(f"Failed processing {mrc_file_path}, continuing with next .mrc stack in line")

if __name__ == "__main__":
    args = sys.argv[1:]
    if not args:
        start_gui()
    else:
        cmd_wrapper()


#!/usr/bin/env python
# coding: utf-8

import torch
import torch.nn as nn
from torch.utils.data import Dataset
from transformers import BertTokenizer
from torch.nn.functional import  log_softmax
import os
import math
import numpy as np
import torch.nn.functional as F
import json
import re

tgt_tokenizer = BertTokenizer("./tgtvocab.txt")
src_tokenizer = BertTokenizer("./srcvocab.txt")

def pre_process():
    with open('./dataset_process/train_input.txt', 'r') as f:
        input_lines = f.readlines()
    with open('./dataset_process/validate_input.txt', 'r') as f:
        input_lines2 = f.readlines()
    with open('./dataset_process/test_input.txt', 'r') as f:
        input_lines3 = f.readlines()

    input_train = input_lines
    input_val = input_lines2
    input_test = input_lines3

    # dataset pre_process :
    input_set = set()
    links = set()
    train_nodes = []
    valid_nodes = []
    test_nodes = []
    for i in range(len(input_train)):
        line = input_train[i]
        nodes = re.findall(r'N(\d+)\[label=\"(.*?)\"\]', line)
        link =  re.findall(r'N(\d+) -> N(\d+)',line)
        links.update(set(link))
        train_nodes.append(list(link))
        nodes_set = set(nodes)
        input_set.update(nodes_set)

    for i in range(len(input_val)):
        line = input_val[i]
        nodes = re.findall(r'N(\d+)\[label=\"(.*?)\"\]', line)
        nodes_set = set(nodes)
        link =  re.findall(r'N(\d+) -> N(\d+)',line)
        links.update(set(link))
        valid_nodes.append(list(link))
        input_set.update(nodes_set)

    for i in range(len(input_test)):
        line = input_test[i]
        nodes = re.findall(r'N(\d+)\[label=\"(.*?)\"\]', line)
        nodes_set = set(nodes)
        link =  re.findall(r'N(\d+) -> N(\d+)',line)
        links.update(set(link))
        test_nodes.append(list(link))
        input_set.update(nodes_set)

    content = {}      # node content
    for i in input_set:
        content[str(i[0])]=i[1]

    train, valid, test = [], [], []   # 句子包含的node
    for i in train_nodes:
        node = set()
        for j in i:
            node.update(j)
        train.append(list(node))
    for i in valid_nodes:
        node = set()
        for j in i:
            node.update(j)
        valid.append(list(node))
    for i in test_nodes:
        node = set()
        for j in i:
            node.update(j)
        test.append(list(node))
    train[7186] = ['7299']
    train[7187] = ['7299']

    return train, valid, test

class TranslationDataset(Dataset):

    def __init__(self, src_list, tgt_list, src_tokenizer, tgt_tokenizer, src_max_len, tgt_max_len):
        self.src_tokens = self.load_tokens(src_list, src_tokenizer, max_len = src_max_len)
        self.tgt_tokens = self.load_tokens(tgt_list, tgt_tokenizer, max_len = tgt_max_len)

    def __getitem__(self, index):
        return self.src_tokens[index], self.tgt_tokens[index]

    def __len__(self):
        return len(self.src_tokens)

    def load_tokens(self, input_list, tokenizer, max_len):
        tokens_list = []  #labels
        for i in input_list:
            tokens_list.append(tokenizer.encode(i, padding='max_length', truncation= True, max_length = max_len))
        return tokens_list

def collate_fn(batch):
    tgt = []
    src = []
    tgt_y = []
    for (_src, _tgt) in batch:
        src.append(_src)
        tgt_y.append(_tgt)
        tgt.append(_tgt)
    tgt_y = torch.tensor(tgt_y)[:,1:]
    src = torch.tensor(src)
    tgt = torch.tensor(tgt)[:,:-1]
    tgt[tgt == 3] = 0
    return src, tgt, tgt_y

class TranslationLoss(nn.Module):
    def __init__(self):
        super(TranslationLoss, self).__init__()
        self.criterion = nn.KLDivLoss(reduction="sum")
        self.padding_idx = 0

    def forward(self, x, target):

        x = log_softmax(x, dim=-1)
        true_dist = torch.zeros(x.size())
        true_dist.scatter_(1, target.data.unsqueeze(1), 1)
        mask = torch.nonzero(target.data == self.padding_idx)
        if mask.dim() > 0:
            true_dist.index_fill_(0, mask.squeeze(), 0.0)
        return self.criterion(x, true_dist.clone().detach())


import cv2
import torch
import random

from datas.data_utils import get_label_file, encode_label

class TrainValDataset(torch.utils.data.Dataset):
    def __init__(self, dataset, train_image_list, val_image_list, transform, class_rgb_values):
        self.dataset = dataset
        self.train_image_list = train_image_list
        self.val_image_list = val_image_list
        self.transform = transform
        self.class_rgb_values = class_rgb_values

    def __getitem__(self, index):
        train_image_file = self.train_image_list[index]
        val_index = random.choice(range(len(self.val_image_list)))
        val_image_file = self.val_image_list[val_index]

        train_label_file = get_label_file(train_image_file, self.dataset)
        val_label_file = get_label_file(val_image_file, self.dataset)

        train_image = cv2.imread(train_image_file)
        val_image = cv2.imread(val_image_file)
        train_label_bgr = cv2.imread(train_label_file)
        val_label_bgr = cv2.imread(val_label_file)
        train_label = encode_label(train_label_bgr, self.class_rgb_values)
        val_label = encode_label(val_label_bgr, self.class_rgb_values)

        train_transformed = self.transform(image=train_image, mask=train_label)
        val_transformed = self.transform(image=val_image, mask=val_label)
        train_image, train_label = train_transformed["image"], train_transformed["mask"]
        val_image, val_label = val_transformed["image"], val_transformed["mask"]

        return train_image, train_label, val_image, val_label

    def __len__(self):
        return len(self.train_image_list)


import os
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.image import load_img, img_to_array
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt

# 载入模型
filepath1 = r'C:\Users\user\Desktop\check_stock\converted_keras'
model = keras.models.load_model(filepath1 + '/your_model_name.h5', compile=False)
# model = keras.models.load_model(filepath1 + '/keraV3.h5', compile=False)
# 定义标签映射字典
label_map = {
    0: -16,
    1: -9,
    2: -5,
    3: -2,
    4: 0,
    5: 1,
    6: 4,
    7: 6,
    8: 10,
    9: 14,
    10: 23,
    11: 999
}

# 定义函数进行图像分类
def classify_image(image_path):
    img = load_img(image_path, target_size=(224, 224))
    img_array = img_to_array(img)
    img_array = np.expand_dims(img_array, axis=0)
    prediction = model.predict(img_array)
    predicted_label_idx = np.argmax(prediction)
    predicted_label = label_map[predicted_label_idx]
    return predicted_label

# 收集真实标签和预测标签
true_labels = []
predicted_labels = []

# 指定包含图像的文件夹路径
base_dir = r'C:\Users\user\Desktop\check_stock\image'
subfolders = os.listdir(base_dir)

# 对每个文件夹中的图像进行分类
for folder in subfolders:
    folder_path = os.path.join(base_dir, folder)
    if os.path.isdir(folder_path):
        image_files = os.listdir(folder_path)
        for image_file in image_files:
            image_path = os.path.join(folder_path, image_file)
            true_label = int(folder)
            predicted_label = classify_image(image_path)
            true_labels.append(true_label)
            predicted_labels.append(predicted_label)

# 计算混淆矩阵
cm = confusion_matrix(true_labels, predicted_labels, labels=list(label_map.values()))

# 计算准确率
accuracy = np.trace(cm) / np.sum(cm)
print("Accuracy:", accuracy)

# 可视化混淆矩阵
disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=list(label_map.values()))
disp.plot(cmap='Blues', values_format='.0f')

plt.title("Confusion Matrix")
plt.xlabel("Predicted Label")
plt.ylabel("True Label")

# 显示准确率
plt.text(0.5, -0.3, f'Accuracy: {accuracy:.2f}', size=12, ha="center", transform=plt.gca().transAxes)

plt.show()


# Router & Template_Render
from flask import  Blueprint, render_template
# 「.env」で設定した環境変数を使用する
import settings
# ログ出力のために、loggerをimportする
from logoutput import logger
import traceback
# Post受信をするためにFlask_requestをimportする
from flask import request, jsonify

# import requests
import pprint

import pandas as pd

# PythonからPythonScriptを呼び出すためのモジュール！ => プログラム内でコマンド実行！
# import subprocess

# Generate Router Instance => Base_URLの設定はここ！
router = Blueprint('router', __name__ )

# ルート, Method: GET
@router.route('/', methods=['GET'])
def index():
    try :
        logger.debug('Flask-API-ルート起動！')
        return render_template('index.html')
    except Exception as error :
        error_msg:str = traceback.format_exc()
        logger.error(f"APIルート: Error:  {error_msg}")

# Robotama_エンドポイント, Method: GET
@router.route('/robotama', methods=['GET'])
def robotama():
  return 'Robotamaなのだ！！'

# FrontAppの情報を確認するためのエンドポイント, Method: GET
@router.route('/front_info', methods=['GET'])
def front_info():
    logger.debug('FrontApp-Info-アクセス！')
    front_end_url = settings.FRONT_APP_URL
    msg = f"FrontAppのURLは: {front_end_url}"
    return msg

# CSVの中身の情報、ColumnやRow_Dataを表示するためのエンドポイント, Method: POST
@router.route('/create_csv_info', methods=['POST'])
def create_csv_info():
    logger.debug('create_csv_info-アクセス！')
    logger.debug(request.files['file'])
    file = request.files['file']

    if file:
        # CSVファイルの読み込み
        df = pd.read_csv(file, encoding='utf-8')
        pprint.pprint(df)
        # DataFrameをJSON形式に変換
        json_data = df.to_json(orient='records')
        logger.debug(json_data)
        return json_data

    else:
        responseData = {
            'error_msg': 'Fileが存在しません！'
        }
        return jsonify(responseData)

# FrontAppの情報を確認するためのエンドポイント, Method: GET
@router.route('/create_word_cloud', methods=['GET'])
def create_word_cloud():
    return 'create_word_cloud'

# @router.before_request
# def before_request(response):
#   response.headers.add('Access-Control-Allow-Origin', '*')
#   response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
#   response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
#   return response

# Requestの後処理
@router.after_request
def after_request(response):
  response.headers.add('Access-Control-Allow-Origin', '*')
  response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
  response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
  return response


import sys
from PyQt5.QtWidgets import QApplication, QSystemTrayIcon, QMenu
from PyQt5.QtGui import QIcon

from PyQt5.QtWidgets import QPushButton, QApplication, QDialog, QLabel, QLineEdit, QGridLayout, QComboBox, QCheckBox
from PyQt5.QtCore import Qt
import sys
import setup_ui

# class setupDialog(QDialog):
#     def __init__(self):
#         super().__init__()
#         self.setupUI_d()

#     def setupUI_d(self):
#         global time_option_key, time_reserve_key, interval, start_time, end_time
#         time_option_key = 1
#         time_reserve_key = 0
#         interval = '600'
#         start_time = '0'
#         end_time = '0'

#         self.setGeometry(1500, 800, 250, 100)
#         self.setWindowTitle("Setup")

#         label1 = QLabel("Check interval(s): ")
#         label2 = QLabel("Reserved action: ")
#         self.lineEdit1 = QLineEdit('600',self)
#         self.cb = QComboBox(self)
#         self.cb.addItem('20:00 ~ 08:00')
#         self.cb.addItem('08:00 ~ 20:00')
#         self.cb.activated[str].connect(self.cb_switch)
#         self.qcb = QCheckBox('Reserve', self)
#         self.qcb.stateChanged.connect(self.reserve_func)
#         self.pushButton1= QPushButton("OK")
#         self.pushButton1.clicked.connect(self.pushButtonClicked)

#         layout = QGridLayout()
#         layout.addWidget(label1, 0, 0)
#         layout.addWidget(self.lineEdit1, 0, 1)
#         layout.addWidget(self.pushButton1, 0, 2)
#         layout.addWidget(label2, 1, 0)
#         layout.addWidget(self.cb, 1, 1)
#         layout.addWidget(self.qcb, 1, 2)
#         self.setLayout(layout)

#     def cb_switch(self, text):
#         global time_option_key
#         if text == '20:00 ~ 08:00':
#             time_option_key = 1
#         else:
#             time_option_key = 2
#         return time_option_key

#     def reserve_func(self, state):
#         global time_reserve_key
#         if state == Qt.Checked:
#             time_reserve_key = 1
#         else:
#             time_reserve_key = 0
#         return time_reserve_key

#     def pushButtonClicked(self):
#         global interval, start_time, end_time
#         interval = self.lineEdit1.text()
#         if time_option_key == 1:
#             start_time = '20:00'
#             end_time = '08:00'
#         else:
#             start_time = '08:00'
#             end_time = '20:00'
#         # self.close()
#         return start_time, end_time, interval

# if __name__ == "__main__":
#     app = QApplication(sys.argv)
#     window = setupDialog()
#     window.show()
#     app.exec_()

import threading
import ctypes
import time

class thread_with_exception(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)

    def run(self):
        popSetup = setupDialog()
        popSetup.exec_()

    def get_id(self):
        # returns id of the respective thread
        if hasattr(self, '_thread_id'):
            return self._thread_id
        for id, thread in threading._active.items():
            if thread is self:
                return id

    def raise_exception(self):
        thread_id = self.get_id()
        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id,
              ctypes.py_object(SystemExit))
        print('stop')
        if res > 1:
            ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id, 0)
            print('Exception raise failure')
        return thread_id

app = QApplication(sys.argv)
trayIcon = QSystemTrayIcon(QIcon('message.ico'), parent = app)
trayIcon.setToolTip('HPC SMS Alarm')
trayIcon.show()

def callSetupdialog():
    popSetup = setup_ui.setupDialog()
    popSetup.exec_()

menu = QMenu()
def menu1():
    global exitAction1, exitAction2 ,exitAction3 ,exitAction4
    exitAction1 = menu.addAction('Start')
    exitAction2 = menu.addAction('Setup')
    exitAction3 = menu.addAction('Exit')
    exitAction1.triggered.connect(menu2)
    exitAction2.triggered.connect(callSetupdialog)
    exitAction3.triggered.connect(app.quit)

    trayIcon.setContextMenu(menu)

def menu2():
    global exitAction1, exitAction2 ,exitAction3 ,exitAction4
    menu.removeAction(exitAction1)
    menu.removeAction(exitAction2)
    menu.removeAction(exitAction3)
    menu.removeAction(exitAction4)
    exitAction5 = menu.addAction('Stop')
    exitAction6 = menu.addAction('Stop')
    exitAction7 = menu.addAction('Setup')
    exitAction8 = menu.addAction('Exit')
    # trayIcon.setContextMenu(menu)

menu1()
sys.exit(app.exec_())

# if __name__ == "__main__":
#     app = QApplication(sys.argv)
#     window = MyWindow()
#     window.show()
#     app.exec_()


import os
import time
import base64

import cv2
from nms import nms
import numpy as np

import pytesseract
import re

from PIL import Image
from .pdf_to_cv import read
from .crop_image import process_image

def deskew(image):
    print("Deskewing Image")
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    gray = cv2.bitwise_not(gray)

    # threshold the image, setting all foreground pixels to
    # 255 and all background pixels to 0
    thresh = cv2.threshold(gray, 0, 255,
                           cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]

    # grab the (x, y) coordinates of all pixel values that
    # are greater than zero, then use these coordinates to
    # compute a rotated bounding box that contains all
    # coordinates
    coords = np.column_stack(np.where(thresh > 0))
    angle = cv2.minAreaRect(coords)[-1]

    # the `cv2.minAreaRect` function returns values in the
    # range [-90, 0); as the rectangle rotates clockwise the
    # returned angle trends to 0 -- in this special case we
    # need to add 90 degrees to the angle
    if angle < -45:
        angle = -(90 + angle)

    # otherwise, just take the inverse of the angle to make
    # it positive
    else:
        angle = -angle

    # rotate the image to deskew it
    (h, w) = image.shape[:2]
    center = (w // 2, h // 2)
    M = cv2.getRotationMatrix2D(center, angle, 1.0)
    rotated = cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE)
    # draw the correction angle on the image so we can validate it

    #Resize Image
    img = cv2.resize(rotated,None,fx=1.5,fy=1.5,interpolation=cv2.INTER_CUBIC)

    return img

def thresholding(img):
    print("Adjusting Image Thresholds")
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    img = cv2.bilateralFilter(img,15,75,75)
    cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 31, 3)
    #plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
    return img

# Add Sanity checks such as masking area ,etc.
def mask(img,dfs):
    masked = img.copy()
    for index,df in dfs.iterrows():
        cv2.rectangle(masked, (df['left'],df['top']),(df['left']+df['width'],df['top']+df['height']),(0,255,0),-1)
    return masked

def mask_image(img ,thresh=15, crop=True):
    #img =cv2.imread(r'test04.png')
    if crop:
        print("Cropping Image [This may cause bugs in already cropped images]")
        PILImage = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
        PILImage = process_image(PILImage)
        img = cv2.cvtColor(np.array(PILImage), cv2.COLOR_RGB2BGR)

    orig = img = deskew(img)
    img = thresholding(img)

    CONFIDENCE_THRESHOLD=thresh
    config = ("--oem 3 --psm 11 -l eng -c preserve_interword_spaces=1" )
    print("Extracting text with confidence threshold: "+str(CONFIDENCE_THRESHOLD))
    extracted_data = pytesseract.image_to_data(img, output_type=pytesseract.Output.DATAFRAME, config=config)
    extracted_data = extracted_data.dropna()
    extracted_data.text = extracted_data.text.map(lambda x: re.sub(r'[^a-zA-Z0-9\/]+', '', x))
    extracted_data = extracted_data[(extracted_data.conf > CONFIDENCE_THRESHOLD)  & (extracted_data.text.str.len()>2)]

    # dob = extracted_data[extracted_data.text.str.match(r'\d+/\d+/\d+')]
    # if dob.empty:
    #     dob = extracted_data[extracted_data.text.str.match(r'(?i)\bDOB\b|\bYEAR\b')].block_num
    #     if not dob.empty:
    #         dob = extracted_data[extracted_data.block_num==int(dob)].tail(1)
    #     else:
    #         print("DOB Not Found")

    aadhar_uid = extracted_data[(extracted_data.text.str.match(r'\d{4}'))]
    # Filter blocks containing more than 2 sets of 4 digit numbers
    aadhar_uid=aadhar_uid.groupby('block_num').filter(lambda x: len(x) >= 2)
    if aadhar_uid.empty:
        print("[ERROR] Unable to process image, no UID found")
        return True, "", ""
    to_mask = aadhar_uid.groupby('block_num').head(2).reset_index(drop=True)
    # print(''.join(aadhar_uid['text'].tolist()))
    aadhar = ''.join(aadhar_uid['text'].tolist())
    try:
        masked_image = mask(orig,to_mask)
    except Exception as e:
        print("[ERROR] Failed Masking Image")
        print(e)
        print("[ERROR] Exception Caught, Continuing")
    print("Masked Successfully")
    return False, masked_image, aadhar


"""
Elastic bounds
===============

This example shows the comparison between different elastic bounds to Nur's cirtical porosity model: Voigt and Reuss Bounds, Voigt–Reuss–Hill Average,Hashin-Shtrikmann bounds
"""
# %%

import numpy as np
import matplotlib.pyplot as plt
plt.rcParams['font.size']=14
plt.rcParams['font.family']='arial'

# %%
import rockphypy # import the module for rock physics
from rockphypy import EM # import the "effective medium" EM module
# import the 'Fluid' module
from rockphypy import Fluid

# %%
#
# Voigt and Reuss Bounds
# ~~~~~~~~~~~~~~~~~~~~~~
# At any given volume fraction of constituents, the effective modulus will fall between the bounds. In the lecture we learn that the **Voigt upper bound** :math:`M_v` of N phases are defined as:
#
# .. math::
#       M_V=\sum_{i=1}^{N}f_iM_i
#
#
# The **Reuss lower bound** is
#
# .. math::
#        \frac{1}{M_R} =\sum_{i=1}^{N}\frac{f_i}{M_i}
#
#
# where :math:`f_i` is the volume fraction of the ith phase and :math:`M_i` is the elastic bulk or shear modulus of the ith phase.
#
#
# Voigt–Reuss–Hill Average Moduli Estimate
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#  This average is simply the arithmetic average of the Voigt upper
# bound and the Reuss lower bound.
#
# .. math::
#       M_{VRH}=\frac{M_V+M_R}{2}
#
#
# :bell:
#   For fluid-solid composite, calculation of Reuss, Voigt and VRH bounds are straightforward. Here we give a generalized function to compute effective moduli of N-phases composite:
#
#
# Hashin-Shtrikmann bounds
# ~~~~~~~~~~~~~~~~~~~~~~~~
# The Voigt and Reuss bounds are simply arithmetic and harmonic averages. The **Hashin-Shtrikman bounds** are stricter than the Reuss-Voigt bounds. The *two-phase* HS bounds can be written as:
#
# .. math::
#       K^{\mathrm{HS} \pm}=K_{1}+\frac{f_{2}}{\left(K_{2}-K_{1}\right)^{-1}+f_{1}\left(K_{1}+\frac{4}{3} \mu_{1}\right)^{-1}}
#
#
# .. math::
#       \mu^{\mathrm{HS} \pm}=\mu_{1}+\frac{f_{2}}{\left(\mu_{2}-\mu_{1}\right)^{-1}+2 f_{1}\left(K_{1}+2 \mu_{1}\right) /\left[5 \mu_{1}\left(K_{1}+\frac{4}{3} \mu_{1}\right)\right]}
#
#
# where the superscript +/− indicates upper or lower bound respectively. :math:`K_1` and :math:`K_2` are the bulk moduli of individual phases; :math:`\mu_1` and :math:`\mu_2` are the shear moduli of individual phases; and :math:`f_1` and :math:`f_2` are the volume fractions of individual phases. The upper and lower bounds are computed by interchanging which material is termed 1 and which is termed 2. The expressions yield the upper bound when the stiffest material is termed 1 and the lower bound when the softest material is termed 1. The expressions shown in the lectures represents the HS bound for *fluid-solid composite*.
#
# Examples
# ~~~~~~~~
# Let's compute effective bulk and shear moduli of a water saturated rock using different bound models.
#
# Here we also make a comparision with Nur's critical porosity model as introduced in the previous example.
#

# %%

# specify model parameters
phi=np.linspace(0,1,100,endpoint=True) # solid volume fraction = 1-phi
K0, G0= 37,44 # moduli of grain material
Kw, Gw= 2.2,0 # moduli of water
# VRH bounds
volumes= np.vstack((1-phi,phi)).T
M= np.array([K0,Kw])
K_v,K_r,K_h=EM.VRH(volumes,M)
# Hashin-Strikmann bound
K_UHS,G_UHS= EM.HS(1-phi, K0, Kw,G0,Gw, bound='upper')
# Critical porosity model
phic=0.4 # Critical porosity
phi_=np.linspace(0.001,phic,100,endpoint=True) # solid volume fraction = 1-phi
K_dry, G_dry= EM.cripor(K0, G0, phi_, phic)# Compute dry-rock moduli
Ksat, Gsat = Fluid.Gassmann(K_dry,G_dry,K0,Kw,phi_)# saturate rock with water

# %%

# plot
plt.figure(figsize=(6,6))
plt.xlabel('Porosity')
plt.ylabel('Bulk modulus [GPa]')
plt.title('V, R, VRH, HS bounds')
plt.plot(phi, K_v,label='K Voigt')
plt.plot(phi, K_r,label='K Reuss = K HS-')
plt.plot(phi, K_h,label='K VRH')
plt.plot(phi, K_UHS,label='K HS+')
plt.plot(phi_, Ksat,label='K CriPor')
plt.legend(loc='best')
plt.grid(ls='--')

# %%
# **Reference**: Mavko, G., Mukerji, T. and Dvorkin, J., 2020. The rock physics handbook. Cambridge university press.
#


#! /usr/bin/python

import dropbox

## testing upload dropbox

access_token="LADJrEFyvzAAAAAAAAAAPmxqPUmb_CQlDVK-iUvXLKyk6DbxuoV_0_UVoqTqZoIa"

dbx = dropbox.Dropbox(access_token)

## account information
dbx.users_get_current_account()

## get all files in dropbox
for entry in dbx.files_list_folder('').entries:
    print(entry.name)

file_name='/home/sree/code/emodrive/images/neutral.jpg'

upload_path="/images/neutral.jpg"
dbx=dropbox.Dropbox(access_token)

with open(file_name, 'rb') as f:
    dbx.files_upload(f.read(),upload_path,mute=True)


from __future__ import absolute_import, print_function, unicode_literals

__metaclass__ = type

import bz2
import gzip
import os
import shutil
import stat
import tempfile
import unittest

try:
    import lzma
except ImportError:
    from backports import lzma

from lp.archivepublisher.utils import RepositoryIndexFile
from lp.soyuz.enums import IndexCompressionType

class TestRepositoryArchiveIndex(unittest.TestCase):

    def setUp(self):
        """Create temporary directories to be used in tests.

        'root': final destination for repository files.
        'temp_root': temporary destination for reporsitory files.
        """
        self.root = tempfile.mkdtemp()
        self.temp_root = tempfile.mkdtemp()

    def tearDown(self):
        """Purge temporary files created on `setUp`."""
        for path in [self.root, self.temp_root]:
            shutil.rmtree(path)

    def getRepoFile(self, filename, compressors=None):
        """Return a `RepositoryIndexFile` for the given filename.

        The `RepositoryIndexFile` is created with the test 'root' and
        'temp_root'.
        """
        if compressors is None:
            compressors = [
                IndexCompressionType.GZIP,
                IndexCompressionType.BZIP2,
                IndexCompressionType.XZ,
                ]
        return RepositoryIndexFile(
            os.path.join(self.root, filename), self.temp_root, compressors)

    def testWorkflow(self):
        """`RepositoryIndexFile` workflow.

        On creation, 2 temporary files are atomically created in the
        'temp_root' location (mkstemp). One for storing the plain contents
        and other for the corresponding compressed contents. At this point,
        no files were created in the 'root' location yet.

        Once the `RepositoryIndexFile` is closed, the files in 'temp_root'
        are closed and moved to 'root' with their expected names.

        Additionally, the resulting files are made readable and writable by
        their group and readable by others.
        """
        repo_file = self.getRepoFile('boing')

        self.assertEqual(0, len(os.listdir(self.root)))
        self.assertEqual(3, len(os.listdir(self.temp_root)))

        repo_file.close()

        self.assertEqual(3, len(os.listdir(self.root)))
        self.assertEqual(0, len(os.listdir(self.temp_root)))

        resulting_files = sorted(os.listdir(self.root))
        self.assertEqual(
            ['boing.bz2', 'boing.gz', 'boing.xz'], resulting_files)

        for filename in resulting_files:
            file_path = os.path.join(self.root, filename)
            mode = stat.S_IMODE(os.stat(file_path).st_mode)
            self.assertTrue(
                (stat.S_IWGRP | stat.S_IRGRP | stat.S_IROTH) & mode)

    def testWrite(self):
        """`RepositoryIndexFile` writing.

        Writes to a `RepositoryIndexFile` happens simultaneously in both
        of its counter-parts (plain and gzipped contents). Once the file
        is closed both resulting files have the same contents, one plain and
        other compressed.
        """
        repo_file = self.getRepoFile('boing')
        repo_file.write('hello')
        repo_file.close()

        gzip_file = gzip.open(os.path.join(self.root, 'boing.gz'))
        gzip_content = gzip_file.read()
        bz2_content = bz2.decompress(
            open(os.path.join(self.root, 'boing.bz2')).read())
        xz_content = lzma.open(os.path.join(self.root, 'boing.xz')).read()

        self.assertEqual(gzip_content, bz2_content)
        self.assertEqual(gzip_content, xz_content)
        self.assertEqual('hello', gzip_content)

        # gzip is compressed as if with "-n", ensuring that the hash
        # doesn't change just because we're compressing at a different
        # point in time. The filename is also blank, but Python's gzip
        # module discards it so it's hard to test.
        self.assertEqual(0, gzip_file.mtime)

    def testCompressors(self):
        """`RepositoryIndexFile` honours the supplied list of compressors."""
        repo_file = self.getRepoFile(
            'boing',
            compressors=[
                IndexCompressionType.UNCOMPRESSED, IndexCompressionType.XZ])
        repo_file.close()
        self.assertEqual(['boing', 'boing.xz'], sorted(os.listdir(self.root)))

    def testUnreferencing(self):
        """`RepositoryIndexFile` unreferencing.

        When a `RepositoryIndexFile` is unreferenced it takes care of
        removing the created files in the 'temp_root'.
        """
        repo_file = self.getRepoFile('boing')

        self.assertEqual(0, len(os.listdir(self.root)))
        self.assertEqual(3, len(os.listdir(self.temp_root)))

        del repo_file

        self.assertEqual(0, len(os.listdir(self.root)))
        self.assertEqual(0, len(os.listdir(self.temp_root)))

    def testRootCreation(self):
        """`RepositoryIndexFile` creates given 'root' path if necessary."""
        missing_root = os.path.join(self.root, 'donotexist')
        compressors = [
            IndexCompressionType.GZIP,
            IndexCompressionType.BZIP2,
            IndexCompressionType.XZ,
            ]
        repo_file = RepositoryIndexFile(
            os.path.join(missing_root, 'boing'), self.temp_root, compressors)

        self.assertFalse(os.path.exists(missing_root))

        repo_file.close()

        self.assertEqual(
            ['boing.bz2', 'boing.gz', 'boing.xz'],
            sorted(os.listdir(missing_root)))

    def testMissingTempRoot(self):
        """`RepositoryIndexFile` cannot be given a missing 'temp_root'."""
        missing_temp_root = os.path.join(self.temp_root, 'donotexist')
        self.assertRaises(
            AssertionError, RepositoryIndexFile,
            os.path.join(self.root, 'boing'), missing_temp_root,
            [IndexCompressionType.UNCOMPRESSED])

    def testRemoveOld(self):
        """`RepositoryIndexFile` removes old index files."""
        old_path = os.path.join(self.root, 'boing')
        with open(old_path, 'w'):
            pass
        self.assertEqual(['boing'], sorted(os.listdir(self.root)))
        repo_file = self.getRepoFile('boing')
        repo_file.close()
        self.assertEqual(
            ['boing.bz2', 'boing.gz', 'boing.xz'],
            sorted(os.listdir(self.root)))


#!/usr/bin/env python
# coding=utf-8

from setuptools import find_packages, setup

from mapi.__version__ import VERSION

with open("readme.md", "r") as fp:
    LONG_DESCRIPTION = fp.read()

with open("requirements.txt", "r") as fp:
    REQUIREMENTS = fp.read().splitlines()

setup(
    author="Jessy Williams",
    author_email="jessy@jessywilliams.com",
    description=(
        "Python library which provides a high-level interface for media "
        "database providers, allowing users to search for television and movie "
        "metadata using a simple interface"
    ),
    include_package_data=True,
    install_requires=REQUIREMENTS,
    license="MIT",
    long_description=LONG_DESCRIPTION,
    long_description_content_type="text/markdown",
    name="mapi",
    packages=find_packages(),
    url="https://github.com/jkwill87/mapi",
    version=VERSION,
)


#Tutorial - http://pythonforengineers.com/build-a-sentiment-analysis-app-with-movie-reviews/

#Dataset: movie reviews corpus from NLTK

import nltk.classify.util
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import movie_reviews
from nltk.corpus import stopwords
from nltk.corpus import wordnet
from nltk.tokenize import word_tokenize

# all_words = movie_reviews.words() #all the words in all the reviews
# print(all_words)

# print(movie_reviews.fileids())	#print list of all files

# freq_dist = nltk.FreqDist(all_words)	#Frequency of all words
# print(freq_dist.most_common(10))	#Most of them are stop words!

##### Now main program #####

#Read https://stackoverflow.com/questions/10059594/a-simple-explanation-of-naive-bayes-classification for how NaiveBayesClassifier work

def naive_bayes_input(words):
	useful_words = [word for word in words if word not in stopwords.words('english')]
	words_dict = dict([(word, True) for word in useful_words])
	return words_dict	#This is how NaiveBayesClassifier expects input

#The sentiment analysis code is just a machine learning algorithm that has been trained to identify positive/negative reviews.

negative_reviews = []
for file in movie_reviews.fileids('neg'):
	words = movie_reviews.words(file)
	negative_reviews.append((naive_bayes_input(words), 'negative'))

positive_reviews = []
for file in movie_reviews.fileids('pos'):
	words = movie_reviews.words(file)
	positive_reviews.append((naive_bayes_input(words), 'positive'))

# print(len(negative_reviews), len(positive_reviews))

train_set = negative_reviews[:800] + positive_reviews[:800]
test_set = negative_reviews[800:] + positive_reviews[800:]

classifier = NaiveBayesClassifier.train(train_set)

accuracy = nltk.classify.util.accuracy(classifier, test_set)
print(accuracy*100)	#Accuracy of 72.5% on the dataset


#!/usr/bin/python3
"""file_storage unittest module"""

import os
import json
import unittest
from datetime import datetime
from models.engine.file_storage import FileStorage
from models.base_model import BaseModel
from models.user import User
from models.state import State
from models.city import City
from models.amenity import Amenity
from models.place import Place
from models.review import Review

class TestFileStorage(unittest.TestCase):
    """test cases for FileStorageclass"""

    def setUp(self):
        """ sets up this each test """
        self.user = User()
        self.user.id = "98"
        self.user.first_name = "Peter"
        self.user.last_name = "Wu"
        self.user.email = "naruto@hokage.com"
        self.storage = FileStorage()

    def tearDown(self):
        """ tears down after each test. resets """
        del self.user

    def test_docstring(self):
        """ checks if docstring is there """
        self.assertIsNotNone(FileStorage.__doc__)

    def test_file_storage_is_dict(self):
        """testing if dict"""
        st = FileStorage()
        st_dict = st.all()
        self.assertFalse(st_dict)
        self.assertEqual(dict, type(st_dict))

    def test_peter_wu(self):
        """ checks peter out. whatta hottie """
        self.assertTrue(self.user)
        self.assertEqual(self.user.first_name, "Peter")
        self.assertFalse(self.user.password)
        self.assertNotEqual(self.user.id, 98)
        self.assertEqual(self.user.id, "98")

    def test_basic_init(self):
        """ tests the init and file path? """
        dank = BaseModel()
        poop = FileStorage()
        poopcopy = poop
        notpoop = FileStorage()
        self.assertEqual(poop, poopcopy)
        self.assertNotEqual(poop, notpoop)
        self.assertTrue(poop)
        self.assertFalse(poop == notpoop)
        self.assertIs(poop, poopcopy)
        self.assertIsNot(poop, notpoop)
        self.assertIsNotNone(poop)
        self.assertIsInstance(poop, FileStorage)
        self.assertNotIsInstance(dank, FileStorage)

    def test_google_doc(self):
        """ more harder tests """
        storage = FileStorage()
        # TEST ALL. NO CELL A
        # Cell B tests init and objects and file path
        self.assertEqual(type(storage._FileStorage__objects), dict)
        self.assertFalse(storage._FileStorage__objects)
        self.assertEqual(type(storage._FileStorage__file_path), str)
        self.assertEqual(storage._FileStorage__file_path, "file.json")

        # Cell C tests all() and type
        test = User()
        test2 = test.id
        key = "User." + test2
        all_storage = storage.all()
        self.assertEqual(type(all_storage), dict)

        # Cell D tests new() and the dict
        my_model = BaseModel()
        self.assertEqual(
                str(type(my_model)), "<class 'models.base_model.BaseModel'>")
        self.assertEqual(
                sorted(my_model.__dict__.keys()),
                ['created_at', 'id', 'updated_at'])

        # Cell E tests save() and json file
        doesFileExist = User()
        self.assertTrue(os.path.isfile("file.json"))

        # Cell F tests reload and the json file
        os.rename("file.json", "CAKEisAlie")
        self.assertFalse(os.path.isfile("file.json"))
        newPoop = Place()
        # self.assertTrue(storage._FileStorage__objects)
        os.rename("CAKEisAlie", "file.json")
        self.assertTrue(os.path.isfile("file.json"))

    def test_methods(self):
        """ tests the public instance mthods """
        bm = FileStorage()
        self.assertTrue(hasattr(bm, "all"))
        self.assertTrue(hasattr(bm, "new"))
        self.assertTrue(hasattr(bm, "save"))
        self.assertTrue(hasattr(bm, "reload"))

    def test_save(self):
        """ tests the save(self) in BaseModel.
        Added this post grading """
        bm = BaseModel()
        self.assertIs(type(bm.id), str)
        self.assertNotEqual(bm.created_at, datetime.utcnow())
        self.assertNotEqual(bm.updated_at, datetime.utcnow())
        old_updated = bm.updated_at
        bm.save()
        self.assertNotEqual(old_updated, bm.updated_at)
        # another way to test save
        os.remove("file.json")
        self.assertFalse(os.path.isfile("file.json"))
        bm.save()
        self.assertTrue(os.path.isfile("file.json"))
        # another way to test save
        os.remove("file.json")
        bm2 = BaseModel()
        bm2.save()
        with open("file.json", mode='r', encoding='utf-8') as f:
            x = json.loads(f.read())
            length1 = len(x)
        bm3 = BaseModel()
        bm3.save()
        with open("file.json", mode='r', encoding='utf-8') as f:
            x = json.loads(f.read())
            length2 = len(x)
        self.assertTrue(length2 > length1)

    def test_save_is_dict(self):
        """ tests to see if the return type of save is a string """
        bm = BaseModel()
        bm.save()
        self.assertIsInstance(bm.to_dict()['created_at'], str)
        self.assertIsInstance(bm.to_dict()['updated_at'], str)

    def test_has_attr(self):
        """ tests if the base model has the attr """
        self.assertTrue(hasattr(BaseModel, "save"))

if __name__ == "__main__":
    unittest.main()


'''

Matasano Crypto Challenges
Set 02 - Challenge 10

IMPLEMENT CBC MODE

'''

import base64
from Crypto.Cipher import AES
from set01challenge02 import xor_strings
from set02challenge09 import pkcs7_padding
from set01challenge06 import slice_target

target = base64.b64decode(open('set02challenge10.txt', 'r').read())
example = b"this is a test to be encrypted and its note the right length"
key = b"YELLOW SUBMARINE"
iv = b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"

class cbc_module():

    def __init__(self, iv, key):
        self.iv = iv
        self.key = key
        self.cipher = AES.new(key, AES.MODE_ECB)
        self.keysize = 16

    def cbc_encrypt(self, plaintext):
        ciphertext = b""
        plaintext = slice_target(plaintext, self.keysize)
        previous_block = self.iv

        if len(plaintext[-1]) != self.keysize:
            plaintext[-1] = pkcs7_padding(plaintext[-1], self.keysize)

        cipherblocks = [b"" for i in plaintext]

        for i in range(len(plaintext)):
            cipherblocks[i] = self.cipher.encrypt(
                xor_strings(plaintext[i], previous_block))
            previous_block = cipherblocks[i]

        ciphertext = b""
        for i in cipherblocks:
            ciphertext += i

        return ciphertext

    def cbc_decrypt(self, ciphertext):
        plaintext = b""
        ciphertext = slice_target(ciphertext, self.keysize)
        previous_block = self.iv
        plainblocks = [b"" for i in ciphertext]

        for i in range(len(ciphertext)):
            plainblocks[i] = xor_strings(self.cipher.decrypt(
                ciphertext[i]), previous_block)
            previous_block = ciphertext[i]

        for i in plainblocks:
            plaintext += i

        return plaintext

def main():
    cipher = cbc_module(iv, key)
    decrypted = cipher.cbc_decrypt(target)
    print(decrypted)
    print()
    encrypted = cipher.cbc_encrypt(decrypted)

    if encrypted != target:
        raise Exception("Error - File value doesn't match source")
    else:
        print("--- Success ---")

if __name__ == "__main__":
    main()


from Main import df
from bokeh.plotting import figure
from bokeh.io import output_file, show
from bokeh.models import HoverTool, ColumnDataSource

df["Start_String"] = df["Start"].dt.strftime("%Y-%m-%d %H:%M:%S")
df["End_String"] = df["End"].dt.strftime("%Y-%m-%d %H:%M:%S")
cds = ColumnDataSource(df)
output_file("Time Stamps.html")
f = figure(plot_width = 1660, plot_height = 875, x_axis_type="datetime", title="Motion Stamps")

hover = HoverTool(tooltips=[("Start", "@Start_String"), ("End", "@End_String")])
f.add_tools(hover)
f.quad(left = "Start", right = "End", bottom = 0, top = 1, color="green", source=cds)
f.yaxis.minor_tick_line_color = None
f.yaxis.ticker.desired_num_ticks = 1
show(f)


import torch
import torch.optim as optim
import torch.nn as nn

from PIL import Image
from visualize_plt import Window

import torchvision
from torchvision.transforms import Compose, CenterCrop, Normalize
from torchvision.transforms import ToTensor, ToPILImage
from transforms import Colorize, Relabel, Tolabel

from FCN import FCN8, FCN16, FCN32
from UNet import UNet
from SegNet import SegNet

from argparse import ArgumentParser

input_transform = Compose([
    CenterCrop(256),
    ToTensor(),
    Normalize([.485, .456, .406], [.229, .224, .225])
])

target_transform = Compose([
    CenterCrop(256),
    Tolabel(),
    Relabel(255,21)
])

CLASSES_NUM = 22
color_transform = Colorize(CLASSES_NUM)
image_transform = ToPILImage()

trainset = torchvision.datasets.VOCSegmentation(root='./data', year='2012',
                                                image_set='train', download=True,
                                                transform=input_transform,target_transform=target_transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=1, shuffle=True, num_workers=2)

testset = torchvision.datasets.VOCSegmentation(root='./data', year='2012',
                                                image_set='val', download=True,
                                                transform=input_transform,target_transform=target_transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=1, shuffe=True, num_workers=2)

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

def train(args, model):

    model.train()
    criterion = nn.CrossEntropyLoss()

    if args.model.startwith('FCN'):
        optimizer = optim.SGD(model.parameters(), 1e-4, .9, 2e-5)
    elif args.model == 'SegNet':
        optimizer = optim.SGD(model.parameters(), 1e-3, .9)
    elif args.model == 'UNet':
        optimizer = optim.Adam(model.parameters())

    window = Window()

    for epoch in range(0, args.epochs):
        losses = []

        for i,data in enumerate(trainloader):
            inputs, labels = data
            inputs, labels = inputs.to(device), labels.to(device)

            optimizer.zero_grad()

            outputs = model(inputs)

            loss = criterion(outputs, labels)
            losses.append(loss.item())

            loss.backward()
            optimizer.step()

            # show the image
            if args.image_period>0 and (i+1) % args.image_period == 0:
                image = inputs[0].cpu().data
                image[0] = image[0] * .229 + .485
                image[1] = image[1] * .224 + .456
                image[2] = image[2] * .225 + .406

                window.show_image(image, 'input( {}-{} )'.format(epoch+1,i+1))
                color_output = color_transform(outputs[0].cpu().max(0)[1].data)
                window.show_image(color_output, 'output( {}-{} )'.format(epoch+1,i+1))
                color_target = color_transform(labels[0].cpu.data)
                window.show_image(color_target, 'target( {}-{} )'.format(epoch+1,i+1))

            # show the average running loss
            if (i+1) % args.loss_period == 0:
                print('[%d, %5d] loss: %.3f' % (epoch, i+1, sum(losses)/len(losses)))

        # save the model state
        if (epoch+1) % args.save_period == 0:
            filename = '{}-{}.pth'.format(args.model, epoch+1)
            torch.save(model.state_dict(), filename)
            print('{} saved'.format(filename))

def evaluate(args, model):

    model.eval()

    input = input_transform(Image.open(args.eval_image))

    with torch.no_grad:
        output = model(input)
    # output =  model(Variable(input, volatile=True).unsqueeze(0))

    color_output = color_transform(output[0].data.max(0)[1])
    window = Window()
    window.show_image(color_output)

    #image_transform(label).save(args.label)

def main(args):
    Net = None
    if args.model == 'FCN8':
        Net = FCN8
    elif args.model == 'FCN16':
        Net = FCN16
    elif args.model == 'FCN32':
        Net = FCN32
    elif args.model == 'UNet':
        Net = UNet
    elif args.model == 'SegNet':
        Net = SegNet

    assert Net is not None, 'model {} is not available'.format(args.model)

    model = Net(CLASSES_NUM)
    model = model.to(device)

    if args.mode == 'train':
        train(args, model)
    elif args.mode == 'eval':
        evaluate(args, model)

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('model', require=True)

    subparsers = parser.add_subparsers(dest='mode')
    subparsers.required = True

    evalParser = subparsers.add_parser('eval')
    evalParser.add_argument('eval_image')

    trainParser = subparsers.add_parser('train')
    trainParser.add_argument('epochs', type=int, default=50)
    trainParser.add_argument('image_period', type=int, default=0)
    trainParser.add_argument('loss_period', type=int, default=50)
    trainParser.add_argument('save_period', type=int, default=10)

    main(parser.parse_args())


import numpy as np
from sklearn.metrics import f1_score
from small_text.active_learner import PoolBasedActiveLearner
from small_text.classifiers import SklearnClassifierFactory

class ActiveLearner():
    def __init__(self, clf_template, query_strategy, train, num_classes=2, num_iterations=30, num_samples=500):
        self.num_classes = num_classes
        self.num_iterations = num_iterations
        self.num_samples = num_samples

        self.train = train
        self.clf_template = clf_template
        self.query_strategy = query_strategy
        self.clf_factory = SklearnClassifierFactory(clf_template, self.num_classes)
        self.active_learner = PoolBasedActiveLearner(self.clf_factory, self.query_strategy, train)
        self.labeled_indices = self.initialize_strategy()

    def initialize_strategy(self):
        # Initialize the model. This is required for model-based query strategies.
        indices_pos_label = np.where(self.train.y == 1)[0]
        indices_neg_label = np.where(self.train.y == 0)[0]

        indices_initial = np.concatenate([np.random.choice(indices_pos_label, 100, replace=False),
                                          np.random.choice(indices_neg_label, 100, replace=False)],
                                         dtype=int)

        self.active_learner.initialize_data(indices_initial, self.train.y[indices_initial])

        return indices_initial

    def evaluate(self, active_learner, train, test):
        y_pred = active_learner.classifier.predict(train)
        y_pred_test = active_learner.classifier.predict(test)

        f1_score_train = f1_score(y_pred, train.y, average='micro')
        f1_score_test = f1_score(y_pred_test, test.y, average='micro')

        print('Train accuracy: {:.2f}'.format(f1_score_train))
        print('Test accuracy: {:.2f}'.format(f1_score_test))
        print('---')
        return f1_score_train, f1_score_test

    def perform_active_learning(self, test):
        """
        This is the main loop in which we perform num_iterations of active learning.
        During each iteration num_samples samples are queried according to the query strategy and then updated.
        The update step reveals the true label to the active learner, i.e. this is a simulation,
        but in a real scenario the user input would be passed to the update function.
        """
        # Set up array to track accuracy scores
        train_accuracy = []
        test_accuracy = []

        # Perform num_iterations of active learning...
        for i in range(self.num_iterations):
            # ...where each iteration consists of labelling num_samples using one
            # of the following strategies: Random Sampling, Least Confidence Sampling,
            # Prediction Entropy Sampling or Discriminative Active Learning
            indices_queried = self.active_learner.query(num_samples=self.num_samples)

            # Simulate user interaction here. Replace this for real-world usage.
            y = self.train.y[indices_queried]

            # Return the labels for the current query to the active learner.
            self.active_learner.update(y)

            self.labeled_indices = np.concatenate([indices_queried, self.labeled_indices])

            print('Iteration #{:d} ({} samples)'.format(i, len(self.labeled_indices)))
            train_acc, test_acc = self.evaluate(self.active_learner, self.train[self.labeled_indices], test)
            train_accuracy = np.append(train_accuracy, train_acc)
            test_accuracy = np.append(test_accuracy, test_acc)

        return train_accuracy, test_accuracy


import numpy as np
import copy
import torch
import torch.nn as nn
import torch.nn.functional as F

from AC_modules.Layers import *

debug = False

class OheNet(nn.Module):
    def __init__(self, map_size, k_in=3, k_out=24, n_features=32, pixel_hidden_dim=128,
                 pixel_n_residuals=4, feature_hidden_dim=64, feature_n_residuals=4, device=None):

        super(OheNet, self).__init__()

        self.n_features = n_features

        self.OHE_conv = Convolution(k_in=k_in, k_out=k_out)
        self.pos_enc = PositionalEncoding(n_kernels=k_out, n_features=n_features, device=device)

        pixel_res_layers = nn.ModuleList([ResidualLayer(map_size**2, pixel_hidden_dim) for _ in range(pixel_n_residuals)])
        self.pixel_res_block = nn.Sequential(*pixel_res_layers)

        self.maxpool = FeaturewiseMaxPool(pixel_axis=2)

        feature_res_layers = nn.ModuleList([ResidualLayer(n_features, feature_hidden_dim) for _ in range(feature_n_residuals)])
        self.feature_res_block = nn.Sequential(*feature_res_layers)

    def forward(self, x):
        """ Input shape (batch_dim, k_in, map_size+2, map_size+2) """

        x = self.OHE_conv(x)
        if debug: print("conv_state.shape: ", x.shape)

        x = self.pos_enc(x)
        if debug: print("After positional enc + projection: ", x.shape)

        x = x.permute(1,2,0)
        if debug: print("x.shape: ", x.shape)

        x = self.pixel_res_block(x) # Interaction between pixels feature-wise
        if debug: print("x.shape: ", x.shape)

        x = self.maxpool(x) # Feature-wise maxpooling
        if debug: print("x.shape: ", x.shape)

        x = self.feature_res_block(x) # Interaction between features -> final representation
        if debug: print("x.shape: ", x.shape)

        return x

class SpatialNet(nn.Module):

    def __init__(self, n_features, state_size, out_channels):
        super(SpatialNet, self).__init__()

        self.size = state_size

        self.linear = nn.Linear(n_features, (state_size-6)*(state_size-6))

        self.conv_block = nn.Sequential(
                                        nn.ConvTranspose2d(in_channels=1,
                                                           out_channels=out_channels,
                                                            kernel_size=3),
                                        nn.ReLU(),
                                        nn.ConvTranspose2d(in_channels=out_channels,
                                                           out_channels=out_channels,
                                                           kernel_size=3),
                                        nn.ReLU(),
                                        nn.ConvTranspose2d(in_channels=out_channels,
                                                              out_channels=out_channels,
                                                              kernel_size=3)
                                        )

    def forward(self, state_rep):
        if debug: print("state_rep.shape: ", state_rep.shape)

        x = F.relu(self.linear(state_rep))
        if debug: print("x.shape (after linear): ", x.shape)

        x = x.reshape(x.shape[0], 1, self.size-6, self.size-6)
        if debug: print("x.shape (after reshape): ", x.shape)

        x = self.conv_block(x)
        if debug: print("x.shape (after conv block): ", x.shape)

        return x


import numpy as np
import os
from ..io import imread

def load_train_images(dir_path):
    """
    Load training images.

    Return
    ------
    X : array-like, shape (n_samples, n_features)
        Training data.

    y : array-like, shape (n_samples)
        Training class labels.
    """

    files = os.listdir(dir_path)

    images = []
    for f in files:
        if f.endswith('.png') or f.endswith('.gif'):
            images.append(f)

    n_samples = len(images)
    arr = imread(dir_path + '/' + images[0])
    dt = arr.dtype
    rows, cols = arr.shape
    n_features = rows * cols

    X = np.zeros((n_samples, n_features), dtype=dt)
    y = np.zeros((n_samples), dtype=np.str)
    for i in np.arange(n_samples):
        image = images[i]
        image_path = dir_path + '/' + image
        arr = imread(image_path)
        X[i, :] = imread(image_path).flatten()
        y[i] = image.split('_')[1]

    return X, y


import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
import random
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import time
import pandas as pd
import torch
from torch.utils.data import Dataset, DataLoader

class CustomDataset(Dataset):
    def __init__(self, data, labels):
        self.data = data
        self.labels = labels

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        x = self.data[index]
        y = self.labels[index]
        # Perform any additional preprocessing or transformations if needed
        return x, y

class ValueNetwork(nn.Module):
    def __init__(self):
        super(ValueNetwork, self).__init__()

        self.conv_layers = nn.Sequential(
            nn.Conv2d(2, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            # nn.Conv2d(32, 32, kernel_size=3, padding=1),
            # nn.ReLU(inplace=True),
            # nn.Conv2d(32, 32, kernel_size=3, padding=1),
            # nn.ReLU(inplace=True),
            # nn.Conv2d(32, 64, kernel_size=3, padding=1),
            # nn.ReLU(inplace=True),
            # nn.Conv2d(64, 64, kernel_size=3, padding=1),
            # nn.ReLU(inplace=True),
            # nn.Conv2d(64, 64, kernel_size=3, padding=1),
            # nn.ReLU(inplace=True),
            # nn.Conv2d(64, 128, kernel_size=3, padding=1),
            # nn.ReLU(inplace=True),
            # nn.Conv2d(128, 128, kernel_size=3, padding=1),
            # nn.ReLU(inplace=True),
            # nn.Conv2d(128, 128, kernel_size=3, padding=1),
            # nn.ReLU(inplace=True),
            # nn.Conv2d(128, 256, kernel_size=3, padding=1),
            # nn.ReLU(inplace=True),
            # nn.Conv2d(256, 256, kernel_size=3, padding=1),
            # nn.ReLU(inplace=True),
            # nn.Conv2d(256, 256, kernel_size=3, padding=1),
            # nn.ReLU(inplace=True),
            # Add more convolutional layers if needed
        )

        self.fc_layers = nn.Sequential(
            nn.Linear(9 * 9 * 256, 256),
            nn.ReLU(inplace=True),
            nn.Linear(256, 1),
            nn.Sigmoid()
        )

    def forward(self, x):
        x = self.conv_layers(x)
        # print(f"Conv: {x.shape}")
        x = x.view(x.size(0), -1)
        # x = x.flatten()
        # print(f"Flatten: {x.shape}")
        x = self.fc_layers(x)
        return x

def load_data():
    path="../Datasets/Hex/dataset270k.txt"
    with open(path) as file:
        data=file.readlines()
    data=[f.strip().split(" ") for f in data]
    X=[list(f[0]) for f in data]
    X=[list(map(int, f)) for f in X]
    # X=[f[:0] for f in X]
    Y=[1 if f[1]=='w' else 0 for f in data ]
    tempData = list(zip(X, Y))
    random.shuffle(tempData)
    X, Y = zip(*tempData)
    print(len(X))
    # dataset = np.empty([len(X), 2,6,6])
    dataset = []
    for i, x in enumerate(X):
        first_layer_array = np.reshape(x[:36], (6, 6))
        second_layer_array = np.reshape(x[36:], (6, 6))
        dataset.append( np.stack((first_layer_array, second_layer_array)))
        # print(dataset[0].shape)
        # break
    return dataset, Y

def load_go_data():
    # go_data = pd.read_csv("/Users/charug18/Drive/Work/PhD/Projects/Go Winner Prediction/Go_binary_data_9x9.csv", delimiter=",")
    go_data = pd.read_csv("Winner Prediction/Go Winner Prediction/Go_binary_data_9x9.csv", delimiter=",")
    data = go_data.iloc[:,0]
    labels = go_data.iloc[:,1]
    go_board = []
    Y=[int(f) for f in labels]
    tempData = list(zip(data, Y))
    random.shuffle(tempData)
    X, Y = zip(*tempData)
    # go_string = []
    for x in X:
        black = [1 if bit == "1" else 0 for bit in x]
        white = [1 if bit == "2" else 0 for bit in x]
        black_array = np.reshape(black, (9, 9))
        white_array = np.reshape(white, (9, 9))
        go_board.append( np.stack((black_array, white_array)))
    return go_board, Y

        # go_string.append( black+white)
def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

value_net = ValueNetwork()
print(f'The model has {count_parameters(value_net):,} trainable parameters.')
exit(0)
X, Y = load_go_data()
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.3, random_state=42)
X_train = torch.tensor(X_train, dtype=torch.float32)
X_test = torch.tensor(X_test, dtype=torch.float32)
Y_train = torch.tensor(Y_train, dtype=torch.float32)
Y_test = torch.tensor(Y_test, dtype=torch.float32)
Y_train = torch.unsqueeze(Y_train, 1)
Y_test = torch.unsqueeze(Y_test, 1)
print(X_train.shape, Y_train.shape)
# exit(0)
# Create an instance of your custom dataset
dataset_train = CustomDataset(X_train, Y_train)
dataset_test = CustomDataset(X_test, Y_test)

# dataset = CustomDataset(flattened_data)

# data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=True)

# # Iterate over the data loader
# for batch in data_loader:
#     # Perform your training or evaluation logic here
#     # Each batch will contain `batch_size` flattened tensors
#     print(batch.shape)
# exit(0)

# Create an instance of the ValueNetwork
value_net = ValueNetwork()

# Define the loss function and optimizer
criterion = nn.BCELoss()
optimizer = optim.SGD(value_net.parameters(), lr=0.01, momentum=0.9)

# Create a data loader for the dataset
batch_size = 128
data_loader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True)
data_loader_test = DataLoader(dataset_test, batch_size=batch_size, shuffle=True)
device = torch.device("cpu")
X_train = X_train.to(device)
Y_train = Y_train.to(device)
X_test = X_test.to(device)
Y_test = Y_test.to(device)
value_net = value_net.to(device)

# Define the number of epochs
num_epochs = 10
num_batches = X_test.shape[0]//batch_size
for epoch in range(num_epochs):
    running_loss = 0.0
    for batch_data, batch_labels in data_loader_train:

        # Forward pass
        output = value_net(batch_data)

        # Compute the loss
        # loss = criterion(output, batch_labels)
        loss = criterion(output, batch_labels)

        # Backpropagation and optimization
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # Update the running loss
        running_loss += loss.item()

    # Print the average loss for each epoch
    epoch_loss = running_loss / len(X_train)
    print(f"Epoch {epoch+1} Loss: {epoch_loss}")

    # Evaluation
    with torch.no_grad():
        # Disable gradient computation
        accuracy = 0.0
        # Set the threshold value
        threshold = 0.5
        start_time = time.time()
        for batch_data, batch_labels in data_loader_test:

            # Forward pass
            output = value_net(batch_data)

            # Convert sigmoid output to binary labels
            binary_labels = torch.where(output > threshold, torch.tensor(1), torch.tensor(0))

            # Compute the accuracy
            accuracy+= accuracy_score(batch_labels, binary_labels)
            # loss = criterion(output, batch_labels)

        end_time = time.time()
        # Update the running acc
        accuracy = accuracy/num_batches

        print(f"Epoch {epoch+1}\tAcc: {accuracy:.2f}\tData: {X_test.shape[0]}\tInference Time: {end_time-start_time}")


import os
import json
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import email_templates
import botocore
import boto3

##############################################################################################
# CONSTANTS
##############################################################################################

# read constants from environmental variables
COGNITO_USER_POOL_ID = os.environ['COGNITO_USER_POOL_ID']
EMAILER_QUEUE_URL = os.environ['EMAILER_QUEUE_URL']

##############################################################################################
# DATABASE SUPPORT FUNCTIONS
##############################################################################################

def get_user_details_by_cognito_id(cognito_id):
    """
    get the customer name and email address from the cognito service by searching for the cognito_id

    Args:
        cognito_id: String containing the unique ID of the user account

    Returns:
        dictionary with users registered given name string and user email address string
    """
    user_details = {
        'given_name': '',
        'email_address': ''
    }

    # get username from cognito_id
    client = boto3.client('cognito-idp')
    cognito_filter = 'sub="' + cognito_id + '"'
    response = client.list_users(UserPoolId=COGNITO_USER_POOL_ID, Limit=1, Filter=cognito_filter)
    if len(response['Users']) > 0:
        for attribute in response['Users'][0]['Attributes']:
            if attribute['Name'] == 'email':
                user_details['email_address'] = attribute['Value']
                print('email address:', user_details['email_address'])
            if attribute['Name'] == 'custom:firstname':
                user_details['given_name'] = attribute['Value']
                print('given name:', user_details['given_name'])
    else:
        print('cognito_id not found: ', cognito_id)

    return user_details

##############################################################################################
# EMAIL FUNCTIONS
##############################################################################################

def send_email(user_details):
    """
    construct and send email to customer using the ses service

    Args:
        user_details: Dict:
            given_name: string: name of customer being emailed
            email_address: string: address of customer being emailed

    Returns:
        none
    """
    given_name = user_details['given_name']
    email_address = user_details['email_address']

    print('notify:', given_name, 'at address:', email_address)

    if (given_name != '') and (email_address != ''):
        client = boto3.client('ses')

        destination = email_address

        body_html = email_templates.sensor_error['body']
        subject = email_templates.sensor_error['subject']
        sender = email_templates.sensor_error['sender']

        if body_html != '':
            # replace the token ### with the users given name
            body_html = body_html.replace('###', given_name)

            charset = "utf-8"
            msg_body = MIMEMultipart('alternative')

            # Encode the text and HTML content and set the character encoding. This step is
            # necessary if you're sending a message with characters outside the ASCII range.
            html_part = MIMEText(body_html.encode(charset), 'html', charset)

            # Add the text and HTML parts to the child container.
            msg_body.attach(html_part)

            msg = MIMEMultipart('mixed')
            # Add subject, from and to lines.
            msg['Subject'] = subject
            msg['From'] = sender
            msg['To'] = destination

            msg.attach(msg_body)

            response = client.send_raw_email(
                Destinations=[destination],
                RawMessage={
                    'Data': msg.as_string(),
                },
                Source=sender,
            )
            print('email send status: ', response)

##############################################################################################
# QUEUE MANAGEMENT FUNCTIONS
##############################################################################################

def delete_sqs_message(event):
    """
    remove message from queue since it has been processed

    Args:
        event: dict:
            receiptHandle: string: address of message we want to delete from the queue

    Returns:
        none
    """
    if 'receiptHandle' in event:
        sqs = boto3.client('sqs')
        try:
            response = sqs.delete_message(
                QueueUrl=EMAILER_QUEUE_URL,
                ReceiptHandle=event['receiptHandle']
            )
            print('remove notification queue entry: ', response)
        except botocore.exceptions.ClientError as err:
            print('Error Message: {}'.format(err.response['Error']['Message']))

##############################################################################################
# ENTRY POINT
##############################################################################################

def lambda_handler(event, context):
    """
    This function is triggered by a the emailer queue
    A message from the queue contains the user ID of the customer we need to notify of a problem
    Construct an email from the template and user details.
    Send the email using the AWS SES service

    Args:
        event: dict
            event data structure contains required parameter named cognito_id
        event: {
            'Records': [
                {
                    'messageId': '1efb833b-43fd-4c9e-963b-ac335754d490',
                    'receiptHandle': 'AQEBl/EorTlj03vFSVDFVDFVHzU0/N3+uEviVFBnDFVDF.....
                    'body': "{
                        'cognitoID':'12345678'
                    }",
                    'attributes': {
                        'ApproximateReceiveCount': '1',
                        'SentTimestamp': '1620458414343',
                        'SenderId': 'AIDASDCACRGSDCJSDC3RE6',
                        'ApproximateFirstReceiveTimestamp': '1620458414386'
                    },
                    'messageAttributes': {},
                    'md5OfBody': '75937062a550dbe8abe1f7e7104f64bc',
                    'eventSource': 'aws:sqs',
                    'eventSourceARN': 'arn:aws:sqs:us-east-1:xxxxxxxx:devicestack1-.....
                    'awsRegion': 'us-east-1'
                }
            ]
        }
        context:
            unused

    Returns:
        non
    """

    for record in event['Records']:
        message = json.loads(record['body'].strip('\n').strip(' '))
        cognito_id = message['cognitoID']
        user_details = get_user_details_by_cognito_id(cognito_id)
        send_email(user_details)
        delete_sqs_message(record)


import time
import numpy as np
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.figure import Figure
import matplotlib.pyplot as plt

def pixtegrate(func,bounds,npoints=10000,sleep=-1.):
    """
    Args:
        func (function): Function to integrate
        bounds (tuple): 2-tuple for lower and upper bounds of range
        npoints (int): Number of points used to sample the function

    Returns:
        integral (float)
    """
    xmin, xmax = bounds
    xvals = np.linspace(xmin,xmax,npoints)
    try:
        # First assume the function operates on arrays
        yvals = func(xvals)
    except:
        # Then assume it operates on a single element
        yvals = np.vectorize(func)(xvals)
    ymin, ymax = yvals.min(), yvals.max()

    # Make a figure with no padding/margins and a black background
    fig,ax = plt.subplots()
    canvas = FigureCanvas(fig)
    fig.subplots_adjust(left=0,bottom=0,right=1,top=1)
    fig.set_facecolor("black")
    ax.set_xmargin(0.)
    ax.set_ymargin(0.)
    ax.axis("off")

    # Color y>=0 red and y<0 blue
    ax.fill_between(xvals,yvals,where=(yvals>=0.),edgecolor="r",facecolor="r",linewidth=0.)
    ax.fill_between(xvals,yvals,where=(yvals<0.),edgecolor="b",facecolor="b",linewidth=0.)

    # Draw and count red/blue pixels
    canvas.draw()
    width, height = fig.get_size_inches() * fig.get_dpi()
    img = np.fromstring(canvas.tostring_rgb(), dtype='uint8').reshape(int(height), int(width), 3)
    nred = (img[:,:,0] > 254).sum()
    nblue = (img[:,:,2] > 254).sum()

    # Count filled fraction of image (red-blue), and normalize to actual bounding box
    integral = (nred-nblue)/(width*height)*((ymax-ymin)*(xmax-xmin))

    # Figure garbage collection
    plt.close()

    if sleep > 0.:
        time.sleep(sleep)

    return integral


''' This module calculates the volume depolarization ratio of the molecular backscatter signal
dected with a polarization lidar. It is an application of the method found in:

A. Behrendt and T. Nakamura, "Calculation of the calibration constant of polarization lidar
and its dependency on atmospheric temperature," Opt. Express, vol. 10, no. 16, pp. 805-817, 2002.

The molecular parameters gamma and epsilon are wavelength dependent, and this
makes the results of the original paper valid only for 532nm. Some new formulas
have been implemented from:

Tomasi, C., Vitale, V., Petkov, B., Lupi, A. & Cacciari, A. Improved
algorithm for calculations of Rayleigh-scattering optical depth in standard
atmospheres. Applied Optics 44, 3320 (2005).

and

Chance, K. V. & Spurr, R. J. D. Ring effect studies: Rayleigh scattering,
including molecular parameters for rotational Raman scattering, and the
Fraunhofer spectrum. Applied Optics 36, 5224 (1997).

For 532nm the results are similare (but not identical)
with the orignal.

Its use::

   # Create a new DepolarizationLidar instance
   my_lidar = DepolarizationLidar(532, 0.7) # Laser wavelegnth, fwhm of the filter
   # Calculate the molecular volume depolarization at temperature T (in deg Kelvin)
   delta_mol = my_lidar.delta_mol_temperature(240) # will return 0.003976

Any comments should be sent to ioannis@inoe.ro

'''
import numpy as np
from matplotlib import pyplot as plt

from . import us_std

from .molecular_properties import gamma_square_N2, gamma_square_O2, epsilon_N2, epsilon_O2
from .constants import hc_k

N2_parameters = {'name': "N_{2}",
                 'B0': 1.989500,
                 'D0': 5.48E-6,
                 'I': 1,
                 'gamma_square': gamma_square_N2,  # Originally 0.509E-48,
                 'epsilon': epsilon_N2,  # Originally 0.161,
                 'g': [6, 3],
                 'relative_concentration': 0.79}  # for even/odd J

O2_parameters = {'name': "O_{2}",
                 'B0': 1.437682,
                 'D0': 4.85E-6,
                 'I': 0,
                 'gamma_square': gamma_square_O2,  # Originally 1.27E-48,
                 'epsilon': epsilon_O2,  # Originally 0.467,
                 'g': [0, 1],
                 'relative_concentration': 0.21}  # for even/odd J

def energy_rotational_norm(J, molecular_parameters):
    ''' Calculates the roational energy of a homonuclear diatomic molecule for
    quantum number J. The molecule is specified by passing a dictionary with
    parameters.

    The result is divided by the Boltzmann constant for ease of calculation.
    '''

    B0 = molecular_parameters['B0']
    D0 = molecular_parameters['D0']

    E_rot = (B0 * J * (J + 1) - D0 * (J ** 2) * ((J + 1) ** 2)) * hc_k
    return E_rot

def delta_n_stokes(J, molecular_parameters):
    '''Calculates the rotational raman shift for the Stokes branch for
    quantum number J. The molecule is specified by passing a dictionary with
    parameters.

    Output in cm-1
    '''

    B0 = molecular_parameters['B0']
    D0 = molecular_parameters['D0']

    delta_n = -B0 * 2 * (2 * J + 3) + D0 * (3 * (2 * J + 3) + (2 * J + 3) ** 3)
    return delta_n

def delta_n_antistokes(J, molecular_parameters):
    '''Calculates the rotational raman shift for the anti-Stokes branch for
    quantum number J. The moelcule is specified by passing a dictionary with
    parameters.

    Output in cm-1
    '''
    B0 = molecular_parameters['B0']
    D0 = molecular_parameters['D0']

    delta_n = B0 * 2 * (2 * J - 1) - D0 * (3 * (2 * J - 1) + (2 * J - 1) ** 3)
    return delta_n

def delta_sigma_stokes(n_incident, J, temperature, molecular_parameters):
    '''Calculates the backscatter coefficient of the Raman scattering for the Stokes
    branch for quantum number J at a temperature T (in Kelvin ). The molecule is specified
    by passing a dictionary with parameters.
    '''

    B0 = molecular_parameters['B0']
    gamma_square_func = molecular_parameters['gamma_square']
    gamma_square = gamma_square_func(n_incident)

    g_index = np.remainder(J, 2)
    g = molecular_parameters['g'][g_index]

    J = float(J)

    b_s = 64 * np.pi ** 4 * hc_k / 15.

    b_s = b_s * (g * B0 * (n_incident +
                           delta_n_stokes(J, molecular_parameters)) ** 4 * gamma_square)
    b_s = b_s / ((2 * molecular_parameters['I'] + 1) ** 2 * temperature)
    b_s = b_s * ((J + 1) * (J + 2)) / (2 * J + 3.)
    b_s = b_s * \
        np.exp(-energy_rotational_norm(J, molecular_parameters) / temperature)
    return b_s

def delta_sigma_antistokes(n_incident, J, temperature, molecular_parameters):
    '''Calculates the backscatter coefficient of the Raman scattering for the Stokes
    branch for quantum number J at a temperature T (in Kelvin ). The moelcule is specified
    by passing a dictionary with parameters.

    In cm^2sr-1.
    '''

    B0 = molecular_parameters['B0']
    gamma_square_func = molecular_parameters['gamma_square']
    gamma_square = gamma_square_func(n_incident)

    g_index = np.remainder(J, 2)
    g = molecular_parameters['g'][g_index]

    J = float(J)

    b_s = 64 * np.pi ** 4 * hc_k / 15.

    b_s = b_s * (g * B0 * (n_incident +
                           delta_n_antistokes(J, molecular_parameters)) ** 4 * gamma_square)
    b_s = b_s / ((2 * molecular_parameters['I'] + 1) ** 2 * temperature)
    b_s = b_s * (J * (J - 1)) / (2 * J - 1.)
    b_s = b_s * \
        np.exp(-energy_rotational_norm(J, molecular_parameters) / temperature)
    return b_s

def delta_mol(n_incident, concentrations, molecular_parameters, relative_transmisions):
    """ Calculates the depolarization ratio of the molecular signal detected by a lidar.

    It's inputs are:

    concentrations         - a list of the relative concentration of the moleculres ( [0.79, 0.21] -
                             for the Nitrogen and Oxygen in the atmosphere)
    molecular_parameters   - a list of dictionaries that describe the molecules.
    relative_transimssions - the ralative part of the intensity of the rotational Raman wings of the
                             molecules detected by the lidar.
    """

    gamma_squares = [parameter['gamma_square'](n_incident)
                     for parameter in molecular_parameters]
    epsilons = [parameter['epsilon'](n_incident) for parameter in molecular_parameters]

    numerator = np.sum([con * gamma_square * (3 * x + 1) for (con, gamma_square, x)
                        in zip(concentrations, gamma_squares, relative_transmisions)])
    denominator = np.sum([con * gamma_square * (3 * x + 1 + 45. / epsilon) for (con, gamma_square, x, epsilon)
                          in zip(concentrations, gamma_squares, relative_transmisions, epsilons)])
    delta_m = 3. / 4. * numerator / denominator
    return delta_m

class DepolarizationLidar:

    def __init__(self, wavelength=532.0, fwhm=0.55):
        '''
        This class calculates the volume depolarization ratio of the molecular
        backscatter signal dected with a polarization lidar.

        It's unputs are:

        wavelength  - The lidar emission wavelength in nm
        fhwm        - The fhwm of the detecting filter

        Available methods:

        delta_mol_temperature(T) - calculate the molecular depolarization ratio at temperature T (in Kelvin)

        '''
        self.J_stokes = np.arange(0, 60)
        self.J_astokes = np.arange(2, 60)

        self.wavelength = float(wavelength)
        self.wavenumber = 10 ** 7 / self.wavelength

        self.optical_filter = FilterFunction(self.wavelength, fwhm)

        self.dn_stokes_N2 = np.array([delta_n_stokes(J, N2_parameters) for J in self.J_stokes])
        self.dn_astokes_N2 = np.array([delta_n_antistokes(J, N2_parameters) for J in self.J_astokes])

        self.wl_stokes_N2 = 1e7 / (self.wavenumber - self.dn_stokes_N2)
        self.wl_astokes_N2 = 1e7 / (self.wavenumber - self.dn_astokes_N2)

        self.dn_stokes_O2 = np.array([delta_n_stokes(J, O2_parameters) for J in self.J_stokes])
        self.dn_astokes_O2 = np.array([delta_n_antistokes(J, O2_parameters) for J in self.J_astokes])

        self.wl_stokes_O2 = 1e7 / (self.wavenumber - self.dn_stokes_O2)
        self.wl_astokes_O2 = 1e7 / (self.wavenumber - self.dn_astokes_O2)

    def delta_mol_temperature(self, T):

        x_N2, x_O2 = self.rotation_contribution_temperature(T)

        delta_m = delta_mol(self.wavenumber, [0.78084, 0.209476], [N2_parameters, O2_parameters], [x_N2, x_O2])

        return delta_m

    def rotation_contribution_temperature(self, T):
        ds_stokes = np.array([delta_sigma_stokes(
            self.wavenumber, J, T, N2_parameters) for J in self.J_stokes])
        ds_astokes = np.array([delta_sigma_antistokes(
            self.wavenumber, J, T, N2_parameters) for J in self.J_astokes])

        x_N2 = (np.sum(self.optical_filter(self.wl_stokes_N2) * ds_stokes) +
                np.sum(self.optical_filter(self.wl_astokes_N2) * ds_astokes)) / (np.sum(ds_stokes) + np.sum(ds_astokes))

        ds_stokes = np.array([delta_sigma_stokes(
            self.wavenumber, J, T, O2_parameters) for J in self.J_stokes])
        ds_astokes = np.array([delta_sigma_antistokes(
            self.wavenumber, J, T, O2_parameters) for J in self.J_astokes])

        x_O2 = (np.sum(self.optical_filter(self.wl_stokes_O2) * ds_stokes) +
                np.sum(self.optical_filter(self.wl_astokes_O2) * ds_astokes)) / (np.sum(ds_stokes) + np.sum(ds_astokes))
        return x_N2, x_O2

    def delta_mol_at_altitude(self, altitudes):
        """ Calculates delta_mol at an altitude (in m)
        """

        atmosphere = us_std.Atmosphere()
        Ts = np.array([atmosphere.temperature(altitude)
                       for altitude in altitudes])

        delta_mols = []

        for T in Ts:
            delta_m = self.delta_mol_temperature(T)
            delta_mols.append(delta_m)

        return delta_mols

    def plot_spectrum(self, T, molecular_parameters, show_filter=True, show_filtered=True, legend=True, suptitle=None,
                      figsize=(8, 5), xlim=None):
        ds_stokes = [delta_sigma_stokes(
            self.wavenumber, J, T, molecular_parameters) for J in self.J_stokes]
        ds_astokes = [delta_sigma_antistokes(
            self.wavenumber, J, T, molecular_parameters) for J in self.J_astokes]

        dn_stokes = np.array(
            [delta_n_stokes(J, molecular_parameters) for J in self.J_stokes])
        dn_astokes = np.array(
            [delta_n_antistokes(J, molecular_parameters) for J in self.J_astokes])

        dl_stokes = 1 / (1 / self.wavelength + np.array(dn_stokes) * 10 ** -7)
        dl_astokes = 1 / \
            (1 / self.wavelength + np.array(dn_astokes) * 10 ** -7)

        fig = plt.figure(figsize=figsize)
        ax1 = fig.add_subplot(111)

        x_step = np.diff(dl_stokes)[0]
        bar_width = x_step / 4.

        bar1 = ax1.bar(dl_stokes, ds_stokes, width=bar_width, color='blue', label='Full')
        ax1.bar(dl_astokes, ds_astokes, width=bar_width, color='blue')

        lines = [bar1[0], ]
        legends = ['Full', ]

        if show_filtered:
            bar2 = ax1.bar(dl_stokes, self.optical_filter(dl_stokes)
                    * ds_stokes, width=bar_width, color='green', alpha=1, label='Filtered')
            ax1.bar(dl_astokes, self.optical_filter(dl_astokes)
                    * ds_astokes, width=bar_width, color='green', alpha=1)
            #ax1.bar(dl_astokes, ds_astokes, width = 0.1, color = 'blue')

            lines.append(bar2[0])
            legends.append('Filtered')

        ax1.set_xlabel('Wavelength [nm])')
        ax1.set_ylabel(r'$\left( \frac{d\sigma}{d\omega}\right)_{\pi}$ [$cm^{2}sr^{-1}$]')
        #ax1.grid(True)

        if show_filter:
            ax2 = ax1.twinx()

            xmin, xmax = ax1.get_xlim()

            filter_wavelengths = np.linspace(xmin, xmax, 1000)
            filter_efficiency = self.optical_filter(filter_wavelengths)

            filter_label = 'Filter fwhm %s nm' % self.optical_filter.fwhm
            line_1, = ax2.plot(filter_wavelengths, filter_efficiency, '-g',
                     label=filter_label)
            ax2.set_ylabel('Filter efficiency')

            ax2.yaxis.label.set_color('green')
            ax2.tick_params(axis='y', colors='green')

            lines.append(line_1)
            legends.append(filter_label)
            plt.ylim(0, 1)

        if suptitle==None:
            fig.suptitle('Rotational Raman spectrum of $%s$' %
                        molecular_parameters['name'])
        else:
            fig.suptitle(suptitle)

        if legend:
            ax1.legend(lines, legends, loc=1)

        if xlim is not None:
            plt.xlim(xlim)

        plt.draw()
        plt.show()

class FilterFunction:

    def __init__(self, wavelength, fwhm):
        '''
        This simple class represents a Gaussian filter function. To generate
        a new filter use::

           my_filter = FilterFunction(wavelength, fwhm)

        with

        wavelength - The central wavelength of the filter in nm
        fwhm       - The fwhm of the filter in nm

        If the the filter is called with a wavelength (in nm) as an argument
        it will return the  efficiency at this wavelength, for example::

           my_filter = FilterFunction(532, 5)
           my_filter(532) # Will return 1.0
           my_filter(535) # Will return 0.3685
        '''
        self.wavelength = wavelength
        self.fwhm = fwhm
        self.c = fwhm / 2.354820045031

    def __call__(self, wavelength):
        #value = np.exp(-(wavelength - self.wavelength) ** 2 / (2 * self.c ** 2))
        value = np.exp((-4 * np.log(2) * (wavelength - self.wavelength) ** 2) / self.fwhm**2)
        return value