########################################################################################################################
# VECNet CI - Prototype
# Date: 09/13/2013
# Institution: University of Notre Dame
# Primary Authors: Robert Jones <Robert.Jones.428@nd.edu>
########################################################################################################################

# Imports that are external to the ts_emod app
from data_services.adapters.EMOD import EMOD_Adapter
from django.http import HttpResponse
import json

def getLocationGraphs(request, location_id=None):
    """
    Function to get graph data from DW by url
     - Input: location ID
     - Output: JSON object of highchart data
     - if called without location ID, returns empty dict
    """
    if location_id == None:
        chart_dict = {}
    else:
        adapter = EMOD_Adapter()
        location_dict = adapter.fetch_locations()
        # get HighChart data object
        try:
            chart_dict = adapter.fetch_weather_chart(location_id, 'chart-div',
                                            start_date=location_dict[int(location_id)]['start_date'],
                                            end_date=location_dict[int(location_id)]['end_date'])
        except:
            chart_dict = {}
    return HttpResponse(json.dumps(chart_dict), mimetype="application/json")


import logging
logger = logging.getLogger(__name__)

import threading
import functools
import collections

import gi
import cairo
gi.require_version('Gtk', '3.0')
from gi.repository import GLib

class SurfaceCache(object):
    """ Pages caching and prerendering made (almost) easy.

    Args:
        doc (:class:`~pympress.document.Document`):  the current document
        max_pages (`int`): The maximum page number.
    """

    #: The actual cache. The `dict`s keys are widget names and its values are
    #: :class:`~collections.OrderedDict`, whose keys are page numbers
    #: and values are instances of :class:`~cairo.ImageSurface`.
    #: In each :class:`~collections.OrderedDict` keys are ordered by
    #: Least Recently Used (get or set), when the size is beyond
    #: :attr:`max_pages`, pages are popped from the start of the cache.
    surface_cache = {}

    #: `dict` containing functions that return a :class:`~cairo.Surface` given a :class:`~cairo.Format`,
    #: width `int` and height `int`, see :meth:`~Gtk.Window.create_similar_image_surface`
    surface_factory = {}

    #: Size of the different managed widgets, as a `dict` of tuples
    surface_size = {}

    #: Type of document handled by each widget. It is a `dict`: its keys are
    #: widget names and its values are document types from ui.
    surface_type = {}

    #: Dictionary of :class:`~threading.Lock` used for managing conccurent
    #: accesses to :attr:`surface_cache` and :attr:`surface_size`
    locks = {}

    #: Set of widgets for which we ignore the max
    unlimited = set()

    #: The current :class:`~pympress.document.Document`.
    doc = None

    #: :class:`~threading.Lock` used to manage conccurent accesses to :attr:`doc`.
    doc_lock = None

    #: Set of active widgets
    active_widgets = set()

    #: maximum number fo pages we keep in cache
    max_pages = 200

    def __init__(self, doc, max_pages):
        self.max_pages = max_pages
        self.doc = doc
        self.doc_lock = threading.Lock()

    def add_widget(self, widget, wtype, prerender_enabled = True, zoomed = False, ignore_max = False):
        """ Add a widget to the list of widgets that have to be managed (for caching and prerendering).

        This creates new entries for ``widget_name`` in the needed internal data
        structures, and creates a new thread for prerendering pages for this widget.

        Args:
            widget (:class:`~Gtk.Widget`):  The widget for which we need to cache
            wtype (`int`):  type of document handled by the widget (see :attr:`surface_type`)
            prerender_enabled (`bool`):  whether this widget is initially in the list of widgets to prerender
            zoomed (`bool`): whether we will cache a zoomed portion of the widget
            ignore_max (`bool`): whether we will cache an unlimited number of slides
        """
        widget_name = widget.get_name().rstrip('0123456789') + ('_zoomed' if zoomed else '')
        with self.locks.setdefault(widget_name, threading.Lock()):
            self.surface_cache[widget_name] = collections.OrderedDict()
            self.surface_size[widget_name] = (-1, -1)
            self.surface_type[widget_name] = wtype
            self.surface_factory[widget_name] = functools.partial(self._create_surface, widget)
            if prerender_enabled and not zoomed:
                self.enable_prerender(widget_name)
            if ignore_max:
                self.unlimited.add(widget_name)

    def swap_document(self, new_doc):
        """ Replaces the current document for which to cache slides with a new one.

        This function also clears the cached pages, since they now belong to an outdated document.

        Args:
            new_doc (:class:`~pympress.document.Document`):  the new document
        """
        with self.doc_lock:
            self.doc = new_doc

        self.clear_cache()

    def disable_prerender(self, widget_name):
        """ Remove a widget from the ones to be prerendered.

        Args:
            widget_name (`str`):  string used to identify a widget
        """
        self.active_widgets.discard(widget_name)

    def enable_prerender(self, widget_name):
        """ Add a widget to the ones to be prerendered.

        Args:
            widget_name (`str`):  string used to identify a widget
        """
        self.active_widgets.add(widget_name)

    def set_widget_type(self, widget_name, wtype):
        """ Set the document type of a widget.

        Args:
            widget_name (`str`):  string used to identify a widget
            wtype (`int`):  type of document handled by the widget (see :attr:`surface_type`)
        """
        with self.locks[widget_name]:
            if self.surface_type[widget_name] != wtype:
                self.surface_type[widget_name] = wtype
                self.surface_cache[widget_name].clear()

    def get_widget_type(self, widget_name):
        """ Get the document type of a widget.

        Args:
            widget_name (`str`):  string used to identify a widget

        Returns:
            `int`: type of document handled by the widget (see :attr:`surface_type`)
        """
        return self.surface_type[widget_name]

    def clear_cache(self, widget_name=None):
        """ Remove all cached values for a given widget. Useful for zoomed views.

        Args:
            widget_name (`str`):  name of the widget that is resized, `None` for all widgets.
        """
        for widget in [widget_name] if widget_name is not None else self.locks:
            with self.locks[widget]:
                self.surface_cache[widget].clear()

    def resize_widget(self, widget_name, width, height):
        """ Change the size of a registered widget, thus invalidating all the cached pages.

        Args:
            widget_name (`str`):  name of the widget that is resized
            width (`int`):  new width of the widget
            height (`int`):  new height of the widget
        """
        with self.locks[widget_name]:
            if (width, height) != self.surface_size[widget_name]:
                self.surface_cache[widget_name].clear()
                self.surface_size[widget_name] = (width, height)

    def get(self, widget_name, page_nb):
        """ Fetch a cached, prerendered page for the specified widget.

        Args:
            widget_name (`str`):  name of the concerned widget
            page_nb (`int`):  number of the page to fetch in the cache

        Returns:
            :class:`~cairo.ImageSurface`: the cached page if available, or `None` otherwise
        """
        with self.locks[widget_name]:
            pc = self.surface_cache[widget_name]
            if page_nb in pc:
                pc.move_to_end(page_nb)
                return pc[page_nb]
            else:
                return None

    def put(self, widget_name, page_nb, val):
        """ Store a rendered page in the cache.

        Args:
            widget_name (`str`):  name of the concerned widget
            page_nb (`int`):  number of the page to store in the cache
            val (:class:`~cairo.ImageSurface`):  content to store in the cache
        """
        with self.locks[widget_name]:
            pc = self.surface_cache[widget_name]
            pc[page_nb] = val
            pc.move_to_end(page_nb)

            while len(pc) > self.max_pages:
                pc.popitem(False)

    def _create_surface(self, widget, fmt, width, height):
        """ Given a widget, create a cairo Image surface with appropriate size and scaling.

        Args:
            widget (:class:`~Gtk.Widget`): the widget for which we’re caching data.
            fmt (:class:`~cairo.Format`): the format for the new surface
            width (`int`): width of the new surface
            height (`int`): height of the new surface

        Returns:
            :class:`~cairo.ImageSurface`: a new image surface
        """
        window = widget.get_window()
        scale = window.get_scale_factor()
        try:
            return window.create_similar_image_surface(fmt, width * scale, height * scale, scale)
        except cairo.Error:
            logger.warning('Failed creating a {} cache surface sized {}x{} scale {} for widget {}'
                           .format(fmt, width * scale, height * scale, scale, widget.get_name()), exc_info=True)
            # Warn here but handle error at the call site
            raise

    def prerender(self, page_nb):
        """ Queue a page for prerendering.

        The specified page will be prerendered for all the registered widgets.

        Args:
            page_nb (`int`):  number of the page to be prerendered
        """
        for name in self.active_widgets:
            GLib.idle_add(self.renderer, name, page_nb)

    def renderer(self, widget_name, page_nb):
        """ Rendering function.

        This function is meant to be scheduled on the GLib main loop. When run,
        it will go through the following steps:

        - check if the job's result is not already available in the cache
        - render it in a new :class:`~cairo.ImageSurface` if necessary
        - store it in the cache if it was not added there since the beginning of
          the process and the widget configuration is still valid

        Args:
            widget_name (`str`):  name of the concerned widget
            page_nb (`int`):  number of the page to store in the cache
        """
        with self.locks[widget_name]:
            if page_nb in self.surface_cache[widget_name]:
                # Already in cache
                return GLib.SOURCE_REMOVE
            ww, wh = self.surface_size[widget_name]
            wtype = self.surface_type[widget_name]

        if ww < 0 or wh < 0:
            logger.warning('Widget {} with invalid size {}x{} when rendering'.format(widget_name, ww, wh))
            return GLib.SOURCE_REMOVE

        with self.doc_lock:
            page = self.doc.page(page_nb)
            if page is None:
                return GLib.SOURCE_REMOVE

        # Render to a ImageSurface
        try:
            surface = self.surface_factory[widget_name](cairo.Format.RGB24, ww, wh)
        except AttributeError:
            logger.warning('Widget {} was not mapped when rendering'.format(widget_name), exc_info = True)
            return GLib.SOURCE_REMOVE
        except cairo.Error:
            return GLib.SOURCE_REMOVE

        context = cairo.Context(surface)
        page.render_cairo(context, ww, wh, wtype)
        del context

        # Save if possible and necessary − using PDF page numbering
        page_nb = page.page_nb
        with self.locks[widget_name]:
            pc = self.surface_cache[widget_name]
            if (ww, wh) == self.surface_size[widget_name] and page_nb not in pc:
                pc[page_nb] = surface

            if widget_name not in self.unlimited:
                pc.move_to_end(page_nb)
                while len(pc) > self.max_pages:
                    pc.popitem(False)

        return GLib.SOURCE_REMOVE

##
# Local Variables:
# mode: python
# indent-tabs-mode: nil
# py-indent-offset: 4
# fill-column: 80
# end:


#!/usr/bin/env python3
# ============================================================================
"""
Microphone file to gather audio data if microphone captures audio in .wav files.

Reads .wav files of audio stream from the folder on line 72 (unless changed, default
    will be T4Train/src/wavs/)

Microphone WAV config.ini setup:
[GLOBAL]
INSTANCES : 20 (recommended at least, increase for more data)
CHANNELS : 2 or 1 (2 for stereo, 1 for mono)
FRAME_LENGTH : 3000 (recommended chunk size)
CURR_ALGO_INDEX : 3 (random forest is recommended)

[DS]
DS_FILE_NUM : 3
SAMPLE_RATE : 48000 (corresponds to a 48 kHz microphone)

[ML]
NUM_BINS : 300 (can change)

* Requires: 1 <= NUM_BINS <= FRAME_LENGTH / 2 (for fft) or FRAMELENGTH (for raw)

Note:
    Lines 178 and 181 are for testing the file by saving stream audio as .wav file, so
    comment out when not testing

Note: Recommended to collect at at least 3-4 times for each label and to start
    making sound before and continue a little after you collect for training
"""
# ============================================================================

from datetime import timedelta
from timeloop import Timeloop
import glob
import numpy as np
import os
import signal
import sys
import time
import utils
import pyaudio
import wave
from scipy.io import wavfile
import configparser

# ============================================================================

# write PID to file
pidnum = os.getpid()
f = open("ds_pidnum.txt", "w")
f.write(str(pidnum))
f.close()

# ============================================================================
# read in configurations
config = configparser.ConfigParser()
config.read('config.ini')

# global variables
CHUNK = int(config['GLOBAL']['FRAME_LENGTH']) # how many samples at a time you want to read
FORMAT = pyaudio.paInt16  # quality of microphone
CHANNELS = int(config['GLOBAL']['CHANNELS'])
RATE = int(config['DS']['SAMPLE_RATE'])  # sampling rate
instances = int(config['GLOBAL']['INSTANCES'])  # number of instances recorded when spacebar is hit
chunks_per_second = int(RATE / CHUNK)  # how many chunks per second
training_data = [[]]
training_data_frame_counter = 0
is_collecting_dataset = False
audio_folder = "wavs/"  #folder within src w/ the wav files

# ============================================================================

print("Starting Microphone WAV")

# open an audio stream
p = pyaudio.PyAudio()

stream = p.open(format=FORMAT,
                channels=CHANNELS,
                rate=RATE,
                input=True,
                frames_per_buffer=CHUNK)

def get_oldest_wav(path):
    """ gets oldest wav file from path"""

    # checking if the wav file exists
    if not os.path.exists(path):
        print("There is no folder named " + audio_folder + ".")
        return None

    # getting all wav files from wav folder
    list_of_files = glob.glob(path + "*.wav")
    if len(list_of_files) == 0:
        print("No wavs were found.")
        return None

    # getting the earlier file
    oldest_file = min(list_of_files, key=os.path.getctime)
    return oldest_file

def save_wav(data, path):
    """ save data as wav file """
    wf = wave.open(path, 'wb')
    wf.setnchannels(CHANNELS)
    wf.setsampwidth(p.get_sample_size(FORMAT))
    wf.setframerate(RATE)
    wf.writeframes(data)
    wf.close()

def save_wav_in_file(data):
    """ save wav in wav folder """

    # naming the file
    curr_time=time.strftime("%Y_%m_%d-%H_%M")
    curr_time_file = curr_time + ".wav"

    # makes directory if directory does not already exist
    if not os.path.exists(audio_folder):
        os.makedirs(audio_folder)

    # saving the wavs
    path = os.path.join(os.getcwd(), audio_folder)
    save_wav(data, os.path.join(path, curr_time_file))
    return

def get_wav_from_file():
    """ gets earliest wav in wavs folder, returns data and
    removes file"""

    # gets path for earliest file & reads data into np array
    folder_path = os.path.join(os.getcwd(), audio_folder)
    wav_path = get_oldest_wav(folder_path)

    fs, data = wavfile.read(wav_path)  # fs == sampling rate
    data = data.T

    # remove the file
    os.remove(wav_path)
    return data

def shape_data(data):
    """ Returns: CHANNELS x (CHUNK + 2) np array of data
        last 2 cols: col 1 is the channel index, col 2 is 0 or 1
                for finished or unfinished chunk of data
    """
    global CHANNELS

    # finding the number of channels from # of rows
    CHANNELS = data.shape[0]

    # initialize column endings matrix
    col_endings = np.zeros((CHANNELS, 2))
    for i in range(CHANNELS):
        col_endings[i, 0] = i
    col_endings[-1, -1] = 1  # last col of last row is 1 for finished frame

    tmpframe = np.hstack((data, col_endings))  # add row endings to tmpframe for ui

    return tmpframe

def microphone_data():
    """" Reads wavs from audio folder into data and saves
        chunk of data with column endings for UI and ML """
    global is_collecting_dataset, training_data_frame_counter, \
        frame, training_data

    try:
        # data = stream.read(CHUNK)  # comment out if microphone sends wavs

        # save the data as a wav in wav folder
        # save_wav_in_file(data)  # comment out if microphone sends wavs

        # read data from wav folder (wavs)
        tmpframe = get_wav_from_file()
        tmpframe = shape_data(tmpframe)

        # saves tmpframe to display points for ui
        np.save('tmpframe', tmpframe)

    except Exception as e:
        print("Could not get data.")
        print(e)
        return

    try:

        # appends chunks to training data
        if is_collecting_dataset and training_data_frame_counter < instances:
            training_data[0].append(tmpframe)
            training_data_frame_counter += 1

        if training_data_frame_counter == instances:
            print('Done collecting training data, saving NOW')
            training_data_frame_counter = 0
            training_data[0] = training_data[0]

            # get label
            f = open("current_label.txt", "r")
            current_label = f.read().strip()
            f.close()

            training_data_file_name = 'training_data_{}.npy'.format(current_label)

            print('Saving Training Data...')

            # appends training data to existing data or saves as new data
            if os.path.exists(os.path.join(os.getcwd(), training_data_file_name)):
                existing_training_data = np.load(training_data_file_name)

                np.save(training_data_file_name,
                        np.append(existing_training_data, training_data, axis=0))
            else:
                np.save(training_data_file_name, training_data)
            training_data = [[]]
            is_collecting_dataset = False

            print('Training Data SAVED!!!')

    except Exception as e:
        print(e)
        return

# MAC/LINUX
def receive_interrupt(signum, stack):
    read_message()

def read_message():
    global is_collecting_dataset
    try:
        f = open("ds_cmd.txt", "r")
        cmd = f.read()
        f.close()
    except Exception as e:
        return
    if cmd == 'SPACEBAR':
        is_collecting_dataset = True
        print("Spacebar Closing")
    elif cmd == 'BYE':
        print("Microphone WAV Closing")

        # closing the system
        f = open("ds_cmd.txt", "w")
        f.write("")
        f.close()

        # closing the audio stream
        stream.stop_stream()
        stream.close()
        p.terminate()

        os._exit(0)
    f = open("ds_cmd.txt", "w")
    f.write("")
    f.close()

if utils.does_support_signals():
    signal.signal(signal.SIGINT, receive_interrupt)

    while True:
        microphone_data()

timeloop = Timeloop()

# add timeloop job to handle commands
@timeloop.job(interval=timedelta(seconds=0.3))
def read_message_wrapper():
    read_message()

# add timeloop job to collect and write microphone data
@timeloop.job(interval=timedelta(seconds=(1/chunks_per_second)))
def microphone_data_wrapper():
    microphone_data()

if not utils.does_support_signals():
    timeloop.start(block=True)

print("Microphone WAV Closing")

# closing the audio stream
stream.stop_stream()
stream.close()
p.terminate()


# python3 and python2
import numpy as np
import cv2
import glob, os
import tqdm
from itertools import repeat
from multiprocessing import Pool
from functools import partial

def align_2p(img, left_eye, right_eye):
    width = 256
    eye_width = 70

    transform = np.matrix([
            [1, 0, left_eye[0]],
            [0, 1, left_eye[1]],
            [0, 0, 1]
    ], dtype='float')

    th = np.pi + -np.arctan2(left_eye[1] - right_eye[1], left_eye[0] - right_eye[0])
    transform *= np.matrix([
            [np.cos(th), np.sin(th), 0],
            [-np.sin(th), np.cos(th), 0],
            [0, 0, 1]
    ], dtype='float')

    scale = np.sqrt((left_eye[1] - right_eye[1]) ** 2 + (left_eye[0] - right_eye[0]) ** 2) / eye_width
    transform *= np.matrix([
            [scale, 0, 0],
            [0, scale, 0],
            [0, 0, 1]
    ], dtype='float')

    transform *= np.matrix([
            [1, 0, -(width - eye_width) / 2],
            [0, 1, -width / 2.42],
            [0, 0, 1]
    ], dtype='float')

    transform = np.linalg.inv(transform)
    jmg = cv2.warpAffine(img, transform[:2], (width, width))
    return jmg

def align_face_2p(img, landmarks):
    left_eye = (landmarks[0], landmarks[1])
    right_eye = (landmarks[2], landmarks[3])
    aligned_img = align_2p(img, left_eye, right_eye)
    return aligned_img

# average landmarks
mean_face_lm5p = np.array([
    [-0.17607, -0.172844],  # left eye pupil
    [0.1736, -0.17356],  # right eye pupil
    [-0.00182, 0.0357164],  # nose tip
    [-0.14617, 0.20185],  # left mouth corner
    [0.14496, 0.19943],  # right mouth corner
])

def _get_align_5p_mat23_size_256(lm):
    # legacy code
    width = 256
    mf = mean_face_lm5p.copy()

    # Assumptions:
    # 1. The output image size is 256x256 pixels
    # 2. The distance between two eye pupils is 70 pixels
    ratio = 70.0 / (
       256.0 * 0.34967
    )  # magic number 0.34967 to compensate scaling from average landmarks

    left_eye_pupil_y = mf[0][1]
    # In an aligned face image, the ratio between the vertical distances from eye to the top and bottom is 1:1.42
    ratioy = (left_eye_pupil_y * ratio + 0.5) * (1 + 1.42)
    mf[:, 0] = (mf[:, 0] * ratio + 0.5) * width
    mf[:, 1] = (mf[:, 1] * ratio + 0.5) * width / ratioy
    mx = mf[:, 0].mean()
    my = mf[:, 1].mean()
    dmx = lm[:, 0].mean()
    dmy = lm[:, 1].mean()
    mat = np.zeros((3, 3), dtype=float)
    ux = mf[:, 0] - mx
    uy = mf[:, 1] - my
    dux = lm[:, 0] - dmx
    duy = lm[:, 1] - dmy
    c1 = (ux * dux + uy * duy).sum()
    c2 = (ux * duy - uy * dux).sum()
    c3 = (dux**2 + duy**2).sum()
    a = c1 / c3
    b = c2 / c3

    kx = 1
    ky = 1

    s = c3 / (c1**2 + c2**2)
    ka = c1 * s
    kb = c2 * s

    transform = np.zeros((2, 3))
    transform[0][0] = kx * a
    transform[0][1] = kx * b
    transform[0][2] = mx - kx * a * dmx - kx * b * dmy
    transform[1][0] = -ky * b
    transform[1][1] = ky * a
    transform[1][2] = my - ky * a * dmy + ky * b * dmx
    return transform

def get_align_5p_mat23(lm5p, size):
    """Align a face given 5 facial landmarks of
    left_eye_pupil, right_eye_pupil, nose_tip, left_mouth_corner, right_mouth_corner

    :param lm5p: nparray of (5, 2), 5 facial landmarks,

    :param size: an integer, the output image size. The face is aligned to the mean face

    :return: a affine transformation matrix of shape (2, 3)
    """
    mat23 = _get_align_5p_mat23_size_256(lm5p.copy())
    mat23 *= size / 256
    return mat23

def align_given_lm5p(img, lm5p, size):
    mat23 = get_align_5p_mat23(lm5p, size)
    return cv2.warpAffine(img, mat23, (size, size))

def align_face_5p(img, landmarks):
    aligned_img = align_given_lm5p(img, np.array(landmarks).reshape((5, 2)), 256)
    return aligned_img

def work(data_dir, out_dir, landmarks, i):
    src_imname = os.path.join(data_dir, 'data', '{:06d}.jpg'.format(i+1))
    des_imname = os.path.join(out_dir, '{:06d}.jpg'.format(i+1))
    img = cv2.imread(src_imname)
    aligned_img = align_face_5p(img, landmarks[i])
    cv2.imwrite(des_imname, aligned_img)
    return 0

def main(data_dir, out_dir, thread_num):
    if not os.path.exists(out_dir):
        os.makedirs(out_dir)

    with open(os.path.join(data_dir, 'list_landmarks_celeba.txt'), 'r') as f:
        landmarks = [list(map(int, x.split()[1:11])) for x in f.read().strip().split('\n')[2:]]

    im_list = glob.glob(os.path.join(data_dir, 'data/*.jpg'))

    pool = Pool(thread_num)
    # pool.starmap(work, zip(range(len(im_list)), repeat(data_dir), repeat(out_dir), repeat(landmarks)))
    partial_work = partial(work, data_dir, out_dir, landmarks)
    pool.map(partial_work, range(len(im_list)))
    pool.close()
    pool.join()

if __name__ == '__main__':
    os.environ["CUDA_VISIBLE_DEVICES"] = ''
    main('./datasets/celebA/', './datasets/celebA/align_5p/', 30)


"""
this script will be exported to the "install.exe" file,
it will automatically install the program and add right-click menus
"""

import os
import shutil
import json
import tkinter as tk
from tkinter import ttk
from tkinter import messagebox
import winreg
from setting import settings as example_settings
from setting import setting_file_name

import ctypes
import sys

def is_admin():
    try:
        return ctypes.windll.shell32.IsUserAnAdmin()
    except:
        return False

# this version I needs click "show more options" to see the option
# def add_context_menu_option(option_name, exe_path, reg_class):
#     """
#     Add a context menu option for the given executable.
#
#     Args:
#     option_name (str): The name of the context menu option.
#     exe_path (str): The path to the executable file.
#     reg_class (str): The registry class for the context menu option.
#     """
#     try:
#         registry_path = fr"Software\\Classes\\{reg_class}\\ShellEx\\{option_name}"
#         command = f'"{exe_path}" "%V"'
#         with winreg.CreateKey(winreg.HKEY_CURRENT_USER, registry_path) as key:
#             winreg.SetValue(key, "", winreg.REG_SZ, option_name)
#
#         registry_path_command = fr"{registry_path}\\command"
#         with winreg.CreateKey(winreg.HKEY_CURRENT_USER, registry_path_command) as key:
#             winreg.SetValue(key, "", winreg.REG_SZ, command)
#     except Exception as e:
#         print(f"Failed to add context menu option '{option_name}': {e}")

def add_context_menu_option(option_name, exe_path, reg_class):
    """
    Add a context menu option for the given executable.

    Args:
    option_name (str): The name of the context menu option.
    exe_path (str): The path to the executable file.
    reg_class (str): The registry class for the context menu option.
    """
    try:
        registry_path = rf"Software\\Classes\\{reg_class}\\shell\\{option_name}"
        command = f'"{exe_path}" "%V"'
        with winreg.CreateKey(winreg.HKEY_CURRENT_USER, registry_path) as key:
            winreg.SetValue(key, "", winreg.REG_SZ, option_name)
            winreg.SetValue(key, "Extended", winreg.REG_SZ, "")

        registry_path_command = rf"{registry_path}\command"
        with winreg.CreateKey(winreg.HKEY_CURRENT_USER, registry_path_command) as key:
            winreg.SetValue(key, "", winreg.REG_SZ, command)
    except Exception as e:
        print(f"Failed to add context menu option '{option_name}': {e}")

def save_settings(settings):
    """
    Save settings as a JSON file in the user's home directory.

    Args:
    settings (dict): The settings to save.
    """
    user_home = os.path.expanduser("~")
    settings_file = os.path.join(user_home, setting_file_name)
    with open(settings_file, "w") as f:
        json.dump(settings, f, indent=4)

def on_install_click(collected_settings_vars):
    """
    Install the script and save the settings.

    Args:
    collected_settings_vars (dict): A dictionary containing the Tkinter variable objects for each setting.
    """
    settings = {k: v.get() for k, v in collected_settings_vars.items()}
    save_settings(settings)

    appdata_path = os.getenv("APPDATA")
    destination_folder = os.path.join(appdata_path, "MultilevelUnzipper")
    os.makedirs(destination_folder, exist_ok=True)

    # copy and replace if exists
    src_folder = "./MultilevelUnzipper"
    for item in os.listdir(src_folder):
        src_item_path = os.path.join(src_folder, item)
        dst_item_path = os.path.join(destination_folder, item)

        if os.path.isfile(src_item_path):
            shutil.copy(src_item_path, dst_item_path)
        elif os.path.isdir(src_item_path):
            if os.path.exists(dst_item_path):
                shutil.rmtree(dst_item_path)
            shutil.copytree(src_item_path, dst_item_path)

    try:
        # ... Rest of the on_install_click function ...

        # Add context menu options
        exe_path = os.path.join(destination_folder, "MultilevelUnzipper.exe")
        add_context_menu_option(
            "Unzip all files under this directory", exe_path, "Directory\\Background"
        )
        add_context_menu_option("Unzip this file", exe_path, "AllFileSystemObjects")

        # ... Rest of the on_install_click function ...
    except Exception as e:
        print(f"Installation failed: {e}")
        return

    # Show a pop-up message when the installation finishes successfully
    messagebox.showinfo(
        "Installation Complete", "MultilevelUnzipper has been installed successfully!"
    )

def create_setting_component(parent, row, setting_key, setting_value):
    """
    Create a setting component based on the setting data type.

    Args:
    parent (Tkinter object): The parent Tkinter object for the component.
    row (int): The row number in the grid where the component will be placed.
    setting_key (str): The name of the setting.
    setting_value (str, int, bool): The default value of the setting.

    Returns:
    Tkinter variable object: The variable object associated with the created component.
    """
    ttk.Label(parent, text=f"{setting_key}:").grid(column=1, row=row, sticky=tk.E)
    setting_var = None

    if isinstance(setting_value, str):
        setting_var = tk.StringVar(value=setting_value)
        ttk.Entry(parent, width=40, textvariable=setting_var).grid(column=2, row=row, sticky=tk.W)
    elif isinstance(setting_value, bool):
        setting_var = tk.BooleanVar(value=setting_value)
        ttk.Checkbutton(parent, variable=setting_var).grid(column=2, row=row, sticky=tk.W)
    elif isinstance(setting_value, int):
        setting_var = tk.IntVar(value=setting_value)
        ttk.Entry(parent, width=15, textvariable=setting_var).grid(column=2, row=row, sticky=tk.W)

    return setting_var

def main():
    """
    The main function that creates and displays the installer GUI.
    """
    root = tk.Tk()
    root.title("My Script Installer")
    root.geometry("400x300")
    root.resizable(False, False)

    mainframe = ttk.Frame(root, padding="20 20 20 20")
    mainframe.grid(column=0, row=0, sticky=(tk.W, tk.E, tk.N, tk.S))

    collected_settings_vars = {}
    for index, (setting_key, setting_value) in enumerate(example_settings.items()):
        collected_settings_vars[setting_key] = create_setting_component(
            mainframe, index + 1, setting_key, setting_value
        )

    # Customize the install button
    install_button_style = ttk.Style()
    install_button_style.configure("TButton", font=("Helvetica", 12, "bold"))
    install_button = ttk.Button(
        mainframe,
        text="Install",
        style="TButton",
        width=20,
        command=lambda: on_install_click(collected_settings_vars),
    )
    install_button.grid(
        column=1, row=len(example_settings) + 1, columnspan=2, pady=(20, 0), sticky=tk.EW
    )

    # Add padding between components for a cleaner look
    install_button = ttk.Button(
        mainframe, text="Install", command=lambda: on_install_click(collected_settings_vars)
    )

    root.mainloop()

if __name__ == "__main__":
    # Check if the script is running with administrator privileges
    if not is_admin():
        ctypes.windll.shell32.ShellExecuteW(None, "runas", sys.executable, sys.argv[0], None, 1)
    else:
        # Your code here
        print("Running with administrator privileges")

    main()


import scrapy
import os

if os.path.exists(os.path.join(os.getcwd(), 'data')):
	pass
else:
	os.mkdir('data')

class CourseDescriptionSpider(scrapy.Spider):

	name = 'hw2Spider'
	start_urls = ['https://www.registrar.ucla.edu/Academics/Course-Descriptions']
	hostname = 'https://www.registrar.ucla.edu'

	def parse(self, response):
		rows = response.css('div.list-letter div.row')
		for row in rows:
			coursenames = row.css('ul tr li a::text').extract()
			links = row.css('ul tr li a::attr("href")').extract()
			for i in range(len(coursenames)):
				link = self.hostname + links[i]
				req = scrapy.Request(url = link, callback = self.parseCoursePage)
				req.meta['c_name'] = coursenames[i]
				yield req

	def parseCoursePage(self, response):
		f = open('data/'+response.meta['c_name'].replace('/',' ')+'.html','wb')
		f.write(response.body)
		f.close()


from django.core.exceptions import ValidationError
from django.test import TestCase
from django.urls import reverse

from items.models import Item, InventoryItem
from orders.models import Order, OrderUpdate, OrderItem

#def create_test_item(item_name):
#  return Item.objects.create(name=item_name, price=1, msrp=2, description="Test item description")
#
#class ItemModelTests(TestCase):
#
#  def test_negative_price(self):
#    test_item = Item(name="Test item", price=-1, msrp=2, description="Test item description")
#    self.assertRaises(ValidationError, test_item.full_clean)
#
#  def test_positive_msrp(self):
#    test_item = Item(name="Test item", price=1, msrp=2, description="Test item description")
#    test_item.full_clean()
#
#  def test_zero_qty(self):
#    test_item = create_test_item('Test item')
#    test_inventory_item = InventoryItem(item_id=test_item.id, qty=0)
#    test_inventory_item.full_clean()

#class ItemIndexViewTests(TestCase):
#
#  def test_no_items(self):
#    response = self.client.get(reverse('items:index'))
#    self.assertEqual(response.status_code, 200)
#    self.assertQuerysetEqual(response.context['item_list'], [])
#
#  def test_multiple_items(self):
#    create_test_item('Test item 1')
#    create_test_item('Test item 2')
#    response = self.client.get(reverse('items:index'))
#    self.assertQuerysetEqual(response.context['item_list'], ['<Item: Test item 1>', '<Item: Test item 2>'])
#
#  def test_items_order(self):
#    create_test_item('Aa')
#    create_test_item('Z')
#    create_test_item('A')
#    create_test_item('5')
#    response = self.client.get(reverse('items:index'))
#    self.assertQuerysetEqual(
#      response.context['item_list'],
#      ['<Item: 5>', '<Item: A>', '<Item: Aa>', '<Item: Z>'])

class OrderCreateViewTests(TestCase):

  def test_valid_order(self):
    data = {
      'email': 'test@test.com',
      'form-TOTAL_FORMS':	'2',
      'form-INITIAL_FORMS':	'0',
      'form-MIN_NUM_FORMS':	'0',
      'form-MAX_NUM_FORMS':	'1000',
      'form-0-name': 'Shipping Name',
      'form-0-line_one': '1234 Shipping St',
      'form-0-line_two': 'Apt 1',
      'form-0-city': 'Shipping City',
      'form-0-state': 'SS',
      'form-0-zip_code': '12345',
      'form-0-country': 'US',
      'form-0-phone': '5551234567',
      'form-0-id': '',
      'form-1-name': 'Billing Name',
      'form-1-line_one': '1234 Billing St',
      'form-1-line_two': 'Apt 2',
      'form-1-city': 'Billing City',
      'form-1-state': 'BS',
      'form-1-zip_code': '67890',
      'form-1-country': 'US',
      'form-1-phone': '5559876543',
      'form-1-id': '',
    }
    response = self.client.post(reverse('orders:new'), data=data)
    self.assertEqual(response.status_code, 302)

    order = Order.objects.get(pk=1)
    self.assertEqual(order.customer.email, 'test@test.com')
    self.assertEqual(order.customer.shipping_address.name, 'Shipping Name')
    self.assertEqual(order.customer.shipping_address.line_one, '1234 Shipping St')
    self.assertEqual(order.customer.shipping_address.line_two, 'Apt 1')
    self.assertEqual(order.customer.shipping_address.city, 'Shipping City')
    self.assertEqual(order.customer.shipping_address.state, 'SS')
    self.assertEqual(order.customer.shipping_address.zip_code, '12345')
    self.assertEqual(order.customer.shipping_address.country, 'US')
    self.assertEqual(order.customer.shipping_address.phone, '5551234567')
    self.assertEqual(order.customer.billing_address.name, 'Billing Name')
    self.assertEqual(order.customer.billing_address.line_one, '1234 Billing St')
    self.assertEqual(order.customer.billing_address.line_two, 'Apt 2')
    self.assertEqual(order.customer.billing_address.city, 'Billing City')
    self.assertEqual(order.customer.billing_address.state, 'BS')
    self.assertEqual(order.customer.billing_address.zip_code, '67890')
    self.assertEqual(order.customer.billing_address.country, 'US')
    self.assertEqual(order.customer.billing_address.phone, '5559876543')

    # TODO: order items?

    # TODO implement shipping calculation
    self.assertEqual(order.shipping, 5)

    order_update = order.orderupdate_set.first()
    self.assertEqual(order_update.status, OrderUpdate.PROCESSING)

  #def test_nonexistent_item(self):
  #  response = self.client.get(reverse('items:detail', args=(1234,)))
  #  self.assertEqual(response.status_code, 404)

  #def test_existing_item(self):
  #  item = create_test_item('Test item')
  #  inventory_item = InventoryItem.objects.create(item_id=item.id, qty=1)
  #  response = self.client.get(reverse('items:detail', args=(item.id,)))
  #  print('response {}'.format(response))
  #  self.assertContains(response, 'Test item')


from typing import Any, Tuple

import torch
from torch import nn

class Seq2SeqEncoder(nn.Module):
    def __init__(
        self, input_size: int, hidden_size: int, device: torch.device
    ) -> None:
        super(Seq2SeqEncoder, self).__init__()
        self.hidden_size = hidden_size

        self.embedding = nn.Embedding(input_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size)
        self.device = device

    def set_hidden_size(self, size: int) -> None:
        self.hidden_size = size

    def forward(self, x, hidden: int) -> Tuple[Any, int]:
        embedded = self.embedding(x).view(1, 1, -1)
        output = embedded
        output, hidden = self.gru(output, hidden)
        return output, hidden

    def initHidden(self) -> torch.Tensor:
        return torch.zeros(1, 1, self.hidden_size, device=self.device)


from bs4 import BeautifulSoup
from Request import get_html, get_response

def pars_info(html_data):

    soup = BeautifulSoup(html_data, 'html.parser')
    cards_company = soup.find_all('div', class_='cart-company-lg')

    dict_keys = ["Название", "Сайт", "Почта", "Телефон",
                 "Вид деятельности"]

    data_base = []
    for card in cards_company:
        data_company = {}
        mail = card.find('a', target="_blank", rel=None)
        site = card.find('a', rel="nofollow", target="_blank")
        dict_value = []

        if mail:

            title = card.find('div', class_='cart-company-lg__title ui-title-inner')
            dict_value.append(title.text)

            if site:
                dict_value.append(site.text)
            else:
                dict_value.append("")

            dict_value.append(mail.text)

            phone = card.find('a', rel="nofollow", class_="text-nowrap")
            if phone:
                phone = phone.text
            else:
                phone = None
            dict_value.append(phone)

            categorys_card = card.find('ul', class_="ui-list-mark")
            categorys = categorys_card.find_all('a')

            category = []
            for i in categorys:
                content = i.text
                category.append(content)
            dict_value.append(category)

            n = 0
            for key in dict_keys:
                data_company[key] = dict_value[n]
                n += 1
            #print('[INFO] Дание о компании: ', data_company)
            data_base.append(data_company)

    return data_base

def pages(link, HEADERS):
    max_page = 1

    response = get_response(link, HEADERS)
    html_data = get_html(response)

    soup = BeautifulSoup(html_data, 'html.parser')

    paginations_class = soup.find('ul', class_='pagination')
    paginations = paginations_class.find_all('a', class_='item-link', text=True)

    if not paginations:
        return max_page
    else:
        max_page = int(paginations[-1].text)
        return max_page


from selenium import webdriver
import time

query_txt = "moneynlifehacker.com"
chrome_path = "D:/Google Drive/chromedriver.exe"
driver = webdriver.Chrome(chrome_path)

url = "https://naver.com"
driver.get(url)
time.sleep(2)

element = driver.find_element_by_id("query")
driver.find_element_by_id("query").click()
element.send_keys(query_txt)
element.send_keys("\n")
time.sleep(2)

driver.find_element_by_link_text("VIEW").click()
time.sleep(2)

from bs4 import BeautifulSoup
html = driver.page_source
soup = BeautifulSoup(html, "html.parser")
content = soup.find('ul',"lst_total").find_all("li")

import sys
f_name = "d:/navertest.txt"
orig_stdout = sys.stdout
file = open(f_name, 'a', encoding="UTF-8")
sys.stdout = file

for i in content :
    print(i.get_text().replace("\n",""))

file.close()
sys.stdout = orig_stdout


#! usr/bin/env python3

## This script is modified from PECO splitting script
## NOTE that it is essential for every excel file to have values in sample_metadata

# %% load-in
# python package for handling excel
import openpyxl
import os
import csv
# this package has a 'here' command literally like r
from pyprojroot import here

def split2csv(wb):
    for sheetName in wb.sheetnames:
        # we don't need our reference sheets
        if sheetName.startswith(('abundance summary', 'metadata', 'codes')):
            pass
        else:
            sheet = wb[sheetName]
            logging.debug('eDNA ' + sheetName)
            # open csv, filename will be region_sheet.csv
            with open(here('./rawdata/2018calvert/seine/fabs_extract/' + version + '_' + sheetName + '.csv'), 'w') as out:
                writer = csv.writer(out)
                # write each row with iter_rows
                for row in sheet.iter_rows(values_only=True):
                    # make it stop when there are no more values
                    if row[0] == None:
                        logging.debug('endsheet')
                        break
                    else:
                        writer.writerow(row)

if __name__ == '__main__':
    version = '2021Nov_FABSMaster'
    path = here('./rawdata/2018calvert/seine/raw/FABSMasterData.xlsx')
    wb = openpyxl.load_workbook(filename=path)
    split2csv(wb)

    version = '2021Nov_FABSMaster2018'
    path = here('./rawdata/2018calvert/seine/raw/FABS_masterdata_2018.xlsx')
    wb = openpyxl.load_workbook(filename=path)
    split2csv(wb)


import pickle
import codecs
import multiprocessing
import codecs
import pandas as pd
from gensim.models.word2vec import Word2Vec

train_file = "data/train_thread_list.csv"
test_file = "data/test_thread_list.csv"
corpus = []

for filename in [train_file,test_file]:
    df = pd.read_csv(filename)
    print("Segment train title/content...")
    for i,data in df.iterrows():
        if i % 10000 == 0:
            print(i)
        apis = eval(data['apilist'])
        # label = data['label']
        corpus.append(' '.join(apis))

class CorpusData:
    def __init__(self, corpus):
        self.corpus = corpus
    def __iter__(self):
        for doc in self.corpus:
            origin_words = doc.split(' ')
            yield origin_words

print("Train word to vector...")
corpus_data = CorpusData(corpus)
model = Word2Vec(corpus_data, size=16, window=5, min_count=5, workers=15)
model.save('w2v_model_16_w5_m5.save')
print("Train w2v completed.")


"""add pwresettoken

Revision ID: 44db46b919f
Revises: 1deecd313c4
Create Date: 2014-01-12 00:26:21.920085

"""

# revision identifiers, used by Alembic.
revision = '44db46b919f'
down_revision = '1deecd313c4'

from alembic import op
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('pwresettoken',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('uid', sa.Integer(), nullable=False),
    sa.Column('token', sa.String(length=32), nullable=False),
    sa.Column('expire_date', sa.DateTime(), nullable=True),
    sa.ForeignKeyConstraint(['uid'], ['users.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('pwresettoken')
    ### end Alembic commands ###


##
#
# File:    ChemRefDataMiscUtilsTests.py
# Author:  jdw
# Date:    16-Mar-2016
# Version: 0.001
#
# Updates:
##
"""
A collection of tests for the ChemRefDataMiscUtils and related classes.   Requires access to
checked out CVS repository of chemical components.

"""
__docformat__ = "restructuredtext en"
__author__ = "John Westbrook"
__email__ = "jwest@rcsb.rutgers.edu"
__license__ = "Creative Commons Attribution 3.0 Unported"
__version__ = "V0.01"

import unittest
import sys
import logging
import os

if __package__ is None or __package__ == "":
    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
    from commonsetup import HERE  # noqa:  F401 pylint: disable=import-error,unused-import
else:
    from .commonsetup import HERE  # noqa: F401 pylint: disable=relative-beyond-top-level

from wwpdb.utils.session.WebRequest import InputRequest
from wwpdb.utils.config.ConfigInfo import ConfigInfo, getSiteId
from wwpdb.apps.chem_ref_data.utils.ChemRefDataMiscUtils import ChemRefDataMiscUtils

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# @unittest.skipUnless(Features().haveCCD(), "Need full CCD for tests")
class ChemRefDataMiscUtilsTests(unittest.TestCase):
    def setUp(self):
        self.__lfh = sys.stderr
        self.__verbose = True
        # Pick up site information from the environment or failover to the development site id.
        self.__siteId = getSiteId(defaultSiteId="WWPDB_DEPLOY_TEST_RU")
        logger.info("Testing with site environment for:  %s", self.__siteId)
        self.__cI = ConfigInfo(self.__siteId)
        self.__topPath = self.__cI.get("SITE_WEB_APPS_TOP_PATH")
        self.__topSessionPath = self.__cI.get("SITE_WEB_APPS_TOP_SESSIONS_PATH")
        #
        # Create a request object and session directories for test cases
        #
        self.__reqObj = InputRequest(paramDict={}, verbose=self.__verbose, log=self.__lfh)
        self.__reqObj.setValue("WWPDB_SITE_ID", self.__siteId)
        self.__reqObj.setValue("TopSessionPath", self.__topSessionPath)
        self.__reqObj.setValue("TopPath", self.__topPath)
        self.__reqObj.setDefaultReturnFormat(return_format="html")
        #

    def tearDown(self):
        pass

    def testGetCompPathList(self):
        """Test case -  get chemical component definition file path list (serial)"""
        logger.info("Starting")
        try:
            mu = ChemRefDataMiscUtils(self.__reqObj, verbose=self.__verbose, log=self.__lfh)
            pathList = mu.getChemCompPathList()
            logger.info("Length of path list %d", len(pathList))

        except:  # noqa: E722 pylint: disable=bare-except
            logger.exception("In test")
            self.fail()

    def testGetCompPathListMulti(self):
        """Test case -  get chemical component definition file path list (multiprocess)"""
        logger.info("Starting")
        try:
            mu = ChemRefDataMiscUtils(self.__reqObj, verbose=self.__verbose, log=self.__lfh)
            pathList = mu.getChemCompPathListMulti(numProc=4)
            logger.info("Length of path list %d", len(pathList))

        except:  # noqa: E722 pylint: disable=bare-except
            logger.exception("In testGetCompPathListMulti")
            self.fail()

    def testUpdateChemRefDataFiles(self):
        """Test case -  create chemical component definition file idList, pathList, and concatenated dictionary."""
        logger.info("Starting")
        ok = False
        try:
            mu = ChemRefDataMiscUtils(self.__reqObj, verbose=self.__verbose, log=self.__lfh)
            ok = mu.updateChemCompSupportFiles(skipIndex=True)  # Do not generate sdb or index file
            self.assertTrue(ok)
        except:  # noqa: E722 pylint: disable=bare-except
            logger.exception("In testUpdateChemRefDataFiles")
            self.fail()

def suitePathList():
    suiteSelect = unittest.TestSuite()
    suiteSelect.addTest(ChemRefDataMiscUtilsTests("testGetCompPathList"))
    suiteSelect.addTest(ChemRefDataMiscUtilsTests("testGetCompPathListMulti"))
    return suiteSelect

def suiteUpdateReferenceFiles():
    suiteSelect = unittest.TestSuite()
    suiteSelect.addTest(ChemRefDataMiscUtilsTests("testUpdateChemRefDataFiles"))
    return suiteSelect

if __name__ == "__main__":
    #
    if True:  # pylint: disable=using-constant-test
        mySuite = suitePathList()
        unittest.TextTestRunner(verbosity=2).run(mySuite)

        mySuite = suiteUpdateReferenceFiles()
        unittest.TextTestRunner(verbosity=2).run(mySuite)


import streamlit as st
import pickle
import pandas as pd
import numpy as np
from xgboost.sklearn import XGBRegressor

st.set_page_config(
    layout='wide')
st.title("Predicted HF")
st.sidebar.title('Features')

#----------------------------------------------------------------------------------------------------
first_kpi, second_kpi = st.columns([10000, 1])

def get_input_data(MIN_df, MAX_df):

	f1 = st.sidebar.slider("Release_Dose_Calc_mg_m", MIN_df.loc["Release_Dose_Calc_mg_m"][0], MAX_df.loc["Release_Dose_Calc_mg_m"][0], float((MIN_df.loc["Release_Dose_Calc_mg_m"][0] + MAX_df.loc["Release_Dose_Calc_mg_m"][0])/2), step = 0.01)
	f2 = st.sidebar.slider("HEADBOX_PRES_psig", MIN_df.loc["HEADBOX_PRES_psig"][0], MAX_df.loc["HEADBOX_PRES_psig"][0], float((MIN_df.loc["HEADBOX_PRES_psig"][0] + MAX_df.loc["HEADBOX_PRES_psig"][0])/2), step = 0.01)
	f3 = st.sidebar.slider("Steam_Box_Pressure_PSIG", MIN_df.loc["Steam_Box_Pressure_PSIG"][0], MAX_df.loc["Steam_Box_Pressure_PSIG"][0], float((MIN_df.loc["Steam_Box_Pressure_PSIG"][0] + MAX_df.loc["Steam_Box_Pressure_PSIG"][0])/2), step = 0.01)
	f4 = st.sidebar.slider("PM15_Winder_Speed_FPM", MIN_df.MIN[3], MAX_df.MAX[3], float((MIN_df.MIN[3] + MAX_df.MAX[3])/2), step = 0.01)
	f5 = st.sidebar.slider("Refiner_1_HPD_T", MIN_df.MIN[4], MAX_df.MAX[4], float((MIN_df.MIN[4] + MAX_df.MAX[4])/2), step = 0.01)
	f6 = st.sidebar.slider("UHLEBOX1_VAC_Inhg", MIN_df.MIN[5], MAX_df.MAX[5], float((MIN_df.MIN[5] + MAX_df.MAX[5])/2), step = 0.01)
	f7 = st.sidebar.slider("Hi_Bright_PM15_Ratio", MIN_df.MIN[6], MAX_df.MAX[6], float((MIN_df.MIN[6] + MAX_df.MAX[6])/2), step = 0.01)
	f8 = st.sidebar.slider("EosYMoiPv_pct", MIN_df.MIN[7], MAX_df.MAX[7], float((MIN_df.MIN[7] + MAX_df.MAX[7])/2), step = 0.01)
	f9 = st.sidebar.slider("Coating_Dose_Calc_mg_m", MIN_df.MIN[8], MAX_df.MAX[8], float((MIN_df.MIN[8] + MAX_df.MAX[8])/2), step = 0.01)
	f10 = st.sidebar.slider("Refiner_3_HP_Control", MIN_df.MIN[9], MAX_df.MAX[9], float((MIN_df.MIN[9] + MAX_df.MAX[9])/2), step = 0.01)
	f11 = st.sidebar.slider("Metso_Rush_Drag_to_DV", MIN_df.MIN[10], MAX_df.MAX[10], float((MIN_df.MIN[10] + MAX_df.MAX[10])/2), step = 0.01)
	f12 = st.sidebar.slider("EUC_PM15_Ratio", MIN_df.MIN[11], MAX_df.MAX[11], float((MIN_df.MIN[11] + MAX_df.MAX[11])/2), step = 0.01)
	f13 = st.sidebar.slider("VirginStock_PM15_Ratio", MIN_df.MIN[12], MAX_df.MAX[12], float((MIN_df.MIN[12] + MAX_df.MAX[12])/2), step = 0.01)
	f14 = st.sidebar.slider("RPW_Pressure_PSI", MIN_df.MIN[13], MAX_df.MAX[13], float((MIN_df.MIN[13] + MAX_df.MAX[13])/2), step = 0.01)
	f15 = st.sidebar.slider("PM_BWT_lbs_3000sqft", MIN_df.MIN[14], MAX_df.MAX[14], float((MIN_df.MIN[14] + MAX_df.MAX[14])/2), step = 0.01)
	f16 = st.sidebar.slider("PM15_Fan_Pump_ORP_pH", MIN_df.MIN[15], MAX_df.MAX[15], float((MIN_df.MIN[15] + MAX_df.MAX[15])/2), step = 0.01)
	f17 = st.sidebar.slider("Refiner_2_HPD_T", MIN_df.MIN[16], MAX_df.MAX[16], float((MIN_df.MIN[16] + MAX_df.MAX[16])/2), step = 0.01)
	f18 = st.sidebar.slider("PM_15_TWS_Flow__ton", MIN_df.MIN[17], MAX_df.MAX[17], float((MIN_df.MIN[17] + MAX_df.MAX[17])/2), step = 0.01)

	input_data = pd.DataFrame.from_dict({'Release_Dose_Calc_mg_m': [f1], 'HEADBOX_PRES_psig': [f2],
		'Steam_Box_Pressure_PSIG': [f3], 'PM15_Winder_Speed_FPM': [f4], 'Refiner_1_HPD_T': [f5],
		'UHLEBOX1_VAC_Inhg': [f6], 'Hi_Bright_PM15_Ratio': [f7], 'EosYMoiPv_pct': [f8],
		'Coating_Dose_Calc_mg_m': [f9], 'Refiner_3_HP_Control': [f10], 'Metso_Rush_Drag_to_DV': [f11],
		'EUC_PM15_Ratio': [f12], 'VirginStock_PM15_Ratio': [f13], 'RPW_Pressure_PSI': [f14],
		'PM_BWT_lbs_3000sqft': [f15], 'PM15_Fan_Pump_ORP_pH': [f16], 'Refiner_2_HPD_T': [f17],
		'PM_15_TWS_Flow__ton': [f18]})

	input_data= pd.DataFrame(input_data)
	return input_data

import time

@st.cache
def load_model(input_data):

	pickle_in = open(r'HF_py_1011.pkl', 'rb')
	xgb = pickle.load(pickle_in)
	pickle_in.close()

	df_1 = input_data
	pred_value = xgb.predict(df_1)
	pred = pred_value.tolist()
	return pred

def main():
	TSA_data = pd.read_csv(r"TSA_TS7.csv")

	list_24 = ['Release_Dose_Calc_mg_m' ,'HEADBOX_PRES_psig' ,'Steam_Box_Pressure_PSIG' ,'PM15_Winder_Speed_FPM' ,'Refiner_1_HPD_T' ,'UHLEBOX1_VAC_Inhg'
		,'Hi_Bright_PM15_Ratio' ,'EosYMoiPv_pct' ,'Coating_Dose_Calc_mg_m' ,'Refiner_3_HP_Control' ,'Metso_Rush_Drag_to_DV' ,'EUC_PM15_Ratio'
		,'VirginStock_PM15_Ratio' ,'RPW_Pressure_PSI' ,'PM_BWT_lbs_3000sqft' ,'PM15_Fan_Pump_ORP_pH' ,'Refiner_2_HPD_T' ,'PM_15_TWS_Flow__ton']

	TSA_18_min = TSA_data[list_24]
	TSA_18_min = TSA_18_min.min()
	MIN_df = TSA_18_min.to_frame(name = 'MIN')

	TSA_18_max = TSA_data[list_24]
	TSA_18_max = TSA_18_max.max()
	MAX_df = TSA_18_max.to_frame(name = 'MAX')

	input_data = get_input_data(MIN_df, MAX_df)
	pred_value = load_model(input_data)
	with first_kpi:

    		number1 = round(pred_value[0], 2)
    		st.markdown(f"
{number1}
", unsafe_allow_html=True)

	st.markdown("
", unsafe_allow_html = True)

if __name__=='__main__':
    main()


import requests
import json

def send_request(video_path, module_url, extract_fps=1):
    json_video = open(video_path, 'rb')
    json_files = {'video': json_video}

    json_data = dict({
        'analysis_type': 'video',
        'video_text': '',
        'extract_fps': extract_fps
    })
    result_response = requests.post(url=module_url, data=json_data, files=json_files)
    result = json.loads(result_response.content)

    json_video.close()

    return result

def save_result2json(result, result_path):
    try :
        with open(result_path, 'w') as result_file:
            json.dump(result, result_file, indent='\t')
        return True
    except:
        return False


import logging

from rest_framework import status
from django.shortcuts import get_object_or_404
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.views import APIView

from app.decorators.decorators import allowed_roles
from app.models.template_scenario import TemplateScenario
from app.serializers.user_scenario import UserScenarioSerializer
from django.core.exceptions import ObjectDoesNotExist

from app.models.user_scenario import UserScenario
from app.models.scenario import ScenarioConfig
from app.models.team import Team
from custom_user.models import User

class UserScenarioViews(APIView):
    permission_classes = (IsAuthenticated,)

    @allowed_roles(["creator", "staff"])
    def post(self, request):
        errors = {}
        user = request.data.get("user")
        config = request.data.get("config")
        team = request.data.get("team")
        serializer = UserScenarioSerializer(data=request.data)
        template = request.data.get("template")
        try:
            template = TemplateScenario.objects.get(id=template)
        except ObjectDoesNotExist:
            errors[
                "template"
            ] = f"No template with id {template} does exist in Database. Template must be provided!"
        if user:
            try:
                user = User.objects.get(id=user)
            except ObjectDoesNotExist:
                errors["user"] = f"No user with id {user} does exist in Database."
        if config:
            try:
                config = ScenarioConfig.objects.get(id=config)
            except ObjectDoesNotExist:
                errors["config"] = f"No config with id {config} does exist in Database."
        if team:
            try:
                team = Team.objects.get(id=team)
            except ObjectDoesNotExist:
                errors["team"] = f"No team with id {team} does exist in Database."
        if errors:
            logging.error("Could not create user scenario. Attributes missing")
            logging.debug(errors)
            return Response(
                {"status": "error", "data": errors}, status=status.HTTP_400_BAD_REQUEST,
            )
        if serializer.is_valid():
            obj = serializer.save()
            obj.template = template
            if user:
                obj.user = user
            if config:
                obj.config = config
            if team:
                obj.team = team
            obj.save()
            return Response(
                {"status": "success", "data": serializer.data},
                status=status.HTTP_200_OK,
            )
        else:
            print("Invalid")
            return Response(
                {"status": "error", "data": serializer.errors},
                status=status.HTTP_400_BAD_REQUEST,
            )

    @allowed_roles(["student", "creator", "staff"])
    def get(self, request, id=None):
        if id:
            item = UserScenario.objects.get(id=id)
            serializer = UserScenarioSerializer(item)
            return Response(
                {"status": "success", "data": serializer.data},
                status=status.HTTP_200_OK,
            )

        items = UserScenario.objects.all()
        serializer = UserScenarioSerializer(items, many=True)
        return Response(
            {"status": "success", "data": serializer.data}, status=status.HTTP_200_OK
        )

    @allowed_roles(["creator", "staff"])
    def patch(self, request, id=None):
        item = UserScenario.objects.get(id=id)
        user = request.data.get("user")
        config = request.data.get("config")
        team = request.data.get("team")
        errors = dict()
        if user:
            try:
                item.user = User.objects.get(id=user)
            except ObjectDoesNotExist:
                errors["user"] = f"No user with id {user} does exist in Database."
        if config:
            try:
                item.config = ScenarioConfig.objects.get(id=config)
            except ObjectDoesNotExist:
                errors["config"] = f"No config with id {config} does exist in Database."
        if team:
            try:
                item.team = Team.objects.get(id=team)
            except ObjectDoesNotExist:
                errors["team"] = f"No team with id {team} does exist in Database."
        if errors:
            return Response(
                {"status": "error", "data": errors}, status=status.HTTP_400_BAD_REQUEST,
            )
        serializer = UserScenarioSerializer(item, data=request.data, partial=True)
        if serializer.is_valid():
            serializer.save()
            return Response({"status": "success", "data": serializer.data})
        else:
            return Response({"status": "error", "data": serializer.errors})

    @allowed_roles(["creator", "staff"])
    def delete(self, request, id=None):
        item = get_object_or_404(UserScenario, id=id)
        item.delete()
        return Response({"status": "success", "data": "Item Deleted"})


# -*- coding: utf-8 -*-
"""
-------------------------------------------------
    File Name:      CreditHebei.py
    Description:    双公示-信用河北-基类
    Author:         Jack Deng
    Date:           2018-01-02
    Version:        v.1.0

    Lastmodified:       2018-01-02  by  Jack Deng
    Lastmodified:       2018-01-08  by  Jack Deng
-------------------------------------------------
"""

import traceback
import uuid

from scrapy.http.request import Request

from hive_framework_milk.commons.utils.selector_util import clean_all_space
from hive_framework_milk.commons.utils.tools import gen_rowkey
from hive_framework_milk.scrapy_spiders.item.parsed_item import ParsedItem
from hive_framework_milk.scrapy_spiders.spiders.spider_seed import Spider_Seed
from .field_mapping import map_field

class CreditHebei(Spider_Seed):
    """
    双公示-信用河北-基类
    """

    name = "CreditHebei"
    seed_key = None
    search_url = None

    custom_settings = {
        'ITEM_PIPELINES': {
             'hive_framework_milk.scrapy_spiders.pipeline.result_pipeline.ResultPipeline': 300
        },
    }

    def custom_init(self, *args, **kwargs):
        """
        检查属性，添加种子队列
        :param args:
        :param kwargs:
        :return:
        """
        super().custom_init(*args, **kwargs)
        if not self.seed_key or not self.search_url:
            raise NotImplementedError("seed_key {} or search_url {} not implemented".format(
                self.seed_key, self.search_url
            ))
        self.seed_queue_names = [self.seed_key]

    def make_request_from_seed(self, data):
        """
        根据种子，产生请求
        :param data:
        :return:
        """
        if isinstance(data, bytes):
            data = data.decode()
        search_url = self.search_url.format(data)
        self.logger1.info("Start search by {}".format(data))
        return Request(
            url=search_url,
            dont_filter=True,
            callback=self.parse,
            errback=self.common_errback,
            meta={'key': data}
        )

    def parse(self, response):
        """
        解析搜索结果页面，yield详情页请求
        :param response:
        :return:
        """
        try:
            key = response.meta["key"]
            # 取一页所有的url
            hrefs = [href.strip() for href in response.xpath("//table//tr//td//a/@href").extract()]
            if not hrefs:
                self.logger1.warning("key {} search failed, please check".format(key))
                return
            for href in hrefs:
                detail_url = response.urljoin(href)
                yield Request(
                    url=detail_url,
                    dont_filter=True,
                    callback=self.parse_detail,
                    errback=self.common_errback,
                    meta={"key": key}
                )
                self.logger1.info("yield detail url , key {}".format(key))
            next_page_href = "".join(response.xpath('//a[@class="next-page"]/@href').extract())
            if next_page_href:# 有下一页就翻页
                next_url = response.urljoin(next_page_href)
                yield Request(
                    url=next_url,
                    dont_filter=True,
                    callback=self.parse,
                    errback=self.common_errback,
                    meta={"key": key}
                )
                self.logger1.warning("yield turn page, key {}".format(key))
            else:
                self.logger1.info("key {} only has one page".format(key))
        except:
            err_msg = traceback.format_exc()
            self.logger1.error("Exception on search {url}, error:{err_msg}".format(
                url=response.url, err_msg=err_msg))

    def parse_detail(self, response):
        """
        解析详情页
        :param response:
        :return:
        """
        try:
            key = response.meta["key"]
            titles_tds = response.xpath("//table//tr//td[1]")
            values_tds = response.xpath("//table//tr//td[last()]")
            titles = [clean_all_space(td.xpath("string(.)").extract()) for td in titles_tds]
            values = ["".join(td.xpath("string(.)").extract()).strip() for td in values_tds]
            if len(titles) != len(values):
                raise Exception("the length of titles and values are not equal, url {}".format(response.url))
            tmp_dict = dict(zip(titles, values))
            res_dict = map_field(tmp_dict)
            if "xzcf" in self.name:
                if "license_status" in res_dict.keys():
                    res_dict["punish_status"] = res_dict.pop("license_status", "")
            item = ParsedItem()
            self.common_item_assembler(response, item)
            item["_id"] = "{}_{}".format(key, uuid.uuid4())
            item["bbd_html"] = ""
            item["_parsed_data"] = res_dict
            item["rowkey"] = gen_rowkey(item, keys=('do_time', 'bbd_type'))
            yield item
            self.logger1.info("one data {} save to mongodb".format(key))
        except:
            err_msg = traceback.format_exc()
            self.logger1.error("Exception on detail {url}, error:{err_msg}".format(
                url=response.url, err_msg=err_msg))

    def common_errback(self, response):
        """
        请求失败时输出日志
        :param response:
        :return:
        """
        self.logger1.error('request {} failed'.format(response.request.url))


import json

class InvertedIndex():
    """
    Class representing an inverted word index.
    """

    def __init__(self):
        """
        Initialses class with empty word index.
        """
        self.word_index = {}

    def create_index(self, url, words):
        """
        Creates an inverted index from the pages than contain
        requested words.

        Args:
            url: url of the crawled page.
            words: words that exist in the page.

        Returns:
            None
        """

        for word in words:
            if word not in self.word_index:
                value = {
                    url: 1
                }
                self.word_index[word] = value
            else:
                if url in self.word_index[word]:
                    self.word_index[word][url] += 1
                else:
                    self.word_index[word][url] = 1

        # w mode - start from empty file each time
        with open("index.json", "w") as output:
            json.dump(obj=self.word_index, fp=output, indent=4)


from multiprocessing import Process
import os

#子进程要执行的代码
def run_proc(name):
	print('Run child Process %s (%s)...' %(name,os.getpid()))
#getpid()方法返回当前进程的id

if __name__=='__main__':
	print('Parent process %s.'%os.getpid())
	#创建一个Process实例，用start方法启动
	p = Process(target = run_proc, args = ('test',))
	#target：要执行函数； args：函数的参数
	print('Child process will start.')
	p.start()
	p.join()#该方法可以等待子进程结束后再继续往下运行，通常用于进程间的同步
	print("Child process end.")


import numpy as np
from astropy.modeling.functional_models import Sersic1D
from .galaxy import coordinates_transfer,indentify_xy
import cv2

def alpha_tanh(r_3d, r_in, r_out, alpha, theta_out):
    theta_out_rad = theta_out*np.pi/180.
    return theta_out_rad*hypotan(r_3d, r_in, r_out, alpha, theta_out_rad)*np.power(0.5*(r_3d/r_out+1.),alpha)

def hypotan(r_3d, r_in, r_out, alpha, theta_out_rad):
    CDEF = 0.23
    A = 2.*CDEF/(np.abs(theta_out_rad)+CDEF)-1.00001
    B = (2.-np.arctanh(A))*r_out/(r_out-r_in)
    return 0.5*np.tanh(B*(r_3d/r_out-1.)+2.)+0.5

def R_2d(x,y):
    return (np.sqrt(x**2+y**2))

def R_3d(x,y,PA,i):
    phi = PA*np.pi/180.
    dis_proj_square = np.abs(np.cos(phi)*y-np.sin(phi)*x)**2
    return np.sqrt(R_2d(x,y)**2+(1./np.cos(i*np.pi/180.)-1)*dis_proj_square)

def xy_3d(x,y,PA,i):
    phi = PA*np.pi/180.
    dis_proj = np.cos(phi)*y-np.sin(phi)*x
    projvec = (1./np.cos(i*np.pi/180.)-1.)*dis_proj
    return x-projvec*np.sin(phi),y+projvec*np.cos(phi)

def R_fourier(x,y,m,am,theta_m,PA,i):
    lower = y < 0.
    phi_map = np.arccos(x/R_2d(x,y))
    phi_map[lower] = 2.*np.pi-phi_map[lower]
    return R_3d(x,y,PA,i)*(1.+am*np.cos(m*(phi_map+theta_m*np.pi/180.)))

def make_fourier(M, seRe, sen, shape, posi, PA, i, r_in, r_out, alpha, theta_out, m, am, theta_m, i_arm, transpar=None):
    '''
    seRe,sen: control the mass distribution of the arms
    i, PA galaxy projection
    r_in, r_out, alpha, theta_out: rotation curve
    m, am, theta_m, i_arm: arm number, relative strength
    '''
    ny = shape[0]
    nx = shape[1]
    xaxis = np.arange(nx)
    yaxis = np.arange(ny)
    xcen = posi[0]
    ycen = posi[1]
    if transpar is not None:
        xcen,ycen = coordinates_transfer(posi[0],posi[1],transpar)
        seRe = seRe /transpar['pixsc']
        r_in = r_in /transpar['pixsc']
        r_out = r_out /transpar['pixsc']
        testPA = PA+transpar['delta_ang']
        if testPA > 90.:
            testPA -= 180.
        elif testPA < -90.:
            testPA += 180.
        PA = testPA
    else:
        xcen,ycen = indentify_xy(posi[0],posi[1])
    xmesh, ymesh = np.meshgrid(xaxis, yaxis)
    x_p = xmesh + 0.5 - xcen
    y_p = ymesh + 0.5 - ycen
    r_2d = R_2d(x_p,y_p)
    r_3d = R_3d(x_p,y_p,PA,i)
    theta_r = alpha_tanh(r_3d, r_in, r_out, alpha, theta_out)
    x_p,y_p = xy_3d(x_p,y_p,PA,i)
    x_i = x_p*np.cos(theta_r) + y_p*np.sin(theta_r)
    y_i = -x_p*np.sin(theta_r) + y_p*np.cos(theta_r)
    #i_arm = 0.
    r_3d_i = R_fourier(x_i,y_i,m,am,theta_m,PA,i_arm)
    s1 = Sersic1D(amplitude=1, r_eff=seRe, n=sen)
    mod = s1(r_3d_i)
    mass_map = M*mod/np.sum(mod)
    return mass_map

def rotate_CV(image, angel , interpolation):
    '''
        input :
        image           :  image                    : ndarray
        angel           :  rotation angel           : int
        interpolation   :  interpolation mode       : cv2 Interpolation object
        returns :
        rotated image   : ndarray

        '''
    #in OpenCV we need to form the tranformation matrix and apply affine calculations
    #
    h,w = image.shape[:2]
    cX,cY = (w//2,h//2)
    M = cv2.getRotationMatrix2D((cX,cY),angel,1)
    rotated = cv2.warpAffine(image,M , (w,h),flags=interpolation)
    return rotated

def reshape_img(posi, image, shapeF, coordinates_transfer_par0, coordinates_transfer_parF):
    '''
        posi: position of center in [dra, ddec]
    '''
    dtheta = coordinates_transfer_parF['delta_ang']-coordinates_transfer_par0['delta_ang']
    xpix,ypix = coordinates_transfer(posi[0],posi[1],coordinates_transfer_parF)
    xpix0,ypix0 = coordinates_transfer(posi[0],posi[1],coordinates_transfer_par0)
    ny,nx = image.shape
    nyF = shapeF[0]
    nxF = shapeF[1]
    xs = xpix*nx/nxF
    ys = ypix*ny/nyF
    T = np.float32([[1, 0, xs-xpix0], [0, 1, ys-ypix0]])
    img_translation = cv2.warpAffine(image, T, (nx, ny),flags = cv2.INTER_CUBIC)
    M = cv2.getRotationMatrix2D((xs,ys),dtheta,1)
    img_rot = cv2.warpAffine(img_translation, M, (nx, ny),flags = cv2.INTER_CUBIC)
    resized = cv2.resize(img_rot, (nxF, nyF), interpolation = cv2.INTER_CUBIC)
    return resized

def non_para(par, c, plist, image):
    npimg = np.zeros_like(image, dtype=float)
    for loop,p in enumerate(plist):
        npimg[p[1],p[0]] = par['pf_{0}_{1}'.format(c,loop)]
    return npimg


# -*- encoding: utf-8 -*-
# @File    : Sharpen.py
# @Time    : 2019/12/3 20:25
# @Author  : 一叶星羽
# @Email   : h0670131005@gmail.com
# @Software: PyCharm

import numpy as np
from scipy import interpolate
from CommonHelpers.Utils import path_to_list, list_to_path

def curve_sharpening(points, kind="linear"):
    x, y = [], []
    for ln in points:
        x.append(ln[0])
        y.append(ln[1])
    x = np.array(x)
    y = np.array(y)

    x_new = np.linspace(x.max(), x.min(), 200)
    func = interpolate.interp1d(x, y, kind=kind)
    y_new = func(x_new)
    new_points = [(x1, y1) for x1 in x_new for y1 in y_new]
    return new_points

def curve_sharpen_path(path, kind):
    print("element: ", path.elementCount())
    points = path_to_list(path)
    print(len(points))
    print(points)
    result = curve_sharpening(points, kind)
    print("------------------------- 锐化结束 ----------------------")
    return list_to_path(result)

if __name__ == '__main__':
    curve_sharpening([(1, 2), (2, 3), (3, 5)])


import argparse
import logging
import os
from collections import OrderedDict
from cityhash import CityHash32 # pylint: disable=no-name-in-module

import tensorflow_io # pylint: disable=unused-import
from tensorflow.compat.v1 import gfile

from fedlearner.common import data_join_service_pb2 as dj_pb
from fedlearner.common.common import set_logger

from fedlearner.data_join import common
from fedlearner.data_join.sort_run_merger import SortRunMergerWriter
from fedlearner.data_join.raw_data_iter_impl.csv_dict_iter import CsvItem

def generate_input_csv(base_dir, start_id, end_id, partition_num):
    for partition_id in range(partition_num):
        dirpath = os.path.join(base_dir, common.partition_repr(partition_id))
        if not gfile.Exists(dirpath):
            gfile.MakeDirs(dirpath)
        assert gfile.IsDirectory(dirpath)
    writer_options = dj_pb.WriterOptions(output_writer='CSV_DICT')
    csv_writers = [SortRunMergerWriter(base_dir, 0,
                                       partition_id, writer_options)
                   for partition_id in range(partition_num)]
    for idx in range(start_id, end_id):
        if idx % 262144 == 0:
            logging.info("Process at index %d", idx)
        partition_id = CityHash32(str(idx)) % partition_num
        raw = OrderedDict()
        raw['raw_id'] = str(idx)
        raw['feat_0'] = str((partition_id << 30) + 0) + str(idx)
        raw['feat_1'] = str((partition_id << 30) + 1) + str(idx)
        raw['feat_2'] = str((partition_id << 30) + 2) + str(idx)
        csv_writers[partition_id].append(CsvItem(raw))
    for partition_id, csv_writer in enumerate(csv_writers):
        fpaths = csv_writer.finish()
        logging.info("partition %d dump %d files", partition_id, len(fpaths))
        for seq_id, fpath in enumerate(fpaths):
            logging.info("  %d. %s", seq_id, fpath)
        logging.info("---------------")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='csv raw data generator cmd.')
    parser.add_argument('--base_dir', type=str, required=True,
                        help='the base dir of fs to store the ' \
                             'generated csv file')
    parser.add_argument('--partition_num', type=int, required=True,
                        help='the partition num of generated data')
    parser.add_argument('--start_id', type=int, required=True,
                        help='the start id of generated csv file')
    parser.add_argument('--end_id', type=int, required=True,
                        help='the end id of generated csv file')
    args = parser.parse_args()
    set_logger()
    generate_input_csv(args.base_dir, args.start_id,
                       args.end_id, args.partition_num)


# Licensed under GPL version 3 - see LICENSE.rst
'''
.. todo::
   Add a mechanism for unit conversion. This expects all distances in mm and all energies in keV.
'''
import os
import numpy as np
from astropy.table import Table, Column, join
import astropy.units as u

from ..math.utils import h2e, e2h
from ..math.polarization import parallel_transport
from .base import OpticalElement, photonlocalcoords
from .aperture import BaseAperture

try:
    from _marx import lib as marx
    from _marx import ffi
    HAS_MARX = True
except ImportError:
    HAS_MARX = False

class MarxError(Exception):
    '''Error in the compiled Marx C module'''
    pass

class MarxMirror(OpticalElement, BaseAperture):
    '''Interface to MARX mirror module

    This class provides an interface to the
    `MARX <http://space.mit.edu/ASC/marx/>`_  mirror module. It requires
    both the MARX source code and compiled MARX binaries as explained in the
    :ref:`Installation instructions for MARXS <sect-installmarxccode>`
    When this model
    is initialized, it requires the path and filename to a MARX setup file.

    The default geometry is such that the focal point is at the origin of the
    coordinate system and the optical axis is along the x-axis, such that
    photons travel from +infinity towards the origin.

    A MarxMirror object does not only act as a mirror, it fulfills the function
    of an aperture at the same time, no further aperture class should be used
    in the same simulation.

    The model reads the relevant MARX mirror model parameters from that file;
    in this way it supports all mirror models that the traditional MARX version
    implements, most notable the "HRMA" and "IXO" mirrors. Details on the
    "HRMA" mirror (which is used to simulate Chandra) can be found at
    http://space.mit.edu/ASC/marx/indetail/hardwaremodel.html#hrma-model . The
    "IXO" mirror model is a simplified version with some generalizations that
    works very similar in general, but is not documented because (from a MARX
    standpoint) it is only meant for developers.

    Parameters
    ----------
    parfile : string
        Path and filename of a MARX parameter file that sets all MARX
        parameters for the mirror model.
    '''

    def __init__(self, parfile, **kwargs):
        # If the state shared between different object that use the s
        # same C module? In that case I need to add a lock so that only
        # one object of this class can exist at any one time.
        if not HAS_MARX:
            raise MarxError('MARX C code is not available. Please see installation instructions.')
        if not os.path.isfile(parfile):
            raise IOError('MARX parameter file {0} does NOT exist.'.format(parfile))
        else:
            self.parfile = parfile
        self.cparfile = marx.pf_open_parameter_file(bytes(parfile, 'utf8'), bytes('r', 'utf8'))
        out = marx.marx_mirror_init(self.cparfile)
        if out < 0:
            raise MarxError('Mirror cannot be initialized. Probably missing parameters or syntax error in {0}.'.format(parfile))

        super().__init__(**kwargs)

    @staticmethod
    def _table2c(photons):
        # Arrays assigned here in python need to keep a reference
        # somewhere, otherwise they would be garbage collected
        # and the pointer would suddenly be invalid.
        # To do so, this function adds them to the photons table header.
        # If this turns out to be too slow, then I have to move it to
        # Cython that can access the native numpy array memory.
        n = len(photons)
        cp = ffi.new('Marx_Photon_Attr_Type[]', n)
        keep_cffi_pointers = {}
        keep_cffi_pointers['cp'] = cp
        for i in range(n):
            cp[i].energy = photons['energy'][i]
            pos = h2e(photons['pos'][i])
            cp[i].x.x = pos[0]
            cp[i].x.y = pos[1]
            cp[i].x.z = pos[2]
            dir = h2e(photons['dir'][i])
            cp[i].p.x = dir[0]
            cp[i].p.y = dir[1]
            cp[i].p.z = dir[2]
            cp[i].arrival_time = photons['time'][i]
        if 'tag' not in photons.colnames:
            photons.add_column(Column(name='tag', data=np.arange(n)))
        for i, t in zip(range(n), photons['tag']):
            cp[i].tag = t

        c_photon_list = ffi.new('Marx_Photon_Type *')
        c_photon_list.attributes = cp
        c_photon_list.n_photons = n
        c_photon_list.max_n_photons = n
        c_photon_list.total_time = np.max(photons['time'])
        c_photon_list.start_time = np.min(photons['time'])

        sorted_index = np.argsort(photons['energy'].data)
        sorted_index = np.ascontiguousarray(sorted_index, dtype=np.uintc)
        keep_cffi_pointers['sorted_index'] = sorted_index  # keep alive
        c_photon_list.sorted_index = ffi.cast('unsigned int*', sorted_index.ctypes.data)

        sorted_energies = np.sort(photons['energy'].data)
        sorted_energies = np.ascontiguousarray(sorted_energies, dtype=float)
        keep_cffi_pointers['sorted_energies'] = sorted_energies  # keep alive
        c_photon_list.sorted_energies = ffi.cast('double*', sorted_energies.ctypes.data)

        c_photon_list.num_sorted = n  # all photons in list are valid
        # Fields not mentioned here are irrelevant for the mirror
        # module and can have any value they are initialized to.

        return c_photon_list, keep_cffi_pointers

    @staticmethod
    def _c2table(c_photon_list):
        '''

        We'll first collect all the data in numpy arrays and make
        a new table. Might not be the most efficient solution,
        but should work for now.
        Make work first, improve later.

        To-Do: keep absorbed?
        '''
        n_valid = c_photon_list.num_sorted
        cp = c_photon_list.attributes

        energy = np.empty(n_valid)
        time = np.empty(n_valid)
        tag = np.empty(n_valid, dtype=int)
        pos = np.empty((n_valid, 3))
        dir = np.empty((n_valid, 3))
        unreflected = np.empty(n_valid, dtype=bool)
        vblocked = np.empty(n_valid, dtype=bool)
        shell = np.empty(n_valid, dtype=int)

        for i in range(n_valid):
            energy[i] = cp[i].energy
            pos[i, 0] = cp[i].x.x
            pos[i, 1] = cp[i].x.y
            pos[i, 2] = cp[i].x.z
            dir[i, 0] = cp[i].p.x
            dir[i, 1] = cp[i].p.y
            dir[i, 2] = cp[i].p.z
            time[i] = cp[i].arrival_time
            tag[i] = cp[i].tag
            unreflected[i] = cp[i].flags & marx.PHOTON_UNREFLECTED
            vblocked[i] = cp[i].flags & marx.PHOTON_MIRROR_VBLOCKED
            shell[i] = cp[i].mirror_shell

        pos = e2h(pos, 1)
        dir = e2h(dir, 0)
        photons = Table([pos, dir, energy, time, tag,
                         unreflected, vblocked, shell],
                        names=['pos', 'dir', 'energy', 'time', 'tag',
                               'unreflected', 'mirror_vblocked', 'mirror_shell'])
        return photons

    @photonlocalcoords
    def _process_photons_in_c(self, photons, verbose):
        '''Wrap the MARX C module'''
        c_photon_list, keep_cffi_pointers = self._table2c(photons)
        out = marx.marx_mirror_reflect(c_photon_list, verbose)
        if out != 0:
            raise MarxError('Error in marx_mirror_reflect.')
        return self._c2table(c_photon_list)

    def __call__(self, photons_in, verbose=0):
        self.add_colpos(photons_in)
        new_photons = self._process_photons_in_c(photons_in, verbose)
        photons = join(new_photons, photons_in, keys='tag',
                       uniq_col_name='{col_name}{table_name}',
                       table_names=['', '_beforemirror'])
        # Probability column needs special treatment, because the probability
        # is not just replaced with the new number, it's multiplicative.
        # First, match tag in to tag after join (which may have fewer photons)
        insorted = np.argsort(photons_in['tag'])
        ypos = np.searchsorted(photons_in['tag'][insorted], photons['tag'])
        indices = insorted[ypos]
        photons['probability'] *= photons_in['probability'][indices]
        photons['probability'][photons['unreflected'] | photons['mirror_vblocked']] = 0
        pol = parallel_transport(photons_in['dir'][insorted], photons['dir'],
                                 photons_in['polarization'][insorted])
        photons['polarization'] = pol
        return photons

    @property
    def area(self):
        '''Area of the aperture.

        This does not take into account any projection effects for
        apertures that are not perpendicular to the optical axis.
        '''
        return marx.Marx_Mirror_Geometric_Area * u.cm**2

# Turn into a test?
# pf = marx.pf_open_parameter_file('./marxs/optics/hrma.par', 'r')
# out = marx.marx_mirror_init(pf)
# out should be 0, but I get 2 ?!?
# Will continue and leave that mystery as a mystery for now, but I bet I'll
# need to get back to it.


"""
Tests for HypernetsWriter class
"""

import unittest
from unittest.mock import patch, MagicMock
from hypernets_processor.data_io.hypernets_writer import HypernetsWriter
from hypernets_processor.context import Context
from hypernets_processor.version import __version__

from xarray import Dataset
import numpy as np
import os
from datetime import datetime as dt
import obsarray

"""___Authorship___"""
__author__ = "Sam Hunt"
__created__ = "21/2/2020"
__version__ = __version__
__maintainer__ = "Sam Hunt"
__email__ = "sam.hunt@npl.co.uk"
__status__ = "Development"

# todo - write test for csv writer
# todo - write test for end to end writer use

class TestHypernetsWriter(unittest.TestCase):
    def test_return_fmt_netcdf(self):
        hw = HypernetsWriter()
        self.assertEqual("nc", hw.return_fmt(fmt="netcdf"))

    def test_return_fmt_netcdf4(self):
        hw = HypernetsWriter()
        self.assertEqual("nc", hw.return_fmt(fmt="netcdf4"))

    def test_return_fmt_csv(self):
        hw = HypernetsWriter()
        self.assertEqual("csv", hw.return_fmt(fmt="csv"))

    def test_return_fmt_invalid(self):
        hw = HypernetsWriter()
        self.assertRaises(NameError, hw.return_fmt, "invalid")

    def test_return_fmt_context(self):
        context = Context()
        context.set_config_value("product_format", "netcdf")
        hw = HypernetsWriter(context)
        self.assertEqual("nc", hw.return_fmt())

    def test_return_fmt_noinput(self):
        hw = HypernetsWriter()
        self.assertRaises(ValueError, hw.return_fmt)

    def test_return_directory(self):
        hw = HypernetsWriter()
        self.assertEqual("directory", hw.return_directory(directory="directory"))

    def test_return_directory_context_toarchive(self):
        context = Context()
        context.set_config_value("archive_directory", "directory")
        context.set_config_value("site", "site")
        context.set_config_value("time", dt(2020, 4, 5, 11, 23, 4, 5))
        context.set_config_value("to_archive", True)

        hw = HypernetsWriter(context)
        self.assertEqual(
            os.path.join("directory", "site", "2020", "4", "5"), hw.return_directory()
        )

    def test_return_directory_context_nottoarchive(self):
        context = Context()
        context.set_config_value("archive_directory", "directory")
        context.set_config_value("site", "site")
        context.set_config_value("time", dt(2020, 4, 5, 11, 23, 4, 5))
        context.set_config_value("to_archive", False)

        hw = HypernetsWriter(context)
        self.assertEqual(os.path.join("directory"), hw.return_directory())

    def test_return_directory_noinput(self):
        hw = HypernetsWriter()
        self.assertRaises(ValueError, hw.return_directory)

    @patch(
        "hypernets_processor.data_io.tests.test_hypernets_writer.HypernetsWriter.return_fmt",
        return_value="nc",
    )
    @patch(
        "hypernets_processor.data_io.tests.test_hypernets_writer.HypernetsWriter.return_directory",
        return_value="directory",
    )
    @patch(
        "hypernets_processor.data_io.tests.test_hypernets_writer.HypernetsWriter._write_netcdf"
    )
    def test_write_netcdf(self, mock_write, mock_dir, mock_fmt):
        ds = Dataset()
        ds.attrs["product_name"] = "test"

        hw = HypernetsWriter()

        hw.write(ds)
        mock_write.assert_called_once_with(
            ds, os.path.join("directory", "test.nc"), compression_level=None
        )

    @patch(
        "hypernets_processor.data_io.tests.test_hypernets_writer.HypernetsWriter.return_fmt",
        return_value="csv",
    )
    @patch(
        "hypernets_processor.data_io.tests.test_hypernets_writer.HypernetsWriter.return_directory",
        return_value="directory",
    )
    @patch(
        "hypernets_processor.data_io.tests.test_hypernets_writer.HypernetsWriter._write_csv"
    )
    def test_write_csv(self, mock_write, mock_dir, mock_fmt):
        ds = Dataset()
        ds.attrs["product_name"] = "test"

        hw = HypernetsWriter()

        hw.write(ds)
        mock_write.assert_called_once_with(ds, os.path.join("directory", "test.csv"))

    def test__write_netcdf(self):

        ds = MagicMock()
        path = "test.nc"

        HypernetsWriter._write_netcdf(ds, path)

        ds.to_netcdf.assert_called_once_with(
            path, encoding={}, engine="netcdf4", format="netCDF4"
        )

    def test_fill_ds(self):
        ds = Dataset()
        ds["array_variable1"] = obsarray.create_var([7, 8], np.float32)
        ds["array_variable2"] = obsarray.create_var([7, 8], np.float32)

        ds["array_variable1"][2, 3] = np.nan
        ds["array_variable2"][2, 3] = np.nan

        HypernetsWriter.fill_ds(ds)

        self.assertTrue(np.all(ds["array_variable1"] == 9.96921e36))
        self.assertTrue(np.all(ds["array_variable2"] == 9.96921e36))

if __name__ == "__main__":
    unittest.main()