import argparse
import datetime
import logging
import os
import re
import sys
from subprocess import call

import h5py
import numpy as np
import pandas as pd
import tifffile as tf

from tools import stats
from imagej.imagej_pandas import ImagejPandas

class LabHDF5NeXusFile():
    def __init__(self, filename='fabio_data_hochegger_lab.nexus.hdf5', imagesfile=None, fileflag='r'):
        self.filename = filename
        self.imagefile = imagesfile

        # open the HDF5 NeXus file
        if fileflag == 'w':
            with h5py.File(self.filename, fileflag) as f:
                # point to the default data to be plotted
                f.attrs['default'] = 'entry'
                # give the HDF5 root some more attributes
                f.attrs['file_name'] = self.filename
                timestamp = datetime.datetime.now().isoformat()
                f.attrs['file_time'] = timestamp
                f.attrs['creator'] = 'fabio@hochegger.lab'
                f.attrs['NeXus_version'] = '4.3.0'
                f.attrs['HDF5_Version'] = h5py.version.hdf5_version
                f.attrs['h5py_version'] = h5py.version.version

    def add_experiment(self, group, experiment_tag, timestamp=None):
        gr = '%s/%s' % (group, experiment_tag)
        timestamp = timestamp if timestamp is not None else datetime.datetime.now().isoformat()

        with h5py.File(self.filename, 'a') as f:
            if gr in f: del f[gr]

            # create the NXentry experiment group
            nxentry = f.create_group(gr)
            nxentry.attrs['NX_class'] = 'NXentry'
            nxentry.attrs['datetime'] = timestamp
            nxentry.attrs['default'] = 'processed'

            nxentry_raw = nxentry.create_group('raw')
            nxentry_raw.attrs['NX_class'] = 'NXgroup'
            nxentry_meas = nxentry.create_group('measurements')
            nxentry_meas.attrs['NX_class'] = 'NXgroup'
            nxentry_sel = nxentry.create_group('selection')
            nxentry_sel.attrs['NX_class'] = 'NXgroup'
            nxentry_proc = nxentry.create_group('processed')
            nxentry_proc.attrs['NX_class'] = 'NXgroup'

        with h5py.File(self.imagefile, 'a') as i:
            rawgr = '%s/%s/raw' % (group, experiment_tag)
            # create the NXentry experiment group
            nxentry = i.require_group(rawgr)
            nxentry.attrs['NX_class'] = 'NXentry'
            nxentry.attrs['datetime'] = timestamp
            i[rawgr].attrs['NX_class'] = 'NXgroup'

    def add_tiff_sequence(self, tiffpath, experiment_tag, run):
        _grp = '%s/%s/raw' % (experiment_tag, run)
        # open the HDF5 NeXus file
        if self.imagefile is None:
            f = h5py.File(self.filename, 'a')
        else:
            f = h5py.File(self.imagefile, 'a')

        # update the NXentry group
        nxdata = f[_grp]
        nxdata.attrs['NX_class'] = 'NXdata'
        nxdata.attrs['signal'] = 'Y'  # Y axis of default plot
        nxdata.attrs['axes'] = 'X'  # X axis of default plot
        nxdata.attrs['units'] = 'um'  # default units

        nframes = len(nxdata.items())
        if nframes == 0:
            with tf.TiffFile(tiffpath, fastij=True) as tif:
                if tif.is_imagej is not None:
                    sizeT, channels = tif.pages[0].imagej_tags.frames, tif.pages[0].imagej_tags.channels
                    sizeZ, sizeX, sizeY = 1, tif.pages[0].image_width, tif.pages[0].image_length
                    logging.info('N of frames=%d channels=%d, sizeZ=%d, sizeX=%d, sizeY=%d' % \
                                 (sizeT, channels, sizeZ, sizeX, sizeY))

                    res = 'n/a'
                    if tif.pages[0].resolution_unit == 'centimeter':
                        # asuming square pixels
                        xr = tif.pages[0].x_resolution
                        res = float(xr[0]) / float(xr[1])  # pixels per cm
                        res = res / 1e4  # pixels per um
                    elif tif.pages[0].imagej_tags.unit == 'micron':
                        # asuming square pixels
                        xr = tif.pages[0].x_resolution
                        res = float(xr[0]) / float(xr[1])  # pixels per um

                    if sizeT > 1:
                        p1a = tif.pages[0].asarray().reshape([sizeT, channels, sizeX, sizeY])
                        for i in range(sizeT):
                            # create a NXentry frame group
                            nxframe = nxdata.create_group('%03d' % i)
                            nxframe.attrs['units'] = 'um'
                            nxframe.attrs['resolution'] = res
                            nxframe.attrs['long_name'] = 'image um (micrometers)'  # suggested X axis plot label

                            # save XY data
                            ch1 = nxframe.create_dataset('channel-1', data=p1a[i][0], dtype=np.uint16)
                            ch2 = nxframe.create_dataset('channel-2', data=p1a[i][1], dtype=np.uint16)
                            ch3 = nxframe.create_dataset('channel-3', data=p1a[i][2], dtype=np.uint16)
                            for ch in [ch1, ch2, ch3]:
                                ch.attrs['CLASS'] = np.string_('IMAGE')
                                ch.attrs['IMAGE_SUBCLASS'] = np.string_('IMAGE_GRAYSCALE')
                                ch.attrs['IMAGE_VERSION'] = np.string_('1.2')

        f.close()

        if self.imagefile is not None and nframes == 0:
            with h5py.File(self.filename, 'a') as f:
                del f[_grp]
                f[_grp] = h5py.ExternalLink(self.imagefile, _grp)

    def add_measurements(self, csvpath, experiment_tag, run):
        dfc = ImagejPandas(csvpath)
        with h5py.File(self.filename, 'a') as f:
            nxmeas = f['%s/%s/measurements' % (experiment_tag, run)]

            dfnt = dfc.df_nuclei.set_index('Frame').sort_index()
            for (nuc_id), filt_nuc_df in dfnt.groupby('Nuclei'):
                nxnucl = nxmeas.create_group('nuclei/N%02d' % nuc_id)
                nxnucl.attrs['NX_class'] = 'NXdata'
                nxnucl.attrs['Nuclei'] = 'N%02d' % nuc_id
                nx = filt_nuc_df['NuclX']
                ny = filt_nuc_df['NuclY']
                fn = filt_nuc_df.reset_index()
                nxnucl.create_dataset('pos', data=fn[['Frame', 'NuclX', 'NuclY']], dtype=nx.dtype)
                nxnucl.create_dataset('frame', data=fn['Frame'], dtype=nx.dtype)
                nxnucl.create_dataset('sample_x', data=nx, dtype=nx.dtype)
                nxnucl.create_dataset('sample_y', data=ny, dtype=ny.dtype)

            dfct = dfc.df_centrosome.set_index('Frame').sort_index()

            for (centr_id), filt_centr_df in dfct.groupby('Centrosome'):
                nxcid = nxmeas.create_group('centrosomes/C%03d' % centr_id)
                nxcid.attrs['NX_class'] = 'NXdata'
                cx = filt_centr_df['CentX']
                cy = filt_centr_df['CentY']
                cn = filt_centr_df.reset_index()
                nxcid.create_dataset('pos', data=cn[['Frame', 'CentX', 'CentY']], dtype=cx.dtype)
                nxcid.create_dataset('frame', data=cn['Frame'], dtype=cx.dtype)
                nxcid.create_dataset('sample_x', data=cx, dtype=cx.dtype)
                nxcid.create_dataset('sample_y', data=cy, dtype=cy.dtype)

            visitedCentrosomes = []
            for _, filt_centr_df in dfct.groupby('Centrosome'):
                nuc_id = filt_centr_df['Nuclei'].unique()[0]
                centrosomesOfNuclei = dfct[dfct['Nuclei'] == nuc_id].groupby('Centrosome')
                if len(centrosomesOfNuclei.groups) >= 2:
                    for i, ((centr_id), filt_centr_df) in enumerate(centrosomesOfNuclei):
                        if centr_id not in visitedCentrosomes:
                            visitedCentrosomes.append(centr_id)
                            self.associate_centrosome_with_nuclei(centr_id, nuc_id, experiment_tag, run, i % 2)
        dfc.merged_df.to_hdf(self.filename, '%s/%s/measurements/pandas_dataframe' % (experiment_tag, run), mode='r+')
        dfc.df_nuclei.to_hdf(self.filename, '%s/%s/measurements/nuclei_dataframe' % (experiment_tag, run), mode='r+')
        self.process_selection_for_run(experiment_tag, run)

    @property
    def dataframe(self):
        df_out = pd.DataFrame()
        with h5py.File(self.filename, 'r') as f:
            for experiment_tag in f:
                for run in f['%s' % experiment_tag]:
                    if 'pandas_dataframe' in f['%s/%s/processed' % (experiment_tag, run)]:
                        selection_key = '%s/%s/processed/pandas_dataframe' % (experiment_tag, run)
                        df = pd.read_hdf(self.filename, key=selection_key, mode='r')
                        df['condition'] = experiment_tag
                        df['run'] = run
                        df_out = df_out.append(df, sort=True)
        df_out = stats.reconstruct_time(df_out)
        df_out.loc[:, ['Frame', 'Centrosome', 'Nuclei']] = df_out[['Frame', 'Centrosome', 'Nuclei']].astype('int32')
        df_out.loc[:, 'Time'] = df_out['Time'].astype('float64')

        return df_out

    @property
    def mask(self):
        df_msk = pd.DataFrame()
        with h5py.File(self.filename, 'r') as f:
            for experiment_tag in f:
                for run in f['%s' % experiment_tag]:
                    if 'pandas_masks' in f['%s/%s/processed' % (experiment_tag, run)]:
                        selection_key = '%s/%s/processed/pandas_masks' % (experiment_tag, run)
                        msk = pd.read_hdf(self.filename, key=selection_key, mode='r')
                        msk.loc[':,condition'] = experiment_tag
                        msk.loc[:, 'run'] = run
                        df_msk = df_msk.append(msk, sort=True)
        return stats.reconstruct_time(df_msk)

    def process_selection_for_run(self, experiment_tag, run):
        with h5py.File(self.filename, 'r') as f:
            nuclei_list = list(f['%s/%s/selection' % (experiment_tag, run)].keys())
            logging.debug(
                'for %s %s there are %d nuclei: %s' % (experiment_tag, run, len(nuclei_list), str(nuclei_list)))
            # don't keep processing if there's nothing to do
            if len(nuclei_list) == 0: return

        merge_key = '%s/%s/measurements/pandas_dataframe' % (experiment_tag, run)
        nuclei_key = '%s/%s/measurements/nuclei_dataframe' % (experiment_tag, run)
        pdhdf_measured = pd.read_hdf(self.filename, key=merge_key, mode='r')
        pdhdf_nuclei = pd.read_hdf(self.filename, key=nuclei_key, mode='r')

        # update centrosome nuclei from selection
        centrosomes_of_nuclei_a = centrosomes_of_nuclei_b = None
        centrosomes_all = list()
        with h5py.File(self.filename, 'r') as f:
            for nuclei_str in nuclei_list:
                if nuclei_str == 'pandas_dataframe' or nuclei_str == 'pandas_masks': continue
                nuclei_id = int(nuclei_str[1:])
                sel_str = '%s/%s/selection/%s' % (experiment_tag, run, nuclei_str)
                centrosomes_of_nuclei_a = [int(c[1:]) for c in f['%s/A' % (sel_str)].keys()]
                centrosomes_of_nuclei_b = [int(c[1:]) for c in f['%s/B' % (sel_str)].keys()]
                for centr_id in centrosomes_of_nuclei_a + centrosomes_of_nuclei_b:
                    pdhdf_measured.loc[pdhdf_measured['Centrosome'] == centr_id, 'Nuclei'] = nuclei_id
                # tag centrosomes with the label given in the GUI
                for centr_id in centrosomes_of_nuclei_a:
                    pdhdf_measured.loc[pdhdf_measured['Centrosome'] == centr_id, 'CentrLabel'] = 'A'
                for centr_id in centrosomes_of_nuclei_b:
                    pdhdf_measured.loc[pdhdf_measured['Centrosome'] == centr_id, 'CentrLabel'] = 'B'

                centrosomes_all.extend(centrosomes_of_nuclei_a + centrosomes_of_nuclei_b)

        # re-merge with nuclei data
        pdhdf_measured.drop(['NuclX', 'NuclY', 'NuclBound'], axis=1, inplace=True)
        pdhdf_measured = pdhdf_measured[pdhdf_measured['Centrosome'].isin(centrosomes_all)]
        df_merge = pdhdf_measured.merge(pdhdf_nuclei, how='left')

        # merge with cell boundary data
        with h5py.File(self.filename, 'r') as f:
            if 'boundary' in f['%s/%s/processed' % (experiment_tag, run)]:
                df_cell = pd.read_hdf(self.filename, key='%s/%s/processed/boundary' % (experiment_tag, run))
                df_cell = df_cell.loc[~df_cell['CellBound'].isnull(),
                                      set(ImagejPandas.MASK_INDEX + ['CellX', 'CellY', 'CellBound',
                                                                     'DistCell', 'SpdCell', 'AccCell'])]

                if 'CellBound' in df_merge.columns and not df_cell.empty:
                    logging.info('clearing CellBound')
                    df_merge.drop(['CellX', 'CellY', 'CellBound'], axis=1, inplace=True)
                df_merge = df_merge.merge(df_cell, how='left')

        df_merge['condition'] = experiment_tag
        df_merge['run'] = run
        logging.debug('nuclei to process: ' + str(df_merge.groupby(ImagejPandas.NUCLEI_INDIV_INDEX).size()))

        try:
            with h5py.File(self.filename, 'r+') as f:
                fproc = f['%s/%s/processed' % (experiment_tag, run)]
                if 'pandas_dataframe' in fproc: del fproc['pandas_dataframe']
                if 'pandas_masks' in fproc: del fproc['pandas_masks']

            df_merge.dropna(how='all', inplace=True)
            df_merge = df_merge[~df_merge['CentrLabel'].isnull()]
            df_interpolated, imask = ImagejPandas.interpolate_data(df_merge)
            df_interpolated = ImagejPandas.vel_acc_nuclei(df_interpolated)
            proc_df = ImagejPandas.dist_vel_acc_centrosomes(df_interpolated)

            maxframe1 = proc_df.loc[proc_df['CentrLabel'] == 'A', 'Frame'].max()
            maxframedc = proc_df['Frame'].max()
            minframe1 = min(maxframe1, maxframedc)

            idx1 = (proc_df['CentrLabel'] == 'A') & (proc_df['Frame'] <= minframe1)
            proc_df.loc[idx1, 'SpeedCentr'] *= -1
            proc_df.loc[idx1, 'AccCentr'] *= -1

            # process interpolated data mask
            mask_df = imask[imask['Nuclei'] > 0]
            mi = mask_df.set_index(ImagejPandas.MASK_INDEX).sort_index()
            mu = mi.unstack('CentrLabel')
            msk = mu.loc[:, ['CentX', 'CentY']].all(axis=1)
            for key in ['Dist', 'Speed', 'Acc']:
                mu.loc[:, (key, 'A')] = mu.loc[:, ('CentX', 'A')]
                mu.loc[:, (key, 'B')] = mu.loc[:, ('CentX', 'B')]
            for key in ['DistCentr', 'SpeedCentr', 'AccCentr']:
                mu.loc[:, (key, 'A')] = msk
            mask_df = mu.stack().reset_index()

            proc_df.to_hdf(self.filename, key='%s/%s/processed/pandas_dataframe' % (experiment_tag, run), mode='r+')
            mask_df.to_hdf(self.filename, key='%s/%s/processed/pandas_masks' % (experiment_tag, run), mode='r+')
        except Exception as e:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            logging.warning('Problem processing %s-%s in line %d of hdf5_nexus.py:\r\n%s' % (
                experiment_tag, run, exc_tb.tb_lineno, e))

    def associate_centrosome_with_nuclei(self, centr_id, nuc_id, experiment_tag, run, centrosome_group=0):
        with h5py.File(self.filename, 'a') as f:
            # link centrosome to current nuclei selection
            source_cpos_addr = '%s/%s/measurements/centrosomes/C%03d/pos' % (experiment_tag, run, centr_id)
            source_npos_addr = '%s/%s/measurements/nuclei/N%02d/pos' % (experiment_tag, run, nuc_id)
            nxcpos = f[source_cpos_addr]

            target_addr = '%s/%s/selection/N%02d' % (experiment_tag, run, nuc_id)
            if source_npos_addr in f:
                if target_addr not in f:
                    nxnuc_ = f.create_group(target_addr)
                    nxnuc_.create_group('A')
                    nxnuc_.create_group('B')
                    nxnpos = f[source_npos_addr]
                    nxnuc_['pos'] = nxnpos

                cstr = 'A' if centrosome_group == 0 else 'B'
                nxnuc_ = f['%s/%s' % (target_addr, cstr)]
                nxnuc_['C%03d' % centr_id] = nxcpos

    def clear_associations(self, experiment_tag, run):
        with h5py.File(self.filename, 'a') as f:
            fsel = f['%s/%s/selection' % (experiment_tag, run)]
            for o in fsel:
                del fsel[o]

    def delete_association(self, of_centrosome, with_nuclei, experiment_tag, run):
        with h5py.File(self.filename, 'a') as f:
            centosomesA = f['%s/%s/selection/N%02d/A' % (experiment_tag, run, with_nuclei)]
            centosomesB = f['%s/%s/selection/N%02d/B' % (experiment_tag, run, with_nuclei)]
            if 'C%03d' % of_centrosome in centosomesA:
                del centosomesA['C%03d' % of_centrosome]
            if 'C%03d' % of_centrosome in centosomesB:
                del centosomesB['C%03d' % of_centrosome]

            with h5py.File(self.filename, 'a') as f:
                fproc = f['%s/%s/processed' % (experiment_tag, run)]
                df_key = '%s/%s/processed/pandas_dataframe' % (experiment_tag, run)
                msk_key = '%s/%s/processed/pandas_masks' % (experiment_tag, run)
                _newdf = _newmsk = None
                if 'pandas_dataframe' in fproc:
                    _newdf = pd.read_hdf(self.filename, key=df_key, mode='r')
                    del fproc['pandas_dataframe']
                if _newdf is not None:
                    _idx = (_newdf['Centrosome'] == of_centrosome) & \
                           (_newdf['Nuclei'] == with_nuclei)
                    _newdf[~_idx].to_hdf(self.filename, key=df_key, mode='r+')
                if 'pandas_masks' in fproc:
                    _newmsk = pd.read_hdf(self.filename, key=msk_key, mode='r')
                    del fproc['pandas_masks']
                if _newmsk is not None:
                    _idx = (_newmsk['Centrosome'] == of_centrosome) & \
                           (_newmsk['Nuclei'] == with_nuclei)
                    _newmsk[~_idx].to_hdf(self.filename, key=msk_key, mode='r+')

    def move_association(self, of_centrosome, from_nuclei, toNuclei, centrosome_group, experiment_tag, run):
        self.delete_association(of_centrosome, from_nuclei, experiment_tag, run)
        self.associate_centrosome_with_nuclei(of_centrosome, toNuclei, experiment_tag, run)

    def is_centrosome_associated(self, centrosome, experiment_tag, run):
        with h5py.File(self.filename, 'r') as f:
            nuclei_list = f['%s/%s/measurements/nuclei' % (experiment_tag, run)]
            sel = f['%s/%s/selection' % (experiment_tag, run)]
            for nuclei in nuclei_list:
                if nuclei in sel:
                    nuc = sel[nuclei]
                    if centrosome in nuc['A'] or centrosome in nuc['B']:
                        return True
        return False

def move_images(filefrom, fileto):
    src = h5py.File(filefrom, 'a')
    dst = h5py.File(fileto, 'a')
    for cond in src.keys():
        for run in src[cond].keys():
            _grp = '%s/%s/raw' % (cond, run)
            if isinstance(src[_grp], h5py.SoftLink):
                logging.debug(_grp, 'group already a soft link')
            if isinstance(src[_grp], h5py.HardLink):
                logging.debug(_grp, 'group already a hard link')
            if isinstance(src[_grp], h5py.Group):
                # Get the name of the parent for the group we want to copy
                group_path = src[_grp].parent.name
                # Check that this group exists in the destination file; if it doesn't, create it
                # This will create the parents too, if they don't exist
                group_id = dst.require_group(group_path)
                # Copy src -> dst
                if _grp not in dst:
                    logging.debug(_grp, 'copying raw images')
                    src.copy(_grp, group_id, name='raw')
                del src[_grp]
                src[_grp] = h5py.ExternalLink(fileto, _grp)
    src.close()
    dst.close()

def process_dir(path, hdf5f):
    condition = os.path.abspath(args.input).split('/')[-1]

    for root, directories, filenames in os.walk(os.path.join(path, 'input')):
        for filename in filenames:
            ext = filename.split('.')[-1]
            if ext == 'tif':
                joinf = os.path.join(root, filename)
                logging.info('--------------------------------------------------------------')
                groups = re.search('^(.+)-(.+).tif$', filename).groups()
                run_id = groups[1]
                run_str = 'run_%s' % run_id
                centdata = os.path.join(path, 'data', 'run-%s-table.csv' % run_id)
                nucldata = os.path.join(path, 'data', 'run-%s-nuclei.csv' % run_id)

                if os.path.isfile(centdata) and os.path.isfile(nucldata) and os.path.isfile(joinf):
                    logging.info('adding raw file: %s' % joinf)
                    hdf5.add_experiment(condition, run_str)
                    logging.info('adding tiff:', joinf)
                    hdf5f.add_tiff_sequence(joinf, condition, run_str)
                    logging.info('adding data file: %s' % centdata)
                    hdf5f.add_measurements(centdata, condition, run_str)

if __name__ == '__main__':
    # process input arguments
    parser = argparse.ArgumentParser(
        description='Creates an HDF5 file for experiments storage.')
    parser.add_argument('input', metavar='I', type=str, help='input directory where the files are')
    args = parser.parse_args()

    # Create hdf5 file if it doesn't exist
    hdf5 = LabHDF5NeXusFile(filename='out/centrosomes.nexus.hdf5',
                            imagesfile='out/centrosomes-images.nexus.hdf5', fileflag='a')
    try:
        process_dir(args.input, hdf5)
        move_images('out/centrosomes.nexus.hdf5', 'out/centrosomes-images.nexus.hdf5')

        logging.info('--------------------------------------------------------------')
        logging.info('shrinking file size...')
        call('h5repack out/centrosomes.nexus.hdf5 out/repack.hdf5', shell=True)
        os.remove('out/centrosomes.nexus.hdf5')
        os.rename('out/repack.hdf5', 'out/centrosomes.nexus.hdf5')
        call('h5repack out/centrosomes-images.nexus.hdf5 out/repack.hdf5', shell=True)
        os.remove('out/centrosomes-images.nexus.hdf5')
        os.rename('out/repack.hdf5', 'out/centrosomes-images.nexus.hdf5')
    finally:
        logging.info('finished.')


# ##### BEGIN GPL LICENSE BLOCK #####
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####

# <pep8 compliant>

import bpy
from bpy.types import (
    Gizmo,
    GizmoGroup,
)
from bpy.props import (
    CollectionProperty,
    IntProperty,
    BoolProperty,
)
from bpy.app.handlers import persistent

bl_info = {
    "name": "VR Scene Inspection",
    "author": "Julian Eisel (Severin)",
    "version": (0, 2, 0),
    "blender": (2, 83, 0),
    "location": "3D View > Sidebar > VR",
    "description": ("View the viewport with virtual reality glasses "
                    "(head-mounted displays)"),
    "support": "OFFICIAL",
    "warning": "This is an early, limited preview of in development "
               "VR support for Blender.",
    "doc_url": "{BLENDER_MANUAL_URL}/addons/3d_view/vr_scene_inspection.html",
    "category": "3D View",
}

@persistent
def ensure_default_vr_landmark(context: bpy.context):
    # Ensure there's a default landmark (scene camera by default).
    landmarks = bpy.context.scene.vr_landmarks
    if not landmarks:
        landmarks.add()
        landmarks[0].type = 'SCENE_CAMERA'

def xr_landmark_active_type_update(self, context):
    wm = context.window_manager
    session_settings = wm.xr_session_settings
    landmark_active = VRLandmark.get_active_landmark(context)

    # Update session's base pose type to the matching type.
    if landmark_active.type == 'SCENE_CAMERA':
        session_settings.base_pose_type = 'SCENE_CAMERA'
    elif landmark_active.type == 'USER_CAMERA':
        session_settings.base_pose_type = 'OBJECT'
    # elif landmark_active.type == 'CUSTOM':
        # session_settings.base_pose_type = 'CUSTOM'

def xr_landmark_active_camera_update(self, context):
    session_settings = context.window_manager.xr_session_settings
    landmark_active = VRLandmark.get_active_landmark(context)

    # Update the anchor object to the (new) camera of this landmark.
    session_settings.base_pose_object = landmark_active.base_pose_camera

def xr_landmark_active_base_pose_location_update(self, context):
    session_settings = context.window_manager.xr_session_settings
    landmark_active = VRLandmark.get_active_landmark(context)

    session_settings.base_pose_location = landmark_active.base_pose_location

def xr_landmark_active_base_pose_angle_update(self, context):
    session_settings = context.window_manager.xr_session_settings
    landmark_active = VRLandmark.get_active_landmark(context)

    session_settings.base_pose_angle = landmark_active.base_pose_angle

def xr_landmark_type_update(self, context):
    landmark_selected = VRLandmark.get_selected_landmark(context)
    landmark_active = VRLandmark.get_active_landmark(context)

    # Only update session settings data if the changed landmark is actually
    # the active one.
    if landmark_active == landmark_selected:
        xr_landmark_active_type_update(self, context)

def xr_landmark_camera_update(self, context):
    landmark_selected = VRLandmark.get_selected_landmark(context)
    landmark_active = VRLandmark.get_active_landmark(context)

    # Only update session settings data if the changed landmark is actually
    # the active one.
    if landmark_active == landmark_selected:
        xr_landmark_active_camera_update(self, context)

def xr_landmark_base_pose_location_update(self, context):
    landmark_selected = VRLandmark.get_selected_landmark(context)
    landmark_active = VRLandmark.get_active_landmark(context)

    # Only update session settings data if the changed landmark is actually
    # the active one.
    if landmark_active == landmark_selected:
        xr_landmark_active_base_pose_location_update(self, context)

def xr_landmark_base_pose_angle_update(self, context):
    landmark_selected = VRLandmark.get_selected_landmark(context)
    landmark_active = VRLandmark.get_active_landmark(context)

    # Only update session settings data if the changed landmark is actually
    # the active one.
    if landmark_active == landmark_selected:
        xr_landmark_active_base_pose_angle_update(self, context)

def xr_landmark_camera_object_poll(self, object):
    return object.type == 'CAMERA'

def xr_landmark_active_update(self, context):
    wm = context.window_manager

    xr_landmark_active_type_update(self, context)
    xr_landmark_active_camera_update(self, context)
    xr_landmark_active_base_pose_location_update(self, context)
    xr_landmark_active_base_pose_angle_update(self, context)

    if wm.xr_session_state:
      wm.xr_session_state.reset_to_base_pose(context)

class VRLandmark(bpy.types.PropertyGroup):
    name: bpy.props.StringProperty(
        name="VR Landmark",
        default="Landmark"
    )
    type: bpy.props.EnumProperty(
        name="Type",
        items=[
            ('SCENE_CAMERA', "Scene Camera",
             "Use scene's currently active camera to define the VR view base "
             "location and rotation"),
            ('USER_CAMERA', "Custom Camera",
             "Use an existing camera to define the VR view base location and "
             "rotation"),
            # Custom base poses work, but it's uncertain if they are really
            # needed. Disabled for now.
            # ('CUSTOM', "Custom Pose",
            #  "Allow a manually definied position and rotation to be used as "
            #  "the VR view base pose"),
        ],
        default='SCENE_CAMERA',
        update=xr_landmark_type_update,
    )
    base_pose_camera: bpy.props.PointerProperty(
        name="Camera",
        type=bpy.types.Object,
        poll=xr_landmark_camera_object_poll,
        update=xr_landmark_camera_update,
    )
    base_pose_location: bpy.props.FloatVectorProperty(
        name="Base Pose Location",
        subtype='TRANSLATION',
        update=xr_landmark_base_pose_location_update,
    )

    base_pose_angle: bpy.props.FloatProperty(
        name="Base Pose Angle",
        subtype='ANGLE',
        update=xr_landmark_base_pose_angle_update,
    )

    @staticmethod
    def get_selected_landmark(context):
        scene = context.scene
        landmarks = scene.vr_landmarks

        return (
            None if (len(landmarks) <
                     1) else landmarks[scene.vr_landmarks_selected]
        )

    @staticmethod
    def get_active_landmark(context):
        scene = context.scene
        landmarks = scene.vr_landmarks

        return (
            None if (len(landmarks) <
                     1) else landmarks[scene.vr_landmarks_active]
        )

class VIEW3D_UL_vr_landmarks(bpy.types.UIList):
    def draw_item(self, context, layout, _data, item, icon, _active_data,
                  _active_propname, index):
        landmark = item
        landmark_active_idx = context.scene.vr_landmarks_active

        layout.emboss = 'NONE'

        layout.prop(landmark, "name", text="")

        icon = (
            'RADIOBUT_ON' if (index == landmark_active_idx) else 'RADIOBUT_OFF'
        )
        props = layout.operator(
            "view3d.vr_landmark_activate", text="", icon=icon)
        props.index = index

class VIEW3D_PT_vr_landmarks(bpy.types.Panel):
    bl_space_type = 'VIEW_3D'
    bl_region_type = 'UI'
    bl_category = "VR"
    bl_label = "Landmarks"
    bl_options = {'DEFAULT_CLOSED'}

    def draw(self, context):
        layout = self.layout
        scene = context.scene
        landmark_selected = VRLandmark.get_selected_landmark(context)

        layout.use_property_split = True
        layout.use_property_decorate = False  # No animation.

        row = layout.row()

        row.template_list("VIEW3D_UL_vr_landmarks", "", scene, "vr_landmarks",
                          scene, "vr_landmarks_selected", rows=3)

        col = row.column(align=True)
        col.operator("view3d.vr_landmark_add", icon='ADD', text="")
        col.operator("view3d.vr_landmark_remove", icon='REMOVE', text="")

        if landmark_selected:
            layout.prop(landmark_selected, "type")

            if landmark_selected.type == 'USER_CAMERA':
                layout.prop(landmark_selected, "base_pose_camera")
            # elif landmark_selected.type == 'CUSTOM':
            #     layout.prop(landmark_selected,
            #                 "base_pose_location", text="Location")
            #     layout.prop(landmark_selected,
            #                 "base_pose_angle", text="Angle")

class VIEW3D_PT_vr_session_view(bpy.types.Panel):
    bl_space_type = 'VIEW_3D'
    bl_region_type = 'UI'
    bl_category = "VR"
    bl_label = "View"

    def draw(self, context):
        layout = self.layout
        session_settings = context.window_manager.xr_session_settings

        layout.use_property_split = True
        layout.use_property_decorate = False  # No animation.

        col = layout.column(align=True, heading="Show")
        col.prop(session_settings, "show_floor", text="Floor")
        col.prop(session_settings, "show_annotation", text="Annotations")

        col = layout.column(align=True)
        col.prop(session_settings, "clip_start", text="Clip Start")
        col.prop(session_settings, "clip_end", text="End")

class VIEW3D_PT_vr_session(bpy.types.Panel):
    bl_space_type = 'VIEW_3D'
    bl_region_type = 'UI'
    bl_category = "VR"
    bl_label = "VR Session"

    def draw(self, context):
        layout = self.layout
        session_settings = context.window_manager.xr_session_settings

        layout.use_property_split = False

        is_session_running = bpy.types.XrSessionState.is_running(context)

        # Using SNAP_FACE because it looks like a stop icon -- I shouldn't
        # have commit rights...
        toggle_info = (
            ("Start VR Session", 'PLAY') if not is_session_running else (
                "Stop VR Session", 'SNAP_FACE')
        )
        layout.operator("wm.xr_session_toggle",
                        text=toggle_info[0], icon=toggle_info[1])

        layout.separator()

        layout.prop(session_settings, "use_positional_tracking")

class VIEW3D_OT_vr_landmark_add(bpy.types.Operator):
    bl_idname = "view3d.vr_landmark_add"
    bl_label = "Add VR Landmark"
    bl_description = "Add a new VR landmark to the list and select it"
    bl_options = {'UNDO', 'REGISTER'}

    def execute(self, context):
        scene = context.scene
        landmarks = scene.vr_landmarks

        landmarks.add()

        # select newly created set
        scene.vr_landmarks_selected = len(landmarks) - 1

        return {'FINISHED'}

class VIEW3D_OT_vr_landmark_remove(bpy.types.Operator):
    bl_idname = "view3d.vr_landmark_remove"
    bl_label = "Remove VR Landmark"
    bl_description = "Delete the selected VR landmark from the list"
    bl_options = {'UNDO', 'REGISTER'}

    def execute(self, context):
        scene = context.scene
        landmarks = scene.vr_landmarks

        if len(landmarks) > 1:
            landmark_selected_idx = scene.vr_landmarks_selected
            landmarks.remove(landmark_selected_idx)

            scene.vr_landmarks_selected -= 1

        return {'FINISHED'}

class VIEW3D_OT_vr_landmark_activate(bpy.types.Operator):
    bl_idname = "view3d.vr_landmark_activate"
    bl_label = "Activate VR Landmark"
    bl_description = "Change to the selected VR landmark from the list"
    bl_options = {'UNDO', 'REGISTER'}

    index: IntProperty(
        name="Index",
        options={'HIDDEN'},
    )

    def execute(self, context):
        scene = context.scene

        if self.index >= len(scene.vr_landmarks):
            return {'CANCELLED'}

        scene.vr_landmarks_active = (
            self.index if self.properties.is_property_set(
                "index") else scene.vr_landmarks_selected
        )

        return {'FINISHED'}

class VIEW3D_PT_vr_viewport_feedback(bpy.types.Panel):
    bl_space_type = 'VIEW_3D'
    bl_region_type = 'UI'
    bl_category = "VR"
    bl_label = "Viewport Feedback"
    bl_options = {'DEFAULT_CLOSED'}

    def draw(self, context):
        layout = self.layout
        view3d = context.space_data

        col = layout.column(align=True)
        col.label(icon='ERROR', text="Note:")
        col.label(text="Settings here may have a significant")
        col.label(text="performance impact!")

        layout.separator()

        layout.prop(view3d.shading, "vr_show_virtual_camera")
        layout.prop(view3d, "mirror_xr_session")

class VIEW3D_GT_vr_camera_cone(Gizmo):
    bl_idname = "VIEW_3D_GT_vr_camera_cone"

    aspect = 1.0, 1.0

    def draw(self, context):
        import bgl

        if not hasattr(self, "frame_shape"):
            aspect = self.aspect

            frame_shape_verts = (
                (-aspect[0], -aspect[1], -1.0),
                (aspect[0], -aspect[1], -1.0),
                (aspect[0], aspect[1], -1.0),
                (-aspect[0], aspect[1], -1.0),
            )
            lines_shape_verts = (
                (0.0, 0.0, 0.0),
                frame_shape_verts[0],
                (0.0, 0.0, 0.0),
                frame_shape_verts[1],
                (0.0, 0.0, 0.0),
                frame_shape_verts[2],
                (0.0, 0.0, 0.0),
                frame_shape_verts[3],
            )

            self.frame_shape = self.new_custom_shape(
                'LINE_LOOP', frame_shape_verts)
            self.lines_shape = self.new_custom_shape(
                'LINES', lines_shape_verts)

        # Ensure correct GL state (otherwise other gizmos might mess that up)
        bgl.glLineWidth(1)
        bgl.glEnable(bgl.GL_BLEND)

        self.draw_custom_shape(self.frame_shape)
        self.draw_custom_shape(self.lines_shape)

class VIEW3D_GGT_vr_viewer_pose(GizmoGroup):
    bl_idname = "VIEW3D_GGT_vr_viewer_pose"
    bl_label = "VR Viewer Pose Indicator"
    bl_space_type = 'VIEW_3D'
    bl_region_type = 'WINDOW'
    bl_options = {'3D', 'PERSISTENT', 'SCALE', 'VR_REDRAWS'}

    @classmethod
    def poll(cls, context):
        view3d = context.space_data
        return (
            view3d.shading.vr_show_virtual_camera and
            bpy.types.XrSessionState.is_running(context) and
            not view3d.mirror_xr_session
        )

    @staticmethod
    def _get_viewer_pose_matrix(context):
        from mathutils import Matrix, Quaternion

        wm = context.window_manager

        loc = wm.xr_session_state.viewer_pose_location
        rot = wm.xr_session_state.viewer_pose_rotation

        rotmat = Matrix.Identity(3)
        rotmat.rotate(rot)
        rotmat.resize_4x4()
        transmat = Matrix.Translation(loc)

        return transmat @ rotmat

    def setup(self, context):
        gizmo = self.gizmos.new(VIEW3D_GT_vr_camera_cone.bl_idname)
        gizmo.aspect = 1 / 3, 1 / 4

        gizmo.color = gizmo.color_highlight = 0.2, 0.6, 1.0
        gizmo.alpha = 1.0

        self.gizmo = gizmo

    def draw_prepare(self, context):
        self.gizmo.matrix_basis = self._get_viewer_pose_matrix(context)

classes = (
    VIEW3D_PT_vr_session,
    VIEW3D_PT_vr_session_view,
    VIEW3D_PT_vr_landmarks,
    VIEW3D_PT_vr_viewport_feedback,

    VRLandmark,
    VIEW3D_UL_vr_landmarks,

    VIEW3D_OT_vr_landmark_add,
    VIEW3D_OT_vr_landmark_remove,
    VIEW3D_OT_vr_landmark_activate,

    VIEW3D_GT_vr_camera_cone,
    VIEW3D_GGT_vr_viewer_pose,
)

def register():
    if not bpy.app.build_options.xr_openxr:
        return

    for cls in classes:
        bpy.utils.register_class(cls)

    bpy.types.Scene.vr_landmarks = CollectionProperty(
        name="Landmark",
        type=VRLandmark,
    )
    bpy.types.Scene.vr_landmarks_selected = IntProperty(
        name="Selected Landmark"
    )
    bpy.types.Scene.vr_landmarks_active = IntProperty(
        update=xr_landmark_active_update,
    )
    # View3DShading is the only per 3D-View struct with custom property
    # support, so "abusing" that to get a per 3D-View option.
    bpy.types.View3DShading.vr_show_virtual_camera = BoolProperty(
        name="Show VR Camera"
    )

    bpy.app.handlers.load_post.append(ensure_default_vr_landmark)

def unregister():
    if not bpy.app.build_options.xr_openxr:
        return

    for cls in classes:
        bpy.utils.unregister_class(cls)

    del bpy.types.Scene.vr_landmarks
    del bpy.types.Scene.vr_landmarks_selected
    del bpy.types.Scene.vr_landmarks_active
    del bpy.types.View3DShading.vr_show_virtual_camera

    bpy.app.handlers.load_post.remove(ensure_default_vr_landmark)

if __name__ == "__main__":
    register()


import requests
import json
import os

BASE_URL = 'http://app.nanonets.com/api/v2/MultiLabelClassification/'
AUTH_KEY = os.environ.get('NANONETS_API_KEY')
MODEL_ID = os.environ.get('NANONETS_MODEL_ID')

def get_model(model_id):
    url = BASE_URL + "Model/%s"%(model_id)
    response = requests.request('GET', url, auth=requests.auth.HTTPBasicAuth(AUTH_KEY,''))
    res =  json.loads(response.text)
    state, status  = res['state'], res['status']
    if state != 5:
        print("The model isn't ready yet, it's status is:", status)
        print("We will send you an email when the model is ready. If your imapatient, run this script again in 10 minutes to check.")
        print("\n\nmore details at:")
        print("https://app.nanonets.com/multilabelclassification/#/classify/"+model_id)
    else:
        print("NEXT RUN: python ./code/prediction.py ./multilabel_data/ImageSets/1067.jpg")

if __name__=="__main__":
    get_model(MODEL_ID)


import sys
import os.path
import logging

def parse_compliance(compliance_aux):
    compliance = {}
    for l in compliance_aux:
        document, requirement, status = l.strip().split(':')
        compliance.setdefault(document, {})
        compliance[document][requirement] = status
    return compliance

def parse_explanations(explanations_aux):
    explanations = {}
    state = 0
    lastd = None
    lastr = None
    for l in explanations_aux:
        if state == 0:
            document, requirement, throwaway = l.strip().split(':')
            explanations.setdefault(document, {})
            explanations[document][requirement] = ''
            lastd = document
            lastr = requirement
            state = 1
        elif state == 1:
            if l.strip() == ':':
                state = 0
            else:
                explanations[lastd][lastr] += l
    return explanations

def table_rows(docname, meaning, compliance_for_document,
        explanations_for_document):
    yield (
        'STIG',
        'STIG ID',
        'IA Controls',
        'Title',
        'Rule ID',
        'Severity',
        'Compliant?',
        'Explanation/Mitigation',
        )
    for rg in meaning.selected_rule_groups():
        for r in rg.rules():
            try:
                compliant = compliance_for_document[r.version]
            except KeyError:
                compliant = '(nothing written)'
            try:
                explanation = explanations_for_document[r.version]
            except KeyError:
                explanation = ''
            yield (
                    docname,
                    r.version,
                    ', '.join(rg.ia_controls),
                    r.title,
                    r.id,
                    r.severity,
                    compliant,
                    explanation,
                    )

def values_to_csv(iterable_of_tuples):
    for row in iterable_of_tuples:
        # HACK: replace double quotes in values with single quotes, so that
        # when we use double quotes to delimit values, quotes inside the
        # values won't screw up the file format
        yield u','.join(u'"%s"' % (x.replace('"', "'")) for x in row)

def compliance2csv(checklists, compliance_aux, explanations_aux):
    """Read aux files created by LaTeX and yield CSV table of compliance.

    meaning is an XCCDFMeaning object.

    compliance_aux and explanations_aux are iterables of lines from a
    the respective files (e.g., open file objects).
    """
    log = logging.getLogger('compliance2csv')
    compliance = parse_compliance(compliance_aux)
    explanations = parse_explanations(explanations_aux)
    for docname, meaning in checklists.items():
        if docname in compliance:
            this_compliance = compliance[docname]
        else:
            log.warning('no notations of compliance found for %s',
                    docname)
            this_compliance = {}
        if docname in explanations:
            this_explanations = explanations[docname]
        else:
            log.info('no explanations of non-compliance '
                    'found for %s. (Maybe there is no '
                    'non-compliance to explain!)',
                    docname)
            this_explanations = {}
        rows = table_rows(docname, meaning, this_compliance, this_explanations)
        for x in values_to_csv(rows):
            yield x


# -*- coding: utf-8 -*-
"""
Created on Thu Jun 16 16:51:54 2022

@author: Ra, S.J., Cho, S.
"""

import pandas as pd
import numpy as np
import sklearn
import os

import matplotlib.pyplot as plt
import joblib
import random

from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.initializers import RandomNormal

from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping

os.chdir(r'C:\Users\YourPCPC\Desktop\DS_MetaLr')
#%% ANN function
def build_ANN(output_size, input_size, hidden_size):
    ann_ = Sequential()
    ann_.add(Input(input_size))
    for nodes in hidden_size:
        ann_.add(Dense(nodes, activation = 'relu',
                       kernel_initializer = RandomNormal(stddev=0.005),
                       bias_initializer = RandomNormal(stddev=0.005)))

    ann_.add(Dense(output_size))

    ann_.compile(
                 loss='mean_squared_error',
                 optimizer=Adam(learning_rate=0.001),
                 metrics=['RootMeanSquaredError', 'MeanAbsoluteError'])
    return ann_

def cv_rmse(mea_y, pred_y):
    rmse_ = np.sqrt(np.mean(np.power((mea_y - pred_y), 2), axis=0))
    cvrmse_ = rmse_/np.mean(mea_y, axis=0)*100
    return cvrmse_

def mbe(mea_y, pred_y):
    mbe_ = np.sum((mea_y - pred_y), axis=0)/np.mean(mea_y, axis=0)/(len(mea_y)-1)*100
    return mbe_

#%% make dir function
def createFolder(directory):
    try:
        if not os.path.exists(directory):
            os.makedirs(directory)
    except OSError:
        print ('Error: Creating directory. ' +  directory)

createFolder('./Model')
#%%
virtual_Data = pd.read_csv('./dataMeta_vs2.csv', header = [0])
virtual_Data['CDU_oper'] = virtual_Data[['CDU1', 'CDU2', 'CDU3', 'CDU4']].sum(axis = 1)
virtual_Data['Fan'] = virtual_Data['Fan']*2

inputData = virtual_Data[['CDU_oper', 'Fan', 'OAT', 'OAH']]
outputData = virtual_Data[['SAT']]

### Make scalers
inputScaler = MinMaxScaler()
outputScaler = MinMaxScaler()

inputScaler = inputScaler.fit(inputData)
outputScaler = outputScaler.fit(outputData)

joblib.dump(inputScaler, './Model/input_scaler.save')
joblib.dump(outputScaler, './Model/output_scaler.save')

### Scale input/output data
inputScaled = inputScaler.transform(inputData)
outputScaled = outputScaler.transform(outputData)

###
inputTrain = inputScaled
outputTrain = outputScaled

#%%
train_input, test_input, train_output, test_output = train_test_split(inputTrain, outputTrain, test_size = 0.3, shuffle = False, random_state = 12)
meta_ANN = build_ANN(outputTrain.shape[1], (inputTrain.shape[1],), (30, 30, 30))

#%%
checkpoint = ModelCheckpoint('./Model/saPred_EP_weight.h5', monitor = 'loss', verbose = 1, save_best_only = True)
#es = EarlyStopping(monitor='loss', mode='min', patience = 30)

meta_ANN.fit(train_input, train_output, epochs = 200, callbacks = [checkpoint])
#%%
meta_ANN.load_weights('./Model/saPred_EP_weight.h5')
meta_ANN.save('./Model/saPred_EP.h5')

#%%
prediction = outputScaler.inverse_transform(meta_ANN.predict(test_input))
true = outputScaler.inverse_transform(test_output)
print(cv_rmse(true, prediction))
#%%
parameters = {'axes.labelsize': 23,
          'axes.titlesize': 23,
          'xtick.labelsize': 23,
          'ytick.labelsize': 23,
          'font.family': 'Times New Roman'}
plt.rcParams.update(parameters)

#%%
fig, ax1 = plt.subplots(1,1, figsize = (20, 5))

ax1.plot(true[0:144*10], color = 'b', label = 'measured')
ax1.plot(prediction[0:144*10], color = 'r', label = 'predicted')

ax1.set_xlabel('data points [10min]')
ax1.set_ylim([12.5, 35.5])
ax1.set_yticks(list(range(13,35,5)))
ax1.set_ylabel(u'temperature [\u00B0C]')

ax1.grid()
ax1.legend(loc = 'upper center', fontsize = 23, ncol = 2)

ax1.set_title('CVRMSE: {}%,       MBE: {}%'.format(np.round(cv_rmse(true,prediction), 2)[0],
                                                   np.round(mbe(true, prediction), 2)[0]))

fig.savefig('./Figure/epModel_test.png', dpi = 350, bbox_inches = 'tight')

#%%
test_input_unscaled = inputScaler.inverse_transform(test_input)

fig, ax1 = plt.subplots(1,1, figsize = (20, 5))

ax1.plot(test_input_unscaled[0:144*10, 0], color = 'g', label = 'CDU operation')

ax1.set_xlabel('data points [10min]')
ax1.set_ylim([-0.2, 5.2])
ax1.set_yticks([0, 1, 2, 3, 4])
ax1.set_ylabel('no. of CDUs')

ax1.grid()
# ax1.legend(loc = 'upper center', fontsize = 23, ncol = 2)
ax2 = ax1.twinx()
ax2.plot(test_input_unscaled[0:144*10, 1], color = 'magenta', label = 'fan on/off')

ax2.set_ylim([-0.2, 5.2])
ax2.set_yticks([0, 2])
ax2.set_yticklabels(['off', 'on'])
ax2.set_ylabel('on/off')

fig.legend(ncol = 2, bbox_to_anchor = (0.5, 0.88), fontsize = 23, loc = 'upper center')
fig.savefig('./Figure/epModel_input_operation.png', dpi = 350, bbox_inches = 'tight')


import torch
from torch.utils.data import ConcatDataset, Subset, random_split

import os
import time
import random

import pickle

from torchvision.datasets import ImageFolder

from config_loader import *
from lidar_preprocessing import preprocess_image

class LidarDataset(ImageFolder):

    # Source: https://github.com/pytorch/vision/issues/2930
    # This creates a dataset with the same functionality as ImageFolder but with our custom targets.
    def __init__(self, name: str) -> None:

        input_folder = os.path.join(DATASET_FOLDER, name)

        with open(os.path.join(DATASET_FOLDER, name, "targets.pickle"), "rb") as file:
            labels = pickle.load(file)

        super().__init__(input_folder, preprocess_image)
        paths, _ = zip(*self.imgs)
        self.targets = [torch.Tensor(labels[path]) for path in paths]
        self.samples = self.imgs = list(zip(paths, self.targets))

    def get_idx_of_turn_samples(self):
        idx = []
        for i, x in enumerate(self.targets):
            if abs(x[0]) > 0.15:
                idx.append(i)
            elif abs(x[0]) > 0.05 and random.random() < 0.3:
                idx.append(i)
            elif random.random() < 0.05:
                idx.append(i)

        return idx

def display_dataset_example(ds: LidarDataset):
    """Displays a few example inputs from the given Lidar Dataset."""
    ds_item = ds.__getitem__(0)
    ds_item['input'].show()
    ds_item = ds.__getitem__(1000)
    ds_item['input'].show()
    ds_item = ds.__getitem__(2000)
    ds_item['input'].show()
    ds_item = ds.__getitem__(3000)
    ds_item['input'].show()
    ds_item = ds.__getitem__(4000)
    ds_item['input'].show()

print("Starting dataset loading...")
start_time = time.time()

datasets = []

for i, x in enumerate(DATASET_LIST):
    ds = LidarDataset(x)
    if DATASET_FILTER_ONLY_TURNS:
        ds = Subset(ds, ds.get_idx_of_turn_samples())
    datasets.append(ds)
    print("Dataset " + str(i) + "/" + str(len(DATASET_LIST)) + " loaded...")
    if DEBUG_GENERATE_ONLY_ONE_DATASET:
        print("DEBUG_REDUCED_DATASET enabled. Ignoring other datasets...")
        break

full_dataset = ConcatDataset(datasets)

print("Concatenating and generating splits...")

train_length = round(0.8*len(full_dataset))
[dataset_train, dataset_test] = random_split(full_dataset, [train_length, len(full_dataset) - train_length])

print("Dataset loading complete!")
print("Loading time: %s" % (time.time() - start_time))


"""add_motd

Revision ID: 19b877061c98
Revises: 1256117ad629
Create Date: 2022-04-05 13:49:44.087517

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

# revision identifiers, used by Alembic.
revision = "19b877061c98"
down_revision = "1256117ad629"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "motd",
        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
        sa.Column("message", sa.Text(), nullable=False),
        sa.Column("date_created", sa.DateTime(), nullable=False),
        sa.PrimaryKeyConstraint("id"),
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("motd")
    # ### end Alembic commands ###


import requests
import logging
from tornado.httpclient import HTTPRequest, AsyncHTTPClient
from tornado import gen
try:
    import ujson as json
except:
    import json as json

__author__ = 'grigory51'

logger = logging.getLogger('bot')

def get_updates(bot):
    client = AsyncHTTPClient()
    requests.post(bot.get_method_url('setWebhook'))
    update_id = [None]

    @gen.coroutine
    def wrapper():
        data = {}
        if update_id[0] is not None:
            data['offset'] = update_id[0]

        req = HTTPRequest(bot.get_method_url('getUpdates', data))
        res = yield client.fetch(req)

        data = json.loads(res.body)
        if data.get('ok'):
            flag = False
            for item in data.get('result', []):
                flag = True
                if update_id[0] is None or item.get('update_id') + 1 > update_id[0]:
                    update_id[0] = item.get('update_id') + 1
                    bot._on_hook(item)
            if flag:
                logger.debug('Processing updates for %s' % (bot.name,))

    return wrapper


import nltk
import wikipedia
import prettytable
import matplotlib.pyplot as plt

from nltk import Tree
from nltk.draw.util import CanvasFrame
from nltk.draw import TreeWidget

from typing import List
from nltk.corpus import wordnet
from nltk.stem import WordNetLemmatizer
from nltk.parse.generate import generate
from nltk.tokenize import word_tokenize, sent_tokenize

what = lambda x: print(nltk.help.upenn_tagset(x))

"""
Exercise 1:
Choose a wikipedia article. You will download and acces the article using this python module: wikipedia.
Use the content property to extract the text.
Print the title of the article and the first N=200 words from the article to verify that all works well.
Print the POS-tagging for the first N=20 sentences.
"""

wikipedia.set_lang('en')
wikipedia_page = wikipedia.page("COVID-19 pandemic")
content = wikipedia_page.content

N = 200
print("Wikipedia Page Title: {}".format(wikipedia_page.title))
words = word_tokenize(content)
print(words[: N])

N = 20
sentences = sent_tokenize(content)
pos_taggs = [nltk.pos_tag(word_tokenize(sentence)) for (i, sentence) in enumerate(sentences) if i < N]
print("POS-tagging for the first N sentences")
print(pos_taggs)

"""
Exercise 2:
Create a function that receives a part of speech tag and returns a list with all the words from the text
(can be given as a parameter too) that represent that part of speech.
Create a function that receives a list of POS tags and returns a list with words having any of the given POS tags
(use the first function in implementing the second one).
"""

def get_words_part_of_speech(content: str = None, tag: str = None) -> List[str]:
    sentences = sent_tokenize(content)
    taggs = [nltk.pos_tag(word_tokenize(sentence)) for sentence in sentences]
    return [word for tagg_sentence in taggs for (word, tagg) in tagg_sentence if tagg == tag]

def get_words_parts_of_speech(content: str = None, tags: List[str] = None) -> List[str]:
    result = []
    for pos in tags:
        result.extend(get_words_part_of_speech(content, pos))
    return result

print("Selected words by a single part of speech")
print(get_words_part_of_speech(content, "CD")[:20])
print("Selected words by multiple part of speech")
print(get_words_parts_of_speech(content, ["DT", "CD"])[:50])

"""
Exercise 3:
Use the function above to print all the nouns (there are multiple tags for nouns),
and, respectively all the verbs (corresponding to all verb tags).
Also, print the percentage of content words (noun+verbs) from the entire text
"""

nouns = get_words_parts_of_speech(content, ["NN", "NNP", "NNPS", "NNS"])
verbs = get_words_parts_of_speech(content, ["VB", "VBD", "VBG", "VBN", "VBP", "VBZ"])

print("All nouns from text")
print(nouns[:50])

print("All verbs from text")
print(verbs[:50])

print("The percent of (nouns + verbs) from all text is: {}%".format((len(nouns) + len(verbs)) * 100 / len(word_tokenize(content))))

"""
Exercise 4:
Print a table of four columns. The columns will be separated with the character "|".
The head of the table will be: Original word | POS | Simple lemmatization | Lemmatization with POS that will compare
the results of lemmatization (WordNetLemmatizer) without giving the part of speech and the lemmatization with the given part of speech for each word.
The table must contain only words that give different results for the two lemmatizations (for example, the word "running" - without POS,
the result will always be running, but with pos="v" it is "run").
The table will contain the results for the first N sentences from the text (each row corresponding to a word).
Try to print only distinct results inside the table
(for example, if a word has two occurnces inside the text, and matches the requirments for appearing in the table, it should have only one corresponding row).
"""

def get_wordnet_pos(tag):
    if tag.startswith('J'):
        return wordnet.ADJ
    elif tag.startswith('V'):
        return wordnet.VERB
    elif tag.startswith('N'):
        return wordnet.NOUN
    elif tag.startswith('R'):
        return wordnet.ADV
    else:
        return wordnet.NOUN

N = 5
table = prettytable.PrettyTable(["Original word", "POS", "Simple lemmatization", "Lemmatization with POS"])
lemmatizer = WordNetLemmatizer()
sentences = [sentence for (i, sentence) in enumerate(sent_tokenize(content)) if i < N]
words_and_pos = [(word, get_wordnet_pos(nltk.tag.pos_tag([word])[0][1])) for sentence in sentences for word in word_tokenize(sentence)]

words_and_pos = list(set(words_and_pos))

for word, tag in words_and_pos:
    simple_lemmatize = lemmatizer.lemmatize(word)
    pos_lemmatize = lemmatizer.lemmatize(word, tag)

    if simple_lemmatize != pos_lemmatize:
        table.add_row([word, tag, simple_lemmatize, pos_lemmatize])

print(table)

"""
Exercise 5:
Print a graphic showing the number of words for each part of speech.
If there are too many different parts of speech, you can print only those with a higher number of corresponding words.
"""

sentences = sent_tokenize(content)
pos_taggs = [nltk.pos_tag(word_tokenize(sentence)) for (i, sentence) in enumerate(sentences)]

counter = {tag : 0 for sentence in pos_taggs for word, tag in sentence}
for sentence in pos_taggs:
    for word, tag in sentence:
        counter[tag] += 1

plt.figure(figsize = (24, 9))
plt.bar(range(len(counter)), list(counter.values()), align = 'center')
plt.xticks(range(len(counter)), list(counter.keys()))
plt.xlabel("Part of speech tag")
plt.ylabel("No. of words")
plt.show()

"""
Exercise 6:
Create your own grammar with different terminal symbols.
Apply recursive descent parsing on a sentence with at least 5 different parts of speech and a tree of at least level 4.
"""

grammar_productions = """
S -> SB VB ADV
SB -> PN | SC | SP
PN -> PP | PNP
VB -> CJ VBI | VBP | VBN
ADV -> CJ VBI UDN
UDN -> LA SC
LA -> 'la'
SC -> 'magazin'
VBI -> 'merg'
PP -> 'Eu'
PNP -> 'ma'
VBP -> 'vreau'
CJ -> 'sa'
"""

sentence = "Eu vreau sa merg la magazin"

grammar = nltk.CFG.fromstring(grammar_productions)

# print(grammar)
# for sentence in generate(grammar, n=10):
#     print(' '.join(sentence))

words = word_tokenize(sentence)
#print(words)

# RecursiveDescentParser, ShiftReduceParser
rdp = nltk.RecursiveDescentParser(grammar)
for tree in rdp.parse(words):
    print(tree)

"""
Exercise 7:
Apply shift reduce parsing on the same sentence and check programatically if the two trees are equal.
Find a sentence with equal trees and a sentence with different results
(we consider the tree different even when it has no sollution for one of the parsers, but has for the other).
"""

sentence = "Eu vreau sa merg la magazin"
words = word_tokenize(sentence)

rdp = nltk.RecursiveDescentParser(grammar)
srp = nltk.ShiftReduceParser(grammar)

tree1 = None
for tree in rdp.parse(words):
    tree1 = tree

print(type(tree1))

tree2 = None
for tree in srp.parse(words):
    tree2 = tree

print(type(tree2))

if tree1 == tree2:
    print("Trees are the same")
else:
    print("Trees are different")


# -*- coding:utf-8 -*-

import mxnet as mx
import log
import logging
import data_tool
import yaml
import sys
import os
import argparse
from model import resnet
from model import resnext

logging.getLogger().setLevel(logging.DEBUG)
logger = log.get_logger(name='train', filename='train.log', filemode='a', level=log.DEBUG, file_and_line=True)

def process(train_params):
    """

    :param train_params:
    :return:
    """
    logger.info('prepare train data')
    features_train, labels_train, features_test, labels_test = data_tool.load_data(
        train_params['train']['traindatapath'], train_params['train']['testdatapath'],
        one_hot=False)

    train_iter = mx.io.NDArrayIter(data=features_train, label=labels_train,
                                   batch_size=train_params['train']['batchsize'], shuffle=True)
    eval_iter = mx.io.NDArrayIter(data=features_test, label=labels_test,
                                  batch_size=train_params['train']['batchsize'])

    logger.info('load symbol %s' % train_params['train']['net'])

    symbol = None
    if train_params['train']['net'] == 'resnext34':
        symbol = resnext.get_symbol(num_classes=train_params['classes'], num_layers=34,
                                    image_shape=train_params['inputshape'])
    elif train_params['train']['net'] == 'resnext50':
        symbol = resnext.get_symbol(num_classes=train_params['classes'], num_layers=50,
                                    image_shape=train_params['inputshape'])
    elif train_params['train']['net'] == 'resnext101':
        symbol = resnext.get_symbol(num_classes=train_params['classes'], num_layers=101,
                                    image_shape=train_params['inputshape'])
    elif train_params['train']['net'] == 'resnet34':
        symbol = resnext.get_symbol(num_classes=train_params['classes'], num_layers=34,
                                    image_shape=train_params['inputshape'])
    elif train_params['train']['net'] == 'resnet50':
        symbol = resnext.get_symbol(num_classes=train_params['classes'], num_layers=50,
                                    image_shape=train_params['inputshape'])
    elif train_params['train']['net'] == 'resnet101':
        symbol = resnext.get_symbol(num_classes=train_params['classes'], num_layers=101,
                                    image_shape=train_params['inputshape'])
    elif train_params['train']['net'] == 'resnet152':
        symbol = resnext.get_symbol(num_classes=train_params['classes'], num_layers=152,
                                    image_shape=train_params['inputshape'])

    logger.info('create model graph at %s' % train_params['train']['graphpath'])
    mx.viz.plot_network(symbol).render(train_params['train']['graphpath'])

    model = mx.mod.Module(symbol=symbol, logger=logger, context=mx.gpu(0))

    logger.info('begin to train==============, epoch: %d, optimizer: %s, learning_rate: %d, batchsize: %d'
                % (train_params['train']['trainepoch'], train_params['train']['optimizer'],
                    train_params['train']['learningrate'], train_params['train']['batchsize']))

    if train_params['preimagnettrain']:
        sym, arg_params, aux_params = get_imagenet_model(train_params['train']['net'])
    if train_params['pretrain']:
        sym, arg_params, aux_params = get_trainned_model(train_params['train']['modelprefix'])

    model.fit(train_data=train_iter,
              eval_data=eval_iter,
              arg_params=arg_params,
              aux_params=aux_params,
              eval_metric=train_params['train']['evalmetric'],
              optimizer=train_params['train']['optimizer'],
              optimizer_params={'learning_rate': train_params['train']['learningrate']},
              epoch_end_callback=mx.callback.do_checkpoint(prefix=train_params['train']['modelprefix'],
                                                           period=train_params['train']['modelepoch']),
              batch_end_callback=mx.callback.Speedometer(batch_size=train_params['train']['batchsize'],
                                                         frequent=1),
              num_epoch=train_params['train']['trainepoch'])

    metric = mx.metric.Accuracy()
    model.score(eval_iter, metric)
    logger.info('train end============== score %f' % metric.get()[1])

def get_imagenet_model(net_name):
    model_prefix = None
    if net_name == 'resnext50':
        model_prefix = 'imagenet/resnext-50'
    elif net_name == 'resnext101':
        model_prefix = 'imagenet/resnext-101'
    elif net_name == 'resnet34':
        model_prefix = 'imagenet/resnet-34'
    elif net_name == 'resnet50':
        model_prefix = 'imagenet/resnet-50'
    elif net_name == 'resnet101':
        model_prefix = 'imagenet/resnet-101'
    elif net_name == 'resnet152':
        model_prefix = 'imagenet/resnet-152'
    sym, arg_params, aux_params = mx.model.load_checkpoint(model_prefix, 0)
    return sym, arg_params, aux_params

def get_trainned_model(model_prefix):
    if model_prefix:
        if model_prefix.rindex('/') > 0:
            index = model_prefix[:model_prefix.rindex('/')]
            file_list = os.listdir(index)
        else:
            file_list = os.listdir('')
        params_file_list = []
        for item in file_list:
            if item.endswith('.params'):
                params_file_list.append(item)
        params_file_list.sort(reverse=True)
        if len(params_file_list) > 0:
            param_file = params_file_list[0]
            from_epoch = int(param_file[param_file.rindex('-')+1:param_file.rindex('.')])
            sym, arg_params, aux_params = mx.model.load_checkpoint(model_prefix,
                                                                   from_epoch)
            logger.info('load pretrainned model from %s %d' % (model_prefix, from_epoch))
            return sym, arg_params, aux_params
    return None, None, None

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='train input',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--parampath', type=str, default='params.yaml',
                        help='path of params.yaml')
    args = parser.parse_args()

    logger.info('load params from %s' % args.parampath)

    params = None
    with open(args.parampath) as f:
        params = yaml.load(f)

    if params is None:
        logger.error('no params found from %s' % args.parampath)
        sys.exit(1)

    process(params)


from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
from sklearn.metrics import classification_report, confusion_matrix
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# drop nan rows
# alternative is to replace with 0 float
# for now we can drop nan rows
def remove_nan(df):
    # is_nan = team_stats.isnull()
    # row_has_nan = is_nan.any(axis=1)
    # rows_with_nan = team_stats[row_has_nan]
    # print(rows_with_nan)
    team_stats = df.dropna()
    return team_stats

"""
    1. Split the data for training and testing using the sklearn.model_selection (train_test_split) module
    2. Use the 'SVC' model to train the model
        - 'fit' is used to train the data
    3. Predict
    4. Display Accuracy
        - use the metric modules
"""
def svm():
    drop_cols = ['result', 'Unnamed: 0']
    df = pd.read_csv('../data/teams_ml_data/combined_team_stats.csv')
    team_stats = remove_nan(df)

    X = team_stats.drop(drop_cols, axis = 1)
    y = team_stats['result']

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)
    svc_classifier = SVC(kernel='linear')
    svc_classifier.fit(X_train, y_train)

    y_pred = svc_classifier.predict(X_test)
    print(confusion_matrix(y_test, y_pred))
    print(classification_report(y_test, y_pred))

def main():
    svm()

if __name__ == "__main__":
    main()

"""
FIRST RUN THROUGH
[[487  12]
 [ 15 478]]
              precision    recall  f1-score   support

           0       0.97      0.98      0.97       499
           1       0.98      0.97      0.97       493

    accuracy                           0.97       992
   macro avg       0.97      0.97      0.97       992
weighted avg       0.97      0.97      0.97       992
"""


import json

import requests

from common_config import *
from utils.img_util import save_sd_img

def text_to_image(prompt, negative_prompt, out_path, steps=SD_DEFAULT_STEPS, batch_size=SD_DEFAULT_BATCH_SIZE):
    url = SD_TXT_TO_IMG_URL
    headers = {
        "accept": "application/json",
        "Content-Type": "application/json"
    }
    data = {
        "prompt": prompt,
        "batch_size": batch_size,
        "n_iter": SD_DEFAULT_N_ITER,
        "steps": steps,
        "width": SD_DEFAULT_WIDTH,
        "height": SD_DEFAULT_HEIGHT,
        "negative_prompt": negative_prompt,
        "cfg_scale": SD_CFG_SCALE,
        "enable_hr": SD_ENABLE_HR,
        "hr_scale": SD_HR_SCALE,
        "hr_upscaler": SD_HR_UPSCALER,
        "denoising_strength": SD_DENOISING_STRENGTH
    }
    res = requests.post(url, headers=headers, data=json.dumps(data))
    save_sd_img(res.json(), out_path)

# if __name__ == '__main__':
#     text_to_image(
#         "Sun WuKong watched in Heaven for more than half a month.,best quality ,masterpiece, illustration, an extremely delicate and beautiful, extremely detailed ,CG ,unity ,8k wallpaper"
#         , NEGATIVE
#         , r"D:\novelai\ai\out.png"
#         , steps=50)


"""
What is the value of the first triangle number to have over five hundred divisors?
"""
from utils import pfactors, n_divisors
from collections import defaultdict

def get_factors_quick(num, cache):
    if cache.has_key(num):
        return cache[num]
    else:
        factors = pfactors(num)
        cache[num] = factors
        return factors

def first_triangle_ndivisors_gt(min_divisors):
    tri = lambda x: x*(x+1)/2

    ndivisors = 1
    curr = 1
    factors_cache = {}
    while ndivisors <= min_divisors:
        curr += 1

        if curr % 2 == 0:
            prime_factors = (get_factors_quick(curr/2, factors_cache) +
                             get_factors_quick(curr+1, factors_cache))
        else:
            prime_factors = (get_factors_quick((curr+1)/2, factors_cache) +
                             get_factors_quick(curr, factors_cache))

        factors_cache[tri(curr)] = prime_factors

        ndivisors = n_divisors(prime_factors)

    return tri(curr)

def p12():
    return first_triangle_ndivisors_gt(500)


import sys
from abc import abstractmethod, ABCMeta

from pyspark import since, keyword_only
from pyspark.ml.wrapper import JavaParams
from pyspark.ml.param import Param, Params, TypeConverters
from pyspark.ml.param.shared import HasLabelCol, HasPredictionCol, HasRawPredictionCol, \
    HasFeaturesCol
from pyspark.ml.common import inherit_doc
from pyspark.ml.util import JavaMLReadable, JavaMLWritable

__all__ = ['Evaluator', 'BinaryClassificationEvaluator', 'RegressionEvaluator',
           'MulticlassClassificationEvaluator', 'ClusteringEvaluator']

@inherit_doc
class Evaluator(Params):
    """
    Base class for evaluators that compute metrics from predictions.

    .. versionadded:: 1.4.0
    """

    __metaclass__ = ABCMeta

    @abstractmethod
    def _evaluate(self, dataset):
        """
        Evaluates the output.

        :param dataset: a dataset that contains labels/observations and
               predictions
        :return: metric
        """
        raise NotImplementedError()

    @since("1.4.0")
    def evaluate(self, dataset, params=None):
        """
        Evaluates the output with optional parameters.

        :param dataset: a dataset that contains labels/observations and
                        predictions
        :param params: an optional param map that overrides embedded
                       params
        :return: metric
        """
        if params is None:
            params = dict()
        if isinstance(params, dict):
            if params:
                return self.copy(params)._evaluate(dataset)
            else:
                return self._evaluate(dataset)
        else:
            raise ValueError("Params must be a param map but got %s." % type(params))

    @since("1.5.0")
    def isLargerBetter(self):
        """
        Indicates whether the metric returned by :py:meth:`evaluate` should be maximized
        (True, default) or minimized (False).
        A given evaluator may support multiple metrics which may be maximized or minimized.
        """
        return True

@inherit_doc
class JavaEvaluator(JavaParams, Evaluator):
    """
    Base class for :py:class:`Evaluator`s that wrap Java/Scala
    implementations.
    """

    __metaclass__ = ABCMeta

    def _evaluate(self, dataset):
        """
        Evaluates the output.
        :param dataset: a dataset that contains labels/observations and predictions.
        :return: evaluation metric
        """
        self._transfer_params_to_java()
        return self._java_obj.evaluate(dataset._jdf)

    def isLargerBetter(self):
        self._transfer_params_to_java()
        return self._java_obj.isLargerBetter()

@inherit_doc
class BinaryClassificationEvaluator(JavaEvaluator, HasLabelCol, HasRawPredictionCol,
                                    JavaMLReadable, JavaMLWritable):
    """
    .. note:: Experimental

    Evaluator for binary classification, which expects two input columns: rawPrediction and label.
    The rawPrediction column can be of type double (binary 0/1 prediction, or probability of label
    1) or of type vector (length-2 vector of raw predictions, scores, or label probabilities).

    >>> from pyspark.ml.linalg import Vectors
    >>> scoreAndLabels = map(lambda x: (Vectors.dense([1.0 - x[0], x[0]]), x[1]),
    ...    [(0.1, 0.0), (0.1, 1.0), (0.4, 0.0), (0.6, 0.0), (0.6, 1.0), (0.6, 1.0), (0.8, 1.0)])
    >>> dataset = spark.createDataFrame(scoreAndLabels, ["raw", "label"])
    ...
    >>> evaluator = BinaryClassificationEvaluator(rawPredictionCol="raw")
    >>> evaluator.evaluate(dataset)
    0.70...
    >>> evaluator.evaluate(dataset, {evaluator.metricName: "areaUnderPR"})
    0.83...
    >>> bce_path = temp_path + "/bce"
    >>> evaluator.save(bce_path)
    >>> evaluator2 = BinaryClassificationEvaluator.load(bce_path)
    >>> str(evaluator2.getRawPredictionCol())
    'raw'

    .. versionadded:: 1.4.0
    """

    metricName = Param(Params._dummy(), "metricName",
                       "metric name in evaluation (areaUnderROC|areaUnderPR)",
                       typeConverter=TypeConverters.toString)

    @keyword_only
    def __init__(self, rawPredictionCol="rawPrediction", labelCol="label",
                 metricName="areaUnderROC"):
        """
        __init__(self, rawPredictionCol="rawPrediction", labelCol="label", \
                 metricName="areaUnderROC")
        """
        super(BinaryClassificationEvaluator, self).__init__()
        self._java_obj = self._new_java_obj(
            "org.apache.spark.ml.evaluation.BinaryClassificationEvaluator", self.uid)
        self._setDefault(metricName="areaUnderROC")
        kwargs = self._input_kwargs
        self._set(**kwargs)

    @since("1.4.0")
    def setMetricName(self, value):
        """
        Sets the value of :py:attr:`metricName`.
        """
        return self._set(metricName=value)

    @since("1.4.0")
    def getMetricName(self):
        """
        Gets the value of metricName or its default value.
        """
        return self.getOrDefault(self.metricName)

    @keyword_only
    @since("1.4.0")
    def setParams(self, rawPredictionCol="rawPrediction", labelCol="label",
                  metricName="areaUnderROC"):
        """
        setParams(self, rawPredictionCol="rawPrediction", labelCol="label", \
                  metricName="areaUnderROC")
        Sets params for binary classification evaluator.
        """
        kwargs = self._input_kwargs
        return self._set(**kwargs)

@inherit_doc
class RegressionEvaluator(JavaEvaluator, HasLabelCol, HasPredictionCol,
                          JavaMLReadable, JavaMLWritable):
    """
    .. note:: Experimental

    Evaluator for Regression, which expects two input
    columns: prediction and label.

    >>> scoreAndLabels = [(-28.98343821, -27.0), (20.21491975, 21.5),
    ...   (-25.98418959, -22.0), (30.69731842, 33.0), (74.69283752, 71.0)]
    >>> dataset = spark.createDataFrame(scoreAndLabels, ["raw", "label"])
    ...
    >>> evaluator = RegressionEvaluator(predictionCol="raw")
    >>> evaluator.evaluate(dataset)
    2.842...
    >>> evaluator.evaluate(dataset, {evaluator.metricName: "r2"})
    0.993...
    >>> evaluator.evaluate(dataset, {evaluator.metricName: "mae"})
    2.649...
    >>> re_path = temp_path + "/re"
    >>> evaluator.save(re_path)
    >>> evaluator2 = RegressionEvaluator.load(re_path)
    >>> str(evaluator2.getPredictionCol())
    'raw'

    .. versionadded:: 1.4.0
    """
    metricName = Param(Params._dummy(), "metricName",
                       """metric name in evaluation - one of:
                       rmse - root mean squared error (default)
                       mse - mean squared error
                       r2 - r^2 metric
                       mae - mean absolute error.""",
                       typeConverter=TypeConverters.toString)

    @keyword_only
    def __init__(self, predictionCol="prediction", labelCol="label",
                 metricName="rmse"):
        """
        __init__(self, predictionCol="prediction", labelCol="label", \
                 metricName="rmse")
        """
        super(RegressionEvaluator, self).__init__()
        self._java_obj = self._new_java_obj(
            "org.apache.spark.ml.evaluation.RegressionEvaluator", self.uid)
        self._setDefault(metricName="rmse")
        kwargs = self._input_kwargs
        self._set(**kwargs)

    @since("1.4.0")
    def setMetricName(self, value):
        """
        Sets the value of :py:attr:`metricName`.
        """
        return self._set(metricName=value)

    @since("1.4.0")
    def getMetricName(self):
        """
        Gets the value of metricName or its default value.
        """
        return self.getOrDefault(self.metricName)

    @keyword_only
    @since("1.4.0")
    def setParams(self, predictionCol="prediction", labelCol="label",
                  metricName="rmse"):
        """
        setParams(self, predictionCol="prediction", labelCol="label", \
                  metricName="rmse")
        Sets params for regression evaluator.
        """
        kwargs = self._input_kwargs
        return self._set(**kwargs)

@inherit_doc
class MulticlassClassificationEvaluator(JavaEvaluator, HasLabelCol, HasPredictionCol,
                                        JavaMLReadable, JavaMLWritable):
    """
    .. note:: Experimental

    Evaluator for Multiclass Classification, which expects two input
    columns: prediction and label.

    >>> scoreAndLabels = [(0.0, 0.0), (0.0, 1.0), (0.0, 0.0),
    ...     (1.0, 0.0), (1.0, 1.0), (1.0, 1.0), (1.0, 1.0), (2.0, 2.0), (2.0, 0.0)]
    >>> dataset = spark.createDataFrame(scoreAndLabels, ["prediction", "label"])
    ...
    >>> evaluator = MulticlassClassificationEvaluator(predictionCol="prediction")
    >>> evaluator.evaluate(dataset)
    0.66...
    >>> evaluator.evaluate(dataset, {evaluator.metricName: "accuracy"})
    0.66...
    >>> mce_path = temp_path + "/mce"
    >>> evaluator.save(mce_path)
    >>> evaluator2 = MulticlassClassificationEvaluator.load(mce_path)
    >>> str(evaluator2.getPredictionCol())
    'prediction'

    .. versionadded:: 1.5.0
    """
    metricName = Param(Params._dummy(), "metricName",
                       "metric name in evaluation "
                       "(f1|weightedPrecision|weightedRecall|accuracy)",
                       typeConverter=TypeConverters.toString)

    @keyword_only
    def __init__(self, predictionCol="prediction", labelCol="label",
                 metricName="f1"):
        """
        __init__(self, predictionCol="prediction", labelCol="label", \
                 metricName="f1")
        """
        super(MulticlassClassificationEvaluator, self).__init__()
        self._java_obj = self._new_java_obj(
            "org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator", self.uid)
        self._setDefault(metricName="f1")
        kwargs = self._input_kwargs
        self._set(**kwargs)

    @since("1.5.0")
    def setMetricName(self, value):
        """
        Sets the value of :py:attr:`metricName`.
        """
        return self._set(metricName=value)

    @since("1.5.0")
    def getMetricName(self):
        """
        Gets the value of metricName or its default value.
        """
        return self.getOrDefault(self.metricName)

    @keyword_only
    @since("1.5.0")
    def setParams(self, predictionCol="prediction", labelCol="label",
                  metricName="f1"):
        """
        setParams(self, predictionCol="prediction", labelCol="label", \
                  metricName="f1")
        Sets params for multiclass classification evaluator.
        """
        kwargs = self._input_kwargs
        return self._set(**kwargs)

@inherit_doc
class ClusteringEvaluator(JavaEvaluator, HasPredictionCol, HasFeaturesCol,
                          JavaMLReadable, JavaMLWritable):
    """
    .. note:: Experimental

    Evaluator for Clustering results, which expects two input
    columns: prediction and features. The metric computes the Silhouette
    measure using the squared Euclidean distance.

    The Silhouette is a measure for the validation of the consistency
    within clusters. It ranges between 1 and -1, where a value close to
    1 means that the points in a cluster are close to the other points
    in the same cluster and far from the points of the other clusters.

    >>> from pyspark.ml.linalg import Vectors
    >>> featureAndPredictions = map(lambda x: (Vectors.dense(x[0]), x[1]),
    ...     [([0.0, 0.5], 0.0), ([0.5, 0.0], 0.0), ([10.0, 11.0], 1.0),
    ...     ([10.5, 11.5], 1.0), ([1.0, 1.0], 0.0), ([8.0, 6.0], 1.0)])
    >>> dataset = spark.createDataFrame(featureAndPredictions, ["features", "prediction"])
    ...
    >>> evaluator = ClusteringEvaluator(predictionCol="prediction")
    >>> evaluator.evaluate(dataset)
    0.9079...
    >>> ce_path = temp_path + "/ce"
    >>> evaluator.save(ce_path)
    >>> evaluator2 = ClusteringEvaluator.load(ce_path)
    >>> str(evaluator2.getPredictionCol())
    'prediction'

    .. versionadded:: 2.3.0
    """
    metricName = Param(Params._dummy(), "metricName",
                       "metric name in evaluation (silhouette)",
                       typeConverter=TypeConverters.toString)
    distanceMeasure = Param(Params._dummy(), "distanceMeasure", "The distance measure. " +
                            "Supported options: 'squaredEuclidean' and 'cosine'.",
                            typeConverter=TypeConverters.toString)

    @keyword_only
    def __init__(self, predictionCol="prediction", featuresCol="features",
                 metricName="silhouette", distanceMeasure="squaredEuclidean"):
        """
        __init__(self, predictionCol="prediction", featuresCol="features", \
                 metricName="silhouette", distanceMeasure="squaredEuclidean")
        """
        super(ClusteringEvaluator, self).__init__()
        self._java_obj = self._new_java_obj(
            "org.apache.spark.ml.evaluation.ClusteringEvaluator", self.uid)
        self._setDefault(metricName="silhouette", distanceMeasure="squaredEuclidean")
        kwargs = self._input_kwargs
        self._set(**kwargs)

    @since("2.3.0")
    def setMetricName(self, value):
        """
        Sets the value of :py:attr:`metricName`.
        """
        return self._set(metricName=value)

    @since("2.3.0")
    def getMetricName(self):
        """
        Gets the value of metricName or its default value.
        """
        return self.getOrDefault(self.metricName)

    @keyword_only
    @since("2.3.0")
    def setParams(self, predictionCol="prediction", featuresCol="features",
                  metricName="silhouette", distanceMeasure="squaredEuclidean"):
        """
        setParams(self, predictionCol="prediction", featuresCol="features", \
                  metricName="silhouette", distanceMeasure="squaredEuclidean")
        Sets params for clustering evaluator.
        """
        kwargs = self._input_kwargs
        return self._set(**kwargs)

    @since("2.4.0")
    def setDistanceMeasure(self, value):
        """
        Sets the value of :py:attr:`distanceMeasure`.
        """
        return self._set(distanceMeasure=value)

    @since("2.4.0")
    def getDistanceMeasure(self):
        """
        Gets the value of `distanceMeasure`
        """
        return self.getOrDefault(self.distanceMeasure)

if __name__ == "__main__":
    import doctest
    import tempfile
    import pyspark.ml.evaluation
    from pyspark.sql import SparkSession
    globs = pyspark.ml.evaluation.__dict__.copy()
    # The small batch size here ensures that we see multiple batches,
    # even in these small test examples:
    spark = SparkSession.builder\
        .master("local[2]")\
        .appName("ml.evaluation tests")\
        .getOrCreate()
    globs['spark'] = spark
    temp_path = tempfile.mkdtemp()
    globs['temp_path'] = temp_path
    try:
        (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
        spark.stop()
    finally:
        from shutil import rmtree
        try:
            rmtree(temp_path)
        except OSError:
            pass
    if failure_count:
        sys.exit(-1)


import os
from discord.ext import commands
import discord
import logging
import requests
from WeatherBot import Weather
from ToolsBot import Tools
from COVIDBot import COVID
from ReadBot import Read
from voice_generator import creat_WAV

BOT_PREFIX = ('!!')

class JapaneseHelpCommand(commands.DefaultHelpCommand):
    def __init__(self):
        super().__init__()
        self.commands_heading = "コマンド:"
        self.no_category = "その他"
        self.command_attrs["help"] = "コマンド一覧と簡単な説明を表示"

    def get_ending_note(self):
        return (f"各コマンドの説明: {BOT_PREFIX}help <コマンド名>\n"
                f"各カテゴリの説明: {BOT_PREFIX}help <カテゴリ名>\n")

client = commands.Bot(command_prefix=BOT_PREFIX, help_command=JapaneseHelpCommand())
voice_client = None

activity = discord.Activity(name='メインモード', type=discord.ActivityType.playing)

formatter = '%(asctime)s:%(levelname)s:%(name)s: %(message)s'

logging.basicConfig(filename='/var/log/Maindiscord.log', level=logging.DEBUG, format=formatter)
logger = logging.getLogger('discord')

@client.event
async def on_message(message):
    global channelid

    #Botとメッセージの送信者が同じ場合は何もしない
    if client.user == message.author:
        return
    if message.content.startswith("こんにちは"):
        m = "こんにちは！" + message.author.name + "さん！\n"
        await message.channel.send(m)
    if message.content.startswith("にゃーん"):
        msg = "にゃ～ん" + message.author.name + "🐈\n"
        await message.channel.send(msg)

    if message.content == "!!join":
        channelid = message.channel.id
        print(channelid)

    if message.content.startswith(BOT_PREFIX):
        pass

    else:
        if message.guild.voice_client:
            if message.channel.id == channelid:
                print(message.content)
                creat_WAV(message.content)
                source = discord.FFmpegPCMAudio("output.wav")
                message.guild.voice_client.play(source)
            else:
                pass
    await client.process_commands(message)

@client.event
async def on_command_error(ctx, error):
    if isinstance(error, discord.ext.commands.errors.MissingPermissions):
        embed = discord.Embed(title=":x: 失敗 -MissingPermissions", description=f"実行者の必要な権限が無いため実行出来ません。", timestamp=ctx.message.created_at, color=discord.Colour.red())
        embed.set_footer(text="'!!help'でヘルプコマンドを参照してください")
        await ctx.send(embed=embed)
    elif isinstance(error, discord.ext.commands.errors.BotMissingPermissions):
        embed = discord.Embed(title=":x: 失敗 -BotMissingPermissions", description=f"Botの必要な権限が無いため実行出来ません。", timestamp=ctx.message.created_at, color=discord.Colour.red())
        embed.set_footer(text="'!!help'でヘルプコマンドを参照してください")
        await ctx.send(embed=embed)
    elif isinstance(error, discord.ext.commands.errors.CommandNotFound):
        embed = discord.Embed(title=":x: 失敗 -CommandNotFound", description=f"不明なコマンドもしくは現在使用不可能なコマンドです。", timestamp=ctx.message.created_at, color=discord.Colour.red())
        embed.set_footer(text="'!!help'でヘルプコマンドを参照してください")
        await ctx.send(embed=embed)
    elif isinstance(error, discord.ext.commands.errors.MemberNotFound):
        embed = discord.Embed(title=":x: 失敗 -MemberNotFound", description=f"指定されたメンバーが見つかりません。", timestamp=ctx.message.created_at, color=discord.Colour.red())
        embed.set_footer(text="'!!help'でヘルプコマンドを参照してください")
        await ctx.send(embed=embed)
    elif isinstance(error, discord.ext.commands.errors.BadArgument):
        embed = discord.Embed(title=":x: 失敗 -BadArgument", description=f"指定された引数がエラーを起こしているため実行出来ません。", timestamp=ctx.message.created_at, color=discord.Colour.red())
        embed.set_footer(text="'!!help'でヘルプコマンドを参照してください")
        await ctx.send(embed=embed)
    elif isinstance(error, discord.ext.commands.errors.MissingRequiredArgument):
        embed = discord.Embed(title=":x: 失敗 -BadArgument", description=f"指定された引数が足りないため実行出来ません。", timestamp=ctx.message.created_at, color=discord.Colour.red())
        embed.set_footer(text="'!!help'でヘルプコマンドを参照してください")
        await ctx.send(embed=embed)
    else:
        raise error

    logger.error(error)

@client.event
async def on_ready():
    logger.info('ユーザー名：' + client.user.name)
    logger.info('ユーザーid：' + str(client.user.id))
    await client.change_presence(activity=activity)

client.add_cog(Weather(bot=client))
client.add_cog(Tools(bot=client))
client.add_cog(COVID(bot=client))
client.add_cog(Read(bot=client))
client.run(os.environ.get("DISCORD_TOKEN"))


from pandas import HDFStore, DataFrame, read_hdf, read_parquet
from pathlib import Path
import yaml

def load_config(path: Path) -> dict:
    """Load the configuration file"""
    return yaml.load(open(path, "r"), Loader=yaml.Loader)

# def open_hdf(path: Path):
#     return HDFStore(path.absolute(), mode="r")

def get_candle_data(path: Path, symbol: str, period: str, candle: str) -> DataFrame:
    if not path:
        return None

    # ohlc_key = f"/OHLCV/{period}/"
    # print("OHLC_key", ohlc_key)
    ohlc_df = read_parquet(path=path / f"OHLCV/{period}/data.parquet").query(f"symbol == '{symbol}'")
    if candle == "ohlc":
        return ohlc_df.sort_values("Date", ascending=False).reset_index(drop=True)  # .drop("Index", axis=1)

    if candle == "ha":
        # ha_key = f"/Signals/{period}/heikin_ashi/"
        ha_df = read_parquet(path=path / f"signals/{period}/heikin_ashi.parquet").query(f"symbol == '{symbol}'")
        # print("ha_df", ha_df)
        # print("ohlc_df", ohlc_df)

        return (
            ha_df.drop("Volume", axis=1)
            .merge(ohlc_df, how="inner", on=["Date", "symbol"])
            .loc[:, ["Date", "symbol", "HA_Open", "HA_High", "HA_Low", "HA_Close", "Volume"]]
            .rename(
                columns={
                    "HA_Open": "Open",
                    "HA_High": "High",
                    "HA_Low": "Low",
                    "HA_Close": "Close",
                }
            )
            .sort_values("Date", ascending=False)
            .reset_index(drop=True)
        )

def load_screener_data(path: Path, period: str, lookback: list[int] = [0, 1]) -> DataFrame:
    if not path:
        return None

    # screener_key = f"Signals/{period}/merged"
    # print("lookback", lookback)
    return (
        read_parquet(path=path / f"signals/{period}/merged.parquet")
        .sort_values("Date", ascending=True)
        .groupby("symbol")
        .tail(lookback[1])
        .groupby("symbol")
        .head(lookback[1] - lookback[0])
        .drop("index", axis=1)
    )

def get_symbol_info(path: Path, symbol: str) -> str:
    if not path or not symbol:
        return ""

    row = read_parquet(path=path / "info.parquet").query(f'symbol == "{symbol}"').iloc[0]
    return "\n".join([f"{k}: {v}" for k, v in row.to_dict().items()])