# -*- coding: utf-8 -*-
"""
Created on Thursday May 26 11:23:00 2016

@author:  Rama Vasudevan, Suhas Somnath, Chris Smith
"""

from __future__ import division, print_function, absolute_import, unicode_literals

from os import path, remove  # File Path formatting
from warnings import warn

import numpy as np  # For array operations
from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat file
import h5py

from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, generatePlotGroups, createSpecVals, maxReadPixels, \
    nf32
from pyUSID.io.translator import Translator
from pyUSID.io.write_utils import INDICES_DTYPE, Dimension
from pyUSID.io.hdf_utils import create_indexed_group, write_main_dataset, write_simple_attrs

class BEodfRelaxationTranslator(Translator):
    """
    Translates old Relaxation data into the new H5 format. This is for the files generated from
    the old BEPSDAQ program utilizing two cards simultaneously.
    At present, this version of the translator only works for Out of field measurements
    It will not work for in-field. This should be fixed at a later date.

    """
    def __init__(self, max_mem_mb=1024):
        super(BEodfRelaxationTranslator, self).__init__(max_mem_mb)
        self.FFT_BE_wave = None
        self.h5_file = None
        self.ds_main = None
        self.mean_resp = None
        self.max_resp = None
        self.min_resp = None

    def translate(self, file_path, show_plots=True, save_plots=True, do_histogram=False):
        """
        Basic method that translates .dat data file(s) to a single .h5 file

        Inputs:
            file_path -- Absolute file path for one of the data files.
            It is assumed that this file is of the OLD data format.

        Outputs:
            Nothing
        """
        file_path = path.abspath(file_path)
        (folder_path, basename) = path.split(file_path)
        (basename, path_dict) = self._parse_file_path(file_path)

        h5_path = path.join(folder_path, basename + '.h5')
        if path.exists(h5_path):
            remove(h5_path)
        self.h5_file = h5py.File(h5_path, 'w')

        isBEPS = True
        parm_dict = self.__getParmsFromOldMat(path_dict['old_mat_parms'])

        ignored_plt_grps = ['in-field']  # Here we assume that there is no in-field.
        # If in-field data is captured then the translator would have to be modified.

        # Technically, we could do away with this if statement, as isBEPS is always true for this translation
        if isBEPS:
            parm_dict['data_type'] = 'BEPSData'

            std_expt = parm_dict['VS_mode'] != 'load user defined VS Wave from file'

            if not std_expt:
                warn('This translator does not handle user defined voltage spectroscopy')
                return

            spec_label = getSpectroscopicParmLabel(parm_dict['VS_mode'])

            # Check file sizes:
        if 'read_real' in path_dict.keys():
            real_size = path.getsize(path_dict['read_real'])
            imag_size = path.getsize(path_dict['read_imag'])
        else:
            real_size = path.getsize(path_dict['write_real'])
            imag_size = path.getsize(path_dict['write_imag'])

        if real_size != imag_size:
            raise ValueError("Real and imaginary file sizes DON'T match!. Ending")

        num_rows = int(parm_dict['grid_num_rows'])
        num_cols = int(parm_dict['grid_num_cols'])
        num_pix = num_rows * num_cols
        tot_bins = real_size / (num_pix * 4)  # Finding bins by simple division of entire datasize

        # Check for case where only a single pixel is missing.
        check_bins = real_size / ((num_pix - 1) * 4)

        if tot_bins % 1 and check_bins % 1:
            warn('Aborting! Some parameter appears to have changed in-between')
            return
        elif not tot_bins % 1:
            #             Everything's ok
            pass
        elif not check_bins % 1:
            tot_bins = check_bins
            warn('Warning:  A pixel seems to be missing from the data.  File will be padded with zeros.')

        tot_bins = int(tot_bins)
        (bin_inds, bin_freqs, bin_FFT, ex_wfm, dc_amp_vec) = self.__readOldMatBEvecs(path_dict['old_mat_parms'])
        """
        Because this is the old data format and there is a discrepancy in the number of bins (they seem to be 2 less
        than the actual number), we need to re-calculate it based on the available data. This is done below.
        """

        band_width = parm_dict['BE_band_width_[Hz]'] * (0.5 - parm_dict['BE_band_edge_trim'])
        st_f = parm_dict['BE_center_frequency_[Hz]'] - band_width
        en_f = parm_dict['BE_center_frequency_[Hz]'] + band_width
        bin_freqs = np.linspace(st_f, en_f, len(bin_inds), dtype=np.float32)

        # Forcing standardized datatypes:
        bin_inds = np.int32(bin_inds)
        bin_freqs = np.float32(bin_freqs)
        bin_FFT = np.complex64(bin_FFT)
        ex_wfm = np.float32(ex_wfm)

        self.FFT_BE_wave = bin_FFT

        (UDVS_labs, UDVS_units, UDVS_mat) = self.__buildUDVSTable(parm_dict)

        # Remove the unused plot group columns before proceeding:
        (UDVS_mat, UDVS_labs, UDVS_units) = trimUDVS(UDVS_mat, UDVS_labs, UDVS_units, ignored_plt_grps)

        spec_inds = np.zeros(shape=(2, tot_bins), dtype=INDICES_DTYPE)

        # Will assume that all excitation waveforms have same number of bins
        # Here, the denominator is 2 because only out of field measruements. For IF + OF, should be 1
        num_actual_udvs_steps = UDVS_mat.shape[0] / 2
        bins_per_step = tot_bins / num_actual_udvs_steps

        # Some more checks
        if bins_per_step % 1:
            warn('Non integer number of bins per step!')
            return
        else:
            bins_per_step = int(bins_per_step)

        num_actual_udvs_steps = int(num_actual_udvs_steps)

        stind = 0
        for step_index in range(UDVS_mat.shape[0]):
            if UDVS_mat[step_index, 2] < 1E-3:  # invalid AC amplitude
                continue  # skip
            spec_inds[0, stind:stind + bins_per_step] = np.arange(bins_per_step, dtype=INDICES_DTYPE)  # Bin step
            spec_inds[1, stind:stind + bins_per_step] = step_index * np.ones(bins_per_step,
                                                                             dtype=INDICES_DTYPE)  # UDVS step
            stind += bins_per_step
        del stind, step_index

        # Some very basic information that can help the processing / analysis crew
        parm_dict['num_bins'] = tot_bins
        parm_dict['num_pix'] = num_pix
        parm_dict['num_udvs_steps'] = num_actual_udvs_steps

        global_parms = dict()
        global_parms['grid_size_x'] = parm_dict['grid_num_cols']
        global_parms['grid_size_y'] = parm_dict['grid_num_rows']
        global_parms['experiment_date'] = parm_dict['File_date_and_time']

        # assuming that the experiment was completed:
        global_parms['current_position_x'] = parm_dict['grid_num_cols'] - 1
        global_parms['current_position_y'] = parm_dict['grid_num_rows'] - 1
        global_parms['data_type'] = parm_dict['data_type']  # self.__class__.__name__
        global_parms['translator'] = 'ODF'
        write_simple_attrs(self.h5_file, global_parms)

        # Create Measurement and Channel groups
        meas_grp = create_indexed_group(self.h5_file, 'Measurement')
        write_simple_attrs(meas_grp, parm_dict)

        chan_grp = create_indexed_group(meas_grp, 'Channel')
        chan_grp.attrs['Channel_Input'] = parm_dict['IO_Analog_Input_1']

        # Create Auxilliary Datasets
        h5_ex_wfm = chan_grp.create_dataset('Excitation_Waveform', data=ex_wfm)

        udvs_slices = dict()
        for col_ind, col_name in enumerate(UDVS_labs):
            udvs_slices[col_name] = (slice(None), slice(col_ind, col_ind + 1))
        h5_UDVS = chan_grp.create_dataset('UDVS',
                                          data=UDVS_mat,
                                          dtype=np.float32)
        write_simple_attrs(h5_UDVS, {'labels': UDVS_labs, 'units': UDVS_units})

        h5_bin_steps = chan_grp.create_dataset('Bin_Steps',
                                               data=np.arange(bins_per_step, dtype=np.uint32),
                                               dtype=np.uint32)

        # Need to add the Bin Waveform type - infer from UDVS
        exec_bin_vec = self.signal_type * np.ones(len(bin_inds), dtype=np.int32)
        h5_wfm_typ = chan_grp.create_dataset('Bin_Wfm_Type',
                                             data=exec_bin_vec,
                                             dtype=np.int32)

        h5_bin_inds = chan_grp.create_dataset('Bin_Indices',
                                              data=bin_inds,
                                              dtype=np.uint32)
        h5_bin_freq = chan_grp.create_dataset('Bin_Frequencies',
                                              data=bin_freqs,
                                              dtype=np.float32)
        h5_bin_FFT = chan_grp.create_dataset('Bin_FFT',
                                             data=bin_FFT,
                                             dtype=np.complex64)
        # Noise floor should be of shape: (udvs_steps x 3 x positions)
        h5_noise_floor = chan_grp.create_dataset('Noise_Floor',
                                                 shape=(num_pix, num_actual_udvs_steps),
                                                 dtype=nf32,
                                                 chunks=(1, num_actual_udvs_steps))

        """
        ONLY ALLOCATING SPACE FOR MAIN DATA HERE!
        Chunk by each UDVS step - this makes it easy / quick to:
            1. read data for a single UDVS step from all pixels
            2. read an entire / multiple pixels at a time
        The only problem is that a typical UDVS step containing 50 steps occupies only 400 bytes.
        This is smaller than the recommended chunk sizes of 10,000 - 999,999 bytes
        meaning that the metadata would be very substantial.
        This assumption is fine since we almost do not handle any user defined cases
        """

        """
        New Method for chunking the Main_Data dataset.  Chunking is now done in N-by-N squares of UDVS steps by pixels.
        N is determined dinamically based on the dimensions of the dataset.  Currently it is set such that individual
        chunks are less than 10kB in size.

        Chris Smith -- csmith55@utk.edu
        """
        pos_dims = [Dimension('X', 'nm', num_cols), Dimension('Y', 'nm', num_rows)]

        # Create Spectroscopic Values and Spectroscopic Values Labels datasets
        spec_vals, spec_inds, spec_vals_labs, spec_vals_units, spec_vals_names = createSpecVals(UDVS_mat, spec_inds,
                                                                                bin_freqs,
                                                                                exec_bin_vec,
                                                                                parm_dict, UDVS_labs,
                                                                                UDVS_units)

        spec_dims = list()
        for row_ind, row_name in enumerate(spec_vals_labs):
            spec_dims.append(Dimension(row_name,
                                            spec_vals_units[row_ind],
                                            spec_vals[row_ind]))

        pixel_chunking = maxReadPixels(10240, num_pix * num_actual_udvs_steps,
                                       bins_per_step, np.dtype('complex64').itemsize)
        chunking = np.floor(np.sqrt(pixel_chunking))
        chunking = max(1, chunking)
        chunking = min(num_actual_udvs_steps, num_pix, chunking)
        self.h5_main = write_main_dataset(chan_grp, (num_pix, tot_bins), 'Raw_Data',
                                          'Piezoresponse', 'V',
                                          pos_dims, spec_dims,
                                          dtype=np.complex64,
                                          chunks=(chunking, chunking * bins_per_step),
                                          compression='gzip')

        self.mean_resp = np.zeros(shape=(self.ds_main.shape[1]), dtype=np.complex64)
        self.max_resp = np.zeros(shape=(self.ds_main.shape[0]), dtype=np.float32)
        self.min_resp = np.zeros(shape=(self.ds_main.shape[0]), dtype=np.float32)

        # Now read the raw data files:
        self._read_data(path_dict['read_real'], path_dict['read_imag'], parm_dict)
        self.h5_file.flush()

        generatePlotGroups(self.ds_main, self.mean_resp, folder_path, basename, self.max_resp,
                           self.min_resp, max_mem_mb=self.max_ram, spec_label=spec_label, show_plots=show_plots,
                           save_plots=save_plots, do_histogram=do_histogram)

        self.h5_file.close()

        return h5_path

    def _read_data(self, real_path, imag_path, parm_dict):
        """
        Reads the imaginary and real data files one pixel at a time andwrites to the H5 dataset.

        Inputs:
            real_path -- file path of the .dat file containing the real component of the data
            imag_path -- file path of the .dat file containing the imaginary component of the data
            parm_dict--dictionary of parameters for the experiment

        Outputs: None
        """
        print('---- reading data one pixel at a time----------')

        num_pix = int(parm_dict['grid_num_rows']) * int(parm_dict['grid_num_cols'])
        # print 'Number of rows is: ', parm_dict['grid_num_rows']
        # print 'Number of cols is: ', parm_dict['grid_num_cols']
        # print 'Rows * cols is:', int(parm_dict['grid_num_rows'])*int(parm_dict['grid_num_cols'])

        if path.getsize(real_path) != path.getsize(imag_path):
            print('Sizes of real and imaginary files NOT matching!!!!')
        if 1.0 * path.getsize(real_path) % num_pix != 0:
            print('Incomplete dataset!!!')

        bytes_per_pix = path.getsize(real_path) / num_pix
        f_real = open(real_path, "rb")
        f_imag = open(imag_path, "rb")

        for pix_ind in range(num_pix):
            print('Reading pixel #{}, file position {}'.format(pix_ind, hex(pix_ind * bytes_per_pix)))
            pix_vec = np.fromstring(f_real.read(int(bytes_per_pix)), dtype='f') + \
                1j * np.fromstring(f_imag.read(int(bytes_per_pix)), dtype='f')

            # Make chronologically correct
            pix_mat = np.reshape(pix_vec, (parm_dict['BE_bins_per_read'],
                                           parm_dict['VS_steps_per_full_cycle'], parm_dict['BE_repeats']))
            pix_mat_temp = np.transpose(pix_mat, (1, 2, 0))
            pix_vec2 = np.reshape(pix_mat_temp, -1)

            # Calculate the mean, min, max
            self.max_resp[pix_ind] = np.max(np.abs(pix_vec2))
            self.min_resp[pix_ind] = np.min(np.abs(pix_vec2))
            self.mean_resp = (1 / (pix_ind + 1)) * (pix_vec2 + pix_ind * self.mean_resp)

            # Write to file now
            self.ds_main[pix_ind, :] = np.complex64(pix_vec2)
            self.h5_file.flush()

        f_real.close()
        f_imag.close()

        print('Finished writing data to .h5')

    @staticmethod
    def __readOldMatBEvecs(file_path):
        """
    Returns information about the excitation BE waveform present in the .mat file

    Inputs:
        filepath -- Absolute filepath of the .mat parameter file

    Outputs:
        Tuple -- (bin_inds, bin_w, bin_FFT, BE_wave, dc_amp_vec_full)\n
        bin_inds -- Bin indices\n
        bin_w -- Excitation bin Frequencies\n
        bin_FFT -- FFT of the BE waveform for the excited bins\n
        BE_wave -- Band Excitation waveform\n
        dc_amp_vec_full -- spectroscopic waveform.
        This information will be necessary for fixing the UDVS for AC modulation for example
        """

        matread = loadmat(file_path, squeeze_me=True)
        BE_wave = matread['BE_wave_1']
        bin_inds = matread['bin_ind_s'] - 1  # Python base 0. note also _s, for this case
        bin_w = matread['bin_w']
        dc_amp_vec_full = matread['dc_amp_vec_full']
        FFT_full = np.fft.fftshift(np.fft.fft(BE_wave))
        bin_FFT = np.conjugate(FFT_full[bin_inds])

        return bin_inds, bin_w, bin_FFT, BE_wave, dc_amp_vec_full

    def _parse_file_path(self, data_filepath):
        """
        Returns the basename and a dictionary containing the absolute file paths for the
        real and imaginary data files, text and mat parameter files in a dictionary

        Parameters
        ----------
        data_filepath : str
            Absolute path of the real / imaginary data file (.dat)

        Returns
        -------
        basename : str
        path_dict : dict

        """
        (folder_path, basename) = path.split(data_filepath)
        (super_folder, basename) = path.split(folder_path)

        if basename.endswith('_c'):
            # Old old data format where the folder ended with a _d for some reason
            base_name = basename[:-2]

        """
        A single pair of real and imaginary files are / were generated for:
            BE-Line and BEPS (compiled version only generated out-of-field or 'read')
        Two pairs of real and imaginary files were generated for later BEPS datasets
            These have 'read' and 'write' prefixes to denote out or in field respectively
        """
        path_dict = dict()

        real_path = path.join(folder_path, base_name + '_sub_real.dat')
        imag_path = path.join(folder_path, base_name + '_sub_imag.dat')

        path_dict['read_real'] = real_path
        path_dict['read_imag'] = imag_path
        path_dict['old_mat_parms'] = data_filepath

        return basename, path_dict

    @staticmethod
    def __getParmsFromOldMat(file_path):
        """
        Formats parameters found in the old parameters .mat file into a dictionary
        as though the dataset had a parms.txt describing it

        Inputs:
            file_path -- absolute filepath of the .mat file containing the parameters

        Outputs -- dictionary containing parameters
        """
        parm_dict = dict()
        matread = loadmat(file_path, squeeze_me=True)

        parm_dict['IO_rate'] = str(int(matread['AO_rate'] / 1E+6)) + ' MHz'

        position_vec = matread['position_vec']
        parm_dict['grid_current_row'] = position_vec[0]
        parm_dict['grid_current_col'] = position_vec[1]
        parm_dict['grid_num_rows'] = int(position_vec[2])
        parm_dict['grid_num_cols'] = int(position_vec[3])

        if position_vec[0] != position_vec[1] or position_vec[2] != position_vec[3]:
            warn('WARNING: Incomplete dataset. Translation not guaranteed!')
            parm_dict['grid_num_rows'] = int(position_vec[0])  # set to number of present cols and rows
            parm_dict['grid_num_cols'] = int(position_vec[1])

        BE_parm_vec_1 = matread['BE_parm_vec_1']
        # Not required for translation but necessary to have
        if BE_parm_vec_1[0] == 3:
            parm_dict['BE_phase_content'] = 'chirp-sinc hybrid'
        else:
            parm_dict['BE_phase_content'] = 'Unknown'
        parm_dict['BE_center_frequency_[Hz]'] = BE_parm_vec_1[1]
        parm_dict['BE_band_width_[Hz]'] = BE_parm_vec_1[2]
        parm_dict['BE_amplitude_[V]'] = BE_parm_vec_1[3]
        parm_dict['BE_band_edge_trim'] = -1 * BE_parm_vec_1[6]  # 150 most likely
        parm_dict['BE_phase_variation'] = BE_parm_vec_1[5]  # 0.01 most likely
        parm_dict['BE_repeats'] = 2 ** int(BE_parm_vec_1[8])
        parm_dict['File_date_and_time'] = 0  # For now ignoring.
        parm_dict['BE_bins_per_read'] = matread['bins_per_band_s']
        assembly_parm_vec = matread['assembly_parm_vec']

        if assembly_parm_vec[2] == 0:
            parm_dict['VS_measure_in_field_loops'] = 'out-of-field'
        elif assembly_parm_vec[2] == 1:
            parm_dict['VS_measure_in_field_loops'] = 'in and out-of-field'
        else:
            parm_dict['VS_measure_in_field_loops'] = 'in-field'

        parm_dict['IO_Analog_Input_1'] = '+/- 10V, FFT'
        if assembly_parm_vec[3] == 0:
            parm_dict['IO_Analog_Input_2'] = 'off'
        else:
            parm_dict['IO_Analog_Input_2'] = '+/- 10V, FFT'

        # num_driving_bands = assembly_parm_vec[0] # 0 = 1, 1 = 2 bands
        # band_combination_order = assembly_parm_vec[1] # 0 parallel 1 series

        VS_parms = matread['SS_parm_vec']
        dc_amp_vec_full = matread['dc_amp_vec_full']

        VS_start_V = VS_parms[4]
        VS_start_loop_amp = VS_parms[5]
        VS_final_loop_amp = VS_parms[6]
        # VS_read_write_ratio = VS_parms[8] #1 <- SS_read_write_ratio

        parm_dict['VS_set_pulse_amplitude_[V]'] = VS_parms[9]  # 0 <- SS_set_pulse_amp
        parm_dict['VS_read_voltage_[V]'] = VS_parms[3]
        parm_dict['VS_steps_per_full_cycle'] = len(dc_amp_vec_full)  # VS_parms[7]
        parm_dict['VS_cycle_fraction'] = 'full'
        parm_dict['VS_cycle_phase_shift'] = 0
        parm_dict['VS_number_of_cycles'] = VS_parms[2]
        parm_dict['FORC_num_of_FORC_cycles'] = 1
        parm_dict['FORC_V_high1_[V]'] = 0
        parm_dict['FORC_V_high2_[V]'] = 0
        parm_dict['FORC_V_low1_[V]'] = 0
        parm_dict['FORC_V_low2_[V]'] = 0

        if VS_parms[0] == 0:
            parm_dict['VS_mode'] = 'DC modulation mode'
            parm_dict['VS_amplitude_[V]'] = 0.5 * (max(dc_amp_vec_full) -
                                                   min(dc_amp_vec_full))  # SS_max_offset_amplitude
            parm_dict['VS_offset_[V]'] = max(dc_amp_vec_full) + min(dc_amp_vec_full)
        elif VS_parms[0] == 1:
            # FORC
            parm_dict['VS_mode'] = 'DC modulation mode'
            parm_dict['VS_amplitude_[V]'] = 1  # VS_parms[1] # SS_max_offset_amplitude
            parm_dict['VS_offset_[V]'] = 0
            parm_dict['VS_number_of_cycles'] = 1
            parm_dict['FORC_num_of_FORC_cycles'] = VS_parms[2]
            parm_dict['FORC_V_high1_[V]'] = VS_start_V
            parm_dict['FORC_V_high2_[V]'] = VS_start_V
            parm_dict['FORC_V_low1_[V]'] = VS_start_V - VS_start_loop_amp
            parm_dict['FORC_V_low2_[V]'] = VS_start_V - VS_final_loop_amp
        elif VS_parms[0] == 2:
            # AC mode
            parm_dict['VS_mode'] = 'AC modulation mode with time reversal'
            parm_dict['VS_amplitude_[V]'] = 0.5 * VS_final_loop_amp
            parm_dict[
                'VS_offset_[V]'] = 0  # this is not correct. Fix manually when it comes to UDVS generation?
        else:
            parm_dict['VS_mode'] = 'Custom'

        return parm_dict

    def __buildUDVSTable(self, parm_dict):
        """
        Generates the UDVS table using the parameters

        Inputs:
            parm_dict -- Dictionary of parameters present in the text files

        Outputs:
            tuple (labels, table)
            labels -- List of strings - Labels for columns in the UDVS table
            table -- UDVS data table
        """

        def translateVal(target, strvals, numvals):
            """
            Internal function - Interprets the provided value using the provided lookup table
            """

            if len(strvals) is not len(numvals):
                return None
            for strval, fltval in zip(strvals, numvals):
                if target == strval:
                    return fltval
            return None  # not found in list

        # Extract values from parm text file
        BE_signal_type = 1
        # This is necessary when normalzing the AI by the AO
        self.harmonic = BE_signal_type
        self.signal_type = BE_signal_type

        BE_amp = parm_dict['BE_amplitude_[V]']

        VS_amp = parm_dict['VS_amplitude_[V]']
        VS_offset = parm_dict['VS_offset_[V]']
        VS_steps = parm_dict['VS_steps_per_full_cycle']
        VS_cycles = parm_dict['VS_number_of_cycles']
        VS_fraction = translateVal(parm_dict['VS_cycle_fraction'], ['full', '1/2', '1/4', '3/4'], [1., 0.5, 0.25, 0.75])
        VS_shift = parm_dict['VS_cycle_phase_shift']
        if VS_shift is not 0:
            VS_shift = translateVal(VS_shift, ['1/4', '1/2', '3/4'], [0.25, 0.5, 0.75])
        VS_in_out_cond = translateVal(parm_dict['VS_measure_in_field_loops'],
                                      ['out-of-field', 'in-field', 'in and out-of-field'], [0, 1, 2])
        VS_ACDC_cond = translateVal(parm_dict['VS_mode'],
                                    ['DC modulation mode', 'AC modulation mode with time reversal',
                                     'load user defined VS Wave from file', 'current mode'],
                                    [0, 2, 3, 4])
        self.expt_type = VS_ACDC_cond
        FORC_cycles = parm_dict['FORC_num_of_FORC_cycles']
        FORC_A1 = parm_dict['FORC_V_high1_[V]']
        FORC_A2 = parm_dict['FORC_V_high2_[V]']
        FORC_B1 = parm_dict['FORC_V_low1_[V]']
        FORC_B2 = parm_dict['FORC_V_low2_[V]']

        # % build vector of voltage spectroscopy values

        if VS_ACDC_cond == 0 or VS_ACDC_cond == 4:  # DC voltage spectroscopy or current mode
            VS_amp_vec_1 = np.arange(0, 1 + 1 / (VS_steps / 4), 1 / (VS_steps / 4))
            VS_amp_vec_2 = np.flipud(VS_amp_vec_1[:-1])
            VS_amp_vec_3 = -VS_amp_vec_1[1:]
            VS_amp_vec_4 = VS_amp_vec_1[1:-1] - 1
            VS_amp_vec = VS_amp * (np.hstack((VS_amp_vec_1, VS_amp_vec_2, VS_amp_vec_3, VS_amp_vec_4)))
            VS_amp_vec = np.roll(VS_amp_vec,
                                 int(np.floor(VS_steps / VS_fraction * VS_shift)))  # apply phase shift to VS wave
            VS_amp_vec = VS_amp_vec[:int(np.floor(VS_steps * VS_fraction))]  # cut VS waveform
            VS_amp_vec = np.tile(VS_amp_vec, VS_cycles)  # repeat VS waveform
            VS_amp_vec = VS_amp_vec + VS_offset

        if FORC_cycles > 1:
            VS_amp_vec = VS_amp_vec / np.max(np.abs(VS_amp_vec))
            FORC_cycle_vec = np.arange(0, FORC_cycles + 1, FORC_cycles / (FORC_cycles - 1))
            FORC_A_vec = FORC_cycle_vec * (FORC_A2 - FORC_A1) / FORC_cycles + FORC_A1
            FORC_B_vec = FORC_cycle_vec * (FORC_B2 - FORC_B1) / FORC_cycles + FORC_B1
            FORC_amp_vec = (FORC_A_vec - FORC_B_vec) / 2
            FORC_off_vec = (FORC_A_vec + FORC_B_vec) / 2

            VS_amp_mat = np.tile(VS_amp_vec, [FORC_cycles, 1])
            FORC_amp_mat = np.tile(FORC_amp_vec, [len(VS_amp_vec), 1]).transpose()
            FORC_off_mat = np.tile(FORC_off_vec, [len(VS_amp_vec), 1]).transpose()
            VS_amp_mat = VS_amp_mat * FORC_amp_mat + FORC_off_mat
            VS_amp_vec = VS_amp_mat.reshape(int(FORC_cycles * VS_cycles * VS_fraction * VS_steps))

        BE_repeats = parm_dict['BE_repeats']
        total_steps = len(VS_amp_vec) * BE_repeats  # Needed for relaxation datasets

        # % Build UDVS table:
        if VS_ACDC_cond is 0 or VS_ACDC_cond is 4:  # relaxation measurements

            num_VS_steps = total_steps * 2  # To account for IF and OOF

            UD_VS_table_label = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod', 'in-field', 'out-of-field']
            UD_VS_table = np.zeros(shape=(num_VS_steps, 7), dtype=np.float32)
            UD_VS_table_unit = ['', 'V', 'A', '', '', 'V', 'V']

            UD_VS_table[:, 0] = np.arange(0, num_VS_steps)  # Python base 0
            for step_num in np.arange(0, VS_steps):
                step_values = (np.arange(int(step_num) * int(BE_repeats) * 2,
                                         (int(step_num) + 1) * int(BE_repeats) * 2))
                UD_VS_table[step_values, 1] = VS_amp_vec[step_num]

            BE_IF_switch = np.abs(np.imag(np.exp(1j * np.pi / 2 * np.arange(1, num_VS_steps + 1))))
            BE_OF_switch = np.abs(np.real(np.exp(1j * np.pi / 2 * np.arange(1, num_VS_steps + 1))))

            if VS_in_out_cond is 0:  # out of field only
                UD_VS_table[:, 2] = BE_amp * BE_OF_switch
            elif VS_in_out_cond is 1:  # in field only
                UD_VS_table[:, 2] = BE_amp * BE_IF_switch
            elif VS_in_out_cond is 2:  # both in and out of field
                UD_VS_table[:, 2] = BE_amp * np.ones(num_VS_steps)

            UD_VS_table[:, 3] = np.ones(num_VS_steps)  # wave type
            UD_VS_table[:, 4] = np.ones(num_VS_steps) * BE_signal_type  # wave mod

            UD_VS_table[:, 5] = float('NaN') * np.ones(num_VS_steps)
            UD_VS_table[:, 6] = float('NaN') * np.ones(num_VS_steps)

            UD_VS_table[BE_IF_switch == 1, 5] = UD_VS_table[BE_IF_switch == 1, 1]
            UD_VS_table[BE_OF_switch == 1, 6] = UD_VS_table[BE_IF_switch == 1, 1]

        return UD_VS_table_label, UD_VS_table_unit, UD_VS_table


from lixian_plugins.api import command

from lixian_config import *
from lixian_encoding import default_encoding
from lixian_cli_parser import command_line_parser
from lixian_cli_parser import with_parser
from lixian_cli_parser import command_line_option, command_line_value
from lixian_commands.util import parse_login, create_client

import urllib2
import json

def get_download_task_info(args, client):

    import lixian_query
    tasks = lixian_query.search_tasks(client, args)
    files = []
    for task in tasks:
        if task['type'] == 'bt':
            subs, skipped, single_file = lixian_query.expand_bt_sub_tasks(task)
            if not subs:
                continue
            if single_file:
                files.append((subs[0]['xunlei_url'], subs[0]['name'], None))
            else:
                for f in subs:
                    files.append((f['xunlei_url'], f['name'], task['name']))
        else:
            files.append((task['xunlei_url'], task['name'], None))
    return files

def export_download_task_info(files, client):
    output = ''
    for url, name, dir in files:
        if type(url) == unicode:
            url = url.encode(default_encoding)
        output += url + '\n'
        output += '  out=' + name.encode(default_encoding) + '\n'
        if dir:
            output += '  dir=' + dir.encode(default_encoding) + '\n'
        output += '  header=Cookie: gdriveid=' + client.get_gdriveid() + '\n'
    return output

def execute_download_aria2rpc(args):
    client = create_client(args)
    task_list = get_download_task_info(args, client)
    # print(export_download_task_info(task_list, client))

    for url, name, dir in task_list:
        if type(url) == unicode:
            url = url.encode(default_encoding)
        if dir:
            dir = dir.encode(default_encoding)

        jsonreq = json.dumps({"jsonrpc": "2.0", "id": "qwer",
                              "method": "aria2.addUri",
                              "params": [
                                         [url],
                                         {
                                             "out": name.encode(default_encoding),
                                             "continue": "true",
                                             "header": ['Cookie: gdriveid=%s' % client.get_gdriveid()]
                                         }
                              ]
                              })
        # print(jsonreq)
        host = '127.0.0.1'
        if args.dev is True:
            host = '192.168.0.2'

        print("Add to aria2c server:%s" % host)
        c = urllib2.urlopen("http://%s:6800/jsonrpc" % host, jsonreq)
        # {u'jsonrpc': u'2.0', u'id': u'qwer', u'result': u'f1257fa333f235e6'}
        result = c.read()
        if result is None or result == "":
            print("\033[31mCann't add aria2 task %s\033[0m" % name)
        else:
            result = json.loads(result.decode(default_encoding))
            print("\033[32mAdd aria2 task[id= %s] %s\033[0m" % (result[u"result"], name))

@command(usage='concurrently download tasks in aria2 with rpc')
@command_line_parser()
@with_parser(parse_login)
@command_line_option('all')
@command_line_option('dev', default=False)
def download_aria2rpc(args):
    '''
    usage: lx download-aria2rpc -j 5 -h dev [id|name]...
    '''
    #print(args)
    execute_download_aria2rpc(args)


#!/usr/bin/env python3
import json
import pickle
import pathlib

import numpy as np
import pandas as pd
from sklearn import preprocessing
from sklearn.model_selection import train_test_split, RandomizedSearchCV
from sklearn.metrics import auc, accuracy_score, confusion_matrix, mean_squared_error
import xgboost as xgb
import argparse
import logging

logging.basicConfig(format='%(asctime)s.%(msecs)03d %(levelname)s {%(module)s} [%(funcName)s : %(lineno)d] %(message)s', datefmt='%Y-%m-%d,%H:%M:%S', level=logging.INFO)
logger = logging.getLogger(__name__)

def _check_path(path):
    p = pathlib.Path(path)
    if not p.exists():
        logger.info(f"Path {path} does not exist. Creating...")
        p.parent.absolute().mkdir(parents=True, exist_ok=True)
        return False
    else:
        logger.info(f"Path {path} already exists.")
        return True

def load_data(features_file_path, labels_file_path):
    X = np.loadtxt(features_file_path, delimiter=',')
    y = pd.read_csv(labels_file_path)

    assert X.shape[0] == y['MTurk_label'].shape[0]

    return X, [int(d) for d  in y['MTurk_label'].to_list()]

def train(X, y,
    cv_results_file_path,
    model_save_path,
    mlpipeline_metrics_path,
    mlpipeline_ui_metadata_path,
    test_score_path,
    test_size=0.3):

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, stratify=y)

    estimator = xgb.XGBClassifier(
        objective="binary:logistic",
        eval_metric="logloss",
        use_label_encoder=False,
        random_state=42)

    parameters={
        'max_depth': range(2, 5, 1),
        'n_estimators': range(10, 100, 10),
        'learning_rate': [0.1, 0.05, 0.01]
    }

    random_search = RandomizedSearchCV(
        estimator=estimator,
        param_distributions=parameters,
        scoring='accuracy',
        random_state=22,
        return_train_score=True
        )

    search_result = random_search.fit(X_train, y_train)

    cv_results = search_result.cv_results_
    best_params = search_result.best_params_
    best_score = search_result.best_score_
    best_model = search_result.best_estimator_

    cv_results = pd.DataFrame(cv_results)

    _check_path(cv_results_file_path)
    cv_results.to_csv(cv_results_file_path)

    logger.info(f"Model: {estimator}")
    logger.info(f"Best CV score is {best_score}")
    logger.info(f"Best parameters are {best_params}")

     # Viz of cv results
    _check_path(mlpipeline_ui_metadata_path)
    metadata = {
        'outputs' : [{
            'type': 'table',
            'storage': 'inline',
            'format': 'csv',
            'header': ['idx'] + list(cv_results.columns.values),
            'source': cv_results.to_csv(header=False)
        }]
    }
    with open(mlpipeline_ui_metadata_path, 'w') as f:
        json.dump(metadata, f)

    # Save model
    _check_path(model_save_path)
    pickle.dump(best_model, open(model_save_path, 'wb'))

    # Save metrics
    _check_path(mlpipeline_metrics_path)
    y_pred = best_model.predict(X_test)
    p = pathlib.Path(mlpipeline_metrics_path)
    metrics = {
        'metrics': [
            {
            'name': 'accuracy_score',
            'numberValue': accuracy_score(y_test, y_pred),
            'format': "PERCENTAGE"
            }
        ]
    }
    with open(mlpipeline_metrics_path, 'w') as f:
        json.dump(metrics, f)

    logger.info(f"Test score is {accuracy_score(y_test, y_pred)}")
    test_score = {'score': accuracy_score(y_test, y_pred)}
    _check_path(test_score_path)
    with open(test_score_path, 'w') as f:
        json.dump(test_score, f)

if __name__ == "__main__":
    try:
        # The component must be stateless
        # All inputs are not hard coded but passed in as params
        parser = argparse.ArgumentParser()
        parser.add_argument('--features_file_path', type=str, action='store')
        parser.add_argument('--labels_file_path', type=str, action='store')
        parser.add_argument('--cv_results_file_path', type=str, action='store')
        parser.add_argument('--model_file_path', type=str, action='store')
        parser.add_argument('--kfp_metrics_path', type=str, action='store')
        parser.add_argument('--mlpipeline_ui_metadata_path', type=str, action='store')
        parser.add_argument('--test_score_path', type=str, action='store')

        FLAGS = parser.parse_args()

        X, y = load_data(FLAGS.features_file_path, FLAGS.labels_file_path)
        train(X, y,
            FLAGS.cv_results_file_path,
            FLAGS.model_file_path,
            FLAGS.kfp_metrics_path,
            FLAGS.mlpipeline_ui_metadata_path,
            FLAGS.test_score_path
            )

    except Exception as e:
        logger.exception(e)


# -*- coding: utf-8 -*-
"""
Created on Wed Jun 19 21:17:34 2019

@author: Soriba
"""
import os
from PIL import Image
import numpy as np
from sklearn.preprocessing import LabelEncoder
from keras.utils import to_categorical
#data preprocessing
def process_data(data_path):
    classes=os.listdir(data_path)
    imgs=[]
    target=[]
    for classe in classes:
        #path to the class folder
        classe_path=os.path.join(data_path,classe)
        images=os.listdir(classe_path)
        for k in range(len(images)):
            img = Image.open(os.path.join(classe_path,images[k]))
            #normalize the pixels
            imgs.append(np.array(img)/255)
            #for each image, we store it's class into the target vector
            target.append(classe)

    nb_imgs=len(imgs)
    s=imgs[0].shape
    #tensor that will be feed to the network
    X= np.ndarray((nb_imgs,*s))
    for i in range(nb_imgs):
        X[i]=imgs[i]

    #target variable
    le=LabelEncoder()
    le.fit(target)
    #labelencoding
    encoded= le.transform(target)
    #OneHotEncoding
    y=to_categorical(encoded)

    return X, y


# -*- coding: utf-8 -*-
# $Author   : LiuShiWen
# $Date     : 2021/7/2 2:59 下午

from configparser import ConfigParser
import os
import sys

project_path = str(os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)))
if sys.platform.startswith("win32"):
    conf_path = os.path.join(project_path, 'Config/config.ini').replace('/', '\\')
    log_path = os.path.join(project_path, 'OutPuts/Logs').replace('/', '\\')
    report_path = os.path.join(project_path, 'OutPuts/Reports').replace('/', '\\')
    img_path = os.path.join(project_path, 'OutPuts/Images').replace('/', '\\')
    UITestData_path = os.path.join(project_path, 'Data/UITestData').replace('/', '\\')
    APITestData_path = os.path.join(project_path, 'Data/APITestData').replace('/', '\\')
else:
    conf_path = os.path.join(project_path, 'Config/.config.ini')
    log_path = os.path.join(project_path, 'OutPuts/Logs')
    report_path = os.path.join(project_path, 'OutPuts/Reports')
    img_path = os.path.join(project_path, 'OutPuts/Images')
    UITestData_path = os.path.join(project_path, 'Data/UITestData')
    APITestData_path = os.path.join(project_path, 'Data/APITestData')
if not os.path.exists(conf_path):
    raise FileNotFoundError("请检查配置文件路径是否存在")
if not os.path.exists(log_path):
    raise FileNotFoundError("请检查日志文件路径是否存在")
if not os.path.exists(report_path):
    raise FileNotFoundError("请检查报告路径是否存在")
if not os.path.exists(img_path):
    raise FileNotFoundError("请检查照片路径是否存在")
if not  os.path.exists(UITestData_path):
    raise FileNotFoundError("请检查UI测试数据路径是否存在")
if not  os.path.exists(APITestData_path):
    raise FileNotFoundError("请检查API测试数据路径是否存在")

class Config(object):
    def __init__(self, conf_path=conf_path, encode="utf-8"):
        if os.path.exists(conf_path):
            self.__cfg_file = conf_path
        else:
            # 此处做其他异常处理或创建配置文件操作
            raise Exception
        self.__config = ConfigParser()
        self.__config.read(self.__cfg_file, encoding=encode)

    # 获取配置文件的所有section
    def get_sections(self):
        return self.__config.sections()

    # 获取指定section的所有option
    def get_options(self, section_name):
        if self.__config.has_section(section_name):
            return self.__config.options(section_name)
        else:
            raise Exception

    # 获取指定section下option的value值
    def get_option_value(self, section_name, option_name):
        if self.__config.has_option(section_name, option_name):
            return self.__config.get(section_name, option_name)

    # 获取指定section下的option的键值对
    def get_all_items(self, section):
        if self.__config.has_section(section):
            return self.__config.items(section)

    # 打印配置文件所有的值
    def print_all_items(self):
        for section in self.get_sections():
            print("[" + section + "]")
            for K, V in self.__config.items(section):
                print(K + "=" + V)

    # 增加section
    def add_new_section(self, new_section):
        if not self.__config.has_section(new_section):
            self.__config.add_section(new_section)
            self.__update_cfg_file()

    # 增加指定section下option
    def add_option(self, section_name, option_key, option_value):
        if self.__config.has_section(section_name):
            self.__config.set(section_name, option_key, option_value)
            self.__update_cfg_file()

    # 删除指定section
    def del_section(self, section_name):
        if self.__config.has_section(section_name):
            self.__config.remove_section(section_name)
            self.__update_cfg_file()

    # 删除指定section下的option
    def del_option(self, section_name, option_name):
        if self.__config.has_option(section_name, option_name):
            self.__config.remove_option(section_name, option_name)
            self.__update_cfg_file()

    # 更新指定section下的option的值
    def update_option_value(self, section_name, option_key, option_value):
        if self.__config.has_option(section_name, option_key):
            self.add_option(section_name, option_key, option_value)

    # 私有方法:操作配置文件的增删改时，更新配置文件的数据
    def __update_cfg_file(self):
        with open(self.__cfg_file, "w") as f:
            self.__config.write(f)

def getEmailOptionValues():
    config = Config()
    email_data = []
    email_username = config.get_option_value('email','username')
    email_authorization_code = config.get_option_value('email','authorization_code')
    email_data.append(email_username)
    email_data.append(email_authorization_code)
    return email_data

if __name__ == '__main__':
    print(getEmailOptionValues())


# baca citra konversi bit
import cv2
import mylib as my

img = cv2.imread('/root/PycharmProject/PemrogamanTingkatLanjut/lenacitra/lenalr.jpeg')
# baris, kolom, layer = img.shape
# pixel berapa komponen berapa
baris, kolom = img.shape[:2]  # slicing a value

stego = img.copy()

# pesan = "Samarinda kota tepian sudah punya bandara sejak tahun 2018"
fileteks = "/root/PycharmProject/PemrogamanTingkatLanjut/lenacitra/teksuji.txt"
pesan = my.readTextFile(fileteks)
print(pesan)
br = kl = 0

for ch in pesan:
    bits = format(ord(ch), '08b')  # biner dari huruf pada pesan

    for bit in bits:
        stgdat = format(stego[br, kl, 0], '08b')[:-1]  #ambil 7 bit dari citra
        stgdat = stgdat + bit  # tabahkan dengan 1 bit pesan
        stego[br, kl, 0] = int(stgdat,2)# simpan data

        kl += 1
        if kl == kolom:
            br += 1
            kl = 0

# save stegoimage
cv2.imwrite("lenastego.png", img)
# cv2.imshow('lenastego', img)
# cv2.imshow('lena', img)
# cv2.waitKey(0)
# cv2.destroyWindow()
# berapa lebar dan tingginya gunakan sh
# shape ( baris, kolom , layer ) | Shape( 255,255,3)
# 0011.0101
# msg.lsb
stegox = stego
# stegox = cv2.imread("lenastego.png")
# baris, kolom = stegox.shape[:2]

jmlhuruf = 647
pesan = ""

br = kl = 0
bits = ""

while len(pesan) < jmlhuruf:
    # ambil satu bit teakhir
    # print(br, kl, end="\t")
    bit = format(stegox[br, kl, 0], '08b')[-1]
    bits = bits + bit

    # konversi menjadi huruf, dan gabungkan
    if (len(bits)==8):
        huruf = chr(int(bits, 2))
        pesan = pesan + huruf
        bits = ""

    # ubah posisi kolom dan baris
    kl += 1
    if kl == kolom:
        br += 1
        kl = 0

print("\n\n")
print(pesan)


import PIL
import tensorflow as tf
import numpy as np
import os
import pickle

from tensorflow.python.keras.models import Model, Sequential
from tensorflow.python.keras.applications import VGG16
from sklearn.preprocessing import StandardScaler

VERBOSE = True

ad_images_path = "../../ad_images/"

def getFileNames(path):
    if VERBOSE:
        print("[MaherBot] Getting filenames from path: " + path)
    return sorted(os.listdir(path))

def loadImage(path):
    temp = PIL.Image.open(path)
    img = temp.copy()
    temp.close()
    return img

filenames = getFileNames(ad_images_path)

ids = list(map(lambda x: x.split(".")[0], filenames))

model = VGG16(include_top=True, weights='imagenet')

input_shape = model.layers[0].output_shape[1:3]

print(model.layers[0].output_shape)
print(input_shape)

imgs = list(map(lambda x: loadImage(ad_images_path + x), filenames))

r_imgs = list(map(lambda x: x.resize(input_shape,PIL.Image.LANCZOS), imgs))

np_imgs = list(map(lambda x: np.array(x), r_imgs))

np_imgs = list(map(lambda x: x.astype('float'), np_imgs))

# for n in np_imgs: print(n.shape)

ids_imgs = list(filter(lambda x: x[1].shape == (224, 224, 3), zip(ids, np_imgs)))

ids = list(map(lambda x: x[0], ids_imgs))

np_imgs = list(map(lambda x: x[1], ids_imgs))

np_imgs = np.array(np_imgs)
print("blabla")
print(np_imgs.shape)

# i.resize(input_shape,PIL.Image.LANCZOS)

# img = np.array(img)
# img = img.astype('float')

layer_name = 'fc2'
fc2_layer_model = Model(inputs=model.input,
                        outputs=model.get_layer(layer_name).output)

features = fc2_layer_model.predict(np_imgs)

scaler = StandardScaler().fit(features)
sfeatures = scaler.transform(features)

results = {}

for i in range(len(ids)):
	results[ids[i]] = sfeatures[i]

with open("data_img_features.pkl", 'wb') as f:
    pickle.dump(results, f, pickle.HIGHEST_PROTOCOL)


# -*- coding: utf-8 -*-
"""
Created on Fri Feb  2 10:35:42 2018

@author: hsf
"""

#from matplotlib import pyplot
#matplotlib.use('qt4agg')
import matplotlib
#matplotlib.use('qt4agg')
import matplotlib.pyplot as ply

#指定默认字体
matplotlib.rcParams['font.sans-serif'] = ['SimHei']
matplotlib.rcParams['font.family']='sans-serif'
#解决负号'-'显示为方块的问题
matplotlib.rcParams['axes.unicode_minus'] = False
matplotlib.rcParams["figure.dpi"] = 100

#绘制柱状图
def bar_card_type(card_type_list, card_types):
    xticks = card_types
    cardGroup = {}
    #对每一类成绩进行频数统计
    for i in card_type_list:
        cardGroup[i] = cardGroup.get(i, 0) + 1
    #创建柱状图
    #第一个参数为柱的横坐标
    #第二个参数为柱的高度
    #参数align为柱的对齐方式，以第一个参数为参考标准
    rects = ply.bar(range(len(card_types)), [cardGroup.get(xtick, 0) for xtick in xticks], align='center')

    #设置柱的文字说明
    #第一个参数为文字说明的横坐标
    #第二个参数为文字说明的内容
    ply.xticks(range(len(card_types)), xticks)
    #设置横坐标的文字说明
    ply.xlabel('Card Type')
    #设置纵坐标的文字说明
    ply.ylabel('Frequency')
    #设置标题
    ply.title(u'Card Type Statistic')
    #绘图
    total = len(card_type_list)
    print(total)
    for rect in rects:
        height = rect.get_height()
        ply.text(rect.get_x() + rect.get_width() / 2, height, \
                 round(height/total*100,3), ha='center', va='bottom')
    ply.show()
#    print(cardGroup)

def bar_player(player_list, players):
    xticks = players
    playerGroup = {}
    #对每一类成绩进行频数统计
    for i in player_list:
        playerGroup[i] = playerGroup.get(i, 0) + 1
    #创建柱状图
    #第一个参数为柱的横坐标
    #第二个参数为柱的高度
    #参数align为柱的对齐方式，以第一个参数为参考标准
    rects = ply.bar(range(len(players)), [playerGroup.get(xtick, 0) for xtick in xticks], align='center')

    #设置柱的文字说明
    #第一个参数为文字说明的横坐标
    #第二个参数为文字说明的内容
    ply.xticks(range(len(playerGroup)), xticks)
    #设置横坐标的文字说明
    ply.xlabel('Card Type')
    #设置纵坐标的文字说明
    ply.ylabel('Frequency')
    #设置标题
    ply.title(u'Card Type Statistic')
    #绘图
    total = len(player_list)
    print(total)
    for rect in rects:
        height = rect.get_height()
        ply.text(rect.get_x() + rect.get_width() / 2, height, \
                 round(height/total*100,3), ha='center', va='bottom')
    ply.show()
#    print(playerGroup)


#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import mock

import nova.conf
from nova import context
from nova import objects
from nova.objects import base as obj_base
from nova.scheduler.client import report
from nova import test
from nova.tests import uuidsentinel as uuids

CONF = nova.conf.CONF

class SchedulerReportClientTestCase(test.NoDBTestCase):

    def setUp(self):
        super(SchedulerReportClientTestCase, self).setUp()
        self.context = context.get_admin_context()
        self.ks_sess_mock = mock.Mock()

        with test.nested(
                mock.patch('keystoneauth1.session.Session',
                           return_value=self.ks_sess_mock),
                mock.patch('keystoneauth1.loading.load_auth_from_conf_options')
        ) as (_auth_mock, _sess_mock):
            self.client = report.SchedulerReportClient()

    @mock.patch('keystoneauth1.session.Session')
    @mock.patch('keystoneauth1.loading.load_auth_from_conf_options')
    def test_constructor(self, load_auth_mock, ks_sess_mock):
        report.SchedulerReportClient()

        load_auth_mock.assert_called_once_with(CONF, 'placement')
        ks_sess_mock.assert_called_once_with(auth=load_auth_mock.return_value)

    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_create_resource_provider')
    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_get_resource_provider')
    def test_ensure_resource_provider_exists_in_cache(self, get_rp_mock,
            create_rp_mock):
        # Override the client object's cache to contain a resource provider
        # object for the compute host and check that
        # _ensure_resource_provider() doesn't call _get_resource_provider() or
        # _create_resource_provider()
        self.client._resource_providers = {
            uuids.compute_node: mock.sentinel.rp
        }

        self.client._ensure_resource_provider(uuids.compute_node)
        self.assertFalse(get_rp_mock.called)
        self.assertFalse(create_rp_mock.called)

    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_create_resource_provider')
    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_get_resource_provider')
    def test_ensure_resource_provider_get(self, get_rp_mock, create_rp_mock):
        # No resource provider exists in the client's cache, so validate that
        # if we get the resource provider from the placement API that we don't
        # try to create the resource provider.
        get_rp_mock.return_value = mock.sentinel.rp

        self.client._ensure_resource_provider(uuids.compute_node)

        get_rp_mock.assert_called_once_with(uuids.compute_node)
        self.assertEqual({uuids.compute_node: mock.sentinel.rp},
                          self.client._resource_providers)
        self.assertFalse(create_rp_mock.called)

    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_create_resource_provider')
    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_get_resource_provider')
    def test_ensure_resource_provider_create_none(self, get_rp_mock,
            create_rp_mock):
        # No resource provider exists in the client's cache, and
        # _create_provider returns None, indicating there was an error with the
        # create call. Ensure we don't populate the resource provider cache
        # with a None value.
        get_rp_mock.return_value = None
        create_rp_mock.return_value = None

        self.client._ensure_resource_provider(uuids.compute_node)

        get_rp_mock.assert_called_once_with(uuids.compute_node)
        create_rp_mock.assert_called_once_with(uuids.compute_node,
                                               uuids.compute_node)
        self.assertEqual({}, self.client._resource_providers)

    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_create_resource_provider')
    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_get_resource_provider')
    def test_ensure_resource_provider_create(self, get_rp_mock,
            create_rp_mock):
        # No resource provider exists in the client's cache and no resource
        # provider was returned from the placement API, so verify that in this
        # case we try to create the resource provider via the placement API.
        get_rp_mock.return_value = None
        create_rp_mock.return_value = mock.sentinel.rp

        self.client._ensure_resource_provider(uuids.compute_node)

        get_rp_mock.assert_called_once_with(uuids.compute_node)
        create_rp_mock.assert_called_once_with(
                uuids.compute_node,
                uuids.compute_node,  # name param defaults to UUID if None
        )
        self.assertEqual({uuids.compute_node: mock.sentinel.rp},
                          self.client._resource_providers)

        create_rp_mock.reset_mock()
        self.client._resource_providers = {}

        self.client._ensure_resource_provider(uuids.compute_node,
                                              mock.sentinel.name)

        create_rp_mock.assert_called_once_with(
                uuids.compute_node,
                mock.sentinel.name,
        )

    def test_get_resource_provider_found(self):
        # Ensure _get_resource_provider() returns a ResourceProvider object if
        # it finds a resource provider record from the placement API
        uuid = uuids.compute_node
        resp_mock = mock.Mock(status_code=200)
        json_data = {
            'uuid': uuid,
            'name': uuid,
            'generation': 42,
        }
        resp_mock.json.return_value = json_data
        self.ks_sess_mock.get.return_value = resp_mock

        result = self.client._get_resource_provider(uuid)

        expected_provider = objects.ResourceProvider(
                uuid=uuid,
                name=uuid,
                generation=42,
        )
        expected_url = '/resource_providers/' + uuid
        self.ks_sess_mock.get.assert_called_once_with(expected_url,
                                                      endpoint_filter=mock.ANY,
                                                      raise_exc=False)
        self.assertTrue(obj_base.obj_equal_prims(expected_provider,
                                                 result))

    def test_get_resource_provider_not_found(self):
        # Ensure _get_resource_provider() just returns None when the placement
        # API doesn't find a resource provider matching a UUID
        resp_mock = mock.Mock(status_code=404)
        self.ks_sess_mock.get.return_value = resp_mock

        uuid = uuids.compute_node
        result = self.client._get_resource_provider(uuid)

        expected_url = '/resource_providers/' + uuid
        self.ks_sess_mock.get.assert_called_once_with(expected_url,
                                                      endpoint_filter=mock.ANY,
                                                      raise_exc=False)
        self.assertIsNone(result)

    @mock.patch.object(report.LOG, 'error')
    def test_get_resource_provider_error(self, logging_mock):
        # Ensure _get_resource_provider() sets the error flag when trying to
        # communicate with the placement API and not getting an error we can
        # deal with
        resp_mock = mock.Mock(status_code=503)
        self.ks_sess_mock.get.return_value = resp_mock

        uuid = uuids.compute_node
        result = self.client._get_resource_provider(uuid)

        expected_url = '/resource_providers/' + uuid
        self.ks_sess_mock.get.assert_called_once_with(expected_url,
                                                      endpoint_filter=mock.ANY,
                                                      raise_exc=False)
        # A 503 Service Unavailable should trigger an error logged and
        # return None from _get_resource_provider()
        self.assertTrue(logging_mock.called)
        self.assertIsNone(result)

    def test_create_resource_provider(self):
        # Ensure _create_resource_provider() returns a ResourceProvider object
        # constructed after creating a resource provider record in the
        # placement API
        uuid = uuids.compute_node
        name = 'computehost'
        resp_mock = mock.Mock(status_code=201)
        self.ks_sess_mock.post.return_value = resp_mock

        result = self.client._create_resource_provider(uuid, name)

        expected_payload = {
            'uuid': uuid,
            'name': name,
        }
        expected_provider = objects.ResourceProvider(
            uuid=uuid,
            name=name,
            generation=1,
        )
        expected_url = '/resource_providers'
        self.ks_sess_mock.post.assert_called_once_with(
                expected_url,
                endpoint_filter=mock.ANY,
                json=expected_payload,
                raise_exc=False)
        self.assertTrue(obj_base.obj_equal_prims(expected_provider,
                                                 result))

    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_get_resource_provider')
    def test_create_resource_provider_concurrent_create(self, get_rp_mock):
        # Ensure _create_resource_provider() returns a ResourceProvider object
        # gotten from _get_resource_provider() if the call to create the
        # resource provider in the placement API returned a 409 Conflict,
        # indicating another thread concurrently created the resource provider
        # record.
        uuid = uuids.compute_node
        name = 'computehost'
        resp_mock = mock.Mock(status_code=409)
        self.ks_sess_mock.post.return_value = resp_mock

        get_rp_mock.return_value = mock.sentinel.get_rp

        result = self.client._create_resource_provider(uuid, name)

        expected_payload = {
            'uuid': uuid,
            'name': name,
        }
        expected_url = '/resource_providers'
        self.ks_sess_mock.post.assert_called_once_with(
                expected_url,
                endpoint_filter=mock.ANY,
                json=expected_payload,
                raise_exc=False)
        self.assertEqual(mock.sentinel.get_rp, result)

    @mock.patch.object(report.LOG, 'error')
    def test_create_resource_provider_error(self, logging_mock):
        # Ensure _create_resource_provider() sets the error flag when trying to
        # communicate with the placement API and not getting an error we can
        # deal with
        uuid = uuids.compute_node
        name = 'computehost'
        resp_mock = mock.Mock(status_code=503)
        self.ks_sess_mock.post.return_value = resp_mock

        result = self.client._create_resource_provider(uuid, name)

        expected_payload = {
            'uuid': uuid,
            'name': name,
        }
        expected_url = '/resource_providers'
        self.ks_sess_mock.post.assert_called_once_with(
                expected_url,
                endpoint_filter=mock.ANY,
                json=expected_payload,
                raise_exc=False)
        # A 503 Service Unavailable should log an error and
        # _create_resource_provider() should return None
        self.assertTrue(logging_mock.called)
        self.assertIsNone(result)

    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                '_ensure_resource_provider')
    @mock.patch.object(objects.ComputeNode, 'save')
    def test_update_resource_stats_saves(self, mock_save, mock_ensure):
        cn = objects.ComputeNode(context=self.context,
                                 uuid=uuids.compute_node,
                                 hypervisor_hostname='host1')
        self.client.update_resource_stats(cn)
        mock_save.assert_called_once_with()
        mock_ensure.assert_called_once_with(uuids.compute_node, 'host1')


import numpy as np
import config
import random
import json
import datetime
import matplotlib.pyplot as plt

def read_price_data(data_file, forex_config, min_len):
    # file_name = "data_price.json"
    print('File ' + data_file + ' Loading.....')
    with open(data_file) as f:
        data = json.load(f)
    price_data = []
    for pair in forex_config.country_pair:
        price_data.append(data[pair][:min_len])
    price_data = np.transpose(np.asarray(price_data), [1, 0, 2])

    price_data = price_data.tolist()
    return price_data

def read_target_data(data_file):
    print('File ' + data_file + ' Loading.....')
    with open(data_file) as f:
        data = json.load(f)
    return data["Target"], len(data["Target"])

def read_date_data(date_file):
    print('File ' + date_file + ' Loading.....')
    with open(date_file) as f:
        date = json.load(f)
    return date['Date']

def read_return_data(data_file):
    print('File ' + data_file + ' Loading.....')
    with open(data_file) as f:
        data = json.load(f)
    return data['Return']

def year_sort(date_file, data_file, target_file, return_file):
    year_list = {}
    year_target_list = {}
    year_return_list = {}
    for i in range(len(data_file)):
        # print (date_file[str(i)])
        date = datetime.datetime.fromtimestamp(date_file[i] / 1000000000)
        if date.year not in year_list.keys():
            year_list[date.year] = []
            year_target_list[date.year] = []
            year_return_list[date.year] = []
        year_list[date.year].append(data_file[i])
        year_target_list[date.year].append(target_file[i])
        year_return_list[date.year].append(return_file[i])

    for key in year_list.keys():
        year_list[key] = zip(year_list[key], year_target_list[key], year_return_list[key])
    return year_list

def year_sort_rnn(date_file, data_file, target_file, return_file, forex_config):
    year_list = {}
    year_target_list = {}
    year_return_list = {}
    continuous_data_num = forex_config.continuous_period_num
    for i in range(len(data_file - continuous_data_num)):
        # print (date_file[str(i)])
        date = datetime.datetime.fromtimestamp(date_file[str(i)] / 1000)
        if date.year not in year_list.keys():
            year_list[date.year] = []
            year_target_list[date.year] = []
            year_return_list[date.year] = []
        year_list[date.year].append([])
        year_target_list[date.year].append([])
        year_return_list[date.year].append([])
        for j in range(continuous_data_num):
            year_list[date.year][-1].append(data_file[i + j])
            year_target_list[date.year][-1].append(target_file[i + j])
            year_return_list[date.year][-1].append(return_file[i + j])
    for key in year_list.keys():
        year_list[key] = zip(year_list[key], year_target_list[key], year_return_list[key])
    return year_list

#####  random pick train years, others will be viewed as testing set #####
def pick_train_test_pair(year_list,train_year_num ,forex_config):
    test_year_num = len(forex_config.years) - train_year_num
    test_years = list(np.random.choice(forex_config.years, test_year_num, replace=False))
    test_data = []
    for test_year in test_years:
        test_data.extend(year_list[test_year])
    key_list = []
    for key in year_list.keys():
        if key in test_years:
            continue
        key_list.append(key)
    print(key_list)
    random_select_num = np.random.choice(len(key_list), train_year_num, replace=False)
    train_data = []
    for i in random_select_num:
        train_data.extend(year_list[key_list[i]])
    return train_data, test_data, test_years

#####  pick train years in order, the year next to last train year will be viewed as testing set #####
def pick_train_test_pair_inorder(year_list,train_year_num ,forex_config):
    train_years = forex_config.years[:train_year_num]
    train_data = []
    for train_year in train_years:
        train_data.extend(year_list[train_year])
    test_data = []
    test_year = forex_config.years[train_year_num]
    test_data.extend(forex_config.year[test_year])
    return train_data,test_data,[test_year]

def get_batch(data, index, forex_config, is_back_test):
    end_index = index + forex_config.batch_size
    if end_index >= len(data[0]):
        end_index = len(data[0]) - 1
    batch_price = data[0][index:end_index]
    batch_price = np.transpose(np.asarray(batch_price), [1, 0, 2])
    if is_back_test:
        batch_target = data[2][index:end_index]
    else:
        batch_target = data[1][index:end_index]
    return batch_price, batch_target

def split_shuffle(train_data, test_data):
    train_data = list(train_data)
    random.shuffle(train_data)
    train_data = list(zip(*train_data))
    test_data = list(test_data)
    test_data = list(zip(*test_data))

    return train_data, test_data

def test(model, data, forex_config):
    total = 0
    correct_num = 0

    for i in range(int(len(data[0]) / forex_config.batch_size)):
        index = i * forex_config.batch_size
        batch_price, batch_target = get_batch(data, index, forex_config, is_back_test=False)
        outputs = model.predict(batch_price)
        total += len(outputs)
        # total += np.sum(max_logit)
        # test_preference = np.zeros(42)
        for j in range(len(outputs)):
            #	test_preference[outputs[j]]+=1
            if batch_target[j][outputs[j]] == 1:  # and max_logit[j] == 1:
                correct_num += 1
                # print ("preference_test {}".format(test_preference))
    rate = float(correct_num) / float(total)
    print("total {}, correct {}, rate {}".format(total, correct_num, rate))

def back_test(model, test_years, data, forex_config, num_step):
    target_period = 48
    start_fund = 100
    base_fund_per_period = start_fund
    total_reward_rate = 0
    total_reward_rate_list = []
    single_return = []
    total_return = []
    temp_total_return = []
    for i in range(int(len(data[0]) / forex_config.batch_size)):
        index = i * forex_config.batch_size
        batch_price, batch_target = get_batch(data, index, forex_config, is_back_test=True)
        outputs = model.predict(batch_price)
        for j in range(len(outputs)):
            total_reward_rate += batch_target[j][outputs[j]]  # reward
            # ---------- profit computation -----------
            single_return.append(batch_target[j][outputs[j]])
            tmp_total_reward_rate = total_reward_rate
            total_reward_rate_list.append(tmp_total_reward_rate)

    for i in range(len(single_return)):
        temp_total_return.append((base_fund_per_period / target_period) * (1 + single_return[i]))

        if i+1 % 48 == 0:
            print(i)
            temp = sum(temp_total_return)
            temp_return = (temp - start_fund)/ start_fund
            total_return.append(temp_return)
            base_fund_per_period = temp
            temp_total_return = []

            if start_fund < 0:
                print('strategy shut down')
                break

    print("total_reward_rate: {}".format(total_return[-1]))
    years_info = ""
    for year in test_years:
        years_info += str(year)[2:]
    fig = plt.figure()

    plt.plot(total_return)
    plt.xlabel('time tick')
    plt.ylabel('return rate')
    fig.savefig('graph/' + years_info + '_' + str(num_step+1) + '.png')

# plt.show()

def train(model, data, test_data, test_years, forex_config):
    limit_step = 5000
    for i in range(limit_step):
        index = (i * forex_config.batch_size) % len(data[0])
        batch_price, batch_target = get_batch(data, index, forex_config, is_back_test=False)
        loss = model.step(batch_price, batch_target)
        print("step " + str(i) + ",loss " + str(loss))
        if (i + 1) % forex_config.steps_per_checkpoint == 0:
            print("test val set...")
            test(model, test_data, forex_config)
            back_test(model, test_years, test_data, forex_config, i)
            print ("test train set...")
            test(model,data,forex_config)
            #back_test(model,data,forex_config)
            model.save(test_years, forex_config)


import string
from collections import Counter

from util.cipher_base import CipherBase

FREQUENCIES = {
	'E': 12.7, 'T': 9.06, 'A': 8.17, 'O': 7.51, 'I': 6.97, 'N': 6.75, 'S': 6.33, 'H': 6.09, 'R': 5.99,
	'D': 5.25, 'L': 4.03, 'C': 2.78, 'U': 2.76, 'M': 2.41, 'W': 2.36, 'F': 2.23, 'G': 2.02, 'Y': 1.97,
	'P': 1.93, 'B': 1.49, 'V': 0.98, 'K': 0.77, 'X': 0.15, 'J': 0.15, 'Q': 0.1, 'Z': 0.05,
}

class AffineCaesarCipher(CipherBase):

	def __init__(self):
		super().__init__()
		self.key_1 = None
		self.key_2 = None

	def encrypt(self, text):
		if not self.key_1 or not self.key_2:
			raise Exception('cipher: keys not set')
		cipher_text = ''
		for symbol in text.lower():
			if symbol in self.ALPHABET_EN:
				index = self.ALPHABET_EN.find(symbol)
				cipher_text += self.ALPHABET_EN[(index * self.key_1 + self.key_2) % self.en_alphabet_len]
			else:
				cipher_text += symbol
		return cipher_text

	def decrypt(self, text):
		if self.key_1 is None or self.key_2 is None:
			raise Exception('decipher: keys not set')
		plaintext = ''
		mod_inverse_of_key_1 = self.mod_inv(self.key_1, self.en_alphabet_len)
		for symbol in text.lower():
			if symbol in self.ALPHABET_EN:
				index = self.ALPHABET_EN.find(symbol)
				plaintext += self.ALPHABET_EN[(index - self.key_2) * mod_inverse_of_key_1 % self.en_alphabet_len]
			else:
				plaintext += symbol
		return plaintext

	def set_keys(self, key_1, key_2):
		self.check_keys(key_1, key_2)
		key_1 = int(key_1)
		key_2 = int(key_2)
		if key_1 < 0 or key_2 < 0 or key_2 > self.en_alphabet_len - 1:
			raise Exception('key_1 must be greater than 0 and key_2 must be between 0 and %s.' % (self.en_alphabet_len - 1))
		if self.e_gcd(key_1, self.en_alphabet_len)[0] != 1:
			raise Exception(
				'key_1 (%s) and the symbol set size (%s) are not relatively prime, choose a different key.'
				% (key_1, self.en_alphabet_len)
			)
		self.key_1 = key_1
		self.key_2 = key_2

	@staticmethod
	def check_keys(key_1, key_2):
		if isinstance(key_1, str) and isinstance(key_2, str):
			digits = string.digits
			for c in key_1:
				if c not in digits:
					raise Exception('first key contains non-numeric characters')

			for c in key_2:
				if c not in digits:
					raise Exception('second key contains non-numeric characters')
		elif isinstance(key_1, int) and isinstance(key_2, int):
			return
		else:
			raise TypeError('invalid key type')

	@staticmethod
	def mod_inv(num, mod):
		for x in range(0, mod + 1):
			if (num * x) % mod == 1:
				return x
		raise Exception('ERROR: modulo {} inverse of {} does not exists!'.format(mod, num))

	def e_gcd(self, a, b):
		if a == 0:
			return b, 0, 1
		else:
			g, y, x = self.e_gcd(b % a, a)
			return g, x - (b // a) * y, y

	@staticmethod
	def freq_attack(text, log):
		alphabet_len = len(AffineCaesarCipher.ALPHABET_EN)
		deciphered = ''
		key_1, key_2 = None, None
		max_sum = 0
		for i in range(alphabet_len):
			if (i % 2 != 0) and (i != 13):
				for j in range(0, alphabet_len):
					curr_sum = 0
					cipher_alg = AffineCaesarCipher()
					cipher_alg.set_keys(i, j)
					current_text = cipher_alg.decrypt(text)
					letters_freq = Counter(current_text.upper())
					for char in current_text.upper():
						if char in AffineCaesarCipher.ALPHABET_EN.upper():
							curr_sum += letters_freq[char] * FREQUENCIES[char]
					if curr_sum > max_sum:
						deciphered = current_text
						max_sum = curr_sum
						key_1 = i
						key_2 = j
		log('Keys <{}, {}>:\n\n{}'.format(key_1, key_2, deciphered))


#!/usr/bin/python3
import os
import re
import sys
import argparse
import configparser

def _argparse():
    parser = argparse.ArgumentParser(description="This is description")
    parser.add_argument('--input', action='store', dest='input_path', type=str, help = "the input folder with kraken2 log files.")
    parser.add_argument('--output', action='store', dest='output_path',type=str,  help = "the output file to write.")
    return parser.parse_args()

def main():
    parser = _argparse()
    abs_path = os.path.abspath(parser.input_path)
    output = open(parser.output_path, "w")
    output.write("sample\ttotal_reads\tconsumed_time\tclassified_reads\tpct\tunclassified_reads\tpct\n")
    pattern1 = re.compile('^(\d+).*in (\d+\.\d+\w).*')
    pattern2 = re.compile('\s+?(\d+)\ssequences\sclassified\s[(](\d+\.\d+%)[)]')
    pattern3 = re.compile('\s+?(\d+)\ssequences\sunclassified\s[(](\d+\.\d+%)[)]')
    files = [abs_path + '/' + each for each in os.listdir(abs_path) if each.endswith("log")]
    for each_file in files:
        print("Reading log file: {}".format(each_file))
        sample_name = os.path.basename(each_file).split(".")[0]
        output_content = []
        with open(each_file, "r") as f:
            for line in f:
                line = line.rstrip("\n")
                match1 = pattern1.match(line)
                if match1:
                    output_content.append(match1.group(1))
                    output_content.append(match1.group(2))
                    continue
                match2 = pattern2.match(line)
                if match2:
                    output_content.append(match2.group(1))
                    output_content.append(match2.group(2))
                    continue
                match3 = pattern3.match(line)
                if match3:
                    output_content.append(match3.group(1))
                    output_content.append(match3.group(2))
                    continue
        output.write("{}\t{}\n".format(sample_name, "\t".join(output_content)))
    print("Summary table written: {}".format(parser.output_path))
    output.close()

if __name__ == '__main__':
    main()


import os
import pandas as pd
import numpy as np
from tqdm import tqdm
import subprocess
import zipfile

PATH_TO_BUILDED_PROJECTS = os.path.abspath('./../../data/builded_projects/unpacked_zips')
OUTPUT_UNZIP = os.path.abspath('./../../data/builded_projects/unpacked_zips')
OUTPUT = os.path.abspath('./../../data/processed/new_class_graphs_10')
CSV_FILES = os.path.abspath('./../../data/processed/csv_files_traces')
PATH_TO_JAVA_PARSER = os.path.abspath('./../../../Java2Graph/javaparser-dloc/target/javaparser-dloc-3.5.14-SNAPSHOT-jar-with-dependencies.jar')
projects = list(filter(lambda x: x!= '.ipynb_checkpoints' and os.path.isdir(os.path.join(PATH_TO_BUILDED_PROJECTS, x)), os.listdir(PATH_TO_BUILDED_PROJECTS)))

for project_name in tqdm(projects):
    if project_name not in ['apache-ignite-2d2044a', 'flowable-flowable-engine-f9f4304']:
        continue

    directory_to_extract_to = os.path.join(OUTPUT_UNZIP, project_name)

    df = pd.read_csv('./../../data/builded_projects/good_methods_names_df.csv')
    df = df[df['repo_name'] == project_name]
    print(project_name)
    for g, df_ind in tqdm(df.groupby(np.arange(len(df)) // 100)):
        df_ind.to_csv(os.path.join(CSV_FILES, project_name + f'{g}.csv'), index=False)

        os.makedirs(os.path.join(OUTPUT, project_name), exist_ok=True)
#         os.makedirs(os.path.join(OUTPUT, project_name, 'call_graph'), exist_ok=True)
#         os.makedirs(os.path.join(OUTPUT, project_name, 'gml'), exist_ok=True)

        build_place = directory_to_extract_to
        output_folder = os.path.join(OUTPUT, project_name)
        csv = os.path.join(CSV_FILES, project_name + f'{g}.csv')
        run_arguments=[ 'java', '-Xmx12g', '-XX:-UseGCOverheadLimit', '-jar', PATH_TO_JAVA_PARSER ]
        run_arguments.append(build_place)
        run_arguments.append(output_folder)
        run_arguments.append(csv)
        run_arguments.append(build_place)
        run_arguments.append(project_name)

        subprocess.call(run_arguments)


from django.shortcuts import render
from rest_framework import generics, status
from .models import Marafon, Category
from .serializers import MarafonSerializer, CreateMarafonSerializer, CategorySerializer, CreateCategorySerializer, UserSerializer
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from django.contrib.auth.models import User

# Create your views here.
class MarafonView(generics.ListAPIView):
    queryset = Marafon.objects.all()
    serializer_class = MarafonSerializer

class CreateMarafonView(APIView):
    serializer_class = CreateMarafonSerializer
    def get(self,request, *args, **kwargs):
        id = request.query_params["id"]
        pr
    def post(self, request, format=None):
        if request.user.is_authenticated:
            if not self.request.session.exists(self.request.session.session_key):
                self.request.session.create()
            serializer = self.serializer_class(data=request.data)
            if serializer.is_valid():
                title = serializer.data.get('title')
                description = serializer.data.get('description')
                category = serializer.data.get('category')
                owner = self.request.user
                marafon = Marafon(owner=owner, title=title,description=description,category=category)
                CreateCategory(category)
                marafon.save()
            return Response(MarafonSerializer(marafon).data, status=status.HTTP_200_OK)
        else:
            return Response({"Unregistered: cannot access anonymous"})

class CategoryView(generics.ListAPIView):
    queryset = Category.objects.all()
    serializer_class = CategorySerializer

def CreateCategory(label_create):
    if Category.objects.filter(label = label_create.strip().title()):
        return
    else:
        category = Category(label = label_create)
        category.save()

class UserCreate(APIView):
    """
    Creates the user.
    """
    def post(self, request, format='json'):
        serializer = UserSerializer(data=request.data)
        if serializer.is_valid():
            user = serializer.save()
            if user:
                token = Token.objects.create(user=user)
                json = serializer.data
                json['token'] = token.key
                return Response(json, status=status.HTTP_201_CREATED)

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


import re

from PyQt5.QtCore import Qt

from backend import matting
from gui.general_window import *
from gui.general import *

import numpy as np

class matting_window(general_window):
    """
    This class describes a matting window.
    """

    def __init__(self, parent=None):
        general_window.__init__(self, lambda x: Image.fromarray((x * 255).astype(np.uint8)),
                                load_before=lambda: self.load_extra_now())
        self.method = matting.Matting()

    def get_int(self, field) -> int:
        """
        Makes sure the pixel format is correct

        Parameters
        ----------
        field : str
            Text to extract the number from

        Returns
        -------
        int
           int from the text
        """
        response = re.findall(r'\d+', field)
        if (len(response) == 0):
            return -1
        return int(response[0])

    def load_extra_now(self):
        """
        Showing the image
        """
        self.setAcceptDrops(True)
        self.label_, self.pixmap_ = self.add_image_draggable(self.method.source.data, (
            lambda x: self.pixmap_converter(x)) if not self.pixmap_converter is None else (
            lambda x: Image.fromarray(255 * x)), self)

        self.boundary_box, self.boundary_group_iamge_end = self.add_input_group("Start coordinates", [
            "x", "y"
        ], [
                lambda x: QTimer.singleShot(100,
                    lambda: self.update_image_size()),
                lambda x: QTimer.singleShot(100,
                    lambda: self.update_image_size())
        ])

        self.update_geometry(self.boundary_box.width(), 90, x=10)

        self.boundary_box_max, self.boundary_group_iamge = self.add_input_group("End coordinates", [
            "x1", "y1"
        ], [
            lambda x: QTimer.singleShot(100,
                lambda: self.update_image_size()),
            lambda x: QTimer.singleShot(100,
                lambda: self.update_image_size())
        ])
        self.update_geometry(self.boundary_box_max.width(), 90, x=10 + self.boundary_box.width(), y=-42)
        self.update_image_size()

    def update_image_size(self):
        """
        Update the image size based on the input size
        """
        x0, y0, x1, y1 = list(
            map(lambda x: self.get_int(x.text()), self.boundary_group_iamge_end + self.boundary_group_iamge))
        if x0 == -1:
            x0 = 0
        if y0 == -1:
            y0 = 0
        if x1 == -1:
            x1 = self.method.source.data.shape[1]
        if y1 == -1:
            y1 = self.method.source.data.shape[0]
        if x0 < x1 and y0 < y1:
            self.area = [[x0, x1], [y0, y1]]
            self.label_.setPixmap(
                pil2pixmap(Image.fromarray((255 * self.method.source.data[y0:y1, x0:x1]).astype(np.uint8))))

    def prepare(self):
        """
        Prepare the method before run
        """
        self.method.working_area = self.area
        self.method.padding = [self.label_.pos().x(), self.label_.pos().y()]
        self.label_.setVisible(False)

    def undo(self):
        """
        When the image is reset we need to show the source image again
        """
        self.label_.setVisible(True)

    def dragEnterEvent(self, event):
        """
        Drag event handler for the source image

        Parameters
        ----------
        event : QDragEnterEvent
            The drag event
        """
        event.accept()

    def dropEvent(self, event):
        """
        Drop event handler for the source image

        Parameters
        ----------
        event : QDropEvent
            The drop event
        """
        position = event.pos()
        self.label_.move(position)

        event.setDropAction(Qt.MoveAction)
        event.accept()


__author__ = 'Bohdan Mushkevych'

import psutil
from subprocess import PIPE

from settings import settings
from system.process_context import ProcessContext
from workers.abstract_cli_worker import AbstractCliWorker

class PigDriver(AbstractCliWorker):
    """Python process that starts Pig processing job, supervises its execution and updates unit_of_work"""

    def __init__(self, process_name):
        super(PigDriver, self).__init__(process_name)

    def _start_process(self, start_timeperiod, end_timeperiod, arguments):
        try:
            input_file = ProcessContext.get_source(self.process_name)

            self.logger.info('start: %s {' % self.process_name)
            p = psutil.Popen([settings['bash_shell'],
                              settings['pig_command'],
                              '-f', '/home/bmushkevych/git/synergy-pig/script.pig',
                              '-p', 'input_file=' + input_file + '/' + start_timeperiod,
                              '-p', 'timeperiod=' + start_timeperiod],
                             close_fds=True,
                             cwd=settings['process_cwd'],
                             stdin=PIPE,
                             stdout=PIPE,
                             stderr=PIPE)
            self.cli_process = p
            self.logger.info('Started %s with pid = %r' % (self.process_name, p.pid))
        except Exception:
            self.logger.error('Exception on starting: %s' % self.process_name, exc_info=True)
        finally:
            self.logger.info('}')


from django.urls import path
from django.conf.urls import url
from .views import (AlumniList,
AlumniDetail,
GraduationCreateView,
GraduationProjectCreateView,
CompanyCreateView,
CompanyUpdateView,
CompanyDeleteView,
JobCreateView,
GraduationUpdateView,
GraduationDeleteView,
GraduationProjectUpdateView,
GraduationProjectDeleteView,
JobUpdateView,
JobDeleteView,
AlumniEdit)

urlpatterns = [
	path('', AlumniList.as_view(), name='AlumniList'),
	path('<int:pk>/', AlumniDetail.as_view(), name='AlumniDetail'),
	path('edit/<int:pk>/', AlumniEdit.as_view(), name='editAlumni'),
	path('<int:pk>/graduation/', GraduationCreateView.as_view(), name='newGraduation'),
	path('graduation/edit/<int:pk>/', GraduationUpdateView.as_view(), name='editGraduation'),
	path('graduation/delete/<int:pk>/', GraduationDeleteView.as_view(), name='deleteGraduation'),
	path('<int:pk>/graduation/project/edit/', GraduationProjectUpdateView.as_view(), name='editProjGraduation'),
	path('<int:pk>/graduation/project/delete/', GraduationProjectDeleteView.as_view(), name='deleteProjGraduation'),
	path('<int:pk>/company/', CompanyCreateView.as_view(), name='newCompany'),
	path('company/edit/<int:pk>/', CompanyUpdateView.as_view(), name='editCompany'),
	path('company/delete/<int:pk>/', CompanyDeleteView.as_view(), name='deleteCompany'),
	path('graduation/project/<int:pk>/', GraduationProjectCreateView.as_view(), name='newProject'),
	path('job/<int:pk>/', JobCreateView.as_view(), name='newJob'),
	path('job/edit/<int:pk>/', JobUpdateView.as_view(), name='editJob'),
	path('job/delete/<int:pk>/', JobDeleteView.as_view(), name='deleteJob'),
]


import datetime
import math
from itertools import groupby
from operator import itemgetter
from datetime import timedelta
from openerp import models, fields, api, exceptions, _

def merge_lists(list1, list2):
    # pomozna funkcija, ki zdruzi attendances in leaves
    # list 1 in 2 sta seznama seznamov ali pa sta prazna
    list1_cols = len(list1[0][1:]) if list1 else 0
    list2_cols = len(list2[0][1:]) if list2 else 0
    dic1 = {col[0]: col[1:] for col in list1}
    dic2 = {col[0]: col[1:] for col in list2}
    for k, value in dic2.items():
        if dic1.has_key(k):
            dic1[k] += value
        else:
            dic1[k] = [0] * list1_cols + value
    list_result = []
    for key, value in dic1.items():
        if len(value) == list1_cols:
            list_result.append([key] + value + [0] * list2_cols)
        else:
            list_result.append([key] + value)
    list_result.sort()
    return list_result

class AttendanceReport(models.TransientModel):
    _inherit = 'attendance_report'

    @api.model
    def attendance_employees(self):
        employee_list = super(AttendanceReport, self).attendance_employees()
        if not self.employee_id:
            # todo: filtriraj se po state je validate
            HrHolidays = self.env['hr.holidays']
            holidays_obj = HrHolidays.search([
                '&',
                '|',
                '&',
                ('date_from', '>=', self.date_from),
                ('date_from', '<=', self.date_to),
                '&',
                ('date_to', '>=', self.date_from),
                ('date_to', '<=', self.date_to),
                ('state', '=', 'validate')
            ])
            holiday_employees = holidays_obj.mapped('employee_id')
            employee_list = list(set(employee_list + holiday_employees)) # odstranim duplikate
        return employee_list

    @api.model
    def was_on_leave(self, employee_id, datetime_day):
        '''
        To sem spremenil,ker moram upostevati, da je na en dan lahko vec leavov
        funkcija zdaj vrne vse leave na dolocen dan in njihovo trajanje
        '''

        day = datetime_day.strftime("%Y-%m-%d")
        holiday_ids = self.env['hr.holidays'].search([('state','=','validate'),
                                                      ('employee_id','=',employee_id),
                                                      ('type','=','remove'),
                                                      ('date_from','<=',day),('date_to','>=',day)])
        return  holiday_ids

    @api.model
    def attendance_employee_pivot_lines(self, employee_id):
        # linijam iz attendance, dodam tiste iz leavov
        self.ensure_one()
        start_hour, start_min, end_hour, end_min, working_hours_on_day = employee_id.get_working_hours()
        working_hours_on_day = working_hours_on_day * 60

        # preberem vrstice, ki jim bom dodal stolpce za leave
        header_line, attendance_lines, summary_line, recapitulation_lines = \
            super(AttendanceReport,self).attendance_employee_pivot_lines(employee_id)

        leave_temp_lines = []
        add_leave_lines = [] # vrstice, ki jih dodam attendances
        leave_types = set() # vsi razlicni leavi
        sum_validated_leaves = 0 # sestevek vseh leavov

        day_from = datetime.datetime.strptime(self.date_from,"%Y-%m-%d")
        day_to = datetime.datetime.strptime(self.date_to,"%Y-%m-%d")
        nb_of_days = (day_to - day_from).days + 1
        for day in range(0, nb_of_days):
            d = day_from + timedelta(days=day)
            leave_ids = self.was_on_leave(employee_id.id, d)
            for leave in leave_ids:
                leave_types.add(leave.holiday_status_id.name)
                leave_day_from = datetime.datetime.strptime(leave.date_from,'%Y-%m-%d %H:%M:%S')
                leave_day_to = datetime.datetime.strptime(leave.date_to,'%Y-%m-%d %H:%M:%S')
                hours = int(math.ceil( (leave_day_to - leave_day_from).seconds / 3600.00 ))

                if hours > working_hours_on_day: # leave je bil daljsi kot stevilo delovnih ur, lahko je trajal vec dni
                    hours = working_hours_on_day
                leave_temp_lines.append({
                    'name': datetime.datetime.strftime(d, '%Y-%m-%d'),
                    'action_desc' : leave.holiday_status_id.name,
                    'time_in_action': hours * 60,
                })
                sum_validated_leaves += hours * 60
        # zacasen seznam je napolnjen, sedaj, ga je potrebno pivotirati
        if not leave_types: # ce ni bilo nobene odsotnosti preprosto vrnem
            return (header_line, attendance_lines, summary_line, recapitulation_lines)

        leave_types = list(leave_types) # rabim list, ker set ne gre sortirat
        leave_types.sort()
        leave_temp_lines.sort(key=itemgetter('name'))
        add_header_line = leave_types                   # dodam obstojeci glavi
        add_summary_line = [0] * len(leave_types)       # dodam obstojecemu povzetku
        for leave_date, leave_group in groupby(leave_temp_lines, itemgetter('name')):
            result_line_head = [leave_date]
            result_line_tail = [0] * len(leave_types)
            for holiday in leave_group:
                position = leave_types.index(holiday['action_desc'])
                result_line_tail[position] += holiday['time_in_action']
                add_summary_line[position] += holiday['time_in_action']
            add_leave_lines.append(result_line_head + result_line_tail)

        header_line += add_header_line
        summary_line += add_summary_line
        attendance_lines = merge_lists(attendance_lines, add_leave_lines) # iz dveh seznamov naredim enega, mergam po datumu
        # na rekapitulacijo dodam sestevek vseh leavov za zaposlenenga
        recapitulation_lines.insert(len(recapitulation_lines) - 1, (_('Validated leaves'), sum_validated_leaves))
        # povecam razliko do polnega delovnega casa, ker grejo validated leavei vsi v delovni cas
        # ker ne morem spreminjati tupla, ga zamenjam z novim
        diff = recapitulation_lines[len(recapitulation_lines) -1][1] +  sum_validated_leaves
        recapitulation_lines[len(recapitulation_lines) - 1] =(_('Difference'), diff)
        return (header_line, attendance_lines, summary_line, recapitulation_lines)