#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Spirou Least-Squares Deconvolution (LSD) analysis module

Created on 2018-08-08 at 14:53

@author: E. Martioli

"""
from __future__ import division
import numpy as np
from scipy import constants
from scipy.optimize import curve_fit
import os

from SpirouDRS import spirouConfig
from SpirouDRS import spirouCore
from SpirouDRS import spirouImage

# =============================================================================
# Define variables
# =============================================================================
# Name of program
__NAME__ = 'spirouLSD.py'
# Get version and author
__version__ = spirouConfig.Constants.VERSION()
__author__ = spirouConfig.Constants.AUTHORS()
__date__ = spirouConfig.Constants.LATEST_EDIT()
__release__ = spirouConfig.Constants.RELEASE()
# -----------------------------------------------------------------------------
# Get Logging function
WLOG = spirouCore.wlog
# Get the parameter dictionary class
ParamDict = spirouConfig.ParamDict
# get the config error
ConfigError = spirouConfig.ConfigError
# Get plotting functions
sPlt = spirouCore.sPlt

# =============================================================================
# Define user functions
# =============================================================================

def lsd_analysis_wrapper(p, loc):
    """
        Function to call functions to perform Least Squares Deconvolution (LSD)
        analysis on the polarimetry data.

        :param p: parameter dictionary, ParamDict containing constants
        Must contain at least:
        LOG_OPT: string, option for logging

        :param loc: parameter dictionary, ParamDict containing data

        :return loc: parameter dictionary, the updated parameter dictionary
        Adds/updates the following:

        """
    # func_name = __NAME__ + '.lsd_analysis_wrapper()'
    name = 'LSDAnalysis'

    # log start of LSD analysis calculations
    wmsg = 'Running function {0} to perform LSD analysis'
    WLOG(p, 'info', wmsg.format(name))

    # load spectral lines
    loc = load_lsd_spectral_lines(p, loc)

    # get wavelength ranges covering spectral lines in the ccf mask
    loc = get_wl_ranges(p, loc)

    # prepare polarimetry data
    loc = prepare_polarimetry_data(p, loc)

    # call function to perform lsd analysis
    loc = lsd_analysis(p, loc)

    return loc

def load_lsd_spectral_lines(p, loc):
    """
    Function to load spectral lines data for LSD analysis.

    :param p: parameter dictionary, ParamDict containing constants
        Must contain at least:
            LOG_OPT: string, option for logging
            IC_POLAR_LSD_CCFLINES: list of strings, list of files containing
                                   spectral lines data
            IC_POLAR_LSD_WLRANGES: array of float pairs for wavelength ranges
            IC_POLAR_LSD_MIN_LINEDEPTH: float, line depth threshold
    :param loc: parameter dictionary, ParamDict to store data

    :return loc: parameter dictionaries,
        The updated parameter dictionary adds/updates the following:
        loc['SELECTED_FILE_CCFLINES']: string, selected filename with CCF lines
        loc['LSD_LINES_WLC']: numpy array (1D), central wavelengths
        loc['LSD_LINES_ZNUMBER']: numpy array (1D), atomic number (Z)
        loc['LSD_LINES_DEPTH']: numpy array (1D), line depths
        loc['LSD_LINES_POL_WEIGHT']: numpy array (1D), line weights =
                                     depth * lande * wlc
    """

    func_name = __NAME__ + '.load_lsd_spectral_lines()'

    # get package name and relative path
    package = spirouConfig.Constants.PACKAGE()
    relfolder = spirouConfig.Constants.LSD_MASK_DIR()
    # get absolute folder path from package and relfolder
    absfolder = spirouConfig.GetAbsFolderPath(package, relfolder)

    # get object temperature from header
    obj_temperature = float(loc['HDR']['OBJTEMP'])
    wmsg = 'Temperature of the object observed: {0} K'
    WLOG(p, '', wmsg.format(obj_temperature))

    # find out which CCFLINE file is most appropriate for source
    temp_diff_min, loc['SELECTED_FILE_CCFLINES'] = 1.e10, 'marcs_t3000g50_all'
    for i in range(len(p['IC_POLAR_LSD_CCFLINES'])):
        filename = p['IC_POLAR_LSD_CCFLINES'][i]
        suffix = filename.split('marcs_t')[1]
        temp_in_file = float(suffix[0:suffix.find('g50_all')])
        temp_diff = np.abs(obj_temperature - temp_in_file)
        if temp_diff < temp_diff_min:
            temp_diff_min = temp_diff
            # get filename corresponding to the closest temperature to object
            loc['SELECTED_FILE_CCFLINES'] = filename

    # get absolute path and filename
    abspath = os.path.join(absfolder, loc['SELECTED_FILE_CCFLINES'])
    # if path exists use it
    if os.path.exists(abspath):
        wmsg = 'Line mask used for LSD computation: {0}'
        WLOG(p, '', wmsg.format(abspath))
        # load spectral lines data from file
        wlcf, znf, depthf, landef = np.loadtxt(abspath, delimiter='  ',
                                               skiprows=1,
                                               usecols=(0, 1, 2, 3),
                                               unpack=True)
    # else raise error
    else:
        emsg1 = 'LSD Line mask file: "{0}" not found, unable to proceed'
        emsg2 = '    function = {0}'.format(func_name)
        eargs = [loc['SELECTED_FILE_CCFLINES']]
        WLOG(p, 'error', [emsg1.format(*eargs), emsg2])
        wlcf, znf, depthf, landef = None, None, None, None

    # initialize data vectors
    wlc, zn, depth, lande = [], [], [], []
    # loop over spectral ranges to select only spectral lines within ranges
    for wlrange in p['IC_POLAR_LSD_WLRANGES']:
        # set initial and final wavelengths in range
        wl0, wlf = wlrange[0], wlrange[1]
        # create wavelength mask to limit wavelength range
        mask = np.where(np.logical_and(wlcf > wl0, wlcf < wlf))
        wlc = np.append(wlc, wlcf[mask])
        zn = np.append(zn, znf[mask])
        depth = np.append(depth, depthf[mask])
        lande = np.append(lande, landef[mask])

    # PS. Below it applies a line depth mask, however the cut in line depth
    # should be done according to the SNR. This will be studied and implemented
    # later. E. Martioli, Aug 10 2018.

    # create mask to cut lines with depth lower than IC_POLAR_LSD_MIN_LINEDEPTH
    dmask = np.where(depth > p['IC_POLAR_LSD_MIN_LINEDEPTH'])
    # apply mask to the data
    wlc, zn, depth, lande = wlc[dmask], zn[dmask], depth[dmask], lande[dmask]

    # calculate weights for calculation of polarimetric Z-profile
    weight = wlc * depth * lande
    weight = weight / np.max(weight)

    # store data into loc dict
    loc['LSD_LINES_WLC'] = wlc
    loc['LSD_LINES_ZNUMBER'] = zn
    loc['LSD_LINES_DEPTH'] = depth
    loc['LSD_LINES_POL_WEIGHT'] = weight

    return loc

def get_wl_ranges(p, loc):
    """
    Function to generate a list of spectral ranges covering all spectral
    lines in the CCF mask, where the width of each individual range is
    defined by the LSD velocity vector

    :param p: parameter dictionary, ParamDict containing constants
        Must contain at least:
            LOG_OPT: string, option for logging
            IC_POLAR_LSD_V0: initial velocity for LSD profile
            IC_POLAR_LSD_VF: final velocity for LSD profile

    :param loc: parameter dictionary, ParamDict to store data
        loc['LSD_LINES_WLC']: numpy array (1D), central wavelengths

    :return loc: parameter dictionaries,
        The updated parameter dictionary adds/updates the following:
        loc['LSD_LINES_WLRANGES']: array of float pairs for wavelength ranges

    """
    # func_name = __NAME__ + '.get_wl_ranges()'

    # speed of light in km/s
    c = constants.c / 1000.
    # set initial and final velocity
    v0, vf = p['IC_POLAR_LSD_V0'], p['IC_POLAR_LSD_VF']
    # define vector of spectral ranges covering only regions around lines
    wlranges_tmp = []
    for w in loc['LSD_LINES_WLC']:
        dwl = w * (vf - v0) / (2. * c)
        wl0 = w - dwl
        wlf = w + dwl
        wlranges_tmp.append([wl0, wlf])
    # initialize final vector of spectral ranges
    loc['LSD_LINES_WLRANGES'] = []
    # initialize current wl0 and wlf
    current_wl0, current_wlf = wlranges_tmp[0][0], wlranges_tmp[0][1]
    # merge overlapping ranges
    for r in wlranges_tmp:
        if r[0] <= current_wlf:
            current_wlf = r[1]
        else:
            loc['LSD_LINES_WLRANGES'].append([current_wl0, current_wlf])
            current_wl0 = r[0]
            current_wlf = r[1]
    # append last range
    loc['LSD_LINES_WLRANGES'].append([current_wl0, current_wlf])

    return loc

def prepare_polarimetry_data(p, loc):
    """
    Function to prepare polarimetry data for LSD analysis.

    :param p: parameter dictionary, ParamDict containing constants
        Must contain at least:
            LOG_OPT: string, option for logging
            IC_POLAR_LSD_NORMALIZE: bool, normalize Stokes I data

    :param loc: parameter dictionary, ParamDict to store data
        Must contain at least:
            loc['WAVE']: numpy array (2D), wavelength data
            loc['STOKESI']: numpy array (2D), Stokes I data
            loc['STOKESIERR']: numpy array (2D), errors of Stokes I
            loc['POL']: numpy array (2D), degree of polarization data
            loc['POLERR']: numpy array (2D), errors of degree of polarization
            loc['NULL2']: numpy array (2D), 2nd null polarization

    :return loc: parameter dictionaries,
        The updated parameter dictionary adds/updates the following:
            loc['LSD_WAVE']: numpy array (1D), wavelength data
            loc['LSD_STOKESI']: numpy array (1D), Stokes I data
            loc['LSD_STOKESIERR']: numpy array (1D), errors of Stokes I
            loc['LSD_POL']: numpy array (1D), degree of polarization data
            loc['LSD_POLERR']: numpy array (1D), errors of polarization
            loc['LSD_NULL']: numpy array (1D), 2nd null polarization

    """

    # func_name = __NAME__ + '.prepare_polarimetry_data()'

    # get the shape of pol
    ydim, xdim = loc['POL'].shape
    # get wavelength ranges to be considered in each spectral order
    ordermask = get_order_ranges()
    # initialize output data vectors
    loc['LSD_WAVE'], loc['LSD_STOKESI'], loc['LSD_STOKESIERR'] = [], [], []
    loc['LSD_POL'], loc['LSD_POLERR'], loc['LSD_NULL'] = [], [], []

    # loop over each order
    for order_num in range(ydim):
        # mask NaN values
        nanmask = np.where(np.logical_and(~np.isnan(loc['STOKESI'][order_num]),
                                          ~np.isnan(loc['POL'][order_num])))

        wl_tmp = loc['WAVE'][order_num][nanmask]
        pol_tmp = loc['POL'][order_num][nanmask]
        polerr_tmp = loc['POLERR'][order_num][nanmask]
        flux_tmp = loc['STOKESI'][order_num][nanmask]
        fluxerr_tmp = loc['STOKESIERR'][order_num][nanmask]
        null_tmp = loc['NULL2'][order_num][nanmask]

        # set order wavelength limits
        wl0, wlf = ordermask[order_num][0], ordermask[order_num][1]
        # create wavelength mask
        mask = np.where(np.logical_and(wl_tmp > wl0, wl_tmp < wlf))

        # test if order is not empty
        if len(wl_tmp[mask]):
            # get masked data
            wl, flux, fluxerr = wl_tmp[mask], flux_tmp[mask], fluxerr_tmp[mask]
            pol, polerr, null = pol_tmp[mask], polerr_tmp[mask], null_tmp[mask]

            if p['IC_POLAR_LSD_NORMALIZE']:
                # measure continuum
                # TODO: Should be in constant file
                kwargs = dict(binsize=30, overlap=15, window=2,
                              mode='median', use_linear_fit=True)
                continuum, xbin, ybin = spirouCore.Continuum(wl, flux, **kwargs)
                # normalize flux
                flux = flux / continuum

            # append data to output vector
            loc['LSD_WAVE'] = np.append(loc['LSD_WAVE'], wl)
            loc['LSD_STOKESI'] = np.append(loc['LSD_STOKESI'], flux)
            loc['LSD_STOKESIERR'] = np.append(loc['LSD_STOKESIERR'], fluxerr)
            loc['LSD_POL'] = np.append(loc['LSD_POL'], pol)
            loc['LSD_POLERR'] = np.append(loc['LSD_POLERR'], polerr)
            loc['LSD_NULL'] = np.append(loc['LSD_NULL'], null)

    # sort data by wavelength
    indices = loc['LSD_WAVE'].argsort()
    loc['LSD_WAVE'] = loc['LSD_WAVE'][indices]
    loc['LSD_STOKESI'] = loc['LSD_STOKESI'][indices]
    loc['LSD_STOKESIERR'] = loc['LSD_STOKESIERR'][indices]
    loc['LSD_POL'] = loc['LSD_POL'][indices]
    loc['LSD_POLERR'] = loc['LSD_POLERR'][indices]
    loc['LSD_NULL'] = loc['LSD_NULL'][indices]

    # apply barycentric RV correction to the wavelength vector
    # TODO: Should be realivistic correction?
    rv_corr = 1.0 + loc['BERVCEN'] / (constants.c / 1000.)
    loc['LSD_WAVE'] = loc['LSD_WAVE'] * rv_corr

    # initialize temporary data vectors
    wl, flux, fluxerr, pol, polerr, null = [], [], [], [], [], []
    # loop over spectral ranges to select only spectral regions of interest
    for wlrange in loc['LSD_LINES_WLRANGES']:
        # set initial and final wavelengths in range
        wl0, wlf = wlrange[0], wlrange[1]

        # create wavelength mask to limit wavelength range
        wlmask = np.where(np.logical_and(loc['LSD_WAVE'] > wl0,
                                         loc['LSD_WAVE'] < wlf))
        wl = np.append(wl, loc['LSD_WAVE'][wlmask])
        flux = np.append(flux, loc['LSD_STOKESI'][wlmask])
        fluxerr = np.append(fluxerr, loc['LSD_STOKESIERR'][wlmask])
        pol = np.append(pol, loc['LSD_POL'][wlmask])
        polerr = np.append(polerr, loc['LSD_POLERR'][wlmask])
        null = np.append(null, loc['LSD_NULL'][wlmask])

    # update loc data vectors
    loc['LSD_WAVE'] = wl
    loc['LSD_STOKESI'] = flux
    loc['LSD_STOKESIERR'] = fluxerr
    loc['LSD_POL'] = pol
    loc['LSD_POLERR'] = polerr
    loc['LSD_NULL'] = null

    return loc

def lsd_analysis(p, loc):
    """
    Function to perform Least Squares Deconvolution (LSD) analysis.

    :param p: parameter dictionary, ParamDict containing constants
        Must contain at least:
            LOG_OPT: string, option for logging

    :param loc: parameter dictionary, ParamDict to store data
        Must contain at least:
            loc['IC_POLAR_LSD_V0']: initial velocity for LSD profile
            loc['IC_POLAR_LSD_VF']: final velocity for LSD profile
            loc['IC_POLAR_LSD_NP']: number of points in the LSD profile
            loc['LSD_WAVE']: numpy array (1D), wavelength data
            loc['LSD_STOKESI']: numpy array (1D), Stokes I data
            loc['LSD_STOKESIERR']: numpy array (1D), errors of Stokes I
            loc['LSD_POL']: numpy array (1D), degree of polarization data
            loc['LSD_POLERR']: numpy array (1D), errors of polarization
            loc['LSD_NULL']: numpy array (1D), 2nd null polarization
            loc['LSD_LINES_WLC']: numpy array (1D), central wavelengths
            loc['LSD_LINES_DEPTH']: numpy array (1D), line depths
            loc['LSD_LINES_POL_WEIGHT']: numpy array (1D), line weights =
                                         depth * lande * wlc

    :return loc: parameter dictionaries,
        The updated parameter dictionary adds/updates the following:
            loc['LSD_VELOCITIES']: numpy array (1D), LSD profile velocities
            loc['LSD_STOKESI']: numpy array (1D), LSD profile for Stokes I
            loc['LSD_STOKESI_MODEL']: numpy array (1D), LSD gaussian model
                                      profile for Stokes I
            loc['LSD_STOKESVQU']: numpy array (1D), LSD profile for Stokes
                                  Q,U,V polarimetry spectrum
            loc['LSD_NULL']: numpy array (1D), LSD profile for null
                                  polarization spectrum

    """

    # func_name = __NAME__ + '.lsd_analysis()'

    # initialize variables to define velocity vector of output LSD profile
    v0, vf, m = p['IC_POLAR_LSD_V0'], p['IC_POLAR_LSD_VF'], p['IC_POLAR_LSD_NP']

    # create velocity vector for output LSD profile
    loc['LSD_VELOCITIES'] = np.linspace(v0, vf, m)

    # create line pattern matrix for flux LSD
    mm, mmp = line_pattern_matrix(loc['LSD_WAVE'], loc['LSD_LINES_WLC'],
                                  loc['LSD_LINES_DEPTH'],
                                  loc['LSD_LINES_POL_WEIGHT'],
                                  loc['LSD_VELOCITIES'])

    # calculate flux LSD profile
    loc['LSD_STOKESI'] = calculate_lsd_profile(loc['LSD_WAVE'],
                                               loc['LSD_STOKESI'],
                                               loc['LSD_STOKESIERR'],
                                               loc['LSD_VELOCITIES'], mm,
                                               normalize=False)

    # fit gaussian to the measured flux LSD profile
    loc['LSD_STOKESI_MODEL'], loc['LSD_FIT_RV'], loc[
        'LSD_FIT_RESOL'] = fit_gaussian_to_lsd_profile(loc['LSD_VELOCITIES'],
                                                       loc['LSD_STOKESI'])

    # calculate polarimetry LSD profile
    loc['LSD_STOKESVQU'] = calculate_lsd_profile(loc['LSD_WAVE'],
                                                 loc['LSD_POL'],
                                                 loc['LSD_POLERR'],
                                                 loc['LSD_VELOCITIES'], mmp)

    # calculate null polarimetry LSD profile
    loc['LSD_NULL'] = calculate_lsd_profile(loc['LSD_WAVE'], loc['LSD_NULL'],
                                            loc['LSD_POLERR'],
                                            loc['LSD_VELOCITIES'], mmp)

    # calculate statistical quantities
    loc['LSD_POL_MEAN'] = np.nanmean(loc['LSD_POL'])
    loc['LSD_POL_STDDEV'] = np.nanstd(loc['LSD_POL'])
    loc['LSD_POL_MEDIAN'] = np.nanmedian(loc['LSD_POL'])
    loc['LSD_POL_MEDABSDEV'] = np.nanmedian(np.abs(loc['LSD_POL'] -
                                                loc['LSD_POL_MEDIAN']))
    loc['LSD_STOKESVQU_MEAN'] = np.nanmean(loc['LSD_STOKESVQU'])
    loc['LSD_STOKESVQU_STDDEV'] = np.nanstd(loc['LSD_STOKESVQU'])
    loc['LSD_NULL_MEAN'] = np.nanmean(loc['LSD_NULL'])
    loc['LSD_NULL_STDDEV'] = np.nanstd(loc['LSD_NULL'])

    return loc

def line_pattern_matrix(wl, wlc, depth, weight, vels):
    """
    Function to calculate the line pattern matrix M given in Eq (4) of paper
    Donati et al. (1997), MNRAS 291, 658-682

    :param wl: numpy array (1D), input wavelength data (size n = spectrum size)
    :param wlc: numpy array (1D), central wavelengths (size = number of lines)
    :param depth: numpy array (1D), line depths (size = number of lines)
    :param weight: numpy array (1D), line polar weights (size = number of lines)
    :param vels: numpy array (1D), , LSD profile velocity vector (size = m)

    :return mm, mmp
        mm: numpy array (2D) of size n x m, line pattern matrix for flux LSD.
        mmp: numpy array (2D) of size n x m, line pattern matrix for polar LSD.
    """

    # set number of points and velocity (km/s) limits in LSD profile
    m, v0, vf = len(vels), vels[0], vels[-1]

    # speed of light in km/s
    c = constants.c / 1000.

    # set number of spectral points
    n = len(wl)

    # initialize line pattern matrix for flux LSD
    mm = np.zeros((n, m))

    # initialize line pattern matrix for polar LSD
    mmp = np.zeros((n, m))

    # set first i=0 -> trick to improve speed
    i0 = 0
    # set values of line pattern matrix M
    for l in range(len(wlc)):
        noi0 = True
        for i in range(i0, n):
            # Calculate line velocity: v = c Δλ / λ
            v = c * (wl[i] - wlc[l]) / wlc[l]
            if v0 <= v <= vf:
                # below is a trick to improve speed
                if noi0:
                    # next spectral line starts with first i of previous line
                    # warning: list of CCF lines must be sorted by wavelength
                    i0 = i
                    noi0 = False
                for j in range(m - 1):
                    if vels[j] <= v < vels[j + 1]:
                        mmp[i][j] += weight[l]
                        mm[i][j] += depth[l]
                        if mm[i][j] > 1.0:
                            mm[i][j] = 1.0
                        break
            elif v > vf:
                break
    return mm, mmp

def calculate_lsd_profile(wl, flux, fluxerr, vels, mm, normalize=False):
    """
    Function to calculate the LSD profile Z given in Eq (4) of paper
    Donati et al. (1997), MNRAS 291, 658-682

    :param wl: numpy array (1D), input wavelength data (size = n)
    :param flux: numpy array (1D), input flux or polarimetry data (size = n)
    :param fluxerr: numpy array (1D), input flux or polarimetry error data
                    (size = n)
    :param vels: numpy array (1D), , LSD profile velocity vector (size = m)
    :param mm: numpy array (2D) of size n x m, line pattern matrix for LSD.
    :param normalize: bool, to calculate a continuum and normalize profile

    :return Z: numpy array (1D) of size m, LSD profile.
    """

    # set number of spectral points
    # noinspection PyUnusedLocal
    n = len(wl)

    # First calculate transpose of M
    mmt = np.matrix.transpose(mm)

    # Initialize matrix for dot product between MT . S^2
    mmt_x_s2 = np.zeros_like(mmt)

    # Then calculate dot product between MT . S^2, where S^2=covariance matrix
    for j in range(np.shape(mmt)[0]):
        mmt_x_s2[j] = mmt[j] / fluxerr ** 2

    # calculate autocorrelation, i.e., MT . S^2 . M
    mmt_x_s2_x_mm = mmt_x_s2.dot(mm)

    # calculate the inverse of autocorrelation using numpy pinv method
    mmt_x_s2_x_mm_inv = np.linalg.pinv(mmt_x_s2_x_mm)

    # calculate cross correlation term, i.e. MT . S^2 . Y
    x_corr_term = mmt_x_s2.dot(flux)

    # calculate velocity profile
    zz = mmt_x_s2_x_mm_inv.dot(x_corr_term)
    # recover last point
    zz[-1] = np.nanmedian(zz[-6:-2])

    if normalize:
        # calculate continuum of LSD profile to remove trend
        cont_z, xbin, ybin = spirouCore.Continuum(vels, zz, binsize=20,
                                                  overlap=5,
                                                  sigmaclip=3.0, window=2,
                                                  mode="median",
                                                  use_linear_fit=False)
        # calculate normalized and detrended LSD profile
        zz /= cont_z

    return zz

def gauss_function(x, a, x0, sigma):
    return a * np.exp(-(x - x0) ** 2 / (2. * sigma ** 2))

def fit_gaussian_to_lsd_profile(vels, zz):
    """
        Function to fit gaussian to LSD Stokes I profile.

        :param vels: numpy array (1D), input velocity data
        :param zz: numpy array (1D), input LSD profile data

        :return z_gauss, RV, resolving_power:
            z_gauss: numpy array (1D), gaussian fit to LSD profile (same size
                    as input vels and Z)
            RV: float, velocity of minimum obtained from gaussian fit
            resolving_power: float, spectral resolving power calculated from
                            sigma of gaussian fit
        """

    # set speed of light in km/s
    c = constants.c / 1000.

    # obtain velocity at minimum, amplitude, and sigma for initial guess
    rvel = vels[np.argmin(zz)]
    amplitude = 1.0 - np.min(zz)
    resolving_power = 50000.
    sig = c / (resolving_power * 2.3548)

    # get inverted profile
    z_inv = 1.0 - zz

    # fit gaussian profile
    guess = [amplitude, rvel, sig]
    # noinspection PyTypeChecker
    popt, pcov = curve_fit(gauss_function, vels, z_inv, p0=guess)

    # initialize output profile vector
    z_gauss = np.zeros_like(vels)

    for i in range(len(z_gauss)):
        # calculate gaussian model profile
        z_gauss[i] = gauss_function(vels[i], *popt)

    # invert fit profile
    z_gauss = 1.0 - z_gauss

    # calculate full width at half maximum (fwhm)
    fwhm = 2.35482 * popt[2]
    # calculate resolving power from mesasured fwhm
    resolving_power = c / fwhm

    # set radial velocity directly from fitted v_0
    rv = popt[1]

    return z_gauss, rv, resolving_power

def get_order_ranges():
    """
    Function to provide the valid wavelength ranges for each order in SPIrou.

    :param: None

    :return orders: array of float pairs for wavelength ranges
    """
    # TODO: Should be moved to file in ...misc/INTROOT/SpirouDRS/data/
    orders = [[963.6, 986.0], [972.0, 998.4], [986.3, 1011], [1000.1, 1020],
              [1015, 1035], [1027.2, 1050], [1042, 1065], [1055, 1078],
              [1070, 1096],
              [1084, 1112.8], [1098, 1128], [1113, 1146], [1131, 1162],
              [1148, 1180],
              [1166, 1198], [1184, 1216], [1202, 1235], [1222, 1255],
              [1243, 1275],
              [1263, 1297], [1284, 1320], [1306, 1342], [1328, 1365],
              [1352, 1390],
              [1377, 1415], [1405, 1440], [1429, 1470], [1456, 1497],
              [1485, 1526],
              [1515, 1557], [1545, 1590], [1578, 1623], [1609, 1657],
              [1645, 1692],
              [1681, 1731], [1722, 1770], [1760, 1810], [1800, 1855],
              [1848, 1900],
              [1890, 1949], [1939, 1999], [1991, 2050], [2044.5, 2105],
              [2104, 2162],
              [2161, 2226], [2225, 2293], [2291, 2362], [2362, 2430],
              [2440, 2510]]
    return orders

def output_lsd_image(p, loc, hdict):
    """
    Function to set up and save output FITS image to store LSD analyis.

    :param p: parameter dictionary, ParamDict containing constants
    :param loc: parameter dictionary, ParamDict to store data
        Must contain at least:
            loc['LSDDATA']: numpy array (2D), LSD analysis data
    :param hdict: dictionary, header dictionary of keywordstores
                  each key is the HEADER key and each value is a list of
                  two values: [HEADER value, HEADER comment]

    :return lsdfits, lsdfitsfitsname:
        lsdfits:         string, output full path
        lsdfitsfitsname: string, output filename
    """

    # construct file names
    lsdfits, tag = spirouConfig.Constants.LSD_POL_FILE(p, loc)
    lsdfitsfitsname = os.path.split(lsdfits)[-1]

    columns = ['velocities', 'stokesI', 'stokesI_model', 'stokesVQU', 'Null']
    values = [loc['LSD_VELOCITIES'], loc['LSD_STOKESI'],
              loc['LSD_STOKESI_MODEL'], loc['LSD_STOKESVQU'],
              loc['LSD_NULL']]

    # save all data into a single array for output FITS
    loc['LSDDATA'] = []
    loc['LSDDATA'] = np.append(loc['LSDDATA'], loc['LSD_VELOCITIES'])
    loc['LSDDATA'] = np.append(loc['LSDDATA'], loc['LSD_STOKESI'])
    loc['LSDDATA'] = np.append(loc['LSDDATA'], loc['LSD_STOKESI_MODEL'])
    loc['LSDDATA'] = np.append(loc['LSDDATA'], loc['LSD_STOKESVQU'])
    loc['LSDDATA'] = np.append(loc['LSDDATA'], loc['LSD_NULL'])

    # add LSD parameters keywords to header
    # add input parameters for LSD analysis
    hdict = spirouImage.AddKey(p, hdict, p['KW_POL_STOKES'], value=loc['STOKES'])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_MASK'],
                               value=loc['SELECTED_FILE_CCFLINES'])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_V0'],
                               value=p['IC_POLAR_LSD_V0'])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_VF'],
                               value=p['IC_POLAR_LSD_VF'])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_NP'],
                               value=p['IC_POLAR_LSD_NP'])

    # add fitted values from gaussian fit to Stokes I LSD
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_FIT_RV'],
                               value=loc['LSD_FIT_RV'])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_FIT_RESOL'],
                               value=loc['LSD_FIT_RESOL'])

    # add statistical quantities from LSD analysis
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_MEANPOL'],
                               value=loc['LSD_POL_MEAN'])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_STDDEVPOL'],
                               value=loc['LSD_POL_STDDEV'])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_MEDIANPOL'],
                               value=loc['LSD_POL_MEDIAN'])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_MEDABSDEVPOL'],
                               value=loc['LSD_POL_MEDABSDEV'])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_STOKESVQU_MEAN'],
                               value=loc['LSD_STOKESVQU_MEAN'])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_STOKESVQU_STDDEV'],
                               value=loc['LSD_STOKESVQU_STDDEV'])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_NULL_MEAN'],
                               value=loc['LSD_NULL_MEAN'])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_NULL_STDDEV'],
                               value=loc['LSD_NULL_STDDEV'])
    # add information about the meaning of data columns
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_COL1'],
                               value=p['IC_POLAR_LSD_DATAINFO'][0])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_COL2'],
                               value=p['IC_POLAR_LSD_DATAINFO'][1])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_COL3'],
                               value=p['IC_POLAR_LSD_DATAINFO'][2])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_COL4'],
                               value=p['IC_POLAR_LSD_DATAINFO'][3])
    hdict = spirouImage.AddKey(p, hdict, p['kw_POL_LSD_COL5'],
                               value=p['IC_POLAR_LSD_DATAINFO'][4])

    hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)

    # Store LSD analysis data in FITS TABLE
    table = spirouImage.MakeTable(p, columns, values)
    spirouImage.WriteTable(p, table, lsdfits, fmt='fits', header=hdict)
    # deal with output onto index.fits
    p = spirouImage.spirouFITS.write_output_dict(p, lsdfits, hdict)
    # Store LSD analysis data in file
    #p = spirouImage.WriteImage(p, lsdfits, loc['LSDDATA'], hdict)
    # return p, lsdfits and lsdfitsname
    return p, lsdfits, lsdfitsfitsname

# =============================================================================
# Start of code
# =============================================================================
# Main code here
if __name__ == "__main__":
    # ----------------------------------------------------------------------
    print("SPIRou Least-Squares Deconvolution Module")

# =============================================================================
# End of code
# =============================================================================


from __future__ import absolute_import
from __future__ import unicode_literals
import gevent
import os
import re
import subprocess

from django.test import SimpleTestCase

class TestRequireJS(SimpleTestCase):

    @classmethod
    def _run_jobs(cls, files, processor):
        jobs = []
        for filename in files:
            jobs.append(gevent.spawn(processor, filename))
        gevent.joinall(jobs)

    @classmethod
    def setUpClass(cls):
        super(TestRequireJS, cls).setUpClass()
        prefix = os.path.join(os.getcwd(), 'corehq')

        proc = subprocess.Popen(["find", prefix, "-name", "*.js"], stdout=subprocess.PIPE)
        (out, err) = proc.communicate()
        cls.js_files = [f for f in out.split("\n") if f
                    and not re.search(r'/_design/', f)
                    and not re.search(r'couchapps', f)
                    and not re.search(r'/vellum/', f)]

        cls.hqdefine_files = []
        cls.requirejs_files = []

        def _categorize_file(filename):
            proc = subprocess.Popen(["grep", "^\s*hqDefine", filename], stdout=subprocess.PIPE)
            (out, err) = proc.communicate()
            if out:
                cls.hqdefine_files.append(filename)
                proc = subprocess.Popen(["grep", "hqDefine.*,.*\[", filename], stdout=subprocess.PIPE)
                (out, err) = proc.communicate()
                if out:
                    cls.requirejs_files.append(filename)

        cls._run_jobs(cls.js_files, _categorize_file)

    def test_files_match_modules(self):
        errors = []

        def _test_file(filename):
            proc = subprocess.Popen(["grep", "hqDefine", filename], stdout=subprocess.PIPE)
            (out, err) = proc.communicate()
            for line in out.split("\n"):
                match = re.search(r'^\s*hqDefine\([\'"]([^\'"]*)[\'"]', line)
                if match:
                    module = match.group(1)
                    if not filename.endswith(module + ".js"):
                        errors.append("Module {} defined in file {}".format(module, filename))

        self._run_jobs(self.hqdefine_files, _test_file)

        if errors:
            self.fail("Mismatched JS file/modules: \n{}".format("\n".join(errors)))

    def test_requirejs_disallows_hqimport(self):
        errors = []

        # Special cases:
        #   Ignore standard_hq_report.js until we migrate UCRs and reports
        #   knockout_bindings should be broken up, in the meantime, ignore
        test_files = [f for f in self.requirejs_files
                      if not f.endswith("reports/js/standard_hq_report.js")
                      and not f.endswith("hqwebapp/js/knockout_bindings.ko.js")]

        def _test_file(filename):
            proc = subprocess.Popen(["grep", "hqImport", filename], stdout=subprocess.PIPE)
            (out, err) = proc.communicate()
            for line in out.split("\n"):
                if line:
                    match = re.search(r'hqImport\([\'"]([^\'"]*)[\'"]', line)
                    if match:
                        errors.append("{} imported in {}".format(match.group(1), filename))
                    else:
                        errors.append("hqImport found in {}: {}".format(filename, line.strip()))

        self._run_jobs(test_files, _test_file)

        if errors:
            self.fail("hqImport used in RequireJS modules: \n{}".format("\n".join(errors)))


# -*- coding: utf-8 -*-
"""
Created on Wed Aug 26 00:14:06 2020

@author: Min
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve,auc
from sklearn.metrics import accuracy_score, f1_score, average_precision_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import precision_recall_curve
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn import preprocessing
from sklearn.cluster import KMeans

loc=r"C:\Users\Min\Downloads\takehome_ds_written.csv"
df=pd.read_csv(loc)
df.head()
df.columns
df.amount_usd_in_cents =round(df.amount_usd_in_cents/100)

df.dtypes
df.describe()

df.isnull().sum()

#perMerchant=df.groupby("merchant").size()

def perc1(sr):
    return np.percentiles(sr, np.linspace(0,100,11))

def perc2(sr):
    return np.percentile(sr, np.linspace(0,100,21))

#plt.scatter(range(perMerchant.shape[0]), perMerchant, sort_values())
#plt.ylim( 1,5000)

np.percentile(df.amount_usd_in_cents, np.linspace(95,100,20))

plt.hist(np.log(df.amount_usd_in_cents))

df.time=pd.to_datetime(df.time)
df['hour']=df.time.dt.hour
df['weekday' ]=df.time.dt.weekday

#df['wknd']=df['weekday' ]<5
df['wknd']=df['weekday' ]>=5

df=df.sort_values(by=['merchant', 'time'])
df['time_1']=df.time.shift(-1)
df['day']=df.time.dt.day

#characterize transaction type using amount usd in cents", tint, heuer', day weekday, song, end of month, early month

tranlist=df.copy(deep=True)
def generatefeatures(tranlist, time_st, time_end):
    from scipy.stats import linregress
    eligible = (tranlist.time >time_st) & (tranlist.time<time_end)
    df=tranlist[eligible]

    df['merchant_1']=df.merchant.shift(-1)
    df['merchant_1_']=df.merchant.shift(1)

    df['tint']=(df.time_1-df.time).dt.days
    df['tint'][df.merchant != df.merchant_1] = None
    df['stt']= df.merchant != df.merchant_1_

    df["endt" ]=df.merchant != df.merchant_1
    df['timeindays']=(time_end - df.time).apply(lambda x: x.days).values

    df['tinactive']=np.nan
    df['tinactive'][df.merchant != df.merchant_1]= df['timeindays'][df.merchant != df.merchant_1]
    df.tint[df.merchant != df.merchant_1]=None
    df['time2']=((df.time-time_st).dt.seconds+(df.time-time_st).dt.days*24*60*60)/3600
    dayAgg=df.groupby(['merchant','timeindays'])['amount_usd_in_cents'].sum().reset_index()

    def percentile(n):
        def percentile_(x):
            return np.percentile(x, n)
        percentile_.__name__ = 'percentile_%s' % n
        return percentile_

    df_agg=df.groupby('merchant').agg({ 'amount_usd_in_cents':['mean','std',percentile(95),percentile(50),percentile(5)],
                                   'tint':['mean','std'],
                                   'day':['mean','std'],
                                   'wknd':['sum'],
                                   'merchant':['count'],
                                   'hour':['mean','std']})
    df_agg.columns=['_'.join(col).strip() for col in df_agg.columns.values]
    df_agg['trend']=df.groupby('merchant').apply(lambda v: linregress(v.time2, v.amount_usd_in_cents)[0])
    df_agg['trendtind']=df.dropna(subset=['tint']).groupby('merchant').apply(lambda v: linregress(v.time2, v.tint)[0])
    df_agg['trendDay']=dayAgg.groupby('merchant').apply(lambda v: linregress(v.timeindays, v.amount_usd_in_cents)[0])
    df_agg['endtime']=df.time[df.endt].values
    df_agg['length']= pd.Series(df.time[df.endt].values - df.time[df.stt].values).apply(lambda x: x.days).values
    df_agg['tinactive']=pd.Series(time_end-df.time[df.endt]).apply(lambda x: x.days).values

    bins=pd.IntervalIndex.from_tuples([(-1,15),(15,31)])
    df['dayCut']=pd.cut(df.day,bins)
    df['dayCut']=df['dayCut'].astype(str)

    bins=pd.IntervalIndex.from_tuples([(-1,8),(8,18),(18,23)])
    df['hourCut']=pd.cut(df.hour,bins)
    df['hourCut']=df['hourCut'].astype(str)

    bins=pd.IntervalIndex.from_tuples([(-1,2),(2,7),( 7,60),(60,1000)])
    df['tintCut']=pd.cut(df.tint,bins)
    df['tintCut']=df['tintCut'].astype(str)

    bins=pd.IntervalIndex.from_tuples([(-1,50),(50,100),(100,1000),(1000,1000000)])
    df['amtCut']=pd.cut(df.amount_usd_in_cents,bins)
    df['amtCut']=df['amtCut'].astype(str)

    df['I']=1
    t1=pd.pivot_table(df, values='I', index='merchant', columns='hourCut',aggfunc=np.sum).fillna(0)
    t1.columns=[x+' transaction hour' for x in t1.columns]
    t1=t1.iloc[:,1:]
    t1_p=t1.divide( df_agg.merchant_count, axis=0)

    df['I']=1
    t2=pd.pivot_table(df, values='I', index='merchant', columns='amtCut',aggfunc=np.sum).fillna(0)
    t2.columns=[x+' $ amount' for x in t2.columns]
    t2=t2.iloc[:,1:]
    t2_p=t2.divide( df_agg.merchant_count, axis=0)

    df['I']=1
    t3=pd.pivot_table(df, values='I', index='merchant', columns='tintCut',aggfunc=np.sum).fillna(0)
    t3.columns=[x+' day bet trans.' for x in t3.columns]
    t3=t3.iloc[:,1:]
    del t3['nan day bet trans.']
    t3_p=t3.divide( df_agg.merchant_count, axis=0)

    df['I']=1
    t4=pd.pivot_table(df, values='I', index='merchant', columns='dayCut',aggfunc=np.sum).fillna(0)
    t4.columns=[x+' day' for x in t4.columns]
    t4=t4.iloc[:,1:]
    t4_p=t4.divide(df_agg.merchant_count, axis=0)

    wk_p=df_agg[['wknd_sum']].divide(df_agg.merchant_count, axis=0)
    df_stat=pd.concat([pd.concat([t1,t2,t3,t4], axis=1), df_agg],axis=1)
    df_stat.columns=[x+'_O' for x in df_stat.columns]

    return(pd.concat([df_stat,t1_p,t2_p,t3_p,t4_p, wk_p], axis=1) )

time_st=pd.to_datetime('2032-12-31 20:07:57')
time_end=pd.to_datetime('2034-12-31 20:07:57')

df_filtered=generatefeatures(df, time_st, time_end)

df_filtered['logged_amt']=np.log(df_filtered.amount_usd_in_cents_mean_O)
df_filtered['logged_int']=np.log(df_filtered.tint_mean_O+1)

featuresCluster=['(18, 23] transaction hour',
       '(8, 18] transaction hour', '(100, 1000] $ amount',
       '(1000, 1000000] $ amount', '(50, 100] $ amount',
       '(2, 7] day bet trans.', '(60, 1000] day bet trans.',
       '(7, 60] day bet trans.', '(15, 31] day',
       'wknd_sum', 'logged_amt', 'logged_int']

featuresClusterO=['(18, 23] transaction hour',
       '(8, 18] transaction hour', '(100, 1000] $ amount',
       '(1000, 1000000] $ amount', '(50, 100] $ amount',
       '(2, 7] day bet trans.', '(60, 1000] day bet trans.',
       '(7, 60] day bet trans.', '(15, 31] day',
       'wknd_sum', 'amount_usd_in_cents_mean_O', 'tint_mean_O']

from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
ss=df_filtered.merchant_count_O>2
(scaler.fit(df_filtered[featuresCluster][ss]))
X=scaler.transform(df_filtered[featuresCluster][ss])
from sklearn.cluster import KMeans

wcss=[]
for i in range(1, 11):
    km = KMeans(n_clusters = i, init = 'k-means++',
                                        random_state = 42)
    km.fit(X)
    wcss.append(km.inertia_)
plt.plot(range(1,11), wcss,'-bo')
plt.title('elbow method')
plt.xlabel('# clusters')
plt.ylabel('wcss')
plt.show()

i=7
km = KMeans(n_clusters = i, init = 'k-means++',
                                        random_state = 42)
km.fit(X)
#import pickle
#pickle.dump( km, open( "km.pkl", "wb" ) )
#km=pickle.load(open( "km.pkl", "rb" ))

df_filtered['label']=np.nan
df_filtered['label'][ss]=km.labels_
df_filtered[featuresClusterO+['label']].groupby('label').mean().to_csv(r'C:\Users\Min\clusters.csv')
df_filteredO=df_filtered.copy(deep=True)

fixedTh=90
mincap=30
maxcap=180
factorStd=4
mischurn=[]
for i in range(3,10):
    midpt=pd.to_datetime('2033-12-31 20:07:57')+pd.Timedelta(i,unit='M')
    eligiblet = (df.time>pd.to_datetime('2032-12-31 20:07:57')) & (df.time < midpt)

    tdf=df[eligiblet]

    tdf['merchant_1_']=tdf.merchant.shift(1)
    tdf['merchant_1']=tdf.merchant.shift(-1)
    tdf['time_1']=tdf.time.shift(-1)
    tdf['tint']= (tdf.time_1-tdf.time).dt.days
    tdf['endt']=tdf.merchant_1 != tdf.merchant
    tdf['tint'][tdf.endt]= None
    tdf['tinactive']=np.nan
    tdf['tinactive'][tdf.endt]=(midpt-tdf.time)[tdf.endt].apply(lambda x: x.days)
    tdf_agg=tdf.groupby('merchant').agg({'tint':['mean','std'],'tinactive':'mean'})

    tdf_agg.columns=['tintM','tintS','inactiveM']
    churn1MID=tdf_agg.index[(tdf_agg.inactiveM > fixedTh)]
    churn2MID=tdf_agg.index[(tdf_agg.inactiveM > (tdf_agg.tintM + tdf_agg.tintS * factorStd).apply(lambda x: max(min(x,maxcap),mincap) ))]

    eligiblet = (df.time> midpt)
    tdf2=df[eligiblet]
    year2MID=tdf2.merchant.unique()

    mischurn.append([midpt, sum(churn1MID.isin(year2MID)), sum(churn2MID.isin(year2MID)),len(churn1MID), len(churn2MID), tdf_agg.shape[0]])

mischurn

fixedTh=90
mincap=30
maxcap=180
factorStd=4
df_filtered.isnull().sum()
import numpy as np
from sklearn.preprocessing import Imputer
imp = Imputer(missing_values=np.nan, strategy='mean')
imp.fit(df_filtered[[  'tint_mean_O', 'tint_std_O' ]].values)
df_filtered[[  'tint_mean_O', 'tint_std_O' ]]=imp.transform(df_filtered[[ 'tint_mean_O', 'tint_std_O'  ]])

churnETA=(df_filtered.tint_mean_O+df_filtered.tint_std_O * factorStd).apply(lambda x: max(min(int(x),maxcap),mincap) )
#perc1(churnETA)

churnind=df_filtered.tinactive_O>churnETA
churnedID=df_filtered.index[churnind]
churnedTime=df_filtered.endtime_O[churnind] + churnETA.apply(lambda x: np.timedelta64(x,'D'))[churnind]

len(churnedID)/df_filtered.shape[0]
df_filtered['churned']=churnind

tmpp=df_filtered.groupby('label')['churned'].agg(['sum','size'])
tmpp.columns=['Churned merchant', 'Total merchants']
tmpp.index=[int(x+1) for x in tmpp.index.values]
ax=tmpp.plot(kind='bar',title='# of merchant counts and churned per cluster')
ax.set_xlabel('cluster')
ax.set_ylabel('# of merchants')

ax=(tmpp.iloc[:,0]/tmpp.iloc[:,1]*100).plot(kind='bar',title='% of churn per cluster')
ax.set_xlabel('cluster')
ax.set_ylabel('% of churn')

churnobsMonths=3
trainMonths=15
churnstart=pd.to_datetime('2034-12-31 20:07:57')-pd.Timedelta(churnobsMonths,unit='M')
time_st=churnstart-pd.Timedelta(trainMonths,unit='M')
time_end=churnstart

churned2034=churnedID[(churnedTime>churnstart)  ]
churned2033=churnedID[churnedTime<churnstart]

df_train_2033=generatefeatures(df, time_st, time_end)
del df_train_2033['endtime_O']
df_train_2033.isnull().sum()

df_train_2033 = df_train_2033[~(df_train_2033.index.isin(churned2033))]
df_train_2033.isnull().sum()

imp = Imputer(missing_values=np.nan, strategy='mean')
imp.fit(df_train_2033.values)
df_train_2033[:]=imp.transform(df_train_2033)

import seaborn as sns
corr=df_train_2033.corr()
kot=corr[corr>0.95]
plt.figure(figsize=(12,8))
sns.heatmap(kot, cmap='Greens')

y=df_train_2033.index.isin(churned2034)
xTrain, xTest, yTrain, yTest = train_test_split(df_train_2033, y, test_size=0.25, random_state=5)

model_params = {
    'n_estimators': [200, 400],
    'max_depth': [5, 20, 50]
}
rf_model = RandomForestClassifier(random_state=1)
grid_clf = GridSearchCV(rf_model, model_params, cv=5)
RF = grid_clf.fit(xTrain, yTrain)
sc_RF=RF.score(xTest, yTest)
RF.best_params_
RF_p=RF.predict_proba(xTest)

precision, recall, thresholds=precision_recall_curve(yTest, RF_p[:,1])
auc0=auc(recall, precision)
false_positive_rate, true_positive_rate, threshold=roc_curve(yTest, RF_p[:,1])
roc_auc=auc(false_positive_rate, true_positive_rate)

plt.plot(false_positive_rate, true_positive_rate, color='b',  label='ROC curve (area = %0.2f)' % roc_auc)
plt.plot([0, 1], [0, 1], color='navy',   linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc="lower right")
plt.show()

plt.plot(recall,precision, color='b',  label='ROC curve (area = %0.2f)' % auc0)
plt.plot([0, 1], [0, 0], color='navy',  linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.legend(loc="lower right")
plt.show()

coef=[]
cols=df_train_2033.columns
for col in cols:
    coef=coef+[np.corrcoef(df_train_2033[col],y)[0,1]]
corcoef=pd.DataFrame({'corr_coef':coef, 'colname':cols})
corcoef['abs_corr_coef']=abs(corcoef.corr_coef)
corcoef=corcoef.sort_values('abs_corr_coef')

ftrImp=pd.DataFrame({'imp_RF':RF.best_estimator_.feature_importances_, 'colname':cols})
ftrImp=ftrImp.merge(corcoef, on='colname',how='left')
ftrImp=ftrImp.sort_values(['imp_RF'], ascending=[0])

f, ax = plt.subplots(figsize=(6, 15))
sns.set_color_codes("pastel")
sns.barplot(y="colname", x="imp_RF", data=ftrImp,
            label="randomForest", color="b")

sns.set_color_codes("muted")
sns.barplot(y="colname", x="corr_coef", data=ftrImp,
            label="PearsonCorrelation", color="g",alpha=0.3)

ax.legend(ncol=2, loc="lower right", frameon=True)
ax.set(xlim=(0, 24), ylabel="",
       xlabel="feature importance")
sns.despine(left=True, bottom=True)

n=20

RF1 = RandomForestClassifier( n_estimators= 400, max_depth=20, class_weight='balanced').fit(xTrain[ftrImp.colname[:n]],yTrain)
sc_RF1=RF1.score(xTest[ftrImp.colname[:n]], yTest)
RF1_p=RF1.predict_proba(xTest[ftrImp.colname[:n]])

precision, recall, thresholds=precision_recall_curve(yTest, RF1_p[:,1])
auc1=auc(recall, precision)
false_positive_rate, true_positive_rate, threshold=roc_curve(yTest, RF1_p[:,1])
roc_auc1 = auc(false_positive_rate, true_positive_rate)

RF2 = RandomForestClassifier( n_estimators= 400, max_depth=20, class_weight='balanced').fit(xTrain.tinactive_O.values.reshape(-1,1),yTrain)
sc_RF2=RF2.score(xTest.tinactive_O.values.reshape(-1,1), yTest)

print([churnobsMonths, trainMonths, sc_RF, sc_RF1, sc_RF2, auc0, auc1, roc_auc, roc_auc1])

#grid={"C":np.logspace(-3,3,7), "penalty":["l1","l2"]}
#logreg=LogisticRegression()
#logreg_cv=GridSearchCV(logreg,grid,cv=10)
#logreg_cv.fit(xTrain,yTrain)
#logreg_cv.score(xTest,yTest)
#

import scikitplot as skplt
skplt.metrics.plot_lift_curve(y_true=yTest, y_probas = RF_p)

df_score=generatefeatures(df, pd.to_datetime('2034-12-31 20:07:57')-pd.Timedelta(trainMonths,unit='M'), pd.to_datetime( '2034-12-31 20:07:57') )
del df_score['endtime_O']
df_score=df_score[~(df_score.index.isin(churnedID))]

imp = Imputer(missing_values=np.nan, strategy='mean')
imp.fit(df_score.values)
df_score[:]=imp.transform(df_score)

p=RF.predict_proba(df_score)

tmp=RF_p[:,1]
tmp.sort()
thresholdP=np.percentile(tmp, 70)

df_score['churnLabel']='Not churned'
df_score['churnLabel'][p[:,1]>=thresholdP]='Likely to churn'
df_score.groupby('churnLabel').size().plot(kind='bar')
sum(df_score['churnLabel']=='Likely to churn' )/df_score.shape[0]

df_score['propensityScore']=p[:,1]
df_score[['propensityScore','churnLabel']].to_csv(r'C:\Users\Min\scored.csv')

df_filtered['churned'].to_csv(r'C:\Users\Min\churned.csv')

df_filtered_O['churned'] =  df_filtered_O.tinactive>90
df_filtered_O['churned'] =  df_filtered['churned']
perc2(df_filtered_O.tint_mean_O.dropna())

sns.distplot(df_filteredO.tint_mean_O[df_filteredO.churned].dropna(), color="dodgerblue", label="churned")
sns.distplot( df_filteredO.tint_mean_O[~df_filteredO.churned].dropna(), color="orange", label="not churned")
plt.legend(loc='upper right')
plt.xlabel('# days between first and latest transactions')
plt.title('Density of inactive days comparison (churned vs. active users)')
plt.show()
df_filtered['loggedTintMean']= np.log(df_filtered.tint_mean_O+1)
fig =  sns.boxplot(x="churned", y="loggedTintMean", data=df_filtered, palette="Set3")
fig.set(  ylabel='logged average # days between transaction')

sns.distplot(df_filteredO.tint_std_O[df_filtered.churned].dropna(), color="dodgerblue", label="churned")
sns.distplot( df_filteredO.tint_std_O[~df_filtered.churned].dropna(), color="orange", label="not churned")
plt.legend(loc='upper right')
plt.xlabel('# days between first and latest transactions')
plt.title('Density of identified account tenure comparison (churned vs. active users)')
plt.show()
df_filtered['loggedTintStd']= np.log(df_filtered.tint_mean_O+1)
fig =  sns.boxplot(x="churned", y="loggedTintStd", data=df_filtered, palette="Set3")
fig.set(  ylabel='# days holding the account')

fig =  sns.boxplot(x="churned", y="length_O", data=df_filtered, palette="Set3")
fig.set(  ylabel='# days holding the account')

fig =  sns.boxplot(x="churned", y="tinactive_O", data=df_filtered, palette="Set3")
fig.set(  ylabel='# days from the latest transaction')

# check if the score stays similar

churnobsMonths=3
trainMonths=15
churnstart=pd.to_datetime('2034-09-30 20:07:57')-pd.Timedelta(churnobsMonths,unit='M')
time_st=churnstart-pd.Timedelta(trainMonths,unit='M')
time_end=churnstart

churned2034=churnedID[(churnedTime>churnstart)  ]
churned2033=churnedID[churnedTime<churnstart]

df_train_2033=generatefeatures(df, time_st, time_end)
del df_train_2033['endtime_O']
df_train_2033.isnull().sum()

df_train_2033 = df_train_2033[~(df_train_2033.index.isin(churned2033))]
df_train_2033.isnull().sum()

imp = Imputer(missing_values=np.nan, strategy='mean')
imp.fit(df_train_2033.values)
df_train_2033[:]=imp.transform(df_train_2033)

y=df_train_2033.index.isin(churned2034)
sc_RF=RF.score(df_train_2033, y)
p1=RF.predict_proba(df_train_2033)

precision, recall, thresholds=precision_recall_curve(yTest, p1[:,1])
auc0=auc(recall, precision)
false_positive_rate, true_positive_rate, threshold=roc_curve(yTest, p1[:,1])
roc_auc=auc(false_positive_rate, true_positive_rate)


# myproject.myapi.utils.py
from channels.auth import AuthMiddlewareStack
from channels.db import database_sync_to_async
from django.contrib.auth.models import AnonymousUser

from rest_framework.authtoken.models import Token
from django.db import close_old_connections

@database_sync_to_async
def get_user(token_key):
    try:
        return Token.objects.get(key=token_key).user
    except Token.DoesNotExist:
        return AnonymousUser()

class TokenAuthMiddleware:
    """
    Token authorization middleware for Django Channels 2
    see:
    https://channels.readthedocs.io/en/latest/topics/authentication.html#custom-authentication
    """

    def __init__(self, inner):
        self.inner = inner

    def __call__(self, scope):
        return TokenAuthMiddlewareInstance(scope, self)

class TokenAuthMiddlewareInstance:
    def __init__(self, scope, middleware):
        self.middleware = middleware
        self.scope = dict(scope)
        self.inner = self.middleware.inner

    async def __call__(self, receive, send):
        close_old_connections()
        headers = dict(self.scope["headers"])
        print(headers[b"cookie"])
        if b"authorization" in headers[b"cookie"]:
            print('still good here')
            cookies = headers[b"cookie"].decode()
            token_key = re.search("authorization=(.*)(; )?", cookies).group(1)
            if token_key:
                self.scope["user"] = await get_user(token_key)

        # inner = self.inner(self.scope)
        return await self.inner(self.scope,receive, send)

TokenAuthMiddlewareStack = lambda inner: TokenAuthMiddleware(AuthMiddlewareStack(inner))


from flask import url_for
import pytest
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.ui import WebDriverWait

def test_profile_no_auth(selenium, base_url):
    selenium.get(
        base_url + url_for('account', id='6e2cd664f6309c64803d5f4fdb0c19d9')
    )

    assert 'Login' in selenium.title

def test_signup_no_auth(selenium, base_url):
    selenium.get(base_url + url_for('signup'))

    assert 'Login' in selenium.title

@pytest.mark.parametrize('selenium', [['admin', 'admin']], indirect=True)
def test_profile(selenium, base_url):
    id = '6e2cd664f6309c64803d5f4fdb0c19d9'

    selenium.get(base_url + url_for('account', id=id))

    assert 'Marina Kuefer - Profil' in selenium.title

    delete = selenium.find_elements_by_css_selector('form[action="{}"]'.format(
        url_for('delete_account', id=id)))
    assert not delete
    selenium.find_element_by_css_selector('a[href="{}"]'.format(
        url_for('edit_account', id=id))).click()
    assert 'Benutzerkonto bearbeiten' in selenium.title

@pytest.mark.parametrize('selenium', [['admin', 'admin']], indirect=True)
def test_signup_edit_delete(selenium, base_url):
    selenium.get(base_url + url_for('signup'))

    assert 'Benutzerkonto erstellen' in selenium.title
    submit = selenium.find_element_by_id('submit')
    submit.click()

    assert 'Benutzerkonto erstellen' in selenium.title
    inputs = selenium.find_elements_by_css_selector(
        'input:required:not([type="email"])')
    for input in inputs:
        input.send_keys('test')
    submit.click()

    assert 'Benutzerkonto erstellen' in selenium.title
    email = selenium.find_element_by_name('email')
    assert email in selenium.find_elements_by_css_selector('input:invalid')
    email.send_keys('test@test.test')
    submit.click()

    wait = WebDriverWait(selenium, 10)
    wait.until(expected_conditions.title_contains('test test - Profil'))
    id = selenium.current_url.split('/')[-1]

    selenium.find_element_by_css_selector('a[href="{}"]'.format(
        url_for('edit_account', id=id))).click()

    for e in selenium.find_elements_by_css_selector(
        'input[name]:not([type="email"])'):
        assert e.get_attribute('value') == 'test'
    email = selenium.find_element_by_css_selector(
        'input[name][type="email"]')
    assert email.get_attribute('value') == 'test@test.test'

    selenium.get(base_url + url_for('account', id=id))
    selenium.find_element_by_css_selector('form[action="{}"] button'.format(
        url_for('delete_account', id=id))).click()
    selenium.get(base_url + url_for('account', id=id))

    assert '410 Gone' in selenium.title


# default imports
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

data = pd.read_csv("./data/house_pricing.csv")
X = data.iloc[:, :-1]
y = data.iloc[:, -1]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=9)

depth_list =  [8, 10, 15, 20, 50, 100, 120, 150, 175, 200]
def decision_classifier_plot(X_train, X_test, y_train, y_test,depth_list):
    test_score=[]
    train_score=[]
# Write your solution here :
    for i in range(len(depth_list)):
        model =DecisionTreeClassifier(max_depth=depth_list[i])
        model.fit(X_train,y_train)
        y_pred=model.predict(X_train)
        y_pred_test=model.predict(X_test)
        train_score.insert(i,accuracy_score(y_train,y_pred))
        test_score.insert(i,accuracy_score(y_test,y_pred_test))
    plt.plot(depth_list, train_score, 'b', label='train set')
    plt.plot(depth_list, test_score, 'g', label='test set')
    plt.legend(loc='best')
    plt.xlabel('depth')
    plt.ylabel('accuracy_score')
    plt.show()


from pygame \
    import sprite, image, transform

from core.text \
    import Text

class Inventory(sprite.Sprite):

    def __init__(self):
        super(Inventory, self).__init__()
        self.empty_path = 'sprites/ui/inventory/empty.png'
        self.full_path = 'sprites/ui/inventory/full.png'

        self.image = image.load(self.empty_path)
        self.rect = self.image.get_rect()
        self.rect.x = 10
        self.rect.y = 490
        self.item = None
        self.empty = True

    def clear(self):
        self.empty = True
        self.image.fill((255, 255, 255))
        self.image = image.load(self.empty_path)
        return self.item

    def add(self, item):
        self.empty = False
        self.item = item

        if item.name == "item":
            name = "Box"
        else:
            name = item.name

        text = Text(20, name)
        self.image = image.load(self.full_path)

        inv_w, inv_h = self.rect.size
        text_w, text_h = text.size

        img_xpos = inv_w/2 - 35/2
        img_ypos = inv_h/2 - 35/2 + 10

        text_xpos = inv_w/2 - text_w/2
        text_ypos = 2

        self.image.blit(transform.scale(item.image, (35, 35)), (img_xpos, img_ypos))
        self.image.blit(text.surface, (text_xpos, text_ypos))


from itertools import groupby
from collections import defaultdict

class Solution:
    def longestLine(self, M):
        """
        :type M: List[List[int]]
        :rtype: int
        """
        if not M or len(M[0]) == 0:
            return 0

        groups = defaultdict(list)
        for r, row in enumerate(M):
            for c, val in enumerate(row):
                groups[0, r] += [val] # rows
                groups[1, c] += [val] # cols
                groups[2, r + c] += [val] # diagonal
                groups[3, r - c] += [val] # anti-diagonal

        return max(list(map(self.ones, groups.values())))

    def ones(self, line: [int]) -> int:
        lens = [len(list(v)) if k == 1 else 0 for k, v in groupby(line)]
        return max(lens)


import OpenGL.GL as gl
import OpenGL.GLUT as glut
import sys
from utility.os_interface import read_file_data
import numpy
from numpy.lib.arraypad import np
import ctypes
import log

class Viewer:
    point_count = 0

    def __init__(self, points, colors, file_iso_function, scale, delta, target_value, vertex_file, time):
        self.point_count = len(points)

        self._init_glut(time)

        # read data
        data = numpy.zeros(self.point_count, dtype=[("position", np.float32, 2),
                                                    ("color", np.float32, 4)])
        data['color'] = colors
        data['position'] = points

        # read shaders
        path = "./shader"
        vertex_code = read_file_data(path, vertex_file)
        fragment_code = read_file_data(path, "fragment1.glsl")

        buffer = self._build_buffer(data)  # TODO ?
        program = self._init_program(vertex_code, fragment_code)
        self._bind_buffers(program, data, buffer)

        # bind uniforms
        self._bind_scale(program, scale)
        self._bind_delta(program, delta)
        self._bind_projection(program)
        self._bind_modelview(program)
        self._bind_target_value(program, target_value)

        glut.glutMainLoop()

    def _display(self):
        gl.glClear(gl.GL_COLOR_BUFFER_BIT)
        # gl.glDrawArrays(gl.GL_TRIANGLE_STRIP, 0, 4)

        gl.glEnable(gl.GL_PROGRAM_POINT_SIZE)
        gl.glClearColor(1.0, 1.0, 1.0, 1.0)

        # for x in range(0, self.point_count/2, 2):
        #  gl.glDrawArrays(gl.GL_LINES, x, x+1)

        gl.glDrawArrays(gl.GL_POINTS, 0, self.point_count)
        glut.glutSwapBuffers()

    def _set_window_size(self, width, height):
        gl.glViewport(0, 0, width, height)

    def _keyboard(self, key, x, y):
        if key == '\033':
            sys.exit()

    def _init_glut(self, time):

        if type(time) != bytes:
            raise TypeError

        glut.glutInit()
        glut.glutInitDisplayMode(glut.GLUT_DOUBLE | glut.GLUT_RGBA)
        glut.glutCreateWindow(time)
        glut.glutReshapeWindow(700, 700)
        glut.glutReshapeFunc(self._set_window_size)
        glut.glutDisplayFunc(self._display)
        glut.glutKeyboardFunc(self._keyboard)

    def _init_program(self, vertex_code, fragment_code):
        program = gl.glCreateProgram()
        print(gl.glGetError())
        vertex = gl.glCreateShader(gl.GL_VERTEX_SHADER)
        print(gl.glGetError())
        fragment = gl.glCreateShader(gl.GL_FRAGMENT_SHADER)
        print(gl.glGetError())
        # Set shaders source
        gl.glShaderSource(vertex, vertex_code)
        print(gl.glGetError())
        gl.glShaderSource(fragment, fragment_code)
        print(gl.glGetError())
        # Compile shaders
        gl.glCompileShader(vertex)
        print(gl.glGetError())
        gl.glCompileShader(fragment)
        print(gl.glGetError())

        # We can now build and link the program

        gl.glAttachShader(program, vertex)
        print(gl.glGetError())
        gl.glAttachShader(program, fragment)
        print(gl.glGetError())
        gl.glLinkProgram(program)
        print(gl.glGetError())

        # We can not get rid of shaders, they won't be used again:

        gl.glDetachShader(program, vertex)
        print(gl.glGetError())
        gl.glDetachShader(program, fragment)
        print(gl.glGetError())

        # Finally, we make program the default program to be ran. We can do it now because we'll use a single in this example:
        print(glut.glutReportErrors())
        if gl.glGetShaderiv(vertex, gl.GL_COMPILE_STATUS) == 0:
            print(gl.glGetShaderInfoLog(vertex))

        if gl.glGetShaderiv(fragment, gl.GL_COMPILE_STATUS) == 0:
            print(gl.glGetShaderInfoLog(fragment))

        if gl.glGetProgramiv(program, gl.GL_LINK_STATUS) == 0:
            print(gl.glGetProgramInfoLog(program))

        gl.glUseProgram(program)
        print(gl.glGetError())

        return program

    def _build_buffer(self, data):
        # Request a buffer slot from GPU
        buffer = gl.glGenBuffers(1)

        # Make this buffer the default one
        gl.glBindBuffer(gl.GL_ARRAY_BUFFER, buffer)

        # Upload data
        gl.glBufferData(gl.GL_ARRAY_BUFFER, data.nbytes, data, gl.GL_DYNAMIC_DRAW)

        return buffer

    def _bind_buffers(self, program, data, buffer):
        stride = data.strides[0]
        offset = ctypes.c_void_p(0)
        loc = gl.glGetAttribLocation(program, "position")

        gl.glEnableVertexAttribArray(loc)
        gl.glBindBuffer(gl.GL_ARRAY_BUFFER, buffer)

        gl.glVertexAttribPointer(loc, 3, gl.GL_FLOAT, False, stride, offset)

        offset = ctypes.c_void_p(data.dtype["position"].itemsize)
        loc = gl.glGetAttribLocation(program, "color")

        gl.glEnableVertexAttribArray(loc)
        gl.glBindBuffer(gl.GL_ARRAY_BUFFER, buffer)
        gl.glVertexAttribPointer(loc, 4, gl.GL_FLOAT, False, stride, offset)

    # Bind Uniform:
    def _bind_scale(self, program, scale):
        loc = gl.glGetUniformLocation(program, "scale")
        gl.glUniform1f(loc, scale)

    def _bind_target_value(self, program, target_value):
        loc = gl.glGetUniformLocation(program, "target_value")
        gl.glUniform1f(loc, target_value)

    def _bind_delta(self, program, delta):
        loc = gl.glGetUniformLocation(program, "delta")
        gl.glUniform1f(loc, delta)

    def _bind_projection(self, program):
        loc = gl.glGetUniformLocation(program, "in_projection_matrix")
        projection = np.matrix([1, 1, 1, 1])
        projection = [1, 1, 1, 1]
        # GLint location, GLsizei count, GLboolean transpose,  	const GLfloat *value
        gl.glUniformMatrix4fv(loc, len(projection), True, projection)

    def _bind_modelview(self, program):
        # loc = gl.glGetUniformLocation(program, "in_modelview_matrix")
        # projection = np.matrix([1, 1, 1, 1])
        # gl.glUniformMatrix4fv(loc, projection)
        return


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('searchapp', '0003_auto_20151129_1758'),
    ]

    operations = [
        migrations.CreateModel(
            name='OReferences',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=120, verbose_name=b'Other References')),
            ],
        ),
        migrations.AddField(
            model_name='paper',
            name='references',
            field=models.ManyToManyField(related_name='_paper_references_+', to='searchapp.Paper', blank=True),
        ),
        migrations.AddField(
            model_name='oreferences',
            name='paper',
            field=models.ForeignKey(to='searchapp.Paper'),
        ),
    ]


import torch
import torch.nn as nn
import torch.nn.functional as F
import dgl.nn.pytorch as dglnn
import dgl.function as fn

class WeightedSAGEConv(nn.Module):
    def __init__(self, in_feat, out_feat):
        super(WeightedSAGEConv, self).__init__()
        self.linear = nn.Linear(in_feat * 2, out_feat)

    def forward(self, g, h, weights):
        with g.local_scope():
            g.ndata['h'] = h
            g.edata['w'] = weights.float()
            g.update_all(message_func=fn.u_mul_e('h', 'w', 'm'), reduce_func=fn.mean('m', 'h_N'))
            h_N = g.ndata['h_N']
            h_total = torch.cat([h, h_N], dim=1)
            return self.linear(h_total)

class RGCNLayer(nn.Module):
    """
    R-GCN layer using basis decomposition
    """
    def __init__(self, in_feat, out_feat, num_rels, num_bases=-1, bias=None,
                 activation=None, is_input_layer=False):
        super(RGCNLayer, self).__init__()
        self.in_feat = in_feat
        self.out_feat = out_feat
        self.num_rels = num_rels
        self.num_bases = num_bases
        self.bias = bias
        self.activation = activation
        self.is_input_layer = is_input_layer

        # sanity check
        if self.num_bases <= 0 or self.num_bases > self.num_rels:
            self.num_bases = self.num_rels

        # weight bases in equation (3)
        self.weight = nn.Parameter(torch.Tensor(self.num_bases, self.in_feat,
                                                self.out_feat))
        if self.num_bases < self.num_rels:
            # linear combination coefficients in equation (3)
            self.w_comp = nn.Parameter(torch.Tensor(self.num_rels, self.num_bases))

        # add bias
        if self.bias:
            self.bias = nn.Parameter(torch.Tensor(out_feat))

        # init trainable parameters
        nn.init.xavier_uniform_(self.weight,
                                gain=nn.init.calculate_gain('relu'))
        if self.num_bases < self.num_rels:
            nn.init.xavier_uniform_(self.w_comp,
                                    gain=nn.init.calculate_gain('relu'))
        if self.bias:
            nn.init.xavier_uniform_(self.bias,
                                    gain=nn.init.calculate_gain('relu'))

    def forward(self, g):
        if self.num_bases < self.num_rels:
            # generate all weights from bases (equation (3))
            weight = self.weight.view(self.in_feat, self.num_bases, self.out_feat)
            weight = torch.matmul(self.w_comp, weight).view(self.num_rels,
                                                        self.in_feat, self.out_feat)
        else:
            weight = self.weight

        if self.is_input_layer:
            def message_func(edges):
                # for input layer, matrix multiply can be converted to be
                # an embedding lookup using source node id
                embed = weight.view(-1, self.out_feat)
                index = edges.data['rel_type'] * self.in_feat + edges.src['id']
                return {'msg': embed[index] * edges.data['norm']}
        else:
            def message_func(edges):
                w = weight[edges.data['rel_type']]
                msg = torch.bmm(edges.src['h'].unsqueeze(1), w).squeeze()
                msg = msg * edges.data['norm']
                return {'msg': msg}

        def apply_func(nodes):
            h = nodes.data['h']
            if self.bias:
                h = h + self.bias
            if self.activation:
                h = self.activation(h)
            return {'h': h}

        g.update_all(message_func, fn.sum(msg='msg', out='h'), apply_func)

class HeteroRGCNLayer(nn.Module):
    """
    original R-GCN layer derived from Kipf's GCN, can be replaced with dgl.nn.pytorch.HeteroGraphConv, but slower by 40%
    """
    def __init__(self, in_size, out_size, etypes):
        super(HeteroRGCNLayer, self).__init__()
        self.weight = nn.ModuleDict({
            name: nn.Linear(in_size, out_size) for name in etypes
        })

    def forward(self, graph, feat_dict):
        funcs = {}

        for srctype, etype, dsttype in graph.canonical_etypes:
            Wh = self.weight[etype](feat_dict[srctype])
            graph.nodes[srctype].data['Wh_%s' % etype] = Wh
            funcs[etype] = (fn.copy_u('Wh_%s' % etype, 'm'), fn.mean('m', 'h'))

        graph.multi_update_all(funcs, 'sum')

        return {ntype: graph.nodes[ntype].data['h'] for ntype in graph.ntypes}


#!/usr/bin/env python
# coding: utf-8

# In[1]:

#importing libraries:
import numpy as np
from flask import Flask, request, jsonify, render_template
import pickle

# In[2]:

app=Flask(__name__)

# In[3]:

model=pickle.load(open('log_model.pkl','rb'))

# In[4]:

@app.route('/')
def home():
    return render_template('individual.html')

#%%
term_to_int={0:"36 Months",1:"64 Months"}

application_type={0:"Individual"}

grade={0:"A",1:"B",2:"C",3:"D",4:"E",5:"F",6:"G"}

sub_grade={0:"A1",1:"A2",2:"A3",3:"A4",4:"A5",
           5:"B1",6:"B2",7:"B3",8:"B4",9:"B5",
           10:"C1",11:"C2",12:"C3",13:"C4",14:"C5",
           15:"D1",16:"D2",17:"D3",18:"D4",19:"D5",
           20:"E1",21:"E2",22:"E3",23:"E4",24:"E5",
           25:"F1",26:"F2",27:"F3",28:"F4",29:"F5",
           30:"G1",31:"G2",32:"G3",33:"G4",34:"G5"}

emp_length={0:"1 Year",1:"10+ Years",2:"2 Years",3:"3 Years",4:"4 Years",5:"5 Year",
            6:"6 Years",7:"7 Years",8:"8 Years",9:"9 Years",10:"< 1 Year"}

home_ownership={0:"Mortage",1:"Others",2:"Own",3:"Rent"}

verification_status={0:"Income Source Verified",1:"Not Verified",2:"Verified"}

purpose={0:"Car",1:"Credit Card",2:"Dept Consolidation",3:"Educational",4:"Home Improvement",
         5:"House",6:"Major Purpose",7:"Medical",8:"Moving",9:"Others",10:"Renewable Energy",
         11:"Small Business",12:"Vacation",13:"Wedding"}

addr_state={0:"Alaska",1:"Alabama",2:"Arkansas",3:"Arizona",4:"California",5:"Colorado",
            6:"Connecticut",7:"District of Columbia",8:"Delaware",9:"Florida",10:"Georgia",
            11:"Hawaii",12:"Iowa",13:"Idaho",14:"Illinois",15:"Indiana",16:"Kansas",
            17:"Kentucky",18:"Louisiana",19:"Massachusetts",20:"Maryland",21:"Maine",
            22:"Michigan",23:"Minnesota",24:"Missouri",25:"Mississippi",26:"Montana",
            27:"North Carolina",28:"North Dakota",29:"Nebraska",30:"New Hampshire",
            31:"New Jersey",32:"New Mexico",33:"Nevada",34:"New York",35:"Ohio",36:"Oklahoma",
            37:"Oregon",38:"Pennsylvania",39:"Rhode Island",40:"South Carolina",41:"South Dakota",
            42:"Tennessee",43:"Texas",44:"Utah",45:"Virginia",46:"Vermont",47:"Washington",
            48:"Wisconsin",49:"West Virginia",50:"Wyoming"}

initial_list_status={0:"Fractional",1:"Whole"}
# In[5]:

@app.route("/Individual", methods=['POST'])
def Individual():
    int_features=[int(x) for x in request.form.values()]
    final_features=[np.array(int_features)]
    prediction=model.predict(final_features)

    data2=request.form["loan_amnt"]
    data3=request.form["term"]
    data4=request.form["int_rate"]
    data5=request.form["installment"]
    data6=request.form["grade"]
    data7=request.form["sub_grade"]
    data8=request.form["emp_length"]
    data9=request.form["home_ownership"]
    data10=request.form["annual_inc"]
    data11=request.form["verification_status"]
    data12=request.form["purpose"]
    data13=request.form["addr_state"]
    data14=request.form["dti"]
    data15=request.form["delinq_2yrs"]
    data16=request.form["inq_last_6mths"]
    data17=request.form["open_acc"]
    data18=request.form["pub_rec"]
    data19=request.form["revol_bal"]
    data20=request.form["revol_util"]
    data21=request.form["total_acc"]
    data22=request.form["initial_list_status"]
    data23=request.form["out_prncp"]
    data24=request.form["out_prncp_inv"]
    data25=request.form["total_pymnt"]
    data26=request.form["total_rec_prncp"]
    data27=request.form["total_rec_int"]
    data28=request.form["total_rec_late_fee"]
    data29=request.form["last_pymnt_amnt"]
    data30=request.form["collections_12_mths_ex_med"]
    data31=request.form["application_type"]
    data32=request.form["acc_now_delinq"]
    data33=request.form["tot_coll_amt"]
    data34=request.form["tot_cur_bal"]

    #create original output dict
    output_dict= dict()
    output_dict['Desired Loan Amount']=data2
    output_dict['Term (in Months)']=term_to_int[int(data3)]
    output_dict['Interest Rate on the loan']=data4
    output_dict['Installment']=data5
    output_dict['Grade']=grade[int(data6)]
    output_dict['Sub Grade']=sub_grade[int(data7)]
    output_dict['Employee Length']=emp_length[int(data8)]
    output_dict['Home Ownership']=home_ownership[int(data9)]
    output_dict['Annual Income']=data10
    output_dict['Verification Status']=verification_status[int(data11)]
    output_dict['Loan Purpose']=purpose[int(data12)]
    output_dict['Address State']=addr_state[int(data13)]
    output_dict['Total Debt Obligation Income']=data14
    output_dict['Delinquency for the past 2 years']=data15
    output_dict['Inquiries in past 6 months (excluding auto and mortgage inquiries)']=data16
    output_dict["Number of open credit lines in the borrower's credit file"]=data17
    output_dict['Number of derogatory public records']=data18
    output_dict['Total credit revolving balance']=data19
    output_dict['Revolving line utilization rate']=data20
    output_dict["Total number of credit lines currently in the borrower's credit file"]=data21
    output_dict['Initial List Status']=initial_list_status[int(data22)]
    output_dict['Remaining outstanding principal for total amount funded']=data23
    output_dict['Remaining outstanding principal for portion of total amount funded by investors']=data24
    output_dict['Payments received to date for total amount funded']=data25
    output_dict['Principal received to date']=data26
    output_dict['Interest received to date']=data27
    output_dict['Late fees received to date']=data28
    output_dict['Last total payment amount received']=data29
    output_dict['Number of collections in 12 months excluding medical collections']=data30
    output_dict['Application Type']=application_type[int(data31)]
    output_dict['Number of accounts on which the borrower is now delinquent']=data32
    output_dict['Total collection amounts ever owed']=data33
    output_dict['Total current balance of all accounts']=data34

    return render_template('individualresult.html',original_input=output_dict,data=prediction)
# In[7]:

if __name__=='__main__':
    app.run(debug=True)

# In[ ]:


from signer import *
from optparse import OptionParser
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import Sequence
from keras.optimizers import Adam
from keras.callbacks import Callback,ModelCheckpoint
from sklearn.externals import joblib

pd.set_option('display.max_rows',None)

op = OptionParser(usage='python3 poc_eval_mp.py [options] [objs-folder] [anit-folder] [meta_embed_file] [model_dest]')
op.add_option('--target_test', dest='target_test', help='an additional target test set for evaluation, used while test_ratio is 0')
op.add_option('--anti_test', dest='anti_test', help='an addtional anit test set for evaluation, used while test_ratio is 0')
op.add_option('--bec_test', dest='bec_test', help='an addtional bec test set for evaluation, used while test_ratio is 0')
op.add_option('--test_ratio', dest='test_ratio', type='float', default=0, help='a ratio for ratio of test set, default is 0')
op.add_option("-v", dest="verbose", action='store_true', default=False, help='a switch to turn on/off detail messages')
op.add_option('-n', dest='enable_nmlzr', action='store_true', default=False, help='a switch to turn on/off the normalizer')
op.add_option('--expt_debug', dest='expt_debug', action='store_true', default=False, help='experiment mode')
options,args  = op.parse_args()

EXPT_DEBUG  = options.expt_debug

#-- Get Data Paths --#
tr_target_path  = args[0]
tr_anti_path    = args[1]
model_path      = args[2] if len(args)>2 else None
tt_target_path  = options.target_test
tt_anti_path    = options.anti_test
tt_bec_path     = options.bec_test

def meta_to_word(t,ignore_length=False,only_initial=False,only_final=False,only_length=False):
    t  = [('<!--signature-->',0) if x=='<!--signature-->' else (list(x.keys())[0],list(x.values())[0]) for x in t]
    if only_initial:
        t  = [x[0] for x,y in t]
    elif only_final:
        t  = [x[-1] for x,y in t]
    elif only_length:
        t  = [x if len(x)==1 else (str(y) if y<31 else '31+') for x,y in t]
    elif ignore_length:
        t  = [x for x,y in t]
    else:
        t  = [x if y<=len(x) else x[0]+'_'*(y-len(x))+x[1:] for x,y in t]
    t  = '<sep>'.join(t)
    return t

def read_meta_path(meta_path):
    filelist  = sorted(glob.glob(os.path.join(meta_path,'*.meta')))
    text      = pd.DataFrame(index=pd.Index([],name='title'))
    for f in filelist:
        title  = os.path.splitext(os.path.split(f)[1])[0]
        t      = read_meta(f)
        text.loc[title,'filename'] = f
        text.loc[title,'raw']  = json.dumps(t,separators=(',',':'))
        text.loc[title,'wrd']  = meta_to_word(t)
        text.loc[title,'wrd_nolen']  = meta_to_word(t,ignore_length=True)
        text.loc[title,'ch0']  = meta_to_word(t,only_initial=True)
        text.loc[title,'ch1']  = meta_to_word(t,only_final=True)
        text.loc[title,'len']  = meta_to_word(t,only_length=True)
    return text

#-- Read Meta --#
tr_target  = read_meta_path(tr_target_path) if tr_target_path else None
tr_anti    = read_meta_path(tr_anti_path) if tr_anti_path else None
tt_target  = read_meta_path(tt_target_path) if tt_target_path else None
tt_anti    = read_meta_path(tt_anti_path) if tt_anti_path else None
tt_bec     = read_meta_path(tt_bec_path) if tt_bec_path else None

#-- Compile Training and Test Data --#
data_tr    = pd.concat([tr_target,tr_anti],0,keys=['target','anti'],names=['grp',],sort=False)
data_tr['bec']  = data_tr.index.get_level_values('grp')!='target'
data_tt    = pd.concat([tt_target,tt_anti,tt_bec],0,keys=['target','anti','bec'],names=['grp',],sort=False)
data_tt['bec']  = data_tt.index.get_level_values('grp')!='target'

#-- Fit Tokenizers --#
MAX_NUM_WORDS  = 20000
tokenizer  = {}
for col in ('wrd','wrd_nolen','ch0','ch1','len'):
    tokenizer[col]  = Tokenizer(num_words=MAX_NUM_WORDS,filters='',lower=False,split='<sep>',char_level=False,oov_token=None)
    tokenizer[col].fit_on_texts(data_tr[col].str.replace('\r\n','\n'))

#-- Text to Tokenizer Word Index --#
X_tr  = {}
for col in ('wrd','wrd_nolen','ch0','ch1','len'):
    X_tr[col]  = tokenizer[col].texts_to_sequences(data_tr[col].str.replace('\r\n','\n'))

max_seq_len = min(max([len(x) for x in X_tr['wrd']]),1000)
for col in ('wrd','wrd_nolen','ch0','ch1','len'):
    X_tr[col]  = pad_sequences(X_tr[col],maxlen=max_seq_len,dtype='int32',padding='post',truncating='post',value=0)

y_tr  = data_tr.bec

X_tt  = {}
for col in ('wrd','wrd_nolen','ch0','ch1','len'):
    X_tt[col]  = tokenizer[col].texts_to_sequences(data_tt[col].str.replace('\r\n','\n'))
    X_tt[col]  = pad_sequences(X_tt[col],maxlen=max_seq_len,dtype='int32',padding='post',truncating='post',value=0)

y_tt  = data_tt.bec

def token_embedding_mc(max_seq_len,tokenizer,channels,output_dim,mask_zero=False):
    embed_model  = {}
    for col in channels:
        embed_model[col]  = token_embedding(max_seq_len,tokenizer[col],output_dim=output_dim,mask_zero=mask_zero)
    #
    embed_mc_output   = Concatenate(axis=2)([embed_model[col].output for col in channels])
    return Model(inputs=[embed_model[col].input for col in channels],outputs=embed_mc_output)

#-----------#
#-- CNN 7 --#
#-----------#
#---- Preprocess: All characters
#---- Channels: (wrd,wrd_nolen,ch0,ch1,len)
#---- Seed: 0
#---- ND: 20, FS: (3,4,5), NF: 64, Dropout: 0.2, Lambda: 0.001, Loss: 'mean_squared_error', AdamDecay: 0.02, MaxEpochs: fr:55 jp:60, BatchSize: 256
keras_seed(0)
MAX_EPOCHS   = 60 if tt_target_path.split(os.sep)[4]=='Trend_jp' else 55
CHANNELS     = ('wrd','wrd_nolen','ch0','ch1','len')
CNN_DROPOUT  = 0.2
REG_LAMBDA   = 0.001
ADAM_DECAY   = 0.02
embed_model_mc    = token_embedding_mc(max_seq_len,tokenizer,channels=CHANNELS,output_dim=20)
tweet_embed_model = YoonKimCNN(embed_model_mc,filter_sizes=(3,4,5),num_filters=64,batch_norm=False,dropout=CNN_DROPOUT)
preds  = Dense(1,activation='sigmoid',
    kernel_regularizer=regularizers.l2(REG_LAMBDA),
    bias_regularizer=regularizers.l2(REG_LAMBDA),
    )(tweet_embed_model.output)
model  = Model(inputs=tweet_embed_model.inputs,outputs=preds)
model.compile(loss='mean_squared_error', #'binary_crossentropy', #
    optimizer=Adam(decay=ADAM_DECAY),
    metrics=['accuracy'])

if EXPT_DEBUG: t0 = time.time()

out  = model.fit([X_tr[col] for col in CHANNELS],y_tr,validation_data=([X_tt[col] for col in CHANNELS],y_tt),epochs=MAX_EPOCHS,batch_size=256,class_weight=None,shuffle=True,verbose=int(EXPT_DEBUG))
hist = pd.DataFrame(out.history,index=range(1,MAX_EPOCHS+1))[['loss','acc','val_loss','val_acc']]
if EXPT_DEBUG:
    print(time.time() - t0)
    print(hist[(hist.index.isin(range(0,MAX_EPOCHS+1,5)))|(hist.val_loss==hist.val_loss.min())|(hist.val_acc==hist.val_acc.max())])

data_tr['bec_prob']  = model.predict([X_tr[col] for col in CHANNELS])
data_tt['bec_prob']  = model.predict([X_tt[col] for col in CHANNELS])

# #--------------------------------------------------------#
# #-- CNN 8 - Baseline CNN w/ space char and len channel --#
# #--------------------------------------------------------#
# #---- Preprocess: All characters
# #---- Channels: (wrd_nolen,len)
# #---- Seed: 0
# #---- ND: 20, FS: (3,4,5), NF: 32, Dropout: 0.02, Lambda: 0.001, Loss: 'binary_crossentropy', AdamDecay: 0, MaxEpochs: fr:85 jp:70, BatchSize: 256
# keras_seed(0)
# MAX_EPOCHS   = 70 if tt_target_path.split(os.sep)[4]=='Trend_jp' else 85
# CHANNELS     = ('wrd_nolen','len')
# CNN_DROPOUT  = 0.02
# REG_LAMBDA   = 0.001
# embed_model_mc    = token_embedding_mc(max_seq_len,tokenizer,channels=CHANNELS,output_dim=20)
# tweet_embed_model = YoonKimCNN(embed_model_mc,filter_sizes=(3,4,5),num_filters=32,batch_norm=False,dropout=CNN_DROPOUT)
# preds  = Dense(1,activation='sigmoid',
#     kernel_regularizer=regularizers.l2(REG_LAMBDA),
#     bias_regularizer=regularizers.l2(REG_LAMBDA),
#     )(tweet_embed_model.output)
# model  = Model(inputs=tweet_embed_model.inputs,outputs=preds)
# model.compile(loss='binary_crossentropy', #'mean_squared_error', #
#     optimizer='adam',
#     metrics=['accuracy'])

# if EXPT_DEBUG: t0 = time.time()

# out  = model.fit([X_tr[col] for col in CHANNELS],y_tr,validation_data=([X_tt[col] for col in CHANNELS],y_tt),epochs=MAX_EPOCHS,batch_size=256,class_weight=None,shuffle=True,verbose=int(EXPT_DEBUG))
# hist = pd.DataFrame(out.history,index=range(1,MAX_EPOCHS+1))[['loss','acc','val_loss','val_acc']]
# if EXPT_DEBUG:
#     print(time.time() - t0)
#     print(hist[(hist.index.isin(range(0,MAX_EPOCHS+1,5)))|(hist.val_loss==hist.val_loss.min())|(hist.val_acc==hist.val_acc.max())])

# data_tr['bec_prob']  = model.predict([X_tr[col] for col in CHANNELS])
# data_tt['bec_prob']  = model.predict([X_tt[col] for col in CHANNELS])

#----------------#
#-- Save Model --#
#----------------#
if model_path:
    joblib.dump({'tokenizer':tokenizer,'hist':hist,},os.path.join(model_path,tr_anti_path+'_cnn7.pkl'))
    model.save(os.path.join(model_path,tr_anti_path+'_cnn7.hd5'),overwrite=True)

#---------------------------#
#-- Evaluation and Output --#
#---------------------------#
BEC_THD  = 0.3
data_tr['bec_hat']   = data_tr.bec_prob>BEC_THD
data_tt['bec_hat']   = data_tt.bec_prob>BEC_THD
res  = pd.concat([
    scores(data_tr.bec,data_tr.bec_hat),
    scores(data_tt.bec,data_tt.bec_hat),
    ],1,keys=('tr','tt'))
if EXPT_DEBUG: print(res)

# res  = pd.Series([
#     accuracy_score(data_tr.loc['target'].bec,data_tr.loc['target'].bec_hat),
#     accuracy_score(data_tr.loc['anti'].bec,data_tr.loc['anti'].bec_hat),
#     accuracy_score(data_tt.loc['target'].bec,data_tt.loc['target'].bec_hat),
#     accuracy_score(data_tt.loc['anti'].bec,data_tt.loc['anti'].bec_hat),
#     ],index=('tr_target','tr_anti','tt_target','tt_anti'))

#-- Output Prediction Results to Stdout --#
if not EXPT_DEBUG:
    # data_tt['filename']  = 'k-fold' + data_tt.filename.str.split('k-fold').str[1]
    for idx,row in data_tt.iterrows():
        print("%(filename)s,%(bec_prob)f,%(bec_hat)d"%row)


from django.shortcuts import render, redirect, get_object_or_404
from django.views.decorators.http import require_POST

from shop.models import *
from orders.models import Order
from .cart import Cart
from .forms import CartAddProductForm, GiftForm
from coupons.forms import CouponApplyForm
import requests

# Можно поменять редирект на карточку товара снова
@require_POST
def cart_add(request, flavour_id):
    cart = Cart(request)
    flavour = Flavour.published.get(id=flavour_id)
    form = CartAddProductForm(request.POST)
    if form.is_valid():
        cd = form.cleaned_data
        cart.add(flavour=flavour,
                 quantity=cd['quantity'],
                 update_quantity=cd['update'])
    return redirect('cart:cart_detail')

@require_POST
def cart_offer_add(request, flavour_id):
    cart = Cart(request)
    flavour = Flavour.published.get(id=flavour_id)
    cart.add(flavour=flavour)
    return cart

def add_offer(request, offer_id):
    products = Offer.published.get(id=offer_id).products.all()
    for product in products:
        flavour_id = product.flavour_by_product()
        cart_offer_add(request, flavour_id)
    cart = Cart(request)
    return redirect('cart:cart_detail')

def cart_remove(request, flavour_id):
    cart = Cart(request)
    flavour = Flavour.published.get(id=flavour_id)
    cart.remove(flavour)
    return redirect('cart:cart_detail')

def cart_detail(request):
    categories = Category.published.all()
    suppliers = Supplier.published.all()
    objectives = Objective.published.all()
    products_rec = Product.published.filter(recommendation=True)
    offers = Offer.published.all()
    cart = Cart(request)
    try:
        orders = Order.published.filter(client=request.user, paid=False)
    except:
        orders = None
    for item in cart:
        item['update_quantity_form'] = CartAddProductForm(
            initial={'quantity': item['quantity'],
                     'update': True})

    coupon_apply_form = CouponApplyForm()
    gift_form = GiftForm()
    list_of_gift = Product.published.filter(available=False)
    gift = None
    sum_for_gift = cart.get_total_price()
    balance = 2500 - sum_for_gift
    balance2 = 5000 - sum_for_gift
    if 2500 <= sum_for_gift <= 5000:
        gift = 2500
    elif sum_for_gift > 5000:
        gift = 5000
    # Проверяем находится ли подарок в корзине перебором
    gift_in_cart = False
    for item in cart:
        flavour = item['flavour']
        # Если недоступный товар в корзине, значит подарок уже добавили
        if flavour.product.available == False:
            gift_in_cart = True
    template = 'cart/detail.html'
    context = locals()
    return render(request, template, context)

@require_POST
def add_gift(request):
    cart = Cart(request)
    gift_form = GiftForm(request.POST)
    product = None
    if gift_form.is_valid():
        gift = gift_form.cleaned_data['gift']
        if gift == 'Батончик':
            product = Product.published.get(name='Батончик(подарок)')
        elif gift == 'Шейкер 3в1':
            product = Product.published.get(name='Шейкер(подарок)')
        elif gift == 'Печенье':
            product = Product.published.get(name='Печенье(подарок)')
        try:
            flavour = product.flavour_for_gift()
            cart.add(flavour=flavour, quantity=1)
        except:
            if gift == 'Батончик+печенье':
                products = Product.published.filter(name__in=('Батончик(подарок)', 'Печенье(подарок)'))
                for product in products:
                    flavour = product.flavour_for_gift()
                    cart.add(flavour=flavour, quantity=1)
    return redirect('cart:cart_detail')


import math
import matplotlib.pyplot as plt
import numpy
import pandas
import scipy

import numpy.linalg as linalg
import sklearn.cluster as cluster
import sklearn.neighbors as neighbors

Spiral = pandas.read_csv('C:\\Users\\Lipi\\Downloads\\MachineLearning\\Week4_LIPI\\jain.csv',
                         delimiter=',')

nObs = Spiral.shape[0]

plt.scatter(Spiral['x'], Spiral['y'], c = Spiral['group'])
plt.xlabel('x')
plt.ylabel('y')
plt.grid(True)
plt.show()

trainData = Spiral[['x','y']]
kmeans = cluster.KMeans(n_clusters=2, random_state=0).fit(trainData)

print("Cluster Centroids = \n", kmeans.cluster_centers_)

Spiral['KMeanCluster'] = kmeans.labels_

for i in range(2):
    print("Cluster Label = ", i)
    print(Spiral.loc[Spiral['KMeanCluster'] == i])

plt.scatter(Spiral['x'], Spiral['y'], c = Spiral['KMeanCluster'])
plt.xlabel('x')
plt.ylabel('y')
plt.grid(True)
plt.show()

# Fourteen nearest neighbors
kNNSpec = neighbors.NearestNeighbors(n_neighbors = 14, algorithm = 'brute', metric = 'euclidean')
nbrs = kNNSpec.fit(trainData)
d3, i3 = nbrs.kneighbors(trainData)

# Retrieve the distances among the observations
distObject = neighbors.DistanceMetric.get_metric('euclidean')
distances = distObject.pairwise(trainData)

# Create the Adjacency matrix
Adjacency = numpy.zeros((nObs, nObs))
for i in range(nObs):
    for j in i3[i]:
        Adjacency[i,j] = math.exp(- (distances[i][j])**2 )

# Make the Adjacency matrix symmetric
Adjacency = 0.5 * (Adjacency + Adjacency.transpose())

# Create the Degree matrix
Degree = numpy.zeros((nObs, nObs))
for i in range(nObs):
    sum = 0
    for j in range(nObs):
        sum += Adjacency[i,j]
    Degree[i,i] = sum

# Create the Laplacian matrix
Lmatrix = Degree - Adjacency

# Obtain the eigenvalues and the eigenvectors of the Laplacian matrix
evals, evecs = linalg.eigh(Lmatrix)

# Series plot of the smallest five eigenvalues to determine the number of clusters
sequence = numpy.arange(1,5,1)
plt.plot(sequence, evals[0:4,], marker = "o")
plt.xlabel('Sequence')
plt.ylabel('Eigenvalue')
plt.xticks(sequence)
plt.grid("both")
plt.show()

# Series plot of the smallest twenty eigenvalues to determine the number of neighbors
sequence = numpy.arange(1,21,1)
plt.plot(sequence, evals[0:20,], marker = "o")
plt.xlabel('Sequence')
plt.ylabel('Eigenvalue')
plt.grid("both")
plt.xticks(sequence)
plt.show()

# Inspect the values of the selected eigenvectors
for j in range(10):
    print('Eigenvalue: ', j)
    print('              Mean = ', numpy.mean(evecs[:,j]))
    print('Standard Deviation = ', numpy.std(evecs[:,j]))
    print('  Coeff. Variation = ', scipy.stats.variation(evecs[:,j]))

Z = evecs[:,[0,1]]

plt.scatter(1e10*Z[:,0], Z[:,1])
plt.xlabel('First Eigenvector')
plt.ylabel('Second Eigenvector')
plt.grid("both")
plt.show()

# Perform 2-cluster K-mean on the first two eigenvectors
kmeans_spectral = cluster.KMeans(n_clusters = 2, random_state = 0).fit(Z)
Spiral['SpectralCluster'] = kmeans_spectral.labels_

plt.scatter(Spiral['x'], Spiral['y'], c = Spiral['SpectralCluster'])
plt.xlabel('x')
plt.ylabel('y')
plt.grid(True)
plt.show()

Old Materials/Lipi Shah/MachineLearning/Week4_LIPI/Week 4 Jain Spiral.py


from django import forms
from django.contrib.auth.forms import AuthenticationForm

from .models import User

class LoginForm(AuthenticationForm):

    def __init__(self, *args, **kwargs):
        super(LoginForm, self).__init__(*args, **kwargs)
        for field_name, field in self.fields.items():
            field.widget.attrs['class'] = 'form-control'
            field.widget.attrs['placeholder'] = field_name

class ProfileEditForm(forms.ModelForm):
    birthday = forms.DateField(
        widget=forms.DateInput(format="%m/%d/%Y",
                               attrs={'placeholder': 'MM/DD/YYYY', 'class': 'form-control date-input'}),
        required=False
    )

    class Meta:
        model = User
        fields = (
            'first_name', 'last_name',
            'handicap', 'birthday', 'location', 'years_of_experience', 'creation_time', 'photo',
        )

    def __init__(self, *args, **kwargs):
        super(ProfileEditForm, self).__init__(*args, **kwargs)
        for field_name, field in self.fields.items():
            field.widget.attrs['class'] = 'form-control'

        self.fields['first_name'].required = self.fields['last_name'].required = True
        self.fields['birthday'].widget.attrs['class'] = 'form-control date-input'


import strax
import straxen
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from argparse import ArgumentParser
import argparse
import cutax
from cutax.cuts.krypton_selections import KrSingleS1S2
from cutax.cuts.krypton_selections import KrDoubleS1SingleS2
import sys
sys.path +=['../../../utils/']

import xomlib

def press_run(run_id):

#    st = cutax.contexts.xenonnt_offline(include_rucio_local=False,include_rucio_remote=True )
    st = cutax.xenonnt_online(_rucio_local_path='/project/lgrandi/rucio', include_rucio_local = True)
    st.storage +=[strax.DataDirectory('/project2/lgrandi/xenonnt/processed', provide_run_metadata=True)]
#    st = cutax.xenonnt_online(output_folder = "/home/gaior/codes/test/.", _rucio_local_path='/project/lgrandi/rucio', include_rucio_local = True)
    st.register([KrSingleS1S2,KrDoubleS1SingleS2])
#    st.storage += [strax.DataDirectory('/project2/lgrandi/xenonnt/processed', provide_run_metadata=True)]
    data_singleS1 = st.get_df(run_id, targets =("event_info_double", "cut_fiducial_volume", "cut_Kr_SingleS1S2"),
                   selection_str= ("cut_fiducial_volume",'cut_Kr_SingleS1S2'))
    data_doubleS1 = st.get_df(run_id, targets =("event_info_double", "cut_fiducial_volume", "cut_Kr_DoubleS1_SingleS2"),
                   selection_str= ("cut_fiducial_volume",'cut_Kr_DoubleS1_SingleS2'))

    ES1a = 32.2
    ES1b = 9.4

    #energie du pic
    ES1 = 41.6

    #Light Yield

    S1_a1 = data_doubleS1['s1_a_area'].values
    S1_b1 = data_doubleS1['s1_b_area'].values
    S1_a = data_singleS1['s1_a_area'].values

    #mean
    S1amoy = np.mean(S1_a1)
    S1bmoy = np.mean(S1_b1)
    S1_41 = np.mean(S1_a)

    #standard deviation
    S1asigma = np.std(S1_a1)
    S1bsigma = np.std(S1_b1)
    S1_41sigma = np.std(S1_a)

    #standard error of mean
    errorS1amoy = S1asigma/np.sqrt(len(S1_a1))
    errorS1bmoy = S1bsigma/np.sqrt(len(S1_b1))
    errorS1_41moy = S1_41sigma/np.sqrt(len(S1_a))

    #light yield
    LYS1a = S1amoy/ES1a
    LYS1b = S1bmoy/ES1b
    LYS1_41 = S1_41/ES1

    #light yield error
    DLyS1a = errorS1amoy/ES1a
    DLyS1b = errorS1bmoy/ES1b
    DLyS1_41 = errorS1_41moy/ES1

    #removing useless datas
    del data_doubleS1
    del data_singleS1
    del S1_a1
    del S1_b1
    del S1_a

    xomresult = xomlib.Xomresult(analysis_name="light_yield",
                                 analysis_version = "v0.0",
                                 variable_name='LYS1_41',
                                 variable_value=LYS1_41,
                                 runid=int(run_id),
                                 data= {"LYS1a":LYS1a, "DLYS1a":DLyS1a,
                                        "LYS1b":LYS1b, "DLYS1b":DLyS1b,
                                        "LYS1_41":LYS1_41, "DLYS1_41":DLyS1_41})
    xomresult.save()
    xomresult.xom_message(success=True)

    # results = []

    # results.append(run_id)
    # results.append(LYS1a)
    # results.append(DLyS1a)
    # results.append(LYS1b)
    # results.append(DLyS1b)
    # results.append(LYS1_41)
    # results.append(DLyS1_41)

    # np.save('/home/pellegriniq/tab22', results)

def main():

    parser = argparse.ArgumentParser()
    parser.add_argument("echo")
    args = parser.parse_args()
    print(args.echo)
    print("start")

    press_run(args.echo)
    print('end')

if __name__ == "__main__":
    main()