text:
#!/usr/bin/env python

import os, sys
import numpy as np
from astropy.io import fits
from glob import glob
import matplotlib.pyplot as plt
from astropy.time import Time
import json
import ephem
import psycopg2
import pandas as pd
import desisurveyops.utils as utils

def calc_sciencelist(night, clobber=False, verbose=False):
    '''
    Create json files with start times, exptimes, and other information
    for spectra for a single night. Outputs have the form specdata20201224.json.

    Parameters
    ----------
    night : int
        night in form 20210320 for 20 March 2021
    clobber : bool
        overwrite existing file
    verbose : bool
        provide verbose output

    Returns
    -------
    none
    '''

    # Output directory
    try:
        outdir = utils.get_outdir()
    except RuntimeError:
        exit(1)

    # Directory with raw data
    try:
        datadir = utils.get_rawdatadir()
    except RuntimeError:
        exit(1)
    nightdir = os.path.join(datadir, str(night))

    # Names for output json file:
    filename = "specdata" + str(night) + ".json"
    specdatafile = os.path.join(outdir, 'NightlyData', filename)

    # See if json file for this night already exists:
    if not os.path.isfile(specdatafile) or clobber:
        # List of all directories and expids:
        expdirs = glob(nightdir + "/*")
        expids = expdirs.copy()
        for i in range(len(expdirs)):
            expids[i] = expdirs[i][expdirs[i].find('000')::]

        # Print warning if there are no data
        if len(expids) == 0:
            print("Warning: No observations found")

        # Get all spec observations:
        specdata = {}
        for expid in expids:
            tmpdir = os.path.join(nightdir, expid)
            scifiles = (glob(tmpdir + "/desi*"))
            if len(scifiles) > 0:
               hhh = fits.open(scifiles[0])
               if hhh[1].header['DATE-OBS'] == None:
                   continue
               specdata[expid] = {}
               specdata[expid]['DATE-OBS'] = Time(hhh[1].header['DATE-OBS']).mjd
               specdata[expid]['OBSTYPE'] = hhh[1].header['OBSTYPE']
               specdata[expid]['FLAVOR'] = hhh[1].header['FLAVOR']
               specdata[expid]['PROGRAM'] = hhh[1].header['PROGRAM']
               specdata[expid]['EXPTIME'] = hhh[1].header['EXPTIME']
               try:
                   specdata[expid]['DOMSHUTU'] = hhh[1].header['DOMSHUTU']
               except KeyError:
                   specdata[expid]['DOMSHUTU'] = 'None'
               try:
                   specdata[expid]['PMCOVER'] = hhh[1].header['PMCOVER']
               except KeyError:
                   specdata[expid]['PMCOVER'] = 'None'
        with open(specdatafile, 'w') as fp:
            json.dump(specdata, fp)

        if verbose:
            print("Wrote", specdatafile)

def calc_guidelist(night, clobber=False, verbose=False):
    '''
    Create json files with start times, exptimes, and other information
    about guiding for a single night. Outputs have the form guidedata20201224.json.

    Parameters
    ----------
    night : int
        night in form 20210320 for 20 March 2021
    clobber : bool
        overwrite existing file
    verbose : bool
        provide verbose output

    Returns
    -------
    none
    '''

    try:
        outdir = utils.get_outdir()
    except RuntimeError:
        exit(1)

    try:
        datadir = utils.get_rawdatadir()
    except RuntimeError:
        exit(1)

    nightdir = os.path.join(datadir, str(night))
    twibeg_mjd, twiend_mjd = utils.get_twilights(int(night))

    # Names for output json file:
    filename = "guidedata" + str(night) + ".json"
    guidedatafile = os.path.join(outdir, 'NightlyData', filename)

    # See if json file for this night already exists:
    if not os.path.isfile(guidedatafile) or clobber:
        # List of all directories and expids:
        expdirs = glob(nightdir + "/*")
        expids = expdirs.copy()
        for i in range(len(expdirs)):
            expids[i] = expdirs[i][expdirs[i].find('000')::]

        # Print warning if there are no data
        if len(expids) == 0:
            print("Warning: No observations found")

        guidedata = {}
        for expid in expids:
            # tmpdir = os.path.join(nightdir, expid, "guide-" + expid + ".fits.fz")
            # guidefiles = (glob(tmpdir + "/guide-" + expid + ".fits.fz"))
            # guidefile = tmpdir + "/guide-" + expid + ".fits.fz"
            guidefile = os.path.join(nightdir, expid, "guide-" + expid + ".fits.fz")
            #if len(guidefiles) > 0:
            if os.path.isfile(guidefile):
                try:
                    hhh = fits.open(guidefile)
                except OSError:
                    if verbose:
                        print("OSError with {}".format(guidefile))
                    continue
                guidedata[expid] = {}
                t1 = Time( utils.find_dateobs(hhh) ).mjd
                try:
                    t2 = Time(hhh['GUIDE0T'].data[-1]['DATE-OBS']).mjd
                except KeyError:
                    t2 = t1  # Need to fix this
                guidedata[expid]['GUIDE-START'] = t1
                guidedata[expid]['GUIDE-STOP'] = t2
                try:
                    guidedata[expid]['DOMSHUTU'] = hhh[0].header['DOMSHUTU']
                except KeyError:
                    guidedata[expid]['DOMSHUTU'] = 'None'
                try:
                    guidedata[expid]['PMCOVER'] = hhh[0].header['PMCOVER']
                except KeyError:
                    guidedata[expid]['PMCOVER'] = 'UNKNOWN'
                try:
                    guidedata[expid]['OBSTYPE'] = hhh[0].header['OBSTYPE']
                except KeyError:
                    guidedata[expid]['OBSTYPE'] = 'None'
                try:
                    guidedata[expid]['FLAVOR'] = hhh[0].header['FLAVOR']
                except KeyError:
                    guidedata[expid]['FLAVOR'] = 'None'
                try:
                    guidedata[expid]['EXPTIME'] = hhh[0].header['EXPTIME']
                except KeyError:
                    guidedata[expid]['EXPTIME'] = 0.
        with open(guidedatafile, 'w') as fp:
            json.dump(guidedata, fp)

        if verbose:
            print("Wrote", guidedatafile)

def get_dometelemetry(night, verbose=False):
    '''
    Retrieve dome telemetry data for a given night.

    Parameters
    ----------
    night : int
        night in form 20210320 for 20 March 2021
    verbose : bool
        print verbose output?

    Returns
    -------
    dome : record array
        dome telemetry stream for night
    '''

    # connect to telemetry database
    conn = psycopg2.connect(database="desi_dev", user="desi_reader")

    # set query limits as 3 hours before/after twilight at beginning/end of night
    twi1, twi2 = utils.get_twilights(night)
    twibeg = Time( Time(twi1, format='mjd'), format='datetime')
    twiend = Time( Time(twi2, format='mjd'), format='datetime')
    query_start = twibeg - (3./24.)
    query_stop = twiend + (3./24.)

    # query for dome data
    domedf = pd.read_sql_query(f"SELECT dome_timestamp,shutter_upper,mirror_cover FROM environmentmonitor_dome WHERE time_recorded >= '{query_start}' AND time_recorded < '{query_stop}'", conn)
    dome = domedf.to_records()

    # convert dome_timestamp to a Time object in MJD
    dome['dome_timestamp'] = np.array([Time(t).mjd for t in dome['dome_timestamp']])

    return dome

def get_ccdtelemetry(night, verbose=False):
    '''
    Retrieve spectrograph ccd telemetry data for a given night.

    Parameters
    ----------
    night : int
        night in form 20210320 for 20 March 2021
    verbose : bool
        print verbose output?

    Returns
    -------
    ccd : record array
        ccd telemetry stream for night
    '''

    # connect to telemetry database
    conn = psycopg2.connect(database="desi_dev", user="desi_reader")

    # set query limits as 3 hours before/after twilight at beginning/end of night
    twi1, twi2 = utils.get_twilights(night)
    twibeg = Time( Time(twi1, format='mjd'), format='datetime')
    twiend = Time( Time(twi2, format='mjd'), format='datetime')
    query_start = twibeg - (3./24.)
    query_stop = twiend + (3./24.)

    specdf = pd.read_sql_query(f"SELECT * FROM spectrographs_ccds WHERE time_recorded >= '{query_start}' AND time_recorded < '{query_stop}'", conn)
    spec_ccds = specdf.to_records()

    # Convert dome_timestamp to a Time object in MJD
    spec_ccds['time_recorded'] = np.array([Time(t).mjd for t in spec_ccds['time_recorded']])

    return spec_ccds

def get_teltelemetry(night, verbose=False):
    '''
    Retrieve telescope telemetry data for a given night.

    Parameters
    ----------
    night : int
        night in form 20210320 for 20 March 2021
    verbose : bool
        print verbose output?

    Returns
    -------
    ccd : record array
        ccd telemetry stream for night
    '''

    # connect to telemetry database
    conn = psycopg2.connect(database="desi_dev", user="desi_reader")

    # set query limits as 3 hours before/after twilight at beginning/end of night
    twi1, twi2 = utils.get_twilights(night)
    twibeg = Time( Time(twi1, format='mjd'), format='datetime')
    twiend = Time( Time(twi2, format='mjd'), format='datetime')
    query_start = twibeg - (3./24.)
    query_stop = twiend + (3./24.)

    # Telescope data:
    teldf = pd.read_sql_query(f"SELECT * FROM tcs_info WHERE time_recorded >= '{query_start}' AND time_recorded < '{query_stop}'", conn)
    tel = teldf.to_records()

    # Convert dome_timestamp to a Time object in MJD
    tel['time_recorded'] = np.array([Time(t).mjd for t in tel['time_recorded']])

    return tel

def get_guidetelemetry(night, verbose=False):
    '''
    Retrieve guider telemetry data for a given night.

    Parameters
    ----------
    night : int
        night in form 20210320 for 20 March 2021
    verbose : bool
        print verbose output?

    Returns
    -------
    guide : record array
        guide telemetry stream for night
    '''

    # connect to telemetry database
    conn = psycopg2.connect(database="desi_dev", user="desi_reader")

    # set query limits as 3 hours before/after twilight at beginning/end of night
    twi1, twi2 = utils.get_twilights(night)
    twibeg = Time( Time(twi1, format='mjd'), format='datetime')
    twiend = Time( Time(twi2, format='mjd'), format='datetime')
    query_start = twibeg - (3./24.)
    query_stop = twiend + (3./24.)

    # query for guider data
    guidedf = pd.read_sql_query(f"SELECT time_recorded,seeing,meanx,meany FROM guider_summary WHERE time_recorded >= '{query_start}' AND time_recorded < '{query_stop}'", conn)
    guide = guidedf.to_records()

    return guide

def calc_domevals(night, verbose=False):
    '''
    Calculate basic dome parameters for a given night.

    Parameters
    ----------
    night : int
        night in form 20210320 for 20 March 2021
    verbose : bool
        print verbose output?

    Returns
    -------
    dome_firstopen : float
        first time the dome opened (MJD)
    dome_lastclose : float
        last time dome closed (MJD)
    fractwiopen : float
        fraction of the time the dome was open between 18 deg twilights
    twiopen_hrs : float
        hours the dome was open between 18 deg twilights
    '''

    twibeg_mjd, twiend_mjd = utils.get_twilights(int(night))
    startdate = int(twibeg_mjd)
    twibeg_hours = 24.*(twibeg_mjd - startdate)
    twiend_hours = 24.*(twiend_mjd - startdate)
    twitot_hours = twiend_hours - twibeg_hours

    # Get dome status:
    dome = get_dometelemetry(night)
    dometime = np.array([ (t - startdate)*24 for t in dome['dome_timestamp']])
    dome_open = np.array([t for t in dome['shutter_upper']], dtype=bool)

    # Compute total time with dome open between twilights:
    nmask = dometime >= twibeg_hours
    nmask = nmask*(dometime <= twiend_hours) # time between twilights
    dmask = nmask*dome_open # dome open
    open_fraction = np.sum(dmask)/np.sum(nmask)
    fractwiopen = open_fraction

    # Total time dome was open between twilights
    twiopen_hrs = open_fraction*twitot_hours

    # get times the dome first opened and last closed
    # shutter_upper == 1 means dome was open
    open_indices = np.where(dome_open)[0]

    if verbose:
        print("len(open_indices) = ", len(open_indices))

    # check if dome didn't open at all
    if len(open_indices) == 0:
        return 0., 0., 0., 0.

    # time stamps when dome first opened and last closed
    firstopen = dome['dome_timestamp'][open_indices[0]]
    lastclose = dome['dome_timestamp'][open_indices[-1]]

    if verbose:
        print("MJD for dome first open, last close:", firstopen, lastclose)
        print("UT for dome first open, last close:", Time(firstopen, format='mjd').datetime, Time(lastclose, format='mjd').datetime )
        print("Dome open: {0:.3f} Dome close: {1:.3f} Fraction of twilight with dome open = {2:.2f}".format(firstopen, lastclose, fractwiopen) )

    return firstopen, lastclose, fractwiopen, twiopen_hrs

def calc_obstimes(night, verbose=False, clobber=False):
    '''
    Calculate start and lengths of science, dither, and guide exposures

    Parameters
    ----------
    night : int
        night in form 20210320 for 20 March 2021
    verbose : bool
        print verbose output?
    clobber : bool
        overwrite existing file

    Returns
    -------
    science_start : float
        start times of science exposures in UT hours
    science_width : float
        length of science exposures in UT hours
    dither_start : float
        start times of dither exposures in UT hours
    dither_width : float
        length of dither exposures in UT hours
    guide_start : float
        start times of guide exposures in UT hours
    guide_width : float
        length of guide exposures in UT hours
    '''

    try:
        outdir = utils.get_outdir()
    except RuntimeError:
        exit(1)

    # Read in the data
    specfilename = "specdata" + str(night) + ".json"
    specdatafile = os.path.join(outdir, 'NightlyData', specfilename)
    if os.path.isfile(specdatafile) and not clobber:
        specdata = utils.read_json(specdatafile)
    else:
        print("Note: {} not found ... creating it".format(specdatafile))
        calc_sciencelist(night)
        specdata = utils.read_json(specdatafile)

    guidefilename = "guidedata" + str(night) + ".json"
    guidedatafile = os.path.join(outdir, 'NightlyData', guidefilename)
    if os.path.isfile(guidedatafile) and not clobber:
        guidedata = utils.read_json(guidedatafile)
    else:
        print("Note: {} not found ... creating it".format(guidedatafile))
        calc_guidelist(night)
        guidedata = utils.read_json(guidedatafile)

    twibeg_mjd, twiend_mjd = utils.get_twilights(int(night))
    startdate = int(twibeg_mjd)

    # Calculate the start and duration for the science observations:
    science_start = []
    science_width = []
    for item in specdata:
        if specdata[item]['OBSTYPE'] == 'SCIENCE' and specdata[item]['FLAVOR'] == 'science' and 'Dither' not in specdata[item]['PROGRAM'] and specdata[item]['DOMSHUTU'] == 'open' and specdata[item]['PMCOVER'] == 'open':
            science_start.append( (specdata[item]['DATE-OBS'] - startdate)*24. )
            science_width.append( specdata[item]['EXPTIME']/3600. )

    # Separately account for time spent on dither tests
    dither_start = []
    dither_width = []
    for item in specdata:
        if specdata[item]['OBSTYPE'] == 'SCIENCE' and specdata[item]['FLAVOR'] == 'science' and 'Dither' in specdata[item]['PROGRAM']:
            dither_start.append( (specdata[item]['DATE-OBS'] - startdate)*24. )
            dither_width.append( specdata[item]['EXPTIME']/3600. )

    # Times for guiding:
    guide_start = []
    guide_width = []
    for item in guidedata:
        # if guidedata[item]['OBSTYPE'] == 'SCIENCE' and guidedata[item]['FLAVOR'] == 'science' and guidedata[item]['PMCOVER'] == 'open' and guidedata[item]['DOMSHUTU'] == 'open':
        if guidedata[item]['OBSTYPE'] == 'SCIENCE' and guidedata[item]['FLAVOR'] == 'science' and guidedata[item]['DOMSHUTU'] == 'open':
            guide_start.append( (guidedata[item]['GUIDE-START'] - startdate)*24. )
            guide_width.append( (guidedata[item]['GUIDE-STOP'] - guidedata[item]['GUIDE-START'])*24. )

    return science_start, science_width, dither_start, dither_width, guide_start, guide_width

def get_totobs(start, length, twibeg_hours, twiend_hours, verbose=False):
    '''
    Calculate the total observing time between twilights:

    Parameters
    ----------
    start : float
        start times of exposures in UT hours
    length : float
        length of exposures in UT hours
    twibeg_hours : float
        time of twilight at the beginning of the night in UT hours
    twiend_hours : float
        time of twilight at the end of the night in UT hours
    verbose : bool
        print verbose output?

    Returns
    -------
    obshours : float
        total observing hours between twilights
    '''

    obshours = 0.
    for i in range(len(start)):
        t1 = start[i]
        t2 = t1 + length[i]
        if t1 <= twibeg_hours and t2 >= twibeg_hours:
            obshours += t2-twibeg_hours
        elif t1 >= twibeg_hours and t2 <= twiend_hours:
            obshours += t2-t1
        elif t1 <= twiend_hours and t2 >= twiend_hours:
            obshours += t1-twiend_hours

    return obshours

def calc_science(night, verbose=False):
    '''
    Calculate quantities about science exposures for a given night

    Parameters
    ----------
    night : int
        night in form 20210320 for 20 March 2021
    verbose : bool
        print verbose output?

    Returns
    -------
    science_first : float
        start time of first spectroscopic exposure in MJD
    science_last : float
        end time of last spectroscopic exposure in MJD
    scitwi_hours : float
        number of hours of spectroscopic exposures between twilights in UT hours
    scitot_hours : float
        total number of hours of spectroscopic exposures in UT hours (not limited by twilight)
    '''

    twibeg_mjd, twiend_mjd = utils.get_twilights(int(night))
    startdate = int(twibeg_mjd)
    twibeg_hours = 24.*(twibeg_mjd - startdate)
    twiend_hours = 24.*(twiend_mjd - startdate)
    twitot_hours = twiend_hours - twibeg_hours

    science_start, science_width, dither_start, dither_width, guide_start, guide_width = calc_obstimes(night)

    # Science hours between twilights
    scitwi_hours = get_totobs(science_start, science_width, twibeg_hours, twiend_hours)

    # Calculate MJD of start of first science exposure:
    if len(science_start) == 0:
        return 0., 0., 0., 0.
    science_first = min(science_start)/24. + startdate
    indx = np.where( np.asarray(science_start) == max(science_start))[0][0]
    science_last = (science_start[indx] + science_width[indx])/24. + startdate

    # Science hours total
    scitot_hours = np.sum(science_width)

    # Open shutter time between twilights
    fracscitwi = scitwi_hours/(twiend_hours - twibeg_hours)

    return science_first, science_last, scitwi_hours, scitot_hours

def get_skydata(expidlist, night, verbose=False):
    '''
    Determine start and lengths of all sky frames associated with expidlist

    Parameters
    ----------
    expidlist : list
        list of expids
    night : int
        night in form 20210320 for 20 March 2021
    verbose : bool
        print verbose output?

    Returns
    -------
    sky_start : list
        start times of each sky exposure in UT hours
    sky_width : list
        lengths of each sky exposure in UT hours
    '''

    try:
        datadir = utils.get_rawdatadir()
    except RuntimeError:
        exit(1)
    nightdir = os.path.join(datadir, str(night))
    twibeg_mjd, twiend_mjd = utils.get_twilights(int(night))
    startdate = int(twibeg_mjd)

    if verbose:
        print(night, startdate, twibeg_mjd, nightdir)

    sky_start = []
    sky_width = []
    for expid in expidlist:
        tmpdir = os.path.join(nightdir, expid.zfill(8))
        skyfiles = (glob(tmpdir + "/sky-" + expid.zfill(8) + ".fits.fz"))
        if len(skyfiles) > 0:
            shh = fits.open(skyfiles[0])
            for i in range(len(shh['SKYCAM0T'].data)):
                tt = Time(shh['SKYCAM0T'].data[i]['DATE-OBS']).mjd
                sky_start.append((tt - startdate)*24 )
                sky_width.append(shh['SKYCAM0T'].data[i]['EXPTIME']/3600.)
    return sky_start, sky_width

def get_guidedata(expidlist, night, acqonly=False, startonly=False):
    '''
    Determine start and lengths of all sky frames associated with expidlist

    Parameters
    ----------
    expidlist : list
        list of expids
    night : int
        night in form 20210320 for 20 March 2021
    acqonly : bool
        report only the times for the acquisition images
    startonly : bool
        report only the start of guiding

    Returns
    -------
    guide_start : list
        start times of each guide exposure in UT hours
    guide_width : list
        lengths of each guide exposure in UT hours
    '''

    try:
        datadir = utils.get_rawdatadir()
    except RuntimeError:
        exit(1)
    nightdir = os.path.join(datadir, str(night))
    twibeg_mjd, twiend_mjd = utils.get_twilights(int(night))
    startdate = int(twibeg_mjd)

    guide_start = []
    guide_width = []
    for expid in expidlist:
        tmpdir = os.path.join(nightdir, expid.zfill(8))
        guidefiles = (glob(tmpdir + "/guide-" + expid.zfill(8) + ".fits.fz"))
        if len(guidefiles) > 0:
            ghh = fits.open(guidefiles[0])
            if acqonly:
                tt = Time(ghh['GUIDE0T'].data[0]['DATE-OBS']).mjd
                guide_start.append((tt - startdate)*24 )
                guide_width.append(ghh['GUIDE0T'].data[0]['EXPTIME']/3600.)
            elif startonly:
                tt = Time(ghh['GUIDE0T'].data[1]['DATE-OBS']).mjd
                guide_start.append((tt - startdate)*24 )
                guide_width.append(ghh['GUIDE0T'].data[1]['EXPTIME']/3600.)
            else:
                for i in range(len(ghh['GUIDE0T'].data)):
                    tt = Time(ghh['GUIDE0T'].data[i]['DATE-OBS']).mjd
                    guide_start.append((tt - startdate)*24 )
                    guide_width.append(ghh['GUIDE0T'].data[i]['EXPTIME']/3600.)
    return guide_start, guide_width

def get_fvcdata(expidlist, night, verbose=False):
    '''
    Determine start and lengths of all FVC frames associated with expidlist

    Parameters
    ----------
    expidlist : list
        list of expids
    night : int
        night in form 20210320 for 20 March 2021

    Returns
    -------
    fvc_start : list
        start times of each FVC exposure in UT hours
    fvc_width : list
        lengths of each FVC exposure in UT hours
    '''

    try:
        datadir = utils.get_rawdatadir()
    except RuntimeError:
        exit(1)
    nightdir = os.path.join(datadir, str(night))
    twibeg_mjd, twiend_mjd = utils.get_twilights(int(night))
    startdate = int(twibeg_mjd)

    fvc_start = []
    fvc_width = []
    for expid in expidlist:
        tmpdir = os.path.join(nightdir, expid.zfill(8))
        fvcfiles = (glob(tmpdir + "/fvc-" + expid.zfill(8) + ".fits.fz"))
        if len(fvcfiles) > 0:
            if verbose:
                print(fvcfiles[0])
            fhh = fits.open(fvcfiles[0])
            tt = Time(fhh['F0000'].header['DATE']).mjd
            fvc_start.append((tt - startdate)*24 )
            fvc_width.append(fhh['F0000'].header['EXPTIME']/3600.)
            try:   # Splits do not have a second FVC image
                tt = Time(fhh['F0001'].header['DATE']).mjd
                fvc_start.append((tt - startdate)*24 )
                fvc_width.append(fhh['F0001'].header['EXPTIME']/3600.)
            except KeyError:
                continue
    return fvc_start, fvc_width

def get_scidata(expidlist, night):
    '''
    Determine start and lengths of all science frames associated with expidlist

    Parameters
    ----------
    expidlist : list
        list of expids
    night : int
        night in form 20210320 for 20 March 2021

    Returns
    -------
    spec_start : list
        start times of each spec exposure in UT hours
    spec_width : list
        lengths of each spec exposure in UT hours
    spec_expid : list
        expid of each spec exposure
    '''

    try:
        datadir = utils.get_rawdatadir()
    except RuntimeError:
        exit(1)
    nightdir = os.path.join(datadir, str(night))
    twibeg_mjd, twiend_mjd = utils.get_twilights(int(night))
    startdate = int(twibeg_mjd)

    spec_start = []
    spec_width = []
    spec_expid = []

    # Identify expids that corresponds to science exposures:
    for expid in expidlist:
        tmpdir = os.path.join(nightdir, expid.zfill(8))
        scifiles = (glob(tmpdir + "/desi*"))
        if len(scifiles) > 0:
            hhh = fits.open(scifiles[0])
            tt = Time(''.join(hhh['SPECTCONS'].data['DATE-OBS'][0])).mjd
            try:
                flavor = hhh[1].header['FLAVOR']
                program = hhh[1].header['PROGRAM']
                obstype = hhh[1].header['OBSTYPE']
                exptime = hhh[1].header['EXPTIME']
            except:
                continue
            if obstype == 'SCIENCE' and flavor == 'science' and 'Dither' not in program and exptime > 10.:
                #spec_expids.append(expid)
                spec_start.append((tt - startdate)*24 )
                spec_width.append(hhh['SPEC'].header['EXPTIME']/3600.)
                spec_expid.append(expid)

    return spec_start, spec_width, spec_expid

def calc_interexp(night, minexptime=300., clobber=False, verbose=False):
    '''
    Calculate the interexposure times

    Parameters
    ----------
    night : int
        night in form 20210320 for 20 March 2021
    minexptime : float
        minimum science exposure time to consider in calculation
    verbose : bool
        print verbose output?
    clobber : bool
        overwrite existing file

    Returns
    -------
    interexp : 2-d float array
        columns are DATE-OBS in MJD, DATE-OBS in UT hrs, EXPTIME in s, EXPID, interexposure time in s
    '''

    # Read in the data
    try:
        outdir = utils.get_outdir()
    except RuntimeError:
        exit(1)

    specfilename = "specdata" + str(night) + ".json"
    specdatafile = os.path.join(outdir, 'NightlyData', specfilename)
    if os.path.isfile(specdatafile) and not clobber:
        specdata = utils.read_json(specdatafile)
    else:
        print("Note: {} not found ... creating it".format(specdatafile))
        calc_sciencelist(night)
        specdata = utils.read_json(specdatafile)

    # Take about the dict
    dateobs_mjd = []
    exptime = []
    expid = []
    for i, key in enumerate(specdata.keys()):
        if specdata[key]['EXPTIME'] > minexptime:
            dateobs_mjd.append(specdata[key]['DATE-OBS'])
            exptime.append(specdata[key]['EXPTIME'])
            expid.append(key)

    twibeg_mjd, twiend_mjd = utils.get_twilights(int(night))
    startdate = int(twibeg_mjd)

    scidata = np.zeros([len(dateobs_mjd), 5])
    for i in range(len(dateobs_mjd)):
        dateobs_hrs = (dateobs_mjd[i] - startdate)*24.
        scidata[i] = np.asarray([dateobs_mjd[i], dateobs_hrs, exptime[i], expid[i], 0.])

    # Sort by expid:
    columnIndex = 3
    sortdata = scidata[scidata[:,columnIndex].argsort()]

    # Calculate interexposure time between each successive pair
    secperday = 24.*3600.
    interexp = []
    for i in range(len(sortdata) - 1):
        sortdata[i][4] = secperday*(sortdata[i+1][0] - sortdata[i][0] - sortdata[i][1]/secperday)
        # interexp.append(secperday*(sortdata[i+1][0] - sortdata[i][0] - sortdata[i][1]/secperday ))

    return sortdata[:-1]

max_stars_repo_name: desihub/desisurveyops

max_issues_repo_name: desihub/desisurveyops

max_forks_repo_name: desihub/desisurveyops

reason: import numpy,from astropy


text:
import sys
sys.path.insert(0, '..')

import agents
import time
import gym
import numpy

import models.dqn_imagination.src.model            as Model
import models.dqn_imagination.src.model_env        as ModelEnv
import models.dqn_imagination.src.config           as Config

from common.Training import *
from common.atari_wrapper import *

path = "models/dqn_imagination/"

env = gym.make("MsPacmanNoFrameskip-v4")
env = AtariWrapper(env)
env.reset()

agent = agents.AgentDQNImagination(env, Model, ModelEnv, Config)

max_iterations = 10*(10**6)

trainig = TrainingIterations(env, agent, max_iterations, path, 10000)
trainig.run()

'''
agent.load(path)
agent.disable_training()
while True:
    reward, done = agent.main()

    env.render()
    time.sleep(0.01)
'''

ext: py

reason: import numpy


text:
import numpy as np
import pyranges as pr
from tests.helpers import assert_df_equal

a = '''Chromosome Start End Strand
chr1    6    12  +
chr1    10    20 +
chr1    22    27 -
chr1    24    30 -'''
b = '''Chromosome Start End Strand
chr1    12    32 +
chr1    14    30 +'''
c = '''Chromosome Start End Strand
chr1    8    15 +
chr1    713800    714800 -
chr1    32    34 -'''

grs = {n: pr.from_string(s) for n, s in zip(["a", "b", "c"], [a, b, c])}
unstranded_grs = {n: gr.unstrand() for n, gr in grs.items()}

features = pr.PyRanges(chromosomes=["chr1"] * 4,
                       starts=[0, 10, 20, 30],
                       ends=[10, 20, 30, 40],
                       strands = ["+","+","+","-"])
unstranded_features = features.unstrand()

def test_strand_vs_strand_same():

    expected_result = pr.from_string("""Chromosome Start End Strand a b c
chr1  0 10  + 1 0 1
chr1 10 20  + 2 2 1
chr1 20 30  + 0 2 0
chr1 30 40  - 0 0 1""")

    res = pr.count_overlaps(grs, features, strandedness="same")
    res = res.apply(lambda df: df.astype({"a": np.int64, "b": np.int64, "c": np.int64}))

    res.print(merge_position=True)

    assert_df_equal(res.df, expected_result.df)

# def test_strand_vs_strand_opposite():

#     expected_result = pr.from_string("""Chromosome Start End Strand a b c
# chr1  0 10  + 1 0 1
# chr1 10 20  + 1 2 1
# chr1 20 30  + 0 2 0
# chr1 30 40  - 0 0 1""")

#     res = pr.count_overlaps(grs, features, strandedness="opposite")

#     print("features")
#     print(features)

#     for name, gr in grs.items():
#         print(name)
#         print(gr)

#     res.print(merge_position=True)

#     assert_df_equal(res.df, expected_result.df)

max_stars_repo_name: SamBryce-Smith/pyranges

max_issues_repo_name: SamBryce-Smith/pyranges

max_forks_repo_name: SamBryce-Smith/pyranges


text:
import numpy as np
import cv2
from matplotlib import pyplot as plt

img = cv2.imread('t.jpg',0)

# Initiate STAR detector
#star = cv2.FeatureDetector_create("STAR")

# Initiate BRIEF extractor
brief = cv2.DescriptorExtractor_create("BRIEF")

# find the keypoints with STAR
#kp = star.detect(img,None)

# compute the descriptors with BRIEF
kp, des = brief.compute(img, kp)

print (brief.getInt('bytes'))
print (des.shape)


text:
import numpy as np

class Env:
    def __init__(self, n_action, p):
        self.n_action = n_action
        self.p = p

    def sample(self, action, n_sample):
        return (np.random.uniform(0, 1, n_sample) < self.p[action]).astype(np.uint8)

def G1():
    def _op(n_action):
        p = np.random.uniform(0.02, 0.05, n_action)
        return p
    return _op

def G2():
    def _op(idx, n_action):
        p_upper = 0.02 + idx * 0.01
        p = np.random.uniform(0.02, p_upper, n_action)
        return p
    return _op

def G3():
    def _op(n_action):
        p = np.zeros(n_action) + 0.05
        return p
    return _op


text:
import unittest

import numpy
import pytest

from chainerx_tests import array_utils
import chainer.testing
import chainerx
import chainerx.testing

from chainerx_tests import dtype_utils
from chainerx_tests import op_utils

@op_utils.op_test(['native:0', 'cuda:0'])
@chainer.testing.parameterize_pytest('shape,indices', [
    # empty indexing
    ((), ()),
    ((3,), ()),
    ((2, 2, 2), ()),
    # integer indexing - non-tuple indexing
    ((3,), 0),
    ((3,), 1),
    ((3,), 2),
    ((3,), -1),
    ((2, 3), 0),
    ((2, 3), 1),
    ((2, 3), numpy.int8(-1)),
    ((2, 3), numpy.int32(0)),
    ((2, 3), numpy.uint64(1)),
    # integer indexining - tuple indexing
    ((3,), (0,)),
    ((3,), (1,)),
    ((3,), (2,)),
    ((3,), (-1,)),
    ((2, 3), (0,)),
    ((2, 3), (1,)),
    ((2, 3), (0, 0)),
    ((2, 3), (1, 1)),
    ((2, 3, 4), (0, -2, 3)),
    ((2, 3, 4), (1, 0)),
    # slice indexing - non-tuple indexing
    ((3,), slice(None)),
    ((3,), slice(2)),
    ((3,), slice(0, 3)),
    ((3,), slice(0, 2)),
    ((3,), slice(1, 3)),
    ((3,), slice(0, 0)),
    ((3,), slice(0, 1)),
    ((3,), slice(2, 0, -1)),
    ((3,), slice(-2, -1)),
    ((3,), slice(2, None, -1)),
    ((3,), slice(None, 0, 1)),
    ((3,), slice(None, -1, -1)),
    ((3,), slice(None, -2, -1)),
    ((6,), slice(0, 6, 2)),
    ((6,), slice(1, 6, 2)),
    ((6,), slice(5, None, -2)),
    # slice indexing - tuple indexing
    ((3,), (slice(None),)),
    ((3,), (slice(2),)),
    ((3,), (slice(0, 3),)),
    ((3,), (slice(0, 2),)),
    ((3,), (slice(1, 3),)),
    ((3,), (slice(0, 0),)),
    ((3,), (slice(0, 1),)),
    ((3,), (slice(2, 0, -1),)),
    ((3,), (slice(-2, -1),)),
    ((3,), (slice(2, None, -1),)),
    ((3,), (slice(None, 0, 1),)),
    ((3,), (slice(None, -1, -1),)),
    ((3,), (slice(None, -2, -1),)),
    ((6,), (slice(0, 6, 2),)),
    ((6,), (slice(1, 6, 2),)),
    ((6,), (slice(5, None, -2),)),
    ((6,), (slice(50, 1, -1),)),
    ((6,), (slice(3, 3, 1),)),
    ((6,), (slice(3, 3, -2),)),
    ((6,), (slice(50, 50, 1),)),
    ((6,), (slice(50, 50, -2),)),
    ((6,), (slice(-50, -50, 1),)),
    ((6,), (slice(-50, -50, -2),)),
    ((2, 3), (slice(None), slice(None))),
    ((2, 3), (slice(1), slice(2))),
    ((2, 3), (slice(0, 2), slice(0, 3))),
    ((2, 3), (slice(0, 2), slice(0, -1))),
    ((2, 3), (slice(0, None, -1), slice(2, 3))),
    ((2, 3), (slice(0, None, None), slice(-2, 0, -1))),
    ((2, 3), (slice(1, 2), slice(0, 2))),
    ((2, 3), (slice(-2, None, -1), slice(0, 3))),
    ((2, 3), (slice(-2, None, -1), slice(-3, None, -1))),
    ((2, 3), (slice(-2, None, -1), slice(None, None, -2))),
    ((2, 3), (slice(1, 2), slice(None, None, 1))),
    ((2, 3), (slice(1, 2), slice(None, None, 2))),
    ((2, 3, 4), (slice(1), slice(-2, 3), slice(1, None, -1))),
    # newaxis indexing - non-tuple indexing
    ((), chainerx.newaxis),
    ((3,), chainerx.newaxis),
    # newaxis indexing - tuple indexing
    ((), (chainerx.newaxis,)),
    ((3,), (chainerx.newaxis,)),
    ((2, 3), (chainerx.newaxis, chainerx.newaxis)),
    # mixed indexing - tuple indexing
    ((2, 3), (0, slice(1, 3))),
    ((4, 3), (slice(1, 3), 1)),
    ((2, 3, 4), (1, slice(2,), slice(1, 3))),
    ((2, 3), (1, chainerx.newaxis, slice(1, 3))),
    ((2, 3, 4), (slice(0, 1), slice(1, 2), slice(1, 3), chainerx.newaxis)),
    ((2, 3, 4), (slice(0, 1), slice(1, 2), chainerx.newaxis, slice(1, 3))),
    ((2, 3, 4), (slice(0, 1), chainerx.newaxis, slice(1, 2), slice(1, 3))),
    ((2, 3, 4), (chainerx.newaxis, slice(0, 1), slice(1, 2), slice(1, 3))),
    ((2, 3, 4),
     (1, slice(2,), chainerx.newaxis, slice(1, 3), chainerx.newaxis)),
])
class TestGetitem(op_utils.NumpyOpTest):
    # TODO(niboshi): Remove this
    check_numpy_strides_compliance = False

    def generate_inputs(self):
        x = numpy.random.uniform(-1, 1, self.shape).astype('float32')
        return x,

    def forward_xp(self, inputs, xp):
        x, = inputs
        y = x[self.indices]
        return y,

@pytest.mark.parametrize_device(['native:0', 'cuda:0'])
def test_getitem_zero_sized_offsets(device):
    a = chainerx.arange(6)

    b = a[3:3]
    # Test pre-conditions.
    assert b.size == 0
    assert b.offset == 12

    # The offset of `c` should be the same as `b` since `b` is empty.
    c = b[2:]
    assert c.size == 0
    assert c.offset == b.offset

@op_utils.op_test(['native:0', 'cuda:0'])
# TODO(hvy): Add cases where axis=None, when supported.
@chainer.testing.parameterize_pytest('shape,indices,axis', [
    # Valid parameters
    ((3,), [0], 0),
    ((3,), [1], 0),
    ((2, 3), [0], 0),
    ((2, 3), [0], 1),
    ((2, 3), [0], -1),
    ((2, 3), [1], 0),
    ((2, 3), [0, -1], 0),
    ((2, 3), [1, 0], 0),
    ((2, 3), [1, 2], 1),
    ((2, 3), [2, 1], 1),
    ((2, 3), [[0], [1]], 0),
    # Invalid: Axis out of bounds
    ((2, 3), [0], 2),
    ((2, 3), [0], -3),
])
@chainer.testing.parameterize_pytest('is_module', [True, False])
@chainer.testing.parameterize_pytest(
    'indices_type', ['list', 'numpy', 'xp'])
# TODO(niboshi): indices_dtype is ignored if indices_type == 'list', which is
# wasteful.
@chainer.testing.parameterize_pytest(
    'indices_dtype', chainerx.testing.integral_dtypes)
class TestTake(op_utils.NumpyOpTest):

    check_numpy_strides_compliance = False
    forward_accept_errors = (chainerx.DimensionError, numpy.AxisError)

    def setup(self):
        if (numpy.dtype(self.indices_dtype).kind == 'u'
                and (numpy.array(self.indices, 'int64') < 0).any()):
            raise unittest.SkipTest(
                'Indices underflows and index out of bounds cannot be tested.')

    def generate_inputs(self):
        a = numpy.random.uniform(-1, 1, self.shape).astype('float32')
        return a,

    def forward_xp(self, inputs, xp):
        indices = self.indices
        axis = self.axis
        indices_type = self.indices_type
        a, = inputs

        assert isinstance(indices, list)
        if indices_type == 'list':
            pass
        elif indices_type == 'numpy':
            indices = numpy.array(indices).astype(self.indices_dtype)
        elif indices_type == 'xp':
            indices = xp.array(indices).astype(self.indices_dtype)
        else:
            assert False, indices_type

        if self.is_module:
            b = xp.take(a, indices, axis)
        else:
            b = a.take(indices, axis)
        return b,

@op_utils.op_test(['native:0', 'cuda:0'])
@chainer.testing.parameterize_pytest('cond_shape,x_shape,y_shape', [
    # Same Shapes
    ((2, 3), (2, 3), (2, 3)),
    # Broadcast Shapes
    ((2, 3), (1, 3), (1, 3)),
    ((2, 3), (2, 1), (1, 3)),
    ((2, 3), (2, 3), (1, 3)),
    ((4, 5), (3, 4, 1), (1, 5)),
    ((1, 4, 5), (3, 4, 1), (3, 1, 5)),
])
@chainer.testing.parameterize_pytest(
    'in_dtypes,out_dtype', dtype_utils.result_dtypes_two_arrays
)
@chainer.testing.parameterize_pytest(
    'condition_dtype', chainerx.testing.all_dtypes)
class TestWhere(op_utils.NumpyOpTest):

    check_numpy_strides_compliance = False

    def setup(self):
        (x_dtype, y_dtype) = self.in_dtypes
        if numpy.dtype(x_dtype).kind != 'f' or \
           numpy.dtype(y_dtype).kind != 'f':
            self.skip_backward_test = True
            self.skip_double_backward_test = True

        if x_dtype == 'float16' or y_dtype == 'float16':
            self.check_backward_options.update({'rtol': 1e-3, 'atol': 1e-3})

    def generate_inputs(self):
        (x_dtype, y_dtype) = self.in_dtypes
        x = array_utils.uniform(self.x_shape, x_dtype)
        y = array_utils.uniform(self.y_shape, y_dtype)
        condition = numpy.random.uniform(0, 1, size=self.cond_shape)
        self.condition = (condition > 0.5).astype(self.condition_dtype)
        return (x, y)

    def forward_xp(self, inputs, xp):
        x, y = inputs
        condition = xp.array(self.condition)
        o = xp.where(condition, x, y)
        o = dtype_utils.cast_if_numpy_array(xp, o, self.out_dtype)
        return o,

@chainerx.testing.numpy_chainerx_array_equal(
    accept_error=(
        chainerx.DimensionError, ValueError))
@pytest.mark.parametrize('cond_shape,x_shape,y_shape', [
    ((2, 3), (3, 4), (2, 3)),
    ((2, 3), (2, 3), (3, 4)),
    ((2, 3), (1, 3), (2, 4))
])
def test_where_invalid_shapes(xp, cond_shape, x_shape, y_shape):
    x = array_utils.create_dummy_ndarray(xp, x_shape, 'float32')
    y = array_utils.create_dummy_ndarray(xp, y_shape, 'float32')
    c = array_utils.create_dummy_ndarray(xp, cond_shape, 'float32')
    return xp.where(c, x, y)

max_stars_repo_name: tkerola/chainer

max_issues_repo_name: tkerola/chainer

max_forks_repo_name: tkerola/chainer


text:
import numpy as nu
import time
import sys
import warnings
warnings.filterwarnings("ignore")
class InvalidLength:
 def __init__(self,value):
  self.Value=value
#-----------------------------------------------------------------#
def FFT(x):
 x = nu.asarray(x,dtype=float)
 N = len(x)
 if N==1:
  return x
 else:
  x_even = FFT(x[::2])
  x_odd = FFT(x[1::2])
  factor = nu.array(nu.exp(-2j * nu.pi * nu.arange(N) / N),dtype=complex)
  fft = nu.concatenate([x_even + factor[:N / 2] * x_odd,x_even + factor[N / 2:] * x_odd])
  fft = nu.around(fft,decimals=8)
  return fft

#-----------------------------------------------------------------#
def dft_mat(x):
 x = nu.asarray(x,dtype=float)
 N = len(x)
 w = nu.mat(nu.zeros(shape=(N,N)),dtype=complex)
 dft = nu.array(0*x,dtype=complex)
 for i in range(0,N):
  for j in range(0,N):
   w[i,j] = nu.exp(-2j*nu.pi*i*j/N)
 for n in range(0,N):
  dft[n] = 0
  for k in range(0,N):
   dft[n] = dft[n] + w[n,k]*x[k]
 dft = nu.around(dft,decimals=8)
 return dft

#-----------------------------------------------------------------#
def dft_sum(x):
 x = nu.asarray(x,dtype=float)
 N = len(x)
 dft = nu.array(0*x,dtype=complex)
 for n in range(0,N):
  dft[n] = 0
  for k in range(0,N):
   dft[n] = dft[n] + nu.exp(-2j*nu.pi*n*k/N)*x[k]
 dft = nu.around(dft,decimals=8)
 return dft

#-----------------------------------------------------------------#
#x=[2, 6, 3, 9, 6, 5, 4, 2]
time1 = None
time2 = None
time3 = None
flag=0
try:
 x     = map(float,sys.argv[1:])
 n     = int(nu.log(len(x))/nu.log(2))
 start = time.time()
 fft   = dft_sum(x)
 stop  = time.time()
 print "DFT by Summation Method\n", fft
 time1 = stop - start
 start = time.time()
 fft   = dft_mat(x)
 stop  = time.time()
 print "DFT by Matrix Method\n", fft
 time2 = stop - start
 if 2**n!=len(x):
  raise InvalidLength(len(x))
except InvalidLength as e:
 print "Length of given sequence is ",e.Value," which is not of the order of 2 so FFT can't be computed"
except OverflowError:
 print "Error:Input Sequence is missing.\nProvide the sequence in command line\nFor eg:\npython",sys.argv[0],"1 2 3 6"
 flag=1
except ValueError:
 print "Error:Please Provide Correct input"
 flag=1
else:
 start = time.time()
 fft   = FFT(x)
 stop  = time.time()
 time3 = stop - start
 print "DFT by FFT Method\n", fft
finally:
 if flag==1:
  exit()
 print "\nTime Required to compute DFT:\n"
 print "Summation Method : ",time1,"\nMatrix Method    : ",time2
 print "FFT Method       : ",time3,"\n"

max_stars_repo_name: suraj-deshmukh/muCodes

max_issues_repo_name: suraj-deshmukh/muCodes

max_forks_repo_name: suraj-deshmukh/muCodes


text:
import sys
import numpy as np
import pandas as pd
#from logger import Logger
from sklearn.model_selection import StratifiedKFold

def Sf_KFold(X, y, n_splits):
    i = 0
    skf = StratifiedKFold(n_splits)
    skf.get_n_splits(X, y)
    for train_index, test_index in skf.split(X, y):
        i += 1
        print("Train: ", train_index, "Test: ", test_index)
        X_train, X_test = X[train_index], X[test_index]
        y_train, y_test = y[train_index], y[test_index]

        dataframe_train = pd.DataFrame({'video_name': X_train, 'label': y_train})
        dataframe_train.to_csv("video_trainset_%d.csv"%i, index=False, sep=',')
        dataframe_test = pd.DataFrame({'video_name': X_test, 'label': y_test})
        dataframe_test.to_csv("video_testset_%d.csv"%i, index=False, sep=',')

    #return X_train, X_test, y_train, y_test
def logger():
    #write log file
    sys.stdout = Logger('test.log', sys.stdout)
    sys.stderr = Logger('test_err.log', sys.stderr)

def main():
    #logger()
    n_splits=5
    all_info_file = 'alldata_single_rm_less_50.csv'
    df = pd.read_csv(all_info_file)
    images_path = df['order']
    labels = df['label']
    Sf_KFold(images_path, labels, n_splits)

if __name__ == "__main__":
    main()

max_stars_repo_name: enoshx/routine

max_issues_repo_name: enoshx/routine

max_forks_repo_name: enoshx/routine


text:
import torch.nn as nn
import torch
from torch.autograd import Variable
import torch.nn.functional as F

import numpy as np

def weights_init(m):
    classname = m.__class__.__name__  #   obtain the class name
    if classname.find('Linear') != -1:
        weight_shape = list(m.weight.data.size())
        fan_in = weight_shape[1]
        fan_out = weight_shape[0]
        w_bound = np.sqrt(6. / (fan_in + fan_out))
        m.weight.data.uniform_(-w_bound, w_bound)
        m.bias.data.fill_(0)
        print("inital  linear weight ")

class word_embedding(nn.Module):
    def __init__(self,vocab_length , embedding_dim):
        super(word_embedding, self).__init__()
        w_embeding_random_intial = np.random.uniform(-1,1,size=(vocab_length ,embedding_dim))
        self.word_embedding = nn.Embedding(vocab_length,embedding_dim)
        self.word_embedding.weight.data.copy_(torch.from_numpy(w_embeding_random_intial))
    def forward(self,input_sentence):
        """
        :param input_sentence:  a tensor ,contain several word index.
        :return: a tensor ,contain word embedding tensor
        """
        sen_embed = self.word_embedding(input_sentence)
        return sen_embed

class RNN_model(nn.Module):
    def __init__(self, batch_sz ,vocab_len ,word_embedding,embedding_dim, lstm_hidden_dim):
        super(RNN_model,self).__init__()

        self.word_embedding_lookup = word_embedding
        self.batch_size = batch_sz
        self.vocab_length = vocab_len
        self.word_embedding_dim = embedding_dim
        self.lstm_dim = lstm_hidden_dim
        #########################################
        # here you need to define the "self.rnn_lstm"  the input size is "embedding_dim" and the output size is "lstm_hidden_dim"
        # the lstm should have two layers, and the  input and output tensors are provided as (batch, seq, feature)
        # ???

        ##########################################
        self.fc = nn.Linear(lstm_hidden_dim, vocab_len )
        self.apply(weights_init) # call the weights initial function.

        self.softmax = nn.LogSoftmax() # the activation function.
        # self.tanh = nn.Tanh()
    def forward(self,sentence,is_test = False):
        batch_input = self.word_embedding_lookup(sentence).view(1,-1,self.word_embedding_dim)
        # print(batch_input.size()) # print the size of the input
        ################################################
        # here you need to put the "batch_input"  input the self.lstm which is defined before.
        # the hidden output should be named as output, the initial hidden state and cell state set to zero.
        # ???

        ################################################
        out = output.contiguous().view(-1,self.lstm_dim)

        out =  F.relu(self.fc(out))

        out = self.softmax(out)

        if is_test:
            prediction = out[ -1, : ].view(1,-1)
            output = prediction
        else:
           output = out
        # print(out)
        return output


text:
"""Tests for the log_normal_distribution module"""

# Copyright 2019 Dean Scarff
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
import unittest
from ..log_normal_distribution import LogNormalDistribution
from .assertions import assert_close

class TestLogNormalDistribution(unittest.TestCase):
    """Tests for the LogNormalDistribution class"""

    def setUp(self):
        np.seterr(all="raise")
        self.assert_close = assert_close.__get__(self, self.__class__)

    def test_get_derivatives_var_1(self):
        """Test LogNormalDistribution(mu, 1).get_derivatives()"""
        mu = np.array([0.0, 1.0, 2.0])
        x = np.array([1.0, 1.0, 1.0])
        dist = LogNormalDistribution(mu, 1.0)
        d1, d2 = dist.get_derivatives(x)
        self.assert_close([0.0, 1.0, 2.0], d1, "d1")
        self.assert_close([-1.0, -1.0, -1.0], d2, "d2")

    def test_get_derivatives_var_2(self):
        """Test LogNormalDistribution(0, 2).get_derivatives()"""
        x = np.exp([0.0, 1.0, 2.0])
        dist = LogNormalDistribution(0.0, 2.0)
        d1, d2 = dist.get_derivatives(x)
        self.assert_close([0.0, -0.5, -1.0], d1, "d1")
        self.assert_close([-0.5, -0.5, -0.5], d2, "d2")


text:
import numpy as np
from keras.models import Sequential
from keras.layers import RepeatVector
from keras import optimizers

inp = 5

model = Sequential()
model.add(RepeatVector(3, input_shape = (inp,)))
model.compile(loss='mse', optimizer='sgd')

input_data = np.ndarray((1, inp))

for i in range(0, inp):
   input_data[0, i] = i * 2 +1

output_data = model.predict(input_data)

print(output_data)
print(output_data.shape)

max_stars_repo_name: andrasberes/NNSharp

max_issues_repo_name: andrasberes/NNSharp

max_forks_repo_name: andrasberes/NNSharp


text:
import numpy as np
import pandas as pd
import math

def run(benchmark_data, no_of_crossvalid_runs, no_of_folds):
    no_of_benchmark_scores = len(benchmark_data.index)
    result = pd.DataFrame(np.zeros((no_of_benchmark_scores, no_of_benchmark_scores)))

    for i in range(0, no_of_benchmark_scores):
        matrix_i = benchmark_data.at[i, 'Score matrix']
        for j in range(i + 1, no_of_benchmark_scores):
            matrix_j = benchmark_data.at[j, 'Score matrix']

            t_value = calculate_t(matrix_i, matrix_j, no_of_crossvalid_runs, no_of_folds)

            result.at[i, j] = t_value
            result.at[j, i] = t_value * -1

    result_bool = check_critical_value(result)
    sum_rows(result_bool)

    result_bool.sort_values(by=['Sum'], ascending=False, inplace=True)

    return result_bool

def calculate_t(matrix_i, matrix_j, no_of_crossvalid_runs, no_of_folds):
    no_of_crossvalid_runs = matrix_i.shape[0]
    no_of_folds = matrix_i.shape[1]

    if ((matrix_i == matrix_j).all()):
        return 0;

    matrix_s = matrix_i - matrix_j
    avg_matrix_s = np.mean(matrix_s)
    t_value = 0

    for i in range(0, no_of_crossvalid_runs):
        for j in range(0, no_of_folds):
            sigma = matrix_s[i, j]
            sigma = sigma - avg_matrix_s
            sigma = sigma * sigma
            sigma = sigma / ((no_of_folds * no_of_crossvalid_runs) - 1)
            t_value = t_value + sigma

    t_value = t_value * ((1 / (no_of_folds * no_of_crossvalid_runs)) + (1 / no_of_folds))
    t_value = math.sqrt(t_value)
    t_value = avg_matrix_s / t_value

    return t_value

def check_critical_value(matrix):
    critical_value = 2.2622

    matrix_size = len(matrix.index)
    result = pd.DataFrame(np.zeros((matrix_size, matrix_size))).astype(int)

    for i in range(0, matrix_size):
        for j in range(0, matrix_size):
            if (matrix.at[i, j] > critical_value):
                result.at[i, j] = 1
            elif (matrix.at[i, j] < (-1 * critical_value)):
                result.at[i, j] = -1
            else:
                result.at[i, j] = 0

    return result

def sum_rows(matrix):
    result = [];
    matrix_size = len(matrix.index)

    for i in range(0, matrix_size):
        sum = 0;
        for j in range(0, matrix_size):
            sum = sum + matrix.at[i, j]

        result.append(sum)

    matrix["Sum"] = result

max_stars_repo_name: baatochan/MyocardialInfarctionDiagnosis

max_issues_repo_name: baatochan/MyocardialInfarctionDiagnosis

max_forks_repo_name: baatochan/MyocardialInfarctionDiagnosis


text:
 # -*-coding:utf-8-*-
import os
import math
import shutil
import logging
import numpy as np
import random
import torch
import torchvision
import torchvision.transforms as transforms

class Cutout(object):
    def __init__(self, n_holes, length):
        self.n_holes = n_holes
        self.length = length

    def __call__(self, img):
        h = img.size(1)
        w = img.size(2)

        mask = np.ones((h, w), np.float32)

        for n in range(self.n_holes):
            y = np.random.randint(h)
            x = np.random.randint(w)

            y1 = np.clip(y - self.length // 2, 0, h)
            y2 = np.clip(y + self.length // 2, 0, h)
            x1 = np.clip(x - self.length // 2, 0, w)
            x2 = np.clip(x + self.length // 2, 0, w)

            mask[y1: y2, x1: x2] = 0.

        mask = torch.from_numpy(mask)
        mask = mask.expand_as(img)
        img = img * mask

        return img

class Logger(object):
    def __init__(self, log_file_name, log_level, logger_name):
        self.__logger = logging.getLogger(logger_name)
        self.__logger.setLevel(log_level)
        file_handler = logging.FileHandler(log_file_name)
        console_handler = logging.StreamHandler()
        formatter = logging.Formatter(
            '[%(asctime)s] - [%(filename)s line:%(lineno)d] : %(message)s')
        file_handler.setFormatter(formatter)
        console_handler.setFormatter(formatter)
        self.__logger.addHandler(file_handler)
        self.__logger.addHandler(console_handler)

    def get_log(self):
        return self.__logger

def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

def data_augmentation(config, is_train=True):
    aug = []
    if is_train:
        # random crop
        if config.augmentation.random_crop:
            aug.append(transforms.RandomCrop(config.input_size, padding=4))
        # horizontal filp
        if config.augmentation.random_horizontal_filp:
            aug.append(transforms.RandomHorizontalFlip())

    aug.append(transforms.ToTensor())
    # normalize  [- mean / std]
    if config.augmentation.normalize:
        if config.dataset == 'cifar10':
            aug.append(transforms.Normalize(
                (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)))
        else:
            aug.append(transforms.Normalize(
                (0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)))

    if is_train and config.augmentation.cutout:
        # cutout
        aug.append(Cutout(n_holes=config.augmentation.holes,
                          length=config.augmentation.length))
    return aug

def save_checkpoint(state, is_best, filename):
    torch.save(state, filename + '.pth.tar')
    if is_best:
        shutil.copyfile(filename + '.pth.tar', filename + '_best.pth.tar')

def load_checkpoint(path, model, optimizer=None):
    if os.path.isfile(path):
        logging.info("=== loading checkpoint '{}' ===".format(path))

        checkpoint = torch.load(path)
        model.load_state_dict(checkpoint['state_dict'], strict=False)

        if optimizer is not None:
            best_prec = checkpoint['best_prec']
            last_epoch = checkpoint['last_epoch']
            optimizer.load_state_dict(checkpoint['optimizer'])
            logging.info("=== done. also loaded optimizer from " +
                         "checkpoint '{}' (epoch {}) ===".format(
                             path, last_epoch + 1))
            return best_prec, last_epoch

def get_data_loader(transform_train, transform_test, config):
    assert config.dataset == 'cifar10' or config.dataset == 'cifar100'
    if config.dataset == "cifar10":
        trainset = torchvision.datasets.CIFAR10(
            root=config.data_path, train=True,
            download=True, transform=transform_train)

        testset = torchvision.datasets.CIFAR10(
            root=config.data_path, train=False,
            download=True, transform=transform_test)
    else:
        trainset = torchvision.datasets.CIFAR100(
            root=config.data_path, train=True,
            download=True, transform=transform_train)

        testset = torchvision.datasets.CIFAR100(
            root=config.data_path, train=False,
            download=True, transform=transform_test)

    train_loader = torch.utils.data.DataLoader(
        trainset, batch_size=config.batch_size,
        shuffle=True, num_workers=config.workers)

    test_loader = torch.utils.data.DataLoader(
        testset, batch_size=config.test_batch,
        shuffle=False, num_workers=config.workers)
    return train_loader, test_loader

def mixup_data(x, y, alpha, device):
    '''Returns mixed inputs, pairs of targets, and lambda'''
    if alpha > 0:
        lam = np.random.beta(alpha, alpha)
    else:
        lam = 1

    batch_size = x.size()[0]
    index = torch.randperm(batch_size).to(device)

    mixed_x = lam * x + (1 - lam) * x[index, :]
    y_a, y_b = y, y[index]
    return mixed_x, y_a, y_b, lam

def mixup_criterion(criterion, pred, y_a, y_b, lam):
    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)

def cutmix_data(input, target, alpha, device):
    lam = np.random.beta(alpha, alpha)
    rand_index = torch.randperm(input.size()[0]).cuda()
    target_a = target
    target_b = target[rand_index]
    bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam)
    input[:, :, bbx1:bbx2, bby1:bby2] = input[rand_index, :, bbx1:bbx2, bby1:bby2]
    # adjust lambda to exactly match pixel ratio
    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))
    # compute output
    input_var = torch.autograd.Variable(input, requires_grad=True)
    targets_a = torch.autograd.Variable(target_a)
    targets_b = torch.autograd.Variable(target_b)
    return input_var, targets_a, targets_b, lam

def cutmix_criterion(criterion, pred, y_a, y_b, lam):
    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)

def imshow(img):
    # img = img / 2 + 0.5
    # npimg = img.numpy()
    plt.imshow(np.transpose(img, (1, 2, 0)))
    plt.show()

def get_area(y1, y2, x1, x2):
    return (y2-y1)*(x2-x1)

def get_holes(points_num):
    center = (16,16)
    r = 3*points_num-3
    perimeter = 2*r*math.pi
    slot=perimeter/points_num
    x_vals=[]
    y_vals=[]
    for i in range(points_num):
        interval = slot*i+random.uniform(0, slot/2)
        angle = interval/r
        x=center[0]+r*math.cos(angle)
        y=center[1]+r*math.sin(angle)
        x_vals.append(round(x))
        y_vals.append(round(y))

    holes = []
    tot_area = 0
    for i in range(len(x_vals)):
        c = 32-3*points_num
        length = random.randint(c-3,c+3)
        x = x_vals[i]
        y = y_vals[i]
        y1 = np.clip(y - length // 2, 0, 32)
        y2 = np.clip(y + length // 2, 0, 32)
        x1 = np.clip(x - length // 2, 0, 32)
        x2 = np.clip(x + length // 2, 0, 32)
        hole = [y1, y2, x1, x2]
        holes.append(hole)

    return holes

def randmix_data(holes, inputs, target, device):
    mask = np.zeros((inputs.size()[0], 3,32, 32), np.float32)
    mask = torch.from_numpy(mask)
    mask = mask.expand_as(inputs)
    permu = []
    targets = []
    lams = []
    tot_area = 0
    r1 = 0.7
    r2 = 1-r1
    # for i in range(len(holes)):
    #     tot_area += holes[i][4]

    for i in range(len(holes)):
        rand_index = torch.randperm(inputs.size()[0])
        permu.append(rand_index)
        targets.append(target[rand_index])
        # lams.append(holes[i][4]/tot_area)
        y21 = holes[i][0]
        y22 = holes[i][1]
        x21 = holes[i][2]
        x22 = holes[i][3]
        mask[:, :, y21: y22, x21: x22] = inputs[rand_index, :, y21: y22, x21: x22]
        lams.append(get_area(y21, y22, x21, x22))
        tot_area+= lams[i]
        if i > 0:
            for k in range (i):
                y11 = holes[k][0]
                y12 = holes[k][1]
                x11 = holes[k][2]
                x12 = holes[k][3]
                if y12<y21 or y11 > y22 or x11>x22 or x12<x21:
                    continue
                elif x12>x22 and y11>=y21 and y12<=y22:
                    mask[:, :, y11:y12, x11:x22] = r1*inputs[rand_index, :, y11:y12, x11:x22] + r2*inputs[permu[k], :, y11:y12, x11:x22]
                    tot_area-=get_area(y11, y12, x11, x22)
                    lams[k]-=r1*get_area(y11, y12, x11, x22)
                    lams[i]-=r2*get_area(y11, y12, x11, x22)
                elif x12>x22 and y11<=y21 and y12>=y22:
                    mask[:, :, y21:y22, x11:x22] = r1*inputs[rand_index, :, y21:y22, x11:x22] + r2*inputs[permu[k], :, y21:y22, x11:x22]
                    tot_area-=get_area(y21, y22, x11, x22)
                    lams[k]-=r1*get_area(y21, y22, x11, x22)
                    lams[i]-=r2*get_area(y21, y22, x11, x22)
                elif x11<x21 and y11>=y21 and y12<=y22:
                    mask[:, :, y11:y12, x21:x12] = r1*inputs[rand_index, :, y11:y12, x21:x12] + r2*inputs[permu[k], :, y11:y12, x21:x12]
                    tot_area-=get_area(y11, y12, x21, x12)
                    lams[k]-=r1*get_area(y11, y12, x21, x12)
                    lams[i]-=r2*get_area(y11, y12, x21, x12)
                elif x11<x21 and y11<=y21 and y12>=y22:
                    mask[:, :, y21:y22, x21:x12] = r1*inputs[rand_index, :, y21:y22, x21:x12] + r2*inputs[permu[k], :, y21:y22, x21:x12]
                    tot_area-=get_area(y21, y22, x21, x12)
                    lams[k]-=r1*get_area(y21, y22, x21, x12)
                    lams[i]-=r2*get_area(y21, y22, x21, x12)
                elif y12>y22 and x11>=x21 and x12 <= x22:
                    mask[:, :, y11:y22, x11:x12] = r1*inputs[rand_index, :, y11:y22, x11:x12] + r2*inputs[permu[k], :, y11:y22, x11:x12]
                    tot_area-=get_area(y11, y22, x11, x12)
                    lams[k]-=r1*get_area(y11, y22, x11, x12)
                    lams[i]-=r2*get_area(y11, y22, x11, x12)
                elif y12>y22 and x11<=x21 and x12 >= x22:
                    mask[:, :, y11:y22, x21:x22] = r1*inputs[rand_index, :, y11:y22, x21:x22] + r2*inputs[permu[k], :, y11:y22, x21:x22]
                    tot_area-=get_area(y11, y22, x21, x22)
                    lams[k]-=r1*get_area(y11, y22, x21, x22)
                    lams[i]-=r2*get_area(y11, y22, x21, x22)
                elif y12>y22 and x11<x21 and x12 > x21:
                    mask[:, :, y11:y22, x21:x12] = r1*inputs[rand_index, :, y11:y22, x21:x12] + r2*inputs[permu[k], :, y11:y22, x21:x12]
                    tot_area-=get_area(y11, y22, x21, x12)
                    lams[k]-=r1*get_area(y11, y22, x21, x12)
                    lams[i]-=r2*get_area(y11, y22, x21, x12)
                elif y12>y22 and x11>x21 and x12 > x22:
                    mask[:, :, y11:y22, x11:x22] = r1*inputs[rand_index, :, y11:y22, x11:x22] + r2*inputs[permu[k], :, y11:y22, x11:x22]
                    tot_area-=get_area(y11, y22, x11, x22)
                    lams[k]-=r1*get_area(y11, y22, x11, x22)
                    lams[i]-=r2*get_area(y11, y22, x11, x22)
                elif y21>y11 and x11>=x21 and x12 <= x22:
                    mask[:, :, y21:y12, x11:x12] = r1*inputs[rand_index, :, y21:y12, x11:x12] + r2*inputs[permu[k], :, y21:y12, x11:x12]
                    tot_area-=get_area(y21, y12, x11, x12)
                    lams[k]-=r1*get_area(y21, y12, x11, x12)
                    lams[i]-=r2*get_area(y21, y12, x11, x12)
                elif y21>y11 and x11<=x21 and x12 >= x22:
                    mask[:, :, y21:y12, x21:x22] = r1*inputs[rand_index, :,y21:y12, x21:x22] + r2*inputs[permu[k], :, y21:y12, x21:x22]
                    tot_area-=get_area(y21, y12, x21, x22)
                    lams[k]-=r1*get_area(y21, y12, x21, x22)
                    lams[i]-=r2*get_area(y21, y12, x21, x22)
                elif y21>y11 and x11<x21 and x12 > x21 and x12 < x22:
                    mask[:, :, y21:y12, x21:x12] = r1*inputs[rand_index, :,y21:y12, x21:x12] + r2*inputs[permu[k], :, y21:y12, x21:x12]
                    tot_area-=get_area(y21, y12, x21, x12)
                    lams[k]-=r1*get_area(y21, y12, x21, x12)
                    lams[i]-=r2*get_area(y21, y12, x21, x12)
                elif y21>y11 and x11>x21 and x12 > x22:
                    mask[:, :, y21:y12, x11:x22] = r1*inputs[rand_index, :,y21:y12, x11:x22] + r2*inputs[permu[k], :, y21:y12, x11:x22]
                    tot_area-=get_area(y21, y12, x11, x22)
                    lams[k]-=r1*get_area(y21, y12, x11, x22)
                    lams[i]-=r2*get_area(y21, y12, x11, x22)
                else:
                    continue

    lams=lams/sum(lams)
    return mask,targets,lams

def randmix_criterion(criterion, pred, targets, lams):
    loss =0
    for i in range(len(targets)):
        loss+=lams[i]*criterion(pred, targets[i])

    return loss

def rand_bbox(size, lam):
    W = size[2]
    H = size[3]
    cut_rat = np.sqrt(1. - lam)
    cut_w = np.int(W * cut_rat)
    cut_h = np.int(H * cut_rat)

    # uniform
    cx = np.random.randint(W)
    cy = np.random.randint(H)

    bbx1 = np.clip(cx - cut_w // 2, 0, W)
    bby1 = np.clip(cy - cut_h // 2, 0, H)
    bbx2 = np.clip(cx + cut_w // 2, 0, W)
    bby2 = np.clip(cy + cut_h // 2, 0, H)

    return bbx1, bby1, bbx2, bby2

def get_current_lr(optimizer):
    for param_group in optimizer.param_groups:
        return param_group['lr']

def adjust_learning_rate(optimizer, epoch, config):
    lr = get_current_lr(optimizer)
    if config.lr_scheduler.type == 'STEP':
        if epoch in config.lr_scheduler.lr_epochs:
            lr *= config.lr_scheduler.lr_mults
    elif config.lr_scheduler.type == 'COSINE':
        ratio = epoch / config.epochs
        lr = config.lr_scheduler.min_lr + \
            (config.lr_scheduler.base_lr - config.lr_scheduler.min_lr) * \
            (1.0 + math.cos(math.pi * ratio)) / 2.0
    elif config.lr_scheduler.type == 'HTD':
        ratio = epoch / config.epochs
        lr = config.lr_scheduler.min_lr + \
            (config.lr_scheduler.base_lr - config.lr_scheduler.min_lr) * \
            (1.0 - math.tanh(
                config.lr_scheduler.lower_bound
                + (config.lr_scheduler.upper_bound
                   - config.lr_scheduler.lower_bound)
                * ratio)
             ) / 2.0
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr
    return lr

max_stars_repo_name: zcablii/data-augment-experiments

max_issues_repo_name: zcablii/data-augment-experiments

max_forks_repo_name: zcablii/data-augment-experiments


text:
# import libraries
import sys
import pandas as pd
import numpy as np
import nltk
import re
import pickle
from sqlalchemy import create_engine
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score,precision_score,recall_score
from sklearn.multioutput import MultiOutputClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import LinearSVC
import warnings
from sklearn.exceptions import DataConversionWarning

nltk.download(['punkt', 'wordnet'])

warnings.filterwarnings(action='ignore', category=DataConversionWarning)
warnings.filterwarnings('ignore')

def load_data(database_filepath):
    """
    Function Description:
        Load data from csv.

    Input:
        database_filepath: Filepath cleaned database file

    Output:
        List (X): List of massages
        List (Y): List of one-hot encoded categories
        List (cat_names): List of catagory names
    """
    engine = create_engine('sqlite:///'+database_filepath)
    df = pd.read_sql_table('ClassifiedMessages', engine)
    X = df.message.values
    Y = df[df.columns[4:]].values
    cat_names=df.columns[4:]

    return X, Y, cat_names

def tokenize(text):
    """
    Function Description:
        Tokanizer for text

    Input:
        text: Text messages

    Output:
        Tokenized words (List)
    """
    #replacing urls
    urls = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', text)
    for i in urls:
        text = text.replace(i, "urlplaceholder")

    # Normalize text
    text = re.sub(r"[^a-zA-Z0-9]", " ", text.lower())

    #Tokenize words
    words = word_tokenize(text)

    #Lemmatize words
    lemmed = [WordNetLemmatizer().lemmatize(w.strip()) for w in words]

    return lemmed

def build_model():
    """
    Function Description:
        Build LinearSVC model using pipeline and then
        use Gridsearch to search for the best paramaters

    Input:
        None

    Output:
        Cross-Validated classifier
    """
    pipeline = Pipeline([
        ('features', FeatureUnion([

            ('text_pipeline', Pipeline([
                ('vect', CountVectorizer(tokenizer=tokenize)),
                ('tfidf', TfidfTransformer())
            ]))
        ])),
        ('clf', MultiOutputClassifier(OneVsRestClassifier(LinearSVC(random_state = 0))))
    ])

    parameters = {
        'features__text_pipeline__vect__max_df': (0.5,0.75, 1.0),
        'features__text_pipeline__tfidf__use_idf': (True, False),
        'clf__estimator__estimator__loss': ['hinge','squared_hinge'],
        'clf__estimator__estimator__multi_class': ['ovr', 'crammer_singer'],
        'clf__estimator__estimator__max_iter': [1000,2000,5000]
    }

    cv = GridSearchCV(pipeline, param_grid=parameters, cv=3, verbose=1, n_jobs=-1)

    return cv

def evaluate_model(model, X_test, Y_test, category_names):
    """
    Function Description:
        Evaulate the model with this function.

    Input:
        model: The build model that needs to be evaluated
        X_test: X_test data
        Y_test: Y_test data
        category_names: List of catagory names

    Prints:
        This function will print the results of the evaluation when called

    Output:
        None
    """
    X_pred = model.predict(X_test)

    eval_dict={'column':[],'f1_score':[],'precision':[],'recall':[]}

    for column in range(len(category_names)):
        f1s=f1_score(Y_test[:,column],X_pred[:,column], average='micro')
        prs=precision_score(Y_test[:,column],X_pred[:,column], average='micro')
        res=recall_score(Y_test[:,column],X_pred[:,column], average='micro')
        eval_dict['column'].append(category_names[column])
        eval_dict['f1_score'].append(f1s)
        eval_dict['precision'].append(prs)
        eval_dict['recall'].append(res)

    print(pd.DataFrame.from_dict(eval_dict))

def save_model(model, model_filepath):
    """
    Function Description:
        Save the model to use for classification at a later stage

    Input:
        model: The build model with best found parameters
        model_filepath: Filepath of where classifier needs to be saved

    Output:
        None
    """
    pickle.dump(model, open(model_filepath, "wb"))

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train)

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()

max_stars_repo_name: RikusFourie/Disaster-Response-Pipeline

max_issues_repo_name: RikusFourie/Disaster-Response-Pipeline

max_forks_repo_name: RikusFourie/Disaster-Response-Pipeline


text:
import os
import pickle
import numpy as np
import copy
import random

from naslib.search_spaces.core.query_metrics import Metric
from naslib.search_spaces.core.graph import Graph
from naslib.utils.utils import get_project_root

class NasBenchNLPSearchSpace(Graph):
    """
    Contains the interface to the tabular benchmark of nas-bench-nlp.
    Note: currently we do not support building a naslib object for
    nas-bench-nlp architectures.
    """

    QUERYABLE = False

    def __init__(self):
        super().__init__()

    def load_labeled_architecture(self, dataset_api=None, max_nodes=25):
        """
        This is meant to be called by a new NasBenchNLPSearchSpace() object.
        It samples a random architecture from the nas-bench-nlp data.
        """
        while True:
            index = np.random.choice(len(dataset_api['nlp_arches']))
            compact = dataset_api['nlp_arches'][index]
            if len(compact[1]) <= max_nodes:
                break
        self.load_labeled = True
        self.set_compact(compact)

    def query(self, metric=None, dataset=None, path=None, epoch=-1, full_lc=False, dataset_api=None):
        """
        Query results from nas-bench-nlp
        """
        metric_to_nlp = {
            Metric.TRAIN_ACCURACY: 'train_losses',
            Metric.VAL_ACCURACY: 'val_losses',
            Metric.TEST_ACCURACY: 'test_losses',
            Metric.TRAIN_TIME: 'wall_times',
            Metric.TRAIN_LOSS: 'train_losses',
        }

        assert self.load_labeled
        """
        If we loaded the architecture from the nas-bench-nlp data (using
        load_labeled_architecture()), then self.compact will contain the architecture spec.
        """
        try:
            assert metric in [Metric.TRAIN_ACCURACY, Metric.TRAIN_LOSS, Metric.VAL_ACCURACY, \
                          Metric.TEST_ACCURACY, Metric.TRAIN_TIME]
        except:
            print('hold')
        query_results = dataset_api['nlp_data'][self.compact]

        sign = 1
        if metric == Metric.TRAIN_LOSS:
            sign = -1

        if metric == Metric.TRAIN_TIME:
            return query_results[metric_to_nlp[metric]]
        elif metric == Metric.HP:
            # todo: compute flops/params/latency for each arch. These are placeholders
            return {'flops': 15, 'params': 0.1, 'latency': 0.01}
        elif full_lc and epoch == -1:
            return [sign * (100 - loss) for loss in query_results[metric_to_nlp[metric]]]
        elif full_lc and epoch != -1:
            return [sign * (100 - loss) for loss in query_results[metric_to_nlp[metric]][:epoch]]
        else:
            # return the value of the metric only at the specified epoch
            return sign * (100 - query_results[metric_to_nlp[metric]][epoch])

    def get_compact(self):
        assert self.compact is not None
        return self.compact

    def get_hash(self):
        return self.get_compact()

    def set_compact(self, compact):
        self.compact = compact

    def sample_random_architecture(self, dataset_api):
        raise NotImplementedError()

    def get_type(self):
        return 'nlp'