'''
Author: Harry Addison
Date: 11/07/2022

License: MIT License

Overview:
        Functions used to perform a search for the next Galactic
        luminous red nova.
        Full method is explained in the paper "Searching for the Next
        Galactic Luminous Red Nova, Harry Addison et. al"
       (https://ui.adsabs.harvard.edu/abs/2022arXiv220607070A/abstract).
'''

import numpy as np
import math
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import multiprocessing

from astroML.plotting.tools import draw_ellipse
from astropy.io import fits
from astropy.table import Table
from astroquery.gaia import Gaia
from extinction import fitzpatrick99
from datetime import datetime
from ztfquery.lightcurve import LCQuery

def ap_abs_mag(ap_mag, plx=0, dist=0, type="plx"):
    '''
    Calculate the absolute magnitude from apparent magnitudes and
    distance/parallax.

    > If you want to use parallax:
        plx = array of parallax data
        type="plx"

        Note: parallaxes must be given in milli-arcseconds (mas).

    > If you want to use distance:
        dist = array of distance data
        type="dist"

        Note: distances must be given in parsecs (pc).
    '''

    if type == "plx":
        abs_mag = ap_mag + 5 * np.log10(plx) - 10

    if type == "dist":
        abs_mag = ap_mag - 5 * np.log10(dist/10)

    return abs_mag

def a_g_to_a_v(a_g):
    '''
    Estimate the V-band extinction correlating to the Gaia G-band
    extinction.

    Input G-band extinction must be in units of mag.
    '''

    # Array of V-band extinctions to convert and compare to the true G-band extinctions.
    a_v_trial = np.linspace(0.00, 10, 1000, endpoint=True)

    # Effective wavelength of the Gaia DR2 G-band.
    g_wave = np.array([6230.00]) #Angstrom

    # Converting the V-band extinctions to G-band.
    a_g_result = []

    for a_v in a_v_trial:
        a_g_result.append(fitzpatrick99(g_wave, a_v, 3.1, unit="aa"))

    # Comparing the the converted V-band extinctions to the true G-band extinctions.
    abs_diff = abs(np.array(a_g_result) - a_g)

    # Selecting the closest converted V-band extinction
    best_ind = np.argmin(abs_diff)
    a_v_best = a_v_trial[best_ind]

    return a_v_best

def redden_correct(data, num_proc):
    '''
    Correcting the Gaia G-RP colour for reddening effects.

    First calculates the reddening based on the G-band extinction.
    Then it corrects for the calculated reddening.

    The input data ("data") is a table consisting of the columns "a_g"
    and "g_rp".

    num_proc = number of processors to use for multiprocessing.
    '''

    p = multiprocessing.Pool(processes = num_proc)

    # Converting the G-band extinction to V-band extinctions
    a_v = p.starmap(a_g_to_a_v, [([data["a_g"][i]]) for i in range(len(data))])

    p.close()
    p.join()

    # Effective wavelength of the Gaia RP-band.
    rp_wave = np.array([7730.00])

    # Converting the V-band extinction to the RP-band.
    a_rp = []
    for i in a_v:
        a_rp.append(fitzpatrick99(rp_wave, i, 3.1, unit="aa")[0])

    # Calculating the reddening.
    a_g_rp = data["a_g"] - a_rp

    # Correcting the colour for the reddening.
    data["g_rp"] = data["g_rp"] - a_g_rp

    return data

def ms_def(mist_data_dir):
    '''
    Defining the oldest limit of the main sequence based on the MIST
    stellar evolution tracks.

    mist_data_dir = data directory containing the data of the MIST
    stellar evolution tracks.
    '''

    # Masses of the stars to be used.
    mass = ["001", "002", "003", "004", "005", "006", "007", "008", "009",
            "010", "011", "012", "013", "014", "015", "016", "017", "018",
            "019", "020", "021", "022", "023", "024", "025", "026"]

    xs = []
    ys = []

    # Loading in the track data and finding the last data point of the main sequence phase.
    for i in range(len(mass)):

        data_path = ("%s%s0000M.track.eep.cmd" %(mist_data_dir, mass[i]))

        eeps_data = Table.read(data_path, format="ascii", header_start=14)

        x = []
        y = []

        for j in range(len(eeps_data)):
            if eeps_data["phase"][j] == 0:
                x.append(eeps_data['Gaia_G_DR2Rev'][j] -
                            eeps_data['Gaia_RP_DR2Rev'][j])

                y.append(eeps_data['Gaia_G_DR2Rev'][j])

        xs.append(x[-1])
        ys.append(y[-1])

    # Fitting a 3rd order polynomial to the last data points of the Main sequence tracks.
    coeffs = np.polyfit(xs, ys, 3)

    return xs, ys, coeffs

def cand_select(data, model, thres_log_likeli, mist_data_dir):
    '''
    Selecting progenitor candidates

    Inputs:
    > data: Table consisting of the columns: "g_rp", "g_mag"
    > model: The produced Gaussian mixture model
    > thres_log_likeli: Threshold log likelihood values
    > mist_data_dir: directory containing the data for the MIST stellar
                     evolution tracks.
    '''

    #Formattigng the data required for "scoring" the data.
    data_format = np.array([data["g_rp"], data["g_mag"]]).transpose()

    # "Scoring" the full data set compared to the model.
    data["log_likeli"] = model.score_samples(data_format)

    # defining the MS edge using the MIST stellar evolution tracks
    xs, ys, coeffs = ms_def(mist_data_dir)

    # Selecting candidates based on thier score, and removing sources below MS and RGB.
    ms_flag = np.ones(len(data)) # 1 = right of MS, 0 = left of MS

    for i in range(len(data)):

        if data["g_rp"][i] < xs[0] and data["g_rp"][i] > xs[-1]:
            y_predict = (coeffs[0] * data["g_rp"][i]**3 +
                         coeffs[1] * data["g_rp"][i]**2 +
                         coeffs[2] * data["g_rp"][i] +
                         coeffs[3])

            if data["g_mag"][i] >= y_predict:
                ms_flag[i] = 0

    cand_data = Table(names=data.colnames)

    for i in range(len(data)):
        if (data["log_likeli"][i] <= thres_log_likeli
            and data["g_mag"][i] <= 1.25
            and not (data["g_rp"][i] >= 0.6 and data["g_mag"][i] >= -2)
            and data["g_rp"][i] <= 1 and not (data["g_rp"][i] <= xs[-1])
            and ms_flag[i] == 1):

            cand_data.add_row(data[i])

    return data, cand_data

def cone_search(data):
    '''
    Conduct a cone search of the Gaia on the given coordinates.

    data is a table containing the columns: "ra", "dec".
    '''

    complete_state = False

    while complete_state == False:
        try:
            # cone serach radius=2 arecseconds around the source at ra and dec.
            query = ('''
                     SELECT count(*)
                     FROM gaiaedr3.gaia_source
                     WHERE
                     CONTAINS(POINT('ICRS',gaiaedr3.gaia_source.ra,
                                    gaiaedr3.gaia_source.dec),
                     CIRCLE('ICRS', %s, %s, 0.0005555555555555556))=1
                     ''' %(data["ra"], data["dec"]))

            job = Gaia.launch_job_async(query, "source check", dump_to_file=False)
            results = job.get_results()

            num_sources = float(results["count_all"])

            complete_state = True

        except:
            print('''ERROR: Gaia query was not successful.
                            Will retry the query.''')

    return num_sources

def neighbour_limitation(data, num_proc):
    '''
    Check to see if a progenitor source has any neighbours within 2
    arcseconds. If so it is removed from the progenitor sample.
    This is nessassary check due to a limitation of ZTF where the survey
    cannot distinguish between close neighbours.

    data is a table of the progenitors' data.
    num_proc is the number of processors to be used for multiprocessing.
    '''

    p = multiprocessing.Pool(processes = num_proc)

    # Cone search on each progenitor retuning the number of sources within 2 arcseconds.
    num_sources = p.starmap(cone_search,
                            [([data[i]]) for i in range(len(data))])

    p.close()
    p.join()

    # Removing progenitors that have another source within 2 arcseconds.
    cand_data = Table(names=data.colnames)

    for i in range(len(data)):
        if num_sources[i] == 1:
            cand_data.add_row(data[i])

    return cand_data

def precursor_selection(progen_data, bandname, error_file_path, data_save_dir,
                        window_width_percent, outlier_thres, mag_change_rate,
                        num_points_thres, time_span_thres):
    '''
    Used to select the precursors from the progenitor sample.

    progen_data = table of the progenitor data
    bandname = ZTF filter band to search for ("g", "i", "r").
    error_file_path = file to where lightcurve query errors are written.
    data_save_dir = directory where lightcurve data is saved.
    window_width_percent = percentage (decimal) of the data used as the
                           number of data points per bin. Used for sigma
                           clipping and sum of differences calculation.
    outlier_thres = percentage (decimal) of times a data point needs to
                    be flagged as an outlier to be removed from the data
    mag_change_rate = minimum brightening rate for classifing precursors
    num_points_thres = minimum number of data points in a lightcurve
                       for the lightcurve to not be discarded.
    time_span_thres = minimum time span of the lightcure for it to not
                      be discarded.

    Outputs:
        > lc_stage_flag: Flag to identify at which point the lightcurve
                         was rejected if it was.
                         Values:
                            > "Precursor Candidate=True/False: False reasoning"

                            > "False: No data"
                               No lightcurve available.

                            > "False: Bad data"
                               Available data was "bad".

                            > "False: Too little data"
                               Lightcurve did not meet time span and/
                               or number of data point constraints.

                            > "False: Insufficient brightening"
                               Lightcurve did not meet brightening
                               requirements.

                            > "True"
                               Lightcurve is of a possible precursor.
    '''

    # Obtaining lc data from ZTF.
    lc_data = lc_query(progen_data["ra"], progen_data["dec"],
                       bandname, error_file_path)

    lc_clean_data, lc_stage_flag = lc_clean(lc_data, window_width_percent,
                                            outlier_thres,
                                            num_points_thres, time_span_thres)

    # Saving the clean lightcurve data to file and running precursor selection
    # (only if there is data).
    if len(lc_clean_data) != 0:
        lc_clean_data.write("%sZTF-LC-Data/%s_band_%d.fits"
                            %(data_save_dir, bandname,
                              progen_data["source_id"]),
                              format="fits", overwrite=True)

        # Analysis the lightcurve using the sum of differences.
        (lc_stage_flag,
         sum_diff, grad_sum_diff) = lc_analysis(lc_data, window_width_percent,
                                                mag_change_rate)

    else:
        sum_diff = None
        grad_sum_diff = None

    return lc_stage_flag, sum_diff, grad_sum_diff

def lc_query(ra, dec, bandname, error_file_path):
    '''
    Cone search obtaining the lightcurve data of the star at the given
    position (ra, dec).

    ra = right accension of star
    dec = declination of the star
    bandname = lightcurve filter band to be obtained ("g", "i", "r").
    error_file_path = file to write errors to.

    Returns a table containing the lightcurve data.
    '''

    query_status = "Incomplete"

    while query_status == "Incomplete":

        try:
            # Querying ZTF for sources within radius = 2 arcsecs of ra, dec.
            lc = LCQuery.from_position(ra, dec, 2, pos="circle",
                                       bandname=bandname)

            # Check to make sure data is returned as connection can be dropped.
            # Code will fail to run if the data hasn't been loaded.
            data_len = len(lc.data)

            query_status = "Complete"

        except:
            # If data is not returned then write error to a file and try again.
            current_time = datetime.now().strftime("%d/%m/%Y %H:%M:%S")

            with open(error_file_path, "a") as f:

                f.write("[%s] Source at RA=%s Dec=%s: "
                        "Possible loss of connection to ZTF servers\n"
                        %(current_time, ra, dec))

            query_status = "Incomplete"

    # Converting data from pandas dataframe to astropy table
    lc_data = Table.from_pandas(lc.data)

    return lc_data

def lc_clean(lc_data, window_width_percent, outlier_thres,
             num_points_thres=25, time_span_thres=365):
    '''
    Cleaning the lightcurve by applying quality constraints to the
    individual data points and the entire lightcurve.

    lc_data = table containing the lightcurve data.
    window_width_percent = percentage (decimal) of the data used as the
                           number of data points per bin. Used for sigma
                           clipping and sum of differences calculation.
    outlier_thres = percentage (decimal) of times a data point needs to
                    be flagged as an outlier to be removed from the data
    num_points_thres = minimum number of data points in a lightcurve
                       for the lightcurve to not be discarded.
    time_span_thres = minimum time span of the lightcure for it to not
                      be discarded.

    Returns a table of the cleaned lightcurve.
    '''
    # Check to see if the lightcurve has any data, if it does then continue cleaning.
    if len(lc_data) == 0:
        lc_stage_flag = "False: No data"

    else:
        # Applying quality constraints to lightcurve data points.
        temp_data = Table(names=lc_data.colnames, dtype=lc_data.dtype)

        for i in range(len(lc_data)):

            # Removing data marked by ZTF as bad observations (catflag!=0 are bad)
            # and data with mag greater than the limiting mag
            if (lc_data["catflags"][i]
                and lc_data["mag"][i] < lc_data["limitmag"][i]):

                temp_data.add_row(lc_data[i])

        lc_data = temp_data

        # Check to see if the lightcurve has any data, if it does then continue cleaning.
        if len(lc_data) == 0:
            lc_stage_flag = "False: Bad data"

        else:
            # Removing possible outliers using sigma clipping
            lc_data, lc_outlier_data = remove_outliers(lc_data,
                                                       window_width_percent,
                                                       outlier_thres)

            # Check to see if the lightcurve has data, if it does then continue cleaning.
            if len(lc_data) == 0:
                lc_stage_flag = "False: Too little data"

            else:
                # Applying constriants on the remaining lightcurve data as a whole.
                # Lightcurves must contain >= 25 data points and span over 1 year.
                lc_time_span = max(lc_data["mjd"]) - min(lc_data["mjd"])#days

                if (len(lc_data) < num_points_thres
                    and lc_time_span < time_span_thres):
                    # Failed constraints, setting lc_data to an empty table
                    lc_data = Table(names=lc_data.colnames,dtype=lc_data.dtype)

                # Making sure that the data is in chronological order.
                lc_data.sort("mjd")

                # Check to see if the lightcurve has any data.
                if len(lc_data) == 0:
                    lc_stage_flag = "False: Too little data"
                else:
                    # If still data don't give flag value as sum of diff to be done next.
                    lc_stage_flag = None

    return lc_data, lc_stage_flag

def remove_outliers(data, window_width_percent, outlier_thres):
    '''
    Remove outliers from a lightcurve using sigma cipping.

    data = table of lightcurve data.
    window_width_percent = percentage (decimal) of the data used as the
                           number of data points per bin. Used for sigma
                           clipping and sum of differences calculation.
    outlier_thres = percentage (decimal) of times a data point needs to
                    be flagged as an outlier to be removed from the data

    Returns tables of the non-outlier data and the outlier data.
    '''

    n = len(data)

    # bin size for the rolling window.
    bin_size = math.ceil(n * window_width_percent)

    outlier_flags = [ [] for _ in range(n) ]

    # going through each rolling window and flagging potential outliers.
    for i in range(n - bin_size + 1):
        # data in the window.
        win_data = data[i : (i + bin_size)]

        win_mag_mean = np.mean(win_data["mag"])

        win_std_dev = np.sqrt((1/len(win_data["mag"])) *
                               sum((win_data["mag"] - win_mag_mean)**2))

        for j in range(bin_size):
            # flagging data that is outside of 3 standard deviations of the mean.
            if (win_data["mag"][j] < (win_mag_mean + 3 * win_std_dev)
                and win_data["mag"][j] > (win_mag_mean - 3 * win_std_dev)):

                outlier_flags[i+j].append(0)

            else:
                outlier_flags[i+j].append(1)

    outlier_data = Table(names=data.colnames, dtype= data.dtype)
    non_outlier_data = Table(names=data.colnames, dtype= data.dtype)

    # Removing data that has been flagged more than the outlier_thres.
    for i in range(n):

        if np.mean(outlier_flags[i]) < outlier_thres:
            non_outlier_data.add_row(data[i])

        else:
            outlier_data.add_row(data[i])

    return non_outlier_data, outlier_data

def lc_analysis(lc_data, window_width_percent, mag_change_rate):
    '''
    Applying sum of differences method to the lightcurve to determine
    if the rate of brightening is precursor like.

    lc_data = table of the lightcurve data.
    window_width_percent = percentage (decimal) of the data used as the
                           number of data points per bin. Used for sigma
                           clipping and sum of differences calculation.
    mag_change_rate = minimum brightening rate for classifing precursors
    '''
    # Calculatinf the rolling mean
    n = len(lc_data)
    bin_size = math.ceil(n*window_width_percent)

    mjd_means = np.array([np.mean((lc_data["mjd"][i : (i + bin_size)]))
                          for i in range(n - bin_size + 1)])

    mag_means = np.array([np.mean((lc_data["mag"][i : (i + bin_size)]))
                          for i in range(n - bin_size + 1)])

    # Calculating sum (and its gradient) of differences between means
    sum_diff = sum(mag_means[1:] - mag_means[:-1])
    grad_sum_diff = (sum_diff / (max(mjd_means)- min(mjd_means)))

    # Applying the rate of change requirement to the grad_sum_diff
    if grad_sum_diff < mag_change_rate:
        lc_stage_flag = "True"

    else:
        lc_stage_flag = "False: Insufficient brightening"

    return lc_stage_flag, sum_diff, grad_sum_diff

def plot_evo_tracks(ax, dir, linestyles=[], colors=[]):
    '''
    Plotting the MIST stellar evolution tracks.

    ax = axis to be plotted to.
    dir = directory containing the MIST stellar evolution track data.
    linestyles = line_styles to be used for plotting. Leave blank to use
                 the defaults.
    colors = colours to be used for plotting. LEave blank to use the
             defaults.
    '''

    # Range of masses to be plotted.
    mass = ["002", "003", "005", "007", "010"]
    # Labels for each track.
    mass_label = [r"$2 \,\rm{M_{\odot}}$", r"$3 \,\rm{M_{\odot}}$",
                  r"$5 \,\rm{M_{\odot}}$", r"$7 \,\rm{M_{\odot}}$",
                  r"$10 \,\rm{M_{\odot}}$"]
    # Default linestyles
    if linestyles == []:
        linestyles = ["solid", "dashed", "dashdot", "dotted", (0, (5, 1))]
    # Default colours
    if colors == []:
        colors = ["purple", "orange", "black", "darkgreen", "red"]

    # Loading in the tracks and plotting the MS to RGB phases.
    for i in range(len(mass)):
        # Laoding the data.
        data_path = ("%s%s0000M.track.eep.cmd" %(dir, mass[i]))

        eeps_data = Table.read(data_path, format="ascii", header_start=14)

        # Selecting the MS to RGB phase
        x = []
        y = []
        reduced_phases = []

        for j in range(len(eeps_data)):
            if eeps_data["phase"][j] >= 0 and eeps_data["phase"][j] <= 3:
                x.append(eeps_data['Gaia_G_DR2Rev'][j] - eeps_data['Gaia_RP_DR2Rev'][j])
                y.append(eeps_data['Gaia_G_DR2Rev'][j])
                reduced_phases.append(eeps_data["phase"][j])

        # Plotting the track
        ax.plot(x, y, linestyle=linestyles[i], label=mass_label[i], c=colors[i])

        # Highlighting MS
        xs = []
        ys = []

        for k in range(len(reduced_phases)):
            if reduced_phases[k] == 0:
                xs.append(x[k])
                ys.append(y[k])

        if len(xs) > 0:
            if mass[i] == "010":
                ax.plot(xs, ys, color="Red", linewidth=4.0, alpha=0.5, label="Main Sequence")
            else:
                ax.plot(xs, ys, color="Red", linewidth=4.0, alpha=0.5)

    # Shrink current axis's height by 12% on the bottom
    box = ax.get_position()
    ax.set_position([box.x0, box.y0 + box.height * 0.12,
                        box.width, box.height * 0.85])

    # Put a legend below current axis
    ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.12), ncol=3)

    return

def plot_cmd_likeli(data, model, thres_log_likeli):
    '''
    Plotting the CMD of the fitting data set with the log likelihood
    values of belonging to the model.

    data = fitting data set after fitting to the model.
    model = Gaussian mixture model.
    thres_log_likeli = threshold log likelihood value.
    '''

    # Creating fake data to plot contours on HRD.
    x = np.linspace(min(data["g_rp"]), max(data["g_rp"]), 100, endpoint=True)
    y = np.linspace(min(data["g_mag"]), max(data["g_mag"]), 100, endpoint=True)

    z = np.zeros((100,100))

    for i in range(100):
        fake_data = []
        for j in range(100):
            fake_data.append([x[i], y[j]])

        fake_data = np.array(fake_data)

        # fiiting the fake data to the model to produce log likelihood contours
        z[i] = model.score_samples(fake_data)

    z = np.transpose(z)

    # CMD of data set with scores.
    fig = plt.figure(figsize=(6, 5))

    plt.scatter(data["g_rp"], data["g_mag"], c=data["log_likeli"],
                cmap="hot", s=2, lw=2, marker="o")

    plt.colorbar(label="Log Likelihood")

    contours = plt.contour(x,y,z,
                          levels=np.array([-100.0,-50.0,-30.0,
                                           -20.0,-10.0, -5.0,0]),
                          colors="black", linestyles="solid")
    plt.clabel(contours, inline=True, fmt="%1.0f")

    thres_contours = plt.contour(x,y,z, levels=np.array([thres_log_likeli]),
                                 colors="b",linestyles="solid")
    plt.clabel(thres_contours, inline=True, fmt="%1.2f")

    plt.xlabel("G-RP (mag)")

    plt.ylabel("Absolute G Magnitude (mag)")
    plt.ylim(top=1.5)
    plt.gca().invert_yaxis()

    plt.title("Candidate Threshold = %s" %(thres_log_likeli))

    return fig

def plot_cmd_train(data, mist_data_dir):
    '''
    Plotting CMD of the training data with density overplotted.
    Also plotting the MIST stellar evolution tracks.

    data = training data set after corrections/processing.
    mist_data_dir = directory containing MIST data.
    '''

    # Histogram to find the density of the data.
    hist, xedges, yedges = np.histogram2d(data["g_rp"], data["g_mag"],
                                          bins=300)

    xidx = np.clip(np.digitize(data["g_rp"], xedges), 0, hist.shape[0]-1)
    yidx = np.clip(np.digitize(data["g_mag"], yedges), 0, hist.shape[1]-1)
    col = hist[xidx, yidx]

    # Scatter plot of g_mag mag vs g_rp, with a density map
    fig = plt.figure(figsize=(6, 5))
    ax = fig.add_subplot(111)

    max_col = max((col+10**-200)**(1/3))
    norm_col = (col+10**-200)**(1/3) / max_col

    im = plt.scatter(data["g_rp"], data["g_mag"],
                     c=norm_col, cmap="hot",
                     lw=0.5, s=0.5)

    cbar = plt.colorbar(im, ticks=[0, (250/max_col**3), (500/max_col**3),
                                      (750/max_col**3), (1000/max_col**3),
                                      (1250/max_col**3)],
                                      label="Sources per Bin")
    cbar.ax.set_yticklabels(["0", "%d"%(250),
                          "%d"%(500), "%d"%(750), "%d"%(1000), "%d"%(1250)])

    ax.set_xlabel("G-RP (mag)")
    ax.set_ylabel("Absolute G Magnitude (mag)")
    ax.invert_yaxis()

    plot_evo_tracks(ax, mist_data_dir,
                        linestyles = ["solid", "dashed", "dashdot",
                                       (0, (5, 1)), "dotted"],
                        colors = ["tab:blue","tab:blue","tab:blue",
                                  "tab:blue","tab:blue"])

    return fig, ax

def plot_cmd_cand(data, mist_data_dir):
    '''
    Plotting CMD of the progenitor candidates.

    data = candidate data.
    mist_data_dir = directory of MIST data.
    '''

    # HRD of candidates with log likelihood values overplotted.
    fig = plt.figure(figsize=(6, 6))
    ax = plt.subplot(111)

    data.add_row([0,0,0,0,0,0,0,0,0,-20])

    im = ax.scatter(data["g_rp"], data["g_mag"], c=data["log_likeli"],
                cmap="hot", s=2, lw=2, marker="o")

    plt.colorbar(im, ax=ax, label="Log Likelihood")

    ax.set_xlabel("G-RP (mag)")

    ax.set_ylabel("Absolute G Magnitude (mag)")
    ax.set_ylim(top=1.5)
    ax.invert_yaxis()

    plot_evo_tracks(ax, mist_data_dir)

    return fig, ax

def plot_model(train_data, model, num_comp):
    '''
    Plotting the Gaussian mixture model.

    train_data = training data set.
    model = constructed Gaussian mixture model.
    num_comp = number of components used in the model.
    '''

    # Computting 2d density for plot
    color_bins = 1000
    mag_bins = 1000

    H, color_bins, mag_bins = np.histogram2d(train_data["g_rp"],
                                             train_data["g_mag"],
                                             (color_bins, mag_bins))

    Xgrid = np.array(list(map(np.ravel,
                              np.meshgrid(0.5 * (color_bins[:-1]
                                                 + color_bins[1:]),
                                          0.5 * (mag_bins[:-1]
                                                 + mag_bins[1:]))))).T

    log_dens = model.score_samples(Xgrid).reshape((1000, 1000))

    # Plotting.
    fig = plt.figure(figsize=(6, 6))

    im = plt.imshow((1.2**(log_dens)),
                     origin='lower', interpolation='nearest', aspect='auto',
                     extent=[color_bins[0], color_bins[-1],
                     mag_bins[0], mag_bins[-1]],
                     cmap="hot_r")

    # Change "scales" to 1 or 2. it represents sigma of the gaussian distribution
    plt.scatter(model.means_[:, 0], model.means_[:, 1], c='w')

    for mu, C, w in zip(model.means_, model.covariances_, model.weights_):
        draw_ellipse(mu, C, scales=[1], fc='none', ec='k')

    plt.ylim(mag_bins[0], mag_bins[-1])

    plt.xlabel("G-RP (mag)")
    plt.ylabel("Absolute G Magnitude (mag)")
    plt.gca().invert_yaxis()

    plt.title("%s Component Model" %num_comp)

    plt.colorbar(im, label="Density")

    return fig

def plot_lc(data, bands):
    '''
    Plotting the given lightcurve.

    data = array of lightcurve tables.
    bands = array of the lightcurve filter bands ("g", "i", "r").
    '''

    fig = plt.figure(figsize=(6, 6))
    ax = plt.subplot(111)

    for i in range(len(data)):

        if bands[i] == "g":
            ax.errorbar(data[i]["hjd"], data[i]["mag"], data[i]["magerr"],
                        fmt=" ", marker = "o", markersize=3,
                        c="green", label="g-band")

        if bands[i] == "i":
            ax.errorbar(data[i]["hjd"], data[i]["mag"], data[i]["magerr"],
                        fmt=" ", marker = "o", markersize=3, c="indigo",
                        label="i-band")

        if bands[i] == "r":
            ax.errorbar(data[i]["hjd"], data[i]["mag"], data[i]["magerr"],
                        fmt=" ", marker = "o", markersize=3, c="red",
                        label="r-band")

    ax.legend()

    plt.gca().invert_yaxis()
    plt.xlabel("HJD (days)")
    plt.ylabel("Apparent Magnitude (mag)")

    return fig, ax


# -*- coding: utf-8 -*-
"""
Created on Fri Feb 14 15:33:40 2020

@author: simed
"""

#%%
from __future__ import division
import argparse
import pandas as pd
import re
import os
import pickle as pkl
import spacy
import numpy as np
from stopwords import stop_words

__authors__ = ['Hammouch Ouassim','El Hajji Mohamed','POKALA Sai Deepesh', 'de BROGLIE Philibert']
__emails__  = ['ouassim.hammouch@student.ecp.fr', 'mohamed.el-hajji@student.ecp.fr','saideepesh.pokala@student-cs.fr','philibert.de-broglie@student-cs.fr']

def text2sentences(path):
    """
        extract the sentences in the path and preprocess them
        Parameters
        ----------
        path : Path of sentences, str

        Returns
        -------
        processed_sentences : List
            A list with preprocessed sentences split in tokens
    """
    #spacy_nlp = spacy.load("en_core_web_sm")

    with open(path, encoding="utf8") as f:
        sentences = f.readlines()
    sentences = [sentence.rstrip("\n") for sentence in sentences]

    processed_sentences = []
    for sentence in sentences:
        sentence = re.sub(r'[^a-zA-Z_\s\']+', '', sentence) # remove special characters, numbers
        sentence = [s for s in sentence.lower().replace("'", " ").split() if s not in stop_words]
        processed_sentences.append(sentence) #all lowercase, replace ' with a space and split at each word

    return processed_sentences

def loadPairs(path):
    data = pd.read_csv(path, delimiter='\t')
    pairs = zip(data['word1'],data['word2'],data['similarity'])
    return pairs

def create_w2id_map(sentences):
    """
       Create a word to id mapping
       Parameters
       ----------
       sentences : List
              List of preprocessed sentences
       Returns
       -------
       w2id : Dict
           word to id mapping
    """
    w2id = {}
    id = 0
    for sentence in sentences :
        for token in sentence :
            if token not in w2id.keys():
                w2id[token]=id
                id +=1
    return w2id

class SkipGram:
    def __init__(self, sentences, nEmbed=100, negativeRate=5, winSize = 5, minCount = 5, lr = 0.05, clip_value = 10, display_rate = 100):
        self.w2id = create_w2id_map(sentences) # word to ID mapping
        self.trainset = sentences # set of sentences
        self.vocab = list(set(self.w2id.keys())) # list of valid words
        self.n = nEmbed
        self.n_neg = negativeRate
        self.winSize = winSize
        self.mincount = minCount
        self.C = np.random.uniform(-1, 1, (len(self.vocab), self.n))
        self.W = np.random.uniform(-1, 1, (self.n, len(self.vocab)))
        self.loss = []
        self.accLoss = 0
        self.lr = lr
        self.trainWords = 1
        self.norms = {"W":[], "C":[]}
        self.clip_value = clip_value
        self.display_rate = display_rate

    def sample(self, omit):
        """
        samples negative words, ommitting those in set omit
        :param omit: set of words
        :return: ids of negative samples
        """

        #TODO : add unigram model sampling (**3/4)
        n = len(self.vocab)
        neg_sample_idxs = []
        for i in range(self.n_neg):
            neg_sample_idx = np.random.randint(n)
            while neg_sample_idx in omit :
                neg_sample_idx = np.random.randint(n)
            neg_sample_idxs.append(neg_sample_idx)

        return neg_sample_idxs

    def forward(self, x):
        """
        Performs the forward pass of a neural network, not used
        Parameters
        ----------
        x : np.array
            data to perform the forward pass
        Returns
        -------
        proba : array
            The probabilities after the softmax layer
        embedding : array
            The hidden layer ( word embeddings)
        pred : array
            The output of the network before the sofrtmax layer
        """
        embedding = np.dot(x, self.w1)
        pred = np.dot(embedding, self.w2)
        proba = self.softmax(pred)
        return proba, embedding, pred

    def train(self):
        """
        	    Train the skipgram model
        	    Returns
        	    -------
        	    None.
        """
        for counter, sentence in enumerate(self.trainset):
            self.trainWords += 1
            sentence = list(filter(lambda word: word in self.vocab, sentence))

            for wpos, word in enumerate(sentence):
                wIdx = self.w2id[word]
                winsize = np.random.randint(self.winSize) + 1
                start = max(0, wpos - winsize)
                end = min(wpos + winsize + 1, len(sentence))

                for context_word in sentence[start:end]:
                    ctxtId = self.w2id[context_word]
                    if ctxtId == wIdx: continue
                    negativeIds = self.sample({wIdx, ctxtId})
                    self.trainWord(wIdx, ctxtId, negativeIds)
                    # print(f"Trained ({word},{context_word})")
                    self.trainWords += 1

            # if counter % self.display_rate == 0:
                # print (' > training %d of %d' % (counter, len(self.trainset)))
                # self.norms["W"].append(np.linalg.norm(self.W))
                # self.norms["C"].append(np.linalg.norm(self.C))

                # print(f"Loss : {self.accLoss / self.trainWords}")
                # self.trainWords = 0
                # self.accLoss = 0.

    def get_one_hot_from_string(self, word):
        """
        Get the one hot vector corresponding to word
        Parameters
        ----------
        word : str
            a word of the vocabulary
        Returns
        -------
        one_hot : array
            One hot vector of the word
        """
        one_hot = np.zeros(len(self.vocab))
        word_index = self.w2id[word]
        one_hot[word_index] = 1
        return one_hot

    def trainWord(self, target_id, context_id, neg_ids):
        """
        Train the model on a batch of  words
        Parameters
        ----------
        target_id : int
            id of the target word
        context_id : int
            id of the context word
        neg_ids : int
            ids of the negative words
        Returns
        -------
        None.
        """

        self.loss = 0
        dw = 0

        w_t = self.W[:,target_id]
        c_p = self.C[context_id,:]
        self.loss -= np.log(self.sigmoid(np.dot(w_t,c_p)))
        # Updates
        dcp = np.clip(self.sigmoid(-1*np.dot(w_t,c_p)), -1*self.clip_value, self.clip_value)
        self.C[context_id, :] = self.C[context_id,:] + self.lr*dcp*w_t
        dw += dcp*c_p

        for neg_id in neg_ids:
            c_n = self.C[neg_id, :]
            self.loss -= np.log(self.sigmoid(-1*np.dot(w_t,c_n)))
        # 	 Updates
            dcn = np.clip(self.sigmoid(np.dot(w_t, c_n)), -1*self.clip_value, self.clip_value)
            self.C[neg_id, :] = self.C[neg_id, :] - self.lr * dcn * w_t
            dw -= dcn*c_n

        self.W[:, target_id] += self.lr*np.clip(dw, -1*self.clip_value, self.clip_value)
        self.accLoss += self.loss

    @staticmethod
    def sigmoid(x):
        return 1/(1+np.exp(-x))

    def softmax(self, x):
        interm = np.exp(x-max(x))
        return interm / interm.sum(axis=0)

    def save(self, path):
        """
        Saves the model on the disk as a dictionary
        Parameters
        ----------
        path : str
            The path to the file which will contain the data.
        Returns
        -------
        None.
        """
        with open(path, "wb") as f:
            pkl.dump(self.__dict__, f)
        # print(f"Saved model to {path} successfully")

    def similarity(self,word1,word2):
        """
            computes similiarity between the two words. unknown words are mapped to one common vector
        :param word1:
        :param word2:
        :return: a float \in [0,1] indicating the similarity (the higher the more similar)
        """

        try:
            id_1, id_2 = self.w2id[word1], self.w2id[word2]
            emb1 = self.W[:, id_1]
            emb2 = self.W[:, id_2]
        except KeyError:
            return 0

        return np.dot(emb1, emb2) / (np.linalg.norm(emb1) * np.linalg.norm(emb2))

    def load(self, path):
        """
        Loads the saved dictionnary onto the model
        Parameters
        ----------
        path : str
            The path for the file.
        Returns
        -------
        None.
        """
        with open(path, "rb") as f:
            self.__dict__ = pkl.load(f)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--text', help='path containing training data', required=True)
    parser.add_argument('--model', help='path to store/read model (when training/testing)', required=True)
    parser.add_argument('--test', help='enters test mode', action='store_true')

    opts = parser.parse_args()

    if not opts.test:
        sentences = text2sentences(opts.text)
        sg = SkipGram(sentences, lr=0.0005, display_rate=50)
        sg.train()
        sg.save(opts.model)

    else:
        pairs = loadPairs(opts.text)
        sg = SkipGram([])
        sg.load(opts.model)
        for a, b, _ in pairs:
            print(sg.similarity(a, b))


from django.urls import path
from . import views

app_name = "ecommerce"

urlpatterns = [

    path('items/', views.ItemsView.as_view(), name='items'),
    path('item/<slug:slug>/', views.ItemView.as_view(), name='item'),
	path('items/add-or-remove/', views.add_or_remove_item, name='add_or_remove'),
	path('checkout/', views.CheckoutView.as_view(), name="checkout"),
	path('pay/', views.stripe_payment, name="pay"),
	path('cart/', views.CartView.as_view(), name="cart"),
	path('orders/', views.OrdersView.as_view(), name="orders"),
	path('order/<int:id>/', views.OrderView.as_view(), name="order"),
	path('update-source/', views.update_source, name="update-source"),
	]


from __future__ import annotations

import os

from jax import config
from jax import numpy as jnp

MODES = ["te0", "te1", "te2", "tm0", "tm1"]

config.update("jax_enable_x64", True)

def _read_effective_index(file_name: str) -> tuple[jnp.ndarray, jnp.ndarray, jnp.ndarray]:
    """Reads effective index data from file.

    Args:
        file_name (str): Name of the file.

    Returns:
        tuple[jnp.ndarray, jnp.ndarray, jnp.ndarray]: Effective index data, width data, wavelength data.
    """

    with open(os.path.join(os.path.dirname(__file__), "data", file_name), "r") as _file:
        _lines = _file.readlines()

    _neff_data = jnp.array(list(map(float, _lines[1][:-2].split(","))))
    _width_data = jnp.array(list(map(float, _lines[3][:-2].split(","))))
    _wav_data = jnp.array(list(map(float, _lines[5][:-2].split(","))))

    _wav_size = _wav_data.shape[0]
    _width_size = _width_data.shape[0]

    if any(mode in file_name for mode in MODES):
        _neff_data = _neff_data.reshape((_width_size, _wav_size))
    else:
        _neff_data = _neff_data.reshape((_wav_size, _width_size))
    return _neff_data, _width_data, _wav_data


import numpy as np
from numba import jit

import warnings
warnings.filterwarnings("ignore")

class GPUM:
    def __init__(self, matrix_a: np.ndarray, matrix_b: np.ndarray) -> None:
        self.matrix_a = matrix_a
        self.matrix_b = matrix_b
        self.check_input()
        self.n = matrix_a.shape[0]

    def check_input(self):
        m, n = self.matrix_a.shape
        k, l = self.matrix_b.shape

        if (m != n) or (k != l) or (m != k):
            raise ValueError
        elif not isinstance(self.matrix_a, np.ndarray) or not isinstance(self.matrix_b, np.ndarray):
            raise TypeError

    @jit(target_backend='cuda')
    def compute(self) -> np.ndarray:
        matrix_c = np.zeros_like(self.matrix_a)
        for i in range(self.n):
            matrix_c[i, :] = self.matrix_a[i, :].dot(self.matrix_b)
        return matrix_c


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json

def test_create_build(app):
    with app.test_client() as c:
        # create build
        r = c.post('/api/build/', data={
            'url': 'https://blurrcat@test.com/repo.git'
        })
        r = json.loads(r.data.decode('utf-8'))
        assert r['id']

def test_create_build_errors(app):
    with app.test_client() as c:
        for data in (
            {},  # missing url
            {'url': 'http://github.com/blurrcat/repo.git'},  # only https
        ):
            r = c.post('/api/build/', data=data)
            assert r.status_code == 400
            r = json.loads(r.data.decode('utf-8'))
            assert r['message']['url']

def test_logs(app):
    with app.test_client() as c:
        r = c.post('/api/build/', data={
            'url': 'https://github.com/blurrcat/repo.git',
            'username': 'blurrcat',
        })
        r = json.loads(r.data.decode('utf-8'))
        since = 10
        r = c.get('/api/build/{}/logs/'.format(r['id']), query_string={
            'since': since,
        })
        assert r.status_code == 200
        r = json.loads(r.data.decode('utf-8'))
        assert 'logs' in r
        assert r['since'] == since
        assert r['since'] + len(r['logs']) == r['next']


#!/home/adonis/anaconda3/bin/python3
# -*- coding: utf-8 -*-

from mayavi import mlab
from scipy import sin, ogrid, array
from pylab import plot, show

# prepare data, hence test scipy elements
x , y = ogrid[-3:3:100j , -3:3:100j]
z = sin(x**2 + y**2)

# test matplotlib
# plot(x, sin(x**2)); show()

#now mayavi2
obj = mlab.surf(x,y,z)
P = mlab.pipeline
scalar_cut_plane = P.scalar_cut_plane(obj, plane_orientation='y_axes')
scalar_cut_plane.enable_contours = True
scalar_cut_plane.contour.filled_contours = True
scalar_cut_plane.implicit_plane.widget.origin = array([  0.00000000e+00,   1.46059210e+00,  -2.02655792e-06])

scalar_cut_plane.warp_scalar.filter.normal = array([ 0.,  1.,  0.])
scalar_cut_plane.implicit_plane.widget.normal = array([ 0.,  1.,  0.])
f = mlab.gcf()
f.scene.camera.azimuth(10)

f.scene.show_axes = True
f.scene.magnification = 4 # or 4
mlab.axes()

# Now animate the data.
dt = 0.01; N = 40
ms = obj.mlab_source
for k in range(N):
    x = x + k*dt
    scalars = sin(x**2 + y**2)
    ms.set(x=x, scalars=scalars)


import numpy as np
import tiledb

def check_and_create(filename, dims,blocksize, dtype):
    if tiledb.object_type(filename) != "array":
        create_array(filename, dims,blocksize,dtype)

def create_array(filename, dims,blocksize,dtype):
    dom = tiledb.Domain(
        tiledb.Dim(name="x", domain=(0, dims[0] - 1), tile=blocksize[0], dtype=np.int32),
        tiledb.Dim(name="y", domain=(0, dims[1] - 1), tile=blocksize[1], dtype=np.int32),
        tiledb.Dim(name="z", domain=(0, dims[2] - 1), tile=blocksize[2], dtype=np.int32)
    )

    attr = tiledb.Attr(dtype=dtype)
    # tiledb.Attr(name=attr, dtype=dtype)
    schema = tiledb.ArraySchema(
        domain=dom, sparse=False, attrs=[attr]
    )

    # Create the (empty) array on disk.
    tiledb.Array.create(filename, schema)

def write_tile_block(filename, position, data):
    size = data.shape
    print("shape: " + str(size))
    print("position: " + str(position))
    with tiledb.DenseArray(filename, mode="w") as A:
        A[position[0]:position[0] + size[0], position[1]:position[1] + size[1], position[2]:position[2] +size[2]] = data

def write_array(grid_position, filename, data):
    size = data.shape
    positions = [size[0] * grid_position[0], size[1] * grid_position[1], grid_position[2]]
    # DenseArray
    with tiledb.DenseArray(filename, mode="w") as A:
        A[positions[0]:positions[0] + size[0], positions[1]:positions[1] + size[1],
        positions[2]:positions[2] + 1] = data

def read_array(file, position, block):
    with tiledb.open(file, 'r') as A:
        return A[position[0]:position[0] + block[0], position[1]:position[1] + block[1], position[2]:position[2] + block[2]]


import os
import sys
import time
import csv
import torch.utils.data
from torch import nn
from torch import optim
from torch.utils.data import SubsetRandomSampler
from torchvision import transforms
from models.ssd_resnet import SSDResnet,  MultiBoxLoss
from helpers import AverageMeter
from data.coco import *
from utils import *
from models.augmentations import *

# ------------------------------- Model Params ------------------------------- #
clip_len = 16 # Number of frames to use per clip
dataset_root = "/media/ynki9/DATA/dev2/coco"
weights_root = "./weights"
model_version = 1
sub_version = 2
local_data = None
model_checkpoint_path = None

# ---------------------------------------------------------------------------- #
#                                Learning Params                               #
# ---------------------------------------------------------------------------- #
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
batch_size = 1
start_epoch = 0
end_epoch = 24
worker_count = 2
learning_rate = 1e-4
print_freq = 200
best_loss = 100
load_checkpoint = True

def main():
    global start_epoch
    global end_epoch, best_loss, model_checkpoint_path

    print(f"Initializing training for model version [{model_version}].")
    print(f"Utilizing COCO Dataset with clip length [{clip_len}] frames.")

    transforms = SSDAugmentation()

    # Training Dataset
    print("Loading Training Dataset.")
    train_dataset = COCODetection(root=dataset_root, transform=transforms)
    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True,
                                                num_workers=worker_count)

    # Validation Dataset
    print("Loading Validation Dataset")
    # validation_dataset = COCODetection(root=dataset_root, transform=transforms)
    # val_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False,
    #                                             num_workers=worker_count, sampler=validation_sampler)

    # initialize model
    print("Generating model version {}.".format(model_version))
    model = SSDResnet(n_classes=len(COCO_CLASSES)).to(device)

    # Initialize Criterion and Back Propagation
    criterion =  MultiBoxLoss(model.priors).to(device)
    optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)

    model_checkpoint_path = "{}/checkpoint_ynet3d_v{}-{}-10.pth.tar".format(weights_root, model_version, sub_version)
    if load_checkpoint and os.path.exists(model_checkpoint_path): # Load saved model parameters
        print("Loading checkpoint: {}".format(model_checkpoint_path))
        checkpoint = torch.load(model_checkpoint_path)
        start_epoch = checkpoint['epoch'] + 1
        epochs_since_improvement = checkpoint['epochs_since_improvement']
        best_loss = checkpoint['best_loss']
        print('\nLoaded checkpoint from epoch %d. Best loss so far is %.3f.\n' % (start_epoch, best_loss))
        model = checkpoint['model']
        optimizer = checkpoint['optimizer']

    # Open Metric Writer
    metric_file_path = "{}/metrics-{}-{}.csv".format(weights_root, model_version, sub_version)
    print("Using Metric File: {}".format(metric_file_path))

    # Perform Training
    print("Performing Training: Epoch {}-{}".format(start_epoch, end_epoch))
    for epoch in range(start_epoch, end_epoch):
        print(f"# --------------------------------- EPOCH {epoch} --------------------------------- #")

        # perform training
        train_loss = train(train_loader=train_loader, model=model, epoch=epoch,
                optimizer=optimizer, criterion=criterion, metric_file_path=metric_file_path)

        # Validate every epoch
        # val_loss, accuracy = validate(val_loader=val_loader,
        #                     model=model,
        #                     criterion=val_criterion)

        # Did validation loss improve?
        is_best = train_loss < best_loss
        best_loss = min(train_loss, best_loss)

        if not is_best:
            epochs_since_improvement += 1
            print("\nEpochs since last improvement: %d\n" % (epochs_since_improvement,))

        else:
            epochs_since_improvement = 0

        # Save checkpoint
        save_checkpoint(epoch, epochs_since_improvement, model, optimizer, train_loss, best_loss, is_best, accuracy)

def train(train_loader, model, epoch, optimizer, criterion, metric_file_path=None):
    model.train()
    batch_time = AverageMeter()
    data_time = AverageMeter()

    # Losses
    losses = AverageMeter()
    localization_losses = AverageMeter()
    classification_losses = AverageMeter()

    start = time.time()

    for i,  (img, gt) in enumerate(train_loader):
        data_time.update(time.time() - start)

        img = img.to(device)
        gt = gt.type(torch.FloatTensor)
        bboxes = gt[:, :, 0:4].to(device)
        labels = gt[:, :, 4:].squeeze(2).to(device)

        # Forward prop.
        optimizer.zero_grad()
        pred_locs, pred_scores = model(img)

        # Loss
        cls_loss, loc_loss = criterion(pred_locs, pred_scores, bboxes, labels)
        loss = cls_loss + loc_loss

        # Backward prop.
        loss.backward()

        # Update model
        optimizer.step()

        losses.update(loss.item())
        localization_losses.update(loc_loss.item())
        classification_losses.update(cls_loss.item())

        batch_time.update(time.time() - start)

        start = time.time()

        # Print status
        if i % print_freq == 0:
            # Record Status
            if metric_file_path != None:
                with open(metric_file_path, 'w+') as csv_file:
                    csv_writer = csv.writer(csv_file, delimiter=',')
                    if csv_writer != None:
                        metrics = [epoch, i, len(train_loader), batch_time.avg, losses.val, classification_losses.val,
                                localization_losses.val]
                        csv_writer.writerow(metrics)

            # Print Status
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Batch Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data Time {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Total Loss {loss.val:.4f} ({loss.avg:.4f})\n'
                  'Classification Loss {classification_loss.val:.4f} ({classification_loss.avg:.4f})\t'
                  'Localization Loss {multibox_loss.val:.4f} ({multibox_loss.avg:.4f})\t'.format(epoch, i, len(train_loader),
                                                                  batch_time=batch_time,
                                                                  data_time=data_time, loss=losses,
                                                                  classification_loss=classification_losses,
                                                                  multibox_loss=localization_losses))

        del img, bboxes, labels, gt  # clear memory
    return losses.val

def validate(val_loader, model, criterion):
    model.eval()  # eval mode disables dropout

    batch_time = AverageMeter()
    data_time = AverageMeter()

    losses = AverageMeter()
    multibox_losses = AverageMeter()
    classification_losses = AverageMeter()

    start = time.time()
    correct = 0
    total = 0

    # begin validation
    with torch.no_grad():
        for i, (frames, labels, bboxes) in enumerate(val_loader):
            data_time.update(time.time() - start)

            frames_flip = model.horizontal_flip(frames).to(device)
            frames = frames.to(device)
            labels = labels.to(device)
            bboxes = bboxes.to(device)

            # Forward prop.
            x_locs, xf_locs, x_action_label, xf_action_label = model(frames, frames_flip)
            # predictions = torch.argmax(raw_pred, dim=1)

            # Loss
            cls_loss, loc_loss, = criterion(x_locs, x_action_label, bboxes, labels)  # scalar
            loss = cls_loss + loc_loss

            # accuracy
            _, accuracy_pred = torch.max(x_action_label.data, 1)
            total += labels.size(0)
            correct += labels.squeeze(1).eq(accuracy_pred).sum().item()

            losses.update(loss.item(), frames.size(0))
            multibox_losses.update(loc_loss.item(), frames.size(0))
            classification_losses.update(cls_loss.item(), frames.size(0))

            batch_time.update(time.time() - start)

            start = time.time()

            # Print status
            if i % print_freq == 0:
                print('Validation: [{0}/{1}]\t'
                      'Batch Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                      'Loss {loss.val:.4f} ({loss.avg:.4f})\n'
                      'Classification Loss {classification_loss.val:.4f} ({classification_loss.avg:.4f})\t'
                      'MultiBox Loss {multibox_loss.val:.4f} ({multibox_loss.avg:.4f})\t'.format(i, len(val_loader),
                                                                      batch_time=batch_time,
                                                                      loss=losses,
                                                                      classification_loss=classification_losses,
                                                                      multibox_loss=multibox_losses))

    print('\n * LOSS - {loss.avg:.3f}\n'.format(loss=losses))
    print('Accuracy of the network: %d %%' % (100 * correct / total))
    return losses.avg, (100 * correct / total)

def save_checkpoint(epoch, epochs_since_improvement, model, optimizer, loss, best_loss, is_best, accuracy):
    state = {'epoch': epoch,
             'epochs_since_improvement': epochs_since_improvement,
             'loss': loss,
             'best_loss': best_loss,
             'model': model,
             'optimizer': optimizer,
             'accuracy': accuracy}
    filename = model_checkpoint_path
    torch.save(state, filename)
    # Store Best Checkpoint
    if is_best:
        torch.save(state, "{}/BEST_imp{}-{}-{}.pth.tar".format(weights_root, model_version, sub_version, epoch))

if __name__ == "__main__":
    print(f"Executing on device: {device}")
    if len(sys.argv) > 1:
        datastore_version = int(sys.argv[1]) # select model
        dataset_root = f"E:\\dev2\\UCF-10{datastore_version}"
    if len(sys.argv) > 2:
        image_per_class = int(sys.argv[2])
    main()


import tensorflow as tf
import numpy as np
import numba

crop_resizemethod = {'bilinear': tf.image.ResizeMethod.BILINEAR,
                     'nearest': tf.image.ResizeMethod.NEAREST_NEIGHBOR}

class Preprocessor:
    def __init__(self, resolution, resize_method, crop_method, channels):
        self.img = tf.placeholder(tf.float32, [None, None, channels])
        self.min_size = tf.placeholder(tf.int32, None)

        self.source = tf.divide(self.img, 255)
        self.source = tf.subtract(tf.multiply(self.source, 2), 1)  # Fit for tanh
        if crop_method == 'center_crop':
            self.process = tf.image.resize_image_with_crop_or_pad(self.source, self.min_size, self.min_size)
        if crop_method == 'resize_only':
            self.process = self.source
        if crop_method == 'pad_resize':
            self.process = \
                tf.image.resize_image_with_pad(self.source, resolution, resolution,
                                               method=crop_resizemethod[resize_method])
        if crop_method == 'random_crop':
            self.process = tf.image.random_crop(self.source, (self.min_size, self.min_size, 3))

        if crop_method != 'pad_resize':
            self.process = \
               tf.image.resize_images(self.process, (resolution, resolution), method=crop_resizemethod[resize_method])
        self.final = tf.transpose(self.process, [2, 0, 1])  # HWC to CHW

        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        self.sess = tf.Session(config=config)

    @numba.jit
    def _get_min_size(self, img):
        return np.min(np.shape(img)[:2])

    def process_image(self, img):
        min_size = self._get_min_size(img)
        img = self.sess.run(self.final, feed_dict={self.img: img, self.min_size: min_size})
        return img


# -*- coding: utf-8 -*-
import numpy as np, random, matplotlib.pyplot as plt, sys, getopt
import scipy as sc
from mpl_toolkits.mplot3d import Axes3D
from mpl_toolkits import mplot3d
from points import *
from LS import *
from scipy.spatial.transform import Rotation as Rot
from matplotlib import cm

plt.rcParams["font.family"] = "Lucida Grande"
plt.rcParams["font.size"] = 16
plt.style.use("ggplot")

# Silly hacks for avoiding singular points and not showing rotation matrix warnings
zero = 1e-3
import warnings
warnings.filterwarnings("ignore")

def Φ_LS(P, T, C, N, α, φ = 0, β = 0, λ = 0):
    e_in=np.array([1,0,0])
    R = Rot.from_euler('ZYZ',[λ,-β,φ]).as_matrix().T
    Rα = Rot.from_euler('z',α).as_matrix()

    e_eye = matmul(Rα,e_in)

    d = 10*np.amax(np.abs(np.linalg.norm(P,axis=1)))*e_in # Displacement
    P = P.dot(R.T) + d
    C = C.dot(R.T) + d
    N = N.dot(R.T)
    vis = (np.dot(e_in,N.T) < 0.)  & (np.dot(e_eye,N.T) < 0.)
    R = np.abs(C.dot(e_in))
    jj = np.argsort(R)

    S = 0.
    for i, j in enumerate(jj):
        if not vis[j]:
            continue
        point = C[j,1:3]
        for jjj in jj[0:i]:
            t = T[jjj]
            if PointInTriangle(point, P[t,1:3]):
                vis[j] = False
        if(vis[j]):
            S += Φ_LS_dA(-N[j], α, e_in, e_eye)
    return S

if __name__=="__main__":
    a = 1
    b = 0.86
    c = 0.82
    n = 250

    P, T = genellip(a, b, c, n, sigma = 0.125, ell = 0.3, gid = 1)
    T, N, C = surface_normals(P, T)
#    P, T, N, C = ellipsoid_points(a, b, c, n)

    α = np.linspace(0, pi, 33)
    φ = np.linspace(0, 2*pi, 30)
    L = np.zeros([α.size, φ.size])

    for i, a in enumerate(α):
        print(i)
        for j, f in enumerate(φ):
           L[i,j] = Φ_LS(P, T, C, N, α=a, φ=f, β=0, λ=0)

    X, Y = np.meshgrid(φ*180/pi, α*180/pi)

    fig = plt.figure(figsize=(12,8))
    ax = fig.gca(projection='3d')
    surf = ax.plot_surface(X, Y, L, cmap=cm.inferno,
                       linewidth=0, antialiased=False)
    plt.ylabel('α (°)')
    plt.xlabel('φ (°)')
    plt.title(r'$Φ_{LS}(α,φ)$')
    plt.ylim([0,180])
    plt.xlim([0,360])
    plt.legend()
    plt.show()


from rest_framework import serializers
from .models import *

class ProduitSerializer(serializers.ModelSerializer):
    class Meta:
         model = Produit
         fields = "__all__"

class VenteSerializer(serializers.ModelSerializer):

    def to_representation(self, instance):
        data = super().to_representation(instance)
        data["produit"] = instance.produit.nom
        data["vente"] = instance.umudandaza.username
        return data

    class Meta:
            model = Vente
            fields = "__all__"
            read_only_fields = "prix_de_vente_total",


import os
import yaml
import appdirs
from importlib import resources
import shutil

config_filename = "config.yaml"

# Using appdirs to determine user-specific config path
config_dir = appdirs.user_config_dir("Open Interpreter")
user_config_path = os.path.join(config_dir, config_filename)

def get_config():
    if not os.path.exists(user_config_path):
        # If user's config doesn't exist, copy the default config from the package
        here = os.path.abspath(os.path.dirname(__file__))
        parent_dir = os.path.dirname(here)
        default_config_path = os.path.join(parent_dir, 'config.yaml')
        # Ensure the user-specific directory exists
        os.makedirs(config_dir, exist_ok=True)
        # Copying the file using shutil.copy
        shutil.copy(default_config_path, user_config_path)

    with open(user_config_path, 'r') as file:
        return yaml.safe_load(file)


from collections import deque

def solution(cacheSize, cities):
    answer = 0
    q = deque()
    if cacheSize == 0:
        return len(cities) * 5
    for city in cities:
        city = city.lower()
        if city not in q:
            if len(q) < cacheSize:
                q.append(city)
            else:
                q.popleft()
                q.append(city)
            answer += 5
        elif city in q:
            q.remove(city)
            q.append(city)
            answer += 1
    print(q)
    return answer

cities = ["Jeju", "Pangyo", "NewYork", "newyork"]
print(solution(2,cities))


import sys
import shutil
import zipfile
import xml.etree.ElementTree as ET

if sys.version_info[0] != 3:
    print("Please run script with Python 3.x")
    sys.exit(1)

args = len(sys.argv)

if args < 2:
    print("Invalid number of arguments! (python", sys.argv[0], "<filename>")
    sys.exit(1)

src = str(sys.argv[1])
tmp_dir = "tmp_dir/"

if args >= 3:
    dst = str(sys.argv[2])
else:
    if '.' in src:
        src_arr = src.split('.')
        dst = src_arr[0] + "_new." + src_arr[1]
    else:
        dst = src + "_new"

# Extract contents from odt to directory
try:
    with zipfile.ZipFile(src, 'r') as zip_ref:
        zip_ref.extractall(tmp_dir)
except FileNotFoundError:
    print("Could not find", src)
    sys.exit(1)
except:
    print("Unexpected error:", sys.exc_info()[0])
    sys.exit(1)

# Namespace declaration
ns = {'style': 'urn:oasis:names:tc:opendocument:xmlns:style:1.0',
      'fo': 'urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0',
      'office': 'urn:oasis:names:tc:opendocument:xmlns:office:1.0',
      'svg': 'urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0',
      'table': 'urn:oasis:names:tc:opendocument:xmlns:table:1.0',
      'loext': 'urn:org:documentfoundation:names:experimental:office:xmlns:loext:1.0',
      'text': 'urn:oasis:names:tc:opendocument:xmlns:text:1.0',
      'draw': 'urn:oasis:names:tc:opendocument:xmlns:drawing:1.0',
      'xlink': 'http://www.w3.org/1999/xlink',
      'officeooo': 'http://openoffice.org/2009/office',
      'grddl': 'http://www.w3.org/2003/g/data-view#',
      'xhtml': 'http://www.w3.org/1999/xhtml',
      'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
      'xsd': 'http://www.w3.org/2001/XMLSchema',
      'xforms': 'http://www.w3.org/2002/xforms',
      'dom': 'http://www.w3.org/2001/xml-events',
      'script': 'urn:oasis:names:tc:opendocument:xmlns:script:1.0',
      'form': 'urn:oasis:names:tc:opendocument:xmlns:form:1.0',
      'math': 'http://www.w3.org/1998/Math/MathML',
      'drawoo': 'http://openoffice.org/2010/draw',
      'calctext': 'urn:org:documentfoundation:names:experimental:calc:xmlns:calcext:1.0',
      'tableooo': 'http://openoffice.org/2009/table',
      'number': 'urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0'}

# Parse XML
for key in ns:
    ET.register_namespace(key, ns[key])

tree = ET.ElementTree()
tree.parse(tmp_dir + "content.xml")
root = tree.getroot()

for table_cell_style in root.findall(".//style:style[@style:family='table-cell']", ns):
    properties = table_cell_style.find("style:table-cell-properties", ns)
    properties.attrib['{%s}border' % ns['fo']] = "0.05pt solid #000000"

tree.write(tmp_dir + "content.xml")

# Compress tmp directory as odt
shutil.make_archive(dst, 'zip', tmp_dir)
shutil.move(dst + '.zip', dst)

# Remove tmp directory
shutil.rmtree(tmp_dir)

print("Successfully added borders to '", src, "'. File saved as '", dst, "'", sep="")


# -*- coding: utf-8 -*-
"""
Created on Thu Nov 18 01:57:54 2021

@author: Necro
"""

from models.dummy_DM import DM_Model
from flask_restful import Resource, reqparse,abort,fields,marshal_with,request
from models.user import UserModel
from models.meal_log import Meal_log
from usda import foodret
import uuid
import datetime
from foodpred import pred

class DM_register(Resource):
    parser = reqparse.RequestParser()
    parser.add_argument('uuid',
    type=str
    )
    parser.add_argument('food',
    type=str
    )
    """
    parser.add_argument('energy', type=int, required=False)
    parser.add_argument('protein', type=int, required=False)
    parser.add_argument('total_lipid', type=int, required=False)
    parser.add_argument('carbohydrate', type=int, required=False)
    parser.add_argument('fiber', type=int, required=False)
    parser.add_argument('sugar', type=int, required=False)
    parser.add_argument('calcium', type=int, required=False)
    parser.add_argument('iron', type=int, required=False)
    parser.add_argument('sodium', type=int, required=False)
    #parser.add_argument('magnesium', type=int, required=False)
    parser.add_argument('vitamin_a', type=int, required=False)
    parser.add_argument('vitamin_c', type=int, required=False)
    parser.add_argument('vitamin_d', type=int, required=False)
    parser.add_argument('saturated_fatty_acid', type=int, required=False)
    parser.add_argument('monounsaturated_fatty_acid', type=int, required=False)
    parser.add_argument('polyunsaturated_fatty_acid', type=int, required=False)
    parser.add_argument('cholesterol', type=int, required=False)
    """

    def get(self):
        data = DM_register.parser.parse_args()
        nutrient = DM_Model.find_by_uuid(data['uuid'])
        #result = NutrientModel.query.filter_by(id = idx).first()

        if nutrient:
            return nutrient.json()

        return {'message' : 'nutrient not found'},404

    ## if uuid exists in the UserModel then NutrientModel is created with that uuid,
    #  if uuid exists in the UserModel and also in the NutrientModel then it updates according to the args passed.

    ##note:for DM_Model if attribute doesnt exist put 0 in that field

    def put(self):
        data1 = DM_register.parser.parse_args()
        ct = datetime.datetime.now()
        if UserModel.find_by_uuid(data1['uuid']):
            if DM_Model.find_by_uuid(data1['uuid'])==None:
                uuid_1 = data1['uuid']

                #comment this section
                file = request.files['image']
                prediction = pred(file)

                food = prediction.replace("_", " ")
                print(food)
                #

                #uncomment this to insert food manually
                #food = data1['food'].replace("_", " ") #to replace "_" with emptyspace

                data = foodret(food)
                ts = str(ct.year)+"-"+str(ct.month)+"-"+str(ct.day)
                nutrient = DM_Model(str(uuid_1),
                                         data['energy'],
                                         data['protein'],
                                         data['total_lipid'],
                                         data['carbohydrate'],
                                         data['fiber'],
                                         data['sugar'],
                                         data['calcium'],
                                         data['iron'],
                                         data['sodium'],
                                         #data['magnesium'],
                                         data['vitamin_a'],
                                         data['vitamin_c'],
                                         data['vitamin_d'],
                                         data['saturated_fatty_acid'],
                                         data['monounsaturated_fatty_acid'],
                                         data['polyunsaturated_fatty_acid'],
                                         data['cholesterol'],
                                         str(1),
                                         ts)

                ##
                nutrient_str = str(data['energy']) + "," + str(data['protein']) + "," + str(data['total_lipid']) + ","+ str(data['carbohydrate']) + ","+ str(data['fiber']) + ","+ str(data['sugar']) + "," + str(data['calcium']) + "," + str(data['iron']) + "," + str(data['sodium']) + "," + str(data['vitamin_a']) + "," + str(data['vitamin_c']) + "," + str(data['vitamin_d']) + "," + str(data['saturated_fatty_acid']) + "," + str(data['monounsaturated_fatty_acid']) + "," + str(data['polyunsaturated_fatty_acid']) + "," + str(data['cholesterol'])
                print(nutrient_str)

                tstime=str(ct.hour)+':'+str(ct.minute)
                meal = Meal_log(str(uuid.uuid4()),data1['uuid'],nutrient_str ,ts,tstime,food)

                ##
                try:
                    nutrient.save_to_db()
                    meal.save_to_db()
                except:
                    return {"message": "An error occurred creating the nutrient."}, 500 # Internal Server Error
            else:
                nutrient = DM_Model.find_by_uuid(data1['uuid'])

                #comment this section
                file = request.files['image']
                prediction = pred(file)
                #food = data1['food'].replace("_", " ") #to replace "_" with emptyspace
                food = prediction.replace("_", " ")
                print(food)
                #

                #uncomment to insert food manually
                #food = data1['food'].replace("_", " ")  #to replace "_" with emptyspace

                data = foodret(food)
                #data = foodret(data1['food'])

                ts = str(ct.year)+"-"+str(ct.month)+"-"+str(ct.day)
                if nutrient.date != ts:
                    nutrient.energy,nutrient.protein,nutrient.total_lipid,nutrient.carbohydrate,nutrient.sugar,nutrient.calcium,nutrient.iron,nutrient.sodium,nutrient.vitamin_a,nutrient.vitamin_c,nutrient.vitamin_d,nutrient.saturated_fatty_acid,nutrient.monounsaturated_fatty_acid,nutrient.polyunsaturated_fatty_acid,nutrient.cholesterol,nutrient.meal_count=[0]*16
                    nutrient.date = ts
                if data['energy']:
                    nutrient.energy = float(nutrient.energy) + float(data['energy'])
                if data['protein']:
                    nutrient.protein = float(nutrient.protein) + float(data['protein'])
                if data['total_lipid']:
                    nutrient.total_lipid = float(nutrient.total_lipid) + float(data['total_lipid'])
                if data['carbohydrate']:
                    nutrient.carbohydrate = float(nutrient.carbohydrate) + float(data['carbohydrate'])
                if data['sugar']:
                    nutrient.sugar = float(nutrient.sugar) + float(data['sugar'])
                if data['calcium']:
                    nutrient.calcium = float(nutrient.calcium) + float(data['calcium'])
                if data['iron']:
                    nutrient.iron = float(nutrient.iron) + float(data['iron'])
                if data['sodium']:
                    nutrient.sodium = float(nutrient.sodium) + float(data['sodium'])
                if data['vitamin_a']:
                    nutrient.vitamin_a = float(nutrient.vitamin_a) + float(data['vitamin_a'])
                if data['vitamin_c']:
                    nutrient.vitamin_c = float(nutrient.vitamin_c) + float(data['vitamin_c'])
                if data['vitamin_d']:
                    nutrient.vitamin_d = float(nutrient.vitamin_d) + float(data['vitamin_d'])
                if data['saturated_fatty_acid']:
                    nutrient.saturated_fatty_acid = float(nutrient.saturated_fatty_acid) + float(data['saturated_fatty_acid'])
                if data['monounsaturated_fatty_acid']:
                    nutrient.monounsaturated_fatty_acid = float(nutrient.monounsaturated_fatty_acid) + float(data['monounsaturated_fatty_acid'])
                if data['polyunsaturated_fatty_acid']:
                    nutrient.polyunsaturated_fatty_acid = float(nutrient.polyunsaturated_fatty_acid) + float(data['polyunsaturated_fatty_acid'])
                if data['cholesterol']:
                    nutrient.cholesterol = float(nutrient.cholesterol) + float(data['cholesterol'])

                nutrient.meal_count = int(nutrient.meal_count)+1

                ##
                nutrient_str = str(data['energy']) + "," + str(data['protein']) + "," + str(data['total_lipid']) + ","+ str(data['carbohydrate']) + ","+ str(data['fiber']) + ","+ str(data['sugar']) + "," + str(data['calcium']) + "," + str(data['iron']) + "," + str(data['sodium']) + "," + str(data['vitamin_a']) + "," + str(data['vitamin_c']) + "," + str(data['vitamin_d']) + "," + str(data['saturated_fatty_acid']) + "," + str(data['monounsaturated_fatty_acid']) + "," + str(data['polyunsaturated_fatty_acid']) + "," + str(data['cholesterol'])
                print(nutrient_str)

                x = nutrient_str.split(",")
                print(x)

                ts = str(ct.year)+"-"+str(ct.month)+"-"+str(ct.day)
                tstime=str(ct.hour)+':'+str(ct.minute)
                meal = Meal_log(str(uuid.uuid4()),data1['uuid'],nutrient_str ,ts,tstime,food)

                ##

                try:
                    nutrient.save_to_db()
                    meal.save_to_db()
                except:
                    return {"message": "An error occurred creating the nutrient."}, 500 # Internal Server Error

        return meal.json(), 201

class DM_List(Resource):
    def get(self):
        return {'dms': [dm.json() for dm in DM_Model.query.all()]}

class DM_delete(Resource):
    #pass uuid from the NutrientModel to delete
    def delete(self):
        data = DM_register.parser.parse_args()
        nutrient = DM_Model.find_by_uuid(data['uuid'])
        #result = NutrientModel.query.filter_by(id = idx).first()

        if nutrient:
            nutrient.delete_from_db()
        return {'message' : 'physical deleted'},404


__metaclass__ = type

__all__ = [
    'db_options',
    'dummy_logger_options',
    'execute_zcml_for_scripts',
    'log',
    'logger',
    'logger_options',
    ]

import atexit
import os
import sys
import threading

from zope.configuration.config import ConfigurationMachine
from zope.security.management import setSecurityPolicy
import zope.sendmail.delivery
import zope.site.hooks

from lp.services.config import config
from lp.services.database.postgresql import ConnectionString
# these are intentional re-exports, apparently, used by *many* files.
from lp.services.scripts.logger import (
    dummy_logger_options,
    log,
    logger,
    logger_options,
    )
from lp.services.webapp.authorization import (
    LaunchpadPermissiveSecurityPolicy,
    LaunchpadSecurityPolicy,
    )
from lp.services.webapp.interaction import (
    ANONYMOUS,
    setupInteractionByEmail,
    )

def execute_zcml_for_scripts(use_web_security=False):
    """Execute the zcml rooted at launchpad/script.zcml

    If use_web_security is True, the same security policy as the web
    application uses will be used. Otherwise everything protected by a
    permission is allowed, and everything else denied.
    """

    # When in testing mode, prevent some cases of erroneous layer usage.
    # But we don't want to import that module in production usage, thus
    # the conditional block.
    if 'lp.testing.layers' in sys.modules:
        from lp.testing.layers import (
                FunctionalLayer, BaseLayer, ZopelessLayer)
        assert not FunctionalLayer.isSetUp, \
                'Setting up Zopeless CA when Zopefull CA is already running'
        assert not BaseLayer.isSetUp or ZopelessLayer.isSetUp, """
                execute_zcml_for_scripts should not be called from tests.
                Instead, your test should use the Zopeless layer.
            """

    if config.isTestRunner():
        scriptzcmlfilename = 'script-testing.zcml'
    else:
        scriptzcmlfilename = 'script.zcml'

    scriptzcmlfilename = os.path.abspath(
        os.path.join(config.root, 'zcml', scriptzcmlfilename))

    from zope.configuration import xmlconfig

    # Hook up custom component architecture calls
    zope.site.hooks.setHooks()

    # Load server-independent site config
    context = ConfigurationMachine()
    xmlconfig.registerCommonDirectives(context)
    context = xmlconfig.file(
        scriptzcmlfilename, execute=True, context=context)

    if use_web_security:
        setSecurityPolicy(LaunchpadSecurityPolicy)
    else:
        setSecurityPolicy(LaunchpadPermissiveSecurityPolicy)

    # Register atexit handler to kill off mail delivery daemon threads, and
    # thus avoid spew at exit.  See:
    # http://mail.python.org/pipermail/python-list/2003-October/192044.html
    # http://mail.python.org/pipermail/python-dev/2003-September/038151.html
    # http://mail.python.org/pipermail/python-dev/2003-September/038153.html

    def kill_queue_processor_threads():
        for thread in threading.enumerate():
            if isinstance(
                thread, zope.sendmail.delivery.QueueProcessorThread):
                thread.stop()
                thread.join(30)
                if thread.isAlive():
                    raise RuntimeError(
                        "QueueProcessorThread did not shut down")
    atexit.register(kill_queue_processor_threads)

    # This is a convenient hack to set up a zope interaction, before we get
    # the proper API for having a principal / user running in scripts.
    setupInteractionByEmail(ANONYMOUS)

def db_options(parser):
    """Add and handle default database connection options on the command line

    Adds -d (--database), -H (--host), -p (--port) and -U (--user)

    Parsed options provide dbname, dbhost and dbuser attributes.

    Generally, scripts will not need this and should instead pull their
    connection details from launchpad.config.config. The database setup and
    maintenance tools cannot do this however.

    dbname and dbhost are also propagated to config.database.dbname and
    config.database.dbhost. This ensures that all systems will be using
    the requested connection details.

    Ensure that command line options propagate to where we say they do

    >>> from optparse import OptionParser
    >>> parser = OptionParser()
    >>> db_options(parser)
    >>> options, args = parser.parse_args(
    ...     ['--dbname=foo', '--host=bar', '--user=baz', '--port=6432'])
    >>> options.dbname
    'foo'
    >>> options.dbhost
    'bar'
    >>> options.dbuser
    'baz'
    >>> options.dbport
    6432
    >>> config.database.rw_main_master
    'dbname=foo user=baz host=bar port=6432'
    >>> config.database.rw_main_slave
    'dbname=foo user=baz host=bar port=6432'

    Make sure that the default user is None

    >>> parser = OptionParser()
    >>> db_options(parser)
    >>> options, args = parser.parse_args([])
    >>> print options.dbuser
    None
    """
    conn_string = ConnectionString(config.database.rw_main_master)

    def update_db_config(**kw):
        connection_string_keys = [
            'rw_main_master',
            'rw_main_slave',
            ]
        config_data = ["[database]"]
        for con_str_key in connection_string_keys:
            con_str = ConnectionString(getattr(config.database, con_str_key))
            for kwarg, kwval in kw.items():
                setattr(con_str, kwarg, kwval)
            config_data.append("%s: %s" % (con_str_key, str(con_str)))
        config.push('update_db_config', '\n'.join(config_data))

    def dbname_callback(option, opt_str, value, parser):
        parser.values.dbname = value
        update_db_config(dbname=value)

    parser.add_option(
            "-d", "--dbname", action="callback", callback=dbname_callback,
            type="string", dest="dbname", default=conn_string.dbname,
            help="PostgreSQL database to connect to."
            )

    def dbhost_callback(options, opt_str, value, parser):
        parser.values.dbhost = value
        update_db_config(host=value)

    parser.add_option(
             "-H", "--host", action="callback", callback=dbhost_callback,
             type="string", dest="dbhost", default=conn_string.host,
             help="Hostname or IP address of PostgreSQL server."
             )

    def dbport_callback(options, opt_str, value, parser):
        value = int(value)
        parser.values.dbport = value
        update_db_config(port=value)

    parser.add_option(
        "-p", "--port", action="callback", callback=dbport_callback,
        type=int, dest="dbport", default=conn_string.port,
        help="Port PostgreSQL server is listening on."
        )

    def dbuser_callback(options, opt_str, value, parser):
        parser.values.dbuser = value
        update_db_config(user=value)

    parser.add_option(
             "-U", "--user", action="callback", callback=dbuser_callback,
             type="string", dest="dbuser", default=None,
             help="PostgreSQL user to connect as."
             )