text:
from O365 import Account
from IPython.display import display
import ipywidgets as widgets
import codecs
import json
import random
import numpy as np
import warnings

# Deactivates deprecation warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)

# email template filepaths
email_template = 'resources/templates/email_template.html'
grade_email_template = 'resources/templates/grade_email_template.html'

def o365_login(id_app, secret):
    """ Function to login in Office 365 app

    Args:
        id_app (string): Azure app id
        secret (string): Azure app secret

    Returns:
        Account: Azure app handler
    """

    credentials = (id_app, secret)
    account = Account(credentials)
    account.authenticate(scopes=['basic', 'message_all'])
    return account

def generate_body(student_name, assignment):
    """ Generates the body of the email to send the assignment

    Args:
        template_path (str): location of the message template
        student_name (str): Name of the student
        assignment (Assigment): Assigment object

    Returns:
        str: email body text
    """

    f = codecs.open(email_template, 'r')
    body = f.read()

    data = assignment.config

    for i in range(len(data)):
        key = "[[" + data['Variable'][i] + "]]"
        body = body.replace(key, str(data['Value'][i]))

    return body.replace('[[name]]', student_name)

def generate_subject(assignment):
    """ Generates email subject

    Args:
        assignment (Assigment): Assigment object

    Returns:
        str: subject string
    """

    data = assignment.config

    key_1 = 'Assignment name'
    name = data.loc[assignment.config['Variable'] == key_1, 'Value']
    name = name.to_string(header=False, index=False)

    key_2 = 'Assignment code'
    code = data.loc[assignment.config['Variable'] == key_2, 'Value']

    code = code.to_string(header=False, index=False)

    return name + " - " + code

def send_email(account, assignment, email, name, attachment=False):
    """ Sends individual emails with assignment to specified student.

    Args:
        account (Account): Azure app handler
        assignment (Assigment): Assigment object
        email (str): email address
        name (str): Student name
        attachment (bool, str, optional): Attachment path.
                If False, email is sent without attachments.
                Defaults to False.

    Returns:
        bool: True if message is sent, False otherwise
    """

    m = account.new_message()
    m.to.add(email)
    m.subject = generate_subject(assignment)
    m.body = generate_body(name, assignment)
    if attachment:
        m.attachments.add(attachment)
    return m.send()

def send_email_list(account, assignment):
    """ Sends assignment emails to the student list

    Args:
        account (Account): Azure app handler
        assignment (Assigment): Assigment object

    Returns:
        bool: list with send status to all emails
    """

    data = assignment.student_list

    max_progess = len(data) - 1
    progress = widgets.FloatProgress(value=0.0, min=0.0, max=max_progess)

    print('------')
    print("Sending emails")
    display(progress)

    sent = []
    for i in range(len(data)):
        email = data['email'][i]
        name = data['name'][i]
        attachment = data['file'][i]
        sent.append(send_email(account,
                               assignment,
                               email,
                               name,
                               attachment))

        progress.value = i

    if all(sent):
        print('Emails sent with no errors')
    else:
        print('Some emails not sent (check function return')

    return sent

def load_credentials(path):
    """ Loads credentials from JSON file

    Args:
        path (str): JSON file with credentilas path

    Returns:
        dict: Dictionary with credentials
    """

    try:
        with open(path, 'r') as fp:
            return json.load(fp)
    except FileNotFoundError:
        print('Credentials file not found')
        return None

def send_test_email(account, assignment, email):
    """ Sends random email sampl to specified email address

    Args:
        account (Account): Azure app handler
        assignment (Assigment): Assigment object
        email (str): Email address
    """

    student = random.randint(0, len(assignment.student_list) - 1)
    name = assignment.student_list['name'][student]
    attachment = assignment.student_list['file'][student]

    send_email(account, assignment, email, name, attachment)

def generate_grading_table(assignment, titles, id):
    """ Generates a table with student answers, correct solutions and
        assignment points per question

    Args:
        assignment (Assigment): Assigment object
        titles (str, list): List with table titles
        id (str): Student id

    Returns:
        str: HTML formated grading table
    """

    answer_names = list(assignment.solutions.columns)[1:]
    answers = assignment.answers[assignment.answers['id'] == id][answer_names]
    if answers.empty:
        answers = ['-'] * len(answer_names)
    else:
        answers = answers.fillna('-')
        answers = answers.iloc[0].to_list()

    id_selection = assignment.student_list['id'] == id
    number = assignment.student_list[id_selection]['number'].item()

    number_selection = assignment.solutions['number'] == number
    solutions = assignment.solutions[number_selection][answer_names]

    solutions_string = []
    for solution in solutions.iloc[0].to_list():
        if np.abs(solution) >= 0.01:
            num = np.round(solution, decimals=4)
            num_format = '{}'
        else:
            num = solution
            num_format = '{:.4e}'

        solutions_string.append(num_format.format(num))

    points = assignment.grades[assignment.grades['id'] == id][answer_names]
    points = points.iloc[0].to_list()

    rows = [answers, solutions_string, points]

    table = '\n
\n'
    table += '\t
\n\t\t 	\n'

    for column in answer_names:
        table += '\t\t{0}	\n'.format(column)
    table += '\t
\n'

    for i, row in enumerate(rows):
        table += '\t
\n\t\t{0}	\n'.format(titles[i])
        for column in row:
            table += '\t\t'
            table += '{0}	\n'.format(column)

        table += '\t
\n'

    table += '
\n'

    return table

def send_grade_list(account, assignment, titles):
    """ Sends grade to the student list.

    Args:
        account (Account): Azure app handler.
        assignment (Assigment): Assigment object.
        titles ([str]): Grading table headers.

    Returns:
        [bool]: list with True if the email was sent, False otherwise.
    """

    grades = assignment.grades

    max_progess = len(grades) - 1
    progress = widgets.FloatProgress(value=0.0, min=0.0, max=max_progess)

    print('------')
    print("Sending emails")
    display(progress)

    sent = []

    for i, id in enumerate(assignment.grades['id']):
        sent.append(send_grade_email(account, assignment, id, titles))

        progress.value = i + 1

    if all(sent):
        print('Emails sent with no errors')
    else:
        print('Some emails not sent (check function return')

    return sent

def generate_grade_body(id, assignment, titles):
    """ Generates the body of the grade sending email

    Args:
        id (str): Student ID
        assignment (Assigment): Assigment object
        titles ([str]): Grading table headers

    Returns:
        [str]: HTML formated email body
    """

    grading_table = generate_grading_table(assignment, titles, id)

    st_list = assignment.student_list

    student_name = st_list[assignment.student_list['id'] == id]['name'].item()

    grades = assignment.grades[assignment.grades['id'] == id]

    grade = np.round(grades['grade'].values[0], decimals=1)
    points = grades['points'].values[0]

    with codecs.open(grade_email_template, 'r') as f:

        body = f.read()
        data = assignment.config

        for i in range(len(data)):
            key = "[[" + data['Variable'][i] + "]]"
            body = body.replace(key, str(data['Value'][i]))

        body = body.replace('[[name]]', student_name)
        body = body.replace('[[grade_table]]', grading_table)
        body = body.replace('[[grade]]', str(grade))
        body = body.replace('[[points]]', str(points))

    return body

def generate_grade_subject(assignment):
    """ Generates the subjet of the grade sending email

    Args:
        assignment (Assigment): Assigment object.

    Returns:
        str: Grade email subject
    """

    config = assignment.config

    name = config.loc[config['Variable'] == 'Assignment name', 'Value']
    name = name.to_string(header=False, index=False)

    code = config.loc[config['Variable'] == 'Assignment code', 'Value']
    code = code.to_string(header=False, index=False)

    return name + " - " + code

def send_grade_email(account, assignment, id, titles, email=False):
    """ Send individual grading email

    Args:
        account (Account): Azure app handler.
        assignment (Assigment): Assigment object.
        id ([type]): [description]
        titles ([str]): Grading table headers.
        email (str or bool, optional): email to send message, if set to
                                       false obtains it from assignment
                                       object. Defaults to False.

    Returns:
        [bool]: Send status.
    """

    st_list = assignment.student_list

    if not email:
        email = st_list[st_list['id'] == id]['email'].item()

    m = account.new_message()
    m.to.add(email)
    m.subject = generate_grade_subject(assignment)
    m.body = generate_grade_body(id, assignment, titles)

    return m.send()

def send_grade_test_email(account, assignment, email, titles):
    """ Sends grading email of a random ID to the specified email

    Args:
        account (Account): Azure app handler.
        assignment (Assigment): Assigment object.
        email (str): email addess
        titles ([str]): Grading table headers.

    Returns:
        [bool]: Send status.
    """
    rand_id = random.choice(assignment.grades['id'].unique())

    return send_grade_email(account, assignment, rand_id, titles, email=email)


text:
from tkinter import *
from tkinter import ttk
import numpy as np
import io
import base64
from PIL import ImageTk, Image
import requests
import os
import matplotlib
from matplotlib import image as mpimg
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg

url = "http://vcm-9111.vm.duke.edu:5000/"
# url = "http://127.0.0.1:5000/"

def editing_window():
    """Editing window

    Initial GUI window that lets user enter their information and
    their requested editing steps.

    Args:
        none

        Returns:
        none
    """
    def enter_data():
        """Collect inputted data

        Collects username, image paths, requested image types to send
        to server.

        Args:
            none

        Returns:
            none
        """
        success_label.grid_forget()
        from client import download_images
        entered_user = user.get()
        print("User: {}".format(entered_user))
        entered_img_paths = img_path.get()
        upload_popup = Toplevel(root)
        if len(entered_img_paths) == 0:
            success_msg = 'No image paths entered'
            popup_window(upload_popup, success_msg, screen_w, screen_h)
            return
        img_paths = process_img_paths(entered_img_paths)
        print("Image paths:")
        for i in img_paths:
            print("\t{}".format(i))
        entered_img_type = img_type.get()
        entered_1 = hist_eq.get()
        entered_2 = contr_stretch.get()
        entered_3 = log_comp.get()
        entered_4 = rev_vid.get()
        req_img_type, proc_steps = convert_inputs(entered_img_type,
                                                  entered_1, entered_2,
                                                  entered_3, entered_4)

        orig_images, filenames, success = get_img_data(img_paths)
        print("Filenames:")
        print("\t{}".format(filenames))
        upload_success = upload_to_server(entered_user, orig_images,
                                          filenames, success,
                                          proc_steps)
        display_img = BooleanVar()
        display_check = ttk.Checkbutton(root, text='Display images',
                                        variable=display_img,
                                        onvalue=True, offvalue=False,
                                        command=lambda:
                                        display_images(display_img.get(),
                                                       entered_user,
                                                       req_img_type,
                                                       img_paths,
                                                       proc_steps,
                                                       root, filenames,
                                                       orig_images,
                                                       success))
        display_check.grid(column=0, row=10, sticky=W)
        orig_file_list = get_file_list(filenames,
                                       req_img_type,
                                       [True, False, False,
                                        False, False], success)
        file_list = get_file_list(filenames, req_img_type, proc_steps,
                                  success)
        popup_window(upload_popup, upload_success, screen_w, screen_h)
        download_btn.config(state=NORMAL)
        download_btn.config(command=lambda:
                            dl_images(root, entered_user, orig_file_list,
                                      '', screen_w, screen_h))
        download_btn2.config(state=NORMAL)
        download_btn2.config(command=lambda:
                             dl_images(root, entered_user, file_list,
                                       '', screen_w, screen_h))
        zip_msg = ttk.Label(root,
                            text='Multiple files saved as download.zip')
        # if success.count(True) > 1:
        zip_msg.grid(column=0, columnspan=2, row=13, sticky=N)
        return

    # Main window
    root = Tk()
    root.title("Image Editor")
    screen_w = root.winfo_screenwidth()
    screen_h = root.winfo_screenheight()
    root_w = 0.3*screen_w
    root_h = 0.3*screen_h
    root.config(height=root_h, width=root_w)
    x = 0.38*screen_w
    y = 0.35*screen_h
    root.geometry('+%d+%d' % (x, y))

    top_label = ttk.Label(root, text='Edit Your Image On Our Server!')
    top_label.grid(column=0, row=0, columnspan=2, sticky=N)

    # Enter username
    user_label = ttk.Label(root, text="Username:")
    user_label.grid(column=0, row=1, sticky=E)
    user = StringVar()
    user_entry = ttk.Entry(root, textvariable=user, width=25)
    user_entry.grid(column=1, row=1, sticky=W)

    instructions = "Separate pathnames with commas."
    instructions_label = ttk.Label(root, text=instructions)
    instructions_label.grid(column=0, row=2, columnspan=2, sticky=N)

    # Enter image paths
    img_label = ttk.Label(root, text="Image paths:")
    img_label.grid(column=0, row=3, sticky=E)
    img_path = StringVar()
    img_path_entry = ttk.Entry(root, textvariable=img_path,
                               width=25)
    img_path_entry.grid(column=1, row=3, sticky=W)

    # Select download image type
    img_type = StringVar()
    type_label = ttk.Label(root, text="Download image type:")
    type_label.grid(column=0, row=4, sticky=E)
    type_dropdown = ttk.Combobox(root, textvariable=img_type)
    type_dropdown['values'] = ('JPEG', 'PNG', 'TIFF')
    type_dropdown.grid(column=1, row=4, sticky=W)
    type_dropdown.current(0)
    type_dropdown.config(state='readonly')

    # Check processing steps
    steps_label = ttk.Label(root, text="Processing steps:")
    steps_label.grid(column=0, row=5, sticky=E)

    hist_eq = BooleanVar()
    hist_eq.set(True)
    contr_stretch = BooleanVar()
    log_comp = BooleanVar()
    rev_vid = BooleanVar()

    hist_check = ttk.Checkbutton(root, text='Histogram Equalization',
                                 variable=hist_eq,
                                 onvalue=True, offvalue=False)
    hist_check.grid(column=1, row=5, sticky=W)
    contr_check = ttk.Checkbutton(root, text='Contrast Stretching',
                                  variable=contr_stretch,
                                  onvalue=True, offvalue=False)
    contr_check.grid(column=1, row=6, sticky=W)
    log_check = ttk.Checkbutton(root, text='Log Compression',
                                variable=log_comp,
                                onvalue=True, offvalue=False)
    log_check.grid(column=1, row=7, sticky=W)
    rev_check = ttk.Checkbutton(root, text='Reverse video',
                                variable=rev_vid,
                                onvalue=True, offvalue=False)
    rev_check.grid(column=1, row=8, sticky=W)

    upload_btn = ttk.Button(root, text='Upload file', command=enter_data,
                            width=10)
    upload_btn.grid(column=0, row=9, columnspan=2, sticky=N)
    success_label = ttk.Label(root, text='')
    download_btn = Button(root,
                          text='Download original image',
                          state=DISABLED)
    download_btn.grid(column=0, columnspan=2, row=11, sticky=N)
    download_btn2 = Button(root,
                           text='Download processed image(s)',
                           state=DISABLED)
    download_btn2.grid(column=0, columnspan=2, row=12, sticky=N)
    s = ttk.Style()
    s.configure('Button', foreground=[('disabled', 'black')])
    # Show GUI window
    root.mainloop()
    return

def convert_inputs(entered_img_type, entered_1, entered_2, entered_3,
                   entered_4):
    """Converts GUI user input into function inputs

    Gets requested image type and processing steps in the desired format.

    Args:
        entered_img_type (str): selected image type from GUI dropdown
        entered_1 (bool): histogram equalization?
        entered_2 (bool): contrast stretching?
        entered_3 (bool): log compression?
        entered_4 (bool): reverse video?

    Returns:
        req_img_type (str): requested image type
        proc_steps (list): processing steps as a list of booleans
    """
    four_steps = [entered_1, entered_2, entered_3, entered_4]
    if entered_img_type == 'JPEG':
        req_img_type = '.jpg'
    elif entered_img_type == 'PNG':
        req_img_type = '.png'
    elif entered_img_type == 'TIFF':
        req_img_type = '.tiff'
    if not any(four_steps):
        proc_steps = [True, False, False, False, False]
    else:
        proc_steps = [False, entered_1, entered_2, entered_3,
                      entered_4]
    return req_img_type, proc_steps

def process_img_paths(input):
    """Processes image paths
    If one path is entered, return list of length 1 containing location.
    If multiple (comma-separated) paths are entered, return list containing
    all paths.

    Args:
        input (str): string containing image path(s)

    Returns:
        paths (list): list containing image path, or separated image paths
    """
    input = input.replace(" ", "")
    paths = input.split(",")
    if '' in paths:
        paths.remove("")
    return paths

def get_img_data(img_paths):
    """Gets image data

    Upload: Extracts data from image paths to upload to server as b64
    strings. Unzips images if necessary.
    Download: Unzips downloaded files.

    Args:
        img_paths (list): list of image paths to process

    Returns:
        images (list): list of numpy arrays containing image data
        filenames (list): list of filenames to use. If zip included and
        unzipping successful, this list contains the image filenames, not
        the zip filename. Otherwise, it just states the zip filename.
        success (list): list of booleans denoting successful processing for
        each of the entries in filenames
    """
    from image import read_img_as_b64, unzip
    images = []
    filenames = []
    success = []
    is_zip = [(re.search('.zip', i) or re.search('.ZIP', i)) for i in
              img_paths]
    for i in range(len(img_paths)):
        curr_path = img_paths[i]
        # Check if image exists
        exists = os.path.isfile(curr_path)
        if exists:
            # Append unzipped images one by one
            if is_zip[i]:
                unzipped_images, names, temp_success = unzip(curr_path)
                if temp_success:
                    for j in range(len(unzipped_images)):
                        images.append(unzipped_images[j])
                        filenames.append(names[j])
                        success.append(True)
                else:
                    images.append('')
                    success.append(False)
            # Append non-zipped images normally
            elif curr_path.lower().endswith(('.png', '.jpg', 'jpeg',
                                            '.tiff')):
                """
                img_obj = Image.open(curr_path)
                img_np = np.array(img_obj)
                images.append(img_np)
                """
                images.append(read_img_as_b64(curr_path))
                success.append(True)
                curr_filename = re.split('/', curr_path)[-1]
                filenames.append(curr_filename)
                # img_obj.close()
            # Don't send data if file is not an image
            else:
                images.append('')
                success.append(False)
        # File not found
        else:
            images.append('')
            filenames.append(img_paths[i])
            success.append(False)
    return images, filenames, success

def get_file_list(filenames, file_ext, proc_steps, success):
    """Gets file list

    This function takes the inputted filenames, requested file ext,
    and requested processing steps. The output file list includes
    information from images that were successfully uploaded.

    Args:
        filenames (list): list of filenames
        file_ext (str): requested file extension
        proc_steps (list): list of processing steps
        success (list): list of upload success
    Returns:
        file_list (list): list of properly formatted image info
        for download_images
    """
    file_list = []
    for i in range(len(filenames)):
        if success[i]:
            file_list.append([filenames[i], file_ext, proc_steps])
    return file_list

def upload_to_server(user, images, filenames, success, proc_steps):
    """Posts image to server

    Posts b64 strings to server

    Args:
        user (str): inputted username
        images (list): list of b64 strings
        filenames (list): attached original filenames
        success (list): whether images were successfully obtained
        proc_steps (list): image processing steps to take

    Returns:
        upload_success (str): message to print below upload button
    """
    from client import add_new_user, upload_images
    status_codes = [400]
    list_for_upload = []
    for i in range(len(filenames)):
        list_for_upload.append([filenames[i], images[i], proc_steps])
    add_new_user(user)
    status_codes = upload_images(user, list_for_upload)
    if isinstance(status_codes, list):
        if all([True if i == 200 else False for i in
                status_codes["code"]]):
            upload_success = "Successfully uploaded"
        else:
            upload_success = "Upload failed for one or more images."
            upload_success += "\nImage display can still be opened"
    elif isinstance(status_codes, dict):
        if all([True if i == 200 else False for i in
                status_codes["code"]]):
            upload_success = "Successfully uploaded"
        else:
            upload_success = "Upload failed for one or more images."
            upload_success += "\nImage display can still be opened"
    return upload_success

def popup_window(window, message, screen_w, screen_h):
    """Popup window

    Message pops up after images are uploaded or downloaded.

    Args:
        window (Toplevel): GUI popup window
        message (str): message string to display

    Returns:
        none
    """
    def ok_click(window):
        """Destroy window when OK button clicked

        Args:
            window (Toplevel): GUI popup window
        Returns:
            none
        """
        window.destroy()
        return

    msg_label = Label(window, text=message+'!')
    msg_label.grid(column=0, row=0, sticky=N, ipadx=0.01*screen_w)
    window.geometry('+%d+%d' % (0.43*screen_w, 0.48*screen_h))
    popup_ok_btn = ttk.Button(window, text='OK',
                              command=lambda: ok_click(window))
    popup_ok_btn.grid(column=0, row=1, sticky=N)
    window.title("Message")
    return

def dl_images(root, username, file_list, path, screen_w, screen_h):
    """Downloads images

    Calls download function and makes popup window.

    Args:
        root (Tk): main window
        username (str): username
        file_list (list): file list for download
        path (str): download path
        screen_w (int): screen width
        screen_h (int): screen height

    Returns:
        success_msg (str): success message
    """
    from client import download_images
    _, status = download_images(username, file_list, path)

    success_msg = ''
    if all([True if i == 200 else False for i in status["code"]]):
        success_msg = "All images downloaded successfully"
    else:
        success_msg = "One or more requested images not\ndownloaded"
    download_popup = Toplevel(root)
    popup_window(download_popup, success_msg, screen_w, screen_h)
    return success_msg

def display_images(run, user, img_type, img_paths, proc_steps, root,
                   filenames, orig_images, success):
    """Display images and histograms in new GUI window

    Converts image arrays to TK objects and displays them in the window.
    Also computes histogram

    Args:
        run (bool): run function or close window
        user (str): username
        img_type (str): image type to show/download
        img_paths (list): list of image paths
        proc_steps (list): list of processing steps
        root (Tk window)
        filenames (list): list of filenames
        orig_images (list): list of uploaded b64 strings
        success (list): whether image extraction was successful

    Returns:
        none
    """
    from image import b64_to_image
    from client import download_images
    global index
    index = 0
    i_row = 0
    if run:
        img_window = Toplevel(root)
        screen_w = root.winfo_screenwidth()
        screen_h = root.winfo_screenheight()
        img_window.config(height=0.75*screen_h, width=0.4*screen_w)
        img_window.title("Image Viewer")
        img_width = round(0.35*screen_h)
        hist_width = round(0.35*screen_h)

        # Initialize Tk labels and frames
        # Filename
        global f_text
        f_text = StringVar()
        f_text.set("Processing ...")
        filename_label = ttk.Label(img_window,
                                   textvariable=f_text)
        filename_label.grid(column=0, row=i_row, columnspan=4,
                            sticky=N)
        # Timestamp
        i_row += 1
        global ts_text
        ts_text = StringVar()
        ts_text.set("Uploaded at ...")
        ts_label = ttk.Label(img_window, textvariable=ts_text)
        ts_label.grid(column=0, row=i_row, columnspan=4, sticky=N)
        ts = []

        # Processing time
        i_row += 1
        global pt_text
        pt_text = StringVar()
        pt_text.set("Processing time ...")
        pt_label = ttk.Label(img_window, textvariable=pt_text)
        pt_label.grid(column=0, row=i_row, columnspan=4, sticky=N)
        pt = []

        # Image size
        i_row += 1
        global sz_text
        sz_text = StringVar()
        sz_text.set("Image size ...")
        sz_label = ttk.Label(img_window, textvariable=sz_text)
        sz_label.grid(column=0, row=i_row, columnspan=4, sticky=N)
        sz = []

        # Image display
        i_row += 1
        orig_label = ttk.Label(img_window, text='Original Image')
        orig_label.grid(column=0, row=i_row, columnspan=2,
                        sticky=N)
        new_label = ttk.Label(img_window, text='Processed Image')
        new_label.grid(column=2, row=i_row, columnspan=2,
                       sticky=N)

        i_row += 1
        orig_img_frame = ttk.Frame(img_window, width=img_width,
                                   height=img_width)
        orig_img_frame.grid(column=0, row=i_row, columnspan=2)
        proc_img_frame = ttk.Frame(img_window, width=img_width,
                                   height=img_width)
        proc_img_frame.grid(column=2, row=i_row, columnspan=2)

        i_row += 1
        orig_hist_label = ttk.Label(img_window,
                                    text='Original Image Histogram')
        orig_hist_label.grid(column=0, row=i_row, columnspan=2,
                             sticky=N)
        new_hist_label = ttk.Label(img_window,
                                   text='Processed Image Histogram')
        new_hist_label.grid(column=2, row=i_row, columnspan=2,
                            sticky=N)

        i_row += 1
        orig_hist_frame = ttk.Frame(img_window, width=hist_width,
                                    height=hist_width)
        orig_hist_frame.grid(column=0, row=i_row, columnspan=2)
        proc_hist_frame = ttk.Frame(img_window, width=hist_width,
                                    height=hist_width)
        proc_hist_frame.grid(column=2, row=i_row, columnspan=2)

        # Display
        tk_images = []
        proc_images = []
        orig_img_label = []
        proc_img_label = []
        orig_hist_plots = []
        proc_hist_plots = []
        new_w = img_width

        # Display original and processed images
        for i in range(len(orig_images)):
            image_to_load = []
            proc_img_to_load = []
            # If the image was successfully extracted
            if success[i]:
                image_string = orig_images[i]
                image_obj = b64_to_image(image_string)
                # Load original image
                try:
                    image_to_load = np.asarray(image_obj)
                    img_to_show = Image.fromarray(image_to_load)
                    w = img_to_show.width
                    h = img_to_show.height
                    final_w, final_h = resize_img_dim(w, h, new_w)
                    img_to_show = img_to_show.resize((final_w,
                                                      final_h),
                                                     Image.ANTIALIAS)
                    tk_images.append(ImageTk.PhotoImage(img_to_show))
                    orig_img_label.append(Label(orig_img_frame,
                                                image=tk_images[-1]))
                    orig_img_label[-1].img = tk_images[-1]
                except:
                    tk_images.append('')
                    orig_img_label.append(Label(orig_img_frame,
                                                text='Image not found'))
                # Load processed image
                try:
                    file_list = [[filenames[i], img_type, proc_steps]]
                    img_info, status = download_images(user,
                                                       file_list,
                                                       'none')
                    img_str = img_info[0]["image"]
                    curr_ts = img_info[0]["timestamp"]
                    curr_pt = img_info[0]["proc_time"]
                    curr_sz = img_info[0]["size"]
                    if isinstance(status, dict):
                        if any([True if i != 200 else False for i in
                                status["code"]]):
                            raise LookupError()
                    else:  # status is an int
                        if status != 200:
                            raise LookupError()
                    proc_img_obj = b64_to_image(img_str)
                    proc_img_to_load = np.asarray(proc_img_obj)
                    img_to_show2 = Image.fromarray(proc_img_to_load)
                    w2 = img_to_show2.width
                    h2 = img_to_show2.height
                    final_w2, final_h2 = resize_img_dim(w2, h2, new_w)
                    img_to_show2 = img_to_show2.resize((final_w,
                                                        final_h),
                                                       Image.ANTIALIAS)
                    proc_images.append(ImageTk.PhotoImage(img_to_show2))
                    proc_img_label.append(Label(proc_img_frame,
                                                image=proc_images[-1]))
                    proc_img_label[-1].img = proc_images[-1]
                    ts.append(curr_ts)
                    pt.append(curr_pt)
                    sz.append(curr_sz)
                except:
                    proc_images.append('')
                    proc_img_label.append(Label
                                          (proc_img_frame,
                                           text='Image not processed'))
                    ts.append('Uploaded at ...')
                    pt.append('Processing time ...')
                    sz.append('Image size ...')

            # Image not successfully extracted
            else:
                tk_images.append('')
                orig_img_label.append(Label(orig_img_frame,
                                            text='Image not found'))
                proc_images.append('')
                proc_img_label.append(Label(proc_img_frame,
                                            text='Image not processed'))
                ts.append('Uploaded at ...')
                pt.append('Processing time ...')
                sz.append('Image size ...')
            # Compute histograms
            orig_hist_plot = plot_histograms(orig_hist_frame,
                                             image_to_load)
            orig_hist_plots.append(orig_hist_plot)
            proc_hist_plot = plot_histograms(proc_hist_frame,
                                             proc_img_to_load)
            proc_hist_plots.append(proc_hist_plot)

        # Display first original image in slideshow
        if tk_images[index] == '':
            f_text.set("Invalid image".format(filenames[index]))
            orig_img_label[index].grid(column=0, row=2,
                                       ipadx=0.35*img_width,
                                       ipady=0.45*img_width,
                                       sticky=(N, S, E, W))
            orig_hist_plots[index].grid(column=0, row=2,
                                        ipadx=0.35*img_width,
                                        ipady=0.45*img_width,
                                        sticky=(N, S, E, W))
        else:
            f_text.set("Filename: {}".format(filenames[index]))
            ts_text.set("Uploaded on {}".format(ts[index]))
            pt_text.set("Processing time: {:.4f}s".format(pt[index]))
            sz_text.set("Image size: {}x{} pixels".format(sz[index][0],
                                                          sz[index][1]))
            filename_label.update()
            ts_label.update()
            pt_label.update()
            sz_label.update()
            orig_img_label[index].grid(column=0, row=2, columnspan=2,
                                       sticky=(N, S, E, W))
            orig_hist_plots[index].grid(column=0, row=2, columnspan=2,
                                        sticky=N)
        # Display first processed image in slideshow
        if proc_images[index] == '':
            proc_img_label[index].grid(column=0, row=2,
                                       ipadx=0.35*img_width,
                                       sticky=(N, S, E, W))
            proc_hist_plots[index].grid(column=0, row=2,
                                        ipadx=0.35*img_width,
                                        sticky=(N, S, E, W))
        else:
            proc_img_label[index].grid(column=0, row=2, columnspan=2,
                                       sticky=(N, S, E, W))
            proc_hist_plots[index].grid(column=0, row=2, columnspan=2,
                                        sticky=N)
        img_window.grid_rowconfigure("all", weight=1)
        img_window.grid_columnconfigure("all", weight=1)

        i_row += 1
        next_btn = ttk.Button(img_window, text='>>',
                              command=lambda: show_next('next',
                                                        tk_images,
                                                        filenames,
                                                        filename_label,
                                                        ts, ts_label,
                                                        pt, pt_label,
                                                        sz, sz_label,
                                                        orig_img_label,
                                                        proc_img_label,
                                                        img_width,
                                                        orig_hist_plots,
                                                        proc_hist_plots,
                                                        hist_width),
                              width=4)
        next_btn.grid(column=2, row=i_row, sticky=E)
        prev_btn = ttk.Button(img_window, text='<<',
                              command=lambda: show_next('prev',
                                                        tk_images,
                                                        filenames,
                                                        filename_label,
                                                        ts, ts_label,
                                                        pt, pt_label,
                                                        sz, sz_label,
                                                        orig_img_label,
                                                        proc_img_label,
                                                        img_width,
                                                        orig_hist_plots,
                                                        proc_hist_plots,
                                                        hist_width),
                              width=4)
        prev_btn.grid(column=1, row=i_row, sticky=W)

    else:
        for widget in root.winfo_children():
            if isinstance(widget, Toplevel):
                widget.destroy()

def resize_img_dim(w, h, new_w):
    """Gets dimensions for resizing input image

    Args:
        w (int): current width of image
        h (int): current height of image
        new_w (int): new width of image

    Returns:
        final_w (int): new width
        final_h (int): new height
    """
    new_h = round(h*new_w/w)
    if new_h <= new_w:
        final_w = new_w
        final_h = new_h
    else:
        final_w = round(new_w*new_w/new_h)
        final_h = new_w

    return final_w, final_h

def show_next(next, images, filenames, filename_label, ts, ts_label,
              pt, pt_label, sz, sz_label, orig_img_label,
              proc_img_label, img_width, orig_hist_plots,
              proc_hist_plots, hist_width):
    """Image slideshow functionality

    Shows images and histograms for next or previous file on button
    click.

    Args:
        next (str): 'next' or 'prev', depending on the button being
        pressed
        images (list): list of original Tk PhotoImages (empty string
        if image could not be uploaded
        filenames (list): list of image filenames
        filename_label (tk Label): label showing filename
        ts (list): list of timestamps
        ts_label (tk Label): label showing upload timestamp
        pt (list): list of processing times
        pt_label (tk Label): label showing processing times
        sz (list): list of image sizes
        sz_label (tk Label): label showing image size
        orig_img_label (tk Label): label showing original image
        proc_img_label (tk Label): label showing processed image
        img_width (int): width of image
        orig_hist_plots (tk Label): label showing histograms for
        original image
        proc_hist_plots (tk Label): label showing histograms for
        processed image
        hist_width (int): width of histogram

    Returns:
        none
    """
    global index
    if next == 'next':
        display_next = index < (len(orig_img_label))-1
    elif next == 'prev':
        display_next = index > 0
    if display_next:
        orig_img_label[index].grid_forget()
        orig_hist_plots[index].grid_forget()
        proc_img_label[index].grid_forget()
        proc_hist_plots[index].grid_forget()
        if next == 'next':
            index += 1
        elif next == 'prev':
            index -= 1
        next_olabel = orig_img_label[index]
        next_plabel = proc_img_label[index]
        next_ohist = orig_hist_plots[index]
        next_phist = proc_hist_plots[index]
        global f_text
        global ts_text
        global pt_text
        global sz_text
        if images[index] != '':
            f_text.set("Filename: {}".format(filenames[index]))
            ts_text.set("Uploaded on {}".format(ts[index]))
            ts_label.update()
            pt_text.set("Processing time: {:.4f}s".format(pt[index]))
            pt_label.update()
            sz_text.set("Image size: {}x{} pixels".
                        format(sz[index][0],
                               sz[index][1]))
            sz_label.update()
            next_olabel.grid(column=0, row=2, columnspan=2, sticky=E)
            next_ohist.grid(column=0, row=2, columnspan=2, sticky=E)
            next_plabel.grid(column=0, row=2, columnspan=2, sticky=W)
            next_phist.grid(column=0, row=2, columnspan=2, sticky=W)
        else:
            f_text.set("Invalid image")
            filename_label.update()
            filename_label.update()
            ts_text.set(ts[index])
            ts_label.update()
            pt_text.set(pt[index])
            pt_label.update()
            sz_text.set(sz[index])
            sz_label.update()
            next_olabel.grid(column=0, row=2, columnspan=2,
                             ipadx=0.35*img_width,
                             ipady=0.45*img_width, sticky=E)
            next_ohist.grid(column=0, row=2, columnspan=2,
                            ipadx=0.35*img_width,
                            ipady=0.45*img_width, sticky=E)
            next_plabel.grid(column=0, row=2, columnspan=2,
                             ipadx=0.35*img_width,
                             ipady=0.45*img_width, sticky=W)
            next_phist.grid(column=0, row=2, ipadx=0.35*img_width,
                            ipady=0.45*img_width, columnspan=2,
                            sticky=W)
    return

def plot_histograms(frame, img_array):
    """Creates histogram widget and plots it.

    Converts image arrays to histograms for display

    Args:
        frame (Tk Frame): frame to put histograms of images
        img_array (np array): np array containing image data

    Returns:
        hist_plots (Widget): histogram widget to implement
    """
    from matplotlib import pyplot as plt
    hist_plots = []
    image_exists = []
    img_label = []

    try:
        img_to_show = Image.fromarray(img_array)
        r, g, b = calc_histograms(img_to_show)

        frame.update()
        w = frame.winfo_width()
        h = frame.winfo_height()

        plt.figure()
        hist_fig, (ax1, ax2, ax3) = plt.subplots(3, 1, sharex=True,
                                                 sharey=False)
        hist_fig.set_tight_layout(True)
        hist_canvas = FigureCanvasTkAgg(hist_fig, master=frame)
        hist_plot = hist_canvas.get_tk_widget()
        # hist_canvas._tkcanvas.bind("<Configure>", hist_dim)
        hist_plot["width"] = 0.98*w
        hist_plot["height"] = 0.98*h
        ax1.set_title("Histograms", fontsize=10)
        ax1.plot(r, 'r')
        ax2.plot(g, 'g')
        ax3.plot(b, 'b')
        plt.xlabel('Intensity')
        ax1.set_ylabel("# red\npixels", fontsize=10)
        ax1.ticklabel_format(axis='y', style='sci', scilimits=(0, 4))
        ax2.set_ylabel("# green\npixels", fontsize=10)
        ax2.ticklabel_format(axis='y', style='sci', scilimits=(0, 4))
        ax3.set_ylabel("# blue\npixels", fontsize=10)
        ax3.ticklabel_format(axis='y', style='sci', scilimits=(0, 4))
        plt.xlabel("Intensity")
        plt.subplots_adjust(wspace=0, hspace=0)
    except:
        hist_plot = Label(frame, text='Histogram not calculated')
    return hist_plot

def calc_histograms(img_to_show):
    """Calculate histograms

    Get R, G, B histogram values from an image.

    Args:
        img_array (image obj): image object to use
    Returns:
        r (list): list of histogram values for red pixels
        g (list): list of histogram values for green pixels
        b (list): list of histogram values for blue pixels
    """
    out_hist = img_to_show.histogram()
    r = out_hist[0:256]
    g = out_hist[256:512]
    b = out_hist[512:768]
    return r, g, b

if __name__ == "__main__":
    editing_window()

ext: py

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""
beta\_nmf_minibatch.py
~~~~~~~~~~~

.. topic:: Contents

  The beta_nmf_minibatch module includes the betaNMF class,
  fit function and theano functions to compute updates and cost.

  Copyright 2014-2016 Romain Serizel
  This software is distributed under the terms of the GNU Public License
  version 3 (http://www.gnu.org/licenses/gpl.txt)"""

import time
import numpy as np
import theano
import base
import theano.tensor as T
import updates
import costs

class BetaNMF(object):
    """BetaNMF class

    Performs nonnegative matrix factorization with mini-batch multiplicative
    updates. GPGPU implementation based on Theano.

    Parameters
    ----------
    data_shape : tuple composed of integers
        The shape of the data to approximate

    n_components : positive integer
        The number of latent components for the NMF model

    beta: arbitrary float (default 2).
        The beta-divergence to consider. Particular cases of interest are
         * beta=2 : Euclidean distance
         * beta=1 : Kullback Leibler
         * beta=0 : Itakura-Saito

    n_iter: positive integer
         number of iterations

    fixed_factors: array of intergers
        Indexes of the factors that are kept fixed during the updates
        * [0] : corresponds to fixed H
        * [1] : corresponds to fixed W

    cache1_size: integer
        Size (in frames) of the first data cache.
        The size is reduced to the closest multiple of the batch_size.
        If set to zero the algorithm tries to fit all the data in cache

    batch_size: integer
        Size (in frames) of the batch for batch processing.
        The batch size has an impact on the parrelization and the memory needed
        to store partial gradients (see Schmidt et al.)

    verbose: integer
        The numer of iterations to wait between two computation and printing
        of the cost

    init_mode : string (default 'random')
        * random : initalise the factors randomly
        * custom : intialise the factors with custom value

    W : array (optionnal)
        Initial wvalue for factor W when custom initialisation is used

    H : array (optionnal)
        Initial wvalue for factor H when custom initialisation is used

    solver : string (default 'mu_batch')
        * mu_batch : mini-batch version of the MU updates.
         (fully equivalent to standard NMF with MU).
        * asg_mu : Asymetric stochatistic gradient for MU [1]_
        * gsg_mu : Greedy stochatistic gradient for  MU [1]_
        * asag_mu : Asymetric stochatistic average gradient [2]_ for MU [1]_
        * gsag_mu : Greedy stochatistic average gradient [2]_ for MU [1]_

    nb_batch_w : interger (default 1)
        number of batches on which W updates is computed
        * 1 : greedy approaches [1]_

    sag_memory : integer (default 0)
        number of batches used to compute the average gradient
        * 0 : SG approaches
        * nb_batches : SAG approaches

    Attributes
    ----------
    nb_cache1 : integer
        number of caches needed to fill the full data

    forget_factor : float
        forgetting factor for SAG

    scores : array
        reconstruction cost and iteration time for each iteration

    factors\_ : list of arrays
        The estimated factors

    w : theano tensor
        factor W

    h_cache1 : theano tensor
        part of the factor H in cache1

    x_cache1 : theano tensor
        data cache

    References
    ----------
    .. [#] R. Serizel, S. Essid, and G. Richard. “Mini-batch stochastic
        approaches for accelerated multiplicative updates in nonnegative matrix
        factorisation with beta-divergence”. Accepted for publication
        In *Proc. of MLSP*, p. 5, 2016.

    .. [#] Schmidt, M., Roux, N. L., & Bach, F. (2013).
        Minimizing finite sums with the stochastic average gradient
        https://hal.inria.fr/hal-00860051/PDF/sag_journal.pdf
    """

    # Constructor
    def __init__(self, data_shape, n_components=50, beta=2, n_iter=50,
                 fixed_factors=None, cache1_size=0,
                 batch_size=100, verbose=0,
                 init_mode='random', W=None, H=None, solver='mu_batch',
                 nb_batch_w=1, sag_memory=0):
        self.data_shape = data_shape
        self.n_components = n_components
        self.batch_size = batch_size
        self.nb_batch = int(np.ceil(np.true_divide(data_shape[0],
                                                   self.batch_size)))
        self.batch_ind = np.zeros((self.nb_batch, self.batch_size))

        if cache1_size > 0:
            cache1_size = min((cache1_size, data_shape[0]))
            if cache1_size < self.batch_size:
                raise ValueError('cache1_size should be at '
                                 'least equal to batch_size')
            self.cache1_size = int(np.ceil(cache1_size/self.batch_size * self.batch_size))
            self.nb_cache1 = int(np.ceil(np.true_divide(self.data_shape[0],
                                                        self.cache1_size)))
        else:
            self.cache1_size = data_shape[0]
            self.nb_cache1 = 1

        self.n_components = np.asarray(n_components, dtype='int32')
        self.beta = theano.shared(np.asarray(beta, theano.config.floatX),
                                  name="beta")
        self.eps = theano.shared(np.asarray(1e-10, theano.config.floatX),
                                  name="eps")
        self.sag_memory = sag_memory
        self.forget_factor = 1./(self.sag_memory + 1)
        self.verbose = verbose
        self.n_iter = n_iter
        self.solver = solver
        self.scores = []
        self.nb_batch_w = nb_batch_w
        if fixed_factors is None:
            fixed_factors = []
        self.fixed_factors = fixed_factors
        fact_ = [base.nnrandn((dim, self.n_components)) for dim in data_shape]
        self.init_mode = init_mode
        if self.init_mode == 'custom':
            fact_[0] = H
            fact_[1] = W
        self.w = theano.shared(fact_[1].astype(theano.config.floatX),
                               name="W", borrow=True, allow_downcast=True)
        self.h_cache1 = theano.shared(fact_[0][:self.cache1_size,
                                               ].astype(theano.config.floatX),
                                      name="H cache1", borrow=True,
                                      allow_downcast=True)
        self.factors_ = fact_
        self.x_cache1 = theano.shared(np.zeros((self.cache1_size,
                                                data_shape[1])).astype(
            theano.config.floatX),
            name="X cache1")
        self.init()

    def check_shape(self):
        """Check that all the matrix have consistent shapes
        """
        batch_shape = self.x_cache1.get_value().shape
        dim = int(self.n_components)
        if self.w.get_value().shape != (self.data_shape[1], dim):
            print("Inconsistent data for W, expected {1}, found {0}".format(
                self.w.get_value().shape,
                (self.data_shape[1], dim)))
            raise SystemExit
        if self.factors_[0].shape != (self.data_shape[0], dim):
            print("Inconsistent shape for H, expected {1}, found {0}".format(
                self.factors_[0].shape,
                (self.data_shape[0], dim)))
            raise SystemExit
        if self.h_cache1.get_value().shape != (batch_shape[0], dim):
            print("Inconsistent shape for h_cache1, expected {1}, found {0}".format(
                self.h_cache1.get_value().shape,
                (batch_shape[0], dim)))
            raise SystemExit

    def fit(self, data, cyclic=False, warm_start=False):
        """Learns NMF model

        Parameters
        ----------
        data : ndarray with nonnegative entries
            The input array

        cyclic : Boolean (default False)
            pick the sample cyclically

        warm_start : Boolean (default False)
            start from previous values
        """
        self.data_shape = data.shape
        if (not warm_start) & (self.init_mode != 'custom'):
            print("cold start")
            self.set_factors(data, fixed_factors=self.fixed_factors)
        self.check_shape()
        self.prepare_batch(False)
        self.prepare_cache1(False)
        div_func = self.get_div_function()
        if self.verbose > 0:
            scores = np.zeros((
                int(np.floor(self.n_iter/self.verbose)) + 2, 2))
        else:
            scores = np.zeros((2, 2))
        if self.solver == 'asag_mu' or self.solver == 'gsag_mu':
            grad_func = self.get_gradient_mu_sag()
            update_func = self.get_updates()
        elif self.solver == 'asg_mu' or self.solver == 'gsg_mu':
            grad_func = self.get_gradient_mu_sg()
            update_func = self.get_updates()
        elif self.solver == 'mu_batch':
            grad_func = self.get_gradient_mu_batch()
            update_func = self.get_updates()
        tick = time.time()
        score = 0

        for cache_ind in range(self.nb_cache1):
            current_cache_ind = np.hstack(self.batch_ind[
                    self.cache1_ind[
                        cache_ind, self.cache1_ind[cache_ind] >= 0]])
            current_cache_ind = current_cache_ind[current_cache_ind >= 0]
            self.x_cache1.set_value(data[current_cache_ind, ].astype(
                    theano.config.floatX))
            self.h_cache1.set_value(self.factors_[0][
                    current_cache_ind, ].astype(theano.config.floatX))
            score += div_func['div_cache1']()
        score_ind = 0
        scores[0, ] = [score, time.time() - tick]

        self.prepare_batch(not cyclic)
        self.prepare_cache1(not cyclic)

        print('Intitial score = %.2f' % score)
        print('Fitting NMF model with %d iterations....' % self.n_iter)
        if self.nb_cache1 == 1:
            current_cache_ind = np.hstack(self.batch_ind[
                self.cache1_ind[
                    0, self.cache1_ind[0] >= 0]])
            current_cache_ind = current_cache_ind[current_cache_ind >= 0]
            self.x_cache1.set_value(data[current_cache_ind, ].astype(
                theano.config.floatX))
            self.h_cache1.set_value(self.factors_[0][
                current_cache_ind, ].astype(theano.config.floatX))
            if self.solver == 'sag':
                self.c1_grad_w.set_value(self.old_grad_w[self.cache1_ind[
                        0, self.cache1_ind[0] >= 0]].astype(
                    theano.config.floatX))
        # main loop
        for it in range(self.n_iter):
            tick = time.time()
            self.prepare_cache1(not cyclic)
            score = 0
            for cache_ind in range(self.nb_cache1):
                if self.nb_cache1 > 1:
                    current_cache_ind = np.hstack(self.batch_ind[
                        self.cache1_ind[
                            cache_ind, self.cache1_ind[cache_ind] >= 0]])
                    current_cache_ind = current_cache_ind[
                        current_cache_ind >= 0]
                    self.x_cache1.set_value(data[current_cache_ind, ].astype(
                        theano.config.floatX))

                    self.h_cache1.set_value(self.factors_[0][
                        current_cache_ind, ].astype(theano.config.floatX))
                if self.solver == 'sag':
                    self.c1_grad_w.set_value(
                        self.old_grad_w[
                            self.cache1_ind[
                                cache_ind,
                                self.cache1_ind[cache_ind] >= 0]].astype(
                            theano.config.floatX))
                for batch_i in range(self.cache1_ind[
                        cache_ind, self.cache1_ind[cache_ind] >= 0].shape[0]):
                    batch_ind = np.arange(batch_i * self.batch_size,
                                          (batch_i + 1) * self.batch_size)
                    batch_ind = batch_ind[
                        batch_ind < current_cache_ind.shape[0]]
                    batch_ind = np.asarray([batch_ind[0],
                                            batch_ind[-1] + 1]).astype(
                        theano.config.floatX)

                    if self.solver == 'mu_batch':
                        self.update_mu_batch_h(batch_ind,
                                               update_func, grad_func)
                    if self.solver == 'asag_mu' or self.solver == 'asg_mu':
                        self.update_mu_sag(batch_ind,
                                           update_func, grad_func)
                    if self.solver == 'gsag_mu' or self.solver == 'gsg_mu':
                        grad_func['grad_h'](batch_ind)
                        update_func['train_h'](batch_ind)
                        if batch_i == 0 and cache_ind == 0:
                            grad_func['grad_w'](batch_ind)
                if self.nb_cache1 > 1:
                    self.factors_[0][current_cache_ind, ] =\
                        self.h_cache1.get_value()
                else:
                    self.factors_[0] = self.h_cache1.get_value()
            if self.solver == 'mu_batch':
                self.update_mu_batch_w(update_func)
            elif self.solver == 'gsag_mu' or self.solver == 'gsg_mu':
                update_func['train_w']()
            if self.nb_cache1 > 1:
                for cache_ind in range(self.nb_cache1):
                    self.x_cache1.set_value(data[np.hstack(self.batch_ind[
                        self.cache1_ind[
                            cache_ind,
                            self.cache1_ind[cache_ind] >= 0]]), ].astype(
                        theano.config.floatX))
                    self.h_cache1.set_value(self.factors_[0][
                        np.hstack(self.batch_ind[
                            self.cache1_ind[
                                cache_ind, self.cache1_ind[cache_ind] >= 0]]),
                        ].astype(theano.config.floatX))
                    if (it+1) % self.verbose == 0:
                        score += div_func['div_cache1']()
            else:
                self.factors_[0] = self.h_cache1.get_value()
                if (it+1) % self.verbose == 0:
                    score = div_func['div_cache1']()
            if (it+1) % self.verbose == 0:
                score_ind += 1
                scores[score_ind, ] = [
                    score, time.time() - tick + scores[score_ind - 1, 1]]
                print('Iteration %d / %d, duration=%.1fms, cost=%f'
                       % (it + 1,
                          self.n_iter,
                          scores[score_ind, 1] * 1000,
                          scores[score_ind, 0]))
                tick = time.time()
        score_ind += 1
        scores[score_ind, ] = [
            score, time.time() - tick + scores[score_ind - 1, 1]]
        print('Iteration %d / %d, duration=%.1fms, cost=%f'
               % (it + 1,
                  self.n_iter,
                  scores[-1, 1] * 1000,
                  scores[-1, 0]))
        return scores

    def get_div_function(self):
        """ compile the theano-based divergence function"""
        div_cache1 = theano.function(inputs=[],
                                     outputs=costs.beta_div(self.x_cache1,
                                                            self.w.T,
                                                            self.h_cache1,
                                                            self.beta),
                                     name="div c1",
                                     allow_input_downcast=True, profile=False)
        return dict(
            div_cache1=div_cache1)

    def get_gradient_mu_sag(self):
        """compile the theano based gradient functions for mu_sag algorithms"""
        tbatch_ind = T.ivector('batch_ind')
        tind = T.iscalar('ind')
        grad_new = updates.gradient_w_mu(
                                    self.x_cache1[tbatch_ind[0]:tbatch_ind[1],
                                                  ],
                                    self.w,
                                    self.h_cache1[tbatch_ind[0]:tbatch_ind[1],
                                                  ],
                                    self.beta)
        up_grad_w = self.forget_factor * grad_new + (
            1 - self.forget_factor) * self.grad_w

        grad_w = theano.function(inputs=[tbatch_ind],
                                 outputs=[],
                                 updates={(self.grad_w, up_grad_w)},
                                 name="grad w",
                                 allow_input_downcast=True)
        grad_new = updates.gradient_h_mu(
                                    self.x_cache1[tbatch_ind[0]:tbatch_ind[1],
                                                  ],
                                    self.w,
                                    self.h_cache1[tbatch_ind[0]:tbatch_ind[1],
                                                  ],
                                    self.beta)

        grad_h = theano.function(inputs=[tbatch_ind],
                                 outputs=[],
                                 updates={(self.c1_grad_h, grad_new)},
                                 name="grad h",
                                 allow_input_downcast=True)
        return dict(
            grad_h=grad_h,
            grad_w=grad_w)

    def get_gradient_mu_sg(self):
        """compile the theano based gradient functions for mu_sg algorithms"""
        tbatch_ind = T.ivector('batch_ind')
        tind = T.iscalar('ind')
        grad_new = updates.gradient_w_mu(
                                    self.x_cache1[tbatch_ind[0]:tbatch_ind[1],
                                                  ],
                                    self.w,
                                    self.h_cache1[tbatch_ind[0]:tbatch_ind[1],
                                                  ],
                                    self.beta)

        grad_w = theano.function(inputs=[tbatch_ind],
                                 outputs=[],
                                 updates={(self.grad_w, grad_new)},
                                 name="grad w",
                                 allow_input_downcast=True)
        grad_new = updates.gradient_h_mu(
                                    self.x_cache1[tbatch_ind[0]:tbatch_ind[1],
                                                  ],
                                    self.w,
                                    self.h_cache1[tbatch_ind[0]:tbatch_ind[1],
                                                  ],
                                    self.beta)

        grad_h = theano.function(inputs=[tbatch_ind],
                                 outputs=[],
                                 updates={(self.c1_grad_h, grad_new)},
                                 name="grad h",
                                 allow_input_downcast=True)
        return dict(
            grad_h=grad_h,
            grad_w=grad_w)

    def get_gradient_mu_batch(self):
        """compile the theano based gradient functions for mu"""
        tbatch_ind = T.ivector('batch_ind')
        tind = T.iscalar('ind')
        grad_new = updates.gradient_w_mu(
                                    self.x_cache1[tbatch_ind[0]:tbatch_ind[1],
                                                  ],
                                    self.w,
                                    self.h_cache1[tbatch_ind[0]:tbatch_ind[1],
                                                  ],
                                    self.beta)

        grad_w = theano.function(inputs=[tbatch_ind],
                                 outputs=[],
                                 updates={(self.grad_w,
                                           self.grad_w + grad_new)},
                                 name="grad w",
                                 allow_input_downcast=True,
                                 on_unused_input='ignore')
        grad_new = updates.gradient_h_mu(
                                    self.x_cache1[tbatch_ind[0]:tbatch_ind[1],
                                                  ],
                                    self.w,
                                    self.h_cache1[tbatch_ind[0]:tbatch_ind[1],
                                                  ],
                                    self.beta)
        grad_h = theano.function(inputs=[tbatch_ind],
                                 outputs=[],
                                 updates={(self.c1_grad_h, grad_new)},
                                 name="grad h",
                                 allow_input_downcast=True)
        return dict(
            grad_h=grad_h,
            grad_w=grad_w)

    def get_updates(self):
        """compile the theano based update functions"""
        tbatch_ind = T.ivector('batch_ind')
        tneg = T.iscalar('neg')
        tpos = T.iscalar('pos')
        up_h = T.set_subtensor(self.h_cache1[tbatch_ind[0]:tbatch_ind[1], ],
                               updates.mu_update(self.h_cache1[
                                    tbatch_ind[0]:tbatch_ind[1], ],
                                    self.c1_grad_h[0, ],
                                    self.c1_grad_h[1, ],
                                    self.eps))
        train_h = theano.function(inputs=[tbatch_ind],
                                  outputs=[],
                                  updates={(self.h_cache1, up_h)},
                                  name="trainH",
                                  allow_input_downcast=True,
                                  on_unused_input='ignore')
        update_w = updates.mu_update(self.w,
                                     self.grad_w[0],
                                     self.grad_w[1],
                                     self.eps)
        train_w = theano.function(inputs=[],
                                  outputs=[],
                                  updates={self.w: update_w},
                                  name="trainW",
                                  allow_input_downcast=True)
        return dict(
            train_h=train_h,
            train_w=train_w)

    def init(self):
        """Initialise theano variable to store the gradients"""
        self.grad_w = theano.shared(
            np.zeros((2,
                      self.data_shape[1],
                      self.n_components)).astype(theano.config.floatX),
            name="gradW", borrow=True,
            allow_downcast=True)
        self.grad_h = np.zeros((2, self.data_shape[0], self.n_components))
        self.c1_grad_h = theano.shared(
            np.zeros((2,
                      self.batch_size,
                      self.n_components)).astype(theano.config.floatX),
            name="c1_gradH", borrow=True,
            allow_downcast=True)

    def prepare_batch(self, randomize=True):
        """Arrange data for batches

        Parameters
        ----------
        randomize : boolean (default True)
            Randomise the data (time-wise) before preparing batch indexes
        """
        ind = - np.ones((self.nb_batch * self.batch_size, ))
        ind[:self.data_shape[0], ] = np.arange(self.data_shape[0])
        if randomize:
            np.random.shuffle(ind[:self.data_shape[0], ])
        self.batch_ind = np.reshape(ind, (self.nb_batch,
                                          self.batch_size)).astype(int)

    def prepare_cache1(self, randomize=True):
        """Arrange data for to fill cache1

        Parameters
        ----------
        randomize : boolean (default True)
            Randomise the data (time-wise) before preparing cahce indexes
        """
        ind = - np.ones((self.nb_cache1 *
                         int(np.ceil(np.true_divide(self.cache1_size,
                                                    self.batch_size)))))
        ind[:self.nb_batch, ] = np.arange(self.nb_batch)
        if randomize:
            np.random.shuffle(ind[:self.nb_batch, ])
        self.cache1_ind = np.reshape(ind, (self.nb_cache1,
                                           int(np.ceil(np.true_divide(
                                                self.cache1_size,
                                                self.batch_size)))
                                           )).astype(int)

    def set_factors(self, data, W=None, H=None, fixed_factors=None):
        """Re-set theano based parameters according to the object attributes.

        Parameters
        ----------
        W : array (optionnal)
            Value for factor W when custom initialisation is used

        H : array (optionnal)
            Value for factor H when custom initialisation is used

        fixed_factors : array  (default Null)
            list of factors that are not updated
                e.g. fixed_factors = [0] -> H is not updated

                fixed_factors = [1] -> W is not updated
        """
        self.data_shape = data.shape
        self.nb_batch = int(np.ceil(np.true_divide(self.data_shape[0],
                                                   self.batch_size)))
        self.batch_ind = np.zeros((self.nb_batch, self.batch_size))

        if self.cache1_size > 0 and self.cache1_size < self.data_shape[0]:
            if self.cache1_size < self.batch_size:
                raise ValueError('cache1_size should be at '
                                 'least equal to batch_size')
            self.cache1_size = self.cache1_size/self.batch_size * self.batch_size
            self.nb_cache1 = int(np.ceil(np.true_divide(self.data_shape[0],
                                                        self.cache1_size)))
        else:
            self.cache1_size = self.data_shape[0]
            self.nb_cache1 = 1

        self.forget_factor = 1./(self.sag_memory + 1)
        fact_ = [base.nnrandn((dim, self.n_components))
                 for dim in self.data_shape]
        if H is not None:
            fact_[0] = H
        if W is not None:
            fact_[1] = W
        if fixed_factors is None:
            fixed_factors = []
        if 1 not in fixed_factors:
            self.w = theano.shared(fact_[1].astype(theano.config.floatX),
                                   name="W", borrow=True, allow_downcast=True)
        if 0 not in fixed_factors:
            self.h_cache1 = theano.shared(
                fact_[0][
                    :self.cache1_size, ].astype(theano.config.floatX),
                name="H cache1", borrow=True,
                allow_downcast=True)
            self.factors_[0] = fact_[0]
        self.factors_ = fact_
        self.x_cache1 = theano.shared(np.zeros((self.cache1_size,
                                                self.data_shape[1])).astype(
            theano.config.floatX),
            name="X cache1")
        self.init()

    def transform(self, data, warm_start=False):
        """Project data X on the basis W

        Parameters
        ----------
        X : array
            The input data
        warm_start : Boolean (default False)
            start from previous values

        Returns
        -------
        H : array
            Activations
        """
        self.fixed_factors = [1]
        if not warm_start:
            print("cold start")
            self.set_factors(data, fixed_factors=self.fixed_factors)
        self.fit(data, warm_start=True)
        return self.factors_[0]

    def update_mu_sag(self, batch_ind, update_func, grad_func):
        """Update current batch with SAG based algorithms

        Parameters
        ----------
        batch_ind : array with 2 elements
            :batch_ind[0]: batch start
            :batch_ind[1]: batch end

        update_func : Theano compiled function
            Update function

        grad_func : Theano compiled function
            Gradient function
        """
        if 0 not in self.fixed_factors:
            grad_func['grad_h'](batch_ind)
            update_func['train_h'](batch_ind)
        if 1 not in self.fixed_factors:
            grad_func['grad_w'](batch_ind)
            update_func['train_w']()

    def update_mu_batch_h(self, batch_ind, update_func, grad_func):
        """Update h for current batch with standard MU

        Parameters
        ----------
        batch_ind : array with 2 elements
            :batch_ind[0]: batch start
            :batch_ind[1]: batch end

        update_func : Theano compiled function
            Update function

        grad_func : Theano compiled function
            Gradient function
        """
        if 0 not in self.fixed_factors:
            grad_func['grad_h'](batch_ind)
            update_func['train_h'](batch_ind)
            grad_func['grad_w'](batch_ind)

    def update_mu_batch_w(self, udpate_func):
        """Update W with standard MU

        Parameters
        ----------
        update_func : Theano compiled function
            Update function
        """
        if 1 not in self.fixed_factors:
            udpate_func['train_w']()
            self.grad_w.set_value(
                np.zeros((
                    2,
                    self.data_shape[1],
                    self.n_components)).astype(
                        theano.config.floatX))

max_stars_repo_name: dasmy/minibatchNMF

max_stars_repo_licenses: Naumen Condor-1.1 MS-PL

max_issues_repo_name: dasmy/minibatchNMF

max_issues_repo_licenses: Naumen Condor-1.1 MS-PL

max_forks_repo_name: dasmy/minibatchNMF

max_forks_repo_licenses: Naumen Condor-1.1 MS-PL

reason: import numpy,import theano


text:
import csv
import networkx as nx
import numpy as np
import scipy.stats as ss
from random import shuffle
import matplotlib.pyplot as plt
import pandas as pd
from itertools import combinations
###########################################################################
def can_nodes_recover(infection_type):
	r"""INoDS can handle the following infection model types = SI, SIR, SIS.
	recovery times imputed in SIR and SIS models
	"""

	infection_type = infection_type.upper()
	if infection_type not in ["SI", "SIR", "SIS"]:
		raise ValueError("INoDS works only for SI, SIR, SIS infection models. User-specified infection model is ill-defined")

	if infection_type =="SI": recovery_prob=False
	else: recovery_prob=True

	return recovery_prob

#################################################################################
def extract_maxtime(edge_filename, health_filename):
	r""" Return min of maximum time across edge_filename and health_filename
	"""
	df = pd.read_csv(edge_filename)

	df.columns = df.columns.str.lower()
	df.columns = [x.strip().replace('_', '') for x in df.columns]
	df2 = pd.read_csv(health_filename)
	df2.columns = df2.columns.str.lower()
	df2.columns = [x.strip().replace('_', '') for x in df2.columns]
	header= list(df)
	if "timestep" not in header:return max(df2['timestep'])
	else: return min(max(df['timestep']), max(df2['timestep']))

#################################################################################
def create_dynamic_network(edge_filename, complete_nodelist, edge_weights_to_binary, normalize_edge_weight, is_network_dynamic, time_max):

	df = pd.read_csv(edge_filename)
	df.columns = df.columns.str.lower()
	header = list(df)

	if [str1 for str1 in header[:2]] != ["node1", "node2"]:
		raise ValueError("The first two columns in network file should be arranged as named as 'node1', 'node2'")
	if "weight" not in header[2]:
		raise ValueError("The third column in network file should contain string = weight")
	if is_network_dynamic and (len(header)<4 or header[3] != "timestep"):
		raise ValueError("Time-stamps are either missing or the column is not labelled as 'timestep'! If network is static then set 'is_network_dynamic' as False")

	##rename the weight column as weight
	df.rename(columns={header[2]: 'weight'}, inplace=True)

	## remove all zero weighted edges
	df = df[df.weight!=0]

	if not is_network_dynamic:
		if "timestep" in header:
			raise ValueError("Network dynamic set as False but the infection data has timesteps!")
		n_edges = len(df.index)
		timelist = [[num]*n_edges for num in xrange(time_max+1)]
		timelist = [val for sublist in timelist for val in sublist]
		df=pd.concat([df]*(time_max+1), ignore_index=True)
		df['timestep']=timelist

	if edge_weights_to_binary:df['weight']=1

	if edge_weights_to_binary and normalize_edge_weight:
		raise ValueError("Cannot convert edge weights to binary AND normalize edge weights! Choose one")
	if not edge_weights_to_binary and normalize_edge_weight:
		## If the user asks for edge weight normalization, then calculate total edge weights
		## at each time step
		#print ("before converting"), min(df['weight']), max(df['weight'])
		max_edgewt = max(df['weight'])/1.
		df['weight'] = df["weight"]/max_edgewt
		#print ("max weight"), max_edgewt, min(df['weight']), max(df['weight'])

	G = {}
	for time1 in range(df["timestep"].min(), df["timestep"].max()+1):  G[time1] = nx.Graph()
	for time1 in G:
		if complete_nodelist is not None:
			complete_nodelist = [str(num) for num in complete_nodelist]
			G[time1].add_nodes_from(complete_nodelist)
		df_sub= df.loc[df['timestep'] == time1]
		df_sub['node1'] = df_sub['node1'].astype(str)
		df_sub['node2'] = df_sub['node2'].astype(str)
		edge_list = list(zip(df_sub.node1, df_sub.node2))
		G[time1].add_edges_from(edge_list)
		edge_attr = dict(zip(zip(df_sub.node1, df_sub.node2), df_sub.weight))
		nx.set_edge_attributes(G[time1],  edge_attr, 'weight')

	return G

##########################################################################
def extract_nodelist(H):

	nodelist = [H[time].nodes() for time in H]
	nodelist = [item for sublist in nodelist for item in sublist]
	nodelist = list(set(nodelist))

	return nodelist
#######################################################################
def check_edge_weights(G):
	""" Code convergence is better if the edge weights are normalized
	at each time step"""

	total_edge_wt = []
	for time1 in G.keys():

		total_wt = sum(nx.get_edge_attributes(G[time1],'weight').values())
		total_wt = int(round(total_wt,1))
		total_edge_wt.append(total_wt)

	if list(set(total_edge_wt))!= [1]:
		print ("Warning: Code convergence is better if the edge weights are normalized at each time step" )

#################################################################
def permute_network(G1, permutation):

	G2 = {}
	jaccard_list = []
	for time in G1.keys():
		G2[time] = nx.Graph()
		G2[time].add_nodes_from(G1[time].nodes())
		num_swaps = int(permutation*len(G1[time].edges()))
		num_orig = len(G1[time].edges()) - num_swaps
		orig_edges = G1[time].edges()
		shuffle(orig_edges)
		track_wt = []
		for num in xrange(num_orig):
			node1, node2 = orig_edges.pop()
			G2[time].add_edge(node1, node2)
			wt= G1[time][node1][node2]["weight"]
			G2[time][node1][node2]["weight"] = wt
			track_wt.append(wt)
		wtlist = [G1[time][node1][node2]["weight"] for node1, node2 in G1[time].edges()]
		#remove edge weights that have been assigned
		for wt in track_wt: wtlist.remove(wt)

		connections = 0 # skip over node pairs that already have an edge
		while connections <num_swaps:
			node1, node2 = np.random.choice(G2[time].nodes(), 2, replace=False)
			if not G2[time].has_edge(node1, node2):
					G2[time].add_edge(node1, node2)
					if len(wtlist)==0:print ("check!!!"), permutation, num_swaps, num_orig, len(G2[time].edges())
					G2[time][node1][node2]["weight"] = wtlist.pop()
					connections+=1

		jaccard_list.append(calculate_jaccard(G1[time], G2[time]))
	#print ("random graph check"), permutation, np.mean(jaccard_list)

	return G2

#############################################################################
def randomize_edges(g):
    """randomize a network using double-edged swaps.
Note: This is used to randomize only the within-edges. A separate algorithm
(randomize_graph_outedges) is used to randomize between-edges"""

    size = g.size() # number of edges in graph
    nx.double_edge_swap(g, nswap = 100*size, max_tries = 10000000*size)
#################################################################
"""
def randomize_network2(G1):

	G = {}
	for time1 in G1:
		wtlist = [G1[time1][node1][node2]["weight"] for node1, node2 in G1[time1].edges()]
		mean_wtlist = np.mean(wtlist)
		degree_list = [G1[time1].degree(node) for node in sorted(G1[time1].nodes())]
		shuffle(degree_list)
		##connect stubs using havel hakimi
		G[time1] = nx.havel_hakimi_graph(degree_list)
		#relabel nodes to match the node names
		mapping = {n1hat: n1 for (n1hat,n1) in zip(G[time1].nodes(), G1[time1].nodes())}
		G[time1] = nx.relabel_nodes(G[time1],mapping, copy=False)
		randomize_edges(G[time1])
		for (n1, n2) in G[time1].edges():
			G[time1][n1][n2]["weight"] = mean_wtlist

	jaccard = calculate_mean_temporal_jaccard(G1, G)
	return G, jaccard
"""
###################################################################
def randomize_network(G1, complete_nodelist, network_dynamic = True):

	r""" Randomize edge connections of each network slice.
	Also, set edge weight to mean.
	"""
	G2 = {}
	if network_dynamic:
		for time in G1.keys():
			G2[time] = nx.Graph()
			if complete_nodelist is not None:
				complete_nodelist = [str(num) for num in complete_nodelist]
				G2[time].add_nodes_from(complete_nodelist)
			else: G2[time].add_nodes_from(G1[time].nodes())
			edge_size = len(G1[time].edges())
			wtlist = [G1[time][node1][node2]["weight"] for node1, node2 in G1[time].edges()]
			mean_wtlist = np.mean(wtlist)

			for num in xrange(len(G1[time].edges())): #for each edge in G1[time]
				#select two random nodes from G2[time]
				condition_met = False # skip over node pairs that already have an edge
				counter=0
				while not condition_met:
					node1, node2 = np.random.choice(G2[time].nodes(), 2, replace=False)
					if not (G2[time].has_edge(node1, node2) or G1[time].has_edge(node1, node2)):
						condition_met = True
						G2[time].add_edge(node1, node2)
						G2[time][node1][node2]["weight"] = mean_wtlist

					else:counter+=1
					if counter > 4* edge_size and not (G2[time].has_edge(node1, node2)):
					##Give up after 2*#edges attempts
						condition_met=True
						G2[time].add_edge(node1, node2)
						G2[time][node1][node2]["weight"] = mean_wtlist

	else:
		init_time = min([time1 for time1 in G1])
		G2[init_time] = nx.Graph()
		if complete_nodelist is not None:
				complete_nodelist = [str(num) for num in complete_nodelist]
				G2[init_time].add_nodes_from(complete_nodelist)
		else: G2[init_time].add_nodes_from(G1[init_time].nodes())
		edge_size = len(G1[init_time].edges())
		wtlist = [G1[init_time][node1][node2]["weight"] for node1, node2 in G1[init_time].edges()]
		mean_wtlist = np.mean(wtlist)

		for num in xrange(len(G1[init_time].edges())): #for each edge in G1[time]
			#select two random nodes from G2[time]
			condition_met = False # skip over node pairs that already have an edge
			counter=0
			while not condition_met:
				node1, node2 = np.random.choice(G2[init_time].nodes(), 2, replace=False)
				if not (G2[init_time].has_edge(node1, node2) or G1[init_time].has_edge(node1, node2)):
					condition_met = True
					G2[init_time].add_edge(node1, node2)
					G2[init_time][node1][node2]["weight"] = mean_wtlist

				else:counter+=1
				if counter > 4* edge_size and not (G2[init_time].has_edge(node1, node2)):
				##Give up after 2*#edges attempts
					condition_met=True
					G2[init_time].add_edge(node1, node2)
					G2[init_time][node1][node2]["weight"] = mean_wtlist

		time_list = [time1 for time1 in G1 if time1 != init_time]
		for time1 in time_list:
			G2[time1] = G2[init_time].copy()

	jaccard = calculate_mean_temporal_jaccard(G1, G2)

	return G2, jaccard
#######################################################################
def stitch_health_data(health_data):
	""" Fill in time steps with same infection status"""
	for node in health_data.keys():
		timelist = health_data[node].keys()
		timelist = sorted(timelist)
		for num in range(1, len(timelist)):
			time2 = timelist[num]
			time1 = timelist[num-1]
			if health_data[node][time1]==health_data[node][time2]:
				for step in range(time1+1, time2): health_data[node][step] = health_data[node][time2]

	return health_data
#######################################################################
def extract_health_data(health_filename, infection_type, nodelist, time_max, diagnosis_lag=False):

	r"""node_health is a dictionary of dictionary. Primary key = node id.
	Secondary key = 0/1. 0 (1) key stores chunk of days when the node is **known** to be healthy (infected).
	 Dates stored as tuple of (start date, end date)"""

	health_data = {}
	for node in nodelist: health_data[str(node)]={}

	with open (health_filename, 'r') as csvfile:
		fileread = csv.reader(csvfile, delimiter = ',')
		next(fileread, None) #skip header
		for row in fileread:
			node = str(row[0])
			timestep = int(row[1])
			diagnosis = int(row[2])
			if node in nodelist and timestep<=time_max:health_data[node][timestep] = diagnosis

	if diagnosis_lag: health_data = stitch_health_data(health_data)

	node_health = {}
	for node in health_data.keys():
		sick_list_node=[]
		healthy_list_node=[]
		node_health[node] = {}
		## sort infection reported into infected (sick_list) and healthy (healthy_list) lists
		for time1 in health_data[node].keys():
			if health_data[node][time1]==1: sick_list_node.append(time1)
			if health_data[node][time1]==0: healthy_list_node.append(time1)

		if len(healthy_list_node)>0:
			healthy_list_node = sorted(healthy_list_node)
			node_health[node][0] = select_healthy_time(healthy_list_node, node, health_data, infection_type)

		if len(sick_list_node)>0:
			sick_list_node = sorted(sick_list_node)
			node_health[node][1]= select_sick_times(sick_list_node, node, health_data)

		if node_health[node].has_key(1):
			for time1, time2 in node_health[node][1]:
				##impute the missing report of sick in health data dictionary
				for day in range(time1, time2+1): health_data[node][day]=1

	return health_data, node_health
##############################################################################
def select_healthy_time(healthy_list_node, node, health_data, infection_type):

	r""" Select chunks of time-periods (from healthy_list_node) for which the node
	     is reported uninfected"""

	healthy_times = []

	#if SIR type of infection
	if infection_type[-1].lower()=='r':
		#min date = if there was no report before the day
		min_date = [time for time in healthy_list_node if len([val for key, val in health_data[node].items() if key<time])==0]

	else:
		#min date = if there is (any report before focal date AND the last report is sick) OR there is no report before the day
		min_date = [time for time in healthy_list_node if (len([val for key, val in health_data[node].items() if key<time])>0 and health_data[node][max([key for key in health_data[node] if key < time])]==1) or len([val for key, val in health_data[node].items() if key<time])==0]

	#max_date = if there is (any report after focal date AND the report is sick) OR there is no report after the day
	max_date = [time for time in healthy_list_node if (len([val for key, val in health_data[node].items() if key> time])>0 and health_data[node][min([key for key in health_data[node] if key > time])]==1) or len([val for key, val in health_data[node].items() if key>time])==0]

	min_date = sorted(min_date)
	max_date = sorted(max_date)

	for day1, day2 in zip(min_date, max_date): healthy_times.append((day1, day2))
	return healthy_times

############################################################################
def select_sick_times(sick_list_node, node, health_data):
	r""" Select chunks of time-periods (from sick_list_node) for which the node
	     is reported sick"""

	sick_times = []
	#min date = if there is (any report before the focal date AND the last report therin is healthy) OR there is no report before the day
	min_date = [time for time in sick_list_node if (len([val for key, val in health_data[node].items() if key<time])>0 and health_data[node][max([key for key in health_data[node] if key < time])]==0) or len([val for key, val in health_data[node].items() if key<time])==0]
	#max date = if there is (any report after the focal date AND the first report theirin is healthy) OR there is no report after the focal date
	max_date = [time for time in sick_list_node if (len([val for key, val in health_data[node].items() if key> time])>0 and health_data[node][min([key for key in health_data[node] if key > time])]==0) or len([val for key, val in health_data[node].items() if key>time])==0]

	min_date = sorted(min_date)
	max_date = sorted(max_date)

	for day1, day2 in zip(min_date, max_date): sick_times.append((day1, day2))
	return sick_times

#########################################################################
def return_contact_days_sick_nodes(node_health, seed_date, G_raw):
	r"""If infection diagnosis is lagged, then true infection day is
	inferred using infectious contact history of the focal node """

	contact_daylist={key:{} for key in G_raw}
	## select all nodes that were reported infected and sort
	for node in sorted([node1 for node1 in node_health.keys() if node_health[node1].has_key(1)]):
		## removing seed nodes
		sick_days = [(time1, time2) for (time1, time2) in sorted(node_health[node][1]) if time1!= seed_date]
		##for all time periods when the node was reported sick
		for time1, time2 in sick_days:
			#default day start
			day_start =1
			if node_health[node].has_key(0):
				##choose all uninfected time-periods before the focal sick period
				healthy_dates = [(healthy_day1, healthy_day2) for healthy_day1, healthy_day2 in node_health[node][0] if healthy_day2 < time1]
				if len(healthy_dates)>0:
					##choose the latest uninfected period
					lower_limit, upper_limit = max(healthy_dates, key=lambda x:x[1])
					##choose the last ever reported time-point of being uninfected
					day_start = upper_limit

			##contact_daylist is dictionary. Primary key = network type. Could be network hypothesis or null network
			## secondary key (node1, time1, time2) indicated focal node and its infected time period
			##values are the time-points when the node could have potentially contracted infection

			for network in G_raw:
				#choose only those days where nodes has contact the previous day

				contact_daylist[network][(node, time1, time2)] =[day for day in range(day_start+1, time1+1) if (day-1) in G_raw[network] and node in G_raw[network][day-1].nodes() and G_raw[network][day-1].degree(node)>0]
				##if there are no contacts then return the entire list
				if len(contact_daylist[network][(node, time1, time2)])==0:
					contact_daylist[network][(node, time1, time2)] = [day for day in range(day_start+1, time1+1)]

	return contact_daylist

#########################################################################
def return_potention_recovery_date(node_health, time_max):
	r""" For SIR/SIS model. Returns the potential time-points of recovery for each
	infected focal node"""

	recovery_daylist = {}
	## select all nodes that were reported infected and sort
	for node in sorted([node1 for node1 in node_health.keys() if node_health[node1].has_key(1)]):
		## sort sick days for the focal node
		sick_days = sorted(node_health[node][1])
		for time1, time2 in sick_days:
			if node_health[node].has_key(0):
				##choose all uninfected time-periods after the focal sick period
				healthy_dates = [(healthy_day1, healthy_day2) for healthy_day1, healthy_day2 in node_health[node][0] if healthy_day2 > time1]

				if len(healthy_dates)>0:
					##choose the first report of uninfection
					lower_limit, upper_limit = min(healthy_dates, key=lambda x:x[1])
					recovery_date = lower_limit
				else: recovery_date = time_max
			else: recovery_date = time_max

			##recovery date can be any time-point between the last report of node "infection" state to the the first
			## report of uninfection afterwards (or time_max of the study)
			recovery_daylist[(node, time1, time2)] = recovery_date

	return recovery_daylist
####################################################################################
def find_seed_date(node_health):
	r"""Find the first time-period where an infection was reported"""

	sick_dates = [val for node in node_health.keys() for key,val in node_health[node].items() if key==1]
	#flatten list
	sick_dates = [item for sublist in sick_dates for item in sublist]
	#pick the first date
	sick_dates=[num[0] for num in sick_dates]

	#sort tuple according to increasing infection dates
	sick_dates = sorted(sick_dates)
	# pick out the first reported infection date
	seed_date = sick_dates[0]

	return seed_date

########################################################################
def calculate_jaccard(g1, g2):

	edges1 = [tuple(sorted(num)) for num in g1.edges()]
	edges2 = [tuple(sorted(num)) for num in g2.edges()]
	w11 = len(list(set(edges1) & set(edges2)))
	w10 = len(list(set(edges1) - set(edges2)))
	w01 =  len(list(set(edges2) - set(edges1)))
	ratio = w11/ (1.*(w11+w10+w01))
	return ratio
########################################################################
def calculate_mean_temporal_jaccard(g1, g2):

	jlist = []
	for time1 in g1:
		edges1 = [tuple(sorted(num)) for num in g1[time1].edges()]
		edges2 = [tuple(sorted(num)) for num in g2[time1].edges()]

		w11 = len(list(set(edges1) & set(edges2)))
		w10 = len(list(set(edges1) - set(edges2)))
		w01 =  len(list(set(edges2) - set(edges1)))
		if (w11+w10+w01)>0:ratio = w11/ (1.*(w11+w10+w01))
		else:ratio=0
		jlist.append(ratio)
	return np.mean(jlist)
########################################################################
def check_init_pars(logl, logp, p0, data):

    ok = True
    p0 = p0.reshape((1, -1, p0.shape[-1]))
    if logp(p0) == -np.inf:
		print ("warning log prior false")
		ok = False

    p1 = p0[0::][0]
    loglist=[]
    for num in xrange(p1.shape[0]):
		loglist.append(logl(p1[num], data))

    if -np.inf in loglist:ok = False

    return ok
########################################################
def compute_diagnosis_lag_truth(graph, contact_datelist, filename):

	diag_date = {}
	infection_date={}
	lag_truths = []
	with open (filename, 'r') as csvfile:
		fileread = csv.reader(csvfile, delimiter = ',')
		next(fileread, None) #skip header
		for row in fileread:
			node = row[0]
			timestep = int(row[1])
			diagnosis = int(row[2])
			if diagnosis==1: infection_date[node] = timestep

	for node, time1, time2 in sorted(contact_datelist):
		daylist = [day for day in contact_datelist[(node, time1, time2)] if graph[day-1].degree(node)>0]
		pos = [pos for pos, date in enumerate(daylist) if date==infection_date[node]][0]
		lag_truths.append(ss.randint.cdf(pos,  0,  len(daylist)))

	return lag_truths

######################################################333
def plot_beta_results(sampler, filename):

        fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(15, 6))
        ax1.plot(sampler.chain[0, :, :, 0].T, color="k", lw=0.1)
        ax1.set_ylabel("Walker positions for $beta$")
        ax1.set_xlabel("Simulation step")

        samples = sampler.chain[0, :, :, 0].reshape((-1, 1))

        ax2.hist(samples, bins=50, histtype="step", normed=True, label="posterior", color="k", linewidth=2)
        ax2.legend(frameon=False, loc="best")
        ax2.set_xlabel("$beta$ posterior")

 	plt.tight_layout()
        plt.savefig(filename)

########################################################################

max_stars_repo_name: greenkidneybean/INoDS-model

max_issues_repo_name: greenkidneybean/INoDS-model

max_forks_repo_name: greenkidneybean/INoDS-model

reason: import numpy,import scipy,import networkx