import sqlite3
import os
import numpy.random as random

# Sets Database Name

database_file = "school.db"

# Tells the program if it should
# delete the database if it fails
# to set up.

database_remove_on_failure = True

# Checks if the file exists

setup = os.path.exists(database_file)

# Initialises the variable

con = None

# Sets debug mode to false
# will avoid showing any errors
# that might occur when running
# the program.

program_debug = False

try:
    # Connects to the database
    con = sqlite3.connect(database_file)
except Exception as error:
    if program_debug:
        print(error)

def setup_database(force=False):
    global con

    # Sets up the database by
    # adding the tables that will
    # be needed in the database
    # for the program to successfully
    # run.

    if force:
        # Force will remove the database
        # and make sure that the program
        # creates a clean version of the
        # database which might be handy
        # for the teacher or program.

        if os.path.exists(database_file):
            if con:
                con.close()
            os.remove(database_file)
        try:
            con = sqlite3.connect(database_file)
        except Exception as err:
            if program_debug:
                print(err)

    try:

        con.execute('''
            CREATE TABLE students(
                ID INTEGER PRIMARY KEY NOT NULL,
                first_name TEXT(30) NOT NULL,
                last_name TEXT(30) NOT NULL,
                class_name TEXT NOT NULL
            )
        ''')

        con.execute('''
            CREATE TABLE arithmetic_test(
                ID INTEGER PRIMARY KEY NOT NULL,
                student_ID INTEGER NOT NULL,
                score_1 REAL NOT NULL,
                score_2 REAL NOT NULL,
                score_3 REAL NOT NULL,
                average_score REAL NOT NULL
            )
        ''')

        # Saves the database

        con.commit()

    except Exception as err:
        if con:
            con.close()
        if database_remove_on_failure:
            os.remove(database_file)
        raise SystemError(f"Failed to setup database file with error: {err}")

if not setup:
    setup_database()

def get_test_id():
    # Sets Min
    largest_id = 100000
    try:
        # Gets ID from table
        for row in con.execute("SELECT ID FROM arithmetic_test"):
            if row[0] > largest_id:
                largest_id = row[0]  # Sets largest ids
        # Increments
        return largest_id + 1
    except Exception as err:
        print(f"Failed to get test id with error: {err}")

def get_student_id():
    # Sets Min
    largest_id = 100000
    try:
        # Gets ID from table
        for row in con.execute("SELECT ID FROM students"):
            if row[0] > largest_id:
                # Sets largest ids
                largest_id = row[0]
        # Increments
        return largest_id + 1
    except Exception as err:
        if program_debug:
            print(err)

def average_score(scores=None):
    if not scores:
        return 0
    else:
        length = len(scores)
        total = 0
        for score in scores:
            total += score
        average = total / length
        return average

def submit_arithmetic_results(sid=0, score_1=0, score_2=0, score_3=0):
    try:
        # Inserts the data into the table and commits it.
        con.execute('''
            INSERT INTO arithmetic_test(ID, student_ID, score_1, score_2, score_3, average_score)
            VALUES(?,?,?,?,?,?)
        ''', (get_test_id(), sid, score_1, score_2, score_3, average_score([score_1, score_2, score_3])))
        con.commit()
        return True
    except Exception as err:
        if program_debug:
            print(err)

        return False

def user_exists(sid=0):
    for row in con.execute(f"SELECT ID FROM students WHERE ID={sid}"):
        if row:
            return True
    return False

def get_score(calculation="", answer=0):
    # Holds the user in the loop
    while True:
        try:
            user_answer = float(input(calculation))
            answer = round(answer, 2)
            user_answer = round(user_answer, 2)
            user_answer = int(user_answer)
            answer = int(answer)

            if user_answer is not answer:
                print(f"\nIncorrect Answer. 0 Points awarded. The answer was {answer}")
                return 0
            else:
                print("\nCorrect Answer. 1 Point has been awarded.")
                return 1
        except Exception as err:
            if program_debug:
                print(err)

def test_user(min_num=0, max_num=10, questions=10, student_id=0):
    # This function uses numpy to be
    # able to generate random numbers
    # between the min_num and max_num
    # the program also uses a random
    # number from 0 to 4 which decides
    # the sign that will be used,
    # the program will check if the user
    # is correct it will give the user
    # a point if not it will not give
    # the user a point.

    scores = []
    count = 0
    # Holds the user in the loop
    while True:
        count += 1
        try:
            question_scores = []
            for question_num in range(0, questions):
                num_1 = random.randint(min_num, max_num)
                num_2 = random.randint(min_num, max_num)
                sign = random.randint(0, 4)
                answer = 0
                if sign == 0:
                    sign = "+"
                    answer = num_1 + num_2
                elif sign == 1:
                    sign = "-"
                    answer = num_1 - num_2
                elif sign == 2:
                    sign = "/"
                    answer = num_1 / num_2
                elif sign == 3:
                    sign = "*"
                    answer = num_1 * num_2

                question_scores.append(get_score(f"\n\n(2 Decimal Place): {str(num_1)} {sign} {str(num_2)} = ", answer))
                total = 0
                for s in question_scores:
                    total += s

                scores.append(total)

                if count == 3:
                    break
        except Exception as err:
            if program_debug:
                print(err)

        if count == 3:
            break

    success = submit_arithmetic_results(sid=student_id, score_1=scores[0], score_2=scores[1], score_3=scores[2])

    if success:
        print("\nAdded test scores.\n")
    else:
        print("\nFailed to add test scores.\n")

def create_student():
    # This function gets the user
    # to input information which
    # is validated, once the user
    # has typed out valid data into
    # the program it will save the data
    # into the database which can be
    # accessed at a later date.

    while True:
        try:

            while True:
                first_name = input("\nFirst Name: ")
                if 4 <= len(first_name) <= 30:
                    break
                else:
                    print("\nInvalid response.\n")
            while True:
                last_name = input("Last Name: ")
                if 4 <= len(last_name) <= 30:
                    break
                else:
                    print("\nInvalid response.\n")
            while True:
                class_name = input("Class Name: ")
                if 4 <= len(class_name) <= 30:
                    break
                else:
                    print("\nInvalid response.\n")

            try:
                sid = get_student_id()
                con.execute('''
                    INSERT INTO students(ID, first_name, last_name, class_name)
                    VALUES(?,?,?,?)
                ''', (sid, first_name, last_name, class_name))
                con.commit()
                print(f"\nSuccessfully signed up!\nStudent ID is {str(sid)}")
                break
            except Exception as err:
                if program_debug:
                    print(err)
                print("\nFailed to sign up, please try again.")
        except Exception as err:
            if program_debug:
                print(err)

def submit_grade():
    # This allows the staff
    # to be able to add the students
    # test scores into the system manually
    # though the teacher could allow the
    # student to be able to take the test instead
    # which could save the teachers time.

    # This will take input from the teacher
    # and assign the data in the database once
    # the data has been validated.

    while True:
        try:
            while True:
                try:
                    has_account = input("Does the student have a student ID? (Y/N) ")
                    if has_account.lower() == "n":
                        create_student()
                        break
                    elif has_account.lower() == "y":
                        break
                except Exception as err:
                    if program_debug:
                        print(err)
            student_id = 0

            while True:
                try:
                    student_id = input("Student ID: ")
                    if student_id.lower() == "create":
                        create_student()
                    else:
                        student_id = int(student_id)

                    if user_exists(student_id):
                        break
                    else:
                        print("\nFailed to find user. Type create if you don't have a student ID.")
                except Exception as err:
                    if program_debug:
                        print(err)

            while True:
                try:
                    score_1 = int(input("Score 1: "))
                    if 0 < score_1 < 30:
                        break
                except Exception as err:
                    if program_debug:
                        print(err)

            while True:
                try:
                    score_2 = int(input("Score 2: "))
                    if 0 < score_2 < 30:
                        break
                except Exception as err:
                    if program_debug:
                        print(err)

            while True:
                try:
                    score_3 = int(input("Score 3: "))
                    if 0 < score_2 < 30:
                        break
                except Exception as err:
                    if program_debug:
                        print(err)

                # Adds the information to
                # the database and checks
                # if the system was able
                # to add the data to the
                # database successfully

            success = submit_arithmetic_results(student_id, score_1, score_2, score_3)
            if success:
                print("\nAdded Entry.")

                more = input("\nDo you wish to add more? (Y/N) ")
                if more.lower() == "n":
                    break
            else:
                print("Failed to add Entry.")
        except Exception as err:
            if program_debug:
                print(err)

def wipe_database():
    # Checks if the user is sure
    # that they want to delete the
    # database and then proceeds
    # depending on what the user
    # wants to do.

    while True:
        confirmation = input("\nAre you sure that want to wipe the database? (Y/N) ")
        if confirmation.lower() == "y":
            setup_database(True)
            break
        elif confirmation.lower() == "n":
            break

def display_test_data():
    # This will display information about the user
    # The program will grab the data from the database
    # and will display it in a user-friendly manner.
    # Uses a loop in order to keep the user in the menu.

    while True:
        try:
            display_options = "\nDisplay Options\n"
            display_options = display_options + "\nOption 1: Alphabetically"
            display_options = display_options + "\nOption 2: Score (highest to lowest)"
            display_options = display_options + "\nOption 3: Average Score (highest to lowest)"
            display_options = display_options + "\nOption 4: Show all data"
            display_options = display_options + "\nOption 5: Exit Menu"
            print(display_options)
            display_choice = int(input("\n\nUser Choice: "))

            if display_choice == 1:

                # Selects data from the database
                # and orders it, this then
                # displays the data for the user
                # to view.

                for main_row in con.execute("SELECT class_name, first_name, last_name, MAX(score_1, score_2, score_3) "
                                            "as high_score FROM students, arithmetic_test ORDER BY class_name, "
                                            "first_name, last_name ASC"):
                    print(f"\n{main_row[0]}: {main_row[1]} {main_row[2]} Score: {main_row[3]}")
            elif display_choice == 2:

                # Selects data from the database
                # and orders it, this then
                # displays the data for the user
                # to view.

                for main_row in con.execute("SELECT class_name, first_name, last_name, MAX(score_1, score_2, score_3) "
                                            "as high_score FROM students, arithmetic_test ORDER BY high_score DESC"):
                    print(f"\n{main_row[0]}: {main_row[1]} {main_row[2]} Score: {main_row[3]}")
            elif display_choice == 3:

                # Selects data from the database
                # and orders it, this then
                # displays the data for the user
                # to view.

                for main_row in con.execute("SELECT class_name, first_name, last_name, average_score FROM students, "
                                            "arithmetic_test ORDER BY average_score DESC"):
                    print(f"\n{main_row[0]}: {main_row[1]} {main_row[2]} Score: {main_row[3]}")
            elif display_choice == 4:

                # Selects data from the database then
                # displays the data for the user
                # to view.

                for main_row in con.execute("SELECT * FROM students"):
                    print(f"\n{main_row[0]} {main_row[1]} {main_row[2]} {main_row[3]}")
                    for row in con.execute("SELECT ID, score_1, score_2, score_3, average_score FROM arithmetic_test"):
                        print(
                            f"Test {row[0]}:\nScore 1: {row[1]}\nScore 2: {row[2]}\n"
                            f"Score 3: {row[3]}\nAverage: {row[4]}")
            elif display_choice == 5:
                break

        except Exception as err:
            print(f"Display Error: {err}")

def main_menu():

    # Creates a loop that the user
    # will be stuck in, this will
    # allow the user to be able
    # to choose from the options
    # available to them.

    while True:
        try:
            menu_options = "\nStudent Grader\n"
            menu_options = menu_options + "\nOption 1: Submit Grade"
            menu_options = menu_options + "\nOption 2: Take Test"
            menu_options = menu_options + "\nOption 3: View Test Data"
            menu_options = menu_options + "\nOption 4: System Statistics"
            menu_options = menu_options + "\nOption 5: Wipe Database"
            menu_options = menu_options + "\nOption 6: Exit Program"
            # Show options
            print(menu_options)
            # Gets user input
            menu_choice = int(input("\n\nUser Choice: "))

            if menu_choice == 1:
                submit_grade()
            elif menu_choice == 2:
                while True:
                    try:
                        signed_up = input("Do you have a student ID? (Y/N) ")
                        if signed_up.lower() == "n":
                            # Signs the user up with a student ID
                            create_student()
                        if signed_up.lower() == "n" or "y":
                            # Requests the student ID
                            sid = int(input("Student ID: "))
                            # Checks if valid User
                            if user_exists(sid):
                                # Tests the User
                                test_user(student_id=sid)
                                break
                    except Exception as err:
                        if program_debug:
                            print(err)

            elif menu_choice == 4:
                display_test_data()
            elif menu_choice == 5:
                wipe_database()
            elif menu_choice == 6:
                break

        except Exception as err:
            if program_debug:
                print(err)

# Checks if the file is being
# run directly if so it will
# open the main menu.

if __name__ == "__main__":
    main_menu()

# Checks if the database
# connection exists if it
# does it will close the
# connection.

if con:
    con.close()


"""Computes principal component analysis.
"""

import pandas
import numpy as np
from sklearn import decomposition

from gen3va.database import database

def from_report(gene_signatures, category):
    data_frames = []
    for gene_signature in gene_signatures:

        # TODO: Fetch all gene signatures with a single DB query.
        genes = []
        values = []
        for rg in gene_signature.combined_genes:
            genes.append(rg.gene.name)
            values.append([rg.value])

        # In principle, there should never be duplicates in our gene lists,
        # but an earlier version of GEO2Enrichr accidentally did not average
        # duplicates. Thus, any extraction ID for which this is the case will
        # fail if this step is not performed.
        indices, data = average_duplicates(np.array(genes), np.array(values))
        new_df = pandas.DataFrame(
            index=indices,
            data=[x[0] for x in data]
        )
        data_frames.append(new_df)

    df = pandas.concat(data_frames, axis=1)
    df = df.fillna(0)
    pca_coords, variance_explained = compute_pca(df.T)

    unspecified_key = 'unspecified'
    series_obj = {}
    for i, (x, y, z) in enumerate(pca_coords):
        sig = gene_signatures[i]
        opt = sig.get_optional_metadata(category)
        if opt:
            if opt.value not in series_obj:
                series_obj[opt.value] = []
            series_obj[opt.value].append({
                'x': x,
                'y': y,
                'z': z,
                'name': sig.name
            })
        else:
            if unspecified_key not in series_obj:
                series_obj[unspecified_key] = []
            series_obj[unspecified_key].append({
                'x': x,
                'y': y,
                'z': z,
                'name': sig.name
            })

    series = []
    for name, data in series_obj.items():
        series.append({
            'name': name,
            'data': data
        })

    pca_obj = {'series': series}

    # This is common with `from_soft_file`. Abstract it?
    max_vals = np.max(pca_coords, axis=0)
    min_vals = np.min(pca_coords, axis=0)
    ranges = np.vstack((max_vals*1.1, min_vals*1.1))
    pca_obj['ranges'] = ranges.tolist()

    titles = ['PC%s (%.2f' %
              (i, pct) + '%' + ' variance captured)' for i, pct in enumerate(variance_explained, start=1)]
    pca_obj['titles'] = titles

    return pca_obj

def compute_pca(df, max_components=3):
    """Performs principal component analysis and returns the first three
    principal components.
    """
    mat = df.as_matrix()
    pca = decomposition.PCA(n_components=None)

    # fit(X) - fit the model with X
    pca.fit(mat)

    # explained_variance_ratio_ - % of variance explained by each of the
    # selected components. Take only the first 3 components.
    variance_explained = pca.explained_variance_ratio_[0:max_components]
    variance_explained *= 100

    # transform(X) - apply the dimensionality reduction on X
    pca_coords = pca.transform(mat)[:, 0:max_components]

    # return the coordinates of the transformed data, plus the % of variance
    # explainced by each component
    return pca_coords, variance_explained

def average_duplicates(genes, values):
    """Finds duplicate genes and averages their expression data.
    """
    # See http://codereview.stackexchange.com/a/82020/59381 for details.
    folded, indices, counts = np.unique(genes, return_inverse=True, return_counts=True)
    output = np.zeros((folded.shape[0], values.shape[1]))
    np.add.at(output, indices, values)
    output /= counts[:, np.newaxis]
    return folded, output


import os
import sys

from PyQt5.QtWidgets import QDialog, QGroupBox, QMainWindow, QFormLayout, QLabel, QVBoxLayout, QHBoxLayout, QPushButton, QProxyStyle, QStyle
from PyQt5.QtWidgets import QComboBox, QWidget, QApplication, QAction, qApp, QButtonGroup, QFileDialog, QMessageBox, QLineEdit, QSizePolicy, QScrollArea, QDateEdit
from PyQt5.QtCore import QSettings, Qt, QSize, QRect, QPoint, QDateTime, QTimer
from PyQt5.QtGui import QIcon, QFont, QPixmap

import main_application.constants.image_filepath_constants as image_constants

if getattr(sys, 'frozen', False):
    application_path = os.path.dirname(sys.executable)
else:
    application_path = os.path.dirname(__file__)

class HomePageWidget(QWidget):
    def __init__(self):
        super().__init__()
        self.init_ui()

    def init_ui(self):
        self.layout = QVBoxLayout()

        self.setup_heading()

        self.button_group = QGroupBox()
        self.layout.addWidget(self.button_group)
        self.v_box = QVBoxLayout()
        self.button_group.setLayout(self.v_box)

        self.setup_ivr_status_button()
        self.setup_generate_ivr_button()
        self.setup_add_recorded_voice_files_button()
        self.setup_analytics_button()
        self.setup_pstn_settings_button()
        self.setup_redirect_settings_button()

        self.v_box.addWidget(self.ivr_status_button)
        self.v_box.addWidget(QLabel())
        self.v_box.addWidget(self.generate_ivr_button)
        self.v_box.addWidget(QLabel())
        self.v_box.addWidget(self.add_recorded_voice_files_button)
        self.v_box.addWidget(QLabel())
        self.v_box.addWidget(self.stats_button)
        self.v_box.addWidget(QLabel())
        self.v_box.addWidget(self.pstn_settings_button)
        self.v_box.addWidget(QLabel())
        self.v_box.addWidget(self.redirect_settings_button)
        self.setLayout(self.layout)

    def setup_redirect_settings_button(self):
        self.redirect_settings_button = QPushButton('Redirect Settings', self)
        self.redirect_settings_button.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        redirect_settings_icon = QIcon(os.path.join(application_path, image_constants.REDIRECT_SETTINGS_ICON))
        self.redirect_settings_button.setIcon(redirect_settings_icon)
        self.redirect_settings_button.setIconSize(QSize(40, 40))
        self.redirect_settings_button.setToolTip(
            "Use This To: Update the numbers that users calling the IVR are redirected to")

    def setup_pstn_settings_button(self):
        self.pstn_settings_button = QPushButton('SIP Trunk Settings', self)
        self.pstn_settings_button.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        pstn_settings_icon = QIcon(os.path.join(application_path, image_constants.PSTN_SETTINGS_ICON))
        self.pstn_settings_button.setIcon(pstn_settings_icon)
        self.pstn_settings_button.setIconSize(QSize(40, 40))
        self.pstn_settings_button.setToolTip(
            "Use This To: Update your PSTN provider settings for your IVR (e.g. Twilio)")

    def setup_analytics_button(self):
        self.stats_button = QPushButton('Analytics', self)
        self.stats_button.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        analytics_icon = QIcon(os.path.join(application_path, image_constants.ANALYTICS_ICON))
        self.stats_button.setIcon(analytics_icon)
        self.stats_button.setIconSize(QSize(40, 40))
        self.stats_button.setToolTip("Use This To: See Analytics about your currently running IVR")

    def setup_add_recorded_voice_files_button(self):
        self.add_recorded_voice_files_button = QPushButton('Add Recorded Voice Files to IVR', self)
        self.add_recorded_voice_files_button.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        wav_icon = QIcon(os.path.join(application_path, image_constants.WAV_ICON))
        self.add_recorded_voice_files_button.setIcon(wav_icon)
        self.add_recorded_voice_files_button.setIconSize(QSize(40, 40))
        self.add_recorded_voice_files_button.setToolTip(
            "Use This To: Add your own recorded voice files to the IVR")

    def setup_generate_ivr_button(self):
        self.generate_ivr_button = QPushButton('Generate IVR', self)
        self.generate_ivr_button.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        ivr_icon = QIcon(os.path.join(application_path, image_constants.IVR_LOGO_ICON))
        self.generate_ivr_button.setIcon(ivr_icon)
        self.generate_ivr_button.setIconSize(QSize(40, 40))
        self.generate_ivr_button.setToolTip("Use This To: Turn a Voiceflow file or project into an IVR")

    def setup_ivr_status_button(self):
        self.ivr_status_button = QPushButton("IVR Server Status", self)
        self.ivr_status_button.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)
        ivr_status_icon = QIcon(os.path.join(application_path, image_constants.SERVER_STATUS_ICON))
        self.ivr_status_button.setIcon(ivr_status_icon)
        self.ivr_status_button.setIconSize(QSize(40, 40))
        self.ivr_status_button.setToolTip("Use This To: Turn the IVR server on or off and view the status")

    def setup_heading(self):
        header_layout = QHBoxLayout()

        nhs_logo = QLabel()
        pixmap = QPixmap(os.path.join(application_path, image_constants.NHS_LOGO))
        pixmap = pixmap.scaled(170, 170, Qt.KeepAspectRatio)
        nhs_logo.setPixmap(pixmap)

        heading_font = QFont('Arial', 16)
        heading_font.setBold(True)

        sub_heading_font = QFont('Arial', 11)
        sub_heading_font.setBold(True)

        sub_heading = QLabel("Create an automated call triage for patients\nI(nteractive) V(oice) R(esponse)")
        sub_heading.setFont(sub_heading_font)

        heading = QLabel("GP IVR")
        heading.setFont(heading_font)

        header_layout.addWidget(nhs_logo)

        heading_layout = QVBoxLayout()
        heading_layout.addWidget(heading)
        heading_layout.addWidget(sub_heading)

        header_layout.addLayout(heading_layout)

        visit_website = QLabel(
            "*For any queries on how to use the program, please refer to the user manual on our website")

        self.layout.addWidget(QLabel())
        self.layout.addLayout(header_layout)
        self.layout.addWidget(QLabel())
        self.layout.addWidget(visit_website)
        self.layout.addWidget(QLabel())


import time
import datetime
from dateutil.relativedelta import relativedelta

def  update_time(blood_cc):
    blood_cc = blood_cc[1:4]
    if blood_cc == '250':
        shift = 2
    else :
        shift = 3
    strptime_time = datetime.datetime.now()
    next_strptime = strptime_time + relativedelta(months=shift)
    next_unixtime = int(time.mktime(next_strptime.timetuple()))
    f = open(r'/opt/time.txt','w')
    f.write(blood_cc +','+str(next_unixtime))
    f.close()

    del next_unixtime , shift
    return '這次捐血時間為 : '+str(strptime_time)[0:10]+'\n捐血cc數為 : '+blood_cc+'\n下次捐血時間 : ' + str(next_strptime)[0:10]


import openml
import numpy as np
from app.neo4j.openml_model import Dataset, Task, Flow, Run, graph
from itertools import combinations
import math

print(graph)

def compare_dataset(d1, d2):
    d_1 = openml.datasets.get_dataset(d1)
    d_2 = openml.datasets.get_dataset(d2)
    mf_1 = d_1.qualities
    mf_2 = d_2.qualities
    norm = []
    mf = list(set(mf_1).intersection(mf_2))

    for f in mf:
        if not math.isnan(mf_1[f]) and not math.isnan(mf_2[f]):
            norm.append(mf_1[f] - mf_2[f])

    norm = np.array(norm, dtype='float64')
    return np.linalg.norm(norm)

def get_datasets(limit):
    datasets = openml.datasets.list_datasets(size=limit)
    return [dataset[1] for dataset in  list(datasets.items())]

def get_pairs(datasets):
    pairs = combinations(datasets, 2)
    return list(pairs)

def get_tasks(did):
    tasks = openml.tasks.list_tasks(data_id=did)
    return [task[1] for task in list(tasks.items())]

def get_flows(offset):
    flows = openml.flows.list_flows(offset=offset)
    return [flow[1] for flow in list(flows.items())]

def get_runs(tid):
    runs = openml.runs.list_runs(task=[tid])
    return runs

def populate(limit):

    datasets = get_datasets(limit)
    pairs = get_pairs(datasets)

    for pair in pairs:
        d1 = pair[0]
        d2 = pair[1]

        if Dataset(did=d1['did']).fetch() != None \
            and Dataset(did=d2['did']).fetch() != None:

            print(f"Datasets {d1['name']} and {d2['name']} already in database.")
            continue
        else:
            distance = compare_dataset(d1, d2)

            print(f"Distance between {d1['name']} and {d2['name']} is {distance}.")

            dataset_1 = Dataset(did=d1['did'],
                                name=d1['name'],
                                file_format=d1['format'])
            dataset_2 = Dataset(did=d2['did'],
                                name=d2['name'],
                                file_format=d2['format'])
            dataset_1.save()
            dataset_2.save()
            dataset_1.add_connections(dataset_2, distance)

def populate_datasets(limit):

    datasets = get_datasets(limit)

    for dataset in datasets:
        datasets = get_datasets(limit)
        current = list(Dataset().all)
        dataset_obj = Dataset(did=dataset['did']).fetch()
        if dataset_obj in current:
            connections = dataset_obj.get_connections()
            others = current[:]
            others.remove(dataset_obj)
            unconnected = list(set(others) - set(connections))
            if len(unconnected) > 1:
                for to_connect in unconnected:
                    print(f"Connecting {to_connect} to {dataset_obj}.")
                    distance = compare_dataset(dataset['did'], to_connect.did)
                    dataset_obj.add_connections(to_connect, distance)
                    dataset_obj.save()
            else:
                print(f"Dataset {dataset_obj} is fully connected.")
        else:
            new = Dataset(
                did=dataset['did'],
                name=dataset['name'],
                file_format=dataset['format'],
            )
            print(f"Created new {new}.")
            new.save()
            for to_connect in current:
                    print(f"Connecting {to_connect} to {new}.")
                    distance = compare_dataset(dataset['did'], to_connect.did)
                    new.add_connections(new, distance)
                    new.save()

def populate_tasks():

    datasets = list(Dataset().all)

    for dataset in datasets:
        current = dataset.get_tasks()
        print(f"On dataset {dataset}")
        new = get_tasks(dataset.did)
        for new_task in new:
            task_obj = Task(tid=new_task['tid']).fetch()
            if task_obj not in current:
                task_obj = Task(
                    tid=new_task['tid'],
                    task_type=new_task['task_type'],
                    task_type_id=new_task['ttid'],
                )
                for key, value in new_task.items():
                    if hasattr(task_obj, key):
                        setattr(task_obj, key, value)
                task_obj.save()
                print(f"Adding new task {task_obj}")
                dataset.add_task(task_obj)

# def populate_flows(offset):
#     flows = get_tasks(offset)
#     for flow in flows:
#         f = flows.get_flow(flow['flow_id'])
#         ti =
#     pass

populate_datasets(50)