import fnmatch
import logging
import os
import sys
import tarfile

from distutils.version import LooseVersion

import h5py
import numpy as np
import torch
import yaml

PRETRAINED_MODEL_LIST = {
    "ljspeech_parallel_wavegan.v1": "1PdZv37JhAQH6AwNh31QlqruqrvjTBq7U",
    "ljspeech_parallel_wavegan.v1.long": "1A9TsrD9fHxFviJVFjCk5W6lkzWXwhftv",
    "ljspeech_parallel_wavegan.v1.no_limit": "1CdWKSiKoFNPZyF1lo7Dsj6cPKmfLJe72",
    "ljspeech_parallel_wavegan.v3": "1-oZpwpWZMMolDYsCqeL12dFkXSBD9VBq",
    "ljspeech_full_band_melgan.v2": "1Kb7q5zBeQ30Wsnma0X23G08zvgDG5oen",
    "ljspeech_multi_band_melgan.v2": "1b70pJefKI8DhGYz4SxbEHpxm92tj1_qC",
    "jsut_parallel_wavegan.v1": "1qok91A6wuubuz4be-P9R2zKhNmQXG0VQ",
    "jsut_multi_band_melgan.v2": "1chTt-76q2p69WPpZ1t1tt8szcM96IKad",
    "csmsc_parallel_wavegan.v1": "1QTOAokhD5dtRnqlMPTXTW91-CG7jf74e",
    "csmsc_multi_band_melgan.v2": "1G6trTmt0Szq-jWv2QDhqglMdWqQxiXQT",
    "arctic_slt_parallel_wavegan.v1": "1_MXePg40-7DTjD0CDVzyduwQuW_O9aA1",
    "jnas_parallel_wavegan.v1": "1D2TgvO206ixdLI90IqG787V6ySoXLsV_",
    "vctk_parallel_wavegan.v1": "1bqEFLgAroDcgUy5ZFP4g2O2MwcwWLEca",
    "vctk_parallel_wavegan.v1.long": "1tO4-mFrZ3aVYotgg7M519oobYkD4O_0-",
    "vctk_multi_band_melgan.v2": "10PRQpHMFPE7RjF-MHYqvupK9S0xwBlJ_",
    "libritts_parallel_wavegan.v1": "1zHQl8kUYEuZ_i1qEFU6g2MEu99k3sHmR",
    "libritts_parallel_wavegan.v1.long": "1b9zyBYGCCaJu0TIus5GXoMF8M3YEbqOw",
    "libritts_multi_band_melgan.v2": "1kIDSBjrQvAsRewHPiFwBZ3FDelTWMp64",
}

def find_files(root_dir, query="*.wav", include_root_dir=True):
    """Find files recursively.

    Args:
        root_dir (str): Root root_dir to find.
        query (str): Query to find.
        include_root_dir (bool): If False, root_dir name is not included.

    Returns:
        list: List of found filenames.

    """
    files = []
    for root, dirnames, filenames in os.walk(root_dir, followlinks=True):
        for filename in fnmatch.filter(filenames, query):
            files.append(os.path.join(root, filename))
    if not include_root_dir:
        files = [file_.replace(root_dir + "/", "") for file_ in files]

    return files

def read_hdf5(hdf5_name, hdf5_path):
    """Read hdf5 dataset.

    Args:
        hdf5_name (str): Filename of hdf5 file.
        hdf5_path (str): Dataset name in hdf5 file.

    Return:
        any: Dataset values.

    """
    if not os.path.exists(hdf5_name):
        logging.error(f"There is no such a hdf5 file ({hdf5_name}).")
        sys.exit(1)

    hdf5_file = h5py.File(hdf5_name, "r")

    if hdf5_path not in hdf5_file:
        logging.error(f"There is no such a data in hdf5 file. ({hdf5_path})")
        sys.exit(1)

    hdf5_data = hdf5_file[hdf5_path][()]
    hdf5_file.close()

    return hdf5_data

def write_hdf5(hdf5_name, hdf5_path, write_data, is_overwrite=True):
    """Write dataset to hdf5.

    Args:
        hdf5_name (str): Hdf5 dataset filename.
        hdf5_path (str): Dataset path in hdf5.
        write_data (ndarray): Data to write.
        is_overwrite (bool): Whether to overwrite dataset.

    """
    # convert to numpy array
    write_data = np.array(write_data)

    # check folder existence
    folder_name, _ = os.path.split(hdf5_name)
    if not os.path.exists(folder_name) and len(folder_name) != 0:
        os.makedirs(folder_name)

    # check hdf5 existence
    if os.path.exists(hdf5_name):
        # if already exists, open with r+ mode
        hdf5_file = h5py.File(hdf5_name, "r+")
        # check dataset existence
        if hdf5_path in hdf5_file:
            if is_overwrite:
                logging.warning("Dataset in hdf5 file already exists. "
                                "recreate dataset in hdf5.")
                hdf5_file.__delitem__(hdf5_path)
            else:
                logging.error("Dataset in hdf5 file already exists. "
                              "if you want to overwrite, please set is_overwrite = True.")
                hdf5_file.close()
                sys.exit(1)
    else:
        # if not exists, open with w mode
        hdf5_file = h5py.File(hdf5_name, "w")

    # write data to hdf5
    hdf5_file.create_dataset(hdf5_path, data=write_data)
    hdf5_file.flush()
    hdf5_file.close()

class HDF5ScpLoader(object):
    """Loader class for a fests.scp file of hdf5 file.

    Examples:
        key1 /some/path/a.h5:feats
        key2 /some/path/b.h5:feats
        key3 /some/path/c.h5:feats
        key4 /some/path/d.h5:feats
        ...
        >>> loader = HDF5ScpLoader("hdf5.scp")
        >>> array = loader["key1"]

        key1 /some/path/a.h5
        key2 /some/path/b.h5
        key3 /some/path/c.h5
        key4 /some/path/d.h5
        ...
        >>> loader = HDF5ScpLoader("hdf5.scp", "feats")
        >>> array = loader["key1"]

        key1 /some/path/a.h5:feats_1,feats_2
        key2 /some/path/b.h5:feats_1,feats_2
        key3 /some/path/c.h5:feats_1,feats_2
        key4 /some/path/d.h5:feats_1,feats_2
        ...
        >>> loader = HDF5ScpLoader("hdf5.scp")
        # feats_1 and feats_2 will be concatenated
        >>> array = loader["key1"]

    """

    def __init__(self, feats_scp, default_hdf5_path="feats"):
        """Initialize HDF5 scp loader.

        Args:
            feats_scp (str): Kaldi-style feats.scp file with hdf5 format.
            default_hdf5_path (str): Path in hdf5 file. If the scp contain the info, not used.

        """
        self.default_hdf5_path = default_hdf5_path
        with open(feats_scp) as f:
            lines = [line.replace("\n", "") for line in f.readlines()]
        self.data = {}
        for line in lines:
            key, value = line.split()
            self.data[key] = value

    def get_path(self, key):
        """Get hdf5 file path for a given key."""
        return self.data[key]

    def __getitem__(self, key):
        """Get ndarray for a given key."""
        p = self.data[key]
        if ":" in p:
            if len(p.split(",")) == 1:
                return read_hdf5(*p.split(":"))
            else:
                p1, p2 = p.split(":")
                feats = [read_hdf5(p1, p) for p in p2.split(",")]
                return np.concatenate([f if len(f.shape) != 1 else f.reshape(-1, 1) for f in feats], 1)
        else:
            return read_hdf5(p, self.default_hdf5_path)

    def __len__(self):
        """Return the length of the scp file."""
        return len(self.data)

    def __iter__(self):
        """Return the iterator of the scp file."""
        return iter(self.data)

    def keys(self):
        """Return the keys of the scp file."""
        return self.data.keys()

    def values(self):
        """Return the values of the scp file."""
        for key in self.keys():
            yield self[key]

class NpyScpLoader(object):
    """Loader class for a fests.scp file of npy file.

    Examples:
        key1 /some/path/a.npy
        key2 /some/path/b.npy
        key3 /some/path/c.npy
        key4 /some/path/d.npy
        ...
        >>> loader = NpyScpLoader("feats.scp")
        >>> array = loader["key1"]

    """

    def __init__(self, feats_scp):
        """Initialize npy scp loader.

        Args:
            feats_scp (str): Kaldi-style feats.scp file with npy format.

        """
        with open(feats_scp) as f:
            lines = [line.replace("\n", "") for line in f.readlines()]
        self.data = {}
        for line in lines:
            key, value = line.split()
            self.data[key] = value

    def get_path(self, key):
        """Get npy file path for a given key."""
        return self.data[key]

    def __getitem__(self, key):
        """Get ndarray for a given key."""
        return np.load(self.data[key])

    def __len__(self):
        """Return the length of the scp file."""
        return len(self.data)

    def __iter__(self):
        """Return the iterator of the scp file."""
        return iter(self.data)

    def keys(self):
        """Return the keys of the scp file."""
        return self.data.keys()

    def values(self):
        """Return the values of the scp file."""
        for key in self.keys():
            yield self[key]

def load_model(checkpoint, config=None):
    """Load trained model.

    Args:
        checkpoint (str): Checkpoint path.
        config (dict): Configuration dict.

    Return:
        torch.nn.Module: Model instance.

    """
    # load config if not provided
    if config is None:
        dirname = os.path.dirname(checkpoint)
        config = os.path.join(dirname, "config.yml")
        with open(config) as f:
            config = yaml.load(f, Loader=yaml.Loader)

    # lazy load for circular error
    import models

    # get model and load parameters
    model_class = getattr(
        models,
        config.get("generator_type", "ParallelWaveGANGenerator")
    )
    model = model_class(**config["generator_params"])
    model.load_state_dict(
        torch.load(checkpoint, map_location="cpu")["model"]["generator"]
    )

    # add pqmf if needed
    if config["generator_params"]["out_channels"] > 1:
        # lazy load for circular error
        from layers import PQMF

        pqmf_params = {}
        if LooseVersion(config.get("version", "0.1.0")) <= LooseVersion("0.4.2"):
            # For compatibility, here we set default values in version <= 0.4.2
            pqmf_params.update(taps=62, cutoff_ratio=0.15, beta=9.0)
        model.pqmf = PQMF(
            subbands=config["generator_params"]["out_channels"],
            **config.get("pqmf_params", pqmf_params),
        )

    return model

def download_pretrained_model(tag, download_dir=None):
    """Download pretrained model form google drive.

    Args:
        tag (str): Pretrained model tag.
        download_dir (str): Directory to save downloaded files.

    Returns:
        str: Path of downloaded model checkpoint.

    """
    assert tag in PRETRAINED_MODEL_LIST, f"{tag} does not exists."
    id_ = PRETRAINED_MODEL_LIST[tag]
    if download_dir is None:
        download_dir = os.path.expanduser("~/.cache/parallel_wavegan")
    output_path = f"{download_dir}/{tag}.tar.gz"
    os.makedirs(f"{download_dir}", exist_ok=True)
    if not os.path.exists(output_path):
        # lazy load for compatibility
        import gdown

        gdown.download(f"https://drive.google.com/uc?id={id_}", output_path, quiet=False)
        with tarfile.open(output_path, 'r:*') as tar:
            for member in tar.getmembers():
                if member.isreg():
                    member.name = os.path.basename(member.name)
                    tar.extract(member, f"{download_dir}/{tag}")
    checkpoint_path = find_files(f"{download_dir}/{tag}", "checkpoint*.pkl")

    return checkpoint_path[0]

def simple_table(item_tuples):

    border_pattern = '+---------------------------------------'
    whitespace = '                                            '

    headings, cells, = [], []

    for item in item_tuples:

        heading, cell = str(item[0]), str(item[1])

        pad_head = True if len(heading) < len(cell) else False

        pad = abs(len(heading) - len(cell))
        pad = whitespace[:pad]

        pad_left = pad[:len(pad)//2]
        pad_right = pad[len(pad)//2:]

        if pad_head:
            heading = pad_left + heading + pad_right
        else:
            cell = pad_left + cell + pad_right

        headings += [heading]
        cells += [cell]

    border, head, body = '', '', ''

    for i in range(len(item_tuples)):

        temp_head = f'| {headings[i]} '
        temp_body = f'| {cells[i]} '

        border += border_pattern[:len(temp_head)]
        head += temp_head
        body += temp_body

        if i == len(item_tuples) - 1:
            head += '|'
            body += '|'
            border += '+'

    print(border)
    print(head)
    print(border)
    print(body)
    print(border)
    print(' ')


# coding: utf-8
import datetime
import json
import re
import urllib
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from service_task.models import SiteInfo

@csrf_exempt
def load_urls(request):
    """
        file with urls like:
        https://docs.djangoproject.com/en/1.10/intro/tutorial02/, https://vk.com/themusicend
    """
    if request.method == 'POST':
        try:
            urls = request.POST.items()[0][0].split(',')
        except Exception:
            return HttpResponse('wrong input')
        wrong_urls = ', wrong_urls: '
        for url in urls:
            try:
                response = urllib.urlopen(url).read()
                title = re.findall(r'<title>.+', response)[0].replace('<title>', '').replace('</title>', '')
                old_info = SiteInfo.objects.filter(title=title).first()
                if old_info:
                    old_info.url = url
                    old_info.timestamp = datetime.datetime.now()
                    old_info.save()
                else:
                    info = SiteInfo(title=title, url=url)
                    info.save()
            except Exception:
                wrong_urls += url

        return HttpResponse('done{}'.format(wrong_urls))
    return HttpResponse('no incoming data')

def get_titles(request):
    info = list()
    for site in SiteInfo.objects.all():
        info.append(dict(url=site.url, title=site.title, timestamp=unicode(site.timestamp)))
    info = json.dumps(info, indent=4, separators=(',', ': '))
    return HttpResponse(info)


import datetime
from dateutil.relativedelta import relativedelta
from prettytable import PrettyTable

def get_all_individual_data_by_key(individuals, id, key):
    meta_data = []
    for meta in individuals[id]:
        if meta[0] == key: meta_data.append(meta[1])
    return meta_data

def get_all_data_of_a_person(families, person_id, key):
    family_ids, child_ids = [], []

    for id in families:
        for meta in families[id]:
            if ((meta[0] == 'WIFE') or (meta[0] == 'HUSB')) and (meta[1] == person_id): family_ids.append(id)

    for id in family_ids:
        for meta in families[id]:
            if meta[0] == key: child_ids.append(meta[1])

    return child_ids

def get_all_descendants_of_all(individuals, families):
    d = {}
    for ind_id in individuals:
        d[ind_id] = get_all_descendants_of_a_person(families, ind_id, 'CHIL', set())

    for ind_id in individuals.keys():
        childs = d[ind_id]
        for child_id in childs:
            childs = childs + d[child_id]

        d[ind_id] = childs
    return d

def get_all_descendants_of_a_person(families, person_id, key, child_ids = set()):
    family_ids = set()

    for id in families:
        for meta in families[id]:
            if ((meta[0] == 'WIFE') or (meta[0] == 'HUSB')) and (meta[1] == person_id):
                family_ids.add(id)

    for id in list(family_ids):
        for meta in families[id]:
            if meta[0] == key:
                child_ids.add(meta[1])

    return list(child_ids)

def get_from_all_records_all_family_data(families, key):
    meta_data = []
    for id in families:
        for meta in families[id]:
            if meta[0] == key: meta_data.append(meta[1])
    return meta_data

def get_all_husband_and_wives(families):
    husband_and_wives = []
    for family_id, family_values in families.items():
        hub = get_family_data_by_key(families, family_id, 'HUSB')
        wife = get_family_data_by_key(families, family_id, 'WIFE')

        if hub and wife:
            husband_and_wives.append((hub, wife))
        else:
            print('ERROR: Husband or Wife ID not present for a pair')
    return husband_and_wives

def get_individual_data_by_key(individuals, id, key):
    meta_data = None
    for meta in individuals[id]:
        if meta[0] == key: meta_data = meta[1]
    return meta_data

def get_family_data_by_key(families, id, key):
    #print(families)
    #return families[id][key]
    #if key not in families[id][0]:
    #    return False
    #return families[id][0][key]
    #meta_data = None
    #for meta in families[id]:
    #    if meta[0] == key: meta_data = meta[1]
    #return meta_data
    family_data_list = families[id]
    for tuple in family_data_list:
        if tuple[0] == key:
            return tuple[1]
    #if execution reaches here key was not found
    return False

def get_marriage_date_by_key(families, id, key):
    meta_data = []

    for meta in families[id]:
        if meta[0] == "MARR":
            meta_data.append(meta[1])
    return meta_data

def get_div_date_by_key(families, id, key):
    meta_data = []

    for meta in families[id]:
        if meta[0] == "DIV":
            meta_data.append(meta[1])
    return meta_data

def is_not_supported_tags(level, tag):
    return (level == 1 and tag == 'DATE') or (level == 2 and tag == 'NAME')

def clean_gedcom_tags(file_path, supported_tags):
    clean_tags = []

    with open(file_path) as file:
        raw_gedcom = file.read().splitlines()

    for tag in raw_gedcom:
        tag_parts = tag.strip().split(" ", 2)

        if len(tag_parts) < 2:
            continue

        if len(tag_parts) > 2 and tag_parts[1] in ['INDI', 'FAM']:
            continue

        if len(tag_parts) > 2 and tag_parts[2] in ['INDI', 'FAM']:
            tag_parts[1], tag_parts[2] = tag_parts[2], tag_parts[1]

        if len(tag_parts) > 2:
            if tag_parts[1] in supported_tags and supported_tags[tag_parts[1]] == tag_parts[0]:
                clean_tags.append((tag_parts[0], tag_parts[1], tag_parts[2]))
            elif tag_parts[0:2] == ["In", ""]:
                clean_tags.append(tag_parts)

        elif len(tag_parts) == 2:
            if tag_parts[1] in supported_tags and supported_tags[tag_parts[1]] == tag_parts[0]:
                clean_tags.append((tag_parts[0], tag_parts[1]))

    clean_tags = clean_tags[1:-1]
    clean_tags = list(filter(('1', 'BIRT').__ne__, clean_tags))

    return clean_tags

def collect_individual_metadata(individuals, clean_tags):
    x = PrettyTable()
    x.field_names = ['ID', 'Name', 'Gender', 'Birthday', 'Age', 'Alive', 'Death', 'Child', 'Spouse']

    for i, tag in enumerate(clean_tags):
        if tag[1] == 'INDI' and tag[0] == '0':
            indi_key = tag[2][1:-1]
            values = []
            j = i + 1
            skip_tag = 0

            while clean_tags[j][1] != 'INDI':
                if clean_tags[j][1] == 'FAM' and clean_tags[j][0] == '0':
                    skip_tag = 1
                    break
                elif clean_tags[j][1] == 'DEAT' and clean_tags[j][2] == 'Y':
                    values.append(('DEAT', clean_tags[j + 1][2]))
                    j += 1
                elif clean_tags[j][1] in ('FAMS', 'FAMC'):
                    values.append((clean_tags[j][1], clean_tags[j][2][1:-1]))
                else:
                    values.append((clean_tags[j][1], clean_tags[j][2]))
                j += 1

            individuals[indi_key] = values

            if skip_tag == 1:
                break

    for key, value in individuals.items():
        name, gender, birth_date, death_date = "", "", "", "-"
        husband_id, wife_id = "", ""
        alive = True
        for i in range(len(value)):
            family_child, family_spouse = "-", "-"
            tag = value[i][0]
            if tag == 'NAME':
                name = value[i][1]
            if tag == 'SEX':
                gender = value[i][1]
            if tag == 'DATE':
                birth_date = value[i][1]
                birth_date = datetime.datetime.strptime(birth_date, '%d %b %Y').date()
            if tag == 'DEAT':
                death_date = value[i][1]
                death_date = datetime.datetime.strptime(death_date, '%d %b %Y').date()
            if tag == 'FAMC':
                family_child = value[i][1]
            if tag == 'FAMS':
                family_spouse = value[i][1]

        if any('DEAT' in i for i in value):
            alive = False
            age = relativedelta(death_date, birth_date).years
        else:
            age = relativedelta(datetime.datetime.now(), birth_date).years

        check_if_birth_before_death(birth_date, death_date)

        x.add_row([key, name, gender, birth_date, age, alive, death_date, family_child, family_spouse])

    print("Individuals")
    print(x)
    x.clear_rows()

def collect_family_metadata(individuals, families, clean_tags):
    y = PrettyTable()

    for i in range(len(clean_tags)):
        values = []
        j = i + 1
        if clean_tags[i][1] == 'FAM' and clean_tags[i][0] == '0':
            family_key = clean_tags[i][2][1:-1]

            while j < len(clean_tags):
                tag = clean_tags[j][1]
                data = clean_tags[j][2][1:-1] if len(clean_tags[j]) > 2 else ""

                if tag not in ['MARR', 'DIV', 'DATE', 'FAM']:
                    values.append((tag, data))
                elif tag == 'MARR' and len(clean_tags[j + 1]) > 2:
                    values.append(('MARR', clean_tags[j + 1][2]))
                elif tag == 'DIV' and len(clean_tags[j + 1]) > 2:
                    values.append(('DIV', clean_tags[j + 1][2]))
                elif tag == 'FAM' and clean_tags[j][0] == '0':
                    break

                j += 1

            families[family_key] = values

    y.field_names = ['ID', 'Married', 'Divorced', 'Husband ID', 'Husband Name', 'Wife ID', 'Wife Name', 'Children']

    for key, value in families.items():
        children = []
        married_date, divorce_date = "", "-"
        husband_id, wife_id = "", ""
        husband_name, wife_name = "", ""

        for tag, data in value:
            if tag == 'HUSB':
                husband_id = data
                husband_name = individuals[husband_id][0][1]
            elif tag == 'WIFE':
                wife_id = data
                wife_name = individuals[wife_id][0][1]
            elif tag == 'CHIL':
                children.append(data)
            elif tag == 'MARR':
                married_date = datetime.datetime.strptime(data, '%d %b %Y').date()
            elif tag == 'DIV':
                divorce_date = datetime.datetime.strptime(data, '%d %b %Y').date()

        check_if_married_before_divorce(married_date, divorce_date)
        check_if_married_before_death(married_date, get_individual_data_by_key(individuals, husband_id, 'DEAT'))
        check_if_married_before_death(married_date, get_individual_data_by_key(individuals, wife_id, 'DEAT'))

        check_if_birth_before_marriage(get_individual_data_by_key(individuals, husband_id, 'DATE'), married_date)
        check_if_birth_before_marriage(get_individual_data_by_key(individuals, wife_id, 'DATE'), married_date)

        check_divorce_before_death(families,individuals) #US06
        # check_birth_after_parent_marriage(families,individuals) #US08
        noBigamy(families, individuals)

        check_age_under_onefifty(individuals) #US07
        check_gender_role(families, individuals) #US21
        recent_births(individuals) #US35

        y.add_row([key, married_date, divorce_date, husband_id, husband_name, wife_id, wife_name, children])

    print("Families")
    print(y)
    y.clear_rows()

# Story Id - US02:
def check_if_birth_before_marriage(birth_date, marriage_date):
    #Birth before Marriage
    if (birth_date == '-') and (marriage_date == '-'):
        return None
    if (birth_date != '-') and (marriage_date == '-'):
        return None

    if (birth_date == '-') and (marriage_date != '-'):
        err = "Birth Date needs to be present if marriage date is present"
        print(err)
        return err

    if birth_date and marriage_date:
        birth_date_formatted = datetime.datetime.strptime(birth_date, '%d %b %Y').date()
        if birth_date_formatted > marriage_date:
            err = "Birth date cannot be after marriage date"
            print(err)
            return err

        if birth_date_formatted == marriage_date:
            err = "Birth date cannot be same as marriage date"
            print(err)
            return err

# # Story Id - US03:
def check_if_birth_before_death(birth_date, death_date):
    #Birth before Death
    if (birth_date == '-') and (death_date == '-'): return None
    if (birth_date != '-') and (death_date == '-'): return None

    if (birth_date == '-') and (death_date != '-'):
        err = "Birth Date needs to be present if death date is present"
        print(err)
        return err

    if birth_date and death_date:
        if birth_date > death_date:
            err = "Birth date cannot be after death date"
            print(err)
            return err

        if birth_date == death_date:
            err = "Birth date cannot be same as death date"
            print(err)
            return err

# Story Id - US04:
def check_if_married_before_divorce(marriage_date, divorce_date):
    # Marriage before Divorce
    if (marriage_date == '-') and (divorce_date == '-'): return None
    if (marriage_date != '-') and (divorce_date == '-'): return None

    if (marriage_date == '-') and (divorce_date != '-'):
        err = "ERROR: US04: Marriage date needs to be present if divorce date is present"
        print(err)
        return err

    if marriage_date and divorce_date:
        if marriage_date > divorce_date:
            err = "ERROR: US04: Marriage date cannot be after divorce date"
            print(err)
            return err

        if marriage_date == divorce_date:
            err = "ERROR: US04: Marriage date cannot be equal to divorce date"
            print(err)
            return err

# Story Id - US05:
def check_if_married_before_death(marriage_date, death_date):
    # Marriage before Death
    if (marriage_date == '-') or (death_date == '-'): return None
    if (marriage_date != '-') and (death_date == '-'): return None

    if marriage_date and death_date:
        death_date_formatted = datetime.datetime.strptime(death_date, '%d %b %Y').date()
        if marriage_date > death_date_formatted:
            err = "ERROR: US05: Marriage date cannot be after death date"
            print(err)
            return err

        if marriage_date == death_date_formatted:
            err = "ERROR: US05: Marriage date cannot be equal to death date"
            print(err)
            return err

def get_individual_name(indi_id, individuals):
    return individuals[indi_id]["NAME"]

def check_divorce_before_death(families, individuals):
    count = 0
    for id in families:
        if "DIV" in families[id]:
            div_date = datetime.datetime.strptime(families[id]["DIV"], "%d %b %Y")
            husb_ID = families[id]["HUSB"]
            wife_ID = families[id]["WIFE"]
            if "DEAT" in individuals[husb_ID] and "DEAT" in individuals[wife_ID]:
                husbDeath = datetime.datetime.strptime(
                    individuals[husb_ID]["DEATH_DATE"], "%d %b %Y"
                )
                wifeDeath = datetime.datetime.strptime(
                    individuals[wife_ID]["DEATH_DATE"], "%d %b %Y"
                )
                if div_date > husbDeath and div_date > wifeDeath:
                    count += 1
                    print(
                        "ERROR: FAMILY: US06: Divorce between "
                        + "'"
                        + get_individual_name(husb_ID, individuals).replace("/", "")
                        + "' and '"
                        + get_individual_name(wife_ID, individuals).replace("/", "")
                        + "' cannot occur after both partners' deaths."
                    )
            elif "DEAT" in individuals[husb_ID]:
                husbDeath = datetime.datetime.strptime(
                    individuals[husb_ID]["DEATH_DATE"], "%d %b %Y"
                )
                if div_date > husbDeath:
                    count += 1
                    print(
                        "ERROR: FAMILY: US06: Divorce between "
                        + "'"
                        + get_individual_name(husb_ID, individuals).replace("/", "")
                        + "' and '"
                        + get_individual_name(wife_ID, individuals).replace("/", "")
                        + "' cannot occur after "
                        + get_individual_name(husb_ID, individuals).replace("/", "")
                        + "'s death."
                    )
            elif "DEAT" in individuals[wife_ID]:
                wifeDeath = datetime.datetime.strptime(
                    individuals[wife_ID]["DEATH_DATE"], "%d %b %Y")
                if div_date > wifeDeath:
                    count += 1
                    print(
                        "ERROR: FAMILY: US06: Divorce between "
                    + "'"
                    + get_individual_name(husb_ID, individuals).replace("/", "")
                    + "' and '"
                    + get_individual_name(wife_ID, individuals).replace("/", "")
                    + "' cannot occur after "
                    + get_individual_name(wife_ID, individuals).replace("/", "")
                    + "'s death."
                )
    if count == 0:
        return True
    else:
        return False

def check_birth_after_parent_marriage(families, individuals):
    is_valid = True
    for id in families:
        #if "MARR" in families[id]:
        if get_family_data_by_key(families,id,"MARR"):
            marriageDate = datetime.datetime.strptime(get_family_data_by_key(families,id,"MARR"), "%d %b %Y").date()
            childID = get_family_data_by_key(families,id,"CHIL")
            if childID:
                childBirthDate = datetime.datetime.strptime(get_individual_data_by_key(individuals,childID,"DATE"), "%d %b %Y").date()
                if childBirthDate < marriageDate:
                    print("ANOMALY: FAMILY: US08: Child ("
                            + childID
                            + ") born before marriage of parents in family: "
                            + id
                            + ".")
                    is_valid = False
        else:
            print("ERROR: FILE: US08: Marriage date not set or properly formatted of family: "
                + id
                + ".")
            is_valid = False
    return is_valid

#Story US09:
def birth_after_parent_death(parentA, parentB, child):
    #Birth after parents death
    if(parentA == '-' or parentB == '-' or child == '-'):
        return None
    childBirth= datetime.datetime(2, 2, 2).date()
    parentADeath= datetime.datetime(9999, 1, 1).date()
    parentBDeath= datetime.datetime(9999, 1, 1).date()
    is_dead = False
    childBirth = datetime.datetime.strptime(child['DATE'], "%d %b %Y").date()
    for id in parentA:
        if id == "DEAT":
            is_dead = True
            parentADeath = temp
        if id == "DATE":
            temp = datetime.datetime.strptime(parentA[id], "%d %b %Y").date()
    for id in parentB:
        if id == "DEAT":
            is_dead = True
            parentBDeath = temp
        if id == "DATE":
            temp = datetime.datetime.strptime(parentB[id], "%d %b %Y").date()
    if (childBirth > parentADeath or childBirth > parentBDeath):
        print("ERROR: US09: Child born after death of parent")
        return True
    else:
        return False

#Story US10:
def married_before_fourteen(marriage_year, birth_year):
    # Marriage before age of 14
    if (marriage_year == '-' or birth_year == '-'):
        return None
    ageAtMarriage = marriage_year-birth_year
    if marriage_year:
        if ageAtMarriage <= 14:
            err = "ERROR: US10: Marriage date is before 14"
            print(err)
            return True
    return False

#Story US11
def noBigamy(families, individuals):
    err = None
    husb_ID = ""
    wife_ID = ""
    myDict = {}
    divDict = {}
    for id in families:
        for item in families[id]:
            if item[0] == "HUSB":
                husb_ID = item[1]
                myDict.setdefault(husb_ID)
                divDict.setdefault(husb_ID)
            elif item[0] == "WIFE":
                wife_ID = item[1]
                myDict.setdefault(wife_ID)
                divDict.setdefault(wife_ID)

    for key, values in myDict.items():
        for id in families:
            fam_husbId = get_family_data_by_key(families, id, 'HUSB')
            fam_wifeId = get_family_data_by_key(families, id, 'WIFE')

            if key == fam_husbId or key == fam_wifeId:
                marr_date = get_marriage_date_by_key(families, id, key)

                if key in myDict:
                    if isinstance(myDict[key], list):
                        myDict[key].append(marr_date)
                    else:
                        myDict[key] = [myDict[key], marr_date]
                else:
                    myDict[key] = [marr_date]

    for key, values in divDict.items():
        for id in families:
            fam_husbId = get_family_data_by_key(families, id, 'HUSB')
            fam_wifeId = get_family_data_by_key(families, id, 'WIFE')
            if key == fam_husbId or key == fam_wifeId:
                div_date = get_div_date_by_key(families, id, key)

                if key in divDict:
                    if isinstance(divDict[key], list):
                        divDict[key].append(div_date)
                    else:
                        divDict[key] = [divDict[key], div_date]
                else:
                    divDict[key] = [div_date]

    updated_myDict = {key: [val for val in value if val is not None] for key, value in myDict.items()}
    updated_divDict = {key: [val for val in value if val is not None] for key, value in divDict.items()}

    for key in myDict:
        if key in divDict:
            my_dates = [date for sublist in myDict[key] if sublist for date in sublist]
            div_dates = [date for sublist in divDict[key] if sublist for date in sublist]
            new_my_dates = [datetime.datetime.strptime(date, '%d %b %Y').date() for date in my_dates]
            new_div_dates = [datetime.datetime.strptime(date, '%d %b %Y').date() for date in div_dates]

            if len(new_my_dates) > 2 and len(new_div_dates) > 1:
                for i in range(len(new_my_dates) - 1):
                    if new_div_dates[i] > new_my_dates[i] and new_div_dates[i] < new_my_dates[i+1]:
                        err = "No Bigamy"

                    else:
                        err = "Bigamy caught"
                    return(err)

            elif (len(new_my_dates) == 2 and (len(new_div_dates) == 1 or len(new_div_dates) == 0)):
                if len(new_div_dates) == 0:
                    err = "Bigamy caught"

                else:
                    lesser_date = min(new_my_dates)
                    greater_date = max(new_my_dates)
                    div_date = new_div_dates[0]

                    if lesser_date < div_date < greater_date:
                        err = "No Bigamy"

                    else:
                        err = "Bigamy caught"
                return err

            else:
                err = "No Bigamy"
    return err

# Story Id: US18
def sibling_should_not_marry(families):
    invalid = False
    husb_wives = get_all_husband_and_wives(families)
    for family_id, family_data in families.items():
        sibling_ids = get_all_individual_data_by_key(families, family_id, 'CHIL')
        for hub_id, wife_id in husb_wives:
            if (hub_id in sibling_ids) and (wife_id in sibling_ids):
                invalid = True
                print(f"ERROR: US18: Siblings {hub_id} and {wife_id} should not marry.")
    return invalid

# Story Id: US14 (No more than five siblings should be born at the same time)
def too_many_siblings_at_same_time(individuals, families):
    invalid = False
    for ind_id in individuals:
        children_ids = get_all_data_of_a_person(families, ind_id, 'CHIL')
        same_date_count = {}
        for child_id in children_ids:
            birth_date = get_individual_data_by_key(individuals, child_id, 'DATE')
            if birth_date in same_date_count:
                same_date_count[birth_date] += 1
            else:
                same_date_count[birth_date] = 0

        for sibling_on_same_time_count in same_date_count.values():
            if sibling_on_same_time_count > 5:
                print(f"ERROR: US14: No more than five siblings should be born at the same time")
                invalid = True
    return invalid

# Story Id: US12 (Mother should be less than 60 years older than her children)
def mother_is_not_too_old_for_child(individuals, families):
    invalid = False
    for ind_id in individuals:
        gender = get_individual_data_by_key(individuals, ind_id, 'SEX')
        if gender == 'F':
            mother_birth_date = get_individual_data_by_key(individuals, ind_id, 'DATE')
            children_ids = get_all_data_of_a_person(families, ind_id, 'CHIL')
            for child_id in children_ids:
                child_birth_date = get_individual_data_by_key(individuals, child_id, 'DATE')
                delta_years = relativedelta(
                    datetime.datetime.strptime(child_birth_date, '%d %b %Y').date(),
                    datetime.datetime.strptime(mother_birth_date, '%d %b %Y').date()
                ).years
                if delta_years >= 60:
                    invalid = True
                    print(f"ERROR: US12: Mother should be less than 60 years older than her children")

    return invalid

# Story Id: US15
def fewer_than_15_siblings(families):
    invalid = False
    for family_id, family_data in families.items():
        siblings = get_all_individual_data_by_key(families, family_id, 'CHIL')
        if len(siblings) > 15:
            invalid = True
            print(f"ERROR: US15: Family has more than 15 siblings")
    return invalid

# Story Id: US17
def parents_should_not_marry_descendants(individuals, families):
    invalid = False
    all_descendants = get_all_descendants_of_all(individuals, families)
    for ind_id in individuals:
        descendants = all_descendants[ind_id]
        for family_id in families:
            husb_id = get_family_data_by_key(families, family_id, 'HUSB')
            wife_id = get_family_data_by_key(families, family_id, 'WIFE')

            if ((ind_id == husb_id) and (wife_id in descendants) or (ind_id == wife_id) and (husb_id in descendants)):
                invalid = True
                print(f"ERROR: US17: Parent is married to their descendant")
    return invalid

# Story Id: US25
def child_should_not_have_same_name_date(individuals, families):
    invalid = False
    for ind_id in individuals:
        children_ids = get_all_data_of_a_person(families, ind_id, 'CHIL')
        s = {}
        for child_id in children_ids:
            child_name = get_individual_data_by_key(individuals, child_id, 'NAME')
            child_birth_date = get_individual_data_by_key(individuals, child_id, 'DATE')
            if f'{child_name}_{child_birth_date}' in s:
                invalid = True
                print(f"ERROR: US25: Two child have same name and birth date")
            else:
                s[f'{child_name}_{child_birth_date}'] = True
    return invalid

#US07
def check_age(birth_date):
    today = datetime.datetime.today()
    birth = datetime.datetime.strptime(birth_date, "%d %b %Y")
    age_from_today = today - birth
    age = age_from_today.days
    return age

def check_age_under_onefifty(individuals):
    is_valid = True
    birth_date = 0
    for id in individuals:
        individual = individuals[id]
        birthday = get_individual_data_by_key(individuals,id,"DATE")
        day_age = check_age(birthday)
        age = day_age/365.25
        if age >= 150:
            is_valid = False
            print("Age is over 150")
    return is_valid

#US21
def check_gender_role(families, individuals):
    is_valid = True
    for id in families:
        husbID = get_family_data_by_key(families,id,"HUSB")
        if husbID:
            gender = get_individual_data_by_key(individuals,husbID,"SEX")
            if gender != "M":
                print("Gender does not match Marriage Role")
                is_valid = False
        wifeID = get_family_data_by_key(families,id,"WIFE")
        if wifeID:
            gender = get_individual_data_by_key(individuals,wifeID,"SEX")
            if gender != "F":
                print("Gender does not match Marriage Role")
                is_valid = False
        return is_valid

#US34
def large_age_difference(families, individuals):
    is_valid = True
    for id in families:

        husbID = get_family_data_by_key(families,id,"HUSB")
        if husbID:
            husb_bday = get_individual_data_by_key(individuals,husbID,"DATE")
            husb_age1 = check_age(husb_bday)
            husb_age = husb_age1/365.25
            #print(husb_age)

        wifeID = get_family_data_by_key(families,id,"WIFE")
        if wifeID:
            wife_bday = get_individual_data_by_key(individuals,wifeID,"DATE")
            wife_age1 = check_age(wife_bday)
            wife_age = wife_age1/365.25
            #print(wife_age)

        if husb_age > 2*wife_age:
            print("Husband ", husbID, "is more than 2 times older than Wife ", wifeID)
        elif wife_age >2*husb_age:
            print("Wife ", wifeID, "is more than 2 times older than Husband ", husbID)
        else:
            print("Husband, ",husbID,"& Wife, ",wifeID,"do not have a large age gap")
            is_valid = False

    return is_valid

#US35
def recent_births(individuals):
    is_valid = True
    for id in individuals:
        individual = individuals[id]
        birthday = get_individual_data_by_key(individuals,id,"DATE")
        day_age = check_age(birthday)
        if day_age > 30:
            is_valid = False
            #print("Born more than 30 days ago")
    return is_valid
    print("Born Recently")

# Story Id: US13
def birth_dates_of_siblings_should_be_more_than_eight_months_apart(individuals, families):
    flag = False
    for family_id, family_data in families.items():
        siblings = get_all_individual_data_by_key(families, family_id, 'CHIL')
        s = []
        for sib_id in siblings:
            sibling_birth_date = get_individual_data_by_key(individuals, sib_id, 'DATE')
            s.append(sibling_birth_date)

        for i in range(1, len(s)):
            date1 = datetime.datetime.strptime(s[i], "%d %b %Y")
            date2 = datetime.datetime.strptime(s[i-1], "%d %b %Y")

            # difference = date1 - date2        #Author: Sarthak Vaidya   Assignment 6 Bad Smell 1
            # months_difference = difference.days // 30
            months_difference = (date1.year - date2.year) * 12 + (date1.month - date2.month) #Assignment 6 bad smell 1 corrected using better logic

            if months_difference > 8:
                flag = True
                print(f"The difference between {s[i-1]} and {s[i]} is greater than 8 months.")
            elif months_difference <= 8:
                flag = False
    return flag

# Story Id: US29
# def list_deceased(individuals):    #Author: Sarthak Vaidya Bad Smell 2 Assignment 6
#     deceased = []
#     for indi_id, indi_data in individuals.items():
#         if(len(indi_data) > 3):
#             if(indi_data[3][0] == 'DEAT'):
#                 deceased.append(indi_data[3][1])

#     for i in range(len(deceased)):
#         print("Dates of deceased people")
#         print(deceased[i])
#     return deceased

def list_deceased(individuals):    #Refactored code removing the bad smell
    deceased = get_deceased_dates(individuals)
    print_deceased_dates(deceased)
    return deceased

def get_deceased_dates(individuals):
    deceased = []
    for indi_id, indi_data in individuals.items():
        if len(indi_data) > 3 and indi_data[3][0] == 'DEAT':
            deceased.append(indi_data[3][1])
    return deceased

def print_deceased_dates(deceased_dates):
    for date in deceased_dates:
        print("Dates of deceased people")
        print(date)

# Story Id - US22:
def unique_id(individuals, families):
    if(individuals == '-' or families == '-'):
        return None
    i = 0
    j = 0
    for ind in individuals:
        for ind2 in individuals:
            if(j >= len(individuals)):
                continue
            elif(i < j):
                 if(ind == ind2):
                     return False
            j = j + 1
        i = i + 1
        j = i + 1
    x = 0
    y = 0
    for fam in families:
        for fam2 in families:
            if(y >= len(families)):
                continue
            elif(x < y):
                if(fam == fam2):
                    return False
            y = y + 1
        x = x + 1
        y = x +1
    return True

# Story Id - US23:
def unique_name_and_birth(individuals):
    if(individuals == '-'):
        return None;
    i = 0
    j = 0
    for ind in individuals:
        ind_name = get_individual_data_by_key(individuals, ind, 'NAME')
        ind_birth = get_individual_data_by_key(individuals, ind, 'DATE')
        for ind2 in individuals:
            ind2_name = get_individual_data_by_key(individuals, ind2, 'NAME')
            ind2_birth = get_individual_data_by_key(individuals, ind2, 'DATE')
            if(j >= len(individuals)):
                continue
            elif(i < j and ind_name != None):
                 if(ind_name == ind2_name and ind_birth == ind2_birth):
                     return False
            j = j + 1
        i = i + 1
        j = i + 1

    return True

# Story Id - US16:
def maleLastNames(families, individuals):
    for family_id, family_data in families.items():
        fam_husb_data = get_individual_data_by_key(families, family_id, "HUSB")
        list_husb_name = get_individual_data_by_key(individuals, fam_husb_data, "NAME").split('/')[1].strip()
        fam_child_data = []
        fam_child_name = None
        for i in range(0, len(family_data)):
            if family_data[i][0] == "CHIL":
                fam_child_data.append(family_data[i][1])

        for i in range(0, len(fam_child_data)):
            child_id = fam_child_data[i].strip("'")
            fam_child_gender = get_all_individual_data_by_key(individuals, child_id, "SEX")
            if fam_child_gender[0] == "M":
                fam_child_name = get_all_individual_data_by_key(individuals, child_id, "NAME")[0].split('/')[1].strip()
            if fam_child_name is not None and list_husb_name != fam_child_name:
                return False
    return True

# Story Id - US33:
def listOrphans(families, individuals):
    today = datetime.datetime.now().date()
    is_orphan = None
    fam_child_data = []
    is_orphan_arr = []
    orphan_name_list = []
    for family_id, family_data in families.items():
        for i in range(0, len(family_data)):
            if family_data[i][0] == "CHIL":
                fam_child_data.append(family_data[i][1])

        if len(fam_child_data) > 0:
            for i in range(len(fam_child_data)):
                child_id = fam_child_data[i].strip("'")
                child_birth_date = get_all_individual_data_by_key(individuals, child_id, "DATE")[0]
                child_name = get_all_individual_data_by_key(individuals, child_id, "NAME")
                child_name = child_name[0].strip('/').replace('/', ' ')
                today = today.strftime("%d %b %Y")
                today = datetime.datetime.strptime(today, "%d %b %Y")
                child_birth_date = datetime.datetime.strptime(child_birth_date, "%d %b %Y")
                age = today - child_birth_date
                years = age.days/365
                years = round(years, 2)

                if years < 18:
                    husband_id = get_individual_data_by_key(families, family_id, 'HUSB')
                    husband_death_date = get_individual_data_by_key(individuals, husband_id, "DEAT")
                    wife_id = get_individual_data_by_key(families, family_id, 'WIFE')
                    wife_death_date = get_individual_data_by_key(individuals, wife_id, "DEAT")
                    if husband_death_date is None or wife_death_date is None:
                        is_orphan = False
                        is_orphan_arr.append(is_orphan)
                    else:
                        is_orphan = True
                        is_orphan_arr.append(is_orphan)
                        orphan_name_list.append(child_name)

                else:
                    is_orphan = False
                    is_orphan_arr.append(is_orphan)

            fam_child_data.clear()

        else:
            is_orphan_arr.append(is_orphan)

    if len(orphan_name_list) > 0:
        print("Orphan Child List")
        for i in range(len(orphan_name_list)):
            print(orphan_name_list[i])

    return is_orphan_arr

# Story Id - US39:
def upcomingAnniversaries(families, individuals):
    today = datetime.datetime.now().date()
    anniversary_list = []
    for family_id, family_data in families.items():
        husband_id = get_individual_data_by_key(families, family_id, 'HUSB')
        husband_death_date = get_individual_data_by_key(individuals, husband_id, "DEAT")
        wife_id = get_individual_data_by_key(families, family_id, 'WIFE')
        wife_death_date = get_individual_data_by_key(individuals, wife_id, "DEAT")
        anniversary_date = get_individual_data_by_key(families, family_id, "MARR")

        if anniversary_date is not None and husband_death_date is None and wife_death_date is None:
            anniversary_date = datetime.datetime.strptime(anniversary_date, "%d %b %Y")
            today = today.strftime("%d %b %Y")
            today = datetime.datetime.strptime(today, "%d %b %Y")

            diffDate = anniversary_date.replace(year = today.year) - today
            diffDays = diffDate.days

            if(diffDays >= 0 and diffDays <= 30):
                anniversary_list.append(anniversary_date)
        else:
            return None

    print("Anniversary List")
    for i in range(len(anniversary_list)):
        print(anniversary_list[i])
    return anniversary_list

# Story Id - US38:
def upcomingBirthdays(individuals):
    today = datetime.datetime.now().date()
    birthday_list = []
    for indi_id, indi_data in individuals.items():
        birth_date = get_individual_data_by_key(individuals, indi_id, 'DATE')
        death_date = get_individual_data_by_key(individuals, indi_id, 'DEAT')

        if birth_date is not None and death_date is None:
            birth_date = datetime.datetime.strptime(birth_date, "%d %b %Y")
            today = today.strftime("%d %b %Y")
            today = datetime.datetime.strptime(today, "%d %b %Y")
            diffDate = birth_date.replace(year = today.year) - today
            diffDays = diffDate.days
            if(diffDays >= 0 and diffDays <= 30):
                birthday_list.append(birth_date)
            else:
                birthday_list.append(None)
        else:
            birthday_list.append(None)

    print("Birthday List")
    for birthday in birthday_list:
        if birthday is not None:
            print(birthday)
    return birthday_list

#US36
def recent_deaths(individuals):
    is_recently_dead = True
    for id in individuals:
        deceaseddate = get_individual_data_by_key(individuals, id,"DEAT")
        if deceaseddate != None: #If Deceased Date is not None
            today = datetime.datetime.today()
            death = datetime.datetime.strptime(deceaseddate, "%d %b %Y")
            death_from_today = today - death
            days_since_death = death_from_today.days
            if days_since_death < 30:
                print(id,"recently passed away")
            else: #If Deceased Date is not greater than 30
                is_recently_dead  = False
        else: #If Deceased Date is None (Alive)
            is_recently_dead = False
    return is_recently_dead

def is_deceased(individuals): #US 29 Alternative
    is_deceased = True
    for id in individuals:
        deceaseddate = get_all_individual_data_by_key(individuals, id, "DEAT")
        if deceaseddate == []:
            #print("Individual ",id," is not deceased")
            is_deceased = False
        else:
            print("Individual ",id," is deceased since", ''.join(deceaseddate))
    return is_deceased

def survivors(families,individuals):
    is_survived_by = True
    all_descendants = get_all_descendants_of_all(individuals,families)
    for ind_id in individuals:
        descendants = all_descendants[ind_id]
        deceaseddate = get_individual_data_by_key(individuals, ind_id, "DEAT")

        if deceaseddate != None: #If Deceased Date is not None (Dead)
            for family_id in families:
                husbID = get_family_data_by_key(families, family_id,"HUSB")
                wifeID = get_family_data_by_key(families, family_id,"WIFE")
                #childID = get_family_data_by_key(families, family_id,"CHIL")
                if ind_id == husbID:
                    wifedeath = get_all_individual_data_by_key(individuals, wifeID, "DEAT")
                    if wifedeath == []:
                        print(ind_id, "Survived by Wife:", wifeID)
                    for descendantID in descendants:
                        deadchilddate = get_all_individual_data_by_key(individuals, descendantID, "DEAT")
                        if deadchilddate == []:
                            alivedescendant = []
                            alivedescendant.append(descendantID)
                            print(ind_id, "Survived by Child:",','.join(alivedescendant))
                            if wifedeath != [] and deadchilddate != []:
                                is_survived_by = False
                elif ind_id == wifeID:
                    husbdeath = get_all_individual_data_by_key(individuals, husbID, "DEAT")
                    if husbdeath == []:
                        print(ind_id, "Survived by Husband:", husbID)
                    for descendantID in descendants:
                        deadchilddate = get_all_individual_data_by_key(individuals, descendantID, "DEAT")
                        if deadchilddate == []:
                            alivedescendant = []
                            alivedescendant.append(descendantID)
                            print(ind_id, "Survived by Child:",','.join(alivedescendant))
                            if husbdeath != [] and deadchilddate != []:
                                is_survived_by = False
    return is_survived_by

#US37
def recent_survivors(families,individuals):
    recently_grieving_family = True
    all_descendants = get_all_descendants_of_all(individuals,families)
    for ind_id in individuals:
        deceaseddate = get_individual_data_by_key(individuals, ind_id,"DEAT")
        if deceaseddate != None: #If Deceased Date is not None
            today = datetime.datetime.today()
            death = datetime.datetime.strptime(deceaseddate, "%d %b %Y")
            death_from_today = today - death
            days_since_death = death_from_today.days
            if days_since_death < 30:
                #print(ind_id,"recently passed away")
                for family_id in families:
                    descendants = all_descendants[ind_id]
                    husbID = get_family_data_by_key(families, family_id,"HUSB")
                    wifeID = get_family_data_by_key(families, family_id,"WIFE")
                    #childID = get_family_data_by_key(families, family_id,"CHIL")
                    if ind_id == husbID:
                        wifedeath = get_all_individual_data_by_key(individuals, wifeID, "DEAT")
                        if wifedeath == []:
                            print(ind_id, "Survived by Wife:", wifeID)
                        for descendantID in descendants:
                            deadchilddate = get_all_individual_data_by_key(individuals, descendantID, "DEAT")
                            if deadchilddate == []:
                                alivedescendant = []
                                alivedescendant.append(descendantID)
                                print(ind_id, "Survived by Child:",','.join(alivedescendant))
                            if wifedeath != [] and deadchilddate != []:
                                recently_grieving_family = False
                    elif ind_id == wifeID:
                        husbdeath = get_all_individual_data_by_key(individuals, husbID, "DEAT")
                        if husbdeath == []:
                            print(ind_id, "Survived by Husband:", husbID)
                        for descendantID in descendants:
                            deadchilddate = get_all_individual_data_by_key(individuals, descendantID, "DEAT")
                            if deadchilddate == []:
                                alivedescendant = []
                                alivedescendant.append(descendantID)
                                print(ind_id, "Survived by Child:",','.join(alivedescendant))
                            if husbdeath != [] and deadchilddate != []:
                                recently_grieving_family = False

    return recently_grieving_family

def parse_gedcom_file(file_name):
    individuals = {}
    families = {}
    supported_tags = {
        'INDI': '0', 'NAME': '1', 'SEX': '1', 'BIRT': '1', 'DEAT': '1', 'FAMC': '1', 'FAMS': '1',
        'FAM': '0',
        'MARR': '1', 'HUSB': '1', 'WIFE': '1', 'CHIL': '1', 'DIV': '1', 'DATE': '2', 'HEAD': '0',
        'TRLR': '0', 'NOTE': '0'
    }
    clean_tags = clean_gedcom_tags(file_name, supported_tags)
    collect_individual_metadata(individuals, clean_tags)
    collect_family_metadata(individuals, families, clean_tags)
    sibling_should_not_marry(families)
    fewer_than_15_siblings(families)
    too_many_siblings_at_same_time(individuals, families)
    mother_is_not_too_old_for_child(individuals, families)
    parents_should_not_marry_descendants(individuals, families)
    child_should_not_have_same_name_date(individuals, families)
    birth_dates_of_siblings_should_be_more_than_eight_months_apart(individuals, families)
    list_deceased(individuals)
    maleLastNames(families, individuals)
    listOrphans(families, individuals)
    upcomingAnniversaries(families, individuals)
    upcomingBirthdays(individuals)
    large_age_difference(families, individuals) #US34
    recent_deaths(individuals) #US36
    recent_survivors(families, individuals) #US37
    return individuals, families

if __name__ == "__main__":
    parse_gedcom_file('sample.ged')


# import libraries
from bs4 import BeautifulSoup
import requests
import pandas as pd
import time

Save_Path = ".../01 Anime Basics.csv"
urlsFile = ".../Page Anime Urls.csv"

# Connect to Website and pull in data
url = "https://myanimelist.net/topanime.php"

# Use two different Agents
headers1 = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36", "X-Amzn-Trace-Id": "Root=1-612b8224-14c65ade7e3e57ee0168ea6b"}
headers2 ={"User-Agent": "Bot of the gods"}

def get_attributes(bs):

    # 01 Get all the available data from this page. Getting these Data for all 50 Anime
    Title = [x.get_text() for x in bs.findAll(class_ = "hoverinfo_trigger fl-l fs14 fw-b anime_ranking_h3")]
    Info = [x.get_text() for x in bs.findAll("div", class_="information di-ib mt4")]
    Score = [x.get_text() for x in bs.findAll("div", class_="js-top-ranking-score-col di-ib al")]
    Rank = [x.get_text() for x in bs.findAll("td", class_="rank ac")]

    Info1 = [i.split("\n") for i in Info]
    Type_Ep = []
    Date = []
    Members = []
    for i in Info1:
        Type_Ep.append(i[1].strip())
        Date.append(i[2].strip())
        Members.append(i[3].strip())

    Type_Ep = [i.split(" ") for i in Type_Ep]

    Anime_Type = []
    Number_of_Episodes = []
    for i in Type_Ep:
        Anime_Type.append(i[0])
        Number_of_Episodes.append(i[1])

    Title = [i.strip() for i in Title]
    Score = [i.strip() for i in Score]
    Rank = [i.strip() for i in Rank]

    # 02 Moving on to the next page for getting more data for each specific Anime
    Anime_Link = [x.find('a')['href'] for x in bs.findAll(class_ = "hoverinfo_trigger fl-l fs14 fw-b anime_ranking_h3")]

    Score_Users = []
    Season = []
    Popularity = []
    Image = []
    Synopsis = []
    English_Title = []
    Status = []
    Broadcast = []
    Producers = []
    Licensors = []
    Studios = []
    Source = []
    Genres = []
    Duration = []
    Rating = []
    Favorites = []
    Related_Anime = []
    Openings = []
    Endings = []

    counter = 0

    # This will run 50 times
    for link in Anime_Link:

        page = requests.get(link, headers=headers1)
        bs = BeautifulSoup(page.text, "html.parser")

        # For avoiding reCAPTCHA
        test = bs.find('div', style="width: 225px")
        try:
           test = test.get_text()
        except:
            print('A wild reCAPTCHA appeared, wait 5 minited!!')
            del test, page, bs
            time.sleep(300)
            page = requests.get(link, headers=headers1)
            bs = BeautifulSoup(page.text, "html.parser")

        # Users
        temp = bs.find('div',itemprop="aggregateRating")
        try:
            temp = temp.find(style='display: none').contents[0]
            Score_Users.append(temp)
        except:
            Score_Users.append(-1)

        # Season
        temp = bs.find('span', class_="information season")
        try:
            temp = temp.find('a').contents[0]
            Season.append(temp)
        except:
            Season.append("N/A")

        # # Popularity
        temp = bs.find('span', class_="numbers popularity")
        try:
            temp = temp.find('strong').contents[0]
            Popularity.append(temp)
        except:
            Popularity.append("-1")

        # Image
        temp = bs.find('div', style="text-align: center;")
        try:
            temp = temp.find('img')['data-src']
            Image.append(temp)
        except:
            Image.append("Ν/Α")

        # Synopsis
        temp = bs.find('p', itemprop="description")
        try:
            temp = temp.get_text()
            Synopsis.append(temp)
        except:
            Synopsis.append("Ν/Α")

        temp = bs.find('div', style="width: 225px")
        temp = temp.get_text()
        list_of_words = temp.split('\n')

        # English_Title
        English_Title_temp = 'N/A'
        for string in list_of_words:
            if string.startswith("English:"):
                English_Title_temp = string
        English_Title.append(English_Title_temp)

        # Status
        try:
            temp = list_of_words[list_of_words.index('Aired:') - 3]
            Status.append(temp)
        except:
            Status.append("Ν/Α")

         # Broadcast
        try:
            temp = list_of_words[list_of_words.index('Broadcast:') + 1]
            Broadcast.append(temp)
        except:
            Broadcast.append("Ν/Α")

        # Producers
        try:
            temp = list_of_words[list_of_words.index('Producers:') + 1]
            Producers.append(temp)
        except:
            Producers.append("Ν/Α")

        # Licensors
        try:
            temp = list_of_words[list_of_words.index('Licensors:') + 1]
            Licensors.append(temp)
        except:
            Licensors.append("Ν/Α")

        # Studios
        try:
            temp = list_of_words[list_of_words.index('Studios:') + 1]
            Studios.append(temp)
        except:
            Studios.append("Ν/Α")

        # Source
        try:
            temp = list_of_words[list_of_words.index('Source:') + 1]
            Source.append(temp)
        except:
            Source.append("Ν/Α")

        # Genres
        try:
            temp = list_of_words[list_of_words.index('Genres:') + 1]
            Genres.append(temp)
        except:
            Genres.append("Ν/Α")

        # Duration
        try:
            temp = list_of_words[list_of_words.index('Duration:') + 1]
            Duration.append(temp)
        except:
            Duration.append("Ν/Α")

     # Rating
        try:
            temp = list_of_words[list_of_words.index('Rating:') + 1]
            Rating.append(temp)
        except:
            Rating.append("Ν/Α")

         # Favorites
        try:
            temp = list_of_words[list_of_words.index('Favorites:') + 1]
            Favorites.append(temp)
        except:
            Favorites.append("Ν/Α")

        # Related_Anime
        temp = bs.find('table', class_="anime_detail_related_anime")

        try:
            temp1 = temp.get_text(separator='\n')
            Related_Anime.append(temp1)
        except:
            Related_Anime.append(temp)

        # Openings
        temp = bs.find('div', class_="theme-songs js-theme-songs opnening")
        try:
            Openings1 = temp.findAll('span', class_="theme-song")
            Openings1 = [x.get_text() for x in Openings1]
        except:
            Openings1 = ''
        Openings.append(Openings1)

        # Ending
        temp = bs.find('div', class_="theme-songs js-theme-songs ending")
        try:
            Endings1 = temp.findAll('span', class_="theme-song")
            Endings1 = [x.get_text() for x in Endings1]
        except:
            Endings1 = ''
        Endings.append(Endings1)

        # For getting live feedback of what is happening
        counter += 1
        print(counter)

    try:
        rows = pd.DataFrame(
        {'Title': Title,
         'Score': Score,
         'Rank': Rank,
         'Date': Date,
         'Members': Members,
         'Number_of_Episodes': Number_of_Episodes,
         'Anime_Type': Anime_Type,
         'Anime_Link': Anime_Link,
          'Score_Users': Score_Users,
          'Season': Season,
          'Popularity': Popularity,
          'Image': Image,
          'Synopsis': Synopsis,
          'English_Title': English_Title,
          'Status': Status,
          'Broadcast': Broadcast,
          'Producers':Producers,
          'Licensors':Licensors,
          'Studios':Studios,
          'Source':Source,
          'Genres':Genres,
          'Duration':Duration,
          'Rating':Rating,
          'Favorites':Favorites,
          'Openings':Openings,
          'Endings':Endings
        })

    except:
        rows = pd.DataFrame(
        {'Title': Title,
         'Score': Score,
         'Rank': Rank,
         'Date': Date,
         'Members': Members,
         'Number_of_Episodes': Number_of_Episodes,
         'Anime_Type': Anime_Type,
         'Anime_Link': Anime_Link,
          'Score_Users': Score_Users,
          'Season': Season,
          'Popularity': Popularity,
          'Image': Image,
          'Synopsis': Synopsis,
          'English_Title': English_Title,
          'Status': Status,
          'Broadcast': Broadcast,
          'Producers':Producers,
          'Licensors':Licensors,
          'Studios':Studios,
          'Source':Source,
          'Genres':Genres,
          'Duration':Duration,
          'Rating':Rating,
          'Favorites':Favorites,
          'Openings':Openings,
          'Endings':Endings
        })
    return rows

rows = pd.DataFrame(
    {'Title': [],
     'Score': [],
     'Rank': [],
     'Date': [],
     'Members': [],
     'Number_of_Episodes': [],
     'Anime_Type': [],
     'Anime_Link': [],
      'Score_Users': [],
      'Season': [],
      'Popularity': [],
      'Image': [],
      'Synopsis': [],
      'English_Title': [],
      'Status': [],
      'Broadcast': [],
      'Producers':[],
      'Licensors':[],
      'Studios':[],
      'Source':[],
      'Genres':[],
      'Duration':[],
      'Rating':[],
      'Favorites':[],
      'Openings':[],
      'Endings':[]

    })

paths = pd.read_csv(urlsFile, encoding= 'unicode_escape')

# Running for each group of 50 Anime
for index, url in paths.iterrows():

    page = requests.get(url['Url'], headers=headers1)
    bs = BeautifulSoup(page.text, "html.parser")
    rows1 = get_attributes(bs)
    frames = [rows, rows1]
    rows = pd.concat(frames, ignore_index=True)

rows.to_csv(Save_Path, index=False)

Code/Web Scraping/01 Web Scraping Basic.py


import torch
from matplotlib import pyplot as plt
import numpy as np
import datetime

from collections import defaultdict
from environments import CirclesSyntheticEnvironment
from policies_energy_based import EBMPolicy
from LSUV import LSUVinit

# ## setting the random seeds, for easy testing and developments
# random_seed = 42
# random.seed(random_seed)
# np.random.seed(random_seed)
from pathlib import Path

if __name__ == "__main__":
    saving_dir = Path("/Users/mleonardi/Projects/research/contextual-bandit-playground/data/")

    pretrain_time = 1_000
    number_of_observations = 1_750
    train_every = 100

    lr = 0.005
    num_epochs = 150
    loss_function_type = "log"
    sample_size = 256
    output_quadratic = False
    alpha = 10
    init_techq = 'xavier'
    use_dropout=True

    for init_techq in ['',]:
        if output_quadratic:
            var = 'QUAD'
        else:
            var = 'LIN'
        exp_cluster = f"FINALONESUPER-{train_every:,}_initializer-{init_techq}_LR-{str(lr).split('.')[-1]}_EPOCH-{num_epochs}_LFT-{loss_function_type}_SS-{sample_size}_ALPHA-{str(alpha).split('.')[-1]}_{var}"

        if not use_dropout:
            exp_cluster += '_no-dropout'

        for _ in range(5):
            environment = CirclesSyntheticEnvironment(
                number_of_different_context=2,
                n_context_features=2,
                number_of_observations=number_of_observations,
                time_perturbation_function=lambda time, mu: mu,
                action_offset=3,
                circle_factor=4.,
                name="2c_4_circ",
            )

            policy = EBMPolicy(
                name=f'TEST',
                lr=lr,
                warm_up=pretrain_time,
                num_epochs=num_epochs,
                loss_function_type=loss_function_type,
                sample_size=sample_size,
                output_quadratic=output_quadratic,
                alpha=alpha,
                feature_size=2,
                init_techq=init_techq,
             )

            c1 = None
            c2 = None
            a1 = None
            a2 = None

            stats_dict = defaultdict(list)
            for i, c in enumerate(environment.generate_contexts()):
                a = policy.get_action(c)
                r, s_r = environment.get_reward(a, c)
                policy.notify_event(c, a, s_r)

                optimal_r, optimal_a, stochastic_r = environment.get_best_reward_action(c)

                stats_dict['a'].append(a)
                stats_dict['r'].append(r)
                stats_dict['s_r'].append(s_r)
                stats_dict['optimal_a'].append(optimal_a)
                stats_dict['optimal_r'].append(optimal_r)
                stats_dict['stochastic_r'].append(stochastic_r)

                if c1 is None:
                    c1 = c
                    a1 = optimal_a

                if c2 is None and optimal_a!= a1:
                    c2 = c
                    a2 = optimal_a

                if i % train_every == 0:
                    print(f'Observed {i} items over {number_of_observations}.')
                    policy.train()

            cumulative_stochastic_reward_policy = np.cumsum(stats_dict['s_r'])
            cumulative_stochastic_reward_oracle = np.cumsum(stats_dict['stochastic_r'])
            stochastic_regret = cumulative_stochastic_reward_oracle - cumulative_stochastic_reward_policy

            plt.figure(figsize=(16, 9))
            x = np.arange(len(stats_dict['a']))

            ax1 = plt.subplot(221)
            ax1.set_title("Played actions")
            ax1.plot(x, stats_dict['a'], label="policy", linestyle="", marker="^")
            ax1.plot(x, stats_dict['optimal_a'], label="oracle", linestyle="", marker="x")

            ####
            ax4 = plt.subplot(222)
            model = policy.ebm_estimator
            number_of_points = 100
            for action, context in zip([a1,a2],[c1, c2]):
                context_plus_reward = torch.FloatTensor(context).repeat(number_of_points, 1)
                actions = np.linspace(0,5, number_of_points)
                energy = model(context_plus_reward, torch.FloatTensor(actions))
                ax4.plot(actions, energy.detach().numpy(), label=f'Action {action}')

            ax4.set_title("Energy function")
            ax4.axvline(x=1)
            ax4.axvline(x=4)

            ax3 = plt.subplot(224)
            ax3.set_title("Cumulative Regrets")
            ax3.plot(
                x,
                stochastic_regret,
                label="stochastic_regret",
                lw=4,
                alpha=0.8,
            )
            ax3.legend()
            ax4.legend()
            ax1.legend()

            ax2 = plt.subplot(223)

            text = (f'pretrain_time:{pretrain_time}\n'+
                    f'number_of_observations: {number_of_observations}\n'+
                    f'train_every: {train_every}\n'+
                    f'lr: {lr}\n'+
                    f'num_epochs: {num_epochs}\n'+
                    f'loss_function_type: {loss_function_type}\n'+
                    f'sample_size: {sample_size}\n'+
                    f'output_quadratic: {output_quadratic}\n'+
                    f'alpha: {alpha}\n\n' +
                    f'STOCHASTIC REGRET: {stochastic_regret[-1]:,}\n'

                    )

            plt.axis([0, 10, 0, 10])

            plt.text(2, 8, text, fontsize=12, ha='left',
                     va='top', wrap=True)
            plt.axis('off')

            path = (saving_dir / "SUPER_EXPS" /exp_cluster/ datetime.datetime.now().strftime('%Y%m%d%H%M%S')).with_suffix(".png")
            path.parents[0].mkdir(exist_ok=True)
            plt.savefig(path)
            plt.close()


from django.db import migrations

from utils import importer
from utils.importer import (
	get_premeth_subject,
	get_tc_school
)

def set_class_subgroup(apps, schema_editor):
	subject = get_premeth_subject(apps)
	school = get_tc_school(apps)

	all_students = importer.schools.Student.all(student_model=apps.get_model('schools', 'Student'))
	seen_classes = set()
	for student in all_students:
		for cls in student.db_student.subjectclass_set.exclude(id__in=seen_classes):
			cls.subgroup = student.class_code[len(cls.subject.name):]
			cls.save()
			seen_classes.add(cls.id)

class Migration(migrations.Migration):

    dependencies = [
        ('schools', '0003_data_import_classes')
    ]

    operations = [
    	migrations.RunPython(set_class_subgroup)
    ]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys
import os
import json
import logging
import tempfile

import pika
import shutil

import pyhelpers.config as configuration
import pyhelpers.logger as logger
import pyhelpers.inject_cu_classname as inject
from pyhelpers.amqpadapter import AmqpAdapter

template_dir = os.path.expanduser("~/.Meta/template")
deployment_base_dir = os.path.expanduser("~/.Meta/deploy")

config = {}

def write_to_file(filename="default.ll", file_content="NULL"):
    output_path = os.path.join(os.getcwd(), filename)
    try:
        with open(output_path, 'w+') as f:
            f.write(file_content)
    except IOError:
        print("ERROR: Could not write to file: '" + output_path + "'!")

def compile_file(id, ir_content):
    logging.info("### Entering Object file Compilation Phase ###")
#    logging.debug(":::: ir_content -> " + ir_content)

    # create temporary working directory
    tempFolder = tempfile.gettempdir()
    os.chdir(tempFolder)
    logging.debug("changed cwd to: " + os.getcwd())

    write_to_file(id + ".ll", ir_content)

    # assembling IR code to bitcode
    logging.info("-> assembling LLVM-IR to LLVM-Bitcode...")
    bc_file = id + ".bc"
    exitcode = os.system('llvm-as ' + id + '.ll')
    if exitcode != 0:
        logging.error("Could not assemble IR file to LLVM-Bitcode! ({})".format(id))
        os.remove(id + '.ll')
        return None

    # compile object file from bitcode via llc (low level compiler)
    object_file = id + ".o"
    logging.info("-> compiling to linkable object file...")
    exitcode = os.system('llc -relocation-model=pic -filetype=obj ' + bc_file + ' -o ' + object_file)
    if exitcode != 0:
        logging.error("Could not compile LLVM-Bitcode to linkable object file! ({})".format(id))
        os.remove(id + '.ll')
        return None

    logging.info("-> finished compilation chain!")

    # Removing temp files
    logging.info("-> removing IR file")
    os.remove(id + '.ll')

    logging.info("-> removing bitcode file")
    os.remove(bc_file)

    output_path = tempFolder + "/" + object_file
    logging.debug("-> object file was stored at: " + output_path)
    return output_path

def compile_service(id, classname, objectfile):
    logging.info("### Entering Project Compilation Phase ###")

    deployment_dir = deployment_base_dir + "/" + id
    build_dir = deployment_dir + "/build"

    # copy files to deployment_dir
    if os.path.exists(deployment_dir):
        logging.debug("-> Cleaning existing deployment_dir..")
        shutil.rmtree(deployment_dir)
    shutil.copytree(template_dir, deployment_dir)

    logging.info("-> copied template from: " + template_dir + " to " + deployment_dir)
    shutil.move(objectfile, deployment_dir + "/computeunit.o")
    logging.info("-> moved object file from: " + objectfile + " to " + deployment_dir + "/computeunit.o")

    # build the service
    os.chdir(build_dir)
    logging.debug("changed cwd to: " + os.getcwd())

    # inject cu class name
    logging.info("-> injecting ComputeUnit classname into main.cpp..")
    inject.inject_class_name("StubComputeUnit", classname, "..")

    # execute build command
    logging.info("-> building the service..")
    exitcode = os.system("cmake ..")
    if exitcode != 0:
        logging.error("Error running CMake! Could not prepare the build directory. ({})".format(id))
        return None
    exitcode = os.system("cmake --build .")
    if exitcode != 0:
        logging.error("Error running CMake! Could not build the service. ({})".format(id))
        return None

    executable = build_dir + "/meta-backend"
    if not os.path.isfile(executable):
        logging.error("Build failed!")
        return None

    logging.debug("-> Build succeded!")
    logging.info("-> Executable is located at: " + executable)
    return executable

def createIR(unit_base_path, id):
    logging.info("Compiling ComputeUnit to LLVM IR..")
    base = unit_base_path
    unit = base + id + ".cpp"
    includes = os.path.expanduser("~/dev/androidcpp/app/libs/meta/core/")

    logging.info("Creating IR file..")
    os.chdir(base)
    os.system("clang++ -stdlib=libc++ -std=c++11 -I" + includes + " -O3 -S -emit-llvm -fPIC " + unit)
    return base + "/" + id + ".ll"

def test(llfile="", id="MyComputeUnit"):
    global template_dir
    global deployment_base_dir

    template_dir = os.path.expanduser("~/dev/androidcpp/app/libs/meta/template")
    deployment_base_dir = os.path.expanduser("~/dev/androidcpp/app/libs/meta/deploy")

    # CREATE IR ########################
    if llfile == "":
        llfile = createIR(os.path.expanduser("~/dev/androidcpp/app/src/main/cpp/cunits/"), id)

    # LOAD IR INTO MEMORY ##########
    logging.info("Loading IR file into memory..")
    try:
        with open(llfile, "r") as f:
            ir_content = f.read()

        # COMPILE ######################
        logging.info("Compiling stuff..")
        # Compile Stuff
        objectfile = compile_file(id, ir_content)
        executable = compile_service(id, id, objectfile)
    except IOError as e:
        print("IOError in test(): {0}".format(e))

    print("Done.")

def onMessage(json_msg):
    logging.info("received ir with id {}".format(json_msg['Id']))

    objectfile = compile_file(json_msg["Id"], json_msg["Content"])
    if objectfile is None:
        logging.error('Could not generate linkable object file for {}! Aborting.'.format(json_msg['Id']))
        return None

    executable = compile_service(json_msg["Id"], json_msg["Classname"], objectfile)
    if executable is None:
        logging.error('Could not compile the service for {}! Aborting.'.format(json_msg['Id']))
        return None

    del json_msg['Classname']
    json_msg["Content"] = executable
    return json_msg

def main():
    logger.configure('compilation')
    if len(sys.argv) > 1:
        if sys.argv[1] == "-cpp":
            unitfolder = sys.argv[2]
            unitname = sys.argv[3]
            ir = createIR(unitfolder, unitname)
            test(ir, unitname)
        elif sys.argv[1] == "--usage":
            print(  "Usage: \n" +
                    "\tCreate standalone service via an LLVM IR file and the id (name) of the ComputeUnit:\n"
                    f"\t\t{sys.argv[0]} irfile id\n\n" +
                    "\tCreate standalone service via an ComputeUnit's source code and the id (name) of the ComputeUnit:\n"
                    f"\t\t{sys.argv[0]} -cpp cufolder id\n\n" +
                    "\tRun the compilation service as a daemon. Input is consumed via rabbitmq:\n"
                    f"\t\t{sys.argv[0]} -daemon\n"
                    )
            return
        else:
            irfile = sys.argv[1]
            unitname = sys.argv[2]
            test(irfile, unitname)
    else:
        config = configuration.load()
        if config is None:
            logging.error('Could not load config! Aborting.')
            sys.exit(1)

        amqp = AmqpAdapter(config, onMessage)
        amqp.subscribe()

# starting main method
if __name__ == '__main__':
    main()


''' Homework 06 - Exercise 01 '''

import matplotlib.pyplot as plt

fruitNames = ["Apples", "Oranges", "Cherries", "Watermelon"]
fruitQuantity_2020 = [25, 25, 10, 18]
fruitQuantity_2021 = [22, 18, 9, 19]

fig, sp = plt.subplots(2, 2, figsize=(12, 10))
chart, scatter, stack, pie = sp.flatten()
fig.canvas.manager.set_window_title("Exercise 01 Subplots")

# First Plot
chart.title.set_text("Plot Plot")
chart.set_xlabel("Fruit Name")
chart.set_ylabel("Fruit Quantity")
chart.plot(fruitNames, fruitQuantity_2020, label='2020', color='b')
chart.plot(fruitNames, fruitQuantity_2021, label='2021', color='c')
chart.legend()

# Second Plot
scatter.title.set_text("Scatter Plot")
scatter.set_xlabel("Fruit Name")
scatter.set_ylabel("Fruit Quantity")
scatter.scatter(fruitNames, fruitQuantity_2020, label='2020', color='b')
scatter.scatter(fruitNames, fruitQuantity_2021, label='2021', color='c')
scatter.legend()

# Third Plot
stack.title.set_text("Stack Plot")
stack.set_xlabel("Fruit Name")
stack.set_ylabel("Fruit Quantity")
stack.stackplot(fruitNames, fruitQuantity_2020, fruitQuantity_2021, labels=["2020", "2021"], colors=['b', 'c'])
stack.legend(loc="upper right")

# Fourth Plot
# summedQuantities = [x + y for x, y in zip(fruitQuantity_2020, fruitQuantity_2021)]
pie.title.set_text("Pie Plot")
pie.pie([x + y for x, y in zip(fruitQuantity_2020, fruitQuantity_2021)], labels=fruitNames)

# Show Plot
plt.suptitle("Plots of Fruit Information", fontsize=20)
plt.show()


"""empty message

Revision ID: a8ccb5b87433
Revises: b3ba90a2a952
Create Date: 2020-10-07 07:47:46.165032

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'a8ccb5b87433'
down_revision = 'b3ba90a2a952'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('document', sa.Column('title', sa.String(length=200), nullable=False))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('document', 'title')
    # ### end Alembic commands ###


from time import sleep
from unittest.mock import MagicMock

import pytest

from pydcop.infrastructure.computations import MessagePassingComputation, \
    Message, message_type
from pydcop.infrastructure.communication import InProcessCommunicationLayer
from pydcop.infrastructure.agents import Agent, AgentException
from pydcop.infrastructure.discovery import Directory, UnknownComputation

@pytest.fixture
def agent():
    agt = Agent('agt1', InProcessCommunicationLayer())
    yield agt
    agt.stop()

@pytest.fixture
def agents():
    # Agent hosting the directory
    agt_dir = Agent('agt_dir', InProcessCommunicationLayer())
    directory = Directory(agt_dir.discovery)
    agt_dir.add_computation(directory.directory_computation)
    agt_dir.discovery.use_directory('agt_dir', agt_dir.address)
    agt_dir.start()
    agt_dir.run(directory.directory_computation.name)

    # standard agents
    agt1 = Agent('agt1', InProcessCommunicationLayer())
    agt1.discovery.use_directory('agt_dir', agt_dir.address)
    agt1.start()

    agt2 = Agent('agt2', InProcessCommunicationLayer())
    agt2.discovery.use_directory('agt_dir', agt_dir.address)
    agt2.start()

    yield agt_dir, agt1, agt2

    agt1.stop()
    agt2.stop()
    agt_dir.stop()

PingMessage = message_type('ping', ['count'])

class PingComputation(MessagePassingComputation):

    def __init__(self, name: str, target: str=None):
        super().__init__(name)
        self.target = target
        self.ping_count = 0
        self._msg_handlers = {
            'ping' : self._on_ping
        }

    def on_start(self):
        if self.target is not None:
            ping_msg = PingMessage(1)
            self.post_msg(self.target, ping_msg)

    def _on_ping(self, var_name, msg, t):
        sleep(0.1)
        self.ping_count = msg.num + 1
        ping_msg = PingMessage(self.ping_count)
        self.post_msg(self.name, var_name, ping_msg)

def wait_run():
    # Small wait, just to give some slack for other threads to run.
    sleep(0.1)

def test_create():
    comm = InProcessCommunicationLayer()
    agent = Agent('agt1', comm)

    assert agent.name == 'agt1'
    assert not agent.computations()
    assert not agent.is_running
    assert agent.communication == comm
    assert agent.address == comm.address

def test_start(agent):

    agent.start()

    assert agent.is_running
    assert not agent.computations()
    # The discovery computation is deployed when starting the agent
    assert set(c.name for c in agent.computations(include_technical=True))\
        == {'_discovery_agt1'}

def test_stop(agent):

    agent._on_stop = MagicMock()

    agent.start()
    assert agent.is_running
    agent.stop()

    assert agent.is_stopping

    # must wait a little for the thread to terminate
    wait_run()

    assert not agent.is_running
    agent._on_stop.assert_called_once_with()

def test_add_computation_before_start(agent):
    ping = PingComputation('agt1')
    agent.add_computation(ping)

    assert agent.computation(ping.name) == ping
    assert not agent.computation(ping.name).is_running

    with pytest.raises(UnknownComputation):
        agent.computation('foo')

def test_add_computation_after_start(agent):
    agent.start()

    ping = PingComputation('agt1')
    agent.add_computation(ping)

    assert agent.computation(ping.name) == ping
    assert not agent.computation(ping.name).is_running

def test_run_computation(agent):
    ping = PingComputation('agt1')
    ping.on_start = MagicMock()

    agent.add_computation(ping)

    with pytest.raises(AgentException):
        agent.run()
    assert not agent.computation(ping.name).is_running
    ping.on_start.assert_not_called()

    agent.start()
    agent.run()
    wait_run()

    assert agent.computation(ping.name).is_running
    ping.on_start.assert_called_once_with()
    ping.on_start.reset_mock()

    # When calling run twice, the computation must not be started twice
    agent.run()
    wait_run()
    ping.on_start.assert_not_called()

def test_run_computation_by_name(agent):
    ping1 = PingComputation('p1')
    ping2 = PingComputation('p2')
    ping1.on_start = MagicMock()

    agent.add_computation(ping1)
    agent.add_computation(ping2)

    agent.start()
    agent.run(ping1.name)
    wait_run()

    # only ping1 was started, ping2 must not be running
    assert agent.computation(ping1.name).is_running
    assert not agent.computation(ping2.name).is_running
    ping1.on_start.assert_called_once_with()

    # starting an already running computation is armless
    agent.run(ping1.name)

    with pytest.raises(UnknownComputation):
        agent.run('bar')

def test_remove_computation(agent):
    ping1 = PingComputation('p1')
    ping1.on_stop = MagicMock()

    agent.add_computation(ping1)
    assert ping1 in agent.computations()
    agent.remove_computation(ping1.name)
    assert ping1 not in agent.computations()
    # p1 was not running, it must not be stopped:
    ping1.on_stop.assert_not_called()

def test_remove_running_computation(agent):
    ping1 = PingComputation('p1')
    ping1.on_stop = MagicMock()

    agent.add_computation(ping1)
    assert ping1 in agent.computations()
    agent.start()
    agent.run()
    wait_run()

    agent.remove_computation(ping1.name)
    assert ping1 not in agent.computations()
    assert not ping1.is_running
    # p1 was running, it must be stopped:
    ping1.on_stop.assert_called_once_with()

def test_pause_computation(agent):
    ping1 = PingComputation('p1')
    # ping1.on_stop = MagicMock()

    agent.add_computation(ping1)
    assert not ping1.is_paused

    with pytest.raises(AgentException):
        agent.pause_computations(ping1.name)

    agent.start()
    wait_run()

    agent.pause_computations(ping1.name)
    assert ping1.is_paused

    agent.unpause_computations(ping1.name)
    assert not ping1.is_paused

def test_pause_several_computations(agent):
    ping1 = PingComputation('p1')
    ping2 = PingComputation('p2')
    # ping1.on_stop = MagicMock()

    agent.add_computation(ping1)
    agent.add_computation(ping2)

    agent.start()
    wait_run()

    agent.pause_computations(ping1.name)
    agent.pause_computations(ping2.name)
    assert ping1.is_paused
    assert ping2.is_paused

    agent.unpause_computations(None)
    assert not ping1.is_paused
    assert not ping2.is_paused

def test_periodic_action(agent):
    mock = MagicMock()

    def cb(*args):
        mock(args)

    agent.set_periodic_action(0.1, cb)
    agent.start()
    sleep(0.25)
    # Depending on the start instant, the cb might be called 2 or 3 times:
    assert 2 <=len(list(mock.mock_calls)) <= 3

def test_periodic_action_not_called(agent):
    mock = MagicMock()
    def cb(*args):
        mock(args)
    agent.set_periodic_action(1, cb)
    agent.start()
    sleep(0.25)
    # Depending on the start instant, the cb might be called 2 or 3 times:

    assert not list(mock.mock_calls)

def test_several_periodic_action(agent):
    mock1 = MagicMock()

    def cb1(*args):
        mock1(args)

    mock2 = MagicMock()

    def cb2(*args):
        mock2(args)

    agent.set_periodic_action(0.1, cb1)
    agent.set_periodic_action(0.1, cb2)
    agent.start()
    sleep(0.25)
    # Depending on the start instant, the cb might be called 2 or 3 times:
    assert 2 <=len(list(mock1.mock_calls)) <= 3
    assert 2 <=len(list(mock2.mock_calls)) <= 3

def test_several_actions_different_period(agent):
    mock1 = MagicMock()

    def cb1(*args):
        mock1(args)

    mock2 = MagicMock()

    def cb2(*args):
        mock2(args)

    agent.set_periodic_action(0.1, cb1)
    agent.set_periodic_action(0.2, cb2)
    agent.start()
    sleep(0.5)
    # Depending on the start instant, the cb might be called 2 or 3 times:
    assert 4 <=len(list(mock1.mock_calls)) <= 5
    assert 2 <=len(list(mock2.mock_calls)) <= 3

def test_remove_action(agent):
    mock1 = MagicMock()

    def cb1(*args):
        mock1(args)

    handle = agent.set_periodic_action(0.1, cb1)
    agent.start()
    sleep(0.5)
    # Depending on the start instant, the cb might be called 2 or 3 times:
    assert 4 <=len(list(mock1.mock_calls)) <= 5

    agent.remove_periodic_action(handle)
    mock1.reset_mock()
    sleep(0.5)
    mock1.assert_not_called()