#!/usr/bin/env python3

'''

Helper for apps that use PDS databases (that were imported into
SQLite3 databases).

Most routines in this module are private to the module (i.e., those
starting with "_").  There's only a handful of public functions.

'''

import datetime
import pathlib
import re

import PDS

##############################################################################
#
# Public values

# Keys for types of emails
pkey  = 'preferred_emails'
npkey = 'non_preferred_emails'

date_never = datetime.date.fromisoformat('1899-12-30')

##############################################################################

# Which database number to use?
# At ECC, the active database is 1.
_database = 1

def _get_db_num():
    return _database

#-----------------------------------------------------------------------------

# These values are not in the database -- they are hard-coded (!)
def _find_member_types():
    member_types = {
        0 : 'Head of Household',
        1 : 'Spouse',
        2 : 'Adult',
        3 : 'Young Adult',
        4 : 'Child',
        5 : 'Other',
    }

    return member_types

#-----------------------------------------------------------------------------

# Normalize some flags to actual Python booleans
def _normalize_boolean(item, src, dest=None) -> None:
    if dest is None:
        dest=src

    if src not in item:
        item[dest] = False
    elif item[src] == '' or item[src] == 0 or not item[src]:
        item[dest] = False
        if src != dest:
            del item[src]
    elif item[src] == 1:
        item[dest] = True
        if src != dest:
            del item[src]

#-----------------------------------------------------------------------------

# Represent filenames with a Pathlib object, so that it's equally accessible
# when running on Windows and Linux.
def _normalize_filename(item, src) -> None:
    if src not in item:
        return

    if not item[src]:
        del item[src]
        return

    item[src] = pathlib.PureWindowsPath(item[src])

#-----------------------------------------------------------------------------

def _normalize_date(item, sentinel=True):
    if item is None or item == '0000-00-00':
        if sentinel:
            return date_never
        else:
            return None
    else:
        return datetime.date.fromisoformat(item)

#-----------------------------------------------------------------------------

# Compute a salutation for the Head of Household ("HoH" and spouse) for each
# familiy.  PDS allows there to be multiple HoHs and multiple spouse Members in
# any given Family, so be sure to account for that.
#
# Make the salutation be of the form:
#
# first/nickname [and first/nickname [...]] last_name [and first/nickname [and first/nickname [...]] last_name [...]
#
# Examples:
#
# - Andrew and Betty Smith
# - Andrew Smith and Betty Johnson
# - Andrew and Betty Smith and Joseph Johnson
# - Andrew and Betty Smith and Joseph and Geraldine Johnson
#
def _compute_family_hoh_and_spouse_salutations(families, log):
    def _add(last_names, member):
        last = member['last']
        if last not in last_names:
            last_names[last] = list()
        last_names[last].append(member)

    #-----------------------------------------------------------

    for fid in sorted(families.keys()):
        family     = families[fid]

        last_names = dict()
        hoh        = list()
        spouses    = list()

        for member in family['members']:
            last = member['last']

            if 'Head' in member['type']:
                hoh.append(member)
                _add(last_names, member)
            if 'Spouse' in member['type']:
                spouses.append(member)
                _add(last_names, member)

        salutation = ''
        for last_name in sorted(last_names.keys()):
            first_names = list()
            for member in last_names[last_name]:
                if 'nickname' in member and member['nickname'] is not None:
                    first_names.append(member['nickname'])
                elif member['first'] is not None:
                    first_names.append(member['first'])
                else:
                    first_names.append("***UNKNOWN***")
                    log.error("Unknown first name")

            if len(salutation) > 0:
                salutation += ' and '
            salutation += f"{' and '.join(first_names)} {last_name}"

        family['hoh_and_spouse_salutation'] = salutation

#-----------------------------------------------------------------------------

def _load_families(pds, columns=None,
                   active_only=True, log=None):
    db_num = _get_db_num()

    if not columns:
        columns = list()
    columns.append('Name')
    columns.append('MailingName')
    columns.append('ParKey')
    columns.append('StreetAddress1')
    columns.append('StreetAddress2')
    columns.append('StreetCityRec')
    columns.append('StreetZip')
    columns.append('StatDescRec')
    columns.append('PictureFile')
    columns.append('EnvelopeUser')
    columns.append('Visitor')
    columns.append('SendNoMail')
    columns.append('PDSInactive{num}'.format(num=db_num))

    where = ('Fam_DB.CensusFamily{db_num}=1'
             .format(db_num=db_num))
    if active_only:
        where += (' AND '
                  '(Fam_DB.PDSInactive{db_num}=0 OR '
                  'FAM_DB.PDSInactive{db_num} is null)'
                  .format(db_num=db_num))

    families = PDS.read_table(pds, 'Fam_DB', 'FamRecNum',
                              columns=columns, log=log,
                              where=where)

    for f in families.values():
        _normalize_boolean(f, src=f'PDSInactive{db_num}', dest="Inactive")
        _normalize_boolean(f, src='SendNoMail')
        _normalize_boolean(f, src='EnvelopeUser')
        _normalize_filename(f, src='PictureFile')

    return families

#-----------------------------------------------------------------------------

def _load_members(pds, columns=None,
                  active_only=True, log=None):
    db_num = _get_db_num()

    if not columns:
        columns = list()
    columns.append('Name')
    columns.append('FamRecNum')
    columns.append('DateOfBirth')
    columns.append('MonthOfBirth')
    columns.append('DayOfBirth')
    columns.append('YearOfBirth')
    columns.append('Gender')
    columns.append('MaritalStatusRec')
    columns.append('MemberType')
    columns.append('PictureFile')
    columns.append('Location')
    columns.append('LanguageRec')
    columns.append('EthnicDescRec')
    columns.append('User3DescRec') # Skills
    columns.append('User4DescRec') # Occupation
    columns.append('Deceased')
    columns.append('PDSInactive{num}'.format(num=db_num))

    where = ('Mem_DB.CensusMember{db_num}=1'
             .format(db_num=db_num))
    if active_only:
        where += (' AND '
                  'Mem_DB.deceased=0 AND '
                  '(Mem_DB.PDSInactive{db_num}=0 OR '
                  'Mem_DB.PDSInactive{db_num} is null)'
                  .format(db_num=db_num))

    members = PDS.read_table(pds, 'Mem_DB', 'MemRecNum',
                             columns=columns, log=log,
                             where=where)

    for m in members.values():
        _normalize_boolean(m, src='Deceased')
        _normalize_boolean(m, src=f'PDSInactive{db_num}', dest="Inactive")
        _normalize_filename(m, src='PictureFile')
        m['date_of_birth'] = _normalize_date(m['DateOfBirth'], sentinel=False)

    return members

#-----------------------------------------------------------------------------

def _link_families_members(families, members):
    # Make a copy because we don't to delete Members from the
    # main/actual list
    members_copy = members.copy()

    for fid, f in families.items():
        family_members = list()
        for mid in members_copy:
            m = members[mid]

            frn = m['FamRecNum']
            if fid == frn:
                family_members.append(m)
                m['family'] = f

        # Delete all the Members we found from the main list of active
        # members (because we already found their families)
        for m in family_members:
            del members_copy[m['MemRecNum']]

        f['members'] = family_members

#-----------------------------------------------------------------------------

def _delete_non_parishioners(families, members):
    to_delete = list()

    # Look for family ParKey >= 10,000
    for fid, f in families.items():
        parkey = int(f['ParKey'])
        if parkey >= 9000 or f['Visitor']:
            f = families[fid]
            for m in f['members']:
                mid = m['MemRecNum']
                del members[mid]

            to_delete.append(fid)

    for fid in to_delete:
        del families[fid]

#-----------------------------------------------------------------------------

def _link_family_emails(families, emails):
    for f in families.values():
        f[pkey]  = list()
        f[npkey] = list()

    for e in emails.values():
        if not e['FamEmail']:
            continue

        fid = e['MemRecNum']
        if fid not in families:
            continue

        f = families[fid]
        if e['EMailOverMail']:
            key = pkey
        else:
            key = npkey

        addr = '{name} <{addr}>'.format(name=f['Name'],
                                        addr=e['EMailAddress'].strip())
        e['full_address'] = addr
        f[key].append(e)

#-----------------------------------------------------------------------------

def _link_family_city_states(families, city_states):
    for f in families.values():
        csid = f['StreetCityRec']
        if csid and csid in city_states:
            f['city_state'] = city_states[csid]['CityState']
        else:
            # Several places in our Python assume that there is a
            # value in the "city_state" entry.  So rather than go
            # check all of those places, just put an empty string
            # there if there actually is no value.
            f['city_state'] = ''

#-----------------------------------------------------------------------------

def _link_family_statuses(families, fam_status_types):
    for f in families.values():
        id = f['StatDescRec']
        if id in fam_status_types:
            f['status'] = fam_status_types[id]['Description']

#-----------------------------------------------------------------------------

def link_family_or_member_phones(family_or_member, phones, phone_types):
    for p in phones.values():
        family_or_member_id = p['Rec']
        if family_or_member_id not in family_or_member:
            continue

        f = family_or_member[family_or_member_id]
        if 'phones' not in f:
            f['phones'] = list()

        ptr = p['PhoneTypeRec']
        phone_type = ''
        if ptr in phone_types:
            phone_type = phone_types[ptr]['Description']

        _normalize_boolean(p, 'Unlisted')
        f['phones'].append({
            'number'   : p['Number'],
            'type'     : phone_type,
            'unlisted' : p['Unlisted'],
        })

#-----------------------------------------------------------------------------

def _link_family_phones(families, phones, phone_types):
    link_family_or_member_phones(families, phones, phone_types)

#-----------------------------------------------------------------------------

def _link_family_keywords(families, keywords, fam_keywords):
    for fk in fam_keywords.values():
        fid = fk['FamRecNum']
        if fid not in families:
            continue

        f = families[fid]
        if 'keywords' not in f:
            f['keywords'] = list()
        keyword = keywords[fk['DescRec']]['Description']
        f['keywords'].append(keyword)

#-----------------------------------------------------------------------------

def _link_member_types(members, types):
    for m in members.values():
        m['type'] = types[m['MemberType']]

#-----------------------------------------------------------------------------

def _link_member_emails(members, emails):
    for m in members.values():
        m[pkey]  = list()
        m[npkey] = list()

    for e in emails.values():
        if e['FamEmail']:
            continue

        mid = e['MemRecNum']
        if mid not in members:
            continue

        m = members[mid]
        if e['EMailOverMail']:
            key = pkey
        else:
            key = npkey

        addr = '{name} <{addr}>'.format(name=m['email_name'],
                                        addr=e['EMailAddress'].strip())
        e['full_address'] = addr
        m[key].append(e)

#-----------------------------------------------------------------------------

def _link_member_phones(members, phones, phone_types):
    link_family_or_member_phones(members, phones, phone_types)

#-----------------------------------------------------------------------------

def _link_member_keywords(members, keywords, mem_keywords):
    for mk in mem_keywords.values():
        mid = mk['MemRecNum']
        if mid not in members:
            continue

        m = members[mid]
        if 'keywords' not in m:
            m['keywords'] = list()
        keyword = keywords[mk['DescRec']]['Description']
        m['keywords'].append(keyword)

#-----------------------------------------------------------------------------

def _link_member_birth_places(members, birth_places):
    for b in birth_places.values():
        mid = b['AskMemNum']
        if mid not in members:
            continue

        m = members[mid]
        m['birth_place'] = b['BirthPlace']

#-----------------------------------------------------------------------------

def _link_member_ministries(members, ministries, mem_ministries, statuses):
    _link_member_mintal(members, 'ministries', ministries, 'MinDescRec',
                        mem_ministries, statuses)

def _link_member_talents(members, talents, mem_talents, statuses):
    _link_member_mintal(members, 'talents', talents, 'TalDescRec',
                        mem_talents, statuses)

def _link_member_mintal(members, desc, things, thing_index_field,
                        mem_things, statuses):
    akey = f'active_{desc}'
    ikey = f'inactive_{desc}'

    for member in members.values():
        member[akey] = list()
        member[ikey] = list()

    for mt in mem_things.values():
        mid = mt['MemRecNum']
        if mid not in members:
            continue
        m = members[mid]

        status_id = mt['StatusDescRec']
        if not status_id:
            continue
        if status_id not in statuses:
            continue
        status = statuses[status_id]
        mem_list_name = akey
        if status['Active'] != 1:
            mem_list_name = ikey

        thing_id = mt[thing_index_field]

        # Deep copy the ministry record so that we can add some more
        # data in it about this specific member
        thing = things[thing_id].copy()
        thing['active'] = status['Active']
        thing['status'] = status['Description']
        thing['start']  = _normalize_date(mt['StartDate'])
        thing['end']    = _normalize_date(mt['EndDate'])

        m[mem_list_name].append(thing)

#-----------------------------------------------------------------------------

def _link_member_marriage_dates(members, mem_dates, mdtid):
    for md in mem_dates.values():
        if md['DescRec'] != mdtid:
            continue

        mid = md['MemRecNum']
        if mid and mid not in members:
            continue
        m = members[mid]
        m['marriage_date'] = _normalize_date(md['Date'])

#-----------------------------------------------------------------------------

training_req_results = {
    0   :   "Pending",
    1   :   "Yes",
    2   :   "No",
    3   :   "Positive",
    4   :   "Negative",
    5   :   "Received",
    6   :   "Incomplete",
    7   :   "Cleared",
    8   :   "Cleared / Restrictions",
    9   :   "Not Cleared",
    10  :   "Illegible",
    11  :   "Submitted",
    12  :   "Inactive",
    13  :   "Expired",
    14  :   "Archived",
}

def _link_member_requirements(members, mem_reqs, req_types):
    for mr in mem_reqs.values():
        mid = mr['MemRecNum']
        if mid not in members:
            continue

        id = mr['ReqResult']
        if id in training_req_results:
            result = training_req_results[id]
        else:
            result = f'Unknown result {id}'

        m = members[mid]
        key = 'requirements'
        if key not in m:
            m[key] = list()

        m[key].append({
            'description' : req_types[mr['ReqDescRec']]['Description'],
            'start_date'  : _normalize_date(mr['ReqDate']),
            'end_date'    : _normalize_date(mr['ExpirationDate']),
            'result'      : result,
            'note'        : mr['ReqNote'],
        })

#-----------------------------------------------------------------------------

def _link_member_id(members, member_source_field, member_dest_field,
                    values, value_source_field='Description'):
    for member in members.values():
        id = member[member_source_field]
        if id and id in values:
            value = values[id]
            member[member_dest_field] = value[value_source_field]

#-----------------------------------------------------------------------------

# Transform the list of all family fund history (i.e., individual
# contributions) to be:
#
# families['funding'][fid][year][fund_id], a dictionary containing:
#
# * 'fund': PDS DB entry from FundSetup_DB
# * 'history': array of entries, one per contribution of the family that year
# on that fund, each entry containing a dictionary of:
#     * 'activity': name of fund from FuncAct (don't both copying over
#        other data -- the fund name is really the only important thing)
#     * 'fund_id': same as fund_id index in "funding"
#     * 'year': same as year index in "fundung"
#     * 'item': detailed dictionary of information about the contribution.
#       'FEAmt', 'FEComment', 'FEDate' are probably the only relevant fields
#       from this dictionary.
def _link_family_funds(funds, fund_periods, fund_activities,
                       families, all_family_funds, all_family_fund_rates,
                       all_family_fund_history, log):
    # Make a cross reference dictionary of funds by fund ID+year.  It will be
    # used below.
    fund_xref = dict()
    for period in fund_periods.values():
        fund_id = period['FundNumber']
        fund_year = period['FundYear']
        fund = funds[period['SetupRecNum']]

        if fund_year not in fund_xref:
            fund_xref[fund_year] = dict()
        if fund_id not in fund_xref[fund_year]:
            fund_xref[fund_year][fund_id] = dict()

        fund_xref[fund_year][fund_id] = fund

    # Similarly, make a family fund rate cross reference dictionary indexed by
    # family fund IDs, to be used for direct lookups, below.
    family_fund_rate_xref = dict()
    for family_fund_rate in all_family_fund_rates.values():
        family_fund_id = family_fund_rate['FundRecNum']
        family_fund_rate_xref[family_fund_id] = family_fund_rate

    # Do the main work of this method in a standalone dictionary for simplicity.
    # We'll link it into the main "families" dictionary at the end.
    funding = dict()
    for item in all_family_fund_history.values():
        # Make sure this family is in the families dictionary (e.g., if we only
        # have the active families, make sure this is an active family)
        fid = item['FEFamRec']
        if fid not in families:
            continue

        # Transform the item date string into a datetime.date
        item['FEDate'] = _normalize_date(item['FEDate'])

        family_fund = all_family_funds[item['FEFundRec']]
        fund_id     = family_fund['FDFund']
        year        = family_fund['FDYear']
        fund        = fund_xref[year][fund_id]

        # Sometimes activity_id will be None.  Thanks PDS!
        activity_id = item['ActRecNum']
        if activity_id and activity_id in fund_activities:
            activity = fund_activities[activity_id]['Activity']
        else:
            activity = 'None'

        # If the family pledged, they'll have a fund_rate.  If not, they won't.
        family_fund_id = family_fund['FDRecNum']
        if family_fund_id in family_fund_rate_xref:
            fund_rate = family_fund_rate_xref[family_fund_id]
        else:
            fund_rate = None

        # Create the multi-levels in the output
        if fid not in funding:
            funding[fid] = dict()
        if year not in funding[fid]:
            funding[fid][year] = dict()
        if fund_id not in funding[fid][year]:
            funding[fid][year][fund_id] = {
                "fund"      : fund,
                "fund_rate" : fund_rate,
                "history"   : list(),
            }

        funding[fid][year][fund_id]['history'].append({
            "fund_id"  : fund_id,
            "year"     : year,
            "activity" : activity,
            "item"     : item,
        })

    # Now assign the results back to families[fid]['funding']
    for fid in funding:
        # Make sure this family is in the families dictionary (e.g., if we only
        # have the active families, make sure this is an active family). This is
        # technicaly redundant with above, but hey -- defensive programming,
        # right?
        if fid not in families:
            continue

        families[fid]['funds'] = funding[fid]

#-----------------------------------------------------------------------------

def _find_member_marriage_date_type(date_types):
    for dtid, dt in date_types.items():
        if dt['Description'] == 'Marriage':
            return dtid

    return None

#-----------------------------------------------------------------------------

# A full Family name will be formatted:
#
#    Last,First(spouse last,spouse first,spouse title,spouse suffix),Title,Suffix
#
# (spouse) information may not be there
# (spouse last) will not be there if the info is the same
#
# If Middle, Nickname, or Maiden are not provided, those terms
# (including "{}", "()", and "[]") are not included.  E.g., if only
# the nickname is provided:
#
#    Squyres,Jeffrey(Jeff)
#
# If Prefix and Suffix are not provided, those terms are not there,
# either (including the commas).  If only Suffix is supplied, then the
# comma will be there for the Prefix, but it will be empty.  Example:
#
#    Squyres,Jeffrey{Michael}(Jeff),,Esq.
#
# There are no cases in Epiphany's database where someone does not
# have both a first and a last name.  So I didn't even bother trying
# to figure out how that would be stored.

def _parse_family_name(name, log=None):
    parts = name.split(',')
    last = parts[0]

    prefix = None
    if len(parts) > 2:
        prefix = parts[2]
        if prefix == '':
            prefix = None

    suffix = None
    if len(parts) > 3:
        suffix = parts[3]

    # The "more" field may have the middle, nickname, and maiden name.
    # Parse those out.
    first = None
    middle = None
    nickname = None
    maiden = None
    if len(parts) > 1:
        more = parts[1]
        result = re.match('([^\(\{\[]+)', more)
        if result:
            first = result[1]
        else:
            first = 'Unknown'

        result = re.search('\{(.+)\}', more)
        if result:
            middle = result[1]

        result = re.search('\((.+)\)', more)
        if result:
            nickname = result[1]

        result = re.search('\[(.+)\]', more)
        if result:
            maiden = result[1]

    if log:
        log.debug("Last: {l}, First: {f}, Middle: {m}, Nickname: {n}, Maiden: {maiden}, Prefix: {pre}, Suffix: {suff}"
                  .format(l=last,f=first,m=middle,n=nickname,maiden=maiden,pre=prefix,suff=suffix))

    return last, first, middle, nickname, maiden, prefix, suffix

#-----------------------------------------------------------------------------

# A full Member name will be formatted:
#
#    Last,First{Middle}(Nickname}[Maiden],Prefix,Suffix
#
# If Middle, Nickname, or Maiden are not provided, those terms
# (including "{}", "()", and "[]") are not included.  E.g., if only
# the nickname is provided:
#
#    Squyres,Jeffrey(Jeff)
#
# If Prefix and Suffix are not provided, those terms are not there,
# either (including the commas).  If only Suffix is supplied, then the
# comma will be there for the Prefix, but it will be empty.  Example:
#
#    Squyres,Jeffrey{Michael}(Jeff),,Esq.
#
# There are no cases in Epiphany's database where someone does not
# have both a first and a last name.  So I didn't even bother trying
# to figure out how that would be stored.

def _parse_member_name(name, log=None):
    parts = name.split(',')
    last = parts[0]

    prefix = None
    if len(parts) > 2:
        prefix = parts[2]
        if prefix == '':
            prefix = None

    suffix = None
    if len(parts) > 3:
        suffix = parts[3]

    # The "more" field may have the middle, nickname, and maiden name.
    # Parse those out.
    first = None
    middle = None
    nickname = None
    maiden = None
    if len(parts) > 1:
        more = parts[1]
        result = re.match('([^\(\{\[]+)', more)
        if result:
            first = result.group(1)
        else:
            first = 'Unknown'

        result = re.search('\{(.+)\}', more)
        if result:
            middle = result.group(1)

        result = re.search('\((.+)\)', more)
        if result:
            nickname = result.group(1)

        result = re.search('\[(.+)\]', more)
        if result:
            maiden = result.group(1)

    if log:
        log.debug("Last: {l}, First: {f}, Middle: {m}, Nickname: {n}, Maiden: {maiden}, Prefix: {pre}, Suffix: {suff}"
                  .format(l=last,f=first,m=middle,n=nickname,maiden=maiden,pre=prefix,suff=suffix))

    return last, first, middle, nickname, maiden, prefix, suffix

def _parse_member_names(members):
    for _, m in members.items():
        name = m['Name']
        (last, first, middle, nickname, maiden,
         prefix, suffix) = _parse_member_name(name)

        m['first']    = first
        m['middle']   = middle
        m['last']     = last
        m['nickname'] = nickname
        m['maiden']   = maiden
        m['prefix']   = prefix
        m['suffix']   = suffix

        field = 'full_name'
        m[field]     = ''
        if prefix:
            m[field] += prefix + ' '
        if first:
            m[field] += first + ' '
        if nickname:
            m[field] += '("' + nickname + '") '
        if middle:
            m[field] += middle + ' '
        if last:
            m[field] += last
        if maiden:
            m[field] += ' (maiden: ' + maiden + ')'
        if suffix:
            m[field] += ', ' + suffix

        if nickname:
            m['email_name'] = '{nick} {last}'.format(nick=nickname, last=last)
        else:
            m['email_name'] = '{first} {last}'.format(first=first, last=last)

#-----------------------------------------------------------------------------

def _make_emails_lower_case(emails):
    key = 'EMailAddress'
    for e in emails.values():
        addr = e[key].lower().strip()
        e[key] = addr

#-----------------------------------------------------------------------------

# Load PDS Families and Members.  Return them as 2 giant hashes,
# appropriately cross-linked to each other.
def load_families_and_members(filename=None, pds=None,
                              active_only=True, parishioners_only=True,
                              log=None):

    if pds and filename:
        raise Exception("Cannot supply both filename *and* PDS SQLite3 cursor -- only supply one or the other")

    if filename:
        pds = PDS.connect(filename)

    city_states = PDS.read_table(pds, 'City_DB', 'CityRec',
                                 columns=['CityState'], log=log)
    statuses    = PDS.read_table(pds, 'StatusType_DB', 'StatusDescRec',
                                 columns=['Description', 'Active'], log=log)
    ministries  = PDS.read_table(pds, 'MinType_DB', 'MinDescRec',
                                 columns=['Description'], log=log)
    talents     = PDS.read_table(pds, 'TalType_DB', 'TalDescRec',
                                 columns=['Description'], log=log)
    birth_places= PDS.read_table(pds, 'Ask_DB', 'AskRecNum',
                                 columns=['AskMemNum', 'BirthPlace'], log=log)
    date_places = PDS.read_table(pds, 'DatePlace_DB', 'DatePlaceRecNum',
                                 log=log)
    date_types  = PDS.read_table(pds, 'DateType_DB', 'DescRec',
                                 columns=['Description'], log=log)
    phone_types = PDS.read_table(pds, 'PhoneTyp_DB', 'PhoneTypeRec',
                                 columns=['Description'], log=log)
    req_types   = PDS.read_table(pds, 'ReqType_DB', 'ReqDescRec',
                                 columns=['Description', 'Expires'], log=log)
    emails      = PDS.read_table(pds, 'MemEMail_DB', 'EMailRec',
                                 columns=['MemRecNum', 'EMailAddress',
                                          'EMailOverMail', 'FamEmail'],
                                 log=log)
    languages   = PDS.read_table(pds, 'LangType_DB', 'LanguageRec',
                                 columns=['Description'],
                                 log=log)
    mem_phones  = PDS.read_table(pds, 'MemPhone_DB', 'PhoneRec',
                                 columns=['Rec', 'Number', 'PhoneTypeRec', 'Unlisted'],
                                 log=log)
    mem_keyword_types = PDS.read_table(pds, 'MemKWType_DB', 'DescRec',
                                 columns=['Description'], log=log)
    mem_keywords= PDS.read_table(pds, 'MemKW_DB', 'MemKWRecNum',
                                 columns=['MemRecNum', 'DescRec'],
                                 log=log)
    mem_ministries=PDS.read_table(pds, 'MemMin_DB', 'MemKWRecNum',
                                  columns=['MinDescRec', 'MemRecNum',
                                           'StatusDescRec', 'StartDate', 'EndDate'],
                                  log=log)
    mem_talents =PDS.read_table(pds, 'MemTal_DB', 'MemKWRecNum',
                                  columns=['TalDescRec', 'MemRecNum',
                                           'StatusDescRec', 'StartDate', 'EndDate'],
                                  log=log)
    mem_dates   = PDS.read_table(pds, 'MemDates_DB', 'MemDateRecNum',
                                 columns=['MemRecNum', 'Date',
                                          'DescRec'],
                                 log=log)
    mem_ethnics = PDS.read_table(pds, 'EthType_DB', 'EthnicDescRec',
                                 columns=['Description'], log=log)
    mem_3kw     = PDS.read_table(pds, 'User3KW_DB', 'User3DescRec',
                                 columns=['Description'], log=log)
    mem_4kw     = PDS.read_table(pds, 'User4KW_DB', 'User4DescRec',
                                 columns=['Description'], log=log)
    mem_reqs    = PDS.read_table(pds, 'MemReq_DB', 'MemReqRecNum',
                                 columns=['MemRecNum', 'ReqDescRec',
                                          'ReqDate', 'ReqResult',
                                          'ReqNote', 'ExpirationDate'])

    relationship_types = PDS.read_table(pds, 'RelType_DB', 'RelDescRec',
                                        columns=['Description'], log=log)
    marital_statuses = PDS.read_table(pds, 'MemStatType_DB', 'MaritalStatusRec',
                                      columns=['Description'], log=log)

    fam_keyword_types = PDS.read_table(pds, 'FamKWType_DB', 'DescRec',
                                 columns=['Description'], log=log)
    fam_keywords= PDS.read_table(pds, 'FamKW_DB', 'FamKWRecNum',
                                 columns=['FamRecNum', 'DescRec'],
                                 log=log)
    fam_status_types = PDS.read_table(pds, 'FamStatType_DB', 'StatDescRec',
                                      columns=['Description'], log=log)
    fam_phones  = PDS.read_table(pds, 'FamPhone_DB', 'PhoneRec',
                                 columns=['Rec', 'Number', 'PhoneTypeRec', 'Unlisted'],
                                 log=log)

    # Descriptions of each fund
    funds = PDS.read_table(pds, 'FundSetup_DB', 'SetupRecNum',
                                      columns=['FundNumber',
                                                'FundKey',
                                                'FundName'], log=log)
    # Each fund also has one or more time periods associated with it
    fund_periods = PDS.read_table(pds, 'FundPeriod_DB', 'FundPeriodRecNum',
                                columns=['SetupRecNum', 'FundNumber',
                                         'FundYear', 'FundStart', 'FundEnd'],
                                log=log)
    # When a Family contributes, each contribution is assocaited with
    # a "funding activity"
    fund_activities = PDS.read_table(pds, 'FundAct_DB', 'ActRecNum',
                                  columns=['FundRecNum',
                                            'GroupName',
                                            'Activity',
                                            'Function',
                                            'GroupOrder',
                                            'pdsorder'], log=log)

    # Families' activities with relation to the established funds (there is one
    # entry for each family for each fund to which that family has contributed).
    fam_funds = PDS.read_table(pds, 'FamFund_DB', 'FDRecNum',
                            columns=['FDFamRec', 'FDYear', 'FDFund',
                                    'FDOrder', 'MemRecNum', 'Comment'],
                            log=log)
    # Pledging information from the family
    fam_fund_rates = PDS.read_table(pds, 'FamFundRate_DB', 'RateRecNum',
                            columns=['FundRecNum', 'FDStartDate', 'FDEndDate',
                                    'FDRate', 'FDRateAdj', 'FDNumber',
                                    'FDPeriod', 'FDTotal',
                                    'Batch', 'BatchDate'])
    # A listing of each individual contribution from each family,
    # cross-referenced to fam_funds.
    fam_fund_history = PDS.read_table(pds, 'FamFundHist_DB', 'FERecNum',
                                columns=['FEDate', 'ActRecNum', 'FEFundRec',
                                        'FEFamRec', 'FEAmt', 'FEBatch',
                                        'MemRecNum', 'FEChk', 'FEComment'],
                                log=log)

    member_types = _find_member_types()
    mdtid        = _find_member_marriage_date_type(date_types)

    _make_emails_lower_case(emails)

    families = _load_families(pds=pds,
                              active_only=active_only,
                              log=log)
    members  = _load_members(pds=pds,
                             active_only=active_only,
                             log=log)

    _link_families_members(families, members)

    if parishioners_only:
        _delete_non_parishioners(families, members)

    _link_family_emails(families, emails)
    _link_family_city_states(families, city_states)
    _link_family_statuses(families, fam_status_types)
    _link_family_phones(families, fam_phones, phone_types)
    _link_family_keywords(families, fam_keyword_types, fam_keywords)

    _parse_member_names(members)
    _link_member_types(members, member_types)
    _link_member_emails(members, emails)
    _link_member_phones(members, mem_phones, phone_types)
    _link_member_keywords(members, mem_keyword_types, mem_keywords)
    _link_member_birth_places(members, birth_places)
    _link_member_ministries(members, ministries, mem_ministries, statuses)
    _link_member_talents(members, talents, mem_talents, statuses)
    _link_member_marriage_dates(members, mem_dates, mdtid)
    _link_member_requirements(members, mem_reqs, req_types)

    _link_member_id(members, 'MaritalStatusRec', 'marital_status', marital_statuses)
    _link_member_id(members, 'LanguageRec', 'language', languages)
    _link_member_id(members, 'EthnicDescRec', 'ethnic', mem_ethnics)
    _link_member_id(members, 'User3DescRec', 'skills', mem_3kw)
    _link_member_id(members, 'User4DescRec', 'occupation', mem_4kw)

    _link_family_funds(funds, fund_periods, fund_activities,
                       families, fam_funds, fam_fund_rates, fam_fund_history,
                       log)

    # Compute family HoH+Spouse salutations
    _compute_family_hoh_and_spouse_salutations(families, log)

    return pds, families, members

##############################################################################

def _get_sorted_addrs(entries):
    addrs = list()
    for entry in entries:
        addrs.append(entry['EMailAddress'].strip())

    return sorted(addrs)

# If a Member or Family has one or more preferred email addresses,
# return them (as an array).  If there are no preferred email
# addresses, return None.
def find_preferred_email(member_or_family):
    mof = member_or_family
    if pkey in mof and len(mof[pkey]) > 0:
        return _get_sorted_addrs(mof[pkey])
    else:
        return [ ]

# Return either the Member/Family preferred email addresses, or, if
# there are no preferred addresses, return the first (by sorted order)
# non-preferred email address (if it exists).  If no email addresses
# exist, return an empty list.
def find_any_email(member_or_family):
    mof = member_or_family
    addrs = find_preferred_email(mof)
    if addrs:
        return addrs
    elif npkey in mof and len(mof[npkey]) > 0:
        addr = _get_sorted_addrs(mof[npkey])[0]
        return [ addr ]
    else:
        return [ ]


"""
Flask web app connects to Mongo database.
Keep a simple list of dated memoranda.

Representation conventions for dates:
   - In the session object, date or datetimes are represented as
   ISO format strings in UTC.  Unless otherwise specified, this
   is the format passed around internally. Note that ordering
   of ISO format strings is consistent with date/time order
   - User input/output is in local (to the server) time
   - Database representation is as MongoDB 'Date' objects
   Note that this means the database may store a date before or after
   the date specified and viewed by the user, because 'today' in
   Greenwich may not be 'today' here.
"""

import flask
from flask import render_template
from flask import request
from flask import url_for
from flask import redirect

import json
import logging

# Date handling
import arrow # Replacement for datetime, based on moment.js
import datetime # But we may still need time
from dateutil import tz  # For interpreting local times

# Mongo database
from pymongo import MongoClient
from bson.objectid import ObjectId

###
# Globals
###
import CONFIG

app = flask.Flask(__name__)

try:
    dbclient = MongoClient(CONFIG.MONGO_URL)
    db = dbclient.memos
    collection = db.dated

except:
    print("Failure opening database.  Is Mongo running? Correct password?")
    sys.exit(1)

import uuid
app.secret_key = str(uuid.uuid4())

###
# Pages
###

@app.route("/")
@app.route("/index")
def index():
  app.logger.debug("Main page entry")
  flask.session['memos'] = get_memos()
  for memo in flask.session['memos']:
      break
      app.logger.debug("Got Memo: " + str(memo))
  return flask.render_template('index.html')

# We don't have an interface for creating memos yet
@app.route("/create")
def create():
	app.logger.debug("Create")
	return flask.render_template('create.html')

@app.errorhandler(404)
def page_not_found(error):
    app.logger.debug("Page not found")
    return flask.render_template('page_not_found.html',
                                 badurl=request.base_url,
                                 linkback=url_for("index")), 404

#################
#
# Url scripts
#
#################

@app.route("/_create")
def func_create():
	app.logger.debug("Creating new memo")
	t_date = request.args.get('Date')
	t_memo = request.args.get('Memo')
	t_offset = request.args.get('offset', type=int)
	app.logger.debug("New memo: "+t_date+","+str(t_offset)+":"+t_memo)
	t_date = arrow.get(t_date, "MM/DD/YYYY hh:mm A")
	t_tz = timezoned(t_offset)
	app.logger.debug(t_tz)
	t_date = t_date.replace(tzinfo=t_tz)
	put_memo(t_date, t_memo)
	return redirect(url_for('index'))

@app.route("/_delete")
def func_delete():
	app.logger.debug("Deleting a memo")
	t_id = request.args.get('id')
	app.logger.debug("Deleting: "+t_id)
	remove_memo(t_id)
	return redirect(url_for('index'))

#################
#
# Functions used within the templates
#
#################

@app.template_filter( 'fmtdate' )
def format_arrow_date( date ):
	try:
		normal = arrow.get( date )
		return normal.to('local').format("ddd MM/DD/YYYY hh:mm A")
	except:
		return "(bad date)"

@app.template_filter( 'humanize' )
def humanize_arrow_date( date ):
    """
    Date is internal UTC ISO format string.
    Output should be "today", "yesterday", "in 5 days", etc.
    Arrow will try to humanize down to the minute, so we
    need to catch 'today' as a special case.
    """
    try:
        then = arrow.get(date).to('local')
        now = arrow.utcnow().to('local')
        if then.date() == now.date():
            human = "Today"
        else:
            human = then.humanize(now)
            if human == "in a day":
                human = "Tomorrow"
    except:
        human = format_arrow_date(date)
    return human.capitalize()

#############
#
# Functions available to the page code above
#
##############
def get_memos():
    """
    Returns all memos in the database, in a form that
    can be inserted directly in the 'session' object.
    """
    records = [ ]
    for record in collection.find( { "type": "dated_memo" } ):
        record['date'] = arrow.get(record['date']).isoformat()
        record['time'] = arrow.get(record['date']).timestamp
        record['_id'] = str(record['_id'])
        record['text'] = str(record['text'].encode('ascii','xmlcharrefreplace'), "utf-8")
        records.append(record)
    records.sort(key=lambda x: x['time'])
    return records

def put_memo(dt, mem):
	"""
	Place memo into database
	Args:
		dt: Datetime (arrow) object
		mem: Text of memo
	"""
	record = {
		"type": "dated_memo",
		"date": dt.to('utc').naive,
		"text": mem
	}
	collection.insert(record)
	return

def remove_memo(mid):
	"""
	Remove memo from database
	Args:
		mid: Memo _id attribute (string)
	"""
	record = {
		"_id": ObjectId(mid)
	}
	collection.remove(record)
	return

def twoChars(num):
	if num < 10:
		return "0"+str(num)
	return str(num)

def timezoned(minutes):
	t_str = ""
	if minutes >= 0:
		t_str += "+"
	else:
		t_str += "-"
		minutes = abs(minutes)
	t_hours = minutes // 60
	t_str += twoChars(t_hours) + ":"
	minutes -= t_hours * 60
	t_str += twoChars(minutes)
	return t_str

if __name__ == "__main__":
    # App is created above so that it will
    # exist whether this is 'main' or not
    # (e.g., if we are running in a CGI script)
    app.debug=CONFIG.DEBUG
    app.logger.setLevel(logging.DEBUG)
    # We run on localhost only if debugging,
    # otherwise accessible to world
    if CONFIG.DEBUG:
        # Reachable only from the same computer
        app.run(port=CONFIG.PORT)
    else:
        # Reachable from anywhere
        app.run(port=CONFIG.PORT,host="0.0.0.0")


import os
import os.path
import sys
import time
import glob
import json

import biothings, config
biothings.config_for_app(config)

from config import DATA_ARCHIVE_ROOT
from biothings.hub.dataload.dumper import HTTPDumper
from biothings.utils.common import iter_n

class ChemblDumper(HTTPDumper):

    SRC_NAME = "chembl"
    SRC_ROOT_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, SRC_NAME)
    SRC_DATA_URL = "https://www.ebi.ac.uk/chembl/api/data/molecule.json"
    SRC_VERSION_URL = "https://www.ebi.ac.uk/chembl/api/data/status.json"

    SCHEDULE = "0 12 * * *"
    CHUNK_MERGE_SIZE = 100 # number of part files merged together after download

    def remote_is_better(self,remotefile,localfile):
        remote_data = json.loads(self.client.get(self.__class__.SRC_VERSION_URL).text)
        assert "chembl_db_version" in remote_data
        assert remote_data["status"] == "UP" # API is working correctly
        self.release = remote_data["chembl_db_version"]
        # get the total count from the first page
        data = json.loads(self.client.get(self.__class__.SRC_DATA_URL).text)
        self.total_count = data["page_meta"]["total_count"]
        if localfile is None:
            # ok we have the release, we can't compare further so we need to download
            return True
        local_data = json.load(open(localfile))
        # comparing strings should work since it's formatted as "ChEMBL_xxx"
        if remote_data["chembl_db_version"] > local_data["chembl_db_version"]:
            return True
        else:
            return False

    def create_todump_list(self, force=False):
        version_filename = os.path.basename(self.__class__.SRC_VERSION_URL)
        try:
            current_localfile = os.path.join(self.current_data_folder,version_filename)
            if not os.path.exists(current_localfile):
                current_localfile = None
        except TypeError:
            # current data folder doesn't even exist
            current_localfile = None
        remote_better = self.remote_is_better(self.__class__.SRC_VERSION_URL,current_localfile)
        if force or current_localfile is None or remote_better:
            new_localfile = os.path.join(self.new_data_folder,version_filename)
            self.to_dump.append({"remote":self.__class__.SRC_VERSION_URL, "local":new_localfile})
            # now we need to scroll the API endpoint. Let's get the total number of records
            # and generate URLs for each call to parallelize the downloads
            for num,i in enumerate(range(0,self.total_count,1000)):
                remote = self.__class__.SRC_DATA_URL + "?limit=1000&offset=" + str(i)
                local = os.path.join(self.new_data_folder,"molecule.part%d" % num)
                self.to_dump.append({"remote":remote, "local":local})

    def post_dump(self, *args, **kwargs):
        self.logger.info("Merging JSON documents in '%s'" % self.new_data_folder)
        # we'll merge 100 files together, that's 100'000 documents. That way we don't have one huge
        # big files and we don't have thousands of them too. We'll also remove metadata (useless now)
        parts = glob.iglob(os.path.join(self.new_data_folder,"molecule.part*"))
        for chunk,cnt in iter_n(parts,self.__class__.CHUNK_MERGE_SIZE,with_cnt=True):
            outfile = os.path.join(self.new_data_folder,"molecule.%s.json" % cnt)
            merged_data = {"molecules" : []}
            for f in chunk:
                data = json.load(open(f))
                merged_data["molecules"].extend(data["molecules"])
            json.dump(merged_data,open(outfile,"w"))
            self.logger.info("Merged %s files" % cnt)
        # now we can delete the parts
        self.logger.info("Deleting part files")
        parts = glob.iglob(os.path.join(self.new_data_folder,"molecule.part*"))
        for f in parts:
            os.remove(f)
        self.logger.info("Post-dump merge done")


from datetime import datetime, date
import os
import csv
import time
import cv2
import numpy as np

class FileManager(Exception):
    pass

# system navigation
def get_items_in_dir(my_path=""):
    my_path = os.getcwd() if my_path == "" or my_path is None else my_path
    files = []
    for roots, dirs, items in os.walk(my_path):
        [files.append(os.path.join(roots, item)) for item in items]
    return files

def save_frames(color_frame, depth_frame, id_crotal=None, cam="cam01"):
    ts = time.time()
    if id_crotal is not None:
        # own path in savings, lamb crotal
        # mypath = os.path.join(os.getcwd(), "savings", id_crotal)
        # mypath = os.path.join(os.getcwd(), "savings")
        # if not os.path.exists(mypath):
        #     os.makedirs(mypath)
        mypath = os.path.dirname(os.getcwd())

        # path_color = ("savings", "color", date.today())
        # path_color = ("savings", "depth", date.today())

        def mkdirs(current_path, paths):
            path = current_path
            for folder in paths:
                path = os.path.join(path, str(folder))
                if not os.path.exists(path):
                    os.mkdir(path)
            return path

        if id_crotal is None:
            path_color = mkdirs(mypath, ("savings", "color", date.today()))
            path_depth = mkdirs(mypath, ("savings", "depth", date.today()))
        else:
            path_color = mkdirs(mypath, ("savings", "color", id_crotal, date.today()))
            path_depth = mkdirs(mypath, ("savings", "depth", id_crotal, date.today()))

        filename = os.path.join(path_color, "{}_{}_{}.png".format(datetime.fromtimestamp(ts), cam, "color"))

        # path_1 = os.path.join(os.getcwd(), "savings", "{}".format(frame_type),
        #                       str(date.today()), "{}.png".format(datetime.fromtimestamp(ts)))
        #
        # path_2 = os.path.join(os.getcwd(), "savings", "{}".format(frame_type),
        #                       str(date.today()), "{}.png".format(datetime.fromtimestamp(ts)))
        #
        # path_color = os.path.join(mypath, "color")
        # path_depth = os.path.join(mypath, "depth")
        # if not os.path.exists(path_color):
        #     os.makedirs(path_color)
        # if not os.path.exists(path_depth):
        #     os.makedirs(path_depth)
        #
        # dirname = str(date.today())
        # # path with date
        # path_color = os.path.join(path_color, dirname)
        # path_depth = os.path.join(path_depth, dirname)
        # if not os.path.exists(path_color):
        #     os.makedirs(path_color)
        # if not os.path.exists(path_depth):
        #     os.makedirs(path_depth)
        #
        # # number of collections of the lamb
        # # dirname = str(max(map(lambda x: int(x) if x.isdigit() else 0,
        # #                       [f for f in os.listdir(mypath) if os.path.isdir(os.path.join(mypath, f))]),
        # #                   default=0) + 1)
        # # mypath = os.path.join(mypath, dirname)
        # # if not os.path.exists(mypath):
        # #     os.makedirs(mypath)
        #
        # # Save files
        # path_color = os.path.join(path_color, "{}_".format(datetime.fromtimestamp(ts)))
        # path_depth = os.path.join(path_depth, "{}_".format(datetime.fromtimestamp(ts)))

        correct, filename = is_new_file_correct(filename)
        if correct:
            cv2.imwrite(filename=filename, img=color_frame)
        else:
            raise FileManager("filename incorrect!!")
        filename = filename.replace("color", "depth")
        correct, filename = is_new_file_correct(filename)
        if correct:
            cv2.imwrite(filename=filename, img=depth_frame)
        else:
            raise FileManager("filename incorrect!!")
    return

# TODO
def read_frames(filename):
    if os.path.exists(filename) and os.path.isfile(filename):

        color_frame = cv2.imread(filename=filename, mode="RGB")
        depth_frame = cv2.imread(filename=filename, mode="RGB")

        return color_frame, depth_frame
    else:
        print("File Manager ERROR")
        print("Error trying to read the saved frames")
        return None

def record_frames(color_frame, depth_frame, frames_saved, id_crotal_aux):
    ts = time.time()
    mypath = os.path.join(os.getcwd(), "savings", "RGBDIJ", id_crotal_aux,
                          "RGBDIJ_lamb_{}.mine".format(datetime.fromtimestamp(ts)))

    # Save RGBDIJ file
    RGBDIJ.save_file(color_frame, depth_frame, mypath)

    # Save PNG (RGB) file
    mypath = mypath.replace("RGBDIJ", "PNG").replace(".mine", ".png")
    correct, mypath = is_new_file_correct(mypath)
    if correct:
        cv2.imwrite(str(mypath.format(datetime.fromtimestamp(ts))), color_frame)
    return frames_saved, id_crotal_aux

# def save_frames(frames_saved, id_crotal, points, mapped_frame):
#     ts = time.time()
#     if id_crotal is not None:
#         mypath = os.path.join(os.getcwd(), "savings", id_crotal)
#         if 2 >= frames_saved > 0 == frames_saved % 2:
#             # date_day = datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H:%M')
#             # # date_time = datetime.datetime.fromtimestamp(ts).strftime('%H:%M:%S')
#             if not os.path.exists(mypath):
#                 os.makedirs(mypath)
#             dirname = str(max(map(lambda x: int(x) if x.isdigit() else 0,
#                                   [f for f in os.listdir(mypath) if os.path.isdir(os.path.join(mypath, f))]),
#                               default=0) + 1)
#             mypath = os.path.join(mypath, dirname)
#             if not os.path.exists(mypath):
#                 os.makedirs(mypath)
#
#             points.export_to_ply(os.path.join(mypath,
#                                               "{}_pcd_lamb.ply".format(datetime.fromtimestamp(ts))), mapped_frame)
#             frames_saved += 1
#
#         elif 60 >= frames_saved > 2 and frames_saved % 2 == 0:
#             dirname = str(max(map(lambda x: int(x) if x.isdigit() else 0,
#                                   [f for f in os.listdir(mypath) if os.path.isdir(os.path.join(mypath, f))])))
#             mypath = os.path.join(mypath, dirname)
#             points.export_to_ply(os.path.join(mypath,
#                                               "{}_pcd_lamb.ply".format(datetime.fromtimestamp(ts))), mapped_frame)
#             frames_saved += 1
#         elif 0 < frames_saved <= 60:
#             frames_saved += 1
#         else:
#             frames_saved = 0
#             id_crotal = None
#     elif frames_saved > 60:
#         frames_saved = 0
#     return frames_saved, id_crotal
#
#
# def take_dataset_frame(color_image, depth_image, frames_saved, id_crotal_aux):
#     ts = time.time()
#     mypath = os.path.join(os.getcwd(), "savings", "RGBDIJ", id_crotal_aux,
#                           "RGBDIJ_lamb_{}.mine".format(datetime.fromtimestamp(ts)))
#
#     # Save RGBDIJ file
#     RGBDIJ.save_file(color_image, depth_image, mypath)
#
#     # Save PNG (RGB) file
#     mypath = mypath.replace("RGBDIJ", "PNG").replace(".mine", ".png")
#     correct, mypath = is_new_file_correct(mypath)
#     if correct:
#         cv2.imwrite(str(mypath.format(datetime.fromtimestamp(ts))), color_image)
#     return frames_saved, id_crotal_aux

def __is_dir_file_correct__(file):
    dirname = os.path.dirname(file)
    if not os.path.exists(dirname):
        os.makedirs(dirname)

    if os.path.exists(dirname) and os.path.isdir(dirname):
        if os.path.exists(dirname):
            return True, dirname
        else:
            return True, os.getcwd()
    else:
        raise Exception("IS_DIR_FILE_CORRECT: error checking the file, file path not right")
        return False, None

def is_file_correct(file):
    dircorrect, dirname = __is_dir_file_correct__(file)
    file = os.path.join(dirname, os.path.basename(file))
    if dircorrect and os.path.exists(file) and os.path.isfile(file):
        return True, file
    else:
        print("IS_FILE_CORRECT: error checking the file, file path not right")
        return False, None

def is_new_file_correct(file):
    dircorrect, dirname = __is_dir_file_correct__(file)
    file = os.path.join(str(dirname), str(os.path.basename(file)))
    if dircorrect:
        if not os.path.exists(file):
            return True, file
        else:
            # TODO
            # changes filename to "file (1), (2) ..."
            # right now it just overrides the file
            return True, file
    else:
        print("IS_FILE_CORRECT: error checking the file, file path not right")
        return False, None

def is_data_correct(data):
    if data is not None:
        return True
    elif data:
        return True
    else:
        print("IS_DATA_CORRECT: error writing the file, the data is empty")
        return False

# CSV
def write_csv(file, data):
    correct, file = is_new_file_correct(file)
    if correct:
        if is_data_correct(data):
            with open(file, mode='w+') as f:
                writer = csv.DictWriter(f, fieldnames=list(data[0].keys()))
                writer.writeheader()
                writer.writerows(data)
        else:
            print("error writing the csv file, data is empty")
    else:
        print("error writing the csv file, file path not right")

def load_csv(file):
    correct, file = is_file_correct(file)
    if correct and os.path.exists(file):
        with open(file) as f:
            reader = csv.reader(f, delimiter=',')
            line_count = 0
            for row in reader:
                if line_count == 0:
                    print("Column names are {}\n".format(row))
                    line_count += 1
                else:
                    print("{0}}: Column names are {1}}\n".format(line_count, row))
                    line_count += 1
            print("Processed {} lines.\n".format(line_count))

def create_csv():
    """
    Creates a csv file with all the files in the current dir and its expected result for the neural network
    :return:
    """
    files = get_items_in_dir()
    data_RGBDIJ = []
    data_PNG = []

    for item in files:
        if ".mine" in item:
            data = data_RGBDIJ
        elif ".png" in item:
            data = data_PNG
        else:
            continue

        item = item.replace("/home/alberto/Documents/workspace/Robolab/RoboLAMB/RoboLAMB/", "")

        if "con_oveja" in item:
            file = {"path": item, "result": "lamb"}
            data.append(file)
        elif "mal_oveja" in item:
            file = {"path": item, "result": "error"}
            data.append(file)
        elif "sin_oveja" in item:
            file = {"path": item, "result": "empty"}
            data.append(file)

    write_csv(os.path.join(os.getcwd(), "out_RGBDIJ.csv"), data_RGBDIJ)
    write_csv(os.path.join(os.getcwd(), "out_PNG.csv"), data_PNG)

# Working with Images
# RGBD, Open3D_PointCloud, RGBDij
class RGBDIJ:
    #
    # def __init__(self, collection):
    #     # self.collection = np.zeros((480, 640, 6), dtype=np.float64)
    #     self._collection = collection

    @classmethod
    def load_file(cls, file_path):
        color_image = np.zeros((480, 640, 3), dtype=np.uint8)
        depth_image = np.zeros((480, 640, 1), dtype=np.uint16)
        correct, file_path = is_file_correct(file_path)
        if correct:
            with open(file_path, "r") as f:
                for line in f:
                    if len(line) < 2:
                        print(line)
                        continue
                    line = line.split(" ")
                    i = int(line[4])
                    j = int(line[5])
                    color_image[i][j][0] = np.uint8(line[0])
                    color_image[i][j][1] = np.uint8(line[1])
                    color_image[i][j][2] = np.uint8(line[2])
                    depth_image[i][j] = np.uint16(line[3])
            return color_image, depth_image
        else:
            raise Exception("cannot load file " + str(file_path))

    @classmethod
    def save_file(cls, color_image, depth_image, file):
        correct, file_path = is_new_file_correct(file)
        if correct:
            with open(file, "w+") as f:
                for (i, j, pos), value in np.ndenumerate(color_image):
                    f.write(str(value) + " ")
                    if pos == 2:
                        f.write(str(depth_image[i][j]) + " " + str(i) + " " + str(j) + " \n")

if __name__ == '__main__':
    create_csv()


import sys, json
print(sys.path)
import numpy as np
from sklearn.externals import joblib

data = json.loads(sys.argv[1])
data = np.array(data['data'])
sav = joblib.load('/var/www/html/house_price.sav')
pred = sav.predict(data.reshape(1,-1))
result = format(int(round(pred[0],0)),',')
print(result)

mL imp/ML Project 1/ML Project 1/house_price_predict.py


from django.shortcuts import render
from rest_framework.response import Response
from rest_framework import generics
from rest_framework.views import APIView
import datetime

from rest_framework import status

from loggingapp.models import PageView
from loggingapp.serializers import PageViewSerializer
from common.views import GenericCRUDView
from party.models import Party
from partner.models import Partner

from django.db.models import Count, Min, Max

# Create your views here.

# top level uri: /session-logs/

# /page-views/
class PageViewCRUD(GenericCRUDView):
  queryset = PageView.objects.all()
  serializer_class = PageViewSerializer

  def get(self, request, format=None):
    params = request.GET
    obj = self.get_queryset()
    if 'startDate' in params:
      obj = obj.filter(pageViewDate__gte=params['startDate'])
    if 'endDate' in params:
      obj = obj.filter(pageViewDate__lte=params['endDate'])
    serializer = self.serializer_class(obj, many=True)
    return Response(serializer.data)

  def post(self,request, format=None):
    serializer = self.serializer_class(data=request.data)
    if serializer.is_valid():
      serializer.save()
      return Response(serializer.data, status=status.HTTP_201_CREATED)
    return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

  def delete(self, request, format=None):
    return Response({'message':'delete is not enabled for Page View'}, status=status.HTTP_400_BAD_REQUEST)

  def update(self, request):
    return Response({'message':'update is not enabled for Page View'}, status=status.HTTP_400_BAD_REQUEST)

# /sessions/counts/
class SessionCountView(generics.GenericAPIView):

  def get(self, request, format=None):
    startDate = request.GET.get('startDate')
    endDate = request.GET.get('endDate')
    ip = request.GET.get('ip')
    partyId = request.GET.get('partyId')

    filters = {}
    if startDate:
      filters['pageViewDate__gte']=startDate
    if endDate:
      filters['pageViewDate__lte']=endDate
    if ip:
      filters['ip']=ip
    if partyId:
      filters['partyId']=partyId

    distinctSessions = PageView.objects.values('sessionId').distinct().filter(**filters)
    return Response({'count':len(distinctSessions)})


# -*- coding: utf-8 -*-
"""
systemconfig.py
~~~~~~~~~~~~~~

this module handles the request from `system configuration tab` of `admin client`

"""
__author__ = 'Chongmyung Park (chongmyung.park@gmail.com)'

import datetime

from flask import request

from pyticas.tool import json
from pyticas_server import protocol as prot
from pyticas_tetres import api_urls_admin, cfg
from pyticas_tetres.admin_auth import requires_auth
from pyticas_tetres.da.actionlog import ActionLogDataAccess
from pyticas_tetres.da.config import ConfigDataAccess
from pyticas_tetres.db.tetres.model import Config
from pyticas_tetres.logger import getLogger
from pyticas_tetres.sched import scheduler, worker
from pyticas_tetres.ttypes import SystemConfigInfo
from pyticas_tetres.util import actionlog, systemconfig
from pyticas_tetres.systasks import actionlog_processor as actionlog_proc

def register_api(app):
    @app.route(api_urls_admin.SYSCFG_GET, methods=['GET', 'POST'])
    @requires_auth
    def tetres_syscfg_get():
        syscfg = systemconfig.get_system_config_info()
        return prot.response_success(obj=syscfg)

    @app.route(api_urls_admin.SYSCFG_UPDATE, methods=['POST'])
    @requires_auth
    def tetres_syscfg_update():
        cfginfo_json = request.form.get('cfg', None)
        if not cfginfo_json:
            return prot.response_invalid_request()

        cfginfo = json.loads(cfginfo_json, SystemConfigInfo)

        if not cfginfo or not isinstance(cfginfo, SystemConfigInfo):
            return prot.response_invalid_request()

        for k, v in cfginfo.__dict__.items():
            if k.startswith('_'):
                continue
            if v is None:
                return prot.response_invalid_request()

        prev_syscfg = systemconfig.set_system_config_info(cfginfo)
        if not prev_syscfg:
            return prot.response_fail('fail to update configuration')

        put_task_to_actionlog(prev_syscfg)

        return prot.response_success()

def put_task_to_actionlog(prev_syscfg):
    """

    :type prev_syscfg: pyticas_tetres.ttypes.SystemConfigInfo
    """
    da_config = ConfigDataAccess()
    syscfg = da_config.get_by_name(cfg.OPT_NAME_SYSCONFIG)
    if not syscfg:
        getLogger(__name__).warning('Cannot find the updated system configuration from `config` table')
        da_config.close_session()
        return

    should_run_actionlog_handler = False
    da_actionlog = ActionLogDataAccess()

    is_data_archive_start_year_extended = False
    is_data_archive_start_year_shrinked = False

    if cfg.DATA_ARCHIVE_START_YEAR != prev_syscfg.data_archive_start_year:
        if cfg.DATA_ARCHIVE_START_YEAR < prev_syscfg.data_archive_start_year:
            is_data_archive_start_year_extended = True
        elif cfg.DATA_ARCHIVE_START_YEAR > prev_syscfg.data_archive_start_year:
            is_data_archive_start_year_shrinked = True

    # cancled already posted action logs
    if is_data_archive_start_year_shrinked or is_data_archive_start_year_extended:
        ex_logs = da_actionlog.search(
            searches=[('target_datatype', ActionLogDataAccess.DT_SYSTEMCONFIG), ('handled', False)],
            op='and',
            cond='match',
            as_model=True)
        for a_log in ex_logs:
            a_log.handled = True
            a_log.handled_date = datetime.datetime.now()
            a_log.status = 'Cancled due to another action'
            a_log.status_updated_date = datetime.datetime.now()

        da_actionlog.commit()

    if is_data_archive_start_year_extended:
        should_run_actionlog_handler = True
        actionlog.add(ActionLogDataAccess.UPDATE,
                      ActionLogDataAccess.DT_SYSTEMCONFIG,
                      Config.__tablename__,
                      syscfg.id,
                      'DATA_ARCHIVE_START_YEAR_EXTENDED: %d -> %d' % (
                          prev_syscfg.data_archive_start_year, cfg.DATA_ARCHIVE_START_YEAR),
                      handled=False)

    elif is_data_archive_start_year_shrinked:
        # add log for re-calculation
        should_run_actionlog_handler = True
        actionlog.add(ActionLogDataAccess.UPDATE,
                      ActionLogDataAccess.DT_SYSTEMCONFIG,
                      Config.__tablename__,
                      syscfg.id,
                      'DATA_ARCHIVE_START_YEAR_SHRINKED: %d -> %d' % (
                          prev_syscfg.data_archive_start_year, cfg.DATA_ARCHIVE_START_YEAR),
                      handled=False)

    if (cfg.INCIDENT_DOWNSTREAM_DISTANCE_LIMIT != prev_syscfg.incident_downstream_distance_limit
            or cfg.INCIDENT_UPSTREAM_DISTANCE_LIMIT != prev_syscfg.incident_upstream_distance_limit):
        should_run_actionlog_handler = True
        actionlog.add(ActionLogDataAccess.UPDATE,
                      ActionLogDataAccess.DT_SYSTEMCONFIG,
                      Config.__tablename__,
                      syscfg.id,
                      ActionLogDataAccess.DT_INCIDENT,
                      handled=False)

    if (cfg.WZ_DOWNSTREAM_DISTANCE_LIMIT != prev_syscfg.workzone_downstream_distance_limit
            or cfg.WZ_UPSTREAM_DISTANCE_LIMIT != prev_syscfg.workzone_upstream_distance_limit):
        should_run_actionlog_handler = True
        actionlog.add(ActionLogDataAccess.UPDATE,
                      ActionLogDataAccess.DT_SYSTEMCONFIG,
                      Config.__tablename__,
                      syscfg.id,
                      ActionLogDataAccess.DT_WORKZONE,
                      handled=False)

    if (cfg.SE_ARRIVAL_WINDOW != prev_syscfg.specialevent_arrival_window
            or cfg.SE_DEPARTURE_WINDOW1 != prev_syscfg.specialevent_departure_window1
            or cfg.SE_DEPARTURE_WINDOW2 != prev_syscfg.specialevent_departure_window2):
        should_run_actionlog_handler = True
        actionlog.add(ActionLogDataAccess.UPDATE,
                      ActionLogDataAccess.DT_SYSTEMCONFIG,
                      Config.__tablename__,
                      syscfg.id,
                      ActionLogDataAccess.DT_SPECIALEVENT,
                      handled=False)

    # restart scheduler
    if (cfg.DAILY_JOB_START_TIME != prev_syscfg.daily_job_start_time
            or cfg.DAILY_JOB_OFFSET_DAYS != prev_syscfg.daily_job_offset_days
            or cfg.WEEKLY_JOB_START_WEEKDAY != prev_syscfg.weekly_job_start_day
            or cfg.WEEKLY_JOB_START_TIME != prev_syscfg.weekly_job_start_time
            or cfg.MONTHLY_JOB_START_DAY != prev_syscfg.monthly_job_start_date
            or cfg.MONTHLY_JOB_START_TIME != prev_syscfg.monthly_job_start_time):
        scheduler.restart()

        if cfg.DAILY_JOB_START_TIME != prev_syscfg.daily_job_start_time:
            actionlog.add(ActionLogDataAccess.UPDATE,
                          ActionLogDataAccess.DT_SYSTEMCONFIG,
                          Config.__tablename__,
                          syscfg.id,
                          'DAILY_JOB_START_TIME is updated : %s -> %s' % (
                          prev_syscfg.daily_job_start_time, cfg.DAILY_JOB_START_TIME),
                          handled=True)

        if cfg.DAILY_JOB_OFFSET_DAYS != prev_syscfg.daily_job_offset_days:
            actionlog.add(ActionLogDataAccess.UPDATE,
                          ActionLogDataAccess.DT_SYSTEMCONFIG,
                          Config.__tablename__,
                          syscfg.id,
                          'DAILY_JOB_OFFSET_DAYS is updated: %s -> %s' % (
                          prev_syscfg.daily_job_offset_days, cfg.DAILY_JOB_OFFSET_DAYS),
                          handled=True)

        if cfg.WEEKLY_JOB_START_WEEKDAY != prev_syscfg.weekly_job_start_day:
            actionlog.add(ActionLogDataAccess.UPDATE,
                          ActionLogDataAccess.DT_SYSTEMCONFIG,
                          Config.__tablename__,
                          syscfg.id,
                          'WEEKLY_JOB_START_WEEKDAY is updated: %s -> %s' % (
                          prev_syscfg.weekly_job_start_day, cfg.WEEKLY_JOB_START_WEEKDAY),
                          handled=True)

        if cfg.WEEKLY_JOB_START_TIME != prev_syscfg.weekly_job_start_time:
            actionlog.add(ActionLogDataAccess.UPDATE,
                          ActionLogDataAccess.DT_SYSTEMCONFIG,
                          Config.__tablename__,
                          syscfg.id,
                          'WEEKLY_JOB_START_TIME is updated: %s -> %s' % (
                          prev_syscfg.weekly_job_start_time, cfg.WEEKLY_JOB_START_TIME),
                          handled=True)

        if cfg.MONTHLY_JOB_START_DAY != prev_syscfg.monthly_job_start_date:
            actionlog.add(ActionLogDataAccess.UPDATE,
                          ActionLogDataAccess.DT_SYSTEMCONFIG,
                          Config.__tablename__,
                          syscfg.id,
                          'MONTHLY_JOB_START_DAY is updated: %s -> %s' % (
                          prev_syscfg.monthly_job_start_date, cfg.MONTHLY_JOB_START_DAY),
                          handled=True)

        if cfg.MONTHLY_JOB_START_TIME != prev_syscfg.monthly_job_start_time:
            actionlog.add(ActionLogDataAccess.UPDATE,
                          ActionLogDataAccess.DT_SYSTEMCONFIG,
                          Config.__tablename__,
                          syscfg.id,
                          'MONTHLY_JOB_START_TIME is updated: %s -> %s' % (
                          prev_syscfg.monthly_job_start_time, cfg.MONTHLY_JOB_START_TIME),
                          handled=True)

    if not should_run_actionlog_handler:
        unhandled = da_actionlog.list(target_datatypes=[ActionLogDataAccess.DT_SYSTEMCONFIG], handled=False)
        if unhandled:
            should_run_actionlog_handler = True

    da_actionlog.close_session()
    da_config.close_session()

    # add actionlog handler to the task queue in the worker process
    if should_run_actionlog_handler:
        getLogger(__name__).debug('System configurations are updated and the handler process is posted')
        worker.add_task(actionlog_proc.run)
    else:
        getLogger(__name__).debug('System configurations are updated and the handler process is NOT posted')


#!/usr/bin/env python
import website.bigfoot.models as models
from website.bigfoot.utils import config, siteaction, livewire_client, dataplus, statix
from django.http import HttpResponse, HttpResponseRedirect
import datetime, os, sys, Image #PIL from Pythonware, thx guys

max_per_file_size = 7340032     #7 MB

archive_types = ['zip', 'tgz', 'tar', 'gz', 'z']
html_types = ['html', 'htm', 'xhtml']
static_image_types = ['tiff', 'tif', 'psd']
image_types = ['gif', 'jpg', 'jpeg', 'bmp', 'png']
music_types = ['mp3', 'mid', 'wav', 'wma', 'ram']
openoffice_types = ['odt', 'odp', 'ods']
video_types = ['mpg', 'wmv', 'avi', 'mpeg', 'mov', 'rv', 'swf']

archive_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/archive-icon.gif')
file_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/file-icon.gif')
html_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/html-icon.gif')
image_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/image-icon.gif')
music_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/music-icon.gif')
openoffice_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/oo-icon.gif')
pdf_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/pdf-icon.gif')
powerpoint_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/ppt-icon.gif')
video_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/video-icon.gif')
word_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/word-icon.gif')
excel_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/xl-icon.gif')
text_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/text-icon.gif')

def getFileTypeNThumbnail(ext):
    if ext in archive_types:
        return 'archive', archive_icon_url
    if ext in html_types:
        return 'html', html_icon_url
    if ext in static_image_types:
        return 'static_image', image_icon_url
    if ext in image_types:
        return 'image', image_icon_url
    if ext in music_types:
        return 'music', music_icon_url
    if ext in openoffice_types:
        return 'openoffice', openoffice_icon_url
    if ext in video_types:
        return 'video', video_icon_url
    if ext == 'doc':
        return 'word', word_icon_url
    if ext in ['ppt', 'pps']:
        return 'powerpoint', powerpoint_icon_url
    if ext == 'xls':
        return 'excel', excel_icon_url
    if ext == 'pdf':
        return 'pdf', pdf_icon_url
    if ext == 'txt':
        return 'text', text_icon_url
    return 'unknown', file_icon_url

def saveFile(request, chat_id, field_name):
    file = None
    ext = None
    size = None
    try:
        if not request.FILES:
            return False, 'invalid_upload', '', '', '', '', ''

        file = request.FILES[field_name]
        filename = file.name.lower()

        filename_parts = filename.split('.')
        if len(filename_parts) == 1:
            ext = ''
        else:
            ext = filename_parts[len(filename_parts) -1]

        if ext in ['exe', 'vbs', 'wmf']:
            return False, 'unsupported_file_type', '', '', '', '', ''

        filecontent = file.read()
        size = len(filecontent)
        if size > max_per_file_size:
            return False, 'large_upload', '', '', '', '', ''

        #save the main file
        file_dir = config.website_base_path + '/chat/data/' + chat_id
        if not os.path.exists(file_dir):
            os.mkdir(file_dir)

        filename = getNewFilename(file_dir, filename)
        file_path = file_dir + '/' + filename
        f = open(file_path, 'wb')
        f.write(filecontent)
        f.close()

        file_type, thumbnail_url = getFileTypeNThumbnail(ext)

        if file_type == 'image':
            try:
                im = Image.open(file_path).convert('RGB')
                im.thumbnail((128,128), Image.ANTIALIAS)
                tmp_path = file_dir + '/preview-' + filename
                im.save(tmp_path, 'JPEG')
                thumbnail_url = dataplus.getStaticUrl(tmp_path)
            except:
                pass

        return True, '', filename, dataplus.getStaticUrl(file_path), file_type, size, thumbnail_url
    except:
        logError('File upload error: ' + str(sys.exc_info()[0]) + ', ' + str(sys.exc_info()[1]))
        return False, 'unknown', '', '', '', '', ''

def getNewFilename(folderPath, filename):
    namepart = ''
    ext = ''
    idx = filename.find('.')
    if idx != -1:
        namepart = filename[:idx]
        ext  = filename[idx:]
    else:
        namepart = filename

    new_filename = filename
    ctr  = 0
    while(os.path.exists(folderPath + '/' + new_filename)):
        ctr += 1
        new_filename = namepart + '(' + str(ctr) + ')' + ext

    return new_filename

def handle(request):
    if request.method == 'POST':
        upload_id = dataplus.dictGetVal(request.REQUEST, 'uploadId')
        if not upload_id:
            return getRenderedResponse('', False, 'invalid_upload_id')

        chat_id = dataplus.dictGetVal(request.REQUEST, 'chatId')
        if not chat_id:
            return getRenderedResponse(upload_id, False, 'invalid_chat_id')

        success, error_code, filename, file_url, file_type, size, thumbnail_url = saveFile(request, chat_id, 'uploadFile')
        if success:
            session_id = request.COOKIES['session_id']
            result = livewire_client.query('addFile', [session_id, chat_id, filename, file_url, file_type, size, thumbnail_url, ''])
            if result.startswith('error:'):
                return getRenderedResponse(upload_id, False, result[6:])

            return getRenderedResponse(upload_id, True, '')
        else:
            return getRenderedResponse(upload_id, False, error_code)

def getRenderedResponse(upload_id, success, error_code):
    return siteaction.render_to_response('chat/fileupload.htm', {  'upload_id':upload_id,
                                                        'success':('false','true')[success],
                                                        'error_code':error_code})

def logError(err):
    try:
        file = open('/apps/socialray/var/chat_fileupload_errors.txt', 'a')
        file.write(str(datetime.datetime.utcnow()) + '\t' + err + '\n')
    except:
        pass
    finally:
        file.close()


import json

from channels.db import database_sync_to_async
from channels.generic.websocket import AsyncWebsocketConsumer
from django.contrib.auth.models import User

from .models import Chat, OneOnOneRoom
from django.shortcuts import get_object_or_404

class ChatConsumer(AsyncWebsocketConsumer):
    async def connect(self):
        self.room_name = self.scope['url_route']['kwargs']['room_name']
        self.room_group_name = 'chat_%s' % self.room_name

        # Join room group
        await self.channel_layer.group_add(
            self.room_group_name,
            self.channel_name
        )

        await self.accept()

    async def disconnect(self, close_code):
        # Leave room group
        await self.channel_layer.group_discard(
            self.room_group_name,
            self.channel_name
        )

    # Receive message from WebSocket
    async def receive(self, text_data):
        text_data_json = json.loads(text_data)
        message = text_data_json['message']
        save_data = await self.save_data(message)
        data = {
            'sender': save_data.sender.first_name,
            'message': save_data.message,
        }
        # Send message to room group
        await self.channel_layer.group_send(
            self.room_group_name,
            {
                'type': 'chat_message',
                'message': data
            }
        )

    # Receive message from room group
    async def chat_message(self, event):
        message = event['message']

        # Send message to WebSocket
        await self.send(text_data=json.dumps({
            'message': message
        }))

    @database_sync_to_async
    def save_data(self, message):
        this_sender = self.scope['user']
        this_room = get_object_or_404(OneOnOneRoom, room_name=self.room_name)
        message = Chat.objects.create(
            message=message,
            room_name=this_room,
            sender=this_sender,
        )
        return message


import os
import re
import argparse
from PIL import Image

def pngs_path(dir_path):
    '''
    # Return: absolute path list
    '''
    folder = os.listdir(dir_path)

    #pngファイル以外のファイルは削除
    pattern = r".*.png"
    p = re.compile(pattern)
    pngs = list(filter(lambda x: p.match(x),folder))
    abs_pngs = list(map(lambda x:os.path.join(dir_path,x),pngs))
    return abs_pngs

def open_image(path_list):
    '''
    # Return: PIL_object
    '''
    return map(lambda x: Image.open(x),path_list)

def trim_images(pil_imgss,x,y,H,W):
    '''
    # Arguments:
        x(int): トリミングする画像の始点x
        y(int): トリミングする画像の始点y
        H(int): トリミングの高さ
        W(int): トリミングの幅

    # Return: PIL object iterator
    '''
    return map(lambda k: k.crop((x,y,x+W,y+H)),pil_imgs)

def resize_images(pil_imgs,H,W):
    '''
    # Arguments:
        H(int): トリミングの高さ
        W(int): トリミングの幅

    # Return: PIL object iterator
    '''
    return map(lambda k: k.resize((W,H)),pil_imgs)

def rotate_images(pil_imgs,degree_lis):
    '''
    # Arguments:
        degree_lis(list): (例)[0,30,60,90]

    # Return: list
    '''
    rotated_imgs =[]
    for img in pil_imgs:
        for deg in degree_lis:
            rotated = img.rotate(deg,resample=Image.NEAREST)
            rotated_imgs.append(rotated)

    return rotated_imgs

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='image_preprocessing')
    parser.add_argument('-t',help='trimming',action='store_true')
    parser.add_argument('-re',help='resize',action='store_true')
    parser.add_argument('-ro',help='rotate',action='store_true')
    args = parser.parse_args()

    '''
    使い方
    1, 画像が入ったフォルダまでの絶対パスを指定
    2, 保存先までの絶対パスを指定
    3, -t -ro -re　目的にあったオプションをつけてプログラムを実行
    '''

    d_path = r'C:\Users\a20083\tkouno\src\temp_img'
    s_path = r'C:\Users\a20083\tkouno\src\temp_img\crop'
    pil_imgs = open_image(pngs_path(d_path))

    if args.t :
        pil_imgs = trim_images(pil_imgs,300,300,600,600)
    if args.re:
        pil_imgs = resize_images(pil_imgs,100,100)
    if args.ro:
        pil_imgs = rotate_images(pil_imgs,[90])

    for num,img in enumerate(pil_imgs):
        img.save(os.path.join(s_path,f'{num}.png'))


# -*- coding: utf-8 -*-
"""RegressionTorchModel Base class for model with no cell specific parameters"""

import matplotlib.pyplot as plt
# +
import numpy as np
import pandas as pd

from cell2location.models.torch_model import TorchModel

class RegressionTorchModel(TorchModel):
    r"""RegressionTorchModel Base class for model with no cell specific parameters

    :param sample_col: str with column name in cell2covar that denotes sample
    :param cell2covar: pd.DataFrame with covariates in columns and cells in rows, rows should be named.
    :param cell_state_mat: Pandas data frame with gene signatures - genes in row, cell states or factors in columns
    :param X_data: Numpy array of gene expression (cols) in spatial locations (rows)
    :param learning_rate: ADAM learning rate for optimising Variational inference objective
    :param n_iter: number of training iterations
    :param total_grad_norm_constraint: gradient constraints in optimisation
    """

    def __init__(
            self,
            sample_id,
            cell2covar: pd.DataFrame,
            X_data: np.ndarray,
            data_type='float32',
            n_iter=200000,
            learning_rate=0.001,
            total_grad_norm_constraint=200,
            verbose=True,
            var_names=None, var_names_read=None,
            obs_names=None, fact_names=None,
            minibatch_size=None, minibatch_seed=[41, 56, 345],
            phi_hyp_prior=None, prior_eps=1e-8,
            nb_param_conversion_eps=1e-8,
            use_cuda=False,
            use_average_as_initial_value=True,
            stratify_cv=None
    ):

        ############# Initialise parameters ################
        # convert covariates to binary matrix
        # test for column types, get dummies for categorical / character, and just copy over continous
        cell2covar_df = pd.get_dummies(cell2covar.loc[:, ~cell2covar.columns.isin([sample_id])])
        cell2sample_df = pd.get_dummies(cell2covar[[sample_id]])
        cell2sample_covar_df = pd.concat([cell2sample_df, cell2covar_df], axis=1)

        fact_names = cell2sample_covar_df.columns
        n_fact = cell2sample_covar_df.shape[1]

        # extract obs names and sample id
        obs_names = cell2covar.index
        sample_id = cell2covar[sample_id]

        super().__init__(X_data, n_fact,
                         data_type, n_iter,
                         learning_rate, total_grad_norm_constraint,
                         verbose, var_names, var_names_read,
                         obs_names, fact_names, sample_id, use_cuda)

        self.nb_param_conversion_eps = nb_param_conversion_eps

        self.cell_factors_df = None
        self.minibatch_size = minibatch_size
        self.minibatch_seed = minibatch_seed
        self.n_cells_total = self.n_cells
        self.which_sample = self.fact_names.isin(cell2sample_df.columns)
        self.n_samples = np.sum(self.which_sample)
        self.n_covar = self.n_fact - self.n_samples

        self.phi_hyp_prior = phi_hyp_prior
        self.prior_eps = prior_eps

        self.cell2sample_df = cell2sample_df
        self.cell2sample_covar_df = cell2sample_covar_df
        # convert to np.ndarray
        self.cell2sample_mat = cell2sample_df.values
        self.cell2sample_covar_mat = cell2sample_covar_df.values

        # find mean and variance for each gene
        self.gene_mean = (self.X_data + self.prior_eps).mean(0).astype(self.data_type).reshape((1, self.n_genes))
        self.noise_gene_mean = (self.gene_mean / 10).astype(self.data_type).reshape((1, self.n_genes))
        self.prior_gene_mean = np.concatenate([self.noise_gene_mean, self.gene_mean], axis=0)

        self.stratify_cv = stratify_cv

        self.extra_data['cell2sample_covar'] = self.cell2sample_covar_mat

        if use_average_as_initial_value:
            # compute initial value for parameters: cluster averages
            self.cell2sample_covar_sig_mat = self.cell2sample_covar_mat / self.cell2sample_covar_mat.sum(0)
            self.clust_average_mat = np.dot(self.cell2sample_covar_sig_mat.T, self.X_data) + self.prior_eps
            self.clust_average_mat[self.which_sample, :] = self.clust_average_mat[self.which_sample, :] / 10

            # aver = get_cluster_averages(adata_snrna_raw, 'annotation_1') + self.prior_eps
            # variances = get_cluster_variances(adata_snrna_raw, 'annotation_1') + self.prior_eps
            # shape = aver ** 2 / variances
            # shape = shape.mean(1).values
            # overdisp_mean = shape.reshape((1, adata_snrna_raw.shape[1]))
            self.gene_E_mat = None  # np.sqrt(1 / overdisp_mean) # get gene_E ~ Exponential()
        else:
            self.clust_average_mat = None
            self.gene_E_mat = None

    # =====================Other functions======================= #
    def plot_gene_budget(self):

        plt.hist(np.log10(self.samples['post_sample_means']['gene_level'][:, 0]), bins=50)
        plt.xlabel('Gene expression level (hierarchical)')
        plt.title('Gene expression level (hierarchical)')
        plt.tight_layout()

    def sample2df(self, gene_node_name='gene_factors'):
        r""" Export cell factors as Pandas data frames.

        :param node_name: name of the cell factor model parameter to be exported
        :param gene_node_name: name of the gene factor model parameter to be exported
        :return: 8 Pandas dataframes added to model object:
                 .covariate_effects, .covariate_effects_sd, .covariate_effects_q05, .covariate_effects_q95
                 .sample_effects, .sample_effects_sd, .sample_effects_q05, .sample_effects_q95
        """

        # export parameters for covariate effects
        cov_ind = ~ self.which_sample
        self.covariate_effects = \
            pd.DataFrame.from_records(self.samples['post_sample_means'][gene_node_name][cov_ind, :].T,
                                      index=self.var_names,
                                      columns=['mean_' + 'cov_effect_' + i for i in self.fact_names[cov_ind]])

        # export parameters for sample effects
        sample_ind = self.which_sample
        self.sample_effects = \
            pd.DataFrame.from_records(self.samples['post_sample_means'][gene_node_name][sample_ind, :].T,
                                      index=self.var_names,
                                      columns=['mean_' + 'sample_effect' + i for i in self.fact_names[sample_ind]])

    def annotate_cell_adata(self, adata):
        r""" Add covariate and sample coefficients to anndata.var

        :param adata: anndata object to annotate
        :return: updated anndata object
        """

        if self.cell_factors_df is None:
            self.sample2df()

        ### Covariate effect
        # add gene factors to adata
        adata.var[self.covariate_effects.columns] = self.covariate_effects.loc[adata.var.index, :]

        ### Sample effects
        # add gene factors to adata
        adata.var[self.sample_effects.columns] = self.sample_effects.loc[adata.var.index, :]

        return (adata)