from __future__ import with_statement, unicode_literals
import sys
import os
import re
import time
import logging

from ..config import config
if config.get('preferences.use-bsddb3') or sys.version_info[0] >= 3:
    from bsddb3 import db
else:
    from bsddb import db

#-------------------------------------------------------------------------
#
# Gramps modules
#
#-------------------------------------------------------------------------
from ..const import GRAMPS_LOCALE as glocale
_ = glocale.translation.gettext
from ..constfunc import cuni, UNITYPE, handle2internal
from ..lib.markertype import MarkerType
from ..lib.nameorigintype import NameOriginType
from ..lib.place import Place
from ..lib.placeref import PlaceRef
from ..lib.placetype import PlaceType
from ..lib.eventtype import EventType
from ..lib.tag import Tag
from ..utils.file import create_checksum
from ..utils.id import create_id
from . import BSDDBTxn
from .write import _mkname, SURNAMES
from .dbconst import (PERSON_KEY, FAMILY_KEY, EVENT_KEY,
                      MEDIA_KEY, PLACE_KEY, REPOSITORY_KEY, SOURCE_KEY)
from gramps.gui.dialog import (InfoDialog)

LOG = logging.getLogger(".upgrade")

def gramps_upgrade_17(self):
    """
    Upgrade database from version 16 to 17.

    1. This upgrade adds tags to event, place, repository, source and
       citation objects.
    2. Data of Source becomes SourceAttributes Secondary Object
    3. Create a place hierarchy.
    4. Add checksum field to media objects.
    5. Rebuild list of custom events.
    """
    length = (len(self.event_map) + len(self.place_map) +
              len(self.repository_map) + len(self.source_map) +
              len(self.citation_map) + len(self.media_map))
    self.set_total(length)

    # ---------------------------------
    # Modify Event
    # ---------------------------------
    # Add new tag_list field.
    self.event_names = set()
    for handle in self.event_map.keys():
        event = self.event_map[handle]
        new_event = list(event)
        event_type = EventType()
        event_type.unserialize(new_event[2])
        if event_type.is_custom():
            self.event_names.add(str(event_type))
        new_event = new_event[:11] + [[]] + new_event[11:]
        new_event = tuple(new_event)
        with BSDDBTxn(self.env, self.event_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_event)
        self.update()

    # ---------------------------------
    # Modify Place
    # ---------------------------------
    # Convert to hierarchical structure and add new tag_list field.
    locations = {}
    self.max_id = 0
    index = re.compile('[0-9]+')
    for handle in self.place_map.keys():
        place = self.place_map[handle]
        match = index.search(place[1])
        if match:
            if self.max_id <= int(match.group(0)):
                self.max_id = int(match.group(0))
        if place[5] is not None:
            locations[get_location(place[5])] = handle

    for handle in list(self.place_map.keys()):
        place = self.place_map[handle]
        new_place = list(place)

        zip_code = ''
        if new_place[5]:
            zip_code = new_place[5][0][6]

        # find title and type
        main_loc = get_location(new_place[5])
        for level, name in enumerate(main_loc):
            if name:
                break

        loc = list(main_loc[:])
        loc[level] = ''

        # find top parent
        parent_handle = None
        for n in range(7):
            if loc[n]:
                tup = tuple([''] * n + loc[n:])
                parent_handle = locations.get(tup, None)
                if parent_handle:
                    break

        # create nodes
        if parent_handle:
            n -= 1
        while n > level:
            if loc[n]:
                title = ', '.join([item for item in loc[n:] if item])
                parent_handle = add_place(self, loc[n], n, parent_handle, title)
                locations[tuple([''] * n + loc[n:])] = parent_handle
            n -= 1

        if parent_handle is not None:
            placeref = PlaceRef()
            placeref.ref = handle2internal(parent_handle)
            placeref_list = [placeref.serialize()]
        else:
            placeref_list = []

        type_num = 7 - level if name else PlaceType.UNKNOWN
        new_place = new_place[:5] + [placeref_list, name, [],
                    PlaceType(type_num).serialize(), zip_code] + \
                    new_place[6:12] + [[]] + new_place[12:]
        new_place = tuple(new_place)
        with BSDDBTxn(self.env, self.place_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_place)
        self.update()

    # ---------------------------------
    # Modify Repository
    # ---------------------------------
    # Add new tag_list field.
    for handle in self.repository_map.keys():
        repository = self.repository_map[handle]
        new_repository = list(repository)
        new_repository = new_repository[:8] + [[]] + new_repository[8:]
        new_repository = tuple(new_repository)
        with BSDDBTxn(self.env, self.repository_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_repository)
        self.update()

    # ---------------------------------
    # Modify Source
    # ---------------------------------
    # Add new tag_list field.
    for handle in self.source_map.keys():
        source = self.source_map[handle]
        new_source = list(source)
        new_source = new_source[:11] + [[]] + new_source[11:]
        new_source = tuple(new_source)
        with BSDDBTxn(self.env, self.source_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_source)
        self.update()

    # ---------------------------------
    # Modify Citation
    # ---------------------------------
    # Add new tag_list field.
    for handle in self.citation_map.keys():
        citation = self.citation_map[handle]
        new_citation = list(citation)
        new_citation = new_citation[:10] + [[]] + new_citation[10:]
        new_citation = tuple(new_citation)
        with BSDDBTxn(self.env, self.citation_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_citation)
        self.update()

    # -------------------------------------------------------
    # Upgrade Source and Citation datamap to SrcAttributeBase
    # -------------------------------------------------------
    for handle in self.source_map.keys():
        source = self.source_map[handle]
        (handle, gramps_id, title, author, pubinfo,
            notelist, medialist, abbrev, change, datamap, reporef_list,
            taglist, private) = source
        srcattributelist = upgrade_datamap_17(datamap)
        new_source = (handle, gramps_id, title, author, pubinfo,
            notelist, medialist, abbrev, change, srcattributelist, reporef_list,
            taglist, private)
        with BSDDBTxn(self.env, self.source_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_source)
        self.update()

    for handle in self.citation_map.keys():
        citation = self.citation_map[handle]
        (handle, gramps_id, datelist, page, confidence, source_handle,
            notelist, medialist, datamap, change, taglist, private) = citation
        srcattributelist = upgrade_datamap_17(datamap)
        new_citation = (handle, gramps_id, datelist, page, confidence, source_handle,
            notelist, medialist, srcattributelist, change, taglist, private)
        with BSDDBTxn(self.env, self.citation_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_citation)
        self.update()

    # ---------------------------------
    # Modify Media
    # ---------------------------------
    # Add new checksum field.
    base_path = self.metadata.get(b'mediapath', '')
    for handle in self.media_map.keys():
        media = self.media_map[handle]
        new_media = list(media)
        if os.path.isabs(new_media[2]):
            full_path = new_media[2]
        else:
            full_path = os.path.join(base_path, new_media[2])
        checksum = create_checksum(full_path)
        new_media = new_media[:5] + [checksum] + new_media[5:]
        new_media = tuple(new_media)
        with BSDDBTxn(self.env, self.media_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_media)
        self.update()

    # Bump up database version. Separate transaction to save metadata.
    with BSDDBTxn(self.env, self.metadata) as txn:
        txn.put(b'version', 17)

def get_location(loc):
    # (street, locality, parish, city, county, state, country)
    if loc is None:
        location = ('',) * 7
    else:
        location = loc[0][:2] + (loc[1],) + loc[0][2:6]
    return location

def add_place(self, name, level, parent, title):
    handle = create_id()
    place = Place()
    place.handle = handle
    self.max_id += 1
    place.gramps_id = self.place_prefix % self.max_id
    place.set_name(name)
    place.set_title(title)
    place.set_type(PlaceType(7-level))
    if parent is not None:
        placeref = PlaceRef()
        placeref.ref = handle2internal(parent)
        place.set_placeref_list([placeref])
    with BSDDBTxn(self.env, self.place_map) as txn:
        if isinstance(handle, UNITYPE):
            handle = handle.encode('utf-8')
        txn.put(handle, place.serialize())
    return handle

def upgrade_datamap_17(datamap):
    """
    In version 16 key value pairs are stored in source and citation. These become
    SrcAttribute
    """
    new_srcattr_list = []
    private = False
    from ..lib.srcattrtype import SrcAttributeType
    for (key, value) in datamap.items():
        the_type = SrcAttributeType(key).serialize()
        new_srcattr_list.append((private, the_type, value))
    return new_srcattr_list

def gramps_upgrade_16(self):
    """
    Upgrade database from version 15 to 16. This upgrade converts all
    SourceRef child objects to Citation Primary objects.

    For each primary object that has a sourceref, what we have to do is:

        (1) create each citation
        (2) update the object to reference the Citations
        (3) remove backlinks for references from object to Source
        (4) add backlinks for references from object to Citations
        (5) add backlinks for references from Citation to Source

    the backlinks are all updated at the end by calling
    :py:meth:`reindex_reference_map <.write.DbBsddb.reindex_reference_map>`
    """
    # Only People, Families, Events, Media Objects, Places, Sources and
    # Repositories need to be updated, because these are the only primary
    # objects that can have source citations.
    length = (len(self.person_map) +
              len(self.event_map) + len(self.family_map) +
              len(self.repository_map) + len(self.media_map) +
              len(self.place_map) + len(self.source_map))
    self.set_total(length)

    # Setup data for upgrade statistics information dialogue
    keyorder = [PERSON_KEY, FAMILY_KEY, EVENT_KEY, MEDIA_KEY,
                PLACE_KEY, REPOSITORY_KEY, SOURCE_KEY]
    key2data = {
            PERSON_KEY : 0,
            FAMILY_KEY : 1,
            EVENT_KEY: 2,
            MEDIA_KEY: 3,
            PLACE_KEY: 4,
            REPOSITORY_KEY: 5,
            SOURCE_KEY : 6,
            }
    key2string = {
        PERSON_KEY      : _('%6d  People        upgraded with %6d citations in %6d secs\n'),
        FAMILY_KEY      : _('%6d  Families      upgraded with %6d citations in %6d secs\n'),
        EVENT_KEY       : _('%6d  Events        upgraded with %6d citations in %6d secs\n'),
        MEDIA_KEY       : _('%6d  Media Objects upgraded with %6d citations in %6d secs\n'),
        PLACE_KEY       : _('%6d  Places        upgraded with %6d citations in %6d secs\n'),
        REPOSITORY_KEY  : _('%6d  Repositories  upgraded with %6d citations in %6d secs\n'),
        SOURCE_KEY      : _('%6d  Sources       upgraded with %6d citations in %6d secs\n'),
        }
    data_upgradeobject = [0] * 7

    # Initialise the citation gramps ID number
    self.cmap_index = 0

    # ---------------------------------
    # Modify Person
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for person_handle in self.person_map.keys():
        person = self.person_map[person_handle]
        try:
            # The parameters are evaluated before deciding whether logging is on
            # or not. Since the retrieval of names is so complex, I think it is
            # safer to protect this with a try except block, even though it
            # seems to work for names being present and not.
            LOG.debug("upgrade person %s %s" % (person[3][4],
                      " ".join([name[0] for name in person[3][5]])))
        except:
            pass
        (handle, gramps_id, gender, primary_name, alternate_names,
         death_ref_index, birth_ref_index, event_ref_list, family_list,
         parent_family_list, media_list, address_list, attribute_list,
         urls, lds_seal_list, source_list, note_list, change, tag_list,
         private, person_ref_list) = person
        if primary_name:
            primary_name = upgrade_name_16(self, primary_name)
        if alternate_names:
            alternate_names = upgrade_name_list_16(
                                    self, alternate_names)
        if address_list:
            address_list = upgrade_address_list_16(
                                    self, address_list)
        if media_list:
            media_list = upgrade_media_list_16(
                                    self, media_list)
        if attribute_list:
            attribute_list = upgrade_attribute_list_16(
                                    self, attribute_list)
        if lds_seal_list:
            lds_seal_list = upgrade_lds_seal_list_16(
                                    self, lds_seal_list)
        if source_list:
            new_citation_list = convert_source_list_to_citation_list_16(
                                    self, source_list)
        else:
            new_citation_list = []
        if person_ref_list:
            person_ref_list = upgrade_person_ref_list_16(
                                    self, person_ref_list)
        if event_ref_list:
            event_ref_list = upgrade_event_ref_list_16(self, event_ref_list)
        if primary_name or alternate_names  or address_list or \
           media_list or attribute_list or lds_seal_list or source_list or \
           person_ref_list or event_ref_list:
            new_person = (handle, gramps_id, gender, primary_name,
                          alternate_names, death_ref_index,
                          birth_ref_index, event_ref_list, family_list,
                          parent_family_list, media_list, address_list,
                          attribute_list, urls, lds_seal_list,
                          new_citation_list, note_list, change, tag_list,
                          private, person_ref_list)
            LOG.debug("      upgrade new_person %s" % [new_person])
            with BSDDBTxn(self.env, self.person_map) as txn:
                if isinstance(handle, UNITYPE):
                    handle = handle.encode('utf-8')
                txn.put(handle, new_person)
        self.update()

    LOG.debug("%d persons upgraded with %d citations in %d seconds. " %
              (len(list(self.person_map.keys())),
               self.cmap_index - start_num_citations,
               time.time() - start_time))
    data_upgradeobject[key2data[PERSON_KEY]] = (len(list(self.person_map.keys())),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)

    # ---------------------------------
    # Modify Media
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for media_handle in self.media_map.keys():
        media = self.media_map[media_handle]
        LOG.debug("upgrade media object %s" % media[4])
        (handle, gramps_id, path, mime, desc,
         attribute_list, source_list, note_list, change,
         date, tag_list, private) = media
        new_citation_list = convert_source_list_to_citation_list_16(
                                   self, source_list)
        new_attribute_list = upgrade_attribute_list_16(
                                   self, attribute_list)

        new_media = (handle, gramps_id, path, mime, desc,
                     new_attribute_list, new_citation_list, note_list,
                     change, date, tag_list, private)
        LOG.debug("      upgrade new_media %s" % [new_media])
        with BSDDBTxn(self.env, self.media_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_media)
        self.update()

    LOG.debug("%d media objects upgraded with %d citations in %d seconds" %
              (len(self.media_map.keys()),
               self.cmap_index - start_num_citations,
               int(time.time() - start_time)))
    data_upgradeobject[key2data[MEDIA_KEY]] = (len(list(self.media_map.keys())),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)

    # ---------------------------------
    # Modify Places
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for place_handle in self.place_map.keys():
        place = self.place_map[place_handle]
        LOG.debug("upgrade place %s" % place[2])
        (handle, gramps_id, title, longi, lat,
         main_loc, alt_loc, urls, media_list, source_list, note_list,
         change, private) = place
        if source_list:
            new_citation_list = convert_source_list_to_citation_list_16(
                                    self, source_list)
        else:
            new_citation_list = []
        if media_list:
            media_list = upgrade_media_list_16(
                                    self, media_list)
        if source_list or media_list:
            new_place = (handle, gramps_id, title,
                         longi, lat, main_loc, alt_loc, urls,
                         media_list, new_citation_list, note_list,
                         change, private)
            LOG.debug("      upgrade new_place %s" % [new_place])
            with BSDDBTxn(self.env, self.place_map) as txn:
                if isinstance(handle, UNITYPE):
                    handle = handle.encode('utf-8')
                txn.put(handle, new_place)
        self.update()

    LOG.debug("%d places upgraded with %d citations in %d seconds. " %
              (len(list(self.place_map.keys())),
               self.cmap_index - start_num_citations,
               time.time() - start_time))
    data_upgradeobject[key2data[PLACE_KEY]] = (len(list(self.place_map.keys())),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)

    # ---------------------------------
    # Modify Families
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for family_handle in self.family_map.keys():
        family = self.family_map[family_handle]
        LOG.debug("upgrade family (gramps_id) %s" % family[1])
        (handle, gramps_id, father_handle, mother_handle,
         child_ref_list, the_type, event_ref_list, media_list,
         attribute_list, lds_seal_list, source_list, note_list,
         change, tag_list, private) = family
        if source_list:
            new_citation_list = convert_source_list_to_citation_list_16(
                                    self, source_list)
        else:
            new_citation_list = []
        if child_ref_list:
            child_ref_list = upgrade_child_ref_list_16(
                                    self, child_ref_list)
        if lds_seal_list:
            lds_seal_list = upgrade_lds_seal_list_16(
                                    self, lds_seal_list)
        if media_list:
            media_list = upgrade_media_list_16(
                                    self, media_list)
        if attribute_list:
            attribute_list = upgrade_attribute_list_16(
                                    self, attribute_list)
        if event_ref_list:
            event_ref_list = upgrade_event_ref_list_16(self, event_ref_list)
        if source_list or media_list or child_ref_list or \
            attribute_list or lds_seal_list or event_ref_list:
            new_family = (handle, gramps_id, father_handle, mother_handle,
                          child_ref_list, the_type, event_ref_list, media_list,
                          attribute_list, lds_seal_list, new_citation_list,
                          note_list, change, tag_list, private)
            LOG.debug("      upgrade new_family %s" % [new_family])
            with BSDDBTxn(self.env, self.family_map) as txn:
                if isinstance(handle, UNITYPE):
                    handle = handle.encode('utf-8')
                txn.put(handle, new_family)
        self.update()

    LOG.debug("%d families upgraded with %d citations in %d seconds. " %
              (len(list(self.family_map.keys())),
               self.cmap_index - start_num_citations,
               time.time() - start_time))
    data_upgradeobject[key2data[FAMILY_KEY]] = (len(list(self.family_map.keys())),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)
    # ---------------------------------
    # Modify Events
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for event_handle in self.event_map.keys():
        event = self.event_map[event_handle]
        LOG.debug("upgrade event %s" % event[4])
        (handle, gramps_id, the_type, date, description, place,
         source_list, note_list, media_list, attribute_list,
         change, private) = event
        if source_list:
            new_citation_list = convert_source_list_to_citation_list_16(
                                    self, source_list)
        else:
            new_citation_list = []
        if attribute_list:
            attribute_list = upgrade_attribute_list_16(
                                    self, attribute_list)
        if media_list:
            media_list = upgrade_media_list_16(
                                    self, media_list)
        if source_list or attribute_list or media_list:
            new_event = (handle, gramps_id, the_type, date, description, place,
                         new_citation_list, note_list, media_list,
                         attribute_list,
                         change, private)
            LOG.debug("      upgrade new_event %s" % [new_event])
            with BSDDBTxn(self.env, self.event_map) as txn:
                if isinstance(handle, UNITYPE):
                    handle = handle.encode('utf-8')
                txn.put(handle, new_event)
        self.update()

    LOG.debug("%d events upgraded with %d citations in %d seconds. " %
              (len(self.event_map.keys()),
               self.cmap_index - start_num_citations,
               time.time() - start_time))
    data_upgradeobject[key2data[EVENT_KEY]] = (len(list(self.event_map.keys())),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)

    # ---------------------------------
    # Modify Repositories
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for repository_handle in self.repository_map.keys():
        repository = self.repository_map[repository_handle]
        LOG.debug("upgrade repository %s" % repository[3])
        (handle, gramps_id, the_type, name, note_list,
         address_list, urls, change, private) = repository
        if address_list:
            address_list = upgrade_address_list_16(
                                    self, address_list)
        if address_list:
            new_repository = (handle, gramps_id, the_type, name, note_list,
                              address_list, urls, change, private)
            LOG.debug("      upgrade new_repository %s" % [new_repository])
            with BSDDBTxn(self.env, self.repository_map) as txn:
                if isinstance(handle, UNITYPE):
                    handle = handle.encode('utf-8')
                txn.put(handle, new_repository)
        self.update()

    LOG.debug("%d repositories upgraded with %d citations in %d seconds. " %
              (len(list(self.repository_map.keys())),
               self.cmap_index - start_num_citations,
               time.time() - start_time))
    data_upgradeobject[key2data[REPOSITORY_KEY]] = (len(list(self.repository_map.keys())),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)

    # ---------------------------------
    # Modify Source
    # ---------------------------------
    start_num_citations = self.cmap_index
    start_time = time.time()
    for source_handle in self.source_map.keys():
        source = self.source_map[source_handle]
        LOG.debug("upgrade source %s" % source[2])
        (handle, gramps_id, title, author,
         pubinfo, note_list, media_list,
         abbrev, change, datamap, reporef_list,
         private) = source
        if media_list:
            media_list = upgrade_media_list_16(
                                        self, media_list)

        new_source = (handle, gramps_id, title, author,
                      pubinfo, note_list, media_list,
                      abbrev, change, datamap, reporef_list,
                      private)
        LOG.debug("      upgrade new_source %s" % [new_source])
        with BSDDBTxn(self.env, self.source_map) as txn:
            txn.put(str(handle), new_source)
        self.update()

    LOG.debug("%d sources upgraded with %d citations in %d seconds" %
              (len(self.source_map.keys()),
               self.cmap_index - start_num_citations,
               int(time.time() - start_time)))
    data_upgradeobject[key2data[SOURCE_KEY]] = (len(self.source_map.keys()),
                                       self.cmap_index - start_num_citations,
                                       time.time() - start_time)

    # ---------------------------------

    # ---------------------------------
    # Example database from repository took:
    # 3403 events upgraded with 8 citations in 23 seconds. Backlinks took 1071 seconds
    # actually 4 of these citations were from:
    # Media upgrade 4 citations upgraded in 4 seconds
    # by only doing the backlinks when there might be something to do,
    # improved to:
    # 3403 events upgraded with 8 citations in 19 seconds. Backlinks took 1348 seconds
    # further improved by skipping debug logging:
    # 3403 events upgraded with 8 citations in 2 seconds. Backlinks took 167 seconds

    #Number of new objects upgraded:
#  2090  People        upgraded with   2092 citations in 2148 secs
#   734  Families      upgraded with    735 citations in 768 secs
#  3403  Events        upgraded with      4 citations in 212 secs
#     7  Media Objects upgraded with      4 citations in 3 secs
#   852  Places        upgraded with      0 citations in 39 secs

# with reduced diagnostics
#Number of new objects upgraded:
#    73  People        upgraded with     76 citations in     74 secs
#    35  Families      upgraded with     36 citations in     31 secs
#  3403  Events        upgraded with      4 citations in      7 secs
#     7  Media Objects upgraded with      4 citations in      3 secs
#   852  Places        upgraded with      0 citations in      1 secs

# without doing any backlinks
#Number of new objects upgraded:
#    73  People        upgraded with     76 citations in     43 secs
#    35  Families      upgraded with     36 citations in     24 secs
#  3403  Events        upgraded with      4 citations in      6 secs
#     7  Media Objects upgraded with      4 citations in      2 secs
#   852  Places        upgraded with      0 citations in      1 secs

# another run about the same code:
#Number of new objects upgraded:
#    73  People        upgraded with     76 citations in     48 secs
#    35  Families      upgraded with     36 citations in     21 secs
#  3403  Events        upgraded with      4 citations in      9 secs
#     7  Media Objects upgraded with      4 citations in      4 secs
#   852  Places        upgraded with      0 citations in      1 secs

# another run
#Number of new objects upgraded:
#    73  People        upgraded with     76 citations in     36 secs
#    35  Families      upgraded with     36 citations in     18 secs
#  3403  Events        upgraded with      4 citations in      9 secs
#     7  Media Objects upgraded with      4 citations in      2 secs
#   852  Places        upgraded with      0 citations in      1 secs

# without incorrect nestetd tranaction structure:
#Number of new objects upgraded:
#    73  People        upgraded with     76 citations in      0 secs
#    35  Families      upgraded with     36 citations in      0 secs
#  3403  Events        upgraded with      4 citations in      0 secs
#     7  Media Objects upgraded with      4 citations in      0 secs
#   852  Places        upgraded with      0 citations in      0 secs

#[[(73, 76, 0.12430405616760254), (35, 36, 0.042523860931396484), (3403, 4, 0.52303886413574219), (7, 4, 0.058229923248291016), (852, 0, 0.14816904067993164)]]

    # Bump up database version. Separate transaction to save metadata.
    with BSDDBTxn(self.env, self.metadata) as txn:
        txn.put(b'version', 16)

    LOG.debug([data_upgradeobject])
    txt = _("Number of new objects upgraded:\n")
    for key in keyorder:
        try:
            txt += key2string[key] % data_upgradeobject[key2data[key]]
        except:
            txt += key2string[key]
    txt += _("\n\nYou may want to run\n"
             "Tools -> Family Tree Processing -> Merge\n"
             "in order to merge citations that contain similar\n"
             "information")
    InfoDialog(_('Upgrade Statistics'), txt, monospaced=True)

def upgrade_media_list_16(self, media_list):
    new_media_list = []
    for media in media_list:
        (privacy, source_list, note_list, attribute_list, ref, rect) = media
        new_citation_list = convert_source_list_to_citation_list_16(
                                        self, source_list)
        new_attribute_list = upgrade_attribute_list_16(
                                        self, attribute_list)
        new_media = (privacy, new_citation_list, note_list, new_attribute_list,
                     ref, rect)
        new_media_list.append((new_media))
    return new_media_list

def upgrade_attribute_list_16(self, attribute_list):
    new_attribute_list = []
    for attribute in attribute_list:
        (privacy, source_list, note_list, the_type,
         value) = attribute
        new_citation_list = convert_source_list_to_citation_list_16(
                                self, source_list)
        new_attribute = (privacy, new_citation_list, note_list,
                         the_type, value)
        new_attribute_list.append((new_attribute))
    return new_attribute_list

def upgrade_child_ref_list_16(self, child_ref_list):
    new_child_ref_list = []
    for child_ref in child_ref_list:
        (privacy, source_list, note_list, ref, frel, mrel) = child_ref
        new_citation_list = convert_source_list_to_citation_list_16(
                                        self, source_list)
        new_child_ref = (privacy, new_citation_list, note_list, ref, frel, mrel)
        new_child_ref_list.append((new_child_ref))
    return new_child_ref_list

def upgrade_lds_seal_list_16(self, lds_seal_list):
    new_lds_seal_list = []
    for lds_seal in lds_seal_list:
        (source_list, note_list, date, type, place,
         famc, temple, status, private) = lds_seal
        new_citation_list = convert_source_list_to_citation_list_16(
                                        self, source_list)
        new_lds_seal = (new_citation_list, note_list, date, type, place,
                        famc, temple, status, private)
        new_lds_seal_list.append((new_lds_seal))
    return new_lds_seal_list

def upgrade_address_list_16(self, address_list):
    new_address_list = []
    for address in address_list:
        (privacy, source_list, note_list, date, location) = address
        new_citation_list = convert_source_list_to_citation_list_16(
                                        self, source_list)
        new_address = (privacy, new_citation_list, note_list, date, location)
        new_address_list.append((new_address))
    return new_address_list

def upgrade_name_list_16(self, name_list):
    new_name_list = []
    for name in name_list:
        new_name = upgrade_name_16(self, name)
        new_name_list.append((new_name))
    return new_name_list

def upgrade_name_16(self, name):
    (privacy, source_list, note, date, first_name, surname_list, suffix,
     title, name_type, group_as, sort_as, display_as, call, nick,
     famnick) = name
    new_citation_list = convert_source_list_to_citation_list_16(
                                    self, source_list)
    new_name = (privacy, new_citation_list, note, date, first_name,
                surname_list, suffix, title, name_type, group_as, sort_as,
                display_as, call, nick, famnick)
    return new_name

def upgrade_person_ref_list_16(self, person_ref_list):
    new_person_ref_list = []
    for person_ref in person_ref_list:
        (privacy, source_list, note_list, ref, rel) = person_ref
        new_citation_list = convert_source_list_to_citation_list_16(
                                        self, source_list)
        new_person_ref = (privacy, new_citation_list, note_list, ref, rel)
        new_person_ref_list.append((new_person_ref))
    return new_person_ref_list

def upgrade_event_ref_list_16(self, event_ref_list):
    new_event_ref_list = []
    for event_ref in event_ref_list:
        (privacy, note_list, attribute_list, ref, role) = event_ref
        new_attribute_list = upgrade_attribute_list_16(
                                        self, attribute_list)
        new_event_ref = (privacy, note_list, new_attribute_list, ref, role)
        new_event_ref_list.append((new_event_ref))
    return new_event_ref_list

def convert_source_list_to_citation_list_16(self, source_list):
    citation_list = []
    for source in source_list:
        (date, private, note_list, confidence, ref, page) = source
        new_handle = create_id()
        new_media_list = []
        new_data_map = {}
        new_change = time.time()
        new_gramps_id = self.citation_prefix % self.cmap_index
        new_citation = (new_handle, new_gramps_id,
                        date, page, confidence, ref, note_list, new_media_list,
                        new_data_map, new_change, private)
        with BSDDBTxn(self.env, self.citation_map) as txn:
            if isinstance(new_handle, UNITYPE):
                new_handle = new_handle.encode('utf-8')
            txn.put(new_handle, new_citation)
        self.cmap_index += 1
        citation_list.append((new_handle))
    return citation_list

def gramps_upgrade_15(self):
    """
    Upgrade database from version 14 to 15. This upgrade adds:

        * tagging
        * surname list
        * remove marker
    """
    length = (len(self.note_map) + len(self.person_map) +
              len(self.event_map) + len(self.family_map) +
              len(self.repository_map) + len(self.media_map) +
              len(self.place_map) + len(self.source_map)) + 10
    self.set_total(length)
    self.tags = {}

    # ---------------------------------
    # Modify Person
    # ---------------------------------
    # Replace the old marker field with the new tag list field.
    for handle in self.person_map.keys():
        person = self.person_map[handle]

        (junk_handle,        #  0
         gramps_id,          #  1
         gender,             #  2
         primary_name,       #  3
         alternate_names,    #  4
         death_ref_index,    #  5
         birth_ref_index,    #  6
         event_ref_list,     #  7
         family_list,        #  8
         parent_family_list, #  9
         media_list,         # 10
         address_list,       # 11
         attribute_list,     # 12
         urls,               # 13
         ord_list,           # 14
         psource_list,       # 15
         pnote_list,         # 16
         change,             # 17
         marker,             # 18
         pprivate,           # 19
         person_ref_list,    # 20
         ) = person

        tag_handle = convert_marker(self, marker)
        if tag_handle:
            tags = [tag_handle]
        else:
            tags = []
        address_list = list(map(convert_address, address_list))
        new_primary_name = convert_name_15(primary_name)
        new_alternate_names = list(map(convert_name_15, alternate_names))
        new_person = (junk_handle,        #  0
                      gramps_id,          #  1
                      gender,             #  2
                      new_primary_name,   #  3
                      new_alternate_names,#  4
                      death_ref_index,    #  5
                      birth_ref_index,    #  6
                      event_ref_list,     #  7
                      family_list,        #  8
                      parent_family_list, #  9
                      media_list,         # 10
                      address_list,       # 11
                      attribute_list,     # 12
                      urls,               # 13
                      ord_list,           # 14
                      psource_list,       # 15
                      pnote_list,         # 16
                      change,             # 17
                      tags,               # 18
                      pprivate,           # 19
                      person_ref_list     # 20
                      )

        with BSDDBTxn(self.env, self.person_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_person)
        self.update()
    #surname is now different, remove secondary index with names
    _db = db.DB(self.env)
    try:
        _db.remove(_mkname(self.full_name, SURNAMES), SURNAMES)
    except db.DBNoSuchFileError:
        pass

    # ---------------------------------
    # Modify Family
    # ---------------------------------
    # Replace the old marker field with the new tag list field.
    for handle in self.family_map.keys():
        family = self.family_map[handle]
        new_family = list(family)
        tag_handle = convert_marker(self, new_family[13])
        if tag_handle:
            new_family[13] = [tag_handle]
        else:
            new_family[13] = []
        new_family = tuple(new_family)
        with BSDDBTxn(self.env, self.family_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_family)
        self.update()

    # ---------------------------------
    # Modify Note
    # ---------------------------------
    # Replace the old marker field with the new tag list field.
    for handle in self.note_map.keys():
        note = self.note_map[handle]
        new_note = list(note)
        tag_handle = convert_marker(self, new_note[6])
        if tag_handle:
            new_note[6] = [tag_handle]
        else:
            new_note[6] = []
        new_note = tuple(new_note)
        with BSDDBTxn(self.env, self.note_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_note)
        self.update()

    # ---------------------------------
    # Modify Media object
    # ---------------------------------
    # Replace the old marker field with the new tag list field.
    for handle in self.media_map.keys():
        media = self.media_map[handle]
        new_media = list(media)
        new_media[10] = []
        new_media = tuple(new_media)
        with BSDDBTxn(self.env, self.media_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_media)
        self.update()

    # ---------------------------------
    # Modify Event
    # ---------------------------------
    # Replace the old marker field with the new tag list field.
    for handle in self.event_map.keys():
        event = self.event_map[handle]
        new_event = list(event)
        new_event = new_event[:11] + new_event[12:]
        #new_event[11] = []
        new_event = tuple(new_event)
        with BSDDBTxn(self.env, self.event_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_event)
        self.update()

    # ---------------------------------
    # Modify Place
    # ---------------------------------
    # Remove the old marker field, set new locality.
    for handle in self.place_map.keys():
        place = self.place_map[handle]
        new_place = list(place)
        if new_place[5] is not None:
            new_place[5] = convert_location(new_place[5])
        new_place[6] = list(map(convert_location, new_place[6]))
        new_place = new_place[:12] + new_place[13:]
        new_place = tuple(new_place)
        with BSDDBTxn(self.env, self.place_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_place)
        self.update()

    # ---------------------------------
    # Modify Source
    # ---------------------------------
    # Remove the old marker field.
    for handle in self.source_map.keys():
        source = self.source_map[handle]
        new_source = list(source)
        new_source = new_source[:11] + new_source[12:]
        new_source = tuple(new_source)
        with BSDDBTxn(self.env, self.source_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_source)
        self.update()

    # ---------------------------------
    # Modify Repository
    # ---------------------------------
    # Remove the old marker field, set new locality.
    for handle in self.repository_map.keys():
        repository = self.repository_map[handle]
        new_repository = list(repository)
        new_repository = new_repository[:8] + new_repository[9:]
        new_repository[5] = list(map(convert_address, new_repository[5]))
        new_repository = tuple(new_repository)
        with BSDDBTxn(self.env, self.repository_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_repository)
        self.update()

    # Bump up database version. Separate transaction to save metadata.
    with BSDDBTxn(self.env, self.metadata) as txn:
        txn.put(b'version', 15)

def convert_marker(self, marker_field):
    """Convert a marker into a tag."""
    marker = MarkerType()
    marker.unserialize(marker_field)
    tag_name = cuni(marker)

    if tag_name != '':
        if tag_name not in self.tags:
            tag = Tag()
            handle = create_id()
            tag.set_handle(handle)
            tag.set_change_time(time.time())
            tag.set_name(tag_name)
            tag.set_priority(len(self.tags))
            with BSDDBTxn(self.env, self.tag_map) as txn:
                if isinstance(handle, UNITYPE):
                    handle = handle.encode('utf-8')
                txn.put(handle, tag.serialize())
            self.tags[tag_name] = handle
        return self.tags[tag_name]
    else:
        return None

def convert_locbase(loc):
    """Convert location base to include an empty locality field."""
    return tuple([loc[0], ''] + list(loc[1:]))

def convert_location(loc):
    """Convert a location into the new format."""
    return (convert_locbase(loc[0]), loc[1])

def convert_address(addr):
    """Convert an address into the new format."""
    return (addr[0], addr[1], addr[2], addr[3], convert_locbase(addr[4]))

def convert_name_15(name):
    (privacy, source_list, note_list, date,
     first_name, surname, suffix, title,
     name_type, prefix, patronymic,
     group_as, sort_as, display_as, call) = name

    connector = ""
    origintype = (NameOriginType.NONE, "")
    patorigintype = (NameOriginType.PATRONYMIC, "")

    if patronymic.strip() == "":
        #no patronymic, create a single surname
        surname_list = [(surname, prefix, True, origintype, connector)]
    else:
        #a patronymic, if no surname or equal as patronymic, a single surname
        if (surname.strip() == "") or (surname == patronymic and prefix == ""):
            surname_list = [(patronymic, prefix, True, patorigintype, connector)]
        else:
            #two surnames, first patronymic, then surname which is primary
            surname_list = [(patronymic, "", False, patorigintype, ""),
                            (surname, prefix, True, origintype, connector)]

    #return new value, add two empty strings for nick and family nick
    return (privacy, source_list, note_list, date,
         first_name, surname_list, suffix, title, name_type,
         group_as, sort_as, display_as, call, "", "")

def gramps_upgrade_14(self):
    """Upgrade database from version 13 to 14."""
    # This upgrade modifies notes and dates
    length = (len(self.note_map) + len(self.person_map) +
              len(self.event_map) + len(self.family_map) +
              len(self.repository_map) + len(self.media_map) +
              len(self.place_map) + len(self.source_map))
    self.set_total(length)

    # ---------------------------------
    # Modify Notes
    # ---------------------------------
    # replace clear text with StyledText in Notes
    for handle in self.note_map.keys():
        note = self.note_map[handle]
        (junk_handle, gramps_id, text, format, note_type,
         change, marker, private) = note
        styled_text = (text, [])
        new_note = (handle, gramps_id, styled_text, format, note_type,
                    change, marker, private)
        with BSDDBTxn(self.env, self.note_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_note)
        self.update()

    # ---------------------------------
    # Modify Event
    # ---------------------------------
    # update dates with newyear
    for handle in self.event_map.keys():
        event = self.event_map[handle]
        (junk_handle, gramps_id, the_type, date, description, place,
         source_list, note_list, media_list, attribute_list,
         change, marker, private) = event
        new_date = convert_date_14(date)
        new_source_list = new_source_list_14(source_list)
        new_media_list = new_media_list_14(media_list)
        new_attribute_list = new_attribute_list_14(attribute_list)
        new_event = (junk_handle, gramps_id, the_type, new_date,
                     description, place, new_source_list, note_list,
                     new_media_list, new_attribute_list, change,marker,private)
        with BSDDBTxn(self.env, self.event_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_event)
        self.update()

    # ---------------------------------
    # Modify Person
    # ---------------------------------
    # update dates with newyear
    for handle in self.person_map.keys():
        person = self.person_map[handle]
        (junk_handle,        #  0
         gramps_id,          #  1
         gender,             #  2
         primary_name,       #  3
         alternate_names,    #  4
         death_ref_index,    #  5
         birth_ref_index,    #  6
         event_ref_list,     #  7
         family_list,        #  8
         parent_family_list, #  9
         media_list,         # 10
         address_list,       # 11
         attribute_list,     # 12
         urls,               # 13
         lds_ord_list,       # 14
         psource_list,       # 15
         pnote_list,         # 16
         change,             # 17
         marker,             # 18
         pprivate,           # 19
         person_ref_list,    # 20
         ) = person

        new_address_list = []
        for address in address_list:
            (privacy, asource_list, anote_list, date, location) = address
            new_date = convert_date_14(date)
            new_asource_list = new_source_list_14(asource_list)
            new_address_list.append((privacy, new_asource_list, anote_list,
                                     new_date, location))
        new_ord_list = []
        for ldsord in lds_ord_list:
            (lsource_list, lnote_list, date, type, place,
             famc, temple, status, lprivate) = ldsord
            new_date = convert_date_14(date)
            new_lsource_list = new_source_list_14(lsource_list)
            new_ord_list.append( (new_lsource_list, lnote_list, new_date, type,
                                  place, famc, temple, status, lprivate))

        new_primary_name = convert_name_14(primary_name)

        new_alternate_names = [convert_name_14(name) for name
                               in alternate_names]

        new_media_list = new_media_list_14(media_list)
        new_psource_list = new_source_list_14(psource_list)
        new_attribute_list = new_attribute_list_14(attribute_list)
        new_person_ref_list = new_person_ref_list_14(person_ref_list)

        new_person = (junk_handle,        #  0
                      gramps_id,          #  1
                      gender,             #  2
                      new_primary_name,       #  3
                      new_alternate_names,    #  4
                      death_ref_index,    #  5
                      birth_ref_index,    #  6
                      event_ref_list,     #  7
                      family_list,        #  8
                      parent_family_list, #  9
                      new_media_list,         # 10
                      new_address_list,       # 11
                      new_attribute_list,     # 12
                      urls,               # 13
                      new_ord_list,          # 14
                      new_psource_list,       # 15
                      pnote_list,         # 16
                      change,             # 17
                      marker,             # 18
                      pprivate,           # 19
                      new_person_ref_list,    # 20
                      )

        with BSDDBTxn(self.env, self.person_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_person)
        self.update()

    # ---------------------------------
    # Modify Family
    # ---------------------------------
    # update dates with newyear
    for handle in self.family_map.keys():
        family = self.family_map[handle]
        (junk_handle, gramps_id, father_handle, mother_handle,
         child_ref_list, the_type, event_ref_list, media_list,
         attribute_list, lds_seal_list, source_list, note_list,
         change, marker, private) = family
        new_child_ref_list = new_child_ref_list_14(child_ref_list)
        new_media_list = new_media_list_14(media_list)
        new_source_list = new_source_list_14(source_list)
        new_attribute_list = new_attribute_list_14(attribute_list)
        new_seal_list = []
        for ldsord in lds_seal_list:
            (lsource_list, lnote_list, date, type, place,
             famc, temple, status, lprivate) = ldsord
            new_date = convert_date_14(date)
            new_lsource_list = new_source_list_14(lsource_list)
            new_seal_list.append( (new_lsource_list, lnote_list, new_date, type,
                                   place, famc, temple, status, lprivate))

        new_family = (junk_handle, gramps_id, father_handle, mother_handle,
                      new_child_ref_list, the_type, event_ref_list, new_media_list,
                      new_attribute_list, new_seal_list, new_source_list, note_list,
                      change, marker, private)

        with BSDDBTxn(self.env, self.family_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_family)
        self.update()

    # ---------------------------------
    # Modify Repository
    # ---------------------------------
    # update dates with newyear
    for handle in self.repository_map.keys():
        repository = self.repository_map[handle]
        # address
        (junk_handle, gramps_id, the_type, name, note_list,
         address_list, urls, change, marker, private) = repository

        new_address_list = []
        for address in address_list:
            (privacy, asource_list, anote_list, date, location) = address
            new_date = convert_date_14(date)
            new_asource_list = new_source_list_14(asource_list)
            new_address_list.append((privacy, new_asource_list, anote_list,
                                     new_date, location))

        new_repository = (junk_handle, gramps_id, the_type, name, note_list,
                          new_address_list, urls, change, marker, private)

        with BSDDBTxn(self.env, self.repository_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_repository)
        self.update()

    # ---------------------------------
    # Modify Media
    # ---------------------------------
    for media_handle in self.media_map.keys():
        media = self.media_map[media_handle]
        (handle, gramps_id, path, mime, desc,
         attribute_list, source_list, note_list, change,
         date, marker, private) = media
        new_source_list = new_source_list_14(source_list)
        new_date = convert_date_14(date)
        new_media = (handle, gramps_id, path, mime, desc,
                     attribute_list, new_source_list, note_list, change,
                     new_date, marker, private)

        with BSDDBTxn(self.env, self.media_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_media)
        self.update()

    # ---------------------------------
    # Modify Place
    # ---------------------------------
    for place_handle in self.place_map.keys():
        place = self.place_map[place_handle]
        (handle, gramps_id, title, longi, lat,
         main_loc, alt_loc, urls, media_list, source_list, note_list,
         change, marker, private) = place
        new_media_list = new_media_list_14(media_list)
        new_source_list = new_source_list_14(source_list)
        new_place = (handle, gramps_id, title, longi, lat,
                     main_loc, alt_loc, urls, new_media_list,
                     new_source_list, note_list, change, marker, private)

        with BSDDBTxn(self.env, self.place_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_place)
        self.update()

    # ---------------------------------
    # Modify Source
    # ---------------------------------
    for source_handle in self.source_map.keys():
        source = self.source_map[source_handle]
        (handle, gramps_id, title, author,
         pubinfo, note_list, media_list,
         abbrev, change, datamap, reporef_list,
         marker, private) = source
        new_media_list = new_media_list_14(media_list)
        new_source = (handle, gramps_id, title, author,
                      pubinfo, note_list, new_media_list,
                      abbrev, change, datamap, reporef_list,
                      marker, private)

        with BSDDBTxn(self.env, self.source_map) as txn:
            if isinstance(handle, UNITYPE):
                handle = handle.encode('utf-8')
            txn.put(handle, new_source)
        self.update()

    # Bump up database version. Separate transaction to save metadata.
    with BSDDBTxn(self.env, self.metadata) as txn:
        txn.put(b'version', 14)

def new_source_list_14(source_list):
    new_source_list = []
    for source in source_list:
        (date, private, note_list, confidence, ref, page) = source
        new_date = convert_date_14(date)
        new_source_list.append((new_date, private, note_list, confidence, ref, page))
    return new_source_list

def new_attribute_list_14(attribute_list):
    new_attribute_list = []
    for attribute in attribute_list:
        (private, asource_list, note_list, the_type, value) = attribute
        new_asource_list = new_source_list_14(asource_list)
        new_attribute_list.append((private, new_asource_list, note_list, the_type, value))
    return new_attribute_list

def new_media_list_14(media_list):
    # ---------------------------------
    # Event Media list
    # ---------------------------------
    new_media_list = []
    for media in media_list:
        (private, source_list, note_list,attribute_list,ref,role) = media
        new_source_list = new_source_list_14(source_list)
        new_attribute_list = new_attribute_list_14(attribute_list)
        new_media_list.append((private, new_source_list, note_list, new_attribute_list, ref, role))
    return new_media_list

def new_person_ref_list_14(person_ref_list):
    new_person_ref_list = []
    for person_ref in person_ref_list:
        (private, source_list, note_list, ref, rel) = person_ref
        new_source_list = new_source_list_14(source_list)
        new_person_ref_list.append((private, new_source_list, note_list, ref, rel))
    return new_person_ref_list

def new_child_ref_list_14(child_ref_list):
    new_child_ref_list = []
    for data in child_ref_list:
        (private, source_list, note_list, ref, frel, mrel) = data
        new_source_list = new_source_list_14(source_list)
        new_child_ref_list.append((private, new_source_list, note_list, ref, frel, mrel))
    return new_child_ref_list

def convert_date_14(date):
    if date:
        (calendar, modifier, quality, dateval, text, sortval) = date
        return (calendar, modifier, quality, dateval, text, sortval, 0)
    else:
        return None

def convert_name_14(name):
    (privacy, source_list, note_list, date,
     first_name, surname, suffix, title,
     name_type, prefix, patronymic,
     group_as, sort_as, display_as, call) = name
    new_date = convert_date_14(date)
    new_source_list = new_source_list_14(source_list)
    return (privacy, new_source_list, note_list, new_date,
            first_name, surname, suffix, title,
            name_type, prefix, patronymic,
            group_as, sort_as, display_as, call)


import statistics

n = int(input())
xs, ys = [], []
for i in range(n):
    x, y = map(int, input().split())
    xs.append(x)
    ys.append(y)

def m_d(x1, y1, x2, y2):
    return abs(x1 - x2) + abs(y1 - y2)

xc, yc = statistics.median(xs), statistics.median(ys)
print(int(sum(m_d(x, y, xc, yc) for x, y in zip(xs, ys))))


from typing import Optional, List

from src.model.frontend_model import Driver
from src.model.internal_model import Driver as InternalDriver
from src.repositories.driver_repository import AbstractDriverRepository

def add_driver_usecase(driver_repository: AbstractDriverRepository, name: str, surname: str, pesel: str,
                       phone: str) -> Driver:
    driver = driver_repository.add(first_name=name, last_name=surname, pesel=pesel, phone=phone)
    return Driver.form_internal(driver)

def get_driver_usecase(driver_repository: AbstractDriverRepository, driver_id) -> Driver:
    return Driver.form_internal(driver_repository.get(driver_id))

def get_all_drivers_usecase(driver_repository: AbstractDriverRepository) -> List[Driver]:
    return [Driver.form_internal(driver) for driver in driver_repository.get_all()]

def update_driver_use_case(driver_repository: AbstractDriverRepository, name: Optional[str], surname: Optional[str],
                           pesel: Optional[str], driver_id: str,
                           phone: Optional[str]
                           ) -> Driver:
    old_driver = driver_repository.get(driver_id)
    new_driver = InternalDriver(
        first_name=name or old_driver.first_name,
        last_name=surname or old_driver.last_name,
        phone=phone or old_driver.phone,
        PESEL=pesel or old_driver.PESEL,
        id=driver_id
    )
    driver_repository.update(driver_id, new_driver)
    return Driver.form_internal(new_driver)


from flask import Flask, render_template
import math
app = Flask(__name__)

@app.route('/static/<content>')
def static_content(content):
    return render_template(content)

@app.route('/<figurageometrica>/<volumenosurfacearea>/<parametro1>', methods=['GET'])
#esto es para figuras que necesitan 1 parametro
def CalculoCon1Parametro(figurageometrica,volumenosurfacearea,parametro1):
  # Su código va aquí
  parametro1 = int(parametro1)
  if parametro1 <= 0:
    return 'El parámetro ingresado es incorrecto.'
  else:
    if figurageometrica=='cube': #esto es si es un cubo
      if volumenosurfacearea=='volumen':
        volumen= (int(parametro1))**3
        return f'El volumen del cubo es: {volumen}'
      elif volumenosurfacearea=='surfacearea':
        surfacearea=6*(int(parametro1))**2
        surfacearea=round(surfacearea,2)
        return f'El area de la supercificie del cubo es: {surfacearea}'
      else:
        return "Ingrese área o volumen, por favor"

    if figurageometrica=='sphere':#esto es si es una esfera
      if volumenosurfacearea=='volumen':
        volumen= (4/3)*math.pi*(parametro1)**3
        return f'El volumen de la esfera es:  {volumen}'
      elif volumenosurfacearea=='surfacearea':
        surfacearea=4*math.pi*(parametro1)**2
        surfacearea=round(surfacearea,2)
        return f'El area de la superficie de la esfera es:  {surfacearea}'
      else:
        return "Opción invalida"

    if figurageometrica=='regular tetrahedron':#esto es si es tetrahedro
      if volumenosurfacearea=='volumen':
        volumen= (1/12)*math.sqrt(2)*(parametro1)**3
        return f'El volumen del tetrahedro regular es: {volumen}'
      elif volumenosurfacearea=='surfacearea':
        surfacearea=math.sqrt(3)*(parametro1)**2
        surfacearea=round(surfacearea,2)
        return f'El area de la superficie del tetrahedro regular es :  {surfacearea}'
      else:
        return "Opción invalida"

@app.route('/<figurageometrica>/<volumenosurfacearea>/<parametro1>/<parametro2>', methods=['GET'])
#esto es para figuras que necesitan 2 parametros
def CalculoCon2parametros(figurageometrica,volumenosurfacearea,parametro1,parametro2):
    # Su código va aquí
    parametro1=int(parametro1)
    parametro2=int(parametro2)
    if parametro1 <=0 or parametro2 <=0:
      return 'Uno o todos los parámetros ingresados son incorrectos.'
    else:
      if figurageometrica=='general cone' or figurageometrica=='general pyramid':
        if volumenosurfacearea=='volumen':
          volumen = (1/3)*parametro1*parametro2
          volumen=round(volumen,2)
          return f'El volumen del general cone es: {volumen}'

      if figurageometrica=='right cicular cone':
        if volumenosurfacearea=='volumen':
          volumen = (1/3)*math.pi*((parametro1)**2)*parametro2
          volumen=round(volumen,2)
          return f'El volumen del right circular cone es: {volumen}'
        elif volumenosurfacearea=='surfacearea':
          surfacearea = math.pi*(parametro1)*(math.sqrt(((parametro1)**2)+((parametro2)**2))) + (math.pi*((parametro1)**2))
          surfacearea=round(surfacearea,2)
          return f'El área de la superficie del rigth circular cone es: {surfacearea}'
        else:
          return("opción invalida")

      if figurageometrica=='right cicular cylinder':
        if volumenosurfacearea=='volumen':
          volumen = math.pi*((parametro1)**2)*parametro2
          volumen=round(volumen,2)
          return f'El volumen del right circular cylinder es: {volumen}'
        elif volumenosurfacearea=='surfacearea':
          surfacearea = (2*math.pi*parametro1*parametro2) + (2*math.pi*((parametro1)**2))
          surfacearea=round(surfacearea,2)
          return f'El área de superficie del right circular cylinder es: {surfacearea}'
        else:
          return("Opción invalida")

      if figurageometrica=='square pyramid':
        if volumenosurfacearea=='volumen':
          volumen = (1/3)*((parametro1)**2)*parametro2
          volumen=round(volumen,2)
          return f'El volumen del {figurageometrica} es: {volumen}'
        elif volumenosurfacearea=='surfacearea':
          surfacearea = parametro1*(parametro1+ (math.sqrt((parametro1**2)+(4*(parametro2**2)))))
          surfacearea=round(surfacearea,2)
          return f'El area de superficie del {figurageometrica} es:  {surfacearea}'
        else:
          return("Opción invalida")

      if figurageometrica=='torus':
        if volumenosurfacearea=='volumen':
          volumen = 2*(math.pi**2)*(parametro1**2)*parametro2
          volumen=round(volumen,2)
          return f'El volumen del {figurageometrica} es:{volumen}'
        elif volumenosurfacearea=='surfacearea':
          surfacearea = 4*(math.pi**2)*parametro1*parametro2
          surfacearea=round(surfacearea,2)
          return f'El área de superficie del {figurageometrica} es: {surfacearea}'
        else:
          return("Opción invalida")

@app.route('/<figurageometrica>/<volumenosurfacearea>/<parametro1>/<parametro2>/<parametro3>', methods=['GET'])
#esto es para figuras que necesitan 3 parametros
def CalculoCon3Parametros(figurageometrica,volumenosurfacearea,parametro1,parametro2,parametro3):
  parametro1=int(parametro1)
  parametro2=int(parametro2)
  parametro3=int(parametro3)
  if parametro1 <=0 or parametro2 <=0 or parametro3 <=0:
      return 'Uno o todos los parámetros ingresados son incorrectos.'
  else:
    if figurageometrica =="rectangulo solido":
      if volumenosurfacearea=="volumen":
        volumen=(parametro1*parametro2*parametro3)
        volumen=round(volumen,2)
        return f'El volumen del rectangulo sólido es: {volumen}'
      elif volumenosurfacearea=="surfacearea":
        surfacearea= 2*(parametro1*parametro2)+2*(parametro1*parametro3)+2*(parametro2*parametro3)
        surfacearea=round(surfacearea,2)
        return f'El área del rectangulo sólido es: {surfacearea}'
      else:
        return "opción invalida"

@app.route('/<figurageometrica>/<volumenosurfacearea>/<parametro1>/<parametro2>/<parametro3>/<parametro4>', methods=['GET'])
#esto es para figuras que necesitan 4 parametros
def calculoCon4Parametros(figurageometrica,volumenosurfacearea,parametro1,parametro2,parametro3, parametro4):
  parametro1=int(parametro1)
  parametro2=int(parametro2)
  parametro3=int(parametro3)
  parametro4=int(parametro4)
  if parametro1 <=0 or parametro2 <=0 or parametro3 <=0 or parametro4 <=0:
      return 'Uno o todos los parámetros ingresados son incorrectos.'
  else:
    if figurageometrica == "frustum of a cone":
      if volumenosurfacearea=="volumen":
        volumen = ((math.pi/3)*((parametro1**2)+(parametro1*parametro2)+(parametro2**2)))*parametro3
        volumen=round(volumen,2)
        return f'El volumen de tronco de un cono es: {volumen}'

      elif volumenosurfacearea== "surfacearea":
        area = (math.pi*parametro4)*(parametro2+parametro1)+(math.pi*parametro1**2)+(math.pi*parametro2**2)
        area= round(area,2)
        return f'El área del tronco de un cono es: {area}'

if __name__ == '__main__':
    app.secret_key = ".."
    app.run(port=8080, threaded=True, host=('127.0.0.1'))


"""View definitions for the surfaces router."""
import logging

import fastapi
from fastapi import status

from src.core import settings, utils
from src.routers.surfaces import controller, schemas

config = settings.get_settings()
LOGGER_NAME = config.LOGGER_NAME

logger = logging.getLogger(LOGGER_NAME)

router = fastapi.APIRouter(prefix="/surfaces", tags=["surfaces"])

@router.get("/hemispheres", responses={status.HTTP_200_OK: {"model": schemas.Surface}})
def get_hemispheres(
    response: fastapi.Response,
    species: str = fastapi.Query(
        ..., example="human", description="The species to fetch the hemispheres for."
    ),
    side: str = fastapi.Query(
        ..., example="left", description="The hemisphere to fetch the surfaces for."
    ),
) -> schemas.Surface:
    """Fetches the human and macaque hemisphere surfaces.

    Args:
        species: The species to fetch the hemispheres for, valid values are
            'human' and 'macaque'.
        side: The hemisphere to fetch the surfaces for, valid values are 'left' and
            'right'.

    Returns:
        A Pydantic BaseClass containing the surface.
    """
    logger.info("Calling GET /surfaces/hemispheres endpoint.")
    response = utils.add_cache_control(response)
    return controller.get_hemispheres(species, side)


import logging
import configparser
class userlogging:
    def loggingOut(self,S_Level,G_Level,formatter=None):
        my_logger=logging.Logger("DSDK")#创建收集器
        my_logger.setLevel(S_Level)#设置收集器的收集级别`
        sh=logging.StreamHandler()#指定日志输出到控制台
        sh.setLevel(G_Level)
        #fh=logging.FileHandler("log.txt",encoding="utf-8")#指定日志输出到文件
        #fh.setLevel(G_Level)
        #设置格式
        formatter_1=logging.Formatter(formatter)#格式化输出
        sh.setFormatter(formatter_1) #控制台日志添加格式化输出
        #fh.setFormatter(formatter_1)#文件日志增加控制台输出
        #对接收集器
        my_logger.addHandler(sh)
        #my_logger.addHandler(fh)
        return my_logger

def read_conf():
    cf=configparser.RawConfigParser()
    cf.read(r"G:\pythonxiangmu1\interface_auto_cases\config\registerconf.conf",encoding="utf-8")
    S_Level=eval(cf.get("LEVEL","S_Level"))
    G_Level=eval(cf.get("LEVEL","S_Level"))
    formatter_1=cf.get("FORMATTER","formatter_1")
    my=userlogging().loggingOut(S_Level[0],G_Level[0],formatter_1)
    return my


from django.contrib import admin
from django.urls import path

from quizapp.views import IndexView, QuizView, QuizDataView, QuizSaveView
from natijaapp.views import ResultView, SignupView, LoginView, logoutView

urlpatterns = [
    path('admin/', admin.site.urls),
    path('index/', IndexView.as_view(), name='index'),
    path('index/<int:pk>', QuizView.as_view(), name='quiz'),
    path('result/', ResultView.as_view(), name='result'),
    path('', SignupView.as_view(), name='signup'),
    path('logout/', logoutView, name='logout'),
    path('login/', LoginView.as_view(), name='login'),
    path('<int:pk>/data/', QuizDataView.as_view(), name='quiz-data'),
    path('<int:pk>/save/', QuizSaveView.as_view(), name='quiz-save'),
]


import json

from fmlaas.request_processor import IDProcessor
from fmlaas.request_processor import AuthContextProcessor
from fmlaas import get_job_table_name_from_env
from fmlaas import get_project_table_name_from_env
from fmlaas.database import DynamoDBInterface
from fmlaas.exception import RequestForbiddenException
from fmlaas.controller.submit_model_update import SubmitModelUpdateController

def lambda_handler(event, context):
    req_json = json.loads(event.get('body'))
    auth_json = event["requestContext"]["authorizer"]

    try:
        id_processor = IDProcessor(req_json)
        project_id = id_processor.get_project_id()
        job_id = id_processor.get_job_id()

        auth_context = AuthContextProcessor(auth_json)

        project_db = DynamoDBInterface(get_project_table_name_from_env())
        job_db = DynamoDBInterface(get_job_table_name_from_env())

        can_submit_model_to_job, presigned_url = SubmitModelUpdateController(project_db,
                                                                             job_db,
                                                                             project_id,
                                                                             job_id,
                                                                             auth_context).execute()

        if not can_submit_model_to_job:
            return {
                "statusCode": 400,
                "body": json.dumps({"error_msg": "Cannot submit model to this job. Either device is not active, or job is complete"})
            }
        else:
            return {
                "statusCode": 200,
                "body": json.dumps({"model_url": presigned_url})
            }
    except ValueError as error:
        return {
            "statusCode": 400,
            "body": json.dumps({"error_msg": str(error)})
        }
    except RequestForbiddenException as error:
        return {
            "statusCode": 403,
            "body": json.dumps({"error_msg": str(error)})
        }


import random, tweepy
import goodreads_scraper, hp_scraper1

# variables.

consumer_key = ""
consumer_secret = ""
access_key = ""
access_secret = ""
fileName = "my_quotes.txt"
last_mention_file = "lastMention.txt"
mentions = []

#1151753922069114880            first mention ID
#1152263154006712322            NEXT TEST ID
#-            next id     ~~~

# setup connection to twitter.
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
# create oject for twiiter.
twitter = tweepy.API(auth)

#function that opens a file containing the last mention id that was responded to.
# it will return this ID as an int.
def getLastMention(file_name) :
    f = open(file_name, 'r')
    last_mention_id = int(f.read())
    f.close()
    return last_mention_id

#funtion that will write that last seen mention id to the mentionID file.
# returns no value.
def setLastMention(file_name, last_mention_id) :
    f = open(file_name, 'w')
    f.write(last_mention_id)
    f.close()
    return

#function that will return a set of mentions from twtter.
# return a python like list.
def getMentions(input_id) :
    print("... gathering mentions\n...\n...")
    mentions = twitter.mentions_timeline(input_id)
    print("... mentions gathered")
    if len(mentions) == 0 :
        print("... No new Mentions to gather since tweet ID :" + str(getLastMention(last_mention_file)))
    return mentions

#function that will respond to the mentions since the last documented mentions.
# needs to access the lastMentions.txt file.
# returns to the calling funciton.
def respondToMentions() :
    #get the last mentions id from the file.
    last_ment_id = getLastMention(last_mention_file)
    newMentions = getMentions(last_ment_id)
    #loop most recent mentions from the twitter timeline.
    for mention in reversed(newMentions) :
        #store tweet ID.
        last_ment_id = mention.id_str
        setLastMention(last_mention_file, last_ment_id)
        #store tweet text.
        last_ment_text = mention.text
        print(last_ment_id + "\t\t"+ last_ment_text)
        #store tweet sender
        at_name = "@" + mention.user.screen_name
        if "#quote" in (last_ment_text).lower() :
            search_tag = ""
            split_list = last_ment_text.split(" ")
            for word in split_list :
                if "@" in word or word.lower() == "#quote" :
                    continue
                if "#" in word :
                    word = word.strip("#")
                search_tag = search_tag + word + "+"
            print('... responding')
            print(search_tag)
            quote = goodreads_scraper.getRandomQuote(str(search_tag))
            if quote == "NONE" :
                status = at_name + " No tweets found on goodreads.com//quotes with tag of : " + search_tag
                twitter.update_status(status, int(last_ment_id))
                print(status)
            else :
                quote = at_name +" "+ quote
                twitter.update_status(quote, int(last_ment_id))
                #print(quote)
            #print(quote)
    print("\n... done")
    return

#call function to get the random quote.
# function will get a random quote from one of the two scrapers randomly.
# then it will post the quote to twitter.
# function will need an input from the user to search good reads for.
def sendQuote(input_tag) :
    # try except to catch any errors then retry to post.
    try :
        # call function and tweet.
        #   function needs an input from the user regarding a quote topic/tag
        #   for testing its Harry Potter.
        randomGR_quote = goodreads_scraper.getRandomQuote(str(input_tag))
        twitter.update_status(randomGR_quote+"\n#QuickQuote#Bot")
        print("... Random Quote Sent.")
        #print("Just posted a tweet, GR!")
        #print(randomGR_quote)
    except tweepy.TweepError :
        sendQuote(input_tag)

#def sendHPQuote() :
    #randomHP_quote = hp_scraper1.getRandomQuote()
    #twitter.update_status(randomHP_quote+"\n#QuickQuote#Bot")
    #print("Just posted a tweet! HP")
    #print(randomHP_quote)

# send a HP quote tweet out!
sendQuote("Inspiration")

# gather most recent menions.
respondToMentions()


from typing import List

class Solution:
    def minPathSum(self, grid: List[List[int]]) -> int:
        m = len(grid)
        n = len(grid[0])
        dp = [grid[0][0] for i in range(n)]

        for i in range(1, n):
            dp[i] = dp[i-1] + grid[0][i]
        for i in range(1, m):
            dp[0] += grid[i][0]
            for j in range(1, n):
                dp[j] = min(dp[j-1], dp[j]) + grid[i][j]
        return dp[n-1]

def test(test_name, grid, expected):
    res = Solution().minPathSum(grid)
    if res == expected:
        print(test_name + ' success.')
    else:
        print(test_name + ' failed.')

if __name__ == "__main__":
    grid1 = [
        [1,3,1],
        [1,5,1],
        [4,2,1]
    ]
    expected1 = 7
    test('test1', grid1, expected1)