#!/usr/bin/env python

import argparse
import bson
import copy
import datetime
import dateutil.parser
import json
import logging
import multiprocessing
import os
import re
import sys
import time

from api import config
from api import util
from api.dao import containerutil
from api.dao.containerstorage import ProjectStorage
from api.jobs.jobs import Job
from api.jobs import gears
from api.types import Origin
from api.jobs import batch

CURRENT_DATABASE_VERSION = 44 # An int that is bumped when a new schema change is made

def get_db_version():

    version = config.get_version()
    if version is None:
        # Attempt to find db version at old location
        version = config.db.version.find_one({'_id': 'version'})
    if version is None or version.get('database') is None:
        return 0
    return version.get('database')

def confirm_schema_match():
    """
    Checks version of database schema

    Returns (0)  if DB schema version matches requirements.
    Returns (42) if DB schema version does not match
                 requirements and can be upgraded.
    Returns (43) if DB schema version does not match
                 requirements and cannot be upgraded,
                 perhaps because code is at lower version
                 than the DB schema version.
    """

    db_version = get_db_version()
    if not isinstance(db_version, int) or db_version > CURRENT_DATABASE_VERSION:
        logging.error('The stored db schema version of %s is incompatible with required version %s',
                       str(db_version), CURRENT_DATABASE_VERSION)
        sys.exit(43)
    elif db_version < CURRENT_DATABASE_VERSION:
        sys.exit(42)
    else:
        sys.exit(0)

def getMonotonicTime():
    # http://stackoverflow.com/a/7424304
    return os.times()[4]

def process_cursor(cursor, closure, context = None):
    """
    Given an iterable (say, a mongo cursor) and a closure, call that closure in parallel over the iterable.
    Call order is undefined. Currently launches N python process workers, where N is the number of vcpu cores.

    Useful for upgrades that need to touch each document in a database, and don't need an iteration order.

    Your closure MUST return True on success. Anything else is logged and treated as a failure.
    A closure that throws an exception will fail the upgrade immediately.
    """

    begin = getMonotonicTime()

    # cores = multiprocessing.cpu_count()
    # pool = multiprocessing.Pool(cores)
    # logging.info('Iterating over cursor with ' + str(cores) + ' workers')

    # # Launch all work, iterating over the cursor
    # # Note that this creates an array of n multiprocessing.pool.AsyncResults, where N is table size.
    # # Memory usage concern in the future? Doesn't seem to be an issue with ~120K records.
    # # Could be upgraded later with some yield trickery.
    # results = [pool.apply_async(closure, (document,)) for document in cursor]

    # # Read the results back, presumably in order!
    # failed = False
    # for res in results:
    # 	result = res.get()
    # 	if result != True:
    # 		failed = True
    # 		logging.info('Upgrade failed: ' + str(result))

    # logging.info('Waiting for workers to complete')
    # pool.close()
    # pool.join()

    logging.info('Proccessing {} items in cursor ...'.format(cursor.count()))

    failed = False
    cursor_size = cursor.count()
    cursor_index = 0.0
    next_percent = 5.0
    percent_increment = 5
    if(cursor_size < 20):
        next_percent = 25.0
        percent_increment = 25
    if(cursor_size < 4):
        next_percent = 50.0
        percent_increment = 50
    for document in cursor:
        if 100 * (cursor_index / cursor_size) >= next_percent:
            logging.info('{} percent complete ...'.format(next_percent))
            next_percent = next_percent + percent_increment
        if context == None:
            result = closure(document)
        else:
            result = closure(document, context)
        cursor_index = cursor_index + 1
        if result != True:
            failed = True
            logging.info('Upgrade failed: ' + str(result))

    if failed is True:
        msg = 'Worker pool experienced one or more failures. See above logs.'
        logging.info(msg)
        raise Exception(msg)

    end = getMonotonicTime()
    elapsed = end - begin
    logging.info('Parallel cursor iteration took ' + ('%.2f' % elapsed))

def upgrade_to_1():
    """
    scitran/core issue #206

    Initialize db version to 1
    """
    config.db.singletons.insert_one({'_id': 'version', 'database': 1})

def upgrade_to_2():
    """
    scitran/core PR #236

    Set file.origin.name to id if does not exist
    Set file.origin.method to '' if does not exist
    """

    def update_file_origins(cont_list, cont_name):
        for container in cont_list:
            updated_files = []
            for file in container.get('files', []):
                origin = file.get('origin')
                if origin is not None:
                    if origin.get('name', None) is None:
                        file['origin']['name'] = origin['id']
                    if origin.get('method', None) is None:
                        file['origin']['method'] = ''
                updated_files.append(file)

            query = {'_id': container['_id']}
            update = {'$set': {'files': updated_files}}
            result = config.db[cont_name].update_one(query, update)

    query = {'$and':[{'files.origin.name': { '$exists': False}}, {'files.origin.id': { '$exists': True}}]}

    update_file_origins(config.db.collections.find(query), 'collections')
    update_file_origins(config.db.projects.find(query), 'projects')
    update_file_origins(config.db.sessions.find(query), 'sessions')
    update_file_origins(config.db.acquisitions.find(query), 'acquisitions')

def upgrade_to_3():
    """
    scitran/core issue #253

    Set first user with admin permissions found as curator if one does not exist
    """
    query = {'curator': {'$exists': False}, 'permissions.access': 'admin'}
    projection = {'permissions.$':1}
    collections = config.db.collections.find(query, projection)
    for coll in collections:
        admin = coll['permissions'][0]['_id']
        query = {'_id': coll['_id']}
        update = {'$set': {'curator': admin}}
        config.db.collections.update_one(query, update)

def upgrade_to_4():
    """
    scitran/core issue #263

    Add '_id' field to session.subject
    Give subjects with the same code and project the same _id
    """

    pipeline = [
        {'$match': { 'subject._id': {'$exists': False}}},
        {'$group' : { '_id' : {'pid': '$project', 'code': '$subject.code'}, 'sids': {'$push': '$_id' }}}
    ]

    subjects = config.db.command('aggregate', 'sessions', pipeline=pipeline)
    for subject in subjects['result']:

        # Subjects without a code and sessions without a subject
        # will be returned grouped together, but all need unique IDs
        if subject['_id'].get('code') is None:
            for session_id in subject['sids']:
                subject_id = bson.ObjectId()
                config.db.sessions.update_one({'_id': session_id},{'$set': {'subject._id': subject_id}})
        else:
            subject_id = bson.ObjectId()
            query = {'_id': {'$in': subject['sids']}}
            update = {'$set': {'subject._id': subject_id}}
            config.db.sessions.update_many(query, update)

def upgrade_to_5():
    """
    scitran/core issue #279

    Ensure all sessions and acquisitions have the same perms as their project
    Bug(#278) discovered where changing a session's project did not update acquisition perms
    """

    projects = config.db.projects.find({})
    for p in projects:
        perms = p.get('permissions', [])

        session_ids = [s['_id'] for s in config.db.sessions.find({'project': p['_id']}, [])]

        config.db.sessions.update_many({'project': p['_id']}, {'$set': {'permissions': perms}})
        config.db.acquisitions.update_many({'session': {'$in': session_ids}}, {'$set': {'permissions': perms}})

def upgrade_to_6():
    """
    scitran/core issue #277

    Ensure all collection modified dates are ISO format
    Bug fixed in 6967f23
    """

    colls = config.db.collections.find({'modified': {'$type': 2}}) # type string
    for c in colls:
        fixed_mod = dateutil.parser.parse(c['modified'])
        config.db.collections.update_one({'_id': c['_id']}, {'$set': {'modified': fixed_mod}})

def upgrade_to_7():
    """
    scitran/core issue #270

    Add named inputs and specified destinations to jobs.

    Before:
    {
        "input" : {
            "container_type" : "acquisition",
            "container_id" : "572baf4e23dcb77ebbe06b3f",
            "filename" : "1_1_dicom.zip",
            "filehash" : "v0-sha384-422bd115d21585d1811d42cd99f1cf0a8511a4b377dd2deeaa1ab491d70932a051926ed99815a75142ad0815088ed009"
        }
    }

    After:
    {
        "inputs" : {
            "dicom" : {
                "container_type" : "acquisition",
                "container_id" : "572baf4e23dcb77ebbe06b3f",
                "filename" : "1_1_dicom.zip"
            }
        },
        "destination" : {
            "container_type" : "acquisition",
            "container_id" : "572baf4e23dcb77ebbe06b3f"
        }
    }
    """

    # The infrastructure runs this upgrade script before populating manifests.
    # For this reason, this one-time script does NOT pull manifests to do the input-name mapping, instead relying on a hard-coded alg name -> input name map.
    # If you have other gears in your system at the time of upgrade, you must add that mapping here.
    input_name_for_gear = {
        'dcm_convert': 'dicom',
        'qa-report-fmri': 'nifti',
        'dicom_mr_classifier': 'dicom',
    }

    jobs = config.db.jobs.find({'input': {'$exists': True}})

    for job in jobs:
        gear_name = job['algorithm_id']
        input_name = input_name_for_gear[gear_name]

        # Move single input to named input map
        input_ = job['input']
        input_.pop('filehash', None)
        inputs = { input_name: input_ }

        # Destination is required, and (for these jobs) is always the same container as the input
        destination = copy.deepcopy(input_)
        destination.pop('filename', None)

        config.db.jobs.update_one(
            {'_id': job['_id']},
            {
                '$set': {
                    'inputs': inputs,
                    'destination': destination
                },
                '$unset': {
                    'input': ''
                }
            }
        )

def upgrade_to_8():
    """
    scitran/core issue #291

    Migrate config, version, gears and rules to singletons collection
    """

    colls = config.db.collection_names()
    to_be_removed = ['version', 'config', 'static']
    # If we are in a bad state (singletons exists but so do any of the colls in to be removed)
    # remove singletons to try again
    if 'singletons' in colls and set(to_be_removed).intersection(set(colls)):
        config.db.drop_collection('singletons')

    if 'singletons' not in config.db.collection_names():
        static = config.db.static.find({})
        if static.count() > 0:
            config.db.singletons.insert_many(static)
        config.db.singletons.insert(config.db.version.find({}))

        configs = config.db.config.find({'latest': True},{'latest':0})
        if configs.count() == 1:
            c = configs[0]
            c['_id'] = 'config'
            config.db.singletons.insert_one(c)

        for c in to_be_removed:
            if c in config.db.collection_names():
                config.db.drop_collection(c)

def upgrade_to_9():
    """
    scitran/core issue #292

    Remove all session and acquisition timestamps that are empty strings
    """

    config.db.acquisitions.update_many({'timestamp':''}, {'$unset': {'timestamp': ''}})
    config.db.sessions.update_many({'timestamp':''}, {'$unset': {'timestamp': ''}})

def upgrade_to_10():
    """
    scitran/core issue #301

    Makes the following key renames, all in the jobs table.
    FR is a FileReference, CR is a ContainerReference:

    job.algorithm_id  --> job.name

    FR.container_type --> type
    FR.container_id   --> id
    FR.filename       --> name

    CR.container_type --> type
    CR.container_id   --> id
    """

    def switch_keys(doc, x, y):
        doc[y] = doc[x]
        doc.pop(x, None)

    jobs = config.db.jobs.find({'destination.container_type': {'$exists': True}})

    for job in jobs:
        switch_keys(job, 'algorithm_id', 'name')

        for key in job['inputs'].keys():
            inp = job['inputs'][key]

            switch_keys(inp, 'container_type', 'type')
            switch_keys(inp, 'container_id',   'id')
            switch_keys(inp, 'filename',       'name')

        dest = job['destination']
        switch_keys(dest, 'container_type', 'type')
        switch_keys(dest, 'container_id',   'id')

        config.db.jobs.update(
            {'_id': job['_id']},
            job
        )

def upgrade_to_11():
    """
    scitran/core issue #362

    Restructures job objects' `inputs` field from a dict with arbitrary keys
    into a list where the key becomes the field `input`
    """

    jobs = config.db.jobs.find({'inputs.type': {'$exists': False}})

    for job in jobs:

        inputs_arr = []
        for key, inp in job['inputs'].iteritems():
            inp['input'] = key
            inputs_arr.append(inp)

        config.db.jobs.update(
            {'_id': job['_id']},
            {'$set': {'inputs': inputs_arr}}
        )

def upgrade_to_12():
    """
    scitran/core PR #372

    Store job inputs on job-based analyses
    """

    sessions = config.db.sessions.find({'analyses.job': {'$exists': True}})

    for session in sessions:
        for analysis in session.get('analyses'):
            if analysis.get('job'):
                job = Job.get(analysis['job'])
                files = analysis.get('files', [])
                files[:] = [x for x in files if x.get('output')] # remove any existing inputs and insert fresh

                for i in getattr(job, 'inputs', {}):
                    fileref = job.inputs[i]
                    contref = containerutil.create_containerreference_from_filereference(job.inputs[i])
                    file_ = contref.find_file(fileref.name)
                    if file_:
                        file_['input'] = True
                        files.append(file_)

                q = {'analyses._id': analysis['_id']}
                u = {'$set': {'analyses.$.job': job.id_, 'analyses.$.files': files}}
                config.db.sessions.update_one(q, u)

def upgrade_to_13():
    """
    scitran/core PR #403

    Clear schema path from db config in order to set abs path to files
    """
    config.db.singletons.find_one_and_update(
        {'_id': 'config', 'persistent.schema_path': {'$exists': True}},
        {'$unset': {'persistent.schema_path': ''}})

def upgrade_to_14():
    """schema_path is no longer user configurable"""
    config.db.singletons.find_one_and_update(
        {'_id': 'config', 'persistent.schema_path': {'$exists': True}},
        {'$unset': {'persistent.schema_path': ''}})

def upgrade_to_15():
    """
    scitran/pull issue #417

    First remove all timestamps that are empty or not mongo date or string format.
    Then attempt to convert strings to dates, removing those that cannot be converted.
    Mongo $type maps: String = 2, Date = 9
    """
    query = {}
    query['$or'] = [
                    {'timestamp':''},
                    {'$and': [
                        {'timestamp': {'$exists': True}},
                        {'timestamp': {'$not': {'$type':2}}},
                        {'timestamp': {'$not': {'$type':9}}}
                    ]}
                ]
    unset = {'$unset': {'timestamp': ''}}

    config.db.sessions.update_many(query, unset)
    config.db.acquisitions.update_many(query, unset)

    query =  {'$and': [
                {'timestamp': {'$exists': True}},
                {'timestamp': {'$type':2}}
            ]}
    sessions = config.db.sessions.find(query)
    for s in sessions:
        try:
            fixed_timestamp = dateutil.parser.parse(s['timestamp'])
        except:
            config.db.sessions.update_one({'_id': s['_id']}, {'$unset': {'timestamp': ''}})
            continue
        config.db.sessions.update_one({'_id': s['_id']}, {'$set': {'timestamp': fixed_timestamp}})

    acquisitions = config.db.acquisitions.find(query)
    for a in acquisitions:
        try:
            fixed_timestamp = dateutil.parser.parse(a['timestamp'])
        except:
            config.db.sessions.update_one({'_id': a['_id']}, {'$unset': {'timestamp': ''}})
            continue
        config.db.sessions.update_one({'_id': a['_id']}, {'$set': {'timestamp': fixed_timestamp}})

def upgrade_to_16():
    """
    Fixes file.size sometimes being a floating-point rather than integer.
    """

    acquisitions = config.db.acquisitions.find({'files.size': {'$type': 'double'}})
    for x in acquisitions:
        for y in x.get('files', []):
            if y.get('size'):
                y['size'] = int(y['size'])
        config.db.acquisitions.update({"_id": x['_id']}, x)

    sessions = config.db.sessions.find({'files.size': {'$type': 'double'}})
    for x in sessions:
        for y in x.get('files', []):
            if y.get('size'):
                y['size'] = int(y['size'])
        config.db.sessions.update({"_id": x['_id']}, x)

    projects = config.db.projects.find({'files.size': {'$type': 'double'}})
    for x in projects:
        for y in x.get('files', []):
            if y.get('size'):
                y['size'] = int(y['size'])
        config.db.projects.update({"_id": x['_id']}, x)

    sessions = config.db.sessions.find({'analyses.files.size': {'$type': 'double'}})
    for x in sessions:
        for y in x.get('analyses', []):
            for z in y.get('files', []):
                if z.get('size'):
                    z['size'] = int(z['size'])
        config.db.sessions.update({"_id": x['_id']}, x)

def upgrade_to_17():
    """
    scitran/core issue #557

    Reassign subject ids after bug fix in packfile code that did not properly match subjects
    """

    pipeline = [
        {'$group' : { '_id' : {'pid': '$project', 'code': '$subject.code'}, 'sids': {'$push': '$_id' }}}
    ]

    subjects = config.db.command('aggregate', 'sessions', pipeline=pipeline)
    for subject in subjects['result']:

        # Subjects without a code and sessions without a subject
        # will be returned grouped together, but all need unique IDs
        if subject['_id'].get('code') is None:
            for session_id in subject['sids']:
                subject_id = bson.ObjectId()
                config.db.sessions.update_one({'_id': session_id},{'$set': {'subject._id': subject_id}})
        else:
            subject_id = bson.ObjectId()
            query = {'_id': {'$in': subject['sids']}}
            update = {'$set': {'subject._id': subject_id}}
            config.db.sessions.update_many(query, update)

def upgrade_to_18():
    """
    scitran/core issue #334

    Move singleton gear doc to its own table
    """

    gear_doc = config.db.singletons.find_one({"_id": "gears"})

    if gear_doc is not None:
        gear_list = gear_doc.get('gear_list', [])
        for gear in gear_list:
            try:
                gears.upsert_gear(gear)
            except Exception as e:
                logging.error("")
                logging.error("Error upgrading gear:")
                logging.error(type(e))
                logging.error("Gear will not be retained. Document follows:")
                logging.error(gear)
                logging.error("")

        config.db.singletons.remove({"_id": "gears"})

def upgrade_to_19():
    """
    scitran/core issue #552

    Add origin information to job object
    """

    update = {
        '$set': {
            'origin' : {'type': str(Origin.unknown), 'id': None}
        }
    }
    config.db.jobs.update_many({'origin': {'$exists': False}}, update)

def upgrade_to_20():
    """
    scitran/core issue #602

    Change dash to underscore for consistency
    """

    query = {'last-seen': {'$exists': True}}
    update = {'$rename': {'last-seen':'last_seen' }}

    config.db.devices.update_many(query, update)

def upgrade_to_21():
    """
    scitran/core issue #189 - Data Model v2

    Field `metadata` renamed to `info`
    Field `file.instrument` renamed to `file.modality`
    Acquisition fields `instrument` and `measurement` removed
    """

    def update_project_template(template):
        new_template = {'acquisitions': []}
        for a in template.get('acquisitions', []):
            new_a = {'minimum': a['minimum']}
            properties = a['schema']['properties']
            if 'measurement' in properties:
                m_req = properties['measurement']['pattern']
                m_req = re.sub('^\(\?i\)', '', m_req)
                new_a['files']=[{'measurement':  m_req, 'minimum': 1}]
            if 'label' in properties:
                l_req = properties['label']['pattern']
                l_req = re.sub('^\(\?i\)', '', l_req)
                new_a['label'] = l_req
            new_template['acquisitions'].append(new_a)

        return new_template

    def dm_v2_updates(cont_list, cont_name):
        for container in cont_list:

            query = {'_id': container['_id']}
            update = {'$rename': {'metadata': 'info'}}

            if cont_name == 'projects' and container.get('template'):
                new_template = update_project_template(json.loads(container.get('template')))
                update['$set'] = {'template': new_template}

            if cont_name == 'sessions':
                update['$rename'].update({'subject.metadata': 'subject.info'})

            measurement = None
            modality = None
            info = None
            if cont_name == 'acquisitions':
                update['$unset'] = {'instrument': '', 'measurement': ''}
                measurement = container.get('measurement', None)
                modality = container.get('instrument', None)
                info = container.get('metadata', None)
                if info:
                    config.db.acquisitions.update_one(query, {'$set': {'metadata': {}}})

            # From mongo docs: '$rename does not work if these fields are in array elements.'
            files = container.get('files')
            if files is not None:
                updated_files = []
                for file_ in files:
                    file_['info'] = {}
                    if 'metadata' in file_:
                        file_['info'] = file_.pop('metadata', None)
                    if 'instrument' in file_:
                        file_['modality'] = file_.pop('instrument', None)
                    if measurement:
                        # Move the acquisition's measurement to all files
                        if file_.get('measurements'):
                            file_['measurements'].append(measurement)
                        else:
                            file_['measurements'] = [measurement]
                    if info and file_.get('type', '') == 'dicom':
                        # This is going to be the dicom header info
                        updated_info = info
                        updated_info.update(file_['info'])
                        file_['info'] = updated_info
                    if modality and not file_.get('modality'):
                        file_['modality'] = modality

                    updated_files.append(file_)
                if update.get('$set'):
                    update['$set']['files'] =  updated_files
                else:
                    update['$set'] = {'files': updated_files}

            result = config.db[cont_name].update_one(query, update)

    query = {'$or':[{'files.metadata': { '$exists': True}},
                    {'metadata': { '$exists': True}},
                    {'files.instrument': { '$exists': True}}]}

    dm_v2_updates(config.db.collections.find(query), 'collections')

    query['$or'].append({'template': { '$exists': True}})
    dm_v2_updates(config.db.projects.find({}), 'projects')

    query['$or'].append({'subject': { '$exists': True}})
    dm_v2_updates(config.db.sessions.find(query), 'sessions')

    query['$or'].append({'instrument': { '$exists': True}})
    query['$or'].append({'measurement': { '$exists': True}})
    dm_v2_updates(config.db.acquisitions.find(query), 'acquisitions')

def upgrade_to_22():
    """
    Add created and modified timestamps to gear docs

    Of debatable value, since infra will load gears on each boot.
    """

    logging.info('Upgrade v22, phase 1 of 3, upgrading gears...')

    # Add timestamps to gears.
    for gear in config.db.gears.find({}):
        now = datetime.datetime.utcnow()

        gear['created']  = now
        gear['modified'] = now

        config.db.gears.update({'_id': gear['_id']}, gear)

        # Ensure there cannot possibly be two gears of the same name with the same timestamp.
        # Plus or minus monotonic time.
        # A very silly solution, but we only ever need to do this once, on a double-digit number of documents.
        # Not worth the effort to, eg, rewind time and do math.
        time.sleep(1)
        logging.info('  Updated gear ' + str(gear['_id']) + ' ...')
        sys.stdout.flush()

    logging.info('Upgrade v22, phase 2 of 3, upgrading jobs...')

    # Now that they're updated, fetch all gears and hold them in memory.
    # This prevents extra database queries during the job upgrade.

    all_gears = list(config.db.gears.find({}))
    gears_map = { }

    for gear in all_gears:
        gear_name = gear['gear']['name']

        gears_map[gear_name] = gear

    # A dummy gear for missing refs
    dummy_gear = {
        'category' : 'converter',
        'gear' : {
            'inputs' : {
                'do-not-use' : {
                    'base' : 'file'
                }
            },
            'maintainer' : 'Noone <nobody@example.example>',
            'description' : 'This gear or job was referenced before gear versioning. Version information is not available for this gear.',
            'license' : 'BSD-2-Clause',
            'author' : 'Noone',
            'url' : 'https://example.example',
            'label' : 'Deprecated Gear',
            'flywheel' : '0',
            'source' : 'https://example.example',
            'version' : '0.0.0',
            'custom' : {
                'flywheel': {
                    'invalid': True
                }
            },
            'config' : {},
            'name' : 'deprecated-gear'
        },
        'exchange' : {
            'git-commit' : '0000000000000000000000000000000000000000',
            'rootfs-hash' : 'sha384:000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000',
            'rootfs-url' : 'https://example.example/does-not-exist.tgz'
        }
    }

    maximum = config.db.jobs.count()
    upgraded = 0

    # Blanket-assume gears were the latest in the DB pre-gear versioning.
    for job in config.db.jobs.find({}):

        # Look up latest gear by name, lose job name key
        gear_name = job['name']
        gear = gears_map.get(gear_name)

        if gear is None:
            logging.info('Job doc ' + str(job['_id']) + ' could not find gear ' + gear_name + ', creating...')

            new_gear = copy.deepcopy(dummy_gear)
            new_gear['gear']['name'] = gear_name

            # Save new gear, store id in memory
            resp = config.db.gears.insert_one(new_gear)
            new_id = resp.inserted_id
            new_gear['_id'] = str(new_id)

            # Insert gear into memory map
            gears_map[gear_name] = new_gear

            logging.info('Created gear  ' + gear_name + ' with id ' + str(new_id) + '. Future jobs with this gear name with not alert.')

            gear = new_gear

        if gear is None:
            raise Exception("We don't understand python scopes ;( ;(")

        # Store gear ID
        job.pop('name', None)
        job['gear_id'] = str(gear['_id'])

        # Save
        config.db.jobs.update({'_id': job['_id']}, job)

        upgraded += 1
        if upgraded % 1000 == 0:
            logging.info('  Processed ' + str(upgraded) + ' jobs of ' + str(maximum) + '...')

    logging.info('Upgrade v22, phase 3 of 3, upgrading batch...')

    maximum = config.db.batch.count()
    upgraded = 0

    for batch in config.db.batch.find({}):

        # Look up latest gear by name, lose job name key
        gear = gears.get_gear_by_name(batch['gear'])
        batch.pop('gear', None)

        # Store gear ID
        batch['gear_id'] = str(gear['_id'])

        # Save
        config.db.batch.update({'_id': batch['_id']}, batch)

        upgraded += 1
        if upgraded % 1000 == 0:
            logging.info('  Processed ' + str(upgraded) + ' batch of ' + str(maximum) + '...')

    logging.info('Upgrade v22, complete.')

def upgrade_to_23():
    """
    scitran/core issue #650

    Support multiple auth providers
    Config 'auth' key becomes map where keys are auth_type
    """

    db_config = config.db.singletons.find_one({'_id': 'config'})
    if db_config:
        auth_config = db_config.get('auth', {})
        if auth_config.get('auth_type'):
            auth_type = auth_config.pop('auth_type')
            config.db.singletons.update_one({'_id': 'config'}, {'$set': {'auth': {auth_type: auth_config}}})

def upgrade_to_24():
    """
    scitran/core issue #720

    Migrate gear rules to the project level
    """

    global_rules = config.db.singletons.find_one({"_id" : "rules"})
    project_ids  = list(config.db.projects.find({},{"_id": "true"}))

    if global_rules is None:
        global_rules = {
            'rule_list': []
        }

    logging.info('Upgrade v23, migrating ' + str(len(global_rules['rule_list'])) + ' gear rules...')

    count = 0
    for old_rule in global_rules['rule_list']:

        logging.info(json.dumps(old_rule))

        gear_name = old_rule['alg']
        rule_name = 'Migrated rule ' + str(count)

        any_stanzas = []
        all_stanzas = []

        for old_any_stanza in old_rule.get('any', []):
            if len(old_any_stanza) != 2:
                raise Exception('Confusing any-rule stanza ' + str(count) + ': ' + json.dumps(old_any_stanza))

            any_stanzas.append({ 'type': old_any_stanza[0], 'value': old_any_stanza[1] })

        for old_all_stanza in old_rule.get('all', []):
            if len(old_all_stanza) != 2:
                raise Exception('Confusing all-rule stanza ' + str(count) + ': ' + json.dumps(old_all_stanza))

            all_stanzas.append({ 'type': old_all_stanza[0], 'value': old_all_stanza[1] })

        # New rule object
        new_rule = {
            'alg': gear_name,
            'name': rule_name,
            'any': any_stanzas,
            'all': all_stanzas
        }

        # Insert rule on every project
        for project in project_ids:
            project_id = project['_id']

            new_rule_obj = copy.deepcopy(new_rule)
            new_rule_obj['project_id'] = str(project_id)

            config.db.project_rules.insert_one(new_rule_obj)

        logging.info('Upgrade v23, migrated rule ' + str(count) + ' of ' + str(len(global_rules)) + '...')
        count += 1

    # Remove obsolete singleton
    config.db.singletons.remove({"_id" : "rules"})
    logging.info('Upgrade v23, complete.')

def upgrade_to_25():
    """
    scitran/core PR #733

    Migrate refresh token from authtokens to seperate collection
    """

    auth_tokens = config.db.authtokens.find({'refresh_token': {'$exists': True}})

    for a in auth_tokens:
        refresh_doc = {
            'uid': a['uid'],
            'token': a['refresh_token'],
            'auth_type': a['auth_type']
        }
        config.db.refreshtokens.insert(refresh_doc)

    config.db.authtokens.update_many({'refresh_token': {'$exists': True}}, {'$unset': {'refresh_token': ''}})

def upgrade_to_26_closure(job):

    gear = config.db.gears.find_one({'_id': bson.ObjectId(job['gear_id'])}, {'gear.name': 1})

    # This logic WILL NOT WORK in parallel mode
    if gear is None:
        logging.info('No gear found for job ' + str(job['_id']))
        return True
    if gear.get('gear', {}).get('name', None) is None:
        logging.info('No gear found for job ' + str(job['_id']))
        return True

    # This logic WILL NOT WORK in parallel mode

    gear_name = gear['gear']['name']

    # Checks if the specific gear tag already exists for the job
    if gear_name in job['tags']:
        return True

    result = config.db.jobs.update_one({'_id': job['_id']}, {'$addToSet': {'tags': gear_name }})

    if result.modified_count == 1:
        return True
    else:
        return 'Parallel failed: update doc ' + str(job['_id']) + ' resulted modified ' + str(result.modified_count)

def upgrade_to_26():
    """
    scitran/core #734

    Add job tags back to the job document, and use a faster cursor-walking update method
    """
    cursor = config.db.jobs.find({})
    process_cursor(cursor, upgrade_to_26_closure)

def upgrade_to_27():
    """
    scitran/core PR #768

    Fix project templates that reference `measurement` instead of `measurements`
    Update all session compliance for affected projects
    """

    projects = config.db.projects.find({'template.acquisitions.files.measurement': {'$exists': True}})

    storage = ProjectStorage()

    for p in projects:
        template = p.get('template', {})
        for a in template.get('acquisitions', []):
            for f in a.get('files', []):
                if f.get('measurement'):
                    f['measurements'] = f.pop('measurement')
        config.log.debug('the template is now {}'.format(template))
        config.db.projects.update_one({'_id': p['_id']}, {'$set': {'template': template}})
        storage.recalc_sessions_compliance(project_id=str(p['_id']))

def upgrade_to_28():
    """
    Fixes session.subject.age sometimes being a floating-point rather than integer.
    """

    sessions = config.db.sessions.find({'subject.age': {'$type': 'double'}})
    logging.info('Fixing {} subjects with age stored as double ...'.format(sessions.count()))
    for session in sessions:
        try:
            session['subject']['age'] = int(session['subject']['age'])
        except:
            session['subject']['age'] = None

        config.db.sessions.update({'_id': session['_id']}, session)

def upgrade_to_29_closure(user):

    avatars = user['avatars']
    if avatars.get('custom') and not 'https:' in avatars['custom']:
        if user['avatar'] == user['avatars']['custom']:
            if(user['avatars'].get('provider') == None):
                config.db.users.update_one({'_id': user['_id']},
                    {'$unset': {'avatar': ""}})
            else:
                config.db.users.update_one({'_id': user['_id']},
                    {'$set': {'avatar': user['avatars'].get('provider')}}
                )
        logging.info('Deleting custom ...')
        config.db.users.update_one({'_id': user['_id']},
            {'$unset': {"avatars.custom": ""}}
        )
    return True

def upgrade_to_29():
    """
    Enforces HTTPS urls for user avatars
    """

    users = config.db.users.find({})
    process_cursor(users, upgrade_to_29_closure)

def upgrade_to_30_closure_analysis(coll_item, coll):
    analyses = coll_item.get('analyses', [])

    for analysis_ in analyses:
        files = analysis_.get('files', [])
        for file_ in files:
            if 'created' not in file_:
                file_['created'] = analysis_.get('created', datetime.datetime(1970, 1, 1))
    result = config.db[coll].update_one({'_id': coll_item['_id']}, {'$set': {'analyses': analyses}})
    if result.modified_count == 1:
        return True
    else:
        return "File timestamp creation failed for:" + str(coll_item)

def upgrade_to_30_closure_coll(coll_item, coll):
    files = coll_item.get('files', [])
    for file_ in files:
        if 'created' not in file_:
            file_['created'] = coll_item.get('created', datetime.datetime(1970, 1, 1))
    result = config.db[coll].update_one({'_id': coll_item['_id']}, {'$set': {'files': files}})
    if result.modified_count == 1:
        return True
    else:
        return "File timestamp creation failed for:" + str(coll_item)

def upgrade_to_30():
    """
    scitran/core issue #759

    give created timestamps that are missing are given based on the parent object's timestamp
    """

    cursor = config.db.collections.find({'analyses.files.name': {'$exists': True},
                                         'analyses.files.created': {'$exists': False}})
    process_cursor(cursor, upgrade_to_30_closure_analysis, context = 'collections')

    cursor = config.db.sessions.find({'analyses.files.name': {'$exists': True},
                                      'analyses.files.created': {'$exists': False}})
    process_cursor(cursor, upgrade_to_30_closure_analysis, context = 'sessions')

    cursor = config.db.sessions.find({'files.name': {'$exists': True}, 'files.created': {'$exists': False}})
    process_cursor(cursor, upgrade_to_30_closure_coll, context = 'sessions')

    cursor = config.db.collections.find({'files.name': {'$exists': True}, 'files.created': {'$exists': False}})
    process_cursor(cursor, upgrade_to_30_closure_coll, context = 'collections')

    cursor = config.db.acquisitions.find({'files.name': {'$exists': True}, 'files.created': {'$exists': False}})
    process_cursor(cursor, upgrade_to_30_closure_coll, context = 'acquisitions')

    cursor = config.db.projects.find({'files.name': {'$exists': True}, 'files.created': {'$exists': False}})
    process_cursor(cursor, upgrade_to_30_closure_coll, context = 'projects')

def upgrade_to_31():
    config.db.sessions.update_many({'subject.firstname_hash': {'$exists': True}}, {'$unset': {'subject.firstname_hash':""}})
    config.db.sessions.update_many({'subject.lastname_hash': {'$exists': True}}, {'$unset': {'subject.lastname_hash':""}})

def upgrade_to_32_closure(coll_item, coll):
    permissions = coll_item.get('permissions', [])
    for permission_ in permissions:
        if permission_.get('site', False):
            del permission_['site']
    result = config.db[coll].update_one({'_id': coll_item['_id']}, {'$set': {'permissions' : permissions}})
    if result.modified_count == 0:
        return "Failed to remove site field"
    return True

def upgrade_to_32():
    for coll in ['acquisitions', 'groups', 'projects', 'sessions']:
        cursor = config.db[coll].find({'permissions.site': {'$exists': True}})
        process_cursor(cursor, upgrade_to_32_closure, context = coll)
    config.db.sites.drop()

def upgrade_to_33_closure(cont, cont_name):
    cont_type = cont_name[:-1]
    if cont.get('analyses'):
        for analysis in cont['analyses']:
            analysis['_id'] = bson.ObjectId(analysis['_id'])
            analysis['parent'] = {'type': cont_type, 'id': cont['_id']}
            analysis['permissions'] = cont['permissions']
            for key in ('public', 'archived'):
                if key in cont:
                    analysis[key] = cont[key]
        config.db['analyses'].insert_many(cont['analyses'])
    config.db[cont_name].update_one(
        {'_id': cont['_id']},
        {'$unset': {'analyses': ''}})
    return True

def upgrade_to_33():
    """
    scitran/core issue #808 - make analyses use their own collection
    """
    for cont_name in ['projects', 'sessions', 'acquisitions', 'collections']:
        cursor = config.db[cont_name].find({'analyses': {'$exists': True}})
        process_cursor(cursor, upgrade_to_33_closure, context=cont_name)

def upgrade_to_34():
    """
    Changes group.roles -> groups.permissions

    scitran/core #662
    """
    config.db.groups.update_many({'roles': {'$exists': True}}, {'$rename': {'roles': 'permissions'}})
    config.db.groups.update_many({'name': {'$exists': True}}, {'$rename': {'name': 'label'}})

def upgrade_to_35_closure(batch_job):
    if batch_job.get('state') in ['cancelled', 'running', 'complete', 'failed']:
        return True
    batch_id = batch_job.get('_id')
    config.db.jobs.update_many({'_id': {'$in': batch_job.get('jobs',[])}}, {'$set': {'batch':batch_id}})
    new_state = batch.check_state(batch_id)
    if new_state:
        result = config.db.batch.update_one({'_id': batch_id}, {'$set': {"state": new_state}})
        if result.modified_count != 1:
            raise Exception('Batch job not updated')
    else:
        result = config.db.batch.update_one({'_id': batch_id}, {'$set': {"state": "running"}})
        if result.modified_count != 1:
            raise Exception('Batch job not updated')
    return True

def upgrade_to_35():
    """
    scitran/core issue #710 - give batch stable states
    """
    cursor = config.db.batch.find({})
    process_cursor(cursor, upgrade_to_35_closure)

def upgrade_to_36_closure(acquisition):

    for f in acquisition['files']:
        if not f.get('mimetype'):
            logging.debug('file with name {} did not have mimetype'.format(f['name']))
            f['mimetype'] = util.guess_mimetype(f['name'])

    result = config.db.acquisitions.update_one({'_id': acquisition['_id']}, {'$set': {'files': acquisition['files']}})
    if result.modified_count != 1:
        raise Exception('Acquisition file not updated')

    return True

def upgrade_to_36():
    """
    scitran/core issue #931 - mimetype not set on packfile uploads
    """
    cursor = config.db.acquisitions.find({'files': { '$gt': [] }, 'files.mimetype': None})
    process_cursor(cursor, upgrade_to_36_closure)

def upgrade_to_37():
    """
    scitran/core issue #916 - group-permission level site info needs to be removed from all levels
    """
    for coll in ['acquisitions', 'groups', 'projects', 'sessions', 'analyses']:
        cursor = config.db[coll].find({'permissions.site': {'$exists': True}})
        process_cursor(cursor, upgrade_to_32_closure, context = coll)

def upgrade_to_38_closure(user):

    # if user has existing API key in correct db location, remove API key stored on user and move on
    # otherwise, migrate api key to new location

    api_key = user['api_key']
    doc = config.db.apikeys.find_one({'uid': user['_id'], 'type': 'user'})

    if not doc:

        # migrate existing API key

        new_api_key_doc = {
            '_id': api_key['key'],
            'created': api_key['created'],
            'last_used': api_key['last_used'],
            'uid': user['_id'],
            'type': 'user'
        }

        config.db.apikeys.insert(new_api_key_doc)

    config.db.users.update_one({'_id': user['_id']}, {'$unset': {'api_key': 0}})

    return True

def upgrade_to_38():
    """
    Move existing user api keys to new 'apikeys' collection
    """
    cursor = config.db.users.find({'api_key': {'$exists': True }})
    process_cursor(cursor, upgrade_to_38_closure)

def upgrade_to_39_closure(job):
    """
    Done in python because:
    " the source and target field for $rename must not be on the same path "
    """

    config_ = job.pop('config', {})
    config.db.jobs.update({'_id': job['_id']}, {'$set': {'config': {'config': config_}}})

    return True

def upgrade_to_39():
    """
    Move old jobs without extra config down one level to match new jobs
    with additional keys.

    Before:
    {
        'config': {
            'a': 'b'
        }
    }

    After:
    {
        'config': {
            'config': {
                'a': 'b'
            }
        }
    }
    """
    cursor = config.db.jobs.find({'config': {'$exists': True }, 'config.config': {'$exists': False }})
    process_cursor(cursor, upgrade_to_39_closure)

def upgrade_to_40_closure(acquisition):
    config.db.acquisitions.update_one({'_id':acquisition['_id']},{'$set':{'timestamp':dateutil.parser.parse(acquisition['timestamp'])}})
    return True

def upgrade_to_40():
    """
    Convert all string acquisition timestamps to type date
    """
    cursor = config.db.acquisitions.find({'timestamp':{'$type':'string'}})
    process_cursor(cursor, upgrade_to_40_closure)

def upgrade_to_41_closure(cont, cont_name):

    files = cont.get('files', [])
    for f in files:
        if 'tags' not in f:
            f['tags'] = []
        if 'measurements' not in f:
            f['measurements'] = []
        if 'origin' not in f:
            f['origin'] = {
                'type': str(Origin.unknown),
                'id': None
            }
        if 'mimetype' not in f:
            f['mimetype'] = util.guess_mimetype(f.get('name'))
        if 'modality' not in f:
            f['modality'] = None
    config.db[cont_name].update_one({'_id': cont['_id']}, {'$set': {'files': files}})
    return True

def upgrade_to_41():
    """
    scitran/core issue #1042 - some "expected" file default keys are not present

    These are the fields that are created on every file object in upload.py
    """

    for cont_name in ['groups', 'projects', 'sessions', 'acquisitions', 'collections', 'analyses']:
        cursor = config.db[cont_name].find({'files': { '$elemMatch': { '$or': [
            {'tags':          {'$exists': False }},
            {'measurements':  {'$exists': False }},
            {'origin':        {'$exists': False }},
            {'mimetype':      {'$exists': False }},
            {'modality':      {'$exists': False }}
        ]}}})
        process_cursor(cursor, upgrade_to_41_closure, context=cont_name)

def upgrade_to_42_closure(cont, cont_name):
    archived = cont.pop('archived')
    update = {'$unset': {'archived': True}}
    if archived:
        cont['tags'] = cont.get('tags', []) + ['hidden']
        update['$set'] = {'tags': cont['tags']}
    config.db[cont_name].update_one({'_id': cont['_id']}, update)
    return True

def upgrade_to_42():
    """
    Change container flag "archived" to container tag "hidden"
    """
    for cont_name in ['groups', 'projects', 'sessions', 'acquisitions']:
        cursor = config.db[cont_name].find({'archived': {'$exists': True}})
        process_cursor(cursor, upgrade_to_42_closure, context=cont_name)

def upgrade_to_43_closure(analysis):
    inputs = [f for f in analysis['files'] if f.get('input')]
    outputs = [f for f in analysis['files'] if f.get('output')]
    for f in inputs + outputs:
        f.pop('input', None)
        f.pop('output', None)
    config.db.analyses.update_one({'_id': analysis['_id']}, {'$set': {'inputs': inputs, 'files': outputs}})
    return True

def upgrade_to_43():
    """
    Remove analysis files' input/output tags and store them separately instead:
       - inputs under `analysis.inputs`
       - outputs under `analysis.files`
    """
    cursor = config.db.analyses.find({'files': {'$exists': True, '$ne': []}})
    process_cursor(cursor, upgrade_to_43_closure)

def upgrade_to_44():
    """
    A rerun of scitran/core issue #263

    A rerun was necessary because a bug was found when moving a session to a new project:
    the subject id should change but it was not, causing subject linking where there should be none

    Add '_id' field to session.subject
    Give subjects with the same code and project the same _id
    """

    pipeline = [
        {'$group' : { '_id' : {'pid': '$project', 'code': '$subject.code'}, 'sids': {'$push': '$_id' }}}
    ]

    subjects = config.db.command('aggregate', 'sessions', pipeline=pipeline)
    for subject in subjects['result']:

        # Subjects without a code and sessions without a subject
        # will be returned grouped together, but all need unique IDs
        if subject['_id'].get('code') is None:
            for session_id in subject['sids']:
                subject_id = bson.ObjectId()
                config.db.sessions.update_one({'_id': session_id},{'$set': {'subject._id': subject_id}})
        else:
            subject_id = bson.ObjectId()
            query = {'_id': {'$in': subject['sids']}}
            update = {'$set': {'subject._id': subject_id}}
            config.db.sessions.update_many(query, update)

###
### BEGIN RESERVED UPGRADE SECTION
###

# Due to performance concerns with database upgrades, some upgrade implementations might be postposed.
# The team contract is that if you write an upgrade touch one of the tables mentioned below, you MUST also implement any reserved upgrades.
# This way, we can bundle changes together that need large cursor iterations and save multi-hour upgrade times.

###
### END RESERVED UPGRADE SECTION
###

def upgrade_schema(force_from = None):
    """
    Upgrades db to the current schema version

    Returns (0) if upgrade is successful
    """

    db_version = get_db_version()

    if force_from:
        if isinstance(db_version,int) and db_version >= force_from:
            db_version = force_from
        else:
            logging.error('Cannot force from future version %s. Database only at version %s', str(force_from), str(db_version))
            sys.exit(43)

    if not isinstance(db_version, int) or db_version > CURRENT_DATABASE_VERSION:
        logging.error('The stored db schema version of %s is incompatible with required version %s',
                       str(db_version), CURRENT_DATABASE_VERSION)
        sys.exit(43)
    elif db_version == CURRENT_DATABASE_VERSION:
        logging.error('Database already up to date.')
        sys.exit(43)

    try:
        while db_version < CURRENT_DATABASE_VERSION:
            db_version += 1
            upgrade_script = 'upgrade_to_'+str(db_version)
            logging.info('Upgrading to version {} ...'.format(db_version))
            globals()[upgrade_script]()
            logging.info('Upgrade to version {} complete.'.format(db_version))
    except KeyError as e:
        logging.exception('Attempted to upgrade using script that does not exist: {}'.format(e))
        sys.exit(1)
    except Exception as e:
        logging.exception('Incremental upgrade of db failed')
        sys.exit(1)
    else:
        config.db.singletons.update_one({'_id': 'version'}, {'$set': {'database': CURRENT_DATABASE_VERSION}})
        sys.exit(0)

if __name__ == '__main__':
    try:
        parser = argparse.ArgumentParser()
        parser.add_argument("function", help="function to be called from database.py")
        parser.add_argument("-f", "--force_from", help="force database to upgrade from previous version", type=int)
        args = parser.parse_args()

        if args.function == 'confirm_schema_match':
            confirm_schema_match()
        elif args.function == 'upgrade_schema':
            if args.force_from:
                upgrade_schema(args.force_from)
            else:
                upgrade_schema()
        else:
            logging.error('Unknown method name given as argv to database.py')
            sys.exit(1)
    except Exception as e:
        logging.exception('Unexpected error in database.py')
        sys.exit(1)


import torch
import diskcache
from pycocotools.coco import COCO
import numpy as np
from scipy import ndimage
import skimage.io as io
import skimage.color as color

def _get_center_point(mask, ann_id, cache):
    """
    This is a rudimentary version of https://arxiv.org/pdf/2304.02643.pdf,
    section D.1.Point Sampling

    From the paper: "The first point is chosen deterministically as the point
    farthest from the object boundary."

    The code below is an approximation of this.

    First, we try to calculate the center of mass. If it's inside the mask, we
    stop here.

    The centroid may be outside of the mask for some mask shapes. In this case
    we do a slow hack, specifically, we check for the
    minumum of the maximum distance from the boundary in four directions
    (up, right, down, left), and take the point with the maximum of these
    minimums. Note: this is not performant for large masks.

    Returns the center point in (x, y) format
    """
    if ann_id in cache:
        return cache[ann_id]

    # try the center of mass, keep it if it's inside the mask
    com_y, com_x = ndimage.center_of_mass(mask)
    com_y, com_x = int(round(com_y, 0)), int(round(com_x, 0))
    if mask[com_y][com_x]:
        cache[ann_id] = (com_x, com_y)
        return (com_x, com_y)

    # if center of mass didn't work, do the slow manual approximation

    # up, right, down, left
    # TODO(future): approximate better by adding more directions
    distances_to_check_deg = [0, 90, 180, 270]

    global_min_max_distance = float('-inf')
    global_coords = None
    # For now, terminate early to speed up the calculation as long as
    # the point sample is gooe enough. This sacrifices the quality of point
    # sampling for speed. In the future we can make this more accurate.
    DISTANCE_GOOD_ENOUGH_THRESHOLD = 20

    # Note: precalculating the bounding box could be somewhat
    #   helpful, but checked the performance gain and it's not much
    #   so leaving it out to keep the code simple.
    # Note: tried binary search instead of incrementing by one to
    #   travel up/right/left/down, but that does not handle masks
    #   with all shapes properly (there could be multiple boundaries).
    for row_idx in range(mask.shape[0]):
        for col_idx in range(mask.shape[1]):
            cur_point = mask[row_idx, col_idx]

            # skip points inside bounding box but outside mask
            if not cur_point:
                continue

            max_distances = []
            for direction in distances_to_check_deg:
                # TODO(future) binary search instead of brute forcing it if we
                # need a speedup, with the cache it doesn't really matter though
                if direction == 0:
                    # UP
                    cur_row_idx = row_idx

                    while cur_row_idx >= 0 and mask[cur_row_idx, col_idx]:
                        cur_row_idx = cur_row_idx - 1
                    cur_row_idx += 1
                    distance = row_idx - cur_row_idx
                    max_distances.append(distance)

                elif direction == 90:
                    # RIGHT
                    cur_col_idx = col_idx

                    while cur_col_idx <= mask.shape[1] - 1 and \
                            mask[row_idx, cur_col_idx]:
                        cur_col_idx += 1
                    cur_col_idx -= 1
                    distance = cur_col_idx - col_idx
                    max_distances.append(distance)

                elif direction == 180:
                    # DOWN
                    cur_row_idx = row_idx
                    while cur_row_idx <= mask.shape[0] - 1 and \
                            mask[cur_row_idx, col_idx]:
                        cur_row_idx = cur_row_idx + 1
                    cur_row_idx -= 1
                    distance = cur_row_idx - row_idx
                    max_distances.append(distance)

                elif direction == 270:
                    # LEFT
                    cur_col_idx = col_idx
                    while cur_col_idx >= 0 and mask[row_idx, cur_col_idx]:
                        cur_col_idx -= 1
                    cur_col_idx += 1
                    distance = col_idx - cur_col_idx
                    max_distances.append(distance)

            min_max_distance = min(max_distances)
            if min_max_distance > global_min_max_distance:
                global_min_max_distance = min_max_distance
                global_coords = (col_idx, row_idx)
            if global_min_max_distance >= DISTANCE_GOOD_ENOUGH_THRESHOLD:
                break

    cache[ann_id] = global_coords
    return global_coords

def build_datapoint(imgId,
                    coco,
                    pixel_mean,
                    pixel_std,
                    coco_root_dir,
                    coco_slice_name,
                    catIds,
                    cache,
                    predictor,
                    pad_input_image_batch):
    img = coco.loadImgs(imgId)[0]

    file_location = f'{coco_root_dir}/{coco_slice_name}/{img["file_name"]}'
    I = io.imread(file_location)
    if len(I.shape) == 2:
        # some images, like img_id==61418, are grayscale
        # convert to RGB to ensure the rest of the pipeline works
        I = color.gray2rgb(I)

    # load and display instance annotations
    annIds = coco.getAnnIds(imgIds=img['id'], catIds=catIds, iscrowd=None)
    anns = coco.loadAnns(annIds)

    # approximate the center point of each mask
    coords_list = []
    gt_masks_list = []
    for ann in anns:
        ann_id = ann['id']
        mask = coco.annToMask(ann)
        gt_masks_list.append(torch.tensor(mask))
        coords = _get_center_point(mask, ann_id, cache)
        coords_list.append(coords)

    image = I

    # predictor_set_image begin
    # Transform the image to the form expected by the model
    input_image = predictor.transform.apply_image(image)
    input_image_torch = torch.as_tensor(input_image)
    input_image_torch = input_image_torch.permute(
        2, 0, 1).contiguous()[None, :, :, :]
    predictor_input_size = input_image_torch.shape[-2:]

    # Preprocess
    x = input_image_torch
    # Normalize colors
    x = (x - pixel_mean) / pixel_std

    if pad_input_image_batch:
        # Pad
        h, w = x.shape[-2:]
        padh = predictor.model.image_encoder.img_size - h
        padw = predictor.model.image_encoder.img_size - w
        x = torch.nn.functional.pad(x, (0, padw, 0, padh))
    else:
        x = x.squeeze(0)

    gt_masks_list = torch.stack(gt_masks_list) if len(gt_masks_list) else None
    return image, coords_list, gt_masks_list, anns, x, predictor_input_size

def build_data(coco_img_ids,
               coco,
               catIds,
               coco_root_dir,
               coco_slice_name,
               point_sampling_cache_dir,
               predictor,
               use_half,
               use_nested_tensor,
               pad_input_image_batch):
    cache = diskcache.Cache(point_sampling_cache_dir)
    # make sure you clear the cache if you change the point sampling algorithm
    # cache.clear()

    pixel_mean = predictor.model.pixel_mean.cpu()
    pixel_std = predictor.model.pixel_std.cpu()

    def build_batch(indicies):
        batch = [[], [], [], [], [], [], [], [], [], [], []]
        batch[3] = [0]
        batch[6] = [0]
        for img_idx in indicies:
            imgId = coco_img_ids[img_idx]

            datapoint = build_datapoint(imgId,
                                        coco,
                                        pixel_mean,
                                        pixel_std,
                                        coco_root_dir,
                                        coco_slice_name,
                                        catIds,
                                        cache,
                                        predictor,
                                        pad_input_image_batch)
            I, coords_list, gt_masks_list, anns, x, predictor_input_size = datapoint
            if len(coords_list) == 0:
                continue
            batch[0].append(x)
            # batch[0].append(x[0])
            coords_list = predictor.transform.apply_coords(
                np.array(coords_list), I.shape[:2])
            coords_list = torch.tensor(coords_list, dtype=torch.float)

            batch[1].append(coords_list.reshape(-1))
            batch[2].append(coords_list.size())
            batch[3].append(coords_list.numel() + batch[3][-1])

            batch[4].append(gt_masks_list.reshape(-1))
            batch[5].append(gt_masks_list.size())
            batch[6].append(gt_masks_list.numel() + batch[6][-1])

            batch[7].append(anns)
            batch[8].append(I)
            batch[9].append(predictor_input_size)
            batch[10].append(img_idx)

        def cat_and_cast(b, use_half):
            b = torch.cat(b) if len(b) > 0 else None
            if use_half is not None and b is not None:
                return b.to(use_half)
            return b

        def to_nested_tensor(data, sizes=None, use_half=None):
            if len(data) == 0:
                return None
            dtype = use_half if use_half is not None else torch.float32

            if sizes is not None:
                data = [d.view(s) for (d, s) in zip(data, sizes)]

            return torch.nested.nested_tensor(data, dtype=dtype, layout=torch.jagged)

        if pad_input_image_batch:
            batch[0] = cat_and_cast(batch[0], use_half)
        else:
            batch[0] = to_nested_tensor(batch[0], use_half=use_half)

        if use_nested_tensor:
            batch[1] = to_nested_tensor(batch[1], batch[2], use_half)
            batch[2] = None
            batch[3] = None
        else:
            batch[1] = cat_and_cast(batch[1], use_half)

        batch[4] = cat_and_cast(batch[4], False)

        return batch

    return build_batch

def setup_coco_img_ids(coco_root_dir, coco_slice_name, coco_category_names, img_id):
    annFile = '{}/annotations/instances_{}.json'.format(
        coco_root_dir, coco_slice_name)

    # initialize COCO api for instance annotations
    coco = COCO(annFile)

    # display COCO categories and supercategories
    cats = coco.loadCats(coco.getCatIds())
    cat_id_to_cat = {cat['id']: cat for cat in cats}
    nms = [cat['name'] for cat in cats]
    # print('COCO categories: \n{}\n'.format(' '.join(nms)))

    # nms = set([cat['supercategory'] for cat in cats])
    # print('COCO supercategories: \n{}'.format(' '.join(nms)))

    if coco_category_names is not None:
        catIds = coco.getCatIds(catNms=coco_category_names)
    else:
        catIds = coco.getCatIds()

    if img_id is not None:
        coco_img_ids = [img_id]
    elif coco_category_names is None:
        coco_img_ids = coco.getImgIds()
    else:
        coco_img_ids = coco.getImgIds(catIds=catIds)

    return coco_img_ids, cat_id_to_cat, catIds, coco


"""A module for running experiments that involve the execution of
commands with different combinations of command line arguments.

This is a somewhat short reference documentation of the interface.
There is also a [longer explanation including various
examples](./index.html).

"""

import sys
import re
import itertools
import os
import subprocess
import signal
import time
from collections import namedtuple
from inspect import signature
from filelock import SoftFileLock
import tqdm
from pathos.multiprocessing import ProcessingPool

def _identity(inp):
    """Identity function just returning the input.

    Default for the modifier functions of ``add()``

    """
    return inp

_Run = namedtuple(
    "_Run",
    [
        "name",
        "command",
        "args",
        "creates_file",
        "stdout_file",
        "stdout_mod",
        "stdout_res",
        "header_string",
        "header_command",
        "header_mod",
        "allowed_return_codes",
        "is_selected",
    ],
)
"""Datatype representing a single run."""

def add(
    name,
    command,
    arguments_descr,
    creates_file=None,
    stdout_file=None,
    stdout_mod=_identity,
    stdout_res=None,
    header_string=None,
    header_command=None,
    header_mod=_identity,
    return_string=None,
    allowed_return_codes=[0],
    combinations_filter=lambda args: True,
):
    """Add a new experiment.

    Based on the experiment description, a set of inidvidual runs is
    generated, where each individual run basically corresponds to a
    set of command line arguments.  The arguments are represented by a
    dictionary, i.e., each argument has a key and a value.

    To describe how the arguments of an individual run can be used in
    several places, we need the concept of a *blob*.  Ultimately, a
    blob is something that will be evaluated to a string by replacing
    wildcards of the form ``[[key]]`` with the value of the
    corresponding argument.  A blob can also be a function, in which
    case it is evaluated (with the arguments as parameter) before
    doing this kind of replacements.  For more details on how a blob
    turns into a string, see the function ``deblob()``.  Note that a
    blob is only defined in the context of an individual run.  Thus,
    whenever we talk about blob, we implicitly have an individual run
    with a specific set of arguments in mind.

    Parameters
    ----------

    name: blob

      Name of the experiment.  An experiment is only run if its name
      (or the name of its ``group``) appears as command line parameter.
      It does not need to be unique among experiments.

    command: blob

      The command that will be called for each run.

    arguments_descr: dictionary

      Dictionary of arguments or lists of arguments.  In case of
      lists, one run for each combination of arguments is generated.
      Each individual argument is a blob, where the blobs are
      evaluated to strings in order of appearance in the dictionary.

    creates_file: blob, optional

      Describes the name of a file that is created by calling the
      command.  This is only used to skip the run if the file already
      exists (at the time when this method is called, not at the time
      when the command is actually run).

    stdout_file: blob, optional

      The filename to which the standard output of the run should be
      written.  There are three different cases, depending on whether
      this file already exists:

      1. If the file exists when calling this function (i.e., before
         performing any runs), the run is skipped.

      1. If the run is not skipped but the file exists when executing
         the run, the standard output is added at the end of the file.

      1. If the file does not exist, it is created beginning with the
         header (if specified) and then the standard output is added.

    stdout_mod: function, default =``identity``

      A function applied to the standard output of the run, before
      writing it to the file.  If the function takes one argument, it
      gets the standard output as string as input.  Otherwise, it
      should take two arguments, the standard output as string and the
      result of a call to ``subprocess.run()``.  The latter gives
      access to additional information such as the return code.  The
      function can return a blob using the special wildcard
      ``[[stdout]]`` (similar to ``stdout_res``).

    stdout_res: blob, optional

      If given, this blob is written to the file instead of the
      standard output itself.  This blob is somewhat special in the
      sense that it evaluated after the run has finished with the
      special argument ``stdout``, i.e., the blob can contain the
      special wildcard ``[[stdout]]``, which will be replaced by the
      standard output (after it was modified by ``stdout_mod``).

    header_string: blob, optional

      A string specifying the header; see input parameter
      ``stdout_file``.

    header_command: blob, optional

      A command that is run to use its standard output as header.

    header_mod: function, default =``identity``

      A function that is applied to the header (specified by
      ``header_string`` or ``header_command``) before writing it to a
      file.  It should take one argument (a string) and return a
      string.

    return_string: blob, optional

      If given, this blob will be evaluated for each run and a list of
      the results is returned.

    allowed_return_codes: list[int], default =``[0]``

      A list of allowed return codes.  If a command returns any other
      code, a warning is printed and the run is aborted.  The empty
      list ``[]`` indicates that any return code should be accepted.

    combinations_filter: function, default = always ``True``

      A function that filters the combinations of arguments.  It
      should take a dictionary of arguments and decide whether it
      represents a valid combination by returning ``True`` of
      ``False``.  The default returns always ``True``, i.e., a run is
      created for every combination.

    Returns
    -------
    None or list[string]
        See documentation of ``return_string``.

    """
    if stdout_mod != _identity and stdout_file is None:
        _print_warning("stdout_mod has no effect if stdout_file is not " "specified")
    if stdout_res is not None and stdout_file is None:
        _print_warning("stdout_res has no effect if stdout_file is not " "specified")
    if header_string is not None and stdout_file is None:
        _print_warning("header_string has no effect if stdout_file is not " "specified")
    if header_command is not None and stdout_file is None:
        _print_warning(
            "header_command has no effect if stdout_file is not " "specified"
        )
    if header_string is not None and header_command is not None:
        _print_warning(
            "header_string and header_command specified" " - Which one should I use?"
        )
    if header_mod != _identity and header_string is None and header_command is None:
        _print_warning(
            "header_mod has no effect if not one of "
            "header_string or header_command are specified"
        )

    # generate the set of arguments
    arguments_descr = {
        k: v if isinstance(v, list) else [v] for k, v in arguments_descr.items()
    }
    arguments_set = [
        dict(zip(arguments_descr.keys(), vals))
        for vals in itertools.product(*arguments_descr.values())
    ]

    return_strings = []
    for args in arguments_set:
        if not combinations_filter(args):
            continue
        for key, val in args.items():
            args[key] = deblob(val, args)

        real_name = deblob(name, args)
        if real_name not in _state.groups[_state.group]:
            _state.groups[_state.group].append(real_name)

        run = _Run(
            name=real_name,
            command=deblob(command, args),
            args=args,
            creates_file=deblob(creates_file, args),
            stdout_file=deblob(stdout_file, args),
            stdout_mod=stdout_mod,
            stdout_res=stdout_res,
            header_string=deblob(header_string, args),
            header_command=deblob(header_command, args),
            header_mod=header_mod,
            allowed_return_codes=allowed_return_codes,
            is_selected=_is_selected(real_name),
        )
        _add_run(run)
        if return_string is not None:
            return_strings.append(deblob(return_string, args))

    if return_string is not None:
        return return_strings

def _add_run(run):
    """Add a single run to the list of runs.

    Checks whether the run should was selected or whether it should be
    skippped and adjusts the corresponding data structures
    accordingly.

    """
    if run.name not in _state.runs_by_name:
        _state.runs_by_name[run.name] = []
        _state.counts_by_name[run.name] = [0, 0]

    if _is_skipped(run):
        _state.counts_by_name[run.name][1] += 1
    else:
        _state.counts_by_name[run.name][0] += 1

    if run.is_selected and not _is_skipped(run):
        _state.runs_by_name[run.name].append(run)

def _is_selected(name):
    """Decides whether a given name was selected.

    A name counts as selected if it is give as command line parameter
    or if it belongs to a group that was given as command line
    parameter.

    """
    if name in sys.argv:
        return True
    for group, names in _state.groups.items():
        if group in sys.argv and name in names:
            return True
    return False

def _is_skipped(run):
    """Decides whether a given run should be skipped.

    A run is skipped if the output file already exists.

    """
    return (run.creates_file is not None and os.path.isfile(run.creates_file)) or (
        run.stdout_file is not None and os.path.isfile(run.stdout_file)
    )

def run():
    """Run the previously declared experiments.

    You should call this exactly once at the end of the file.

    If ``dry_run`` is given as command line parameter, then the runs are
    not executed but the commands printed to ``stdout``.

    """
    global _state
    _state.run_was_called = True
    _state.time_start_run = time.time()

    _print_runs()

    if _is_selected("dry_run"):
        _run_dry()
        _state = _State()
        return

    _print_section("\nrunning the experiments:")

    for name, runs in _state.runs_by_name.items():
        if len(runs) == 0:
            continue
        # run in parallel
        orig_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
        pool = ProcessingPool(nodes=_cores)
        signal.signal(signal.SIGINT, orig_sigint_handler)
        try:
            for _ in tqdm.tqdm(
                pool.uimap(_run_run, runs),
                desc=name.ljust(_max_name_len()),
                total=len(runs),
            ):
                pass
        except KeyboardInterrupt:
            _print_warning("aborted during experiment " + name)
    _state.run_completed = True
    _state = _State()

def use_cores(nr_cores):
    """Set the number of cores used to run the experiments.

    Parameters
    ----------

    nr_cores: int

      The number of cores that should be used.

    """
    global _cores
    _cores = nr_cores

_cores = 4
"""The number of cores used to run the experiments."""

def _run_dry():
    """Perform a dry run."""
    _print_section("\ndry run: just printing, no doing")
    for name, runs in _state.runs_by_name.items():
        if len(runs) == 0:
            continue
        # print the runs
        _print_section("\ncommands for experiment " + name)
        for run in runs:
            print(run.command)

def group(group_name):
    """Set the current group.

    Each experiment created with ``add()`` is added to the group for
    which this function was last called.

    Parameters
    ----------

    group_name: string

      The name of the group.

    """
    _state.group = group_name
    if group_name not in _state.groups:
        _state.groups[group_name] = []

def section(title):
    """Print a section title.

    Parameters
    ----------

    title: string

      The title that should be printed.

    """
    _print_section(title)

def deblob(blob, args):
    """Transforms a blob into a string.

    This function is meant for internal use but understanding what it
    does might be useful.  A blob is transferred into a string in the
    following steps.

    1. If ``blob`` is a function, it is called with ``args`` as parameter.

    1. The result (or ``blob`` itself, if 1. did not apply) is converted
    to a string (using ``str()``).

    1. Every pattern of the form ``[[key]]`` is replaced by the value of
    the corresponding argument in ``args``.

    Step 3 assumes that every pattern of the form ``[[key]]`` has a
    corresponding argument with this key.

    Parameters
    ----------

    blob: blob

      The blob that should be turned into a string.

    args: dictionary

      The named arguments of the current run.

    """
    if blob is None:
        return None

    if callable(blob):
        blob = blob(args)

    blob = str(blob)

    keys = [m.group(1) for m in re.finditer(r"\[\[([^\[\]]*)\]\]", blob)]
    result = blob
    for key in keys:
        if key not in args:
            _print_warning("No value for [[" + key + "]] found to replace in " + blob)
            continue
        result = result.replace("[[" + key + "]]", args[key])
    return result

def _run_run(run):
    """Actually run a run."""
    res = _execute(run.command)

    # check return codes
    if (
        res.returncode not in run.allowed_return_codes
        and run.allowed_return_codes != []
    ):
        _print_warning(
            "unexpected return code ("
            + str(res.returncode)
            + ") for command: "
            + res.args
            + "\n"
            + res.stderr.strip()
            + ""
        )
        return

    if run.stdout_file is None:
        return

    filename = run.stdout_file
    lock = filename + ".lock"
    if os.path.dirname(filename) != "":
        os.makedirs(os.path.dirname(filename), exist_ok=True)

    # create new file with header
    if not os.path.isfile(filename) and (
        run.header_command is not None or run.header_string is not None
    ):
        header = run.header_string
        if run.header_command is not None:
            header = _execute(run.header_command).stdout.strip()
        header = run.header_mod(header)

        with SoftFileLock(lock):
            if not os.path.isfile(filename):
                with open(filename, "w") as out:
                    print(header, file=out, flush=True)

    # write to stdout
    stdout = res.stdout.strip()
    mod = run.stdout_mod
    output = mod(stdout) if len(signature(mod).parameters) == 1 else mod(stdout, res)
    if mod != _identity:
        run.args["stdout"] = stdout
        output = deblob(output, run.args)
    if run.stdout_res is not None:
        run.args["stdout"] = output
        output = deblob(run.stdout_res, run.args)

    with SoftFileLock(run.stdout_file + ".lock"):
        with open(run.stdout_file, "a") as out:
            print(output, file=out, flush=True)

def _execute(command):
    """Execute a command line command and return the result.

    This is a wrapper for ``subprocess.run()``.

    """
    return subprocess.run(
        command,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        universal_newlines=True,
    )

def _print_warning(string):
    """Print a warning message."""
    print(_color_warning("\nWARNING: " + string.replace("\n", "\n\t")), file=sys.stderr)

def _print_section(string):
    """Print section heading."""
    print("\033[1m" + string + "\033[0m")

def _print_runs():
    """Print summary of all specified run."""
    format_str = "{:<" + str(_max_name_len() + 5) + "}{:>10}{:>10}{:>10}"
    _print_section(format_str.format("", "todo", "skipped", "total"))
    for group, names in _state.groups.items():
        if len(names) == 0:
            continue
        print(_color_selected(group, _is_selected(group)))
        for name in names:
            prefix = " └─ " if name == names[-1] else " ├─ "
            count = _state.counts_by_name[name]
            print(
                _color_selected(
                    format_str.format(
                        prefix + name, count[0], count[1], count[0] + count[1]
                    ),
                    _is_selected(name),
                )
            )

def _color_warning(string):
    """Return the string but with ansi colors representing a warning."""
    return "\u001b[33m" + string + "\u001b[0m"

def _color_selected(string, selected):
    """Return the string but with ansi colors for (un)selected experiments."""
    col = "\u001b[32;1m" if selected else "\u001b[91;2m"
    return col + string + "\u001b[0m"

def _max_name_len():
    """Gives the length of the longest used name.

    Includes group and experiment names.

    """
    if _state.runs_by_name == {}:
        return 0
    return max([len(name) for name in _state.runs_by_name])

class _State:
    """Internal state."""

    def __init__(self):
        self.runs_by_name = dict()
        self.counts_by_name = dict()
        self.group = "ungrouped"
        self.groups = {self.group: []}
        self.time_start = time.time()
        self.time_start_run = time.time()
        self.run_was_called = False
        self.run_completed = False

    def __del__(self):
        if self.runs_by_name == {}:
            return

        if not self.run_completed:
            return

        if not self.run_was_called:
            _print_warning(
                "Some runs were added without calling run(). "
                "Did you forget to call run() at the end of the script?"
            )
            return

        total_runs = sum(
            [count[0] + count[1] for count in self.counts_by_name.values()]
        )
        print(
            "time for gathering {0:d} runs: {1:.2f} seconds".format(
                total_runs, self.time_start_run - self.time_start
            )
        )

        print(
            "time for running the experiments: {0:.2f} seconds".format(
                time.time() - self.time_start_run
            )
        )
        print()

_state = _State()
"""
Instance of the internal state.
"""


from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import numpy as np

data =  datasets.load_iris()

features = data.data

target = data.target

X = features[:100,]
y = target[:100]

X_train, X_test, y_train, y_test = train_test_split(X,y, test_size = 0.2, random_state = 10)

from perceptron import Perceptron

clf = Perceptron()

train = clf.fit(X_train, y_train)

predict = clf.predict(X_test)

accuracy =  accuracy_score(predict, y_test)

print('perceptron\'s Prediction \n', predict)

print('The accuracy of perceptron prediction is',accuracy)

print()
from LogisticRegression import LogisticRegression

lg = LogisticRegression(eta = 0.01)

model = lg.fit(X_train, y_train)

predict2 = lg.predict(X_test)

accuracy2 =  accuracy_score(predict2, y_test)

print('The accuracy of Logistic Regression\'s prediction is',accuracy2)

print('logistic Regression Prediction \n', predict2)

print('y_test \n', y_test)

print('The estimated sigmoid function of the predicted values are \n',lg.estimate(X_test))

algorithms & models/Iris _Logistic Regression and perceptron.py


#!/usr/bin/env python3
#
## @file
# checkout_pin_command.py
#
# Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#

import os

from git import Repo

from edkrepo.commands.edkrepo_command import EdkrepoCommand, OverrideArgument, SourceManifestRepoArgument
import edkrepo.commands.arguments.checkout_pin_args as arguments
import edkrepo.commands.humble.checkout_pin_humble as humble
from edkrepo.common.common_cache_functions import get_repo_cache_obj
from edkrepo.common.common_repo_functions import sparse_checkout_enabled, reset_sparse_checkout, sparse_checkout
from edkrepo.common.common_repo_functions import check_dirty_repos, checkout_repos, combinations_in_manifest
from edkrepo.common.humble import SPARSE_CHECKOUT, SPARSE_RESET, SUBMODULE_DEINIT_FAILED
from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException, EdkrepoProjectMismatchException
from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos
from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo, get_manifest_repo_path
from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest
from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
from edkrepo_manifest_parser.edk_manifest import ManifestXml
from project_utils.submodule import deinit_full, maintain_submodules
import edkrepo.common.ui_functions as ui_functions

class CheckoutPinCommand(EdkrepoCommand):
    def __init__(self):
        super().__init__()

    def get_metadata(self):
        metadata = {}
        metadata['name'] = 'checkout-pin'
        metadata['help-text'] = arguments.COMMAND_DESCRIPTION
        metadata['alias'] = 'chp'
        args = []
        metadata['arguments'] = args
        args.append({'name' : 'pinfile',
                     'positional' : True,
                     'position' : 0,
                     'required' : True,
                     'help-text' : arguments.PIN_FILE_HELP})
        args.append(OverrideArgument)
        args.append(SourceManifestRepoArgument)
        return metadata

    def run_command(self, args, config):
        workspace_path = get_workspace_path()
        manifest = get_workspace_manifest()

        manifest_repo = find_source_manifest_repo(manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo)
        cfg, user_cfg, conflicts = list_available_manifest_repos(config['cfg_file'], config['user_cfg_file'])
        if manifest_repo in cfg:
            manifest_repo_path = config['cfg_file'].manifest_repo_abs_path(manifest_repo)
        elif manifest_repo in user_cfg:
            manifest_repo_path = config['user_cfg_file'].manifest_repo_abs_path(manifest_repo)
        else:
            manifest_repo_path = None

        pin_path = self.__get_pin_path(args, workspace_path, manifest_repo_path, manifest)
        pin = ManifestXml(pin_path)
        manifest_sources = manifest.get_repo_sources(manifest.general_config.current_combo)
        check_dirty_repos(manifest, workspace_path)
        for source in manifest_sources:
            local_path = os.path.join(workspace_path, source.root)
            repo = Repo(local_path)
            origin = repo.remotes.origin
            origin.fetch()
        self.__pin_matches_project(pin, manifest, workspace_path)
        sparse_enabled = sparse_checkout_enabled(workspace_path, manifest_sources)
        if sparse_enabled:
            ui_functions.print_info_msg(SPARSE_RESET, header = False)
            reset_sparse_checkout(workspace_path, manifest_sources)
        submodule_combo = pin.general_config.current_combo
        try:
            deinit_full(workspace_path, manifest, args.verbose)
        except Exception as e:
            ui_functions.print_error_msg(SUBMODULE_DEINIT_FAILED, header=False)
            if args.verbose:
                ui_functions.print_error_msg(e, header = False)
        pin_repo_sources = pin.get_repo_sources(pin.general_config.current_combo)
        try:
            checkout_repos(args.verbose, args.override, pin_repo_sources, workspace_path, manifest, manifest_repo_path)
            manifest.write_current_combo(humble.PIN_COMBO.format(args.pinfile))
        finally:
            cache_path = None
            cache_obj = get_repo_cache_obj(config)
            if cache_obj is not None:
                cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
            maintain_submodules(workspace_path, pin, submodule_combo, args.verbose, cache_path)
            if sparse_enabled:
                ui_functions.print_info_msg(SPARSE_CHECKOUT, header = False)
                sparse_checkout(workspace_path, pin_repo_sources, manifest)

    def __get_pin_path(self, args, workspace_path, manifest_repo_path, manifest):
        pin_path = None
        if not args.pinfile.endswith('.xml'):
            pin_name = '{}.xml'.format(args.pinfile)
        else:
            pin_name = args.pinfile

        if os.path.isabs(args.pinfile) and os.path.isfile(args.pinfile):
            pin_path = os.path.normpath(args.pinfile)
        elif manifest_repo_path is not None:
            pin_path = self.__find_pin_in_manifest_repo(pin_name, manifest_repo_path, manifest.general_config.pin_path)
        else:
            pin_path = self.__find_pin_in_workspace(workspace_path, pin_name)

        if pin_path:
            return pin_path
        else:
            raise EdkrepoInvalidParametersException(humble.NOT_FOUND)

    def __find_pin_in_manifest_repo(self, pin_name, manifest_repo_path, pin_path):
        expected_path_in_manifest_repo = os.path.normpath(os.path.join(manifest_repo_path, pin_path, pin_name))
        path_if_at_root_of_man_repo = os.path.normpath(os.path.join(manifest_repo_path, pin_name))
        if os.path.isfile(expected_path_in_manifest_repo):
            return expected_path_in_manifest_repo
        elif os.path.isfile(path_if_at_root_of_man_repo): # Corner case to catch pins that may have been placed in the root dir of the manifest repo.
            return path_if_at_root_of_man_repo
        else:
            return None

    def __find_pin_in_workspace(self, workspace_path, pin_name):
        # Before walking the entire workspace attempt to locate the pin at the root and in the repo/ dir as a performance improvement.
        path_at_wkspc_root = os.path.normpath(os.path.join(workspace_path, pin_name))
        path_in_repo_dir = os.path.normpath(os.path.join(workspace_path, 'repo', pin_name))
        if os.path.isfile(path_at_wkspc_root):
            return path_at_wkspc_root
        elif os.path.isfile(path_in_repo_dir):
            return path_in_repo_dir
        elif os.path.dirname(pin_name) is None:
             for dirpath, dirnames, filenames in os.walk(workspace_path):
                if pin_name in filenames:
                    return os.path.join(dirpath, pin_name)
        else:
            return None

    def __pin_matches_project(self, pin, manifest, workspace_path):
        if pin.project_info.codename != manifest.project_info.codename:
            raise EdkrepoProjectMismatchException(humble.MANIFEST_MISMATCH)
        elif not set(pin.remotes).issubset(set(manifest.remotes)):
            raise EdkrepoProjectMismatchException(humble.MANIFEST_MISMATCH)
        elif pin.general_config.current_combo not in combinations_in_manifest(manifest):
            ui_functions.print_warning_msg(humble.COMBO_NOT_FOUND.format(pin.general_config.current_combo), header=True)
        combo_name = pin.general_config.current_combo
        pin_sources = pin.get_repo_sources(combo_name)
        pin_root_remote = {source.root:source.remote_name for source in pin_sources}
        try:
            # If the pin and the project manifest have the same combo get the
            # repo sources from that combo. Otherwise get the default combo's
            # repo sources
            manifest_sources = manifest.get_repo_sources(combo_name)
        except ValueError:
            manifest_sources = manifest.get_repo_sources(manifest.general_config.default_combo)
        manifest_root_remote = {source.root:source.remote_name for source in manifest_sources}
        if set(pin_root_remote.items()).isdisjoint(set(manifest_root_remote.items())):
            raise EdkrepoProjectMismatchException(humble.MANIFEST_MISMATCH)
        pin_root_commit = {source.root:source.commit for source in pin_sources}
        for source in pin_sources:
            source_repo_path = os.path.join(workspace_path, source.root)
            repo = Repo(source_repo_path)
            if repo.commit(pin_root_commit[source.root]) is None:
                raise EdkrepoProjectMismatchException(humble.NOT_FOUND)


# -*- coding: utf-8 -*-
"""
Created on Mon Jul  1 14:12:19 2019

@author: rahul
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

dataset=pd.read_csv("heart.csv")
dataset.shape
x=dataset.iloc[:,0:13]
y=dataset.iloc[:,13]
from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.2)

from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
x_train = sc.fit_transform(x_train)
x_test = sc.transform(x_test)

import tensorflow as tf

model = tf.keras.models.Sequential([
  tf.keras.layers.Dense(128, activation='relu',input_dim=13),
  tf.keras.layers.Dense(64, activation='relu'),
  tf.keras.layers.Dense(1,activation='sigmoid')
])

model.compile(optimizer = 'adam',
              loss = 'binary_crossentropy',
              metrics=['accuracy'])
model.fit(x_train, y_train, batch_size = 10, nb_epoch = 10)
y_pred=model.predict(x_test)
y_pred1=(y_pred>0.5).astype(int)


from django.db import connections
from django.db.utils import DEFAULT_DB_ALIAS, load_backend
import logging

_log = logging.getLogger('control_stock_app')

def create_connection(alias=DEFAULT_DB_ALIAS):
    """Retorna una nueva conexion a la base

    Args:
        alias ([string]): [Nombre default de la base de datos del diccionario de conexiones]

    Returns:
        [django.db.connection]: [Retorna un objeto de conexion de backend.oracle]
    """

    connections.ensure_defaults(alias)
    connections.prepare_test_settings(alias)
    db = connections.databases[alias]
    backend = load_backend(db['ENGINE'])

    return backend.DatabaseWrapper(db, alias)

def get_data_from_procedure(connection, proc_name, **kwargs):

    """
    Funcion que retorna lista de datos desde un procedimiento almacenado

    :param connection: conexion django
    :param proc_name: nombre del procedimiento almacenado a consultar
    :param kwargs: (proc_params: indica la lista de parametros que se asocian al procedimiento almacenado)
    :return: Lista con los datos en formato diccionario
    """

    lst_resultado = []

    with connection.cursor() as cur:

        try:

            _proc_params = kwargs.get('proc_params')

            if _proc_params is None:

                # ejecuto el store procedure sin parametros
                cur.callproc(proc_name)

            else:

                # asigno el cursor a la variable de salida
                #_proc_params['out_param'] = rawCursor

                # convierto a lista los elementos del diccionario (no necesito el key)
                _params = list(_proc_params.values())

                # ejecuto el store procedure con los parametros
                cur.callproc(proc_name, _params)

            # convierto el resultado a lista de diccionarios
            lst_resultado = get_dict_from_cursor(cur)

        except Exception as e:
            _log.error("Error en ejecucion de proceso get_data_from_procedure")
            _log.error("Procedimiento: " + proc_name)
            _log.error(str(e))

        finally:
            try:
                cur.close()
                # _log.debug("cursor cerrado")

            except Exception as e:
                _log.error('Error al cerrar cursor: Procedimiento: ' + proc_name)
                _log.error(str(e))

    return lst_resultado

def set_data_from_procedure(connection, proc_name, **kwargs):

    """
    Funcion que inserta/actualiza datos desde un procedimiento almacenado

    :param connection: conexion django
    :param proc_name: nombre del procedimiento almacenado a ejecutar
    :param kwargs: (proc_params: indica la lista de parametros que se asocian al procedimiento almacenado)
    """

    with connection.cursor() as cur:

        _proc_params = kwargs.get('proc_params')

        try:

            if _proc_params is None:

                # ejecuto el store procedure sin parametros
                cur.callproc(proc_name)

            else:

                # convierto a lista los elementos del diccionario (no necesito el key)
                _params = list(_proc_params.values())

                # ejecuto el store procedure con los parametros
                cur.callproc(proc_name, _params)

        except Exception as e:
            _log.error("Error ejecutandoSP: " + proc_name)
            _log.error(str(e))

        finally:
            cur.close()

def get_dict_from_cursor(cursor):

    """
    Retorna el resultado del cursor en una lista de diccionario
    :param cursor: cursor resultado de la ejecucion del query
    :return: lista de diccionario
    """

    lst_result = []

    try:

        # se obtienen los nombres de las columnas asociadas al resultado de la consulta
        cols = [col[0] for col in cursor.description]

        # por cada registro se arma el diccionario y se lo agrega a la lista
        for datarow in cursor:
            # print(datarow)
            lst_result.append(dict(zip(cols, datarow)))

    except Exception as e:
        _log.error("")
        _log.error("Error en el proceso: get_dict_from_cursor")
        _log.error(str(e))
        _log.error("")

    return lst_result


from flask_app.config.mysqlconnection import connectToMySQL
from flask import flash
import re

DATABASE_SCHEMA = 'adventure_awaits_2_db'

class Image:
    def __init__(self, data):
        self.id = data['id']
        self.title = data['title']
        self.base64 = data['base64']
        self.created_at = data['created_at']
        self.updated_at = data['updated_at']

# C !!!!!!!!!!!!!!!!!!!!!!!!
    @classmethod
    def create(cls, info):
        query = 'INSERT INTO images (base64, title) VALUES (%(base64)s, %(title)s)'
        data = {
            'base64' : info['base64'],
            'title' : info['title'],
        }

        new_images_id = connectToMySQL(DATABASE_SCHEMA).query_db(query, data)

        return new_images_id

    @classmethod
    def create_image_link(cls, info):
        query = 'INSERT INTO posts_has_images (post_id, image_id) VALUES (%(post_id)s, %(image_id)s)'
        data = {
            'post_id' : info['post_id'],
            'image_id' : info['image_id'],
        }

        return connectToMySQL(DATABASE_SCHEMA).query_db(query, data)

# R !!!!!!!!!!!!!!!!!!!!!!!!
    @classmethod
    def get_all(cls):
        query = 'SELECT * FROM images;'
        results = connectToMySQL(DATABASE_SCHEMA).query_db(query)
        if len(results) > 0:
            all_images = []
            for images in results:
                all_images.append(cls(images))
            return all_images

    @classmethod
    def get_all_linked(cls, post_id):
        query = 'SELECT * FROM posts_has_images JOIN images ON images.id = image_id WHERE post_id = %(post_id)s;'
        data = {
            'post_id': post_id
        }
        results = connectToMySQL(DATABASE_SCHEMA).query_db(query, data)
        print(results)
        if results:
            all_images = []
            for image in results:
                all_images.append(Image.get_one(image['id']))
            return all_images
        return results

    @classmethod
    def get_one(cls, id):
        query = 'SELECT * FROM images WHERE id = %(id)s;'
        data = {
            'id': id
        }
        results = connectToMySQL(DATABASE_SCHEMA).query_db(query, data)
        one_images = []
        if len(results) > 0:
           return cls(results[0])
        return results

# U !!!!!!!!!!!!!!!!!!!!!!!!
    @classmethod
    def update_one(cls, info):
        # TODO: add the ability to update pw
        query = 'UPDATE images SET link=%(link)s WHERE id=%(id)s'
        data = {
            'link': info['link'],
            'id': info['id'],
        }
        return connectToMySQL(DATABASE_SCHEMA).query_db(query,data)

# D !!!!!!!!!!!!!!!!!!!!!!!!
    @classmethod
    def delete_one(cls, id):
        query = 'DELETE FROM images WHERE id=%(id)s'
        data = {
            'id': id
        }
        connectToMySQL(DATABASE_SCHEMA).query_db(query,data)
        return id

# ******************************************* VALIDATIONS ********************************************
    @staticmethod
    def validate_images(user_data):
        is_valid = True

        if len(user_data['title']) < 3:
            is_valid = False
            flash('Title name must be greater than 3 characters')

        if len(user_data['base64']) < 3:
            is_valid = False
            flash('Must pick a image to upload')

        return is_valid