#Embedded file name: dropbox/client/photoimporter.py
from __future__ import absolute_import
import collections
import datetime
import errno
import itertools
import json
import operator
import pprint
import re
import struct
import threading
import time
from contextlib import contextmanager
from hashlib import md5
import arch
import dropbox.fsutil as fsutil
from client_api.hashing import DROPBOX_MAX_BLOCK_SIZE, BetterDropboxHasher
from dropbox.attrs import attr_dict_from_whitelist, Attributes, get_attr_data, unfreeze_attr_dict
from dropbox.build_common import get_build_number
from dropbox.camera import PhotoImportCanceled, PhotoImportDisconnected, PhotoImportSelectiveSync, PhotoImportDeviceLocked, PhotoImportExceptionBase, PhotoImportLowDropboxSpace, PhotoImportNoConnectionError, PhotoImportAlbumCreationError
from dropbox.camera.util import is_apple_device
from dropbox.callbacks import Observable, ObservableIterator
from dropbox.debugging import easy_repr
from dropbox.dbexceptions import LowDiskSpaceError
from dropbox.functions import batch, handle_exceptions, safe_str, split_extension
from dropbox.i18n import trans
from dropbox.lock_ordering import NonRecursiveLock
from dropbox.metadata.metadata import get_metadata_for_plat
from dropbox.metadata.transforms import try_rotate_image_file
from dropbox.native_event import AutoResetEvent
from dropbox.path import ServerPath
from dropbox.timehelper import tz_offset_string
from dropbox.trace import TRACE, unhandled_exc_handler, report_bad_assumption, reraise_exc_handler
IMPORT_FREE_SPACE_BUFFER = 50 * 1024 * 1024
CU_HASH_8_BYTES = 8 * 1024
NUM_READ_RETRIES = 10
XATTR_EXIF_MANUFACTURER = 'exif_manufacturer'
XATTR_EXIF_MODEL = 'exif_model'
XATTR_EXIF_DATETIME = 'exif_datetime'
XATTR_DEVICE_MANUFACTURER = 'device_manufacturer'
XATTR_DEVICE_MODEL = 'device_model'
XATTR_DEVICE_NAME = 'device_name'
XATTR_DEVICE_UID = 'device_uid'
XATTR_DEVICE_SERIALNUM = 'device_serialnum'
XATTR_CLIENT_IMPORT_TIME = 'client_import_time'
XATTR_CLIENT_PLATFORM = 'client_platform'
XATTR_CLIENT_BUILDSTRING = 'client_buildstring'
XATTR_CLIENT_USERID = 'client_userid'
XATTR_CLIENT_TIMEOFFSET = 'client_timeoffset'
XATTR_FILE_DATETIME = 'file_datetime'
XATTR_FILE_MTIME = 'file_mtime'
XATTR_FILE_HASHFULL = 'file_hashfull'
XATTR_FILE_NAMEORIG = 'file_nameorig'
CAMERA_XATTRS = [XATTR_EXIF_MANUFACTURER,
 XATTR_EXIF_MODEL,
 XATTR_EXIF_DATETIME,
 XATTR_DEVICE_MANUFACTURER,
 XATTR_DEVICE_MODEL,
 XATTR_DEVICE_NAME,
 XATTR_DEVICE_UID,
 XATTR_DEVICE_SERIALNUM,
 XATTR_CLIENT_IMPORT_TIME,
 XATTR_CLIENT_PLATFORM,
 XATTR_CLIENT_BUILDSTRING,
 XATTR_CLIENT_USERID,
 XATTR_CLIENT_TIMEOFFSET,
 XATTR_FILE_DATETIME,
 XATTR_FILE_MTIME,
 XATTR_FILE_HASHFULL]
CAMERA_XATTRS_PLAT = 'dropbox_camera_upload'
XATTR_COLLECTION_GIDS = 'collection_gids'
COLLECTION_XATTRS_PLAT = 'dropbox_collection_gid'
PHOTOIMPORT_EXCEPTIONS_RERAISE = (PhotoImportExceptionBase, LowDiskSpaceError)
EXIF_IMAGE_ORIENTATION = 'Image Orientation'
EXIF_IMAGE_MAKE = 'Image Make'
EXIF_IMAGE_MODEL = 'Image Model'
EXIF_LENGTH = 'EXIF ExifImageLength'
EXIF_WIDTH = 'EXIF ExifImageWidth'
EXIF_THUMBNAIL = 'Thumbnail Compression'
EXIF_THUMBNAIL_OFFSET = 'Thumbnail JPEGInterchangeFormat'
EXIF_THUMBNAIL_ORIENTATION = 'Thumbnail Orientation'
EXIF_JPEG_THUMBNAIL = 'JPEGThumbnail'
EXIF_TIFF_THUMBNAIL = 'TIFFThumbnail'
EXIF_DATETIME = 'EXIF DateTimeOriginal'
EXIF_CUSTOMRENDERED = 'EXIF CustomRendered'
PHOTO_EXIF_WHITELIST = {'exif': {EXIF_IMAGE_ORIENTATION: {},
          EXIF_IMAGE_MAKE: {},
          EXIF_IMAGE_MODEL: {},
          EXIF_LENGTH: {},
          EXIF_WIDTH: {},
          EXIF_THUMBNAIL: {},
          EXIF_THUMBNAIL_OFFSET: {},
          EXIF_THUMBNAIL_ORIENTATION: {},
          EXIF_JPEG_THUMBNAIL: {},
          EXIF_TIFF_THUMBNAIL: {},
          EXIF_DATETIME: {},
          EXIF_CUSTOMRENDERED: {}}}
EXIF_TO_CAMERA_ATTR = {EXIF_IMAGE_MAKE: XATTR_EXIF_MANUFACTURER,
 EXIF_IMAGE_MODEL: XATTR_EXIF_MODEL,
 EXIF_DATETIME: XATTR_EXIF_DATETIME}
DEFAULT_UPLOAD_LOCATION = u'/Camera Uploads'
EXIF_DATETIME_STRPTIME_FMTS = ['%Y:%m:%d %H:%M:%S', '%Y:%m:%d:%H:%M:%S', '%Y:%m:%d %I:%M:%S %p']

def parse_exif_datetime(data, strptime_fn):
    normalized_data = data.replace('/', ':').replace('-', ':').replace('.', '')
    for i, pattern in enumerate(EXIF_DATETIME_STRPTIME_FMTS):
        try:
            return strptime_fn(normalized_data, pattern)
        except ValueError:
            if i == len(EXIF_DATETIME_STRPTIME_FMTS) - 1:
                raise
            continue

def _nexus_datetime_from_path(taken_at, path):
    try:
        match = re.search('img_(\\d{8}_\\d{6})(_\\d+)?\\.jpg', path.lower())
        if match:
            fname_datetime = datetime.datetime.strptime(match.group(1), '%Y%m%d_%H%M%S')
            delta = fname_datetime - taken_at
            if abs(delta) > datetime.timedelta(seconds=60):
                taken_at = fname_datetime
    except ValueError:
        pass
    except Exception:
        unhandled_exc_handler()

    return taken_at.strftime('%Y:%m:%d %H:%M:%S')

def _motorola_datetime_from_path(taken_at, path):
    try:
        match = re.search('(\\d{4}).(\\d{2}).(\\d{2}).(\\d{2}).(\\d{2}).(\\d{2})(_\\d+)?\\.jpg', path.lower())
        if match:
            fname_datetime = datetime.datetime(*map(int, match.group(1, 2, 3, 4, 5, 6)))
            delta = fname_datetime - taken_at
            if abs(delta) > datetime.timedelta(seconds=60):
                taken_at = fname_datetime
    except ValueError:
        pass
    except Exception:
        unhandled_exc_handler()

    return taken_at.strftime('%Y:%m:%d %H:%M:%S')

def update_device_specific_deets(details):
    exif_data = details.exif
    if EXIF_DATETIME in exif_data:
        try:
            taken_at = parse_exif_datetime(exif_data[EXIF_DATETIME], datetime.datetime.strptime)
            if (exif_data.get('Image Make', '').lower(), exif_data.get('Image Model', '').lower()) in (('lge', 'nexus 4'), ('samsung', 'galaxy nexus')):
                exif_data[EXIF_DATETIME] = _nexus_datetime_from_path(taken_at, details.f_name)
            elif exif_data.get('Image Make', '').lower() == 'motorola':
                exif_data[EXIF_DATETIME] = _motorola_datetime_from_path(taken_at, details.f_name)
        except ValueError:
            TRACE('Unable to parse %r into datetime.', exif_data[EXIF_DATETIME])
        except Exception:
            unhandled_exc_handler()

class PhotoDetails(object):
    __use_advanced_allocator__ = 1048576
    __slots__ = ('blocklist', 'size', 'mtime', 'path', 'cu_hash_8', 'cu_hash_full', 'exif', 'f_obj', 'f_name', 'f_basename', 'f_ext', 'f_time', 'photo_name', 'photo_time', 'member_of_albums', 'rotation_result')

    def __init__(self, f_obj):
        self.mtime = time.time()
        self.f_obj = f_obj
        self.f_time = f_obj.time()
        self.size = f_obj.size()
        self.f_name = f_obj.name()
        self.f_basename, self.f_ext = split_extension(self.f_name)
        self.member_of_albums = None

    def __repr__(self):
        return easy_repr(self, 'f_obj', 'f_time', 'size', 'f_name')

OUT_OF_QUOTA = 'out of quota'
OUT_OF_DISK_SPACE = 'out of disk space'
NOT_OUT_OF_SPACE = 'not out of space'
MAYBE_OUT_OF_SPACE = 'maybe out of space'

class PhotoImporter(threading.Thread):
    STARTING, SCANNING, TRANSFERRING, DONE = range(4)
    N = 0
    lock = NonRecursiveLock()

    def __init__(self, device, uploader, app):
        with PhotoImporter.lock:
            thread_name = 'CAMERAIMPORTER-%d' % PhotoImporter.N
            PhotoImporter.N += 1
            self.importer_id = PhotoImporter.N
        super(PhotoImporter, self).__init__(name=thread_name)
        self.device = device
        self.uploader = uploader
        self.photodb = self.uploader.photodb
        self.app = app
        self.fs = self.app.sync_engine.fs
        self.state = Observable(self.STARTING, handle_exc=unhandled_exc_handler)
        self.total_bytes = Observable(0, handle_exc=unhandled_exc_handler)
        self.cur_bytes = Observable(0, handle_exc=unhandled_exc_handler)
        self.transferred_files = Observable([], handle_exc=unhandled_exc_handler)
        self.error = Observable(None, handle_exc=unhandled_exc_handler)
        self.found_files = Observable(0, handle_exc=unhandled_exc_handler)
        self.remaining_files = set([])
        self.skipped_files = []
        self._cancel = False
        self._disconnected = False
        self._event = AutoResetEvent()
        self.has_imported = False
        self.cu_hashes_full = set()
        self._show_finding_progress = False
        self._dest_path_verified = False
        self._files_to_read = None

    def _prep_for_file_read(self):
        pass

    def cancel(self):
        self._cancel = True
        self._event.set()
        return False

    def disconnected(self):
        self._disconnected = True
        self._event.set()
        return False

    def device_ready(self):
        self._event.set()

    def check_cancel(self, timeout = 0):
        self._event.wait(timeout=timeout)
        if self._cancel:
            raise PhotoImportCanceled()
        if self._disconnected:
            raise PhotoImportDisconnected()

    def _get_dest_path_sp(self):
        return ServerPath.from_ns_rel(self.app.sync_engine.main_root_ns, self.uploader.location or DEFAULT_UPLOAD_LOCATION)

    def run(self):
        try:
            if self.app.quota < self.app.in_use:
                raise PhotoImportLowDropboxSpace()
            self.state.set(self.STARTING)
            if not self.app.sync_engine.check_if_initial_list_is_done():
                ev = threading.Event()
                if not self.app.sync_engine.check_if_initial_list_is_done(ev.set):
                    ev.wait()
            status = self.app.sync_engine.status
            while not status.try_set_status_label('importing', True, fail_if_set=['moving']):
                TRACE('Currently moving Dropbox folder, import thread is sleeping. Status: %r', status.get_status())
                self.check_cancel(1)

            wait_for_list_results_thunk = self.uploader.demand_freshness()
            try:
                while not self.device.ready:
                    if not self._show_finding_progress and self.device.percent != 100 and self.device.percent < 40:
                        self._show_finding_progress = True
                        self.total_bytes.set(100)
                    if self._show_finding_progress:
                        self.cur_bytes.set(self.device.percent)
                    TRACE('Device is still connecting.')
                    self.check_cancel(1)

                self._prep_for_file_read()
                sync_engine = self.app.sync_engine
                self.dest_path_sp = self._get_dest_path_sp()
                self.dest_path = sync_engine.server_to_local(self.dest_path_sp)
                self.cache_path = sync_engine.verify_cache_path()
                self.sel_sync = sync_engine._ignore_set_should_ignore(self.dest_path_sp)
                self.import_time = time.time()
                self.device_attr_dict = self._create_device_attr_dict(self.device)
                with self.device.files() as files:

                    def update_found_files(num_found):
                        self.found_files.set(num_found)
                        if num_found % 50 == 0:
                            self.check_cancel()

                    files_observed = ObservableIterator(files, handle_exc=reraise_exc_handler)
                    files_observed.register(update_found_files)
                    if wait_for_list_results_thunk:
                        wait_for_list_results_thunk()
                        wait_for_list_results_thunk = None
                    self._files_to_read = self._filter_device_files((PhotoDetails(f) for f in files_observed))
                    if not self._files_to_read:
                        return
                    available_quota, available_disk_space = self.get_available_space()
                    import_space_usage = self.get_import_space_usage(self._files_to_read, available_quota, available_disk_space, num_imported=self.photodb.num_imported())
                    self.import_normally = self.can_import_normally(import_space_usage)
                    if not self.import_normally:
                        self.get_available_space(trace=True)
                    self._transfer_files()
            finally:
                status.set_status_label('importing', False)
                TRACE("Setting status label 'importing' to false!")

        except Exception as e:
            TRACE('!! Photo import raised exception %r', e)
            if not isinstance(e, PHOTOIMPORT_EXCEPTIONS_RERAISE):
                unhandled_exc_handler()
            if arch.photouploader.is_disconnected_error(e):
                TRACE('Exception seems to be the result of a device disconnection. Not bubbling import error.')
            else:
                self.error.set(e)
        else:
            self.error.set(None)
        finally:
            self.state.set(self.DONE)
            self.device.release()

    @staticmethod
    def _filter_seen_photos(device_files, seen_photos):
        if not seen_photos:
            return (list(device_files), [], [])
        TRACE('%d photos have been imported from this device before. Pruning.', len(seen_photos))
        entries_seen = []
        new_files = []
        for details in device_files:
            photo = (details.f_name, details.f_time)
            if details.f_time is not None and photo in seen_photos:
                seen_photos.remove(photo)
                entries_seen.append(details)
            else:
                new_files.append(details)

        return (new_files, list(seen_photos), entries_seen)

    @contextmanager
    def _handle_photo_exceptions(self):
        try:
            yield
        except PHOTOIMPORT_EXCEPTIONS_RERAISE:
            raise
        except (IOError, OSError) as e:
            if e.errno == errno.ENOSPC:
                raise LowDiskSpaceError(0, 0)
            self.device.handle_disconnect_exceptions(e)
            unhandled_exc_handler()
        except Exception as e:
            self.device.handle_disconnect_exceptions(e)
            unhandled_exc_handler()

    @staticmethod
    def update_photo_name_and_time(details, default = None):
        try:
            st_time = parse_exif_datetime(details.exif.get(EXIF_DATETIME, ''), time.strptime)
            details.photo_name = time.strftime(u'%Y-%m-%d %H.%M.%S', st_time)
            details.photo_time = time.mktime(st_time)
        except Exception:
            TRACE('Failed to rename photo by EXIF time, falling back to default!')
            st_time = details.f_time.timetuple() if details.f_time else time.localtime(default)
            details.photo_name = time.strftime(u'%Y-%m-%d %H.%M.%S', st_time)
            details.photo_time = time.mktime(st_time)

    def update_photo_details(self, details):
        with self.fs.open(details.path, 'r') as tmp:
            details.exif = get_metadata_for_plat('exif', tmp, PHOTO_EXIF_WHITELIST).get('exif', {})
        update_device_specific_deets(details)
        self.update_photo_name_and_time(details, default=self.import_time)
        if 'HDR' in details.exif.get(EXIF_CUSTOMRENDERED, ''):
            details.photo_name += u' HDR'

    def _filter_device_files(self, device_files):
        TRACE('Looking for previously imported files from this device.')
        files, entries_to_remove, self.skipped_files = self._filter_seen_photos(device_files, self.photodb.get_seen_photos(self.device.uid))
        if (files or self.skipped_files) and entries_to_remove:
            TRACE('Pruning %d entries from the database.', len(entries_to_remove))
            self.photodb.del_seen_photos(self.device.uid, entries_to_remove)
        TRACE('Found and processing %r new files from device! (%r seen photos skipped, %r seen photos removed)', len(files), len(self.skipped_files), len(entries_to_remove))
        found_nothing = not files and not self.skipped_files
        if arch.constants.platform == 'win' and is_apple_device(self.device) and found_nothing:
            raise PhotoImportDeviceLocked()
        if self.device.locked and found_nothing:
            raise PhotoImportDeviceLocked()
        TRACE('%r potentially new files will be read from device!', len(files))
        return files

    def _transfer_files(self):
        files = self._files_to_read
        self.total_bytes.set(sum((deet.size for deet in files)))
        self.has_imported = self._has_imported_before()
        self.remaining_files = set(files)
        self.state.set(self.SCANNING)
        cache = self.cache_path.join(datetime.date.today().strftime('camera-import-%Y-%m-%d'), str(self.importer_id))
        retries = [0]
        pending_files = collections.deque()
        by_timestamp = []
        self.pending_size_in_cache = 0

        def progress(n):
            self.cur_bytes.set(self.cur_bytes.get() + n)

        self.app.safe_makedirs(unicode(cache))
        try:
            for i, details in enumerate(files):
                with self._handle_photo_exceptions():
                    self.check_cancel()
                    self.app.sync_engine.verify_disk_space(cache, details.size)
                    bytes_read = self._read_and_hash(details, cache, retries, self.check_cancel, progress)
                    if bytes_read == 0:
                        TRACE('Skipping 0 byte file %r', details)
                        self.skipped_files.append(details)
                        self.remaining_files.remove(details)
                        continue
                    self.update_photo_details(details)
                    if self._handle_existing_hashes(details, files, i):
                        continue
                    if self.sel_sync:
                        raise PhotoImportSelectiveSync(self.dest_path_sp)
                    if self._end_photo_group(pending_files, details):
                        self._maybe_write_photos(pending_files, by_timestamp)
                        pending_files.clear()
                    pending_files.append(details)

            if pending_files:
                with self._handle_photo_exceptions():
                    self._maybe_write_photos(pending_files, by_timestamp)
                    pending_files.clear()
        except (LowDiskSpaceError, PhotoImportLowDropboxSpace, PhotoImportSelectiveSync):
            raise
        except Exception:
            self.complete_maybe_import(by_timestamp)
            raise
        else:
            self.complete_maybe_import(by_timestamp)
        finally:
            try:
                fsutil.rmtree(self.fs, cache, ignore_errors=True)
            except Exception:
                unhandled_exc_handler()

    def _end_photo_group(self, pending_files, curr_deets):
        return pending_files and pending_files[0].photo_name != curr_deets.photo_name

    def complete_maybe_import(self, by_timestamp):
        if not self.import_normally:
            for by_basename in by_timestamp:
                with self._handle_photo_exceptions():
                    self.check_space(sum((deet.size for f_basename, deet_list in by_basename for deet in deet_list)))
                    self._rename_photos(by_basename)

    def _read_and_hash(self, details, dest_dir, retries, check_cancel, progress_cb):
        dest = None
        bytes_read = 0
        assert struct.calcsize('I') == 4
        mask = 2 ** (8 * struct.calcsize('I')) - 1
        try:
            with details.f_obj.open(self.fs, dest_dir) as read:
                cu_hash_full = md5()
                cu_hash_8 = md5()
                cu_hash_8_left = CU_HASH_8_BYTES
                fail_fast = getattr(details.f_obj, 'fail_fast', False)
                needs_write = getattr(details.f_obj, 'needs_write', True)
                cu_hash_8.update(struct.pack('>I', details.size & mask))
                while True:
                    check_cancel()
                    try:
                        buf = read(DROPBOX_MAX_BLOCK_SIZE)
                    except Exception:
                        retries[0] += 1
                        if retries[0] >= NUM_READ_RETRIES or fail_fast:
                            raise
                        check_cancel(1)
                        try:
                            buf = read(DROPBOX_MAX_BLOCK_SIZE)
                        except Exception:
                            raise
                        else:
                            retries[0] = 0

                    else:
                        retries[0] = 0

                    if not buf:
                        break
                    l_buf = len(buf)
                    progress_cb(l_buf)
                    bytes_read += l_buf
                    if needs_write:
                        if dest is None:
                            dest = fsutil.mkstemp(self.fs, suffix=details.f_ext, dir=dest_dir)
                            details.path = self.fs.make_path(dest.name)
                        dest.write(buf)
                    size = len(buf)
                    cu_hash_full.update(buf)
                    if cu_hash_8_left:
                        m_8 = min(cu_hash_8_left, size)
                        cu_hash_8.update(buffer(buf, 0, m_8))
                        cu_hash_8_left -= m_8

                if not needs_write:
                    details.path = self.fs.make_path(details.f_obj.path)
        finally:
            if dest:
                dest.close()

        details.cu_hash_8 = cu_hash_8.hexdigest()
        details.cu_hash_full = cu_hash_full.hexdigest()
        if bytes_read != details.size:
            report_bad_assumption("Bytes read don't match details")
        return bytes_read

    def get_import_space_usage(self, *args, **kwargs):
        return self.import_space_usage(*args, **kwargs)

    @staticmethod
    def import_space_usage(files, available_quota, available_disk, num_imported):
        max_import_size = sum((f.size for f in files))
        min_import_size = sum((f.size for f in sorted(files, key=operator.attrgetter('size'))[:-num_imported])) if num_imported else max_import_size
        TRACE('Determining if importing normally: max import size: %r, num imported: %r, min import size: %r', max_import_size, num_imported, min_import_size)
        if available_quota >= max_import_size and available_disk >= max_import_size:
            return NOT_OUT_OF_SPACE
        if available_quota < min_import_size:
            return OUT_OF_QUOTA
        if available_disk < min_import_size:
            return OUT_OF_DISK_SPACE
        return MAYBE_OUT_OF_SPACE

    def can_import_normally(self, import_space_usage):
        if import_space_usage == NOT_OUT_OF_SPACE:
            TRACE('Definitely not going over quota.  Importing normally.')
            return True
        if import_space_usage == OUT_OF_QUOTA:
            raise PhotoImportLowDropboxSpace()
        elif import_space_usage == OUT_OF_DISK_SPACE:
            raise LowDiskSpaceError(0, 0)
        elif import_space_usage == MAYBE_OUT_OF_SPACE:
            TRACE("!! May go over quota, can't import normally.  Importing into the cache temporarily!")
            return False

    def get_available_space(self, pending_size_in_cache = 0, trace = False):
        quota = self.app.quota
        unearned = self.uploader.cu_quota_unearned
        in_use = self.app.in_use
        pending_size = self.photodb.get_pending_size() + pending_size_in_cache
        if pending_size < 0:
            report_bad_assumption('In get_available_space, pending size is negative! photodb.pending: %r, pending_in_cache: %r', self.photodb.get_pending_size(), pending_size_in_cache)
        quota_remaining = quota + unearned - in_use - pending_size - IMPORT_FREE_SPACE_BUFFER
        free_disk_space = self.fs.get_disk_free_space(self.app.config['dropbox_path'])
        disk_space_remaining = free_disk_space - IMPORT_FREE_SPACE_BUFFER
        if trace:
            pending = self.photodb.pending()
            TRACE('quota(%s) + unearned(%s) - in_use(%s) - pending(%s (%s)) - buffer(%s) = %s', quota, unearned, in_use, pending_size, len(pending), IMPORT_FREE_SPACE_BUFFER, quota_remaining)
            TRACE('free disk space(%s) - buffer(%s) = %s', free_disk_space, IMPORT_FREE_SPACE_BUFFER, disk_space_remaining)
            TRACE('Pending = %s', pprint.pformat(pending))
        return (quota_remaining, disk_space_remaining)

    def check_space(self, size, pending_size_in_cache = 0):
        avail_quota, avail_disk = self.get_available_space(pending_size_in_cache)
        if avail_quota < size:
            self.get_available_space(pending_size_in_cache, trace=True)
            raise PhotoImportLowDropboxSpace()
        if avail_disk < size:
            self.get_available_space(pending_size_in_cache, trace=True)
            raise LowDiskSpaceError(0, 0)

    def generate_path_from_deets(self, details, n):
        fn = u'%s%s%s' % (details.photo_name, u'-%d' % n if n else u'', details.f_ext.lower())
        return self.dest_path.join(fn)

    def _handle_dup(self, details):
        self.skipped_files.append(details)
        self.remaining_files.remove(details)
        fsutil.safe_remove(self.fs, details.path)
        if details.f_time:
            self.uploader.add_seen_photos(self.device.uid, ((details.f_name, details.f_time),))

    def _handle_existing_hashes(self, details, files, cur_index):
        is_dup = False
        if details.cu_hash_full in self.cu_hashes_full or self.uploader.exists(details.cu_hash_full):
            is_dup = True
            TRACE('Photo %r, cu_hash_full %r exists in our photo database already!  Not importing', details.f_name, details.cu_hash_full)
            self._handle_dup(details)
        else:
            self.cu_hashes_full.add(details.cu_hash_full)
            self._rotate_and_hash(details)
        return is_dup

    def _maybe_write_photos(self, files, by_timestamp):
        try:
            self.check_cancel()
            size = sum((deets.size for deets in files))
            self.check_space(size, self.pending_size_in_cache)
            details_by_f_basename = collections.defaultdict(list)
            for details in files:
                details_by_f_basename[details.f_basename].append(details)

            by_basename = sorted(details_by_f_basename.iteritems())
            if self.import_normally:
                self._rename_photos(by_basename)
            else:
                by_timestamp.append(by_basename)
                self.pending_size_in_cache += size
        except PHOTOIMPORT_EXCEPTIONS_RERAISE:
            raise
        except Exception:
            unhandled_exc_handler()

    def _rename_photos(self, ordered_by_f_basename):
        self._verify_dest_path(ordered_by_f_basename)
        suffix = 0 if len(ordered_by_f_basename) == 1 else 1
        to_add_seen = []
        for f_name, details_list in ordered_by_f_basename:
            self.state.set(self.TRANSFERRING)
            self.check_cancel()
            for n in itertools.count(suffix):
                paths = [ self.generate_path_from_deets(details, n) for details in details_list ]
                if any((fsutil.is_exists(self.fs, path) for path in paths)):
                    continue
                else:
                    break
            else:
                raise Exception('Too many files with the same name')

            suffix = n + 1
            for path, details in itertools.izip(paths, details_list):
                details.photo_time += n * 0.001
                try:
                    self._write_photo_to_path(details, path)
                except Exception:
                    unhandled_exc_handler()
                    continue

                if not self.uploader.exists(details.cu_hash_full):
                    self.uploader.add_photo(details)
                self.transferred_files.append(details)
                self.remaining_files.remove(details)
                if details.f_time:
                    to_add_seen.append((details.f_name, details.f_time))
                TRACE('Downloaded image to %r, blocklist %r', path, details.blocklist)

        if to_add_seen:
            self.uploader.add_seen_photos(self.device.uid, to_add_seen)

    def _has_imported_before(self):
        return self.photodb.has_imported()

    def _verify_dest_path(self, file_deets_pairs = None, force = False):
        if self._dest_path_verified and not force:
            return
        dirs_created = []
        fsutil.makedirs(self.fs, self.dest_path, dirs_created)
        if dirs_created:
            try:
                self.app.sync_engine._retag_special_folders()
            except Exception:
                unhandled_exc_handler()

        else:
            self.has_imported = self.has_imported or self._has_imported_before()
            if not self.has_imported:
                reason = trans(u'Old')
                self.app.sync_engine._resolve_conflict(self.dest_path, self.dest_path.basename, reason)
                fsutil.makedirs(self.fs, self.dest_path)
        self.has_imported = True
        self._dest_path_verified = True

    def _rotate_and_hash(self, details):
        exif_attrs = details.exif

        def on_success(path):
            if self.uploader.rotation_type == self.uploader.ROTATION_TYPE_V1:
                path = self.fs.make_path(path)
                old_path = details.path
                new_temp = details.path.append(u'.rot')
                self.fs.rename(path, new_temp)
                details.path = new_temp
                fsutil.safe_remove(self.fs, old_path)

        event_result = try_rotate_image_file(unicode(details.path), ext=details.f_ext, exif_attrs=exif_attrs, on_error=unhandled_exc_handler, on_success=on_success, cache_dir=unicode(self.cache_path))
        TRACE('Rotation event result: %r rotation type: %r', event_result, self.uploader.rotation_type)
        details.rotation_result = event_result
        dbhash = BetterDropboxHasher()
        with self.fs.open(details.path, sequential=True) as f:
            while True:
                s = f.read(DROPBOX_MAX_BLOCK_SIZE)
                if s == '':
                    break
                dbhash.update(s)

        details.blocklist = dbhash.digest()

    def _write_photo_to_path(self, details, newpath):
        self.fs.rename(details.path, newpath)
        details.path = newpath
        try:
            self._write_attrs(details)
        except Exception:
            unhandled_exc_handler()

        exif_attrs = details.exif
        try:
            self.report_camera_event('photo-rotate', self.device, result=details.rotation_result, length=exif_attrs.get(EXIF_LENGTH, ''), width=exif_attrs.get(EXIF_WIDTH, ''), orientation=exif_attrs.get(EXIF_IMAGE_ORIENTATION, ''), thumbnail=EXIF_THUMBNAIL in exif_attrs, rotation_type=self.uploader.rotation_type)
        except Exception:
            unhandled_exc_handler()

        try:
            self.fs.set_file_mtime(details.path, details.photo_time)
        except Exception:
            unhandled_exc_handler()

    def _write_attrs(self, details):
        camera_import_attrs = self._create_import_attrs(details)
        self.app.sync_engine.attr_handler.write_attributes(camera_import_attrs, None, details.path)

    def _create_device_attr_dict(self, device):
        attr_dict = {}
        for xattr in CAMERA_XATTRS:
            if xattr.startswith('device'):
                try:
                    device_attr_name = xattr.split('_')[1]
                except Exception:
                    TRACE('!!Camera xattr name %r not formatted properly!  Expected device_[propertyname]', xattr)
                    unhandled_exc_handler()
                else:
                    device_attr = getattr(self.device, device_attr_name, None)
                    if device_attr:
                        if not isinstance(device_attr, str):
                            device_attr = device_attr.encode('utf8')
                        attr_dict[xattr] = device_attr

        plat_string = arch.util.get_platform_info()
        plat_string = plat_string[0] + ' ' + plat_string[3]
        import_attr_dict = {XATTR_CLIENT_IMPORT_TIME: str(int(self.import_time * 1000)),
         XATTR_CLIENT_TIMEOFFSET: tz_offset_string(datetime.datetime.now()),
         XATTR_CLIENT_PLATFORM: plat_string,
         XATTR_CLIENT_BUILDSTRING: get_build_number(),
         XATTR_CLIENT_USERID: str(self.app.uid)}
        attr_dict.update(import_attr_dict)
        TRACE('Readable general import attrs: %r', attr_dict)
        return attr_dict

    def _get_additional_import_attrs(self, *args, **kwargs):
        return (None, None)

    def _create_import_attrs(self, details):
        old_attrs = self.app.sync_engine.attr_handler.read_attributes(details.path)
        old_attrs_dict = unfreeze_attr_dict(old_attrs.attr_dict)
        attr_dict = self._create_file_attr_dict(details)
        TRACE('Readable camera upload file attrs: %r, file: %r', attr_dict, details.path)
        attr_dict.update(self.device_attr_dict)
        attr_dict = attr_dict_from_whitelist(attr_dict, CAMERA_XATTRS_PLAT)
        formatted = old_attrs_dict
        formatted.update(dict([(CAMERA_XATTRS_PLAT, attr_dict)]))
        additional_plats, additional_attrs = self._get_additional_import_attrs(details, old_attrs_dict)
        if additional_attrs:
            formatted.update(additional_attrs)
        new_data_plats = set(old_attrs.data_plats)
        new_data_plats.add(CAMERA_XATTRS_PLAT)
        if additional_plats:
            new_data_plats.add(additional_plats)
        attrs = Attributes(attr_dict=formatted, data_plats=new_data_plats)
        TRACE('Encoded attrs: %r', attrs)
        return attrs

    def _create_file_attr_dict(self, details):
        ret = {}
        for exif_attr_name, attr in details.exif.items():
            if exif_attr_name in EXIF_TO_CAMERA_ATTR:
                ret[EXIF_TO_CAMERA_ATTR[exif_attr_name]] = attr

        manufacturer = ret.get(XATTR_EXIF_MANUFACTURER, None)
        model = ret.get(XATTR_EXIF_MODEL, None)
        if manufacturer and model:
            manufacturer = manufacturer.split()[0]
            if model.startswith(manufacturer):
                model = model[len(manufacturer):].strip()
            ret[XATTR_EXIF_MANUFACTURER] = manufacturer
            ret[XATTR_EXIF_MODEL] = model
        if details.photo_time:
            try:
                photo_time = datetime.datetime.fromtimestamp(details.photo_time)
                _datetime = photo_time.strftime('%Y:%m:%d %H:%M:%S')
                if photo_time.microsecond:
                    _datetime += '.%.3d' % (photo_time.microsecond / 1000)
            except Exception:
                unhandled_exc_handler()
            else:
                ret[XATTR_FILE_DATETIME] = _datetime

        ret[XATTR_FILE_HASHFULL] = details.cu_hash_full
        try:
            basename = self.app.sync_engine.arch.make_path(unicode(details.f_name)).basename
        except Exception:
            basename = unicode(details.f_name)

        ret[XATTR_FILE_NAMEORIG] = safe_str(basename)
        return ret

def get_albums_by_photo_dict(albums_list):
    by_photo_id = collections.defaultdict(list)
    for album in albums_list:
        for photo_id in album.photos_list:
            by_photo_id[photo_id].append(album)

    return by_photo_id

class PhotoGalleryImporter(PhotoImporter):
    ALBUM_BATCH_SIZE = 250

    def __init__(self, *args, **kwargs):
        self.check_space_callback = kwargs.pop('check_space_callback')
        self.import_albums = kwargs.pop('import_albums')
        self.create_event_subdirs = kwargs.pop('create_subdirs')
        super(PhotoGalleryImporter, self).__init__(*args, **kwargs)
        self.albums_by_photo = {}
        self.event_dirname_by_id = {}

    def _get_additional_import_attrs(self, details, old_attrs):
        if details.member_of_albums:
            existing_albums = []
            try:
                try:
                    attr = old_attrs[COLLECTION_XATTRS_PLAT][XATTR_COLLECTION_GIDS]
                except KeyError:
                    pass
                else:
                    existing_albums = json.loads(get_attr_data(attr))

            except Exception:
                unhandled_exc_handler()

            albums = [ album.server_collection_gid for album in details.member_of_albums ]
            albums += existing_albums
            attr_dict = dict([(XATTR_COLLECTION_GIDS, json.dumps(albums))])
            TRACE('IPHOTOIMPORT: Album membership file attrs: %r', attr_dict)
            attr_dict = attr_dict_from_whitelist(attr_dict, COLLECTION_XATTRS_PLAT)
            return (COLLECTION_XATTRS_PLAT, dict([(COLLECTION_XATTRS_PLAT, attr_dict)]))
        return (None, None)

    def _get_dest_path_sp(self):
        default_location = '/' + trans(u'Photos from iPhoto')
        return ServerPath.from_ns_rel(self.app.sync_engine.main_root_ns, default_location)

    def _has_imported_before(self):
        return self.photodb.get_device_last_import(self.device.uid) is not None

    def update_photo_details(self, details):
        super(PhotoGalleryImporter, self).update_photo_details(details)
        details.member_of_albums = self.albums_by_photo.get(details.f_obj.id)

    @staticmethod
    def update_photo_name_and_time(details, default = None):
        try:
            st_time = details.f_obj.iphoto_time()
            if st_time is None:
                st_time = parse_exif_datetime(details.exif.get(EXIF_DATETIME, ''), time.strptime)
            details.photo_name = time.strftime(u'%Y-%m-%d %H.%M.%S', st_time)
            details.photo_time = time.mktime(st_time)
        except Exception:
            TRACE('Failed to rename photo by EXIF or iPhoto time, falling back to default!')
            st_time = details.f_time.timetuple() if details.f_time else time.localtime(default)
            details.photo_name = time.strftime(u'%Y-%m-%d %H.%M.%S', st_time)
            details.photo_time = time.mktime(st_time)

    def _prep_for_file_read(self):
        if self.create_event_subdirs:
            self.events = self.device.events()
            TRACE('IPHOTOIMPORT: Got %d events', len(self.events))
        if self.import_albums:
            albums = self.device.albums()
            server_gids = []
            if albums:
                albums_list = albums.values()
                server_id_by_iphoto_id = self.photodb.get_iphoto_album_server_cgid_mapping()
                if server_id_by_iphoto_id:
                    TRACE('IPHOTOIMPORT: Got saved iPhoto album to server gid mapping! %d collections previously created on server!', len(server_id_by_iphoto_id))
                else:
                    TRACE('IPHOTOIMPORT: Creating %d collections on the server!', len(albums))
                    for album_batch in batch(albums_list, self.ALBUM_BATCH_SIZE):
                        try:
                            ret = self.app.conn.create_collections([ album.name for album in album_batch ])
                        except Exception as e:
                            unhandled_exc_handler()
                            if self.app.conn.is_transient_error(e):
                                raise PhotoImportNoConnectionError()
                            else:
                                raise PhotoImportAlbumCreationError()
                        else:
                            server_gids.extend(ret['collection_gids'])
                            self.check_cancel()

                    for album, server_gid in zip(albums_list, server_gids):
                        server_id_by_iphoto_id[album.uid] = server_gid

                    self.photodb.save_iphoto_album_server_cgid_mapping(server_id_by_iphoto_id)
                for iphoto_album_uid, server_gid in server_id_by_iphoto_id.iteritems():
                    albums[iphoto_album_uid].server_collection_gid = server_gid

                albums_list = [ album for album in albums.itervalues() if album.server_collection_gid is not None ]
                self.albums_by_photo = get_albums_by_photo_dict(albums_list)
            else:
                self.albums_by_photo = {}

    def get_import_space_usage(self, *args, **kwargs):
        import_space_usage = self.import_space_usage(*args, **kwargs)
        if self.check_space_callback:
            if import_space_usage != NOT_OUT_OF_SPACE:
                self.get_available_space(trace=True)
            self.check_space_callback(import_space_usage, len(self._files_to_read))
            self.check_cancel()
        return import_space_usage

    def _handle_existing_hashes(self, details, files, cur_index):
        other_path = None
        is_dup = False
        if details.cu_hash_full in self.cu_hashes_full:
            is_dup = True
            TRACE('Photo %r, cu_hash_full %r was already imported in this session!  Not importing', details.f_name, details.cu_hash_full)
        else:
            self._rotate_and_hash(details)
            other_path = self.app.sync_engine.find_blocklist_in_dir(details.blocklist, self.dest_path_sp)
            if other_path:
                other_path = self.app.sync_engine.server_to_local(other_path.server_path)
                is_dup = True
                TRACE('Photo %r, blocklist %r exists in the camera uploads folder at %s already!  Not importing', details.f_name, details.blocklist, other_path)
        if is_dup:
            self._handle_dup(details)
            if details.member_of_albums:
                TRACE('Duplicate photo was in iPhoto albums! Adding existing photo to albums')
                if details.cu_hash_full in self.cu_hashes_full:
                    for other_deets in reversed(files[:cur_index]):
                        if other_deets.cu_hash_full == other_deets.cu_hash_full and other_deets not in self.skipped_files:
                            TRACE('Found existing file in this import with same hash at %s', other_deets.path)
                            if other_deets.member_of_albums is None:
                                other_deets.member_of_albums = []
                            other_deets.member_of_albums.extend(details.member_of_albums)
                            if other_deets.path.dirname == self.dest_path and fsutil.is_exists(other_deets.path):
                                other_path = other_deets.path
                            break

                if other_path:
                    TRACE('Found existing file in folder with same blocklist: %s', other_path)
                    details.path = other_path
                    self._write_attrs(details)
        else:
            self.cu_hashes_full.add(details.cu_hash_full)
        return is_dup

    @handle_exceptions
    def run(self):
        if self.create_event_subdirs:
            try:
                self.event_dirname_by_id = self.photodb.get_iphoto_event_dirname_by_id()
                self.event_dirname_by_id = dict(((event_id, self.fs.make_path(dirname)) for event_id, dirname in self.event_dirname_by_id.iteritems()))
            except Exception:
                unhandled_exc_handler()
                self.event_dirname_by_id = {}

        super(PhotoGalleryImporter, self).run()
        if self.check_space_callback:
            if not self._files_to_read:
                self.check_space_callback(NOT_OUT_OF_SPACE, 0)
        else:
            self.photodb.clear_iphoto_album_server_cgid_mapping()

    def _end_photo_group(self, pending_files, curr_deets):
        if self.create_event_subdirs:
            return pending_files and (pending_files[0].photo_name != curr_deets.photo_name or pending_files[0].f_obj.event_id() != curr_deets.f_obj.event_id())
        else:
            return pending_files and pending_files[0].photo_name != curr_deets.photo_name

    def _verify_dest_path(self, file_deets_pairs, force = False):
        super(PhotoGalleryImporter, self)._verify_dest_path(force=force)
        if self.create_event_subdirs:
            deetslist = file_deets_pairs[0][1]
            event_id = deetslist[0].f_obj.event_id()
            dirs_created = []
            event_dir = self.event_dirname_by_id.get(event_id)
            if event_dir:
                TRACE('IPHOTOIMPORT: Found existing event folder %r', unicode(event_dir))
                if not fsutil.is_exists(self.fs, event_dir):
                    fsutil.makedirs(self.fs, event_dir, dirs_created)
            else:
                try:
                    event_dir = self.events[event_id].name
                except KeyError:
                    unhandled_exc_handler()
                    event_dir = trans(u'Untitled event')
                else:
                    TRACE('IPHOTOIMPORT: Creating folder for event %r', event_dir)
                    event_dir = normalize_event_name(event_dir)

                for suffix in itertools.count():
                    event_dirname = event_dir if suffix == 0 else '%s (%d)' % (event_dir, suffix)
                    event_dir_path = self.dest_path.join(event_dirname)
                    fsutil.makedirs(self.fs, event_dir_path, dirs_created)
                    if dirs_created:
                        break

                TRACE('IPHOTOIMPORT: Creating folder for event %r at path %r', event_id, unicode(event_dir_path))
                self.photodb.save_iphoto_event_dirname_by_id(event_id, unicode(event_dir_path))
                self.event_dirname_by_id[event_id] = event_dir_path

    def generate_path_from_deets(self, details, n):
        fn = u'%s%s%s' % (details.photo_name, u'-%d' % n if n else u'', details.f_ext.lower())
        if self.create_event_subdirs:
            dirpath = self.event_dirname_by_id[details.f_obj.event_id()]
        else:
            dirpath = self.dest_path
        return dirpath.join(fn)

STRIPPED_CHAR_PLACEHOLDER = '-'

def normalize_event_name(name):
    normalized_name = re.sub('[\\\\/:*?"<>|]+', STRIPPED_CHAR_PLACEHOLDER, name)
    normalized_name = re.sub('^\\.+', STRIPPED_CHAR_PLACEHOLDER, normalized_name)
    if not normalized_name or normalized_name != name and not normalized_name.strip(STRIPPED_CHAR_PLACEHOLDER):
        normalized_name = trans(u'Untitled event')
    return normalized_name


import numpy as np
import cv2
from skimage.filters import threshold_otsu
from skimage import morphology
from skimage.color import rgb2lab
from skimage.io import imsave
import os
import gdal
import matplotlib.pyplot as plt

def imread_gdal(data_name,level):

    gdalObj = gdal.Open(data_name)

    rOverview = gdalObj.GetRasterBand (1)
    gOverview = gdalObj.GetRasterBand (2)
    bOverview = gdalObj.GetRasterBand (3)
    if level>=0:
        rOverview = rOverview.GetOverview(level)
        gOverview = gOverview.GetOverview(level)
        bOverview = bOverview.GetOverview(level)
    rOverview = rOverview.ReadAsArray(0,0, rOverview.XSize, rOverview.YSize)
    gOverview = gOverview.ReadAsArray(0,0, gOverview.XSize, gOverview.YSize)
    bOverview = bOverview.ReadAsArray(0,0, bOverview.XSize, bOverview.YSize)

    return np.stack((rOverview ,gOverview ,bOverview),axis=2)

p=r'D:\MPI_CBG\data_plice\data\normal_tif'

file_names=[]
for folder_path in p:
    for root, dirs, files in os.walk(folder_path):
        for name in files:
            if name.endswith("normal.tif"):
                file_names.append(root+'\\'+name)

for file_name in file_names:
    i+=1

    print(str(i) + '/'+ str(len(file_names)) + '  ' + file_name)

    img_s=imread_gdal(file_name,5)

    plt.figure()
    plt.imshow(img_s)
    plt.title(file_name)


import datetime
import openpyxl
def check_date(y, m, d):
    correctDate= None
    try:
        newDate= datetime.datetime(y,m,d)
        correctDate= "Valid Date"
        myfile= openpyxl.load_workbook('event.xlsx')
        sheet= myfile.get_sheet_by_name('Sheet1')
        sheet['A2']= 'My Birthday'
        event_date= str(y)+ '-'+str(m)+'-'+str(d)
        sheet['B2'] = event_date
        myfile.save('event.xlsx')
    except ValueError:
        correctDate = "Invalid Date"
    return correctDate
day= int(input("Enter Day: "))
mon= int(input("Enter Month: "))
yr= int(input("Enter Year: "))
res=check_date(yr,mon,day)
print(res)

14 July (Live Session Submission)/LabEx2.py


from instabot import Bot
from os.path import exists
from datetime import datetime
import os
import time
import random

os.system("rm -rf config")

bot = Bot()
bot.login(username="", password="")

dir_image = "img/"

my_hashtags = """
#Migraine #Migraines #MigraineAttack #MigrainesSuck #MigraineTriggers #MigraineProblems #MigraineLife #AbdominalMigraines #Headache #MenstrualMigraine  #Headaches #ChronicMigraine #ChronicHeadache   #HeadacheFromHell
#Losangeles #LAgram #USA #America#California #UnitedStates #city #citylife#view #bigcity
"""

my_caption = ""
for content_file in os.listdir(dir_image):
    if content_file.endswith("jpg"):

        caption_file = content_file[:-4] + ".txt"
        if exists(dir_image + caption_file):
            print("Start Time =", datetime.now())
            random_seconds = random.randint(560, 999)
            time.sleep(random_seconds)
            with open(dir_image + caption_file) as f:
                lines = f.readlines()
                buffer_lines = []
                for line in lines:
                    # print("LINE")
                    # print(line)
                    new_line = line.replace("", "")
                    new_line = new_line.replace("", "")
                    new_line = new_line.replace("", "")
                    buffer_lines.append(new_line)
            buffer_lines.pop()
            buffer_lines.append(my_hashtags)
            my_caption = "".join(buffer_lines)
            bot.upload_photo(dir_image + content_file, caption=my_caption)
            print(dir_image + content_file, " was uploaded.")
            print("End Time =", datetime.now())

'''

'''


# cnn으로 변경
# 파라미터 변경해보고
# 노드 개수, activation 추가
# epochs = [1,2,3]
# learning_rate 추가

import numpy as np
from tensorflow.keras.datasets import mnist
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers  import Dense, Dropout, Input, Conv1D, MaxPooling1D, Flatten

# 1. 데이터
(x_train, y_train), (x_test, y_test) = mnist.load_data()

from tensorflow.keras.utils import to_categorical
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)

x_train = x_train.reshape(60000, 28*28).astype('float32')/255
x_test = x_test.reshape(10000, 28*28).astype('float32')/255

# print(x_train.shape, y_train.shape)         # (60000, 28, 28) (60000, 10)

# 2. 모델
def build_model(drop=0.5, optimizer='adam'):
    '''
    inputs = Input(shape=(28*28), name='input')
    x = Dense(512, activation='relu', name='hidden1')(inputs)
    x = Dropout(drop)(x)
    x = Dense(256, activation='relu', name='hidden2')(inputs)
    x = Dropout(drop)(x)
    x = Dense(128, activation='relu', name='hidden3')(inputs)
    x = Dropout(drop)(x)
    outputs = Dense(10, activation='softmax', name='outputs')(x)
    model = Model(inputs=inputs, outputs=outputs)
    model.compile(optimizer=optimizer, metrics=['acc'],
                    loss='categorical_crossentropy')
    '''

    inputs = Input(shape=(28*28,1), name='input')
    x = Conv1D(filters = 2, kernel_size=3, activation= 'relu')(inputs)
    x = Dropout(drop) (x)
    x = MaxPooling1D(2,2) (x)
    x = Dropout(drop) (x)
    x = Flatten() (x)
    x = Dense(2, activation='relu', name='hidden3')(x)
    x = Dropout(drop)(x)
    outputs = Dense(10, activation='softmax', name='outputs')(x)
    model = Model(inputs=inputs, outputs=outputs)
    model.compile(optimizer=optimizer, metrics=['acc'],
                    loss='categorical_crossentropy')

    return model

def create_hyperparameter():
    batches = [1000, 2000, 3000, 4000, 5000]
    optimizers = ['rmsprop'] # , 'adam', 'adadelta']
    dropout = [0.3, 0.4, 0.5]
    return {"batch_size" : batches, "optimizer" : optimizers,
            "drop" : dropout}

hyperparameters = create_hyperparameter()
# print(hyperparameters)

from tensorflow.keras.wrappers.scikit_learn import KerasClassifier
# model2 = build_model()        # 아래처럼 써준다
model2 = KerasClassifier(build_fn=build_model, verbose=1)

from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
# model = RandomizedSearchCV(model2, hyperparameters, cv=5)
model = GridSearchCV(model2, hyperparameters, cv=2)

model.fit(x_train, y_train, verbose=1, epochs=2) #, validation_split=0.2)

print(model.best_params_)
print(model.best_estimator_)
print(model.best_score_)
acc = model.score(x_test, y_test)
print("최종 스코어 : ", acc)

'''
최종 스코어 :  0.4219000041484833
'''


import os
import sys
import logging
import psycopg2

from configuration import DB_CONFIG, DSN

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
MIGRATIONS_DIR = os.path.join(BASE_DIR, 'sql/')
UTILS = ['bootstrap', 'migration']

def bootstrap(db_config: dict) -> None:
    """
    Function for creating all needed for work tables
    in already created database.

    :param db_config: database configuration
    """

    logging.info('Run bootstrap')
    try:
        with psycopg2.connect(dsn=DSN.format(**db_config)) as connection:
            logging.info('Connected to PostgresDB')
            with connection.cursor() as cursor:
                with open(MIGRATIONS_DIR + '002_tables.sql', 'r') as bs:
                    cursor.execute(bs.read())
    except psycopg2.OperationalError as e:
        sys.exit(f'System error: {e}')
    logging.info('Bootstrap is finished')

def migrations(db_config: dict) -> None:
    """
    Function for migrate database

    :param db_config:  database configuration
    :return:
    """

    logging.info(f'Run sql')
    pass

if __name__ == '__main__':
    # Set logging
    logging.basicConfig(
        level=logging.DEBUG,
        format='[{asctime}][{levelname}] - {name}: {message}', style='{'
    )

    # Set configurations
    db_config = DB_CONFIG

    if len(sys.argv) > 1 and sys.argv[1] != 'help':
        util = sys.argv[1]
        if util in UTILS and util == 'bootstrap':
            bootstrap(db_config=db_config)

        elif util in UTILS and util == 'migration':
            migrations(db_config=db_config)

        else:
            print(f'Unknown util {util}')
            print('Usage main.py {util} ')
            print('Module: %s' % (' | '.join(UTILS)))

    else:
        print('Set util')
        print('Usage main.py {util} ')
        print('Module: %s' % (' | '.join(UTILS)))


# -*- coding: UTF-8 -*-

# standard
import configparser
from distutils.command.config import config

# module
import gdrivepdfdl

def main():

    # 初期設定
    settings_file_path = 'settings.ini'
    config = configparser.ConfigParser()

    if (not config.read(settings_file_path)):
        print(f'{settings_file_path} is not found.')
        return

    # [bin]
    proxy_bat_path        = config['bin']['proxy_bat_path']
    edge_driver_path      = config['bin']['edge_driver_path']
    # [userdata]
    edge_profile_path     = config['userdata']['edge_profile_path']
    gdrive_url            = config['userdata']['gdrive_url']
    # [settings]
    mime_type             = config['settings']['mime_type']
    mode                  = config['settings']['mode']
    width                 = int(config['settings']['width'])
    rendering_waiting_sec = int(config['settings']['rendering_waiting_sec'])
    access_waiting_sec    = int(config['settings']['access_waiting_sec'])
    output_dir            = config['settings']['output_dir']

    print(f"Proxy bat path          : {proxy_bat_path}")
    print(f"Edge driver path        : {edge_driver_path}")
    print(f"Edge profile path       : {edge_profile_path}")
    print(f"Google drive folder url : {gdrive_url}")
    print(f"mime_type               : {mime_type}")
    print(f"q(quality) or s(speed)  : {mode}")
    print(f"width                   : {width}")
    print(f"rendering wait sec      : {rendering_waiting_sec}")
    print(f"access wait sec         : {access_waiting_sec}")
    print(f"output dir              : {output_dir}")

    gdh = gdrivepdfdl.GDriveHar(proxy_bat_path, edge_driver_path, edge_profile_path, gdrive_url)

    pdf_data = gdh.get(mode, mime_type, width, rendering_waiting_sec, access_waiting_sec)
    urls = gdrivepdfdl.Utility.to_base64_list(pdf_data['har'], mime_type)
    gdrivepdfdl.Utility.save_image(output_dir, urls,  pdf_data['pdf_infos'], mime_type, True)

if __name__ == '__main__':
    main()


import sympy
from sympy import *

#State what the program does
print("This program uses the Newton's Method technique to approximate zeroes of functions.\n\n")

#Collect input
function = input("What is the function? Write in terms of x. Example functions:\n y = 2*x**3 + x**2 - x + 1\n y = exp(x) - 5*x\n\n")
function = function.split("=")[1] #take the expression, ignore the "y = " portion
x = sympy.symbols("x") #define x for expression
function = sympy.parse_expr(function) #convert into sympy expression so derivative can be taken automatically
print("Function: ", function)
derivative = sympy.diff(function)
print("Derivative : ", derivative)
initial_guess = float(input("What is your initial guess for the function's zero?\n"))
#provide option to use Newton's method for a desired accuracy or given number of iterations
option_for_iterations = input("Type 'Q' to provide a desired accuracy or 'N' to provide a set number of iterations.\n").upper()
if(option_for_iterations == "Q"):
  global desired_accuracy
  desired_accuracy = float(input("What is the desired accuracy?\n"))
elif(option_for_iterations == "N"):
  global given_num_iterations
  given_num_iterations = int(input("Enter number of iterations.\n"))
else:
  print("Invalid Input.")
  exit()

#Create equations
def f(x_n):
  result = function.subs(x, x_n)
  return result
def df(x_n):
  result = derivative.subs(x, x_n)
  return result
def newton_formula(x):
  x_n_plus_1 = x - (f(x)/df(x))
  return x_n_plus_1

#Iterations
iterate = True #boolean to keep applying Newton's method
numIterations = 0
xn = initial_guess

while(iterate == True):
  print("\nX" + str(numIterations) + " = " + str(xn))
  print("Yn = ", f(xn))
  print("Y'n = ", df(xn))
  print("Yn/Y'n = ", f(xn)/df(xn))
  x_n_plus_1 = newton_formula(xn)
  print("X" + str(numIterations+1) + " = " + str(x_n_plus_1))
  numIterations += 1
  #check condition to stop iterating based on the option the user chose (accuracy vs given iterations)
  if((option_for_iterations == "Q" and abs(x_n_plus_1 - xn) < desired_accuracy)) or(option_for_iterations == "N" and (numIterations >= given_num_iterations)):
    iterate = False
  xn = x_n_plus_1

print("\nThe zero of the given function is: ", xn)
print("Note: Took ", numIterations, " iterations")


import json
import os.path
import pprint
import re
import sys

import openpyxl

#既定のアセットパッケージリスト
DEFINED_ASSET_PACKAGES = ["Castle_01_Package", "Castle_02_Package", "Castle_03_Package", "Castle_04_Package", "Castle_05_Package", "Castle_06_Package", "Forest_01_Package", "Forest_02_Package", "Forest_03_Package", "Forest_04_Package", "Forest_05_Package", "Forest_06_Package", "Shrine_01_Package", "Shrine_04_Package", "Tunnel_01_Package", "Tunnel_05_Package"]
#各パッケージごとの既定オブジェクトとセル上での記号の対応
DEFINED_OBJECTS_DICT = {
    "Castle": {
            "扉": "Door", "鍵": "ControlledDoor", "障": "Obstacle",
            "⇒": "OnewayDoor", "鳥": "Bird", "柱": "Pillar1",
            "燭": "Candlestick", "毒": "PoisonZone", "ス": "Script",
            "麻": "ParalyzeZone", "沈": "SilenceZone",
            "ダ": "DamageFloor", "→": "ShiftZone", "←": "ShiftZone",
            "↑": "ShiftZone", "↓": "ShiftZone", "転": "Teleporter",
            "上": "UpStairs", "天": "CeilingHole", "穴": "Chute",
                #CeilingHole: 上方向にはしご設置可能
            "下": "DownStairs", "地": "FloorHole",
                #FloorHole: 下方向にはしご設置可能
            "壁": "WallScript", "敵": "Enemy", "宝": "Item",
            "音": "MusicChanger", "": "Fence", "柱": "Pillar2",
            "出": "Exit"
                #Pillar2はセルのフォント設定がイタリック体
        },
    "Tunnel": {
            "扉": "Door", "鍵": "ControlledDoor", "宝": "Item",
            "⇒": "OnewayDoor", "-": "FenceCenter", "柱": "Pillar",
            "手": "Hand", "灯": "Emissive", "毒": "PoisonZone",
            "麻": "ParalyzeZone", "沈": "SilenceZone",
            "透": "DamageFloorInvisible", "ダ": "DamageFloor",
            "→": "ShiftZone", "←": "ShiftZone", "ス": "Script",
            "↑": "ShiftZone", "↓": "ShiftZone", "転": "Teleporter",
            "上": "UpStairs", "天": "CeilingHole", "穴": "Chute",
            "下": "DownStairs", "地": "FloorHole",
            "壁": "WallScript", "敵": "Enemy", "障": "Crucifixion",
            "音": "MusicChanger", "": "Fence", "出": "Exit"
        },
    "Shrine": {
            "扉": "Door", "鍵": "ControlledDoor", "碑": "Obelisk",
            "⇒": "OnewayDoor", "柱": "Pillar", "灯": "Light",
            "毒": "PoisonZone", "麻": "ParalyzeZone",
            "沈": "SilenceZone", "ダ": "DamageFloor",
            "→": "ShiftZone", "←": "ShiftZone", "ス": "Script",
            "↑": "ShiftZone", "↓": "ShiftZone", "転": "Teleporter",
            "上": "UpStairs", "天": "CeilingHole", "穴": "Chute",
            "下": "DownStairs", "地": "FloorHole",
            "壁": "WallScript", "敵": "Enemy", "宝": "Item",
            "音": "MusicChanger", "": "Fence", "出": "Exit"
        },
    "Forest": {
            "扉": "Door", "鍵": "ControlledDoor", "灯": "Emissive",
            "墓": "GraveMarker", "⇒": "OnewayDoor", "柱": "Pillar1",
            "木": "Tree", "透": "DamageFloorInvisible",
            "毒": "PoisonZone", "麻": "ParalyzeZone",
            "沈": "SilenceZone", "ダ": "DamageFloor",
            "→": "ShiftZone", "←": "ShiftZone", "ス": "Script",
            "↑": "ShiftZone", "↓": "ShiftZone", "転": "Teleporter",
            "上": "UpStairs", "天": "CeilingHole", "穴": "Chute",
            "下": "DownStairs", "地": "FloorHole",
            "壁": "WallScript", "敵": "Enemy", "宝": "Item",
            "音": "MusicChanger", "": "Fence", "柱": "Pillar2",
            "出": "Exit"
        }
}
#セルに書いてあっても無視するオブジェクトのリスト
IGNORE_OBJECT_LIST = ["ControlledDoor", "Teleporter", "Script", "UpStairs", "CeilingHole", "Chute", "DownStairs", "FloorHole", "WallScript", "Enemy", "Item", "MusicChanger"]
#マップのサイズはゲーム内で既定なのでそのまま使う
WIDTH = 30
HEIGHT = 30
#セル名縦列から数字への変換テーブル
cell_tbl = {"A": 0, "B": 1, "C": 2, "D": 3, "E": 4, "F": 5, "G": 6, "H": 7, "I": 8, "J": 9, "K": 10, "L": 11, "M": 12, "N": 13, "O": 14, "P": 15, "Q": 16, "R": 17, "S": 18, "T": 19, "U": 20, "V": 21, "W": 22, "X": 23, "Y": 24, "Z": 25, "AA": 26, "AB": 27, "AC": 28, "AD": 29}
#数字からセル名への逆変換テーブル
cell_rtbl = {v: k for k, v in cell_tbl.items()}
re_splitter = re.compile("(?P<x>[A-Z]+)(?P<y>[0-9]+)")
#床タイプ(黄: 床, 青: 浅瀬, 緑: 深瀬, 黒: 奈落, 白: 壁)
tiletype_tbl = {"FFFFFF00": 0, "FF0000FF": 1, "FF00FF00": 2, "FF000000": 3, "FFFFFFFF": 4}
tiletype_rtbl = {v: k for k, v in tiletype_tbl.items()}
#Point(int, int)からblocksシートのブロックセル名への変換
pointdict2coodinate = lambda d: "%s%s" % (cell_rtbl[d["x"]], (HEIGHT - d["y"]))
#(int, int)からセル名への変換
point2coodinate = lambda x, y: "%s%s" % (openpyxl.utils.cell.get_column_letter(x+1), y+1)

#mapObjectEntryを作成
def create_object(d, name, point, facing, components=[]):
    if "_" not in name:
        name = "%s_%s%s" % (name, point, facing)
    #if name.split("_")[0] not in DEFINED_OBJECTS:
    defined = DEFINED_OBJECTS_DICT[d["assetPackage"]["package"].split("_")[0]].values()
    if name.split("_")[0] not in defined:
        print("[create_object]Error: %s is not valid object name." % name)
        exit(-1)
    point = re_splitter.fullmatch(point).groupdict()
    point["x"] = int(cell_tbl[point["x"]])
    point["y"] = 30 - int(point["y"])
    return {"name": name, "point": point, "facing": int(facing), "components": components}

#フェンスをmapObjectEntriesに追加
def add_fence(cell, d):
    l = [cell.border.top, cell.border.right, cell.border.bottom, cell.border.left]
    for i in range(0, 4):
        if l[i] is None or l[i].style is None:
            continue
        d["mapObjectEntries"].append(create_object(d, "Fence", cell.coordinate, i))

#セルに書かれた値を元にオブジェクトを生成してmapObjectEntriesに追加
def add_object(cell, d):
    asset_type = d["assetPackage"]["package"].split("_")[0]
    obj_dict = DEFINED_OBJECTS_DICT[asset_type]
    name = obj_dict[cell.value[0]]
    point = cell.coordinate
    facing = int(cell.value[1])
    #Pillar1/2の別があるのはCastleとForestのみ
    if name.startswith("Pillar") and asset_type in ("Castle", "Forest"):
        if cell.font.i:
            name = "Pillar2"
        else:
            name = "Pillar1"
    #詳細に設定しなければならないオブジェクトはセルに書いてあっても無視する
    if name not in IGNORE_OBJECT_LIST:
        d["mapObjectEntries"].append(create_object(d, name, point, facing))
    #一つのセルに複数のオブジェクトが入っていたらvalueを変更してもう一度回す
    if(cell.value > 2):
        cell.value = cell.value[2:]
        add_object(cell, d)

#objectsシートからmapObjectEntriesを作成
def read_objects(ws, d):
    for row in ws.rows:
        if not row[0].value:
            continue
        name = row[0].value
        point = row[1].value
        facing = row[2].value
        components = []
        if len(row) > 3 and row[3].value:
            key = ("name", "value")
            for x in row[3:]:
                if x.value and ":" in x.value:
                    components.append(dict(zip(key, x.value.split(":"))))
        d["mapObjectEntries"].append(create_object(d, name, point, facing, components))

#blocksシートからブロックデータとフェンス等のオブジェクトを読む
def read_blocks(ws, d):
    defined = DEFINED_OBJECTS_DICT[d["assetPackage"]["package"].split("_")[0]].values()
    #Excelの縦行のカウントとゲームでの縦行のカウントはスタート位置が逆
    for y in reversed(range(1, HEIGHT + 1)):
        #print(ws[y], len(ws[y]))
        for x in range(0, WIDTH):
            cell = ws[y][x]
            b = {"TileType": 0, "TileVariationId": 0, "Attributes": 0, "BattleStageType": 0, "SurfaceType": 0}
            #print("%s, %s" % (cell.coordinate, cell.fill.bgColor.rgb))
            b["TileType"] = tiletype_tbl[cell.fill.fgColor.rgb]
            d["blocks"].append(b)
            if cell.border.left or cell.border.right or cell.border.top or cell.border.bottom:
                add_fence(cell, d)
            if cell.value and cell.value[0] in defined:
                add_object(cell, d)

#infoシートからデータを読む
def read_info(ws, d):
    d["mapName"] = ws["A1"].value
    d["assetPackage"] = {"package": ws["A2"].value}
    d["width"] = WIDTH
    d["height"] = HEIGHT
    d["table"] = []

#マップのxlsxファイルかを超適当に判断する
def is_dngmapxlsx(xls_name):
    wb = openpyxl.load_workbook(xls_name)
    try:
        return True if wb["info"]["A1"].value != "m_Name" else False
    except KeyError:
        return False

#LibreOfficeで書いたXLSXファイルをdngmap.jsonに変換
def xls2json(xls_name, json_name, force=False):
    if not force and os.path.exists(json_name):
        print("output file %s is already exists." % json_name)
        exit(-1)
    #print("convert %s to %s" % (xls_name, json_name))
    #データテーブルのxlsxファイルを開いたらサイレントに閉じる
    if not is_dngmapxlsx(xls_name):
        return
    wb = openpyxl.load_workbook(xls_name)
    d = {}
    d["blocks"] = []
    d["mapObjectEntries"] = []
    read_info(wb["info"], d)
    #pprint.pp(d)
    read_blocks(wb["blocks"], d)
    read_objects(wb["objects"], d)
    #pprint.pp(d["mapObjectEntries"])
    with open(json_name, "wt") as f:
        json.dump(d, f)

#シートの幅を調整
def adjust_sheet_dimention(ws):
    for col in ws.columns:
        l = len(str(col[0].value))
        for c in col[1:]:
            l = len(str(c.value)) if len(str(c.value)) > l else l
        idx = col[0].column_letter
        sz = col[0].font.size / 10.0
        #1だけ余白を開けておかないとセルによっては幅の調整がいまいちな感じになる
        ws.column_dimensions[idx].width = l * sz + (1 if l else 0)

#objectsシートにマップオブジェクト（除くblocksシートに描く分）を書いていく
def create_objects_sheet(ws, d):
    i = 0
    for obj in sorted(d["mapObjectEntries"], key=lambda x: x["name"]):
        #Fence等は自動でmapObjectEntriesから削除されるようになった
        #if obj["name"].startswith("Fence") or :
            #continue
        ws[point2coodinate(0, i)].value = obj["name"]
        ws[point2coodinate(1, i)].value = pointdict2coodinate(obj["point"])
        ws[point2coodinate(2, i)].value = obj["facing"]
        if obj["components"]:
            j = 3
            for x in obj["components"]:
                if x["name"] == "Destination":
                    point = {k: int(v) for k, v in zip(["x", "y"], x["value"].split(","))}
                    ws[point2coodinate(j, i)].value = "%s:%s" % (x["name"], pointdict2coodinate(point))
                else:
                    ws[point2coodinate(j, i)].value = "%s:%s" % (x["name"], x["value"])
                j += 1
        i += 1

#blocksシートにフェンスを描く
def add_fence_block(ws, obj):
    point = pointdict2coodinate(obj["point"])
    facing = obj["facing"]
    #print(obj["point"], point, facing)
    color = openpyxl.styles.colors.Color(rgb="FF888888")
    side = openpyxl.styles.borders.Side(style="thick", color=color)
    border = ws[point].border
    #top
    if facing == 0:
        ws[point].border = openpyxl.styles.borders.Border(top=side, right=border.right, bottom=border.bottom, left=border.left, diagonal=border.diagonal, diagonalUp=border.diagonalUp, diagonalDown=border.diagonalDown)
    #right
    elif facing == 1:
        ws[point].border = openpyxl.styles.borders.Border(top=border.top, right=side, bottom=border.bottom, left=border.left, diagonal=border.diagonal, diagonalUp=border.diagonalUp, diagonalDown=border.diagonalDown)
    #bottom
    elif facing == 2:
        ws[point].border = openpyxl.styles.borders.Border(top=border.top, right=border.right, bottom=side, left=border.left, diagonal=border.diagonal, diagonalUp=border.diagonalUp, diagonalDown=border.diagonalDown)
    #left
    elif facing == 3:
        ws[point].border = openpyxl.styles.borders.Border(top=border.top, right=border.right, bottom=border.bottom, left=side, diagonal=border.diagonal, diagonalUp=border.diagonalUp, diagonalDown=border.diagonalDown)

#[Obsolete]: blocksシートに柱を描く：／ならPillar1、＼ならPillar2
#blocksシートに柱を描く：普通の字体ならPillar1、イタリックならPillar2
def add_pillar_block(ws, obj):
    name = obj["name"]
    point = pointdict2coodinate(obj["point"])
    if name.startswith("Pillar2") and asset_type in ("Castle", "Forest"):
        ws[point].font = openpyxl.styles.fonts.Font(i=True)
    ws[point].value = "柱0"
    return

    name = obj["name"]
    point = pointdict2coodinate(obj["point"])
    print(obj["point"], point, facing)
    color = openpyxl.styles.colors.Color(rgb="FF880088")
    side = openpyxl.styles.borders.Side(style="thick", color=color)
    border = ws[point].border
    if name.startswith("Pillar1"):
        ws[point].border =  openpyxl.styles.borders.Border(top=border.top, right=border.right, bottom=border.bottom, left=border.left, diagonal=side, diagonalUp=True, diagonalDown=border.diagonalDown)
    else:
        ws[point].border =  openpyxl.styles.borders.Border(top=border.top, right=border.right, bottom=border.bottom, left=border.left, diagonal=side, diagonalUp=border.diagonalUp, diagonalDown=True)

#blocksシートのセルにオブジェクトを書き込む
def add_object_in_cell(ws, symbol, obj):
    point = pointdict2coodinate(obj["point"])
    facing = obj["facing"]
    value = "%s%s" % (symbol, facing)
    if len(value) > 2:
        print("[add_object_in_cell]Error: %s@%s is not valid symbol." % (value, point))
        exit(-1)
    if ws[point].value:
        ws[point].value += value
    else:
        ws[point].value = value

#blocksシートに書き込む必要のあるオブジェクトを変換
def add_block_objects(wb, d):
    blocks = wb["blocks"]
    objects = wb["objects"]
    defined = DEFINED_OBJECTS_DICT[d["assetPackage"]["package"].split("_")[0]]
    rdefined = {v: k for k, v in defined.items()}
    del_idx = []
    for obj in d["mapObjectEntries"]:
        name = obj["name"].split("_")[0]
        if name == "Fence":
            add_fence_block(blocks, obj)
        #Pillar1/2:facingにかかわらず常にブロック右上に配置
        elif name.startswith("Pillar"):
            add_pillar_block(blocks, obj)
        elif name not in rdefined:
            print("[add_block_objects]Error: %s is not valid object name." % name)
            exit(-1)
        else:
            #obj["name"] = rdefined[name]
            add_object_in_cell(blocks, rdefined[name], obj)
        if name not in IGNORE_OBJECT_LIST:
            del_idx.append(d["mapObjectEntries"].index(obj))
    del_idx.sort(reverse=True)
    for i in del_idx:
        del d["mapObjectEntries"][i]

#タイルブロックをblocksシートに変換
def create_blocks_sheet(ws, d):
    #幅と高さを適当に調整@LibreOffice
    for k in cell_tbl.values():
        ws.row_dimensions[k].height = 15
    for k in cell_tbl.keys():
        ws.column_dimensions[k].width = 2.7
    for y in range(0, HEIGHT):
        #print(cell_rtbl[y-1], ":", end="")
        for x in range(0, WIDTH):
            rgb = tiletype_rtbl[d["blocks"][(HEIGHT - y - 1) * HEIGHT + x]["TileType"]]
            ws[point2coodinate(x, y)].fill = openpyxl.styles.fills.PatternFill(patternType=openpyxl.styles.fills.FILL_SOLID, fgColor=openpyxl.styles.colors.Color(rgb=rgb))
            #ws[point2coodinate(x, y)].fill.fgColor.rgb = ws[point2coodinate(x, y)].fill.bgColor.rgb = rgb

#infoシートを作成
def create_info_sheet(ws, d):
    ws["A1"] = d["mapName"]
    try:
        ws["A2"] = d["assetPackage"]["package"]
    except KeyError:
        if d["mapName"].startswith("Castle"):
            d["assetPackage"]["package"] = "Castle_01_Package"
        elif d["mapName"].startswith("Tunnel"):
            d["assetPackage"]["package"] = "Tunnel_01_Package"
        elif d["mapName"].startswith("Shrine"):
            d["assetPackage"]["package"] = "Shrine_01_Package"
        elif d["mapName"].startswith("Forest"):
            d["assetPackage"]["package"] = "Forest_01_Package"
        print("[create_info_sheet]cannnot find valid asset package. use alternative package %s." % d["assetPackage"]["package"])
        ws["A2"] = d["assetPackage"]["package"]

#json->xlsxの変換を行う
def json2xlsx(json_name, xls_name, force=False):
    if not force and os.path.exists(xls_name):
        print("output file %s is already exists." % xls_name)
        exit(-1)
    #print("convert %s to %s" % (json_name, xls_name))
    d = json.load(open(json_name, encoding="utf8"))
    wb = openpyxl.Workbook()
    #デフォルトで作成されるシートを除去
    wb.remove(wb.worksheets[0])
    wb.create_sheet("info")
    wb.create_sheet("blocks")
    wb.create_sheet("objects")
    create_info_sheet(wb["info"], d)
    create_blocks_sheet(wb["blocks"], d)
    add_block_objects(wb, d)
    create_objects_sheet(wb["objects"], d)
    adjust_sheet_dimention(wb["info"])
    adjust_sheet_dimention(wb["objects"])
    wb.save(xls_name)
    #print("done")

if __name__ == "__main__":
    if len(sys.argv) < 4 or sys.argv[1] in ("-h", "--help", "/?") or sys.argv[3][3:] not in ("encode", "decode"):
        print("DngmapConv.py <in> <out> -t=decode/encode [-f/--force]")
        exit()
    if sys.argv[1] == sys.argv[2]:
        print("input and output are same file.")
        exit()
    force = False
    if len(sys.argv) > 4:
        for a in sys.argv[4:]:
            if a in ("-f", "--force"):
                force = True
            else:
                print("invalid argument : %s" % a)
                exit()
    if sys.argv[3][3:] == "encode":
        json2xlsx(sys.argv[1], sys.argv[2], force=force)
    else:
        xls2json(sys.argv[1], sys.argv[2], force=force)


#!/usr/bin/python
# -*- coding: UTF-8 -*-
from __future__ import absolute_import
from __future__ import with_statement
from __future__ import division
from __future__ import nested_scopes
from __future__ import generators
from __future__ import unicode_literals
from __future__ import print_function

try:
    from pycuda.elementwise import ElementwiseKernel
except:
    import sys
    def missing(*args,**kwargs):
        if 'sphinx' in sys.modules:
                print('Please locate and install PyCuda')
        else:
            raise ValueError('Please locate and install PyCuda')
    ElementwiseKernel = missing

##############################################################################
# A plotting helper
##############################################################################

gpubarlinekerna = ElementwiseKernel(
        "float *x, float low, float high, float *z",
        "z[i] = x[i]>=low&&x[i]<high?1.0:0.0",
        "gpubarlinekerna")

gpubarlinekernb = ElementwiseKernel(
        "float *p, float *x, float *z",
        "z[i]=p[i]>0?x[i]:0.0",
        "gpubarlinekernb")

gpubarlinekernc = ElementwiseKernel(
        "float *p, float *x, float mean, float *z",
        "z[i]=p[i]>0?pow(x[i]-mean,2):0.0",
        "gpubarlinekernc")

def gpubarlinedata(xdata,ydata,bins,minval=None,maxval=None):
    if maxval==None: maxval=gpumax(xdata)
    if minval==None: minval=gpumin(xdata)
    binsize= (maxval-minval)/float(bins)
    inbin  = gpuarray.empty_like(xdata)
    select = gpuarray.empty_like(xdata)
    xmeans = []
    ymeans = []
    errors = []
    for i in xrange(bins):
        lo=minval+binsize*i;
        hi=minval+binsize*(i+1);
        gpubarlinekerna(xdata,lo,hi,inbin)
        N=gpusum(inbin)
        if N>1:
            gpubarlinekernb(inbin,ydata,select)
            my=gpusum(select)/float(N)
            gpubarlinekernb(inbin,xdata,select)
            mx=gpusum(select)/float(N)
            gpubarlinekernc(inbin,ydata,my,select)
            s=sqrt(gpusum(select)/(N*(N-1)))
            xmeans.append(mx)
            ymeans.append(my)
            errors.append(s)
    return (xmeans,ymeans,errors)

def sebarline(datasets,bins,min=None,max=None,lx="",ly="",title=""):
    fig = plt.figure()
    ax = fig.add_subplot(111)
    for (x,y) in datasets:
        xm,ym,err=gpubarlinedata(x,y,bins,min,max)
        plt.errorbar(xm,ym,yerr=map(lambda x:2*x,err))
    ax.set_xlabel(textf(lx))
    ax.set_ylabel(textf(ly))
    ax.set_title(textf(title))
    fig.show()

def sebarline2(datasets,lx="",ly="",title=""):
    fig = plt.figure()
    ax = fig.add_subplot(111)
    for (x,y) in datasets:
        ym=cmap(gpumean)(y)
        ys=cmap(gpusem)(y)*2
        plt.errorbar(x,ym,yerr=ys)
    ax.set_xlabel(textf(lx))
    ax.set_ylabel(textf(ly))
    ax.set_title(textf(title))
    fig.show()

def gpuhistogram(xdata,ydata,bins,minval=None,maxval=None):
    if maxval==None: maxval=gpumax(xdata)
    if minval==None: minval=gpumin(xdata)
    binsize= (maxval-minval)/float(bins)
    inbin  = gpuarray.empty_like(xdata)
    N = []
    for i in xrange(bins):
        gpubarlinekerna(xdata,minval+binsize*i,minval+binsize*(i+1),inbin)
        N.append(gpusum(inbin))
    return N


from __future__ import print_function, division

import logging
import argparse
import sys

class Robot(object):
    def __init__(self, client, market_id):
        self.c = client
        self.market_id = market_id
        self.ie = InstructionEngine(client, market_id)

class InstructionEngine(object):
    def __init__(self, client, market_id):
        self.c = client
        self.market_id = market_id

    def update_bets(self, sel_id, backs, lays):
        _to_betfair_odds = lambda x: (self.c.set_betfair_odds(x[0]), round(x[1], 2))
        backs = set(map(_to_betfair_odds, backs))
        lays = set(map(_to_betfair_odds, lays))

        curr_bets = self.c.get_mu_bets(self.market_id, status='U')
        if isinstance(curr_bets, str):
            curr_bets = []
        else:
            curr_bets = filter(lambda x: x['selectionId'] == sel_id, curr_bets)

        cancel_ids = []
        for bet in curr_bets:
            try:
                if bet['betType'] == 'B':
                    backs.remove((bet['price'], bet['size']))
                else:
                    lays.remove((bet['price'], bet['size']))
            except KeyError:
                logging.info('[sel_id=%8s] Cancelling %s bet [bet_id=%s, GBP %.2f @ %.2f (p=%.3f)]' %
                             (sel_id, bet['betType'], bet['betId'], bet['size'], bet['price'], 1 / bet['price']))
                cancel_ids.append(bet['betId'])
        self.c.cancel_bets(cancel_ids)

        bets = []
        for back in backs:
            logging.info('[sel_id=%8s] Placing new BACK bet: GBP %.2f @ %.2f (p=%.3f)' % (sel_id, back[1], back[0], 1.0 / back[0]))
            bets.append(self._bet(sel_id, 'B', back[0], back[1]))
        for lay in lays:
            logging.info('[sel_id=%8s] Placing new  LAY bet: GBP %.2f @ %.2f (p=%.3f)' % (sel_id, lay[1], lay[0], 1.0 / lay[0]))
            bets.append(self._bet(sel_id, 'L', lay[0], lay[1]))
        if len(bets) > 0:
            self.c.place_bets(bets)

    def _bet(self, sel_id, bet_type, price, size):
            return {"marketId": self.market_id,
                    "selectionId": sel_id,
                    "betType": bet_type, # 'B' or 'L'
                    "price": str(price),
                    "size": str(size),
                    "betCategoryType": "E", # "E", "M" or "L"
                    "betPersistenceType": "NONE", # "NONE", "SP" or "IP"
                    "bspLiability": "0", # should be "0" if unused
                    "asianLineId": "0" # should be "0" if unused
                    }

    def pnl(self):
        pass


from datetime import datetime
from pathlib import Path
from sys import exit
from time import time

from influxdb_client import InfluxDBClient, Point, WritePrecision
from influxdb_client.client.write_api import SYNCHRONOUS
import yaml
import plugins

from synge import get_logger

class InfluxDB(plugins.PlugIns):
    def __init__(self, conf):
        self.name = 'influxdb'
        self.type = 'load'
        self.cfg = conf
        self.setup = None
        self.dbclient = None

        self.log = get_logger(self.__class__.__name__)

    def _load_setup(self):
        setupfile=Path(Path(__file__).parent, 'setup.yml')
        if setupfile.is_file():
            with open(setupfile, 'r') as ymlfile:
                setup = yaml.load(ymlfile, Loader=yaml.FullLoader)
        else:
            self.log.error(f'No setup file {setupfile}')
            exit(1)
        return setup[self.name]

    def _setup(self):
        conf_needed = [r for r in self.setup['conf_needed']]
        conf_rows = [r for r in self.cfg.yaml[self.name]]

        if all(item in conf_rows for item in conf_needed):
            self.dbclient = InfluxDBClient(url=f'http://{self.cfg.yaml[self.name]["host"]}:{self.cfg.yaml[self.name]["port"]}', token=self.cfg.yaml[self.name]["access_token"], org=self.cfg.yaml[self.name]["org"])
        else:
            self.log.error(f'Please fill all needed fields in configuration file.')
            exit(1)

    def run(self, datapoints):
        self.setup = self._load_setup()
        self._setup()

        self.log.debug('Starting InfluxDB sync')
        start = time()
        with self.dbclient as client:
            write_api = client.write_api(write_options=SYNCHRONOUS)
            data = "mem,host=host1 used_percent=23.43234543"
            write_api.write(self.cfg.yaml[self.name]['bucket'], self.cfg.yaml[self.name]['org'], datapoints)
        # for q in self.flow:
        #     self._fetch_data(q['category'], q['type'], day)
        end = time()
        self.log.debug(f'InfluxDB syncing was successful, took {end - start} seconds')


import json
import requests
from requests_oauthlib import OAuth1Session
from .tw_message import tw_message
from .dataset.tweet import Tweet
from lib.settings import settings

class Twitter:
    """twitterAPI通信処理
    """
    def __init__(self):
        """初期処理
        Args:
            self(lib.twitter):
        Return:
            None
        """
        token = settings.get_setting()
        CK = token['CONSUMER_KEY']
        CS = token['CONSUMER_SECRET']
        AT = token['ACCESS_TOKEN']
        ATS = token['ACCESS_TOKEN_SECRET']
        self.twitter = OAuth1Session(CK, CS, AT, ATS)

    def read_list_tl(self):
        """リストのタイムライン読み込み

        Args:
            self(lib.Twitter):
        Returns:

        """
        message = tw_message()
        url = "https://api.twitter.com/1.1/lists/statuses.json" #タイムライン取得エンドポイント

        # list_id決め打ちで取得しに行く
        params = {'list_id' : 1118826235000279040, 'count': 200} #取得数

        tweet = Tweet()
        tid = tweet.get_maxtimestamp_tweet()
        if tid is not None:
            params['since_id'] = tid
        print(params)
        res = self.twitter.get(url, params = params)

        if res.status_code == 200: #正常通信出来た場合
            timelines = json.loads(res.text)

            for line in timelines:
                message.add_message(line['user']['name'], line['user']['screen_name'],\
                    line['id_str'], line['created_at'])
            return message
        else: #正常通信出来なかった場合
            raise requests.exceptions.ConnectionError

    # リストID取得 list_idを単に取るだけ、実際のbotでは使わない。
    def get_list_id(self, list_name):
        message = tw_message()
        url = "https://api.twitter.com/1.1/lists/list.json"

        # params = {'user_id': user_name}
        params = {}
        res = self.twitter.get(url, params = params)
        if res.status_code == 200: #正常通信出来た場合
            data = json.loads(res.text)
            for line in data:
                if line['full_name'] == "@" + list_name + "/list":
                    print(line['id_str'])
        else: #正常通信出来なかった場合
            raise requests.exceptions.ConnectionError


import RPi.GPIO as GPIO
import time, os, sys, glob
from datetime import datetime
import picamera
import subprocess

#define necessary parameters
pinNum = 18	#set GPIO_18 as the input of 'HC-SR501'

#define camera function
def createH264(name):
	camera = picamera.PiCamera()
	camera.resolution = (1024, 768)
	camera.framerate = 30
	camera.awb_mode = 'auto'

	camera.start_recording(name + '.h264')
	camera.wait_recording(5)
	camera.stop_recording()
#end of createVideo()

#define function for deleting .h264 file
def deleteH264(name):
	if os.path.isfile(name + '.h264'):
		os.remove(name + '.h264')
def deleteH264_all():
	files = glob.iglob(os.path.join('./', '*.h264'))
	for file in files:
		if os.path.isfile(file):
			os.remove(file)

#start main processing...
GPIO.setmode(GPIO.BCM)
GPIO.setup(pinNum, GPIO.IN)
print("Setting up all sensors...")
time.sleep(5)

#start main processing...
try:
	while True:
		#if detected...
		if GPIO.input(pinNum):
			#first, create .h264 video
			now = datetime.now()
			fileName = '%s-%s-%s_%s:%s:%s' % (now.year, now.month, now.day, now.hour, now.minute, now.second)
			print("Human is detected! Recording 5-sec video...")
			createH264(fileName)

			#Then, convert *.264 -> *.mp4
			if subprocess.call(['MP4Box','-add', fileName + '.h264', fileName + '.mp4']) == 0:
				print("Created video named \'%s.mp4\'" % fileName)
				deleteH264(fileName)
			else:
				print("Error: failed converting...")
				deleteH264(fileName)
		#if NOT detected...
		else:
			print("Nothing...")

		time.sleep(1)
	#end of while

#list of possible exceptions
except KeyboardInterrupt:
	print("Pressed Keyboard.")
except TimeoutError:
	print("TIMEOUT!! Check out the error and try again...")
except:
	print("Error: unknown error...")

GPIO.cleanup()
deleteH264_all()
print("Good bye~")
sys.exit()


#!/usr/bin/python
# -*- coding: utf-8 -*-

import logging
import numpy
import pyaudio
import aubio
import time

class AudioInput(object):
    sample_rate = 44100
    buffersize = 512
    max_energy = 5000

    _read_listeners = []

    def __init__(self, sample_rate=None, buffersize=None):
        self.sample_rate = sample_rate or self.sample_rate
        self.buffersize = buffersize or self.buffersize
        self.window_size = self.buffersize * 2
        self.stream = None

        self.onset = aubio.onset(
            'specflux', self.window_size, self.buffersize, self.sample_rate)
        self.onset.set_threshold(0.3)
        self.onset.set_silence(-20.)
        self.tempo = aubio.tempo(
            'default', self.window_size, self.buffersize, self.sample_rate)

        self.energy = aubio.specdesc('specflux', self.buffersize * 2)
        self.pv = aubio.pvoc(self.buffersize * 2, self.buffersize)

        self.pitch = aubio.pitch(
            "yinfft", self.window_size, self.buffersize, self.sample_rate)
        self.pitch.set_unit("midi")
        self.pitch.set_tolerance(0.8)

        self.py_audio = pyaudio.PyAudio()

    def init_stream(self, input_device_index=None, channels=None):
        if not input_device_index or not channels:
            input_device_index, channels = self._get_input_device()

        logging.info("Using input device %s with %s channels" % (
            input_device_index, channels
        ))

        self.stream = self.py_audio.open(
            format=pyaudio.paFloat32,
            channels=2,
            rate=self.sample_rate,
            input=True,
            frames_per_buffer=self.buffersize,
            input_device_index=input_device_index)

    def _get_input_device(self):
        valid_devices = []
        for i in range(self.py_audio.get_device_count()):
            data = self.py_audio.get_device_info_by_index(i)
            if data.get('maxInputChannels', 0) > 0:
                valid_devices.append(
                    (data['maxInputChannels'], data.get('index'),
                        data.get('name'), data.get('defaultSampleRate'))
                )
                print(data)
        if len(valid_devices) == 0:
            logging.error('AudioInput: Couldn\'t find a valid input device')
            raise Exception('AudioInput: invalid input device')
        if len(valid_devices) > 1:
            logging.info(
                'AudioInput: Found the following devices, using the last one:'
                '\n%s' % '\n'.join([
                    '%s. %s (i:%s ch:%s SR:%s)' % (
                        i, info[2], info[1], info[0], info[3]
                    )
                    for i, info in enumerate(valid_devices)
                ])
            )
        return valid_devices[-1][1], valid_devices[-1][0]

    def register_read_listener(self, callback):
        self._read_listeners.append(callback)
        return len(self._read_listeners)

    def unregister_read_listener(self, callback):
        self._read_listeners.remove(callback)

    def read_loop(self):
        _time = time.time()
        while True:
            f = numpy.fromstring(
                self.stream.read(self.buffersize), dtype=numpy.dtype('<f'))

            data = {
                'is_beat': self.tempo(f),
                'onset': self.onset(f),
                'left_energy': 30,
                'energy': self.energy(self.pv(f[:self.buffersize]))[0],
                'pitch_midi': int(self.pitch(f)[0])
            }
            data['energy_norm'] = min(1, data['energy'] / 1500)
            data['pitch_norm'] = min(1, data['pitch_midi'] / 137)

            time_diff = time.time() - _time
            _time = time.time()
            for callback in self._read_listeners:
                callback(data, time_diff=time_diff)

    def drop_stream(self):
        self.stream.stop_stream()
        self.stream.close()
        self.py_audio.terminate()

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    t = AudioInput()
    t.init_stream()
    t.read_loop()


from check_on_results.check_general import check_interface
from check_decorators.print_name import printNameDecorator
from utils.assertions import check, raise_assertion
from utils.find_output import find_dated_output_folder

@printNameDecorator
class check_logs_content(check_interface):
    def __init__(self, study_path, log_level="warns", faulty_weeks=[]):
        super().__init__(study_path)
        self.log_level = log_level
        self.expected_faulty_weeks = faulty_weeks

        check(log_level in ["warns", "fatal"], "Unknown log level : %s" % log_level)

    def run(self):
        path_to_output_folder = find_dated_output_folder(self.study_path)
        path_to_log_file = find_logs_file(path_to_output_folder)

        # Build lines to be found in logs file
        lines_to_find = []
        match_lines = []
        for weeks_of_year in self.expected_faulty_weeks:
            year = weeks_of_year.year
            week_label = "week" if len(weeks_of_year.weeks) == 1 else "weeks"
            concatenated_weeks = ""
            for week_number in weeks_of_year.weeks:
                concatenated_weeks += " " + str(week_number)
            lines_to_find.append("[solver][" + self.log_level + "] Year " + str(year) + " failed at "
                                 + week_label + concatenated_weeks + ".")

            match_lines.append(False)

        # Search for expected lines in log file content and store matches
        log_file = open(str(path_to_log_file), 'r')
        for line in log_file:
            for i in range(len(lines_to_find)):
                if lines_to_find[i] in line:
                    match_lines[i] = True

        # Report of lines not found
        for i in range(len(lines_to_find)):
            assert match_lines[i], "'" + lines_to_find[i] + "'" + " log not found in simulation.log"

    def name(self):
        return "unfeasible problem logs"

def find_logs_file(path_to_output_folder):
    found_files = []
    for path in path_to_output_folder.rglob('simulation.log'):
        found_files.append(path)
    if len(found_files) == 0:
        raise_assertion("Found no log file for simulation")
    elif len(found_files) > 1:
        raise_assertion("Found too many log files for simulation")
    return found_files[0]


import matplotlib.pyplot as plt
import numpy as np

import random
def randomcolor():
    colorArr = ['1','2','3','4','5','6','7','8','9','A','B','C','D','E','F']
    color = ""
    for i in range(6):
        color += colorArr[random.randint(0,14)]
    return "#"+color

filePath_redteam = "E:/zyt/junior1/DDS/FINAL-PROJECT/dataset/redteam.txt/redteam.txt"
with open(filePath_redteam, "r") as f:
    most_active_attacker_dict = {}
    most_poor_victim_for_each_attacker = {}
    most_poor_victim_for_all_attackers = {}

    for line in f:
        ts, host, src, dst = line.strip('\n').split(',')

        if src not in most_active_attacker_dict:
            most_active_attacker_dict[src] = 0
            most_poor_victim_for_each_attacker[src] = {}
        most_active_attacker_dict[src] += 1
        if dst not in most_poor_victim_for_each_attacker[src]:
            most_poor_victim_for_each_attacker[src][dst] = 0
        most_poor_victim_for_each_attacker[src][dst] += 1

        if dst not in most_poor_victim_for_all_attackers:
            most_poor_victim_for_all_attackers[dst] = 0
        most_poor_victim_for_all_attackers[dst] += 1

    for a in sorted(most_active_attacker_dict.items(), key=lambda s: s[1], reverse=True):
        print("attacker => count: %4d | src: %s | number of victims: %4d"
              % (a[1], a[0], len(most_poor_victim_for_each_attacker[a[0]])))
        # print("number of victims: " + str(len(most_poor_victim_for_each_attacker[a[0]])))
        for v in sorted(most_poor_victim_for_each_attacker[a[0]].items(), key=lambda s: s[1], reverse=True):
            if v[1] < most_poor_victim_for_all_attackers[v[0]]:
                print("  victim => count: %4d | dst: %6s   << total: %4d" % (
                    v[1], v[0], most_poor_victim_for_all_attackers[v[0]]))
            else:
                print("  victim => count: %4d | dst: %6s" % (v[1], v[0]))
        print("\n")

    # plot

    # 构建数据
    x_data = []
    y_data = []
    for a in sorted(most_active_attacker_dict.items(), key=lambda s: s[1], reverse=True):
        x_data.append(a[0])

    for all in sorted(most_poor_victim_for_all_attackers.items(), key=lambda s: s[1], reverse=True):
        # print(all[0])
        temp_list = []
        for a in sorted(most_active_attacker_dict.items(), key=lambda s: s[1], reverse=True):
            if all[0] in most_poor_victim_for_each_attacker[a[0]]:
                num = most_poor_victim_for_each_attacker[a[0]][all[0]]
                print(num)
                temp_list.append(num)

            else:
                temp_list.append(0)

        y_data.append(temp_list)

    # 绘图
    l = len(y_data)
    for i in range(l-1):
        y_data[i + 1][0] = y_data[i][0]+y_data[i + 1][0]
        y_data[i + 1][1] = y_data[i][1]+y_data[i + 1][1]
        y_data[i + 1][2] = y_data[i][2]+y_data[i + 1][2]
        y_data[i + 1][3] = y_data[i][3]+y_data[i + 1][3]
    count = 0
    y_data.reverse()
    for all in sorted(most_poor_victim_for_all_attackers.items(), key=lambda s: s[1], reverse=False):
        ttt = y_data[0]
        plt.bar(x=x_data, height=y_data[count], label=all[0], color=randomcolor(), alpha=0.8)
        # plt.bar(x=x_data, height=y_data[count], color='steelblue', alpha=0.8)
        count = count + 1
    # 设置标题
    plt.title("Distribution of attack frequency")
    # 为两条坐标轴设置名称
    plt.xlabel("Attacker")
    plt.ylabel("Number of attack items")
    # 显示图例
    # plt.legend()
    plt.show()

# with open("./data/dns.txt", "r") as f:
#     for line in f:
#         time, src, resolved = line.split(',')
#         print(time, src, resolved)