from django.core.files.storage import FileSystemStorage
from django.core.management.base import CommandError, NoArgsCommand
from staticutils.versioning import get_file_version, get_versioned_path
from django.contrib.staticfiles import finders
import os
from django.utils.encoding import smart_str
from optparse import make_option
from glob import iglob
from django.conf import settings
from django.core.files.storage import get_storage_class

class Command(NoArgsCommand):
    help = "Purges all files from `VERSIONED_STATIC_ROOT` that are not "\
           "current versions of static files. (Files where the version "\
           "*matches* the source file's current version are not removed, "\
           "unless the `--all` flag is given.)"

    option_list = NoArgsCommand.option_list + (
        make_option('-n', '--dry-run', action='store_true', dest='dry_run',
            default=False, help="Do everything except modify the filesystem."),
        make_option('-i', '--ignore', action='append', default=[],
            dest='ignore_patterns', metavar='PATTERN',
            help="Ignore files or directories matching this glob-style "
                "pattern. Use multiple times to ignore more."),
        make_option('--all',
            action='store_true',
            dest='all',
            default=False,
            help='Removes *everything* from `VERSIONED_STATIC_ROOT`, not just '\
                 'old versions of static files.'),
        make_option('--noinput',
            action='store_true',
            dest='interactive',
            default=False,
            help='Does not prompt for confirmation when deleting.'),
        make_option('--no-default-ignore', action='store_false',
            dest='use_default_ignore_patterns', default=True,
            help="Don't ignore the common private glob-style patterns 'CVS', "
                "'.*' and '*~'."),
    )

    def __init__(self, *args, **kwargs):
        super(Command, self).__init__(*args, **kwargs)
        self.deleted_files = []
        self.unmodified_files = []

        # Force storage to be a filesystem storage for VERSIONED_STATIC_ROOT.
        self.storage = get_storage_class(settings.STATICFILES_STORAGE)()
        self.versioned_storage = FileSystemStorage(location=settings.VERSIONED_STATIC_ROOT)

    def handle_noargs(self, **options):
        self.verbosity = int(options.get('verbosity', 1))

        ignore_patterns = options['ignore_patterns']
        if options['use_default_ignore_patterns']:
            ignore_patterns += ['CVS', '.*', '*~']
        ignore_patterns = list(set(ignore_patterns))

        # Warn before doing anything more.
        if options.get('interactive'):
            confirm = raw_input(u"""
You have requested to delete versioned files at the destination
location as specified in your settings file.

This will remove existing files.
Are you sure you want to do this?

Type 'yes' to continue, or 'no' to cancel: """)
            if confirm != 'yes':
                raise CommandError("Collecting static files cancelled.")

        # Get output filenames for up-to-date static files.
        self.valid_outpaths = set()
        if not options['all']:
            for finder in finders.get_finders():
                for path, storage in finder.list(ignore_patterns):
                    # Prefix the relative path if source storage contains it
                    if getattr(storage, 'prefix', None):
                        prefixed_path = os.path.join(storage.prefix, path)
                    else:
                        prefixed_path = path

                    current_version = get_file_version(path, storage)
                    current_outpath = get_versioned_path(
                        prefixed_path, current_version)

                    self.valid_outpaths.add(current_outpath)

        # Find everything currently in VERSIONED_STATIC_ROOT and remove
        # anything not in `self.valid_outpaths`
        def cleardirs(cd, subdirs, files):
            for d in subdirs:
                new_cd = os.path.join(cd, d)
                new_subdirs, new_files = self.versioned_storage.listdir(
                    os.path.join(cd, d)
                )
                cleardirs(new_cd, new_subdirs, new_files)
            for f in files:
                filepath = os.path.join(cd, f)
                if filepath not in self.valid_outpaths:
                    if options['dry_run']:
                        self.log(u"Pretending to delete '%s'" % filepath)
                    else:
                        self.log(u"Deleting '%s'" % filepath)
                        self.versioned_storage.delete(filepath)
                    self.deleted_files.append(filepath)
                else:
                    self.log(u"Skipping up-to-date file '%s'" % filepath)
                    self.unmodified_files.append(filepath)

        subdirs, files = self.versioned_storage.listdir("")
        cleardirs("", subdirs, files)

        actual_count = len(self.deleted_files)
        unmodified_count = len(self.unmodified_files)
        if self.verbosity >= 1:
            self.stdout.write(smart_str(u"\n%s versioned file%s %s%s.\n"
                              % (actual_count, actual_count != 1 and 's' or '',
                                 'deleted',
                                 unmodified_count and ' (%s unmodified)'
                                 % unmodified_count or '')))

    def log(self, msg, level=2):
        """
        Small log helper
        """
        msg = smart_str(msg)
        if not msg.endswith("\n"):
            msg += "\n"
        if self.verbosity >= level:
            self.stdout.write(msg)

    def remove_old_versions(self, path, prefixed_path, source_storage, **kwargs):
        # For iglob matching to work, we actually need the full path
        # of the output file.
        outpath = os.path.join(settings.VERSIONED_STATIC_ROOT, prefixed_path)

        current_version = get_file_version(path, source_storage)
        current_outpath = get_versioned_path(outpath, current_version)

        # Match any possible version of the output file.
        for matching_path in iglob(get_versioned_path(outpath, "*")):
            # Remove VERSIONED_STATIC_ROOT just for output log, etc.
            rel_matching_path = matching_path.replace(
                settings.VERSIONED_STATIC_ROOT, "", 1
            )[1:]
            if kwargs['all'] or (matching_path != current_outpath):
                self.deleted_files.append(matching_path)
                if kwargs['dry_run']:
                    self.log(u"Pretending to delete '%s' (Current is '%s')" % (
                        rel_matching_path, current_version
                    ))
                else:
                    self.log(u"Deleting '%s' (Current is '%s')" % (
                        rel_matching_path, current_version
                    ))
                    os.unlink(matching_path)
                    self.versioned_storage.delete(matching_path)
            else:
                self.unmodified_files.append(matching_path)
                self.log(u"Skipping '%s'. (Current version.)" % rel_matching_path)


# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# Expects to be run in an environment with sudo and no interactive password
# prompt, such as within the Chromium OS development chroot.
from __future__ import print_function

import logging
import os
import time

from hw_driver import HwDriverError
import serial

class _HandlerTemplate(object):
  """Template for all handlers to support common open/close operations."""

  def __init__(self):
    # The base subclasses need to handle opening themselves up.
    self._logger = logging.getLogger(type(self).__name__)
    self._open = False

  def is_open(self):
    """Query whether keyboard handler is open for use."""
    return self._open

  def open(self):
    """Open the keyboard handler for use."""
    self._open = True

  def close(self):
    """Close the keyboard handler for use."""
    self._open = False

class NoopHandler(_HandlerTemplate):
  """Noop keyboard handler that always keeps the keyboard closed.

  For some use-cases a proper keyboard handler cannot be initialized e.g.
  missing hardware. If those cases are not necessarily blocking, use a
  NoopHandler. It will warn every time open/close are being used that
  they are noops, but will not issue an exception.

  As open open() and close() just print warnings, |self._open| stays False
  and is_open() always returns False
  """

  _BASE_WRN = ('Using a noop keyboard handler. Check logs to see why it is '
               'in use and address issue if full keyboard functionality is '
               'needed.')

  def open(self):
    """Print warning only."""
    self._logger.warn(self._BASE_WRN)

  def close(self):
    """Print warning only."""
    self._logger.warn(self._BASE_WRN)

class _BaseHandler(_HandlerTemplate):
  """Base class for keyboard handlers.
    """
  # Power button press delays in seconds.
  #
  # The EC specification says that 8.0 seconds should be enough
  # for the long power press.  However, some platforms need a bit
  # more time.  Empirical testing has found these requirements:
  #   Alex: 8.2 seconds
  #   ZGB:  8.5 seconds
  # The actual value is set to the largest known necessary value.
  #
  # TODO(jrbarnette) Being generous is the right thing to do for
  # existing platforms, but if this code is to be used for
  # qualification of new hardware, we should be less generous.
  LONG_DELAY = 8.5
  SHORT_DELAY = 0.2
  NORMAL_TRANSITION_DELAY = 1.2

  # Maximum number of times to re-read power button on release.
  RELEASE_RETRY_MAX = 5

  # Default minimum time interval between 'press' and 'release'
  # keyboard events.
  SERVO_KEY_PRESS_DELAY = 0.1

  KEY_MATRIX = None

  def __init__(self, servo):
    """Sets up the servo communication infrastructure.

        @param servo: A Servo object representing
                           the host running servod.
        """
    super(_BaseHandler, self).__init__()
    # TODO(fdeng): crbug.com/298379
    # We should move servo object out of servo object
    # to minimize the dependencies on the rest of Autotest.
    self._servo = servo
    board = self._servo.get_board()

  def power_long_press(self):
    """Simulate a long power button press."""
    # After a long power press, the EC may ignore the next power
    # button press (at least on Alex).  To guarantee that this
    # won't happen, we need to allow the EC one second to
    # collect itself.
    # TODO(waihong): Make this delay as one of board specific configs.
    self.power_key(self.LONG_DELAY)
    time.sleep(1.0)

  def power_normal_press(self):
    """Simulate a normal power button press."""
    self.power_key()

  def power_short_press(self):
    """Simulate a short power button press."""
    self.power_key(self.SHORT_DELAY)

  def power_key(self, press_secs=''):
    """Simulate a power button press.

        Args:
          press_secs: Time in seconds to simulate the keypress.
        """
    if press_secs is '':
      press_secs = self.NORMAL_TRANSITION_DELAY

    # Check if pwr_button control available, by setting it to
    # its current value. Use pwr_button control by default.
    # Otherwise, use pwr_button_hold which calls a single EC
    # console command to toggle power button, for the CCD case.
    servo_type = self._servo.get('servo_type')
    is_ccd = 'ccd' in servo_type and 'servo_micro' not in servo_type

    if is_ccd:
      use_hold_command = True
    else:
      try:
        value = self._servo.get('pwr_button')
        self._servo.set('pwr_button', value)
        use_hold_command = False
      except HwDriverError:
        use_hold_command = True

    if use_hold_command:
      self.power_key_hold(press_secs)
    else:
      self.power_key_press_release(press_secs)

  def power_key_hold(self, press_secs):
    """Simulate a power button by a single EC console command.

        Args:
          press_secs: Time in seconds to simulate the keypress.
        """
    # Convert to milliseconds
    self._servo.set('pwr_button_hold', int(press_secs * 1000))

  def power_key_press_release(self, press_secs):
    """Simulate a power button by setting it to press and then release.

        Args:
          press_secs: Time in seconds to simulate the keypress.
        """
    self._logger.info('Pressing power button for %.4f secs', press_secs)
    self._servo.set_get_all(
        ['pwr_button:press',
         'sleep:%.4f' % press_secs, 'pwr_button:release'])
    # TODO(tbroch) Different systems have different release times on the
    # power button that this loop addresses.  Longer term we may want to
    # make this delay platform specific.
    retry = 1
    while True:
      value = self._servo.get('pwr_button')
      if value == 'release' or retry > self.RELEASE_RETRY_MAX:
        break
      self._logger.info('Waiting for pwr_button to release, retry %d.', retry)
      retry += 1
      time.sleep(self.SHORT_DELAY)

  def ctrl_d(self, press_secs=''):
    """Simulate Ctrl-d simultaneous button presses."""
    NotImplementedError()

  def ctrl_u(self, press_secs=''):
    """Simulate Ctrl-u simultaneous button presses."""
    NotImplementedError()

  def ctrl_s(self, press_secs=''):
    """Simulate Ctrl-s simultaneous button presses."""
    NotImplementedError()

  def ctrl_enter(self, press_secs=''):
    """Simulate Ctrl-enter simultaneous button presses."""
    NotImplementedError()

  def ctrl_key(self, press_secs=''):
    """Simulate Enter key button press."""
    NotImplementedError()

  def arrow_up(self, press_secs=''):
    """Simulate ArrowUp key button press."""
    NotImplementedError()

  def arrow_down(self, press_secs=''):
    """Simulate ArrowDown key button press."""
    NotImplementedError()

  def enter_key(self, press_secs=''):
    """Simulate Enter key button press."""
    NotImplementedError()

  def refresh_key(self, press_secs=''):
    """Simulate Refresh key (F3) button press."""
    NotImplementedError()

  def ctrl_refresh_key(self, press_secs=''):
    """Simulate Ctrl and Refresh (F3) simultaneous press.

        This key combination is an alternative of Space key.
        """
    NotImplementedError()

  def imaginary_key(self, press_secs=''):
    """Simulate imaginary key button press.

        Maps to a key that doesn't physically exist.
        """
    NotImplementedError()

  def sysrq_x(self, press_secs=''):
    """Simulate Alt VolumeUp X simultaneous press.

        This key combination is the kernel system request (sysrq) X.
        """
    NotImplementedError()

  def arb_key(self, press_secs=''):
    """Simulate an arbitrary key press.
        """
    NotImplementedError()

  def arb_key_config(self, key):
    """Set key for an arbitrary key press.
        """
    self._arb_key = key

class MatrixKeyboardHandler(_BaseHandler):
  """Matrix keyboard handler for DUT with internal keyboards.

    It works on mostly all devices, with or without Chrome EC.
    """
  KEY_MATRIX = {
      'ctrl_refresh': ['0', '0', '0', '1'],
      'ctrl_d': ['0', '1', '0', '0'],
      'd': ['0', '1', '1', '1'],
      'ctrl_enter': ['1', '0', '0', '0'],
      'enter': ['1', '0', '1', '1'],
      'ctrl': ['1', '1', '0', '0'],
      'refresh': ['1', '1', '0', '1'],
      'unused': ['1', '1', '1', '0'],
      'none': ['1', '1', '1', '1']
  }

  def __init__(self, servo):
    """Sets up the servo communication infrastructure.

        @param servo: A Servo object representing
                           the host running servod.
        """
    super(MatrixKeyboardHandler, self).__init__(servo)
    self.open()

  def _press_keys(self, key):
    """Simulate button presses.

        Note, key presses will remain on indefinitely. See
            _press_and_release_keys for release procedure.
        """
    (m1_a1, m1_a0, m2_a1, m2_a0) = self.KEY_MATRIX[key]
    self._servo.set_get_all([
        'kbd_m2_a0:%s' % m2_a0,
        'kbd_m2_a1:%s' % m2_a1,
        'kbd_m1_a0:%s' % m1_a0,
        'kbd_m1_a1:%s' % m1_a1, 'kbd_en:on'
    ])

  def _press_and_release_keys(self, key, press_secs=''):
    """Simulate button presses and release."""
    if press_secs is '':
      press_secs = self.SERVO_KEY_PRESS_DELAY
    self._press_keys(key)
    time.sleep(press_secs)
    self._servo.set('kbd_en', 'off')

  def ctrl_d(self, press_secs=''):
    """Simulate Ctrl-d simultaneous button presses."""
    self._press_and_release_keys('ctrl_d', press_secs)

  def ctrl_enter(self, press_secs=''):
    """Simulate Ctrl-enter simultaneous button presses."""
    self._press_and_release_keys('ctrl_enter', press_secs)

  def ctrl_key(self, press_secs=''):
    """Simulate Enter key button press."""
    self._press_and_release_keys('ctrl', press_secs)

  def enter_key(self, press_secs=''):
    """Simulate Enter key button press."""
    self._press_and_release_keys('enter', press_secs)

  def refresh_key(self, press_secs=''):
    """Simulate Refresh key (F3) button press."""
    self._press_and_release_keys('refresh', press_secs)

  def ctrl_refresh_key(self, press_secs=''):
    """Simulate Ctrl and Refresh (F3) simultaneous press.

        This key combination is an alternative of Space key.
        """
    self._press_and_release_keys('ctrl_refresh', press_secs)

  def imaginary_key(self, press_secs=''):
    """Simulate imaginary key button press.

        Maps to a key that doesn't physically exist.
        """
    self._press_and_release_keys('unused', press_secs)

class StoutHandler(MatrixKeyboardHandler):
  """Stout keyboard handler for DUT with internal keyboards.

    """

  KEY_MATRIX = {
      'ctrl_d': ['0', '0', '0', '1'],
      'd': ['0', '0', '1', '1'],
      'unused': ['0', '1', '1', '1'],
      'rec_mode': ['1', '0', '0', '0'],
      'ctrl_enter': ['1', '0', '0', '1'],
      'enter': ['1', '0', '1', '1'],
      'ctrl': ['1', '1', '0', '1'],
      'refresh': ['1', '1', '1', '0'],
      'ctrl_refresh': ['1', '1', '1', '1'],
      'none': ['1', '1', '1', '1']
  }

  def __init__(self, servo):
    """Sets up the servo communication infrastructure.

        @param servo: A Servo object representing
                           the host running servod.
        """
    super(StoutHandler, self).__init__(servo)
    self.open()

class ParrotHandler(MatrixKeyboardHandler):
  """Parrot keyboard handler for DUT with internal keyboards.

    """

  KEY_MATRIX = {
      'ctrl_d': ['0', '0', '1', '0'],
      'd': ['0', '0', '1', '1'],
      'ctrl_enter': ['0', '1', '1', '0'],
      'enter': ['0', '1', '1', '1'],
      'ctrl_refresh': ['1', '0', '0', '1'],
      'unused': ['1', '1', '0', '0'],
      'refresh': ['1', '1', '0', '1'],
      'ctrl': ['1', '1', '1', '0'],
      'none': ['1', '1', '1', '1']
  }

  def __init__(self, servo):
    """Sets up the servo communication infrastructure.

        @param servo: A Servo object representing
                           the host running servod.
        """
    super(ParrotHandler, self).__init__(servo)
    self.open()

class ChromeECHandler(_BaseHandler):
  """Chrome EC keyboard handler for DUT with Chrome EC.
    """

  # en-US key matrix (from "kb membrane pin matrix.pdf")
  KEY_MATRIX = {
      # key: (row, col)
      '`': (3, 1),
      '1': (6, 1),
      '2': (6, 4),
      '3': (6, 2),
      '4': (6, 3),
      '5': (3, 3),
      '6': (3, 6),
      '7': (6, 6),
      '8': (6, 5),
      '9': (6, 9),
      '0': (6, 8),
      '-': (3, 8),
      '=': (0, 8),
      'q': (7, 1),
      'w': (7, 4),
      'e': (7, 2),
      'r': (7, 3),
      't': (2, 3),
      'y': (2, 6),
      'u': (7, 6),
      'i': (7, 5),
      'o': (7, 9),
      'p': (7, 8),
      '[': (2, 8),
      ']': (2, 5),
      '\\': (3, 11),
      'a': (4, 1),
      's': (4, 4),
      'd': (4, 2),
      'f': (4, 3),
      'g': (1, 3),
      'h': (1, 6),
      'j': (4, 6),
      'k': (4, 5),
      'l': (4, 9),
      ';': (4, 8),
      '\'': (1, 8),
      'z': (5, 1),
      'x': (5, 4),
      'c': (5, 2),
      'v': (5, 3),
      'b': (0, 3),
      'n': (0, 6),
      'm': (5, 6),
      ',': (5, 5),
      '.': (5, 9),
      '/': (5, 8),
      ' ': (5, 11),
      '<right>': (6, 12),
      '<alt_r>': (0, 10),
      '<down>': (6, 11),
      '<tab>': (2, 1),
      '<f10>': (0, 4),
      '<shift_r>': (7, 7),
      '<ctrl_r>': (4, 0),
      '<esc>': (1, 1),
      '<backspace>': (1, 11),
      '<f2>': (3, 2),
      '<alt_l>': (6, 10),
      '<ctrl_l>': (2, 0),
      '<f1>': (0, 2),
      '<search>': (0, 1),
      '<f3>': (2, 2),
      '<f4>': (1, 2),
      '<f5>': (3, 4),
      '<f6>': (2, 4),
      '<f7>': (1, 4),
      '<f8>': (2, 9),
      '<f9>': (1, 9),
      '<up>': (7, 11),
      '<shift_l>': (5, 7),
      '<enter>': (4, 11),
      '<left>': (7, 12)
  }

  def __init__(self, servo):
    """Sets up the servo communication infrastructure.

        @param servo: A Servo object representing
                           the host running servod.
        """
    super(ChromeECHandler, self).__init__(servo)
    base_board = self._servo.get_base_board()
    if base_board:
      self._ec_uart_regexp = base_board + '_ec_uart_regexp'
      self._ec_uart_cmd = base_board + '_ec_uart_cmd'
    else:
      self._ec_uart_regexp = 'ec_uart_regexp'
      self._ec_uart_cmd = 'ec_uart_cmd'
    self.open()

  def _send_command(self, command):
    """Send command through UART.

        This function opens UART pty when called, and then command is sent
        through UART.

        @param command: The command to send.
        """
    self._servo.set(self._ec_uart_regexp, 'None')
    self._servo.set(self._ec_uart_cmd, command)

  def _press_and_release_keys(self, keys, press_secs=''):
    """Simulate a key combination press and release.

        The key combination (multiple keys) are all pressed and then
        all released.

        @param keys: A list of key names, which are the keys of KEY_MATRIX.
        """
    if press_secs is '':
      press_secs = self.SERVO_KEY_PRESS_DELAY
    for key in keys:
      # Send EC command: kbpress col row pressed
      self._send_command('kbpress %d %d 1' % (self.KEY_MATRIX[key][1],
                                              self.KEY_MATRIX[key][0]))
    time.sleep(press_secs)
    for key in keys:
      # Send EC command: kbpress col row pressed
      self._send_command('kbpress %d %d 0' % (self.KEY_MATRIX[key][1],
                                              self.KEY_MATRIX[key][0]))

  def ctrl_d(self, press_secs=''):
    """Simulate Ctrl-d simultaneous button presses."""
    self._press_and_release_keys(['<ctrl_l>', 'd'], press_secs)

  def ctrl_u(self, press_secs=''):
    """Simulate Ctrl-u simultaneous button presses."""
    self._press_and_release_keys(['<ctrl_l>', 'u'], press_secs)

  def ctrl_s(self, press_secs=''):
    """Simulate Ctrl-s simultaneous button presses."""
    self._press_and_release_keys(['<ctrl_l>', 's'], press_secs)

  def ctrl_enter(self, press_secs=''):
    """Simulate Ctrl-enter simultaneous button presses."""
    self._press_and_release_keys(['<enter>'], press_secs)

  def ctrl_key(self, press_secs=''):
    """Simulate Enter key button press."""
    self._press_and_release_keys(['<ctrl_l>'], press_secs)

  def arrow_up(self, press_secs=''):
    """Simulate ArrowUp key button press."""
    self._press_and_release_keys(['<up>'], press_secs)

  def arrow_down(self, press_secs=''):
    """Simulate ArrowDown key button press."""
    self._press_and_release_keys(['<down>'], press_secs)

  def enter_key(self, press_secs=''):
    """Simulate Enter key button press."""
    self._press_and_release_keys(['<enter>'], press_secs)

  def refresh_key(self, press_secs=''):
    """Simulate Refresh key (F3) button press."""
    self._press_and_release_keys(['<f3>'], press_secs)

  def ctrl_refresh_key(self, press_secs=''):
    """Simulate Ctrl and Refresh (F3) simultaneous press.

        This key combination is an alternative of Space key.
        """
    self._press_and_release_keys(['<ctrl_l>', '<f3>'], press_secs)

  def sysrq_x(self, press_secs=''):
    """Simulate Alt VolumeUp X simultaneous press.

        This key combination is the kernel system request (sysrq) x.
        """
    self._press_and_release_keys(['<alt_l>', '<f10>', 'x'], press_secs)

  def arb_key(self, press_secs=''):
    """Simulate an arbitrary key press."""
    self._press_and_release_keys([self._arb_key], press_secs)

class USBkm232Handler(_BaseHandler):
  """Keyboard handler for devices without internal keyboard."""

  MAX_RSP_RETRIES = 10
  USB_QUEUE_DEPTH = 6
  CLEAR = '\x38'
  KEYS = {
      #row 1
      '`': 1,
      '1': 2,
      '2': 3,
      '3': 4,
      '4': 5,
      '5': 6,
      '6': 7,
      '7': 8,
      '8': 9,
      '9': 10,
      '0': 11,
      '-': 12,
      '=': 13,
      '<undef1>': 14,
      '<backspace>': 15,
      '<tab>': 16,
      'q': 17,
      'w': 18,
      'e': 19,
      'r': 20,
      't': 21,
      'y': 22,
      'u': 23,
      'i': 24,
      'o': 25,
      'p': 26,
      '[': 27,
      ']': 28,
      '\\': 29,
      # row 2
      '<capslock>': 30,
      'a': 31,
      's': 32,
      'd': 33,
      'f': 34,
      'g': 35,
      'h': 36,
      'j': 37,
      'k': 38,
      'l': 39,
      ';': 40,
      '\'': 41,
      '<undef2>': 42,
      '<enter>': 43,
      # row 3
      '<lshift>': 44,
      '<undef3>': 45,
      'z': 46,
      'x': 47,
      'c': 48,
      'v': 49,
      'b': 50,
      'n': 51,
      'm': 52,
      ',': 53,
      '.': 54,
      '/': 55,
      '[clear]': 56,
      '<rshift>': 57,
      # row 4
      '<lctrl>': 58,
      '<undef5>': 59,
      '<lalt>': 60,
      ' ': 61,
      '<ralt>': 62,
      '<undef6>': 63,
      '<rctrl>': 64,
      '<undef7>': 65,
      '<mouse_left>': 66,
      '<mouse_right>': 67,
      '<mouse_up>': 68,
      '<mouse_down>': 69,
      '<lwin>': 70,
      '<rwin>': 71,
      '<win apl>': 72,
      '<mouse_lbtn_press>': 73,
      '<mouse_rbtn_press>': 74,
      '<insert>': 75,
      '<delete>': 76,
      '<mouse_mbtn_press>': 77,
      '<undef16>': 78,
      '<larrow>': 79,
      '<home>': 80,
      '<end>': 81,
      '<undef23>': 82,
      '<uparrow>': 83,
      '<downarrow>': 84,
      '<pgup>': 85,
      '<pgdown>': 86,
      '<mouse_scr_up>': 87,
      '<mouse_scr_down>': 88,
      '<rarrow>': 89,
      # numpad
      '<numlock>': 90,
      '<num7>': 91,
      '<num4>': 92,
      '<num1>': 93,
      '<undef27>': 94,
      '<num/>': 95,
      '<num8>': 96,
      '<num5>': 97,
      '<num2>': 98,
      '<num0>': 99,
      '<num*>': 100,
      '<num9>': 101,
      '<num6>': 102,
      '<num3>': 103,
      '<num.>': 104,
      '<num->': 105,
      '<num+>': 106,
      '<numenter>': 107,
      '<undef28>': 108,
      '<mouse_slow>': 109,
      # row 0
      '<esc>': 110,
      '<mouse_fast>': 111,
      '<f1>': 112,
      '<f2>': 113,
      '<f3>': 114,
      '<f4>': 115,
      '<f5>': 116,
      '<f6>': 117,
      '<f7>': 118,
      '<f8>': 119,
      '<f9>': 120,
      '<f10>': 121,
      '<f11>': 122,
      '<f12>': 123,
      '<prtscr>': 124,
      '<scrllk>': 125,
      '<pause/brk>': 126,
  }

  def __init__(self, servo, serial_device):
    """Constructor for usbkm232 class."""
    super(USBkm232Handler, self).__init__(servo)
    if serial_device is None:
      raise Exception('No device specified when '
                      'initializing usbkm232 keyboard handler')
    self.serial_device = serial_device
    self.serial = None
    self.open()
    self._logger.info('USBKM232: %s', self.serial_device)

  def open(self):
    """Open serial connection to serial_device."""
    if self.is_open():
      return
    self.serial = serial.Serial(self.serial_device, 9600, timeout=0.1)
    self.serial.interCharTimeout = 0.5
    self.serial.timeout = 0.5
    self.serial.writeTimeout = 0.5
    super(USBkm232Handler, self).open()

  def close(self):
    """Close usbkm232 device, and assert rst on atmega if necessary."""
    if not self.is_open():
      return
    self.serial.close()
    super(USBkm232Handler, self).close()

  def _test_atmega(self):
    """Send and receive a key from the atmega to verify it is present.

       Returns:
         Raises exception if no correct response is received.
       """
    self.serial.write(chr(0))
    rsp = self.serial.read(1)
    if not rsp or (ord(rsp) != 0xff):
      self._logger.error('Presence check response from atmega KB emu: rsp: %s',
                         rsp)
      self._logger.error('Atmega KB offline: failed to communicate.')

  def _press(self, press_ch):
    """Encode and return character to press using usbkm232.

        Args:
          press_ch: character to press

        Returns:
          Proper encoding to send to the uart side of the usbkm232 to create the
          desired key press.
        """
    return '%c' % self.KEYS[press_ch]

  def _release(self, release_ch):
    """Encode and return character to release using usbkm232.

        This value is simply the _press_ value + 128

        Args:
          release_ch: character to release

        Returns:
          Proper encoding to send to the uart side of the usbkm232 to create the
          desired key release.
        """
    return '%c' % (self.KEYS[release_ch] | 0x80)

  def _rsp(self, orig_ch):
    """Check response after sending character to usbkm232.

        The response is the one's complement of the value sent.  This method
        blocks until proper response is received.

        Args:
          orig_ch: original character sent.

        Raises:
          Exception: if response was incorrect or timed out
        """
    count = 0
    rsp = self.serial.read(1)
    while (len(rsp) != 1 or ord(orig_ch) != (~ord(rsp) & 0xff)) \
            and count < self.MAX_RSP_RETRIES:
      rsp = self.serial.read(1)
      print('re-read rsp')
      count += 1

    if count == self.MAX_RSP_RETRIES:
      raise Exception('Failed to get correct response from usbkm232')
    print('usbkm232: response [-] = \\0%03o 0x%02x' % (ord(rsp), ord(rsp)))

  def _write(self, mylist, check=False, clear=True):
    """Write list of commands to usbkm232.

        Args:
          mylist: list of encoded commands to send to the uart side of the
            usbkm232
          check: boolean determines whether response from usbkm232 should be
            checked.
          clear: boolean determines whether keytroke clear should be sent at end
            of the sequence.
        """
    # TODO(tbroch): USB queue depth is 6 might be more efficient to write
    #               more than just one make/break
    for i, write_ch in enumerate(mylist):
      print('usbkm232: writing  [%d] = \\0%03o 0x%02x' % \
            (i, ord(write_ch), ord(write_ch)))
      self.serial.write(write_ch)
      if check:
        self._rsp(write_ch)
      time.sleep(.05)

    if clear:
      print('usbkm232: clearing keystrokes')
      self.serial.write(self.CLEAR)
      if check:
        self._rsp(self.CLEAR)

  def writestr(self, mystr):
    """Write string to usbkm232.

        Args:
          mystr: string to send across the usbkm232
          """
    rlist = []
    for write_ch in mystr:
      rlist.append(self._press(write_ch))
      rlist.append(self._release(write_ch))
    self._write(rlist)

  def ctrl_d(self, press_secs=''):
    """Press and release ctrl-d sequence."""
    self._write([self._press('<lctrl>'), self._press('d')])

  def ctrl_u(self, press_secs=''):
    """Press and release ctrl-u sequence."""
    self._write([self._press('<lctrl>'), self._press('u')])

  def ctrl_s(self, press_secs=''):
    """Press and release ctrl-s sequence."""
    self._write([self._press('<lctrl>'), self._press('s')])

  def arrow_up(self, press_secs=''):
    """Press and release ArrowUp key."""
    self._write([self._press('<uparrow>')])

  def arrow_down(self, press_secs=''):
    """Press and release ArrowDown key."""
    self._write([self._press('<downarrow>')])

  def enter_key(self, press_secs=''):
    """Press and release enter"""
    self._write([self._press('<enter>')])

  def ctrl_key(self, press_secs=''):
    """Simulate Enter key button press."""
    self._write([self._press('<lctrl>')])

  def crtl_enter(self):
    """Press and release ctrl+enter"""
    self._write([self._press('<lctrl>'), self._press('<enter>')])

  def space_key(self):
    """Press and release space key."""
    self._write([self._press(' ')])

  def refresh_key(self, press_secs=''):
    """Simulate Refresh key (F3) button press."""
    self._write([self._press('<f3>')])

  def ctrl_refresh_key(self, press_secs=''):
    """Simulate Ctrl and Refresh (F3) simultaneous press.
        This key combination is an alternative of Space key.
        """
    self._write([self._press('<lctrl>'), self._press('<f3>')])

  def sysrq_x(self, press_secs=''):
    """Simulate Alt VolumeUp X simultaneous press.

        This key combination is the kernel system request (sysrq) x.
        """
    self._write([self._press('<lalt>'), self._press('<f10>'), self._press('x')])

  def arb_key(self, press_secs=''):
    """Simulate an arbitrary key press."""
    self._write([self._press(self._arb_key)])

  def tab(self):
    """Press and release tab"""
    self._write([self._press('<tab>')])

class ServoUSBkm232Handler(USBkm232Handler):
  """Keyboard handler for devices without internal keyboard."""

  def __init__(self, servo, legacy):
    """
    Args:
      servo: Servo device used to execute controls
      legacy: bool, true for servo v2, v3 as they require more setup.
    """
    servo.set('atmega_rst', 'on')
    servo.set('at_hwb', 'off')
    servo.set('atmega_rst', 'off')
    serial = servo.get('atmega_pty')
    self.legacy = legacy
    super(ServoUSBkm232Handler, self).__init__(servo, serial)

  def open(self):
    """Take atmega out of reset, and potentially do legacy setup."""
    if self.is_open():
      return
    # Ensure that the atmega is not in reset
    self._servo.set('atmega_rst', 'off')
    # Do proper setup for legacy devices
    if self.legacy:
      self._servo.set('atmega_baudrate', '9600')
      self._servo.set('atmega_bits', 'eight')
      self._servo.set('atmega_parity', 'none')
      self._servo.set('atmega_sbits', 'one')
      self._servo.set('usb_mux_sel4', 'on')
      self._servo.set('usb_mux_oe4', 'on')
    # Give the board enough time to boot up.
    time.sleep(1)
    super(ServoUSBkm232Handler, self).open()
    self._test_atmega()

  def close(self):
    """Set the atmega to reset before closing the serial port."""
    if not self.is_open():
      return
    # If using the atmega, ensure that the atmega is in reset.
    self._servo.set('atmega_rst', 'on')
    super(ServoUSBkm232Handler, self).close()


# encoding=utf-8
import re
import datetime

from scrapy.selector import Selector
from scrapy.http import Request
from scrapy.spiders import CrawlSpider

from zg_news.items import ZgNewsItem

class Spider(CrawlSpider):
    name = "zg_newsSpider"
    host = "https://www.chinanews.com/"
    start_requests=[]

    def start_requests(self):
        start = input("Enter your startTime(20181101):")
        end= input("Enter your endTime(20181101):")
        index=int(end)
        while index>=int(start):
            cur='%d'%index
            if int(cur[4]+cur[5])<=12 and 31 >= int(cur[6] + cur[7]) >= 1:
                url="https://www.chinanews.com/scroll-news/"+cur[0]+cur[1]+cur[2]+cur[3]+"/"+cur[4]+cur[5]+cur[6]+cur[7]+"/news.shtml"
                yield Request(url=url, callback=self.parse,dont_filter=True)
            index = index - 1

    def parse(self, response):
        """ 抓取新闻链接 """

        selector = Selector(response)
        flag=response.text
        if "查看往日回顾" in flag:
            type_arr= selector.xpath('//*[@class="dd_lm"]/a/text()').extract()
            url_arr= selector.xpath('//*[@class="dd_bt"]/a/@href').extract()
            if type_arr and url_arr:
                for type, url in zip(type_arr, url_arr):
                    if "shipin" not in url and "tp" not in url:
                        url="https://www.chinanews.com"+url
                        yield Request(url=url, meta={'type': type}, callback=self.parse1)

    def parse1(self,response):
        """抓取新闻html"""
        ZgNewsItems = ZgNewsItem()
        ZgNewsItems["type"]=response.meta['type']
        ZgNewsItems["url"]=response.url
        ZgNewsItems["html"]=response.text
        ZgNewsItems["datetime"]=''
        ZgNewsItems["source"]='中国新闻网'
        ZgNewsItems["content"]=''
        ZgNewsItems["editor"]=''
        ZgNewsItems["title"]=''
        selector = Selector(response)
        text0 = selector.xpath('//*[@class="left-t"]/text()').extract_first()

        #获得时间和来源网站
        if text0:
            if "来源" in text0:
                ss=text0.split("来源：")
                ZgNewsItems["datetime"]=ss[0].split(" ")[1].strip()
                ZgNewsItems["source"]=ss[1]

        #获得新闻内容
        text1=selector.xpath('//*[@class="left_zw"]/p/text()').extract()
        if text1:
            str0=''
            ZgNewsItems["content"]=str0.join(text1)

        #获得新闻编辑
        text2=selector.xpath('//*[@class="left_name"]/text()').extract()
        if text2:
            str1=''
            ZgNewsItems["editor"]=str1.join(text2).strip()

        #获得新闻标题
        text3 = selector.xpath('//*[@id="cont_1_1_2"]/h1/text()').extract_first()
        if text3:
            ZgNewsItems["title"]=text3.lstrip().rstrip()
        yield ZgNewsItems


"""
Mask R-CNN
The main Mask R-CNN model implemenetation.

"""

import datetime
import re
import os
import logging
import multiprocessing

from mrcnn.utils import *
from mrcnn.data_generator import *

import keras
import keras.backend as K
import keras.layers as KL
import keras.models as KM
import keras.engine as KE

import tensorflow as tf

############################################################
#  MaskRCNN Class
############################################################
class MaskRCNN():

    def __init__(self, mode, config, model_dir):
        """
        mode: Either "training" or "inference"
        config: A Sub-class of the Config class
        model_dir: Directory to save training logs and trained weights
        """
        assert mode in ['training', 'inference']
        self.mode = mode
        self.config = config
        self.model_dir = model_dir
        self.set_log_dir()
        self.keras_model = self.build(mode=mode, config=config)

    def build(self, mode, config):
        """
        Build Mask R-CNN architecture.
        input_shape: The shape of the input image
        mode: Either "training" or "inference". The inputs and outputs of the model differ accordingly.
        """
        assert mode in ['training', 'inference']

        # Image size must be dividable by 2 multiple times
        h, w = config.IMAGE_SHAPE[:2]
        if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6):
            raise ValueError("Image size must be dividable by 2 at least 6 times "
                            "to avoid fractions when downscaling and upscaling."
                            "For example, use 256, 320, 384, 448, 512, ... etc. ")

        # Inputs
        input_image = KL.Input(shape=[None, None, 3], name="input_image")
        input_image_meta = KL.Input(shape=[config.IMAGE_META_SIZE], name="input_image_meta")

        if mode == "training":
            # RPN GT
            input_rpn_match = KL.Input(shape=[None, 1], name="input_rpn_match", dtype=tf.int32)
            input_rpn_bbox = KL.Input(shape=[None, 4], name="input_rpn_bbox", dtype=tf.float32)

            # Detection GT (class IDs, bounding boxes, and masks)
            # 1. GT class IDs (zero padded)
            input_gt_class_ids = KL.Input(shape=[None], name="input_gt_class_ids", dtype=tf.int32)

            # 2. GT Boxes in pixels (zero padded)
            input_gt_boxes = KL.Input(shape=[None, 4], name="input_gt_boxes", dtype=tf.float32)
            # Normalize coordinates
            gt_boxes = KL.Lambda(lambda x: norm_boxes_graph(x, K.shape(input_image)[1:3]))(input_gt_boxes)

            # 3. GT Masks (zero padded)
            # [batch, height, width, MAX_GT_INSTANCES]
            if config.USE_MINI_MASK:
                input_gt_masks = KL.Input(shape=[config.MINI_MASK_SHAPE[0],
                                                 config.MINI_MASK_SHAPE[1], None],
                                          name="input_gt_masks", dtype=bool)
            else:
                input_gt_masks = KL.Input(shape=[config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1], None],
                                          name="input_gt_masks", dtype=bool)

        elif mode == "inference":
            # Anchors in normalized coordinates
            input_anchors = KL.Input(shape=[None, 4], name="input_anchors")

        # Build the shared convolutional layers.
        # Bottom-up layers
        # Returns a list of the last layers of each stages, 5 in total.
        # Don't create the thead(stage 5), so we pick the 4th item in the list.
        _, C2, C3, C4, C5 = resnet_graph(input_image, config.BACKBONE, stage5=True, train_bn=config.TRAIN_BN)

        # Top-down Layers
        P5 = KL.Conv2D(256, (1, 1), name='fpn_c5p5')(C5)
        P4 = KL.Add(name='fpn_p4add')([
            KL.UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5),
            KL.Conv2D(256, (1, 1), name='fpn_c4p4')(C4)])
        P3 = KL.Add(name="fpn_p3add")([
            KL.UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4),
            KL.Conv2D(256, (1, 1), name='fpn_c3p3')(C3)])
        P2 = KL.Add(name="fpn_p2add")([
            KL.UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3),
            KL.Conv2D(256, (1, 1), name='fpn_c2p2')(C2)])
        # Attach 3x3 conv to all P layers to get the final feature maps.
        P2 = KL.Conv2D(256, (3, 3), padding="SAME", name="fpn_p2")(P2)
        P3 = KL.Conv2D(256, (3, 3), padding="SAME", name="fpn_p3")(P3)
        P4 = KL.Conv2D(256, (3, 3), padding="SAME", name="fpn_p4")(P4)
        P5 = KL.Conv2D(256, (3, 3), padding="SAME", name="fpn_p5")(P5)
        # P6 is used for the 5th anchor scale in RPN. Generated by
        # subsampled from P5 with stride of 2
        P6 = KL.MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5)

        # Note that P6 is used in PRN, but not in classifier heads
        rpn_feature_maps = [P2, P3, P4, P5, P6]
        mrcnn_feature_maps = [P2, P3, P4, P5]

        # Anchor
        if mode == "training":
            anchors = self.get_anchors(config.IMAGE_SHAPE)
            # Duplicate across the batch dimension because Keras requires it
            anchors = np.broadcast_to(anchors, (config.BATCH_SIZE,) + anchors.shape)
            # A hack to get around Keras's bad support for constants
            anchors = KL.Lambda(lambda x: tf.constant(anchors), name="anchors")(input_image)
        else:
            anchors = input_anchors

        # RPN Model
        rpn = build_rpn_model(config.RPN_ANCHOR_STRIDE, len(config.RPN_ANCHOR_RATIOS), 256)

        # Loop through pyramid layers
        layer_output = []
        for p in rpn_feature_maps:
            layer_output.append(rpn([p]))
        # Convert from list of lists of level outputs to list of lists
        # Concatenate layer outputs
        # of outputs across levels.
        # e.g. [[a1, b1, c1], [a2, b2, c2]] => [[a1, a2], [b1, b2], [c1, c2]]
        output_name = ["rpn_class_logits", "rpn_class", "rpn_bbox"]
        outputs = list(zip(*layer_output))
        outputs = [KL.Concatenate(axis=1, name=n)(list(o))
                   for o, n in zip(outputs, output_name)]

        rpn_class_logits, rpn_class, rpn_bbox = outputs
        # Generate proposals
        # Proposals are [batch, N, (y1, x1, y2, x2)] in normalized coordinates
        # and zero padded.
        proposal_count = config.POST_NMS_ROIS_TRAINING if mode == "training" \
            else config.POST_NMS_ROIS_INFERENCE
        rpn_rois = ProposalLayer(
            proposal_count = proposal_count,
            num_threshold = config.RPN_NMS_THRESHOLD,
            name = "ROI",
            config = config
        )([rpn_class, rpn_bbox, anchors])

    def find_last(self):
        """Finds the last checkpoint file of the last trained model in the
        model directory.
        Returns:
            log_dir: The directory where events and weights are saved
            checkpoint_path: the path to the last checkpoint file
        """
        dir_names = next(os.walk(self.model_dir))[1]
        key = self.config.NAME.lower()
        dir_names = filter(lambda f: f.startswith(key), dir_names)
        dir_names = sorted(dir_names)
        if not dir_names:
            return None, None
        # Pick the last directory
        dir_name = os.path.join(self.model_dir, dir_names[-1])
        checkpoints = next(os.walk(dir_name))[1]
        checkpoints = filter(lambda f: f.startswith('mask_rcnn'), checkpoints)
        checkpoints = sorted(checkpoints)
        if not checkpoints:
            return dir_name, None
        checkpoint = os.path.join(dir_name, checkpoints[-1])
        return dir_names, checkpoint

    def load_weight(self, filepath, by_name=False, exclude=None):
        import h5py
        from keras.engine import topology

        if h5py is None:
            raise ImportError('`load_weights` requires h5py.')
        f = h5py.File(filepath, mode='r')
        if 'layer_names' not in f.attrs and 'model_weights' in f:
            f = f['model_weights']

        # In multi-GPU training, get layers of inner model
        keras_model = self.keras_model
        layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model") else keras_model.layers

        # Exclude some layers
        if exclude:
            by_name = True
            layers = filter(lambda l: l.name not in exclude, layers)

        if by_name:
            topology.load_weights_from_hdf5_group_by_name(f, layers)
        else:
            topology.load_weights_from_hdf5_group(f, layers)
        if hasattr(f, 'close'):
            f.close()

        # Update the log directory
        self.set_log_dir(filepath)

    def set_log_dir(self, model_path=None):
        """Sets the model log directory and epoch counter.

        model_path: If None, or a format different from what this code uses
            then set a new log directory and start epochs from 0. Otherwise,
            extract the log directory and the epoch counter from the file
            name.
        """
        # Set date and epoch counter as if starting a new model
        self.epoch = 0
        now = datetime.datetime.now()

        # If we have a model path with date and epochs use them
        if model_path:
            regex = r".*/\w+(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})/mask\_rcnn\_\w+(\d{4})\.h5"
            m = re.match(regex, model_path)
            if m:
                now = datetime.datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)),
                                        int(m.group(4)), int(m.group(5)))
                self.epoch = int(m.group(6)) - 1 + 1

        # Directory for training logs
        self.log_dir = os.path.join(self.model_dir, "{}{:%Y%m%dT%H%M}".format(self.config.NAME.lower(), now))

        # Path to save after each epoch. Include placeholders that get filled by Keras.
        self.checkpoint_path = os.path.join(self.log_dir, "mask_rcnn_{}_*epoch*.h5").format(self.config.NAME.lower())
        self.checkpoint_path = self.checkpoint_path.replace("*epoch*", "{epoch:04d}")

    def get_imagenet_weights(self):
        """Downloads ImageNet trained weights from Keras.
        Returns path to weights file.
        """
        from keras.utils.data_utils import get_file
        TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/'\
                                 'releases/download/v0.2/'\
                                 'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'
        weights_path = get_file('resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',
                                TF_WEIGHTS_PATH_NO_TOP,
                                cache_subdir='models',
                                md5_hash='a268eb855778b3df3c7506639542a6af')
        return weights_path

    def train(self, train_dataset, val_dataset, learning_rate, epochs, layers, augmentation=None):
        """Train the model.
        train_dataset, val_dataset: Training and validation Dataset objects.
        learning_rate: The learning rate to train with
        epochs: Number of training epochs. Note that previous training epochs
                are considered to be done alreay, so this actually determines
                the epochs to train in total rather than in this particaular
                call.
        layers: Allows selecting wich layers to train. It can be:
            - A regular expression to match layer names to train
            - One of these predefined values:
              heaads: The RPN, classifier and mask heads of the network
              all: All the layers
              3+: Train Resnet stage 3 and up
              4+: Train Resnet stage 4 and up
              5+: Train Resnet stage 5 and up
        augmentation: Optional. An imgaug (https://github.com/aleju/imgaug)
            augmentation. For example, passing imgaug.augmenters.Fliplr(0.5)
            flips images right/left 50% of the time. You can pass complex
            augmentations as well. This augmentation applies 50% of the
            time, and when it does it flips images right/left half the time
            and adds a Gausssian blur with a random sigma in range 0 to 5.

                augmentation = imgaug.augmenters.Sometimes(0.5, [
                    imgaug.augmenters.Fliplr(0.5),
                    imgaug.augmenters.GaussianBlur(sigma=(0.0, 5.0))
                ])
        """
        assert self.mode == "training", "Create model in training mode."

        # Pre-defined layer regular expressions
        layer_regex = {
            # all layers but the backbone
            "heads": r"(mrcnn\_.*)|(rpn\_.*)|(fpn\_.*)",
            # From a specific Resnet stage and up
            "3+": r"(res3.*)|(bn3.*)|(res4.*)|(bn4.*)|(res5.*)|(bn5.*)|(mrcnn\_.*)|(rpn\_.*)|(fpn\_.*)",
            "4+": r"(res4.*)|(bn4.*)|(res5.*)|(bn5.*)|(mrcnn\_.*)|(rpn\_.*)|(fpn\_.*)",
            "5+": r"(res5.*)|(bn5.*)|(mrcnn\_.*)|(rpn\_.*)|(fpn\_.*)",
            # All layers
            "all": ".*",
        }
        if layers in layer_regex.keys():
            layers = layer_regex[layers]

        # Data generators
        train_generator = data_generator(train_dataset, self.config, shuffle=True, augmentation=augmentation,
                                         batch_size=self.config.BATCH_SIZE)
        val_generator = data_generator(val_dataset, self.config, shuffle=True, batch_size=self.config.BATCH_SIZE)

        # Callbacks
        callbacks = [
            keras.callbacks.TensorBoard(log_dir=self.log_dir, histogram_freq=0, write_graph=True, write_images=False),
            keras.callbacks.ModelCheckpoint(self.checkpoint_path, verbose=0, save_weights_only=True)
        ]

        # Train
        log("\nStarting at epoch {}. LR={}\n".format(self.epoch, learning_rate))
        log("Checkpoint Path: {}".format(self.checkpoint_path))
        self.set_trainable(layers)
        self.compile(learning_rate, self.config.LEARNING_MOMENTUM)

        # Work-around for Windows: Keras fails on Windows when using
        # multiprocessing workers. See discussion here:
        # https://github.com/matterport/Mask_RCNN/issues/13#issuecomment-353124009
        if os.name is 'nt':
            workers = 0
        else:
            workers = multiprocessing.cpu_count()

        self.keras_model.fit_generator(
            train_generator,
            initial_epoch=self.epoch,
            epochs=epochs,
            steps_per_epoch=self.config.STEPS_PER_EPOCH,
            callbacks=callbacks,
            validation_data=val_generator,
            validation_steps=self.config.VALIDATION_STEPS,
            max_queue_size=100,
            workers=workers,
            use_multiprocessing=True,
        )
        self.epoch = max(self.epoch, epochs)

    def set_trainable(self, layer_regex, keras_model=None, indent=0, verbose=1):
        """Sets model layers as trainable if their names match
        the given regular expression.
        """
        # Print message on the first call (but not on recursive calls)
        if verbose > 1 and keras_model is None:
            log("Selecting layers to train")

        keras_model = keras_model or self.keras_model

        # In multi-GPU training, we wrap the model. Get layers
        # of the inner model because they have the weights.
        layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model")\
            else keras_model.layers
        for layer in layers:
            # Is the layer a model?
            if layer.__class__.__name__ == 'Model':
                print("In model: ", layer.name)
                self.set_trainable(layer_regex, keras_model=layer, indent=indent + 4)
                continue

            if not layer.weights:
                continue
            # Is it trainable?
            trainable = bool(re.fullmatch(layer_regex, layer.name))
            # Update layer. If layer is a container, update inner layer .
            if layer.__class__.__name__ == 'TimeDistributed':
                layer.layer.trainable = trainable
            else:
                layer.trainable = trainable
            # Print trainable layer names
            if trainable and verbose > 0:
                log("{}{:20}   ({})".format(" " * indent, layer.name,
                                            layer.__class__.__name__))

    def compile(self, learning_rate, momentum):
        """Gets the model ready for training. Adds losses, regularization, and
        metrics. Then calls the Keras compile() function.
        TODO: Change the loss_names for keypoint
        """
        # Optimizer object
        optimizer = keras.optimizers.SGD(lr=learning_rate, momentum=momentum, clipnorm=self.config.GRADIENT_CLIP_NORM)

        # Add Losses
        # First, clear previously set losses to avoid duplication
        self.keras_model._losses = []
        self.keras_model._per_input_losses = {}
        loss_names = ["rpn_class_loss", "rpn_bbox_loss", "mrcnn_class_loss",
                      "mrcnn_bbox_loss", "mrcnn_mask_loss"]
        for name in loss_names:
            layer = self.keras_model.get_layer(name)
            if layer.output in self.keras_model.losses:
                continue
            self.keras_model.add_loss(tf.reduce_mean(layer.output), keep_dims=True)

        # Add L2 Regularization
        # Skip gamma and beta weights of batch normalization layers
        reg_losses = [keras.regularizers.l2(self.config.WEIGHT_DECAY)(w) / tf.cast(tf.size(w), tf.float32)
                      for w in self.keras_model.trainable_weights
                      if 'gamma' not in w.name and 'beta' not in w.name]
        self.keras_model.add_loss(tf.add_n(reg_losses))

        # Compile
        self.keras_model.compile(optimizer=optimizer, loss=[None] * len(self.keras_model.outputs))

        # Add metrics for losses
        for name in loss_names:
            if name in self.keras_model.metrics_names:
                continue
            layer = self.keras_model.get_layer(name)
            self.keras_model.metrics_names.append(name)
            self.keras_model.metrics_tensors.append(tf.reduce_mean(layer.output, keep_dims=True))

    def get_anchors(self, image_shape):
        """Returns anchor pyramid for the given image size."""
        backbone_shapes = compute_backbone_shapes(self.config, image_shape)
        #  Cache anchors and resue if image shape is the same
        if not hasattr(self, "_anchor_cache"):
            self._anchor_cache = []
        if not tuple(image_shape) in self._anchor_cache:
            # Generate Anchors
            a = generate_pyramid_anchors(
                self.config.RPN_ANCHOR_SCALES,
                self.config.RPN_ANCHOR_RATIOS,
                backbone_shapes,
                self.config.BACKBONE_STRIDES,
                self.config.RPN_ANCHOR_STRIDE
            )
        # Keep a copy of the latest anchors in pixel coordinates because
        # it's used in inspect_model notebooks.
        # TODO: inspect_model???
        self.anchor = a
        # Normalize coordinates
        self._anchor_cache[tuple(image_shape)] = norm_boxes(a, image_shape[:2])
        return self._anchor_cache[tuple(image_shape)]

############################################################
#  Resnet Graph
############################################################
def conv_block(input_tensor, kernel_size, filters, stage, block,
               strides=(2, 2), use_bias=True, train_bn=True):
    """conv_block is the block that has a conv layer at shortcut
    # Arguments
        input_tensor: input tensor
        kernel_size: defualt 3, the kernel size of middle conv layer at main path
        filters: list of integers, the nb_filters of 3 conv layer at main path
        stage: integer, current stage label, used for generating layer names
        block: 'a','b'..., current block label, used for generating layer names
        use_bias: Boolean. To use or not use a bias in conv layers.
        train_bn: Boolean. Train or freeze Batch Norm layres
    Note that from stage 3, the first conv layer at main path is with subsample=(2,2)
    And the shortcut should have subsample=(2,2) as well
    """
    nb_filter1, nb_filter2, nb_filter3 = filters
    conv_name_base = 'res' + str(stage) + block + '_branch'
    bn_name_base = 'bn' + str(stage) + block + '_branch'

    x = KL.Conv2D(nb_filter1, (1, 1), strides=strides, name=conv_name_base + '2a', use_bias=use_bias)(input_tensor)
    x = BatchNorm(name=bn_name_base + '2a')(x, training=train_bn)
    x = KL.Activation('relu')(x)

    x = KL.Conv2D(nb_filter2, (kernel_size, kernel_size), padding='same', name=conv_name_base + '2b', use_bias=use_bias)(x)
    x = BatchNorm(name=bn_name_base + '2b')(x, training=train_bn)
    x = KL.Activation('relu')(x)

    x = KL.Conv2D(nb_filter3, (1, 1), name=conv_name_base + '2c', use_bias=use_bias)(x)
    x = BatchNorm(name=bn_name_base + '2c')(x, training=train_bn)

    shortcut = KL.Conv2D(nb_filter3, (1, 1), strides=strides, name=conv_name_base + '1', use_bias=use_bias)(input_tensor)
    shortcut = BatchNorm(name=bn_name_base + '1')(shortcut, training=train_bn)

    x = KL.add()([x, shortcut])
    x = KL.Activation('relu', name='res' + str(stage) + block + '_out')(x)
    return x

def identity_block(input_tensor, kernel_size, filters, stage, block,
                   use_bias=True, train_bn=True):
    """The identity_block is the block that has no conv layer at shortcut
     # Arguments
         input_tensor: input tensor
         kernel_size: defualt 3, the kernel size of middle conv layer at main path
         filters: list of integers, the nb_filters of 3 conv layer at main path
         stage: integer, current stage label, used for generating layer names
         block: 'a','b'..., current block label, used for generating layer names
         use_bias: Boolean. To use or not use a bias in conv layers.
         train_bn: Boolean. Train or freeze Batch Norm layres
     """
    nb_filter1, nb_filter2, nb_filter3 = filters
    conv_name_base = 'res' + str(stage) + block + '_branch'
    bn_name_base = 'bn' + str(stage) + block + '_branch'

    x = KL.Conv2D(nb_filter1, (1, 1), name=conv_name_base + '2a', use_bias=use_bias)(input_tensor)
    x = BatchNorm(name=bn_name_base + '2a')(x, training=train_bn)
    x = KL.Activation('relu')(x)

    x = KL.Conv2D(nb_filter2, (kernel_size, kernel_size), padding='same', name=conv_name_base + '2b', use_bias=use_bias)(x)
    x = BatchNorm(name=bn_name_base + '2b')(x, training=train_bn)
    x = KL.Activation('relu')(x)

    x = KL.Conv2D(nb_filter3, (1, 1), name=conv_name_base + '2c', use_bias=use_bias)(x)
    x = BatchNorm(name=bn_name_base + '2c')(x, training=train_bn)

    x = KL.Add()([x, input_tensor])
    x = KL.Activation('relu', name='res' + str(stage) + block + '_out')(x)
    return x

def resnet_graph(input_image, architecture, stage5=False, train_bn=True):
    """Build a ResNet graph.
        architecture: Can be resnet50 or resnet101
        stage5: Boolean. If False, stage5 of the network is not created
        train_bn: Boolean. Train or freeze Batch Norm layres
    """
    assert architecture in ["resnet50", "resnet101"]
    # Stage 1
    x = KL.ZeroPadding2D((3, 3))(input)
    x = KL.Conv2D(64, (7, 7), strides=(2, 2), name='conv1', use_bias=True)(x)
    X = BatchNorm(name='bn_conv1')(x, training=train_bn)
    x = KL.Activation('relu')(x)
    C1 = x = KL.MaxPooling2D((3, 3), strides=(2, 2), padding="same")(x)

    # Stage 2
    x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1), train_bn=train_bn)
    x = identity_block(x, 3, [64, 64, 256], stage=2, block='b', train_bn=train_bn)
    C2 = x = identity_block(x, 3, [64, 64, 256], stage=2, block='c', train_bn=train_bn)

    # Stage 3
    x = conv_block(x, 3, [128, 128, 512], stage=3, block='a', train_bn=train_bn)
    x = identity_block(x, 3, [128, 128, 512], stage=3, block='b', train_bn=train_bn)
    x = identity_block(x, 3, [128, 128, 512], stage=3, block='c', train_bn=train_bn)
    C3 = x = identity_block(x, 3, [128, 128, 512], stage=3, block='d', train_bn=train_bn)

    # Stage 4
    x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a', train_bn=train_bn)
    block_count  = {"resnet50": 5, "resnet101": 22}[architecture]
    for i in range(block_count):
        x = identity_block(x, 3, [256, 256, 1024], stage=4, block=chr(98+i), train_bn=train_bn)
    C4 = x

    # Stage 5
    if stage5:
        x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a', train_bn=train_bn)
        x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b', train_bn=train_bn)
        C5 = x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c', train_bn=train_bn)
    else:
        C5 = None
    return [C1, C2, C3, C4, C5]

############################################################
#  Region Proposal Network (RPN)
############################################################
def rpn_graph(feature_map, anchors_per_location, anchor_stride):
    """Builds the computation graph of Region Proposal Network.

    feature_map: backbone features [batch, height, width, depth]
    anchors_per_location: number of anchors per pixel in the feature map
    anchor_stride: Controls the density of anchors. Typically 1 (anchors for
                   every pixel in the feature map), or 2 (every other pixel).

    Returns:
        rpn_logits: [batch, H, W, 2] Anchor classifier logits (before softmax)
        rpn_probs: [batch, H, W, 2] Anchor classifier probabilities.
        rpn_bbox: [batch, H, W, (dy, dx, log(dh), log(dw))] Deltas to be
                  applied to anchors.
    """
    # TODO: check if stride of 2 causes alignment issues if the featuremap
    #       is not even.
    # Shared convolutional base of the RPN
    shared = KL.Conv2D(512, (3, 3), padding='same', activation='relu', strides=anchor_stride,
                       name="rpn_conv_shared")(feature_map)

    # Anchor Score. [batch, height, width, anchors per location * 2]
    x = KL.Conv2D(2 * anchors_per_location, (1, 1), padding='valid', activation='linear', name='rpn_class_raw')(shared)

    # Reshape to [batch, anchors, 2]
    rpn_class_logits = KL.Lambda(lambda t: tf.reshape(t, [tf.shape(t)[0], -1, 2]))(x)

    # Softmax on last dimension of BG/FG
    rpn_probs = KL.Activation("softmax", name="rpn_class_xxx")(rpn_class_logits)

    # Bounding Box refinement. [batch, H, W, anchors per location, depth]
    # where depth is [x, y, log(w), log(h)]
    x = KL.Conv2D(anchors_per_location * 4, (1, 1), padding="valid", acitivation='linear', name='rpn_bbox_pred')(shared)

    # Reshape to [batch, anchors, 4]
    rpn_bbox = KL.Lambda(lambda t: tf.reshape(t, [tf.shape(t)[0], -1, 4]))(x)

    return [rpn_class_logits, rpn_probs, rpn_bbox]

def build_rpn_model(anchor_stride, anchors_per_location, depth):
    """Builds a Keras model of the Region Proposal Network.
    It wraps the RPN graph so it can be used multiple times with shared
    weights.

    anchors_per_location: number of anchors per pixel in the feature map
    anchor_stride: Controls the density of anchors. Typically 1 (anchors for
                   every pixel in the feature map), or 2 (every other pixel).
    depth: Depth of the backbone feature map.

    Returns a Keras Model object. The model outputs, when called, are:
    rpn_logits: [batch, H, W, 2] Anchor classifier logits (before softmax)
    rpn_probs: [batch, W, W, 2] Anchor classifier probabilities.
    rpn_bbox: [batch, H, W, (dy, dx, log(dh), log(dw))] Deltas to be
                applied to anchors.
    """
    input_feature_map = KL.Input(shape=[None, None, depth], name="input_rpn_feature_map")
    outputs = rpn_graph(input_feature_map, anchors_per_location, anchor_stride)
    return KM.Model([input_feature_map], outputs, name="rpn_model")

############################################################
#  Proposal Layer
############################################################
class ProposalLayer(KE.Layer):
    """Receives anchor scores and selects a subset to pass as proposals
    to the second stage. Filtering is done based on anchor scores and
    non-max suppression to remove overlaps. It also applies bounding
    box refinement deltas to anchors.

    Inputs:
        rpn_probs: [batch, anchors, (bg prob, fg prob)]
        rpn_bbox: [batch, anchors, (dy, dx, log(dh), log(dw))]
        anchors: [batch, (y1, x1, y2, x2)] anchors in normalized coordinates

    Returns:
        Proposals in normalized coordinates [batch, rois, (y1, x1, y2, x2)]
    """
    def __init__(self, proposal_count, nms_threshold, config=None, **kwargs):
        sum(ProposalLayer, self).__init__(**kwargs)
        self.config = config
        self.proposal_count = proposal_count
        self.nms_threshold = nms_threshold

    def call(self, inputs):
        # Box scores. Use the foreground class confidence. [Batch, num_rois, 1]
        scores = inputs[0][:, :, 1]
        # Box deltas [batch, num_rois, 4]
        deltas = inputs[1]
        deltas = deltas * np.reshape(self.config.RPN_BBOX_STD_DEV, [1, 1, 4])
        # Anchors
        anchors = inputs[2]

        # Improve performance by trimming to top anchors by score
        # and doing the rest on the smaller subset.
        pre_nms_limit = tf.minimum(6000, tf.shape(anchors)[1])
        ix = tf.nn.top_k(scores, pre_nms_limit, sorted=True, name="top_anchors").indices
        scores = batch_slice([scores, ix], lambda x, y: tf.gather(x, y),
                             self.config.IMAGES_PER_GPU)
        deltas = batch_slice([deltas, ix], lambda x, y: tf.gather(x, y),
                             self.config.IMAGES_PER_GPU)
        pre_nms_anchors = batch_slice([anchors, ix], lambda a, x: tf.gather(a, x),
                                      self.config.IMAGES_PER_GPU,
                                      names=["pre_nms_anchors"])

        # Apply deltas to anchors to get refined

    def compute_output_shape(self, input_shape):
        return (None, self.proposal_count, 4)


import logging
import requests
import moment
import utils
import time
import json

class weather(utils.utils):
    def message_callback(self,ch, method, properties, body):
        logging.info('messgae received weather')
        time.sleep(1)
        self.__get_weather(body)
        ch.basic_ack(delivery_tag=method.delivery_tag)  # 执行完再ack消息
        logging.info('weather messgae is ack on :'+str(moment.now()))

    def __get_weather(self,body):
        params = self.get_params(body)
        req = requests.get(params.request_url)
        data = json.loads(req.text)
        if(data['status'] == 200):
            wea = data['data']['forecast'][0]
            params.content='天气：'+wea['type']+' '+wea['high']+','+wea['low']+'   '+wea['notice']
        else:
            params.content='错误'

        self.notification(params)
        self.notification_to_zhouyu(params)

    def notification_to_zhouyu(self,params):
        url='http://push.devzhou.t.cn/Push/'+params.title+'/'+params.content+'?url='+params.red_url
        requests.get(url)
        logging.info('messgae send successfully')


"""empty message

Revision ID: b3cae1b7fe28
Revises: 87ff7184d522
Create Date: 2018-04-07 11:28:49.409601

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'b3cae1b7fe28'
down_revision = '87ff7184d522'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('sessions',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('expiry', sa.DateTime(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('sessions')
    # ### end Alembic commands ###


import random

from termcolor import colored

from Database.user import User
from constants import Constants
from request import Request

try:
    from Database.update import *
except ImportError:
    from .Database.update import *
try:
    from signature import Signature
except ImportError:
    from .signature import Signature
try:
    from geography import Geo
except ImportError:
    from .geography import Geo
try:
    from session import Session
except ImportError:
    from .session import Session

class Follow:

    def __init__(self, username=None, link=None, proxy=None):
        self.user = User(username=username)
        self.account = self.user.account
        self.device = self.user.device
        self.session = Session(username=username, proxy=proxy)
        self.request = Request()
        self.client_username = link.split('?')[0].split('/')[3]
        self.client_id = None
        self.x_pigeon = Signature.generate_UUID(True)
        self.geo = Geo(proxy=proxy)

    def users_info(self):

        self.session.set_headers(auth=True)
        self.request.send_request(
            endpoint=Constants.API_URL1 + 'users/{client_username}/usernameinfo/?from_module=deep_link_util'.format(
                client_username=self.client_username),
            account=self.account,
            device=self.device,
            session=self.session
        )

    def feed_reels_media(self):
        self.session.set_headers(auth=True, is_post=True)

        body = dict()
        body["supported_capabilities_new"] = Constants.NEW_SUPPORTED_CAPABILITIES
        body["source"] = "profile"
        body["_csrftoken"] = self.account.get('csrftoken')
        body["_uid"] = self.account.get('user_id')
        body["_uuid"] = self.device.get("uuid")
        body["user_ids"] = list('highlight:17958815605168824')  # TODO

        self.request.send_request(endpoint=Constants.API_URL1 + 'feed/reels_media/',
                                  post=body,
                                  session=self.session
                                  )

    def get_invite_suggestions(self):
        body = dict()
        body['count_only'] = '1'
        body['_csrftoken'] = self.account.get('csrftoken')
        body['_uuid'] = self.device.get('uuid')

        self.session.set_headers(is_post=True, auth=True)

        return self.request.send_request(endpoint=Constants.API_URL1 + "fb/get_invite_suggestions/",
                                         post=body,
                                         with_signature=False,
                                         session=self.session
                                         )

    def qp_batch_fetch(self):
        body = dict()
        body["surfaces_to_triggers"] = Constants.SURFACES_TO_TRIGGERS_OTHER_PROFILE_PAGE
        body["surfaces_to_queries"] = Constants.SURFACES_TO_QUERIES_OTHER_PROFILE_PAGE
        body["vc_policy"] = "default"
        body["_csrftoken"] = self.account.get('csrftoken')
        body["_uid"] = self.account.get('user_id')
        body["_uuid"] = self.device.get('uuid')
        body["scale"] = Constants.BATCH_SCALE
        body["version"] = Constants.BATCH_VERSION

        self.session.set_headers(is_post=True, auth=True)

        self.request.send_request(endpoint=Constants.API_URL1 + 'qp/batch_fetch/',
                                  post=body,
                                  session=self.session
                                  )

    def feed_user(self, max_id=None):
        param = dict()
        param['exclude_comment'] = 'true'
        if not max_id:
            param['max_id'] = max_id
        param['only_fetch_first_carousel_media'] = 'false'
        self.session.set_headers(auth=True)
        self.request.send_request(endpoint=Constants.API_URL1 + 'feed/user/{}/'.format(self.client_id),
                                  params=param,
                                  session=self.session
                                  )

    def highlights(self):
        param = dict()
        param['supported_capabilities_new'] = Constants.NEW_SUPPORTED_CAPABILITIES117
        param['phone_id'] = self.device.get('phone_id')
        param['battery_level'] = random.randint(30, 100)
        param['is_charging'] = '0'
        param['will_sound_on'] = '0'

        self.session.set_headers(prefix=True, auth=True)
        self.request.send_request(endpoint=Constants.API_URL1 + 'highlights/{client_id}/highlights_tray/?'.format(
            client_id=self.client_id),
                                  params=param,
                                  session=self.session
                                  )

    def feed_user_story(self):
        param = dict()
        param['supported_capabilities_new'] = Constants.NEW_SUPPORTED_CAPABILITIES117
        self.session.set_headers(auth=True)
        self.request.send_request(
            endpoint=Constants.API_URL1 + 'feed/user/{client_id}/story/'.format(client_id=self.client_id),
            params=param,
            session=self.session
        )

    def create_friendship(self):
        body = dict()
        body['_csrftoken'] = self.account.get('csrftoken')
        body['user_id'] = self.client_id
        body['radio_type'] = 'wifi-none'
        body['_uid'] = self.account.get('user_id')
        body['device_id'] = self.device.get('android_device_id')
        body['_uuid'] = self.device.get('uuid')

        self.session.set_headers(is_post=True, auth=True)
        self.request.send_request(
            endpoint=Constants.API_URL1 + 'friendships/create/{client_id}/'.format(client_id=self.client_id),
            post=body,
            session=self.session)

    def discover_chaining(self):
        param = dict()
        param['target_id'] = self.client_id
        self.session.set_headers(auth=True)
        self.request.send_request(endpoint=Constants.API_URL1 + 'discover/chaining/',
                                  params=param,
                                  session=self.session)

    def friendships_show(self):
        self.session.set_headers(auth=True)
        self.request.send_request(Constants.API_URL1 + 'friendships/show/{}/'.format(self.client_id),
                                  session=self.session)

    def follow_by_link(self):
        self.get_invite_suggestions()  # TODO
        self.qp_batch_fetch()  # Done
        self.users_info()  # Done
        self.client_id = self.request.last_json['user']['pk']
        self.friendships_show()  # Done
        self.feed_user()  # Done
        try:
            max_id = self.request.last_json['items'][0]['id']
        except Exception as e:
            print(e)
        else:
            self.feed_user(max_id=max_id)
        self.highlights()  # Done
        self.feed_user_story()  # Done
        # self.feed_reels_media() # TODO
        self.create_friendship()  # Done
        self.discover_chaining()  # Done
        print(colored(self.client_username + ' is now followed by ' + self.account.get('username'), 'yellow',
                      attrs=['bold']))

        # Update account cookie
        update_cookie(self.account.get('username'), self.request.cookie)

if __name__ == "__main__":
    link = "https://instagram.com/akzhol_makhmudov?igshid=1widv7x46l29k"
    f = Follow('ainura.bolotbek',
               link=link) #, proxy = {'http': 'http://192.168.0.102:1111', 'https': 'https://192.168.0.102:1111'})
    # f.feed_user_story()

    f.follow_by_link()


from selenium import webdriver
import time
import pandas

url1 = "http://yanxiu.qlteacher.com/project/yey2018/learning/learning"
lll = []
driver = webdriver.Chrome()
uname = ''
pwd = ''
driver.get("http://id.qlteacher.com/login")
time.sleep(10)
driver.find_element_by_name("username").send_keys(uname)
driver.find_element_by_name("password").send_keys(pwd)
time.sleep(10)
driver.find_element_by_tag_name('button').click()
time.sleep(10)
driver.get(url1)
time.sleep(10)
print(driver.page_source)
time.sleep(10)
for link in driver.find_elements_by_xpath("//a[@type='button']"):
    print(link.get_attribute("href"))
    lll.append([link.get_attribute("href")])
h = pandas.DataFrame(lll)
h.to_csv('C:\\Users\Administrator\Desktop\oxox.csv',encoding='utf-8')

登陆以后获取专题链接.py

登陆以后获取专题链接.py


from pyspark import SparkFiles
path = os.path.join(tempdir, "test.txt")
with open(path, "w") as testFile:
   _ = testFile.write("100")
sc.addFile(path)
def func(iterator):
   with open(SparkFiles.get("test.txt")) as testFile:
       fileVal = int(testFile.readline())
       return [x * fileVal for x in iterator]
sc.parallelize([1, 2, 3, 4]).mapPartitions(func).collect()


import random, string, json
import os.path,pickle
import uuid
from pdfbase.internal import PDFPlugin
from interface import implements
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from google.cloud import storage
import requests
from requests.auth import HTTPDigestAuth
from urllib.parse import urlencode
from queuelib import FifoDiskQueue

# implement interface

class GoogleDocsSheetsPlugin(implements(PDFPlugin)):

    def __init__(self):
        #get googledoc-config.json file content and then save this dtaa to class config variable
        with open(os.path.dirname(__file__) + '/googledoc-config.json') as json_file:
            config = json.load(json_file)
            self.config = config

    def get_token(self):
        creds = None
        SCOPES = [
                'https://www.googleapis.com/auth/spreadsheets.readonly',
                'https://www.googleapis.com/auth/drive'
            ]
        # The file token.pickle stores the user's access and refresh tokens, and is
        # created automatically when the authorization flow completes for the first
        # time.
        if os.path.exists(os.path.dirname(__file__) + '/token.pickle'):
            with open(os.path.dirname(__file__) + '/token.pickle', 'rb') as token:
                creds = pickle.load(token)
        # If there are no (valid) credentials available, let the user log in.
        if not creds or not creds.valid:
            if creds and creds.expired and creds.refresh_token:
                creds.refresh(Request())
            else:
                flow = InstalledAppFlow.from_client_secrets_file(
                    os.path.dirname(__file__)+'/credentials.json', SCOPES)
                creds = flow.run_local_server(port=0)
            # Save the credentials for the next run
            with open(os.path.dirname(__file__) + '/token.pickle', 'wb') as token:
                pickle.dump(creds, token)
        return creds
    def getSessionCookie(self):
        error = cookie = None
        r = requests.get(
            self.raw_data['SESSIONCOOKIEBASEURL'],
            auth=HTTPDigestAuth(self.raw_data['ODKUSERNAME'],
                                self.raw_data['ODKPASSWORD']),
            timeout=10)  # Sending the digest authorization request
        headers = str(r.headers)

        if r.status_code == 200:  # Check if request is OK
            startIndex = headers.find('JSESSIONID=')
            endIndex = headers.find(';')
            sessionCookie = headers[
                startIndex + 11:
                endIndex]  # Getting the value of json cookie from the string
            if len(sessionCookie) == 32:  # Making sure got the cookie value right
                cookie = sessionCookie  # Saving the cookie value
            else:
                error = "No session cookie found"
        else:
            error = "Authorization error"

        return error, cookie

    def get_sheetvalues(self,sheetID, varMapping):
        error = None
        try:
            SPREADSHEET_ID = sheetID
            RANGE_NAME = varMapping
            # call the class get_token method to access google sheet
            creds = self.get_token()

            service = build('sheets', 'v4', credentials=creds)

            # Call the Sheets API
            sheet = service.spreadsheets()
            #get the sheet data
            result = sheet.values().get(
                spreadsheetId=SPREADSHEET_ID,
                range=RANGE_NAME).execute()
            values = result.get('values', [])

            if not values:
                error = "No Mapping details found"
            else:
                mappingValues = values
        except Exception as e:
            error = "Failed to fetch mapping detials"
            mappingValues = None

        return mappingValues, error

    # this method return all the tags on the basis of which we filter the request
    def get_tags(self):
        tags = dict()
        tags["SHEETID"] = self.config["SHEETID"]
        tags["SHEETNAME"] = self.config["SHEETNAME"]
        tags["RANGE"] = self.config["RANGE"]
        tags["MAPPINGDETAILS"] = self.config["MAPPINGDETAILS"]
        tags["OPTIONSSHEET"] = self.config["OPTIONSSHEET"]
        tags["DOCTEMPLATEID"] = self.config["DOCTEMPLATEID"]
        tags["APPLICATIONID"] = self.config["APPLICATIONID"]
        return tags
    #this method fetches the data from google sheet and return it as raw_data and also send tag
    def fetch_data(self):
        error = None
        tags = None
        try:
            if self.config['RANGE']:
                RANGE_NAME = self.config['SHEETNAME'] + "!" + self.config['RANGE']
            else :
                RANGE_NAME = self.config['SHEETNAME']
            #call class method which return sheet data and error if permission is not there
            getValueMapping = self.get_sheetvalues(self.config['SHEETID'],RANGE_NAME)

            mappingError = getValueMapping[1]  # Error in fetching mapping
            mappingValues = getValueMapping[0]  # mapping values list
            if not mappingError:
                raw_data = mappingValues
                # Create a JSON from data.
                column_names = raw_data[0]
                final_data = []
                for data in raw_data[2:]:
                    single_data = dict()
                    counter = 0
                    for col in column_names:
                        single_data[col] = data[counter]
                        counter += 1
                    tags = self.get_tags()
                    all_data = dict()
                    all_data['req_data'] = single_data
                    all_data.update(self.config) # merge tags with sheet each row data
                    #final_data.append(raw_data)
                    raw_data = dict()
                    raw_data['reqd_data'] = all_data
                    raw_data['tags'] = tags
                    q = FifoDiskQueue(os.path.dirname(__file__)+'/../../queuedata')
                    q.push(json.dumps(raw_data).encode('utf-8'))
                    q.close()

            else:
                error = "No Mapping details found"

        except Exception as e:
            import traceback
            error = "Failed to fetch mapping detials"
            mappingValues = None

        return error

    def fetch_mapping(self,data):
        error = None
        rawData = None
        try:
            getValueMapping = self.get_sheetvalues(data['SHEETID'],data['MAPPINGDETAILS'])
            mappingError = getValueMapping[1]  # Error in fetching mapping
            mappingValues = getValueMapping[0]  # mapping values list
            getOptionsMapping = self.get_sheetvalues(data['SHEETID'],
                                                         data['OPTIONSSHEET'])
            optionsError = getOptionsMapping[1]  # Error in fetching options
            optionsMapping = getOptionsMapping[0]  # options mapping list

            if not mappingError and not optionsError:
               rawData = dict()
               rawData['value_mapping'] = mappingValues
               rawData['options_mapping'] = optionsMapping
               data.update(rawData)
               rawData = data
               self.raw_data = rawData
            else:
                error = str(mappingError) + str(optionsError)

        except Exception as e:
            error = "Failed to fetch mapping detials"
            print(e)

        return rawData, error
    def mapData(self,allData, mappingValues, optionsMapping):
        error = None
        finalData = None
        try:
            mappingValues.pop(0)  # removing the header row of mapping sheet

            finalData = []  # List to hold the final values
            optionsMapping.pop(0)  # removing the header row of options sheet
            for row in mappingValues:
                if row[1] == 'options':
                    # row[2] option value name
                    # str(allData[row[2]]) option value a,b,c,d
                    # optionsMapping[] the list with valuename and options value
                    thisList = ''.join(
                        str(e) for e in optionsMapping)  # Converted list to string
                    newList = thisList.split(str(
                        row[2]), 1)[1]  # List split to start with required string
                    indexStart = newList.find(str(row[2]))  # Finding the substring
                    indexEnd = newList.find("]")  # Find the stopping point
                    strToCheck = newList[:
                                         indexEnd]  # The semi final string to find values from obtained
                    optionValueStart = strToCheck.find(str(allData[row[2]]))
                    if optionValueStart is -1:
                        allData[row[
                            2]] = 'NO_TEXT_FOUND'  # If the particular option is not found
                        finalData.append(allData[row[2]])  # Appending the received data to the final list
                    else:
                        length = len(allData[row[2]])
                        optionValueEnd = strToCheck.find("'", optionValueStart)
                        optionValue = strToCheck[optionValueStart + length + 2:
                                                 optionValueEnd]
                        finalData.append(
                            optionValue
                        )  # Appending the correct option to the final list
                else:
                    if not allData[row[2]]:
                        allData[row[2]] = 'NO_TEXT_FOUND'  # If data is None

                    finalData.append(allData[row[
                        2]])  # Appending the received data to the final list
        except Exception as e:
            import traceback
            print(traceback.format_exc())
            error = "Failed to map data"
        return finalData,error

    def get_config(self):
        return self.config

    def generateFileDrive(self,url):
        asUrl = url
        error = documentId = fileName = pdfUrl = None
        try:
            #call the app script url
            contents = requests.get(url, timeout=60).json()
            if contents.get("error") != "null":
                error = contents.get('error')
            if error=="undefined":
                error = None
            documentId = contents.get("documentId")
            fileName = contents.get("fileName")
            pdfUrl = contents.get("url")
        except Exception as e:
            import traceback
            error = "Failed to get response from App Script"

        return documentId, fileName, pdfUrl, error

    def build_PDF(self,raw_data,file_name):
        error = None
        fileDownloaded = ''
        fileName=''
        pdfUrl=''
        try:
            data = raw_data['req_data']
            mappingValues = raw_data['value_mapping']
            optionsMapping = raw_data['options_mapping']
            mappedData = self.mapData(data, mappingValues, optionsMapping)
            mappingError = mappedData[1]
            finalData = mappedData[0]
            if not mappingError:
                # URL of google app script
                finalDataStr = json.dumps(finalData)
                payload = {
                    "fileName": file_name,
                    "mylist": finalDataStr,
                    "templateId": raw_data['DOCTEMPLATEID']

                }  # Encoding the url with payload
                if('ODKUSERNAME' in self.raw_data.keys() and self.raw_data['ODKUSERNAME'] and 'ODKPASSWORD' in self.raw_data.keys() and self.raw_data['ODKPASSWORD']):
                    callSessionCookie = self.getSessionCookie()

                    if not callSessionCookie[0]:
                        sessionCookie = callSessionCookie[1]
                    else:
                        error = callSessionCookie[0]
                    payload['sessionCookie'] =  sessionCookie
                    payload['username'] =  self.raw_data['ODKUSERNAME']
                    payload['password'] =  self.raw_data['ODKPASSWORD']

                gasUrl = self.config['URL'] + urlencode(payload)

                appScriptResponse = self.generateFileDrive(gasUrl)  # Calling the GAS url and Getting the GAS response
                error = appScriptResponse[3]

                if not error:
                    google_doc_name = appScriptResponse[1]
                    pdfUrl = appScriptResponse[2]
                    fileName = appScriptResponse[1] + '.pdf'

            else:
                error = mappingError

        except Exception as e:
            error = "Failed to generate pdf"
            print(e)
            import traceback
            print(traceback.format_exc())

        return fileName,error,pdfUrl

    def upload_PDF(self, key,file):
        # Uploads a file to the local server.
        error = ''
        try:
            response = requests.get(file)
            with open(self.config['DIRPATH']+key, 'wb') as f:
                f.write(response.content)
            self.delete_file_drive(file)
        except Exception as e:
            import traceback
            error = "Failed to download file from drive"

        return key,error

    def retrieve_PDF(self, key):
        filedata = ''
        error = None
        file_name = self.config['DIRPATH'] + key+'.pdf'
        try:
            with open(file_name, 'rb') as f:
                filedata = f.read()
        except Exception as e:
            error = 'File not found'

        return filedata,error

    def delete_file_drive(self,file):
        # Google drive API to Permanently delete a file, skipping the trash.
        error = done = None
        try:
            creds = None
            creds = self.get_token()
            service = build('drive', 'v3', credentials=creds)
            doc_id = file.split('/')
            file_id = doc_id[5]  # find file id from url here
            service.files().delete(fileId=file_id).execute()
            done = True
        except:
            error = 'Failed to delete file'

        return error, done


from pyramid.response import Response
from pyramid.view import view_config

from sqlalchemy.exc import DBAPIError

from .models import (
    DBSession,
    MyModel,
    )

@view_config(route_name='home', renderer='templates/home.pt')
def home(request):
    try:
        one = DBSession.query(MyModel).filter(MyModel.name == 'one').first()
        if request.POST:
            # Of course we won't return the post directly
            return {"items": request.POST}
    except DBAPIError:
        return Response(conn_err_msg, content_type='text/plain', status_int=500)
    return {'one': one, 'project': 'todo', "items":""}

conn_err_msg = """\
Pyramid is having a problem using your SQL database.  The problem
might be caused by one of the following things:

1.  You may need to run the "initialize_todo_db" script
    to initialize your database tables.  Check your virtual
    environment's "bin" directory for this script and try to run it.

2.  Your database server may not be running.  Check that the
    database server referred to by the "sqlalchemy.url" setting in
    your "development.ini" file is running.

After you fix the problem, please restart the Pyramid application to
try it again.
"""


from math import sqrt, fabs
from fractions import Fraction as fr

x = sqrt(3)
xf = fr(str(x))

print("x  = %1.30f" % x)
print("xf = %s" % xf)

tmp = 0
for i in range(10, int(1e6)+1, 10):
	xf_approx = xf.limit_denominator(i)
	xfloat = float(xf_approx)
	diff = fabs(x - xfloat)

	# Convert Fraction(nom,den) to Floats
	xf_string = str(xf_approx)
	j = xf_string.find('/')
	nom = float(xf_string[0:j])
	den = float(xf_string[j+1:])

	if (den != tmp):
		print("   = %15s = %1.15f  (%1.0e)" % (xf_approx, xfloat, diff))
		tmp = den