from __future__ import absolute_import, division, print_function, unicode_literals

import hashlib
import hmac
import math
import os
import re
import shlex
import signal
import socket
import subprocess
import sys
import tempfile
import threading
import time

from datetime import datetime
from calendar import timegm
from threading import Event

from flent import util
from flent.build_info import DATA_DIR
from flent.util import classname, ENCODING, Glob
from flent.loggers import get_logger

mswindows = (sys.platform == "win32")

try:
    from defusedxml.xmlrpc import monkey_patch
    monkey_patch()
    del monkey_patch
    XML_DEFUSED = True
except ImportError:
    XML_DEFUSED = False

try:
    # python 2
    import xmlrpclib as xmlrpc
except ImportError:
    import xmlrpc.client as xmlrpc

logger = get_logger(__name__)

if mswindows:
    def _handle_exitstatus(sts):
        raise NotImplementedError(
            "Subprocess management currently doesn't work on Windows")
else:
    # helper function from subprocess module
    def _handle_exitstatus(sts, _WIFSIGNALED=os.WIFSIGNALED,
                           _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED,
                           _WEXITSTATUS=os.WEXITSTATUS):
        # This method is called (indirectly) by __del__, so it cannot
        # refer to anything outside of its local scope."""
        if _WIFSIGNALED(sts):
            return -_WTERMSIG(sts)
        elif _WIFEXITED(sts):
            return _WEXITSTATUS(sts)
        else:
            # Should never happen
            raise RuntimeError("Unknown child exit status!")

def get(name):
    cname = classname(name, "Runner")
    if cname not in globals():
        raise RuntimeError("Runner not found: '%s'" % name)
    return globals()[cname]

class RunnerBase(object):

    transformed_metadata = []

    def __init__(self, name, settings, idx=None, start_event=None,
                 finish_event=None, kill_event=None, parent=None, **kwargs):
        super(RunnerBase, self).__init__()
        self.name = name
        self.settings = settings
        self.idx = idx
        self.test_parameters = {}
        self._raw_values = []
        self.command = None
        self.returncode = 0
        self.out = self.err = ''
        self._parent = parent
        self.result = None

        self.start_event = start_event
        self.kill_event = kill_event if kill_event is not None else Event()
        self.finish_event = finish_event

        self._child_runners = []

        self._pickled = False

        self.metadata = {'RUNNER': self.__class__.__name__, 'IDX': idx}

    def __getstate__(self):
        state = {}

        for k, v in self.__dict__.items():
            if k not in ('start_event', 'kill_event', 'finish_event',
                         'kill_lock', 'stdout', 'stderr') and \
                         not k.startswith("_"):
                state[k] = v

        state['_pickled'] = True

        return state

    # Emulate threading interface to fit into aggregator usage.
    def start(self):
        if self._pickled:
            raise RuntimeError("Attempt to run a pickled runner")

        for c in self._child_runners:
            c.start()

        s = super(RunnerBase, self)
        if hasattr(s, 'start'):
            s.start()

            logger.debug("Started %s idx %d ('%s')", self.__class__.__name__,
                         self.idx, self.name)

    def join(self, timeout=None):
        s = super(RunnerBase, self)
        if hasattr(s, 'join'):
            s.join(timeout)

        for c in self._child_runners:
            c.join(timeout)

    def is_alive(self):
        alive = []
        s = super(RunnerBase, self)
        if hasattr(s, "is_alive"):
            alive.append(s.is_alive())

        alive.extend([c.is_alive() for c in self._child_runners])
        return any(alive)

    def kill(self, graceful=False):
        for c in self._child_runners:
            c.kill(graceful)
        self.kill_event.set()

    def run(self):
        if self.start_event is not None:
            self.start_event.wait()
        self._run()
        self.finish_event.set()
        logger.debug("%s %s finished", self.__class__.__name__,
                     self.name, extra={'runner': self})

    def add_child(self, cls, *args):
        logger.debug("%s: Adding child %s", self.name, cls.__name__)
        c = cls(*args,
                name="%s :: child %d" % (self.name, len(self._child_runners)),
                settings=self.settings,
                idx=self.idx,
                start_event=self.start_event,
                finish_event=self.finish_event,
                kill_event=self.kill_event,
                parent=self)
        self._child_runners.append(c)

    @property
    def child_results(self):
        for c in self._child_runners:
            res = c.result
            if res and hasattr(res, 'items'):
                yield res

    def get_raw_values(self):
        if not self._child_runners:
            return self._raw_values

        vals = list(self._raw_values)
        for c in self._child_runners:
            vals.extend(c.raw_values)
        return sorted(vals, key=lambda v: v['t'])

    def set_raw_values(self, val):
        self._raw_values = val

    raw_values = property(get_raw_values, set_raw_values)

class TimerRunner(RunnerBase, threading.Thread):

    def __init__(self, timeout, **kwargs):
        super(TimerRunner, self).__init__(**kwargs)
        self.timeout = timeout
        self.command = 'Timeout after %f seconds' % self.timeout

    def _run(self):
        self.kill_event.wait(self.timeout)

class FileMonitorRunner(RunnerBase, threading.Thread):

    def __init__(self, filename, length, interval, delay, **kwargs):
        super(FileMonitorRunner, self).__init__(**kwargs)
        self.filename = filename
        self.length = length
        self.interval = interval
        self.delay = delay
        self.metadata['FILENAME'] = self.filename
        self.command = 'File monitor for %s' % self.filename

    def _run(self):
        if self.delay:
            self.kill_event.wait(self.delay)

        if not self.kill_event.is_set():
            start_time = current_time = time.time()

            result = []

            # Add an extra interval to comparison to avoid getting one too few
            # samples due to small time differences.
            while (current_time < start_time + self.length + self.interval and
                   not self.kill_event.is_set()):
                try:
                    with open(self.filename, 'r') as fp:
                        val = fp.read()
                    self.out += val
                    try:
                        val = float(val)
                        result.append((current_time, val))
                    except ValueError:
                        val = val.strip()
                    self._raw_values.append({'t': current_time, 'val': val})
                except IOError as e:
                    self.err += "Error opening file {}: {}\n".format(
                        self.filename, e)
                finally:
                    self.kill_event.wait(self.interval)
                    current_time = time.time()
            if result:
                self.result = result
            else:
                self.returncode = 1

class ProcessRunner(RunnerBase, threading.Thread):
    """Default process runner for any process."""
    silent = False
    supports_remote = True
    _env = {}

    def __init__(self, command, delay, remote_host, **kwargs):
        super(ProcessRunner, self).__init__(**kwargs)

        self.remote_host = remote_host
        self.delay = delay
        self.killed = False
        self.pid = None
        self.returncode = None
        self.kill_lock = threading.Lock()
        self.test_parameters = {}
        self.stdout = None
        self.stderr = None

        if 'units' in kwargs:
            self.metadata['UNITS'] = kwargs['units']

        if isinstance(command, dict):
            self.command = self.find_binary(**command)
            self.command_args = command
        else:
            self.command = command
            self.command_args = {}
        self.args = shlex.split(self.command)
        self.metadata['COMMAND'] = self.command

        # Rudimentary remote host capability. Note that this is modifying the
        # final command, so all the find_* stuff must match on the local and
        # remote hosts. I.e. the same binaries must exist in the same places.
        if remote_host:
            if not self.supports_remote:
                raise RuntimeError(
                    "%s (idx %d) does not support running on remote hosts." % (
                        self.__class__.__name__, self.idx))
            command = "ssh %s '%s'" % (remote_host, command)
            self.metadata['REMOTE_HOST'] = remote_host

    def handle_usr2(self, signal, frame):
        if self.start_event is not None:
            self.start_event.set()

    def fork(self):
        # Use named temporary files to avoid errors on double-delete when
        # running on Windows/cygwin.
        try:
            self.stdout = tempfile.NamedTemporaryFile(
                prefix="flent-", delete=False)
            self.stderr = tempfile.NamedTemporaryFile(
                prefix="flent-", delete=False)
        except OSError as e:
            if e.errno == 24:
                raise RuntimeError(
                    "Unable to create temporary files because too many "
                    "files are open. Try increasing ulimit.")
            else:
                raise RuntimeError("Unable to create temporary files: %s" % e)

        pid = os.fork()

        if pid == 0:
            os.dup2(self.stdout.fileno(), 1)
            os.dup2(self.stderr.fileno(), 2)
            self.stdout.close()
            self.stderr.close()
            signal.signal(signal.SIGTERM, signal.SIG_DFL)

            try:
                if self.start_event is not None:
                    signal.signal(signal.SIGUSR2, self.handle_usr2)
                    self.start_event.wait()
                    signal.signal(signal.SIGUSR2, signal.SIG_DFL)

                time.sleep(self.delay)
            except:
                os._exit(0)

            env = dict(os.environ)
            env.update(self._env)
            prog = self.args[0]
            os.execvpe(prog, self.args, env)
        else:
            logger.debug("Forked %s as pid %d", self.args[0], pid)
            self.pid = pid

    def kill(self, graceful=False):
        super(ProcessRunner, self).kill(graceful)
        if graceful:
            # Graceful shutdown is done on a best-effort basis, and may results
            # in some errors from the test tools. We don't print these, since
            # they are expected.
            self.silent = True
        else:
            with self.kill_lock:
                self.killed = True
            self.cleanup_tmpfiles()
        if self.pid is not None:
            sig = signal.SIGINT if graceful else signal.SIGTERM
            logger.debug("Sending signal %d to pid %d.", sig, self.pid)
            try:
                os.kill(self.pid, sig)
            except OSError as e:
                pass

    def cleanup_tmpfiles(self):
        for f in self.stdout, self.stderr:
            if f is not None:
                try:
                    f.close()
                except OSError:
                    pass
                try:
                    os.unlink(f.name)
                except OSError:
                    pass

    def is_killed(self):
        with self.kill_lock:
            return self.killed

    def start(self):
        if mswindows:
            raise RuntimeError(
                "Process management currently doesn't work on Windows, "
                "so running tests is not possible.")
        self.fork()
        super(ProcessRunner, self).start()

    def run(self):
        """Runs the configured job. If a delay is set, wait for that many
        seconds, then open the subprocess, wait for it to finish, and collect
        the last word of the output (whitespace-separated)."""

        if self.start_event is not None:
            self.start_event.wait()
            try:
                os.kill(self.pid, signal.SIGUSR2)
            except OSError:
                pass

        if self.kill_event is None:
            pid, sts = os.waitpid(self.pid, 0)
        else:
            pid, sts = os.waitpid(self.pid, os.WNOHANG)
            while (pid, sts) == (0, 0):
                self.kill_event.wait(1)
                if self.kill_event.is_set():
                    try:
                        os.kill(self.pid, signal.SIGTERM)
                    except OSError:
                        pass
                pid, sts = os.waitpid(self.pid, os.WNOHANG)

        self.finish_event.set()

        self.returncode = _handle_exitstatus(sts)

        # Even with locking, kill detection is not reliable; sleeping seems to
        # help. *sigh* -- threading.
        time.sleep(0.2)

        if self.is_killed():
            return

        self.stdout.seek(0)
        self.out += self.stdout.read().decode(ENCODING)
        try:
            # Close and remove the temporary file. This might fail, but we're
            # going to assume that is okay.
            filename = self.stdout.name
            self.stdout.close()
            os.unlink(filename)
        except OSError:
            pass

        self.stderr.seek(0)
        self.err += self.stderr.read().decode(ENCODING)
        try:
            filename = self.stderr.name
            self.stderr.close()
            os.unlink(filename)
        except OSError:
            pass

        if self.returncode and not self.silent:
            logger.warning("Program exited non-zero.",
                           extra={'runner': self})

        self.result = self.parse(self.out)
        if not self.result and not self.silent:
            logger.warning("Command produced no valid data.",
                           extra={'runner': self})

        logger.debug("%s %s finished", self.__class__.__name__,
                     self.name, extra={'runner': self})

    def parse(self, output):
        """Default parser returns the last (whitespace-separated) word of
        output as a float."""

        return float(output.split()[-1].strip())

DefaultRunner = ProcessRunner

class SilentProcessRunner(ProcessRunner):
    silent = True

    def parse(self, output):
        return None

class DitgRunner(ProcessRunner):
    """Runner for D-ITG with a control server."""
    supports_remote = False

    def __init__(self, duration, interval, **kwargs):
        super(DitgRunner, self).__init__(**kwargs)
        if 'control_host' in kwargs:
            control_host = kwargs['control_host']
        else:
            control_host = self.settings.CONTROL_HOST or self.settings.HOST
        self.proxy = xmlrpc.ServerProxy("http://%s:%s"
                                        % (control_host,
                                           self.settings.DITG_CONTROL_PORT),
                                        allow_none=True)
        self.ditg_secret = self.settings.DITG_CONTROL_SECRET
        self.duration = duration
        self.interval = interval

    def start(self):
        try:
            interval = int(self.interval * 1000)
            hm = hmac.new(self.ditg_secret.encode(
                'UTF-8'), digestmod=hashlib.sha256)
            hm.update(str(self.duration).encode('UTF-8'))
            hm.update(str(interval).encode('UTF-8'))
            params = self.proxy.request_new_test(
                self.duration, interval, hm.hexdigest(), True)
            if params['status'] != 'OK':
                if 'message' in params:
                    raise RuntimeError(
                        "Unable to request D-ITG test. "
                        "Control server reported error: %s" % params['message'])
                else:
                    raise RuntimeError(
                        "Unable to request D-ITG test. "
                        "Control server reported an unspecified error.")
            self.test_id = params['test_id']
            self.out += "Test ID: %s\n" % self.test_id
        except (xmlrpc.Fault, socket.error) as e:
            raise RuntimeError(
                "Error while requesting D-ITG test: '%s'. "
                "Is the control server listening (see man page)?" % e)
        self.command = self.command.format(
            signal_port=params['port'], dest_port=params['port'] + 1)
        self.args = shlex.split(self.command)

        super(DitgRunner, self).start()

    def parse(self, output):
        data = ""
        utc_offset = 0
        results = {}
        try:
            # The control server has a grace period after the test ends, so we
            # don't know exactly when the test results are going to be ready. We
            # assume that it will be within ten seconds.
            for i in range(10):
                res = self.proxy.get_test_results(self.test_id)
                if res['status'] == 'OK':
                    data = res['data']
                    self.out += data
                    utc_offset = res['utc_offset']
                    break
                time.sleep(1)
            if res['status'] != 'OK':
                if 'message' in res:
                    self.err += "Error while getting results. " \
                                "Control server reported error: %s.\n" \
                                % res['message']
                else:
                    self.err += "Error while getting results. " \
                                "Control server reported unknown error.\n"
        except xmlrpc.Fault as e:
            self.err += "Error while getting results: %s.\n" % e

        # D-ITG *should* output about 50 bytes of data per data point. However,
        # sometimes it runs amok and outputs megabytes of erroneous data. So, if
        # the length of the data is more than ten times the expected value,
        # abort rather than try to process the data.
        if len(data) > (self.duration / self.interval) * 500:
            self.err += "D-ITG output too much data (%d bytes).\n" % len(data)
            return results

        if 'raw' in res:
            self.parse_raw(res['raw'])

        for line in data.splitlines():
            if not line.strip():
                continue
            parts = [float(i) for i in line.split()]
            timestamp = parts.pop(0) + utc_offset
            for i, n in enumerate(('bitrate', 'delay', 'jitter', 'loss')):
                if n not in results:
                    results[n] = []
                results[n].append([timestamp, parts[i]])

        return results

    def parse_raw(self, data):
        raw_values = []

        for line in data.splitlines():
            parts = re.split(r">?\s*", line)
            vals = dict(zip(parts[::2], parts[1::2]))
            times = {}
            for v in ('txTime', 'rxTime'):
                t, microsec = vals[v].split(".")
                h, m, s = t.split(":")
                # FIXME: This is definitely going to break if a test is run
                # around midnight
                dt = datetime.utcnow().replace(hour=int(h),
                                               minute=int(m),
                                               second=int(s),
                                               microsecond=int(microsec))
                times[v] = float(timegm(dt.timetuple())) + dt.microsecond / 10**6

            raw_values.append({
                't': times['rxTime'],
                'val': 1000.0 * (times['rxTime'] - times['txTime']),
                'seq': int(vals['Seq']),
                'size': int(vals['Size'])
            })

        self.raw_values = raw_values

class NetperfDemoRunner(ProcessRunner):
    """Runner for netperf demo mode."""
    transformed_metadata = ('MEAN_VALUE',)
    netperf = {}

    def parse(self, output):
        """Parses the interim result lines and returns a list of (time,value)
        pairs."""

        result = []
        raw_values = []
        lines = output.strip().splitlines()
        avg_dur = None
        alpha = 0.5
        for line in lines:
            if line.startswith("Interim"):
                parts = line.split()

                # Calculate an EWMA of the netperf sampling duration and exclude
                # data points from a sampling period that is more than an order
                # of magnitude higher or lower than this average; these are
                # probably the result of netperf spitting out a measurement at
                # the end of a run after having lost the measurement flow during
                # the run, or a very short interval giving a very high bandwidth
                # measurement
                dur = float(parts[5])
                time = float(parts[9])
                value = float(parts[2])
                if avg_dur is None:
                    avg_dur = dur

                raw_values.append({'dur': dur, 't': time, 'val': value})

                if dur < avg_dur * 10.0 and dur > avg_dur / 10.0:
                    result.append([time, value])
                    avg_dur = alpha * avg_dur + (1.0 - alpha) * dur
        self.raw_values = raw_values
        try:
            self.metadata['MEAN_VALUE'] = float(lines[-1])
        except (ValueError, IndexError):
            pass

        return result

    def find_binary(self, **args):

        if not self.netperf:
            netperf = util.which('netperf', fail=True)

            # Try to figure out whether this version of netperf supports the -e
            # option for socket timeout on UDP_RR tests, and whether it has been
            # compiled with --enable-demo. Unfortunately, the --help message is
            # not very helpful for this, so the only way to find out is try to
            # invoke it and check for an error message. This has the side-effect
            # of having netperf attempt a connection to localhost, which can
            # stall, so we kill the process almost immediately.

            proc = subprocess.Popen([netperf, '-l', '1', '-D', '-0.2',
                                     '--', '-e', '1'],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            # should be enough time for netperf to output any error messages
            time.sleep(0.1)
            proc.kill()
            out, err = proc.communicate()
            if "Demo Mode not configured" in str(out):
                raise RuntimeError("%s does not support demo mode." % netperf)

            if "invalid option -- '0'" in str(err):
                raise RuntimeError(
                    "%s does not support accurate intermediate time reporting. "
                    "You need netperf v2.6.0 or newer." % netperf)

            self.netperf['executable'] = netperf
            self.netperf['-e'] = False

            if "netperf: invalid option -- 'e'" not in str(err):
                self.netperf['-e'] = True

        args['binary'] = self.netperf['executable']

        if args['marking']:
            args['marking'] = "-Y {0}".format(args['marking'])

        for c in 'local_bind', 'control_local_bind':
            if args[c]:
                args[c] = "-L {0}".format(args[c])

        if args['test'] == "UDP_RR" and self.netperf["-e"]:
            args['socket_timeout'] = "-e {0:d}".format(args['socket_timeout'])
        else:
            args['socket_timeout'] = ""

        if args['test'] in ("TCP_STREAM", "TCP_MAERTS", "omni"):
            args['format'] = "-f m"

            if args['test'] == 'TCP_STREAM' and self.settings.SOCKET_STATS:
                ss_args = {'host': self.remote_host or 'localhost',
                           'interval': args['interval'],
                           'length': args['length'],
                           'target': args['host'],
                           'ip_version': args['ip_version']}

                self.add_child(SsRunner, [args['control_port']],
                               ss_args, self.delay, None)

        return "{binary} -P 0 -v 0 -D -{interval:.2f} -{ip_version} {marking} " \
            "-H {control_host} -p {control_port} -t {test} -l {length:d} " \
            "{format} {control_local_bind} {extra_args} -- {socket_timeout} " \
            "{local_bind} -H {host} {extra_test_args}".format(**args)

class RegexpRunner(ProcessRunner):

    """Runner that matches each line to one or more regular expressions,
    returning the values from the first matched.

    The regular expressions must define symbolic groups 'time' and 'value'."""

    regexes = []
    metadata_regexes = []

    # Parse is split into a stateless class method in _parse to be able to call
    # it from find_binary.
    def parse(self, output):
        result, raw_values, metadata = self._parse(output)
        self.raw_values = raw_values
        self.metadata.update(metadata)
        return result

    @classmethod
    def _parse(cls, output):
        result = []
        raw_values = []
        metadata = {}
        lines = output.split("\n")
        for line in lines:
            for regexp in cls.regexes:
                match = regexp.match(line)
                if match:
                    result.append([float(match.group('t')),
                                   float(match.group('val'))])
                    rw = match.groupdict()
                    for k, v in rw.items():
                        try:
                            rw[k] = float(v)
                        except ValueError:
                            pass
                    raw_values.append(rw)
                    break  # only match one regexp per line
            for regexp in cls.metadata_regexes:
                match = regexp.match(line)
                if match:
                    for k, v in match.groupdict().items():
                        try:
                            metadata[k] = float(v)
                        except ValueError:
                            metadata[k] = v
        return result, raw_values, metadata

class PingRunner(RegexpRunner):
    """Runner for ping/ping6 in timestamped (-D) mode."""

    # Ping will change the comma separator for command line arguments based on
    # locale, which will break sub-second intervals. Override locale settings to
    # avoid this breaking stuff.
    _env = {"LC_ALL": "C", "LANG": "C"}

    # For some reason some versions of ping output icmp_req and others icmp_seq
    # for sequence numbers.
    regexes = [re.compile(r'^\[(?P<t>[0-9]+\.[0-9]+)\]'
                          r'(?:.*icmp_.eq=(?P<seq>[0-9]+))?'
                          r'.*time=(?P<val>[0-9]+(?:\.[0-9]+)?) ms$'),
               re.compile(r'^\[(?P<t>[0-9]+\.[0-9]+)\].*:'
                          r'(?: \[(?P<seq>[0-9]+)\])?.*, '
                          r'(?P<val>[0-9]+(?:\.[0-9]+)?) ms \(.*\)$')]
    metadata_regexes = [re.compile(r'^.*min/avg/max(?:/mdev)? = '
                                   r'(?P<MIN_VALUE>[0-9]+(?:\.[0-9]+)?)/'
                                   r'(?P<MEAN_VALUE>[0-9]+(?:\.[0-9]+)?)/'
                                   r'(?P<MAX_VALUE>[0-9]+(?:\.[0-9]+)?).*$')]
    transformed_metadata = ('MEAN_VALUE', 'MIN_VALUE', 'MAX_VALUE')

    @classmethod
    def find_binary(cls, ip_version, interval, length, host,
                    marking=None, local_bind=None):
        """Find a suitable ping executable, looking first for a compatible
        `fping`, then falling back to the `ping` binary. Binaries are checked
        for the required capabilities."""

        if ip_version == 6:
            suffix = "6"
        else:
            suffix = ""

        fping = util.which('fping' + suffix)
        ping = util.which('ping' + suffix)
        pingargs = []

        if fping is not None:
            proc = subprocess.Popen([fping, '-h'],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            out, err = proc.communicate()
            # check for presence of timestamp option
            if "print timestamp before each output line" in str(out):
                return "{binary} -D -p {interval:.0f} -c {count:.0f} " \
                    "{marking} {local_bind} {host}".format(
                        binary=fping,
                        interval=interval * 1000,  # fping expects interval in ms
                        # since there's no timeout parameter for fping,
                        # calculate a total number of pings to send
                        count=length // interval + 1,
                        marking="-O {0}".format(marking) if marking else "",
                        local_bind=("-I {0}".format(local_bind)
                                    if local_bind else ""),
                        host=host)
            elif "must run as root?" in str(err):
                logger.warning(
                    "Found fping but it seems to be missing "
                    "permissions (no SUID?). Not using.")

        if ping is None and ip_version == 6:
            # See if we have a combined ping binary (new versions of iputils)
            ping6 = util.which("ping")
            proc = subprocess.Popen([ping6, '-h'],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            out, err = proc.communicate()
            if hasattr(err, 'decode'):
                err = err.decode(ENCODING)
            if '-6' in err:
                ping = ping6
                pingargs = ['-6']

        if ping is not None:
            # Ping can't handle hostnames for the -I parameter, so do a lookup
            # first.
            if local_bind:
                local_bind = util.lookup_host(local_bind, ip_version)[4][0]

            # Try parsing the output of 'ping' and complain if no data is
            # returned from the parser. This should pick up e.g. the ping on OSX
            # not having a -D option and allow us to supply a better error
            # message.
            proc = subprocess.Popen([ping, '-D', '-n',
                                     '-c', '1', 'localhost'] + pingargs,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            out, err = proc.communicate()
            if hasattr(out, 'decode'):
                out = out.decode(ENCODING)
            if not cls._parse(out)[0]:
                raise RuntimeError(
                    "Cannot parse output of the system ping binary ({ping}). "
                    "Please install fping v3.5+.".format(ping=ping))

            return "{binary} -n -D -i {interval:.2f} -w {length:d} {marking} " \
                "{local_bind} {pingargs} {host}".format(
                    binary=ping,
                    interval=max(0.2, interval),
                    length=length,
                    marking="-Q {0}".format(marking) if marking else "",
                    local_bind="-I {0}".format(local_bind) if local_bind else "",
                    host=host,
                    pingargs=" ".join(pingargs))

        raise RuntimeError("No suitable ping tool found.")

class HttpGetterRunner(RegexpRunner):

    regexes = [re.compile(
        r'^\[(?P<t>[0-9]+\.[0-9]+)\].*in (?P<val>[0-9]+(?:\.[0-9]+)?) seconds.$')]
    metadata_regexes = [re.compile(r'^.*min/avg/max(?:/mdev)? = '
                                   r'(?P<MIN_VALUE>[0-9]+(?:\.[0-9]+)?)/'
                                   r'(?P<MEAN_VALUE>[0-9]+(?:\.[0-9]+)?)/'
                                   r'(?P<MAX_VALUE>[0-9]+(?:\.[0-9]+)?).*$')]
    transformed_metadata = ('MEAN_VALUE', 'MIN_VALUE', 'MAX_VALUE')

class IperfCsvRunner(ProcessRunner):
    """Runner for iperf csv output (-y C), possibly with unix timestamp patch."""

    transformed_metadata = ('MEAN_VALUE',)

    def __init__(self, **kwargs):
        if 'udp' in kwargs:
            self.udp = kwargs['udp']
            del kwargs['udp']
        else:
            self.udp = False
        super(IperfCsvRunner, self).__init__(**kwargs)

    def parse(self, output):
        result = []
        raw_values = []
        lines = output.strip().split("\n")
        dest = None
        for line in lines[:-1]:  # The last line is an average over the whole test
            parts = line.split(",")
            if len(parts) < 9:
                continue

            timestamp = parts[0]
            bandwidth = parts[8]

            if dest is None:
                dest = parts[3]

            # Newer versions of iperf2 emits sub-second timestamps if given the
            # --enhancedreports argument. Since we detect this in find_iperf, we
            # only support this format.

            try:
                sec, mil = timestamp.split(".")
                dt = datetime.strptime(sec, "%Y%m%d%H%M%S")
                timestamp = time.mktime(dt.timetuple()) + float(mil) / 1000
                val = float(bandwidth)
                result.append([timestamp, val])
                raw_values.append({'t': timestamp, 'val': val})
            except ValueError:
                pass

        self.raw_values = raw_values
        try:
            parts = lines[-1].split(",")
            # src and dest should be reversed if this was a reply from the
            # server. Track this for UDP where it may be missing.
            if parts[1] == dest or not self.udp:
                self.metadata['MEAN_VALUE'] = float(parts[8])
            else:
                self.metadata['MEAN_VALUE'] = None
        except (ValueError, IndexError):
            pass
        return result

    @classmethod
    def find_binary(cls, host, interval, length, ip_version, local_bind=None,
                    no_delay=False, udp=False, bw=None):
        iperf = util.which('iperf')

        if iperf is not None:
            proc = subprocess.Popen([iperf, '-h'],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            out, err = proc.communicate()

            if "--enhancedreports" in str(err):
                if udp:
                    udp_args = "--udp --bandwidth {}".format(bw if bw else "100M")
                else:
                    udp_args = ""
                return "{binary} --enhancedreports --reportstyle C --format m " \
                    "--client {host} --time {length} --interval {interval} " \
                    "{local_bind} {no_delay} {udp} {ip6}".format(
                        host=host,
                        binary=iperf,
                        length=length,
                        interval=interval,
                        # --help output is wrong
                        ip6="--ipv6_domain" if ip_version == 6 else "",
                        local_bind="--bind {0}".format(
                            local_bind) if local_bind else "",
                        no_delay="--nodelay" if no_delay else "",
                        udp=udp_args)
            else:
                logger.warning(
                    "Found iperf binary, but it does not have "
                    "an --enhancedreport option. Not using.")

        raise RuntimeError("No suitable Iperf binary found.")

class ParseException(Exception):
    pass

class SsRunner(ProcessRunner):
    """Runner for iterated `ss -t -i -p`. Depends on same partitial output
    separationa and time stamping as TcRunner."""

    # Keep track of runners to avoid duplicates (relies on this being a class
    # variable, and so the same dictionary instance across all instances of the
    # class).
    _duplicate_map = {}

    ip_v4_addr_sub_re = "([0-9]{1,3}.){3}[0-9]{1,3}(:\d+)"
    # ref.: to commented, untinkered version: ISBN 978-0-596-52068-7
    ip_v6_addr_sub_re = "(?:(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}|" \
                        "(?=(?:[A-F0-9]{0,4}:){0,7}[A-F0-9]{0,4})" \
                        "(([0-9A-F]{1,4}:){1,7}|:)((:[0-9A-F]{1,4})" \
                        "{1,7}|:))(:\d+)"

    time_re = re.compile(r"^Time: (?P<timestamp>\d+\.\d+)", re.MULTILINE)
    cwnd_re = re.compile(r"cwnd:(?P<cwnd>\d+)", re.MULTILINE)
    rtt_re = re.compile(r"rtt:(?P<rtt>\d+\.\d+/\d+\.\d+)", re.MULTILINE)
    pid_re = re.compile(r"pid=(?P<pid>\d+)", re.MULTILINE)
    ports_ipv4_re = re.compile(r"" + "(?P<src_p>" + ip_v4_addr_sub_re + ")" +
                               "\s+" + "(?P<dst_p>" + ip_v4_addr_sub_re + ")")
    ports_ipv6_re = re.compile(r"" + "(?P<src_p>" + ip_v6_addr_sub_re + ")" +
                               "\s+" + "(?P<dst_p>" + ip_v6_addr_sub_re + ")",
                               re.IGNORECASE)
    ss_header_re = re.compile(r"" + "State\s+Recv-Q\s+Send-Q\s+Local")

    src_p = []
    dst_p = []
    # ref.: upstream ss
    ss_states = ["UNKNOWN", "ESTAB", "SYN-SENT", "SYN-RECV", "FIN-WAIT-1",
                 "FIN-WAIT-2", "TIME-WAIT", "UNCONN", "CLOSE-WAIT", "LAST-ACK",
                 "LISTEN", "CLOSING"]
    ss_states_re = re.compile(r"|".join(ss_states))

    def __init__(self, exclude_ports, command, *args, **kwargs):
        self.exclude_ports = exclude_ports
        super(SsRunner, self).__init__(command, *args, **kwargs)

        dup_key = (command['host'], command['interval'], command['length'],
                   command['target'], command['ip_version'],
                   tuple(self.exclude_ports))

        if dup_key in self._duplicate_map:
            logger.debug("Found duplicate SsRunner, reusing output")
            self._dup_runner = self._duplicate_map[dup_key]
            self.command = "%s (duplicate)" % self.command
        else:
            self._dup_runner = None
            self._duplicate_map[dup_key] = self
            self._dup_key = dup_key

    def fork(self):
        if self._dup_runner is None:
            super(SsRunner, self).fork()

    def run(self):
        if self._dup_runner is None:
            super(SsRunner, self).run()
            del self._duplicate_map[self._dup_key]
            return

        self._dup_runner.join()
        logger.debug("%s %s finished", self.__class__.__name__,
                     self.name, extra={'runner': self})

        self.out = self._dup_runner.out
        self.err = self._dup_runner.err

        self.result = self.parse(self.out)
        if not self.result and not self.silent:
            logger.warning("Command produced no valid data.",
                           extra={'runner': self})

    def filter_np_parent(self, part):
        sub_part = []
        sub_parts = self.ss_states_re.split(part)
        sub_parts = [sp for sp in sub_parts if sp.strip()
                     and not self.ss_header_re.search(sp)]

        for sp in sub_parts:
            pid = self.pid_re.search(sp)
            if None is pid:
                continue
            pid_str = pid.group('pid')

            f_ports = self.ports_ipv4_re.search(sp)
            if None is f_ports:
                f_ports = self.ports_ipv6_re.search(sp)

            if None is f_ports:
                raise ParseException()

            dst_p = int(f_ports.group('dst_p').split(":")[-1])

            if self.par_pid == pid_str and dst_p not in self.exclude_ports:
                sub_part.append(sp)

        if 1 != len(sub_part):
            raise ParseException()

        return sub_part[0]

    def parse_part(self, part):
        sub_part = self.filter_np_parent(part)

        timestamp = self.time_re.search(part)

        cwnd = self.cwnd_re.search(sub_part)
        rtt = self.rtt_re.search(sub_part)

        if None in [timestamp, cwnd, rtt]:
            raise ParseException()

        timestamp = float(timestamp.group('timestamp'))
        cwnd = int(cwnd.group('cwnd'))
        rtt, rtt_var = [float(x) for x in rtt.group('rtt').split('/')]

        raw_values = {'t': timestamp, 'tcp_cwnd': cwnd,
                      'tcp_rtt': rtt, 'tcp_rtt_var': rtt_var}
        self._raw_values.append(raw_values)

        return raw_values

    def parse(self, output):
        self.par_pid = str(self._parent.pid)
        results = {}
        parts = output.split("\n---\n")
        for part in parts:
            try:
                res_dict = self.parse_part(part)
                t = res_dict['t']
                for k, v in res_dict.items():
                    if k == 't':
                        continue
                    if k not in results:
                        results[k] = [[t, v]]
                    else:
                        results[k].append([t, v])

            except ParseException:
                continue

        return results

    def find_binary(self, ip_version, host, interval, length, target):
        script = os.path.join(DATA_DIR, 'scripts', 'ss_iterate.sh')
        if not os.path.exists(script):
            raise RuntimeError("Cannot find ss_iterate.sh.")

        bash = util.which('bash')
        if not bash:
            raise RuntimeError("Socket stats requires a Bash shell.")

        resol_target = util.lookup_host(target, ip_version)[4][0]
        if ip_version == 6:
            resol_target = "[" + str(resol_target) + "]"

        filt = ""
        for p in self.exclude_ports:
            filt = "{} and dport != {}".format(filt, p)

        return "{bash} {script} -I {interval:.2f} " \
            "-c {count:.0f} -H {host} -t '{target}' -f '{filt}'".format(
                bash=bash,
                script=script,
                interval=interval,
                count=length // interval + 1,
                host=host,
                target=resol_target,
                filt=filt)

class TcRunner(ProcessRunner):
    """Runner for iterated `tc -s qdisc`. Expects iterations to be separated by
    '\n---\n and a timestamp to be present in the form 'Time: xxxxxx.xxx' (e.g.
    the output of `date '+Time: %s.%N'`)."""

    time_re = re.compile(r"^Time: (?P<timestamp>\d+\.\d+)", re.MULTILINE)
    split_re = re.compile(r"^qdisc ", re.MULTILINE)
    qdisc_res = [
        re.compile(r"Sent (?P<sent_bytes>\d+) bytes (?P<sent_pkts>\d+) pkt "
                   r"\(dropped (?P<dropped>\d+), "
                   r"overlimits (?P<overlimits>\d+) "
                   r"requeues (?P<requeues>\d+)\)"),
        re.compile(r"backlog (?P<backlog_bytes>\d+)b "
                   r"(?P<backlog_pkts>\d+)p "
                   r"requeues (?P<backlog_requeues>\d+)"),

        # codel
        re.compile(r"count (?P<count>\d+) "
                   r"lastcount (?P<lastcount>\d+) "
                   r"ldelay (?P<delay>[0-9\.]+[mu]?s) "
                   r"(?P<dropping>dropping)? ?"
                   r"drop_next (?P<drop_next>-?[0-9\.]+[mu]?s)"),
        re.compile(r"maxpacket (?P<maxpacket>\d+) "
                   r"ecn_mark (?P<ecn_mark>\d+) "
                   r"drop_overlimit (?P<drop_overlimit>\d+)"),
        # fq_codel
        re.compile(r"maxpacket (?P<maxpacket>\d+) "
                   r"drop_overlimit (?P<drop_overlimit>\d+) "
                   r"new_flow_count (?P<new_flow_count>\d+) "
                   r"ecn_mark (?P<ecn_mark>\d+)"),
        re.compile(r"new_flows_len (?P<new_flows_len>\d+) "
                   r"old_flows_len (?P<old_flows_len>\d+)"),

        # pie
        re.compile(r"prob (?P<prob>[0-9\.]+) "
                   r"delay (?P<delay>[0-9\.]+[mu]?s) "
                   r"avg_dq_rate (?P<avg_dq_rate>\d+)"),
        re.compile(r"pkts_in (?P<pkts_in>\d+) "
                   r"overlimit (?P<overlimit_pie>\d+) "
                   r"dropped (?P<dropped_pie>\d+) "
                   r"maxq (?P<maxq>\d+) "
                   r"ecn_mark (?P<ecn_mark>\d+)"),
    ]

    cake_tin_re = r"(Tin \d|Bulk|Best Effort|Video|Voice)"
    cake_alltins_re = re.compile(r"(\s*" + cake_tin_re + r")+")
    cake_1tin_re = re.compile(cake_tin_re)
    cake_keys = ["av_delay", "sp_delay", "pkts", "bytes",
                 "drops", "marks", "sp_flows", "bk_flows", "max_len"]

    # Normalise time values (seconds, ms, us) to milliseconds and bit values
    # (bit, Kbit, Mbit) to bits
    def parse_val(self, v):
        if v is None:
            return None
        elif v.endswith("us"):
            return float(v[:-2]) / 1000
        elif v.endswith("ms"):
            return float(v[:-2])
        elif v.endswith("s"):
            return float(v[:-1]) * 1000
        elif v.endswith("Kbit"):
            return float(v[:-4]) * 1000
        elif v.endswith("Mbit"):
            return float(v[:-4]) * 10**6
        elif v.endswith("bit"):
            return float(v[:-3])
        else:
            try:
                return float(v)
            except ValueError:
                return v

    def parse(self, output):
        results = {}
        parts = output.split("\n---\n")
        for part in parts:
            # Split out individual qdisc entries (in case there are more than
            # one). If so, discard the root qdisc and sum the rest.
            qdiscs = [i for i in self.split_re.split(part)
                      if not i.startswith("ingress")]
            if len(qdiscs) > 2:
                part = "qdisc ".join([i for i in qdiscs
                                      if 'root' not in i])

            matches = {}
            timestamp = self.time_re.search(part)
            if timestamp is None:
                continue
            timestamp = float(timestamp.group('timestamp'))

            for r in self.qdisc_res:
                m = r.search(part)
                # Keep searching from the end of the last match until no more
                # matches are found; this should find all qdisc outputs in case
                # there are several qdiscs installed on the interface. The
                # values for the qdiscs are summed for the result (discarding
                # what should be the root qdisc as per above).
                while m is not None:
                    for k, v in list(m.groupdict().items()):
                        v = self.parse_val(v)
                        if k not in matches or not isinstance(v, float):
                            matches[k] = v
                        else:
                            matches[k] += v
                    m = r.search(part, m.end(0))

            m = self.cake_alltins_re.search(part)
            if m:
                tins = self.cake_1tin_re.findall(m.group(0))
                start = m.end()
                for key in self.cake_keys:
                    m = re.search(
                        "^  %s(:?\s*([0-9\.kmbitus]+)){%d}\s*$" % (key,
                                                                   len(tins)),
                        part[start:],
                        re.IGNORECASE | re.MULTILINE)
                    if m:
                        k = "cake_%s" % key
                        matches[k] = dict(
                            zip(tins, map(self.parse_val,
                                          m.group(0).split()[1:])))

            # The cake stats, being multi-dimensional, is not actually plottable
            # yet. For now, add in an 'ecn_marks' key that is the sum of all
            # cake tins, for comparability with other qdiscs
            if "cake_marks" in matches and "ecn_mark" not in matches:
                matches['ecn_mark'] = sum(matches['cake_marks'].values())

            for k, v in matches.items():
                if not isinstance(v, float):
                    continue
                if k not in results:
                    results[k] = [[timestamp, v]]
                else:
                    results[k].append([timestamp, v])
            matches['t'] = timestamp
            self._raw_values.append(matches)
        return results

    @classmethod
    def find_binary(cls, interface, interval, length, host='localhost'):
        script = os.path.join(DATA_DIR, 'scripts', 'tc_iterate.sh')
        if not os.path.exists(script):
            raise RuntimeError("Cannot find tc_iterate.sh.")

        bash = util.which('bash')
        if not bash:
            raise RuntimeError("TC stats requires a Bash shell.")

        if interface is None:
            logger.warning(
                "Warning: No interface given for tc runner. "
                "Defaulting to 'eth0'.")
            interface = 'eth0'

        return "{bash} {script} -i {interface} -I {interval:.2f} " \
            "-c {count:.0f} -H {host}".format(
                bash=bash,
                script=script,
                interface=interface,
                interval=interval,
                count=length // interval + 1,
                host=host)

class CpuStatsRunner(ProcessRunner):
    """Runner for getting CPU usage stats from /proc/stat. Expects iterations to be
    separated by '\n---\n and a timestamp to be present in the form 'Time:
    xxxxxx.xxx' (e.g. the output of `date '+Time: %s.%N'`).

    """
    time_re = re.compile(r"^Time: (?P<timestamp>\d+\.\d+)", re.MULTILINE)
    value_re = re.compile(r"^\d+ \d+ (?P<load>\d+\.\d+)$", re.MULTILINE)

    def parse(self, output):
        results = {}
        parts = output.split("\n---\n")
        for part in parts:
            # Split out individual qdisc entries (in case there are more than
            # one). If so, discard the root qdisc and sum the rest.
            timestamp = self.time_re.search(part)
            if timestamp is None:
                continue
            timestamp = float(timestamp.group('timestamp'))
            value = self.value_re.search(part)

            if value is None:
                continue
            matches = {}

            for k, v in list(value.groupdict().items()):
                v = float(v)
                if k not in matches:
                    matches[k] = v
                else:
                    matches[k] += v

            for k, v in matches.items():
                if not isinstance(v, float):
                    continue
                if k not in results:
                    results[k] = [[timestamp, v]]
                else:
                    results[k].append([timestamp, v])
            matches['t'] = timestamp
            self._raw_values.append(matches)
        return results

    @classmethod
    def find_binary(cls, interval, length, host='localhost'):
        script = os.path.join(DATA_DIR, 'scripts', 'stat_iterate.sh')
        if not os.path.exists(script):
            raise RuntimeError("Cannot find stat_iterate.sh.")

        bash = util.which('bash')
        if not bash:
            raise RuntimeError("CPU stats requires a Bash shell.")

        return "{bash} {script} -I {interval:.2f} " \
            "-c {count:.0f} -H {host}".format(
                bash=bash,
                script=script,
                interval=interval,
                count=length // interval + 1,
                host=host)

class WifiStatsRunner(ProcessRunner):
    """Runner for getting WiFi debug stats from /sys/kernel/debug. Expects
    iterations to be separated by '\n---\n and a timestamp to be present in the
    form 'Time: xxxxxx.xxx' (e.g. the output of `date '+Time: %s.%N'`).

    """
    time_re = re.compile(r"^Time: (?P<timestamp>\d+\.\d+)", re.MULTILINE)
    station_re = re.compile(
        r"^Station: (?P<mac>(?:[0-9a-f]{2}:){5}[0-9a-f]{2})\n", re.MULTILINE)
    airtime_re = re.compile(
        r"^Airtime:\nRX: (?P<rx>\d+) us\nTX: (?P<tx>\d+) us", re.MULTILINE)

    def __init__(self, **kwargs):
        if 'stations' in kwargs:
            if kwargs['stations'] in (["all"], ["ALL"]):
                self.stations = []
                # disabled as it doesn't work properly yet
                self.all_stations = False
            else:
                self.stations = kwargs['stations']
                self.all_stations = False
            del kwargs['stations']
        else:
            self.stations = []

        super(WifiStatsRunner, self).__init__(**kwargs)

    def parse(self, output):
        results = {}
        parts = output.split("\n---\n")
        for part in parts:
            matches = {}
            timestamp = self.time_re.search(part)
            if timestamp is None:
                continue
            timestamp = float(timestamp.group('timestamp'))

            # Split by station regex: First entry is everything before the
            # per-station stats, the rest is a pair of (<station mac>, contents).
            station_parts = self.station_re.split(part)[1:]
            stations = {}

            for s, v in zip(station_parts[::2], station_parts[1::2]):
                if s not in self.stations:
                    if self.all_stations:
                        self.stations.append(s)
                    else:
                        continue
                sv = {}
                airtime = self.airtime_re.search(v)
                if airtime is not None:
                    sv['airtime_rx'] = float(airtime.group('rx'))
                    sv['airtime_tx'] = float(airtime.group('tx'))

                rcs = v.find("RC stats:\n")

                # For now, just parse the average aggregation size, which is the
                # last field of each csv line output.
                if rcs > -1:
                    nl = v.find("\n", rcs + 10)
                    if nl > -1:
                        line = v[rcs + 10:nl]
                        sv['avg_aggr_size'] = float(line.split(",")[-1])
                stations[s] = sv

                # Flatten for results array
                for k, v in sv.items():
                    if not isinstance(v, float):
                        continue
                    rk = "::".join([k, s])
                    if rk not in results:
                        results[rk] = [[timestamp, v]]
                    else:
                        results[rk].append([timestamp, v])

            for k, v in matches.items():
                if not isinstance(v, float):
                    continue
                if k not in results:
                    results[k] = [[timestamp, v]]
                else:
                    results[k].append([timestamp, v])
            matches['t'] = timestamp
            matches['stations'] = stations
            self._raw_values.append(matches)

        if self.all_stations:
            self.test_parameters['wifi_stats_stations'] = ",".join(self.stations)
        return results

    @classmethod
    def find_binary(cls, interface, interval, length, host='localhost'):
        script = os.path.join(DATA_DIR, 'scripts', 'wifistats_iterate.sh')
        if not os.path.exists(script):
            raise RuntimeError("Cannot find wifistats_iterate.sh.")

        bash = util.which('bash')
        if not bash:
            raise RuntimeError("WiFi stats requires a Bash shell.")

        return "{bash} {script} -i {interface} -I {interval:.2f} " \
            "-c {count:.0f} -H {host}".format(
                bash=bash,
                script=script,
                interface=interface,
                interval=interval,
                count=length // interval + 1,
                host=host)

class NetstatRunner(ProcessRunner):
    """Runner for getting TCP stats from /proc/net/netstat. Expects
    iterations to be separated by '\n---\n and a timestamp to be present in the
    form 'Time: xxxxxx.xxx' (e.g. the output of `date '+Time: %s.%N'`).

    """
    time_re = re.compile(r"^Time: (?P<timestamp>\d+\.\d+)", re.MULTILINE)
    tcpext_header_re = re.compile(
        r"^TcpExt: (?P<header>[A-Z][0-9a-zA-Z ]+)\n", re.MULTILINE)
    tcpext_data_re = re.compile(r"^TcpExt: (?P<data>[0-9 ]+)\n", re.MULTILINE)

    def parse(self, output):
        results = {}
        parts = output.split("\n---\n")
        for part in parts:
            matches = {}
            timestamp = self.time_re.search(part)
            if timestamp is None:
                continue
            timestamp = float(timestamp.group('timestamp'))
            hdr = self.tcpext_header_re.search(part)
            data = self.tcpext_data_re.search(part)

            if hdr is None or data is None:
                continue

            h = hdr.group("header").split()
            d = data.group("data").split()

            if len(h) != len(d):
                continue

            matches = dict(zip(h, [float(i) for i in d]))

            for k, v in matches.items():
                if not isinstance(v, float):
                    continue
                if k not in results:
                    results[k] = [[timestamp, v]]
                else:
                    results[k].append([timestamp, v])
            matches['t'] = timestamp
            self._raw_values.append(matches)
        return results

    @classmethod
    def find_binary(cls, interval, length, host='localhost'):
        script = os.path.join(DATA_DIR, 'scripts', 'netstat_iterate.sh')
        if not os.path.exists(script):
            raise RuntimeError("Cannot find netstat_iterate.sh.")

        bash = util.which('bash')
        if not bash:
            raise RuntimeError("Capturing netstat requires a Bash shell.")

        return "{bash} {script} -I {interval:.2f} -c {count:.0f} " \
            "-H {host}".format(
                bash=bash,
                script=script,
                interval=interval,
                count=length // interval + 1,
                host=host)

class NullRunner(RunnerBase):
    pass

class ComputingRunner(RunnerBase):
    command = "Computed"
    supported_meta = ['MEAN_VALUE']
    copied_meta = ['UNITS']

    def __init__(self, apply_to=None, post=False, **kwargs):
        super(ComputingRunner, self).__init__(**kwargs)
        if apply_to is None:
            self.keys = []
        else:
            self.keys = apply_to
        self.metadata['COMPUTED_LATE'] = post

    def compute_result(self, res):
        if not self.keys:
            return res

        new_res = []
        keys = Glob.expand_list(self.keys, res.series_names, [self.name])

        for r in res.zipped(keys):
            values = [v for v in r[1:] if v is not None]
            if not values:
                new_res.append(None)
            else:
                new_res.append(self.compute(values))

        meta = res.meta('SERIES_META') if 'SERIES_META' in res.meta() else {}
        meta[self.name] = self.metadata
        for mk in self.supported_meta:
            vals = []
            for k in keys:
                if k in meta and mk in meta[k] and meta[k][mk] is not None:
                    vals.append(meta[k][mk])
            if vals:
                try:
                    meta[self.name][mk] = self.compute(vals)
                except TypeError:
                    meta[self.name][mk] = None

        for mk in self.copied_meta:
            vals = []
            for k in keys:
                if k in meta and mk in meta[k]:
                    vals.append(meta[k][mk])
            if vals:
                # If all the source values of the copied metadata are the same,
                # just use that value, otherwise include all of them.
                meta[self.name][mk] = vals if len(set(vals)) > 1 else vals[0]

        res.add_result(self.name, new_res)
        return res

    def compute(self, values):
        """Compute the function on the values this runner should be applied to.

        Default implementation returns None."""
        return None

class AverageRunner(ComputingRunner):
    command = "Average (computed)"

    def compute(self, values):
        return math.fsum(values) / len(values)

class SmoothAverageRunner(ComputingRunner):
    command = "Smooth average (computed)"

    def __init__(self, smooth_steps=5, **kwargs):
        super(SmoothAverageRunner, self).__init__(**kwargs)
        self._smooth_steps = smooth_steps
        self._avg_values = []

    def compute(self, values):
        self._avg_values.append(math.fsum(values) / len(values))
        while len(self._avg_values) > self._smooth_steps:
            self._avg_values.pop(0)
        return math.fsum(self._avg_values) / len(self._avg_values)

class SumRunner(ComputingRunner):
    command = "Sum (computed)"

    def compute(self, values):
        return math.fsum(values)

class DiffMinRunner(ComputingRunner):
    command = "Diff from min (computed)"

    def compute_result(self, res):
        if not self.keys:
            return res

        key = self.keys[0]

        data = [i for i in res[key] if i is not None]
        if not data:
            res.add_result(self.name, [None] * len(res[key]))
        else:
            min_val = min(data)
            res.add_result(
                self.name, [i - min_val if i is not None else None
                            for i in res[key]])
        return res

class FairnessRunner(ComputingRunner):
    command = "Fairness (computed)"

    def compute(self, values):
        if not len(values):
            return None
        valsum = math.fsum([x**2 for x in values])
        if not valsum:
            return None
        return math.fsum(values)**2 / (len(values) * valsum)


import mock

from analysis.analysis_testcase import AnalysisTestCase
from analysis.clusterfuzz_data import ClusterfuzzData
from analysis.clusterfuzz_parser import ClusterfuzzParser
from analysis.stacktrace import CallStack
from analysis.stacktrace import StackFrame
from analysis.stacktrace import Stacktrace
from analysis.type_enums import CrashClient
from analysis.type_enums import SanitizerType
from libs.deps.chrome_dependency_fetcher import ChromeDependencyFetcher
from libs.deps.dependency import Dependency
from libs.deps.dependency import DependencyRoll

class CusterfuzzDataTest(AnalysisTestCase):
  """Tests ``ClusterfuzzData`` class."""

  def testProperties(self):
    """Tests ``ClusterfuzzData`` specific properties."""
    raw_crash_data = self.GetDummyClusterfuzzData(sanitizer='ASAN')
    crash_data = ClusterfuzzData(raw_crash_data)
    self.assertEqual(crash_data.crashed_address,
                     raw_crash_data['customized_data']['crashed_address'])
    self.assertEqual(crash_data.crashed_type,
                     raw_crash_data['customized_data']['crashed_type'])
    self.assertEqual(crash_data.sanitizer,
                     SanitizerType.ADDRESS_SANITIZER)
    self.assertEqual(crash_data.job_type,
                     raw_crash_data['customized_data']['job_type'])
    self.assertEqual(crash_data.regression_range,
                     raw_crash_data['customized_data']['regression_range'])
    self.assertEqual(crash_data.testcase,
                     raw_crash_data['customized_data']['testcase'])

  @mock.patch('analysis.clusterfuzz_parser.ClusterfuzzParser.Parse')
  def testParseStacktraceFailed(self, mock_parse):
    """Tests that ``stacktrace`` is None when failed to pars stacktrace."""
    mock_parse.return_value = None
    crash_data = ClusterfuzzData(self.GetDummyClusterfuzzData())
    self.assertIsNone(crash_data.stacktrace)

  def testParseStacktraceSucceeded(self):
    """Tests parsing ``stacktrace``."""
    crash_data = ClusterfuzzData(self.GetDummyClusterfuzzData())
    stack = CallStack(0)
    stacktrace = Stacktrace([stack], stack)
    with mock.patch(
        'analysis.clusterfuzz_parser.ClusterfuzzParser.Parse') as mock_parse:
      mock_parse.return_value = stacktrace
      self._VerifyTwoStacktracesEqual(crash_data.stacktrace, stacktrace)

  def testParseStacktraceReturnsCache(self):
    """Tests that ``stacktrace`` returns cached ``_stacktrace`` value."""
    crash_data = ClusterfuzzData(self.GetDummyClusterfuzzData())
    stack = CallStack(1)
    stacktrace = Stacktrace([stack], stack)
    crash_data._stacktrace = stacktrace
    self._VerifyTwoStacktracesEqual(crash_data.stacktrace, stacktrace)

  def testDependencies(self):
    """Tests ``dependencies`` property."""
    dep = Dependency('src/', 'https://repo', 'rev1')
    crash_data = ClusterfuzzData(self.GetDummyClusterfuzzData(
        dependencies=[{'dep_path': dep.path,
                       'repo_url': dep.repo_url,
                       'revision': dep.revision}]))

    self.assertEqual(len(crash_data.dependencies), 1)
    self.assertTrue(dep.path in crash_data.dependencies)
    self.assertEqual(crash_data.dependencies[dep.path].path, dep.path)
    self.assertEqual(crash_data.dependencies[dep.path].repo_url, dep.repo_url)
    self.assertEqual(crash_data.dependencies[dep.path].revision, dep.revision)

  def testDependenciesReturnsCache(self):
    """Tests that ``dependencies`` returns cached ``_dependencies`` value."""
    crash_data = ClusterfuzzData(self.GetDummyClusterfuzzData())
    deps = {'src/': Dependency('src/', 'https://repo', 'rev')}
    crash_data._dependencies = deps
    self.assertEqual(crash_data.dependencies, deps)

  def testDependencyRollsReturnsCache(self):
    """Tests that ``dependency_rolls`` returns cached ``_dependency_rolls``."""
    crash_data = ClusterfuzzData(self.GetDummyClusterfuzzData())
    dep_roll = {'src/': DependencyRoll('src/', 'https://repo', 'rev0', 'rev3')}
    crash_data._dependency_rolls = dep_roll
    self.assertEqual(crash_data.dependency_rolls, dep_roll)

  def testDependencyRolls(self):
    """Tests ``regression_rolls`` property."""
    dep_roll = DependencyRoll('src/', 'https://repo', 'rev1', 'rev6')
    crash_data = ClusterfuzzData(self.GetDummyClusterfuzzData(
        dependency_rolls=[{'dep_path': dep_roll.path,
                           'repo_url': dep_roll.repo_url,
                           'old_revision': dep_roll.old_revision,
                           'new_revision': dep_roll.new_revision}]))

    self.assertEqual(len(crash_data.dependency_rolls), 1)
    self.assertTrue(dep_roll.path in crash_data.dependency_rolls)
    self.assertEqual(crash_data.dependency_rolls[dep_roll.path].path,
                     dep_roll.path)
    self.assertEqual(crash_data.dependency_rolls[dep_roll.path].repo_url,
                     dep_roll.repo_url)
    self.assertEqual(crash_data.dependency_rolls[dep_roll.path].old_revision,
                     dep_roll.old_revision)
    self.assertEqual(crash_data.dependency_rolls[dep_roll.path].new_revision,
                     dep_roll.new_revision)


from dataclasses import dataclass

@dataclass(order=True)
class LinkBuilder():
    url: str
    client_id: str
    scope: str
    redirect_uri: str
    response_type: str
    state: str

    def get(self) -> str:
        link = self.url + '?' + ''.join([
            key + '=' + self.__dict__[key] + '&'
            for key in self.__dict__ if key != 'url'
        ])
        return link[0:-1]


from datetime import datetime
from pydantic import validate_arguments

class PainelDTO:
    @validate_arguments
    def __init__(self, id_configuracao:int, id_secretaria:int, status:bool, data_criado:datetime, total_ativo:int, total_presentes:int, total_ausentes:int, total_presentes_curso:int, total_ativo_curso:int, total_ausente_curso:int):
        self.id_configuracao = id_configuracao
        self.id_secretaria = id_secretaria
        self.status = status
        self.data_criado = data_criado
        self.total_ativo = total_ativo
        self.total_presentes = total_presentes
        self.total_ausentes = total_ausentes
        self.total_presentes_curso = total_presentes_curso
        self.total_ativo_curso = total_ativo_curso
        self.total_ausente_curso = total_ausente_curso


import bs4
import requests

from supybot import callbacks, i18n
from supybot.commands import wrap, optional

_ = i18n.PluginInternationalization('GoogleTranslate')

class GoogleTranslate(callbacks.Plugin):
    """
    This plugin lets you use Google Translate without an API key.
    """

    @wrap(['somethingWithoutSpaces', 'somethingWithoutSpaces', 'text'])
    def translate(self, irc, msg, args, source_lang, target_lang, text):
        """<source language> <target language> <text>

        Translates <text> from <source language> to <target language> using Google Translate. Use language codes, e.g. "en" for English and "es" for Spanish. To auto-detect the source language, use "auto".
        """

        r = requests.get('https://translate.google.com/m', params={
            'sl': source_lang,
            'hl': target_lang,
            'q': text
        })

        if not r.ok:
            irc.error(_('HTTP Error %d: %s'), r.status_code, r.reason.title(),
                      Raise=True)

        soup = bs4.BeautifulSoup(r.text)

        result = soup.find('div', class_='t0')

        if not result.text:
            irc.error(_('Nothing returned by Google Translate'), Raise=True)

        irc.reply(result.text)

Class = GoogleTranslate


from datetime import datetime
from dateutil.relativedelta import relativedelta

from definitions import *
from Questrade import QuestradeWrapper, Symbol

import numpy as np
from pandas.tseries.holiday import USFederalHolidayCalendar as FedHoliday

class SimpleModelDataSetBuilder(object):

    def __init__(self, symbols=('FB', 'AAPL', 'AMZN', 'NFLX', 'GOOG', 'AMD', 'MU', 'SNAP', 'MSFT', 'NVDA')):
        self.window_size = 250
        self.data_points_per_stock = 300
        self.threshold = 0.0025
        self.dataset_name = 'simple_model_dataset'

        self.end_date = datetime.today().replace(hour=0, minute=0, second=0, microsecond=0)
        self.symbols = [symbol.upper() for symbol in symbols]
        self.dataset_path = os.path.join(DATASET_DIR, self.dataset_name + '.npz')

    def iso(self, dt):
        return '%d-%02d-%02dT%02d:%02d:00-04:00' % (dt.year, dt.month, dt.day, dt.hour, dt.minute)

    def dt(self, iso):
        return datetime.strptime(iso[:-13], '%Y-%m-%dT%H:%M:%S')

    def prepare_dataset(self):
        LOGGER = logging.getLogger()

        LOGGER.info('Checking if dataset already exists')
        if os.path.isfile(self.dataset_path):
            LOGGER.info('Dataset found')
            return self.dataset_path

        LOGGER.info('Building dataset')

        LOGGER.info('Authenticating with Questrade')
        qt = QuestradeWrapper()
        qt.authenticate()

        LOGGER.info('Getting symbol IDs from Questrade')
        symbol_ids = [Symbol(qt.search_symbol(symbol)) for symbol in self.symbols]

        LOGGER.info('Downloading data from Questrade')
        data_lst = {x.symbolId: {'min': [], 'hour': [], 'day': []} for x in symbol_ids}
        data_map = {x.symbolId: {'min': {}, 'hour': {}, 'day': {}} for x in symbol_ids}
        min_start = self.end_date - relativedelta(months=1)
        hour_start = self.end_date - relativedelta(months=4)
        day_start = self.end_date - relativedelta(years=2)
        for sym_id in symbol_ids:
            data_lst[sym_id.symbolId]['min'] = qt.get_candles(sym_id, self.iso(min_start), self.iso(self.end_date), 'OneMinute')
            data_lst[sym_id.symbolId]['hour'] = qt.get_candles(sym_id, self.iso(hour_start), self.iso(self.end_date), 'OneHour')
            data_lst[sym_id.symbolId]['day'] = qt.get_candles(sym_id, self.iso(day_start), self.iso(self.end_date), 'OneDay')

        LOGGER.info('Converting datetime')
        for sym_id in symbol_ids:
            for candle in data_lst[sym_id.symbolId]['min']:
                candle.end = self.dt(candle.end)
            for candle in data_lst[sym_id.symbolId]['hour']:
                candle.end = self.dt(candle.end)
            for candle in data_lst[sym_id.symbolId]['day']:
                candle.end = self.dt(candle.end)

        LOGGER.info('Inserting data into map')
        for sym_id in symbol_ids:
            for candle in data_lst[sym_id.symbolId]['min']:
                if candle.end.minute == 0 or candle.end.minute == 15 or candle.end.minute == 30 or candle.end.minute == 45:
                    data_map[sym_id.symbolId]['min'][candle.end] = data_lst[sym_id.symbolId]['min'].index(candle)
            for candle in data_lst[sym_id.symbolId]['hour']:
                data_map[sym_id.symbolId]['hour'][candle.end] = data_lst[sym_id.symbolId]['hour'].index(candle)
            for candle in data_lst[sym_id.symbolId]['day']:
                data_map[sym_id.symbolId]['day'][candle.end] = data_lst[sym_id.symbolId]['day'].index(candle)

        # Prepare some stuff for the upcoming loop
        x = np.zeros((self.data_points_per_stock * len(symbol_ids), self.window_size, 12), dtype=np.float32)
        y = np.zeros((self.data_points_per_stock * len(symbol_ids), 1), dtype=np.int32)
        hour_minute_samples = [(9, 30), (9, 45), (15, 0)] + [(x,y) for x in range(10, 14+1) for y in range(0, 45+1, 15)]
        hour_minute_samples.sort(reverse=True)
        us_holidays = FedHoliday().holidays(self.end_date - relativedelta(years=3), self.end_date).to_pydatetime()

        LOGGER.info('Forming training data')
        matrix_index = 0
        date_iterator = self.end_date
        while True:
            date_iterator = date_iterator - relativedelta(days=1)

            # Skip weekends and holidays
            if date_iterator.weekday() >= 5 or date_iterator in us_holidays:
                continue

            # 15-minute intervals -> 9:30, 9:45, 10:00, 10:15, ..., 14:30, 14:45, 15:00
            for hour, minute in hour_minute_samples:
                end_datetime = date_iterator.replace(hour=hour, minute=minute)

                for sym_id in symbol_ids:

                    try:
                        min_end_index = data_map[sym_id.symbolId]['min'][end_datetime]
                        min_start_index = min_end_index - self.window_size
                        hour_end_index = data_map[sym_id.symbolId]['hour'][end_datetime.replace(minute=0)]
                        hour_start_index = hour_end_index - self.window_size
                        day_end_index = data_map[sym_id.symbolId]['day'][end_datetime.replace(hour=0, minute=0)]
                        day_start_index = day_end_index - self.window_size
                    except KeyError:
                        continue

                    min_data = data_lst[sym_id.symbolId]['min'][min_start_index:min_end_index]
                    hour_data = data_lst[sym_id.symbolId]['hour'][hour_start_index:hour_end_index]
                    day_data = data_lst[sym_id.symbolId]['day'][day_start_index:day_end_index]

                    # Insert training data into x matrix
                    for i in range(len(min_data)):
                        for data, index in ((min_data, 0), (hour_data, 1), (day_data, 2)):
                            x[matrix_index][i][(index * 4) + 0] = data[i].close
                            x[matrix_index][i][(index * 4) + 1] = data[i].volume
                            x[matrix_index][i][(index * 4) + 2] = ((16 - data[i].end.hour) * 60) + 60 - data[i].end.minute
                            x[matrix_index][i][(index * 4) + 3] = data[i].end.weekday()

                    # Insert label data into y vector
                    label_data = [data_lst[sym_id.symbolId]['min'][x] for x in range(min_end_index, min_end_index+61, 60)]
                    hour_change = (label_data[1].close - label_data[0].close) / label_data[0].close
                    y[matrix_index] = 0 if hour_change < (-self.threshold) else 2 if hour_change > self.threshold else 1

                    matrix_index += 1
                    LOGGER.info('Collected %d data points' % matrix_index)

                    if matrix_index >= self.data_points_per_stock * len(symbol_ids): break
                if matrix_index >= self.data_points_per_stock * len(symbol_ids): break
            if matrix_index >= self.data_points_per_stock * len(symbol_ids): break

        LOGGER.info('Saving dataset')
        np.savez(self.dataset_path, x=x, y=y)

        LOGGER.info('Dataset is ready')
        return self.dataset_path

if __name__ == '__main__':
    simple_model_dataset_builder = SimpleModelDataSetBuilder()
    simple_model_dataset_builder.prepare_dataset()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue May 28 07:25:48 2019

@author: rehan
"""
import re

from bs4 import BeautifulSoup
import requests
from sys import byteorder
from array import array
from struct import pack
import pyaudio
import wave
from gtts import gTTS
from pygame import mixer # Load the required library
import threading
from imageai.Detection import ObjectDetection
import os
import collections
from PIL import Image
import pytesseract
import argparse
import cv2
import os
from __main__ import *
import face_recognition
import argparse
import pickle
import cv2
import sys
from gtts import gTTS
import urllib.request
import requests
import base64
try:
    from PIL import Image
except ImportError:
    import Image
import pytesseract

import base64
import requests

import cv2
import numpy as np
from keras.models import load_model
import sys
import warnings

import tensorflow as tf
from keras.backend.tensorflow_backend import set_session

from pydub import AudioSegment
from pydub.playback import play

os.chdir('/Users/rehan/Documents/FYP/PROJECT/face-recognition-opencv')

pytesseract.pytesseract.tesseract_cmd = r"/usr/local/Cellar/tesseract/4.0.0_1/bin/tesseract"

ansList = []

##Satart Section
''' Keras took all GPU memory so to limit GPU usage, I have add those lines'''

config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.1
set_session(tf.Session(config=config))
''' Keras took all GPU memory so to limit GPU usage, I have add those lines'''
## End section

faceCascade = cv2.CascadeClassifier('haarcascade_frontalface_alt2.xml')
model = load_model('model_5-49-0.62.hdf5')

#OBJECT

detector = ObjectDetection()
detector.setModelTypeAsRetinaNet()
detector.setModelPath( "/Users/rehan/Documents/FYP/PROJECT/resnet50_coco_best_v2.0.1.h5")

detector.loadModel()

#os.getcwd()
#os.chdir('face-recognition-opencv')

def faceRecognition(imagePath):
    language = 'en'
    print("inner module starting");
    args = {}
    args["encodings"] = "encodings.pickle"
    args["detection_method"] = "hog"
    # load the known faces and embeddings
    #print("[INFO] loading encodings...")
    data = pickle.loads(open(args["encodings"], "rb").read())

    args["image"] = imagePath
    # load the input image and convert it from BGR to RGB
    image = cv2.imread(args["image"])
    rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

    # detect the (x, y)-coordinates of the bounding boxes corresponding
    # to each face in the input image, then compute the facial embeddings
    # for each face
    #print("[INFO] recognizing faces...")
    boxes = face_recognition.face_locations(rgb,
    	model=args["detection_method"])
    encodings = face_recognition.face_encodings(rgb, boxes)

    # initialize the list of names for each face detected
    names = []

    # loop over the facial embeddings
    for encoding in encodings:
    	# attempt to match each face in the input image to our known
    	# encodings
    	matches = face_recognition.compare_faces(data["encodings"],
    		encoding)
    	name = "Unknown"
    	# check to see if we have found a match
    	if True in matches:
    		# find the indexes of all matched faces then initialize a
    		# dictionary to count the total number of times each face
    		# was matched
    		matchedIdxs = [i for (i, b) in enumerate(matches) if b]
    		counts = {}

    		# loop over the matched indexes and maintain a count for
    		# each recognized face face
    		for i in matchedIdxs:
    			name = data["names"][i]
    			counts[name] = counts.get(name, 0) + 1

    		# determine the recognized face with the largest number of
    		# votes (note: in the event of an unlikely tie Python will
    		# select first entry in the dictionary)
    		name = max(counts, key=counts.get)

    	# update the list of names
    	if '/' in name:
    		name = name.rsplit('/', 1)[-1]
    	names.append(name)
    # loop over the recognized faces

    if (len(names)==0):
        return('We could\'t find any faces')
    else:
        retAns = "We detected "+' and '.join(names)

        return(retAns)

    #return(names)
    #sys.exit()
    #for ((top, right, bottom, left), name) in zip(boxes, names):
    #	# draw the predicted face name on the image
    #	cv2.rectangle(image, (left, top), (right, bottom), (0, 255, 0), 2)
    #	y = top - 15 if top - 15 > 15 else top + 15
    #	cv2.putText(image, name, (left, y), cv2.FONT_HERSHEY_SIMPLEX,
    #		0.75, (0, 255, 0), 2)

    # show the output image
    #cv2.imshow("Image", image)
    #cv2.waitKey(0)
def retAns(mylist):
    counter=collections.Counter(mylist)

    noOb = len(mylist)

    if noOb > 0:

        if noOb>1:
            ans= 'There are a total of '+str(noOb)+' objects in this picture.\n'
        else:
            return('There is just one object in this picture. A '+mylist[0]+'.\n')

        if len(counter.keys())==1:
            ans = ans+'All of those are '+list(counter.keys())[0]+'s.\n'
            return(ans)
        for i in counter.keys():
            if counter[i]>1:

                ans=ans+str(counter[i])+' of those are '+i+'.\n'
            else:
                ans=ans+str(counter[i])+' of those is a '+i+'.\n'

    else:
        ans = 'We couldn\'t detect any objects in this picture. Please try again!'

    return(ans)

def objectRecognition(imagePath,detector):
    execution_path = os.getcwd()
    #custom_objects = detector.CustomObjects(person=True)

    results={}
    ob=[]
    obPercent=[]
    #detections, extracted_images = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , "in/"+f), output_image_path=os.path.join(execution_path , "out/"+f), extract_detected_objects=True)

    obList=[]

    detections = detector.detectObjectsFromImage(input_image=imagePath, output_image_path=os.path.join(execution_path , "output/objectRecognition.JPG"))
    ob=[]
    obPercent=[]
    for eachObject in detections:
        #results[]=eachObject["percentage_probability"]

        ob.append(eachObject["name"])
        obPercent.append(eachObject["percentage_probability"])
    obList.append(ob)
    return(retAns(ob))

def textRecognition(imagePath):

    os.system('convert '+imagePath+' -resize 400% -type Grayscale output/text.tif')

    return(pytesseract.image_to_string(Image.open('output/text.tif')))
    #return(pytesseract.image_to_string(Image.open('output/text.tif')))

import speech_recognition as sr
r = sr.Recognizer()

#r.recognize_google()
count = 0
def captureImage():
    global count
    import cv2
    video_capture = cv2.VideoCapture(0)
    # Check success
    if not video_capture.isOpened():
        raise Exception("Could not open video device")
    # Read picture. ret === True on success
    ret, frame = video_capture.read()

    fName = str(count)+'.jpg'
    cv2.imwrite('output/'+fName,frame)
    count=count+1
    # Close device
    video_capture.release()
    execution_path = os.getcwd()

    return(execution_path+'/output/'+fName)

def saveData(userName):
    language = 'en'
    cascPath = 'haarcascade_frontalface_default.xml'
    faceCascade = cv2.CascadeClassifier(cascPath)
    people = []
    saving = False
    temp_path = ""
    name = ""
    video_capture = cv2.VideoCapture(0)
    i = 0
    j = 0
    while True:
        # Capture frame-by-frame
        ret, frame = video_capture.read()

        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

        faces = faceCascade.detectMultiScale(
            gray,
            scaleFactor=1.3,
            minNeighbors=5,
           # minSize=(30, 30)
           # flags=cv2.cv.CV_HAAR_SCALE_IMAGE
        )
        #path = 'F:/FYP/face-recognition-opencv/face-recognition-opencv/unknown'
        # Draw a rectangle around the faces
        #if not os.path.exists('F:/FYP/face-recognition-opencv/face-recognition-opencv/unknown'):
         #   os.makedirs('F:/FYP/face-recognition-opencv/face-recognition-opencv/unknown')
       # print("what")
        for (x, y, w, h) in faces:
          #  print("fa")
            cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)
            crop_img = frame[y:y+h, x:x+w].copy()
            imNameCopy = "{0}".format(i)
            i = i + 1
            if not saving:
     #           cv2.imwrite(os.path.join(path , imNameCopy+".jpg"), crop_img)
     #           import recognize_faces_image
      #          if 'Unknown' in recognize_faces_image.names or recognize_faces_image.names==[]:
       #             print("Give unknown a name!")
        #            name = input("Give a name")
                    temp_path = '/Users/rehan/Documents/FYP/PROJECT/face-recognition-opencv/profiles/'+userName
                    if not os.path.exists(temp_path):
                        os.makedirs(temp_path)
                    #else:
                     #   temp_len = len(os.listdir('F:/FYP/face-recognition-opencv/face-recognition-opencv/profiles/'))
                      #  print("Profile with this name already exists!")
                       # os.makedirs(temp_path+str(temp_len))
                        #print("Profile made instead with name"+name+str(temp_len))
                        #temp_path = 'F:/FYP/face-recognition-opencv/face-recognition-opencv/profiles/'+name +str(temp_len)

                    saving = True
     #           else:
      #              for a in recognize_faces_image.names:
       #                 myobj = gTTS(text=a+" is around you!", lang=language, slow=False)
        #                myobj.save("welcome.mp3")

                        # Playing the converted file
         #               playsound("welcome.mp3")
          #              sys.exit()
            else:
                cv2.imwrite(os.path.join(temp_path , imNameCopy+".jpg"), crop_img)
                temp_len = len(os.listdir(temp_path))
                temp_len = temp_len + 1
                cv2.imwrite(os.path.join(temp_path , name+str(temp_len)+".jpg"), crop_img)
                j = j + 1
                if j>20:
                    #import encode_faces
                    saving = False
                    video_capture.release()
                    return

def encodeModel():
    import encode_faces

def test_image(addr):
    target = ['angry','disgust','fear','happy','sad','surprise','neutral']
    font = cv2.FONT_HERSHEY_SIMPLEX

    im = cv2.imread(addr)
    gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
    faces = faceCascade.detectMultiScale(gray,scaleFactor=1.1)
    resultExp=[]
    for (x, y, w, h) in faces:
            cv2.rectangle(im, (x, y), (x+w, y+h), (0, 255, 0), 2,5)
            face_crop = im[y:y+h,x:x+w]
            face_crop = cv2.resize(face_crop,(48,48))
            face_crop = cv2.cvtColor(face_crop, cv2.COLOR_BGR2GRAY)
            face_crop = face_crop.astype('float32')/255
            face_crop = np.asarray(face_crop)
            face_crop = face_crop.reshape(1, 1,face_crop.shape[0],face_crop.shape[1])
            result = target[np.argmax(model.predict(face_crop))]
            resultExp.append(result)
            cv2.putText(im,result,(x,y), font, 1, (200,0,0), 3, cv2.LINE_AA)

    #cv2.imshow('result', im)
    cv2.imwrite('output/result_emotion_detection_app.jpg',im)
    outP="No emotions detected"
    if(len(resultExp)>0):
        outP=""
        counter=collections.Counter(resultExp)
        for ci in list(counter.keys()):
            if(counter[ci]>1):
                outP=outP+str(counter[ci])+" are "+ci+". "
            else:
                outP=outP+str(counter[ci])+" is "+ci+". "
    return(outP)

def recordAudio(mpname,s):

    import pyaudio
    import wave

    chunk = 1024  # Record in chunks of 1024 samples
    sample_format = pyaudio.paInt16  # 16 bits per sample
    channels = 2
    fs = 44100  # Record at 44100 samples per second
    seconds = s
    filename = mpname

    p = pyaudio.PyAudio()  # Create an interface to PortAudio

    print('Recording')

    stream = p.open(format=sample_format,
                    channels=channels,
                    rate=fs,
                    frames_per_buffer=chunk,
                    input=True)

    frames = []  # Initialize array to store frames

    # Store data in chunks for 3 seconds
    for i in range(0, int(fs / chunk * seconds)):
        data = stream.read(chunk)
        frames.append(data)

    # Stop and close the stream
    stream.stop_stream()
    stream.close()
    # Terminate the PortAudio interface
    p.terminate()

    print('Finished recording')

    # Save the recorded data as a WAV file
    wf = wave.open(filename, 'wb')
    wf.setnchannels(channels)
    wf.setsampwidth(p.get_sample_size(sample_format))
    wf.setframerate(fs)
    wf.writeframes(b''.join(frames))
    wf.close()

    AUDIO_FILE = mpname
    # use the audio file as the audio source
    try:

        with sr.AudioFile(AUDIO_FILE) as source:
            audio = r.record(source)  # read the entire audio file

        ans=r.recognize_google(audio)
    except:
        ans=''
    if 'break' ==ans:
        main()
    return(ans)

r = sr.Recognizer()

def sayText(mytext,fName):
    myobj = gTTS(text=mytext, lang='en', slow=False)

    myobj.save(fName)

        # Playing the converted file

    song = AudioSegment.from_mp3(fName)
    play(song)

AUDIO_FILE = "demo.wav"

noteCount = 0
def noteTaking():
    global noteCount
    song = AudioSegment.from_mp3("startRecording.mp3")
    play(song)

    note = recordAudio('notes/'+str(noteCount)+'.wav',3)

    song = AudioSegment.from_mp3("endRecording.mp3")
    play(song)

    song = AudioSegment.from_mp3("nameRecording.mp3")
    play(song)

    #name = recordAudio('notes/'+str(noteCount)+'.wav',5)

    while(True):
        name = recordAudio('notes/'+str(noteCount)+'.wav',5)
        print(name)
        try:
            f=open('notes/'+str(name)+'.txt','w+')
            f.write(note)
            f.close()
            noteCount= noteCount +1
            break
        except:
            song = AudioSegment.from_mp3("errorRecording.mp3")
            play(song)

def readArticle():
    url = "https://www.dawn.com/"

# Getting the webpage, creating a Response object.
    response = requests.get(url)

    # Extracting the source code of the page.
    data = response.text
    soup = BeautifulSoup(data, 'html')
    mydivs = soup.findAll("h2", {"class": "story__title"})

    result=""

    for tArticle in range(0,5):
        result=result+ str(mydivs[tArticle].text)+". "

    sayText(result,'newsArticle.mp3')

url = "https://www.tennisexpress.com/search.cfm?searchKeyword=BB6892"

# Getting the webpage, creating a Response object.
response = requests.get(url)

# Extracting the source code of the page.
data = response.text

def noteSearch():

    allFiles=os.listdir('notes/')

    song = AudioSegment.from_mp3("searchRecording.mp3")
    play(song)

    name = recordAudio('tempQuery.wav',5)

    for t in allFiles:
        if name in t:
            song = AudioSegment.from_mp3("foundRecording.mp3")
            play(song)

            fR=open('notes/'+t,'r')
            readFile=fR.read()

            sayText(readFile,'tempQuery.mp3')
            return
    song = AudioSegment.from_mp3("notFoundRecording.mp3")
    play(song)

def main(speak = True):

    while(True):
        print("please speak a word into the microphone")

        if(speak):
            song = AudioSegment.from_mp3("homeRecording.mp3")
            play(song)
            speak=False

        ans = recordAudio("demo.wav",5)
        print(ans)

        if('detect' in ans or 'recognize' in ans or 'face' in ans or 'object' in ans or 'objects' in ans or 'text' in ans or 'read' in ans):
            imagePath = captureImage()
            #imagePath='/Users/rehan/Documents/FYP/PROJECT/face-recognition-opencv/2019_05_27_13_24_34.JPG'
            if('face' in ans or 'faces' in ans):
                output=faceRecognition(imagePath)
                output=output+". "+test_image(imagePath)

            elif('objects' in ans or 'object' in ans):
                output=objectRecognition(imagePath,detector)
            elif('text' in ans or 'read' in ans):
                output = textRecognition(imagePath)
                output=output.replace('\n',' ')
                if (len(output)>100):
                    output=output[0:500]
                if(len(output)==0):
                    output='No text recognized'
            else:
                output='No output'

            sayText(output,'welcome.mp3')
            print(output)
        elif('save' in ans or 'store' in ans):

            asp=ans.split(' ')
            if((len(asp)>1)):
                saveUser = 'Saving user as '+asp[1]+'. Please make sure there is only the user in the camera frame'

                sayText(saveUser,'sUser.mp3')

                saveData(asp[1])

                sayText('User Saved','sUser2.mp3')

        elif('note' in ans or 'notes' in ans):

            if('search' in ans):
                noteSearch()
            elif('record' or 'recording' in ans):
                noteTaking()
        elif('headlines' in ans or 'headline' in ans or 'news' in ans or 'dawn' in ans):
            readArticle()

        print("done - result written to demo.wav")

# use the audio file as the audio source

#objectRecognition(imp,detector)
#textRecognition(imp)
#faceRecognition(imp)


import gzip
import json
from sklearn.model_selection import train_test_split
import numpy as np
from tensorflow.python.keras import models, layers
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score
import re
from keras import backend as BK

###########################################################
# taking the input, applying the tanh function and scaling
# to output from -1 to 1 to the wanted scale
# Parameters:
#   - x (input float)
#   - target_min (lower bound of the wanted scale)
#   - target_max (upper bound of the wanted scale)
# Returns
#   - scaled input
def mapping_to_target_range(x, target_min=1, target_max=5):
    x02 = BK.tanh(x) + 1
    scale = (target_max - target_min) / 2.
    return x02 * scale + target_min

###########################################################
# Main supervised learning with neural network method.
# Loads the dataset, applies preprocessing, trains the
# model, saves the model parameters and prints the
# classification report
# Parameters:
#   - load (Bool if the weights should be loaded)
# Returns
#   -
def supervised_learning(load):
    # Loading dataset
    data = []
    g = gzip.open("Software.json.gz", 'r')
    print("Loading dataset ...")
    for l in g:
        data.append(json.loads(l))
    N = 100000
    print("The dataset used has ", len(data), "entries! Of this dataset", N, "entries are used to train the model.")

    reviews = []
    ratings = []
    print("Text preprocessing ...")
    for d in data[:N]:
        if d.get('reviewText') is None:
            continue
        # remove all unwanted chars
        review = re.compile(r'[^a-z0-9\s]').sub(r'', re.compile(r'[\W]').sub(r' ', d.get('reviewText').lower()))
        reviews.append(review)
        rating = float(d.get('overall'))
        ratings.append(rating)

    # vectorized the input texts
    max_features = 200000
    tokenizer = Tokenizer(num_words=max_features)
    tokenizer.fit_on_texts(reviews)
    reviews = tokenizer.texts_to_sequences(reviews)

    # calculating the maximal review length & pad all inputs to the same length for the neural network
    max_length = max(len(train_r) for train_r in reviews)
    reviews = tf.keras.preprocessing.sequence.pad_sequences(reviews, maxlen=max_length)

    # split the data into training set, test set and validation set
    print("Splitting dataset ...")
    train_reviews, test_reviews, train_ratings, test_ratings = train_test_split(np.array(reviews), np.array(ratings),
                                                                                test_size=0.1)
    train_reviews, validation_reviews, train_ratings, validation_ratings = train_test_split(train_reviews,
                                                                                train_ratings,
                                                                                test_size=0.2)

    # Create the neural network. Input size was calculated above
    input = layers.Input(shape=(max_length,))
    x = layers.Embedding(max_features, 64)(input)
    # three times, use a convolutional layer, normalization and max pooling layer
    x = layers.Conv1D(64, 5, activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.MaxPool1D(3)(x)
    x = layers.Conv1D(64, 5, activation='relu')(x)
    x = layers.BatchNormalization()(x)
    x = layers.MaxPool1D(5)(x)
    x = layers.Conv1D(64, 5, activation='relu')(x)
    x = layers.GlobalMaxPool1D()(x)
    x = layers.Flatten()(x)
    # two normally connected layers to condense the output to a single number
    x = layers.Dense(64, activation='relu')(x)
    output = layers.Dense(1, activation=mapping_to_target_range)(x)
    model = models.Model(inputs=input, outputs=output)

    # Adam (a stochastic gradient descent variant) as optimization function
    opt = tf.keras.optimizers.Adam(learning_rate=0.001)

    # compiling the model. As error the MSE is specified since the output and target are floats
    model.compile(optimizer=opt, loss='mean_squared_error')

    # loading model weights if wanted
    if load:
        print("\nLoading previous model weights:\n")
        model.load_weights('weights/supervisedLearning')

    # training the model
    print("Training Model:\n")
    model.fit(train_reviews, train_ratings, batch_size=64, epochs=3,
              validation_data=(validation_reviews, validation_ratings))

    # calculating the predictions on the test set
    test_pred = model.predict(test_reviews)

    # printing the classification report
    print(classification_report(test_ratings,np.round(test_pred)))

    # testing the model with 3 examples (positive, negative, neutral review)
    print("\nTesting model: ")
    x = "I really like this book. It is one of the best I have read."
    x = re.compile(r'[^a-z\s]').sub(r'', re.compile(r'[\W]').sub(r' ', x.lower()))
    x = tokenizer.texts_to_sequences(np.array([x]))
    x = tf.keras.preprocessing.sequence.pad_sequences(x, maxlen=max_length)
    result = model.predict(x)
    print("'I really like this book. It is one of the best I have read.' got the rating: ", round(result[0][0]))

    x = "I really hate this book. It is one of the worst I have read."
    x = re.compile(r'[^a-z\s]').sub(r'', re.compile(r'[\W]').sub(r' ', x.lower()))
    x = tokenizer.texts_to_sequences(np.array([x]))
    x = tf.keras.preprocessing.sequence.pad_sequences(x, maxlen=max_length)
    result = model.predict(x)
    print("'I really hate this book. It is one of the worst I have read.' got the rating: ", round(result[0][0]))

    x = "This book is ok. It is very average."
    x = re.compile(r'[^a-z\s]').sub(r'', re.compile(r'[\W]').sub(r' ', x.lower()))
    x = tokenizer.texts_to_sequences(np.array([x]))
    x = tf.keras.preprocessing.sequence.pad_sequences(x, maxlen=max_length)
    result = model.predict(x)
    print("'This book is ok. It is very average.' got the rating: ", round(result[0][0]))

    # saving the model weights
    print("\n\nSaving model weights ...")
    model.save_weights('weights/supervisedLearning')