import collections
import os
import threading

from tensorflow.core.protobuf import debug_event_pb2
from tensorflow.python.framework import errors
from tensorflow.python.framework import tensor_util
from tensorflow.python.lib.io import file_io
from tensorflow.python.lib.io import tf_record
from tensorflow.python.util import compat

DebugEventWithOffset = collections.namedtuple(
    "DebugEventWithOffset", "debug_event offset")

class DebugEventsReader:
  """Reader class for a tfdbg v2 DebugEvents directory."""

  # Number of digests after which a read lock is released and re-acquired during
  # serial reading of digests for SourceFiles, Execution, and
  # GraphExecutionTrace. This allows us to avoid releasing and re-acquiring the
  # lock too often (i.e., after each digest) and to minimize performance
  # penalty.
  _READER_RELEASE_PER = 100

  _METADATA_SUFFIX = ".metadata"
  _SOURCE_FILE_SUFFIX = ".source_files"
  _STACK_FRAMES_SUFFIX = ".stack_frames"
  _GRAPHS_SUFFIX = ".graphs"
  _EXECUTION_SUFFIX = ".execution"
  _GRAPH_EXECUTION_TRACES_SUFFIX = ".graph_execution_traces"

  def __init__(self, dump_root):
    if not file_io.is_directory(dump_root):
      raise ValueError("Specified dump_root is not a directory: %s" % dump_root)
    self._dump_root = dump_root
    self._metadata_paths = self._load_metadata_files()

    prefixes = [
        metadata_path[:-len(self._METADATA_SUFFIX)]
        for metadata_path in self._metadata_paths
    ]
    prefix = prefixes[0]  # This is the prefix of the main file set.
    self._source_files_path = compat.as_bytes(prefix + self._SOURCE_FILE_SUFFIX)
    self._stack_frames_path = compat.as_bytes(prefix +
                                              self._STACK_FRAMES_SUFFIX)
    self._graphs_path = compat.as_bytes(prefix + self._GRAPHS_SUFFIX)
    self._execution_path = compat.as_bytes(prefix + self._EXECUTION_SUFFIX)
    # There can be multiple .graph_execution_trace files each belonging
    # to a file set generated on an individual host, in the case of
    # a distributed TensorFlow job.
    # This is different from the other debug event files in the file set.
    self._graph_execution_traces_paths = [
        compat.as_bytes(prefix + self._GRAPH_EXECUTION_TRACES_SUFFIX)
        for prefix in prefixes
    ]
    self._readers = dict()  # A map from file path to reader.
    # A map from file path to current reading offset.
    self._reader_offsets = dict()
    # Lock for reader creation.
    self._readers_lock = threading.Lock()
    # Locks for read operation on individual readers.
    self._reader_read_locks = dict()

    self._offsets = dict()

  def _load_metadata_files(self):
    """Load and parse metadata files in the dump root.

    Check that all metadata files have a common tfdbg_run_id, and raise
    a ValueError if their tfdbg_run_ids differ.

    Returns:
      A list of metadata file paths in ascending order of their starting
        wall_time timestamp.
    """

    metadata_paths = file_io.get_matching_files(
        os.path.join(self._dump_root, "*%s" % self._METADATA_SUFFIX))
    if not metadata_paths:
      raise ValueError("Cannot find any tfdbg metadata file in directory: %s" %
                       self._dump_root)
    wall_times = []
    run_ids = []
    tensorflow_versions = []
    file_versions = []
    for metadata_path in metadata_paths:
      reader = tf_record.tf_record_random_reader(metadata_path)
      try:
        record = reader.read(0)[0]
        debug_event = debug_event_pb2.DebugEvent.FromString(record)
        wall_times.append(debug_event.wall_time)
        run_ids.append(debug_event.debug_metadata.tfdbg_run_id)
        tensorflow_versions.append(
            debug_event.debug_metadata.tensorflow_version
        )
        file_versions.append(debug_event.debug_metadata.file_version)
      except Exception as e:
        raise errors.DataLossError(
            None,
            None,
            "Error reading tfdbg metadata from paths %s" % metadata_paths,
        ) from e
      finally:
        reader.close()
    self._starting_wall_time = wall_times[0]
    self._tfdbg_run_id = run_ids[0]
    self._tensorflow_version = tensorflow_versions[0]
    self._file_version = file_versions[0]
    if len(metadata_paths) == 1:
      # Fast path for a common case (only one DebugEvent file set.)
      return metadata_paths

    num_no_id = len([run_id for run_id in run_ids if not run_id])
    if num_no_id:
      paths_without_run_id = [
          metadata_path
          for metadata_path, run_id in zip(metadata_paths, run_ids)
          if not run_id
      ]
      raise ValueError(
          "Found %d tfdbg metadata files and %d of them do not "
          "have tfdbg run ids. The metadata files without run ids are: %s" %
          (len(run_ids), num_no_id, paths_without_run_id))
    elif len(set(run_ids)) != 1:
      raise ValueError(
          "Unexpected: Found multiple (%d) tfdbg2 runs in directory %s" %
          (len(set(run_ids)), self._dump_root))
    # Return the metadata files in ascending order of their timestamps.
    paths_and_timestamps = sorted(
        zip(metadata_paths, wall_times), key=lambda t: t[1])
    self._starting_wall_time = paths_and_timestamps[0][1]
    return [path[0] for path in paths_and_timestamps]

  def starting_wall_time(self):
    """Get the starting timestamp of the instrumented TensorFlow program.

    When there are multiple hosts (i.e., multiple tfdbg file sets), the earliest
    timestamp among the file sets is returned. It is assumed to be the job that
    starts first (e.g., the coordinator).

    Returns:
      Starting timestamp in seconds since the epoch, as a float.
    """
    return self._starting_wall_time

  def tfdbg_run_id(self):
    """Get the run ID of the instrumented TensorFlow program."""
    return self._tfdbg_run_id

  def tensorflow_version(self):
    """Get the version string of TensorFlow that the debugged program ran on."""
    return self._tensorflow_version

  def tfdbg_file_version(self):
    """Get the tfdbg file format version."""
    return self._file_version

  def __enter__(self):
    return self

  def __exit__(self, exception_type, exception_value, traceback):
    del exception_type, exception_value, traceback  # Unused
    self.close()

  def _generic_iterator(self, file_path):
    """A helper method that makes an iterator given a debug-events file path.

    Repeated calls to this method create iterators that remember the last
    successful reading position (offset) for each given `file_path`. So the
    iterators are meant for incremental reading of the file.

    Args:
      file_path: Path to the file to create the iterator for.

    Yields:
      A tuple of (offset, debug_event_proto) on each `next()` call.
    """
    yield_count = 0
    reader = self._get_reader(file_path)
    read_lock = self._reader_read_locks[file_path]
    read_lock.acquire()
    try:
      while True:
        current_offset = self._reader_offsets[file_path]
        try:
          record, self._reader_offsets[file_path] = reader.read(current_offset)
        except (errors.DataLossError, IndexError):
          # We ignore partial read exceptions, because a record may be
          # truncated. The PyRandomRecordReader throws an `IndexError` when
          # offset goes out of bound.
          break
        yield DebugEventWithOffset(
            debug_event=debug_event_pb2.DebugEvent.FromString(record),
            offset=current_offset)
        yield_count += 1
        # The read lock must be periodically released to allow for concurrent
        # random reads. But we do so at a number of reads, instead of after
        # every single read, in order to minimize the performance penalty.
        if yield_count % self._READER_RELEASE_PER == 0:
          read_lock.release()
          read_lock.acquire()
    finally:
      read_lock.release()

  def _get_reader(self, file_path):
    """Get a random-access reader for TFRecords file at file_path."""
    file_path = compat.as_bytes(file_path)
    # The following code uses the double-checked locking pattern to optimize
    # the common case (where the reader is already initialized).
    if file_path not in self._readers:  # 1st check, without lock.
      with self._readers_lock:
        if file_path not in self._readers:  # 2nd check, with lock.
          self._readers[file_path] = tf_record.tf_record_random_reader(
              file_path)
          self._reader_read_locks[file_path] = threading.Lock()
          self._reader_offsets[file_path] = 0
    return self._readers[file_path]

  def source_files_iterator(self):
    return self._generic_iterator(self._source_files_path)

  def stack_frames_iterator(self):
    return self._generic_iterator(self._stack_frames_path)

  def graphs_iterator(self):
    return self._generic_iterator(self._graphs_path)

  def read_source_files_event(self, offset):
    """Read a DebugEvent proto at given offset from the .source_files file."""
    with self._reader_read_locks[self._source_files_path]:
      proto_string = self._get_reader(self._source_files_path).read(offset)[0]
    return debug_event_pb2.DebugEvent.FromString(proto_string)

  def read_graphs_event(self, offset):
    """Read a DebugEvent proto at a given offset from the .graphs file.

    Args:
      offset: Offset to read the DebugEvent proto from.

    Returns:
      A DebugEventProto.

    Raises:
      `errors.DataLossError` if offset is at a wrong location.
      `IndexError` if offset is out of range of the file.
    """
    return debug_event_pb2.DebugEvent.FromString(
        self._get_reader(self._graphs_path).read(offset)[0])

  def execution_iterator(self):
    return self._generic_iterator(self._execution_path)

  def read_execution_event(self, offset):
    """Read a DebugEvent proto at a given offset from the .execution file.

    Args:
      offset: Offset to read the DebugEvent proto from.

    Returns:
      A DebugEventProto.

    Raises:
      `errors.DataLossError` if offset is at a wrong location.
      `IndexError` if offset is out of range of the file.
    """
    with self._reader_read_locks[self._execution_path]:
      proto_string = self._get_reader(self._execution_path).read(offset)[0]
    return debug_event_pb2.DebugEvent.FromString(proto_string)

  def graph_execution_traces_iterators(self):
    return [
        self._generic_iterator(path)
        for path in self._graph_execution_traces_paths
    ]

  def read_graph_execution_traces_event(self, locator):
    """Read DebugEvent at given offset from given .graph_execution_traces file.

    Args:
      locator: A (file_index, offset) tuple that locates the DebugEvent
        containing the graph execution trace.

    Returns:
      A DebugEventProto.

    Raises:
      `errors.DataLossError` if offset is at a wrong location.
      `IndexError` if offset is out of range of the file.
    """
    file_index, offset = locator
    graph_execution_traces_path = self._graph_execution_traces_paths[file_index]
    with self._reader_read_locks[graph_execution_traces_path]:
      proto_string = self._get_reader(graph_execution_traces_path).read(
          offset)[0]
    return debug_event_pb2.DebugEvent.FromString(proto_string)

  def close(self):
    with self._readers_lock:
      file_paths = list(self._readers.keys())
      for file_path in file_paths:
        self._readers[file_path].close()
        del self._readers[file_path]

class BaseDigest:
  """Base class for digest.

  Properties:
    wall_time: A timestamp for the digest as a `float` (unit: s).
    locator: A datum that allows tracng the digest to its original
      location. It can be either of the two:
       1. Bytes offset from the beginning of the file as a single integer,
          for the case of all digests of the same kind coming from the same
          file.
       2. A tuple of a file index and a byte offset. This applies to case
          in which the same type of debugger data may come from multiple files,
          e.g., graph execution traces.
  """

  def __init__(self, wall_time, locator):
    self._wall_time = wall_time
    self._locator = locator

  @property
  def wall_time(self):
    return self._wall_time

  @property
  def locator(self):
    return self._locator

  def to_json(self):
    return {"wall_time": self.wall_time}

class ExecutionDigest(BaseDigest):
  """Light-weight digest summarizing top-level execution event.

  Use `DebugDataReader.read_execution(execution_digest)` to load the more
  detailed data object concerning the execution event (`Execution`).

  Properties:
    op_type: Type name of the executed op. In the case of the eager execution of
      an individual op, it is the name of the op (e.g., "MatMul").
      In the case of the execution of a tf.function (FuncGraph), this is the
      internally-generated name of the function (e.g.,
      "__inference_my_func_123").
    output_tensor_device_ids: IDs of the devices on which the output tensors of
      the execution reside. For no-output execution, this is `None`.
  """

  def __init__(self,
               wall_time,
               locator,
               op_type,
               output_tensor_device_ids=None):
    super().__init__(wall_time, locator)
    self._op_type = op_type
    self._output_tensor_device_ids = _tuple_or_none(output_tensor_device_ids)

  @property
  def op_type(self):
    return self._op_type

  @property
  def output_tensor_device_ids(self):
    return self._output_tensor_device_ids

  def to_json(self):
    output = super().to_json()
    output.update({
        "op_type": self.op_type,
        "output_tensor_device_ids": self.output_tensor_device_ids,
    })
    return output

def _tuple_or_none(data):
  return tuple(data) if data else None

class Execution(ExecutionDigest):
  """Detailed data relating to a top-level execution event.

  The execution is of an individual op or a tf.function, which may have any
  number of output tensors.

  Properties (beyond the base class `ExecutionDigest`):
    host_name: Name of the host on which the execution happened.
    stack_frame_ids: Reference IDs for stack frames, ordered from bottommost to
      topmost. Use `DebugDataReader.read_execution_stack_trace()` to load the
      detailed stack frames (filepath, lineno and function name).
    tensor_debug_mode: TensorDebugMode enum value, as an `int`.
    graph_id: ID of the executed FuncGraph (applicable only the execution of a
      tf.function). `None` for the eager execution of an individual op.
    input_tensor_ids: IDs of the input (eager) tensor(s) for this execution, if
      any. If the eager execution has no input tensor, this is `None`. Else,
      this is a `tuple` of `int`s.
    output_tensor_ids: IDs of the output (eager) tensor(s) from this execution,
      if any. If the eager execution produces no output tensor, this is `None`.
      Else, this is a `tuple` of `int`s.
    debug_tensor_values: Values of the debug tensor(s), applicable only to
      non-FULL_TENSOR tensor debug mode. A tuple of list of numbers. Each
      element of the tuple corresponds to an output tensor of the execution.
      See documentation of the various TensorDebugModes for the semantics of the
      numbers. If the eager execution produces no output tensor, this is
      `None`. Else, this is a `tuple` of `list`s.
  """

  def __init__(self,
               execution_digest,
               host_name,
               stack_frame_ids,
               tensor_debug_mode,
               graph_id=None,
               input_tensor_ids=None,
               output_tensor_ids=None,
               debug_tensor_values=None):
    super().__init__(
        execution_digest.wall_time,
        execution_digest.locator,
        execution_digest.op_type,
        output_tensor_device_ids=execution_digest.output_tensor_device_ids)
    self._host_name = host_name
    self._stack_frame_ids = tuple(stack_frame_ids)
    self._tensor_debug_mode = tensor_debug_mode
    self._graph_id = graph_id
    self._input_tensor_ids = _tuple_or_none(input_tensor_ids)
    self._output_tensor_ids = _tuple_or_none(output_tensor_ids)
    self._debug_tensor_values = _tuple_or_none(debug_tensor_values)

  @property
  def host_name(self):
    return self._host_name

  @property
  def stack_frame_ids(self):
    return self._stack_frame_ids

  @property
  def tensor_debug_mode(self):
    return self._tensor_debug_mode

  @property
  def graph_id(self):
    return self._graph_id

  @property
  def input_tensor_ids(self):
    return self._input_tensor_ids

  @property
  def num_outputs(self):
    return len(self._output_tensor_ids) if self._output_tensor_ids else 0

  @property
  def output_tensor_ids(self):
    return self._output_tensor_ids

  @property
  def debug_tensor_values(self):
    return self._debug_tensor_values

  def to_json(self):
    output = super().to_json()
    output.update({
        "host_name": self.host_name,
        "stack_frame_ids": self.stack_frame_ids,
        "tensor_debug_mode": self.tensor_debug_mode,
        "graph_id": self.graph_id,
        "input_tensor_ids": self.input_tensor_ids,
        "output_tensor_ids": self.output_tensor_ids,
        "debug_tensor_values": self.debug_tensor_values,
    })
    return output

class DebuggedGraph:
  """Data object representing debugging information about a tf.Graph.

  Includes `FuncGraph`s.

  Properties:
    name: Name of the graph (if any). May be `None` for non-function graphs.
    graph_id: Debugger-generated ID for the graph.
    inner_graph_ids: A list of the debugger-generated IDs for the graphs
      enclosed by this graph.
    outer_graph_id: If this graph is nested within an outer graph, ID of the
      outer graph. If this is an outermost graph, `None`.
  """

  def __init__(self,
               name,
               graph_id,
               outer_graph_id=None):
    self._name = name
    self._graph_id = graph_id
    self._outer_graph_id = outer_graph_id
    self._inner_graph_ids = []
    # A dictionary from op name to GraphOpCreationDigest.
    self._op_by_name = dict()
    # A dictionary mapping op to immediate downstream consumers.
    self._op_consumers = collections.defaultdict(list)

  def add_inner_graph_id(self, inner_graph_id):
    """Add the debugger-generated ID of a graph nested within this graph.

    Args:
      inner_graph_id: The debugger-generated ID of the nested inner graph.
    """
    assert isinstance(inner_graph_id, str)
    self._inner_graph_ids.append(inner_graph_id)

  def add_op(self, graph_op_creation_digest):
    """Add an op creation data object.

    Args:
      graph_op_creation_digest: A GraphOpCreationDigest data object describing
        the creation of an op inside this graph.
    """
    if graph_op_creation_digest.op_name in self._op_by_name:
      raise ValueError(
          "Duplicate op name: %s (op type: %s)" %
          (graph_op_creation_digest.op_name, graph_op_creation_digest.op_type))
    self._op_by_name[
        graph_op_creation_digest.op_name] = graph_op_creation_digest

  def add_op_consumer(self, src_op_name, src_slot, dst_op_name, dst_slot):
    """Add a consuming op for this op.

    Args:
      src_op_name: Name of the op of which the output tensor is being consumed.
      src_slot: 0-based output slot of the op being consumed.
      dst_op_name: Name of the consuming op (e.g., "Conv2D_3/BiasAdd")
      dst_slot: 0-based input slot of the consuming op that receives the tensor
        from this op.
    """
    self._op_consumers[src_op_name].append((src_slot, dst_op_name, dst_slot))

  @property
  def name(self):
    return self._name

  @property
  def graph_id(self):
    return self._graph_id

  @property
  def outer_graph_id(self):
    return self._outer_graph_id

  @property
  def inner_graph_ids(self):
    return self._inner_graph_ids

  def get_tensor_id(self, op_name, output_slot):
    """Get the ID of a symbolic tensor in this graph."""
    return self._op_by_name[op_name].output_tensor_ids[output_slot]

  def get_op_creation_digest(self, op_name):
    """Get the GraphOpCreationDigest for a op in the graph."""
    return self._op_by_name[op_name]

  def get_op_consumers(self, src_op_name):
    """Get all the downstream consumers of this op.

    Only data (non-control) edges are tracked.

    Args:
      src_op_name: Name of the op providing the tensor being consumed.

    Returns:
      A list of (src_slot, dst_op_name, dst_slot) tuples. In each item of
      the list:
        src_slot: 0-based output slot of the op of which the output tensor
          is being consumed.
        dst_op_name: Name of the consuming op (e.g., "Conv2D_3/BiasAdd")
        dst_slot: 0-based input slot of the consuming op that receives
          the tensor from this op.
    """
    return self._op_consumers[src_op_name]

  def to_json(self):
    return {
        "name": self.name,
        "graph_id": self.graph_id,
        "outer_graph_id": self._outer_graph_id,
        "inner_graph_ids": self._inner_graph_ids,
    }

class DebuggedDevice:
  """Debugger data regarding a device involved in the debugged program.

  Properties:
    device_name: Name of the device, as a str.
    device_id: An integer ID for the device, unique for each device within
      the scope of the debugged TensorFlow program.
  """

  def __init__(self,
               device_name,
               device_id):
    self._device_name = device_name
    self._device_id = device_id

  @property
  def device_name(self):
    return self._device_name

  @property
  def device_id(self):
    return self._device_id

  def to_json(self):
    return {
        "device_name": self._device_name,
        "device_id": self._device_id,
    }

class GraphOpCreationDigest(BaseDigest):
  """Data object describing the creation of an op inside a graph.

  For size efficiency, this digest object does not contain any stack frames or
  any references to them. To obtain the stack frames, use
  `DataReader.read_graph_op_creation_stack_trace()`.

  Properties (beyond the base class):
    graph_id: Debugger-generated ID of the immediately-enclosing graph.
    op_type: Type name of the op (e.g., "MatMul").
    op_name: Name of the op (e.g., "dense_1/MatMul").
    output_tensor_ids: Debugger-generated IDs for the output(s) of the op.
      If the op produces no output tensor, this is `None`. Else, this is a
      `tuple` of `int`s.
    input_names: Names of the input tensors to the op.
    device_name: The name of the device that the op is placed on (if available).
    host_name: Name of the host on which the op is created.
    stack_frame_ids: IDs of the frames of the stack trace at which the op
      is created.
  """

  def __init__(self,
               wall_time,
               locator,
               graph_id,
               op_type,
               op_name,
               output_tensor_ids,
               host_name,
               stack_frame_ids,
               input_names=None,
               device_name=None):
    super().__init__(wall_time, locator)
    self._graph_id = graph_id
    self._op_type = op_type
    self._op_name = op_name
    self._output_tensor_ids = _tuple_or_none(output_tensor_ids)
    self._host_name = host_name
    self._stack_frame_ids = stack_frame_ids
    self._input_names = _tuple_or_none(input_names)
    self._device_name = device_name

  @property
  def graph_id(self):
    return self._graph_id

  @property
  def op_type(self):
    return self._op_type

  @property
  def op_name(self):
    return self._op_name

  @property
  def output_tensor_ids(self):
    return self._output_tensor_ids

  @property
  def num_outputs(self):
    return len(self._output_tensor_ids) if self.output_tensor_ids else 0

  @property
  def input_names(self):
    return self._input_names

  @property
  def device_name(self):
    return self._device_name

  @property
  def host_name(self):
    return self._host_name

  @property
  def stack_frame_ids(self):
    return self._stack_frame_ids

  def to_json(self):
    output = super().to_json()
    output.update({
        "graph_id": self.graph_id,
        "op_type": self.op_type,
        "op_name": self.op_name,
        "output_tensor_ids": self.output_tensor_ids,
        "host_name": self.host_name,
        "stack_frame_ids": self.stack_frame_ids,
        "input_names": self.input_names,
        "device_name": self.device_name,
    })
    return output

class GraphExecutionTraceDigest(BaseDigest):
  """Light-weight summary of a intra-graph tensor execution event.

  Use `DebugDataReader.read_graph_execution_trace()` on this object to read more
  detailed data (`GraphExecutionTrace`).

  Properties (beyond the base class):
    op_type: Type name of the executed op (e.g., "Conv2D").
    op_name: Name of the op (e.g., "conv_2d_3/Conv2D").
    output_slot: Output slot index of the tensor.
    graph_id: The debugger-generated ID of the innermost (immediately-enclosing)
      graph.
  """

  def __init__(self, wall_time, locator, op_type, op_name, output_slot,
               graph_id):
    super().__init__(wall_time, locator)
    self._op_type = op_type
    self._op_name = op_name
    self._output_slot = output_slot
    self._graph_id = graph_id

  @property
  def op_type(self):
    return self._op_type

  @property
  def op_name(self):
    return self._op_name

  @property
  def output_slot(self):
    return self._output_slot

  @property
  def graph_id(self):
    return self._graph_id

  def to_json(self):
    output = super().to_json()
    output.update({
        "op_type": self.op_type,
        "op_name": self.op_name,
        "output_slot": self.output_slot,
        "graph_id": self.graph_id,
    })
    return output

class GraphExecutionTrace(GraphExecutionTraceDigest):
  """Detailed data object describing an intra-graph tensor execution.

  Attributes (in addition to GraphExecutionTraceDigest):
    graph_ids: The debugger-generated IDs of the graphs that enclose the
      executed op (tensor), ordered from the outermost to the innermost.
    graph_id: The debugger-generated ID of the innermost (immediately-enclosing)
      graph.
    tensor_debug_mode: TensorDebugMode enum value.
    debug_tensor_value: Debug tensor values (only for non-FULL_TENSOR
      tensor_debug_mode). A list of numbers. See the documentation of the
      TensorDebugModes for the semantics of the numbers.
    device_name: Device on which the tensor resides (if available)
  """

  def __init__(self,
               graph_execution_trace_digest,
               graph_ids,
               tensor_debug_mode,
               debug_tensor_value=None,
               device_name=None):
    super().__init__(graph_execution_trace_digest.wall_time,
                     graph_execution_trace_digest.locator,
                     graph_execution_trace_digest.op_type,
                     graph_execution_trace_digest.op_name,
                     graph_execution_trace_digest.output_slot,
                     graph_execution_trace_digest.graph_id)
    self._graph_ids = tuple(graph_ids)
    self._tensor_debug_mode = tensor_debug_mode
    self._debug_tensor_value = debug_tensor_value
    self._device_name = device_name

  @property
  def graph_ids(self):
    return self._graph_ids

  @property
  def graph_id(self):
    return self._graph_ids[-1]

  @property
  def tensor_debug_mode(self):
    return self._tensor_debug_mode

  @property
  def debug_tensor_value(self):
    return _tuple_or_none(self._debug_tensor_value)

  @property
  def device_name(self):
    return self._device_name

  def to_json(self):
    output = super().to_json()
    output.update({
        "graph_ids": self.graph_ids,
        "tensor_debug_mode": self.tensor_debug_mode,
        "debug_tensor_value": self.debug_tensor_value,
        "device_name": self.device_name,
    })
    return output

def _parse_tensor_value(tensor_proto, return_list=False):
  """Helper method for reading a tensor value from a tensor proto.

  The rationale for the distinction between `True` and `False value of
  `return_list` is as follows:
  - `return_list=True` is used for TensorDebugMode values other than
    FULL_TENSOR, e.g., CONCISE_HEALTH, SHAPE and FULL_HEATLH. Under
    those modes, the value is guaranteed (by contract) to be a 1D float64
    tensor.
  - `return_list=False` is used for the FULL_HEALTH TensorDebugMode
    specifically. Instead, we use `numpy.ndarray` to maximally preserve
    the shape, dtype and value information regarding the underlying tensor
    value. Under that mode, we don't use a python list to represent the
    tensor value because that can lead to loss of information (e.g., both
    float16 and float32 dtypes get mapped to Python floats).

  Args:
    tensor_proto: The TensorProto instance from which the tensor value will be
      loaded.
    return_list: Whether the return value will be a nested Python list that
      comes out from `numpy.ndarray.tolist()`.

  Returns:
    If parsing is successful, the tensor value as a `numpy.ndarray` or the
      nested Python list converted from it.
    If parsing fails, `None`.
  """
  try:
    ndarray = tensor_util.MakeNdarray(tensor_proto)
    return ndarray.tolist() if return_list else ndarray
  except TypeError:
    # Depending on tensor_debug_mode, certain dtype of tensors don't
    # have logged debug tensor values.
    return None

def _execution_digest_from_debug_event_proto(debug_event, locator):
  """Convert a DebugEvent proto into an ExecutionDigest data object."""
  return ExecutionDigest(
      debug_event.wall_time,
      locator,
      debug_event.execution.op_type,
      output_tensor_device_ids=(debug_event.execution.output_tensor_device_ids
                                or None))

def _execution_from_debug_event_proto(debug_event, locator):
  """Convert a DebugEvent proto into an Execution data object."""
  execution_proto = debug_event.execution

  debug_tensor_values = None
  if (execution_proto.tensor_debug_mode ==
      debug_event_pb2.TensorDebugMode.FULL_TENSOR):
    pass  # TODO(cais): Build tensor store.
  elif (execution_proto.tensor_debug_mode !=
        debug_event_pb2.TensorDebugMode.NO_TENSOR):
    debug_tensor_values = []
    for tensor_proto in execution_proto.tensor_protos:
      # TODO(cais): Refactor into a helper method.
      debug_tensor_values.append(
          _parse_tensor_value(tensor_proto, return_list=True))
  return Execution(
      _execution_digest_from_debug_event_proto(debug_event, locator),
      execution_proto.code_location.host_name,
      tuple(execution_proto.code_location.stack_frame_ids),
      execution_proto.tensor_debug_mode,
      graph_id=execution_proto.graph_id,
      input_tensor_ids=tuple(execution_proto.input_tensor_ids),
      output_tensor_ids=tuple(execution_proto.output_tensor_ids),
      debug_tensor_values=_tuple_or_none(debug_tensor_values))

class DebugDataReader:
  """A reader that reads structured debugging data in the tfdbg v2 format.

  The set of data read by an object of this class concerns the execution history
  of a tfdbg2-instrumented TensorFlow program.

  Note:
    - An object of this class incrementally reads data from files that belong to
      the tfdbg v2 DebugEvent file set. Calling `update()` triggers the reading
      from the last-successful reading positions in the files.
    - This object can be used as a context manager. Its `__exit__()` call
      closes the file readers cleanly.
  """

  def __init__(self, dump_root):
    self._reader = DebugEventsReader(dump_root)

    # TODO(cais): Implement pagination for memory constraints.
    self._execution_digests = []

    # Mapping (host_name, file_path) tuple to offset in the .source_files file.
    self._host_name_file_path_to_offset = collections.OrderedDict()
    # A dict mapping id to (host_name, file_path, lineno, func) tuple.
    self._stack_frame_by_id = dict()
    # Stores unprocessed stack frame IDs. This is necessary to handle the
    # case in which reading of the .stack_frames file gets ahead of the reading
    # of the .source_files file.
    self._unprocessed_stack_frames = dict()
    # A dict mapping id to DebuggedDevice objects.
    self._device_by_id = dict()
    # A dict mapping id to DebuggedGraph objects.
    self._graph_by_id = dict()
    self._graph_op_digests = []
    # TODO(cais): Implement pagination for memory constraints.
    self._graph_execution_trace_digests = []

    self._monitors = []

  def _add_monitor(self, monitor):
    self._monitors.append(monitor)

  def _load_source_files(self):
    """Incrementally read the .source_files DebugEvent file."""
    source_files_iter = self._reader.source_files_iterator()
    for debug_event, offset in source_files_iter:
      source_file = debug_event.source_file
      self._host_name_file_path_to_offset[
          (source_file.host_name, source_file.file_path)] = offset

  def _load_stack_frames(self):
    """Incrementally read the .stack_frames file.

    This must be called after _load_source_files().
    It assumes that the following contract is honored by the writer of the tfdbg
    v2 data file set:
      - Before a stack frame is written to the .stack_frames file, the
        corresponding source file information must have been written to the
        .source_files file first.
    """
    stack_frames_iter = self._reader.stack_frames_iterator()
    for debug_event, _ in stack_frames_iter:
      stack_frame_with_id = debug_event.stack_frame_with_id
      file_line_col = stack_frame_with_id.file_line_col
      self._unprocessed_stack_frames[stack_frame_with_id.id] = file_line_col
    # We do the processing in a separate stage, because the reading in the
    # .source_files file may sometimes get ahead of the .source_files file.
    unprocessed_stack_frame_ids = tuple(self._unprocessed_stack_frames.keys())
    for stack_frame_id in unprocessed_stack_frame_ids:
      file_line_col = self._unprocessed_stack_frames[stack_frame_id]
      if len(self._host_name_file_path_to_offset) > file_line_col.file_index:
        host_name, file_path = list(self._host_name_file_path_to_offset.keys())[
            file_line_col.file_index]
        self._stack_frame_by_id[stack_frame_id] = (
            host_name, file_path, file_line_col.line, file_line_col.func)
      del self._unprocessed_stack_frames[stack_frame_id]

  def _load_graphs(self):
    """Incrementally read the .graphs file.

    Compiles the DebuggedGraph and GraphOpCreation data.
    """
    graphs_iter = self._reader.graphs_iterator()
    for debug_event, offset in graphs_iter:
      if debug_event.graph_op_creation.ByteSize():
        op_creation_proto = debug_event.graph_op_creation
        op_digest = GraphOpCreationDigest(
            debug_event.wall_time,
            offset,
            op_creation_proto.graph_id,
            op_creation_proto.op_type,
            op_creation_proto.op_name,
            tuple(op_creation_proto.output_tensor_ids),
            op_creation_proto.code_location.host_name,
            tuple(op_creation_proto.code_location.stack_frame_ids),
            input_names=tuple(op_creation_proto.input_names))
        self._graph_op_digests.append(op_digest)
        debugged_graph = self._graph_by_id[op_creation_proto.graph_id]
        debugged_graph.add_op(op_digest)
        for dst_slot, input_name in enumerate(op_creation_proto.input_names):
          src_op_name, src_slot = input_name.split(":")
          debugged_graph.add_op_consumer(src_op_name, int(src_slot),
                                         op_creation_proto.op_name, dst_slot)

      elif debug_event.debugged_graph.ByteSize():
        graph_proto = debug_event.debugged_graph
        graph = DebuggedGraph(
            graph_proto.graph_name or None,
            graph_proto.graph_id,
            outer_graph_id=graph_proto.outer_context_id or None)
        self._graph_by_id[graph_proto.graph_id] = graph
        if graph_proto.outer_context_id:
          self._graph_by_id[
              graph_proto.outer_context_id].add_inner_graph_id(graph.graph_id)
      elif debug_event.debugged_device.ByteSize():
        device_proto = debug_event.debugged_device
        self._device_by_id[device_proto.device_id] = DebuggedDevice(
            device_proto.device_name, device_proto.device_id)

  def _load_graph_execution_traces(self):
    """Incrementally load the .graph_execution_traces file."""
    for i, traces_iter in enumerate(
        self._reader.graph_execution_traces_iterators()):
      for debug_event, offset in traces_iter:
        self._graph_execution_trace_digests.append(
            self._graph_execution_trace_digest_from_debug_event_proto(
                debug_event, (i, offset)))
        if self._monitors:
          graph_execution_trace = (
              self._graph_execution_trace_from_debug_event_proto(
                  debug_event, (i, offset)))
          for monitor in self._monitors:
            monitor.on_graph_execution_trace(
                len(self._graph_execution_trace_digests) - 1,
                graph_execution_trace)

  def _graph_execution_trace_digest_from_debug_event_proto(
      self, debug_event, locator):
    trace_proto = debug_event.graph_execution_trace
    op_name = trace_proto.op_name
    op_type = self._lookup_op_type(trace_proto.tfdbg_context_id, op_name)
    return GraphExecutionTraceDigest(
        debug_event.wall_time, locator, op_type, op_name,
        trace_proto.output_slot,
        debug_event.graph_execution_trace.tfdbg_context_id)

  def _graph_execution_trace_from_debug_event_proto(self, debug_event, locator):
    """Convert a DebugEvent proto into a GraphExecutionTrace data object."""
    trace_proto = debug_event.graph_execution_trace
    graph_ids = [trace_proto.tfdbg_context_id]
    # Walk up the chain of outer contexts (graphs), so as to include all of
    # their IDs
    while True:
      graph = self.graph_by_id(graph_ids[0])
      if graph.outer_graph_id:
        graph_ids.insert(0, graph.outer_graph_id)
      else:
        break

    if (trace_proto.tensor_debug_mode ==
        debug_event_pb2.TensorDebugMode.FULL_TENSOR):
      debug_tensor_value = None
    else:
      debug_tensor_value = _parse_tensor_value(
          trace_proto.tensor_proto, return_list=True)
    return GraphExecutionTrace(
        self._graph_execution_trace_digest_from_debug_event_proto(
            debug_event, locator),
        graph_ids=graph_ids,
        tensor_debug_mode=trace_proto.tensor_debug_mode,
        debug_tensor_value=debug_tensor_value,
        device_name=trace_proto.device_name or None)

  def _lookup_op_type(self, graph_id, op_name):
    """Lookup the type of an op by name and the immediately enclosing graph.

    Args:
      graph_id: Debugger-generated ID of the immediately-enclosing graph.
      op_name: Name of the op.

    Returns:
      Op type as a str.
    """
    return self._graph_by_id[graph_id].get_op_creation_digest(op_name).op_type

  def _load_execution(self):
    """Incrementally read the .execution file."""
    execution_iter = self._reader.execution_iterator()
    for debug_event, offset in execution_iter:
      self._execution_digests.append(
          _execution_digest_from_debug_event_proto(debug_event, offset))
      if self._monitors:
        execution = _execution_from_debug_event_proto(debug_event, offset)
        for monitor in self._monitors:
          monitor.on_execution(len(self._execution_digests) - 1, execution)

  def update(self):
    """Perform incremental read of the file set."""
    self._load_source_files()
    self._load_stack_frames()
    self._load_graphs()
    self._load_graph_execution_traces()
    self._load_execution()

  def source_file_list(self):
    """Get a list of source files known to the debugger data reader.

    Returns:
      A tuple of `(host_name, file_path)` tuples.
    """
    return tuple(self._host_name_file_path_to_offset.keys())

  def source_lines(self, host_name, file_path):
    """Read the line-by-line content of a source file.

    Args:
      host_name: Host name on which the source file is located.
      file_path: File path at which the source file is located.

    Returns:
      Lines of the source file as a `list` of `str`s.
    """
    offset = self._host_name_file_path_to_offset[(host_name, file_path)]
    return list(self._reader.read_source_files_event(offset).source_file.lines)

  def starting_wall_time(self):
    """Wall timestamp for when the debugged TensorFlow program started.

    Returns:
      Stating wall time as seconds since the epoch, as a `float`.
    """
    return self._reader.starting_wall_time()

  def tensorflow_version(self):
    """TensorFlow version used in the debugged TensorFlow program.

    Note: this is not necessarily the same as the version of TensorFlow used to
    load the DebugEvent file set.

    Returns:
      TensorFlow version used by the debugged program, as a `str`.
    """
    return self._reader.tensorflow_version()

  def tfdbg_run_id(self):
    """Get the debugger run ID of the debugged TensorFlow program."""
    return self._reader.tfdbg_run_id()

  def outermost_graphs(self):
    """Get the number of outer most graphs read so far."""
    return [graph for graph in self._graph_by_id.values()
            if not graph.outer_graph_id]

  def graph_by_id(self, graph_id):
    """Get a DebuggedGraph object by its ID."""
    return self._graph_by_id[graph_id]

  def device_name_by_id(self, device_id):
    """Get the name of a device by the debugger-generated ID of the device."""
    return self._device_by_id[device_id].device_name

  def device_name_map(self):
    """Get a map mapping device IDs to device names."""
    return {device_id: self._device_by_id[device_id].device_name
            for device_id in self._device_by_id}

  def graph_op_digests(self, op_type=None):
    """Get the list of the digests for graph-op creation so far.

    Args:
      op_type: Optional op type to filter the creation events with.

    Returns:
      A list of `GraphOpCreationDigest` objects.
    """
    if op_type is not None:
      return [digest for digest in self._graph_op_digests
              if digest.op_type == op_type]
    else:
      return self._graph_op_digests

  def graph_execution_traces(self, digest=False, begin=None, end=None):
    """Get all the intra-graph execution tensor traces read so far.

    Args:
      digest: Whether the results will be returned in the more light-weight
        digest form.
      begin: Optional beginning index for the requested traces or their digests.
        Python-style negative indices are supported.
      end: Optional ending index for the requested traces or their digests.
        Python-style negative indices are supported.

    Returns:
      If `digest`: a `list` of `GraphExecutionTraceDigest` objects.
      Else: a `list` of `GraphExecutionTrace` objects.
    """
    digests = self._graph_execution_trace_digests
    if begin is not None or end is not None:
      begin = begin or 0
      end = end or len(digests)
      digests = digests[begin:end]
    if digest:
      return digests
    else:
      return [self.read_graph_execution_trace(digest) for digest in digests]

  def num_graph_execution_traces(self):
    """Get the number of graph execution traces read so far."""
    return len(self._graph_execution_trace_digests)

  def executions(self, digest=False, begin=None, end=None):
    """Get `Execution`s or `ExecutionDigest`s this reader has read so far.

    Args:
      digest: Whether the results are returned in a digest form, i.e.,
        `ExecutionDigest` format, instead of the more detailed `Execution`
        format.
      begin: Optional beginning index for the requested execution data objects
        or their digests. Python-style negative indices are supported.
      end: Optional ending index for the requested execution data objects or
        their digests. Python-style negative indices are supported.

    Returns:
      If `digest`: a `list` of `ExecutionDigest` objects.
      Else: a `list` of `Execution` objects.
    """
    digests = self._execution_digests
    if begin is not None or end is not None:
      begin = begin or 0
      end = end or len(digests)
      digests = digests[begin:end]
    if digest:
      return digests
    else:
      # TODO(cais): Optimizer performance removing repeated file open/close.
      return [self.read_execution(digest) for digest in digests]

  def num_executions(self):
    """Get the number of execution events read so far."""
    return len(self._execution_digests)

  def read_execution(self, execution_digest):
    """Read a detailed Execution object."""
    debug_event = self._reader.read_execution_event(execution_digest.locator)
    return _execution_from_debug_event_proto(debug_event,
                                             execution_digest.locator)

  def read_graph_execution_trace(self, graph_execution_trace_digest):
    """Read the detailed graph execution trace.

    Args:
      graph_execution_trace_digest: A `GraphExecutionTraceDigest` object.

    Returns:
      The corresponding `GraphExecutionTrace` object.
    """
    debug_event = self._reader.read_graph_execution_traces_event(
        graph_execution_trace_digest.locator)
    return self._graph_execution_trace_from_debug_event_proto(
        debug_event, graph_execution_trace_digest.locator)

  def read_execution_stack_trace(self, execution):
    """Read the stack trace of a given Execution object.

    Args:
      execution: The Execution object of interest.

    Returns:
      1. The host name.
      2. The stack trace, as a list of (file_path, lineno, func) tuples.
    """
    host_name = self._stack_frame_by_id[execution.stack_frame_ids[0]][0]
    return (host_name, [
        self._stack_frame_by_id[frame_id][1:]
        for frame_id in execution.stack_frame_ids])

  def read_graph_op_creation_stack_trace(self, graph_op_creation_digest):
    """Read the stack trace of a given graph op creation object.

    Args:
      graph_op_creation_digest: The GraphOpCreationDigest object of interest.

    Returns:
      A tuple consisting of:
        1. The host name.
        2. The stack trace, as a list of (file_path, lineno, func) tuples.
    """
    return graph_op_creation_digest.host_name, [
        self._stack_frame_by_id[frame_id][1:]
        for frame_id in graph_op_creation_digest.stack_frame_ids
    ]

  # TODO(cais): Add graph_execution_digests() with an ExecutionDigest
  #   as a kwarg, to establish the association between top-level and intra-graph
  #   execution events.

  def execution_to_tensor_values(self, execution):
    """Read the full tensor values from an Execution or ExecutionDigest.

    Args:
      execution: An `ExecutionDigest` or `ExecutionDigest` object.

    Returns:
      A list of numpy arrays representing the output tensor values of the
        execution event.
    """
    debug_event = self._reader.read_execution_event(execution.locator)
    return [_parse_tensor_value(tensor_proto)
            for tensor_proto in debug_event.execution.tensor_protos]

  def graph_execution_trace_to_tensor_value(self, trace):
    """Read full tensor values from an Execution or ExecutionDigest.

    Args:
      trace: An `GraphExecutionTraceDigest` or `GraphExecutionTrace` object.

    Returns:
      A numpy array representing the output tensor value of the intra-graph
        tensor execution event.
    """
    debug_event = self._reader.read_graph_execution_traces_event(trace.locator)
    return _parse_tensor_value(debug_event.graph_execution_trace.tensor_proto)

  def symbolic_tensor_id(self, graph_id, op_name, output_slot):
    """Get the ID of a symbolic tensor.

    Args:
      graph_id: The ID of the immediately-enclosing graph.
      op_name: Name of the op.
      output_slot: Output slot as an int.

    Returns:
      The ID of the symbolic tensor as an int.
    """
    return self._graph_by_id[graph_id].get_tensor_id(op_name, output_slot)

  def graph_execution_trace_to_tensor_id(self, trace):
    """Get symbolic tensor ID from a GraphExecutoinTraceDigest object."""
    return self.symbolic_tensor_id(
        trace.graph_id, trace.op_name, trace.output_slot)

  def __enter__(self):
    return self

  def __exit__(self, exception_type, exception_value, traceback):
    del exception_type, exception_value, traceback  # Unused
    self._reader.close()


from textwrap import dedent
from clusterjob import JobScript
import logging
import os
try:
    from unittest.mock import Mock
except ImportError:
    from mock import Mock

logging.basicConfig(level=logging.DEBUG)

def test_init():
    body = "echo 'Hello World'"
    jobscript = JobScript(body, backend='slurm', jobname='printenv',
                          queue='test', time='00:05:00', nodes=1, threads=1,
                          mem=100, stdout='printenv.out',
                          stderr='printenv.err')
    assert jobscript.backends == ['lpbs', 'lsf', 'pbs', 'pbspro', 'sge',
            'slurm']
    for key in ['jobname', 'queue', 'time', 'nodes', 'threads', 'mem',
            'stdout', 'stderr']:
        assert key in jobscript.resources
    assert 'backend' in jobscript.__dict__
    assert str(jobscript).strip() == dedent(r'''
    #!/bin/bash
    #SBATCH --job-name=printenv
    #SBATCH --mem=100
    #SBATCH --nodes=1
    #SBATCH --partition=test
    #SBATCH --error=printenv.err
    #SBATCH --output=printenv.out
    #SBATCH --cpus-per-task=1
    #SBATCH --time=00:05:00
    echo 'Hello World'
    ''').strip()

def test_write_expand_tilde(tmpdir, monkeypatch):
    """Check that when writing out a jobscript to file, the filename passed to
    the `_write_script` method has '~' expaned when writing to a local file,
    and *not* expanded when wriging to a remote file (there, the expansion will
    be performed by SSH)
    """
    monkeypatch.setattr(JobScript, '_write_script', Mock())
    body = 'sleep 180'
    job = JobScript(body, jobname='test_clj')
    job.rootdir = '~/jobs/'
    job.workdir = 'job1'
    job.backend = 'slurm'
    job.shell = '/bin/bash'
    job.write()
    job._write_script.assert_called_with(
            '#!/bin/bash\n#SBATCH --job-name=test_clj\nsleep 180',
            os.path.expanduser('~/jobs/job1/test_clj.slr'), None)
    job.remote = 'remote'
    job.write()
    job._write_script.assert_called_with(
            '#!/bin/bash\n#SBATCH --job-name=test_clj\nsleep 180',
            '~/jobs/job1/test_clj.slr', 'remote')

def test_write(tmpdir, monkeypatch):
    body = 'sleep 180'
    job = JobScript(body, jobname='test_clj')
    job.backend = 'slurm'
    filename = str(tmpdir.join('job.slr'))
    assert not os.path.isfile(filename)
    job.write(filename=filename)
    assert os.path.isfile(filename)
    job.remote = 'remote'
    monkeypatch.setattr(JobScript, '_run_cmd', Mock())
    monkeypatch.setattr(JobScript, '_upload_file', Mock())
    job.write()
    assert job._upload_file.call_count == 1


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue May 18 15:21:34 2021

@author: ogk
"""

import os
import h5py
import param
import numpy as np
import matplotlib.pyplot as plt

save_path = '/home/ogk/Documents/OGK/Trajectory/new_label_v/5000/'

dataset_path = '/home/ogk/Documents/dataset/data/data__I_download/'
if os.path.isdir(dataset_path) != True:
    print('\n>> Not find dataset folder path: '+dataset_path)
    print('>> Please Check folder_path && Computer working system !\n')

stopping = 50
# session = 8
for session in range(1, 38):
    print('session=', session)
    List_File = os.listdir(dataset_path)
    List_File.sort()
    file_name = List_File[session-1]
    print('\n Dataset :',dataset_path+str(file_name))
    mat_file = h5py.File(dataset_path+str(file_name), 'r') # read mat
    FINGER_POS = mat_file[list(mat_file.keys())[3]]
    pos_x = FINGER_POS[1][::param.bin_width]
    pos_y = FINGER_POS[2][::param.bin_width]

    TIMES = mat_file[list(mat_file.keys())[5]]
    time_bin = (TIMES[0])[::param.bin_width]
    time_bin = time_bin[:-1]

    vel_x = (pos_x[1:] - pos_x[:-1])*(-10)/(0.004*param.bin_width)
    vel_y = (pos_y[1:] - pos_y[:-1])*(-10)/(0.004*param.bin_width)
    vel_x = np.resize(vel_x, (len(vel_x), 1))
    vel_y = np.resize(vel_y, (len(vel_y), 1))
    velocity = np.hstack((vel_x, vel_y))

    avg_v = np.sqrt(np.power(vel_x, 2)+np.power(vel_y, 2))
    avg_v[avg_v < 5] = 0
    mom_f = np.ones(stopping)
    avg_v.resize(len(avg_v))
    avg_v = np.convolve(mom_f, avg_v, 'same')
    avg_v.resize(len(avg_v), 1)
    nonzero = np.nonzero(avg_v)
    stop_t1 = np.where(avg_v==0)[0]
    stop_t2 = np.where(avg_v!=0)[0]
    fuck1 = stop_t1[1:] - stop_t1[:-1] - 1
    fuck2 = stop_t2[1:] - stop_t2[:-1] - 1
    timing1 = np.where(fuck1!=0)[0] + 1
    timing2 = np.where(fuck2!=0)[0] + 1
    ttt1 = []
    for i in timing1:
        ttt1.append(stop_t1[i])
    ttt2 = []
    for i in timing2:
        ttt2.append(stop_t2[i])

    # if len(ttt2) > 3:
    #     if len(ttt1) == len(ttt2):
    #         ttt = np.array([ttt1, ttt2]).T
    #         ttt = np.vstack([[0, timing1[0]], ttt])
    #     else:
    #         ttt = np.array([ttt1, ttt2[1:]]).T

    # stoping_t = np.zeros(ttt.shape)
    # for i in range(2):
    #     for j in range(len(ttt)):
    #         stoping_t[j, i] = time_bin[ttt[j, i]]

    new_v = []
    for i in nonzero[0]:
        new_v.append(velocity[i, :])
    new_v = np.array(new_v)
    end_t = 0.064*(len(new_v))
    new_t = np.arange(0, end_t, 0.064)

    time = 1 # mins
    duration_1 = int(time*(60*1000)/64)
    time = 3 # mins
    duration_3 = int(time*(60*1000)/64)
    time = 5 # mins
    duration_5 = int(time*(60*1000)/64)

    t = time_bin
    v = velocity
    plt.figure(figsize=(16, 10))
    plt.margins(x=0)
    plt.grid(True)
    plt.title('Session '+str(session), fontsize=24)
    plt.plot(t, v[:, 0], linewidth=0.5, alpha=0.7)
    plt.plot(t, v[:, 1], linewidth=0.5, alpha=0.7)
    plt.legend(['Vx', 'Vy'], fontsize=24, loc='upper right')
    plt.xlabel('Time(sec)', fontsize=20)
    plt.ylabel('Velocity(mm/sec)', fontsize=20)
    plt.axvline(t[duration_1], color='g')
    plt.axvline(t[duration_3], color='g')
    plt.axvline(t[duration_5], color='g')
    plt.text(t[duration_1]-20, -400, r'1 min', fontdict={'size':'20','color':'b'})
    plt.text(t[duration_3]-20, -400, r'3 mins', fontdict={'size':'20','color':'b'})
    plt.text(t[duration_5]-20, -400, r'5 mins', fontdict={'size':'20','color':'b'})

    if len(ttt1) == len(ttt2):
        if len(ttt1) > 0 or len(ttt2) > 0:
            ttt = np.array([ttt1, ttt2]).T
            ttt = np.vstack([[0, timing1[0]], ttt])
        else:
            if len(stop_t1) > 0 and len(stop_t2) > 0:
                ttt = np.array([[stop_t1[0]], [stop_t2[0]]]).T
            else:
                ttt = np.array([[0, 0]])
    elif len(ttt1) < len(ttt2):
        ttt1 = [timing2[0], *ttt1]

        ttt = np.array([ttt1, ttt2]).T
        # ttt = np.vstack([[0, timing1[0]], ttt])
    else:
        # ttt2 = [timing1[0], *ttt2]
        ttt = np.array([ttt1[:-1], ttt2]).T
        ttt = np.vstack([[0, timing1[0]], ttt])

    stoping_t = np.zeros(ttt.shape)
    for i in range(2):
        for j in range(len(ttt)):
            stoping_t[j, i] = time_bin[ttt[j, i]]

    for i in range(len(stoping_t)):
        plt.axvspan(stoping_t[i, 0], stoping_t[i, 1], color='red', alpha=0.3)
    plt.xlim(t[0], t[5000])
    # plt.ylim(0, 1000)

    plt.savefig(save_path+str(session)+'.png')
    plt.cla()
    plt.clf()
    plt.close()


from __future__ import annotations

from collections import defaultdict
from typing import Generic, Hashable, Iterator, Protocol, Set, TypeVar

from .sorted_collection import SortedCollection

IndexableT = TypeVar("IndexableT", bound="Indexable")
HashableT = TypeVar("HashableT", bound=Hashable)

class Indexable(Hashable, Protocol):
    def __lt__(self: IndexableT, other: IndexableT) -> bool:
        pass

class ColumnIndex(Generic[HashableT]):
    def __init__(self) -> None:
        self._index: defaultdict[Indexable, Set[HashableT]] = defaultdict(set)
        self._sorted_values: SortedCollection[Indexable] = SortedCollection()

    def get_sorted_items(
        self,
        *,
        reverse: bool = False,
    ) -> Iterator[Hashable]:
        if reverse:
            keys = self._sorted_values.reverse()
        else:
            keys = iter(self._sorted_values)
        for value in keys:
            yield from self._index[value]

    def remove_item(self, key: Indexable, id_: HashableT) -> None:
        """Remove an item from the index"""
        ids = self._index[key]
        ids.remove(id_)
        if not ids:
            self._sorted_values.remove(key)

    def add_item(self, key: Indexable, id_: HashableT) -> None:
        """Add an item to the index"""
        self._index[key].add(id_)
        self._sorted_values.add(key)

    def __getitem__(self, key: Indexable) -> Set[HashableT]:
        return self._index[key]


import matplotlib.pyplot as plt
import seaborn as sns
sns.set(style="darkgrid")

def datos(nombre_archivo):
    valores = open(nombre_archivo, "r")
    valores = valores.read()
    valores = [float(x) for x in valores.split()]
    return valores

def bloques(valores,intervalo):
    res = []
    valor = 0
    for elem in range(0,len(valores)):
        if elem % intervalo == 0 and elem != 0:
            res.append(valor/intervalo)
            valor = 0
        valor += valores[elem]
    res.append(valor/intervalo)
    return res

nada = [None]

bipartitos = bloques(datos("bipartitoout"),1)
noconexo = bloques(datos("noconexoout"),1)
conexo = bloques(datos("conexoout"),1)
completo = bloques(datos("completoout"),1)

plt.semilogy()
plt.plot(nada+bipartitos, label="Bipartitos")
plt.plot(nada+noconexo, label="No Conexos")
plt.plot(nada+conexo, label="Conexos")
plt.plot(nada+completo, label="Completos")

plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
#plt.legend( ('Relaciones'), loc = 'upper left')
plt.grid(True)
plt.grid(color = '0.5', linestyle = '--', linewidth = 1)
plt.axis('tight')
#plt.axis([0, 20, 0, 0.00002])
plt.xlabel(r"Aumento de variables", fontsize = 15, color = 'black')
plt.ylabel(r"Tiempo en Segundos", fontsize = 15, color = 'black')
plt.title(' Familias de Grafos',fontsize = 17, color = 'black', verticalalignment = 'baseline', horizontalalignment = 'center')
plt.show()


import xml.etree.ElementTree as ET
import xml.dom.minidom
import serial
import time
import re
import Adafruit_BBIO.ADC as ADC
import struct
import math
from shutil import copyfile
from Adafruit_PCA9685 import PCA9685 as PWM

###########################################
########### Declare Functions #############
###########################################

def init_pwm(i2c_address, pwm_freq):
	pwm = PWM(i2c_address) 		#create a pwm object for PCA9685 at i2c_address
	pwm.set_pwm_freq(pwm_freq) 	#set desired pwm freq

	return pwm

def init_temp():
	ADC.setup()
	time.sleep(0.001)

def init_pwm_values(pwm_xml_init_file):
	pwm_xml_init = open(pwm_xml_init_file, 'r')	#open initialization xml for pwm values
	time.sleep(0.001)
	pwm_init_values = pwm_xml_init.read()		#read the initial pwm values into pwmInitValues
	time.sleep(0.001)
	pwm_xml_init.close()						#close initialization xml
	time.sleep(0.001)

	return pwm_init_values

def init_temp_xml():
	temp_data = ET.Element('data')
	temp_item = ET.SubElement(temp_data, 'temp')
	temp_item_container = ET.SubElement(temp_item, 'item')
	temp_item_container.set('name','Temperature')
	temp_item_container.text = "0.0"

	return temp_data

def read_pwm_values(pwm_init_values, pwm_xml_current_file, ser):

#	Added April 11th by Brian Malbec to try and prevent crashes
	try:
		time.sleep(0.01)
		pwm_xml = open(pwm_xml_current_file, 'w+')	#open the xml for current pwm values
		time.sleep(0.001)
		pwm_current_values = ser.readline()					#read the incoming values
		time.sleep(0.001)

		if not pwm_current_values:							#if no incoming data
			pwm_current_values = pwm_init_values				#set the pwm values to initial values

		pwm_xml.write(pwm_current_values)
		time.sleep(0.001)
		pwm_xml.close()							#close the xml file
		time.sleep(0.001)

	except ET.ParseError:
		time.sleep(0.01)
		pwm_xml = open(pwm_xml_current_file, 'w+')
		time.sleep(0.01)
		pwm_xml.write(pwm_init_values)
		time.sleep(0.01)
		pwm_xml.close()
		time.sleep(0.01)

def parse_pwm_values(pwm_xml):
	try:
		time.sleep(0.009)
		pwm_tree = ET.parse(pwm_xml)	#parse the current values
		time.sleep(0.001)
		pwm_root = pwm_tree.getroot()				#get the root of the parse tree

		left_x = float(pwm_root[0][0].text)
		left_y = float(pwm_root[0][1].text)
		right_x = float(pwm_root[0][2].text)
		right_y = float(pwm_root[0][3].text)
		d_left = int(pwm_root[0][4].text)
		d_right = int(pwm_root[0][5].text)
		d_up = int(pwm_root[0][6].text)
		d_down = int(pwm_root[0][7].text)

		return (left_x, left_y, right_x, right_y,
			d_left, d_right, d_up, d_down)

	except ET.ParseError:
		return (0, 0, 0, 0, 0, 0, 0, 0)

def calculate_motor_speeds(left_x, left_y, right_x, right_y,
			   d_left, d_right, d_up, d_down,
			   prev_value_LX, prev_value_LY, prev_value_RX, prev_value_RY):

#	Added April 11th by Brian Malbec to prevent crashes
	try:
		old_min = -1
		old_max = 1
		new_min = 2300
		new_max = 3700

		max_jump = 100

		ceiling = new_max
		floor = new_min

		#temp = new_min
		new_max = (new_max - new_min) / 2
		new_min = 0

		old_range = old_max - old_min
		new_range = new_max - new_min

		#stopped=temp+new_max		commented out bc "temp" is redundant of "floor", and it's only used here
		stopped = floor + new_max
		threshold = new_max / 2

		new_value_LX = int(((left_x - old_min) * new_range) / old_range) + new_min
		new_value_LY = int(((left_y - old_min) * new_range) / old_range) + new_min
		new_value_RX = int(((right_x - old_min) * new_range) / old_range) + new_min
		new_value_RY = int(((right_y - old_min) * new_range) / old_range) + new_min

		motor1 = stopped
		motor2 = stopped
		motor3 = stopped
		motor4 = stopped
		motor5 = stopped
		motor6 = stopped

	#	if abs(newValueLX - prevValueLX) > threshold:
	#		newValueLX = ramp_formula(prevValueLX, newValueLX)
	#	if abs(newValueLY - prevValueLY) > threshold:
	#		newValueLY = ramp_formula(prevValueLY, newValueLY)
	#	if abs(newValueRX - prevValueRX) > threshold:
	#		newValueRX = ramp_formula(prevValueRX, newValueRX)
	#	if abs(newValueRY - prevValueRY) > threshold:
	#		newValueRY = ramp_formula(prevValueRY, newValueRY)

		if (new_value_LX - prev_value_LX) > max_jump:
			new_value_LX = prev_value_LX + max_jump
		else:
			if (prev_value_LX - new_value_LX) > max_jump:
				new_value_LX = prev_value_LX - max_jump

		if (new_value_LY - prev_value_LY) > max_jump:
			new_value_LY = prev_value_LY + max_jump
		else:
			if (prev_value_LY - new_value_LY) > max_jump:
				new_value_LY = prev_value_LY - max_jump

		if (new_value_RX - prev_value_RX) > max_jump:
			new_value_RX = prev_value_RX + max_jump
		else:
			if (prev_value_RX - new_value_RX) > max_jump:
				new_value_RX = prev_value_RX - max_jump

		if (new_value_RY - prev_value_RY) > max_jump:
			new_value_RY = prev_value_RY + max_jump
		else:
			if (prev_value_RY - new_value_RY) > max_jump:
				new_value_RY = prev_value_RY - max_jump

		prev_value_LX = new_value_LX
		prev_value_LY = new_value_LY
		prev_value_RX = new_value_RX
		prev_value_RY = new_value_RY

	#	LEFT
		if new_value_LX > threshold:
			#print "\t\tLEFT"
			left_val = new_value_LX - threshold
			motor1 += left_val
			motor2 -= left_val
			motor3 -= left_val
			motor4 += left_val

	#	RIGHT
		if new_value_LX < threshold:
			#print "\t\tRIGHT"
			right_val = threshold - new_value_LX
			motor1 -= right_val
			motor2 += right_val
			motor3 += right_val
			motor4 -= right_val

	#	FORWARD
		if new_value_LY > threshold:
			#print "\t\tFORWARD"
			fwd_val = new_value_LY - threshold
			motor1 -= fwd_val
			motor2 -= fwd_val
			motor3 -= fwd_val
			motor4 -= fwd_val

	#	BACKWARD
		if new_value_LY < threshold:
			#print "\t\tBACKWARD"
			back_val = threshold - new_value_LY
			motor1 += back_val
			motor2 += back_val
			motor3 += back_val
			motor4 += back_val

	#	ROTATE LEFT
		if new_value_RX > threshold:
			#print "\t\tROTATE LEFT"
			rotate_left_val = new_value_RX - threshold
			motor1 += rotate_left_val
			motor2 -= rotate_left_val
			motor3 += rotate_left_val
			motor4 -= rotate_left_val

	#	ROTATE RIGHT
		if new_value_RX < threshold:
			#print "\t\tROTATE RIGHT"
			rotate_right_val = threshold - new_value_RX
			motor1 -= rotate_right_val
			motor2 += rotate_right_val
			motor3 -= rotate_right_val
			motor4 += rotate_right_val

	#	ASCEND
		if new_value_RY > threshold:
			#print "\t\tASCEND"
			ascend_val = 2 * (new_value_RY - threshold)
			motor5 -= ascend_val
			motor6 -= ascend_val

	#	DESCEND
		if new_value_RY < threshold:
			#print "\t\tDESCEND"
			descend_val = 2 * (threshold - new_value_RY)
			motor5 += descend_val
			motor6 += descend_val

	#	Restrict final motor values within the allowed bounds
		if motor1 > ceiling:
			motor1 = ceiling
		if motor1 < floor:
			motor1 = floor
		if motor2 > ceiling:
			motor2 = ceiling
		if motor2 < floor:
			motor2 = floor
		if motor3 > ceiling:
			motor3 = ceiling
		if motor3 < floor:
			motor3 = floor
		if motor4 > ceiling:
			motor4 = ceiling
		if motor4 < floor:
			motor4 = floor
		if motor5 > ceiling:
			motor5 = ceiling
		if motor5 < floor:
			motor5 = floor
		if motor6 > ceiling:
			motor6 = ceiling
		if motor6 < floor:
			motor6 = floor

		return (motor1, motor2, motor3, motor4, motor5, motor6,
			prev_value_LX, prev_value_LY, prev_value_RX, prev_value_RY)

	except ET.ParseError:
		return (0,0,0,0,0,0,0,0,0,0)

def set_motor_speeds(pwm, motor1, motor2, motor3, motor4, motor5, motor6,
		     d_left, d_right, d_up, d_down,
		     servo_turn, servo_grip, servo_grip_min, servo_grip_max, servo_turn_min, servo_turn_max):

#	Added April 11th by Brian Malbec to prevent crashes
	try:
		#time.sleep(0.1)
		pwm.set_pwm(0,0,motor1)
		#time.sleep(0.1)
		pwm.set_pwm(5,0,motor2)
		#time.sleep(0.1)
		pwm.set_pwm(2,0,motor3)
		#time.sleep(0.1)
		pwm.set_pwm(3,0,motor4)
		#time.sleep(0.1)
		pwm.set_pwm(4,0,motor5)
		#time.sleep(0.1)
		pwm.set_pwm(1,0,motor6)
		#time.sleep(0.1)

	#		Rotate Left
		if d_left == 1:
			servo_turn -=500
		else:
	#		Rotate Right
			if d_right == 1:
				servo_turn +=500

	#		Open
		if d_up == 1:
			servo_grip -= 500
		#	servo_grip = servo_min
		else:
	#		Close
			if d_down == 1:
				servo_grip += 500
		#		servo_grip = servo_max

	##########	Added on March 23, 2018 by Brian Malbec because without it,
	##########	code may not know "servo_min" and "servo_max" to be global variables
		#servo_max = 4000
		#servo_min = 1200
		#servo_max = 3900
		#servo_min = 1400
	##########################################################################

	#		Restrict servo values within the allowed bounds
		if servo_turn > servo_turn_max:
			servo_turn = servo_turn_max
		if servo_turn < servo_turn_min:
			servo_turn = servo_turn_min
		if servo_grip > servo_grip_max:
			servo_grip = servo_grip_max
		if servo_grip < servo_grip_min:
			servo_grip = servo_grip_min

		servo_turn_local = servo_turn
		servo_grip_local = servo_grip

	#		Set the PWM of each servo
		pwm.set_pwm(7,0,servo_grip_local)
		pwm.set_pwm(6,0,servo_turn_local)

		return servo_turn_local, servo_grip_local

	except ET.ParseError:
		return 2500, 2500

def new_temp_xml(raw_temp):

	temp_data = ET.Element('data')
	temp_item = ET.SubElement(temp_data, 'temp')
	temp_item_container = ET.SubElement(temp_item, 'item')
	temp_item_container.set('name','Temperature')
	temp_item_container.text = str(raw_temp)

	return temp_data

def send_temp(temp_pin, ser):

#	Added April 11th by Brian Malbec to prevent crashes
	try:
		raw_temp = ADC.read(temp_pin)
		time.sleep(0.001)
		millivolts = raw_temp * 1800
		temp_C = float("{0:.1f}".format((millivolts - 803) / 8))
		temp_data = new_temp_xml(temp_C)
		temp_XML = ET.tostring(temp_data)
		ser.write(temp_XML)
		ser.write('\n')

	except ET.ParseError:
		pass

##############################################
############# Set Parameters #################
##############################################

temp_pin = "AIN1"

i2c_address = 0x40
pwm_freq = 485

pwm_xml_init_file = '/home/ubuntu/testfile5.xml'
pwm_xml_current_file = '/home/ubuntu/testfile4.xml'

ser = serial.Serial('/dev/ttyO4', 38400, timeout=0.15)

prev_value_LX = 3000
prev_value_LY = 3000
prev_value_RX = 3000
prev_value_RY = 3000

servo_turn = 2000
servo_grip = 2000
#servo_max = 4000
#servo_min = 1200
#	Added April 23rd by Brian, making a wider range for the servo to turn & grip
servo_grip_max = 5000
servo_grip_min = 2000

servo_turn_max = 6000
servo_turn_min = 0
#servo_max = 3900
#servo_min = 1400

###############################################
####### Call Initialization Functions #########
###############################################
pwm = init_pwm(i2c_address, pwm_freq)
init_temp()
temp_data = init_temp_xml()
pwm_init_values = init_pwm_values(pwm_xml_init_file)

##############################################
################### Main #####################
##############################################
while True:
	try:
		read_pwm_values(pwm_init_values, pwm_xml_current_file, ser)
		#lx, ly, rx, ry,
		left_x, left_y, right_x, right_y, d_left, d_right, d_up, d_down = parse_pwm_values(pwm_xml_current_file)

		motor1, motor2, motor3, motor4, motor5, motor6, prev_value_LX, prev_value_LY, prev_value_RX, prev_value_RY = calculate_motor_speeds(left_x, left_y, right_x, right_y, d_left, d_right, d_up, d_down, prev_value_LX, prev_value_LY, prev_value_RX, prev_value_RY)

		servo_turn, servo_grip = set_motor_speeds(pwm, motor1, motor2, motor3, motor4, motor5, motor6, d_left, d_right, d_up, d_down, servo_turn, servo_grip, servo_grip_min, servo_grip_max, servo_turn_min, servo_turn_max)

		send_temp(temp_pin, ser)

	except ET.ParseError:
		pass


#
# Synopsis: Report unused entities
#
# Categories: Coding Standards, Project Report
#
# Languages: Ada, Fortran, C, Java
#
# Description:
#  Reports unused entities, sorted either by filename (default) or by
#  entity kind.  The file/line location of the definition is also reported.
#  When run within Understand (from Options->Run A Command), double click
#  on the output line to visit the entity definition in the source.
#  Requires an existing Understand database.
#
#
# Written by Jason Quinn
#
# 1/26/21

import understand
import sys
import re
import argparse

# Usage:
parser = argparse.ArgumentParser()
parser.add_argument("db", help="Specify Understand database")
parser.add_argument("-kind", help="(optional) Specify a kind text string. For instance Enumerator.")
args = parser.parse_args()

# Get the declaration ref
def getDeclRef(ent):
    decl = None

    # Language specific
    if re.search('ada', ent.language()):
        decl = ent.ref("declarein ~spec ~body ~instance ~formal ~incomplete ~private ~stub")
        if not decl:
            decl = ent.ref("spec declarein")
        if not decl:
            decl = ent.ref("body declarein")
        if not decl:
            decl = ent.ref("instance declarein")
        if not decl:
            decl = ent.ref("formal declarein")
        if not decl:
            decl = ent.ref("incomplete declarein")
        if not decl:
            decl = ent.ref("private declarein")
        if not decl:
            decl = ent.ref("stub declarein")

    # Fortran, Java, and C\C++
    else:
        decl = ent.ref("definein")
        if not decl:
            decl = ent.ref("declarein")

    return decl

# Check if it is a duplicate ref
def duplicateRef(ref1, ref2):
    if not (ref1 and ref2):
        return False

    if (ref1.file().longname() == ref2.file().longname()) and (ref1.line() == ref2.line()) and (ref1.column() == ref2.column()):
        return True

    return False

if __name__ == '__main__':
    # Open Database
    db = understand.open(args.db)

    if args.kind:
        entKinds = args.kind
    else:
        entKinds = """ada object ~task ~unknown ~unresolved,\
	ada constant ~unknown ~unresolved,\
	ada parameter,\
	ada function ~unknown ~unresolved,\
	ada procedure ~unknown ~unresolved,\
	ada type ~unknown ~unresolved,\
	fortran variable,\
	fortran function ~unresolved ~intrinsic,\
	fortran subroutine ~unresolved ~instrinsic,\
	fortran type ~unknown,\
	c object ~unknown ~unresolved,\
	c parameter,\
	c function ~unknown ~unresolved,\
	c type ~unknown ~unresolved ~unnamed,\
	java variable,\
	java parameter,\
	java method,\
	java class ~anonymous,\
	java interface"""

    ents = db.ents(entKinds)
    unused = []
    for ent in ents:
        if re.search('standard', ent.library()) or ent.refs("ada rename") or ent.refs("java definein", "java abstract method") or (ent.kind().check("java method") and ent.name() == "main"):
            continue
        # Filter out ents with the following refs
        if not ent.refs("""associationby,\
                        base,\
                        callby ~inactive,\
                        derive,\
                        java callby,\
                        java castby,\
                        java createby,\
                        java dotrefby,\
                        java extendby,\
                        java implementby,\
                        java modifyby,\
                        java override,\
                        java setby ~init,\
                        java throwby,\
                        java typedby,\
                        java useby,\
                        derivefrom,\
                        instanceof,\
                        modifyby,\
                        refby,\
                        renameby,\
                        subtypefrom,\
                        typedby,\
                        useby ~inactive,\
                        usepackageby,\
                        usetypeby,\
                        withby"""):

            declRef = getDeclRef(ent)

            # Skip over unresolved entities
            if not declRef:
                continue

            # If not used, check for setbys - may be set where defined but otherwise unused
            setOnDecl = False
            setByRef = []
            if ent.kind().check("object, parameter, variable"):
                setByRef = ent.refs("Setby")
                if len(setByRef) == 1:
                    if duplicateRef(setByRef[0], declRef):
                        setOnDecl = True

            # If there are no setby references, or only one at the point of definition, call it unused
            if len(setByRef) < 1 or setOnDecl:
                unused.append(declRef)

    # Sort unused by kind and print
    if args.kind:
        lastKind = ""
        unused.sort(key=lambda x: x.file().longname())
        for ref in unused:
            thisKind = ref.scope().kindname()

            if thisKind != lastKind:
                print(f'\n{thisKind}')
                lastKind = thisKind

            print('\t', end='')
            print(f'{ref.scope().longname()}', end='')
            print('\t', end='')
            print(f'[File: {ref.file().longname()} Line: {ref.line()}', end='')
            endRef = ref.scope().refs("end")
            if endRef:
                print(f'-{endRef[0].line()}')

            else:
                print()

    # Sort unused by file and print
    else:
        lastFile = ""
        unused.sort(key=lambda x: x.file().longname())
        for ref in unused:
            thisFile = ref.file().longname()

            if thisFile != lastFile:
                print(f'\n{thisFile}')
                lastFile = thisFile

            print('\t', end='')
            print(f'{ref.scope().longname()}', end='')
            print('\t', end='')
            print(f'[File: {ref.file().longname()} Line: {ref.line()}', end='')

            endRef = ref.scope().refs("end")

            if endRef:
                print(f'-{endRef[0].line()}')

            else:
                print()


# -*- coding: utf-8 -*-
"""
Created on Wed Mar 14 20:42:21 2018

@author: wuwangchuxin
"""
#import time
import numpy as np
import pandas as pd
import pickle
import os
import statsmodels.api as sm
#os.chdir(r'E:\alpha_min')

#start=time.clock()
def standard_progress():
    filenameList = os.listdir(r'E:\alpha_min')
#    filenameList = ['alpha_001.csv']
    for filename in filenameList:
        #首先对因子值进行空值删除和标准化处理
#        start=time.clock()
        data = pd.read_csv(filename)
        data = pd.concat([data.iloc[:,0],data.iloc[:,7:]],axis=1)
        data_c = data.fillna(0)
        data_b = data_c.iloc[:,1:]
        data_d = np.array(data_b)
        x_mean = data_d.mean(axis = 0)
        x_std = data_d.std(axis = 0)
        for i in range(len(data_c.columns)-1):
            for j in range(len(data_c)-1):
                if data_d[j][i] > (x_mean[i]+1.65*x_std[i]):
                    data_d[j][i] = (x_mean[i]+1.65*x_std[i])
                elif data_d[j][i] < (x_mean[i]+1.65*x_std[i]):
                    data_d[j][i] = (x_mean[i]-1.65*x_std[i])
        data_d=pd.DataFrame(data_d,index=list(data_c['code']),columns=list(data_c.columns)[1:])
        data_d.reset_index(inplace=True)
        data_d = data_d.rename(columns={'index':'code'})
        # 存到移动硬盘里
        output = open(r'G:\short_period_mf\alpha_min_stand\standard_%s.pickle'%filename[:9],'wb')
        pickle.dump(data_d,output)
        output.close()
#        end = time.clock()
    return None

def Netual_process(alpha_data,industry):
    new_data = pd.merge(alpha_data,industry,on = ['code'])
    new_data2 = pd.DataFrame(new_data.code)
    for i in range(len(new_data.columns.tolist())-1):
        y = new_data.iloc[:,i+1].as_matrix()
        X = new_data.iloc[:,-30:].as_matrix()
        model = sm.OLS(X,y)
        results = model.fit()
        Betas = results.params #这里没有常数项，所有参数均为回归系数
        new_factors = pd.DataFrame(y - X.dot(Betas.T))
        new_data2 = pd.concat([new_data2,new_factors],axis = 1)
    output = open(r'G:\short_period_mf\netual_process\netual_%s'%saf[-16:],'wb')
    pickle.dump(new_data2,output)
    output.close()
    return None

def IC_computing(path,industry):
    data = pd.read_pickle(path)
    return_data = pd.read_pickle(r'G:\short_period_mf\dailyreturn.pickle')
    return_data = return_data.rename(columns={'symbol':'code'})
    new_data = pd.merge(data,return_data,on = ['code'])
    dailyReturn = new_data.daily_return
    factors = Netual_process(data,industry)
    IC = dailyReturn.corr(factors)
    output = open(r'G:\short_period_mf\ic_value\ic_%s'%saf[-16:],'wb')
    pickle.dump(IC,output)
    output.close()
    return None

industry2=pd.read_pickle(r'C:\Users\wuwangchuxin\Desktop\yinhua_min\data\industry.pkl')
industry3 = industry2.drop_duplicates()
standard_alpha = os.listdir(r'G:\short_period_mf\alpha_min_stand')
for saf in standard_alpha:
    path = r'G:\short_period_mf\alpha_min_stand\%s'%saf
    IC_computing(path,industry)

for saf in standard_alpha:
    alpha_d = pd.read_pickle(r'G:\short_period_mf\alpha_min_stand\%s'%saf)
    Netual_process(alpha_d,industry3)
#
#paths = os.listdir(r'G:\short_period_mf\netual_process')
#for pat in paths:
#    IC_computing(pat,industry)

#
#def IC_computing():
#    data = pd.read_pickle(r'G:\alpha_min_stand\standard_Alpha_001.pickle')
#    return_data = pd.read_pickle('dailyreturn.pickle')
#    return_data = return_data.reset_index()
#    new_data = pd.merge(data,return_data,on = ['code'])
#    dailyReturn = new_data.daily_return
#    factor = new_data.ix[:,0:-1].mean(1)
##    print(factor)
#    IC = dailyReturn.corr(factor)
##    print(IC)
#    return None
#
#
#def dfEWREGBETA(df1, sr, n, halflife=60):
#	df1 = df1.ewm(halflife).mean()
#	sr2 = sr.ewm(halflife).mean()
#    condition_1 = isinstance(sr, np.ndarray)
#    condition_2 = isinstance(sr, pd.core.series.Series)
#    temp = df1.copy()
#    if condition_1:
#        temp.rolling(n).apply(lambda y: stats.linregress(x=sr, y=y)[0])
#    if condition_2:
#        for i in range(0, len(df1) - n):
#            temp.iloc[i:i + n, :].apply(lambda y: stats.linregress(x=sr.iloc[i:i + n], y=y)[0])
#    return temp


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import os
import requests
import time

from .models import DatasetType, SourceType
from marquez_client import errors
from marquez_client import log
from marquez_client.constants import (
    DEFAULT_HOST, DEFAULT_PORT, DEFAULT_TIMEOUT_MS, DEFAULT_NAMESPACE_NAME
)
from marquez_client.version import VERSION
from six.moves.urllib.parse import quote

_API_PATH = 'api/v1'

_USER_AGENT = f'marquez-python/{VERSION}'
_HEADERS = {'User-Agent': _USER_AGENT}

class MarquezClient(object):
    def __init__(self, host=None, port=None,
                 timeout_ms=None, namespace_name=None):
        host = host or os.environ.get('MARQUEZ_HOST', DEFAULT_HOST)
        port = port or os.environ.get('MARQUEZ_PORT', DEFAULT_PORT)
        self._timeout = self._to_seconds(timeout_ms or os.environ.get(
            'MARQUEZ_TIMEOUT_MS', DEFAULT_TIMEOUT_MS)
        )
        self._namespace_name = namespace_name or os.environ.get(
            'MARQUEZ_NAMESPACE', DEFAULT_NAMESPACE_NAME
        )
        self._api_base = f'http://{host}:{port}/{_API_PATH}'

    @property
    def namespace(self):
        return self._namespace_name

    def create_namespace(self, namespace_name, owner_name, description=None):
        if not namespace_name:
            raise ValueError('namespace_name must not be None')
        if not owner_name:
            raise ValueError('owner_name must not be None')

        payload = {
            'ownerName': owner_name
        }

        if description:
            payload['description'] = description

        return self._put(
            self._url('/namespaces/{0}', namespace_name),
            payload=payload
        )

    def get_namespace(self, namespace_name):
        if not namespace_name:
            raise ValueError('namespace_name must not be None')

        return self._get(self._url('/namespaces/{0}', namespace_name))

    def list_namespaces(self, limit=None, offset=None):
        return self._get(
            self._url('/namespaces'),
            params={
                'limit': limit,
                'offset': offset
            }
        )

    def create_job(self, job_name, job_type, location=None, input_dataset=None,
                   output_dataset=None, description=None, context=None,
                   namespace_name=None):
        if not job_name:
            raise ValueError('job_name must not be None')
        if not job_type:
            raise ValueError('job_type must not be None')

        if not namespace_name:
            namespace_name = self._namespace_name

        payload = {
            'inputs': input_dataset or [],
            'outputs': output_dataset or [],
            'type': job_type
        }

        if context:
            payload['context'] = context
        if location:
            payload['location'] = location
        if description:
            payload['description'] = description

        return self._put(
            self._url('/namespaces/{0}/jobs/{1}', namespace_name, job_name),
            payload=payload
        )

    def get_job(self, job_name, namespace_name=None):
        if not job_name:
            raise ValueError('job_name must not be None')

        if not namespace_name:
            namespace_name = self._namespace_name

        return self._get(
            self._url('/namespaces/{0}/jobs/{1}', namespace_name, job_name)
        )

    def list_jobs(self, limit=None, offset=None, namespace_name=None):
        if not namespace_name:
            namespace_name = self._namespace_name

        return self._get(
            self._url('/namespaces/{0}/jobs', namespace_name),
            params={
                'limit': limit,
                'offset': offset
            }
        )

    def create_job_run(self, job_name, nominal_start_time=None,
                       nominal_end_time=None, run_args=None,
                       mark_as_running=False, namespace_name=None):
        if not job_name:
            raise ValueError('job_name must not be None')

        if not namespace_name:
            namespace_name = self._namespace_name

        payload = {}

        if nominal_start_time:
            payload['nominalStartTime'] = nominal_start_time

        if nominal_end_time:
            payload['nominalEndTime'] = nominal_end_time

        if run_args:
            payload['runArgs'] = run_args

        response = self._post(
            self._url('/namespaces/{0}/jobs/{1}/runs',
                      namespace_name, job_name),
            payload=payload)

        if mark_as_running:
            run_id = response['runId']
            response = self.mark_job_run_as_started(run_id)

        return response

    def get_job_run(self, run_id):
        if not run_id:
            raise ValueError('run_id must not be None')

        return self._get(self._url('/jobs/runs/{0}', run_id))

    def list_job_runs(self, job_name, limit=None,
                      offset=None, namespace_name=None):
        if not namespace_name:
            namespace_name = self._namespace_name

        return self._get(
            self._url(
                '/namespaces/{0}/jobs/{1}/runs',
                namespace_name,
                job_name),
            params={
                'limit': limit,
                'offset': offset
            }
        )

    def mark_job_run_as_started(self, run_id):
        return self._mark_job_run_as(run_id, 'start')

    def mark_job_run_as_completed(self, run_id):
        return self._mark_job_run_as(run_id, 'complete')

    def mark_job_run_as_failed(self, run_id):
        return self._mark_job_run_as(run_id, 'fail')

    def mark_job_run_as_aborted(self, run_id):
        return self._mark_job_run_as(run_id, 'abort')

    def _mark_job_run_as(self, run_id, action):
        if not run_id:
            raise ValueError('run_id must not be None')

        return self._post(
            self._url('/jobs/runs/{0}/{1}', run_id, action), payload={}
        )

    # Sources API
    def create_source(self, source_name, source_type, connection_url,
                      description=None):
        if not source_name:
            raise ValueError('source_name must not be None')
        if not connection_url:
            raise ValueError('connection_url must not be None')
        if not isinstance(source_type, SourceType):
            raise ValueError(f'source_type must be instance of SourceType')

        payload = {'type': source_type.value,
                   'connectionUrl': connection_url}

        if description:
            payload['description'] = description

        return self._put(self._url('/sources/{0}', source_name),
                         payload=payload)

    def get_source(self, source_name):
        if not source_name:
            raise ValueError('source_name must not be None')

        return self._get(self._url('/sources/{0}', source_name))

    def list_sources(self, limit=None, offset=None):
        return self._get(
            self._url('/sources'),
            params={
                'limit': limit,
                'offset': offset
            }
        )

    # Datasets API

    def create_dataset(self, dataset_name, dataset_type,
                       physical_name, source_name,
                       description=None, run_id=None,
                       schema_location=None,
                       namespace_name=None):
        if not dataset_name:
            raise ValueError('dataset_name must not be None')
        if not isinstance(dataset_type, DatasetType):
            raise ValueError('dataset_type must be instance of DatasetType')
        if not physical_name:
            raise ValueError('physical_name must not be None')
        if not source_name:
            raise ValueError('source_name must not be None')

        if dataset_type == DatasetType.STREAM and not schema_location:
            raise ValueError('STREAM type datasets must have schema_location')

        if not namespace_name:
            namespace_name = self._namespace_name

        payload = {
            'type': dataset_type.value,
            'physicalName': physical_name,
            'sourceName': source_name,
        }

        if description:
            payload['description'] = description

        if run_id:
            payload['runId'] = run_id

        if schema_location:
            payload['schemaLocation'] = schema_location

        return self._put(
            self._url('/namespaces/{0}/datasets/{1}', namespace_name,
                      dataset_name),
            payload=payload
        )

    def get_dataset(self, dataset_name, namespace_name=None):
        if not dataset_name:
            raise ValueError('dataset_name must not be None')

        if not namespace_name:
            namespace_name = self._namespace_name

        return self._get(
            self._url('/namespaces/{0}/datasets/{1}',
                      namespace_name, dataset_name)
        )

    def list_datasets(self, namespace_name=None, limit=None, offset=None):
        if not namespace_name:
            namespace_name = self._namespace_name

        return self._get(
            self._url('/namespaces/{0}/datasets', namespace_name),
            params={
                'limit': limit,
                'offset': offset
            }
        )

    # Common

    def _url(self, path, *args):
        encoded_args = [quote(arg.encode('utf-8'), safe='') for arg in args]
        return f'{self._api_base}{path.format(*encoded_args)}'

    def _post(self, url, payload, as_json=True):
        now_ms = self._now_ms()

        response = requests.post(
            url=url, headers=_HEADERS, json=payload, timeout=self._timeout)
        log.info(f'{url}', method='POST', payload=json.dumps(
            payload), duration_ms=(self._now_ms() - now_ms))

        return self._response(response, as_json)

    def _put(self, url, payload=None, as_json=True):
        now_ms = self._now_ms()

        response = requests.put(
            url=url, headers=_HEADERS, json=payload, timeout=self._timeout)
        log.info(f'{url}', method='PUT', payload=json.dumps(
            payload), duration_ms=(self._now_ms() - now_ms))

        return self._response(response, as_json)

    def _get(self, url, params=None, as_json=True):
        now_ms = self._now_ms()

        response = requests.get(
            url, params=params, headers=_HEADERS, timeout=self._timeout)
        log.info(f'{url}', method='GET',
                 duration_ms=(self._now_ms() - now_ms))

        return self._response(response, as_json)

    @staticmethod
    def _now_ms():
        return int(round(time.time() * 1000))

    def _response(self, response, as_json):
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError as e:
            self._raise_api_error(e)

        return response.json() if as_json else response.text

    def _raise_api_error(self, e):
        # TODO: https://github.com/MarquezProject/marquez-python/issues/55
        raise errors.APIError() from e

    @staticmethod
    def _to_seconds(timeout_ms):
        return float(timeout_ms) / 1000.0


import sys
import xml.etree.ElementTree as etree

def get_attr_number(node):
    # your code goes here
    res = 0
    for child in node.iter():
        #print(child.tag, child.attrib)
        res += len(child.attrib)
    return res

if __name__ == '__main__':
    # sys.stdin.readline()
    # xml = sys.stdin.read()
    xml = """
        <feed xml:lang='en'>
        <title>HackerRank</title>
        <subtitle lang='en'>Programming challenges</subtitle>
        <link rel='alternate' type='text/html' href='http://hackerrank.com/'/>
        <updated>2013-12-25T12:00:00</updated>
        <entry>
            <author gender='male'>Harsh</author>
            <question type='hard'>XML 1</question>
            <description type='text'>This is related to XML parsing</description>
        </entry>
        </feed>
    """
    tree = etree.ElementTree(etree.fromstring(xml))
    root = tree.getroot()
    print(get_attr_number(root))


#Python 3.8.1 (tags/v3.8.1:1b293b6, Dec 18 2019, 22:39:24) [MSC v.1916 32 bit (Intel)] on win32
#Type "help", "copyright", "credits" or "license()" for more information.
# coding: utf-8

# In[2]:

import pickle
from sklearn.tree import DecisionTreeClassifier
import numpy as np
from sklearn.neighbors import KNeighborsClassifier

# In[3]:

def create_knn(fold_num,max_neighbors):
    x_train,y_train = read_train_csv(fold_num)
    for num in range(0,max_neighbors):
        name_1= "knn/knn_model_"+str(fold_num)+"k="+str(num)+".txt"
        knn = KNeighborsClassifier(n_neighbors = num+1)
        knn.fit(x_train, y_train)
        with open(name_1, 'wb') as f1:
            pickle.dump(knn, f1)
        f1.close()

    # knn = KNeighborsClassifier(n_neighbors=n)
    # knn.fit(x_train,y_train)
    # name_1= "knn/knn_model_"+str(fold_num)+"n="+str()+".txt"
    # with open(name_1, 'wb') as f1:
    #     pickle.dump(knn, f1)
    # f1.close()

# In[1]:

def create_dtc(fold_num):
    x_train,y_train = read_train_csv(fold_num)
    dtc = DecisionTreeClassifier()
    dtc.fit(x_train, y_train)
    name_2 = "dtc/dtc_model_"+str(fold_num)+".txt"
    with open(name_2, 'wb') as f2:
        pickle.dump(dtc, f2)
    f2.close()

# In[ ]:
def read_train_csv(num):
    x_train = np.genfromtxt("C:/Users/Harsh/Desktop/hw5/train.csv/x_train_"+str(num)+".csv",delimiter=",")
    y_train = np.genfromtxt("C:/Users/Harsh/Desktop/hw5/train.csv/y_train_"+str(num)+".csv",delimiter=",")
    return x_train,y_train

# In[ ]:

# In[ ]:


import streamlit as st
import pydeck as pdk
# CREATING FUNCTION FOR MAPS

def draw_map(data, lat, lon, zoom):
    st.write(pdk.Deck(
        map_style="mapbox://styles/mapbox/light-v9",
        initial_view_state={
            "latitude": lat,
            "longitude": lon,
            "zoom": zoom,
            "pitch": 50,
        },
        layers=[
            pdk.Layer(
                "HexagonLayer",
                data=data,
                get_position=["lon", "lat"],
                radius=10000,
                elevation_scale=40,
                elevation_range=[0, 10000],
                pickable=True,
                extruded=True,
            ),
        ]
    ))


from sklearn import datasets, metrics, svm

mnist = datasets.load_digits()

images = mnist.images

data_size = len(images)

#Preprocessing images
images = images.reshape(len(images), -1)
labels = mnist.target

#Initialize Logistic Regression
SVM_classifier = svm.SVC(gamma=0.001)
#Training the data on only 75% of the dataset. Rest of the 25% will be used in testing the Logistic Regression
SVM_classifier.fit(images[:int((data_size / 4) * 3)], labels[:int((data_size / 4) * 3)])

#Testing the data
predictions = SVM_classifier.predict(images[int((data_size / 4)):])
target = labels[int((data_size/4)):]

#Print the performance report of the Logistic Regression model that we learnt
print("Performance Report: \n %s \n" % (metrics.classification_report(target, predictions)))