import collections
import csv
import dataclasses
import os
import tempfile
from typing import List, Optional, Tuple, Union

import flatbuffers

from mediapipe.tasks.metadata import metadata_schema_py_generated as metadata_fb
from mediapipe.tasks.python.metadata import metadata
from mediapipe.tasks.python.metadata.metadata_writers import metadata_info
from mediapipe.tasks.python.metadata.metadata_writers import writer_utils

_INPUT_IMAGE_NAME = 'image'
_INPUT_IMAGE_DESCRIPTION = 'Input image to be processed.'
_INPUT_REGEX_TEXT_NAME = 'input_text'
_INPUT_REGEX_TEXT_DESCRIPTION = ('Embedding vectors representing the input '
                                 'text to be processed.')
_OUTPUT_CLASSIFICATION_NAME = 'score'
_OUTPUT_CLASSIFICATION_DESCRIPTION = 'Score of the labels respectively.'
_OUTPUT_SEGMENTATION_MASKS_NAME = 'segmentation_masks'
_OUTPUT_SEGMENTATION_MASKS_DESCRIPTION = (
    'Masks over the target objects with high accuracy.'
)
# Detection tensor result to be grouped together.
_DETECTION_GROUP_NAME = 'detection_result'
# File name to export score calibration parameters.
_SCORE_CALIBATION_FILENAME = 'score_calibration.txt'

@dataclasses.dataclass
class CalibrationParameter:
  """Parameters for score calibration [1].

  Score calibration is performed on an output tensor through sigmoid functions.
  One of the main purposes of score calibration is to make scores across classes
  comparable, so that a common threshold can be used for all output classes.

  For each index in the output tensor, this applies:
    * `f(x) = scale / (1 + e^-(slope * g(x) + offset))` if `x > min_score` or if
      no `min_score` has been specified.
    * `f(x) = default_score` otherwise or if no scale, slope and offset have
      been specified.

  [1]:
    https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L434
  """
  scale: float
  slope: float
  offset: float
  min_score: Optional[float] = None

@dataclasses.dataclass
class LabelItem:
  """Label item for labels per locale.

  Attributes:
    filename: The file name to save the labels.
    names: A list of label names.
    locale: The specified locale for labels.
  """
  filename: str
  names: List[str]
  locale: Optional[str] = None

@dataclasses.dataclass
class ScoreThresholding:
  """Parameters to performs thresholding on output tensor values [1].

  Attributes:
    global_score_threshold: The recommended global threshold below which results
      are considered low-confidence and should be filtered out.  [1]:
    https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L468
  """
  global_score_threshold: float

@dataclasses.dataclass
class RegexTokenizer:
  """Parameters of the Regex tokenizer [1] metadata information.

  [1]:
    https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L500

  Attributes:
    delim_regex_pattern: the regular expression to segment strings and create
      tokens.
    vocab_file_path: path to the vocabulary file.
  """
  delim_regex_pattern: str
  vocab_file_path: str

@dataclasses.dataclass
class BertTokenizer:
  """Parameters of the Bert tokenizer [1] metadata information.

  [1]:
    https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L477

  Attributes:
    vocab_file_path: path to the vocabulary file.
  """
  vocab_file_path: str

@dataclasses.dataclass
class SentencePieceTokenizer:
  """Parameters of the sentence piece tokenizer tokenizer [1] metadata information.

  [1]:
    https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L485

  Attributes:
    sentence_piece_model_path: path to the sentence piece model file.
    vocab_file_path: path to the vocabulary file.
  """
  sentence_piece_model_path: str
  vocab_file_path: Optional[str] = None

class Labels(object):
  """Simple container holding classification labels of a particular tensor.

  Example usage:
    # The first added label list can be used as category names as needed.
    labels = Labels()
      .add(['/m/011l78', '/m/031d23'])
      .add(['cat', 'dog], 'en')
      .add(['chat', 'chien], 'fr')
  """

  def __init__(self) -> None:
    self._labels = []  # [LabelItem]

  @property
  def labels(self) -> List[LabelItem]:
    return self._labels

  def add(self,
          labels: List[str],
          locale: Optional[str] = None,
          exported_filename: Optional[str] = None) -> 'Labels':
    """Adds labels in the container.

    Args:
      labels: A list of label names, e.g. ['apple', 'pear', 'banana'].
      locale: The specified locale for labels.
      exported_filename: The file name to export the labels. If not set,
        filename defaults to 'labels.txt'.

    Returns:
      The Labels instance, can be used for chained operation.
    """
    if not labels:
      raise ValueError('The list of labels is empty.')

    # Prepare the new item to be inserted
    if not exported_filename:
      exported_filename = 'labels'
      if locale:
        exported_filename += f'_{locale}'
      exported_filename += '.txt'
    item = LabelItem(filename=exported_filename, names=labels, locale=locale)

    # Insert the new element at the end of the list
    self._labels.append(item)
    return self

  def add_from_file(self,
                    label_filepath: str,
                    locale: Optional[str] = None,
                    exported_filename: Optional[str] = None) -> 'Labels':
    """Adds a label file in the container.

    Args:
      label_filepath: File path to read labels. Each line is a label name in the
        file.
      locale: The specified locale for labels.
      exported_filename: The file name to export the labels. If not set,
        filename defaults to 'labels.txt'.

    Returns:
      The Labels instance, can be used for chained operation.
    """

    with open(label_filepath, 'r') as f:
      labels = f.read().split('\n')
      return self.add(labels, locale, exported_filename)

class ScoreCalibration:
  """Simple container holding score calibration related parameters."""

  # A shortcut to avoid client side code importing metadata_fb
  transformation_types = metadata_fb.ScoreTransformationType

  def __init__(self,
               transformation_type: metadata_fb.ScoreTransformationType,
               parameters: List[Optional[CalibrationParameter]],
               default_score: int = 0):
    self.transformation_type = transformation_type
    self.parameters = parameters
    self.default_score = default_score

  @classmethod
  def create_from_file(cls,
                       transformation_type: metadata_fb.ScoreTransformationType,
                       file_path: str,
                       default_score: int = 0) -> 'ScoreCalibration':
    """Creates ScoreCalibration from the file.

    Args:
      transformation_type: type of the function used for transforming the
        uncalibrated score before applying score calibration.
      file_path: file_path of the score calibration file [1]. Contains
        sigmoid-based score calibration parameters, formatted as CSV. Lines
        contain for each index of an output tensor the scale, slope, offset and
        (optional) min_score parameters to be used for sigmoid fitting (in this
        order and in `strtof`-compatible [2] format). Scale should be a
        non-negative value. A line may be left empty to default calibrated
        scores for this index to default_score. In summary, each line should
        thus contain 0, 3 or 4 comma-separated values.
      default_score: the default calibrated score to apply if the uncalibrated
        score is below min_score or if no parameters were specified for a given
        index.
      [1]:
        https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L133
      [2]:
        https://en.cppreference.com/w/c/string/byte/strtof

    Returns:
      A ScoreCalibration object.
    Raises:
      ValueError: if the score_calibration file is malformed.
    """
    with open(file_path, 'r') as calibration_file:
      csv_reader = csv.reader(calibration_file, delimiter=',')
      parameters = []
      for row in csv_reader:
        if not row:
          parameters.append(None)
          continue

        if len(row) != 3 and len(row) != 4:
          raise ValueError(
              f'Expected empty lines or 3 or 4 parameters per line in score'
              f' calibration file, but got {len(row)}.')

        if float(row[0]) < 0:
          raise ValueError(
              f'Expected scale to be a non-negative value, but got '
              f'{float(row[0])}.')

        parameters.append(
            CalibrationParameter(
                scale=float(row[0]),
                slope=float(row[1]),
                offset=float(row[2]),
                min_score=None if len(row) == 3 else float(row[3])))

    return cls(transformation_type, parameters, default_score)

def _fill_default_tensor_names(
    tensor_metadata_list: List[metadata_fb.TensorMetadataT],
    tensor_names_from_model: List[str]):
  """Fills the default tensor names."""
  # If tensor name in metadata is empty, default to the tensor name saved in
  # the model.
  for tensor_metadata, name in zip(tensor_metadata_list,
                                   tensor_names_from_model):
    tensor_metadata.name = tensor_metadata.name or name

def _pair_tensor_metadata(
    tensor_md: List[metadata_info.TensorMd],
    tensor_names_from_model: List[str]) -> List[metadata_info.TensorMd]:
  """Pairs tensor_md according to the tensor names from the model."""
  tensor_names_from_arg = [
      md.tensor_name for md in tensor_md or [] if md.tensor_name is not None
  ]
  if not tensor_names_from_arg:
    return tensor_md

  if collections.Counter(tensor_names_from_arg) != collections.Counter(
      tensor_names_from_model):
    raise ValueError(
        'The tensor names from arguments ({}) do not match the tensor names'
        ' read from the model ({}).'.format(tensor_names_from_arg,
                                            tensor_names_from_model))
  pairs_tensor_md = []
  name_md_dict = dict(zip(tensor_names_from_arg, tensor_md))
  for name in tensor_names_from_model:
    pairs_tensor_md.append(name_md_dict[name])
  return pairs_tensor_md

def _create_metadata_buffer(
    model_buffer: bytearray,
    general_md: Optional[metadata_info.GeneralMd] = None,
    input_md: Optional[List[metadata_info.TensorMd]] = None,
    output_md: Optional[List[metadata_info.TensorMd]] = None,
    input_process_units: Optional[List[metadata_fb.ProcessUnitT]] = None,
    output_group_md: Optional[List[metadata_info.TensorGroupMd]] = None,
    custom_metadata_md: Optional[List[metadata_info.CustomMetadataMd]] = None,
) -> bytearray:
  """Creates a buffer of the metadata.

  Args:
    model_buffer: valid buffer of the model file.
    general_md: general information about the model.
    input_md: metadata information of the input tensors.
    output_md: metadata information of the output tensors.
    input_process_units: a lists of metadata of the input process units [1].
    output_group_md: a list of metadata of output tensor groups [2];
    custom_metadata_md: a lists of custom metadata.
    [1]:
      https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L655
    [2]:
      https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L677

  Returns:
    A buffer of the metadata.

  Raises:
    ValueError: if the tensor names from `input_md` and `output_md` do not
    match the tensor names read from the model.
  """
  # Create input metadata from `input_md`.
  if input_md:
    input_md = _pair_tensor_metadata(
        input_md, writer_utils.get_input_tensor_names(model_buffer))
    input_metadata = [m.create_metadata() for m in input_md]
  else:
    num_input_tensors = writer_utils.get_subgraph(model_buffer).InputsLength()
    input_metadata = [metadata_fb.TensorMetadataT()] * num_input_tensors

  _fill_default_tensor_names(input_metadata,
                             writer_utils.get_input_tensor_names(model_buffer))

  # Create output metadata from `output_md`.
  if output_md:
    output_md = _pair_tensor_metadata(
        output_md, writer_utils.get_output_tensor_names(model_buffer))
    output_metadata = [m.create_metadata() for m in output_md]
  else:
    num_output_tensors = writer_utils.get_subgraph(model_buffer).OutputsLength()
    output_metadata = [metadata_fb.TensorMetadataT()] * num_output_tensors
  _fill_default_tensor_names(output_metadata,
                             writer_utils.get_output_tensor_names(model_buffer))

  # Create the subgraph metadata.
  subgraph_metadata = metadata_fb.SubGraphMetadataT()
  subgraph_metadata.inputTensorMetadata = input_metadata
  subgraph_metadata.outputTensorMetadata = output_metadata
  if input_process_units:
    subgraph_metadata.inputProcessUnits = input_process_units
  if custom_metadata_md:
    subgraph_metadata.customMetadata = [
        m.create_metadata() for m in custom_metadata_md
    ]
  if output_group_md:
    subgraph_metadata.outputTensorGroups = [
        m.create_metadata() for m in output_group_md
    ]

  # Create the whole model metadata.
  if general_md is None:
    general_md = metadata_info.GeneralMd()
  model_metadata = general_md.create_metadata()
  model_metadata.subgraphMetadata = [subgraph_metadata]

  # Get the metadata flatbuffer.
  b = flatbuffers.Builder(0)
  b.Finish(
      model_metadata.Pack(b),
      metadata.MetadataPopulator.METADATA_FILE_IDENTIFIER)
  return b.Output()

class MetadataWriter(object):
  """Generic Metadata writer.

  Example usage:

  For an example model which requires two inputs: image and general feature
  inputs, and generates one output: classification.

  with open(model_path, 'rb') as f:
    writer = MetadataWriter.create(f.read())
    model_content, metadata_json_content = writer
        .add_genernal_info('model_name', 'model description')
        .add_image_input()
        .add_feature_input()
        .add_classification_output(Labels().add(['A', 'B']))
        .populate()
  """

  @classmethod
  def create(cls, model_buffer: bytearray) -> 'MetadataWriter':
    return cls(model_buffer)

  def __init__(self, model_buffer: bytearray) -> None:
    self._model_buffer = model_buffer
    self._general_md = None
    self._input_mds = []
    self._input_process_units = []
    self._output_mds = []
    self._output_group_mds = []
    self._associated_files = []
    self._custom_metadata_mds = []
    self._temp_folder = tempfile.TemporaryDirectory()

  def __del__(self):
    if os.path.exists(self._temp_folder.name):
      self._temp_folder.cleanup()

  def add_general_info(
      self,
      model_name: str,
      model_description: Optional[str] = None) -> 'MetadataWriter':
    """Adds a general info metadata for the general metadata informantion."""
    # Will overwrite the previous `self._general_md` if exists.
    self._general_md = metadata_info.GeneralMd(
        name=model_name, description=model_description)
    return self

  color_space_types = metadata_fb.ColorSpaceType

  def add_feature_input(self,
                        name: Optional[str] = None,
                        description: Optional[str] = None) -> 'MetadataWriter':
    """Adds an input tensor metadata for the general basic feature input."""
    input_md = metadata_info.TensorMd(name=name, description=description)
    self._input_mds.append(input_md)
    return self

  def add_image_input(
      self,
      norm_mean: List[float],
      norm_std: List[float],
      color_space_type: Optional[int] = metadata_fb.ColorSpaceType.RGB,
      name: str = _INPUT_IMAGE_NAME,
      description: str = _INPUT_IMAGE_DESCRIPTION) -> 'MetadataWriter':
    """Adds an input image metadata for the image input.

    Args:
      norm_mean: The mean value used to normalize each input channel. If there
        is only one element in the list, its value will be broadcasted to all
        channels. Also note that norm_mean and norm_std should have the same
        number of elements. [1]
      norm_std: The std value used to normalize each input channel. If there is
        only one element in the list, its value will be broadcasted to all
        channels. [1]
      color_space_type: The color space type of the input image. [2]
      name: Name of the input tensor.
      description: Description of the input tensor.

    Returns:
      The MetadataWriter instance, can be used for chained operation.

    [1]:
      https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L389
    [2]:
      https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L198
    """
    input_md = metadata_info.InputImageTensorMd(
        name=name,
        description=description,
        norm_mean=norm_mean,
        norm_std=norm_std,
        color_space_type=color_space_type,
        tensor_type=self._input_tensor_type(len(self._input_mds)))

    self._input_mds.append(input_md)
    return self

  def add_regex_text_input(
      self,
      regex_tokenizer: RegexTokenizer,
      name: str = _INPUT_REGEX_TEXT_NAME,
      description: str = _INPUT_REGEX_TEXT_DESCRIPTION) -> 'MetadataWriter':
    """Adds an input text metadata for the text input with regex tokenizer.

    Args:
      regex_tokenizer: information of the regex tokenizer [1] used to process
        the input string.
      name: Name of the input tensor.
      description: Description of the input tensor.

    Returns:
      The MetadataWriter instance, can be used for chained operation.

    [1]:
      https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L500
    """
    tokenizer_md = metadata_info.RegexTokenizerMd(
        delim_regex_pattern=regex_tokenizer.delim_regex_pattern,
        vocab_file_path=regex_tokenizer.vocab_file_path)
    input_md = metadata_info.InputTextTensorMd(
        name=name, description=description, tokenizer_md=tokenizer_md)
    self._input_mds.append(input_md)
    self._associated_files.append(regex_tokenizer.vocab_file_path)
    return self

  def add_bert_text_input(self, tokenizer: Union[BertTokenizer,
                                                 SentencePieceTokenizer],
                          ids_name: str, mask_name: str,
                          segment_name: str) -> 'MetadataWriter':
    """Adds an metadata for the text input with bert / sentencepiece tokenizer.

    `ids_name`, `mask_name`, and `segment_name` correspond to the `Tensor.name`
    in the TFLite schema, which help to determine the tensor order when
    populating metadata.

    Args:
      tokenizer: information of the tokenizer used to process the input string,
        if any. Supported tokenziers are: `BertTokenizer` [1] and
        `SentencePieceTokenizer` [2].
      ids_name: name of the ids tensor, which represents the tokenized ids of
        the input text.
      mask_name: name of the mask tensor, which represents the mask with `1` for
        real tokens and `0` for padding tokens.
      segment_name: name of the segment ids tensor, where `0` stands for the
        first sequence, and `1` stands for the second sequence if exists.

    Returns:
      The MetadataWriter instance, can be used for chained operation.

    Raises:
      ValueError: if the type tokenizer is not BertTokenizer or
        SentencePieceTokenizer.

    [1]:
      https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L477
    [2]:
      https://github.com/google/mediapipe/blob/f8af41b1eb49ff4bdad756ff19d1d36f486be614/mediapipe/tasks/metadata/metadata_schema.fbs#L485
    """
    if isinstance(tokenizer, BertTokenizer):
      tokenizer_md = metadata_info.BertTokenizerMd(
          vocab_file_path=tokenizer.vocab_file_path)
    elif isinstance(tokenizer, SentencePieceTokenizer):
      tokenizer_md = metadata_info.SentencePieceTokenizerMd(
          sentence_piece_model_path=tokenizer.sentence_piece_model_path,
          vocab_file_path=tokenizer.vocab_file_path)
    else:
      raise ValueError(
          f'The type of tokenizer, {type(tokenizer)}, is unsupported')
    bert_input_md = metadata_info.BertInputTensorsMd(
        self._model_buffer,
        ids_name,
        mask_name,
        segment_name,
        tokenizer_md=tokenizer_md)

    self._input_mds.extend(bert_input_md.input_md)
    self._associated_files.extend(
        bert_input_md.get_tokenizer_associated_files())
    self._input_process_units.extend(
        bert_input_md.create_input_process_unit_metadata())
    return self

  def add_classification_output(
      self,
      labels: Optional[Labels] = None,
      score_calibration: Optional[ScoreCalibration] = None,
      score_thresholding: Optional[ScoreThresholding] = None,
      name: str = _OUTPUT_CLASSIFICATION_NAME,
      description: str = _OUTPUT_CLASSIFICATION_DESCRIPTION
  ) -> 'MetadataWriter':
    """Add a classification head metadata for classification output tensor.

    Example usage:
      writer.add_classification_output(
        Labels()
          .add(['/m/011l78', '/m/031d23'])
          .add(['cat', 'dog], 'en')
          .add(['chat', 'chien], 'fr')
          )

    Args:
      labels: an instance of Labels helper class.
      score_calibration: an instance of ScoreCalibration helper class.
      score_thresholding: an instance of ScoreThresholding.
      name: Metadata name of the tensor. Note that this is different from tensor
        name in the flatbuffer.
      description: human readable description of what the output is.

    Returns:
      The current Writer instance to allow chained operation.
    """
    calibration_md = self._create_score_calibration_md(score_calibration)
    score_thresholding_md = None
    if score_thresholding:
      score_thresholding_md = metadata_info.ScoreThresholdingMd(
          score_thresholding.global_score_threshold)

    label_files = self._create_label_file_md(labels)
    output_md = metadata_info.ClassificationTensorMd(
        name=name,
        description=description,
        label_files=label_files,
        tensor_type=self._output_tensor_type(len(self._output_mds)),
        score_calibration_md=calibration_md,
        score_thresholding_md=score_thresholding_md,
    )
    self._output_mds.append(output_md)
    return self

  def add_detection_output(
      self,
      labels: Optional[Labels] = None,
      score_calibration: Optional[ScoreCalibration] = None,
      group_name: str = _DETECTION_GROUP_NAME,
  ) -> 'MetadataWriter':
    """Adds a detection head metadata for detection output tensor of models with postprocessing.

    Args:
      labels: an instance of Labels helper class.
      score_calibration: an instance of ScoreCalibration helper class.
      group_name: name of output tensor group.

    Returns:
      The current Writer instance to allow chained operation.
    """
    calibration_md = self._create_score_calibration_md(score_calibration)
    label_files = self._create_label_file_md(labels)
    detection_output_mds = metadata_info.DetectionOutputTensorsMd(
        self._model_buffer,
        label_files=label_files,
        score_calibration_md=calibration_md,
    ).output_mds
    self._output_mds.extend(detection_output_mds)
    # Outputs are location, category, score, number of detections.
    if len(detection_output_mds) != 4:
      raise ValueError('The size of detections output should be 4.')
    # The first 3 tensors (location, category, score) are grouped.
    group_md = metadata_info.TensorGroupMd(
        name=group_name,
        tensor_names=[output_md.name for output_md in detection_output_mds[:3]],
    )
    self._output_group_mds.append(group_md)
    return self

  def add_raw_detection_output(
      self,
      labels: Optional[Labels] = None,
      output_tensors_order: metadata_info.RawDetectionOutputTensorsOrder = metadata_info.RawDetectionOutputTensorsOrder.UNSPECIFIED,
  ) -> 'MetadataWriter':
    """Adds a detection head metadata for detection output tensor of models without postprocessing.

    Args:
      labels: an instance of Labels helper class.
      output_tensors_order: the order of the output tensors. For models of
        out-of-graph non-maximum-suppression only.

    Returns:
      The current Writer instance to allow chained operation.
    """
    label_files = self._create_label_file_md(labels)
    detection_output_mds = metadata_info.RawDetectionOutputTensorsMd(
        self._model_buffer,
        label_files=label_files,
        output_tensors_order=output_tensors_order,
    ).output_mds
    self._output_mds.extend(detection_output_mds)
    # Outputs are location, score.
    if len(detection_output_mds) != 2:
      raise ValueError('The size of detections output should be 2.')
    return self

  def add_segmentation_output(
      self,
      labels: Optional[Labels] = None,
      name: str = _OUTPUT_SEGMENTATION_MASKS_NAME,
      description: str = _OUTPUT_SEGMENTATION_MASKS_DESCRIPTION,
  ) -> 'MetadataWriter':
    """Adds a segmentation head metadata for segmentation output tensor.

    Args:
      labels: an instance of Labels helper class.
      name: Metadata name of the tensor. Note that this is different from tensor
        name in the flatbuffer.
      description: human readable description of what the output is.

    Returns:
      The current Writer instance to allow chained operation.
    """
    label_files = self._create_label_file_md(labels)
    output_md = metadata_info.SegmentationMaskMd(
        name=name,
        description=description,
        label_files=label_files,
    )
    self._output_mds.append(output_md)
    return self

  def add_feature_output(self,
                         name: Optional[str] = None,
                         description: Optional[str] = None) -> 'MetadataWriter':
    """Adds an output tensor metadata for the general basic feature output."""
    output_md = metadata_info.TensorMd(name=name, description=description)
    self._output_mds.append(output_md)
    return self

  def add_custom_metadata(
      self, custom_metadata_md: metadata_info.CustomMetadataMd
  ) -> 'MetadataWriter':
    self._custom_metadata_mds.append(custom_metadata_md)
    return self

  def populate(self) -> Tuple[bytearray, str]:
    """Populates metadata into the TFLite file.

    Note that only the output tflite is used for deployment. The output JSON
    content is used to interpret the metadata content.

    Returns:
      A tuple of (model_with_metadata_in_bytes, metadata_json_content)
    """
    # Populates metadata and associated files into TFLite model buffer.
    populator = metadata.MetadataPopulator.with_model_buffer(self._model_buffer)
    metadata_buffer = _create_metadata_buffer(
        model_buffer=self._model_buffer,
        general_md=self._general_md,
        input_md=self._input_mds,
        output_md=self._output_mds,
        input_process_units=self._input_process_units,
        custom_metadata_md=self._custom_metadata_mds,
        output_group_md=self._output_group_mds,
    )
    populator.load_metadata_buffer(metadata_buffer)
    if self._associated_files:
      populator.load_associated_files(self._associated_files)
    populator.populate()
    tflite_content = populator.get_model_buffer()

    displayer = metadata.MetadataDisplayer.with_model_buffer(tflite_content)
    metadata_json_content = displayer.get_metadata_json()

    return tflite_content, metadata_json_content

  def _input_tensor_type(self, idx):
    return writer_utils.get_input_tensor_types(self._model_buffer)[idx]

  def _output_tensor_type(self, idx):
    return writer_utils.get_output_tensor_types(self._model_buffer)[idx]

  def _export_labels(self, filename: str, index_to_label: List[str]) -> str:
    filepath = os.path.join(self._temp_folder.name, filename)
    with open(filepath, 'w') as f:
      f.write('\n'.join(index_to_label))
    self._associated_files.append(filepath)
    return filepath

  def _export_calibration_file(self, filename: str,
                               calibrations: List[CalibrationParameter]) -> str:
    """Stores calibration parameters in a csv file."""
    filepath = os.path.join(self._temp_folder.name, filename)
    with open(filepath, 'w') as f:
      for item in calibrations:
        if item:
          if item.scale is None or item.slope is None or item.offset is None:
            raise ValueError('scale, slope and offset values can not be set to '
                             'None.')
          elif item.min_score is not None:
            f.write(f'{item.scale},{item.slope},{item.offset},{item.min_score}')
          else:
            f.write(f'{item.scale},{item.slope},{item.offset}')
        f.write('\n')

    self._associated_files.append(filepath)
    return filepath

  def _create_score_calibration_md(
      self, score_calibration: ScoreCalibration
  ) -> Optional[metadata_info.ScoreCalibrationMd]:
    """Creates the ScoreCalibrationMd object."""
    if score_calibration is None:
      return None
    return metadata_info.ScoreCalibrationMd(
        score_transformation_type=score_calibration.transformation_type,
        default_score=score_calibration.default_score,
        file_path=self._export_calibration_file(
            _SCORE_CALIBATION_FILENAME, score_calibration.parameters
        ),
    )

  def _create_label_file_md(
      self, labels: Optional[Labels] = None
  ) -> Optional[List[metadata_info.LabelFileMd]]:
    """Creates a list of LabelFileMd objects."""
    label_files = None
    if labels:
      label_files = []
      for item in labels.labels:
        label_files.append(
            metadata_info.LabelFileMd(
                self._export_labels(item.filename, item.names),
                locale=item.locale,
            )
        )
    return label_files

class MetadataWriterBase:
  """Base MetadataWriter class which contains the apis exposed to users.

  MetadataWriter for Tasks e.g. image classifier / object detector will inherit
  this class for their own usage.
  """

  def __init__(self, writer: MetadataWriter) -> None:
    self.writer = writer

  def populate(self) -> Tuple[bytearray, str]:
    """Populates metadata into the TFLite file.

    Note that only the output tflite is used for deployment. The output JSON
    content is used to interpret the metadata content.

    Returns:
      A tuple of (model_with_metadata_in_bytes, metadata_json_content)
    """
    return self.writer.populate()


#!/usr/bin/env python3

import argparse
import pathlib

import pandas as pd

import gradescope_mean

parser = argparse.ArgumentParser(
    description='grade synthesis from gradescope (see doc at: https://github.com/matthigger/gradescope_mean/blob/main/readme.md)')
parser.add_argument('f_scope', type=str,
                    help='gradescope output csv (Assignments > Download Grades > CSV)')
parser.add_argument('--config', dest='f_config', action='store',
                    default=None,
                    help='yaml configuration (see doc at: https://github.com/matthigger/gradescope_mean/blob/main/doc/config.md)')
parser.add_argument('--plot', dest='plot_flag', action='store',
                    default=True,
                    help='html of histogram plot per assignment category ('
                         'none generated if no html output file specified)')
parser.add_argument('--late_csv', dest='f_late_csv', action='store',
                    default=None,
                    help='csv of late days applied per assignment')
parser.add_argument('--per_student', dest='per_stud', action='store_true',
                    help='outputs csv per student')

def main(args=None):
    if args is None:
        args = parser.parse_args()

    # load config
    folder = pathlib.Path(args.f_scope).resolve().parent
    if args.f_config is None:
        config = gradescope_mean.Config.cli_copy_config(folder)
    else:
        config = gradescope_mean.Config.from_file(args.f_config)

    # process
    gradebook, df_grade_full = config(f_scope=args.f_scope)

    # output
    df_grade_full.to_csv(str(folder / 'grade_full.csv'))

    # outputs csv per student
    if args.per_stud:
        _folder = folder / 'per_student'
        _folder.mkdir(exist_ok=True)
        for idx, row in df_grade_full.iterrows():
            _df = pd.DataFrame(row)
            last = row['last name']
            first = row['first name']
            _df.to_csv(_folder / f'{last}_{first}.csv')

    # print late days to csv
    if args.f_late_csv is not None:
        f_late = folder / args.f_late_csv
        gradebook.df_lateday.to_csv(f_late.with_suffix('.csv'))

    # plot
    if args.plot_flag:
        fig = gradescope_mean.plot_hist(df_grade_full=df_grade_full,
                                        cat_weight_dict=config.cat_weight_dict)
        f_html = folder / f'hist.html'
        fig.write_html(str(f_html), include_plotlyjs='cdn')

if __name__ == '__main__':
    main()


import requests
from bs4 import BeautifulSoup
import pandas as pd
import re

URL = "https://en.wikipedia.org/wiki/List_of_jet_airliners"

response = requests.get(URL)  # page

soup = BeautifulSoup(response.text, 'html.parser')  # scraped

tables = soup.find_all('table', {"class":"wikitable sortable"})  #all tables from page
total = [] # array to contain plane objects

for table in tables:

    rows = table.find_all('tr')  #each row in plane

    columns = [v.text.replace('\n', "") for v in rows[0].find_all('th')]  #finds headers, puts them into columns list
    df = pd.DataFrame(columns=columns)  #empty column data structure

    for i in range(1, len(rows)):
        tds = rows[i].find_all('td')  #each table cell

        values = [re.sub("[\(\[].*?[\)\]]", "", td.text.replace('\n', "").replace(',', "")) for td in tds]  #adds table cell to a values array

        df = df.append(pd.Series(values, index=columns),ignore_index=True)  #puts values array into df datastructure under respective column

    total.append(df)  #add df to total array
final = pd.concat(total, ignore_index=True)  #adds all of total to a final data structure

#column formating - combining columns with *slightly* different titles
final["Origin"] = final["Origin"].astype(str).replace('nan', "") + final["Country"].astype(str).replace('nan', "")
final["Engines"] = final["Engines[a]"].astype(str).replace('nan', "") + final["Engines[c]"].astype(str).replace('nan', "") + final["Engines[f]"].astype(str).replace('nan', "")
final["In Service"] = final["In Service [1][b]"].astype(str).replace('nan', "") + final["In Service (2020)[1][d]"].astype(str).replace('nan', "")
final = final.drop(["Origin", "Country", "Engines[a]", "Engines[c]", "Engines[f]","In Service [1][b]","In Service (2020)[1][d]"], axis=1)

#row formatting - deleting cancelled or only planned planes
final = final[~final["First Flight"].str.contains("planned")]
final = final[~final["First Flight"].str.contains("cancelled")]
final = final[~final["Airline service entry"].str.contains("planned")]
final = final[~final["Airline service entry"].str.contains("cancelled")]

#export to csv file
final.to_csv(r'planes.csv')


import sys
from lxml import etree

xmlfile = sys.argv[1]
master_tp_file = sys.argv[2]
take_tp_files = sys.argv[3:]

def main():
    with open(xmlfile) as fh:
        doc = etree.parse(fh)

    #clip, asset-clip, ref-clip are all ok
    clips = doc.xpath("library/event/project/sequence/spine/clip|library/event/project/sequence/spine/asset-clip|library/event/project/sequence/spine/ref-clip")

    log(f"Number of clips in xml : {len(clips)}")
    log(f"Number of Take sp files: {len(take_tp_files)}")
    assert len(clips) == len(take_tp_files)

    master_tp = readTimePointFile(master_tp_file)

    nr = 0
    while len(clips) > nr:
        take_tp = readTimePointFile(take_tp_files[nr])
        assert len(master_tp) == len(take_tp)
        addTimepointsToClip(clips[nr], master_tp, take_tp)
        nr += 1

    etree.indent(doc, "    ")
    print(str(etree.tostring(doc, xml_declaration=True), "utf-8"))

def log(msg):
    sys.stderr.write(f"{msg}\n")

def readTimePointFile(filename):
    with open(filename) as fh:
        lines = fh.readlines()
    tps = []
    for line in lines:
        tps.append(float(line.split("\t")[0]))
    return tps

def addTimepointsToClip(clip, master_tp, take_tp):
    log(clip)
    #log(take_tp)

    timeMaps = clip.xpath(".//timeMap")
    for timeMap in timeMaps:
        log(timeMap)
        addTimepointsToTimeMap(timeMap, master_tp, take_tp)

def addTimepointsToTimeMap(timeMap, master_tp, take_tp):
    #first_timept = timeMap[0]
    second_timept = timeMap[1]
    zeropoint_string = second_timept.get("time")

    #save last timepoint in timeMap, to append finally
    last_timept = timeMap[-1]
    timeMap.remove(last_timept)

    #remove second last timepoint in timeMap, it will be replaced by last in tp
    #print(dir(timeMap))
    second_last_timept = timeMap[-2]
    timeMap.remove(second_last_timept)

    (zeropoint, fps) = zeropoint_string[:-1].split("/")
    #log(fps)
    factor = 50/int(fps)
    #log(factor)
    zeropoint_50f = int(int(zeropoint)*factor)
    log(f"zeropoint_50f: {zeropoint_50f}")

    nr = 0
    while len(master_tp) > nr:
        master_timepoint = master_tp[nr]
        take_timepoint = take_tp[nr]

        master_timepoint_50f = f"{int(master_timepoint*50+zeropoint_50f)}/50s"
        take_timepoint_50f = f"{int(take_timepoint*50+zeropoint_50f)}/50s"

        new_timept = etree.Element("timept")
        new_timept.set("interp", "linear")
        #new_timept.set("value", master_timepoint_50f)
        #new_timept.set("time", take_timepoint_50f)
        new_timept.set("time", master_timepoint_50f)
        new_timept.set("value", take_timepoint_50f)

        #log(etree.tostring(new_timept))
        timeMap.append(new_timept)

        nr += 1

    timeMap.append(last_timept)

main()


from typing import Dict, Tuple

import numpy as np

import visualization
from map import SimulationMap

class AutoExpandingMap:
    def render_chunk(self, chunk_x, chunk_y):
        self.assert_exist_chunk((chunk_x, chunk_y))
        img = np.zeros((self.chunk_y * 16, self.chunk_x * 16, 3), dtype=np.uint8)
        for x in range(self.chunk_x):
            for y in range(self.chunk_y):
                img[y * 16:(y + 1) * 16, x * 16:(x + 1) * 16, :] = self.tilemap[
                    self.chunks[(chunk_x, chunk_y)].type_map[x, y]]
        return img

    def __init__(self, chunk_shape):
        self.chunks: Dict[Tuple[int, int], SimulationMap] = {}
        self.chunk_x, self.chunk_y = chunk_shape
        self.tilemap = visualization.TileSet(16)

    def __getitem__(self, args):
        x, y = args
        chunk_x, inchunk_x = divmod(x, self.chunk_x)
        chunk_y, inchunk_y = divmod(y, self.chunk_y)
        chunk_index = (chunk_x, chunk_y)
        self.assert_exist_chunk(chunk_index)
        return self.chunks[chunk_index].height_map[inchunk_x, inchunk_y], \
               self.chunks[chunk_index].type_map[inchunk_x, inchunk_y]

    def assert_exist_chunk(self, chunk_index):
        if chunk_index not in self.chunks:
            self.chunks[chunk_index] = SimulationMap((self.chunk_x, self.chunk_y), *chunk_index)


#import numpy #删除列表重复的
import requests
from bs4 import BeautifulSoup
import re
import urllib3
urllib3.disable_warnings()  # 忽略https证书告警

def DNS_Climb_censys(DNS):
    # try:
    amount = 4  # 默认页数

    Dns_List = []
    headers={
        'Host': 'search.censys.io',
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Firefox/91.0',
        'Accept': '*/*',
        'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
        'Accept-Encoding': 'gzip, deflate',
        'X-Requested-With': 'XMLHttpRequest',
        'Connection': 'close',
        'Referer': 'https://search.censys.io',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-origin',
    }

    for i in range(int(amount)):
        url=f"https://search.censys.io/certificates/_search?q={DNS}&page={i}"
        html=requests.get(url=url,headers=headers,verify=False)
        soup = BeautifulSoup(html.text, 'lxml')
        dns_ = soup.find_all(text=re.compile(rf"{DNS}"))
        for i in dns_: # 临时发挥写的现在我也懵看不太懂
            a=re.sub('\*=', '', i)
            a=a.split(',')[0]
            a = a.split('=')
            if len(a)==2:
                if not (a in Dns_List):  # 取反如果Dns_List列表里面有就不添加
                    Dns_List.append(a[1])

    #Dns_List=numpy.unique(Dns_List)
    return Dns_List

    # except Exception as bc:
    #     print("有错误！错误提示" + str(bc))

if __name__=="__main__":
    print(DNS_Climb_censys('baidu.com'))


import copy
import random
from typing import Tuple, Union

import numpy as np
import gym
from gym import spaces
from tetris_environment.tetris_engine import UnrenderedTetrisGame
from tetris_environment.rendering_tetris_engine import RenderingTetrisGame

SCREEN_WIDTH, SCREEN_HEIGHT = 200, 400

class TetrisEnv(gym.Env):
    metadata = {'render.modes': ['human', 'rgb_array']}

    def __init__(self, type: str, render: bool = False, low: int = -3, high: int = 3):

        if type not in ("fourer", 'regular', 'extended fourer'):
            raise RuntimeError("Invalid Tetris type")

        # open up a game state to communicate with emulator
        if not render:
            self.game_state = UnrenderedTetrisGame(type)
            self.game_type = UnrenderedTetrisGame
        else:
            self.game_state = RenderingTetrisGame(type)
            self.game_type = RenderingTetrisGame
            # raise RuntimeError()

        self._action_set = self.game_state.get_action_set()
        self.action_space = spaces.Discrete(len(self._action_set))
        self.observation_space = spaces.Box(low=low, high=high, shape=(9,), dtype=int)
        self.viewer = None
        self.type = type
        self.rendering = render

    def step(self, a: int) -> Tuple[tuple, float, bool, dict]:
        """
        Takes one step in the environment using the defined action a
        :param a: the action to take
        :return: a 4-tuple consisting of
                • the current encoded state (see get_encoded_state())
                • the reward received
                • whether or not the end of an epîsode has been reached
                • several observations with their own labels
                    - label "score" (int): the total score received during this step and this step alone
                    - label "new_piece (bool): is True when a new falling piece was added to the board
                    - label "lines_cleared" (int): the total amount of lines cleared during this step
        """
        self._action_set = np.zeros([len(self._action_set)])
        self._action_set[a] = 1
        _, reward, terminal, observations = self.game_state.frame_step(self._action_set)
        state = self.get_encoded_state()
        return state, reward, terminal, observations

    @property
    def n_actions(self):
        return len(self._action_set)

    # return: (states, observations)
    def reset(self):
        do_nothing = np.zeros(len(self._action_set))
        do_nothing[0] = 1
        # self.observation_space = spaces.Box(low=0, high=255, shape=(SCREEN_WIDTH, SCREEN_HEIGHT, 3))
        self.observation_space = spaces.Box(low=-3, high=3, shape=(9,), dtype=int)
        self.game_state.frame_step(do_nothing)
        state = self.get_encoded_state()
        return state

    def render(self, mode='human', close=False):

        if not self.rendering:
            return

        if close:
            if self.viewer is not None:
                self.viewer.close()
                self.viewer = None
            return
        img = self.game_state.get_image()
        if mode == 'rgb_array':
            return img
        elif mode == 'human':
            from gym.envs.classic_control import rendering
            if self.viewer is None:
                self.viewer = rendering.SimpleImageViewer()
            self.viewer.imshow(img)

    def get_encoded_state(self, board=None) -> tuple:
        """
        Encodes the state space from the 10-by-20 (for a normal Tetris game) board to an integer array of size 9 by
        only noting the height differences between adjacent columns. If this difference is greater than 3 or
        smaller than -3, the value is equal to 3 or -3 respectively.
        :param board: the Tetris board to be encoded. If none is provided, the board of the class is used
        :return: a tuple of size board_width - 1 containing h_(i+1)-h_i in the i'th spot.
        """
        if board is None:
            board = self.game_state

        board_width = board.board_width
        state = [0 for _ in range(board_width - 1)]  # list of zeroes

        low = -3
        high = 3

        for i in range(board_width - 1):
            height_diff = board.get_column_height(i + 1) - board.get_column_height(i)
            if height_diff < low:
                height_diff = low
            elif height_diff > high:
                height_diff = high
            state[i] = height_diff

        return tuple(state)  # finally, convert to hashable type, i.e. tuple

    def all_possible_placements(self):
        """
        Move the piece from all the way left to all the way right, and rotate it in all possible ways.
        Then return the state achieved by dropping the piece down and the first (of several) actions to
        take to achieve this state
        :return:
        """
        all_possible_positions = []

        piece = self.game_state.fallingPiece
        if piece is not None:
            for width in range(-4, self.game_state.board_width + 4):
                shape = piece['shape']
                for rotation in range(len(self.game_state.pieces.get(shape))):
                    new_piece = copy.deepcopy(piece)
                    new_piece['x'] = width
                    new_piece['rotation'] = rotation
                    if self.game_state.is_valid_position(piece=new_piece):
                        all_possible_positions.insert(random.randint(0, len(all_possible_positions)), new_piece)

            all_possible_placements = []
            for piece in all_possible_positions:
                board = copy.deepcopy(self.game_state.board)
                new_game = self.game_type(board=board, type=self.type)
                new_game.fallingPiece = piece
                actions = self._get_actions(old_board=self.game_state, new_board=new_game)
                move_down = np.zeros(6)
                move_down[4] = 1
                new_game.frame_step(move_down)
                state = self.get_encoded_state(new_game)
                all_possible_placements.append((state, actions))

            return all_possible_placements
        else:
            return []

    @staticmethod
    def _get_actions(old_board, new_board):
        """
        Finds the actions required to position the falling piece on :param old_board in the same way as
        the falling piece on :param new_board
        :return: a list of all actions required (first rotations, then lateral movement). The last action added
                is the move_down action
        :raise AssertionError if the shapes of the falling pieces are different
        """
        # Define actions
        move_left = 1
        move_right = 3
        move_down = 4
        rotate_clockwise = 5

        actions = []

        curr_piece = old_board.fallingPiece
        target_piece = new_board.fallingPiece

        assert curr_piece['shape'] == target_piece['shape']

        if curr_piece['rotation'] != target_piece['rotation']:
            for _ in range(abs(curr_piece["rotation"] - target_piece["rotation"])):
                actions.append(rotate_clockwise)
        if curr_piece['x'] < target_piece['x']:
            for _ in range(target_piece['x'] - curr_piece['x']):
                actions.append(move_right)
        if curr_piece['x'] > target_piece['x']:
            for _ in range(curr_piece['x'] - target_piece['x']):
                actions.append(move_left)
        if len(actions) == 0:
            # piece placements are identical, so no actions are added.
            pass

        actions.append(move_down)

        return actions

    def get_falling_piece(self) -> Union[tuple, None]:
        """
        :return: a tuple of the form (piece.shape, piece.x, piece.y, piece.rotation)
        """
        return self.game_state.get_falling_piece()

    @property
    def no_move(self):
        return 0


from pycaret.datasets import get_data
from pycaret.classification import *
from Species import Species
from Model import Model
from FileService import FileService
from Prints import Prints
from StoreService import StoreService
from StoreRepository import StoreRepository
from StoreDTO import StoreDTO
from Menu import Menu

class Prediction(StoreService, StoreRepository, Model, object):

    fileName = ''
    fileNameWithExt = ''

    PrintsService = None

    def __init__(self, *args):
        pass
        self.PrintsService = Prints()
        self.PrintsService.print_options()

    def input_features(self):
        data = input('ENTER FEATURES: ')
        self.features = data.split()
        if len(self.features) < 8:
            print("INVALID INPUT FEATURES")
            self.input_features()

    def print_features(self):
        self.PrintsService.print_features(self.features)

    def input_prediction(self):

        self.PrintsService.print_options()

        winning = input("ENTER WINNING RESULT: ")

        if winning == "": self.input_prediction()
        if winning in [member.value for member in Menu]:

            if winning == Menu("RESET").value:
                self.features = []
                print(self.features)
                self.input_prediction()

            elif winning == Menu("BACK").value:
                self.features = self.featuresHistory.copy()
                print(self.features)
                self.input_prediction()

            elif winning == Menu("QUIT").value:
                quit()

        self.featuresHistory = self.features.copy()
        self.features.append(winning)

        if len(self.features) > 24:
            self.features.pop(0)
        else:
            print(len(self.features))
            print(self.features)
            self.input_prediction()

        row = self.features.copy()

        prev_rediction = self.prediction_list.copy()

        self.saveToFile(row, self.fileNameWithExt)
        self.predictFeature() # starts predicting

        # update stats
        self.setStat(Species(int(winning)).name, 1)
        self.print_features()

        setVeggies = self.PrintsService.set_veggies(self.prediction_list)
        self.set_prediction_list(setVeggies)
        self.PrintsService.numeric_to_alpha_prediction(setVeggies)

        self.setWinLossStat(winning, prev_rediction)

        # get win and percentage
        print(f"\n")

        predict_logic = Prints()
        self.prediction_list =  predict_logic.prediction_logic(self.prediction)

        print(f"PREVIOUS WINNING RESULT: {Species(int(winning)).name}")

        self.setVeggieHitCount(winning, self.prediction_list)
        print(f"\n")

        self.resetStatValue(Species(int(winning)).name, 0)
        for idx in range(10): self.setStat(Species(int(idx)).name, 1)
        self.printStats()

        self.input_prediction()

    def predictFeature(self):
        data = self.predictNext(data=get_data(self.fileName))
        for item in data:
            self.prediction.append(item)

    def set_prediction_list(self, data):
        self.prediction_list = data.copy()

    def printStats(self):

        TOTAL_ROUNDS = self.getTotalRounds()
        WIN_PERCENT = self.getWinPercent()
        LOSS_PERCENT = self.getLossPercent()

        print(f"TOTAL ROUNDS: {TOTAL_ROUNDS}")
        print(f"TOTAL WIN: {self.TOTAL_WIN} {WIN_PERCENT}%, \t TOTAL LOSS: {self.TOTAL_LOSS} {LOSS_PERCENT}%")
        print(f"WINNING STREAK: {self.STREAK_WIN}, \t LOSING STREAK: {self.STREAK_LOSS}")
        print(f"VEGGIE STRIKE: {self.VEGE_STRIKE}, \t VEGGIE SPARE: {self.VEGE_SPARE}")
        print(f"\n")
        print(f"VEGGIE TOP: {self.VEGE_TOP}, \t VEGGIE MID: {self.VEGE_MID}, \t VEGGIE BOT: {self.VEGE_BOT}")
        print(f"\n")
        print(f"HOT_DOG: {self.HOT_DOG}, BBQ: {self.BBQ}, HAM: {self.HAM}, STEAK: {self.STEAK}")
        print(f"TOMATO: {self.TOMATO}, CABBAGE: {self.CABBAGE}, CORN: {self.CORN}, CARROT: {self.CARROT}")
        print(f"\n")


import base64
import boto3
import gzip
import json
import logging
import os
import re
import time

from datetime import datetime, timezone

logging.basicConfig()
logging.getLogger().setLevel(level=logging.INFO)

## fail fast on bad configuration
kinesis_client = boto3.client('kinesis')
kinesis_stream = os.environ['DESTINATION_STREAM_NAME']

python_logging_regex    = re.compile(r'\[([A-Z]+)]\s+([0-9]{4}-.*Z)\s+([-0-9a-fA-F]{36})\s+(.*)')

lambda_start_regex      = re.compile(r'START RequestId:\s+([-0-9a-fA-F]{36})\s+Version:\s+(.+)')
lambda_finish_regex     = re.compile(r'END RequestId:\s+([-0-9a-fA-F]{36})')
lambda_report_regex     = re.compile(r'REPORT RequestId:\s+([-0-9a-fA-F]{36})\s+Duration:\s+([0-9.]+)\s+ms\s+Billed Duration:\s+([0-9]+)\s+ms\s+Memory Size:\s+([0-9]+)\s+MB\s+Max Memory Used:\s+([0-9]+)\s+MB')
lambda_extended_regex   = re.compile(r'.*Init Duration:\s+([0-9.]+)\s+ms')
lambda_xray_regex       = re.compile(r'.*XRAY TraceId:\s+([0-9a-fA-F-]+)\s+SegmentId:\s+([0-9a-fA-F]+)\s+Sampled:\s+(true|false)')

def lambda_handler(event, context):
    output_messages = extract_messages(event)
    logging.info(f'total number of messages to output: {len(output_messages)}')
    logging.debug(f'output messages: {json.dumps(output_messages)}')
    write_to_kinesis(output_messages)

def extract_messages(event):
    """ Source records may contain multiple log messages, so we can't just use a list
        comprehension to extract them. We also want to skip control records, and any
        records that we can't process for unforeseen reasons. Thus this function.
        """
    output_messages = []
    for record in event['Records']:
        output_messages += process_input_record(record)
    return output_messages

def process_input_record(record):
    try:
        payload = record['kinesis']['data']
        decoded = gzip.decompress(base64.b64decode(payload))
        data = json.loads(decoded)
        message_type = data.get('messageType')
        if message_type == 'DATA_MESSAGE':
            log_group = data['logGroup']
            log_stream = data['logStream']
            events = data.get('logEvents', [])
            logging.info(f'processing {len(events)} events from group "{log_group}" / stream "{log_stream}"')
            logging.debug(f'input events: {json.dumps(events)}')
            return [transform_log_event(log_group, log_stream, event) for event in events]
        elif message_type == 'CONTROL_MESSAGE':
            logging.debug('skipping control message')
        elif message_type:
            logging.warning(f'unexpected message type: {message_type}')
    except:
        logging.error(f'failed to process record; keys = {record.keys()}', exc_info=True)
    # fall-through for any unprocessed messages (exception or unhandled message type)
    return []

def transform_log_event(log_group, log_stream, event):
    """ Turns the message into JSON if it isn't already, recognizing standard logging
        formats; adds tracking fields.
        """
    message = event.get('message', '').strip()
    result = try_parse_json(message)
    if not result:
        result = try_parse_python_log(message)
    if not result:
        result = try_parse_lambda_status(message)
    if not result:
        result = {
            'level': 'INFO',
            'message': message
        }

    result['source'] = {
        'logGroup': log_group,
        'logStream': log_stream
    }
    opt_add_timestamp(result, event)
    return result

def try_parse_json(message):
    if message.startswith('{') and message.endswith('}'):
        try:
            return json.loads(message)
        except:
            pass

def try_parse_python_log(message):
    """ Attempts to parse the passed message as output from the Lambda Python logger, as
        documented here: https://docs.aws.amazon.com/lambda/latest/dg/python-logging.html
        """
    match = python_logging_regex.match(message)
    if match:
        return {
            'level':        match.group(1),
            'timestamp':    match.group(2),
            'lambda':       { 'requestId': match.group(3) },
            'message':      match.group(4)
        }

def try_parse_lambda_status(message):
    """ Attempts to parse the message as one of the standard Lambda status messages.
        """
    try:
        if message.startswith('START RequestId:'):
            match = lambda_start_regex.match(message)
            if match:
                data = {
                    'requestId':    match.group(1),
                    'version':      match.group(2)
                }
        elif message.startswith('END RequestId:'):
            match = lambda_finish_regex.match(message)
            if match:
                data = {
                    'requestId':    match.group(1)
                }
        elif message.startswith('REPORT RequestId:'):
            message = message.replace('\n', '\t')
            match = lambda_report_regex.match(message)
            if match:
                data = {
                    'requestId':    match.group(1),
                    'durationMs':   float(match.group(2)),
                    'billedMs':     float(match.group(3)),
                    'maxMemoryMb':  int(match.group(4)),
                    'usedMemoryMb': int(match.group(5))
                }
                # initialization stats are only reported for first invocation
                match = lambda_extended_regex.match(message)
                if match:
                    data['initializationMs'] = float(match.group(1))
                # x-ray stats are only reported if enabled
                match = lambda_xray_regex.match(message)
                if match:
                    data['xrayTraceId'] = match.group(1)
                    data['xraySegment'] = match.group(2)
                    data['xraySampled'] = match.group(3)
        if data:
            return {
                'level':   'INFO',
                'message': message,
                'lambda':  data
            }
    except:
        pass

def opt_add_timestamp(data, event):
    """ If the passed data field already has a "timestamp" element, it's returned
        unchanged. Otherwise the log event's timestamp is formatted and added to
        the message.
        """
    if data.get('timestamp'):
        return
    dt = datetime.fromtimestamp(event['timestamp'] / 1000.0, tz=timezone.utc)
    data['timestamp'] = dt.isoformat()

def write_to_kinesis(listOfEvents):
    """ Makes a best-effort attempt to write all messages to Kinesis, batching them
        as needed to meet the limits of PutRecords.
        """
    records = prepare_records(listOfEvents)
    while records:
        records = process_batch(records)
        if (records):
            time.sleep(2) # an arbitrary sleep; we should rarely hit this
    return

def prepare_records(listOfEvents):
    records = []
    for event in listOfEvents:
        partition_key = event.get('cloudwatch', {}).get('logStream', 'DEFAULT')
        records.append({
            'PartitionKey': partition_key,
            'Data': json.dumps(event)
        })
    return records

def process_batch(records):
    """ Forms a batch from the provided records and attempts to send it; any records
        that won't fit in the batch will be returned, as well as any that couldn't
        be sent (we return unattempted records first to give them the best chance of
        being sent if there are persistent errors).
        """
    to_be_sent, to_be_returned = build_batch(records)
    logging.info(f'sending batch of {len(to_be_sent)} records with {len(to_be_returned)} remaining')
    try:
        response = kinesis_client.put_records(
            StreamName=kinesis_stream,
            Records=to_be_sent
        )
        return process_response(response, to_be_sent) + to_be_returned
    except kinesis_client.exceptions.ProvisionedThroughputExceededException:
        logging.warn(f'received throughput-exceeded on stream {kinesis_stream}; retrying all messages')
        return to_be_sent + to_be_returned

def build_batch(records):
    """ Creates a batch of records based on Kinesis limits; returns both the batch
        and any remaining records that didn't fit in the batch.
        """
    rec_count = 0
    byte_count = 0

    while rec_count < min(len(records), 500) and byte_count < 1048576:
        record = records[rec_count]
        rec_count += 1
        byte_count += len(record['Data']) + len(record['PartitionKey'])

    # we already added the record before we knew the size, so we'll compensate
    if byte_count > 1048576:
        rec_count = rec_count - 1

    # this should never happen: it would require a max-size log event and a
    # long partition key
    if rec_count == 0:
        logging.warn('throwing out too-long message')
        return [], records[1:]

    return records[:rec_count], records[rec_count:]

def process_response(response, records):
    """ Examines the response from PutRecords, returning any records that couldn't be sent.
        """
    if response['FailedRecordCount'] == 0:
        return []

    result = []
    dropped_record_count = 0
    for ii in range(len(response['Records'])):
        entry = response['Records'][ii]
        errorCode = entry.get('ErrorCode')
        if errorCode == 'ProvisionedThroughputExceededException':
            result.append(records[ii])
        elif errorCode:
            dropped_record_count += 1

    if dropped_record_count > 0:
        logging.warn(f'dropped {dropped_record_count} records due to Kinesis internal errors')
    if len(result) > 0:
        logging.info(f'requeueing {len(result)} records due to throughput-exceeded')

    return result


# import the necessary packages
import numpy as np
import imutils
import cv2
from copy import*

#TO-DO: FIND THE REASON WHY IT DOESNT WORK FOR PERSPECTIVE TRANFORMATION (CHECK BOUNDING RECT)
#       FIX COMBINING ALGORITHM

class Stitcher:
	def __init__(self):
		# determine if we are using OpenCV v3.X
		self.isv3 = imutils.is_cv3()

	#DOESNT WORK WITH PERSPECTIVE
	def stitch(self, images, ratio=0.75, reprojThresh=4.0,
		showMatches=False,resize = 400):
		#copy of images
		imgs = []
		for image in images:
			img = cv2.imread(image)
			if resize is not None:
				img = imutils.resize(img, width=resize)
			imgs.append(img)
		globalXOffset = 0
		globalYOffset = 0
		#keypoints
		ks = []
		#features
		fs = []
		#homographies, in the form of i+1 respect to i
		hs = []
		#find keypoints and features
		for image in imgs:
			kp,ft = self.detectAndDescribe(image)
			ks.append(kp)
			fs.append(ft)

		#calculate homographies
		for i in range(len(imgs)-1):
			hs.append((self.matchKeypoints(ks[i+1], ks[i],
			fs[i+1], fs[i], ratio, reprojThresh))[1])

		lt,rb = self.calcBoundingRect(imgs,hs)

		xOffset, yOffset = int(lt[0]),int(lt[1])
		w,h = int(rb[0])-xOffset, int(rb[1])-yOffset

		#create empty image of appropriate size
		main = np.zeros((h,w,3), np.uint8)

		#add identity matrix as the first homography matrix
		hs.insert(0,np.identity(3))

		#instantiate base homography matrix
		homography = np.identity(3)

		for i in range(len(imgs)):
			#update base homography matrix
			homography = homography.dot(hs[i])

			#create a copy of the base homography matrix to
			#translate it without affecting the other homographies
			hTranslated = deepcopy(homography)
			hTranslated[0][2] += -xOffset
			hTranslated[1][2] += -yOffset

			#warpPerspective
			result = cv2.warpPerspective(imgs[i], hTranslated,
			(w, h))
			#cv2.imshow('result'+str(i),result)
			#cv2.imshow('im1',imgs[-1])
			#cv2.imshow('im2',imgs[-2])

			#stich them with masking
			anded = cv2.bitwise_and(result,main)
			gray = cv2.cvtColor(anded, cv2.COLOR_BGR2GRAY)
			ret,binary = cv2.threshold(gray,1,255,cv2.THRESH_BINARY_INV)
			binary = cv2.cvtColor(binary,cv2.COLOR_GRAY2RGB)
			maskedMain = cv2.bitwise_and(main,binary)
			main = cv2.bitwise_or(maskedMain,result)

		cv2.imshow('main',main)
		return result

	def detectAndDescribe(self, image):
		# convert the image to grayscale
		gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

		# check to see if we are using OpenCV 3.X
		if self.isv3:
			# detect and extract features from the image
			descriptor = cv2.xfeatures2d.SIFT_create()
			(kps, features) = descriptor.detectAndCompute(image, None)

		# otherwise, we are using OpenCV 2.4.X
		else:
			# detect keypoints in the image
			detector = cv2.FeatureDetector_create("SIFT")
			kps = detector.detect(gray)

			# extract features from the image
			extractor = cv2.DescriptorExtractor_create("SIFT")
			(kps, features) = extractor.compute(gray, kps)

		# convert the keypoints from KeyPoint objects to NumPy
		# arrays
		kps = np.float32([kp.pt for kp in kps])

		# return a tuple of keypoints and features
		return (kps, features)

	def matchKeypoints(self, kpsA, kpsB, featuresA, featuresB,
		ratio, reprojThresh):
		# compute the raw matches and initialize the list of actual
		# matches
		matcher = cv2.DescriptorMatcher_create("BruteForce")
		rawMatches = matcher.knnMatch(featuresA, featuresB, 2)
		matches = []

		# loop over the raw matches
		for m in rawMatches:
			# ensure the distance is within a certain ratio of each
			# other (i.e. Lowe's ratio test)
			if len(m) == 2 and m[0].distance < m[1].distance * ratio:
				matches.append((m[0].trainIdx, m[0].queryIdx))

		# computing a homography requires at least 4 matches
		if len(matches) > 4:
			# construct the two sets of points
			ptsA = np.float32([kpsA[i] for (_, i) in matches])
			ptsB = np.float32([kpsB[i] for (i, _) in matches])

			# compute the homography between the two sets of points
			(H, status) = cv2.findHomography(ptsA, ptsB, cv2.RANSAC,
				reprojThresh)

			# return the matches along with the homograpy matrix
			# and status of each matched point
			return (matches, H, status)

		# otherwise, no homograpy could be computed
		return None

	def drawMatches(self, imageA, imageB, kpsA, kpsB, matches, status):
		# initialize the output visualization image
		(hA, wA) = imageA.shape[:2]
		(hB, wB) = imageB.shape[:2]
		vis = np.zeros((max(hA, hB), wA + wB, 3), dtype="uint8")
		vis[0:hA, 0:wA] = imageA
		vis[0:hB, wA:] = imageB

		# loop over the matches
		for ((trainIdx, queryIdx), s) in zip(matches, status):
			# only process the match if the keypoint was successfully
			# matched
			if s == 1:
				# draw the match
				ptA = (int(kpsA[queryIdx][0]), int(kpsA[queryIdx][1]))
				ptB = (int(kpsB[trainIdx][0]) + wA, int(kpsB[trainIdx][1]))
				cv2.line(vis, ptA, ptB, (0, 255, 0), 1)

		# return the visualization
		return vis

	def getBoundingRect(self,x0,y0,w0,h0,x1,y1,w1,h1):
		return (min(x0,x1),min(y0,y1),max(x0+w0,x1+w1),max(y0+h0,y1+h1))

	def negToZero(self,number):
		if number < 0:
			return 0
		return number

	def calcBoundingRect(self,imgs,hs):
		points = []
		h = np.identity(3)
		for i in range(len(imgs)):
			width,height = (imgs[i].shape[1],imgs[i].shape[0])
			lt = h.dot(np.array([0, 0, 1]))
			lb = h.dot(np.array([0, height,1]))
			rt = h.dot(np.array([width, 0, 1]))
			rb = h.dot(np.array([width, height,1]))
			points.extend([lt,lb,rt,rb])
			if i < len(imgs)-1:
				h = h.dot(hs[i])
		lt,rb = ((min(points,key = lambda t: t[0])[0],min(points,key = lambda t: t[1])[1]),
				(max(points,key = lambda t: t[0])[0],max(points,key = lambda t: t[1])[1]))
		return (lt,rb)


import logging
logging.basicConfig(filename ="tupleHomework.log", level = logging.DEBUG, format ="%(asctime)s %(levelname)s %(message)s")

class tuple_t:
    def __init__(self,b):
        logging.info("only for tuple")
        self.b = b

    def extract_tuple(self):
        try:
            logging.info("Extracting tuples from the collection")
            for i in self.b:
                if type(i) == tuple:
                    logging.info(i)
        except Exception as e:
            logging.error(e)

tupleObj = tuple_t(b = [[1,2,3,4] , (2,3,4,5,6) , (3,4,5,6,7) , set([23,4,5,45,4,4,5,45,45,4,5]) , {'k1' :"sudh" , "k2" : "ineuron","k3":"kumar" , 3:6 , 7:8} , ["ineuron" , "data science "]])
tupleObj.extract_tuple()


import argparse
import json
import os
import time
from pathlib import Path

import clip
import kornia.augmentation as k_transforms
import torch
import torchvision.transforms as tv_transforms
from omegaconf import OmegaConf
from sample_language_nn import sample_pairs
from torch.nn.functional import normalize
from tqdm import tqdm

from lgssl.datasets.builder import build_loader

def extract_vision_embeddings(dataset_cfg, model_name, model_weights):
    print("---- get dataset ----")
    curr_time = time.time()
    dataset_name = dataset_cfg.name
    n_workers = len(os.sched_getaffinity(0))
    loader = build_loader(dataset_cfg, shuffle=False, n_workers=n_workers)
    print(f"Loaded dataset {dataset_name} in {time.time() - curr_time:.2f}sec")

    print("---- generate embeddings ----")
    curr_time = time.time()
    if model_name == "clip":
        clip_model, _ = clip.load(model_weights, "cuda")
        model = clip_model.encode_image
        img_mean = [0.48145466, 0.4578275, 0.40821073]
        img_std = [0.26862954, 0.26130258, 0.27577711]
    else:
        model = torch.hub.load("pytorch/vision", model_name, weights=model_weights)
        model.fc = torch.nn.Identity()
        model.heads = torch.nn.Identity()
        model = model.cuda().eval()
        model = torch.nn.DataParallel(model)
        img_mean = [0.485, 0.456, 0.406]
        img_std = [0.229, 0.224, 0.225]

    dataset = []
    embeddings = []
    gpu_augment = torch.nn.Sequential(
        tv_transforms.ConvertImageDtype(torch.float32),
        k_transforms.Normalize(mean=img_mean, std=img_std),
    )

    # set embedding size for now
    dataset_size = len(loader.dataset)
    embedding_size = None
    prev_end = 0

    for i, batch in enumerate(tqdm(loader)):
        uids = batch["uid"]
        imgs = batch["image_0"]
        imgs = gpu_augment(imgs)
        caps = batch["caption_0"]
        batch_size = imgs.shape[0]

        for _j in range(batch_size):
            tar_i, id_i = uids[_j].split("-")
            dataset.append((tar_i, id_i, caps[_j]))

        with torch.inference_mode():
            emb_i = model(imgs.cuda())
            if type(emb_i) is list:
                emb_i = emb_i[0]

            if embedding_size is None:
                assert len(emb_i.shape) == 2
                embedding_size = emb_i.shape[1]
                embeddings = torch.zeros(
                    dataset_size, embedding_size, dtype=torch.float
                )
            else:
                assert embedding_size == emb_i.shape[1], emb_i.shape

            # get indices
            start_i = prev_end
            end_i = prev_end + batch_size
            prev_end = end_i
            assert end_i <= dataset_size
            embeddings[start_i:end_i] = normalize(emb_i, p=2, dim=-1).detach().cpu()
            del emb_i

    del model
    torch.cuda.empty_cache()

    embeddings = embeddings[:prev_end]
    assert len(dataset) == prev_end
    print(f"Gathered embeddings in {time.time() - curr_time:.2f}sec")

    return dataset, embeddings

def generate_dict(dataset_cfg, model_name, model_weights):

    # extract embeddings
    dataset, embeddings = extract_vision_embeddings(
        dataset_cfg, model_name, model_weights
    )

    # sample pairs
    nn_pairs = sample_pairs(dataset, embeddings)

    # change name
    model_weights = model_weights.replace("/", "").replace("-", "").lower()
    encoder = f"vis_{model_name}-{model_weights}"

    # save dict
    dataset_name = dataset_cfg.name
    save_dir = Path(__file__).parent / "../data/data_dicts"
    save_path = save_dir / f"{dataset_name}_{encoder}.json"

    json.dump(nn_pairs, save_path.open("w"))
    print(f"Saved nearest neighbors to: {str(save_path.resolve())}")

if __name__ == "__main__":
    # --- define data ---

    parser = argparse.ArgumentParser()
    parser.add_argument("dataset", type=str)
    parser.add_argument("model", type=str)
    parser.add_argument("weights", type=str)
    args = parser.parse_args()

    # define dataset config
    dataset_cfg = OmegaConf.create(
        {
            "_target_": "lgssl.datasets.simclr_dataset.SimCLRDataset",
            "name": args.dataset,
            "sampling": "self",
            "augmentation": "center_crop",
            "batch_size": 2048,
        }
    )

    generate_dict(dataset_cfg, args.model, args.weights)


import abc
import attr
import requests

from event_notifier.exceptions import EventNotifierNotificationDispatchException

class ANotifierBackend(object, metaclass=abc.ABCMeta):
    """Abstract class for notifier backends"""
    @abc.abstractmethod
    def dispatch_notification(self, notification): # pragma: no cover
        """Sends a passsed-in notification

        :param notification: a Notification object to be dispatched
        :type notification: event_notifier.Notification.Notification
        :raises EventNotifierNotificationDispatchException: When an error is encountered with dispatching the notification.
        :returns: True if the notification was dispatched correctly
        :rtype: bool
        """
        raise NotImplementedError

@attr.s
class PushoverNotifierBackend(ANotifierBackend):
    """A backend to send Pushover notifications"""
    token = attr.ib()
    user = attr.ib()
    device = attr.ib(default=None)
    # def __init__(self, token, user, device=None):
    #     """Initializes the PushoverNotifier backend
    #
    #     :param token: The Pushover API token ("APP_TOKEN").
    #     :param user: The Pushover user/group key to whom the notification will be sent
    #     :param device: Optional. The user's device name to send the message directly to that device, rather than all of the user's devices (multiple devices may be separated by a comma)
    #     :type token: str
    #     :type user: str
    #     :type device: str
    #     """
    #     self.token = token
    #     self.user = user
    #     self.device = device

    def dispatch_notification(self, notification):
        """Sends a passed-in notification

        :param notification: a Notification object to be dispatched
        :type notification: event_notifier.Notification.Notification
        :raises EventNotifierNotificationDispatchException: When an error is encountered with dispatching the notification. Passes the response object from the HTTP POST request to the exception.
        :returns: True if the notification was dispatched correctly
        :rtype: bool
        """
        payload = {
            'token': self.token,
            'user': self.user,
            'title': notification.subject,
            'message': notification.message,
            'device': self.device,
            'url': notification.url,
            'url_title': notification.url_title,
            'priority': notification.priority,
            'timestamp': int(notification.timestamp.timestamp()),
            'sound': notification.sound
        }
        headers = {'Content-type': 'application/x-www-form-urlencoded'}
        r = requests.post(
            'https://api.pushover.net:443/1/messages.json',
            data = payload,
            headers = headers
        )
        if r.status_code != 200:
            raise EventNotifierNotificationDispatchException(r)
        return True


import torch
import torch.nn as nn
import torch.nn.functional as F
from .layers import conv2d, deconv2d, attention_block

"""
https://www.kaggle.com/code/xxc025/attention-u-net/notebook
"""

class UNet(nn.Module):
    def __init__(self, inc, num_cls, start_fm):
        super(UNet, self).__init__()

        #(Double) Convolution 1
        self.double_conv1 = conv2d(inc, start_fm)
        #Max Pooling 1
        self.maxpool1 = nn.MaxPool2d(kernel_size=2)

        #Convolution 2
        self.double_conv2 = conv2d(start_fm, start_fm * 2)
        #Max Pooling 2
        self.maxpool2 = nn.MaxPool2d(kernel_size=2)

        #Convolution 3
        self.double_conv3 = conv2d(start_fm * 2, start_fm * 4)
        #Max Pooling 3
        self.maxpool3 = nn.MaxPool2d(kernel_size=2)

        #Convolution 4
        self.double_conv4 = conv2d(start_fm * 4, start_fm * 8)
        #Max Pooling 4
        self.maxpool4 = nn.MaxPool2d(kernel_size=2)

        #Convolution 5
        self.double_conv5 = conv2d(start_fm * 8, start_fm * 16)

        self.convtranas6 = nn.ConvTranspose2d(start_fm * 16, start_fm * 8, 2, 2)
        self.deconv6 = deconv2d(start_fm * 8, start_fm * 8)
        self.atten_block6 = attention_block(start_fm * 8)
        self.conv6 = conv2d(start_fm * 16, start_fm * 8)

        self.convtranas7 = nn.ConvTranspose2d(start_fm * 8, start_fm * 4, 2, 2)
        self.deconv7 = deconv2d(start_fm * 4, start_fm * 4)
        self.atten_block7 = attention_block(start_fm * 4)
        self.conv7 = conv2d(start_fm * 8, start_fm * 4)

        self.convtranas8 = nn.ConvTranspose2d(start_fm * 4, start_fm * 2, 2, 2)
        self.deconv8 = deconv2d(start_fm * 2, start_fm * 2)
        self.atten_block8 = attention_block(start_fm * 2)
        self.conv8 = conv2d(start_fm * 4, start_fm * 2)

        self.convtranas9 = nn.ConvTranspose2d(start_fm * 2, start_fm, 2, 2)
        self.deconv9 = deconv2d(start_fm, start_fm)
        self.atten_block9 = attention_block(start_fm)
        self.conv9 = conv2d(start_fm * 2, start_fm)

        # One by One Conv
        self.one_by_one = nn.Conv2d(start_fm, num_cls, kernel_size=3, padding=1)
        self.final_act = nn.Sigmoid()

    def forward(self, inputs):
        # 1
        conv1 = self.double_conv1(inputs)
        maxpool1 = self.maxpool1(conv1)

        # 2
        conv2 = self.double_conv2(maxpool1)
        maxpool2 = self.maxpool2(conv2)

        # 3
        conv3 = self.double_conv3(maxpool2)
        maxpool3 = self.maxpool3(conv3)

        # 4
        conv4 = self.double_conv4(maxpool3)
        maxpool4 = self.maxpool4(conv4)

        # center
        conv5 = self.double_conv5(maxpool4)

        # 6
        conv5 = self.convtranas6(conv5)
        up6 = self.deconv6(conv5)
        conv6 = self.atten_block6(up6, conv4)
        up6 = torch.cat([up6, conv6], 1)
        ex_conv6 = self.conv6(up6)

        # 7
        ex_conv6 = self.convtranas7(ex_conv6)
        up7 = self.deconv7(ex_conv6)
        conv7 = self.atten_block7(up7, conv3)
        up7 = torch.cat([up7, conv7], 1)
        ex_conv7 = self.conv7(up7)

        # 8
        ex_conv7 = self.convtranas8(ex_conv7)
        up8 = self.deconv8(ex_conv7)
        conv8 = self.atten_block8(up8, conv2)
        up8 = torch.cat([up8, conv8], 1)
        ex_conv8 = self.conv8(up8)

        # 9
        ex_conv8 = self.convtranas9(ex_conv8)
        up9 = self.deconv9(ex_conv8)
        conv9 = self.atten_block9(up9, conv1)
        up9 = torch.cat([up9, conv9], 1)
        ex_conv9 = self.conv9(up9)

        one_by_one = self.one_by_one(ex_conv9)
        x = self.final_act(one_by_one)
        return x

if __name__=="__main__":
    start_fm = 64
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    H = W = 224
    model = UNet(inc=3, num_cls=1, start_fm=start_fm)
    inp = torch.rand((1, 3, 224, 224), dtype=torch.float32).to(device)
    out = model(inp)
    #print(out.shape)

    # summary(model, (1, 3, 224, 224))


from django.shortcuts import get_object_or_404
from django.shortcuts import render
from django.views.generic import ListView
from .models import Post
from .forms import PostForm
from django.contrib.auth.decorators import login_required
# Create your views here.
class PostListView(ListView):
    queryset = Post.objects.all()
    context_object_name = 'posts'
    template_name = 'posts/post_list.html'

def post_detail(request, year, month, day, post):
    post = get_object_or_404(
        Post,
        slug=post,
        publish__year=year,
        publish__month=month,
        publish__day=day,
    )

    return render(
        request,
        'posts/post_detail.html',
        {'post': post}
    )
@login_required
def new_post(request):
    new_post = False
    if request.method == 'POST':
        post_form = PostForm(data=request.POST)
        if post_form.is_valid():
            new_post = post_form.save(commit=False)
            new_post.author = request.user
            new_post.status = 'Published'
            new_post.save()
    else:
        post_form = PostForm()

    return render(
        request,
        'posts/new_post.html',
        {'post_form': post_form,
         'new_post': new_post}
    )


import matplotlib.pyplot as plt
import numpy as np
from sklearn.datasets import make_circles
from sklearn.datasets import make_moons
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.tree import DecisionTreeClassifier

plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
datasets = [make_moons(n_samples=200, noise=0.2, random_state=1), make_circles(n_samples=200, noise=0.2, factor=0.5, random_state=1)]
i = 1
for dataset in datasets:
    X, y = dataset
    X = StandardScaler().fit_transform(X)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
    x1_min, x1_max = X[:, 0].min() - 0.5, X[:, 0].max() + 0.5
    x2_min, x2_max = X[:, 1].min() - 0.5, X[:, 1].max() + 0.5
    x_axis, y_axis = np.meshgrid(np.arange(x1_min, x1_max, 0.2), np.arange(x2_min, x2_max, 0.2))
    ax = plt.subplot(len(datasets), 2, i)
    if i == 1:
        ax.set_title("月亮形数据")
    elif i == 3:
        ax.set_title("环形数据")
    ax.scatter(X_train[:, 0], X_train[:, 1], edgecolors='k', marker='*')
    ax.scatter(X_test[:, 0], X_test[:, 1], alpha=0.6)
    ax.set_xlim(x_axis.min(), x_axis.max())
    ax.set_ylim(y_axis.min(), y_axis.max())
    ax.set_xticks(())
    ax.set_yticks(())
    i = i + 1
    ax = plt.subplot(len(datasets), 2, i)
    model = DecisionTreeClassifier(max_depth=5)
    model.fit(X_train, y_train)
    score = model.score(X_test, y_test)
    Z = model.predict_proba(np.c_[x_axis.ravel(), y_axis.ravel()])
    Z = Z[:, 1]
    Z = Z.reshape(x_axis.shape)
    ax.contourf(x_axis, y_axis, Z, alpha=.8)
    ax.scatter(X_train[:, 0], X_train[:, 1], edgecolors='k')
    ax.scatter(X_test[:, 0], X_test[:, 1], alpha=0.6)
    ax.set_xlim(x_axis.min(), x_axis.max())
    ax.set_ylim(y_axis.min(), y_axis.max())
    ax.set_xticks(())
    ax.set_yticks(())
    if i == 2:
        ax.set_title("决策树")
    elif i == 4:
        ax.set_title("决策树")
    ax.text(x_axis.max() - .3, y_axis.min() + .3, (f'{100 * score}%'), size=15, horizontalalignment='right')
    i = i + 1
plt.tight_layout()
plt.show()


# -*- coding: utf-8 -*-

# Copyright 2018 IBM.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================

from qiskit.aqua import Pluggable
from abc import abstractmethod
import copy
from qiskit.aqua import AquaError

class AlgorithmInput(Pluggable):

    _PROBLEM_SET = ['energy', 'excited_states', 'eoh', 'search', 'svm_classification', 'ising']

    @abstractmethod
    def __init__(self):
        super().__init__()
        if 'problems' not in self.configuration or len(self.configuration['problems']) <= 0:
            raise AquaError('Algorithm Input missing or empty configuration problems')

        for problem in self.configuration['problems']:
            if problem not in AlgorithmInput._PROBLEM_SET:
                raise AquaError('Problem {} not in known problem set {}'.format(problem, AlgorithmInput._PROBLEM_SET))

    @property
    def all_problems(self):
        return copy.deepcopy(self._PROBLEM_SET)

    @property
    def problems(self):
        """
        Gets the set of problems that this input form supports
        """
        return self.configuration.problems

    @abstractmethod
    def to_params(self):
        """
        Convert the derived algorithminput class fields to a dictionary where the values are in a
        form that can be saved to json
        Returns:
            Dictionary of input fields
        """
        raise NotImplementedError()

    @abstractmethod
    def from_params(self, params):
        """
        Load the dictionary into the algorithminput class fields. This dictionary being that as
        created by to_params()
        Args:
            params: A dictionary as originally created by to_params()
        """
        raise NotImplementedError()


import torch as tr
import torchvision as tv
import numpy as np
import random, os, math
from tqdm import trange
from pathlib import Path
from datetime import datetime
from tensorboardX import SummaryWriter
from config import get_config
from models import Generator, Discriminator
from vgg import FeatureExtractor

from processing import get_dataset, downsample, reverse_normalize, normalize, randomcrop, tt

device = tr.device('cuda' if tr.cuda.is_available() else 'cpu')
cfg = get_config()

class SRGAN(object):
    def __init__(self, cfg):
        super(SRGAN, self).__init__()

        # Networks
        self.generator = Generator(cfg)
        self.discriminator = Discriminator(cfg)

        # Optimizers
        self.optim_gen = tr.optim.Adam(self.generator.parameters(), lr=cfg.learning_rate, betas=(cfg.beta1, 0.999))
        self.optim_disc = tr.optim.Adam(self.discriminator.parameters(), lr=cfg.learning_rate, betas=(cfg.beta1, 0.999))

        # loss models
        self.mse_loss = tr.nn.MSELoss()
        self.bce_loss = tr.nn.BCELoss(reduction='sum')
        self.feature_extractor = FeatureExtractor()

        # Get image dataset (cropped)
        dataset = get_dataset(cfg.data_dir)
        self.dataloader = tr.utils.data.DataLoader(dataset, batch_size=cfg.batch_size, shuffle=True)

        # For logging results to tensorboard
        self.global_step = 0
        self.build_writers()

    def build_writers(self):
        if not Path(cfg.save_dir).is_dir():
            os.mkdir(cfg.save_dir)
        if cfg.extension is None:
            cfg.extension = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')

        self.save_path = cfg.save_dir + cfg.extension

        log_path = cfg.log_dir + cfg.extension
        self.writer = SummaryWriter(log_path)

    def logger(self, name, loss):
        if self.global_step % cfg.log_freq == 0:
            # Log vars
            self.writer.add_scalar(name, loss, self.global_step)

    def log_state(self, name, image):
        if self.global_step % (cfg.log_freq * 5) == 0:
            image = reverse_normalize(image)
            self.writer.add_image(name, image, self.global_step)

    def pretrain(self):
        '''Pretrain the generator network using a MSE loss function. This is done so that SRGAN generator does not fall into a local optima'''

        if os.path.isfile('./pretrained_models/pretrained.pt'):
            print("loaded_checkpoint: ./pretrained_models/pretrained.pt")
            checkpoint = tr.load('./pretrained_models/pretrained.pt')
            self.generator.load_state_dict(checkpoint['generator_state'])
            self.optim_gen.load_state_dict(checkpoint['generator_optimizer'])
            self.generator.train()

        print('Pretraining Generator')

        ds = tr.FloatTensor(cfg.batch_size, cfg.num_channels, cfg.cropsize//cfg.factor, cfg.cropsize//cfg.factor)

        for epoch in trange(cfg.pretrain_epochs):

            # Batch
            for i, data in enumerate(self.dataloader):

                # mini-batch data
                hr, _ = data

                if hr.size(0) < cfg.batch_size:
                    break

                # Downsample images to low resolution
                # Removing NORMALIZE -> [0,1] instead of ~ N(0.485, 0.29)
                # remove in downscale if so
                for j in range(cfg.batch_size):
                    ds[j] = downsample(hr[j])
                    hr[j] = normalize(hr[j])

                # Generate the super resolution image
                sr = self.generator(ds)

                # https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch
                self.generator.zero_grad()

                loss = self.mse_loss(sr, hr)
                loss.backward()
                self.optim_gen.step()

                self.logger('Pretrain/Generator loss', loss)

                self.log_state('Pretrain/Downsampled', ds[0])
                self.log_state('Pretrain/Generated', sr[0])
                self.log_state('Pretrain/Original', hr[0])
                self.global_step += 1

            if epoch % cfg.save_freq == 0:
                tr.save({
                    'generator_state': self.generator.state_dict(),
                    'generator_optimizer': self.optim_gen.state_dict()
                }, './pretrained_models/pretrained.pt')

    def train(self):
        '''Train the SRGAN using the just pretrained or past pretrained generator.'''

        if os.path.isfile('./pretrained_models/pretrained.pt'):
            print("loaded_checkpoint: ./pretrained_models/pretrained.pt")
            checkpoint = tr.load('./pretrained_models/pretrained.pt')
            self.generator.load_state_dict(checkpoint['generator_state'])
            self.optim_gen.load_state_dict(checkpoint['generator_optimizer'])
            self.generator.train()

        print('Training SRGAN')

        ds = tr.FloatTensor(cfg.batch_size, cfg.num_channels, cfg.cropsize//cfg.factor, cfg.cropsize//cfg.factor)

        # Restart global step for SRGAN tape
        self.global_step = 0

        real_label = tr.ones((cfg.batch_size,1))
        fake_label = tr.zeros((cfg.batch_size,1))

        for epoch in trange(cfg.epochs):

            # Mini-batchs
            for i, data in enumerate(self.dataloader):
                # Generate data
                hr, _ = data

                # Prevent error if the dataloader runs out of images
                if hr.size(0) < cfg.batch_size:
                    break

                # mini-batch data (ToTensor() normalizes between [0,1])
                # randn() instead of rand() for normalized input [-1,1]
                nz = tr.rand((hr.size(0),hr.size(1),hr.size(2)//cfg.factor, hr.size(3)//cfg.factor))
                for j in range(cfg.batch_size):
                    ds[j] = downsample(hr[j])
                    hr[j] = normalize(hr[j])
                    nz[j] = normalize(nz[j])

                ## Begin Training Discriminator
                # ----------------------------------------------------------------
                self.discriminator.zero_grad()
                # Why zero_grad
                # https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch

                # BCELoss use explained here
                # https://pytorch.org/tutorials/beginner/dcgan_faces_tutorial.html

                # train real
                truth = self.discriminator(hr)
                loss_truth = self.bce_loss(truth, real_label)
                loss_truth.backward()

                # train fake
                noise = self.generator(nz)
                fake = self.discriminator(noise.detach())
                loss_fake = self.bce_loss(fake, fake_label)
                loss_fake.backward()

                loss_disc = loss_truth + loss_fake
                self.optim_disc.step()

                # Begin Training Generator
                # ----------------------------------------------------------------
                self.generator.zero_grad()

                # Generate the super resolution image and judge it
                sr = self.generator(ds)
                generated = self.discriminator(sr)

                # Calculate Perceptual Loss
                true_features = self.feature_extractor(hr)
                generated_features = self.feature_extractor(sr)
                content_loss = self.mse_loss(generated_features, true_features)
                adversarial_loss = self.bce_loss(generated, real_label)
                loss_gen = content_loss + (10e-3 * adversarial_loss)

                # Optimize Generator
                loss_gen.backward()
                self.optim_gen.step()

                # Log Losses
                self.logger('SRGAN/Content Loss', content_loss)
                self.logger('SRGAN/Adversarial Loss', adversarial_loss)
                self.logger('SRGAN/Discriminator Loss', loss_disc)
                self.logger('SRGAN/Generator Loss', loss_gen)

                # Log Images
                self.log_state('SRGAN/Original', hr[0])
                self.log_state('SRGAN/Downsampled', ds[0])
                self.log_state('SRGAN/Generated', sr[0])

                # Increment Tape
                self.global_step += 1

            if epoch % cfg.save_freq == 0:
                tr.save({
                    'generator_state': self.generator.state_dict(),
                    'generator_optimizer': self.optim_gen.state_dict()
                }, self.save_path+'.pt')

            # Get image dataset (cropped)
            dataset = get_dataset(cfg.data_dir)
            self.dataloader = tr.utils.data.DataLoader(dataset, batch_size=cfg.batch_size, shuffle=True)

def main():
    srgan = SRGAN(cfg)

    if cfg.pretrain is True:
        srgan.pretrain()

    srgan.train()

if __name__ == '__main__':
    main()


from django import forms
from .models import OrderItem, ColorVariation, Product, SizeVariation, Address
from django.contrib.auth import get_user_model

User = get_user_model()

class AddToCartForm(forms.ModelForm):
    color = forms.ModelChoiceField(queryset=ColorVariation.objects.none())
    size = forms.ModelChoiceField(queryset=SizeVariation.objects.none())

    class Meta:
        model = OrderItem
        fields = ['quantity', 'color', 'size']

    def __init__(self, *args, **kwargs):
        product_id = kwargs.pop('product_id')
        product = Product.objects.get(id=product_id)
        super().__init__(*args, **kwargs)

        self.fields['color'].queryset=product.available_colours.all()
        self.fields['size'].queryset=product.available_sizes.all()

class CheckoutViewForm(forms.Form):
    billing_county = forms.CharField(required = False)
    billing_postal_code = forms.IntegerField(required = False)
    billing_phone_number1 = forms.IntegerField(required = False)
    billing_phone_number2 = forms.IntegerField(required = False)

    shipping_county = forms.CharField(required = False)
    shipping_postal_code = forms.IntegerField(required = False)
    shipping_phone_number1 = forms.IntegerField(required = False)
    shipping_phone_number2 = forms.IntegerField(required = False)

    selected_billing_address = forms.ModelChoiceField(queryset=Address.objects.none(), required = False)
    selected_shipping_address = forms.ModelChoiceField(queryset=Address.objects.none(), required = False)

    def __init__(self, *args, **kwargs):
        user_id = kwargs.pop('user_id')
        user = User.objects.get(id=user_id)
        super().__init__(*args, **kwargs)

        user = User.objects.get(id=user_id)

        billing_address_queryset = Address.objects.filter(
            user = user,
            Address_type = 'B'
        )
        shipping_address_queryset = Address.objects.filter(
            user = user,
            Address_type = 'S'
        )

        self.fields['selected_billing_address'].queryset= billing_address_queryset
        self.fields['selected_shipping_address'].queryset=shipping_address_queryset

    def clean(self):
        data = self.cleaned_data
        selected_billing_address = data.get('selected_billing_address', None)

        if selected_billing_address is None:
            if not data.get('billing_county', None):
                self.add_error('billing_county', 'this field must be filled')

            if not data.get('billing_postal_code', None):
                self.add_error('billing_postal_code', 'this field must be filled')

            if not data.get('billing_phone_number1', None):
                self.add_error('billing_phone_number1', 'this field must be filled')

            if not data.get('billing_phone_number2', None):
                self.add_error('billing_phone_number2', 'this field must be filled')

        selected_shipping_address = data.get('selected_shipping_address', None)
        if selected_shipping_address is None:
            if not data.get('shipping_county', None):
                self.add_error('shipping_county', 'this field must be filled')

            if not data.get('shipping_postal_code', None):
                self.add_error('shipping_postal_code', 'this field must be filled')

            if not data.get('shipping_phone_number1', None):
                self.add_error('shipping_phone_number1', 'this field must be filled')

            if not data.get('shipping_phone_number2', None):
                self.add_error('shipping_phone_number2', 'this field must be filled')